Sphinx uses C to expand the xmlpipe2 Data Source

Because sphtracing is used as the full-text search engine, but many of my data is stored on the hard disk, it would be too painful to read the data into the database, and the data volume is large, the MySQL database stores the data only for full-text retrieval, which is too wasteful. Therefore, the xmlpipe2 data source is decided to be used. Because of the high I/O capability of the C language, it is simply used to solve the efficiency problem, C language that is never written. The Code is as follows:

 

# Include <stdio. h> <br/> # include <stdlib. h> <br/> # include <string. h> <br/> # include <unistd. h> <br/> # include <MySQL. h> <br/> # include <iconv. h> </P> <p> # include "dictionary. H "<br/> # include" iniparser. H "</P> <p> // configuration file name <br/> # define ini_file_name" sanshi_xmlpipe.ini "<br/> // delimiter number of multiple SQL statements <br/> # define SQL _sign "|" <br/> // set the separator between the name and field value in the field <br/> # define attribute_sign ": "<br/> // set the delimiter between multiple fields <br/> # define dield _ Sign ", "<br/> // set the length of the file to be read <br/> # define readfile_max_len 1024 <br/> // set the path + maximum length of the file <br/> # define file_name_max_len 1024 </P> <p> typedef struct tag_sanshi_ini_config <br/>{< br/> char * db_name; <br/> char * db_user; <br/> char * db_pwd; <br/> char * db_host; <br/> char * SQL _query_pre; <br/> char * SQL _query; <br/> char * SQL _query_post; <br/> char * sphinx_schema; <br/> char * sphinx_id; <br/> char * sphinx_file; <Br/> char * sphinx_other_field; <br/> char * file_dir; <br/>} sanshi_ini_config; </P> <p> sanshi_ini_config conf = {<br/> "DB: db_name", <br/> "DB: db_user", <br/> "DB: db_pwd ", <br/>" DB: db_host ", <br/>" SQL: SQL _query_pre ", <br/>" SQL: SQL _query ", <br/>" SQL: SQL _query_post ", <br/>" XML: schema ", <br/>" XML: index_id ", <br/>" XML: file_field ", <br/>" XML: other_field ", <br/>" file: base_dir "<br/>}; </P> <p> typedef struct t Ag_sanshi_field <br/>{< br/> char * field; <br/> int ID; <br/>} sanshi_field; </P> <p> sanshi_field parser_field (char * field_str) <br/>{< br/> sanshi_field temp; <br/> char * Buf = strstr (field_str, attribute_sign); <br/> temp. id = atoi (BUF + strlen (attribute_sign); <br/> Buf [0] = '/0'; <br/> temp. field = field_str; <br/> // temp. id = BUF + strlen (attribute_sign); <br/> // Buf = strstr (field_str, attribute_sig N); <br/> // temp. id = atoi (BUF + strlen (attribute_sign); <br/> // printf ("% S % d/N", field_str, temp. field, temp. ID); <br/>/* <br/> char * P; <br/> char field_tmp [strlen (field_str) + 1]; <br/> strcpy (field_tmp, field_str); <br/> // temp. field = strtok (field_str, attribute_sign); <br/> // field_str = strtok (null, attribute_sign); <br/> // temp. id = atoi (p); <br/> printf ("% s/n", field_tmp); <br/> field_str = NULL; <Br/> */<br/> return temp; <br/>}</P> <p> void print_file_content (char * file_name) <br/>{< br/> file * FP; <br/> char line [readfile_max_len]; <br/> fp = fopen (file_name, "R "); <br/> If (FP! = NULL) <br/>{< br/> while (fgets (line, readfile_max_len, FP )! = NULL) <br/>{< br/> printf (line); <br/>}< br/> fclose (FP ); <br/>}</P> <p> void exec_mysql_query (MySQL * mysql_con, char * SQL _str) <br/>{< br/> char * token = strtok (SQL _str, SQL _sign); <br/> while (Token! = NULL) <br/>{< br/> int query_error_no = 0; <br/> query_error_no = mysql_query (mysql_con, token); <br/> If (query_error_no! = 0) <br/>{< br/> printf ("error SQL = % S/nerror NO = % d/nerror MSG = % s/n", Token, query_error_no, mysql_error (mysql_con); <br/> mysql_close (mysql_con); <br/> exit (0 ); <br/>}< br/> mysql_free_result (mysql_store_result (mysql_con); <br/> // printf ("Exec SQL: % s/n", token ); <br/> token = strtok (null, SQL _sign); <br/>}</P> <p> int main (INT argc, char * argv []) <br/>{< br/> dictionary * ini; </P> <p> MySQL MYS Ql, * mysql_con; <br/> mysql_res * result; <br/> mysql_row row; <br/> int query_error_no, sphinx_id; </P> <p> sanshi_field file_field; </P> <p> ini = iniparser_load (ini_file_name); <br/> // get ini config MySQL set <br/> Conf. db_name = iniparser_getstring (INI, Conf. db_name, "test"); <br/> Conf. db_user = iniparser_getstring (INI, Conf. db_user, "root"); <br/> Conf. db_pwd = iniparser_getstring (INI, Conf. db_pwd, ""); <br/> con F. db_host = iniparser_getstring (INI, Conf. db_host, "localhost"); <br/> Conf. SQL _query_pre = iniparser_getstring (INI, Conf. SQL _query_pre, null); <br/> Conf. SQL _query = iniparser_getstring (INI, Conf. SQL _query, null); <br/> Conf. SQL _query_post = iniparser_getstring (INI, Conf. SQL _query_post, null); <br/> Conf. sphinx_schema = iniparser_getstring (INI, Conf. sphinx_schema, null); <br/> sphinx_id = iniparser_getint (INI, c Onf. sphinx_id, 0); <br/> Conf. sphinx_file = iniparser_getstring (INI, Conf. sphinx_file, null); <br/> file_field = parser_field (Conf. sphinx_file); </P> <p> Conf. sphinx_other_field = iniparser_getstring (INI, Conf. sphinx_other_field, null); <br/> Conf. file_dir = iniparser_getstring (INI, Conf. file_dir ,". /"); </P> <p> // printf ("db_name = % S/T db_user = % S/T db_pwd = % S/T db_host = % s/n", Conf. db_name, Conf. db_user, Conf. db_p WD, Conf. db_host); <br/> // MySQL connect <br/> mysql_init (& MySQL); <br/> mysql_con = mysql_real_connect (& MySQL, Conf. db_host, Conf. db_user, Conf. db_pwd, Conf. db_name, 0, null, 0); <br/> If (mysql_con = NULL) <br/>{< br/> printf ("error: connect MySQL fail! Plaese check INI file in set/n % s/n ", mysql_error (& MySQL); <br/> exit (0 ); <br/>}< br/> // printf ("MySQL connect suc! /N "); </P> <p> // exec SQL <br/> exec_mysql_query (mysql_con, Conf. SQL _query_pre); </P> <p> query_error_no = mysql_query (mysql_con, Conf. SQL _query); </P> <p> If (query_error_no! = 0) <br/>{< br/> printf ("error SQL = % S/nerror NO = % d/nerror MSG = % s/n", Conf. SQL _query, query_error_no, mysql_error (mysql_con); <br/> mysql_close (mysql_con); <br/> exit (0 ); <br/>}< br/> // printf ("Exec SQL _query: % s/n", Conf. SQL _query); <br/> result = mysql_store_result (mysql_con); <br/> // echo xml header <br/> printf ("<? XML version =/"1.0/" encodeing =/"UTF-8/"?> /N <sphset: docset>/n % s/n ", Conf. sphinx_schema); <br/> // printf ("% s/n", Conf. sphinx_other_field); <br/> while (ROW = mysql_fetch_row (result) <br/>{< br/> printf ("<sphsyntax: document ID =/"% d/">/N ", (row [sphinx_id]? Row [sphinx_id]: 0); <br/> char * field_str; <br/> char field_tmp [strlen (Conf. sphinx_other_field) + 1]; <br/> char temp_file_name [file_name_max_len]; <br/> memcpy (field_tmp, Conf. sphinx_other_field, strlen (Conf. sphinx_other_field) + 1); <br/> field_str = strtok (field_tmp, dield_sign); <br/> while (field_str! = NULL) <br/>{< br/> // printf ("% s/n", field_str); <br/> sanshi_field other_field = parser_field (field_str ); <br/> printf ("

 

I wrote C for the first time. Some local optimization departments are sufficient. I hope you can give me some advice.

 

The configuration file is as follows:

[DB] <br/> db_host = 127.0.0.1 <br/> db_name = test <br/> db_pwd = 123456 <br/> db_user = root <br/> [SQL] <br /> SQL _query_pre = select * From log | select * From log <br/> SQL _query = select * From log <br/> SQL _query_post = <br/> [XML] <br/> schema =/<br/> <Sphinx: schema>/<br/> <Sphinx: field name = "logactiontype"/>/<br/> <Sphinx: field name = "logdatatype"/>/<br/> <Sphinx: ATTR name = "logtime" type = "timestamp"/>/<br/> <sphype: ATTR name = "logip" type = "int" bits = "16" default = "1"/>/<br/> </Sphinx: schema> <br/> index_id = 0 <br/> file_field = logactiontype: 1 <br/> other_field = logdatatype: 2, logtime: 6, logip: 4 <br/> base_dir =

 

Note: The INI parsing of this program uses the source code of iniparser3.0b, that is, the dependent two hfiles.

# Include "dictionary. H"

# Include "iniparser. H"

The compiled command is:

Gcc-I/data/APP/MySQL/include/MySQL/-L/data/APP/MySQL/lib/MySQL/-l mysqlclient-g-o sanshi sanshi_xmlpipe.c dictionary. c iniparser. c

Note that the library path of msql is already named