Collect PHP-PHP source code

Make full use of the powerful string processing capabilities of regular expressions. the functions are simple and can meet the needs of general applications. the functions are constantly improved. the use process is: set an initial url, add navigation rules, add collection fields and rules, and save the output to make full use of the powerful string processing capabilities of regular expressions. this feature is easy to use and can meet the needs of general applications, the function is constantly improving. usage process: set an initial url, add navigation rules, add collection fields and rules, and save the output.

 Timeout = 30;}/*** run collection ** @ return array */function run () {$ begintime = $ this-> microtime_float (); $ cnt = 1; foreach ($ this-> startUrls as $ starturl) {/*** parse the page number range in the starting address */if (preg_match ("~ \ {(\ D +), (\ d + )\}~ ", $ Starturl, $ pagenum) {$ pagebegin = intval ($ pagenum [1]); $ pageend = intval ($ pagenum [2]); (; $ pagebegin <= $ pageend; $ pagebegin ++) {$ starturl = str_replace ($ pagenum [0], $ pagebegin, $ starturl ); $ urllists = $ this-> getLists ($ this-> layout_arr [0] ['pattern'], $ this-> getContent ($ starturl )); foreach ($ urllists as $ url) {if ($ this-> limit> 0 & $ cnt <= $ this-> limit) | $ this-> limit = 0) {$ this-> filterContent ($ t His-> getContent ($ url, $ starturl); $ cnt ++ ;}}}} else {$ urllists = $ this-> getLists ($ this-> layout_arr [0] ['pattern'], $ this-> getContent ($ starturl )); foreach ($ urllists as $ url) {if ($ this-> limit> 0 & $ cnt <= $ this-> limit) | $ this-> limit = 0) {$ this-> filterContent ($ this-> getContent ($ url, $ starturl )); $ cnt ++ ;}}$ this-> runtime = $ this-> microtime_float ()-$ begintime; return $ this-> result;}/*** from text Extract the url list ** @ param string $ pattern * @ param string $ content * @ return Array */function getLists ($ pattern = '', $ content = '') {if (strpos ($ pattern, '{*}') = false) return array ($ pattern ); $ pattern = preg_quote ($ pattern); $ pattern = str_replace ('\{\* \}',' ([^ \ '\ ">] *)', $ pattern); $ pattern = "~ ". $ Pattern ."~ Is "; preg_match_all ($ pattern, $ content, $ preg_rs); return array_unique ($ preg_rs [0]);} /*** get the html content of the specified url, including the header ** @ param string $ url * @ return string */function getContent ($ url, $ referer = '') {$ url = $ this-> urlRtoA ($ url, $ referer); preg_match ("/(http: \/) ([^: \/] *):? (\ D *)(\/?. *)/I ", $ url, $ preg_rs); $ host = $ preg_rs [2]; $ port = empty ($ preg_rs [3])? 80: $ preg_rs [3]; $ innerUrl = $ preg_rs [4]; $ fsp = fsockopen ($ host, $ port, $ errno, $ errstr, $ this-> timeout); if (! $ Fsp) $ this-> log ($ errstr. '('. $ errno. '); $ output = "GET $ url HTTP/1.0 \ r \ nHost: $ host \ r \ n"; if (! Isset ($ this-> putHead ['access']) $ this-> putHead ['access'] = "*/*"; if (! Isset ($ this-> putHead ['user-agent']) $ this-> putHead ['user-agent'] = 'mozilla/4.0 (compatible; MSIE 6.0; windows NT 5.2) '; if (! Isset ($ this-> putHead ['referer']) {$ this-> putHead ['referer'] = ($ referer = '')? 'Http ://'. $ host: $ referer;} foreach ($ this-> putHead as $ headname => $ headvalue) {$ output. = trim ($ headname ). ':'. trim ($ headvalue ). "\ r \ n";} $ output. = "Connection: close \ r \ n"; fwrite ($ fsp, $ output); $ content = ''; while (! Feof ($ fsp) {$ content. = fgets ($ fsp, 256);} fclose ($ fsp); $ this-> getHead ($ content); $ this-> httpContent = $ content; if (strtoupper ($ this-> charset )! = 'Utf-8') {$ content = iconv ($ this-> charset, 'utf-8', $ content);} else if (! Empty ($ this-> httpHead ['charset']) & $ this-> httpHead ['charset']! = 'Utf-8') {$ content = iconv ($ this-> httpHead ['charset'], 'utf-8', $ content );} $ this-> httpreferer = $ referer; return $ content ;} /*** extract all fields from the content according to the rule * @ param Array * @ return Array */function filterContent ($ content = '') {$ rs = array (); foreach ($ this-> field_arr as $ field => $ fieldinfo) {$ rs [$ field] = $ this-> getPregField ($ fieldinfo, $ content );} $ this-> result [] = $ rs;}/*** convert relative paths to absolute paths ** @ param string $ relative * @ Param string $ referer * @ return string */function urlRtoA ($ relative, $ referer) {/*** remove # Subsequent parts */$ pos = strpos ($ relative, '#'); if ($ pos> 0) $ relative = substr ($ relative, 0, $ pos);/*** if the detection path is an absolute address, return */if (preg_match ("~ ^ (Http | ftp )://~ I ", $ relative) return $ relative;/*** parse the reference address to obtain the protocol, host, and other information */preg_match ("~ (Http | ftp): // ([^/] *) (. */) ([^/#] *) ~ I ", $ referer, $ preg_rs); $ parentdir = $ preg_rs [1]; $ petrol = $ preg_rs [2]. ': //'; $ host = $ preg_rs [3];/*** if it starts with/*/if (preg_match ("~ ^ /~ I ", $ relative) return $ petrol. $ host. $ relative; return $ parentdir. $ relative;}/*** extract a field based on the rule ** @ param string $ pattern * @ param string $ content * @ return string */function getPregField ($ fieldinfo, $ content) {/*** when the rule is a fixed value, the fixed value */if (strpos ($ fieldinfo ['pattern'], '{' is returned directly. $ fieldinfo ['field']. '}') = false) return $ fieldinfo ['pattern']; if ($ fieldinfo ['isregular '] = 'true ') {$ pattern = $ fieldinfo ['pattern' ]; $ Pattern = str_replace ('{'. $ fieldinfo ['field']. '}', '(? P <'. $ fieldinfo ['field'].'> .*?) ', $ Pattern);} else {$ pattern = preg_quote ($ fieldinfo ['pattern']); $ pattern = str_replace ('\{'. $ fieldinfo ['field']. '\}','(? P <'. $ fieldinfo ['field'].'> .*?) ', $ Pattern);} $ pattern = "~ ". $ Pattern ."~ Is "; preg_match ($ pattern, $ content, $ preg_rs); $ fieldresult = $ preg_rs [$ fieldinfo ['field']; /*** remove the line break */$ fieldresult = preg_replace ("~ [\ R \ n] * ~ Is ",'', $ fieldresult);/*** perform secondary replacement for the collected results according to the rules */$ replace_arr = $ fieldinfo ['replace ']; if (is_array ($ replace_arr) {$ replace_arr [0] = "~ ". $ Replace_arr [0]." ~ S "; $ fieldresult = preg_replace ($ replace_arr [0], $ replace_arr [1], $ fieldresult );} /*** recursively collect fields with the next page */if ($ this-> pagelimit = 0) {if ($ fieldinfo ['nextpage']! = '') {$ Pattern = $ fieldinfo ['nextpage']; $ pattern = str_replace ('{nextpage }','(? P
 
  
[^ \ '\ ">] *?) ', $ Pattern); $ pattern = "~ ". $ Pattern ."~ Is "; if (preg_match ($ pattern, $ content, $ preg_rs) & $ preg_rs ['nextpage']! = '') {$ Fieldresult. = $ this-> getPregField ($ fieldinfo, $ this-> getContent ($ preg_rs ['nextpage'], $ this-> httpreferer) ;}} if (! Empty ($ fieldinfo ['callback']) $ fieldresult = $ fieldinfo ['callback'] ($ fieldresult); return $ fieldresult ;} /*** add a collection field and rule ** @ param string $ field * @ param string $ pattern */function addField ($ field, $ pattern, $ replace_arr = '', $ isregular = 'false', $ nextpage = '', $ callback ='') {$ rs = array ('field' => $ field, 'pattern' => $ pattern, 'replace '=> $ replace_arr, 'isreferer' => $ isregular, 'nextpage' => $ nextpage, 'callback' => $ callback); $ this-> field_arr [$ field] = $ rs;}/*** output **/function output () {echo "The result is:
  
"; Echo" runtime: $ this-> runtime S
  

  
";  print_r($this->result);  echo "
";}/*** Output to the XLS file ** @ param string $ file */function saveXls ($ file = 'spider_result.xls ') {$ fp = fopen ($ file, 'W'); if ($ fp) {foreach ($ this-> result as $ result) {$ line = implode ("\ t", $ result ). "\ n"; fputs ($ fp, $ line) ;}} fclose ($ fp); echo 'The result has been saved '. $ file. '.
  
Cost time :'. $ this-> runtime;} function saveSql ($ table = 'Spider _ result', $ file = 'Spider _ result. SQL ') {$ fp = fopen ($ file, 'w'); if ($ fp) {foreach ($ this-> field_arr as $ fieldinfo) {$ SQL _key. = ',''. $ fieldinfo ['field']. ''' ;}$ SQL _key = substr ($ SQL _key, 1); foreach ($ this-> result as $ result) {$ SQL _value = array (); foreach ($ result as $ key => $ value) {$ SQL _value [] = "'". $ this-> addslash ($ value ). "'";} $ line = "insert into' $ table' ($ SQL _key) VALUES (". join (',', $ SQL _value ). "); \ r \ n"; fputs ($ fp, $ line) ;}} fclose ($ fp); echo 'The result has been saved '. $ file. '.
  
Cost time :'. $ this-> runtime;}/*** get the header information of the response content ** @ param string $ content * @ return array */function getHead ($ content) {$ head = explode ("\ r \ n", $ content); $ head = $ head [0]; // echo $ head; if (! Preg_match ("~ Charset \ = (. *) \ r \ n ~ I ", $ head, $ preg_rs) preg_match ('~ Charset = ([^ \ "\ '] *) ~ I ', $ content, $ preg_rs); $ this-> httpHead ['charset'] = strtoupper (trim ($ preg_rs [1]); // preg_match ("~ Charset \ = (.*)~ I ", $ head, $ preg_rs); return $ this-> httpHead ;} /*** set the encoding of the collection page * manually call this function before the collection fails to be automatically recognized by the program ** @ param string $ charset */function setCharset ($ charset) {$ this-> charset = strtoupper ($ charset);}/*** set the link page address of the first layer ** @ param array $ url_arr */function setStartUrls ($ url_arr) {$ this-> startUrls = $ url_arr;}/*** add a first-level link page address ** @ param string $ url */function addStartUrl ($ url) {$ this-> startUrls [] = $ url;}/*** add a collection level ** @ par Am integer $ deep * @ param string $ layout * @ param boolean $ isSimple * @ param boolean $ isPageBreak * @ param string $ pattern */function addLayer ($ deep, $ layout, $ pattern = '', $ isSimple = 'false', $ isPageBreak = 'false ') {$ this-> layout_arr [$ deep] = array ('layout '=> $ layout, 'issimple' => $ isSimple, 'ispagebreak' => $ isPageBreak, 'pattern' => $ pattern);}/*** custom head * @ param string $ namespace * @ param string $ value * /Function setHead ($ name, $ value) {$ this-> putHead [$ name] = $ value;}/*** clear html code * @ param string $ content; * @ param string $ cleartags * @ return string */function clearHtml ($ content, $ cleartags = 'p') {$ cleartags_arr = explode ('|', $ cleartags ); foreach ($ cleartags_arr as $ cleartag) {$ pattern = '~ <\/? '. $ Cleartag.' [^>] *> ~ Is '; $ content = preg_replace ($ pattern, '', $ content);} return $ content;}/*** log **/function log ($ str) {echo $ str."
  
\ N ";}/*** get Collection running time ** @ return float */function getRuntime () {return $ this-> runtime;} function microtime_float () {list ($ usec, $ sec) = explode ("", microtime (); return (float) $ usec + (float) $ sec );} function addslash ($ string) {return addslashes ($ string) ;}$ spider = new spider (); $ spider-> addStartUrl ('http: // response }'); $ spider-> setCharset ('gb2312'); $ spider-> addLayer (0, 'list', '/shuntian/blog/item/{* mongo.html '); $ spider-> addField ('title ','
  {Title}', Array (' _ Shun's sky-shooting's Sky', ''); $ spider-> addField ('body ','
  
{Body}
'); $ Spider-> addField ('autor', 'shooting'); $ spider-> run (); $ spider-> saveSql ();
 

The above is the collected content. For more information, see The PHP Chinese website (www.php1.cn )!