利用curl，Regex做的一個php蜘蛛抓取器

鳳網fcms內容管理系統 get.php 抓取架構，對網頁內容的分析處理並進行相關替換 std.php 通用正則 news_67_com.php 對http://news.67.com 的抓取分析器 先抓列表，再抓內容頁。 還欠缺監控，統計，錯誤處理功能。個人覺得還是比較好玩。 include_once dirname(__FILE__) . '/std.php'; $site = array( 'aname' => '中國娛樂網', 'domain' => 'news.67.com', 'dirname' => '目錄名稱，用於匹配基於目錄不同的本文', 'gettype' => 'default', //擷取主檔案 'creg' => '/(?si)(.*?)\<\!--文章 end-->/', 'code' => 'utf-8', 'sub' => '擷取子目錄正則', 'content' => 'tag1', 'img_upload'=> array('tag1' => ''), //下一頁 'reg_next' => '/(?is)下一頁\>\><\/a>/', 'key0' => '/(?is)/', 'key0_ap' => array(array(',', '|'), ' '), 'tag0' => '/(?is) ([^<^>]*?)<\/h1>/', 'tag0_arp'=> array( array( '/(?is)\(組圖\)/', '/(?is)\(圖\)/', '/(?is)\(圖\.\./', '/(?is)\(組圖\.\./', '/(?is)\./', '/(?is)(《|》)/', ), array( '', '', '', '', '', '', ) ), 'tag1' => '/(?is)(.*?)/', 'tag1_brp'=> array( array( '/(?is)（.*?）/', '/(?is)\(.*?\)/', '/(?is)\s* .*?(.*?)<\/p>\s*/', '/(?is)\s* \s*/', '/(?is)\s* \s*/', '/(?is)　/', '/(?is) /', '/(?is)\s* \s*/', '/(?is)\s* \s*/', '/(?is)\s*\s*/', '/(?is)\s*<\/center>\s*/', '/(?is)\s* \s*/', ), array( '', '', ' ', ' ', ' ', '', '', ' ', ' ', ' ', ' ', ' ' ), ), 'tag1_arp'=> array( array( '/(?is) <\/p>/', '/(?is)<\/strong>/' ), array( '', '' ), ), 'strip' => array('tag1' => ''), 'tag2' => '/(?is)導讀：\s*(.*?)\s*<\/div>/', 'tag2_arp'=> array( array( '/(?is)　/' ), array( '' ), ), 'tag3' => '/(?is)(中國娛樂網)/', 'tag4' => '/(?is)日期：(\d+-\d+-\d+ \d+:\d+:\d+).*?<\/div>/', ); $map = array( 'tag' => 'key0', 'title' => 'tag0', 'content' => 'tag1', 'summary'=> 'tag2', 'source' => 'tag3', 'pub_date' => 'tag4', ); $site_list = array( 'aname' => '中國娛樂網', 'domain' => 'www.67.com', 'gettype' => 'default', 'creg' => '/(?si)(.*?)/', 'code' => 'gbk', 'reg_next' => '/(?si) 下一頁<\/a><\/li>/', //連結 'tag0' => '/(?is).*?[^<^>]*?<\/a>.*?<\/div>/', //標題 'tag1' => '/(?is).*?([^<^>]*?)<\/a>.*?<\/div>/', 'tag1_arp'=> array( array( '/(?is)\(組圖\)/', '/(?is)\(圖\)/', '/(?is)\(圖\.\./', '/(?is)\(組圖\.\./', '/(?is)\./', '/(?is)(《|》)/', ), array( '', '', '', '', '', '', ) ), ); $list_map = array( 'url' => 'tag0', 'title' => 'tag1', ); $site_list_sub = array(); 複製代碼 global $std; $std = array( 'url' => '[0-9a-zA-Z\.\:\-\/%_#;&]+', 'img' => '/(?is)/', ); 複製代碼 /** * test.php * * @author xzfred * @copyright 2009 fengone.com * @created 2010-12-07 . * @version $Id: php.php 3 2008-10-10 07:49:21Z fred $ * SVNPath $HeadURL: http://192.168.0.16/svn/vim/skeletons/php.php $ */ /* include_once "std.php"; include_once "lady_163_com.php"; */ include_once $GLOBALS['g_dir_core'] . "get.php"; //================================================================================ include_once DIR_HOST_TAG . '/tuku_ent_china_com.php'; $obj = new FcHtmlParse($site); $c = $obj->parse(file_get_contents("http://tuku.ent.china.com/fun/html/2011-08-23/181703.xml")); echo "\n\n\n ===================\n"; echo $c['field']['tag1'][0]; echo "\n\n\n ===================\n"; var_dump($c); exit(); //列表測試 $obj = new FcHtmlParse($site_list); $c = $obj->parse(file_get_contents("http://tuku.ent.china.com/fun/html/3569_1.html")); var_dump($c); exit(); /* $obj = new FcHtmlGet($site); $c = $obj->getPage('http://star.pclady.com.cn/entertainment/ss/1106/703240.html'); var_dump($c); $obj = new FcHtmlGet($site); $c = $obj->getPage('http://star.pclady.com.cn/entertainment/ss/1106/703240.html'); var_dump($c); $obj = new FcHtmlParse($site); $img_obj = new FcHtmlImgUpload($site); $data = file_get_contents("e:/b.html"); $c = $obj->parse($data); $ic = $img_obj->upload($c['tag']['tag1'][0]); var_dump($ic); $data = file_get_contents("e:/a.html"); $c = $obj->parse($data); $ic = $img_obj->upload($c['tag']['tag1'][0]); var_dump($ic); */ //var_dump($c['tag']['tag1']); 複製代碼 include_once $GLOBALS['g_dir_core'] . 'host/std.php'; $site = array( 'aname' => '中華網娛樂圖庫', 'domain' => 'tuku.ent.china.com', 'dirname' => '目錄名稱，用於匹配基於目錄不同的本文', 'gettype' => 'default', //擷取主檔案 'creg' => '/(?si)(.*?<\/list>)/', 'code' => 'utf-8', 'sub' => '擷取子目錄正則', 'content' => 'tag1', 'img_upload'=> array('tag1' => ''), //下一頁 'reg_next' => '/(?is)下一頁<\/a>/', 'key0' => '/(?is)/', 'key0_ap' => array(array(',', '|'), ' '), 'tag0' => '/(?is)title="([^"]*?)"/', 'tag0_arp' => array( array( '/(?is)\(圖\)/', '/(?is)\"/', '/(?is)獨家：/', '/(?is)獨家:/', '/(?is)(《|》)/', ), array( '', '', '', '', '', ) ), 'tag1' => '/(?is)(.*?)<\/list>/', 'tag1_brp'=> array( array( '/(?is)\s*\s*/' ), array( ' $1 $3 ' ) ), 'tag1_arp'=> array( array( '/(?is) ([^<^>]*?)<\/p>/', '/(?is)\<br\/\>/', ), array( ' $1 ', '', ) ), 'strip' => array('tag1' => ''), //網名 'tag3' => '/(?is)([^<^>]*?)<\/span>/', 'tag4' => '/(?is)(中華網)/' ); $map = array( 'tag' => 'key0', 'title' => 'tag0', 'content' => 'tag1', 'author' => 'tag3', 'source' => 'tag4' ); $site_list = array( 'aname' => '中華網娛樂圖庫', 'domain' => 'tuku.ent.china.com', 'gettype' => 'default', 'creg' => '/(?si)(.*?)<\/div>/', 'code' => 'utf-8', 'reg_next' => '/(?si) 下一頁<\/a><\/li>/', //連結 'tag0' => '/(?is).*?<\/span>[^<^>]*?<\/a><\/div>/', 'tag0_brp'=> array( array( '/(?is)\.htm/', ), array( '.xml' ) ), //標題 'tag1' => '/(?is).*?<\/span>([^<^>]*?)<\/a><\/div>/', 'tag1_arp' => array( array( '/(?is)\(圖\)/', '/(?is)\"/', '/(?is)獨家：/', '/(?is)獨家:/', '/(?is)(《|》)/', ), array( '', '', '', '', '', ) ), ); $list_map = array( 'url' => 'tag0', 'title' => 'tag1', ); $site_list_sub = array(); 複製代碼

