自己寫的一個php基於phpQuery的通用採集類

還是小菜，第一次分享代碼哈，這是自己以前寫的一個php的採集類，自己一直在用，自我感覺很簡單很強大，只要懂一點點選取器的知識就可以採集任何頁面了，也支援https頁面，做簡單的採集足夠用了。

<?php  /**  *通用列表採集類  *版本V1.3  *作者:JAE  *部落格:http://blog.jaekj.com  */    require_once '../phpQuery/phpQuery/phpQuery.php';    class QueryList{                private $pageURL;         private $regArr = array();         public $jsonArr = array();         private $regRange;         private $html;         /************************************************         * 參數: 頁面地址 選取器數組 塊選取器         * 【選取器數組】說明：格式array("名稱"=>array("選取器","類型"),.......)         * 【類型】說明：值 "text" ,"html" ,"屬性"          *【塊選取器】：指 先按照規則 選出 幾個大塊 ，然後再分別再在塊裡面 進行相關的選擇         *************************************************/         function QueryList($pageURL,$regArr=array(),$regRange='')         {             $this->pageURL = $pageURL;                 //為了能擷取https://               $ch = curl_init();                curl_setopt($ch, CURLOPT_URL,$this->pageURL);                curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);                 curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false);                curl_setopt($ch,CURLOPT_RETURNTRANSFER,1);                 $this->html = curl_exec($ch);               curl_close($ch);                            if(!empty($regArr))             {                              $this->regArr = $regArr;                 $this->regRange = $regRange;                 $this->getList();             }                        }         function setQuery($regArr,$regRange='')         {             $this->jsonArr=array();             $this->regArr = $regArr;             $this->regRange = $regRange;             $this->getList();         }        private function getList()         {                          $hobj = phpQuery::newDocumentHTML($this->html);             if(!empty($this->regRange))             {             $robj = pq($hobj)->find($this->regRange);                          $i=0;             foreach($robj as $item)             {                                  while(list($key,$reg_value)=each($this->regArr))                 {                     $iobj = pq($item)->find($reg_value[0]);                                           switch($reg_value[1])                       {                           case 'text':                                 $this->jsonArr[$i][$key] = trim(pq($iobj)->text());                                 break;                           case 'html':                                 $this->jsonArr[$i][$key] = trim(pq($iobj)->html());                                 break;                           default:                                $this->jsonArr[$i][$key] = pq($iobj)->attr($reg_value[1]);                                break;                                                   }                 }                 //重設數組指標                 reset($this->regArr);                 $i++;              }             }             else             {            while(list($key,$reg_value)=each($this->regArr))             {                $lobj = pq($hobj)->find($reg_value[0]);                                                         $i=0;                   foreach($lobj as $item)                   {                       switch($reg_value[1])                       {                           case 'text':                                 $this->jsonArr[$i++][$key] = trim(pq($item)->text());                                 break;                           case 'html':                                 $this->jsonArr[$i++][$key] = trim(pq($item)->html());                                 break;                           default:                                $this->jsonArr[$i++][$key] = pq($item)->attr($reg_value[1]);                                break;                                                   }                                                              }                                      }           }         }           function getJSON()         {             return json_encode($this->jsonArr);         }         }

 

 

<?phprequire 'Query/QueryList.class.php';//採集OSC的代碼分享列表，標題 連結 作者$url = "http://www.oschina.net/code/list";$reg = array("title"=>array(".code_title a:eq(0)","text"),"url"=>array(".code_title a:eq(0)","href"),"author"=>array("img","title"));$rang = ".code_list li";$hj = new QueryList($url,$reg,$rang);$arr = $hj->jsonArr;print_r($arr);//如果還想采當前頁面右邊的 TOP40活躍貢獻者 映像，得到JSON資料,可以這樣寫$reg = array("portrait"=>array(".hot_top img","src"));$hj->setQuery($reg);$json = $hj->getJSON();echo $json . "<hr/>";//采OSC內容頁內容$url = "http://www.oschina.net/code/snippet_186288_23816";$reg = array("title"=>array(".QTitle h1","text"),"con"=>array(".Content","html"));$hj = new QueryList($url,$reg);$arr = $hj->jsonArr;print_r($arr);//就舉這麼多例子吧，是不是用來做採集很方便

<?php /**  *自己寫的百度和Google搜尋API  *版本V2.0  *作者:JAE  *部落格:http://blog.jaekj.com  **/require_once 'QueryList_class.php';   class Searcher   {      private $searcher;      private $key;      private $num;      private $page;      private $regArr ;      private $regRange ;      private $regZnum;      public $jsonArr;      //參數 搜尋引擎 搜尋索引鍵 返回的結果條數 第幾頁      function Searcher($searcher,$key,$num,$page)      {          if($searcher=='baidu')          {              $this->regArr = array("title"=>array("h3.t a,#ting_singlesong_box a","text"),"tCon"=>array("div.c-abstract,font:slice(0,2),div#weibo,table tr:eq(0),div.c-abstract-size p:eq(0),div.vd_sitcom_new_tinfo","text"),"url"=>array("h3.t a,#ting_singlesong_box a","href"));              $this->regRange = 'table.result,table.result-op';              $this->regZnum=array("zNum"=>array("span.nums","text"));          }          else if($searcher=='google')          {              $this->regArr = array("title"=>array("h3.r a","text"),"tCon"=>array("span.st","text"),"url"=>array("h3.r a","href"));              $this->regRange = 'li.g';              $this->regZnum=array("zNum"=>array("div#resultStats","text"));          }          $this->searcher = $searcher;          $this->key = $key;          $this->num  = $num;          $this->page = $page-1;          $this->getList();      }      private function getList()      {            $s = urlencode($this->key);            $num = $this->num;            $start = $this->num*$this->page;            if($this->searcher=='baidu')            {                $url = "http://www.baidu.com/s?pn=$start&rn=$num&wd=$s";                 $reg_znum='/[\d,]+/';            }            else if($this->searcher=='google')            {                $url="https://www.google.com.hk/search?filter=0&lr=&newwindow=1&safe=images&hl=en&as_qdr=all&num=$num&start=$start&q=$s";                $reg_znum='/([\d,]+) result(s)?/';            }           $searcherObj = new QueryList($url,$this->regArr,$this->regRange);          for($i=0;$i<count($searcherObj->jsonArr);$i++)          {              if($this->searcher=='baidu')              {                 $searcherObj->jsonArr[$i]['url'] = $this->getBaiduRealURL($searcherObj->jsonArr[$i]['url']);              }              else if($this->searcher=='google')              {                  $searcherObj->jsonArr[$i]['url'] = $this->getGoogleRealURL($searcherObj->jsonArr[$i]['url']);              }          }          $this->jsonArr = $searcherObj->jsonArr ;                    //擷取總共結果條數                   $searcherObj->setQuery($this->regZnum);          $zNum = $searcherObj->jsonArr[0]['zNum'];          preg_match($reg_znum,$zNum,$arr)?$zNum=$arr[0]:$zNum=0;          $zNum = (int)str_replace(',','',$zNum);          //計算總頁數            $zPage = ceil($zNum/$this->num);             $this->jsonArr=array('num'=>$this->num,'page'=>((int)$this->page+1),'zNum'=>$zNum,'zPage'=>$zPage,"s"=>"$this->key",'other'=>array('author'=>'JAE','QQ'=>'734708094','blog'=>'http://blog.jaekj.com'),'data'=>$this->jsonArr);                           }       function getJSON()      {          return json_encode($this->jsonArr);      }    private  function getBaiduRealURL($url)     {         //得到百度跳轉的真正地址        $header = get_headers($url,1);        if (strpos($header[0],'301')  strpos($header[0],'302'))         {            if(is_array($header['Location']))             {                //return $header['Location'][count($header['Location'])-1];                return $header['Location'][0];            }            else            {                return $header['Location'];            }        }        else        {            return $url;        }     }     private function getGoogleRealURL($url)     {          $reg_url = '/q=(.+)&/U';         return  preg_match($reg_url,$url,$arr)?urldecode($arr[1]):$url;               } }// $hj = new Searcher('google','oschina',20,2); // print_r( $hj->jsonArr);//效果示範地址//http://blog.jaekj.com//jae/demo/searcher/Searcher_class.php?searcher=baidu&s=jaekj&num=20&page=1