PHP Single Thread crawler

Code:

/*** @desc: Single thread crawler * @author [Lee] <[<[email protected]>]>* @property * 1, callcontent Gets the callback function for the contents of the given URL page * 2, Calltodo processing business logic callback function such as: the content of the crawled into the database * @method * Run crawler * @param depth depth default @return vo  Id*/class crawl{public $callcontent = ' getcontent ';  # Gets the callback function for the contents of the given URL page public $calltodo = ' Todo ';  # Callback functions for processing business logic such as: To store the crawled content into the database private $url;    # Internal properties: url/* @desc in the current process: an internal method that invokes the callback function to get the page content @param URL passed into the callback function's parameters @return RET page content */private function getcontent ($url) {    $callback = $this->callcontent;    $ret = Call_user_func ($callback, $url); return $ret;}     /* @desc: Internal method, call callback function for business processing @param content passed to callback function parameter */private function todo ($content) {$callback = $this->calltodo; Call_user_func ($callback, $content);} /* @desc: Internal method, get hyperlinks in page @param content page contents @return URLs gets the hyperlink */private function Geturl ($content) {$preg = '/<[a|    A].*?href=[\ ' \ "]{0,1} ([^>\ ' \" \]*). *?>/';    $bool = Preg_match_all ($preg, $content, $res);    $urls = Array (); if ($bool) {        $urls = $res [1];    } $urls = Array_unique ($urls); return $urls;} /* @desc: Internal method, fix incomplete URL @param url original url @param url fix good URL */private function Reviseurl ($url) {$info = Parse_url ($url)    ;    $scheme = $info ["scheme"]?: ' http ';    $user = $info ["User"];    $pass = $info ["Pass"];    $host = $info ["Host"];    $port = $info ["Port"];    $path = $info ["Path"]; $url = $scheme.    ‘://‘; if ($user && $pass) {$url. = $user. ":" . $pass.    "@";    } $url. = $host; if ($port) {$url. = ":".    $port;    } $url. = $path; return $url;} /* @desc: Construct method, initialize URL */public function __construct ($url) {$this->url = $url;}    /* @desc: Main method, execute program @param depth Mining depth default 2 */public function run ($depth = 2) {$url = $this->url;        if ($depth > 0) {$depth--;        $content = $this->getcontent ($url);        Business processing started $this->todo ($content);        End of Business processing $urls = $this->geturl ($content);  $url = $this->reviseurl ($url);      if (Is_array ($urls) &&!empty ($urls)) {foreach ($urls as $u) {if (Preg_match ('/^                http/', $u)) {$returl = $u; } else {$real = $url. ‘/‘ .                    $u;                $returl = $real;                } $crawl = new crawl ($returl);            $crawl->run ($depth); }        }    }}}

Test:

$crawl = new crawl(‘www.baidu.com‘);$crawl->run();/*@desc：获取内容的回调*/function getcontent($url){$http = new http($url);$ret = $http->get()->run();return $ret;}/*@desc：处理业务逻辑的回调*/function todo($content){echo ‘执行内容‘.PHP_EOL;}

Output:

执行内容执行内容执行内容执行内容执行内容执行内容执行内容执行内容执行内容执行内容执行内容执行内容执行内容执行内容执行内容执行内容执行内容执行内容执行内容执行内容执行内容执行内容执行内容执行内容

PHP Single Thread crawler