Home > Tutorials > PHP Tutorials

A PHP spider crawler made with curl, regular expressions

Source: Internet
Author: User
Developer on Alibaba Coud: Build your first app with APIs, SDKs, and tutorials on the Alibaba Cloud. Read more ＞
Phoenix Network FCMS Content Management system
Get.php crawl frame, analysis and processing of web content and related substitution
Std.php Universal Regular
news_67_com.php to Http://news.67.com's crawl analyzer
Catch the list first, then grab the content page.
Also lack of monitoring, statistics, error handling functions. Personally think it is more fun.
  2. Include_once dirname (__file__). '/std.php ';
  6. $site = Array (
  8. ' Aname ' = ' China Entertainment Network ',
  10. ' Domain ' = ' news.67.com ',
  12. ' dirname ' \ = ' directory name, used to match the text based on different directories ',
  14. ' GetType ' = ' default ',
  16. Get Master File
  18. ' Creg ' = '/(SI) (. *?) \<\!--article end-->/',
  20. ' Code ' = ' Utf-8 ',
  22. ' Sub ' = ' Get subdirectory regular ',
  24. ' Content ' = ' tag1 ',
  26. ' img_upload ' = = Array (' tag1 ' = ') ',
  28. Next page
  30. ' Reg_next '/' = '/(? is) next page \>\><\/a>/',
  32. ' Key0 ' = '/' (? is)/',
  34. ' Key0_ap ' = = Array (array (', ', ' | '), "),
  35. ' tag0 ' = '/(? is)

    ([^<^>]*?) <\/h1>/',

  37. ' Tag0_arp ' = Array (
  39. Array
  41. '/(? is) \ (Photos \)/',
  43. '/(? is) \ (Figure \)/',
  45. '/(? is) \ (Figure \.\./',
  47. '/(? is) \ (Photos \.\./',
  49. '/(? is) \./',
  51. '/(? is) ("|")/',
  53. ),
  55. Array
  57. '', '', '', '', '', '',
  59. )
  61. ),
  63. ' Tag1 ' = '/(? is) (. *?) /',
  65. ' TAG1_BRP ' = Array (
  67. Array
  69. '/(? is) (. *?)/',
  71. '/(? is) \ (. *?\)/',
  73. '/(? is) \s*

    .*? (.*?) <\/p>\s*/',

  75. '/(? is) \s*

    \s*/',

  77. '/(? is) \s*

    \s*/',

  79. '/(? is)/',
  81. '/(? is)
    /',
  83. '/(? is) \s*

    \s*/',

  85. '/(? is) \s*

    \s*/',

  86. '/(? is) \s* \s*/',
  88. '/(? is) \s*<\/center>\s*/',
  90. '/(? is) \s*

    \s*/',

  92. ),
  94. Array
  96. '', '', '

    ', '

    ', '

    ', '', '', '

    ', '

    ', '

    ', '

    ', '

    '

  98. ),
  100. ),
  102. ' Tag1_arp ' = Array (
  104. Array
  106. '/(? is)

    <\/p>/',

  108. '/(? is)<\/strong>/'
  110. ),
  112. Array
  114. '', ''
  116. ),
  118. ),
  120. ' Strip ' = = Array (' tag1 ' = ') ',
  122. ' Tag2 ' + '/(? is) Guide: \s* (. *?) \s*<\/div>/',
  124. ' Tag2_arp ' = Array (
  126. Array
  128. '/(? is)/'
  130. ),
  132. Array
  134. ''
  136. ),
  138. ),
  140. ' Tag3 ' + '/(? is) (China Entertainment Network)/',
  142. ' Tag4 ' = '/(? is) Date: (\d+-\d+-\d+ \d+:\d+:\d+). *?<\/div>/',
  144. );
  148. $map = Array (
  150. ' Tag ' = ' key0 ',
  152. ' Title ' = ' tag0 ',
  154. ' Content ' = ' tag1 ',
  156. ' Summary ' = ' Tag2 ',
  158. ' Source ' = ' tag3 ',
  160. ' Pub_date ' = ' tag4 ',
  162. );
  166. $site _list = Array (
  168. ' Aname ' = ' China Entertainment Network ',
  170. ' Domain ' = ' www.67.com ',
  172. ' GetType ' = ' default ',
  174. ' Creg ' = '/(SI) (. *?) /',
  176. ' Code ' = ' GBK ',
  178. ' Reg_next ' = '/' (? si)
  180. Next page <\/a><\/li>/',
  182. Link
  184. ' tag0 ' = '/(? is). *? [^<^>]*?<\/a>.*?<\/div>/',
  186. Title
  188. ' Tag1 ' = '/(? is). *? ([^<^>]*?) <\/a>.*?<\/div>/',
  190. ' Tag1_arp ' = Array (
  192. Array
  194. '/(? is) \ (Photos \)/',
  196. '/(? is) \ (Figure \)/',
  198. '/(? is) \ (Figure \.\./',
  200. '/(? is) \ (Photos \.\./',
  202. '/(? is) \./',
  204. '/(? is) ("|")/',
  206. ),
  208. Array
  210. '', '', '', '', '', '',
  212. )
  214. ),
  216. );
  220. $list _map = Array (
  222. ' url ' = ' tag0 ',
  224. ' Title ' = ' Tag1 ',
  226. );
  230. $site _list_sub = Array ();

Copy Code
    1. Global $STD;
    2. $STD = Array (
    3. ' url ' = ' [0-9a-za-z\.\:\-\/%_#;&]+ ',
    4. ' img ' = '/' (? is)/',
    5. );
Copy Code
  2. /**
  4. * test.php
  6. *
  8. * @author xzfred
  10. * @copyright fengone.com
  12. * @created 2010-12-07.
  14. * @version $Id: php.php 3 2008-10-10 07:49:21z Fred $
  16. * Svnpath $HeadURL: http://192.168.0.16/svn/vim/skeletons/php.php $
  18. */
  20. /*
  22. Include_once "std.php";
  24. Include_once "lady_163_com.php";
  26. */
  28. Include_once $GLOBALS [' G_dir_core ']. "Get.php";
  32. //================================================================================
  34. Include_once Dir_host_tag. '/tuku_ent_china_com.php ';
  36. $obj = new Fchtmlparse ($site);
  38. $c = $obj->parse (file_get_contents ("Http://tuku.ent.china.com/fun/html/2011-08-23/181703.xml"));
  40. echo "\n\n\n ===================\n";
  42. echo $c [' field '] [' tag1 '][0];
  44. echo "\n\n\n ===================\n";
  46. Var_dump ($c);
  50. Exit ();
  52. List test
  54. $obj = new Fchtmlparse ($site _list);
  56. $c = $obj->parse (file_get_contents ("http://tuku.ent.china.com/fun/html/3569_1.html"));
  58. Var_dump ($c);
  60. Exit ();
  62. /*
  66. $obj = new Fchtmlget ($site);
  68. $c = $obj->getpage (' http://star.pclady.com.cn/entertainment/ss/1106/703240.html ');
  70. Var_dump ($c);
  74. $obj = new Fchtmlget ($site);
  76. $c = $obj->getpage (' http://star.pclady.com.cn/entertainment/ss/1106/703240.html ');
  78. Var_dump ($c);
  82. $obj = new Fchtmlparse ($site);
  84. $img _obj = new Fchtmlimgupload ($site);
  88. $data = file_get_contents ("e:/b.html");
  90. $c = $obj->parse ($data);
  92. $ic = $img _obj->upload ($c [' tag '] [' Tag1 '][0]);
  94. Var_dump ($ic);
  98. $data = file_get_contents ("e:/a.html");
  100. $c = $obj->parse ($data);
  102. $ic = $img _obj->upload ($c [' tag '] [' Tag1 '][0]);
  104. Var_dump ($ic);
  106. */
  110. Var_dump ($c [' tag '] [' tag1 ']);
Copy Code
  2. Include_once $GLOBALS [' G_dir_core ']. ' host/std.php ';
  6. $site = Array (
  8. ' Aname ' and ' China Network Entertainment Gallery ',
  10. ' Domain ' = ' tuku.ent.china.com ',
  12. ' dirname ' \ = ' directory name, used to match the text based on different directories ',
  14. ' GetType ' = ' default ',
  16. Get Master File
  18. ' Creg ' = '/(SI) ( .*?<\/list>)/',
  20. ' Code ' = ' Utf-8 ',
  22. ' Sub ' = ' Get subdirectory regular ',
  24. ' Content ' = ' tag1 ',
  26. ' img_upload ' = = Array (' tag1 ' = ') ',
  28. Next page
  30. ' Reg_next '/' = '/(? is) next page <\/a>/',
  32. ' Key0 ' = '/' (? is)/',
  34. ' Key0_ap ' = = Array (array (', ', ' | '), "),
  36. ' tag0 ' = '/(? is) title= "([^"]*?) " /',
  38. ' Tag0_arp ' = Array (
  40. Array
  42. '/(? is) \ (Figure \)/',
  44. '/(? is) \ "/',
  46. '/(? is) Exclusive:/',
  48. '/(? is) Exclusive:/',
  50. '/(? is) ("|")/',
  52. ),
  54. Array
  56. '', '', '', '', '',
  58. )
  60. ),
  62. ' Tag1 ' = '/(? is) (. *?) <\/list>/',
  64. ' TAG1_BRP ' = Array (
  66. Array
  68. '/(? is) \s*\s*/'
  70. ),
  72. Array
  74. '

  76. $

  78. $

    '
  80. )
  82. ),
  84. ' Tag1_arp ' = Array (
  86. Array
  88. '/(? is)

    ([^<^>]*?) <\/p>/',

  90. '/(? is) \<br\/\>/',
  92. ),
  94. Array
  96. '

    $

    ',
  98. '',
  100. )
  102. ),
  104. ' Strip ' = = Array (' tag1 ' = ') ',
  106. Network Name
  108. ' Tag3 ' = '/(? is)([^<^>]*?) <\/span>/',
  110. ' Tag4 ' + '/(? is) (China net)/'
  112. );
  116. $map = Array (
  118. ' Tag ' = ' key0 ',
  120. ' Title ' = ' tag0 ',
  122. ' Content ' = ' tag1 ',
  124. ' Author ' = ' Tag3 ',
  126. ' Source ' = ' tag4 '
  128. );
  132. $site _list = Array (
  134. ' Aname ' and ' China Network Entertainment Gallery ',
  136. ' Domain ' = ' tuku.ent.china.com ',
  138. ' GetType ' = ' default ',
  140. ' Creg ' = '/(SI) (. *?) <\/div>/',
  142. ' Code ' = ' Utf-8 ',
  144. ' Reg_next ' = '/' (? si)
  146. Next page <\/a><\/li>/',
  148. Link
  150. ' tag0 ' = '/(? is). *?<\/span>[^<^>]*?<\/a><\/div>/',
  152. ' TAG0_BRP ' = Array (
  154. Array
  156. '/(? is) \.htm/',
  158. ),
  160. Array
  162. '. Xml '
  164. )
  166. ),
  168. Title
  170. ' Tag1 ' = '/(? is).*?<\/span> ([^<^>]*?) <\/a><\/div>/',
  172. ' Tag1_arp ' = Array (
  174. Array
  176. '/(? is) \ (Figure \)/',
  178. '/(? is) \ "/',
  180. '/(? is) Exclusive:/',
  182. '/(? is) Exclusive:/',
  184. '/(? is) ("|")/',
  186. ),
  188. Array
  190. '', '', '', '', '',
  192. )
  194. ),
  196. );
  200. $list _map = Array (
  202. ' url ' = ' tag0 ',
  204. ' Title ' = ' Tag1 ',
  206. );
  210. $site _list_sub = Array ();
Copy Code

    • This article is an English version of an article which is originally in the Chinese language on aliyun.com and is provided for information purposes only. This website makes no representation or warranty of any kind, either expressed or implied, as to the accuracy, completeness ownership or reliability of the article or any translations thereof. If you have any concerns or complaints relating to the article, please send an email, providing a detailed description of the concern or complaint, to info-contact@alibabacloud.com. A staff member will contact you within 5 working days. Once verified, infringing content will be removed immediately.

    Contact Us

    The content source of this page is from Internet, which doesn't represent Alibaba Cloud's opinion; products and services mentioned on that page don't have any relationship with Alibaba Cloud. If the content of the page makes you feel confusing, please write us an email, we will handle the problem within 5 days after receiving your email.

    If you find any instances of plagiarism from the community, please send an email to: info-contact@alibabacloud.com and provide relevant evidence. A staff member will contact you within 5 working days.

    A Free Trial That Lets You Build Big!

    Start building with 50+ products and up to 12 months usage for Elastic Compute Service

    Get Started for Free

    • Sales Support

      1 on 1 presale consultation

      Chat Contact Sales

    • After-Sales Support

      24/7 Technical Support 6 Free Tickets per Quarter Faster Response

      Open a Ticket

    • Alibaba Cloud offers highly flexible support services tailored to meet your exact needs.

      Learn More