Python web crawler collection of associative words example _python

Python Crawler _ collection of associative word code

Copy Code code as follows:

#coding: Utf-8
Import Urllib2
Import Urllib
Import re
Import time
From random import choice
#特别提示, the proxy IP in the list below may fail, please switch to a valid proxy IP
IPList = [' 27.24.158.153:81 ', ' 46.209.70.74:8080 ', ' 60.29.255.88:8888 ']

List1 = ["group", "technology"]
for item in List1:
    ip= Choice (iplist)
    GJC = urllib . quote (item)
    URL = http://sug.so.360.cn/suggest/word?callback=suggest_so&encodein= utf-8&encodeout=utf-8&word= "+GJC
    headers = {
                 "get": URL,
                 "Host": "sug.so.360.cn",
                 "Referer": "http://www.so.com/",
                 "user-agent": "smozilla/5.0 (Macintosh; Intel Mac OS X 10_8_4) applewebkit/537.17 (khtml, like Gecko) chrome/24.0.1312.56 safari/537.17 ",
   & nbsp;           }

Proxy_support = Urllib2. Proxyhandler ({' http ': ' http://' +ip})

Opener = Urllib2.build_opener (Proxy_support)
Urllib2.install_opener (opener)
req = Urllib2. Request (URL)

For key in headers:
Req.add_header (Key,headers[key])

html = Urllib2.urlopen (req). Read ()

SS = Re.findall ("\" (. *?) \ "", HTML)
For item in SS:
Print Item
Time.sleep (2)