python多線程抓取網頁資訊

#!/usr/env  python
#-*- coding: utf-8  -*-
import urllib 
import urllib2 
import random 
import requests
import os,sys 
import Queue
import threading
import time
import MySQLdb
from sgmllib import SGMLParser 
import re
queue = Queue.Queue()
out_queue = Queue.Queue()
num=0

class ThreadUrl(threading.Thread):
    
    def __init__(self, queue, out_queue):
        threading.Thread.__init__(self)
        self.queue = queue
        self.out_queue = out_queue

    def run(self):
        while True:
            
host = self.queue.get()
print host
try:
html=requests.get(host)

result=html.content
html.close()
self.out_queue.put(result)

            #place chunk into out queue
except:
print time.sleep(5)

            #signals to queue job is done
self.queue.task_done()

class DatamineThread(threading.Thread):
   
    def __init__(self, out_queue):
        threading.Thread.__init__(self)
        self.out_queue = out_queue

    def run(self):
        while True:
            
result = self.out_queue.get()
pattern=re.compile('<div class="appdiscrib">[\s\S]*?<h4>(.+?)</h4>')
data0=re.findall(pattern,result)

pattern=re.compile('版 本 號(.+?)</li>')
data1=re.findall(pattern,result)
pattern=re.compile('開 發 者(.+?)</li>')
data2=re.findall(pattern,result)
pattern=re.compile('發布時間(.+?)</li>')
data3=re.findall(pattern,result)
pattern=re.compile('檔案大小(.+?)</li>')
data4=re.findall(pattern,result)
pattern=re.compile('支援韌體(.+?)</li>')
data5=re.findall(pattern,result)
pattern=re.compile('應用介紹</h3>[\s\S]*?<div class="intro">([\s\S]*?)</div>')
data6=re.findall(pattern,result)
for items in data6:
pass#print re.sub('<br />',' ',items)
sql="insert into address(name,version,developer,pubtime,filesize,support,introduction) values(%s,%s,%s,%s,%s,%s,%s)"
for items in data6:

if(data5):
values=(data0[0],data1[0],data2[0],data3[0],data4[0],data5[0],re.sub('<br />',' ',items))
else:
values=(data0[0],data1[0],data2[0],data3[0],data4[0],'NULL',re.sub('<br />',' ',items))
#print values
#print sql % values

try:

conn=MySQLdb.connect(host='localhost',user='root',passwd='123456',db='addressbookdb',charset="utf8")
cursor=conn.cursor() 
cursor.execute(sql,values)
conn.commit()
except:
print "error2"


try:
cursor.close()
conn.close()
except:
print "error3"

pattern=re.compile(' <div class="appTitle clearfix">[\s\S]*?<img src=(.+?)/>')
data=re.findall(pattern,result)
for j in data:
print j
global num
      
try:
temp=requests.get(j[1:-2])
f=file("picture/"+str(num),"w+")
num=num+1
print num
f.write(temp.content)
except:
print "error4"
 
           
self.out_queue.task_done()
def main():

for k in range(1,2539):
print k


try:
url="http://apk.gfan.com/apps_7_1_"+str(k)+".html"

html=requests.get(url)

result=html.content
html.close()
pattern=re.compile('<a href="([http://apk.gfan.com]?/Product/App\d{1,8}.html)"')
dataresult=re.findall(pattern,result)
dataresult=list(set(dataresult))


for a in range(20):
w = ThreadUrl(queue, out_queue)
w.setDaemon(True)
w.start()
for i in dataresult:
host="http://apk.gfan.com"+i

queue.put(host)
for a in range(20):
dt = DatamineThread(out_queue)
dt.setDaemon(True)
dt.start()
except:
time.sleep(5)


queue.join()
out_queue.join()





#sql="select * from address"
#cursor.execute(sql)
#conn.commit()
#finalresult=cursor.fetchall()
#if finalresult:
#for x in finalresult:
#pass #print x[0:]


    
if  __name__=="__main__":
       main()