Code One:
#!/usr/bin/python #-*-Coding:utf-8-*-#encoding =utf-8 Import threading Import Queue import sys import URLLIB2 Import re import mysqldb # database variable setting # db_host = ' 127.0.0.1 ' db_user = ' XXXX ' db_passwd = ' XXXXXXXX ' db_name = ' xxx X # variable Setting # Thread_limit = 3 jobs = queue.queue (5) Singlelock = Threading. Lock () info = Queue.queue () def WorkerBee (inputlist): for x in Xrange (thread_limit):   ; print ' Thead {0} started. '. Format (x) t = Spider ()
t.start () for i in Inputlist: try: jobs.put (i, Block=True, timeout=5) except:
singlelock.acquire () print "The queue is full!" singlelock.release () # Wait for the threads to finish singlelock.acquire () # Acquire the lock so we can print print ' waiting for threads to finish. '   &NBSP ; singlelock.release () # Release the lock Jobs.join () # This command waits for
All threads to finish. # while not Jobs.empty (): # print Jobs.get () DEF GetTitle (url,time=10): response = Urllib2.urlopen (url,timeout=time) html = Response.read () response.clOSE () reg = R ' <title> (. *?) </title> ' title = Re.compile (reg). FindAll (html) # title = Title [0].decode (' gb2312 ', ' replace '). Encode (' utf-8 ') title = title[0] Return title class Spider (Threading. Thread): def Run (self): while 1: try: job = Jobs.get (true,1) singlelock.acquire () title = GetTitle (job[1]) info.put ([Job[0],title], Block=true,timeout=5) # print ' This {0} is {1} '. Format (job[1],title) singlelock.release () jobs.task_done () except:
break; if __name__ = = ' __main__ ': con = None urls = [] try: con = MySQLdb.connect (DB_HOST,DB_USER,DB_PASSWD,DB_ NAME) cur = Con.cursor () cur.execute (' SELECT id,url from ' table_name ' WHERE ' status ' =0 LIMIT ') &nBsp; rows = Cur.fetchall () for Row in rows: # print row urls.append ([row[0],row[1]])
workerbee (URLs) while not Info.empty (): print Info.get () finally: if Con: con.close ()
Code two:
#!/usr/bin/python #-*-Coding:utf-8-*-#encoding =utf-8 #Filename: robot.py Import Threading,queue,sys,urllib2,re # Variable Setting # Thread_limit = 3 #设置线程数 jobs = Queue.queue (5) #设置队列长度 Singlelock = Threading. Lock () #设置一个线程锁, avoid repeated calls to URLs = [' http://games.sina.com.cn/w/n/2013-04-28/1634703505.shtml ', ' http:// Games.sina.com.cn/w/n/2013-04-28/1246703487.shtml ', ' http://games.sina.com.cn/w/n/2013-04-28/1028703471.shtml ', ' http://games.sina.com.cn/w/n/2013-04-27/1015703426.shtml ', ' http://games.sina.com.cn/w/n/2013-04-26/ 1554703373.shtml ', ' http://games.sina.com.cn/w/n/2013-04-26/1512703346.shtml ', ' http://games.sina.com.cn/w/n/ 2013-04-26/1453703334.shtml ', ' http://games.sina.com.cn/w/n/2013-04-26/1451703333.shtml ', ' http:// Games.sina.com.cn/w/n/2013-04-26/1445703329.shtml ', ' http://games.sina.com.cn/w/n/2013-04-26/1434703322.shtml ', ' http://games.sina.com.cn/w/n/2013-04-26/1433703321.shtml ', ' http://games.sina.com.cn/w/n/2013-04-26/ 1433703320.shtml ', ' http://games.sina.com.cn/w/n/2013-04-26/1429703318.shtml ', ' http://games.sina.com.cn/w/n/2013-04-26/1429703317.shtml ', ' http://games.sina.com.cn/w/n/ 2013-04-26/1409703297.shtml ', ' http://games.sina.com.cn/w/n/2013-04-26/1406703296.shtml ', ' http:// Games.sina.com.cn/w/n/2013-04-26/1402703292.shtml ', ' http://games.sina.com.cn/w/n/2013-04-26/1353703286.shtml ', ' http://games.sina.com.cn/w/n/2013-04-26/1348703284.shtml ', ' http://games.sina.com.cn/w/n/2013-04-26/ 1327703275.shtml ', ' http://games.sina.com.cn/w/n/2013-04-26/1239703265.shtml ', ' http://games.sina.com.cn/w/n/ 2013-04-26/1238703264.shtml ', ' http://games.sina.com.cn/w/n/2013-04-26/1231703262.shtml ', ' http:// Games.sina.com.cn/w/n/2013-04-26/1229703261.shtml ', ' http://games.sina.com.cn/w/n/2013-04-26/1228703260.shtml ', ' http://games.sina.com.cn/w/n/2013-04-26/1223703259.shtml ', ' http://games.sina.com.cn/w/n/2013-04-26/ 1218703258.shtml ', ' http://games.sina.com.cn/w/n/2013-04-26/1202703254.shtml ', ' http://games.sina.com.cn/w/n/ 2013-04-26/1159703251.shtml ', ' http://games.sina.com.cn/w/n/2013-04-26/1139703233.shtml '] def workerbee (inputlist): For x in Xrange (thread_limit): print ' Thead {0} started. '. Format (x) T = Spider () t.start () for I in Inputlist:try:jobs.put (i, Block=true, timeout=5) excep
T:singlelock.acquire () print "The queue is full!" Singlelock.release () # Wait for the ' threads to finish Singlelock.acquire () # Acquire ' lock so we can print P
Rint "Waiting for threads to finish."
Singlelock.release () # Release of the Lock Jobs.join () # This command waits for all threads to finish. # While not Jobs.empty (): # Print Jobs.get () def getTitle (url,time=10): Response = Urllib2.urlopen (url,timeout=time html = Response.read () response.close () Reg = R ' <title> (. *?) </title> ' title = Re.compile (reg). FindAll (HTML) title = Title[0].decode (' gb2312 ', ' replace '). Encode (' Utf-8 ') re Turn title class Spider (Threading. Thread): def run (self): while 1:try:job= Jobs.get (true,1) singlelock.acquire () title = GetTitle (Job) print ' This {0} is {1} '. Format (job,t
Itle) Singlelock.release () Jobs.task_done () Except:break; if __name__ = = ' __main__ ': WorkerBee (URLs)