Hehe, the first two sections don't seem to have much to do with Python. This section is all about code,
This is my first time to write Python, a lot of places are messy, mainly to see the logical process.
I've got a big head for the coded format. Remove the page do not know what the encoding, so first find CharSet, and then to Unicode. Unified under Unicode operation, but the database is UTF8, Windows console must be GBK, but I IDE console must be UTF8. So there will be debug this variable exists ... The main purpose is to control the output encoding.
This program has run for 24 hours, then distributed on 10 machines deployed on a long period of time is basically no problem.
After that, you will crawl the Web page 100,000 times a day.
The source code is as follows:
Content crawling and tools
[Python] View plain copy ' ' created on 2010-9-15 @author: chenggong ' import urllib2 import re import socket debug = 0 ' ' tool class ' class Tools (): #log函数 @staticmethod def writelog (level,info,notify=false): if DEBUG == 0: try: print "[" +level+ "]" +info.decode (' UTF-8 '). Encode (' GBK ') except: print "[" +level+ "]" + Info.encode (' GBK ') else: print "[" +level+ "]" +info #if notify: # print "[notify] report Administrator!!" #转unicode @staticmethod def tounicode ( S,charset): if ( charset == "" ) : return s else: try: u = unicode ( s, charset ) except: u = "" return u #正则抓取 # @param single Whether to crawl only one @staticmethod def getfrompatten (patten,src,single=false): rst = ""; p = re.compile (PATten,re. S) all = p.findall (SRC) for matcher in all: rst += matcher + " " if ( single ): break return rst.strip () "" ' Web content crawler ' class pagegripper (): url_open_ timeout = 10 #网页超时时间 MAX_RETRY = 3 #最大重试次数 def __init__ (self): socket.setdefaulttimeout (self. url_open_timeout) #获取字符集 def getcharset (self,s): rst = tools.getfrompatten (U ' charset= (. *?) "', s,true) if rst != "": if rst == "UTF8": rst = "Utf-8" return rst #尝试获取页面 def downloadurl (self,url): charset = "" page = "" retry = 0 while true: try: fp = urllib2.urlopen (URL) break except urllib2. httperror,e: #状态错误 tools.writelog (' error ', ' HTTP status error code= ' +e.code) raise urllib2. HTtperror             EXCEPT URLLIB2 . urlerror,e: #网络错误超时 tools.writelog (' Warn ', ' page access timeout, retry ... ') retry+=1 if ( retry > self. max_retry ): tools.writelog (' warn ', ' exceeding maximum retries, giving up ') raise urllib2. urlerror while true: line = Fp.readline () if charset == "": charset = self.getcharset (line) if not line: break page += tools.tounicode (Line,charset) fp.close () return page #获取页面 d ef getpageinfo (Self,url): tools.writelog ( "Info", "Start crawling Web page,url= " +url) info = "" try: info = self.downloadurl (URL) except: raise tools.writelog ("Debug", "Web Crawl Success") return info ' Content extraction class ' ' class infogripper (): pagegripper = pagegripper () &nbSp def __init__ (self): Tools.writelog (' Debug ', "Crawler launch") #抓取标题 def griptitle (self,data): title = tools.getfrompatten (U ' box2t sp ">