A small crawler implemented by python that has been continuously searched by Baidu

I learned python on the third day. I want to write something to commemorate it. so I wrote crawler for a long time, but it is not a good worm. I can only talk about the keywords of the web page and store them locally, however, I think it is basically a bug that uses the BeautifulSoup library in the text to process html document analysis, because I only extracted the keyword of the title, therefore, we can use regular expressions instead. another library is jieba, which is used for Chinese word segmentation. another library is chardet, which is used to determine character encoding. We wanted to use multiple threads, but I thought I was confused and gave up.

The code is as follows:

# Coding: UTF-8
Import re
Import urllib
Import urllib2
Import sys
Import time
Import Queue
Import thread
Import threading
Import jieba
Import chardet
From BeautifulSoup import BeautifulSoup as BS

DEEP = 1000
LOCK = threading. Lock ()
PATH = "c: \ test \\"
UrlQueue = Queue. Queue ()
Def pachong ():
Url = 'http: // www.baidu.com'
Return url

Def getPageUrl (html ):
ReUrl = re. compile (r' <\ s * [Aa] {1} \ s + [^>] *? [Hh] [Rr] [Ee] [Ff] \ s * = \ s * [\ "\ ']? ([^> \ "\ '] +) [\" \']?. *?> ')
Urls = reUrl. findall (html)
For url in urls:
If len (url)> 10:
If url. find ('javascript ') =-1:
UrlQueue. put (url)

Def getContents (url ):
Try:
Url = urllib2.quote (url. split ('#') [0]. encode ('utf-8'), safe = "%/: = &?~ # +! $,; '@ () * [] ")
Req = urllib2.urlopen (url)
Res = req. read ()
Code = chardet. detect (res) ['encoding']
# Print
# Print code
Res = res. decode (str (code), 'ignore ')
Res = res. encode ('gb2312', 'ignore ')
Code = chardet. detect (res) ['encoding']
# Print code
# Print res
Return res
Failed T urllib2.HTTPError, e:
Print e. code
Return None
Failed T urllib2.URLError, e:
Print str (e)
Return None

Def writeToFile (html, url ):
Fp = file (PATH + str (time. time () + '.html ', 'w ')
Fp. write (html)
Fp. close ()


Def getKeyWords (html ):
Code = chardet. detect (html) ['encoding']
If code = 'ISO-8859-2 ':
Html. decode ('gbk', 'ignore'). encode ('gb2312', 'ignore ')
Code = chardet. detect (html) ['encoding']
Soup = BS (html, fromEncoding = "gb2312 ")
TitleTag = soup. title
TitleKeyWords = titleTag. contents [0]
CutWords (titleKeyWords)

Def cutWords (contents ):
Print contents
Res = jieba. cut_for_search (contents)
Res = '\ n'. join (res)
Print res
Res = res. encode ('gb2312 ')
KeyWords = file (PATH + 'cutKeyWors.txt ', 'A ')
KeyWords. write (res)
KeyWords. close ()

Def start ():

While urlQueue. empty () = False:
Url = urlQueue. get ()
Html = getContents (url)
GetPageUrl (html)
GetKeyWords (html)
# WriteToFile (html, url)

If _ name _ = '_ main __':
StartUrl = pachong ()
UrlQueue. put (startUrl)
Start ()