Original Python crawler Code
Mainly used in the URLLIB2, BeautifulSoup module
#Encoding=utf-8ImportReImportRequestsImportUrllib2ImportdatetimeImportMySQLdb fromBs4ImportBeautifulSoupImportsysreload (SYS) sys.setdefaultencoding ("Utf-8")classSplider (object):def __init__(self):PrintU'start crawling content ...' ##用来获取网页源代码 defGetSource (Self,url): Headers= {'user-agent':'mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_3) applewebkit/537.36 (khtml, like Gecko) chrome/50.0.2652.0 safari/537.36'} req= Urllib2. Request (url=url,headers=headers) Socket=Urllib2.urlopen (req) content=Socket.read () socket.close ( )returncontent##changepage用来生产不同页数的链接 defchangepage (self,url,total_page): Now_page= Int (Re.search ('page/(\d+)', Url,re. S). Group (1)) Page_group= [] forIinchRange (now_page,total_page+1): Link= Re.sub ('page/(\d+)','page/%d'%I,url,re. S) page_group.append (link)returnPage_group#Get Word content defGetchildrencon (self,child_url): Conobj={} content=Self.getsource (child_url) Soup= BeautifulSoup (Content,'Html.parser', from_encoding='Utf-8') Content= Soup.find ('Div',{'class':'c-article_content'}) img= Re.findall ('src= "(. *?)"', str (content), re. S) conobj['Con'] =content.get_text () conobj['img'] = (';'). Join (IMG)returnConobj##获取内容 defgetcontent (self,html_doc): Soup= BeautifulSoup (Html_doc,'Html.parser', from_encoding='Utf-8') Tag= Soup.find_all ('Div',{'class':'Promo-feed-headline'}) Info={} i=0 forLinkinchTag:info[i]={} Title_desc= Link.find ('H3') info[i]['title'] =title_desc.get_text () post_date= Link.find ('Div',{'class':'post-date'}) Pos_d= post_date['data-date'][0:10] info[i]['Content_time'] =pos_d info[i]['Source'] ='WhoWhatWear'Source_link= Link.find ('a', Href=re.compile (R"Section=fashion-trends")) Source_url='http://www.whowhatwear.com'+source_link['href'] info[i]['Source_url'] =Source_url in_content=Self.getsource (source_url) In_soup= BeautifulSoup (In_content,'Html.parser', from_encoding='Utf-8') Soup_content= In_soup.find (' Section',{'class':'widgets-list-content'}) info[i]['content'] = Soup_content.get_text (). Strip ('\ n') Text_con= In_soup.find (' Section',{'class':'text'}) Summary= Text_con.get_text (). Strip ('\ n')ifText_con.text! = NoneElseNULL info[i]['Summary'] = summary[0:200]+'...'; Img_list= Re.findall ('src= "(. *?)"', str (soup_content), re. S) info[i]['IMGs'] = (';'). Join (img_list) info[i]['Create_time'] = Datetime.datetime.now (). Strftime ("%y-%m-%d%h:%m:%s") I+=1#Print Info #exit () returnInfodefSaveinfo (self,content_info): Conn= MySQLdb.connect (host='127.0.0.1', user='Root', passwd='123456', port=3306,db='Test', charset='UTF8') Cursor=conn.cursor () foreachinchContent_info: forKvincheach.items (): SQL="INSERT INTO T_fashion_spider2 (' title ', ' summary ', ' content ', ' content_time ', ' IMGs ', ' source ', ' source_url ', ' Create_time ') VALUES ('%s ', '%s ', '%s ', '%s ', '%s ', '%s ', '%s ', '%s ')"% (Mysqldb.escape_string (v['title']), mysqldb.escape_string (v['Summary']), mysqldb.escape_string (v['content']), v['Content_time'],v['IMGs'],v['Source'],v['Source_url'],v['Create_time']) cursor.execute (SQL) Conn.commit () Cursor.close () conn.close ()if __name__=='__main__': ClassInfo=[] P_num= 5URL='HTTP://WWW.WHOWHATWEAR.COM/SECTION/FASHION-TRENDS/PAGE/1'Jikesplider=Splider () all_links=jikesplider.changepage (url,p_num) forLinkinchall_links:PrintU'Working with page:'+Link HTML=jikesplider.getsource (link)info =jikesplider.getcontent (HTML) classinfo.append (info) jikesplider.saveinfo (ClassInfo)
Python crawler Code