Crawl 58 of the same City's second-hand trading data with Python crawlers

Crawled 14W data, deposited in mongodb, with the charts Library to display statistical results, here to show a schematic

Module 1 get the category URL list

BeautifulSouprequests,pymongomain_url = ' http://bj.58.com/sale.shtml ' client = Pymongo. Mongoclient (' localhost ', 27017) tc_58 = client[' 58tc ']tab_link_list = tc_58[' link_list ']web_data = Requests.get (main_ Url) soup = BeautifulSoup (web_data.text, ' lxml ') sub_menu_link = soup.select (' ul.ym-submnu > li > B > A ') link_list = []count = 0sub_menu_link:    link = ' http://bj.58.com ' + link.get (' href ')    #print (link)     link = = ' http://bj.58.com/shoujihao/':        pass    link = = ' http://bj.58.com/ Tongxunyw/':        Pass    link = = ' http://bj.58.com/tiaozao/':        count + = 1         count = = 1:            data = {' link ': link}            link_list.append (data)    Else:        data = {' link ': link}        link_list.append (data)link_list:    tab_link_list.insert (i)

Module 2 Get each item detail information

 fromBs4ImportBeautifulSoupImportRequests,re,pymongo,sys fromMultiprocessingImportPoolclient = Pymongo. Mongoclient (' localhost ', 27017) tc_58 = client[' 58tc ']# detail_link = tc_58[' Detail_link ']tab_link_list = tc_58[' link_ List ']# tc_58_data = client[' 58tcData 'defGetdetailurl (Page_url,Tab): url_list = [] Web_data = Requests.get (Page_url) soup = BeautifulSoup (web_data.text, ' lxml ') detail_url = soup.select (' div.infocon > Table > Tbody > TR ; td.t > a[onclick] ') #获取详细页面url forUrlinchDetail_url:url_list.append (url.get (' href '). split ('? ') [0]) #插入mongodb count = 0 client = Pymongo. Mongoclient (' localhost ', 27017) tc_58 = client[' 58tc '] tab_list = tc_58[Tab+ ' _list '] forIinchUrl_list:count + = 1 tab_list.insert ({' link ': i})returnCountoriginal_price_patt = Re.compile (' original price: (. +) ')defGetInfo (Detail_url):Try: Web_data = Requests.get (Detail_url) soup = BeautifulSoup (web_data.text, ' lxml ') title = Soup.title.text.strip () View_count = Soup.select (' body > Div.content > div > Div.box_left > Div.info_lubotu.clearfix > Div.box_left_top > P > Span.lo Ok_time ') [0].text want_count = soup.select (' body > div.content > div > Div.box_left > div.info_lubotu.c Learfix > Div.box_left_top > P > Span.want_person ') [0].text current_price = soup.select (' body > Div.con Tent > div > Div.box_left > Div.info_lubotu.clearfix > Div.info_massege.left > Div.price_li > Span > I ') Current_price = Current_price[0].textifCurrent_priceElse NoneOriginal_price = Soup.select (' body > div.content > div > Div.box_left > Div.info_lubotu.clearfix > Div.info _massege.left > Div.price_li > span > b ') original_price = Original_price[0].textifOriginal_priceElse NoneOriginal_price = Re.findall (original_price_patt,original_price)ifOriginal_priceElse NoneLocation = Soup.select (' body > div.content > div > Div.box_left > Div.info_lubotu.clearfix > Div.info_masse Ge.left > Div.palce_li > Span > I ') [0].text tag = soup.select (' body > div.content > div > Div.box _left > Div.info_lubotu.clearfix > Div.info_massege.left > Div.biaoqian_li ') tag = List (tag[0].stripped_st RingsifTagElse NoneSeller_name = Soup.select (' body > div.content > div > Div.box_right > Div.personal.jieshao_div > Div.person Al_jieshao > P.personal_name ') [0].text # level = Soup.select (' body > div.content > div > Div.box_right        > Div.personal.jieshao_div > Div.personal_jieshao > Span ') # level = str (level[0]). split (' \ n ') # # full_count = 0 # half_count = 0 # for J at Level: # if ' <span class= ' icon_png ' >< /SPAN> ' = = J: # Full_count + = 1 # elif ' <span class= "icon_png smallscore" ></span> ' = = J: # Half_count + = 1 Full_count = Len (soup.find_all (' span ', class_= ' icon_png ')) half_co        UNT = Len (soup.find_all (' span ', class_= ' icon_png smallscore ')) level_count = {' full ': full_count, ' half ': half_count}  desc = soup.select (' body > div.content > div > Div.box_left > Div:nth-of-type (3) > div > div > P ') desc = desc[0].text  ifDescElse Nonedata = {' title ': title, ' view_count ': view_count, ' want_count ': want_count, ' cur Rent_price ': current_price, ' original_price ': original_price, ' location ': location, ' tag ': ta g, ' seller_name ': seller_name, # ' level ': level, ' level_count ': level_count, ' desc ':d esc, ' link ':Detail_url}returnDataexcept: Print (sys.exc_info () [0], sys.exc_info () [1])return None# for I in Tab_link_list.find ({},{' link ': 1, ' _id ': 0}): # print (i[' link ') # getdetailurl (i[' link ')) #规律每个页面最多70页defInsertdetaillin (sub_menu_list): Patt = re.compile ('. +?com/([a-z]+)/') tab_list = [] forIinchsub_menu_list. Find ({},{' link ': 1, ' _id ': 0}): #for i in [{' link ': ' http://bj.58.com/shouji/'}]: i = i[' link '] sub_menu_nam E = Re.findall (patt,i) [0] print (sub_menu_name+ ': ', end= ') url_list = [] forJinchRange (1,71): link = i + ' pn ' + str (j) url_list.append (link) cnt = 0 forKinchurl_list:cnt = cnt + getdetailurl (k, sub_menu_name) print (str (cnt) + ' lines inserted ')ifcnt! = 0:tab_list.append (sub_menu_name+ ' _list ')returntab_list# for i in Tab_link_list.find ({},{' link ': 1, ' _id ': 0}): # print (i) #insertDetailLin (tab_link_list) Allmencollectionname = tc_58.collection_names () #allMenCollectionName. Remove (' detail_link ') Allmencollectionname.remove (' link_list ')defInsertData (Tab_name): client = Pymongo. Mongoclient (' localhost ', 27017) tc_58 = client[' 58tc '] tc_58_data = client[' 58tcDataNew '] fenlei =Tab_name[: -5] fenlei = tc_58_data[fenlei+ ' _data ') tab_name = tc_58[Tab_name] #print (tab_name) forIinchTab_name. Find ({},{' link ': 1, ' _id ': 0}): data = getInfo (i[' link ')) fenlei.insert (data)defgetcontinuingly (Fenlei): client = Pymongo. Mongoclient (' localhost ', 27017) tc_58_data = client[' 58tcDataNew '] tc_58 = client[' 58tc '] fenlei_data = Tc_58_data [Fenlei+ ' _data '] fenlei_list = tc_58[Fenlei+ ' _list '] db_urls = [item[' link '] forIteminchFenlei_data.find ()] index_url = [item[' link '] forIteminchFenlei_list.find ()] x=set (db_urls) y=set (index_url) rest_of_urls = y-xreturnList (rest_of_urls)defstartgetcontinuingly (Fenlei): client = Pymongo. Mongoclient (' localhost ', 27017) tc_58_data = client[' 58tcDataNew '] fenlei = tc_58_data[Fenlei+ ' _data '] #rest_of_urls = getcontinuingly (' chuang ') rest_of_urls = getcontinuingly (Fenlei) #print (rest_of_urls) forIinchRest_of_urls:data = GetInfo (i) fenlei.insert (data) # startgetcontinuingly (' bijiben ') pool = Pool () pool.map (in Sertdata,allmencollectionname) #pool. map (insertdata,[' chuang_list ') #insertData (allmencollectionname)

Module 3 Analysis

 fromCollectionsImportCounterImportPymongo,chartsdefGettotalcount (Database,Host=None,Port=None): client = Pymongo. Mongoclient (Host,Port) db = client[Database] Tab_list = db.collection_names () #print (tab_list) count = 0 forIinchTab_list:count = Count + db[i].find ({}). count () print (count)returnCount#gettotalcount (' 58tcDataNew ') #14700defGetareabyclassify (classify,Database= ' 58tcDataNew ',Host=None,Port=None): client = Pymongo. Mongoclient (Host,Port) db = client[Database] classify =classify+ ' _data ' #location_list = [i[' location '][3:] if i[' location ']! = ' and i[' location '][:2] = = ' Beijing ' else None for I in db[' bijiben_data '].find (filter={},projection={' location ': 1, ' _id ': 0})] location_list = [i[' location '][3:] forIinchdb[' yueqi_data '].find (filter={}, projection={' location ': 1, ' _id ': 0})ifi[' Location ']! = ' andi[' location '][:2] = = ' Beijing ' andi[' location '][3:]! = '] Loc_name = List (set (location_list)) dic_count = {} forIinchloc_name:dic_count[i] = Location_list.count (i)returndic_count# bijiben_area_count = getareabyclassify (classify= ' yueqi ') # print (bijiben_area_count) # Danche_area_count = Getareabyclassify (classify= ' Danche ') # sum_area_count = Counter (bijiben_area_count) + Counter (danche_area_count) # Print (sum_area_count)defMyCounter (L,Database= ' 58tcDataNew ',Host=None,Port=None): client = Pymongo. Mongoclient (Host,Port) db = client[Database] Tab_list = db.collection_names () dic_0 = {} forIinchTab_list:loc = i[:-5] + ' _area_count ' dic_0[loc] = 0if notL:returnCounter (dic_0)Else:returnCounter (L[0]) + myCounter (L[1:])defGetallcount (Database= ' 58tcDataNew ',Host=None,Port=None): client = Pymongo. Mongoclient (Host,Port) db = client[Database] Tab_list = db.collection_names () Dic_all_count = {} forIinchTab_list:dic = getareabyclassify (i[:-5]) loc = i[:-5] + ' _area_count ' dic_all_count[loc] = dic di C_val = [dic_all_count[x] forXinchdic_all_count] my = myCounter (dic_val) dic_all_count[' total_area_count '] = dict (my)returnDic_all_countdic_all_count = Getallcount () # print (dic_all_count[' bijiben_area_count ') # print (dic_all_count[' Total _area_count ']) # #tmp_list = [] forIinchdic_all_count[' total_area_count ': data = {' name ': i, ' data ': [dic_all_count[' total_area_count '][i]], ' Type ': ' column '} tmp_list.append (data) options = {' chart ': {' zoomtype ': ' xy '}, ' title ': {' text ': ' Beijing 5 8 Second-hand trade information published area map '}, ' subtitle ': {' text ': ' data source: 58.com '}, ' xaxis ': {' categories ': [']}, ' YAxis ': {' title ': {' text ': ' quantity '}}, ' plotoptions ': {' column ': {' datalabels ': {' enabled ':True}}}}charts.plot (tmp_list,show= ' inline ', Options=options)

Crawl 58 of the same City's second-hand trading data with Python crawlers