Bs+json parsing

#-*-Coding=utf-8-*-import urllib2from beautifulsoup import beautifulsoup as Bs3import jsonimport codecs# character detection, used to detect its true Code format import chardet#save content to FileDef save_to_file (filename, content): F = open (filename, ' w+ ') assert (f) f.write ( Content) F.close () def parse_key_link (content): Old_code_name = chardet.detect (content) [' Encoding ']print (' Old_code_ name[key_link]=%s '% (Old_code_name,)) js = Json.loads (Content.decode (old_code_name)) for I in js[' result ' [' Items ']: Print (' name=%s, link=%s '% (i[' name '].encode (old_code_name), i[' link '].encode (old_code_name))) def parse_config ( Content): Old_code_name = chardet.detect (content) [' Encoding ']print (' old_code_name[config]=%s '% (Old_code_name,)) JS = Json.loads (Content.decode (old_code_name)) for I in js[' result ' [' paramtypeitems ']:p rint (' name=%s '% (i[' name ']. Encode (Old_code_name),) I1 = i[' Paramitems ']for J in I1:print (' name=%s '% (j[' name '].encode (old_code_name),)) J1 = j[' va Lueitems ']for K in J1:print (' specid=%d,value=%s '% (k[' specid '],k[' value '].Encode (old_code_name)) def parse_option (content): Old_code_name = chardet.detect (content) [' Encoding ']print (' old_ code_name[option]=%s '% (Old_code_name,)) js = Json.loads (Content.decode (old_code_name)) for I in js[' result ' [' Configtypeitems ']:p rint (' name=%s '% (i[' name '].encode (old_code_name),)) I1 = i[' Configitems ']for J in I1:print (' name=% S '% (j[' name '].encode (old_code_name),)) J1 = j[' Valueitems ']for K in J1:print (' specid=%d,value=%s '% (k[' specid '],k[' V Alue '].encode (old_code_name))) def parse_color (content): Old_code_name = chardet.detect (content) [' Encoding ']print (' old_code_name[color]=%s '% (Old_code_name,)) js = Json.loads (Content.decode (old_code_name)) for I in js[' result ' [' Specitems ']:p rint (' specid=%d '% (i[' specid '),) I1 = i[' Coloritems ']for J in I1:print (' id=%d,name=%s,value=%s,picnum=% d '% (j[' id '],j[' name '].encode (old_code_name), j[' value '].encode (old_code_name), j[' Picnum ']) def parse_innercolor ( Content): Old_code_name = chardet.detect (content) [' Encoding ']print (' Old_code_name[innercolor]=%s '% (Old_code_name,)) js = Json.loads (Content.decode (old_code_name)) for I in js[' result ' [' Specitems ']: Print (' specid=%d '% (i[' specid '),) I1 = i[' Coloritems ']for j in i1:j1 = j[' values ']for K in J1:print (' Id=%d,name=%s,valu e=%s,picnum=%d '% (j[' id '],j[' name '].encode (old_code_name), K.encode (old_code_name), j[' Picnum ')) def Parse_json_ Data (content): Name_list = [' keylink ', ' config ', ' option ', ' Color ', ' innercolor ']parse_list = [Parse_key_link, Parse_ Config, parse_option, Parse_color, Parse_innercolor]assert (len (content) = = Len (parse_list)) for I in range (len (content) ):p Arse_list[i] (Content[i]) def parse_content (content): #content是GB2312的编码soup = BS3 (content) Key_text = ' var Levelid ' elem_lib = soup.find (' script ', Text=lambda (x): Key_text in X) #str_script是utf-8 Encoding str_script = str (elem_ lib.string) #print (Chardet.detect (str_script)) #由于命令行是cp936 GBK encoding If the encoding does not conform to the inability to print STRGBK = Str_script.decode (' Utf-8 '). Encode (' gb2312 ') #print (STRGBK) #移除html的转义字符 &NBSP;STRGBK = Strgbk.replace ('   ', ') d =Strgbk.splitlines () List_data = []for i in D:if i.isspace (): continue# filter unwanted variable if len (i) < 100:continue# take out JSON data idx = I.F IND (' {') if idx = = -1:continue# removed last; k = I[idx:-1]list_data.append (k) Parse_json_data (list_data) def crawler_4_autohome (): Autohome_url = ' http://car.autohome.com.cn/config/series/657.html ' #uft -8content = Urllib2.urlopen (url=autohome_ URL). Read () #print (chardet.detect (content)) parse_content (content) if __name__ = = ' __main__ ': Crawler_4_autohome ()

Source:

http://download.csdn.net/detail/davidsu33/8447189

Bs+json parsing