Sina Weibo data mining recipe three: search (Selenium)

#!/usr/bin/python #-*-Coding:utf-8-*-"Created on 2014-12-30@author:beyondzhou@name:decode_search_entities.py" # Decode entities from Search Resultdef decode_search_entities (): # import from search import Weibo_search F        Rom entities Import Weibo_entities # do the search (only get to the pages) Subject = Weibo_search (topic= ' iphone ')  # Decode entities (mids, names, texts, dates, reposts, comments, likes) = Weibo_entities (subject) # Output    Mids print ' \noutput mids! '    For mid in mids:print mid # Output names print ' \noutput names! '    For name in Names:print name # Output texts print ' \noutput texts! '    For text in texts:print text # Output dates print ' \noutput dates! '    For Date_ in Dates:print date_ # Output reposts number print ' \noutput reposts! ' For repost in Reposts:print repost # Output comment number print ' \noutput comments!' for comment in Comments:print comment # Output likes number print ' \noutput likes! ' For like in likes:print if __name__ = = ' __main__ ': decode_search_entities ()

#!/usr/bin/python #-*-coding:utf-8-*-"Created on 2014-12-30@author:beyondzhou@name:search.py"     import Selenium.webdriver.support.ui as Uiimport timefrom Selenium import webdriver# use Non-api to access sin A searchdef weibo_search (topic= ' Weibo '):    driver = webdriver. Phantomjs (executable_path=r ' E:\eclipse\Weibo\tool\phantomjs ')     wait = UI. Webdriverwait (driver,30)     driver.get (' http://passport.weibo.com/')     Now_handle = driver.current_window_handle    print ' Now_handle: ', now_handle    print driver.current_url    username = driver.find_element_by_id (' username ')     password = driver.find_element_by_id (' password ')     SBTN = driver.find_element_by_class_name (' smb_btn ')     #save_btn = driver.find_element_by_id (' Save_btn ')     Safe_login = driver.find_element_by_id (' safe_login ')     Username.send_keys (") # Send username    PASSWORd.send_keys (") #send password    Safe_login.click ()      Sbtn.submit ()     Time.sleep (5)     page = Driver.page_source.encode (' utf-8 ')     open (R ' e:\automation\login.html ', ' W '). Write (page)      driver.find_element_by_link_text (U "Weibo"). Click ()     Time.sleep (5)     # deal with handles    all_handles = driver.window_handles    print ' all_handles: ', All_ handles    # Switch to new windows    #driver. Switch_to_window (Driver.window_handles[-1])   & nbsp For handle in all_handles:        If handle are not now_handle:          &NBSP ; Driver.switch_to_window (handle)     page = Driver.page_source.encode (' utf-8 ')     open (R ' e:\ Automation\weibo.html ', ' W '). Write (page)     Time.sleep (5)     Driver.find_element_by_xpath ("// input[@node-type= ' Searchinput '). Clear ()     Driver.find_elemEnt_by_xpath ("//input[@node-type= ' searchinput ']"). Send_keys (topic)     Driver.find_element_by_link_text ("F"). Click ()     Time.sleep (5)     page = Driver.page_source.encode (' utf-8 ')     pages =  page    Open (R ' e:\automation\searchResult.html ', ' W '). Write (page)     # get page 2, only can get 2 pages (This was a bug)     for I in range (1):        Wait.until (Lambda Driver:driver.find_element_ By_link_text (U "Next"))         driver.find_element_by_link_text (U "next Page"). Click ()         Time.sleep (5)         page = Driver.page_source.encode (' utf-8 ')       &NBS P Pages + = page        open (r ' e:\automation\searchResult%s.html '% i, ' W '). Write (Driver.page_ Source.encode (' Utf-8 '))             driver.quit ()     print ' Search done! '         return pages

# decode entities from Sina Weibo HTML sourcesdef weibo_entities (subject):        # Generate Soup     from BeautifulSoup import beautifulsoup    soup = BeautifulSoup (subject,fromencoding= "Utf-8") & nbsp       # decode Weibo mid    mids = []    Mids_source = Soup.findall ("div", attrs={ "Action-type": "Feed_list_item"})     for index in range (len (mids_source)):        Mids.append (mids_source[index][' mid ')     print ' mids entities done! '           names = []    Names_source = Soup.findall ("A", attrs={"class": "W_texta W_FB"})     for index in Range (len (names_source)):        Names.append (names_source[index].text)     print ' names Entities done! '         texts = []    Texts_source = Soup.findall ("P", attrs={"class": "Comment_txt"})     for index in RAnge (Len (texts_source)):        Texts.append (texts_source[index].text)     print ' texts Entities done! '         dates = []    Dates_source = Soup.findall ("div", attrs={"class": "Feed_from w_t Extb "})     for index in range (len (dates_source)):        try:        &NB Sp   TMP = dates_source[index].a["title"]        except keyerror:            TMP = ' among 5 minutes '         Dates.append (TMP)     print ' dates entities done! '         reposts = []    Reposts_source = Soup.findall ("A", attrs={"Action-type": "Feed _list_forward "})     for index in range (len (reposts_source)):        if reposts_source[ Index].em.string is none:            TMP = 0        else:    & nbsp     &NBSP TMP = reposts_source[index].em.string        reposts.append (TMP)     print ' reposts Entities done! '         comments = []    Comments_source = Soup.findall ("A", attrs={"Action-type": " Feed_list_comment "})     for index in range (len (comments_source)):        if Comments_ Source[index].em is None or comments_source[index].em.string is none:            TMP = 0&nbs P       else:            TMP = comments_source[index].em.string    &N Bsp   Comments.append (TMP)     print ' comments entities done! '         likes = []    Likes_source = Soup.findall ("A", attrs={"Action-type": "Feed_ List_like "})     for index in range (len (likes_source)):        If Likes_source[index].em is N One or likes_source[index].em.string is none:           TMP = 0        else:            TMP = Likes_source[index]. em.string        likes.append (TMP)     print ' Likes entities done! '         Return mids, names, texts, dates, reposts, comments, likes

Result: Now_handle:1d316630-8fa5-11e4-b234-a1009e62cc2ehttp://passport.weibo.com/all_handles: [u ' 1d316630-8fa5-11e4-b234-a1009e62cc2e ', U ' 2457d7f0-8fa5-11e4-b234-a1009e62cc2e ']i= 0search done!mids entities done! Names entities Done!texts entities Done!dates entities done!reposts entities done!comments entities Done!likes entities do ne! Output mids! 3793356192594744379327148875698737933561842055393793356184205479379335618001052337933561637408663793356155350919379335614 2032064379335613018080437933561084724233793356087723484379335608352875337933560753794973793356049746415379335604555171237 9335603319207837933560287731603793356008524805379335599102053537933559406838413793355915515511379335587356577037933558653 9926237933558486200723793355823229119379335578989332237933557267504073793355722777596379335570599877037933557015825343793 355664537972379335566453795837933556603428733793355622100907379335556315408137933555599249843793355554985052379335550862270937933555051273493 7933554676356133793355420533970Output Names!_ Small Circle fayecometyee sometimes have stone waiting Ysu92@<span style= "White-space:pre" >< /span>   ke zhen aimarkitty cat bb roo 1689 run bar Migin Liu Hongbo love convulsions candy-sl a small cartoon bunny girl Vivian Angang daughter my family cherish June first grow @<span style= " White-space:pre "></span>   play manager Ice Dew Xuan International van Christmas Mieg cranes traveling thousands of miles Luya climbing two fiddling with a day-love who @<span style=" White-space:pre "> </span>   net shes nh baby circle Faye Zhungchun Kaimairi Bird sound sweet wenq qilian ice erotic microblog small nest at Le @<span style= "White-space:pre" ></span >   Lin update shes nh-baby aristocratic hen low-key de sexy Gao Lichen ice surplus green clean has been-small circle Faye Adzuki Bean 1682 Qiu-chun Yan photosynthetic green ice dance extraordinary to refueling shes nh _baby stern kuipers @<span style= " White-space:pre "></span>   dream World official Weibo Maggie Venci resorting branch 88888 foil @<span style=" White-space:pre "></span >   China Hong E-commerce platform foil @<span style= "White-space:pre" ></span>   China telecom Guangdong customer service but Wei _777output texts! Casio tr500 The day can be shipped Casio tr150/Casio tr300 New unsealed invoice fully packaged Casio tr350 basic color Complete (idle) (transfer) Casio tr200/casio tr350s sale Apple 6iphone6 iphone6 plus all colorsThe Spot national warranty Casio digital Camera Support Inspection Recruitment Agent interested parties Please Me: faye1912 forwarding concerns iphone6/tr//ipad/zr/toaster random tomorrow pumping @ forward draw platform run it sister Christmas Carnival # Casio selfie artifact # Hong Kong iphone6# Apple 6# Selfie artifact iphone 6Plus Casio self-timer artifact tr500,tr350s~ price Beautiful, licensed, national UNPROFOR TR150 TR200 TR350 tr350s full arrival shopkeeper 64956648[Update] also found two wrong words (and went to use 5 of the wrong two). Mental illness, call on the phone, as for the Weibo? The credit card is not the self-brush of course can get a refund, as for the envelope also drying? This is to show off. Do you have an iphone? Did you have a credit card when the envelope was off? The car was smashed hiking the end of noon yesterday. Hawaii is as big as Texas! The back of the truck was also opened and the purse and the Beatspill were all taken away. For the first time in life, call 911. Later phoned USAA to report the loss. I was told that my card had been brushed by more than 100. After confirming that they readily promised refound to me. Remind everyone to pay attention to safety during the holidays. Praise USAA One more! @ North America Savings Express | Williston ... Casio TR150 TR200 TR350 tr350s TR500 Full arrival agents still worry about the goods? IPhone6 recruit various agents!! Provide physical store supply! Immediate Cash Back: 2902515669 hot goods Apple 6 now has three colors complete are the country Line/Hong Kong 16g/64g after-sale inspection (all colors) unopened not activated. ~ Selfie artifact ~ ~ Another gift package includes a camera bag and a 16g card. Tr150iphone6 iphone6plus tr100 tr200 tr300 iphone5s also at the same time big hot dai Rationale of the preferred add the shop number vip662233 your dream # Casio tr350s# mint green, pink Casio tr500,tr350 Country Line (full color). The gift pack includes a camera bag and an SD card. Also big hot tr150#iphone6 #iPhone6PLUS tr200 tr300 iphone5s casio selfie artifact: keei12345678 overseas idle dfqw# Casio # #卡西欧自拍神器 #tr150 TR200 TR350 tr350s TR500 Full Arrival agent still worry about the goods? Apple 6iphone6 iphone5s recruit a variety of agents!! Cash Back now: y666368 support installment # lead year Awards # See you at the end of the year! Stakeholders Yes! Report Cash per person up to 2015 oceans! and a iph.One6, IPad, gold bars and so on, such as the Real Hao award! -Money is so capricious! Http://t.cn/RzcK4Ml Christmas Gift Not afraid no surprise # Casio selfie artifact # Guo Hong Kong #iphone6# overseas idle selfie artifact #iphone 6Plus casio selfie artifact tr500,tr350s~ price beautiful, licensed, national UNPROFOR TR150 TR200 TR350 tr350s Full arrival shopkeeper still worry about the goods? Big gift: 7093112 Special wayward gifts shining shiny shiny no longer to worry about taking pictures to help you solve the problem # Casio tr500#tr350s (Peacock Lan Ying Bebe Dream Red Honey Orange, Fresh green, sky blue. Lavender powder) Voice box gift version, with 16g of memory, Wi-Fi, anywhere share photos! Our store #iphone6 #iPhone6 iphone6plus warranty One year free postage: No. 2902515669 outside the fashion trend of the Casio self-timer artifact TR500 Middle School students, high school students dream of self-timer artifact tr150 tr350 tr350s now # Overseas Investment # Special # overseas Micro-Business # engage in activities iphone6 iphone5s price let you satisfied understand: 2998808841 I and my friends said I was a iphone6, they do not believe, let me show them a look. "Send IPhone6" request is very simple, as long as the attention of me and @ Little Monster Assistant, forwarded this microblog, there is a chance to get iPhone6 one. January 1, 2015 forwarding platform pumping one! Conveniently forward, no trouble, maybe IPhone6 is yours?! #卡西欧TR350s # Casio tr500,tr350 tr150 tr200 full range of country line abundant (color complete) gift Package camera bag and SD card. Apple 6iphone6 iphone6plus iphone5s casio selfie artifact: aa108866# Selfie artifact # #卡西欧TR500 #,tr350s~ price Beautiful, licensed, national UNPROFOR ~iphone6 iphone6plus Casio TR150 TR200 TR350 tr350s Full arrival agents still worry about the goods? Recruit various agents! : keei12345678 your dream # Casio tr350s# mint green, pink Casio tr500,tr350 country line (full color). The gift pack includes a camera bag and an SD card. Also big hot tr150#iphone6 #iPhone6PLUS tr200 tr300 iphone5s Casio self-timer artifact: 64956648 Overseas Idle # Casio self-timer artifact #tr500 New, the country does not, unpacking, overseas idle Casio TR350STR350 Hong Kong abundant (full color) gift box version. Self-timer artifact Gift Pack and SD card. Also big hot TR150? #苹果专卖iPhone6 iphone6plus#, tr100 tr200 tr300 idle self-timer artifact 64956648 Wayward Staging # We sent IPhone6 the request is very simple # support, daily mood is not the same, feel kind and familiar with a lot of HTTP// T.cn/rzm3xyv "We send iPhone6 the request is very simple" really give back to fans, small part think now the best prize is iPhone6. From now until December 31, follow us, forward the micro-blog, there will be a chance to win IPhone6 (prizes may need to wait)! Smoke one every month. Do not bother, or try it, in case of it? HTTP://T.CN/RZM3XYV new Country line Casio Hong Kong IPhone6 Apple 6 iphone6plus Casio selfie artifact Casio TR500 Casio tr350s~ price Beautiful licensed national UNPROFOR Casio TR150 Casio TR200 Casio TR350 casio tr350s casio TR300 Casio TR100 full arrival shopkeeper still worry about the goods? Recruit various agents: duyan8827 new country line Casio Hong Kong IPhone6 Apple 6 iphone6plus Casio selfie artifact Casio TR500 Casio tr350s~ price Beautiful licensed national UNPROFOR Casio TR150 Casio TR200 casio TR350 casio tr350s casio TR300 Casio TR100 full arrival shopkeeper still worry about the goods are not? Recruit all kinds of agents: faye1912 New Country Line #卡西欧神器 # Hong Kong #iphone6# #苹果6 # selfie artifact #iphone 6Plus casio selfie artifact tr500,tr350s~ price beautiful, licensed, national UNPROFOR TR150 TR200 TR350 tr350s full arrival shopkeeper still worry about not goods: y666368 casio TR150 TR200 TR350 tr350s TR500 ZR1000 ZR1200 Full arrival agents still worry about the goods? IPhone6 iphone5s recruit various agents!! Provide physical store supply! Instant cashback: 77093112 super Big Bang # Casio tr150# mint green, pink Casio tr350s,tr350 country Line abundant (color complete) perfect packaging gift version. The selfie artifact gift package includes a camera bag and an SD card. Tr150iphone6 iphone6plus tr100 tr200 tr300 iphone5s Samsung NOTE4 Low price sales agent's firstChoose keei12345678# Casio # #卡西欧自拍神器 #tr150 TR200 TR350 tr350s TR500 ZR1000 ZR1200 Full arrival agents still worry about the goods? Apple 6iphone6 iphone5s recruit various agents!! Provide warehouse supply! Immediately return to cash: aa108866 support installment theft number of so intentions, the following cottage Iphone6plus is how to actually I was lovelorn, but even if you hate me I still love you, hubby! @ Constructed Casio digital Camera Apple 6 port version IPhone6 (4.7 inch)/IPhone6 plus (5.5 inch) 16g/64g Beauty camera Casio tr100/Casio tr150/Casio tr350/Casio tr200/Casio tr300/Casio tr350s/Casio tr500 hurriedly snapped up ah my source for the Hong Kong version/Ching this time for the sale of affordable agent prestige: duyan8827 Country Line Hong Kong Apple 6iphone6 iphoneplus Casio selfie artifact TR500 tr350s tr350,tr150 Country Line ( 7 colors) Not activated. Non-unpacking the national UNPROFOR Fair Trade tr200 tr300 tr100, our store to recruit agent concessions more breaking. y666368 New Country Line Hong Kong IPhone6 Apple 6 iphone6plus Casio selfie artifact tr500,tr350s~ price beautiful, licensed, National UNPROFOR Casio tr150tr200tr350tr350s full arrival shopkeeper also worry about not goods: y666368# Casio # #卡西欧自拍神器 #tr150 TR200 TR350 tr350s TR500 Full arrival agents still worry about the goods? Apple 6iphone6 iphone5s recruit a variety of agents!! Immediate Cashback: aa108866 support installment spot Iphone6/iphone6 plus casio tr350s/tr350 stable Sourcing agents preferred Original packaging accessories complete 16G original memory card original camera package Casio TR500 Casio tr150 Casio tr200 ( Pre-owned) (idle) Casio tr100 Casio tr300 all support counters inspection beauty camera selfie artifact recruit Agent Prestige: faye1912 Casio digital camera Apple 6 port version IPhone6 (4.7 inch)/IPhone6 plus (5.5 inch) 16g/64g Beauty camera Casio tr100/Casio tr150/Casio tr350/Casio tr200/Casio tr300/Casio tr350s/Casio tr500 hurriedly snapped up ah my source for the Hong Kong version/Ching this time for the sale of cheap recruit agent: vip662233 you want to eat 100 pieces of spicy soup immediately let # overseas purchase # to you recently Casio new products tr500 domineering arrival # overseas favorite #tr350s TR  150# Personal Idle help turn tr350 National UNPROFOR Iphone6 National guarantee iphone6 plus all kinds of complete recruit agent keei12345678 self-portrait artifact dream Red Fresh green once meet you oliol look, see, then look, iPhone, millet, bag, phone call, good want t t~~~! #对面 # points for free gifts, all of them are mine! If you want it, poke it! Selfie artifact # Casio tr500#,tr350s~ price Beautiful, licensed, national UNPROFOR ~iphone6 iphone6plus Casio TR150 TR200 TR350 tr350s Comprehensive arrival recruit all kinds of agents! : 2998808841 Hong Kong Apple 6iphone6 iphoneplus Casio self-timer artifact TR500 tr350s tr350,tr150 Country Line (7 colors) not activated. Unopened national support inspection tr200 tr300 tr100 mating symbol ^_^ Free WiFi card package and so on, we recruit agent offers more breaking has been imitated has never been surpassed please join the strength of the team:d uyan8827# "dream World 2" Qing gong test welcome clothes? Send iphone6# look forward to a small surprise. The sword out of the river, the name moves the world! Oriental Martial Arts round Game "dream World 2" hot public test, December 26 "Crouching Tiger Hidden Dragon" new suit opened. Now invites the world heroes, a total of dream lakes! Follow @ Dream World official Weibo, forward this Weibo and @ three or more friends and have a chance to win the IPhone6 award! Starter Pack Pickup: http://t.cn/R7wstIQhttp://t.cn/Rzk7CYx Hot goods TR500 Casio TR350S,TR350 cameras are all state-of-the-line after-sale inspection (all colors) unopened not activated. ~ Selfie Artifact ~ ~ Another gift package includes a camera bag and a 16g card. Tr150iphone6 iphone6plus tr100 tr200 tr300 iphone5s is also the preferred choice for large heat proxies. Csoip6. Or can be a private signal outside the fashion trend of the Casio self-timer artifact TR500 Middle School students, high school students dream of selfie artifact tr150 tr350 tr350s now # Overseas Investment # Special # overseas Micro-Merchants # engage in activities iphone6 iphone5s price let you satisfied let you dream not far away, There's not much chance, second-handIdle # together understand: xiaomippp# in the exhibition Hung 1th shop 12.21 Big Rob Festival, forward to win iphone# good activity is a lot, cause let me lost my eyes, tonight's efforts, is to let my home can also enjoy a happy @ Day Loadline head a# China exhibition Hung 1th shop 12.21 Big Rob Festival, forward to win iphone# from now until December 31, pay attention to @ in the Expo Hung e-commerce platform, forward this micro-bo, and @ 1 friends, you can participate in the lottery. http://t.cn/RzB3Yda# points easily, forward to win IPhone6 (phase II) #不管中不中奖, I have to support the work of small series, we refuel together!!! @ Like the broken memories of flowers easy to grab, easy to beat, points good gift easy to rob! Telecom points generated before December 31, 2012 will expire 0! Good gift good points, you are going to →http://t.cn/r7t0k41 here to redeem it? Forward this Weibo and @ a friend who has the opportunity to win #iphone6# for free #有钳就是任性 #! HTTP://T.CN/RZEUXM1, Ohio man Myers has recently uploaded an interesting video on the internet, in which he successfully drew the hungry Leopard frog with his iphone, he said, according to the US-based Chinese web report. Output dates!2014-12-30 05:542014-12-30 00:172014-12-30 05:54among 5 minutes2014-12-30 05:542014-12-30 05:542014-12-30 05:542014-12-30 05:542014-12-30 05:542014-12-30 05:542014-12-30 05:532014-12-30 05:532014-12-30 05:53among 5 minutes2014-12-30 05:532014-12-30 05:532014-12-30 05:532014-12-30 05:532014-12-30 05:53among 5 minutes2014-12-30 05:532014-12-30 05:532014-12-30 05:532014-12-30 05:532014-12-30 05:532014-12-30 05:522014-12-30 05:52among 5 minutes2014-12-30 05:522014-12-30 05:522014-12-30 05:522014-12-30 05:522014-12-30 05:522014-12-30 05:522,014-12-30 05:522014-12-30 05:522014-12-30 05:522014-12-30 05:522014-12-30 05:51among 5 minutes2014-12-30 05:512014-12-30 05:512014-12-30 05:51among 5 minutes2014-12-30 05:51among 5 minutes2014-12-30 05:512014-12-30 05:51Output reposts! 03502000000000000000000000000000000000050000Output Comments!00000000000000000000000000000000000020000output likes!01560300000000010460000062000000008251600000000000260002505500

Sina Weibo data mining recipe three: Search (selenium)