Beautiful soup is a Python library that parses HTML and XML, and it can parse files in the way you like, and find and modify the parse tree. It can handle nonstandard tags well and generate parse trees (parse tree). It provides simple and commonly used navigation (navigating), searches, and modifies the parse tree operation.
Use the URLLIB2 and BS4 modules to crawl HTML page data, respectively, for title, content, stock name, stock ID, publication time, and number of onlookers.
Example:
| The code is as follows |
Copy Code |
| ##-coding:utf-8-##
Import time
From BS4 import BeautifulSoup
Import Urllib2
Import Pymongo
Import re
Import datetime def update ():
Datas = {}
Connection = Pymongo. Connection (' 192.168.1.2 ', 27017)
#连接mongodb
db = Connection.test_hq
#创建或连接test_hq库
For I in Soup.find_all ("div", class_= "item"):
datas[' _id '] = str (i.h2.a[' href ']). Split ('/') [ -1].split ('. ') [0]
#获取html页面名称为id号
datas[' title '] = I.h2.get_text ()
#获取标题
Url2 = i.h2.a[' href ']
#获取标题内容url地址
HTML2 = Urllib2.urlopen (URL2)
HTML_DOC2 = Html2.read ()
soup2 = BeautifulSoup (HTML_DOC2)
datas[' content '] = Soup2.find (attrs={"name": "description"}) [' content ']
#获取文章内容
Stock_name = []
stock_id = []
For name in Re.findall (U "[u4e00-u9fa5]+", I.find (class_= "Stocks"). Get_text ()):
Stock_name.append (name)
#获取影响股票名称, the array method holds the corresponding stock ID number, MONGO supports array insertion
datas[' stock_name '] = Stock_name
For ID in Re.findall ("d+", I.find (class_= "Stocks"). Get_text ()):
Stock_id.append (ID)
#获取影响股票id
datas[' stock_id '] = stock_id
datas[' update_time ' = Datetime.datetime.strptime (Re.search ("w+.*w+", I.find (class_= "FL Date"). Span.get_text (). Group (), '%y-%m-%d%h:%m ')-Datetime.timedelta (hours=8)
#获取发布时间, converting to MONGO time format
datas[' onlooker ' = Int (Re.search ("d+", I.find (class_= "Icons Ic-wg"). Get_text ()). Group ())
#获取围观数
Db.test.save (Datas)
#插入数据库 Def get_data ():
title = str (soup.h2.a[' href ']). Split ('/') [ -1].split ('. ') [0]
#获取html页面名称做更新判断
With open (' Update.txt ', ' R ') as F:
Time = F.readline ()
If title = = Time:
print ' Currently no update ', title
Else
With open (' Update.txt ', ' W ') as F:
F.write (title)
Update ()
While True:
if __name__ = = ' __main__ ':
url = ' http://www.ipython.me/qingbao/'
html = urllib2.urlopen (URL)
Html_doc = Html.read ()
Soup = BeautifulSoup (Html_doc)
Get_data ()
Time.sleep (30)
#每30秒刷新一次 |
