使用BeautifulSoup爬蟲程式擷取百度搜尋結果的標題和url樣本

熟悉Java的jsoup包的話，對於Python的BeautifulSoup庫應該很容易上手。

代碼如下:

#coding: utf-8
import sys
import urllib
import urllib2
from BeautifulSoup import BeautifulSoup

question_word = "吃貨 程式員"
url = "http://www.baidu.com/s?wd=" + urllib.quote(question_word.decode(sys.stdin.encoding).encode('gbk'))
htmlpage = urllib2.urlopen(url).read()
soup = BeautifulSoup(htmlpage)
print len(soup.findAll("table", {"class": "result"}))
for result_table in soup.findAll("table", {"class": "result"}):
a_click = result_table.find("a")
print "-----標題----\n" + a_click.renderContents()#標題
print "----連結----\n" + str(a_click.get("href"))#連結
print "----描述----\n" + result_table.find("div", {"class": "c-abstract"}).renderContents()#描述
print

