Python simple crawler writing, python Crawler
1. Mainly learn how to write this program
A. Read the explanation website
B. Find the relevant page
C. Find the image link element.
D. Save the image to the folder.
.....
Every step is decomposed and implemented using functions. The code is easy to implement.
# An error is reported when the code runs as soon as possible and must be modified
Import urllib. requestimport osdef url_open (url): # Read description req = urllib. request. request (url) # req. add_header (\ 'user-Agent \ ', \ 'mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.125 Safari/537.36 \') response = urllib. request. urlopen (req) html = response. read () return htmldef get_page (url): # Find the relevant page html = url_open (url) a = html. find ('current-comment-page') B = htm L. find (a) return html [a: B] def find_imgs (url): # find the image link element html = url_open (url) img_addrs = [] a = html. find ('img src = ') while! =-1: B = html.find('.jpg ', a, a + 255') if B! =-1: img_addrs.append (html [a + 9: B + 4]) else: B = a + 9 a = html. find ('img src = ', B') return img_addrsdef save_imgs (folder, img_addrs): # Save the image to the folder for each in img_addrs: filename = each. split ('\'/\ '') with open (filename, 'wb') as f: img = url_open (each) f. write (img) def download_mm (folder = 'ooxx', pages = 10): OS. mkdir (folder) OS. chdir (folder) url = 'HTTP: // jandan.net/ooxx/'page_num = int (get_page (url) for I in range (pages ): page_num-= I page_url = url + 'page-'+ str (page_num) +' # comments 'img_addrs = find_imgs (page_url) save_imgs (img_addrs) if _ name _ = '_ main _': download_mm ()