Python batch download watercress image code

Some pictures were found during Douban, so I was too lazy to dig them one by one. I have previously written c # and python versions for downloading, so I changed the Python code, make it easy to use.

#-*-Coding: utf8 -*-
Import urllib2, urllib, socket
Import re
Import requests
From lxml import etree
Import OS, time

DEFAULT_DOWNLOAD_TIMEOUT = 30

Class AppURLopener (urllib. FancyURLopener ):
Version = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT )"

Def check_save_path (save_path ):
If not OS. path. exists (save_path ):
OS. makedirs (save_path)

Def get_image_name (image_link ):
File_name = OS. path. basename (image_link)
Return file_name

Def save_image1 (image_link, save_path ):
File_name = get_image_name (image_link)
File_path = save_path + "\" + file_name
Print ("prepare to download {0} to {1}". format (image_link, file_path ))
Try:
Urllib. _ urlopener = AppURLopener ()
Socket. Setdefatimetimeout (DEFAULT_DOWNLOAD_TIMEOUT)
Urllib. urlretrieve (url = image_link, filename = save_path)
Return True
Failed T Exception, ex:
Print (ex. args)
Print ("file download error: {0}". format (ex. message ))
Return False

Def save_image (image_link, save_path ):
File_name = get_image_name (image_link)
File_path = save_path + "\" + file_name
Print ("prepare to download {0} to {1}". format (image_link, file_path ))
Try:
File_handler = open (file_path, "wb ")
Image_handler = urllib2.urlopen (url = image_link, timeout = DEFAULT_DOWNLOAD_TIMEOUT). read ()
File_handler.write (image_handler)
Return True
Failed T Exception, ex:
Print ("file download error: {0}". format (ex. message ))
Return False

Def get_thumb_picture_link (thumb_page_link ):
Try:
Html_content = urllib2.urlopen (url = thumb_page_link, timeout = DEFAULT_DOWNLOAD_TIMEOUT). read ()
Html_tree = etree. HTML (html_content)
# Print (str (html_tree ))
Link_tmp_list = html_tree.xpath ('// div [@ class = "photo_wrap"]/a [@ class = "photolst_photo"]/img/@ src ')
Page_link_list = []
For link_tmp in link_tmp_list:
Page_link_list.append (link_tmp)
Return page_link_list
Failed T Exception, ex:
Print (ex. message)
Return []

Def download_pictures (album_link, min_page_id, max_page_id, picture_count_per_page, save_path ):
Check_save_path (save_path)
Min_page_id = 0
While min_page_id <max_page_id:
Thumb_page_link = album_link + "? Start = {0} ". format (min_page_id * picture_count_per_page)
Thumb_picture_links = get_thumb_picture_link (thumb_page_link)
For thumb_picture_link in thumb_picture_links:
Full_picture_link = thumb_picture_link.replace ("photo/thumb", "photo/large ")
Save_flag = save_image (image_link = full_picture_link, save_path = save_path)
If not save_flag:
Full_picture_link = thumb_picture_link.replace ("photo/thumb", "photo/photo ")
Save_image (image_link = full_picture_link, save_path = save_path)
Time. sleep (1)
Min_page_id + = 1
Print ("download completed ")

# Set the local folder for saving images
Save_path = "J: \ douban \ meiren2"
# Set the album address. Note that it ends with a backslash.
Album_link = "https://www.douban.com/photos/album/43697061"
# Set the total number of album pages
Max_page_id = 9
# Set the number of images on each page. The default value is 18.
Picture_count_per_page = 18
Download_pictures (album_link, max_page_id, picture_count_per_page, save_path)

========================================================== ==================================

Compared with urllib2, urllib is really difficult. If you do not set the User-Agent, the download speed will be extremely slow. In addition, you need to call the socket module to set the timeout time, in the end, I may step on other traps. For example, I was blocked by Douban when I went down. Therefore, we recommend using urllib2.