標籤:size 語句 == output mode mod images .com 防盜鏈
此指令碼用於爬網站的下載連結,最終輸出到txt文檔中。
如果是沒有防盜鏈設定的網站,也可以使用指令碼中的下載函數嘗試直接下載。
本指令碼是為了短期特定目標設計的,如果使用它爬其它特徵的資源連結需自行修改配置語句。
python初學者,請多多指正。
# -*- coding: utf-8 -*- import reimport urllibimport osimport urllib2import requestsimport time#download the filedef download(page, url):local_filename =url.split(‘/‘)[-1] + page + ‘.jpg‘r = requests.get(url, stream=True)with open(local_filename, ‘wb‘) as f:for chunk in r.iter_content(chunk_size = 1024): if chunk: # filter out keep-alive new chunksf.write(chunk) f.flush() return local_filename# download(‘123‘,‘http://cdn.woi3d.com/openfiles/86509/88619.jpg-preview‘)# download(‘11‘, ‘http://www.cnblogs.com/images/logo_small.gif‘)#turn the data array into urls arraydef print_urls(urls):output_urls = []for link in urls:start_link = link.find(‘"‘)end_link = link.find(‘"‘, start_link+1)output_link = link[start_link+1: end_link]if output_link.find(‘http‘) == -1:output_link = ‘http://www.woi3d.com‘ + output_linkif link.count(‘"‘) > 2:continueelse:output_urls.append(output_link)return output_urlsdef output_download_link_page(page):url = pages = urllib.urlopen(url).read()urls = []img_urls = ‘no image on‘ + pagenew_stl_urls = []title = re.findall(r‘<h1>.+<\/h1>‘, s, re.I)if len(title) != 0:title = title[0]else:title = ‘no title‘img_urls = print_urls(re.findall(r‘href=".*?\.jpg.*?"‘, s, re.I))if len(img_urls) != 0:img_urls = img_urls[0]else:img_urls = ‘no image‘ + pagestl_urls = print_urls (set(re.findall(r‘href="/download/.*?"‘, s, re.I)))for url in stl_urls:#url = urllib2.urlopen(url).urlurl = requests.get(url).urlnew_stl_urls.append(url)urls.append(title) urls.append(img_urls) urls = urls + new_stl_urlsreturn urls#print output_download_link_page(‘http://www.woi3d.com/thing/46876‘)#output all links to downloaddef output_all_pages(site):s = urllib.urlopen(site).read()page = re.findall(r‘href="/thing/.*?"‘, s, re.I)page = set(page)return print_urls(page)#output all the sites to downloaddef generate_sites(start, end):sites = []for num in range(start, end):sites.append(‘http://www.woi3d.com/popular?query=&pg=‘ + str(num))return sites#write all the results to a txt filefile_new = open (‘1.txt‘, ‘r+‘)url_pakage = []sites = generate_sites(40, 46)count = 0for site in sites:print sitefile_new.write( ‘\n‘ + site)pages = output_all_pages(site)for page in pages:urls = output_download_link_page(page)#if len(urls) >= 10:continuecount = count + 1for url in urls:file_new.write(url + ‘\n‘)print ‘done‘time.sleep(10)file_new.close()print ‘all done. all..‘ + str(count) + ‘..models‘
python 網站資源連結簡易爬蟲