Using Node. js to capture beautiful pictures in batches through sitemap. xml, node. jssitemap. xml

Source: Internet
Author: User

Using Node. js to capture beautiful pictures in batches through sitemap. xml, node. jssitemap. xml

I have read many versions before, and I have made one myself.

1. You can specify the directory to which the file is saved.
2. store articles in different directories
3. allows you to set the maximum number of parallel downloads.

Next time you are free to download the entire site.

Package. json

{ "name": "me2sex-images", "version": "0.0.1", "description": "Batch download images from http://me2-sex.lofter.com", "main": "index.js", "author": "Fay", "license": "MIT", "dependencies": {  "async": "^0.9.0",  "cheerio": "^0.18.0",  "mkdirp": "^0.5.0",  "request": "^2.51.0",  "url": "^0.10.2",  "xml2js": "^0.4.4" }}

Index. js

Var node = {async: require ('async'), cheerio: require ('cheerio '), fs: require ('fs'), mkdirp: require ('mkdirp '), path: require ('path'), request: require ('request'), url: require ('url'), xml2js: require ('xml2js '),}; var Me2SexImages = {/*** configuration option */options: {// sitemap address sitemap: 'http: // sexy.faceks.com/sitemap.xml', // save it to this folder saveTo: '/Users/Fay/Pictures/me2sex', // maximum downLimit: 5 ,}, Posts: [],/*** start download (program entry function) */start: function () {var self = this; var async = node. async; async. waterfall ([self. wrapTask (self. sitemapXML), self. wrapTask (self. sitemapJSON), self. wrapTask (self. downAllImages),], function (err, result) {if (err) {console. log ('error: % s', err. message);} else {console. log ('success: downloaded successfully') ;}}},/*** Package task, make sure that the context of the original task points to a specific object * @ param {Function} task conforms to asycs. j The task Function * @ param {Any} context in s call mode * @ param {Array} exArgs additional parameter * @ return {Function} conforms to asycs. js call method of the task function */wrapTask: function (task, context, exArgs) {var self = this; return function () {var args = []. slice. call (arguments); args = exArgs? ExArgs. concat (args): args; task. apply (context | self, args) ;}},/*** get site sitemap. xml */sitemapXML: function (callback) {console. log ('starts to download sitemap. xml '); node. request (this. options. sitemap, function (err, res, body) {if (! Err) console. log ('Download sitemap. xml SUCCESS '); callback (err, body) ;},/*** sets sitemap. xml to json */sitemapJSON: function (sitemapXML, callback) {var self = this; console. log ('start parsing sitemap. xml '); node. xml2js. parseString (sitemapXML, {explicitArray: false}, function (err, json) {if (! Err) {self. posts = json. urlset. url; self. posts. shift (); console. log ('parse sitemap. xml succeeded. A total of % d pages ', self. posts. length);} callback (err, self. posts) ;};},/*** download the whole site image */downAllImages: function (callback) {var self = this; var async = node. async; console. log ('start batch download'); async. eachSeries (self. posts, self. wrapTask (self. downPostImages), callback) ;},/*** download a single post image * @ param {Object} post article */downPos TImages: function (post, callback) {var self = this; var async = node. async; async. waterfall ([self. wrapTask (self. mkdir, self, [post]), self. wrapTask (self. getPost), self. wrapTask (self. parsePost), self. wrapTask (self. downImages),], callback);}, mkdir: function (post, callback) {var path = node. path; var url = node. url. parse (post. loc); post. dir = path. join (this. options. saveTo, path. basename (url. pat Hname); console. log ('Prepare to create directory: % s', post. dir); if (node. fs. existsSync (post. dir) {callback (null, post); console. log ('Directory: % s already exists ', post. dir); return;} node. mkdirp (post. dir, function (err) {callback (err, post); console. log ('Directory: % s created successfully ', post. dir);}) ;},/*** get post content */getPost: function (post, callback) {console. log ('start request page: % s', post. loc); node. request (post. loc, function (err, res, body) {if (! Err) post.html = body; callback (err, post); console. log ('request page succeeded: % s', post. loc) ;};},/*** parse post and obtain the image list in post */parsePost: function (post, callback) {var $ = post. $ = node.cheerio.load(post.html); post. images = $ ('. img '). map (function () {return $ (this ). attr ('bigimgsrc ');}). toArray (); callback (null, post) ;},/*** download the image in the post image list */downImages: function (post, callback) {console. log ('% d sister picture found, ready to download... ', post. images. length); node. async. eachLimit (post. images, this. options. downLimit, this. wrapTask (this. downImage, this, [post]), callback) ;},/*** download a single image */downImage: function (post, imgsrc, callback) {var url = node. url. parse (imgsrc); var fileName = node. path. basename (url. pathname); var toPath = node. path. join (post. dir, fileName); console. log ('start downloading image: % s, save to: % s, file name: % s', imgsrc, post. dir, fileName); node. request (imgsrc ). pipe (node. fs. createWriteStream (toPath )). on ('close', function () {console. log ('image download succeeded: % s', imgsrc); callback ();}). on ('error', callback) ;}}; Me2SexImages. start ();

The above is all the content of this article. I hope you will like it.

Related Article

Contact Us

The content source of this page is from Internet, which doesn't represent Alibaba Cloud's opinion; products and services mentioned on that page don't have any relationship with Alibaba Cloud. If the content of the page makes you feel confusing, please write us an email, we will handle the problem within 5 days after receiving your email.

If you find any instances of plagiarism from the community, please send an email to: info-contact@alibabacloud.com and provide relevant evidence. A staff member will contact you within 5 working days.

A Free Trial That Lets You Build Big!

Start building with 50+ products and up to 12 months usage for Elastic Compute Service

  • Sales Support

    1 on 1 presale consultation

  • After-Sales Support

    24/7 Technical Support 6 Free Tickets per Quarter Faster Response

  • Alibaba Cloud offers highly flexible support services tailored to meet your exact needs.