Using Node. js to capture beautiful pictures in batches through sitemap. xml _ javascript skills

This article mainly introduces how to use Node. js to capture beautiful pictures in batches through sitemap. xml and related code. If you need it, you can refer to it. I have read many versions before, and I have made one myself.

1. You can specify the directory to which the file is saved.
2. store articles in different directories
3. allows you to set the maximum number of parallel downloads.

Next time you are free to download the entire site.

Package. json

{ "name": "me2sex-images", "version": "0.0.1", "description": "Batch download images from http://me2-sex.lofter.com", "main": "index.js?1.1.9", "author": "Fay", "license": "MIT", "dependencies": {  "async": "^0.9.0",  "cheerio": "^0.18.0",  "mkdirp": "^0.5.0",  "request": "^2.51.0",  "url": "^0.10.2",  "xml2js": "^0.4.4" }}

Index. js

Var node = {async: require ('async'), cheerio: require ('cheerio '), fs: require ('fs'), mkdirp: require ('mkdirp '), path: require ('path'), request: require ('request'), url: require ('url'), xml2js: require ('xml2js '),}; var Me2SexImages = {/*** configuration option */options: {// sitemap URL sitemap :' http://sexy.faceks.com/sitemap.xml ', // Save to this folder saveTo:'/Users/Fay/Pictures/me2sex ', // The maximum parallel downLimit: 5,}, posts: [], /*** start download (program entry function) */start: function () {var self = this; var async = node. async; async. waterfall ([self. wrapTask (self. sitemapXML), self. wrapTask (self. sitemapJSON), self. wrapTask (self. downAllImages),], function (err, result) {if (err) {console. log ('error: % s', err. message);} else {console. log ('success: Downloaded successfully ') ;}};},/*** package the task to ensure that the context of the original task points to a specific object * @ param {Function} The task conforms to asycs. js call method: The task Function * @ param {Any} context * @ param {Array} exArgs additional parameter * @ return {Function} conforms to asycs. js call method of the task function */wrapTask: function (task, context, exArgs) {var self = this; return function () {var args = []. slice. call (arguments); args = exArgs? ExArgs. concat (args): args; task. apply (context | self, args) ;}},/*** get site sitemap. xml */sitemapXML: function (callback) {console. log ('starts to download sitemap. xml '); node. request (this. options. sitemap, function (err, res, body) {if (! Err) console. log ('Download sitemap. xml SUCCESS '); callback (err, body) ;},/*** sets sitemap. xml to json */sitemapJSON: function (sitemapXML, callback) {var self = this; console. log ('start parsing sitemap. xml '); node. xml2js. parseString (sitemapXML, {explicitArray: false}, function (err, json) {if (! Err) {self. posts = json. urlset. url; self. posts. shift (); console. log ('parse sitemap. xml succeeded. A total of % d pages ', self. posts. length);} callback (err, self. posts) ;};},/*** download the whole site image */downAllImages: function (callback) {var self = this; var async = node. async; console. log ('start batch download'); async. eachSeries (self. posts, self. wrapTask (self. downPostImages), callback) ;},/*** download a single post image * @ param {Object} post article */downPos TImages: function (post, callback) {var self = this; var async = node. async; async. waterfall ([self. wrapTask (self. mkdir, self, [post]), self. wrapTask (self. getPost), self. wrapTask (self. parsePost), self. wrapTask (self. downImages),], callback);}, mkdir: function (post, callback) {var path = node. path; var url = node. url. parse (post. loc); post. dir = path. join (this. options. saveTo, path. basename (url. pat Hname); console. log ('Prepare to create directory: % s', post. dir); if (node. fs. existsSync (post. dir) {callback (null, post); console. log ('Directory: % s already exists ', post. dir); return;} node. mkdirp (post. dir, function (err) {callback (err, post); console. log ('Directory: % s created successfully ', post. dir);}) ;},/*** get post content */getPost: function (post, callback) {console. log ('start request page: % s', post. loc); node. request (post. loc, function (err, res, body) {if (! Err) post.html = body; callback (err, post); console. log ('request page succeeded: % s', post. loc) ;};},/*** parse post and obtain the image list in post */parsePost: function (post, callback) {var $ = post. $ = node.cheerio.load(post.html); post. images = $ ('. img '). map (function () {return $ (this ). attr ('bigimgsrc ');}). toArray (); callback (null, post) ;},/*** download the image in the post image list */downImages: function (post, callback) {console. log ('% d sister picture found, ready to download... ', post. images. length); node. async. eachLimit (post. images, this. options. downLimit, this. wrapTask (this. downImage, this, [post]), callback) ;},/*** download a single image */downImage: function (post, imgsrc, callback) {var url = node. url. parse (imgsrc); var fileName = node. path. basename (url. pathname); var toPath = node. path. join (post. dir, fileName); console. log ('start downloading image: % s, save to: % s, file name: % s', imgsrc, post. dir, fileName); node. request (imgsrc ). pipe (node. fs. createWriteStream (toPath )). on ('close', function () {console. log ('image download succeeded: % s', imgsrc); callback ();}). on ('error', callback) ;}}; Me2SexImages. start ();

The above is all the content of this article. I hope you will like it.