Using Node. js to capture beautiful pictures in batches through sitemap. xml, node. jssitemap. xml

Using Node. js to capture beautiful pictures in batches through sitemap. xml, node. jssitemap. xml

I have read many versions before, and I have made one myself.

1. You can specify the directory to which the file is saved.
2. store articles in different directories
3. allows you to set the maximum number of parallel downloads.

Next time you are free to download the entire site.

Package. json

{ "name": "me2sex-images", "version": "0.0.1", "description": "Batch download images from http://me2-sex.lofter.com", "main": "index.js", "author": "Fay", "license": "MIT", "dependencies": {  "async": "^0.9.0",  "cheerio": "^0.18.0",  "mkdirp": "^0.5.0",  "request": "^2.51.0",  "url": "^0.10.2",  "xml2js": "^0.4.4" }}

Index. js

Var node = {async: require ('async'), cheerio: require ('cheerio '), fs: require ('fs'), mkdirp: require ('mkdirp '), path: require ('path'), request: require ('request'), url: require ('url'), xml2js: require ('xml2js '),}; var Me2SexImages = {/*** configuration option */options: {// sitemap address sitemap: 'http: // sexy.faceks.com/sitemap.xml', // save it to this folder saveTo: '/Users/Fay/Pictures/me2sex', // maximum downLimit: 5 ,}, Posts: [],/*** start download (program entry function) */start: function () {var self = this; var async = node. async; async. waterfall ([self. wrapTask (self. sitemapXML), self. wrapTask (self. sitemapJSON), self. wrapTask (self. downAllImages),], function (err, result) {if (err) {console. log ('error: % s', err. message);} else {console. log ('success: downloaded successfully') ;}}},/*** Package task, make sure that the context of the original task points to a specific object * @ param {Function} task conforms to asycs. j The task Function * @ param {Any} context in s call mode * @ param {Array} exArgs additional parameter * @ return {Function} conforms to asycs. js call method of the task function */wrapTask: function (task, context, exArgs) {var self = this; return function () {var args = []. slice. call (arguments); args = exArgs? ExArgs. concat (args): args; task. apply (context | self, args) ;}},/*** get site sitemap. xml */sitemapXML: function (callback) {console. log ('starts to download sitemap. xml '); node. request (this. options. sitemap, function (err, res, body) {if (! Err) console. log ('Download sitemap. xml SUCCESS '); callback (err, body) ;},/*** sets sitemap. xml to json */sitemapJSON: function (sitemapXML, callback) {var self = this; console. log ('start parsing sitemap. xml '); node. xml2js. parseString (sitemapXML, {explicitArray: false}, function (err, json) {if (! Err) {self. posts = json. urlset. url; self. posts. shift (); console. log ('parse sitemap. xml succeeded. A total of % d pages ', self. posts. length);} callback (err, self. posts) ;};},/*** download the whole site image */downAllImages: function (callback) {var self = this; var async = node. async; console. log ('start batch download'); async. eachSeries (self. posts, self. wrapTask (self. downPostImages), callback) ;},/*** download a single post image * @ param {Object} post article */downPos TImages: function (post, callback) {var self = this; var async = node. async; async. waterfall ([self. wrapTask (self. mkdir, self, [post]), self. wrapTask (self. getPost), self. wrapTask (self. parsePost), self. wrapTask (self. downImages),], callback);}, mkdir: function (post, callback) {var path = node. path; var url = node. url. parse (post. loc); post. dir = path. join (this. options. saveTo, path. basename (url. pat Hname); console. log ('Prepare to create directory: % s', post. dir); if (node. fs. existsSync (post. dir) {callback (null, post); console. log ('Directory: % s already exists ', post. dir); return;} node. mkdirp (post. dir, function (err) {callback (err, post); console. log ('Directory: % s created successfully ', post. dir);}) ;},/*** get post content */getPost: function (post, callback) {console. log ('start request page: % s', post. loc); node. request (post. loc, function (err, res, body) {if (! Err) post.html = body; callback (err, post); console. log ('request page succeeded: % s', post. loc) ;};},/*** parse post and obtain the image list in post */parsePost: function (post, callback) {var $ = post. $ = node.cheerio.load(post.html); post. images = $ ('. img '). map (function () {return $ (this ). attr ('bigimgsrc ');}). toArray (); callback (null, post) ;},/*** download the image in the post image list */downImages: function (post, callback) {console. log ('% d sister picture found, ready to download... ', post. images. length); node. async. eachLimit (post. images, this. options. downLimit, this. wrapTask (this. downImage, this, [post]), callback) ;},/*** download a single image */downImage: function (post, imgsrc, callback) {var url = node. url. parse (imgsrc); var fileName = node. path. basename (url. pathname); var toPath = node. path. join (post. dir, fileName); console. log ('start downloading image: % s, save to: % s, file name: % s', imgsrc, post. dir, fileName); node. request (imgsrc ). pipe (node. fs. createWriteStream (toPath )). on ('close', function () {console. log ('image download succeeded: % s', imgsrc); callback ();}). on ('error', callback) ;}}; Me2SexImages. start ();

The above is all the content of this article. I hope you will like it.