Nodejs uses phantomjs to download the webpage

Nodejs uses phantomjs to download the webpage

This article mainly introduces how nodejs uses phantomjs to download webpages. For more information, see.

The function is actually very simple. You can use phantomjs.exe to collect url-loaded resources and use sub-processes to start nodejs to load all resources. For css resources, match the css content, download the url Resource

Of course, the function is still very simple. In the case of responsive design and asynchronous loading, there are still a lot of resources that cannot be downloaded and need to be processed according to the actual situation.

First, download nodejs and phantomjs.

The following is down. js executed by phantomjs.exe.

?

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35

Var page = require ('webpage'). create (), System = require ('system '); Var spawn = require ("child_process"). spawn   If (system. args. length = 1 ){ Console. log ('usage: netsniff. js <some URL> '); Phantom. exit (1 ); } Else { Var urls = []; Page. address = system. args [1]; Page. onResourceReceived = function (res ){ If (res. stage = 'start '){ Urls. push (res. url ); } }; Page. open (page. address, function (status ){ Var har; If (status! = 'Success '){ Console. log ('fail to load the address '); Phantom. exit (1 ); } Else { Console. log ('down resource' + urls. length + 'urls .'); Var child = spawn ("node", ["-- harmony", "downHtml. js", urls. join (',')]) Child. stdout. on ("data", function (data ){ Console. log (data ); }) Child. stderr. on ("data", function (data ){ Console. log (data ); }) Child. on ("exit", function (code ){ Phantom. exit (); }) } }); }

Below is the corresponding node running downHtml. js

?

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83

"Use strict "; Var fs = require ('fs '); Var http = require ('http '); Var path = require ('path '); Var r_url = require ('url ');   Var dirCache ={}; // check whether the cache is reduced Function makedir (pathStr, callback ){ If (dirCache [pathStr] = 1 ){ Callback (); } Else { Fs. exists (pathStr, function (exists ){ If (exists = true ){ DirCache [pathStr] = 1; Callback (); } Else { Makedir (path. dirname (pathStr), function (){ Fs. mkdir (pathStr, function (){ DirCache [pathStr] = 1; Callback (); }) }); } }) } };   Var reg =/[:,] \ s * url \ (['"]?. *? (\ 1) \)/g Var reg2 = // \ (['"]?) (.*?) (\ 1 )\)/ Var isDownMap = {}; Var downImgFromCss = function (URL ){ Http. get (URL, function (res ){ // Console. log (path. resolve (process. cwd (), 'index.min.css ')) // Res. pipe (fs. createWriteStream (path. resolve (process. cwd (), 'index.min.css '))); Var body = ""; Res. setEncoding ('utf8 '); Res. on ('data', function (chunk ){ Body + = chunk; }); Res. on ('end', function (){ Var match = body. match (reg ); For (var I = 0, len = match. length; I <len; I ++ ){ Var m = match [I]. match (reg2 ); If (m & m [2]) { Var url = m [2]; Let imgUrl = r_url.resolve (URL, url ); If (! IsDownMap [imgUrl]) { Var uo = r_url.parse (imgUrl ); Let filepath = CWD + '/' + uo. hostname + uo. pathname; Makedir (path. dirname (filepath), function (){ Http. get (imgUrl, function (res ){ Res. pipe (fs. createWriteStream (filepath )); }) }) IsDownMap [imgUrl] = 1; } } } }); }); }   Var URLS = process. argv [2]. split (','); Var CWD = process. cwd (); // Download resources URLS. forEach (function (URL ){ Var uo = r_url.parse (URL ); Var filepath; If (uo. pathname = '/' | uo. pathname = ''){ Filepath = CWD + '/' + uo. hostname + '/index.html '; } Else { Filepath = CWD + '/' + uo. hostname + uo. pathname; } Makedir (path. dirname (filepath), function (){ Http. get (URL, function (res ){ If (URL.indexOf('.css ')! =-1 | (res. headers ["content-type"] & res. headers ["content-type"]. indexOf ('text/css ')! =-1 )){ Console. log ('down images form css file: '+ URL + '.'); DownImgFromCss (URL ); } Res. pipe (fs. createWriteStream (filepath )); }) }); });

Put down. js downHtml. js in the same folder and run it through the following cmd:

D: \ phantomjs-2.0.0-windows \ bin \ phantomjs.exe down. js http://www.youku.com/

The above is all the content of this article. I hope you will like it.

Note <>: For more exciting tutorials, please pay attention to the help house programming