JavaScript programming practices

Capture 178 novels with nodejs

I read a light novel on the 178 network before the light was turned off yesterday. As a result, half of the stories were cut off. In my anger, I decided to use nodejs to write a script to capture the remaining novels to the TXT file, so that I could download the desired novels more than 10 seconds before the lights are turned off. The Code is as follows:

VaR HTTP = require ('http'); var FS = require ('fs'); var $ = require ('jquery '); vaR iconv = require ('iconv-lite '); var wopition = {flags: 'w', encoding: 'utf-8', mode: 0666} var weburl = 'HTTP: // xs.dmzj.com '; var starturl ='/1224/4376/27626.shtml '; var filewritestream = fs.createwritestream('novel2.txt', wopition); var savenovel = function (HTML) {var $ Doc = $ (HTML); var novel = $ Doc. find ('div. novel_text '). text ();/ /You need to add a line feed filewritestream. write (novel);} var gethtml = function (URL, callback) {HTTP. get (URL, function (RES) {res. setencoding ('binary '); var html = ''; console. log ('get response: '+ Res. statuscode); Res. on ('data', function (data) {HTML + = data ;}). on ('end', function () {var Buf = new buffer (HTML, 'binary '); var STR = iconv. decode (BUF, 'utf-8'); console. log ('Download: '+ URL); callback (Str. tostring ());});}). on ('error', F Unction (e) {console. log ('get error: '+ E. message)}) ;}; var get178text = function (URL) {gethtml (URL, function (data) {var $ Doc = $ (data); savenovel (data ); vaR pagenums = $ Doc. find ('div. pages '). find ('span option '). each (function (I, p) {var $ T = $ (p); if (I! = 0 & $ T. length> 0) {gethtml (weburl + $ T. ATTR ('value'), function (data) {savenovel (data) ;};}); var pages = $ Doc. find ('div. pages_novel '). find ('a # next_chapter '). ATTR ('href '); var cururl = weburl + pages; If (cururl. tostring () = URL. tostring () {// console. log ('Download complete'); return;} get178text (cururl) ;});} get178text (weburl + starturl );

Nodejs write crawler is really super convenient... It only takes tens of minutes to write code. It takes a long time to install the jquery module of nodejs in windows.

After I finish writing the story, I find that the novels on the 1949th can be directly downloaded...

It doesn't matter. The technology house should constantly explore its potential for the things you love.

When I have time to improve it, I will summarize the usage of filters in jquery:

1. Save Novels in HTML format to improve the reading experience

2. Download images at the same time

In the webpage source code, the HTML encoding of Chinese characters and the conversion of specific Chinese characters are often seen on the webpage while viewing the HTML source code. "& #35745; & #31639; & #26426; & #25805; & #20316; & #31995; & #32479; "Chinese characters are displayed on the webpage. Each character in HTML is encoded accordingly. You can use VaR char = string. fromcharcode (CODE) to convert a number into a character:

// Function () {var STR = "& #35745; & #31639; & #26426; & #25805; & #20316; & #31995; & #32479; "; var pattern =/\ D {1,}/g; var chars; chars = Str. match (pattern); var Cn = ''; For (VAR I = 0; I <chars. length; ++ I) {var S = chars [I]; S = string. fromcharcode (s); CN + = s;} console. log (CN );}());

If Regexp has a flag, the match () method performs a global search and finds all matching substrings in stringobject. The JS regular expression is so simple:

var fs = require('fs');var fileWriteStream = fs.createWriteStream('NJSON.JS',{ flags: 'w',encoding: 'utf-8',mode: 0666 });var fileReadStream = fs.createReadStream("JSON.txt",{flags : 'r',encoding : 'utf-8',mode : 0666});//fileReadStream.pipe(fileWriteStream);fileReadStream.on('data',function(data){data =  data.toString().replace(/&#\d+;/g,function(word){return String.fromCharCode(word.substr(2,word.length - 3));});console.log(data);fileWriteStream.write(data);});

Use VaR code = Char. charcodeat () to convert the character to a number:

(Function () {var Cn = 'computer OS '; var STR = ''; For (VAR I = 0; I <CN. length; ++ I) {var code = cn [I]. charcodeat (); STR + = '& #' + code + ';'} console. log (STR );}());