Node.js Basic module HTTP, web Analytics tools Cherrio implement crawler _node.js

First, the preface
Said is the reptile preliminary study, actually did not use the reptile related third party class library, mainly used the Node.js Basic module HTTP, the Web page analysis tool Cherrio. Use HTTP to get the URL path directly to the page resource, and then use Cherrio analysis. Here I have studied the case myself to knock again, deepen understanding. In the process of coding, I first took the JQ object directly with a foreach traversal, the direct error, because JQ does not correspond to this method, only JS array can be invoked.

Second, the knowledge point
①:superagent to grab the Web tools. I haven't used it for a while.
②:cherrio Web analytics Tools, you can understand it as a service-side jquery, because the syntax is the same.
Effect chart

1, crawl the entire Web page

2, the analysis of the data, provided by examples for the case implementation examples.

Research on the source of crawler

var http=require (' http ');
 
var cheerio=require (' Cheerio ');
 
var url= ' http://www.imooc.com/learn/348 '; /**************************** printed data structure [{chaptertitle: ', videos:[{title: ', ID: '}]} *********************** /function Printcourseinfo (coursedata) {Coursedata.foreach (function (item) {var chaptertitle=
  Item.chaptertitle;
  Console.log (chaptertitle+ ' \ n ');
  Item.videos.forEach (function (video) {Console.log (' "' +video.id+ '" ' +video.title+ ');
})
 });
 
 /************* analyzes the data crawled from the Web page **************/function filterchapter (HTML) {var coursedata=[];
 var $=cheerio.load (HTML);
 var chapters=$ ('. Chapter ');
  Chapters.each (function (item) {var chapter=$ (this); var chaptertitle=chapter.find (' strong '). Text ();
 
  Locate the chapter title var videos=chapter.find ('. Video '). Children (' Li ');
 
  var chapterdata={chaptertitle:chaptertitle, videos:[]};
   Videos.each (function (item) {var video=$ (this). Find ('. Studyvideo ');
   var title=video.text (); var id=video.attr (' href '). Split ('/video ') [1];
 ChapterData.videos.push ({title:title, id:id})}) Coursedata.push (Chapterdata);
 
 });
return coursedata;
 
 } http.get (Url,function (res) {var html= ';
 Res.on (' Data ', function (data) {html+=data;
  }) Res.on (' End ', function () {var coursedata=filterchapter (HTML);
 Printcourseinfo (Coursedata);
 }). On (' Error ', function () {Console.log (' Get Course data error ');})

Resources:
Https://github.com/alsotang/node-lessons/tree/master/lesson3

http://www.imooc.com/video/7965