Htmlagilitypack C # Crawler

Main program

classProgram {Static voidMain (string[] args) {            varTMPSTR1 ="http://www.****.com/txtxz/{0}/down.html"; WebClient WC=NewWebClient ();  for(inti =54422; I <54423; i++)            {                varstr =String.Format (TMPSTR1, i); String result=WC.                Downloadstring (str); Bookdataparser Parser=Newbookdataparser (result, str); varitem =parser.                Getbookdata (); if(Item = =NULL) {Console.WriteLine ("fail to get in url{0}", i); Continue; } WC. DownloadFile (item. Downlink, item. Title+". txt");            Console.WriteLine (item); }Console.readkey (); }    }

BookDataParser.cs

/// <summary>    ///BookData parsing/// </summary>     Public classBookdataparser {PrivateString _content;//HTML content        PrivateHTMLDocument _doc;//Htmlagilitypack working with document classes        PrivateString _url;//current link, only for construction bookdata        /// <summary>        ///Construction Bookdataparser/// </summary>        /// <param name= "Content" >HTML content</param>        /// <param name= "url" >Current Link</param>         PublicBookdataparser (stringContentstringURL) {_url=URL; _content=content; _doc=NewHTMLDocument (); _doc.        loadhtml (content); }        /// <summary>        ///Parse return bookdata/// </summary>        /// <returns></returns>         Publicbookdata Getbookdata () {if(! Vaildcontent ()) {return NULL; } varBookData =NewBookData (); //Title            varTitlenode = _doc. Documentnode.selectsinglenode ("//div[@id = ' titlename ']/h1"); varstr = titleNode.ChildNodes.First (n = n.name = ="#text").            InnerHtml; Bookdata.title= str. Substring (0Str. IndexOf ("TXT")); varInfonode = _doc. Documentnode.selectnodes ("//div[@class = ' txt_info ']/span"); //Author            varSTR2 = infonode[0].            InnerText; Bookdata.authorname= str2. Substring (str2. IndexOf (':')+1); //class            varSTR3 = infonode[1].            InnerText; Bookdata.class= Str3. Substring (STR3. IndexOf (":") +1); //Date            varSTR4 =Infonode.last ().            InnerText; Bookdata.uploadtime= DateTime.Parse (STR4. Substring (STR4. IndexOf (":") +1)); //Description            varDescnode = _doc. Documentnode.selectsinglenode ("//div[@class = ' infos_txt ')"); Bookdata.description=GetDesc (Descnode); //downlink            varLinkNode1 = _doc. Documentnode.selectsinglenode ("//div[@class = ' Pan_url ']/a[last ()]"); Bookdata.downlink= Linknode1.getattributevalue ("href",NULL); //InfoLinkBookdata.infolink =_url; returnBookData; }        /// <summary>        ///handling Descnode getting a description/// </summary>        /// <param name= "Descnode" >node that describes the information</param>        /// <returns>Book description</returns>        Private stringGetDesc (Htmlnode descnode) {StringBuilder sb=NewStringBuilder (); foreach(varNodeinchdescnode.childnodes) {if(node. name=="#text")                {                    varstr =node.                    InnerText; if(!string.isnullorwhitespace (str)) {sb. Append (Httputility.htmldecode (str).                    Trim ()); }                }                if(node. name=="BR") {sb. Append ("\ n"); }            }            returnsb.        ToString (); }        /// <summary>        ///Verify that content is empty or illegal/// </summary>        /// <returns></returns>         Public BOOLvaildcontent () {if(_content = =NULL)return false; varnode = _doc. Documentnode.selectsinglenode ("//div[class= ' Blocktitle ']"); returnnode = =NULL|| Node. Innertext.startswith ("error occurred"); }    }

Htmlagilitypack C # Crawler