HtmlAgilityPack. dll crawlers can download Baidu music in batches (C # source code)

Define the structure of a song [csharp] public class sMp3 {public sMp3 () {} public string strSid; // sid public string strDownPage; // download page public string strEdition; // album name public string strSinger; // artist name public string strUrl; //} store the List of all the song addresses to be resolved in the List <sMp3> listSongs = new List <sMp3> (); [csharp] Create an HtmlAgilityPack. the HtmlDocument object is used to obtain and parse HTML pages. htmlDocument hdoc_Main = new HtmlAgilityPack. htmlDocument (); The path to the XPath to be operated [csharp] // homepage string inline = "http://music.baidu.com/"; string Baidu_Music_MainPage_Label = "a"; string Baidu_Music_MainPage_Value = "sid"; string inline = ". /html [1]/body [1]/div [4]/div [1]/div [1]/div [2]/div [3]/div [1] /div [1] "; // download the page such as http://music.baidu.com/song/31496563/download string Baidu_Music_DownMusic_strXpath = ". /html [1]/body [1]/div [1]/div [4] "; string Baidu_Music_DownMusic_Label =" a "; string Baidu_Music_DownMusic_Value =" href "; string Baidu_Music_DownMusic_strXpath_Title = ". /html [1]/body [1]/div [1]/div [2] "; string Baidu_Music_DownMusic_Title =" title "; how do these Xpath paths come from .. you can view the source code ,. you can also use the previously released XPathTool. One of the core functions of the address http://blog.csdn.net/witch_soya/article/details/8486893XPathTool http://download.csdn.net/detail/witch_soya/4978587: getLabelVal this function accepts an HtmlAgilityPack. htmlDocument, in fact, specifies the HTML file to be parsed. strXpath is the specified Xpath statement, so that part of the HTML content can be captured through Xpath. Then, use the strLabel and strValue parameters to extract the specified attribute value of the required tag from the extracted content. For example, HtmlAgilityPack. htmlDocument loads an html document and uses strXpath to obtain a region in the HTML document. If you specify strLabel as "a" strValue as "href, the returned Arraylist is all the hyperlink addresses of the label in the region [csharp] private ArrayList getLabelVal (HtmlAgilityPack. htmlDocument dc, string strXpath, string strLabel, string strValue) {ArrayList Arr_Label_Val = new ArrayList (); // obtain the specified node HtmlNode node = dc. documentNode. selectSingleNode (strXpath); if (node = null) {return Null;} string strXPathLabel_Val = "descendant:" + strLabel; # region try {// HtmlNodeCollection atts = node. selectNodes ("// * [@ background or @ lowsrc or @ src or @ href]"); // you can obtain the full-text-based // HtmlNodeCollection hrefs = node. selectNodes ("// a [@ href]"); // you can obtain the HtmlNodeCollection hrefs = node based on the current node. selectNodes (strXPathLabel_Val); if (hrefs = null) {return null;} foreach (HtmlNode href in hrefs) {if (Href. attributes [strValue] = null) {continue;} // The sid String strSid = href of the song is obtained here. attributes [strValue]. value; Arr_Label_Val.Add (strSid) ;}} catch (System. exception ex) {MessageBox. show (ex. toString ();} finally {// f2.Show () ;}# endregion return Arr_Label_Val;} [csharp] // obtain the Baidu music address private void button#click (object sender, EventArgs e) {HtmlWeb hw = new HtmlWeb (); string url = Baidu_Music_Ma InPage_strUrl; try {hdoc_Main = hw. load (url);} catch (System. exception ex) {MessageBox. show (ex. toString (); return;} // parse the sid Arr_sid = getLabelVal (hdoc_Main, empty, empty, Baidu_Music_MainPage_Value) of the tag obtained on the Baidu music homepage; if (Arr_sid = null) {MessageBox. show ("Resolution homepage label error"); return ;}for (int I = 0; I <Arr_sid.Count; I ++) {sMp3 sMp3Song = new sMp3 (); sMp3 Song. strSid = Arr_sid [I]. toString (); // Assembly address string strDownLoadPage = "http://music.baidu.com/song/" + Arr_sid [I] + "/download"; sMp3Song. strDownPage = strDownLoadPage; listSongs. add (sMp3Song);} // parse and download the music webpage Thread th = new Thread (DownLoadArrPage); th. start ();} [csharp] // parse the webpage public delegate void MyInvoke (string str1, string str2, string str3); private void DownLoadArrPage () {MyInvoke mi = new M YInvoke (UpdateForm); HtmlWeb hw = new HtmlWeb (); for (int I = 0; I <listSongs. count (); I ++) {string strUrl = (listSongs [I]. strDownPage ). toString (); HtmlAgilityPack. htmlDocument hdoc_DownPage = hw. load (strUrl); ArrayList Arr_DownLoadUrl = getLabelVal (hdoc_DownPage, callback, Baidu_Music_DownMusic_Label, callback); // In fact, only one hyperlink data returned by this Arry_DownLoadUrl does not need to be traversed. ListSongs [I]. strUrl = Arr_DownLoadUrl [0]. toString (); ArrayList Arr_Title = getLabelVal (hdoc_DownPage, callback, Baidu_Music_DownMusic_Label, Baidu_Music_DownMusic_Title); listSongs [I]. strEdition = Arr_Title [0]. toString (); listSongs [I]. strSinger = Arr_Title [1]. toString ();} // What is stored in the Arry_Downloadurl?/data/music/file? Links = http://zhangmenshiting.baidu.com/data2/music/31626527/3149656368400128.mp3? Xcode = 4b1b6b4117b45f71b5949db22586f5d7 for (int I = 0; I <listSongs. count (); I ++) {string strUrl = (listSongs [I]. strUrl ). toString (); // Regular Expression // @ "demo_class.asp \? Sort = ([^ x00-xff] {4}) & id \ = ([a-z0-9] +) "; Regex reg = new Regex (@" http ://(. * [a-zA-Z0-9 _]) "); var result = reg. match (strUrl ). groups; foreach (var item in result) {// listSongs [I]. strUrl = item; if (item. toString (). contains ("http: //") = false) {continue;} // Add the content to the first list box this. beginInvoke (mi, new Object [] {listSongs [I]. strEdition, listSongs [I]. strSinger, item. toString ()}); listSongs [I]. strUrl = item. toString ();}}}