C # Write crawler, V2.1,

C # Write crawler, V2.1,

This is a small patch for 2.0, and there is almost no interaction for 2.0. a progress bar and a text box are added this time. The main error is webResponse because of the obtained link.

In this case

 try                {                    webResponse = (HttpWebResponse)webRequest.GetResponse();                }                catch(WebException ex)                {                    webResponse = (HttpWebResponse)ex.Response;                }

Intercept error information. We will not handle it here. In the future, we will directly determine the statecode attribute to determine whether to execute the following program.

Another change is that we used to save the obtained webpage to text.

WebRequest myRequest = WebRequest.Create("http://image.baidu.com/search/index?tn=baiduimage&ipn=r&ct=201326592&cl=2&lm=-1&st=-1&fm=result&fr=&sf=1&fmq=1466307565574_R&pv=&ic=0&nc=1&z=&se=1&showtab=0&fb=0&width=&height=&face=0&istype=2&ie=utf-8&word=" + Uri.EscapeDataString(keyWord));            HttpWebResponse myResponse = (HttpWebResponse)myRequest.GetResponse();            if (myResponse.StatusCode == HttpStatusCode.OK)            {                Stream strm = myResponse.GetResponseStream();                StreamReader sr = new StreamReader(strm);                string line = sr.ReadToEnd();

Put it all in string.

The last point is to remove the DownloadPage method. As shown above, the DownloadPage function can be implemented in the Click Event of the button. There is no need to do one thing twice.

The following is the front-end page:

Background code:

Using Newtonsoft. json; using Newtonsoft. json. linq; using System. collections. generic; using System. componentModel; using System. data; using System. drawing; using System. IO; using System. linq; using System. net; using System. text; using System. text. regularExpressions; using System. threading. tasks; using System. windows. forms; namespace Baidu image crawler V2. _ 1 {public partial class Form1: Form {public delegate vo Id AsynFunction (string s, int I); public Form1 () {InitializeComponent ();} private static string [] getLinks (string html, out int counts) {const string pattern = @ "http: // ([\ w-] + \.) + [\ w-] + (/[\ w -. /? % & =] *)? "; Regex r = new Regex (pattern, RegexOptions. ignoreCase); // create the regular expression mode MatchCollection m = r. matches (html); // obtain the matching result string [] links = new string [m. count]; int count = 0; for (int I = 0; I <m. count; I ++) {if (isValiable (m [I]. toString () {links [count] = m [I]. toString (); // extract the result count ++;} counts = count; return links;} private void button#click (object sender, EventArgs e) {string keyWord = this. textBox1.Text; WebRequest myRequest = WebRequest. create (" http://image.baidu.com /Search/index? Tn = baiduimage & ipn = r & ct = 201326592 & cl = 2 & lm =-1 & st =-1 & fm = result & fr = & sf = 1 & fmq = 1466307565574_R & pv = & ic = 0 & nc = 1 & z = & se = 1 & showtab = 0 & fb = 0 & width = & height = & face = 0 & istype = 2 & ie = UTF-8 & word = "+ Uri. escapeDataString (keyWord); HttpWebResponse myResponse = (HttpWebResponse) myRequest. getResponse (); if (myResponse. statusCode = HttpStatusCode. OK) {Stream strm = myResponse. getResponseStream (); StreamReader sr = new Strea MReader (strm); string line = sr. readToEnd (); int counts = 0; string [] str = getLinks (line, out counts); this. progressBar1.Maximum = counts; for (int I = 0; I <counts; I ++) {AsynFunction fun = new AsynFunction (savePicture); fun. beginInvoke (str [I], I, ar => {fun. endInvoke (ar); this. progressBar1.BeginInvoke (new Action () => {this. progressBar1.Value = progressBar1.Maximum;}); this. textBox2.BeginIn Voke (new Action () => {StringBuilder sb = new StringBuilder (); sb. append (Environment. newLine); // sb. append (str [I]. toString (); sb. append ("Download ended"); this. textBox2.Text + = sb. toString () ;}) ;}, fun ;}} private static bool isValiable (string url) {if (url. contains (". jpg ") | url. contains (". gif ") | url. contains (". png ") {return true; // obtain resources such as images} return false;} public void savePicture (string Path, int I) {if (path! = "" & Path! = Null) {DataClasses1DataContext db = new DataClasses1DataContext (); Uri url = new Uri (path); HttpWebRequest webRequest = (HttpWebRequest) HttpWebRequest. Create (url); webRequest. Referer =" http://image.baidu.com "; WebRequest. timeout = 30000; // sets the connection Timeout time for webRequest. allowAutoRedirect = true; webRequest. headers. set ("Pragma", "no-cache"); webRequest. userAgent = "Mozilla-Firefox-Spider (Wenanry)"; HttpWebResponse webResponse; try {webResponse = (HttpWebResponse) webRequest. getResponse ();} catch (WebException ex) {webResponse = (HttpWebResponse) ex. response;} if (webResponse! = Null & webResponse. StatusCode = HttpStatusCode. OK) {if (isValiable (path) // you can store an image in the database if it is an image. {Bitmap myImage = new Bitmap (webResponse. getResponseStream (); MemoryStream MS = new MemoryStream (); myImage. save (MS, System. drawing. imaging. imageFormat. jpeg); var p = new pictureUrl {pictureUrl1 = ms. toArray ()}; db. pictureUrl. insertOnSubmit (p); db. submitChanges (); this. progressBar1.BeginInvoke (new Action () => {this. progressBar1.Value = I;}); this. textBox2.BeginInvoke (new Action () => {StringBuilder sb1 = new StringBuilder (); sb1.Append (path); sb1.Append ("image download start" + Environment. newLine); this. textBox2.Text + = sb1.ToString () ;}}} private void button2_Click (object sender, EventArgs e) {this. close ();}}}