類比web訪問有登入且有驗證碼的登入後抓取資料，web抓取

類比web訪問有登入且有驗證碼的登入後抓取資料，web抓取
類比web訪問有登入且有驗證碼的登入後抓取資料

1 取驗證碼1 在表單上放一個picturebox (imgValidate)存放擷取的驗證碼圖片，
2 用瀏覽器的開發人員工具firefox (f12) 分析出驗證碼的網址
private void GetValidateImage()
        {
            cookies = new CookieContainer();
            string strUrl = "http://www.xxx.com/ValidateCodePicture.aspx?Key="+strValidCode;  //驗證碼頁面 strValidCode這個隨機碼要先取出來

            CookieContainer cc = new CookieContainer();
            HttpWebRequest request = (HttpWebRequest)WebRequest.Create(strUrl);

            //set request args
            request.Method = "Get";
            request.CookieContainer = cc;
            request.KeepAlive = true;

            //request.ContentType = "application/x-www-form-urlencoded; charset=UTF-8";
            request.ContentType = "text/html";

            //類比goole瀏覽器訪問
            request.UserAgent =
                "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36";
            //request.Referer = strUrl;
            request.Headers.Add("x-requested-with:XMLHttpRequest");
            request.Headers.Add(HttpRequestHeader.AcceptLanguage, "zh-CN,zh;q=0.8,en;q=0.6,nl;q=0.4,zh-TW;q=0.2");
            //request.ContentLength = postdataByte.Length;  text/html; charset=utf-8
            request.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8";
            request.AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip |
                                             DecompressionMethods.None;
            //支援跳轉頁面，查詢結果將是跳轉後的頁面
            ////request.AllowAutoRedirect = true;

            request.Headers.Add("Accept-Encoding", "gzip, deflate");
            if (request.Method == "POST")
            {
                (request as HttpWebRequest).ContentType = "application/x-www-form-urlencoded";
            }

            HttpWebResponse response = (HttpWebResponse)request.GetResponse();

            MemoryStream ms = null;
            using (var stream = response.GetResponseStream())
            {
                Byte[] buffer = new Byte[response.ContentLength];
                int offset = 0, actuallyRead = 0;
                do
                {
                    actuallyRead = stream.Read(buffer, offset, buffer.Length - offset);
                    offset += actuallyRead;
                }
                while (actuallyRead > 0);
                ms = new MemoryStream(buffer);
            }
            response.Close();

            cookies = request.CookieContainer; //儲存cookies
            strCookies = request.CookieContainer.GetCookieHeader(request.RequestUri); //把cookies轉換成字串

            Bitmap sourcebm = new Bitmap((Stream)ms);//初始化Bitmap圖片
            imgValidate.Image = sourcebm;
        }

2 取js賦值的內容  有的網頁用查看網頁原始碼的方式看不到控制項的值，需要用到下面的方法
  即用C#內建的webbrowse來載入網頁，再用webBrowser1.Document來取對應控制項的值，如
  tring strMsg2 = webBrowser1.Document.GetElementById("hdValidateCodeID").OuterHtml;
3 取得要提交的參數  如果是asp.net的網頁還有提交”__EVENTTARGET“,"__EVENTARGUMENT","__VIEWSTATE"這三個參數，這個也可以在開發人員工具-網路-參數裡看到
可以用httpRequest先取得原始碼再分析出
這裡用的是webbrowse裡載入好的
  
        private void GetViewState()
        {
            string strMsg = webBrowser1.Document.GetElementById("__VIEWSTATE").OuterHtml;
            //取viewstate value
            //<INPUT id=__VIEWSTATE type=hidden value=/wEPDwUKMTg0NTk3Mjg2N2Rk name=__VIEWSTATE>
            MatchCollection mc = Regex.Matches(strMsg, "id=__VIEWSTATE.*(?<viewstate>value[^>]*)", RegexOptions.IgnoreCase);

            if (mc.Count > 0)
            {
                foreach (Match m in mc)
                {
                    strViewState = m.Groups["viewstate"].Value.ToString().Trim();
                    if (strViewState.Length > 0)
                    {
                        strViewState = strViewState.Replace("value=", "").Replace("\"", "").Replace("\\", "").Replace("name=__VIEWSTATE","").Replace(" ","");
                    }
                }
            }

            //<INPUT id=hdValidateCodeID type=hidden value=c1b52d3a-1f8b-1dc4-0d44-32a4b46ef8af name=hdValidateCodeID>
            string strMsg2 = webBrowser1.Document.GetElementById("hdValidateCodeID").OuterHtml;
            MatchCollection mc2 = Regex.Matches(strMsg2, "id=hdValidateCodeID.*(?<validatecode>value[^>]*)", RegexOptions.IgnoreCase);

            if (mc2.Count > 0)
            {
                foreach (Match m in mc2)
                {
                    strValidCode = m.Groups["validatecode"].Value.ToString().Trim();
                    if (strValidCode.Length > 0)
                    {
                        strValidCode = strValidCode.Replace("value=", "").Replace("\"", "").Replace("\\", "").Replace("/", "").Replace("name=hdValidateCodeID","").Replace(" ","");
                    }
                }
            }
            txtValidCode.Text = strValidCode;
            txtViewState.Text = strViewState;

            //String 的Cookie　要轉成　Cookie型的　並放入CookieContainer中  
            string cookieStr = webBrowser1.Document.Cookie;
            string[] cookstr = cookieStr.Split(';');

            foreach (string str in cookstr)
            {
                try
                {
                    string[] cookieNameValue = str.Split('=');
                    Cookie ck = new Cookie(cookieNameValue[0].Trim().ToString(), cookieNameValue[1].Trim().ToString());
                    ck.Domain = "XXX.com"; //必須寫對  
                    myCookieContainer.Add(ck);
                }
                catch
                {
                }
            }  
        }
  
4 登入並且存取cookie提交參數，並存下cookie,供後續用
private void Login()
        {
            cookies = new CookieContainer();
            string strUrl = "http://www.xxx.com/Login.aspx";  //驗證碼頁面

            HttpWebRequest request = (HttpWebRequest)WebRequest.Create(strUrl);

            //set request args
            request.Method = "POST";
            request.CookieContainer = myCookieContainer;
            request.KeepAlive = true;

            //request.ContentType = "application/x-www-form-urlencoded; charset=UTF-8";
            request.ContentType = "text/html";


            //類比goole瀏覽器訪問
            request.UserAgent =
                "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36";
            //request.Referer = strUrl;
            request.Headers.Add("x-requested-with:XMLHttpRequest");
            request.Headers.Add(HttpRequestHeader.AcceptLanguage, "zh-CN,zh;q=0.8,en;q=0.6,nl;q=0.4,zh-TW;q=0.2");
            //request.ContentLength = postdataByte.Length;  text/html; charset=utf-8
            request.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8";
            request.AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip |
                                             DecompressionMethods.None;
            //支援跳轉頁面，查詢結果將是跳轉後的頁面
            ////request.AllowAutoRedirect = true;

            request.Headers.Add("Accept-Encoding", "gzip, deflate");
            if (request.Method == "POST")
            {
                (request as HttpWebRequest).ContentType = "application/x-www-form-urlencoded";
            }

            //---begin
           
            string postData = string.Format("txtUserName={0}&txtPassword={1}&txtValidateCode={2}&hdValidateCodeID={3}&ddlLanguage=CN&btnLogin=登入&__EVENTTARGET=&__EVENTARGUMENT=&__VIEWSTATE={4}", txtUserName.Text, txtPassword.Text, txtValidate.Text,strValidCode,strViewState);  //這裡按照前面FireBug中查到的POST字串做相應修改。
            byte[] postdatabyte = Encoding.UTF8.GetBytes(postData);
           
            request.ContentLength = postdatabyte.Length;

            using (Stream stream = request.GetRequestStream())
            {
                stream.Write(postdatabyte, 0, postdatabyte.Length);
            }
            //---end---

            HttpWebResponse response = (HttpWebResponse)request.GetResponse();

            //StreamReader reader = new StreamReader(response.GetResponseStream(), Encoding.GetEncoding("gb2312"));
            StreamReader reader = new StreamReader(response.GetResponseStream(), Encoding.UTF8);
            string strMsg = reader.ReadToEnd();

            response.Close();

            cookies = request.CookieContainer; //儲存cookies,後面再請求其它網頁就可用這個cookie,不用在登入了
            lbLogin.Text = "已登入";
            btnSearchResume.Enabled = true;

        }