C # collection program source code Introduction

I wrote a collection program for my work. If I offend your website, I am sorry !! Ah ~! I am just a common programmer. namespace CJ {public partial class Form1: Form {public int proxy = 0; public int keyi = 0; public int keyj = 0; public int keym = 0; public int keyn = 0; public int sum = 0; public string newurl = ""; public string cururl = ""; public string dirname = ""; public string curdir = ""; public string responseFromServer = ""; public string filename = ""; public string SQL = ""; public str Ing mulu = ""; StringBuilder sbs = new StringBuilder (); List <Class1> cls = new List <Class1> (); public ArrayList al = new ArrayList (); public string insertdl = "insert into mzinedl values ("; public string insertxl = "insert into mzinexl values ("; public string insertinfo = "insert into mzineinfo values ("; public string insertwz = "insert into mzinewz values ("; public Form1 () {InitializeComponent () ;} /// <Summary> /// Save the webpage // </summary> /// <param name = "FILE_NAME"> file path </param> /// <param name = "data"> data </param> public void TextToFile (string FILE_NAME, string data) {if (File. exists (FILE_NAME) {return;} using (StreamWriter sw = File. createText (FILE_NAME) {sw. write (data); sw. close ();}} /// <summary> /// download the object /// </summary> /// <param name = "PageUrl"> URL </param> /// <param name = "filenam E "> Save file path </param> public void DownFile (string PageUrl, string filename) {if (! Directory. exists (filename) {Directory. createDirectory (filename);} string path = PageUrl. substring (PageUrl. lastIndexOf ("/") + 1); string dirname = filename + "\" + path; if (File. exists (dirname) {return;} else {try {WebClient wc = new WebClient (); WebProxy wp = new WebProxy (al [proxy]. toString (), true); wc. proxy = wp; wc. downloadFile (PageUrl, dirname);} catch (WebException ex) {if (ex. stat Us = WebExceptionStatus. connectFailure) {// unable to connect to the remote server, -- change proxy IP // MessageBox. show (ex. toString (); proxy ++; if (proxy> = al. count) {al = ReadIPproxy ("e: \ test.txt"); // initialize the proxy IP} DownFile (PageUrl, filename);} else if (ex. status = WebExceptionStatus. timeout) {// Timeout -- change proxy IP // MessageBox. show (ex. toString (); proxy ++; if (proxy> = al. count) {al = ReadIPproxy ("e: \ test.txt"); // initialize the proxy IP} DownFile (PageUrl, filename);} else if (ex. status = WebExceptionStatus. protocolError) {// file not found -- Jump Out // MessageBox. show (ex. toString (); return ;}}}} /// <summary> /// read the file /// </summary> /// <param name = "FILE_NAME"> file path </param> /// <returns> data </returns> public ArrayList ReadIPproxy (string FILE_NAME) {using (StreamReader sr = File. openText (FILE_NAME) {String input; while (input = sr. readLine ())! = Null) {al. add (input);} sr. close ();} return al;} // <summary> // database // </summary> public void Executesql () {SqlHelper. executeNonQuery (SqlHelper. sqlstr, CommandType. text, sbs. toString (), null );} /// <summary> /// read the file /// </summary> /// <param name = "FILE_NAME"> file path </param> /// <returns> data </returns> public string FileToText (string FILE_NAME) {string data; using (StreamReader sr = File. ope NText (FILE_NAME) {data = sr. readToEnd (); sr. close () ;}return data ;} /// <summary> /// Save the SQL // </summary> /// <param name = "SQL"> </param> public void SaveSqls (string SQL) {sbs. append (SQL ). append ("") ;}/// <summary> /// when the request fails, repeated operations // </summary> /// <param name = "PageUrl"> </param> /// <returns> </returns> public string ToServer (string PageUrl) {string responseFromServer = ""; try {while (1 = = 1) {WebRequest request = WebRequest. create (PageUrl); WebProxy wp = new WebProxy (al [proxy]. toString (), true); request. proxy = wp; request. timeout = 1000*60; HttpWebResponse response = (HttpWebResponse) request. getResponse (); Stream dataStream = response. getResponseStream (); StreamReader reader = null; try {reader = new StreamReader (dataStream, System. text. encoding. default); responseFromServer = Reader. readToEnd () ;}catch {proxy ++; if (proxy> = al. count) {al = ReadIPproxy ("e: \ test.txt"); // initialize the proxy IP} ToServer (PageUrl) ;}; reader. close (); dataStream. close (); response. close (); if (responseFromServer. contains ("refresh") | responseFromServer = "") {proxy ++; if (proxy> = al. count) {al = ReadIPproxy ("e: \ test.txt"); // initialize the proxy IP} // ToServer (PageUrl) ;}else {break ;}} catch (webti Ti On ex) {if (ex. status = WebExceptionStatus. protocolError) {responseFromServer = "" ;}else {proxy ++; if (proxy> = al. count) {al = ReadIPproxy ("e: \ test.txt"); // initialize the proxy IP} ToServer (PageUrl) ;}} return responseFromServer ;} /// <summary> /// Save the XML file /// </summary> public void SaveXmls () {string pathxml = ""; foreach (Class1 c in cls) {Class1 s = c; pathxml = s. address; if (! File. exists (pathxml) {XmlSerializer xs = new XmlSerializer (typeof (Class1); Stream stream = new FileStream (pathxml, FileMode. create, FileAccess. write, FileShare. readWrite); xs. serialize (stream, s); stream. close ();}}} /// <summary> /// remove the HTMl Tag /// </summary> /// <param name = "Html"> </param> /// <param name = "RegStr"> </param> // <returns> </returns> public static string Remove (string Html) {// R Egex Reg = new Regex (RegStr); // foreach (Match m in Reg. matches (Html) // {// Html = Html. replace (m. value, ""); //} // return Html. trim (); string regesstr = "<. *?> "; Return Regex. replace (Html, regesstr, string. empty, RegexOptions. ignoreCase);} public static string FilterScript (string content) {string regexstr = @ "<(script) [^>] *> (\ s * | .) * </\ 1> "; return Regex. replace (content, regexstr, string. empty, RegexOptions. ignoreCase );} /// <summary> /// mark all the risks that have been omitted /// </summary> /// <param name = "html"> </param> /// <returns> </returns> public string wipeScript (string html) {System. text. regularExpressions. regex regex1 = new System. text. regularExpressions. regex (@ "(<script) {1,} [^ <>] *> [^ \ 0] * (<\/script>) {1,}", System. text. regularExpressions. regexOptions. ignoreCase); System. text. regularExpressions. regex regex2 = new System. text. regularExpressions. regex (@ "href * = * [\ s \ S] * script *:", System. text. regularExpressions. regexOptions. ignoreCase); System. text. regularExpressions. R Egex regex3 = new System. text. regularExpressions. regex (@ "on [\ s \ S] * =", System. text. regularExpressions. regexOptions. ignoreCase); System. text. regularExpressions. regex regex4 = new System. text. regularExpressions. regex (@ "<iframe [\ s \ S] + </iframe *>", System. text. regularExpressions. regexOptions. ignoreCase); System. text. regularExpressions. regex regex5 = new System. text. regularExpressions. regex (@ "<frameset [\ S \ S] + </frameset *> ", System. text. regularExpressions. regexOptions. ignoreCase); html = regex1.Replace (html, ""); // filter <script> </script> tags html = regex2.Replace (html ,""); // filter href = javascript: (<A>) attribute html = regex3.Replace (html, "_ disibledevent ="); // filter other controls on... event html = regex4.Replace (html, ""); // filter iframe html = regex5.Replace (html, ""); // filter frameset return html;} public void HtmlSource (string ur Lpri) {// file path to be written filename = "E :\\ Guan 2 \ magazine.html"; if (! Directory. exists ("E: \ 2") {Directory. createDirectory ("E :\\ 2");} if (File. exists (filename) {responseFromServer = FileToText (filename); // Yes} else {responseFromServer = ToServer (urlpri); // does not exist} sum ++; if (responseFromServer! = "") {// Analyze the content TextToFile (filename, responseFromServer); MatchCollection mc = Regex. matches (responseFromServer, @ "href =" "/magazine /(. *) ""> <B> (. *) </B> ", RegexOptions. ignoreCase); foreach (Match m in mc) {newurl = m. groups [1]. value; dirname = m. groups [2]. value; int key = ++ keyi; SQL = insertdl + key + ", '" + dirname + "')"; SaveSqls (SQL); cururl = urlpri + newurl; curdir = "E :\\ Guan 2 \\" + dirname; one (c Ururl, curdir, key) ;}savexmls (); Executesql (); this. textBox1.Text = sum. ToString (); MessageBox. Show ("collection successful! ") ;}} Public void one (string urlpri, string _ dirname, int _ key) {// file path to be written filename = _ dirname +" \ "+ urlpri. substring (urlpri. lastIndexOf ("/") + 1); if (! Directory. exists (_ dirname) {Directory. createDirectory (_ dirname);} if (File. exists (filename) {responseFromServer = FileToText (filename);} else {responseFromServer = ToServer (urlpri);} sum ++; if (responseFromServer! = "") {TextToFile (filename, responseFromServer); MatchCollection mc = Regex. matches (responseFromServer, @ "href = ""\. \. /(. * list.html) "" [\ s \ S] *? (. *?) ", RegexOptions. ignoreCase); foreach (Match m in mc) {newurl = m. groups [1]. value; dirname = m. groups [2]. value; cururl = "http://www.zydg.net/magazine/" + newurl; curdir = _ dirname + "\" + dirname; two (cururl, curdir, _ key );}}} public void two (string urlpri, string _ dirname, int _ key) {filename = urlpri. substring (0, urlpri. lastIndexOf ("/"); filename = filename. substring (filename. lastIndexOf ("/") + 1) + ". html"; filename = _ dirname + "\" + filename; if (! Directory. exists (_ dirname) {Directory. createDirectory (_ dirname);} if (File. exists (filename) {responseFromServer = FileToText (filename);} else {responseFromServer = ToServer (urlpri);} sum ++; if (responseFromServer! = "") {TextToFile (filename, responseFromServer); Match mc = Regex. Match (responseFromServer, @ "journal \ s + period :(.*?) <Br> [\ s \ S] *? Compilation \ s + series :(.*?) <Br> [\ s \ S] *? Output \ s + :(.*?) <Br> [\ s \ S] *? Contact Number :(.*?) <Br> [\ s \ S] *? E-mail :(.*?) <Br> [\ s \ S] *? Social media \ s + address :(.*?) <Br> [\ s \ S] *? Mail \ s + Editor :(.*?) <Br> [\ s \ S] *? Mail Code :(.*?) <Br> [\ s \ S] *? Overseas issue code :(.*?) <Br> [\ s \ S] *? International Standard Publication No :(.*?) <Br> [\ s \ S] *? Unified Chinese publication number :(.*?) </Td> ", RegexOptions. ignoreCase); Match content = Regex. match (responseFromServer, @ "journal \ s + things \ s + \ s + :::... ([\ s \ S] *?)... ::: \ S + indexed period number list ", RegexOptions. multiline); int key = ++ keyj; SQL = insertxl + keyj + "," + _ key + ", '" + dirname + "', '" + mc. groups [1]. value + "','" + mc. groups [2]. value + "','" + mc. groups [3]. value + "','" + mc. groups [4]. value + "','" + mc. groups [5]. value + "','" + mc. groups [6]. value + "','" + mc. groups [7]. value + "','" + mc. groups [8]. value + "','" + mc. groups [9]. value + "','" + mc. groups [10]. value + "','" + Mc. groups [11]. value + "','" + Remove (content. groups [1]. value) + "')"; SaveSqls (SQL); MatchCollection mc2 = Regex. matches (responseFromServer, @ "href = '(. *?) '\ S + target. *> (.*?) </A> ", RegexOptions. ignoreCase); foreach (Match m2 in mc2) {newurl = m2.Groups [1]. value; dirname = m2.Groups [2]. value. replace ("year ","-"). replace ("Number ",""). replace ("period", ""); cururl = urlpri. substring (0, urlpri. lastIndexOf ("/") + 1) + newurl; curdir = _ dirname + "\" + dirname; three (cururl, curdir, key, dirname );}}} public void three (string urlpri, string _ dirname, int _ key, string qishu) {// path of the file to be written Filename = _ dirname + "\" + urlpri. Substring (urlpri. LastIndexOf ("/") + 1); if (! Directory. exists (_ dirname) {Directory. createDirectory (_ dirname);} if (File. exists (filename) {responseFromServer = FileToText (filename);} else {responseFromServer = ToServer (urlpri);} sum ++; if (responseFromServer! = "") {TextToFile (filename, responseFromServer); Match m = Regex. Match (responseFromServer, @ "src = 'face _(.*?) '", RegexOptions. IgnoreCase); string photoName =" "; if (m. Groups [1]. Value. Trim ()! = "") {PhotoName = urlpri. substring (0, urlpri. lastIndexOf ("/") + 1) + "face _" + m. groups [1]. value; DownFile (photoName, _ dirname);} int key = ++ keym; SQL = insertinfo + key + "," + _ key + ", '"+ qishu +"', '"+ _ dirname +" \ "+" face _ "+ m. groups [1]. value + "')"; SaveSqls (SQL); MatchCollection mc2 = Regex. matches (responseFromServer, @ "href = '(\ d).html ?) '[\ S \ S] *? <Font \ s + color = black> (.*?) </A> | & lt; font [^>] *?> [(. +?)] ", RegexOptions. ignoreCase); foreach (Match m2 in mc2) {newurl = m2.Groups [1]. value; string muName = m2.Groups [3]. value; if (muName = "") {muName = mulu;} string lstr = m2.Groups [2]. value; string s1 = ""; string s2 = ""; if (lstr! = "") {If (lstr. contains (". ") {s1 = lstr. substring (0, lstr. indexOf (". "); s2 = lstr. substring (lstr. lastIndexOf (". ") + 1);} else {s1 = lstr; s2 =" ";} int k2 = ++ keyn; SQL = insertwz + k2 +", "+ key + ", '"+ muName +"', '"+ s1 +"', '"+ s2 +"') "; SaveSqls (SQL); cururl = urlpri. substring (0, urlpri. lastIndexOf ("/") + 1) + newurl; curdir = _ dirname; four (cururl, curdir, k2);} mulu = muName ;}} Public void four (string urlpri, string _ dirname, int _ key) {filename = _ dirname + "\" + urlpri. substring (urlpri. lastIndexOf ("/") + 1); if (! Directory. exists (_ dirname) {Directory. createDirectory (_ dirname);} if (File. exists (filename) {responseFromServer = FileToText (filename);} else {responseFromServer = ToServer (urlpri);} sum ++; if (responseFromServer! = "") {TextToFile (filename, responseFromServer); // The analysis content matches m = Regex. Match (responseFromServer, @ "body start --> (? <Text> [\ s \ S] *?) <! -- End of the text ", RegexOptions. ignoreCase); string content = m. groups ["text"]. value; // get all content of the body string c = FilterScript (content); c = Remove (c); // get the filtered body content // Match MS = Regex. match (c, @ "Start text --> (? <Text> [\ s \ S] *?) <! -- End of the text ", RegexOptions. ignoreCase); // set the name of the XML file to be saved. string xmlname = urlpri. substring (urlpri. lastIndexOf ("/") + 1, urlpri. lastIndexOf (". ")-urlpri. lastIndexOf ("/"); string pathxml = _ dirname + "\" + xmlname + "xml "; // pass the path and name together to Class1 cs = new Class1 (_ key, c, pathxml); cls. add (cs); // serialization successful MatchCollection mc = Regex. matches (responseFromServer, @ "(. *) "" \ s + hspace | HreF = "" ([^>] * PDF) "") ", RegexOptions. ignoreCase); foreach (Match m2 in mc) {string imgurl = m2.Groups ["imgs"]. value. trim (); // obtain the name of a single image string zhuurl = urlpri. substring (0, urlpri. lastIndexOf ("/") + 1); if (imgurl! = "") {String jurl = zhuurl + imgurl; // obtain the absolute DownFile (jurl, _ dirname) of the image;} string url = m2.Groups ["pdfs"]. value. trim (); // get the name of a single PDF if (URL! = "") {String jurl = zhuurl + response URL; // obtain the absolute pdf path DownFile (jurl, _ dirname) ;}}} private void btnOK_Click (object sender, eventArgs e) {al = ReadIPproxy ("e: \ test.txt"); // initialize the proxy IP HtmlSource ("http://www.zydg.net/magazine/");} private void button#click (object sender, EventArgs e) {Application. exit ();}}}