Accumulate [C #] --- capture the data you want on the page,

Accumulate [C #] --- capture the data you want on the page,

Effect:

Description: This function captures a foreign website that detects PM2.5. Read the website data in real time and save it to the database. Refresh every hour.

Address: http://beijing.usembassy-china.org.cn/070109air.html

Filtered address: http://utils.usembassy.gov/feed2js/feed2js.php? Src = http % 3A % 2F % 2Fwww.stateair.net % 2 Fweb % 2 Frss % 2F1% 2F1. xml & amp; desc = 1 & amp; num = 7 & amp; targ = y & amp; utf = y & amp; pc = y & amp; words = 40 & amp;

Ideas: Capture all the data on the page, save it to the txt file, read the txt file in one row, use split and substring to capture the desired data, and save it to the database, when you insert a database, check whether the database already exists. If not, insert the database.

Code:

Using System; using System. collections. generic; using System. configuration; using System. data; using System. data. sqlClient; using System. IO; // using System. linq; using System. net; using System. text; using System. text. regularExpressions; // using System. threading. tasks;/********************************* created: qingapple * Creation Time: * description: obtain PM2.5 *****************************/namespace GetUSAData {class in the United States of America Program {// public static string GetURL = System. Configuration. ConfigurationSettings. deleettings ["GetURL"]; // get the data address public static string GetURL = "http://utils.usembassy.gov/feed2js/feed2js.php? Src = http % 3A % 2F % 2Fwww.stateair.net % 2 Fweb % 2 Frss % 2F1% 2F1. xml & amp; desc = 1 & amp; num = 7 & amp; targ = y & amp; utf = y & amp; pc = y & amp; words = 40 & amp; "; public static string txtURL = System. configuration. configurationSettings. deleetask[ "txtURL"]; // The Path to the txt file is public static string conn = ConfigurationManager. connectionStrings ["ConnectionString"]. toString (); static void Main (string [] args) {LoadGO ();} public static void LoadGO () {GetUSA (); List <string []> getlist = Read (txtURL); // Delete txt if (File. exists (txtURL) {// delete the File if it Exists. delete (txtURL);} if (getlist. count> 0) {for (int I = getlist. count-1; I>-1; I --) {DateTime dtime = DateTime. parse (getlist [I] [0]. toString (); string getTime = dtime. toString ("yyyy-MM-dd HH: mm"); string controlTime = dtime. toString ("yyyy-MM-dd"); float LatestHourdata1 = float. parse (get List [I] [2]); int LatestHourdata2 = Convert. toInt32 (getlist [I] [3]); float Avgdata1 = 0; int Avgdata2 = 0; string Avgdata3 = getlist [I] [4]. toString (); List <SqlParameter> listWhere = new List <SqlParameter> (); listWhere. add (new SqlParameter ("@ strDatetime", controlTime); string sqlSelect = @ "SELECT count (*) as allcount, sum (LatestHourdata1) as LatestHourdata1_avg, sum (LatestHourdata2) as LatestHourdata 2_avg FROM T_twitter where ([LatestHourdata1] is not null or [LatestHourdata2] is not null or [Avgdata1] is not null or [AvgData2] is not null) and CONVERT (varchar (100), [datetime], 23) = @ strDatetime "; DataTable sumDT = ControlDB (sqlSelect, listWhere," select "); // The sum of the query results is used to calculate the daily average if (sumDT. rows. count> 0) {foreach (DataRow itemDR in sumDT. rows) {int allcount = Convert. toInt32 (itemDR ["allcount"]. toStr Ing (); // total number of current dates in the database if (allcount> 0) {if (itemDR ["LatestHourdata1_avg"]! = Null) {Avgdata1 = float. parse (itemDR ["LatestHourdata1_avg"]. toString (); // total LatestHourdata1_avg in the database Avgdata1 = (Avgdata1 + LatestHourdata1)/(allcount + 1); // (total database + latest) /(total number of databases + 1) = daily average value} if (itemDR ["LatestHourdata2_avg"]! = Null) {Avgdata2 = Convert. toInt32 (itemDR ["LatestHourdata2_avg"]. toString (); // total LatestHourdata2_avg in the database Avgdata2 = (Avgdata2 + LatestHourdata2)/(allcount + 1); // (total database + latest) /(total number of databases + 1) = daily average} // determine the average severity of PM2.5 based on website Rules if (Avgdata2> = 0 & Avgdata2 <= 50) {Avgdata3 = "Good (at 24-hour exposure at this level)";} else if (Avgdata2 >=51 & Avgdata2 <= 100) {Avgdata3 = "Moderate (at 24-hour Exposure at this level) ";} else if (Avgdata2 >=101 & Avgdata2 <= 150) {Avgdata3 =" Unhealthy for Sensitive Groups (at 24-hour exposure at this level) ";}else if (Avgdata2 >=151 & Avgdata2 <= 200) {Avgdata3 =" Unhealthy (at 24-hour exposure at this level )";} else if (Avgdata2> = 201 & Avgdata2 <= 300) {Avgdata3 = "Very Unhealthy (at 24-hour exposure at this level)";} else {Avgdata3 = "Hazardous (at 24-hour exposure at this level)" ;}} else {Avgdata1 = LatestHourdata1; Avgdata2 = LatestHourdata2 ;}}} list <SqlParameter> pars = new List <SqlParameter> (); pars. add (new SqlParameter ("@ whereDatetime", getTime); pars. add (new SqlParameter ("@ datetime", getTime); pars. add (new SqlParameter ("@ LatestHourdata1", LatestHourdata1); pars. add (new SqlParameter ("@ LatestHourdata2", LatestH Ourdata2); pars. add (new SqlParameter ("@ LatestHourdata3", getlist [I] [4]. toString (); pars. add (new SqlParameter ("@ Avgdata1", Avgdata1); pars. add (new SqlParameter ("@ Avgdata2", Avgdata2); pars. add (new SqlParameter ("@ Avgdata3", Avgdata3); string SQL = @ "if not exists (select * from dbo. t_twitter where datetime = @ whereDatetime) begininsert T_twitter (datetime, LatestHourdata1, LatestHourdata2, LatestHou Rdata3, Avgdata1, AvgData2, AvgData3) VALUES (@ datetime, @ LatestHourdata1, @ override, @ override, @ Avgdata1, @ Avgdata2, @ Avgdata3) end "; ControlDB (SQL, pars, ""); // insert data }}/// <summary> // obtain the page data and save it to txt // </summary> public static void GetUSA () {WebRequest request = WebRequest. create (GetURL); WebResponse response = request. getResponse (); StreamReader reader = new StreamReader (response. getRe Eclipsestream (), Encoding. getEncoding ("gb2312"); // reader. readToEnd () indicates obtaining the source code FileStream fs = new FileStream (txtURL, FileMode. create); byte [] data = System. text. encoding. default. getBytes (reader. readToEnd (); // starts writing to fs. write (data, 0, data. length); // clears the buffer and closes the stream fs. flush (); fs. close ();} /// <summary> /// read the txt file by path /// </summary> /// <param name = "path"> txt path </param> // /<returns> </returns> public Static List <string []> Read (string path) {List <string []> list = new List <string []> (); StreamReader sr = new StreamReader (path, encoding. default); String line; while (line = sr. readLine ())! = Null) {int I = line. toString (). indexOf ("title"); if (I> 0) {string titleStr = line. toString (). substring (I + 7); // capture the value string [] titlelist = titleStr after the title. split ('"'); // use" to intercept string titledata = titlelist [0]; string [] datalist = titledata. split ('&'); // use & to intercept string data = datalist [0]; string [] datastrlist = data. split (new char [] {';'}, StringSplitOptions. removeEmptyEntries); // uses; to intercept list. add (datastrlist) ;}} sr. close (); return list ;} /// <summary> /// add query table // </summary> /// <returns> </returns> public static DataTable ControlDB (string SQL, list <SqlParameter> par, string type) {DataAccess controData = new DataAccess (); DataTable resultDT = new DataTable (); if (type = "select") {resultDT = controData. getDataTable (SQL, par. toArray ();} else {int result = controData. executeSql (SQL, par. toArray () ;}return resultDT ;}}}

Download Demo:

Http://files.cnblogs.com/files/xinchun/GetUSAData.zip