Implementation of batch update of Asp.net Big Data

Premise: for operations with a large amount of data, it is a waste of resources to operate the database every time data is collected.

The data is collected to the memory table and synchronized to the physical database by updating the large data volume. In this example, the number of records on a page is

Example (Simple Model)

--- Data collection PAGE method ---
Using system;
Using system. Collections. Generic;
Using system. Web;
Using system. Web. UI;
Using system. Web. UI. webcontrols;
Using statistics. Common;
Using statistics. BLL;

Namespace statistics. Web
{
Public partial class CLICK: system. Web. UI. Page
{
Protected void page_load (Object sender, eventargs E)
{
Response. cachecontrol = "no-Cache ";
String referurl = string. empty;
If (request. urlreferrer! = NULL)
{
Referurl = request. urlreferrer. tostring ();
}
Adurl = input. Filter (request. querystring ["url"]);
String IP = statistics. Common. IP. getclientip ();
}
Datetime createtime = datetime. now;
Statistics. updatestatistics (referurl, IP, createtime, 2 );
}

Response. Redirect (adurl );
}
}
}
--- Statistics. updatestatistics ---
Using system;
Using system. Collections. Generic;
Using system. text;
Using system. Data;
Using system. Threading;
Using statistics. sqlserverdal;

Namespace statistics. BLL
{
Public class statistics
{

Public static void updatestatistics (string URL, string IP, datetime createtime)
{
Try
{

Lock (typeof (Statistics ))
{

Datarow DR = memorydata. mongods. Tables [0]. newrow ();

Dr. beginedit ();
Dr ["url"] = URL;
Dr ["ip"] = IP;
Dr ["createtime"] = createtime;

Dr. endedit ();

If (memorydata. dsflag = 1)
{
Memorydata. globaldsone. Tables [0]. Rows. Add (dr. itemarray );
}
Else if (memorydata. dsflag = 2)
{
Memorydata. globaldstwo. Tables [0]. Rows. Add (dr. itemarray );
}
Else
{
Memorydata. globaldsthree. Tables [0]. Rows. Add (dr. itemarray );
}
}

}
Catch (exception E)
{

}

}

}
}
--- Create a memory table, insert data, and insert data in batches ---
Using system;
Using system. diagnostics;
Using system. Collections. Generic;
Using system. text;
Using system. Data;
Using system. Data. sqlclient;
Using statistics. dbutility;
Namespace statistics. sqlserverdal
{
Public class memorydata
{
Public static dataset globaldsone;
Public static dataset globaldstwo;
Public static dataset globaldsthree;
Public static dataset regionds;
Public static int dsflag = 1;

/// <Summary>
/// Insert memory statistics into the sea table statisticsinfo
/// </Summary>
/// <Returns> </returns>
Public static void shiftmemorydata ()
{
Dataset DS = NULL;
If (dsflag = 1)
{
If (globaldsone! = NULL & globaldsone. Tables [0]. Rows. Count> 0)
{
Dsflag = 2;
DS = globaldsone. Copy ();
Globaldsone. Clear ();
Globaldsone. Dispose ();
GC. Collect ();
}

}
Else if (dsflag = 2)
{
If (globaldstwo! = NULL & globaldstwo. Tables [0]. Rows. Count> 0)
{
Dsflag = 3;
DS = globaldstwo. Copy ();
Globaldstwo. Clear ();
Globaldstwo. Dispose ();
GC. Collect ();
}
}
Else
{
If (globaldsthree! = NULL & globaldsthree. Tables [0]. Rows. Count> 0)
{
Dsflag = 1;
DS = globaldsthree. Copy ();
Globaldsthree. Clear ();
Globaldsthree. Dispose ();
GC. Collect ();
}
}
If (Ds! = NULL & Ds. Tables [0]. Rows. Count> 0)
{
Long elapsedseconds = sqlbulkcopyinsert (Ds. Tables [0])/1000;
DS. Clear ();
DS. Dispose ();
GC. Collect ();
}

}

/// <Summary>
/// Use sqlbulkcopy to insert data
/// </Summary>
/// <Param name = "datatable"> </param>
/// <Returns> </returns>
Private Static long sqlbulkcopyinsert (datatable DT)
{
Stopwatch stopwatch = new stopwatch ();
Stopwatch. Start ();

Sqlbulkcopy = new sqlbulkcopy (pubconstant. connectionstring );
Sqlbulkcopy. destinationtablename = "tempstatistics ";
Sqlbulkcopy. batchsize = DT. Rows. count;
Sqlconnection = new sqlconnection (pubconstant. connectionstring );
Sqlconnection. open ();
Sqlbulkcopy. writetoserver (DT );
Sqlbulkcopy. Close ();
Sqlconnection. Close ();

Stopwatch. Stop ();
DT. Clear ();
DT. Dispose ();
GC. Collect ();
Return stopwatch. elapsedmilliseconds;
}

/// <Summary>
/// Create a table structure in the memory
/// </Summary>

Public static dataset createschemastatistics ()
{
Dataset DS = new dataset ();
Datatable dt = new datatable ();
DS. Tables. Add (DT );

Datacolumn dcurl = new datacolumn ();
Dcurl. allowdbnull = false;
Dcurl. Caption = "url ";
Dcurl. columnname = "url ";
Dcurl. datatype = system. type. GetType ("system. String ");
DS. Tables [0]. Columns. Add (dcurl );

Datacolumn dcip = new datacolumn ();
Dcip. allowdbnull = false;
Dcip. Caption = "ip ";
Dcip. columnname = "ip ";
Dcip. datatype = system. type. GetType ("system. String ");
DS. Tables [0]. Columns. Add (dcip );

Datacolumn dccreatetime = new datacolumn ();
Dccreatetime. allowdbnull = false;
Dccreatetime. Caption = "createtime ";
Dccreatetime. columnname = "createtime ";
Dccreatetime. datatype = system. type. GetType ("system. datetime ");
DS. Tables [0]. Columns. Add (dccreatetime );

Return Ds;

}

Public static void initmemorystatistics ()
{

Dataset DS = createschemastatistics ();
Globaldsone = Ds. Copy ();
Globaldstwo = Ds. Copy ();
Globaldsthree = Ds. Copy ();
DS = Ds. Copy ();
}
}
}
--- Initialize and regularly update data in global. CS ---
Protected void application_start (Object sender, eventargs E)
{
Globaldata. createmeoryds ();
Globaldata. loadresourcedata ();
Globaldata. loadpublishdata ();
Timer. createtimer ();
}
--- Timer. createtimer ---
Using system;
Using system. Collections. Generic;
Using system. text;
Using statistics. sqlserverdal;
Using system. configuration;

Namespace statistics. BLL
{
Public class Timer
{

Public static void createtimer ()
{
Int memorydatashifttimer = int. parse (configurationmanager. receivettings

["Memorydatashiftminute"]);
System. Timers. Timer time = new system. Timers. Timer (memorydatashifttimer * 1000

* 60 );
Time. elapsed + = new system. Timers. elapsedeventhandler (timerelapsed );
Time. Enabled = true;
GC. keepalive (time );

}

Public static void timerelapsed (Object sender, system. Timers. elapsedeventargs E)
{

Memorydata. shiftmemorydata ();
}
}
}
The scheduled interval can be set as needed. In this example, it is set to 1 minute.
--- End ---
This is the implementation method and step. It is relatively simple to write. If there are better implementation methods, please share them together. Thank you!