Use C # To remove duplicate data from the 0.1 million-level Array

10 thousand test data entries

First Algorithm (4 minutes)

 

[Csharp]
///

/// List generic deduplication
///
/// Generics composed of one-dimensional arrays
/// Repeated items to be removed
Private void RemoveRepeaterUrl (ref List List, int ItemID)
{
For (int I = 0; I <list. Count; I ++)
{
For (int j = (I + 1); j <list. Count; j ++)
{
If (list [j] [ItemID] = list [I] [ItemID])
{
List. RemoveAt (j); // remove identical items
I = 0; // deduplication from the beginning. If there are a large number of repeated items in the array, only one deduplication cannot solve the problem. Using this method will make the efficiency slow by 1/3.
J = 0;
}
}
}
}

 

///

/// List generic deduplication
///
/// Generics composed of one-dimensional arrays
/// Repeated items to be removed
Private void RemoveRepeaterUrl (ref List List, int ItemID)
{
For (int I = 0; I <list. Count; I ++)
{
For (int j = (I + 1); j <list. Count; j ++)
{
If (list [j] [ItemID] = list [I] [ItemID])
{
List. RemoveAt (j); // remove identical items
I = 0; // deduplication from the beginning. If there are a large number of repeated items in the array, only one deduplication cannot solve the problem. Using this method will make the efficiency slow by 1/3.
J = 0;
}
}
}
}

 

Second algorithm (4S)

[Csharp]
// Remove duplicate data and return list
 
// Use the containsKeys method of the hashtable class in the class library to determine whether the data exists in hashtable. If the data does not exist
 
// Add the data to the new List and clear the hashtable.
 
Public List GetUnque (List List)
{
List List1 = new List ();
Hashtable hash = new Hashtable ();
Foreach (string stu in list)
{
String [] kk1 = stu. Split (new string [] {"--"}, StringSplitOptions. RemoveEmptyEntries );
String comword = kkk1.length = 3? Kk1 [2]: "";
If (! Hash. ContainsKey (comword ))
{
Hash. Add (comword, comword );
List1.Add (stu );
}
}
Hash. Clear ();
Hash = null;
Return list1;
}

// Remove duplicate data and return list

// Use the containsKeys method of the hashtable class in the class library to determine whether the data exists in hashtable. If the data does not exist

// Add the data to the new List and clear the hashtable.

Public List GetUnque (List List)
{
List List1 = new List ();
Hashtable hash = new Hashtable ();
Foreach (string stu in list)
{
String [] kk1 = stu. Split (new string [] {"--"}, StringSplitOptions. RemoveEmptyEntries );
String comword = kkk1.length = 3? Kk1 [2]: "";
If (! Hash. ContainsKey (comword ))
{
Hash. Add (comword, comword );
List1.Add (stu );
}
}
Hash. Clear ();
Hash = null;
Return list1;
}