MySQL data deduplication prevents insertion:)

INSERT into table (ID, name, age) VALUES (1, "A", +) on duplicate key update name=values (name), Age=values (age)

/* Insert Data: If there are duplicates, select Update; */

Insert ignore into ' testtable ' (' Mpass ', ' Pass ') select Mpass,pass from Rr_pass_0 limit 0,1000000replace into ' testtable ' ( ' Mpass ', ' Pass ') select Mpass,pass from Rr_pass_0 limit 0,10

Set PRIMARY key: Discard if duplicate data is selected;

SELECT *, COUNT (distinct name) from the table group by name

Querying for duplicate data

Write an import script these days; leave a note;

<?php error_reporting (0); Ini_set (' Memory_limit ', ' 1024M '); header (' Content-type:text/html;charset=utf-8 '); $db _ Host = "127.0.0.1"; $db _user = "root"; $db _pass = ""; $db _database = "QQ"; $table = "import_table"; Imported Library $tables = "Query_tablle"; Query the library//62300000,100000$dump_num = "100"; According to a 10W standard; $db = new db ($db _host, $db _user, $db _pass, $db _database);//$test = $db->query ("INSERT into testtables (a) VALUES (' 145646464 '), $start _num = 5305950;//start position $tables _query = $db->query ("SELECT count (*) from $tables");// Total Queries $tables_count_array = $db->sql_array ($tables _query); $tables _count_num = $tables _count_array[0]; $counts = Intval ($tables _count_num/$dump _num); $counts = $counts + 1; $counts = 10000;for ($i =0; $i < $counts; $i + +) {//echo (" Select ' Pass ' from Rr1_pass limit $start _num, $dump _num ")." <br \> "; $start _time = Microtime (true); $sql _oking = $db->query (" Select ' Pass ' from ' $tables ' limit $start _num,$ Dump_num "), while ($sql _oking_array = $db->sql_array ($sql _oking)) {$pass = $sql_oking_array[' pass '; $check = $db->query ("Select pass from ' $table ' where pass = \" $pass \ "Limit 0,1"); $check = $db- >sql_nums ($check); if (!empty $pass &&! $check) {$MD 5 = MD5 ($pass); $MD 5_2 = MD5 ($pass); $db->query (" Insert into $table (Md5,md5_2,pass) VALUES (' ". $md 5." ', ' ". $md 5_2." ', ' ". $pass." ') " ;}} $end _time = Microtime (true), $time = $end _time-$start _time, echo "$start _num, $dump _num db_write Runtime:". $time. " s\r\n "; $start _num = $start _num+ $dump _num;} Class Db{private $db _host;private $db _user;private $db _pass;private $db _conn;private $db _database;private $sql; Private $result;p ublic function __construct ($db _host, $db _user, $db _pass, $db _database, $db _conn = ") {$this->db_ host = $db _host; $this->db_user = $db _user; $this->db_pass = $db _pass; $this->db_database = $db _database; $this- >db_conn = $db _conn; $this->connect ();} Public Function Connect () {$this->db_conn = @mysql_connect ($this->db_host, $this->db_user, $this->db_pass ); if (! $this->db_conn{echo "MySQL connect not okay!";} if (!mysql_select_db ($this->db_database, $this->db_conn)) {echo "Select Database not okay!";}} Public Function Query ($sql) {$this->sql = $sql; $result = mysql_query ($this->sql, $this->db_conn); $this result = $result; #if (! $this->result) {echo "SQL query is not okay!";} exit;return $this->result;} Public Function Sql_array ($str = ") {if ($str! =") {return mysql_fetch_array ($STR);} else{return mysql_fetch_array ($this->result);}} Public Function Sql_nums ($sql) {return mysql_num_rows ($sql);} Public Function free () {@mysql_free_result ($this->result);} Public Function __destruct () {if (!empty ($this->result)) {$this->free ();} Mysql_close ($this->db_conn);}? >

　　

MySQL data deduplication prevents insertion:)