High-performance IP database format qqzeng-ip.dat, ip. dat

High-performance IP database format qqzeng-ip.dat, ip. dat
High Performance IP database format qqzeng-ip.dat

Encoding:UTF8Byte order:Little-Endian

Multiple fields are returned.(For example, Asia | China | Hong Kong | Kowloon | youqiwang | New World TELECOM | 810200 | Hong Kong | HK | 114.17495 | 22.327115)

------------------------File structure---------------------------

// File Header 16Byte (4-4-4-4)
[Location of the first stream in the index area] [location of the last stream in the index area] [location of the first stream in the prefix area] [location of the last stream in the prefix area]

// Content areaUnlimited length
[Region information] [region information]... Unique

// Index Area 12Byte (4-4-3-1)
[Start IP address] [end IP address] [region stream location] [stream length]

// Prefix Area 9Byte (1-4-4)
[0-255] [index start Index] [index end Index]

------------------------File structure---------------------------

Advantages:The index is divided into [Starting IP] [ending IP] [regional offset] [length], which reduces the parsing of multi-level offset jump steps and length and improves efficiency;
Based on the first ip byte as the prefix, the first index and the last index prefixed with this number are parsed to narrow the query interval,
Then, we can quickly find the corresponding interval using a binary search in this interval, improving the efficiency by several levels.

Compression:The original txt is 15 M, and the generated dat structure is 2.45 M.

Performance:Normal Computer Testing, resolution, 1 million ip takes 2.012439 seconds, 1 million takes 21.10258 seconds, good computer testing will be more efficient

Comparison:Simpler and more efficient than other dat

Create:Qqzeng-ip Address:

 

Public class IPSearch {private Dictionary <uint, PrefixIndex> prefixDict; private byte [] indexBuffer; private byte [] data; long firstStartIpOffset; // The first stream position in the index area is long lastStartIpOffset; // The Position of the last stream in the index area long prefixStartOffset; // The Position of the first stream in the prefix area long prefixEndOffset; // The Position of the last stream in the prefix area long ipCount; // number of ip segments long prefixCount; // Number of prefixes /// <summary> /// initialize binary dat data /// </summary> /// <param name = "dataPath"> </param> public I PSearch (string dataPath) {using (FileStream fs = new FileStream (dataPath, FileMode. open, FileAccess. read, FileShare. read) {data = new byte [fs. length]; fs. read (data, 0, data. length);} firstStartIpOffset = BytesToLong (data [0], data [1], data [2], data [3]); lastStartIpOffset = BytesToLong (data [4], data [5], data [6], data [7]); prefixStartOffset = BytesToLong (data [8], data [9], data [10], data [11]); pre FixEndOffset = BytesToLong (data [12], data [13], data [14], data [15]); // The prefixCount value is not fixed to 256. You can customize the external version of the domestic version for the global version or some part of the global version for the future. ipCount = (lastStartIpOffset-firstStartIpOffset)/12 + 1; // The index block contains 12 bytes in each group. prefixCount = (prefixEndOffset-prefixStartOffset)/9 + 1; // 9 bytes in each group of prefix blocks // The index interval corresponding to the initialization prefix indexBuffer = new byte [prefixCount * 9]; Array. copy (data, prefixStartOffset, indexBuffer, 0, prefixCount * 9); pref IxDict = new Dictionary <uint, PrefixIndex> (); for (var k = 0; k <prefixCount; k ++) {int I = k * 9; uint prefix = (uint) indexBuffer [I]; long start_index = BytesToLong (indexBuffer [I + 1], indexBuffer [I + 2], indexBuffer [I + 3], indexBuffer [I + 4]); long end_index = BytesToLong (indexBuffer [I + 5], indexBuffer [I + 6], indexBuffer [I + 7], indexBuffer [I + 8]); prefixDict. add (prefix, new PrefixIndex () {Prefix = prefix, start_index = start_index, end_index = end_index}) ;}} public static uint IpToInt (string ip) {byte [] bytes = IPAddress. parse (ip ). getAddressBytes (); return (uint) bytes [3] + (uint) bytes [2]) <8) + (uint) bytes [1]) <16) + (uint) bytes [0]) <24);} public static string IntToIP (uint ip_Int) {return new IPAddress (ip_Int ). toString () ;}/// <summary> /// query multidimensional field information by ip address /// </s Ummary> // <param name = "ip"> ip address (123.4.5.6) </param> // <returns> Asia | China | Hong Kong | Kowloon | youqiwang | New World Telecommunications | 810200 | Hong Kong | HK | 114.17495 | 22.327115 </returns> public string Query (string ip) {uint intIP = IpToInt (ip); uint high = 0; uint low = 0; uint startIp = 0; uint endIp = 0; uint local_offset = 0; uint local_length = 0; uint ip_prefix_value = uint. parse (ip. split ('. ') [0]); if (prefixDict. containsKey (ip_prefix_valu E) {low = (uint) prefixDict [ip_prefix_value]. start_index; high = (uint) prefixDict [ip_prefix_value]. end_index;} else {return "";} uint my_index = low = high? Low: BinarySearch (low, high, intIP); GetIndex (my_index, out startIp, out endIp, out local_offset, out local_length); if (startIp <= intIP) & (endIp> = intIP) {return GetLocal (local_offset, local_length);} else {return "";}} /// <summary> /// binary approximation algorithm /// </summary> public uint BinarySearch (uint low, uint high, uint k) {uint M = 0; while (low <= high) {uint mid = (low + high)/2; uint endipNum = GetEndIp (mid); if (endipNum> = k) {M = mid; // mid may be solved by high = mid-1;} else low = mid + 1;} return M ;} /// <summary> /// Parse in the index area /// </summary> /// <param name = "left"> ip left index </param> /// <param name = "startip"> returns the starting ip value </param> /// <param name = "endip"> returns the ending ip value </param> /// <param name = "local_offset"> return the stream location of the address information </param> /// <param name = "local_length"> return the stream length of the address information </ param> private void GetIndex (uint left, out uint startip, out uint endip, out uint local_offset, out uint local_length) {long left_offset = firstStartIpOffset + (left * 12); startip = BytesToLong (data [left_offset], data [1 + left_offset], data [2 + left_offset], data [3 + left_offset]); endip = BytesToLong (data [4 + left_offset], data [5 + left_offset], data [6 + left_offset], data [7 + left_offset]); local_offset = (uint) data [8 + left_offset] + (uint) data [9 + left_offset]) <8) + (uint) data [10 + left_offset]) <16); local_length = (uint) data [11 + left_offset];} /// <summary> /// obtain only the value of the ending ip address /// </summary> /// <param name = "left"> left index in the index area </param> // <returns> returns the value of the ending ip address </returns> private uint GetEndIp (uint left) {long left_offset = firstStartIpOffset + (left * 12); return BytesToLong (data [4 + left_offset], data [5 + left_offset], data [6 + left_offset], data [7 + left_offset]);} /// <summary> /// return address information /// </summary> /// <param name = "local_offset"> stream position of address information </param>/ // <param name = "local_length"> stream length of address information </param> // <returns> </returns> private string GetLocal (uint local_offset, uint local_length) {byte [] buf = new byte [local_length]; Array. copy (data, local_offset, buf, 0, local_length); return Encoding. UTF8.GetString (buf, 0, (int) local_length );} /// <summary> /// bytes to the integer section order // </summary> /// <param name = "a"> </param> // <param name = "B"> </param> // <param name = "c"> </param> // <param name = "d"> </param >/// <returns> </returns> private uint BytesToLong (byte, byte B, byte c, byte d) {return (uint) a <0) | (uint) B <8) | (uint) c <16) | (uint) d <24) ;}/ * (call example): IPSearch finder = new IPSearch ("qqzeng-ip.dat"); string result = finder. query ("1.2.3.4 "); --> result = "Asia | China | Hong Kong | Kowloon | youqiwang | New World Telecommunications | 810200 | Hong Kong | HK | 114.17495 | 22.327115" */public class PrefixIndex {public uint prefix {get; set;} public long start_index {get; set;} public long end_index {get; set ;}// multi-language parsing will be added in the future