C # detect byte stream Encoding

Namespace microshaoft
{
Using system;
Using system. IO;
Using system. net;
# Region class identifyencoding .....
/// <Summary>
/// Detects character encoding classes
/// <Seealso CREF = "stream"/>
/// <Seealso CREF = "Uri"/>
/// <Seealso CREF = "fileinfo"/>
/// </Summary>
/// <Remarks>
/// <! [CDATA [
/// <Strong> identifyencoding </strong> is used to detect <see CREF = "Uri"/>, <see CREF = "fileinfo"/>, <see CREF = "sbyte"/> encoding of byte arrays.
/// Create by lion <br/>
/// <Br/>
/// Support. NET Framework v1.1.4322 <br/>
/// Website: www.lionsky.net (lion-A at sohu.com) <br/>
///]>
/// </Remarks>
Public class identifyencoding
{
# Region fields .....
// Frequency tables to hold the GB, big5, and EUC-TW character
// Frequencies
Internal static int [] [] gbfreq = new int [94] [];
Internal static int [] [] gbkfreq = new int [0, 126] [];
Internal static int [] [] big5freq = new int [94] [];
Internal static int [] [] euc_twfreq = new int [94] [];
Internal static string [] nicename = new string []
{
"Gb2312", "GBK", "HZ", "big5", "CNS 11643"
, "ISO 2022cn", "UTF-8", "Unicode", "ASCII", "other"
};
# Endregion
# Region methods .....
/// <Summary>
/// Initialize the <see CREF = "identifyencoding"/> instance
/// </Summary>
Public identifyencoding ()
{
Initialize_frequencies ();
}
# Region getencodingname .....
/// <Summary>
/// Determine the encoding type from the specified <see CREF = "Uri"/>
/// </Summary>
/// <Param name = "testurl"> <see CREF = "Uri"/> </param>
/// <Returns> return encoding type ("gb2312", "GBK", "HZ", "big5", "CNS 11643", "ISO 2022cn", "UTF-8 ", "Unicode", "ASCII", "other") </returns>
/// <Example>
/// The following example shows how to call the <see CREF = "getencodingname"/> method:
/// <Code>
/// Identifyencoding ide = new identifyencoding ();
/// Response. Write (IDE. getencodingname (New uri ("http://china5.nikkeibp.co.jp/china/news/com/200307/pr_com200307170131.html ")));
/// </Code>
/// </Example>
Public Virtual string getencodingname (URI testurl)
{
Sbyte [] rawtext = new sbyte [2, 1024];
Int bytesread = 0, byteoffset = 0;
Stream chinesestream;
Try
{
Chinesestream = webrequest. Create (testurl. absoluteuri). getresponse (). getresponsestream ();
While (bytesread = readinput (chinesestream, ref rawtext, byteoffset, rawtext. Length-byteoffset)> 0)
{
Byteoffset + = bytesread;
}
Chinesestream. Close ();
}
Catch (exception E)
{
Console. Error. writeline ("error loading or using URL" + E. tostring ());
}
Return getencodingname (rawtext );
}
/// <Summary>
/// Determine the encoding type from the specified <see CREF = "fileinfo"/>
/// </Summary>
/// <Param name = "testfile"> <see CREF = "fileinfo"/> </param>
/// <Returns> return encoding type ("gb2312", "GBK", "HZ", "big5", "CNS 11643", "ISO 2022cn", "UTF-8 ", "Unicode", "ASCII", "other") </returns>
/// <Example>
/// The following example shows how to call the <see CREF = "getencodingname"/> method:
/// <Code>
/// Identifyencoding ide = new identifyencoding ();
/// Response. Write (IDE. getencodingname (New fileinfo (@ "C: \ test.txt ")));
/// </Code>
/// </Example>
Public Virtual string getencodingname (fileinfo testfile)
{
Filestream chinesefile;
Sbyte [] rawtext;
Rawtext = new sbyte [(INT) filelength (testfile)];
Try
{
Chinesefile = new filestream (testfile. fullname, filemode. Open, fileaccess. Read );
Readinput (chinesefile, ref rawtext, 0, rawtext. Length );
}
Catch (exception E)
{
Console. Error. writeline ("error:" + E );
}
Return getencodingname (rawtext );
}
/// <Summary>
/// Determine the encoding type from the specified <see CREF = "sbyte"/> byte array
/// </Summary>
/// <Param name = "rawtext"> <see CREF = "fileinfo"/> </param>
/// <Returns> return encoding type ("gb2312", "GBK", "HZ", "big5", "CNS 11643", "ISO 2022cn", "UTF-8 ", "Unicode", "ASCII", "other") </returns>
/// <Example>
/// The following example shows how to call the <see CREF = "getencodingname"/> method:
/// <Code>
/// Identifyencoding ide = new identifyencoding ();
/// Response. write (IDE. getencodingname (identifyencoding. tosbytearray (system. text. encoding. getencoding ("gb2312 "). getbytes ("Lion interactive network (www.lionsky.net )"))));
/// </Code>
/// </Example>
Public Virtual string getencodingname (sbyte [] rawtext)
{
Int [] scores;
Int index, maxscore = 0;
Int encoding_guess = 0;
Scores = new int [10];
// Analyze the encoding Probability
Scores [0] = gb2312probability (rawtext );
Scores [1] = gbkprobability (rawtext );
Scores [2] = hzprobability (rawtext );
Scores [3] = big5probability (rawtext );
Scores [4] = enctwprobability (rawtext );
Scores [5] = iso2022cnprobability (rawtext );
Scores [6] = utf8probability (rawtext );
Scores [7] = unicodeprobability (rawtext );
Scores [8] = asciiprobability (rawtext );
Scores [9] = 0;
// Tabulate scores
For (Index = 0; index <10; index ++)
{
If (scores [Index]> maxscore)
{
Encoding_guess = index;
Maxscore = scores [Index];
}
}
// Return other if nothing scored above 50
If (maxscore <= 50)
{
Encoding_guess = 9;
}
Return nicename [encoding_guess];
}
# Endregion
# Region about probability .....
# Region gb2312probability
/// <Summary>
/// Determine the possibility of gb2312 Encoding
/// </Summary>
/// <Param name = "rawtext"> byte array to be judged <see CREF = "sbyte"/> </param>
/// <Returns> returns the possibility of a value between 0 and 100. </returns>
Internal virtual int gb2312probability (sbyte [] rawtext)
{
Int I, rawtextlen = 0;
Int dbchars = 1, gbchars = 1;
Long gbfreq = 0, totalfreq = 1;
Float rangeval = 0, freqval = 0;
Int row, column;
// Stage 1: Check to see if characters fit into acceptable ranges
Rawtextlen = rawtext. length;
For (I = 0; I <rawtextlen-1; I ++)
{
If (rawtext [I]> = 0)
{
// Asciichars ++;
}
Else
{
Dbchars ++;
If (sbyte) Identity (0xa1) <= rawtext [I] & rawtext [I] <= (sbyte) Identity (0xf7) & (sbyte) Identity (0xa1) <= rawtext [I + 1] & rawtext [I + 1] <= (sbyte) Identity (0xfe ))
{
Gbchars ++;
Totalfreq + = 500;
Row = rawtext [I] + 256-0xa1;
Column = rawtext [I + 1] + 256-0xa1;
If (gbfreq [row] [column]! = 0)
{
Gbfreq + = gbfreq [row] [column];
}
Else if (15 <= row & Row <55)
{
Gbfreq + = 200;
}
}
I ++;
}
}
Rangeval = 50 * (float) gbchars/(float) dbchars );
Freqval = 50 * (float) gbfreq/(float) totalfreq );
Return (INT) (rangeval + freqval );
}
# Endregion
# Region gbkprobability .....
/// <Summary>
/// Determine the possibility of GBK Encoding
/// </Summary>
/// <Param name = "rawtext"> byte array to be judged <see CREF = "sbyte"/> </param>
/// <Returns> returns the possibility of a value between 0 and 100. </returns>
Internal virtual int gbkprobability (sbyte [] rawtext)
{
Int I, rawtextlen = 0;
Int dbchars = 1, gbchars = 1;
Long gbfreq = 0, totalfreq = 1;
Float rangeval = 0, freqval = 0;
Int row, column;
// Stage 1: Check to see if characters fit into acceptable ranges
Rawtextlen = rawtext. length;
For (I = 0; I <rawtextlen-1; I ++)
{
If (rawtext [I]> = 0)
{
// Asciichars ++;
}
Else
{
Dbchars ++;
If (sbyte) Identity (0xa1) <= rawtext [I] & rawtext [I] <= (sbyte) Identity (0xf7) & (sbyte) Identity (0xa1) <= rawtext [I + 1] & rawtext [I + 1] <= (sbyte) Identity (0xfe ))
{
Gbchars ++;
Totalfreq + = 500;
Row = rawtext [I] + 256-0xa1;
Column = rawtext [I + 1] + 256-0xa1;
If (gbfreq [row] [column]! = 0)
{
Gbfreq + = gbfreq [row] [column];
}
Else if (15 <= row & Row <55)
{
Gbfreq + = 200;
}
}
Else if (sbyte) Identity (0x81) <= rawtext [I] & rawtext [I] <= (sbyte) Identity (0xfe) & (sbyte) identity (0x80) <= rawtext [I + 1] & rawtext [I + 1] <= (sbyte) Identity (0xfe) | (sbyte) 0x40 <= rawtext [I + 1] & rawtext [I + 1] <= (sbyte) 0x7e )))
{
Gbchars ++;
Totalfreq + = 500;
Row = rawtext [I] + 256-0x81;
If (0x40 <= rawtext [I + 1] & rawtext [I + 1] <= 0x7e)
{
Column = rawtext [I + 1]-0x40;
}
Else
{
Column = rawtext [I + 1] + 256-0x80;
}
If (gbkfreq [row] [column]! = 0)
{
Gbfreq + = gbkfreq [row] [column];
}
}
I ++;
}
}
Rangeval = 50 * (float) gbchars/(float) dbchars );
Freqval = 50 * (float) gbfreq/(float) totalfreq );
Return (INT) (rangeval + freqval)-1;
}
# Endregion
# Region hzprobability .....
/// <Summary>
/// Determine the possibility of HZ Encoding
/// </Summary>
/// <Param name = "rawtext"> byte array to be judged <see CREF = "sbyte"/> </param>
/// <Returns> returns the possibility of a value between 0 and 100. </returns>
Internal virtual int hzprobability (sbyte [] rawtext)
{
Int I, rawtextlen;
Int hzchars = 0, dbchars = 1;
Long hzfreq = 0, totalfreq = 1;
Float rangeval = 0, freqval = 0;
Int hzstart = 0, hzend = 0;
Int row, column;
Rawtextlen = rawtext. length;
For (I = 0; I <rawtextlen; I ++)
{
If (rawtext [I] = '~ ')
{
If (rawtext [I + 1] = '{')
{
Hzstart ++;
I + = 2;
While (I <rawtextlen-1)
{
If (rawtext [I] = 0x0a | rawtext [I] = 0x0d)
{
Break;
}
Else if (rawtext [I] = '~ '& Rawtext [I + 1] = '}')
{
Hzend ++;
I ++;
Break;
}
Else if (0x21 <= rawtext [I] & rawtext [I] <= 0x77) & (0x21 <= rawtext [I + 1] & rawtext [I + 1] <= 0x77 ))
{
Hzchars + = 2;
Row = rawtext [I]-0x21;
Column = rawtext [I + 1]-0x21;
Totalfreq + = 500;
If (gbfreq [row] [column]! = 0)
{
Hzfreq + = gbfreq [row] [column];
}
Else if (15 <= row & Row <55)
{
Hzfreq ++ = 200;
}
}
Else if (byte) 0xa1 <= rawtext [I] & rawtext [I] <= (byte) 0xf7) & (byte) 0xa1 <= rawtext [I + 1] & rawtext [I + 1] <= (byte) 0xf7 ))
{
Hzchars + = 2;
Row = rawtext [I] + 256-0xa1;
Column = rawtext [I + 1] + 256-0xa1;
Totalfreq + = 500;
If (gbfreq [row] [column]! = 0)
{
Hzfreq + = gbfreq [row] [column];
}
Else if (15 <= row & Row <55)
{
Hzfreq ++ = 200;
}
}
Dbchars + = 2;
I + = 2;
}
}
Else if (rawtext [I + 1] = '}')
{
Hzend ++;
I ++;
}
Else if (rawtext [I + 1] = '~ ')
{
I ++;
}
}
}
If (hzstart> 4)
{
Rangeval = 50;
}
Else if (hzstart> 1)
{
Rangeval = 41;
}
Else if (hzstart> 0)
{
// Only 39 in case the sequence happened to occur
Rangeval = 39; // in otherwise non-Hz text
}
Else
{
Rangeval = 0;
}
Freqval = 50 * (float) hzfreq/(float) totalfreq );
Return (INT) (rangeval + freqval );
}
# Endregion
# Region big5probability .....
/// <Summary>
/// Determine the possibility of big5 Encoding
/// </Summary>
/// <Param name = "rawtext"> byte array to be judged <see CREF = "sbyte"/> </param>
/// <Returns> returns the possibility of a value between 0 and 100. </returns>
Internal virtual int big5probability (sbyte [] rawtext)
{
Int I, rawtextlen = 0;
Int dbchars = 1, bfchars = 1;
Float rangeval = 0, freqval = 0;
Long bffreq = 0, totalfreq = 1;
Int row, column;
// Check to see if characters fit into acceptable ranges
Rawtextlen = rawtext. length;
For (I = 0; I <rawtextlen-1; I ++)
{
If (rawtext [I]> = 0)
{
// Asciichars ++;
}
Else
{
Dbchars ++;
If (sbyte) Identity (0xa1) <= rawtext [I] & rawtext [I] <= (sbyte) Identity (0xf9) & (sbyte) 0x40 <= rawtext [I + 1] & rawtext [I + 1] <= (sbyte) 0x7e) | (sbyte) Ident