Improvements to interchange between UTF-8 and gb2312 [original]

Improvements in interchange between UTF-8 and gb2312

Author: Li Tianzhu

Download source code

Recently, when I was working on a small program, I suddenly encountered the problem of Chinese character encoding conversion. Question about how to convert between UTF-8 and gb2312. In the VC knowledge base to see Wu Kang Bin's article "UTF-8 and gb2312 Interchange", the article is easy to understand, the Code is not long. Saving me a lot of time to find materials. Thank you. :)
During the code reading process, Wu kangbin used many string conversions to perform binary computation. This involved a large number of Io operations, and the efficiency must be relatively low. In addition, the workload for encoding conversion is often very large, so the efficiency issue is very important. In addition, there are many memory leaks in the code, which may be because the author attaches too much importance to implementation and does not pay attention to these details.
Start with the question. In the UTF-8, and Unicode conversion, the binary operation, instead of the string conversion. UTF-8 a Chinese character, with 3 bytes, and Unicode with 2 bytes; the relationship is as follows:

UTF-8 coding: [, A5, A6, A7, A8], [, B3, B4, B5, B6, B7, B8], [, C3, C4, c5, C6, C7, C8];

Corresponding unicode encoding:

[A5,A6,A7,A8,B3,B4,B5,B6],               [B7,B8,C3,C4,C5,C6,C7,C8]

Therefore, we only need to perform a bit operation to achieve the goal. For example:

// Convert the UTF-8 to Unicode void cchinesecodelib: utf_8tounicode (wchar * pout, char * ptext) {char * uchar = (char *) pout; uchar [1] = (ptext [0] & 0x0f) <4) + (ptext [1]> 2) & 0x0f ); uchar [0] = (ptext [1] & 0x03) <6) + (ptext [2] & 0x3f); return ;}

// Unicode conversion to UTF-8 void cchinesecodelib: unicodetoutf_8 (char * pout, wchar * ptext) {// pay attention to the order of wchar high and low characters, low byte before, high bytes are in the back char * pchar = (char *) ptext; pout [0] = (0xe0 | (pchar [1] & 0xf0)> 4 )); pout [1] = (0x80 | (pchar [1] & 0x0f) <2) + (pchar [0] & 0xc0)> 6 ); pout [2] = (0x80 | (pchar [0] & 0x3f); return ;}

// Convert Unicode to gb2312 void cchinesecodelib: bytes (char * pout, unsigned short udata) {widechartomultibyte (cp_acp, null, & udata, 1, pout, sizeof (wchar ), null, null); return ;}

// Convert gb2312 to Unicode void cchinesecodelib: gb2312tounicode (wchar * pout, char * gbbuffer) {: multibytetowidechar (cp_acp, mb_precomposed, gbbuffer, 2, pout, 1 ); return ;}

// Convert gb2312 to UTF-8 void cchinesecodelib: gb2312toutf_8 (string & pout, char * ptext, int Plen) {char Buf [4]; char * rst = new char [Plen + (Plen> 2) + 2]; memset (BUF, 0, 4); memset (RST, 0, Plen + (Plen> 2) + 2); int I = 0; Int J = 0; while (I <Plen) {// if it is an English copy, you can (* (ptext + I)> = 0) {rst [J ++] = ptext [I ++];} else {wchar pbuffer; gb2312tounicode (& pbuffer, ptext + I); unicodetoutf_8 (BUF, & pbuffer); unsigned short int TMP = 0; TMP = rst [J] = Buf [0]; TMP = rst [J + 1] = Buf [1]; TMP = rst [J + 2] = Buf [2]; j + = 3; I + = 2 ;}} rst [J] = '/0 '; // return result pout = RST; Delete [] RST; return ;}

// Convert the UTF-8 to gb2312 void cchinesecodelib: utf_8togb2312 (string & pout, char * ptext, int Plen) {char * newbuf = new char [Plen]; char ctemp [4]; memset (ctemp, 0, 4); int I = 0; Int J = 0; while (I <Plen) {If (ptext [I]> 0) {newbuf [J ++] = ptext [I ++];} else {wchar wtemp; utf_8tounicode (& wtemp, ptext + I); unicodetogb2312 (ctemp, wtemp ); newbuf [J] = ctemp [0]; newbuf [J + 1] = ctemp [1]; I + = 3; j + = 2 ;}} newbuf [J] = '/0'; pout = newbuf; Delete [] newbuf; return ;}

The code is compiled under Win2k, VC ++ 60, and tested. Let's talk about it so much. Let's take a look at the code. You can't laugh at it. You are welcome to discuss it together ....

Certificate -----------------------------------------------------------------------------------------------------------------------------------------

Source code:

// Chinesecodelib. h: interface for the cchinesecodelib class.
//
//////////////////////////////////////// //////////////////////////////
# Include <string>
Using namespace STD;

/*
Function: conversion between gb2312 and UTF-8 Encoding
Author: litz
Email: mycro@163.com
Reference: Mr. Wu kangbin's article "interchange between UTF-8 and gb2312"
Http://www.vckbase.com/document/viewdoc? Id = 1397
*/

# If! Defined (_ cchinesecodelib_h _)
# DEFINE _ cchinesecodelib_h _

Class cchinesecodelib
{
Public:
Static void utf_8togb2312 (string & pout, char * ptext, int Plen );
Static void gb2312toutf_8 (string & pout, char * ptext, int Plen );
// Unicode to UTF-8
Static void unicodetoutf_8 (char * pout, wchar * ptext );
// Convert gb2312 to Unicode
Static void gb2312tounicode (wchar * pout, char * gbbuffer );
// Convert Unicode to gb2312
Static void unicodetogb2312 (char * pout, unsigned short udata );
// Convert the UTF-8 to Unicode
Static void utf_8tounicode (wchar * pout, char * ptext );

Cchinesecodelib ();
Virtual ~ Cchinesecodelib ();
};

# Endif //! Defined (_ cchinesecodelib_h _)

//-----------------------------------------------------------------------------

// Chinesecodelib. cpp: Implementation of the cchinesecodelib class.
//
//////////////////////////////////////// //////////////////////////////

# Include "stdafx. H"
# Include "chinesecodelib. H"

//////////////////////////////////////// //////////////////////////////
// Construction/destruction
//////////////////////////////////////// //////////////////////////////

Cchinesecodelib: cchinesecodelib ()
{

}

Cchinesecodelib ::~ Cchinesecodelib ()
{

}

Void cchinesecodelib: utf_8tounicode (wchar * pout, char * ptext)
{
Char * uchar = (char *) pout;
 
Uchar [1] = (ptext [0] & 0x0f) <4) + (ptext [1]> 2) & 0x0f );
Uchar [0] = (ptext [1] & 0x03) <6) + (ptext [2] & 0x3f );

Return;
}

Void cchinesecodelib: unicodetogb2312 (char * pout, unsigned short udata)
{
Widechartomultibyte (cp_acp, null, & udata, 1, pout, sizeof (wchar), null, null );
Return;
}

Void cchinesecodelib: gb2312tounicode (wchar * pout, char * gbbuffer)
{
: Multibytetowidechar (cp_acp, mb_precomposed, gbbuffer, 2, pout, 1 );
Return;
}

Void cchinesecodelib: unicodetoutf_8 (char * pout, wchar * ptext)
{
// Pay attention to the order of wchar high and low characters. The lower byte is in the front and the higher byte is in the back
Char * pchar = (char *) ptext;

Pout [0] = (0xe0 | (pchar [1] & 0xf0)> 4 ));
Pout [1] = (0x80 | (pchar [1] & 0x0f) <2) + (pchar [0] & 0xc0)> 6 );
Pout [2] = (0x80 | (pchar [0] & 0x3f ));
 
Return;
}

Void cchinesecodelib: gb2312toutf_8 (string & pout, char * ptext, int Plen)
{
Char Buf [4];
Char * rst = new char [Plen + (Plen> 2) + 2];
 
Memset (BUF, 0, 4 );
Memset (RST, 0, Plen + (Plen> 2) + 2 );
 
Int I = 0;
Int J = 0;
While (I <Plen)
{
// Directly copy data in English
If (* (ptext + I)> = 0)
{
RST [J ++] = ptext [I ++];
}
Else
{
Wchar pbuffer;
Gb2312tounicode (& pbuffer, ptext + I );

Unicodetoutf_8 (BUF, & pbuffer );

Unsigned short int TMP = 0;
TMP = rst [J] = Buf [0];
TMP = rst [J + 1] = Buf [1];
TMP = rst [J + 2] = Buf [2];


J + = 3;
I + = 2;
}
}
RST [J] = '/0 ';

// Return results
Pout = RST;
Delete [] RST;
 
Return;
}

Void cchinesecodelib: utf_8togb2312 (string & pout, char * ptext, int Plen)
{
Char * newbuf = new char [Plen];
Char ctemp [4];
Memset (ctemp, 0, 4 );

Int I = 0;
Int J = 0;

While (I <Plen)
{
If (ptext [I]> 0)
{
Newbuf [J ++] = ptext [I ++];
}
Else
{
Wchar wtemp;
Utf_8tounicode (& wtemp, ptext + I );

Unicodetogb2312 (ctemp, wtemp );

Newbuf [J] = ctemp [0];
Newbuf [J + 1] = ctemp [1];

I + = 3;
J + = 2;
}
}
Newbuf [J] = '/0 ';

Pout = newbuf;
Delete [] newbuf;

Return;
}