Implementation Method: It is implemented by extending the standard word divider in e.net.
Source code StandardTokenizer of the e.net standard tokenizer
The e.net standard word divider has a good experience in English word segmentation. In terms of mail, IP address, and symbol processing, it is very good. Unfortunately, it does not support Chinese phrase segmentation. So I modified the core code to extend it and support Chinese word segmentation.
Objective: To enable it to add a cut word for a Chinese phrase.
Effect:
Original sentence: "I am Chinese! I am chiness! Email: youpeizun126@126.com; IP: 172.17. 34.168"
Word splitting effect:
Me/Yes/Chinese/China/people/Email/youpeizun126@126.com/IP/172.17.34.168
Task to be completed:
1. Load dictionary
2. intercept a continuous Chinese field
3. Perform continuous word segmentation.
The following is a flowchart of the extended e.net standard word divider that supports Chinese word segmentation.
Next, I will expand the core code written by the e.net standard tokenizer, which mainly includes three functions, which are used to load the dictionary, load the continuous Chinese fields, and perform the Chinese phrase tokenization algorithm.
Core Chinese phrase segmentation code
/**//*
# Add a Chinese dictionary to region
Public void LoadDirectory (string path)
{
If (! File. Exists ("words.txt "))
Return;
TextReader tr_words = new StreamReader ("words.txt", System. Text. Encoding. Default );
System. Diagnostics. Debug. Write ("begin read words ");
If (directory = null)
{
Directory = new System. Collections. Hashtable ();
Try
{
String word = null;
While (word = tr_words.ReadLine ())! = Null)
{
Try
{
If (directory [word] = null)
{
Directory. Add (word, word );
}
}
Catch (SystemException ex _)
{
}
}
}
Catch (SystemException ex)
{
}
}
# Endregion
}
# Region intercepts a continuous Chinese field
Private void InitChinessText ()
{
Textlengh = 0;
Cn_index = 0;
Chinesstext [0] = token. image;
Textlengh ++;
Cn_start = token. beginColumn;
IsCnToken = true;
Bool isCN = true;
While (isCN & textlengh <255)
{Token = token_source.GetNextToken ();
If (token. kind! = 0)
{
IsCN = Char. GetUnicodeCategory (token. image, 0). Equals (System. Globalization. UnicodeCategory. OtherLetter );
}
Else
IsCN = false;
If (isCN)
{
Chinesstext [textlengh] = token. image;
Textlengh ++;
}
Else
{
Cn_end_token = token;
}
}
If (textlengh> = 4)
{
Wordlengh = 4;
}
Else
Wordlengh = textlengh;
}
# Endregion
# Region implements a Chinese Word Segmentation Algorithm
Private string GetNextTokenText ()
{String text = null;
If (wordlengh = 4)
{
Text = chinesstext [cn_index] + chinesstext [cn_index + 1] + chinesstext [cn_index + 2] + chinesstext [cn_index + 3];
If (directory [text]! = Null)
{
}
Wordlengh --;
}
If (wordlengh = 3)
{
Text = chinesstext [cn_index] + chinesstext [cn_index + 1] + chinesstext [cn_index + 2];
Wordlengh --;
If (directory [text]! = Null)
{
Goto return _;
}
}
If (wordlengh = 2)
{
Text = chinesstext [cn_index] + chinesstext [cn_index + 1];
Wordlengh --;
If (directory [text]! = Null)
{
Goto return _;
}
}
If (wordlengh = 1)
{
Text = chinesstext [cn_index];
Cn_index ++;
If (textlengh-cn_index)> = 4)
{
Wordlengh = 4;
}
Else
If (textlengh-cn_index) = 0)
{
IsCnToken = false;
Jj_ntk = cn_end_token.kind;
Token = new Token ();
Token. next = cn_end_token;
}
Else
{
Wordlengh = textlengh-cn_index;
}
}
Return _:
Return text;
}
# Endregion
*/
Thank you for reading.
