Simple Chinese word segmentation using PHP

The following is a simple word segmentation program based on a dictionary file I found online.

The following is a simple word segmentation program based on a dictionary file I found online.

(Note: The dictionary is in gdbm format, and the key is the word value, which is the word frequency. about 40 thousand common words are used)

Complete Program demo and download see: http://root.twomice.net/my_php4/dict/chinese_segment.php

// Simple implementation of the Chinese word segmentation system
// Sentence unit: any character with an ascii value <128
// Common dubyte symbols: ". ,? "";:! ￥ ...... % $ # @ ^ & * () [] {}| \/"'
// You can consider adding the following common Chinese characters: the sum is not allowed (however, there are special characters such as "playing" "Zheng He"...: p)

// Calculation time
Function getmicrotime (){
List ($ usec, $ sec) = explode ("", microtime ());
Return (float) $ usec + (float) $ sec );
}
$ Time_start = getmicrotime ();

// Lexicographic class
Class ch_dictionary {
Var $ _ id;

Function ch_dictionary ($ fname = ""){
If ($ fname! = ""){
$ This-> load ($ fname );
}
}

// Load the dictionary (gdbm data file) according to the file name)
Function load ($ fname ){
$ This-> _ id = dba_popen ($ fname, "r", "gdbm ");
If (! $ This-> _ id ){
Echo "failed to open the dictionary. ($ fname)
\ N ";
Exit;
}
}

// Return frequency based on words.-1 is returned if no words are returned.
Function find ($ word ){
$ Freq = dba_fetch ($ word, $ this-> _ id );
If (is_bool ($ freq) $ freq =-1;
Return $ freq;
}
}

// Word Segmentation: (reverse)
// Cut the input string into sentences in the forward direction, and then combine the word segmentation to return an array composed of words.
Class ch_word_split {
Var $ _ mb_mark_list; // The full-angle punctuation of common split sentences
Var $ _ word_maxlen; // maximum possible length of a single word (Chinese characters)
Var $ _ dic; // Dictionary...
Var $ _ ignore_mark; // true or false

Function ch_word_split (){
$ This-> _ mb_mark_list = array (",","",". ","! ","? ",":","...... "," (",") ");
$ This-> _ word_maxlen = 12; // 12 Chinese characters
$ This-> _ dic = NULL;
$ This-> _ ignore_mark = true;
}

// Set the dictionary
Function set_dic ($ fname ){
$ This-> _ dic = new ch_dictionary ($ fname );
}

Function set_ignore_mark ($ set ){
If (is_bool ($ set) $ this-> _ ignore_mark = $ set;
}

// Cut the string into sentences and then split it into words
Function string_split ($ str, $ func = ""){
$ Ret = array ();

If ($ func = "" |! Function_exists ($ func) $ func = "";

$ Len = strlen ($ str );
$ Qtr = "";

For ($ I = 0; $ I <$ len; $ I ++ ){
$ Char = $ str [$ I];

If (ord ($ char) <0xa1 ){
// Read a half-width character
If (! Empty ($ qtr )){
$ Tmp = $ this-> _ sen_split ($ qtr );
$ Qtr = "";

If ($ func! = "") Call_user_func ($ func, $ tmp );
Else $ ret = array_merge ($ ret, $ tmp );
}

// If it is a word or number, read data to> = 0xa1 based on char
If ($ this-> _ is_alnum ($ char )){
Do {
If ($ I + 1) >=$ len) break;
$ Char2 = substr ($ str, $ I + 1, 1 );
If (! $ This-> _ is_alnum ($ char2) break;

$ Char. = $ char2;
$ I ++;
} While (1 );

If ($ func! = "") Call_user_func ($ func, array ($ char ));
Else $ ret [] = $ char;
}
Elseif ($ char = ''| $ char =" \ t "){
// Nothing.
Continue;
}
Elseif (! $ This-> _ ignore_mark ){
If ($ func! = "") Call_user_func ($ func, array ($ char ));
Else $ ret [] = $ char;
}
}
Else {
// Double byte characters.
$ I ++;
$ Char. = $ str [$ I];

If (in_array ($ char, $ this-> _ mb_mark_list )){
If (! Empty ($ qtr )){
$ Tmp = $ this-> _ sen_split ($ qtr );
$ Qtr = "";

If ($ func! = "") Call_user_func ($ func, $ tmp );
Else $ ret = array_merge ($ ret, $ tmp );
}

If (! $ This-> _ ignore_mark ){
If ($ func! = "") Call_user_func ($ func, array ($ char ));
Else $ ret [] = $ char;
}
}
Else {
$ Qtr. = $ char;
}
}
}

If (strlen ($ qtr)> 0 ){
$ Tmp = $ this-> _ sen_split ($ qtr );

If ($ func! = "") Call_user_func ($ func, $ tmp );
Else $ ret = array_merge ($ ret, $ tmp );
}

// Return value
If ($ func = ""){
Return $ ret;
}
Else {
Return true;
}
}

// Cut sentences into words and reverse
Function _ sen_split ($ sen ){
$ Len = strlen ($ sen)/2;
$ Ret = array ();

For ($ I = $ len-1; $ I> = 0; $ I --){
// For example, this is a word splitting program.

// Obtain the last word first
$ W = substr ($ sen, $ I * 2, 2 );

// The final term length
$ Wlen = 1;

// Start reverse matching to the maximum length.
$ Lf = 0; // last freq
For ($ j = 1; $ j <= $ this-> _ word_maxlen; $ j ++ ){
$ O = $ I-$ j;
If ($ o <0) break;
$ W2 = substr ($ sen, $ o * 2, ($ j + 1) * 2 );

$ Tmp_f = $ this-> _ dic-> find ($ w2 );
// Echo "{$ I}. {$ j}: $ w2 (f: $ tmp_f) \ n ";
If ($ tmp_f> $ lf ){
$ Lf = $ tmp_f;
$ Wlen = $ j + 1;
$ W = $ w2;
}
}
// Offset $ I according to $ wlen
$ I = $ I-$ wlen + 1;
Array_push ($ ret, $ w );
}

$ Ret = array_reverse ($ ret );
Return $ ret;
}

// Determine whether the character is a letter or number _-[0-9a-z _-]
Function _ is_alnum ($ char ){
$ Ord = ord ($ char );
If ($ ord = 45 | $ ord = 95 | ($ ord> = 48 & $ ord <= 57 ))
Return true;
If ($ ord >=97 & $ ord <= 122) | ($ ord >=65 & $ ord <= 90 ))
Return true;
Return false;
}
}

// Callback function after word segmentation
Function call_back ($ ar ){
Foreach ($ ar as $ tmp ){
Echo $ tmp ."";
// Flush ();
}
}

// Instance (read from sample.txt if there is no input ):
$ Wp = new ch_word_split ();
$ Wp-> set_dic ("dic. db ");

If (! Isset ($ _ REQUEST ['testdat ']) | empty ($ _ REQUEST ['testdat']) {
$ Data = file_get_contents ("sample.txt ");
}
Else {
$ Data = & $ _ REQUEST ['testdat '];
}

// Output
Echo "simple word segmentation demonstration \ n ";
Echo "\ n ";
Echo "word splitting result (". strlen ($ data). "chars ):
\ N\ N "; </p> <p> // Set whether to ignore the unreturned word segmentation symbol (punctuation, common words) <br/> $ wp-> set_ignore_mark (false ); </p> <p> // run splitting. if the callback function is not set, an array composed of words is returned. <br/> $ wp-> string_split ($ data, "call_back"); </p> <p> $ time_end = getmicrotime (); <br/> $ time = $ time_end-$ time_start; </p> <p> echo"
\ N time consumed for this word segmentation: $ time seconds
\ N ";
?>



Appendix:

Source code of this program: chinese_segment.php (simple implementation)

Required Dictionary: dic. db (gdbm format)

Original address: http://www.phpchina.com /? 3870/action_viewspace_itemid_2609.html