[Recommended] jsp Chinese word segmentation program

The code is as follows:

Copy code

Publicclass MM2 { Privatestaticfinal Log log = LogFactory. getLog (MM2.class );   Privatestatic HashMap <String, Integer> dictionary = null; Privatestaticfinalint WORD_MAX_LENGTH = 9; Private Reader;   Static  { LoadDictionary ();  }   Public MM2 (Reader reader)  { This. reader = reader;  }   // Cut sentences consisting of Chinese characters, letters, and numbers Public ArrayList <Sentence> getSentence () throws IOException  { ArrayList <Sentence> list = new ArrayList <Sentence> (); StringBuffer cb = new StringBuffer (); Int d = reader. read (); Int offset = 0; Boolean B = false; While (d>-1)  { Int type = Character. getType (d ); If (type = 2 | type = 9 | type = 5)  { D = toAscii (d ); Cb. append (char) d );  } Else  { B = true;  } D = reader. read (); If (d =-1 | B)  { If (d =-1) offset ++; B = false; Char [] ioBuffer = newchar [cb. length ()]; Cb. getChars (0, cb. length (), ioBuffer, 0 ); Sentence sen = new Sentence (ioBuffer, offset-cb.length ()); List. add (sen ); Cb. setLength (0 );  } Offset ++;  } Return list;  }   // Cut sentences into words Public ArrayList <Token> getToken (ArrayList <Sentence> list) throws IOException  { ArrayList <Token> tokenlist = new ArrayList <Token> (); For (Sentence sen: list)  { StringBuffer word = new StringBuffer (); Int offset = sen. getStartOffset (); Int bufferIndex = 0; Char c; Boolean B = false; While (bufferIndex <sen. getText (). length)  { Offset ++; C = sen. getText () [bufferIndex ++]; If (word. length () = 0) Word. append (c ); Else  { String temp = (word. toString () + c). intern (); If (dictionary. containsKey (temp) & dictionary. get (temp) = 1) Word. append (c ); Elseif (dictionary. containsKey (temp) & bufferIndex <sen. getText (). length) Word. append (c ); Else  { BufferIndex --; Offset --; While (word. length ()> 1 & dictionary. get (word. toString ())! = Null & dictionary. get (word. toString () = 2)  { Word. deleteCharAt (word. length ()-1 ); BufferIndex --; Offset --;  } B = true;  }  } If (B | bufferIndex = sen. getText (). length)  { Token token = new Token (word. toString (), offset-word.length (), offset, "word "); Word. setLength (0 ); Tokenlist. add (token ); B = false;  }  }  } Return tokenlist;  }   // Combine a single English or number. Public ArrayList <Token> getNewToken (ArrayList <Token> list) throws IOException  { ArrayList <Token> tokenlist = new ArrayList <Token> (); Token word = null; For (int I = 0; I <list. size (); I ++)  { Token t = list. get (I ); If (t. getWord (). length () = 1 & Character. getType (int) t. getWord (). charAt (0 ))! = 5)  { If (word = null) Word = t; Elseif (word. getEnd () = t. getStart ())  { Word. setEnd (t. getEnd ()); Word. setWord (word. getWord () + t. getWord ());  } Else  { Tokenlist. add (word ); Word = t;  }  } Elseif (word! = Null)  { Tokenlist. add (word ); Word = null; Tokenlist. add (t );  } Else Tokenlist. add (t );  } If (word! = Null) Tokenlist. add (word ); Return tokenlist;  }   // Convert dual-angle to single-angle Publicstaticint toAscii (int codePoint)  { If (codePoint> = 65296 & codePoint <= 65305) // 0-9 | (CodePoint> = 65313 & codePoint <= 65338) // A-Z | (CodePoint> = 65345 & codePoint <= 65370) // a-z  )  { CodePoint-= 65248;  } Return codePoint;  }   // Load the dictionary Publicstaticvoid loadDictionary ()  { If (dictionary = null)  { Dictionary = new HashMap <String, Integer> (); InputStream is = null; BufferedReader br = null; Try  { Is = new FileInputStream (new File (MM2.class. getClassLoader (). getResource ("dictionary.txt"). toURI ())); Br = new BufferedReader (new InputStreamReader (is, "UTF-8 ")); String word = null; While (word = br. readLine ())! = Null)  { Word = word. toLowerCase (); If (word. indexOf ("#") =-1) & (word. length () <= WORD_MAX_LENGTH ))  { Dictionary. put (word. intern (), 1 ); Int I = word. length ()-1; While (I> = 2)  { String temp = word. substring (0, I). intern (); If (! Dictionary. containsKey (temp )) Dictionary. put (temp, 2 ); I --;  }  }  }  } Catch (Exception e)  { Log.info (e );  } Finally  { Try  { If (br! = Null) Br. close (); If (is! = Null) Is. close ();  } Catch (IOException e)  { Log.info (e );  }  }  }  }   Publicstatic String [] segWords (Reader input)  { ArrayList <String> list = new ArrayList <String> (); Try  { MM2 f = new MM2 (input ); ArrayList <Token> tlist = f. getNewToken (f. getToken (f. getSentence ())); For (Token t: tlist)  { List. add (t. getWord ());  }  } Catch (IOException e)  { Log.info (e );  } Return (String []) list. toArray (new String [0]);  }   Publicstaticvoid main (String [] args)  { String [] cc = MM2.segWords (new StringReader ("ibm business machine t60p". toLowerCase ())); For (String c: cc)  { System. out. println (c );  }  } }