Chinese Word Segmentation source code-Java

package org.apache.lucene.analysis.cn;/** * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements.  See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License.  You may obtain a copy of the License at * *     http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */import java.io.Reader;import org.apache.lucene.analysis.*;/** * Title: ChineseTokenizer * Description: Extract tokens from the Stream using Character.getType() *              Rule: A Chinese character as a single token * Copyright:   Copyright (c) 2001 * Company: * * The difference between thr ChineseTokenizer and the * CJKTokenizer (id=23545) is that they have different * token parsing logic. *  * Let me use an example. If having a Chinese text * "C1C2C3C4" to be indexed, the tokens returned from the * ChineseTokenizer are C1, C2, C3, C4. And the tokens * returned from the CJKTokenizer are C1C2, C2C3, C3C4. * * Therefore the index the CJKTokenizer created is much * larger. * * The problem is that when searching for C1, C1C2, C1C3, * C4C2, C1C2C3 ... the ChineseTokenizer works, but the * CJKTokenizer will not work. * * @author Yiyi Sun * @version 1.0 * */public final class ChineseTokenizer extends Tokenizer {    public ChineseTokenizer(Reader in) {        input = in;    }    private int offset = 0, bufferIndex=0, dataLen=0;    private final static int MAX_WORD_LEN = 255;    private final static int IO_BUFFER_SIZE = 1024;    private final char[] buffer = new char[MAX_WORD_LEN];    private final char[] ioBuffer = new char[IO_BUFFER_SIZE];    private int length;    private int start;    private final void push(char c) {        if (length == 0) start = offset-1;            // start of token        buffer[length++] = Character.toLowerCase(c);  // buffer it    }    private final Token flush() {        if (length>0) {            //System.out.println(new String(buffer, 0, length));            return new Token(new String(buffer, 0, length), start, start+length);        }        else            return null;    }    public final Token next() throws java.io.IOException {        length = 0;        start = offset;        while (true) {            final char c;            offset++;            if (bufferIndex >= dataLen) {                dataLen = input.read(ioBuffer);                bufferIndex = 0;            }            if (dataLen == -1) return flush();            else                c = ioBuffer[bufferIndex++];            switch(Character.getType(c)) {            case Character.DECIMAL_DIGIT_NUMBER:            case Character.LOWERCASE_LETTER:            case Character.UPPERCASE_LETTER:                push(c);                if (length == MAX_WORD_LEN) return flush();                break;            case Character.OTHER_LETTER:                if (length>0) {                    bufferIndex--;                    offset--;                    return flush();                }                push(c);                return flush();            default:                if (length>0) return flush();                break;            }        }    }}

/////////////////////////////////////////

package org.apache.lucene.analysis.cn;/** * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements.  See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License.  You may obtain a copy of the License at * *     http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */import java.util.Hashtable;import org.apache.lucene.analysis.*;/** * Title: ChineseFilter * Description: Filter with a stop word table *              Rule: No digital is allowed. *                    English word/token should larger than 1 character. *                    One Chinese character as one Chinese word. * TO DO: *   1. Add Chinese stop words, such as /ue400 *   2. Dictionary based Chinese word extraction *   3. Intelligent Chinese word extraction * * Copyright:    Copyright (c) 2001 * Company: * @author Yiyi Sun * @version 1.0 * */public final class ChineseFilter extends TokenFilter {    // Only English now, Chinese to be added later.    public static final String[] STOP_WORDS = {    "and", "are", "as", "at", "be", "but", "by",    "for", "if", "in", "into", "is", "it",    "no", "not", "of", "on", "or", "such",    "that", "the", "their", "then", "there", "these",    "they", "this", "to", "was", "will", "with"    };    private Hashtable stopTable;    public ChineseFilter(TokenStream in) {        super(in);        stopTable = new Hashtable(STOP_WORDS.length);        for (int i = 0; i < STOP_WORDS.length; i++)            stopTable.put(STOP_WORDS[i], STOP_WORDS[i]);    }    public final Token next() throws java.io.IOException {        for (Token token = input.next(); token != null; token = input.next()) {            String text = token.termText();          // why not key off token type here assuming ChineseTokenizer comes first?            if (stopTable.get(text) == null) {                switch (Character.getType(text.charAt(0))) {                case Character.LOWERCASE_LETTER:                case Character.UPPERCASE_LETTER:                    // English word/token should larger than 1 character.                    if (text.length()>1) {                        return token;                    }                    break;                case Character.OTHER_LETTER:                    // One Chinese character as one Chinese word.                    // Chinese word extraction to be added later here.                    return token;                }            }        }        return null;    }}

/////////////////////////

package org.apache.lucene.analysis.cn;/** * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements.  See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License.  You may obtain a copy of the License at * *     http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */import java.io.Reader;import org.apache.lucene.analysis.Analyzer;import org.apache.lucene.analysis.TokenStream;/** * Title: ChineseAnalyzer * Description: *   Subclass of org.apache.lucene.analysis.Analyzer *   build from a ChineseTokenizer, filtered with ChineseFilter. * Copyright:   Copyright (c) 2001 * Company: * @author Yiyi Sun * @version 1.0 * */public class ChineseAnalyzer extends Analyzer {    public ChineseAnalyzer() {    }    /**    * Creates a TokenStream which tokenizes all the text in the provided Reader.    *    * @return  A TokenStream build from a ChineseTokenizer filtered with ChineseFilter.    */    public final TokenStream tokenStream(String fieldName, Reader reader) {        TokenStream result = new ChineseTokenizer(reader);        result = new ChineseFilter(result);        return result;    }}