Lucene word Breaker

The base class for the Lucene parser is Analyzer,analyzer contains two core components: Tokenizer and Tokenfilter. The custom parser must implement an abstract method Createcomponents (String) of the Analyzer class to define the tokenstreamcomponents. When calling method Tokenstream (String, Reader), tokenstreamcomponents is reused.

The custom parser first needs to inherit the Analyzer class with the following code:

 Public classHanalyzerextendsAnalyzer {/** Inactive words are not used by default **/    Private Booleanusestopwords; PrivateChararrayset stopwords;  PublicHanalyzer () {usestopwords=false; }         PublicHanalyzer (Chararrayset stopwords) {usestopwords=true;  This. Stopwords =stopwords; } @Overrideprotectedtokenstreamcomponents createcomponents (String fieldName) {Lettertokenizer Tokenizer=NewLettertokenizer (); if(usestopwords) {return NewTokenstreamcomponents (Tokenizer,NewHstoptokenfilter (Tokenizer, stopwords)); }        return Newtokenstreamcomponents (Tokenizer); }}

Analyzer two core components: Tokenizer and Tokenfilter, implemented as follows:

/** Word breaker, need to define token attribute Chartermattribute Offsetattribute **/ Public classLettertokenizerextendsTokenizer {/** Word meta Text properties **/     Private FinalChartermattribute Termatt = AddAttribute (chartermattribute.class); /** Word Element Displacement Property **/     Private FinalOffsetattribute Offsetatt = AddAttribute (offsetattribute.class); /** Token text maximum length **/    Private Static Final intMax_word_len = 255; /** Buffer Size **/    Private Static Final intIo_buffer_size = 4096; Private Char[] Iobuffer =New Char[Io_buffer_size]; /** Token delimiter set **/    Private Char[] Splitchars = {', ', ', '. ', '! '}; /** Position of the current string in the original string **/    Private intOffset = 0; /** The position of the current character in this read string **/    Private intBufferindex = 0; /** Length of each read String **/    Private intDatalen = 0; @Override Public BooleanIncrementtoken ()throwsIOException {clearattributes (); //clears all properties of the previous token        intlength = 0;//the length of the word        intStart =Bufferindex; Char[]buffer =Termatt.buffer ();  while(true) {            if(Bufferindex >= Datalen) {//Continue reading data from input when the word breaker is processed to the end of IobufferOffset + =Datalen; Datalen=Input.read (Iobuffer); if(Datalen = =-1) {//at reader end of readerDatalen = 0; if(Length > 0) {//Although the data has been read from input, the characters processed by Iobuffer have not generated tokens                         Break; } Else {                        return false; }} Bufferindex= 0;//point to start position of Iobuffer            }            /**handling characters read by Iobuffer*/            Final CharCH = iobuffer[bufferindex++]; if(Istokenchar (CH)) {//ch delimiter, form token, jump out of loop                if(length = = 0) {Start= offset + bufferIndex-1; } Else if(Length = =buffer.length) {buffer= Termatt.resizebuffer (length + 1); }                if(Length = =Max_word_len) {                     Break; }                 Break; } Else{buffer[length+ +] = normalize (CH);//Chartermattribute Text Assignment}} termatt.setlength (length); Offsetatt.setoffset (Correctoffset (start), Correctoffset (Start+length)); return true; }    /** Normalized---> lowercase **/    protected CharNormalizeCharch) {        returncharacter.tolowercase (CH); }            /** If the character ch is a delimiter, return true **/    protected BooleanIstokenchar (Charch) {         for(Charc:splitchars) {            if(ch = =c) {return true; }        }        return false; }    }

/** Filter tokenstream, need to change token's Positionincrementattribute attribute **/ Public classHstoptokenfilterextendsTokenfilter {/** Tokenstream Stream Token Text properties **/    Private FinalChartermattribute Termatt = AddAttribute (chartermattribute.class); /** Current token and previous token shift difference attribute **/    PrivatePositionincrementattribute Posincratt = AddAttribute (positionincrementattribute.class); Private intskippedpositions; /** Disable Word Collection **/    PrivateChararrayset stopwords; protectedhstoptokenfilter (Tokenstream input) {Super(input); }         Publichstoptokenfilter (tokenstream input, Chararrayset stopwords) { This(input);  This. Stopwords =stopwords; } @Override Public BooleanIncrementtoken ()throwsIOException {clearattributes (); //clear all properties of last tokenskippedpositions = 0;  while(Input.incrementtoken ()) {if(Filter ()) {//filter out current token, modify SkippedpositionsSkippedpositions + =posincratt.getpositionincrement (); } Else{//The current token is not filtered and if the previous token is filtered, the Positionincrementattribute attribute of the current token needs to be modified                if(Skippedpositions! = 0) {posincratt.setpositionincrement (posincratt.getpositionincrement ()+skippedpositions); }                return true; }        }        return false; }    Private Booleanfilter () {returnStopwords.contains (Termatt.buffer (), 0, Termatt.length ()); }}

With the custom Hanalyzer, you can complete the text analysis, with the following examples:

 Public classMain { Public Static voidMain (String []args) {Hanalyzer Analyzer=NewHanalyzer (); Tokenstream TS=NULL; Try{TS= Analyzer.tokenstream ("MyField",NewStringReader ("I am a student. My name is tom! ")); //get the Word element Location propertyOffsetattribute offset = Ts.addattribute (offsetattribute.class); //get Word meta Text propertiesChartermattribute term = ts.addattribute (chartermattribute.class); //Reset tokenstream (reset StringReader)Ts.reset (); //iterate to get the result of a word             while(Ts.incrementtoken ()) {System.out.println (Offset.startoffset () )+ "-" + offset.endoffset () + ":" +term.tostring ()); }            //Close Tokenstream (Close StringReader)Ts.end (); } Catch(IOException e) {e.printstacktrace (); }    }    }

Lucene word Breaker