Lucene BASICS (III)-Chinese Word Segmentation and highlight, lucene Word Segmentation

Lucene BASICS (III)-Chinese Word Segmentation and highlight, lucene Word Segmentation
Lucene word divider and highlightingWord Divider

In lucene, documents are indexed Based on word segmentation. Different Word divider indexes have different effects. In the previous example, the standard word divider is used, which has a good effect on English, however, the effect of Chinese word segmentation is not very good. He can directly perform Word Segmentation Based on Chinese characters without the concept of words.

To use word segmentation, you only need to convert the Analyzer instance into a third-party word divider.

There are many Chinese word segmentation. Here, IKAnalyzer is used as an example,
Https://git.oschina.net/wltea/IK-Analyzer-2012FF now has a tutorial in it.

Highlight

Import the lucene-highlighter-xxx.jar to highlight the query results

// Html tag highlighted by keyword, need to import lucene-highlighter-xxx.jar

SimpleHTMLFormatter simpleHTMLFormatter = new SimpleHTMLFormatter ("<span style = 'color: red'>", "</span> ");

Highlighter highlighter = new Highlighter (simpleHTMLFormatter, new QueryScorer (query ));

For (int I = 0; I

Document doc = isearcher.doc(hits? I =.doc );

// Add highlighted content

TokenStream tokenStream = analyzer. tokenStream ("content", new StringReader (doc. get ("content ")));

String content = highlighter. getBestFragment (tokenStream, doc. get ("content"); System. out. println (content );

}

 

Lucene Chinese Word Divider

Package quota e_demo04;

Import java. io. IOException;
Import java. io. StringReader;

Import org. apache. lucene. analysis. Analyzer;
Import org. apache. lucene. analysis. TokenStream;
Import org.apache.e.doc ument. Document;
Import org.apache.e.doc ument. Field;
Import org.apache.e.doc ument. TextField;
Import org. apache. lucene. index. CorruptIndexException;
Import org. apache. lucene. index. DirectoryReader;
Import org. apache. lucene. index. IndexWriter;
Import org. apache. lucene. index. IndexWriterConfig;
Import org. apache. lucene. index. IndexWriterConfig. OpenMode;
Import org. apache. lucene. queryparser. classic. ParseException;
Import org. apache. lucene. queryparser. classic. QueryParser;
Import org. apache. lucene. search. IndexSearcher;
Import org. apache. lucene. search. Query;
Import org. apache. lucene. search. QueryWrapperFilter;
Import org. apache. lucene. search. ScoreDoc;
Import org. apache. lucene. search. highlight. Highlighter;
Import org. apache. lucene. search. highlight. InvalidTokenOffsetsException;
Import org. apache. lucene. search. highlight. QueryScorer;
Import org. apache. lucene. search. highlight. SimpleHTMLFormatter;
Import org. apache. lucene. store. Directory;
Import org. apache. lucene. store. RAMDirectory;
Import org. apache. lucene. util. Version;
Import org. wltea. analyzer. lucene. IKAnalyzer;

/**
* Chinese word segmentation, IKAnalayzer, highlighting index results
*
* @ Author YipFun
*/
Public class extends edemo04
{
Private static final Version version = Version. paie_4_9;
Private Directory directory = null;
Private DirectoryReader ireader = null;
Private IndexWriter iwriter = null;
Private IKAnalyzer analyzer;

// Test Data
Private String [] content = {"Hello, I am a member of the Communist Party of China", "People's Republic of China", "the Chinese people have stood up since then", "Lucene is a good full-text retrieval tool ", "Chinese word segmentation for full-text search "};

/**
* Constructor
*/
Public writable edemo04 ()
{
Directory = new RAMDirectory ();
}

Private IKAnalyzer getAnalyzer ()
{
If (analyzer = null)
{
Return new IKAnalyzer ();
} Else
{
Return analyzer;
}
}

/**
* Create an index
*/
Public void createIndex ()
{
Document doc = null;
Try
{
IndexWriterConfig iwConfig = new IndexWriterConfig (version, getAnalyzer ());
IwConfig. setOpenMode (OpenMode. CREATE_OR_APPEND );
Iwriter = new IndexWriter (directory, iwConfig );
For (String text: content)
{
Doc = new Document ();
Doc. add (new TextField ("content", text, Field. Store. YES ));
Iwriter. addDocument (doc );
}

} Catch (IOException e)
{
E. printStackTrace ();
} Finally
{
Try
{
If (iwriter! = Null)
Iwriter. close ();
} Catch (IOException e)
{
E. printStackTrace ();
}
}

}

Public IndexSearcher getSearcher ()
{
Try
{
If (ireader = null)
{
Ireader = DirectoryReader. open (directory );
} Else
{
DirectoryReader tr = DirectoryReader. openIfChanged (ireader );
If (tr! = Null)
{
Ireader. close ();
Ireader = tr;
}
}
Return new IndexSearcher (ireader );
} Catch (CorruptIndexException e)
{
E. printStackTrace ();
} Catch (IOException e)
{
E. printStackTrace ();
}
Return null;
}

Public void searchByTerm (String field, String keyword, int num) throws InvalidTokenOffsetsException
{
IndexSearcher isearcher = getSearcher ();
Analyzer analyzer = getAnalyzer ();
// Use QueryParser to Query analyzer to construct a Query object
QueryParser qp = new QueryParser (version, field, analyzer );
// What is the result of this sentence?
Qp. setDefaultOperator (QueryParser. OR_OPERATOR );
Try
{
Query query = qp. parse (keyword );
ScoreDoc [] hits;

// Several Methods of searcher
Hits = isearcher. search (query, null, num). scoreDocs;

// Html tag highlighted by keyword, need to import lucene-highlighter-xxx.jar
SimpleHTMLFormatter simpleHTMLFormatter = new SimpleHTMLFormatter ("<span style = 'color: red'>", "</span> ");
Highlighter highlighter = new Highlighter (simpleHTMLFormatter, new QueryScorer (query ));

For (int I = 0; I {
Document doc = isearcher.doc(hits? I =.doc );
// Add highlighted content
TokenStream tokenStream = analyzer. tokenStream ("content", new StringReader (doc. get ("content ")));
String content = highlighter. getBestFragment (tokenStream, doc. get ("content "));
System. out. println (content );
}

} Catch (IOException e)
{
E. printStackTrace ();
} Catch (ParseException e)
{
E. printStackTrace ();
}
}

/**
* Filter Query
*
* @ Param field
* @ Param keyword
* @ Param num
* @ Throws InvalidTokenOffsetsException
*/
Public void searchByTermFilter (String field, String keyword, int num) throws InvalidTokenOffsetsException
{
IndexSearcher isearcher = getSearcher ();
Analyzer analyzer = getAnalyzer ();
// Use QueryParser to Query analyzer to construct a Query object
QueryParser qp = new QueryParser (version, field, analyzer );
// What is the result of this sentence?
Qp. setDefaultOperator (QueryParser. OR_OPERATOR );
Try
{
Query query = qp. parse (keyword );
Query q2 = qp. parse ("full-text search ");
ScoreDoc [] hits;

QueryWrapperFilter filter = new QueryWrapperFilter (q2 );
// Several Methods of searcher
Hits = isearcher. search (query, filter, num). scoreDocs;

// Html tag highlighted by keyword, need to import lucene-highlighter-xxx.jar
SimpleHTMLFormatter simpleHTMLFormatter = new SimpleHTMLFormatter ("<span style = 'color: red'>", "</span> ");
Highlighter highlighter = new Highlighter (simpleHTMLFormatter, new QueryScorer (query ));

For (int I = 0; I {
Document doc = isearcher.doc(hits? I =.doc );
// Add highlighted content
TokenStream tokenStream = analyzer. tokenStream ("content", new StringReader (doc. get ("content ")));
String content = highlighter. getBestFragment (tokenStream, doc. get ("content "));
System. out. println (content );
}

} Catch (IOException e)
{
E. printStackTrace ();
} Catch (ParseException e)
{
E. printStackTrace ();
}
}

Public static void main (String [] args) throws InvalidTokenOffsetsException
{
System. out. println ("start ");
Incluedemo04 ld = new incluedemo04 ();
Ld. createIndex ();
Long start = System. currentTimeMillis ();
Ld. searchByTerm ("content", "people", 500 );
System. out. println ("end search use" + (System. currentTimeMillis ()-start) + "ms ");
}

}

Running result:

 

Start: ext. dic

Load the extended stopword dictionary: stopword. dic

<Span style = 'color: red'> People's Republic of China </span>

<Span style = 'color: red'> people in China </span> have stood up.

End search use 129 ms