Write a search engine by yourself (changsouba course 4 # Word Segmentation #) (Java, Lucene, hadoop)

Basic Principles of Word Segmentation: 1. Word Segmentation is a technology used to filter and group texts by language features based on algorithms. 2. The word splitting object is text, not an image animation script. 3. Word Segmentation: filtering and grouping. 4. Filtering mainly filters out words or words that have no practical significance in the text. 5. grouping is performed based on the words added to the word segmentation database. The following describes how to use the [java] package com. qianyan. analyzer; import java. io. IOException; import java. io. stringReader; import org. apache. lucene. analysis. analyzer; import org. apache. lucene. analysis. simpleAnalyzer; import org. apache. lucene. analysis. token; import org. apache. lucene. analysis. tokenStream; import org. apache. lucene. analysis. whitespaceAnalyzer; import org. apache. lucene. analysis. cjk. CJKAnalyzer; imp Ort org.apache.e.analysis.cn. chineseAnalyzer; import org. apache. lucene. analysis. standard. standardAnalyzer; public class TestAnalyzer {public static void main (String [] args) throws IOException {Analyzer analyzer = new StandardAnalyzer (); // standard filtering disabled times // Analyzer analyzer = new SimpleAnalyzer (); // simple filtering of spaces and symbols // Analyzer analyzer = new WhitespaceAnalyzer (); // filter space // Analyzer analyzer = new Chinese Analyzer (); // The Chinese word divider in lucene splits each character into a filter symbol. // Analyzer analyzer = new CJKAnalyzer (); // split the Chinese character into two characters. The English and standard functions are the same. String input = "this is test lucene analyzer class! "; TokenStream tokenStream = analyzer. tokenStream (" ", new StringReader (input); Token token = new Token (); while (null! = TokenStream. next (token) System. out. println (token. term () ;}} for beginners, we only need to master these classic word divider. However, in the actual development process, some third-party Chinese Word Segmentation packages based on lucene word segmentation are provided to meet our needs. Here we will only introduce the "ding word segmentation package", the command draws on the idiom "ding Jie niu. Ding jieniu, an ancient Chinese idiom from Zhuangzi, is a metaphor that has been practiced over and over again to grasp the objective laws of things. It is easy to do things and easy to use. After decompression, we need to add 2 jar packages to the project, unzip the paoding-analysis.jar under the Directory and the commons-logging.jar under lib. Copy the dic folder to the src directory of our project. [Java] package com. qianyan. analyzer; import java. io. IOException; import java. io. stringReader; import net. paoding. analysis. analyzer. paodingAnalyzer; import org. apache. lucene. analysis. analyzer; import org. apache. lucene. analysis. token; import org. apache. lucene. analysis. tokenStream; public class TestPaodingAnalyzer {public static void main (String [] args) throws IOException {Analyzer analyzer = new Pa OdingAnalyzer (); String input = "I Love Tiananmen, Beijing! "; TokenStream ts = analyzer. tokenStream (" ", new StringReader (input); Token token = new Token (); while (null! = (Token = ts. next (null) System. out. println (token. term () ;}you can see through this example that the paoding analyzer is quite powerful and Its syntax is not described here, if you are interested, read the decompressed Chinese operation manual. Next, let's take a look at how to create an index based on the paoding Analyzer: [java] package com. qianyan. index; import java. io. IOException; import net. paoding. analysis. analyzer. paodingAnalyzer; import org. apache. lucene. analysis. analyzer; import org.apache.e.doc ument. document; import org.apache.e.doc ument. field; import org. apache. lucene. index. indexWriter; import org. apache. lucene. store. directory; import org. apache. lucene. store. FSDirector Y; public class TestPaodingIndex {public static void main (String [] args) throws IOException {String [] ids = {"1", "2", "3 ", "4"}; String [] names = {"Zhang San", "Li Si", "Li Wu", "Zhao Liu "}; string [] addresses = {"living in Beijing", "Nanjing", "Beijing Haidian", "Nanning"}; String [] birthdays = {"19820720", "19840203 ", "19770409", "19830130"}; Analyzer analyzer = new PaodingAnalyzer (); String indexDir = "E:/luceneindex"; Directory dir = FSDirect Ory. getDirectory (indexDir); IndexWriter writer = new IndexWriter (dir, analyzer, true, IndexWriter. maxFieldLength. UNLIMITED); for (int I = 0; I <ids. length; I ++) {Document document = new Document (); document. add (new Field ("id", ids [I], Field. store. YES, Field. index. ANALYZED); document. add (new Field ("name", names [I], Field. store. YES, Field. index. ANALYZED); document. add (new Field ("address", addresse S [I], Field. store. YES, Field. index. ANALYZED); document. add (new Field ("birthday", birthdays [I], Field. store. YES, Field. index. ANALYZED); writer. addDocument (document);} writer. optimize (); writer. close () ;}then let's look at the simple search class [java] package com. qianyan. search; import java. io. IOException; import org.apache.e.doc ument. document; import org. apache. lucene. index. term; import org. apache. lucene. search. indexS Earcher; import org. apache. lucene. search. prefixQuery; import org. apache. lucene. search. scoreDoc; import org. apache. lucene. search. termQuery; import org. apache. lucene. search. topDocs; import org. apache. lucene. search. wildcardQuery; import org. apache. lucene. store. directory; import org. apache. lucene. store. FSDirectory; public class TestPaodingSearch {public static void main (String [] args) throws IOExcept Ion {String indexDir = "E:/javaseindex"; Directory dir = FSDirectory. getDirectory (indexDir); IndexSearcher searcher = new IndexSearcher (dir); ScoreDoc [] hits = null;/* Term term = new Term ("address", "Beijing "); termQuery query = new TermQuery (term); * // * Term term = new Term ("name", "Zhang"); PrefixQuery query = new PrefixQuery (term ); */Term term = new Term ("name", "Li *"); WildcardQuery query = new Wildca RdQuery (term); TopDocs topDocs = searcher. search (query, 100); hits = topDocs. scoreDocs; for (int I = 0; I