測試lucene的所有分詞介面

07-27

測試lucene的所有分詞介面(原創)- -Tag： Lucene 中文分詞

Lucene本身提供了幾個分詞介面,我後來有給寫了一個分詞介面.

功能遞增如下:

WhitespaceAnalyzer:僅僅是去除空格，對字元沒有lowcase化,不支持中文

SimpleAnalyzer:功能強於WhitespaceAnalyzer,將除去letter之外的符號全部過濾掉,並且將所有的字元lowcase化,不支持中文

StopAnalyzer:StopAnalyzer的功能超越了SimpleAnalyzer，在SimpleAnalyzer的基礎上增加了去除StopWords的功能,不支持中文

StandardAnalyzer:英文的處理能力同於StopAnalyzer.支持中文採用的方法為單字切分.

ChineseAnalyzer:來自於Lucene的sand box.性能類似於StandardAnalyzer,缺點是不支持中英文混和分詞.

CJKAnalyzer:chedong寫的CJKAnalyzer的功能在英文處理上的功能和StandardAnalyzer相同但是在漢語的分詞上，不能過濾掉標點符號，即使用二元切分

TjuChineseAnalyzer:我寫的,功能最為強大.TjuChineseAnlyzer的功能相當強大,在中文分詞方面由於其調用的為ICTCLAS的java介面.所以其在中文方面性能上同與ICTCLAS.其在英文分詞上採用了Lucene的StopAnalyzer,可以去除stopWords,而且可以不區分大小寫,過濾掉各類標點符號.

程序調試於:JBuilder 2005

package org.apache.lucene.analysis;

//Author:zhangbufeng//TjuAILab(天津大學人工智慧實驗室)//2005.9.22.11:00

import java.io.*;import junit.framework.*;

import org.apache.lucene.*;import org.apache.lucene.analysis.*;import org.apache.lucene.analysis.StopAnalyzer;import org.apache.lucene.analysis.standard.*;import org.apache.lucene.analysis.cn.*;import org.apache.lucene.analysis.cjk.*;import org.apache.lucene.analysis.tjucn.*;import com.xjt.nlp.word.*;public class TestAnalyzers extends TestCase {

public TestAnalyzers(String name) { super(name); }

public void assertAnalyzesTo(Analyzer a, String input, String[] output) throws Exception { //前面的"dummy"好像沒有用到 TokenStream ts = a.tokenStream("dummy", new StringReader(input)); StringReader readerInput=new StringReader(input); for (int i=0; i Token t = ts.next(); //System.out.println(t); assertNotNull(t); //使用下面這條語句即可以輸出Token的每項的text,並且用空格分開 System.out.print(t.termText); System.out.print(" "); assertEquals(t.termText(), output[i]); } System.out.println(" "); assertNull(ts.next()); ts.close(); }public void outputAnalyzer(Analyzer a ,String input) throws Exception{ TokenStream ts = a.tokenStream("dummy",new StringReader(input)); StringReader readerInput = new StringReader(input); while(true){ Token t = ts.next(); if(t!=null){ System.out.print(t.termText); System.out.print(" "); } else break;

}System.out.println(" ");ts.close();}

public void testSimpleAnalyzer() throws Exception { //學習使用SimpleAnalyzer(); //SimpleAnalyzer將除去letter之外的符號全部過濾掉,並且將所有的字元lowcase化 Analyzer a = new SimpleAnalyzer(); assertAnalyzesTo(a, "foo bar FOO BAR", new String[] { "foo", "bar", "foo", "bar" }); assertAnalyzesTo(a, "foo bar . FOO <> BAR", new String[] { "foo", "bar", "foo", "bar" }); assertAnalyzesTo(a, "foo.bar.FOO.BAR", new String[] { "foo", "bar", "foo", "bar" }); assertAnalyzesTo(a, "U.S.A.", new String[] { "u", "s", "a" }); assertAnalyzesTo(a, "C++", new String[] { "c" }); assertAnalyzesTo(a, "B2B", new String[] { "b", "b" }); assertAnalyzesTo(a, "2B", new String[] { "b" }); assertAnalyzesTo(a, ""QUOTED" word", new String[] { "quoted", "word" }); assertAnalyzesTo(a,"zhang ./ bu <> feng", new String[]{"zhang","bu","feng"}); ICTCLAS splitWord = new ICTCLAS(); String result = splitWord.paragraphProcess("我愛共產黨 i LOVE chanchan"); assertAnalyzesTo(a,result, new String[]{"我","愛","共產黨","i","love","chanchan"});

}

public void testWhiteSpaceAnalyzer() throws Exception { //WhiterspaceAnalyzer僅僅是去除空格，對字元沒有lowcase化 Analyzer a = new WhitespaceAnalyzer(); assertAnalyzesTo(a, "foo bar FOO BAR", new String[] { "foo", "bar", "FOO", "BAR" }); assertAnalyzesTo(a, "foo bar . FOO <> BAR", new String[] { "foo", "bar", ".", "FOO", "<>", "BAR" }); assertAnalyzesTo(a, "foo.bar.FOO.BAR", new String[] { "foo.bar.FOO.BAR" }); assertAnalyzesTo(a, "U.S.A.", new String[] { "U.S.A." }); assertAnalyzesTo(a, "C++", new String[] { "C++" });

assertAnalyzesTo(a, "B2B", new String[] { "B2B" }); assertAnalyzesTo(a, "2B", new String[] { "2B" }); assertAnalyzesTo(a, ""QUOTED" word", new String[] { ""QUOTED"", "word" });

assertAnalyzesTo(a,"zhang bu feng", new String []{"zhang","bu","feng"}); ICTCLAS splitWord = new ICTCLAS(); String result = splitWord.paragraphProcess("我愛共產黨 i love chanchan"); assertAnalyzesTo(a,result, new String[]{"我","愛","共產黨","i","love","chanchan"}); }

public void testStopAnalyzer() throws Exception { //StopAnalyzer的功能超越了SimpleAnalyzer，在SimpleAnalyzer的基礎上 //增加了去除StopWords的功能 Analyzer a = new StopAnalyzer(); assertAnalyzesTo(a, "foo bar FOO BAR", new String[] { "foo", "bar", "foo", "bar" }); assertAnalyzesTo(a, "foo a bar such FOO THESE BAR", new String[] { "foo", "bar", "foo", "bar" }); assertAnalyzesTo(a,"foo ./ a bar such ,./<> FOO THESE BAR ", new String[]{"foo","bar","foo","bar"}); ICTCLAS splitWord = new ICTCLAS(); String result = splitWord.paragraphProcess("我愛共產黨 i Love chanchan such"); assertAnalyzesTo(a,result, new String[]{"我","愛","共產黨","i","love","chanchan"});

} public void testStandardAnalyzer() throws Exception{ //StandardAnalyzer的功能最為強大，對於中文採用的為單字切分 Analyzer a = new StandardAnalyzer(); assertAnalyzesTo(a,"foo bar Foo Bar", new String[]{"foo","bar","foo","bar"}); assertAnalyzesTo(a,"foo bar ./ Foo ./ BAR", new String[]{"foo","bar","foo","bar"}); assertAnalyzesTo(a,"foo ./ a bar such ,./<> FOO THESE BAR ", new String[]{"foo","bar","foo","bar"}); assertAnalyzesTo(a,"張步峰是天大學生", new String[]{"張","步","峰","是","天","大","學","生"}); //驗證去除英文的標點符號 assertAnalyzesTo(a,"張,/步/,峰,.是.,天大<>學生", new String[]{"張","步","峰","是","天","大","學","生"}); //驗證去除中文的標點符號 assertAnalyzesTo(a,"張。、步。、峰是。天大。學生", new String[]{"張","步","峰","是","天","大","學","生"}); } public void testChineseAnalyzer() throws Exception{ //可見ChineseAnalyzer在功能上和standardAnalyzer的功能差不多，但是可能在速度上慢於StandardAnalyzer Analyzer a = new ChineseAnalyzer();

//去空格 assertAnalyzesTo(a,"foo bar Foo Bar", new String[]{"foo","bar","foo","bar"}); assertAnalyzesTo(a,"foo bar ./ Foo ./ BAR", new String[]{"foo","bar","foo","bar"}); assertAnalyzesTo(a,"foo ./ a bar such ,./<> FOO THESE BAR ", new String[]{"foo","bar","foo","bar"}); assertAnalyzesTo(a,"張步峰是天大學生", new String[]{"張","步","峰","是","天","大","學","生"}); //驗證去除英文的標點符號 assertAnalyzesTo(a,"張,/步/,峰,.是.,天大<>學生", new String[]{"張","步","峰","是","天","大","學","生"}); //驗證去除中文的標點符號 assertAnalyzesTo(a,"張。、步。、峰是。天大。學生", new String[]{"張","步","峰","是","天","大","學","生"}); //不支持中英文寫在一起 // assertAnalyzesTo(a,"我愛你 i love chanchan", /// new String[]{"我","愛","你","i","love","chanchan"});

} public void testCJKAnalyzer() throws Exception { //chedong寫的CJKAnalyzer的功能在英文處理上的功能和StandardAnalyzer相同 //但是在漢語的分詞上，不能過濾掉標點符號，即使用二元切分 Analyzer a = new CJKAnalyzer(); assertAnalyzesTo(a,"foo bar Foo Bar", new String[]{"foo","bar","foo","bar"}); assertAnalyzesTo(a,"foo bar ./ Foo ./ BAR", new String[]{"foo","bar","foo","bar"}); assertAnalyzesTo(a,"foo ./ a bar such ,./<> FOO THESE BAR ", new String[]{"foo","bar","foo","bar"});

// assertAnalyzesTo(a,"張,/步/,峰,.是.,天大<>學生", // new String[]{"張步","步峰","峰是","是天","天大","大學","學生"}); //assertAnalyzesTo(a,"張。、步。、峰是。天大。學生", // new String[]{"張步","步峰","峰是","是天","天大","大學","學生"}); //支持中英文同時寫 assertAnalyzesTo(a,"張步峰是天大學生 i love", new String[]{"張步","步峰","峰是","是天","天大","大學","學生","i","love"});

} public void testTjuChineseAnalyzer() throws Exception{ /** * TjuChineseAnlyzer的功能相當強大,在中文分詞方面由於其調用的為ICTCLAS的java介面. * 所以其在中文方面性能上同與ICTCLAS.其在英文分詞上採用了Lucene的StopAnalyzer,可以去除 * stopWords,而且可以不區分大小寫,過濾掉各類標點符號. */ Analyzer a = new TjuChineseAnalyzer(); String input = "體育訊　在被尤文淘汰之後，皇馬主帥博斯克拒絕接受媒體對球隊後防線的批評，同時還為自己排出的首發陣容進行了辯護。"+ "「失利是全隊的責任，而不僅僅是後防線該受指責，」博斯克說，「我並不認為我們踢得一塌糊塗。」「我們進入了半決賽，而且在晉級的道路上一路奮 "+ "戰。即使是今天的比賽我們也有幾個翻身的機會，但我們面對的對手非常強大，他們踢得非常好。」「我們的球迷應該為過去幾個賽季里我們在冠軍杯中的表現感到驕傲。」"+ "博斯克還說。對於博斯克在首發中排出了久疏戰陣的坎比亞索，賽後有記者提出了質疑，認為完全應該將隊內的另一 "+ "名球員帕文派遣上場以加強後衛線。對於這一疑議，博斯克拒絕承擔所謂的「責任」，認為球隊的首發沒有問題。「我們按照整個賽季以來的方式做了，"+ "對於人員上的變化我沒有什麼可說的。」對於球隊在本賽季的前景，博斯克表示皇馬還有西甲聯賽的冠軍作為目標。「皇家馬德里在冠軍 "+ "杯中戰鬥到了最後，我們在聯賽中也將這麼做。」"+ "A Java User Group is a group of people who share a common interest in Java technology and meet on a regular basis to share"+ " technical ideas and information. The actual structure of a JUG can vary greatly - from a small number of friends and coworkers"+ " meeting informally in the evening, to a large group of companies based in the same geographic area. "+ "Regardless of the size and focus of a particular JUG, the sense of community spirit remains the same. ";

outputAnalyzer(a,input); //此處我已經對大文本進行過測試,不會有問題效果很好 outputAnalyzer(a,"我愛共產黨 ,，。 I love China 我喜歡唱歌 "); assertAnalyzesTo(a,"我愛共產黨 ,，。I love China 我喜歡唱歌", new String[]{"愛","共產黨","i","love","china","喜歡","唱歌"}); }}