third_codes/elasticsearch-analysis-ik
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

merge code

Browse Source
This commit is contained in:
medcl 2016-04-10 22:17:59 +08:00
commit ca2bfe5732
6 changed files with 372 additions and 363 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
src/main/java/org/wltea/analyzer/dic/Dictionary.java
View File

@ -79,7 +79,7 @@ public class Dictionary {
* 配置对象 * 配置对象
*/ */
private Configuration configuration; private Configuration configuration;
public static final ESLogger logger=Loggers.getLogger("ik-analyzer"); public static ESLogger logger=Loggers.getLogger("ik-analyzer");
private static ScheduledExecutorService pool = Executors.newScheduledThreadPool(1); private static ScheduledExecutorService pool = Executors.newScheduledThreadPool(1);
@ -103,7 +103,7 @@ public class Dictionary {
* @return Dictionary * @return Dictionary
*/ */
public static synchronized Dictionary initial(Configuration cfg){ public static synchronized Dictionary initial(Configuration cfg){
if(singleton == null){
synchronized(Dictionary.class){ synchronized(Dictionary.class){
if(singleton == null){ if(singleton == null){
singleton = new Dictionary(); singleton = new Dictionary();
@ -127,6 +127,7 @@ public class Dictionary {
return singleton; return singleton;
} }
} }
}
return singleton; return singleton;
} }
@ -227,7 +228,7 @@ public class Dictionary {
try { try {
is = new FileInputStream(file.toFile()); is = new FileInputStream(file.toFile());
} catch (FileNotFoundException e) { } catch (FileNotFoundException e) {
e.printStackTrace(); logger.error(e.getMessage(), e);
} }
try { try {
@ -296,8 +297,10 @@ public class Dictionary {
logger.error("ik-analyzer",e); logger.error("ik-analyzer",e);
}finally{ }finally{
try { try {
if(is != null){
is.close(); is.close();
is = null; is = null;
}
} catch (IOException e) { } catch (IOException e) {
logger.error("ik-analyzer",e); logger.error("ik-analyzer",e);
} }
@ -315,14 +318,11 @@ public class Dictionary {
for(String location:remoteExtDictFiles){ for(String location:remoteExtDictFiles){
logger.info("[Dict Loading] " + location); logger.info("[Dict Loading] " + location);
List<String> lists = getRemoteWords(location); List<String> lists = getRemoteWords(location);
/** Redundant Nullcheck as the list is initialized in the getRemoteWords method
//如果找不到扩展的字典，则忽略 //如果找不到扩展的字典，则忽略
if(lists == null){ if(lists == null){
logger.error("[Dict Loading] "+location+"加载失败"); logger.error("[Dict Loading] "+location+"加载失败");
continue; continue;
}*/ }
for(String theWord:lists){ for(String theWord:lists){
if (theWord != null && !"".equals(theWord.trim())) { if (theWord != null && !"".equals(theWord.trim())) {
//加载扩展词典数据到主内存词典中 //加载扩展词典数据到主内存词典中
@ -393,7 +393,7 @@ public class Dictionary {
try { try {
is = new FileInputStream(file.toFile()); is = new FileInputStream(file.toFile());
} catch (FileNotFoundException e) { } catch (FileNotFoundException e) {
e.printStackTrace(); logger.error(e.getMessage(), e);
} }
try { try {
@ -455,8 +455,10 @@ public class Dictionary {
}finally{ }finally{
try { try {
if(is != null){
is.close(); is.close();
is = null; is = null;
}
} catch (IOException e) { } catch (IOException e) {
logger.error("ik-analyzer",e); logger.error("ik-analyzer",e);
} }
@ -469,14 +471,11 @@ public class Dictionary {
for(String location:remoteExtStopWordDictFiles){ for(String location:remoteExtStopWordDictFiles){
logger.info("[Dict Loading] " + location); logger.info("[Dict Loading] " + location);
List<String> lists = getRemoteWords(location); List<String> lists = getRemoteWords(location);
/** Redundant Nullcheck as the list is initialized in the getRemoteWords method
//如果找不到扩展的字典，则忽略 //如果找不到扩展的字典，则忽略
if(lists == null){ if(lists == null){
logger.error("[Dict Loading] "+location+"加载失败"); logger.error("[Dict Loading] "+location+"加载失败");
continue; continue;
}*/ }
for(String theWord:lists){ for(String theWord:lists){
if (theWord != null && !"".equals(theWord.trim())) { if (theWord != null && !"".equals(theWord.trim())) {
//加载远程词典数据到主内存中 //加载远程词典数据到主内存中
@ -555,8 +554,10 @@ public class Dictionary {
logger.error("ik-analyzer",e); logger.error("ik-analyzer",e);
}finally{ }finally{
try { try {
if(is != null){
is.close(); is.close();
is = null; is = null;
}
} catch (IOException e) { } catch (IOException e) {
logger.error("ik-analyzer",e); logger.error("ik-analyzer",e);
} }

6
src/main/java/org/wltea/analyzer/dic/Monitor.java
View File

@ -7,9 +7,13 @@ import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpHead; import org.apache.http.client.methods.HttpHead;
import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients; import org.apache.http.impl.client.HttpClients;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.Loggers;
public class Monitor implements Runnable { public class Monitor implements Runnable {
public static ESLogger logger= Loggers.getLogger("ik-analyzer");
private static CloseableHttpClient httpclient = HttpClients.createDefault(); private static CloseableHttpClient httpclient = HttpClients.createDefault();
/* /*
* 上次更改时间 * 上次更改时间
@ -87,7 +91,7 @@ public class Monitor implements Runnable {
response.close(); response.close();
} }
} catch (IOException e) { } catch (IOException e) {
e.printStackTrace(); logger.error(e.getMessage(), e);
} }
} }
} }

12
src/main/java/org/wltea/analyzer/help/Sleep.java
View File

@ -5,7 +5,7 @@ import org.elasticsearch.common.logging.Loggers;
public class Sleep { public class Sleep {
public static final ESLogger logger= Loggers.getLogger("ik-analyzer"); public static ESLogger logger= Loggers.getLogger("ik-analyzer");
public enum Type{MSEC,SEC,MIN,HOUR}; public enum Type{MSEC,SEC,MIN,HOUR};
public static void sleep(Type type,int num){ public static void sleep(Type type,int num){
@ -15,20 +15,20 @@ public class Sleep {
Thread.sleep(num); Thread.sleep(num);
return; return;
case SEC: case SEC:
Thread.sleep(num*1000L); Thread.sleep(num*1000);
return; return;
case MIN: case MIN:
Thread.sleep(num*60*1000L); Thread.sleep(num*60*1000);
return; return;
case HOUR: case HOUR:
Thread.sleep(num*60*60*1000L); Thread.sleep(num*60*60*1000);
return; return;
default: default:
logger.error("输入类型错误，应为MSEC,SEC,MIN,HOUR之一"); System.err.println("输入类型错误，应为MSEC,SEC,MIN,HOUR之一");
return; return;
} }
} catch (InterruptedException e) { } catch (InterruptedException e) {
e.printStackTrace(); logger.error(e.getMessage(), e);
} }
} }

10
src/main/java/org/wltea/analyzer/query/SWMCQueryBuilder.java
View File

@ -34,6 +34,8 @@ import org.apache.lucene.queryparser.classic.ParseException;
import org.apache.lucene.queryparser.classic.QueryParser; import org.apache.lucene.queryparser.classic.QueryParser;
import org.apache.lucene.search.Query; import org.apache.lucene.search.Query;
import org.apache.lucene.util.Version; import org.apache.lucene.util.Version;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.Loggers;
import org.wltea.analyzer.core.IKSegmenter; import org.wltea.analyzer.core.IKSegmenter;
import org.wltea.analyzer.core.Lexeme; import org.wltea.analyzer.core.Lexeme;
@ -45,6 +47,8 @@ import org.wltea.analyzer.core.Lexeme;
*/ */
public class SWMCQueryBuilder { public class SWMCQueryBuilder {
public static ESLogger logger= Loggers.getLogger("ik-analyzer");
/** /**
* 生成SWMCQuery * 生成SWMCQuery
* @param fieldName * @param fieldName
@ -78,7 +82,7 @@ public class SWMCQueryBuilder {
lexemes.add(l); lexemes.add(l);
} }
}catch(IOException e){ }catch(IOException e){
e.printStackTrace(); logger.error(e.getMessage(), e);
} }
return lexemes; return lexemes;
} }
@ -135,7 +139,7 @@ public class SWMCQueryBuilder {
Query q = qp.parse(keywordBuffer_Short.toString()); Query q = qp.parse(keywordBuffer_Short.toString());
return q; return q;
} catch (ParseException e) { } catch (ParseException e) {
e.printStackTrace(); logger.error(e.getMessage(), e);
} }
}else{ }else{
@ -145,7 +149,7 @@ public class SWMCQueryBuilder {
Query q = qp.parse(keywordBuffer.toString()); Query q = qp.parse(keywordBuffer.toString());
return q; return q;
} catch (ParseException e) { } catch (ParseException e) {
e.printStackTrace(); logger.error(e.getMessage(), e);
} }
} }
} }

8
src/main/java/org/wltea/analyzer/sample/IKAnalzyerDemo.java
View File

@ -44,7 +44,7 @@ import org.wltea.analyzer.lucene.IKAnalyzer;
*/ */
public class IKAnalzyerDemo { public class IKAnalzyerDemo {
public static final ESLogger logger= Loggers.getLogger("ik-analyzer"); public static ESLogger logger= Loggers.getLogger("ik-analyzer");
public static void main(String[] args){ public static void main(String[] args){
//构建IK分词器，使用smart分词模式 //构建IK分词器，使用smart分词模式
@ -67,20 +67,20 @@ public class IKAnalzyerDemo {
ts.reset(); ts.reset();
//迭代获取分词结果 //迭代获取分词结果
while (ts.incrementToken()) { while (ts.incrementToken()) {
logger.info(offset.startOffset() + " - " + offset.endOffset() + " : " + term.toString() + " | " + type.type()); System.out.println(offset.startOffset() + " - " + offset.endOffset() + " : " + term.toString() + " | " + type.type());
} }
//关闭TokenStream（关闭StringReader） //关闭TokenStream（关闭StringReader）
ts.end(); // Perform end-of-stream operations, e.g. set the final offset. ts.end(); // Perform end-of-stream operations, e.g. set the final offset.
} catch (IOException e) { } catch (IOException e) {
e.printStackTrace(); logger.error(e.getMessage(), e);
} finally { } finally {
//释放TokenStream的所有资源 //释放TokenStream的所有资源
if(ts != null){ if(ts != null){
try { try {
ts.close(); ts.close();
} catch (IOException e) { } catch (IOException e) {
e.printStackTrace(); logger.error(e.getMessage(), e);
} }
} }
} }

20
src/main/java/org/wltea/analyzer/sample/LuceneIndexAndSearchDemo.java
View File

@ -64,7 +64,7 @@ import org.wltea.analyzer.lucene.IKAnalyzer;
*/ */
public class LuceneIndexAndSearchDemo { public class LuceneIndexAndSearchDemo {
public static final ESLogger logger= Loggers.getLogger("ik-analyzer"); public static ESLogger logger= Loggers.getLogger("ik-analyzer");
/** /**
* 模拟： * 模拟：
@ -110,39 +110,39 @@ public class LuceneIndexAndSearchDemo {
QueryParser qp = new QueryParser(fieldName, analyzer); QueryParser qp = new QueryParser(fieldName, analyzer);
qp.setDefaultOperator(QueryParser.AND_OPERATOR); qp.setDefaultOperator(QueryParser.AND_OPERATOR);
Query query = qp.parse(keyword); Query query = qp.parse(keyword);
logger.info("Query = " + query); System.out.println("Query = " + query);
//搜索相似度最高的5条记录 //搜索相似度最高的5条记录
TopDocs topDocs = isearcher.search(query , 5); TopDocs topDocs = isearcher.search(query , 5);
logger.info("命中：" + topDocs.totalHits); System.out.println("命中：" + topDocs.totalHits);
//输出结果 //输出结果
ScoreDoc[] scoreDocs = topDocs.scoreDocs; ScoreDoc[] scoreDocs = topDocs.scoreDocs;
for (int i = 0; i < topDocs.totalHits; i++){ for (int i = 0; i < topDocs.totalHits; i++){
Document targetDoc = isearcher.doc(scoreDocs[i].doc); Document targetDoc = isearcher.doc(scoreDocs[i].doc);
logger.info("内容：" + targetDoc.toString()); System.out.println("内容：" + targetDoc.toString());
} }
} catch (CorruptIndexException e) { } catch (CorruptIndexException e) {
e.printStackTrace(); logger.error(e.getMessage(), e);
} catch (LockObtainFailedException e) { } catch (LockObtainFailedException e) {
e.printStackTrace(); logger.error(e.getMessage(), e);
} catch (IOException e) { } catch (IOException e) {
e.printStackTrace(); logger.error(e.getMessage(), e);
} catch (ParseException e) { } catch (ParseException e) {
e.printStackTrace(); logger.error(e.getMessage(), e);
} finally{ } finally{
if(ireader != null){ if(ireader != null){
try { try {
ireader.close(); ireader.close();
} catch (IOException e) { } catch (IOException e) {
e.printStackTrace(); logger.error(e.getMessage(), e);
} }
} }
if(directory != null){ if(directory != null){
try { try {
directory.close(); directory.close();
} catch (IOException e) { } catch (IOException e) {
e.printStackTrace(); logger.error(e.getMessage(), e);
} }
} }
} }