third_codes/elasticsearch-analysis-ik
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

merge code

Browse Source
This commit is contained in:
medcl 2016-04-10 22:17:59 +08:00
commit ca2bfe5732
6 changed files with 372 additions and 363 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
src/main/java/org/wltea/analyzer/dic/Dictionary.java
View File

@ -79,7 +79,7 @@ public class Dictionary {
* 配置对象
*/
private Configuration configuration;
public static final ESLogger logger=Loggers.getLogger("ik-analyzer");
public static ESLogger logger=Loggers.getLogger("ik-analyzer");
private static ScheduledExecutorService pool = Executors.newScheduledThreadPool(1);
@ -103,7 +103,7 @@ public class Dictionary {
* @return Dictionary
*/
public static synchronized Dictionary initial(Configuration cfg){
if(singleton == null){
synchronized(Dictionary.class){
if(singleton == null){
singleton = new Dictionary();
@ -127,6 +127,7 @@ public class Dictionary {
return singleton;
}
}
}
return singleton;
}
@ -227,7 +228,7 @@ public class Dictionary {
try {
is = new FileInputStream(file.toFile());
} catch (FileNotFoundException e) {
e.printStackTrace();
logger.error(e.getMessage(), e);
}
try {
@ -296,8 +297,10 @@ public class Dictionary {
logger.error("ik-analyzer",e);
}finally{
try {
if(is != null){
is.close();
is = null;
}
} catch (IOException e) {
logger.error("ik-analyzer",e);
}
@ -315,14 +318,11 @@ public class Dictionary {
for(String location:remoteExtDictFiles){
logger.info("[Dict Loading] " + location);
List<String> lists = getRemoteWords(location);
/** Redundant Nullcheck as the list is initialized in the getRemoteWords method
//如果找不到扩展的字典，则忽略
if(lists == null){
logger.error("[Dict Loading] "+location+"加载失败");
continue;
}*/
}
for(String theWord:lists){
if (theWord != null && !"".equals(theWord.trim())) {
//加载扩展词典数据到主内存词典中
@ -393,7 +393,7 @@ public class Dictionary {
try {
is = new FileInputStream(file.toFile());
} catch (FileNotFoundException e) {
e.printStackTrace();
logger.error(e.getMessage(), e);
}
try {
@ -455,8 +455,10 @@ public class Dictionary {
}finally{
try {
if(is != null){
is.close();
is = null;
}
} catch (IOException e) {
logger.error("ik-analyzer",e);
}
@ -469,14 +471,11 @@ public class Dictionary {
for(String location:remoteExtStopWordDictFiles){
logger.info("[Dict Loading] " + location);
List<String> lists = getRemoteWords(location);
/** Redundant Nullcheck as the list is initialized in the getRemoteWords method
//如果找不到扩展的字典，则忽略
if(lists == null){
logger.error("[Dict Loading] "+location+"加载失败");
continue;
}*/
}
for(String theWord:lists){
if (theWord != null && !"".equals(theWord.trim())) {
//加载远程词典数据到主内存中
@ -555,8 +554,10 @@ public class Dictionary {
logger.error("ik-analyzer",e);
}finally{
try {
if(is != null){
is.close();
is = null;
}
} catch (IOException e) {
logger.error("ik-analyzer",e);
}

6
src/main/java/org/wltea/analyzer/dic/Monitor.java
View File

@ -7,9 +7,13 @@ import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpHead;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.Loggers;
public class Monitor implements Runnable {
public static ESLogger logger= Loggers.getLogger("ik-analyzer");
private static CloseableHttpClient httpclient = HttpClients.createDefault();
/*
* 上次更改时间
@ -87,7 +91,7 @@ public class Monitor implements Runnable {
response.close();
}
} catch (IOException e) {
e.printStackTrace();
logger.error(e.getMessage(), e);
}
}
}

12
src/main/java/org/wltea/analyzer/help/Sleep.java
View File

@ -5,7 +5,7 @@ import org.elasticsearch.common.logging.Loggers;
public class Sleep {
public static final ESLogger logger= Loggers.getLogger("ik-analyzer");
public static ESLogger logger= Loggers.getLogger("ik-analyzer");
public enum Type{MSEC,SEC,MIN,HOUR};
public static void sleep(Type type,int num){
@ -15,20 +15,20 @@ public class Sleep {
Thread.sleep(num);
return;
case SEC:
Thread.sleep(num*1000L);
Thread.sleep(num*1000);
return;
case MIN:
Thread.sleep(num*60*1000L);
Thread.sleep(num*60*1000);
return;
case HOUR:
Thread.sleep(num*60*60*1000L);
Thread.sleep(num*60*60*1000);
return;
default:
logger.error("输入类型错误，应为MSEC,SEC,MIN,HOUR之一");
System.err.println("输入类型错误，应为MSEC,SEC,MIN,HOUR之一");
return;
}
} catch (InterruptedException e) {
e.printStackTrace();
logger.error(e.getMessage(), e);
}
}

12
src/main/java/org/wltea/analyzer/query/SWMCQueryBuilder.java
View File

@ -34,6 +34,8 @@ import org.apache.lucene.queryparser.classic.ParseException;
import org.apache.lucene.queryparser.classic.QueryParser;
import org.apache.lucene.search.Query;
import org.apache.lucene.util.Version;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.Loggers;
import org.wltea.analyzer.core.IKSegmenter;
import org.wltea.analyzer.core.Lexeme;
@ -45,6 +47,8 @@ import org.wltea.analyzer.core.Lexeme;
*/
public class SWMCQueryBuilder {
public static ESLogger logger= Loggers.getLogger("ik-analyzer");
/**
* 生成SWMCQuery
* @param fieldName
@ -78,7 +82,7 @@ public class SWMCQueryBuilder {
lexemes.add(l);
}
}catch(IOException e){
e.printStackTrace();
logger.error(e.getMessage(), e);
}
return lexemes;
}
@ -87,7 +91,7 @@ public class SWMCQueryBuilder {
/**
* 根据分词结果生成SWMC搜索
* @param fieldName
// * @param pathOption
// * @param pathOption
* @param quickMode
* @return
*/
@ -135,7 +139,7 @@ public class SWMCQueryBuilder {
Query q = qp.parse(keywordBuffer_Short.toString());
return q;
} catch (ParseException e) {
e.printStackTrace();
logger.error(e.getMessage(), e);
}
}else{
@ -145,7 +149,7 @@ public class SWMCQueryBuilder {
Query q = qp.parse(keywordBuffer.toString());
return q;
} catch (ParseException e) {
e.printStackTrace();
logger.error(e.getMessage(), e);
}
}
}

8
src/main/java/org/wltea/analyzer/sample/IKAnalzyerDemo.java
View File

@ -44,7 +44,7 @@ import org.wltea.analyzer.lucene.IKAnalyzer;
*/
public class IKAnalzyerDemo {
public static final ESLogger logger= Loggers.getLogger("ik-analyzer");
public static ESLogger logger= Loggers.getLogger("ik-analyzer");
public static void main(String[] args){
//构建IK分词器，使用smart分词模式
@ -67,20 +67,20 @@ public class IKAnalzyerDemo {
ts.reset();
//迭代获取分词结果
while (ts.incrementToken()) {
logger.info(offset.startOffset() + " - " + offset.endOffset() + " : " + term.toString() + " | " + type.type());
System.out.println(offset.startOffset() + " - " + offset.endOffset() + " : " + term.toString() + " | " + type.type());
}
//关闭TokenStream（关闭StringReader）
ts.end(); // Perform end-of-stream operations, e.g. set the final offset.
} catch (IOException e) {
e.printStackTrace();
logger.error(e.getMessage(), e);
} finally {
//释放TokenStream的所有资源
if(ts != null){
try {
ts.close();
} catch (IOException e) {
e.printStackTrace();
logger.error(e.getMessage(), e);
}
}
}

20
src/main/java/org/wltea/analyzer/sample/LuceneIndexAndSearchDemo.java
View File

@ -64,7 +64,7 @@ import org.wltea.analyzer.lucene.IKAnalyzer;
*/
public class LuceneIndexAndSearchDemo {
public static final ESLogger logger= Loggers.getLogger("ik-analyzer");
public static ESLogger logger= Loggers.getLogger("ik-analyzer");
/**
* 模拟：
@ -110,39 +110,39 @@ public class LuceneIndexAndSearchDemo {
QueryParser qp = new QueryParser(fieldName, analyzer);
qp.setDefaultOperator(QueryParser.AND_OPERATOR);
Query query = qp.parse(keyword);
logger.info("Query = " + query);
System.out.println("Query = " + query);
//搜索相似度最高的5条记录
TopDocs topDocs = isearcher.search(query , 5);
logger.info("命中：" + topDocs.totalHits);
System.out.println("命中：" + topDocs.totalHits);
//输出结果
ScoreDoc[] scoreDocs = topDocs.scoreDocs;
for (int i = 0; i < topDocs.totalHits; i++){
Document targetDoc = isearcher.doc(scoreDocs[i].doc);
logger.info("内容：" + targetDoc.toString());
System.out.println("内容：" + targetDoc.toString());
}
} catch (CorruptIndexException e) {
e.printStackTrace();
logger.error(e.getMessage(), e);
} catch (LockObtainFailedException e) {
e.printStackTrace();
logger.error(e.getMessage(), e);
} catch (IOException e) {
e.printStackTrace();
logger.error(e.getMessage(), e);
} catch (ParseException e) {
e.printStackTrace();
logger.error(e.getMessage(), e);
} finally{
if(ireader != null){
try {
ireader.close();
} catch (IOException e) {
e.printStackTrace();
logger.error(e.getMessage(), e);
}
}
if(directory != null){
try {
directory.close();
} catch (IOException e) {
e.printStackTrace();
logger.error(e.getMessage(), e);
}
}
}