merge code
This commit is contained in:
commit
ca2bfe5732
@ -79,7 +79,7 @@ public class Dictionary {
|
|||||||
* 配置对象
|
* 配置对象
|
||||||
*/
|
*/
|
||||||
private Configuration configuration;
|
private Configuration configuration;
|
||||||
public static final ESLogger logger=Loggers.getLogger("ik-analyzer");
|
public static ESLogger logger=Loggers.getLogger("ik-analyzer");
|
||||||
|
|
||||||
private static ScheduledExecutorService pool = Executors.newScheduledThreadPool(1);
|
private static ScheduledExecutorService pool = Executors.newScheduledThreadPool(1);
|
||||||
|
|
||||||
@ -103,7 +103,7 @@ public class Dictionary {
|
|||||||
* @return Dictionary
|
* @return Dictionary
|
||||||
*/
|
*/
|
||||||
public static synchronized Dictionary initial(Configuration cfg){
|
public static synchronized Dictionary initial(Configuration cfg){
|
||||||
|
if(singleton == null){
|
||||||
synchronized(Dictionary.class){
|
synchronized(Dictionary.class){
|
||||||
if(singleton == null){
|
if(singleton == null){
|
||||||
singleton = new Dictionary();
|
singleton = new Dictionary();
|
||||||
@ -127,6 +127,7 @@ public class Dictionary {
|
|||||||
return singleton;
|
return singleton;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
return singleton;
|
return singleton;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -227,7 +228,7 @@ public class Dictionary {
|
|||||||
try {
|
try {
|
||||||
is = new FileInputStream(file.toFile());
|
is = new FileInputStream(file.toFile());
|
||||||
} catch (FileNotFoundException e) {
|
} catch (FileNotFoundException e) {
|
||||||
e.printStackTrace();
|
logger.error(e.getMessage(), e);
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
@ -296,8 +297,10 @@ public class Dictionary {
|
|||||||
logger.error("ik-analyzer",e);
|
logger.error("ik-analyzer",e);
|
||||||
}finally{
|
}finally{
|
||||||
try {
|
try {
|
||||||
|
if(is != null){
|
||||||
is.close();
|
is.close();
|
||||||
is = null;
|
is = null;
|
||||||
|
}
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
logger.error("ik-analyzer",e);
|
logger.error("ik-analyzer",e);
|
||||||
}
|
}
|
||||||
@ -315,14 +318,11 @@ public class Dictionary {
|
|||||||
for(String location:remoteExtDictFiles){
|
for(String location:remoteExtDictFiles){
|
||||||
logger.info("[Dict Loading] " + location);
|
logger.info("[Dict Loading] " + location);
|
||||||
List<String> lists = getRemoteWords(location);
|
List<String> lists = getRemoteWords(location);
|
||||||
|
|
||||||
/** Redundant Nullcheck as the list is initialized in the getRemoteWords method
|
|
||||||
//如果找不到扩展的字典,则忽略
|
//如果找不到扩展的字典,则忽略
|
||||||
if(lists == null){
|
if(lists == null){
|
||||||
logger.error("[Dict Loading] "+location+"加载失败");
|
logger.error("[Dict Loading] "+location+"加载失败");
|
||||||
continue;
|
continue;
|
||||||
}*/
|
}
|
||||||
|
|
||||||
for(String theWord:lists){
|
for(String theWord:lists){
|
||||||
if (theWord != null && !"".equals(theWord.trim())) {
|
if (theWord != null && !"".equals(theWord.trim())) {
|
||||||
//加载扩展词典数据到主内存词典中
|
//加载扩展词典数据到主内存词典中
|
||||||
@ -393,7 +393,7 @@ public class Dictionary {
|
|||||||
try {
|
try {
|
||||||
is = new FileInputStream(file.toFile());
|
is = new FileInputStream(file.toFile());
|
||||||
} catch (FileNotFoundException e) {
|
} catch (FileNotFoundException e) {
|
||||||
e.printStackTrace();
|
logger.error(e.getMessage(), e);
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
@ -455,8 +455,10 @@ public class Dictionary {
|
|||||||
|
|
||||||
}finally{
|
}finally{
|
||||||
try {
|
try {
|
||||||
|
if(is != null){
|
||||||
is.close();
|
is.close();
|
||||||
is = null;
|
is = null;
|
||||||
|
}
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
logger.error("ik-analyzer",e);
|
logger.error("ik-analyzer",e);
|
||||||
}
|
}
|
||||||
@ -469,14 +471,11 @@ public class Dictionary {
|
|||||||
for(String location:remoteExtStopWordDictFiles){
|
for(String location:remoteExtStopWordDictFiles){
|
||||||
logger.info("[Dict Loading] " + location);
|
logger.info("[Dict Loading] " + location);
|
||||||
List<String> lists = getRemoteWords(location);
|
List<String> lists = getRemoteWords(location);
|
||||||
|
|
||||||
/** Redundant Nullcheck as the list is initialized in the getRemoteWords method
|
|
||||||
//如果找不到扩展的字典,则忽略
|
//如果找不到扩展的字典,则忽略
|
||||||
if(lists == null){
|
if(lists == null){
|
||||||
logger.error("[Dict Loading] "+location+"加载失败");
|
logger.error("[Dict Loading] "+location+"加载失败");
|
||||||
continue;
|
continue;
|
||||||
}*/
|
}
|
||||||
|
|
||||||
for(String theWord:lists){
|
for(String theWord:lists){
|
||||||
if (theWord != null && !"".equals(theWord.trim())) {
|
if (theWord != null && !"".equals(theWord.trim())) {
|
||||||
//加载远程词典数据到主内存中
|
//加载远程词典数据到主内存中
|
||||||
@ -555,8 +554,10 @@ public class Dictionary {
|
|||||||
logger.error("ik-analyzer",e);
|
logger.error("ik-analyzer",e);
|
||||||
}finally{
|
}finally{
|
||||||
try {
|
try {
|
||||||
|
if(is != null){
|
||||||
is.close();
|
is.close();
|
||||||
is = null;
|
is = null;
|
||||||
|
}
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
logger.error("ik-analyzer",e);
|
logger.error("ik-analyzer",e);
|
||||||
}
|
}
|
||||||
|
@ -7,9 +7,13 @@ import org.apache.http.client.methods.CloseableHttpResponse;
|
|||||||
import org.apache.http.client.methods.HttpHead;
|
import org.apache.http.client.methods.HttpHead;
|
||||||
import org.apache.http.impl.client.CloseableHttpClient;
|
import org.apache.http.impl.client.CloseableHttpClient;
|
||||||
import org.apache.http.impl.client.HttpClients;
|
import org.apache.http.impl.client.HttpClients;
|
||||||
|
import org.elasticsearch.common.logging.ESLogger;
|
||||||
|
import org.elasticsearch.common.logging.Loggers;
|
||||||
|
|
||||||
public class Monitor implements Runnable {
|
public class Monitor implements Runnable {
|
||||||
|
|
||||||
|
public static ESLogger logger= Loggers.getLogger("ik-analyzer");
|
||||||
|
|
||||||
private static CloseableHttpClient httpclient = HttpClients.createDefault();
|
private static CloseableHttpClient httpclient = HttpClients.createDefault();
|
||||||
/*
|
/*
|
||||||
* 上次更改时间
|
* 上次更改时间
|
||||||
@ -87,7 +91,7 @@ public class Monitor implements Runnable {
|
|||||||
response.close();
|
response.close();
|
||||||
}
|
}
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
e.printStackTrace();
|
logger.error(e.getMessage(), e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -5,7 +5,7 @@ import org.elasticsearch.common.logging.Loggers;
|
|||||||
|
|
||||||
public class Sleep {
|
public class Sleep {
|
||||||
|
|
||||||
public static final ESLogger logger= Loggers.getLogger("ik-analyzer");
|
public static ESLogger logger= Loggers.getLogger("ik-analyzer");
|
||||||
|
|
||||||
public enum Type{MSEC,SEC,MIN,HOUR};
|
public enum Type{MSEC,SEC,MIN,HOUR};
|
||||||
public static void sleep(Type type,int num){
|
public static void sleep(Type type,int num){
|
||||||
@ -15,20 +15,20 @@ public class Sleep {
|
|||||||
Thread.sleep(num);
|
Thread.sleep(num);
|
||||||
return;
|
return;
|
||||||
case SEC:
|
case SEC:
|
||||||
Thread.sleep(num*1000L);
|
Thread.sleep(num*1000);
|
||||||
return;
|
return;
|
||||||
case MIN:
|
case MIN:
|
||||||
Thread.sleep(num*60*1000L);
|
Thread.sleep(num*60*1000);
|
||||||
return;
|
return;
|
||||||
case HOUR:
|
case HOUR:
|
||||||
Thread.sleep(num*60*60*1000L);
|
Thread.sleep(num*60*60*1000);
|
||||||
return;
|
return;
|
||||||
default:
|
default:
|
||||||
logger.error("输入类型错误,应为MSEC,SEC,MIN,HOUR之一");
|
System.err.println("输入类型错误,应为MSEC,SEC,MIN,HOUR之一");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
} catch (InterruptedException e) {
|
} catch (InterruptedException e) {
|
||||||
e.printStackTrace();
|
logger.error(e.getMessage(), e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -34,6 +34,8 @@ import org.apache.lucene.queryparser.classic.ParseException;
|
|||||||
import org.apache.lucene.queryparser.classic.QueryParser;
|
import org.apache.lucene.queryparser.classic.QueryParser;
|
||||||
import org.apache.lucene.search.Query;
|
import org.apache.lucene.search.Query;
|
||||||
import org.apache.lucene.util.Version;
|
import org.apache.lucene.util.Version;
|
||||||
|
import org.elasticsearch.common.logging.ESLogger;
|
||||||
|
import org.elasticsearch.common.logging.Loggers;
|
||||||
import org.wltea.analyzer.core.IKSegmenter;
|
import org.wltea.analyzer.core.IKSegmenter;
|
||||||
import org.wltea.analyzer.core.Lexeme;
|
import org.wltea.analyzer.core.Lexeme;
|
||||||
|
|
||||||
@ -45,6 +47,8 @@ import org.wltea.analyzer.core.Lexeme;
|
|||||||
*/
|
*/
|
||||||
public class SWMCQueryBuilder {
|
public class SWMCQueryBuilder {
|
||||||
|
|
||||||
|
public static ESLogger logger= Loggers.getLogger("ik-analyzer");
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 生成SWMCQuery
|
* 生成SWMCQuery
|
||||||
* @param fieldName
|
* @param fieldName
|
||||||
@ -78,7 +82,7 @@ public class SWMCQueryBuilder {
|
|||||||
lexemes.add(l);
|
lexemes.add(l);
|
||||||
}
|
}
|
||||||
}catch(IOException e){
|
}catch(IOException e){
|
||||||
e.printStackTrace();
|
logger.error(e.getMessage(), e);
|
||||||
}
|
}
|
||||||
return lexemes;
|
return lexemes;
|
||||||
}
|
}
|
||||||
@ -87,7 +91,7 @@ public class SWMCQueryBuilder {
|
|||||||
/**
|
/**
|
||||||
* 根据分词结果生成SWMC搜索
|
* 根据分词结果生成SWMC搜索
|
||||||
* @param fieldName
|
* @param fieldName
|
||||||
// * @param pathOption
|
// * @param pathOption
|
||||||
* @param quickMode
|
* @param quickMode
|
||||||
* @return
|
* @return
|
||||||
*/
|
*/
|
||||||
@ -135,7 +139,7 @@ public class SWMCQueryBuilder {
|
|||||||
Query q = qp.parse(keywordBuffer_Short.toString());
|
Query q = qp.parse(keywordBuffer_Short.toString());
|
||||||
return q;
|
return q;
|
||||||
} catch (ParseException e) {
|
} catch (ParseException e) {
|
||||||
e.printStackTrace();
|
logger.error(e.getMessage(), e);
|
||||||
}
|
}
|
||||||
|
|
||||||
}else{
|
}else{
|
||||||
@ -145,7 +149,7 @@ public class SWMCQueryBuilder {
|
|||||||
Query q = qp.parse(keywordBuffer.toString());
|
Query q = qp.parse(keywordBuffer.toString());
|
||||||
return q;
|
return q;
|
||||||
} catch (ParseException e) {
|
} catch (ParseException e) {
|
||||||
e.printStackTrace();
|
logger.error(e.getMessage(), e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -44,7 +44,7 @@ import org.wltea.analyzer.lucene.IKAnalyzer;
|
|||||||
*/
|
*/
|
||||||
public class IKAnalzyerDemo {
|
public class IKAnalzyerDemo {
|
||||||
|
|
||||||
public static final ESLogger logger= Loggers.getLogger("ik-analyzer");
|
public static ESLogger logger= Loggers.getLogger("ik-analyzer");
|
||||||
|
|
||||||
public static void main(String[] args){
|
public static void main(String[] args){
|
||||||
//构建IK分词器,使用smart分词模式
|
//构建IK分词器,使用smart分词模式
|
||||||
@ -67,20 +67,20 @@ public class IKAnalzyerDemo {
|
|||||||
ts.reset();
|
ts.reset();
|
||||||
//迭代获取分词结果
|
//迭代获取分词结果
|
||||||
while (ts.incrementToken()) {
|
while (ts.incrementToken()) {
|
||||||
logger.info(offset.startOffset() + " - " + offset.endOffset() + " : " + term.toString() + " | " + type.type());
|
System.out.println(offset.startOffset() + " - " + offset.endOffset() + " : " + term.toString() + " | " + type.type());
|
||||||
}
|
}
|
||||||
//关闭TokenStream(关闭StringReader)
|
//关闭TokenStream(关闭StringReader)
|
||||||
ts.end(); // Perform end-of-stream operations, e.g. set the final offset.
|
ts.end(); // Perform end-of-stream operations, e.g. set the final offset.
|
||||||
|
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
e.printStackTrace();
|
logger.error(e.getMessage(), e);
|
||||||
} finally {
|
} finally {
|
||||||
//释放TokenStream的所有资源
|
//释放TokenStream的所有资源
|
||||||
if(ts != null){
|
if(ts != null){
|
||||||
try {
|
try {
|
||||||
ts.close();
|
ts.close();
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
e.printStackTrace();
|
logger.error(e.getMessage(), e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -64,7 +64,7 @@ import org.wltea.analyzer.lucene.IKAnalyzer;
|
|||||||
*/
|
*/
|
||||||
public class LuceneIndexAndSearchDemo {
|
public class LuceneIndexAndSearchDemo {
|
||||||
|
|
||||||
public static final ESLogger logger= Loggers.getLogger("ik-analyzer");
|
public static ESLogger logger= Loggers.getLogger("ik-analyzer");
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 模拟:
|
* 模拟:
|
||||||
@ -110,39 +110,39 @@ public class LuceneIndexAndSearchDemo {
|
|||||||
QueryParser qp = new QueryParser(fieldName, analyzer);
|
QueryParser qp = new QueryParser(fieldName, analyzer);
|
||||||
qp.setDefaultOperator(QueryParser.AND_OPERATOR);
|
qp.setDefaultOperator(QueryParser.AND_OPERATOR);
|
||||||
Query query = qp.parse(keyword);
|
Query query = qp.parse(keyword);
|
||||||
logger.info("Query = " + query);
|
System.out.println("Query = " + query);
|
||||||
|
|
||||||
//搜索相似度最高的5条记录
|
//搜索相似度最高的5条记录
|
||||||
TopDocs topDocs = isearcher.search(query , 5);
|
TopDocs topDocs = isearcher.search(query , 5);
|
||||||
logger.info("命中:" + topDocs.totalHits);
|
System.out.println("命中:" + topDocs.totalHits);
|
||||||
//输出结果
|
//输出结果
|
||||||
ScoreDoc[] scoreDocs = topDocs.scoreDocs;
|
ScoreDoc[] scoreDocs = topDocs.scoreDocs;
|
||||||
for (int i = 0; i < topDocs.totalHits; i++){
|
for (int i = 0; i < topDocs.totalHits; i++){
|
||||||
Document targetDoc = isearcher.doc(scoreDocs[i].doc);
|
Document targetDoc = isearcher.doc(scoreDocs[i].doc);
|
||||||
logger.info("内容:" + targetDoc.toString());
|
System.out.println("内容:" + targetDoc.toString());
|
||||||
}
|
}
|
||||||
|
|
||||||
} catch (CorruptIndexException e) {
|
} catch (CorruptIndexException e) {
|
||||||
e.printStackTrace();
|
logger.error(e.getMessage(), e);
|
||||||
} catch (LockObtainFailedException e) {
|
} catch (LockObtainFailedException e) {
|
||||||
e.printStackTrace();
|
logger.error(e.getMessage(), e);
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
e.printStackTrace();
|
logger.error(e.getMessage(), e);
|
||||||
} catch (ParseException e) {
|
} catch (ParseException e) {
|
||||||
e.printStackTrace();
|
logger.error(e.getMessage(), e);
|
||||||
} finally{
|
} finally{
|
||||||
if(ireader != null){
|
if(ireader != null){
|
||||||
try {
|
try {
|
||||||
ireader.close();
|
ireader.close();
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
e.printStackTrace();
|
logger.error(e.getMessage(), e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if(directory != null){
|
if(directory != null){
|
||||||
try {
|
try {
|
||||||
directory.close();
|
directory.close();
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
e.printStackTrace();
|
logger.error(e.getMessage(), e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
Loading…
x
Reference in New Issue
Block a user