Use logger to log exceptions instead of printStackTrace(...)
This commit is contained in:
Ayman Abdel Ghany
parent
3f0214a8e3
commit
5ec0bd5bd6
@ -1,7 +1,7 @@
|
||||
/**
|
||||
* IK 中文分词 版本 5.0
|
||||
* IK Analyzer release 5.0
|
||||
*
|
||||
*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
@ -20,8 +20,8 @@
|
||||
* 源代码由林良益(linliangyi2005@gmail.com)提供
|
||||
* 版权声明 2012，乌龙茶工作室
|
||||
* provided by Linliangyi and copyright 2012 by Oolong studio
|
||||
*
|
||||
*
|
||||
*
|
||||
*
|
||||
*/
|
||||
package org.wltea.analyzer.dic;
|
||||
|
||||
@ -62,37 +62,37 @@ public class Dictionary {
|
||||
*/
|
||||
private static Dictionary singleton;
|
||||
|
||||
private DictSegment _MainDict;
|
||||
private DictSegment _MainDict;
|
||||
|
||||
private DictSegment _SurnameDict;
|
||||
private DictSegment _SurnameDict;
|
||||
|
||||
private DictSegment _QuantifierDict;
|
||||
private DictSegment _QuantifierDict;
|
||||
|
||||
private DictSegment _SuffixDict;
|
||||
private DictSegment _SuffixDict;
|
||||
|
||||
private DictSegment _PrepDict;
|
||||
private DictSegment _PrepDict;
|
||||
|
||||
private DictSegment _StopWords;
|
||||
|
||||
private DictSegment _StopWords;
|
||||
|
||||
|
||||
/**
|
||||
* 配置对象
|
||||
*/
|
||||
private Configuration configuration;
|
||||
public static ESLogger logger=Loggers.getLogger("ik-analyzer");
|
||||
|
||||
private static ScheduledExecutorService pool = Executors.newScheduledThreadPool(1);
|
||||
|
||||
public static final String PATH_DIC_MAIN = "ik/main.dic";
|
||||
public static final String PATH_DIC_SURNAME = "ik/surname.dic";
|
||||
public static final String PATH_DIC_QUANTIFIER = "ik/quantifier.dic";
|
||||
public static final String PATH_DIC_SUFFIX = "ik/suffix.dic";
|
||||
public static final String PATH_DIC_PREP = "ik/preposition.dic";
|
||||
public static final String PATH_DIC_STOP = "ik/stopword.dic";
|
||||
|
||||
private Dictionary(){
|
||||
public static ESLogger logger=Loggers.getLogger("ik-analyzer");
|
||||
|
||||
}
|
||||
private static ScheduledExecutorService pool = Executors.newScheduledThreadPool(1);
|
||||
|
||||
public static final String PATH_DIC_MAIN = "ik/main.dic";
|
||||
public static final String PATH_DIC_SURNAME = "ik/surname.dic";
|
||||
public static final String PATH_DIC_QUANTIFIER = "ik/quantifier.dic";
|
||||
public static final String PATH_DIC_SUFFIX = "ik/suffix.dic";
|
||||
public static final String PATH_DIC_PREP = "ik/preposition.dic";
|
||||
public static final String PATH_DIC_STOP = "ik/stopword.dic";
|
||||
|
||||
private Dictionary(){
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* 词典初始化
|
||||
@ -107,30 +107,30 @@ public class Dictionary {
|
||||
synchronized(Dictionary.class){
|
||||
if(singleton == null){
|
||||
singleton = new Dictionary();
|
||||
singleton.configuration=cfg;
|
||||
singleton.loadMainDict();
|
||||
singleton.loadSurnameDict();
|
||||
singleton.loadQuantifierDict();
|
||||
singleton.loadSuffixDict();
|
||||
singleton.loadPrepDict();
|
||||
singleton.loadStopWordDict();
|
||||
|
||||
//建立监控线程
|
||||
for(String location:cfg.getRemoteExtDictionarys()){
|
||||
//10 秒是初始延迟可以修改的 60是间隔时间 单位秒
|
||||
pool.scheduleAtFixedRate(new Monitor(location), 10, 60, TimeUnit.SECONDS);
|
||||
}
|
||||
for(String location:cfg.getRemoteExtStopWordDictionarys()){
|
||||
pool.scheduleAtFixedRate(new Monitor(location), 10, 60, TimeUnit.SECONDS);
|
||||
}
|
||||
|
||||
return singleton;
|
||||
singleton.configuration=cfg;
|
||||
singleton.loadMainDict();
|
||||
singleton.loadSurnameDict();
|
||||
singleton.loadQuantifierDict();
|
||||
singleton.loadSuffixDict();
|
||||
singleton.loadPrepDict();
|
||||
singleton.loadStopWordDict();
|
||||
|
||||
//建立监控线程
|
||||
for(String location:cfg.getRemoteExtDictionarys()){
|
||||
//10 秒是初始延迟可以修改的 60是间隔时间 单位秒
|
||||
pool.scheduleAtFixedRate(new Monitor(location), 10, 60, TimeUnit.SECONDS);
|
||||
}
|
||||
for(String location:cfg.getRemoteExtStopWordDictionarys()){
|
||||
pool.scheduleAtFixedRate(new Monitor(location), 10, 60, TimeUnit.SECONDS);
|
||||
}
|
||||
|
||||
return singleton;
|
||||
}
|
||||
}
|
||||
}
|
||||
return singleton;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 获取词典单子实例
|
||||
* @return Dictionary 单例对象
|
||||
@ -141,7 +141,7 @@ public class Dictionary {
|
||||
}
|
||||
return singleton;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 批量加载新词条
|
||||
* @param words Collection<String>词条列表
|
||||
@ -156,7 +156,7 @@ public class Dictionary {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 批量移除（屏蔽）词条
|
||||
*/
|
||||
@ -170,7 +170,7 @@ public class Dictionary {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 检索匹配主词典
|
||||
* @return Hit 匹配结果描述
|
||||
@ -178,15 +178,15 @@ public class Dictionary {
|
||||
public Hit matchInMainDict(char[] charArray){
|
||||
return singleton._MainDict.match(charArray);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 检索匹配主词典
|
||||
* @return Hit 匹配结果描述
|
||||
*/
|
||||
public Hit matchInMainDict(char[] charArray , int begin, int length){
|
||||
return singleton._MainDict.match(charArray, begin, length);
|
||||
return singleton._MainDict.match(charArray, begin, length);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 检索匹配量词词典
|
||||
* @return Hit 匹配结果描述
|
||||
@ -194,8 +194,8 @@ public class Dictionary {
|
||||
public Hit matchInQuantifierDict(char[] charArray , int begin, int length){
|
||||
return singleton._QuantifierDict.match(charArray, begin, length);
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* 从已匹配的Hit中直接取出DictSegment，继续向下匹配
|
||||
* @return Hit
|
||||
@ -204,16 +204,16 @@ public class Dictionary {
|
||||
DictSegment ds = matchedHit.getMatchedDictSegment();
|
||||
return ds.match(charArray, currentIndex, 1 , matchedHit);
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* 判断是否是停止词
|
||||
* @return boolean
|
||||
*/
|
||||
public boolean isStopWord(char[] charArray , int begin, int length){
|
||||
public boolean isStopWord(char[] charArray , int begin, int length){
|
||||
return singleton._StopWords.match(charArray, begin, length).isMatch();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* 加载主词典及扩展词典
|
||||
*/
|
||||
@ -224,13 +224,13 @@ public class Dictionary {
|
||||
//读取主词典文件
|
||||
Path file = PathUtils.get(configuration.getDictRoot(), Dictionary.PATH_DIC_MAIN);
|
||||
|
||||
InputStream is = null;
|
||||
try {
|
||||
is = new FileInputStream(file.toFile());
|
||||
} catch (FileNotFoundException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
InputStream is = null;
|
||||
try {
|
||||
is = new FileInputStream(file.toFile());
|
||||
} catch (FileNotFoundException e) {
|
||||
logger.error(e.getMessage(), e);
|
||||
}
|
||||
|
||||
try {
|
||||
BufferedReader br = new BufferedReader(new InputStreamReader(is , "UTF-8"), 512);
|
||||
String theWord = null;
|
||||
@ -240,26 +240,26 @@ public class Dictionary {
|
||||
_MainDict.fillSegment(theWord.trim().toCharArray());
|
||||
}
|
||||
} while (theWord != null);
|
||||
|
||||
} catch (IOException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
|
||||
}finally{
|
||||
} catch (IOException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
|
||||
}finally{
|
||||
try {
|
||||
if(is != null){
|
||||
is.close();
|
||||
is = null;
|
||||
is.close();
|
||||
is = null;
|
||||
}
|
||||
} catch (IOException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
logger.error("ik-analyzer",e);
|
||||
}
|
||||
}
|
||||
//加载扩展词典
|
||||
this.loadExtDict();
|
||||
//加载远程自定义词库
|
||||
this.loadRemoteExtDict();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* 加载用户配置的扩展词典到主词库表
|
||||
*/
|
||||
@ -270,13 +270,13 @@ public class Dictionary {
|
||||
InputStream is = null;
|
||||
for(String extDictName : extDictFiles){
|
||||
//读取扩展词典文件
|
||||
logger.info("[Dict Loading] " + extDictName);
|
||||
logger.info("[Dict Loading] " + extDictName);
|
||||
Path file = PathUtils.get(configuration.getDictRoot(), extDictName);
|
||||
try {
|
||||
is = new FileInputStream(file.toFile());
|
||||
} catch (FileNotFoundException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
}
|
||||
try {
|
||||
is = new FileInputStream(file.toFile());
|
||||
} catch (FileNotFoundException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
}
|
||||
|
||||
//如果找不到扩展的字典，则忽略
|
||||
if(is == null){
|
||||
@ -287,29 +287,29 @@ public class Dictionary {
|
||||
String theWord = null;
|
||||
do {
|
||||
theWord = br.readLine();
|
||||
if (theWord != null && !"".equals(theWord.trim())) {
|
||||
if (theWord != null && !"".equals(theWord.trim())) {
|
||||
//加载扩展词典数据到主内存词典中
|
||||
_MainDict.fillSegment(theWord.trim().toCharArray());
|
||||
}
|
||||
} while (theWord != null);
|
||||
|
||||
|
||||
} catch (IOException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
}finally{
|
||||
logger.error("ik-analyzer",e);
|
||||
}finally{
|
||||
try {
|
||||
if(is != null){
|
||||
is.close();
|
||||
is = null;
|
||||
is.close();
|
||||
is = null;
|
||||
}
|
||||
} catch (IOException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
}
|
||||
logger.error("ik-analyzer",e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* 加载远程扩展词典到主词库表
|
||||
*/
|
||||
@ -331,14 +331,14 @@ public class Dictionary {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 从远程服务器上下载自定义词条
|
||||
*/
|
||||
private static List<String> getRemoteWords(String location){
|
||||
|
||||
|
||||
List<String> buffer = new ArrayList<String>();
|
||||
RequestConfig rc = RequestConfig.custom().setConnectionRequestTimeout(10*1000)
|
||||
.setConnectTimeout(10*1000).setSocketTimeout(60*1000).build();
|
||||
@ -350,7 +350,7 @@ public class Dictionary {
|
||||
try {
|
||||
response = httpclient.execute(get);
|
||||
if(response.getStatusLine().getStatusCode()==200){
|
||||
|
||||
|
||||
String charset = "UTF-8";
|
||||
//获取编码，默认为utf-8
|
||||
if(response.getEntity().getContentType().getValue().contains("charset=")){
|
||||
@ -376,49 +376,49 @@ public class Dictionary {
|
||||
}
|
||||
return buffer;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* 加载用户扩展的停止词词典
|
||||
*/
|
||||
private void loadStopWordDict(){
|
||||
//建立主词典实例
|
||||
_StopWords = new DictSegment((char)0);
|
||||
_StopWords = new DictSegment((char)0);
|
||||
|
||||
//读取主词典文件
|
||||
//读取主词典文件
|
||||
Path file = PathUtils.get(configuration.getDictRoot(), Dictionary.PATH_DIC_STOP);
|
||||
|
||||
InputStream is = null;
|
||||
try {
|
||||
is = new FileInputStream(file.toFile());
|
||||
} catch (FileNotFoundException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
InputStream is = null;
|
||||
try {
|
||||
is = new FileInputStream(file.toFile());
|
||||
} catch (FileNotFoundException e) {
|
||||
logger.error(e.getMessage(), e);
|
||||
}
|
||||
|
||||
try {
|
||||
BufferedReader br = new BufferedReader(new InputStreamReader(is , "UTF-8"), 512);
|
||||
String theWord = null;
|
||||
do {
|
||||
theWord = br.readLine();
|
||||
if (theWord != null && !"".equals(theWord.trim())) {
|
||||
_StopWords.fillSegment(theWord.trim().toCharArray());
|
||||
}
|
||||
} while (theWord != null);
|
||||
try {
|
||||
BufferedReader br = new BufferedReader(new InputStreamReader(is , "UTF-8"), 512);
|
||||
String theWord = null;
|
||||
do {
|
||||
theWord = br.readLine();
|
||||
if (theWord != null && !"".equals(theWord.trim())) {
|
||||
_StopWords.fillSegment(theWord.trim().toCharArray());
|
||||
}
|
||||
} while (theWord != null);
|
||||
|
||||
} catch (IOException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
} catch (IOException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
|
||||
}finally{
|
||||
try {
|
||||
if(is != null){
|
||||
is.close();
|
||||
is = null;
|
||||
}
|
||||
} catch (IOException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
}
|
||||
}
|
||||
}finally{
|
||||
try {
|
||||
if(is != null){
|
||||
is.close();
|
||||
is = null;
|
||||
}
|
||||
} catch (IOException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
//加载扩展停止词典
|
||||
@ -426,15 +426,15 @@ public class Dictionary {
|
||||
if(extStopWordDictFiles != null){
|
||||
is = null;
|
||||
for(String extStopWordDictName : extStopWordDictFiles){
|
||||
logger.info("[Dict Loading] " + extStopWordDictName);
|
||||
logger.info("[Dict Loading] " + extStopWordDictName);
|
||||
|
||||
//读取扩展词典文件
|
||||
file=PathUtils.get(configuration.getDictRoot(), extStopWordDictName);
|
||||
try {
|
||||
is = new FileInputStream(file.toFile());
|
||||
} catch (FileNotFoundException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
}
|
||||
//读取扩展词典文件
|
||||
file=PathUtils.get(configuration.getDictRoot(), extStopWordDictName);
|
||||
try {
|
||||
is = new FileInputStream(file.toFile());
|
||||
} catch (FileNotFoundException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
}
|
||||
//如果找不到扩展的字典，则忽略
|
||||
if(is == null){
|
||||
continue;
|
||||
@ -446,26 +446,26 @@ public class Dictionary {
|
||||
theWord = br.readLine();
|
||||
if (theWord != null && !"".equals(theWord.trim())) {
|
||||
//加载扩展停止词典数据到内存中
|
||||
_StopWords.fillSegment(theWord.trim().toCharArray());
|
||||
_StopWords.fillSegment(theWord.trim().toCharArray());
|
||||
}
|
||||
} while (theWord != null);
|
||||
|
||||
|
||||
} catch (IOException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
|
||||
logger.error("ik-analyzer",e);
|
||||
|
||||
}finally{
|
||||
try {
|
||||
if(is != null){
|
||||
is.close();
|
||||
is = null;
|
||||
is.close();
|
||||
is = null;
|
||||
}
|
||||
} catch (IOException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
logger.error("ik-analyzer",e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
//加载远程停用词典
|
||||
List<String> remoteExtStopWordDictFiles = configuration.getRemoteExtStopWordDictionarys();
|
||||
for(String location:remoteExtStopWordDictFiles){
|
||||
@ -484,10 +484,10 @@ public class Dictionary {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 加载量词词典
|
||||
*/
|
||||
@ -496,12 +496,12 @@ public class Dictionary {
|
||||
_QuantifierDict = new DictSegment((char)0);
|
||||
//读取量词词典文件
|
||||
Path file = PathUtils.get(configuration.getDictRoot(), Dictionary.PATH_DIC_QUANTIFIER);
|
||||
InputStream is = null;
|
||||
try {
|
||||
is = new FileInputStream(file.toFile());
|
||||
} catch (FileNotFoundException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
}
|
||||
InputStream is = null;
|
||||
try {
|
||||
is = new FileInputStream(file.toFile());
|
||||
} catch (FileNotFoundException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
}
|
||||
try {
|
||||
BufferedReader br = new BufferedReader(new InputStreamReader(is , "UTF-8"), 512);
|
||||
String theWord = null;
|
||||
@ -511,134 +511,134 @@ public class Dictionary {
|
||||
_QuantifierDict.fillSegment(theWord.trim().toCharArray());
|
||||
}
|
||||
} while (theWord != null);
|
||||
|
||||
|
||||
} catch (IOException ioe) {
|
||||
logger.error("Quantifier Dictionary loading exception.");
|
||||
|
||||
|
||||
}finally{
|
||||
try {
|
||||
if(is != null){
|
||||
is.close();
|
||||
is = null;
|
||||
is.close();
|
||||
is = null;
|
||||
}
|
||||
} catch (IOException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
logger.error("ik-analyzer",e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private void loadSurnameDict(){
|
||||
private void loadSurnameDict(){
|
||||
|
||||
_SurnameDict = new DictSegment((char)0);
|
||||
_SurnameDict = new DictSegment((char)0);
|
||||
Path file = PathUtils.get(configuration.getDictRoot(), Dictionary.PATH_DIC_SURNAME);
|
||||
InputStream is = null;
|
||||
try {
|
||||
is = new FileInputStream(file.toFile());
|
||||
} catch (FileNotFoundException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
}
|
||||
if(is == null){
|
||||
throw new RuntimeException("Surname Dictionary not found!!!");
|
||||
}
|
||||
try {
|
||||
BufferedReader br = new BufferedReader(new InputStreamReader(is , "UTF-8"), 512);
|
||||
String theWord;
|
||||
do {
|
||||
theWord = br.readLine();
|
||||
if (theWord != null && !"".equals(theWord.trim())) {
|
||||
_SurnameDict.fillSegment(theWord.trim().toCharArray());
|
||||
}
|
||||
} while (theWord != null);
|
||||
} catch (IOException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
}finally{
|
||||
try {
|
||||
if(is != null){
|
||||
is.close();
|
||||
is = null;
|
||||
}
|
||||
} catch (IOException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
}
|
||||
}
|
||||
}
|
||||
InputStream is = null;
|
||||
try {
|
||||
is = new FileInputStream(file.toFile());
|
||||
} catch (FileNotFoundException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
}
|
||||
if(is == null){
|
||||
throw new RuntimeException("Surname Dictionary not found!!!");
|
||||
}
|
||||
try {
|
||||
BufferedReader br = new BufferedReader(new InputStreamReader(is , "UTF-8"), 512);
|
||||
String theWord;
|
||||
do {
|
||||
theWord = br.readLine();
|
||||
if (theWord != null && !"".equals(theWord.trim())) {
|
||||
_SurnameDict.fillSegment(theWord.trim().toCharArray());
|
||||
}
|
||||
} while (theWord != null);
|
||||
} catch (IOException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
}finally{
|
||||
try {
|
||||
if(is != null){
|
||||
is.close();
|
||||
is = null;
|
||||
}
|
||||
} catch (IOException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private void loadSuffixDict(){
|
||||
private void loadSuffixDict(){
|
||||
|
||||
_SuffixDict = new DictSegment((char)0);
|
||||
_SuffixDict = new DictSegment((char)0);
|
||||
Path file = PathUtils.get(configuration.getDictRoot(), Dictionary.PATH_DIC_SUFFIX);
|
||||
InputStream is = null;
|
||||
try {
|
||||
is = new FileInputStream(file.toFile());
|
||||
} catch (FileNotFoundException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
}
|
||||
if(is == null){
|
||||
throw new RuntimeException("Suffix Dictionary not found!!!");
|
||||
}
|
||||
try {
|
||||
InputStream is = null;
|
||||
try {
|
||||
is = new FileInputStream(file.toFile());
|
||||
} catch (FileNotFoundException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
}
|
||||
if(is == null){
|
||||
throw new RuntimeException("Suffix Dictionary not found!!!");
|
||||
}
|
||||
try {
|
||||
|
||||
BufferedReader br = new BufferedReader(new InputStreamReader(is , "UTF-8"), 512);
|
||||
String theWord;
|
||||
do {
|
||||
theWord = br.readLine();
|
||||
if (theWord != null && !"".equals(theWord.trim())) {
|
||||
_SuffixDict.fillSegment(theWord.trim().toCharArray());
|
||||
}
|
||||
} while (theWord != null);
|
||||
} catch (IOException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
}finally{
|
||||
try {
|
||||
is.close();
|
||||
is = null;
|
||||
} catch (IOException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
}
|
||||
}
|
||||
}
|
||||
BufferedReader br = new BufferedReader(new InputStreamReader(is , "UTF-8"), 512);
|
||||
String theWord;
|
||||
do {
|
||||
theWord = br.readLine();
|
||||
if (theWord != null && !"".equals(theWord.trim())) {
|
||||
_SuffixDict.fillSegment(theWord.trim().toCharArray());
|
||||
}
|
||||
} while (theWord != null);
|
||||
} catch (IOException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
}finally{
|
||||
try {
|
||||
is.close();
|
||||
is = null;
|
||||
} catch (IOException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private void loadPrepDict(){
|
||||
private void loadPrepDict(){
|
||||
|
||||
_PrepDict = new DictSegment((char)0);
|
||||
_PrepDict = new DictSegment((char)0);
|
||||
Path file = PathUtils.get(configuration.getDictRoot(), Dictionary.PATH_DIC_PREP);
|
||||
InputStream is = null;
|
||||
try {
|
||||
is = new FileInputStream(file.toFile());
|
||||
} catch (FileNotFoundException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
}
|
||||
if(is == null){
|
||||
throw new RuntimeException("Preposition Dictionary not found!!!");
|
||||
}
|
||||
try {
|
||||
InputStream is = null;
|
||||
try {
|
||||
is = new FileInputStream(file.toFile());
|
||||
} catch (FileNotFoundException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
}
|
||||
if(is == null){
|
||||
throw new RuntimeException("Preposition Dictionary not found!!!");
|
||||
}
|
||||
try {
|
||||
|
||||
BufferedReader br = new BufferedReader(new InputStreamReader(is , "UTF-8"), 512);
|
||||
String theWord;
|
||||
do {
|
||||
theWord = br.readLine();
|
||||
if (theWord != null && !"".equals(theWord.trim())) {
|
||||
BufferedReader br = new BufferedReader(new InputStreamReader(is , "UTF-8"), 512);
|
||||
String theWord;
|
||||
do {
|
||||
theWord = br.readLine();
|
||||
if (theWord != null && !"".equals(theWord.trim())) {
|
||||
|
||||
_PrepDict.fillSegment(theWord.trim().toCharArray());
|
||||
}
|
||||
} while (theWord != null);
|
||||
} catch (IOException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
}finally{
|
||||
try {
|
||||
is.close();
|
||||
is = null;
|
||||
} catch (IOException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void reLoadMainDict(){
|
||||
logger.info("重新加载词典...");
|
||||
_PrepDict.fillSegment(theWord.trim().toCharArray());
|
||||
}
|
||||
} while (theWord != null);
|
||||
} catch (IOException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
}finally{
|
||||
try {
|
||||
is.close();
|
||||
is = null;
|
||||
} catch (IOException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void reLoadMainDict(){
|
||||
logger.info("重新加载词典...");
|
||||
// 新开一个实例加载词典，减少加载过程对当前词典使用的影响
|
||||
Dictionary tmpDict = new Dictionary();
|
||||
tmpDict.configuration = getSingleton().configuration;
|
||||
@ -647,6 +647,6 @@ public class Dictionary {
|
||||
_MainDict = tmpDict._MainDict;
|
||||
_StopWords = tmpDict._StopWords;
|
||||
logger.info("重新加载词典完毕...");
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -7,9 +7,13 @@ import org.apache.http.client.methods.CloseableHttpResponse;
|
||||
import org.apache.http.client.methods.HttpHead;
|
||||
import org.apache.http.impl.client.CloseableHttpClient;
|
||||
import org.apache.http.impl.client.HttpClients;
|
||||
import org.elasticsearch.common.logging.ESLogger;
|
||||
import org.elasticsearch.common.logging.Loggers;
|
||||
|
||||
public class Monitor implements Runnable {
|
||||
|
||||
public static ESLogger logger= Loggers.getLogger("ik-analyzer");
|
||||
|
||||
private static CloseableHttpClient httpclient = HttpClients.createDefault();
|
||||
/*
|
||||
* 上次更改时间
|
||||
@ -19,12 +23,12 @@ public class Monitor implements Runnable {
|
||||
* 资源属性
|
||||
*/
|
||||
private String eTags;
|
||||
|
||||
|
||||
/*
|
||||
* 请求地址
|
||||
*/
|
||||
private String location;
|
||||
|
||||
private String location;
|
||||
|
||||
public Monitor(String location) {
|
||||
this.location = location;
|
||||
this.last_modified = null;
|
||||
@ -38,16 +42,16 @@ public class Monitor implements Runnable {
|
||||
* ④如果有变化，重新加载词典
|
||||
* ⑤休眠1min，返回第①步
|
||||
*/
|
||||
|
||||
|
||||
public void run() {
|
||||
|
||||
//超时设置
|
||||
RequestConfig rc = RequestConfig.custom().setConnectionRequestTimeout(10*1000)
|
||||
.setConnectTimeout(10*1000).setSocketTimeout(15*1000).build();
|
||||
|
||||
|
||||
HttpHead head = new HttpHead(location);
|
||||
head.setConfig(rc);
|
||||
|
||||
|
||||
//设置请求头
|
||||
if (last_modified != null) {
|
||||
head.setHeader("If-Modified-Since", last_modified);
|
||||
@ -55,17 +59,17 @@ public class Monitor implements Runnable {
|
||||
if (eTags != null) {
|
||||
head.setHeader("If-None-Match", eTags);
|
||||
}
|
||||
|
||||
|
||||
CloseableHttpResponse response = null;
|
||||
try {
|
||||
|
||||
|
||||
response = httpclient.execute(head);
|
||||
|
||||
|
||||
//返回200 才做操作
|
||||
if(response.getStatusLine().getStatusCode()==200){
|
||||
|
||||
|
||||
if (!response.getLastHeader("Last-Modified").getValue().equalsIgnoreCase(last_modified)
|
||||
||!response.getLastHeader("ETag").getValue().equalsIgnoreCase(eTags)) {
|
||||
||!response.getLastHeader("ETag").getValue().equalsIgnoreCase(eTags)) {
|
||||
|
||||
// 远程词库有更新,需要重新加载词典，并修改last_modified,eTags
|
||||
Dictionary.getSingleton().reLoadMainDict();
|
||||
@ -87,9 +91,9 @@ public class Monitor implements Runnable {
|
||||
response.close();
|
||||
}
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
logger.error(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@ -5,8 +5,8 @@ import org.elasticsearch.common.logging.Loggers;
|
||||
|
||||
public class Sleep {
|
||||
|
||||
public static ESLogger logger= Loggers.getLogger("ik-analyzer");
|
||||
|
||||
public static ESLogger logger= Loggers.getLogger("ik-analyzer");
|
||||
|
||||
public enum Type{MSEC,SEC,MIN,HOUR};
|
||||
public static void sleep(Type type,int num){
|
||||
try {
|
||||
@ -24,13 +24,13 @@ public class Sleep {
|
||||
Thread.sleep(num*60*60*1000);
|
||||
return;
|
||||
default:
|
||||
logger.error("输入类型错误，应为MSEC,SEC,MIN,HOUR之一");
|
||||
System.err.println("输入类型错误，应为MSEC,SEC,MIN,HOUR之一");
|
||||
return;
|
||||
}
|
||||
} catch (InterruptedException e) {
|
||||
e.printStackTrace();
|
||||
logger.error(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@ -1,7 +1,7 @@
|
||||
/**
|
||||
* IK 中文分词 版本 5.0
|
||||
* IK Analyzer release 5.0
|
||||
*
|
||||
*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
@ -20,7 +20,7 @@
|
||||
* 源代码由林良益(linliangyi2005@gmail.com)提供
|
||||
* 版权声明 2012，乌龙茶工作室
|
||||
* provided by Linliangyi and copyright 2012 by Oolong studio
|
||||
*
|
||||
*
|
||||
*/
|
||||
package org.wltea.analyzer.query;
|
||||
|
||||
@ -34,6 +34,8 @@ import org.apache.lucene.queryparser.classic.ParseException;
|
||||
import org.apache.lucene.queryparser.classic.QueryParser;
|
||||
import org.apache.lucene.search.Query;
|
||||
import org.apache.lucene.util.Version;
|
||||
import org.elasticsearch.common.logging.ESLogger;
|
||||
import org.elasticsearch.common.logging.Loggers;
|
||||
import org.wltea.analyzer.core.IKSegmenter;
|
||||
import org.wltea.analyzer.core.Lexeme;
|
||||
|
||||
@ -45,6 +47,8 @@ import org.wltea.analyzer.core.Lexeme;
|
||||
*/
|
||||
public class SWMCQueryBuilder {
|
||||
|
||||
public static ESLogger logger= Loggers.getLogger("ik-analyzer");
|
||||
|
||||
/**
|
||||
* 生成SWMCQuery
|
||||
* @param fieldName
|
||||
@ -62,7 +66,7 @@ public class SWMCQueryBuilder {
|
||||
Query _SWMCQuery = getSWMCQuery(fieldName , lexemes , quickMode);
|
||||
return _SWMCQuery;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 分词切分，并返回结链表
|
||||
* @param keywords
|
||||
@ -78,16 +82,16 @@ public class SWMCQueryBuilder {
|
||||
lexemes.add(l);
|
||||
}
|
||||
}catch(IOException e){
|
||||
e.printStackTrace();
|
||||
logger.error(e.getMessage(), e);
|
||||
}
|
||||
return lexemes;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* 根据分词结果生成SWMC搜索
|
||||
* @param fieldName
|
||||
// * @param pathOption
|
||||
// * @param pathOption
|
||||
* @param quickMode
|
||||
* @return
|
||||
*/
|
||||
@ -100,7 +104,7 @@ public class SWMCQueryBuilder {
|
||||
int lastLexemeLength = 0;
|
||||
//记录最后词元结束位置
|
||||
int lastLexemeEnd = -1;
|
||||
|
||||
|
||||
int shortCount = 0;
|
||||
int totalCount = 0;
|
||||
for(Lexeme l : lexemes){
|
||||
@ -110,15 +114,15 @@ public class SWMCQueryBuilder {
|
||||
keywordBuffer_Short.append(' ').append(l.getLexemeText());
|
||||
shortCount += l.getLength();
|
||||
}
|
||||
|
||||
|
||||
if(lastLexemeLength == 0){
|
||||
keywordBuffer.append(l.getLexemeText());
|
||||
keywordBuffer.append(l.getLexemeText());
|
||||
}else if(lastLexemeLength == 1 && l.getLength() == 1
|
||||
&& lastLexemeEnd == l.getBeginPosition()){//单字位置相邻，长度为一，合并)
|
||||
keywordBuffer.append(l.getLexemeText());
|
||||
}else{
|
||||
keywordBuffer.append(' ').append(l.getLexemeText());
|
||||
|
||||
|
||||
}
|
||||
lastLexemeLength = l.getLength();
|
||||
lastLexemeEnd = l.getEndPosition();
|
||||
@ -128,16 +132,16 @@ public class SWMCQueryBuilder {
|
||||
QueryParser qp = new QueryParser(fieldName, new StandardAnalyzer());
|
||||
qp.setDefaultOperator(QueryParser.AND_OPERATOR);
|
||||
qp.setAutoGeneratePhraseQueries(true);
|
||||
|
||||
|
||||
if(quickMode && (shortCount * 1.0f / totalCount) > 0.5f){
|
||||
try {
|
||||
//System.out.println(keywordBuffer.toString());
|
||||
Query q = qp.parse(keywordBuffer_Short.toString());
|
||||
return q;
|
||||
} catch (ParseException e) {
|
||||
e.printStackTrace();
|
||||
logger.error(e.getMessage(), e);
|
||||
}
|
||||
|
||||
|
||||
}else{
|
||||
if(keywordBuffer.length() > 0){
|
||||
try {
|
||||
@ -145,10 +149,10 @@ public class SWMCQueryBuilder {
|
||||
Query q = qp.parse(keywordBuffer.toString());
|
||||
return q;
|
||||
} catch (ParseException e) {
|
||||
e.printStackTrace();
|
||||
logger.error(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,7 +1,7 @@
|
||||
/**
|
||||
* IK 中文分词 版本 5.0.1
|
||||
* IK Analyzer release 5.0.1
|
||||
*
|
||||
*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
@ -20,8 +20,8 @@
|
||||
* 源代码由林良益(linliangyi2005@gmail.com)提供
|
||||
* 版权声明 2012，乌龙茶工作室
|
||||
* provided by Linliangyi and copyright 2012 by Oolong studio
|
||||
*
|
||||
*
|
||||
*
|
||||
*
|
||||
*/
|
||||
package org.wltea.analyzer.sample;
|
||||
|
||||
@ -44,47 +44,47 @@ import org.wltea.analyzer.lucene.IKAnalyzer;
|
||||
*/
|
||||
public class IKAnalzyerDemo {
|
||||
|
||||
public static ESLogger logger= Loggers.getLogger("ik-analyzer");
|
||||
|
||||
public static ESLogger logger= Loggers.getLogger("ik-analyzer");
|
||||
|
||||
public static void main(String[] args){
|
||||
//构建IK分词器，使用smart分词模式
|
||||
Analyzer analyzer = new IKAnalyzer(true);
|
||||
|
||||
|
||||
//获取Lucene的TokenStream对象
|
||||
TokenStream ts = null;
|
||||
TokenStream ts = null;
|
||||
try {
|
||||
ts = analyzer.tokenStream("myfield", new StringReader("WORLD ,.. html DATA</html>HELLO"));
|
||||
// ts = analyzer.tokenStream("myfield", new StringReader("这是一个中文分词的例子，你可以直接运行它！IKAnalyer can analysis english text too"));
|
||||
//获取词元位置属性
|
||||
OffsetAttribute offset = ts.addAttribute(OffsetAttribute.class);
|
||||
//获取词元文本属性
|
||||
CharTermAttribute term = ts.addAttribute(CharTermAttribute.class);
|
||||
//获取词元文本属性
|
||||
TypeAttribute type = ts.addAttribute(TypeAttribute.class);
|
||||
|
||||
|
||||
//重置TokenStream（重置StringReader）
|
||||
ts.reset();
|
||||
OffsetAttribute offset = ts.addAttribute(OffsetAttribute.class);
|
||||
//获取词元文本属性
|
||||
CharTermAttribute term = ts.addAttribute(CharTermAttribute.class);
|
||||
//获取词元文本属性
|
||||
TypeAttribute type = ts.addAttribute(TypeAttribute.class);
|
||||
|
||||
|
||||
//重置TokenStream（重置StringReader）
|
||||
ts.reset();
|
||||
//迭代获取分词结果
|
||||
while (ts.incrementToken()) {
|
||||
logger.info(offset.startOffset() + " - " + offset.endOffset() + " : " + term.toString() + " | " + type.type());
|
||||
System.out.println(offset.startOffset() + " - " + offset.endOffset() + " : " + term.toString() + " | " + type.type());
|
||||
}
|
||||
//关闭TokenStream（关闭StringReader）
|
||||
ts.end(); // Perform end-of-stream operations, e.g. set the final offset.
|
||||
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
logger.error(e.getMessage(), e);
|
||||
} finally {
|
||||
//释放TokenStream的所有资源
|
||||
if(ts != null){
|
||||
try {
|
||||
ts.close();
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
try {
|
||||
ts.close();
|
||||
} catch (IOException e) {
|
||||
logger.error(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@ -1,7 +1,7 @@
|
||||
/**
|
||||
* IK 中文分词 版本 5.0
|
||||
* IK Analyzer release 5.0
|
||||
*
|
||||
*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
@ -20,8 +20,8 @@
|
||||
* 源代码由林良益(linliangyi2005@gmail.com)提供
|
||||
* 版权声明 2012，乌龙茶工作室
|
||||
* provided by Linliangyi and copyright 2012 by Oolong studio
|
||||
*
|
||||
*
|
||||
*
|
||||
*
|
||||
*/
|
||||
package org.wltea.analyzer.sample;
|
||||
|
||||
@ -58,14 +58,14 @@ import org.wltea.analyzer.lucene.IKAnalyzer;
|
||||
/**
|
||||
* 使用IKAnalyzer进行Lucene索引和查询的演示
|
||||
* 2012-3-2
|
||||
*
|
||||
*
|
||||
* 以下是结合Lucene4.0 API的写法
|
||||
*
|
||||
*/
|
||||
public class LuceneIndexAndSearchDemo {
|
||||
|
||||
public static ESLogger logger= Loggers.getLogger("ik-analyzer");
|
||||
|
||||
public static ESLogger logger= Loggers.getLogger("ik-analyzer");
|
||||
|
||||
/**
|
||||
* 模拟：
|
||||
* 创建一个单条记录的索引，并对其进行搜索
|
||||
@ -74,20 +74,20 @@ public class LuceneIndexAndSearchDemo {
|
||||
public static void main(String[] args){
|
||||
//Lucene Document的域名
|
||||
String fieldName = "text";
|
||||
//检索内容
|
||||
//检索内容
|
||||
String text = "IK Analyzer是一个结合词典分词和文法分词的中文分词开源工具包。它使用了全新的正向迭代最细粒度切分算法。";
|
||||
|
||||
|
||||
//实例化IKAnalyzer分词器
|
||||
Analyzer analyzer = new IKAnalyzer(true);
|
||||
|
||||
|
||||
Directory directory = null;
|
||||
IndexWriter iwriter = null;
|
||||
IndexReader ireader = null;
|
||||
IndexSearcher isearcher = null;
|
||||
try {
|
||||
//建立内存索引对象
|
||||
directory = new RAMDirectory();
|
||||
|
||||
directory = new RAMDirectory();
|
||||
|
||||
//配置IndexWriterConfig
|
||||
IndexWriterConfig iwConfig = new IndexWriterConfig(analyzer);
|
||||
iwConfig.setOpenMode(OpenMode.CREATE_OR_APPEND);
|
||||
@ -98,53 +98,53 @@ public class LuceneIndexAndSearchDemo {
|
||||
doc.add(new TextField(fieldName, text, Field.Store.YES));
|
||||
iwriter.addDocument(doc);
|
||||
iwriter.close();
|
||||
|
||||
|
||||
|
||||
|
||||
//搜索过程**********************************
|
||||
//实例化搜索器
|
||||
//实例化搜索器
|
||||
ireader = DirectoryReader.open(directory);
|
||||
isearcher = new IndexSearcher(ireader);
|
||||
|
||||
String keyword = "中文分词工具包";
|
||||
isearcher = new IndexSearcher(ireader);
|
||||
|
||||
String keyword = "中文分词工具包";
|
||||
//使用QueryParser查询分析器构造Query对象
|
||||
QueryParser qp = new QueryParser(fieldName, analyzer);
|
||||
qp.setDefaultOperator(QueryParser.AND_OPERATOR);
|
||||
Query query = qp.parse(keyword);
|
||||
logger.info("Query = " + query);
|
||||
|
||||
System.out.println("Query = " + query);
|
||||
|
||||
//搜索相似度最高的5条记录
|
||||
TopDocs topDocs = isearcher.search(query , 5);
|
||||
logger.info("命中：" + topDocs.totalHits);
|
||||
System.out.println("命中：" + topDocs.totalHits);
|
||||
//输出结果
|
||||
ScoreDoc[] scoreDocs = topDocs.scoreDocs;
|
||||
for (int i = 0; i < topDocs.totalHits; i++){
|
||||
Document targetDoc = isearcher.doc(scoreDocs[i].doc);
|
||||
logger.info("内容：" + targetDoc.toString());
|
||||
}
|
||||
|
||||
System.out.println("内容：" + targetDoc.toString());
|
||||
}
|
||||
|
||||
} catch (CorruptIndexException e) {
|
||||
e.printStackTrace();
|
||||
logger.error(e.getMessage(), e);
|
||||
} catch (LockObtainFailedException e) {
|
||||
e.printStackTrace();
|
||||
logger.error(e.getMessage(), e);
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
logger.error(e.getMessage(), e);
|
||||
} catch (ParseException e) {
|
||||
e.printStackTrace();
|
||||
logger.error(e.getMessage(), e);
|
||||
} finally{
|
||||
if(ireader != null){
|
||||
try {
|
||||
ireader.close();
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
logger.error(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
if(directory != null){
|
||||
try {
|
||||
directory.close();
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
logger.error(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user