merge code
This commit is contained in:
medcl
commit
ca2bfe5732
@ -62,37 +62,37 @@ public class Dictionary {
|
||||
*/
|
||||
private static Dictionary singleton;
|
||||
|
||||
private DictSegment _MainDict;
|
||||
private DictSegment _MainDict;
|
||||
|
||||
private DictSegment _SurnameDict;
|
||||
private DictSegment _SurnameDict;
|
||||
|
||||
private DictSegment _QuantifierDict;
|
||||
private DictSegment _QuantifierDict;
|
||||
|
||||
private DictSegment _SuffixDict;
|
||||
private DictSegment _SuffixDict;
|
||||
|
||||
private DictSegment _PrepDict;
|
||||
private DictSegment _PrepDict;
|
||||
|
||||
private DictSegment _StopWords;
|
||||
private DictSegment _StopWords;
|
||||
|
||||
|
||||
/**
|
||||
* 配置对象
|
||||
*/
|
||||
private Configuration configuration;
|
||||
public static final ESLogger logger=Loggers.getLogger("ik-analyzer");
|
||||
public static ESLogger logger=Loggers.getLogger("ik-analyzer");
|
||||
|
||||
private static ScheduledExecutorService pool = Executors.newScheduledThreadPool(1);
|
||||
private static ScheduledExecutorService pool = Executors.newScheduledThreadPool(1);
|
||||
|
||||
public static final String PATH_DIC_MAIN = "ik/main.dic";
|
||||
public static final String PATH_DIC_SURNAME = "ik/surname.dic";
|
||||
public static final String PATH_DIC_QUANTIFIER = "ik/quantifier.dic";
|
||||
public static final String PATH_DIC_SUFFIX = "ik/suffix.dic";
|
||||
public static final String PATH_DIC_PREP = "ik/preposition.dic";
|
||||
public static final String PATH_DIC_STOP = "ik/stopword.dic";
|
||||
public static final String PATH_DIC_MAIN = "ik/main.dic";
|
||||
public static final String PATH_DIC_SURNAME = "ik/surname.dic";
|
||||
public static final String PATH_DIC_QUANTIFIER = "ik/quantifier.dic";
|
||||
public static final String PATH_DIC_SUFFIX = "ik/suffix.dic";
|
||||
public static final String PATH_DIC_PREP = "ik/preposition.dic";
|
||||
public static final String PATH_DIC_STOP = "ik/stopword.dic";
|
||||
|
||||
private Dictionary(){
|
||||
private Dictionary(){
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 词典初始化
|
||||
@ -103,28 +103,29 @@ public class Dictionary {
|
||||
* @return Dictionary
|
||||
*/
|
||||
public static synchronized Dictionary initial(Configuration cfg){
|
||||
if(singleton == null){
|
||||
synchronized(Dictionary.class){
|
||||
if(singleton == null){
|
||||
singleton = new Dictionary();
|
||||
singleton.configuration=cfg;
|
||||
singleton.loadMainDict();
|
||||
singleton.loadSurnameDict();
|
||||
singleton.loadQuantifierDict();
|
||||
singleton.loadSuffixDict();
|
||||
singleton.loadPrepDict();
|
||||
singleton.loadStopWordDict();
|
||||
|
||||
synchronized(Dictionary.class){
|
||||
if(singleton == null){
|
||||
singleton = new Dictionary();
|
||||
singleton.configuration=cfg;
|
||||
singleton.loadMainDict();
|
||||
singleton.loadSurnameDict();
|
||||
singleton.loadQuantifierDict();
|
||||
singleton.loadSuffixDict();
|
||||
singleton.loadPrepDict();
|
||||
singleton.loadStopWordDict();
|
||||
//建立监控线程
|
||||
for(String location:cfg.getRemoteExtDictionarys()){
|
||||
//10 秒是初始延迟可以修改的 60是间隔时间 单位秒
|
||||
pool.scheduleAtFixedRate(new Monitor(location), 10, 60, TimeUnit.SECONDS);
|
||||
}
|
||||
for(String location:cfg.getRemoteExtStopWordDictionarys()){
|
||||
pool.scheduleAtFixedRate(new Monitor(location), 10, 60, TimeUnit.SECONDS);
|
||||
}
|
||||
|
||||
//建立监控线程
|
||||
for(String location:cfg.getRemoteExtDictionarys()){
|
||||
//10 秒是初始延迟可以修改的 60是间隔时间 单位秒
|
||||
pool.scheduleAtFixedRate(new Monitor(location), 10, 60, TimeUnit.SECONDS);
|
||||
return singleton;
|
||||
}
|
||||
for(String location:cfg.getRemoteExtStopWordDictionarys()){
|
||||
pool.scheduleAtFixedRate(new Monitor(location), 10, 60, TimeUnit.SECONDS);
|
||||
}
|
||||
|
||||
return singleton;
|
||||
}
|
||||
}
|
||||
return singleton;
|
||||
@ -183,7 +184,7 @@ public class Dictionary {
|
||||
* @return Hit 匹配结果描述
|
||||
*/
|
||||
public Hit matchInMainDict(char[] charArray , int begin, int length){
|
||||
return singleton._MainDict.match(charArray, begin, length);
|
||||
return singleton._MainDict.match(charArray, begin, length);
|
||||
}
|
||||
|
||||
/**
|
||||
@ -223,12 +224,12 @@ public class Dictionary {
|
||||
//读取主词典文件
|
||||
Path file = PathUtils.get(configuration.getDictRoot(), Dictionary.PATH_DIC_MAIN);
|
||||
|
||||
InputStream is = null;
|
||||
try {
|
||||
is = new FileInputStream(file.toFile());
|
||||
} catch (FileNotFoundException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
InputStream is = null;
|
||||
try {
|
||||
is = new FileInputStream(file.toFile());
|
||||
} catch (FileNotFoundException e) {
|
||||
logger.error(e.getMessage(), e);
|
||||
}
|
||||
|
||||
try {
|
||||
BufferedReader br = new BufferedReader(new InputStreamReader(is , "UTF-8"), 512);
|
||||
@ -241,16 +242,16 @@ public class Dictionary {
|
||||
} while (theWord != null);
|
||||
|
||||
} catch (IOException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
logger.error("ik-analyzer",e);
|
||||
|
||||
}finally{
|
||||
}finally{
|
||||
try {
|
||||
if(is != null){
|
||||
is.close();
|
||||
is = null;
|
||||
is.close();
|
||||
is = null;
|
||||
}
|
||||
} catch (IOException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
logger.error("ik-analyzer",e);
|
||||
}
|
||||
}
|
||||
//加载扩展词典
|
||||
@ -269,13 +270,13 @@ public class Dictionary {
|
||||
InputStream is = null;
|
||||
for(String extDictName : extDictFiles){
|
||||
//读取扩展词典文件
|
||||
logger.info("[Dict Loading] " + extDictName);
|
||||
logger.info("[Dict Loading] " + extDictName);
|
||||
Path file = PathUtils.get(configuration.getDictRoot(), extDictName);
|
||||
try {
|
||||
is = new FileInputStream(file.toFile());
|
||||
} catch (FileNotFoundException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
}
|
||||
try {
|
||||
is = new FileInputStream(file.toFile());
|
||||
} catch (FileNotFoundException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
}
|
||||
|
||||
//如果找不到扩展的字典，则忽略
|
||||
if(is == null){
|
||||
@ -286,21 +287,23 @@ public class Dictionary {
|
||||
String theWord = null;
|
||||
do {
|
||||
theWord = br.readLine();
|
||||
if (theWord != null && !"".equals(theWord.trim())) {
|
||||
if (theWord != null && !"".equals(theWord.trim())) {
|
||||
//加载扩展词典数据到主内存词典中
|
||||
_MainDict.fillSegment(theWord.trim().toCharArray());
|
||||
}
|
||||
} while (theWord != null);
|
||||
|
||||
} catch (IOException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
}finally{
|
||||
logger.error("ik-analyzer",e);
|
||||
}finally{
|
||||
try {
|
||||
is.close();
|
||||
is = null;
|
||||
if(is != null){
|
||||
is.close();
|
||||
is = null;
|
||||
}
|
||||
} catch (IOException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
}
|
||||
logger.error("ik-analyzer",e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -315,14 +318,11 @@ public class Dictionary {
|
||||
for(String location:remoteExtDictFiles){
|
||||
logger.info("[Dict Loading] " + location);
|
||||
List<String> lists = getRemoteWords(location);
|
||||
|
||||
/** Redundant Nullcheck as the list is initialized in the getRemoteWords method
|
||||
//如果找不到扩展的字典，则忽略
|
||||
if(lists == null){
|
||||
logger.error("[Dict Loading] "+location+"加载失败");
|
||||
continue;
|
||||
}*/
|
||||
|
||||
}
|
||||
for(String theWord:lists){
|
||||
if (theWord != null && !"".equals(theWord.trim())) {
|
||||
//加载扩展词典数据到主内存词典中
|
||||
@ -384,41 +384,41 @@ public class Dictionary {
|
||||
*/
|
||||
private void loadStopWordDict(){
|
||||
//建立主词典实例
|
||||
_StopWords = new DictSegment((char)0);
|
||||
_StopWords = new DictSegment((char)0);
|
||||
|
||||
//读取主词典文件
|
||||
//读取主词典文件
|
||||
Path file = PathUtils.get(configuration.getDictRoot(), Dictionary.PATH_DIC_STOP);
|
||||
|
||||
InputStream is = null;
|
||||
try {
|
||||
is = new FileInputStream(file.toFile());
|
||||
} catch (FileNotFoundException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
InputStream is = null;
|
||||
try {
|
||||
is = new FileInputStream(file.toFile());
|
||||
} catch (FileNotFoundException e) {
|
||||
logger.error(e.getMessage(), e);
|
||||
}
|
||||
|
||||
try {
|
||||
BufferedReader br = new BufferedReader(new InputStreamReader(is , "UTF-8"), 512);
|
||||
String theWord = null;
|
||||
do {
|
||||
theWord = br.readLine();
|
||||
if (theWord != null && !"".equals(theWord.trim())) {
|
||||
_StopWords.fillSegment(theWord.trim().toCharArray());
|
||||
}
|
||||
} while (theWord != null);
|
||||
try {
|
||||
BufferedReader br = new BufferedReader(new InputStreamReader(is , "UTF-8"), 512);
|
||||
String theWord = null;
|
||||
do {
|
||||
theWord = br.readLine();
|
||||
if (theWord != null && !"".equals(theWord.trim())) {
|
||||
_StopWords.fillSegment(theWord.trim().toCharArray());
|
||||
}
|
||||
} while (theWord != null);
|
||||
|
||||
} catch (IOException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
} catch (IOException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
|
||||
}finally{
|
||||
try {
|
||||
if(is != null){
|
||||
is.close();
|
||||
is = null;
|
||||
}
|
||||
} catch (IOException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
}
|
||||
}
|
||||
}finally{
|
||||
try {
|
||||
if(is != null){
|
||||
is.close();
|
||||
is = null;
|
||||
}
|
||||
} catch (IOException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
//加载扩展停止词典
|
||||
@ -426,15 +426,15 @@ public class Dictionary {
|
||||
if(extStopWordDictFiles != null){
|
||||
is = null;
|
||||
for(String extStopWordDictName : extStopWordDictFiles){
|
||||
logger.info("[Dict Loading] " + extStopWordDictName);
|
||||
logger.info("[Dict Loading] " + extStopWordDictName);
|
||||
|
||||
//读取扩展词典文件
|
||||
file=PathUtils.get(configuration.getDictRoot(), extStopWordDictName);
|
||||
try {
|
||||
is = new FileInputStream(file.toFile());
|
||||
} catch (FileNotFoundException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
}
|
||||
//读取扩展词典文件
|
||||
file=PathUtils.get(configuration.getDictRoot(), extStopWordDictName);
|
||||
try {
|
||||
is = new FileInputStream(file.toFile());
|
||||
} catch (FileNotFoundException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
}
|
||||
//如果找不到扩展的字典，则忽略
|
||||
if(is == null){
|
||||
continue;
|
||||
@ -446,19 +446,21 @@ public class Dictionary {
|
||||
theWord = br.readLine();
|
||||
if (theWord != null && !"".equals(theWord.trim())) {
|
||||
//加载扩展停止词典数据到内存中
|
||||
_StopWords.fillSegment(theWord.trim().toCharArray());
|
||||
_StopWords.fillSegment(theWord.trim().toCharArray());
|
||||
}
|
||||
} while (theWord != null);
|
||||
|
||||
} catch (IOException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
logger.error("ik-analyzer",e);
|
||||
|
||||
}finally{
|
||||
try {
|
||||
is.close();
|
||||
is = null;
|
||||
if(is != null){
|
||||
is.close();
|
||||
is = null;
|
||||
}
|
||||
} catch (IOException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
logger.error("ik-analyzer",e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -469,14 +471,11 @@ public class Dictionary {
|
||||
for(String location:remoteExtStopWordDictFiles){
|
||||
logger.info("[Dict Loading] " + location);
|
||||
List<String> lists = getRemoteWords(location);
|
||||
|
||||
/** Redundant Nullcheck as the list is initialized in the getRemoteWords method
|
||||
//如果找不到扩展的字典，则忽略
|
||||
if(lists == null){
|
||||
logger.error("[Dict Loading] "+location+"加载失败");
|
||||
continue;
|
||||
}*/
|
||||
|
||||
}
|
||||
for(String theWord:lists){
|
||||
if (theWord != null && !"".equals(theWord.trim())) {
|
||||
//加载远程词典数据到主内存中
|
||||
@ -497,12 +496,12 @@ public class Dictionary {
|
||||
_QuantifierDict = new DictSegment((char)0);
|
||||
//读取量词词典文件
|
||||
Path file = PathUtils.get(configuration.getDictRoot(), Dictionary.PATH_DIC_QUANTIFIER);
|
||||
InputStream is = null;
|
||||
try {
|
||||
is = new FileInputStream(file.toFile());
|
||||
} catch (FileNotFoundException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
}
|
||||
InputStream is = null;
|
||||
try {
|
||||
is = new FileInputStream(file.toFile());
|
||||
} catch (FileNotFoundException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
}
|
||||
try {
|
||||
BufferedReader br = new BufferedReader(new InputStreamReader(is , "UTF-8"), 512);
|
||||
String theWord = null;
|
||||
@ -519,125 +518,127 @@ public class Dictionary {
|
||||
}finally{
|
||||
try {
|
||||
if(is != null){
|
||||
is.close();
|
||||
is = null;
|
||||
is.close();
|
||||
is = null;
|
||||
}
|
||||
} catch (IOException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
logger.error("ik-analyzer",e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private void loadSurnameDict(){
|
||||
private void loadSurnameDict(){
|
||||
|
||||
_SurnameDict = new DictSegment((char)0);
|
||||
_SurnameDict = new DictSegment((char)0);
|
||||
Path file = PathUtils.get(configuration.getDictRoot(), Dictionary.PATH_DIC_SURNAME);
|
||||
InputStream is = null;
|
||||
try {
|
||||
is = new FileInputStream(file.toFile());
|
||||
} catch (FileNotFoundException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
}
|
||||
if(is == null){
|
||||
throw new RuntimeException("Surname Dictionary not found!!!");
|
||||
}
|
||||
try {
|
||||
BufferedReader br = new BufferedReader(new InputStreamReader(is , "UTF-8"), 512);
|
||||
String theWord;
|
||||
do {
|
||||
theWord = br.readLine();
|
||||
if (theWord != null && !"".equals(theWord.trim())) {
|
||||
_SurnameDict.fillSegment(theWord.trim().toCharArray());
|
||||
}
|
||||
} while (theWord != null);
|
||||
} catch (IOException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
}finally{
|
||||
try {
|
||||
InputStream is = null;
|
||||
try {
|
||||
is = new FileInputStream(file.toFile());
|
||||
} catch (FileNotFoundException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
}
|
||||
if(is == null){
|
||||
throw new RuntimeException("Surname Dictionary not found!!!");
|
||||
}
|
||||
try {
|
||||
BufferedReader br = new BufferedReader(new InputStreamReader(is , "UTF-8"), 512);
|
||||
String theWord;
|
||||
do {
|
||||
theWord = br.readLine();
|
||||
if (theWord != null && !"".equals(theWord.trim())) {
|
||||
_SurnameDict.fillSegment(theWord.trim().toCharArray());
|
||||
}
|
||||
} while (theWord != null);
|
||||
} catch (IOException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
}finally{
|
||||
try {
|
||||
if(is != null){
|
||||
is.close();
|
||||
is = null;
|
||||
}
|
||||
} catch (IOException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private void loadSuffixDict(){
|
||||
|
||||
_SuffixDict = new DictSegment((char)0);
|
||||
Path file = PathUtils.get(configuration.getDictRoot(), Dictionary.PATH_DIC_SUFFIX);
|
||||
InputStream is = null;
|
||||
try {
|
||||
is = new FileInputStream(file.toFile());
|
||||
} catch (FileNotFoundException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
}
|
||||
if(is == null){
|
||||
throw new RuntimeException("Suffix Dictionary not found!!!");
|
||||
}
|
||||
try {
|
||||
|
||||
BufferedReader br = new BufferedReader(new InputStreamReader(is , "UTF-8"), 512);
|
||||
String theWord;
|
||||
do {
|
||||
theWord = br.readLine();
|
||||
if (theWord != null && !"".equals(theWord.trim())) {
|
||||
_SuffixDict.fillSegment(theWord.trim().toCharArray());
|
||||
}
|
||||
} while (theWord != null);
|
||||
} catch (IOException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
}finally{
|
||||
try {
|
||||
is.close();
|
||||
is = null;
|
||||
} catch (IOException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (IOException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private void loadSuffixDict(){
|
||||
private void loadPrepDict(){
|
||||
|
||||
_SuffixDict = new DictSegment((char)0);
|
||||
Path file = PathUtils.get(configuration.getDictRoot(), Dictionary.PATH_DIC_SUFFIX);
|
||||
InputStream is = null;
|
||||
try {
|
||||
is = new FileInputStream(file.toFile());
|
||||
} catch (FileNotFoundException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
}
|
||||
if(is == null){
|
||||
throw new RuntimeException("Suffix Dictionary not found!!!");
|
||||
}
|
||||
try {
|
||||
|
||||
BufferedReader br = new BufferedReader(new InputStreamReader(is , "UTF-8"), 512);
|
||||
String theWord;
|
||||
do {
|
||||
theWord = br.readLine();
|
||||
if (theWord != null && !"".equals(theWord.trim())) {
|
||||
_SuffixDict.fillSegment(theWord.trim().toCharArray());
|
||||
}
|
||||
} while (theWord != null);
|
||||
} catch (IOException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
}finally{
|
||||
try {
|
||||
is.close();
|
||||
is = null;
|
||||
} catch (IOException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private void loadPrepDict(){
|
||||
|
||||
_PrepDict = new DictSegment((char)0);
|
||||
_PrepDict = new DictSegment((char)0);
|
||||
Path file = PathUtils.get(configuration.getDictRoot(), Dictionary.PATH_DIC_PREP);
|
||||
InputStream is = null;
|
||||
try {
|
||||
is = new FileInputStream(file.toFile());
|
||||
} catch (FileNotFoundException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
}
|
||||
if(is == null){
|
||||
throw new RuntimeException("Preposition Dictionary not found!!!");
|
||||
}
|
||||
try {
|
||||
InputStream is = null;
|
||||
try {
|
||||
is = new FileInputStream(file.toFile());
|
||||
} catch (FileNotFoundException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
}
|
||||
if(is == null){
|
||||
throw new RuntimeException("Preposition Dictionary not found!!!");
|
||||
}
|
||||
try {
|
||||
|
||||
BufferedReader br = new BufferedReader(new InputStreamReader(is , "UTF-8"), 512);
|
||||
String theWord;
|
||||
do {
|
||||
theWord = br.readLine();
|
||||
if (theWord != null && !"".equals(theWord.trim())) {
|
||||
BufferedReader br = new BufferedReader(new InputStreamReader(is , "UTF-8"), 512);
|
||||
String theWord;
|
||||
do {
|
||||
theWord = br.readLine();
|
||||
if (theWord != null && !"".equals(theWord.trim())) {
|
||||
|
||||
_PrepDict.fillSegment(theWord.trim().toCharArray());
|
||||
}
|
||||
} while (theWord != null);
|
||||
} catch (IOException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
}finally{
|
||||
try {
|
||||
is.close();
|
||||
is = null;
|
||||
} catch (IOException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
}
|
||||
}
|
||||
}
|
||||
_PrepDict.fillSegment(theWord.trim().toCharArray());
|
||||
}
|
||||
} while (theWord != null);
|
||||
} catch (IOException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
}finally{
|
||||
try {
|
||||
is.close();
|
||||
is = null;
|
||||
} catch (IOException e) {
|
||||
logger.error("ik-analyzer",e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void reLoadMainDict(){
|
||||
logger.info("重新加载词典...");
|
||||
public void reLoadMainDict(){
|
||||
logger.info("重新加载词典...");
|
||||
// 新开一个实例加载词典，减少加载过程对当前词典使用的影响
|
||||
Dictionary tmpDict = new Dictionary();
|
||||
tmpDict.configuration = getSingleton().configuration;
|
||||
@ -646,6 +647,6 @@ public class Dictionary {
|
||||
_MainDict = tmpDict._MainDict;
|
||||
_StopWords = tmpDict._StopWords;
|
||||
logger.info("重新加载词典完毕...");
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -7,9 +7,13 @@ import org.apache.http.client.methods.CloseableHttpResponse;
|
||||
import org.apache.http.client.methods.HttpHead;
|
||||
import org.apache.http.impl.client.CloseableHttpClient;
|
||||
import org.apache.http.impl.client.HttpClients;
|
||||
import org.elasticsearch.common.logging.ESLogger;
|
||||
import org.elasticsearch.common.logging.Loggers;
|
||||
|
||||
public class Monitor implements Runnable {
|
||||
|
||||
public static ESLogger logger= Loggers.getLogger("ik-analyzer");
|
||||
|
||||
private static CloseableHttpClient httpclient = HttpClients.createDefault();
|
||||
/*
|
||||
* 上次更改时间
|
||||
@ -65,7 +69,7 @@ public class Monitor implements Runnable {
|
||||
if(response.getStatusLine().getStatusCode()==200){
|
||||
|
||||
if (!response.getLastHeader("Last-Modified").getValue().equalsIgnoreCase(last_modified)
|
||||
||!response.getLastHeader("ETag").getValue().equalsIgnoreCase(eTags)) {
|
||||
||!response.getLastHeader("ETag").getValue().equalsIgnoreCase(eTags)) {
|
||||
|
||||
// 远程词库有更新,需要重新加载词典，并修改last_modified,eTags
|
||||
Dictionary.getSingleton().reLoadMainDict();
|
||||
@ -87,7 +91,7 @@ public class Monitor implements Runnable {
|
||||
response.close();
|
||||
}
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
logger.error(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -5,7 +5,7 @@ import org.elasticsearch.common.logging.Loggers;
|
||||
|
||||
public class Sleep {
|
||||
|
||||
public static final ESLogger logger= Loggers.getLogger("ik-analyzer");
|
||||
public static ESLogger logger= Loggers.getLogger("ik-analyzer");
|
||||
|
||||
public enum Type{MSEC,SEC,MIN,HOUR};
|
||||
public static void sleep(Type type,int num){
|
||||
@ -15,20 +15,20 @@ public class Sleep {
|
||||
Thread.sleep(num);
|
||||
return;
|
||||
case SEC:
|
||||
Thread.sleep(num*1000L);
|
||||
Thread.sleep(num*1000);
|
||||
return;
|
||||
case MIN:
|
||||
Thread.sleep(num*60*1000L);
|
||||
Thread.sleep(num*60*1000);
|
||||
return;
|
||||
case HOUR:
|
||||
Thread.sleep(num*60*60*1000L);
|
||||
Thread.sleep(num*60*60*1000);
|
||||
return;
|
||||
default:
|
||||
logger.error("输入类型错误，应为MSEC,SEC,MIN,HOUR之一");
|
||||
System.err.println("输入类型错误，应为MSEC,SEC,MIN,HOUR之一");
|
||||
return;
|
||||
}
|
||||
} catch (InterruptedException e) {
|
||||
e.printStackTrace();
|
||||
logger.error(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -34,6 +34,8 @@ import org.apache.lucene.queryparser.classic.ParseException;
|
||||
import org.apache.lucene.queryparser.classic.QueryParser;
|
||||
import org.apache.lucene.search.Query;
|
||||
import org.apache.lucene.util.Version;
|
||||
import org.elasticsearch.common.logging.ESLogger;
|
||||
import org.elasticsearch.common.logging.Loggers;
|
||||
import org.wltea.analyzer.core.IKSegmenter;
|
||||
import org.wltea.analyzer.core.Lexeme;
|
||||
|
||||
@ -45,6 +47,8 @@ import org.wltea.analyzer.core.Lexeme;
|
||||
*/
|
||||
public class SWMCQueryBuilder {
|
||||
|
||||
public static ESLogger logger= Loggers.getLogger("ik-analyzer");
|
||||
|
||||
/**
|
||||
* 生成SWMCQuery
|
||||
* @param fieldName
|
||||
@ -78,7 +82,7 @@ public class SWMCQueryBuilder {
|
||||
lexemes.add(l);
|
||||
}
|
||||
}catch(IOException e){
|
||||
e.printStackTrace();
|
||||
logger.error(e.getMessage(), e);
|
||||
}
|
||||
return lexemes;
|
||||
}
|
||||
@ -87,7 +91,7 @@ public class SWMCQueryBuilder {
|
||||
/**
|
||||
* 根据分词结果生成SWMC搜索
|
||||
* @param fieldName
|
||||
// * @param pathOption
|
||||
// * @param pathOption
|
||||
* @param quickMode
|
||||
* @return
|
||||
*/
|
||||
@ -135,7 +139,7 @@ public class SWMCQueryBuilder {
|
||||
Query q = qp.parse(keywordBuffer_Short.toString());
|
||||
return q;
|
||||
} catch (ParseException e) {
|
||||
e.printStackTrace();
|
||||
logger.error(e.getMessage(), e);
|
||||
}
|
||||
|
||||
}else{
|
||||
@ -145,7 +149,7 @@ public class SWMCQueryBuilder {
|
||||
Query q = qp.parse(keywordBuffer.toString());
|
||||
return q;
|
||||
} catch (ParseException e) {
|
||||
e.printStackTrace();
|
||||
logger.error(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -44,46 +44,46 @@ import org.wltea.analyzer.lucene.IKAnalyzer;
|
||||
*/
|
||||
public class IKAnalzyerDemo {
|
||||
|
||||
public static final ESLogger logger= Loggers.getLogger("ik-analyzer");
|
||||
public static ESLogger logger= Loggers.getLogger("ik-analyzer");
|
||||
|
||||
public static void main(String[] args){
|
||||
//构建IK分词器，使用smart分词模式
|
||||
Analyzer analyzer = new IKAnalyzer(true);
|
||||
|
||||
//获取Lucene的TokenStream对象
|
||||
TokenStream ts = null;
|
||||
TokenStream ts = null;
|
||||
try {
|
||||
ts = analyzer.tokenStream("myfield", new StringReader("WORLD ,.. html DATA</html>HELLO"));
|
||||
// ts = analyzer.tokenStream("myfield", new StringReader("这是一个中文分词的例子，你可以直接运行它！IKAnalyer can analysis english text too"));
|
||||
//获取词元位置属性
|
||||
OffsetAttribute offset = ts.addAttribute(OffsetAttribute.class);
|
||||
//获取词元文本属性
|
||||
CharTermAttribute term = ts.addAttribute(CharTermAttribute.class);
|
||||
//获取词元文本属性
|
||||
TypeAttribute type = ts.addAttribute(TypeAttribute.class);
|
||||
OffsetAttribute offset = ts.addAttribute(OffsetAttribute.class);
|
||||
//获取词元文本属性
|
||||
CharTermAttribute term = ts.addAttribute(CharTermAttribute.class);
|
||||
//获取词元文本属性
|
||||
TypeAttribute type = ts.addAttribute(TypeAttribute.class);
|
||||
|
||||
|
||||
//重置TokenStream（重置StringReader）
|
||||
//重置TokenStream（重置StringReader）
|
||||
ts.reset();
|
||||
//迭代获取分词结果
|
||||
while (ts.incrementToken()) {
|
||||
logger.info(offset.startOffset() + " - " + offset.endOffset() + " : " + term.toString() + " | " + type.type());
|
||||
System.out.println(offset.startOffset() + " - " + offset.endOffset() + " : " + term.toString() + " | " + type.type());
|
||||
}
|
||||
//关闭TokenStream（关闭StringReader）
|
||||
ts.end(); // Perform end-of-stream operations, e.g. set the final offset.
|
||||
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
logger.error(e.getMessage(), e);
|
||||
} finally {
|
||||
//释放TokenStream的所有资源
|
||||
if(ts != null){
|
||||
try {
|
||||
ts.close();
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
try {
|
||||
ts.close();
|
||||
} catch (IOException e) {
|
||||
logger.error(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
@ -64,7 +64,7 @@ import org.wltea.analyzer.lucene.IKAnalyzer;
|
||||
*/
|
||||
public class LuceneIndexAndSearchDemo {
|
||||
|
||||
public static final ESLogger logger= Loggers.getLogger("ik-analyzer");
|
||||
public static ESLogger logger= Loggers.getLogger("ik-analyzer");
|
||||
|
||||
/**
|
||||
* 模拟：
|
||||
@ -74,7 +74,7 @@ public class LuceneIndexAndSearchDemo {
|
||||
public static void main(String[] args){
|
||||
//Lucene Document的域名
|
||||
String fieldName = "text";
|
||||
//检索内容
|
||||
//检索内容
|
||||
String text = "IK Analyzer是一个结合词典分词和文法分词的中文分词开源工具包。它使用了全新的正向迭代最细粒度切分算法。";
|
||||
|
||||
//实例化IKAnalyzer分词器
|
||||
@ -101,7 +101,7 @@ public class LuceneIndexAndSearchDemo {
|
||||
|
||||
|
||||
//搜索过程**********************************
|
||||
//实例化搜索器
|
||||
//实例化搜索器
|
||||
ireader = DirectoryReader.open(directory);
|
||||
isearcher = new IndexSearcher(ireader);
|
||||
|
||||
@ -110,39 +110,39 @@ public class LuceneIndexAndSearchDemo {
|
||||
QueryParser qp = new QueryParser(fieldName, analyzer);
|
||||
qp.setDefaultOperator(QueryParser.AND_OPERATOR);
|
||||
Query query = qp.parse(keyword);
|
||||
logger.info("Query = " + query);
|
||||
System.out.println("Query = " + query);
|
||||
|
||||
//搜索相似度最高的5条记录
|
||||
TopDocs topDocs = isearcher.search(query , 5);
|
||||
logger.info("命中：" + topDocs.totalHits);
|
||||
System.out.println("命中：" + topDocs.totalHits);
|
||||
//输出结果
|
||||
ScoreDoc[] scoreDocs = topDocs.scoreDocs;
|
||||
for (int i = 0; i < topDocs.totalHits; i++){
|
||||
Document targetDoc = isearcher.doc(scoreDocs[i].doc);
|
||||
logger.info("内容：" + targetDoc.toString());
|
||||
System.out.println("内容：" + targetDoc.toString());
|
||||
}
|
||||
|
||||
} catch (CorruptIndexException e) {
|
||||
e.printStackTrace();
|
||||
logger.error(e.getMessage(), e);
|
||||
} catch (LockObtainFailedException e) {
|
||||
e.printStackTrace();
|
||||
logger.error(e.getMessage(), e);
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
logger.error(e.getMessage(), e);
|
||||
} catch (ParseException e) {
|
||||
e.printStackTrace();
|
||||
logger.error(e.getMessage(), e);
|
||||
} finally{
|
||||
if(ireader != null){
|
||||
try {
|
||||
ireader.close();
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
logger.error(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
if(directory != null){
|
||||
try {
|
||||
directory.close();
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
logger.error(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user