third_codes/elasticsearch-analysis-ik
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #49 from arronli/master

Browse Source 
Update  org.wltea.analyzer.dic.Monito and org.wltea.analyzer.dic.Monitor
This commit is contained in:
Medcl 2014-12-04 17:49:25 +08:00
commit d60e7a6877
3 changed files with 66 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
config/ik/IKAnalyzer.cfg.xml
View File

@ -7,7 +7,7 @@
<!--用户可以在这里配置自己的扩展停止词字典-->
<entry key="ext_stopwords">custom/ext_stopword.dic</entry>
<!--用户可以在这里配置远程扩展字典 -->
<entry key="remote_ext_dict">words_location</entry>
<!--用户可以在这里配置远程扩展停止词字典-->
<entry key="remote_ext_stopwords">words_location</entry>
<!-- <entry key="remote_ext_dict">words_location</entry> -->
<!--用户可以在这里配置远程扩展停止词字典-->
<!-- <entry key="remote_ext_stopwords">words_location</entry> -->
</properties>

45
src/main/java/org/wltea/analyzer/dic/Dictionary.java
View File

@ -25,6 +25,20 @@
*/
package org.wltea.analyzer.dic;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
@ -35,11 +49,6 @@ import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.Loggers;
import org.wltea.analyzer.cfg.Configuration;
import java.io.*;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
/**
* 词典管理类,单子模式
*/
@ -69,13 +78,18 @@ public class Dictionary {
*/
private Configuration configuration;
private ESLogger logger=null;
private static ScheduledExecutorService pool = Executors.newScheduledThreadPool(1);
public static final String PATH_DIC_MAIN = "ik/main.dic";
public static final String PATH_DIC_SURNAME = "ik/surname.dic";
public static final String PATH_DIC_QUANTIFIER = "ik/quantifier.dic";
public static final String PATH_DIC_SUFFIX = "ik/suffix.dic";
public static final String PATH_DIC_PREP = "ik/preposition.dic";
public static final String PATH_DIC_STOP = "ik/stopword.dic";
private Dictionary(){
logger = Loggers.getLogger("ik-analyzer");
}
@ -100,17 +114,16 @@ public class Dictionary {
singleton.loadPrepDict();
singleton.loadStopWordDict();
//建立监控线程
for(String location:cfg.getRemoteExtDictionarys()){
Thread monitor = new Thread(new Monitor(location));
monitor.start();
}
for(String location:cfg.getRemoteExtStopWordDictionarys()){
Thread monitor = new Thread(new Monitor(location));
monitor.start();
}
return singleton;
//建立监控线程
for(String location:cfg.getRemoteExtDictionarys()){
//10 秒是初始延迟可以修改的 60是间隔时间 单位秒
pool.scheduleAtFixedRate(new Monitor(location), 10, 60, TimeUnit.SECONDS);
}
for(String location:cfg.getRemoteExtStopWordDictionarys()){
pool.scheduleAtFixedRate(new Monitor(location), 10, 60, TimeUnit.SECONDS);
}
return singleton;
}
}
}

76
src/main/java/org/wltea/analyzer/dic/Monitor.java
View File

@ -2,15 +2,11 @@ package org.wltea.analyzer.dic;
import java.io.IOException;
import org.apache.http.Header;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpHead;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.wltea.analyzer.help.Sleep;
import org.wltea.analyzer.help.Sleep.Type;
public class Monitor implements Runnable {
@ -44,53 +40,49 @@ public class Monitor implements Runnable {
*/
public void run() {
//超时设置
RequestConfig rc = RequestConfig.custom().setConnectionRequestTimeout(10*1000)
.setConnectTimeout(10*1000).setSocketTimeout(15*1000).build();
while (true) {
HttpHead head = new HttpHead(location);
head.setConfig(rc);
HttpHead head = new HttpHead(location);
head.setConfig(rc);
//设置请求头
if (last_modified != null) {
head.setHeader("If-Modified-Since", last_modified);
}
if (eTags != null) {
head.setHeader("If-None-Match", eTags);
}
CloseableHttpResponse response = null;
try {
//设置请求头
if (last_modified != null) {
head.setHeader("If-Modified-Since", last_modified);
}
if (eTags != null) {
head.setHeader("If-None-Match", eTags);
}
response = httpclient.execute(head);
CloseableHttpResponse response = null;
try {
response = httpclient.execute(head);
//返回304 Not Modified，词库未更新
if(response.getStatusLine().getStatusCode()==304){
continue;
}else if(response.getStatusLine().getStatusCode()==200){
if (!response.getLastHeader("Last-Modified").getValue().equalsIgnoreCase(last_modified)
||!response.getLastHeader("ETags").getValue().equalsIgnoreCase(eTags)) {
// 远程词库有更新,需要重新加载词典，并修改last_modified,eTags
Dictionary.getSingleton().reLoadMainDict();
last_modified = response.getLastHeader("Last-Modified")==null?null:response.getLastHeader("Last-Modified").getValue();
eTags = response.getLastHeader("ETags")==null?null:response.getLastHeader("ETags").getValue();
}
//返回200 才做操作
if(response.getStatusLine().getStatusCode()==200){
if (!response.getLastHeader("Last-Modified").getValue().equalsIgnoreCase(last_modified)
||!response.getLastHeader("ETags").getValue().equalsIgnoreCase(eTags)) {
// 远程词库有更新,需要重新加载词典，并修改last_modified,eTags
Dictionary.getSingleton().reLoadMainDict();
last_modified = response.getLastHeader("Last-Modified")==null?null:response.getLastHeader("Last-Modified").getValue();
eTags = response.getLastHeader("ETags")==null?null:response.getLastHeader("ETags").getValue();
}
} catch (ClientProtocolException e) {
e.printStackTrace();
}
} catch (Exception e) {
e.printStackTrace();
}finally{
try {
response.close();
} catch (IOException e) {
e.printStackTrace();
}finally{
try {
response.close();
} catch (IOException e) {
e.printStackTrace();
}
Sleep.sleep(Type.SEC, 60);
}
}
}
}
}