Merge pull request #49 from arronli/master

Update  org.wltea.analyzer.dic.Monito and org.wltea.analyzer.dic.Monitor
This commit is contained in:
Medcl 2014-12-04 17:49:25 +08:00
commit d60e7a6877
3 changed files with 66 additions and 61 deletions

View File

@ -7,7 +7,7 @@
<!--用户可以在这里配置自己的扩展停止词字典--> <!--用户可以在这里配置自己的扩展停止词字典-->
<entry key="ext_stopwords">custom/ext_stopword.dic</entry> <entry key="ext_stopwords">custom/ext_stopword.dic</entry>
<!--用户可以在这里配置远程扩展字典 --> <!--用户可以在这里配置远程扩展字典 -->
<entry key="remote_ext_dict">words_location</entry> <!-- <entry key="remote_ext_dict">words_location</entry> -->
<!--用户可以在这里配置远程扩展停止词字典--> <!--用户可以在这里配置远程扩展停止词字典-->
<entry key="remote_ext_stopwords">words_location</entry> <!-- <entry key="remote_ext_stopwords">words_location</entry> -->
</properties> </properties>

View File

@ -25,6 +25,20 @@
*/ */
package org.wltea.analyzer.dic; package org.wltea.analyzer.dic;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import org.apache.http.client.ClientProtocolException; import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.config.RequestConfig; import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.CloseableHttpResponse;
@ -35,11 +49,6 @@ import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.logging.Loggers;
import org.wltea.analyzer.cfg.Configuration; import org.wltea.analyzer.cfg.Configuration;
import java.io.*;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
/** /**
* 词典管理类,单子模式 * 词典管理类,单子模式
*/ */
@ -69,13 +78,18 @@ public class Dictionary {
*/ */
private Configuration configuration; private Configuration configuration;
private ESLogger logger=null; private ESLogger logger=null;
private static ScheduledExecutorService pool = Executors.newScheduledThreadPool(1);
public static final String PATH_DIC_MAIN = "ik/main.dic"; public static final String PATH_DIC_MAIN = "ik/main.dic";
public static final String PATH_DIC_SURNAME = "ik/surname.dic"; public static final String PATH_DIC_SURNAME = "ik/surname.dic";
public static final String PATH_DIC_QUANTIFIER = "ik/quantifier.dic"; public static final String PATH_DIC_QUANTIFIER = "ik/quantifier.dic";
public static final String PATH_DIC_SUFFIX = "ik/suffix.dic"; public static final String PATH_DIC_SUFFIX = "ik/suffix.dic";
public static final String PATH_DIC_PREP = "ik/preposition.dic"; public static final String PATH_DIC_PREP = "ik/preposition.dic";
public static final String PATH_DIC_STOP = "ik/stopword.dic"; public static final String PATH_DIC_STOP = "ik/stopword.dic";
private Dictionary(){ private Dictionary(){
logger = Loggers.getLogger("ik-analyzer"); logger = Loggers.getLogger("ik-analyzer");
} }
@ -102,12 +116,11 @@ public class Dictionary {
//建立监控线程 //建立监控线程
for(String location:cfg.getRemoteExtDictionarys()){ for(String location:cfg.getRemoteExtDictionarys()){
Thread monitor = new Thread(new Monitor(location)); //10 秒是初始延迟可以修改的 60是间隔时间 单位秒
monitor.start(); pool.scheduleAtFixedRate(new Monitor(location), 10, 60, TimeUnit.SECONDS);
} }
for(String location:cfg.getRemoteExtStopWordDictionarys()){ for(String location:cfg.getRemoteExtStopWordDictionarys()){
Thread monitor = new Thread(new Monitor(location)); pool.scheduleAtFixedRate(new Monitor(location), 10, 60, TimeUnit.SECONDS);
monitor.start();
} }
return singleton; return singleton;

View File

@ -2,15 +2,11 @@ package org.wltea.analyzer.dic;
import java.io.IOException; import java.io.IOException;
import org.apache.http.Header;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.config.RequestConfig; import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpHead; import org.apache.http.client.methods.HttpHead;
import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients; import org.apache.http.impl.client.HttpClients;
import org.wltea.analyzer.help.Sleep;
import org.wltea.analyzer.help.Sleep.Type;
public class Monitor implements Runnable { public class Monitor implements Runnable {
@ -44,10 +40,11 @@ public class Monitor implements Runnable {
*/ */
public void run() { public void run() {
//超时设置 //超时设置
RequestConfig rc = RequestConfig.custom().setConnectionRequestTimeout(10*1000) RequestConfig rc = RequestConfig.custom().setConnectionRequestTimeout(10*1000)
.setConnectTimeout(10*1000).setSocketTimeout(15*1000).build(); .setConnectTimeout(10*1000).setSocketTimeout(15*1000).build();
while (true) {
HttpHead head = new HttpHead(location); HttpHead head = new HttpHead(location);
head.setConfig(rc); head.setConfig(rc);
@ -61,12 +58,11 @@ public class Monitor implements Runnable {
CloseableHttpResponse response = null; CloseableHttpResponse response = null;
try { try {
response = httpclient.execute(head); response = httpclient.execute(head);
//返回304 Not Modified词库未更新 //返回200 才做操作
if(response.getStatusLine().getStatusCode()==304){ if(response.getStatusLine().getStatusCode()==200){
continue;
}else if(response.getStatusLine().getStatusCode()==200){
if (!response.getLastHeader("Last-Modified").getValue().equalsIgnoreCase(last_modified) if (!response.getLastHeader("Last-Modified").getValue().equalsIgnoreCase(last_modified)
||!response.getLastHeader("ETags").getValue().equalsIgnoreCase(eTags)) { ||!response.getLastHeader("ETags").getValue().equalsIgnoreCase(eTags)) {
@ -78,9 +74,7 @@ public class Monitor implements Runnable {
} }
} }
} catch (ClientProtocolException e) { } catch (Exception e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace(); e.printStackTrace();
}finally{ }finally{
try { try {
@ -88,8 +82,6 @@ public class Monitor implements Runnable {
} catch (IOException e) { } catch (IOException e) {
e.printStackTrace(); e.printStackTrace();
} }
Sleep.sleep(Type.SEC, 60);
}
} }
} }