third_codes/elasticsearch-analysis-ik
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

Also load config from /etc/elasticsearch/analysis-ik (#197)

Browse Source 
Support install by `bin/plugin`, dealing with config files reallocation
This commit is contained in:
Robert LU 2016-05-25 17:07:25 +08:00 committed by Medcl
parent 7e29998ab9
commit 26fe905cc6
17 changed files with 238 additions and 227 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
README.md
View File

@ -169,7 +169,8 @@ Result
### Dictionary Configuration ### Dictionary Configuration
#### `plugins/elasticsearch-analysis-ik-*/config/ik/IKAnalyzer.cfg.xml` `IKAnalyzer.cfg.xml` can be located at `{conf}/analysis-ik/config/IKAnalyzer.cfg.xml`
or `{plugins}/elasticsearch-analysis-ik-*/config/IKAnalyzer.cfg.xml`
```xml ```xml
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>

0
config/ik/IKAnalyzer.cfg.xml → config/IKAnalyzer.cfg.xml
View File

0
config/ik/custom/ext_stopword.dic → config/custom/ext_stopword.dic
View File

0
config/ik/custom/mydict.dic → config/custom/mydict.dic
View File

0
config/ik/custom/single_word.dic → config/custom/single_word.dic
View File

0
config/ik/custom/single_word_full.dic → config/custom/single_word_full.dic
View File

0
config/ik/custom/single_word_low_freq.dic → config/custom/single_word_low_freq.dic
View File

0
config/ik/custom/sougou.dic → config/custom/sougou.dic
View File

0
config/ik/main.dic → config/main.dic
View File

0
config/ik/preposition.dic → config/preposition.dic
View File

0
config/ik/quantifier.dic → config/quantifier.dic
View File

0
config/ik/stopword.dic → config/stopword.dic
View File

0
config/ik/suffix.dic → config/suffix.dic
View File

0
config/ik/surname.dic → config/surname.dic
View File

6
src/main/java/org/elasticsearch/plugin/analysis/ik/AnalysisIkPlugin.java
View File

@ -24,13 +24,15 @@ import static java.rmi.Naming.bind;
public class AnalysisIkPlugin extends Plugin { public class AnalysisIkPlugin extends Plugin {
public static String PLUGIN_NAME = "analysis-ik";
@Override public String name() { @Override public String name() {
return "analysis-ik"; return PLUGIN_NAME;
} }
@Override public String description() { @Override public String description() {
return "ik analysis"; return PLUGIN_NAME;
} }
@Override @Override

34
src/main/java/org/wltea/analyzer/cfg/Configuration.java
View File

@ -20,12 +20,13 @@ import java.util.Properties;
public class Configuration { public class Configuration {
private static String FILE_NAME = "ik/IKAnalyzer.cfg.xml"; private static String FILE_NAME = "IKAnalyzer.cfg.xml";
private static final String EXT_DICT = "ext_dict"; private static final String EXT_DICT = "ext_dict";
private static final String REMOTE_EXT_DICT = "remote_ext_dict"; private static final String REMOTE_EXT_DICT = "remote_ext_dict";
private static final String EXT_STOP = "ext_stopwords"; private static final String EXT_STOP = "ext_stopwords";
private static final String REMOTE_EXT_STOP = "remote_ext_stopwords"; private static final String REMOTE_EXT_STOP = "remote_ext_stopwords";
private static ESLogger logger = Loggers.getLogger("ik-analyzer"); private static ESLogger logger = Loggers.getLogger("ik-analyzer");
private Path conf_dir;
private Properties props; private Properties props;
private Environment environment; private Environment environment;
@ -34,16 +35,24 @@ public class Configuration {
props = new Properties(); props = new Properties();
environment = env; environment = env;
conf_dir = environment.configFile().resolve(AnalysisIkPlugin.PLUGIN_NAME);
Path fileConfig = PathUtils.get(getDictRoot(), FILE_NAME); Path configFile = conf_dir.resolve(FILE_NAME);
InputStream input = null; InputStream input = null;
try { try {
input = new FileInputStream(fileConfig.toFile()); logger.info("try load config from {}", configFile);
input = new FileInputStream(configFile.toFile());
} catch (FileNotFoundException e) { } catch (FileNotFoundException e) {
conf_dir = this.getConfigInPluginDir();
configFile = conf_dir.resolve(FILE_NAME);
try {
logger.info("try load config from {}", configFile);
input = new FileInputStream(configFile.toFile());
} catch (FileNotFoundException ex) {
// We should report origin exception
logger.error("ik-analyzer", e); logger.error("ik-analyzer", e);
} }
}
if (input != null) { if (input != null) {
try { try {
props.loadFromXML(input); props.loadFromXML(input);
@ -64,7 +73,7 @@ public class Configuration {
if (filePaths != null) { if (filePaths != null) {
for (String filePath : filePaths) { for (String filePath : filePaths) {
if (filePath != null && !"".equals(filePath.trim())) { if (filePath != null && !"".equals(filePath.trim())) {
Path file = PathUtils.get("ik", filePath.trim()); Path file = PathUtils.get(filePath.trim());
extDictFiles.add(file.toString()); extDictFiles.add(file.toString());
} }
@ -101,7 +110,7 @@ public class Configuration {
if (filePaths != null) { if (filePaths != null) {
for (String filePath : filePaths) { for (String filePath : filePaths) {
if (filePath != null && !"".equals(filePath.trim())) { if (filePath != null && !"".equals(filePath.trim())) {
Path file = PathUtils.get("ik", filePath.trim()); Path file = PathUtils.get(filePath.trim());
extStopWordDictFiles.add(file.toString()); extStopWordDictFiles.add(file.toString());
} }
@ -130,8 +139,13 @@ public class Configuration {
} }
public String getDictRoot() { public String getDictRoot() {
return PathUtils.get( return conf_dir.toAbsolutePath().toString();
new File(AnalysisIkPlugin.class.getProtectionDomain().getCodeSource().getLocation().getPath()).getParent(),"config") }
.toAbsolutePath().toString();
private Path getConfigInPluginDir() {
return PathUtils
.get(new File(AnalysisIkPlugin.class.getProtectionDomain().getCodeSource().getLocation().getPath())
.getParent(), "config")
.toAbsolutePath();
} }
} }

46
src/main/java/org/wltea/analyzer/dic/Dictionary.java
View File

@ -56,7 +56,6 @@ import org.wltea.analyzer.cfg.Configuration;
*/ */
public class Dictionary { public class Dictionary {
/* /*
* 词典单子实例 * 词典单子实例
*/ */
@ -74,7 +73,6 @@ public class Dictionary {
private DictSegment _StopWords; private DictSegment _StopWords;
/** /**
* 配置对象 * 配置对象
*/ */
@ -83,23 +81,21 @@ public class Dictionary {
private static ScheduledExecutorService pool = Executors.newScheduledThreadPool(1); private static ScheduledExecutorService pool = Executors.newScheduledThreadPool(1);
public static final String PATH_DIC_MAIN = "ik/main.dic"; public static final String PATH_DIC_MAIN = "main.dic";
public static final String PATH_DIC_SURNAME = "ik/surname.dic"; public static final String PATH_DIC_SURNAME = "surname.dic";
public static final String PATH_DIC_QUANTIFIER = "ik/quantifier.dic"; public static final String PATH_DIC_QUANTIFIER = "quantifier.dic";
public static final String PATH_DIC_SUFFIX = "ik/suffix.dic"; public static final String PATH_DIC_SUFFIX = "suffix.dic";
public static final String PATH_DIC_PREP = "ik/preposition.dic"; public static final String PATH_DIC_PREP = "preposition.dic";
public static final String PATH_DIC_STOP = "ik/stopword.dic"; public static final String PATH_DIC_STOP = "stopword.dic";
private Dictionary() { private Dictionary() {
} }
/** /**
* 词典初始化 * 词典初始化 由于IK Analyzer的词典采用Dictionary类的静态方法进行词典初始化
* 由于IK Analyzer的词典采用Dictionary类的静态方法进行词典初始化 * 只有当Dictionary类被实际调用时，才会开始载入词典， 这将延长首次分词操作的时间 该方法提供了一个在应用加载阶段就初始化字典的手段
* 只有当Dictionary类被实际调用时，才会开始载入词典， *
* 这将延长首次分词操作的时间
* 该方法提供了一个在应用加载阶段就初始化字典的手段
* @return Dictionary * @return Dictionary
*/ */
public static synchronized Dictionary initial(Configuration cfg) { public static synchronized Dictionary initial(Configuration cfg) {
@ -133,6 +129,7 @@ public class Dictionary {
/** /**
* 获取词典单子实例 * 获取词典单子实例
*
* @return Dictionary 单例对象 * @return Dictionary 单例对象
*/ */
public static Dictionary getSingleton() { public static Dictionary getSingleton() {
@ -144,7 +141,9 @@ public class Dictionary {
/** /**
* 批量加载新词条 * 批量加载新词条
* @param words Collection<String>词条列表 *
* @param words
* Collection<String>词条列表
*/ */
public void addWords(Collection<String> words) { public void addWords(Collection<String> words) {
if (words != null) { if (words != null) {
@ -173,6 +172,7 @@ public class Dictionary {
/** /**
* 检索匹配主词典 * 检索匹配主词典
*
* @return Hit 匹配结果描述 * @return Hit 匹配结果描述
*/ */
public Hit matchInMainDict(char[] charArray) { public Hit matchInMainDict(char[] charArray) {
@ -181,6 +181,7 @@ public class Dictionary {
/** /**
* 检索匹配主词典 * 检索匹配主词典
*
* @return Hit 匹配结果描述 * @return Hit 匹配结果描述
*/ */
public Hit matchInMainDict(char[] charArray, int begin, int length) { public Hit matchInMainDict(char[] charArray, int begin, int length) {
@ -189,15 +190,16 @@ public class Dictionary {
/** /**
* 检索匹配量词词典 * 检索匹配量词词典
*
* @return Hit 匹配结果描述 * @return Hit 匹配结果描述
*/ */
public Hit matchInQuantifierDict(char[] charArray, int begin, int length) { public Hit matchInQuantifierDict(char[] charArray, int begin, int length) {
return singleton._QuantifierDict.match(charArray, begin, length); return singleton._QuantifierDict.match(charArray, begin, length);
} }
/** /**
* 从已匹配的Hit中直接取出DictSegment，继续向下匹配 * 从已匹配的Hit中直接取出DictSegment，继续向下匹配
*
* @return Hit * @return Hit
*/ */
public Hit matchWithHit(char[] charArray, int currentIndex, Hit matchedHit) { public Hit matchWithHit(char[] charArray, int currentIndex, Hit matchedHit) {
@ -205,9 +207,9 @@ public class Dictionary {
return ds.match(charArray, currentIndex, 1, matchedHit); return ds.match(charArray, currentIndex, 1, matchedHit);
} }
/** /**
* 判断是否是停止词 * 判断是否是停止词
*
* @return boolean * @return boolean
*/ */
public boolean isStopWord(char[] charArray, int begin, int length) { public boolean isStopWord(char[] charArray, int begin, int length) {
@ -309,7 +311,6 @@ public class Dictionary {
} }
} }
/** /**
* 加载远程扩展词典到主词库表 * 加载远程扩展词典到主词库表
*/ */
@ -340,8 +341,8 @@ public class Dictionary {
private static List<String> getRemoteWords(String location) { private static List<String> getRemoteWords(String location) {
List<String> buffer = new ArrayList<String>(); List<String> buffer = new ArrayList<String>();
RequestConfig rc = RequestConfig.custom().setConnectionRequestTimeout(10*1000) RequestConfig rc = RequestConfig.custom().setConnectionRequestTimeout(10 * 1000).setConnectTimeout(10 * 1000)
.setConnectTimeout(10*1000).setSocketTimeout(60*1000).build(); .setSocketTimeout(60 * 1000).build();
CloseableHttpClient httpclient = HttpClients.createDefault(); CloseableHttpClient httpclient = HttpClients.createDefault();
CloseableHttpResponse response; CloseableHttpResponse response;
BufferedReader in; BufferedReader in;
@ -377,8 +378,6 @@ public class Dictionary {
return buffer; return buffer;
} }
/** /**
* 加载用户扩展的停止词词典 * 加载用户扩展的停止词词典
*/ */
@ -420,7 +419,6 @@ public class Dictionary {
} }
} }
// 加载扩展停止词典 // 加载扩展停止词典
List<String> extStopWordDictFiles = configuration.getExtStopWordDictionarys(); List<String> extStopWordDictFiles = configuration.getExtStopWordDictionarys();
if (extStopWordDictFiles != null) { if (extStopWordDictFiles != null) {
@ -485,7 +483,6 @@ public class Dictionary {
} }
} }
} }
/** /**
@ -527,7 +524,6 @@ public class Dictionary {
} }
} }
private void loadSurnameDict() { private void loadSurnameDict() {
_SurnameDict = new DictSegment((char) 0); _SurnameDict = new DictSegment((char) 0);
@ -564,7 +560,6 @@ public class Dictionary {
} }
} }
private void loadSuffixDict() { private void loadSuffixDict() {
_SuffixDict = new DictSegment((char) 0); _SuffixDict = new DictSegment((char) 0);
@ -600,7 +595,6 @@ public class Dictionary {
} }
} }
private void loadPrepDict() { private void loadPrepDict() {
_PrepDict = new DictSegment((char) 0); _PrepDict = new DictSegment((char) 0);