third_codes/elasticsearch-analysis-ik
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

bug fix,prefix blank char caused ArrayIndexOutOfBoundsException

Browse Source
This commit is contained in:
medcl 2013-12-13 16:39:30 +08:00
parent 35700686c9
commit e2fb31a55e
7 changed files with 27 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
pom.xml
View File

@ -6,7 +6,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch-analysis-ik</artifactId>
<version>1.2.4</version>
<version>1.2.5</version>
<packaging>jar</packaging>
<description>IK Analyzer for ElasticSearch</description>
<inceptionYear>2009</inceptionYear>
@ -31,7 +31,7 @@
</parent>
<properties>
<elasticsearch.version>0.90.6</elasticsearch.version>
<elasticsearch.version>0.90.2</elasticsearch.version>
</properties>
<repositories>

10
src/main/java/org/wltea/analyzer/core/CJKSegmenter.java
View File

@ -25,12 +25,12 @@
*/
package org.wltea.analyzer.core;
import java.util.LinkedList;
import java.util.List;
import org.wltea.analyzer.dic.Dictionary;
import org.wltea.analyzer.dic.Hit;
import java.util.LinkedList;
import java.util.List;
/**
* 中文-日韩文子分词器
@ -58,7 +58,7 @@ class CJKSegmenter implements ISegmenter {
//处理词段队列
Hit[] tmpArray = this.tmpHits.toArray(new Hit[this.tmpHits.size()]);
for(Hit hit : tmpArray){
hit = Dictionary.getSingleton().matchWithHit(context.getSegmentBuff(), context.getCursor() , hit);
hit = Dictionary.getSingleton().matchWithHit(String.valueOf(context.getSegmentBuff()).toLowerCase().toCharArray(), context.getCursor() , hit);
if(hit.isMatch()){
//输出当前的词
Lexeme newLexeme = new Lexeme(context.getBufferOffset() , hit.getBegin() , context.getCursor() - hit.getBegin() + 1 , Lexeme.TYPE_CNWORD);
@ -77,7 +77,7 @@ class CJKSegmenter implements ISegmenter {
//*********************************
//再对当前指针位置的字符进行单字匹配
Hit singleCharHit = Dictionary.getSingleton().matchInMainDict(context.getSegmentBuff(), context.getCursor(), 1);
Hit singleCharHit = Dictionary.getSingleton().matchInMainDict(String.valueOf(context.getSegmentBuff()).toLowerCase().toCharArray(), context.getCursor(), 1);
if(singleCharHit.isMatch()){//首字成词
//输出当前的词
Lexeme newLexeme = new Lexeme(context.getBufferOffset() , context.getCursor() , 1 , Lexeme.TYPE_CNWORD);

10
src/main/java/org/wltea/analyzer/core/IKSegmenter.java
View File

@ -23,16 +23,16 @@
*/
package org.wltea.analyzer.core;
import java.io.IOException;
import java.io.Reader;
import java.util.ArrayList;
import java.util.List;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.env.Environment;
import org.wltea.analyzer.cfg.Configuration;
import org.wltea.analyzer.dic.Dictionary;
import java.io.IOException;
import java.io.Reader;
import java.util.ArrayList;
import java.util.List;
/**
* IK分词器主类
*

4
src/main/java/org/wltea/analyzer/dic/DictSegment.java
View File

@ -114,8 +114,8 @@ class DictSegment implements Comparable<DictSegment>{
}
//设置hit的当前处理位置
searchHit.setEnd(begin);
Character keyChar = new Character(charArray[begin]);
Character keyChar = new Character(charArray[begin]);
DictSegment ds = null;
//引用实例变量为本地变量，避免查询时遇到更新的同步问题

6
src/main/java/org/wltea/analyzer/dic/Dictionary.java
View File

@ -152,7 +152,7 @@ public class Dictionary {
* @return Hit 匹配结果描述
*/
public Hit matchInMainDict(char[] charArray , int begin, int length){
return singleton._MainDict.match(String.valueOf(charArray).trim().toLowerCase().toCharArray(), begin, length);
return singleton._MainDict.match(String.valueOf(charArray).toLowerCase().toCharArray(), begin, length);
}
/**
@ -160,7 +160,7 @@ public class Dictionary {
* @return Hit 匹配结果描述
*/
public Hit matchInQuantifierDict(char[] charArray , int begin, int length){
return singleton._QuantifierDict.match(String.valueOf(charArray).trim().toLowerCase().toCharArray(), begin, length);
return singleton._QuantifierDict.match(String.valueOf(charArray).toLowerCase().toCharArray(), begin, length);
}
@ -179,7 +179,7 @@ public class Dictionary {
* @return boolean
*/
public boolean isStopWord(char[] charArray , int begin, int length){
return singleton._StopWords.match(String.valueOf(charArray).trim().toLowerCase().toCharArray(), begin, length).isMatch();
return singleton._StopWords.match(String.valueOf(charArray).toLowerCase().toCharArray(), begin, length).isMatch();
}
/**

8
src/main/java/org/wltea/analyzer/lucene/IKTokenizer.java
View File

@ -26,9 +26,6 @@
*/
package org.wltea.analyzer.lucene;
import java.io.IOException;
import java.io.Reader;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
@ -38,6 +35,9 @@ import org.elasticsearch.env.Environment;
import org.wltea.analyzer.core.IKSegmenter;
import org.wltea.analyzer.core.Lexeme;
import java.io.IOException;
import java.io.Reader;
/**
* IK分词器 Lucene Tokenizer适配器类
* 兼容Lucene 4.0版本
@ -80,7 +80,7 @@ public final class IKTokenizer extends Tokenizer {
if(nextLexeme != null){
//将Lexeme转成Attributes
//设置词元文本
termAtt.append(nextLexeme.getLexemeText());
termAtt.append(nextLexeme.getLexemeText().toLowerCase());
//设置词元长度
termAtt.setLength(nextLexeme.getLength());
//设置词元位移

16
src/main/java/org/wltea/analyzer/query/IKQueryExpressionParser.java
View File

@ -24,20 +24,16 @@
*/
package org.wltea.analyzer.query;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.*;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.util.BytesRef;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import java.util.Stack;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TermRangeQuery;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.util.BytesRef;
/**
* IK简易查询表达式解析
* 结合SWMCQuery算法
@ -66,7 +62,7 @@ public class IKQueryExpressionParser {
*/
public Query parseExp(String expression , boolean quickMode){
Query lucenceQuery = null;
if(expression != null && !"".equals(expression.trim())){
if(expression != null && !"".equals(expression)){
try{
//文法解析
this.splitElements(expression);