mirror of
https://github.com/yanyiwu/cppjieba.git
synced 2025-07-18 00:00:12 +08:00
big change: add RuneStr for the position of word in string
This commit is contained in:
yanyiwu
parent
abcc0af034
commit
339e3ca772
@ -10,7 +10,7 @@
|
||||
#include <limits>
|
||||
#include "limonp/StringUtil.hpp"
|
||||
#include "limonp/Logging.hpp"
|
||||
#include "TransCode.hpp"
|
||||
#include "Unicode.hpp"
|
||||
#include "Trie.hpp"
|
||||
|
||||
namespace cppjieba {
|
||||
@ -48,12 +48,12 @@ class DictTrie {
|
||||
return true;
|
||||
}
|
||||
|
||||
const DictUnit* Find(Unicode::const_iterator begin, Unicode::const_iterator end) const {
|
||||
const DictUnit* Find(unicode::RuneStrArray::const_iterator begin, unicode::RuneStrArray::const_iterator end) const {
|
||||
return trie_->Find(begin, end);
|
||||
}
|
||||
|
||||
void Find(Unicode::const_iterator begin,
|
||||
Unicode::const_iterator end,
|
||||
void Find(unicode::RuneStrArray::const_iterator begin,
|
||||
unicode::RuneStrArray::const_iterator end,
|
||||
vector<struct Dag>&res,
|
||||
size_t max_word_len = MAX_WORD_LENGTH) const {
|
||||
trie_->Find(begin, end, res, max_word_len);
|
||||
@ -124,7 +124,7 @@ class DictTrie {
|
||||
const string& word,
|
||||
double weight,
|
||||
const string& tag) {
|
||||
if (!TransCode::Decode(word, node_info.word)) {
|
||||
if (!unicode::DecodeRunesInString(word, node_info.word)) {
|
||||
XLOG(ERROR) << "Decode " << word << " failed.";
|
||||
return false;
|
||||
}
|
||||
|
||||
@ -7,7 +7,7 @@
|
||||
#include "limonp/Logging.hpp"
|
||||
#include "DictTrie.hpp"
|
||||
#include "SegmentBase.hpp"
|
||||
#include "TransCode.hpp"
|
||||
#include "Unicode.hpp"
|
||||
|
||||
namespace cppjieba {
|
||||
class FullSegment: public SegmentBase {
|
||||
@ -29,17 +29,19 @@ class FullSegment: public SegmentBase {
|
||||
vector<string>& words) const {
|
||||
PreFilter pre_filter(symbols_, sentence);
|
||||
PreFilter::Range range;
|
||||
vector<Unicode> uwords;
|
||||
uwords.reserve(sentence.size());
|
||||
vector<unicode::WordRange> wrs;
|
||||
wrs.reserve(sentence.size()/2);
|
||||
while (pre_filter.HasNext()) {
|
||||
range = pre_filter.Next();
|
||||
Cut(range.begin, range.end, uwords);
|
||||
Cut(range.begin, range.end, wrs);
|
||||
}
|
||||
TransCode::Encode(uwords, words);
|
||||
words.clear();
|
||||
words.reserve(wrs.size());
|
||||
unicode::GetStringsFromWordRanges(wrs, words);
|
||||
}
|
||||
void Cut(Unicode::const_iterator begin,
|
||||
Unicode::const_iterator end,
|
||||
vector<Unicode>& res) const {
|
||||
void Cut(unicode::RuneStrArray::const_iterator begin,
|
||||
unicode::RuneStrArray::const_iterator end,
|
||||
vector<unicode::WordRange>& res) const {
|
||||
//resut of searching in trie tree
|
||||
LocalVector<pair<size_t, const DictUnit*> > tRes;
|
||||
|
||||
@ -56,15 +58,19 @@ class FullSegment: public SegmentBase {
|
||||
dictTrie_->Find(begin, end, dags);
|
||||
for (size_t i = 0; i < dags.size(); i++) {
|
||||
for (size_t j = 0; j < dags[i].nexts.size(); j++) {
|
||||
size_t nextoffset = dags[i].nexts[j].first;
|
||||
assert(nextoffset < dags.size());
|
||||
const DictUnit* du = dags[i].nexts[j].second;
|
||||
if (du == NULL) {
|
||||
if (dags[i].nexts.size() == 1 && maxIdx <= uIdx) {
|
||||
res.push_back(Unicode(1, dags[i].rune));
|
||||
unicode::WordRange wr = {begin + i, begin + nextoffset};
|
||||
res.push_back(wr);
|
||||
}
|
||||
} else {
|
||||
wordLen = du->word.size();
|
||||
if (wordLen >= 2 || (dags[i].nexts.size() == 1 && maxIdx <= uIdx)) {
|
||||
res.push_back(du->word);
|
||||
unicode::WordRange wr = {begin + i, begin + nextoffset};
|
||||
res.push_back(wr);
|
||||
}
|
||||
}
|
||||
maxIdx = uIdx + wordLen > maxIdx ? uIdx + wordLen : maxIdx;
|
||||
|
||||
@ -105,7 +105,7 @@ struct HMMModel {
|
||||
XLOG(ERROR) << "emitProb illegal.";
|
||||
return false;
|
||||
}
|
||||
if (!TransCode::Decode(tmp2[0], unicode) || unicode.size() != 1) {
|
||||
if (!unicode::DecodeRunesInString(tmp2[0], unicode) || unicode.size() != 1) {
|
||||
XLOG(ERROR) << "TransCode failed.";
|
||||
return false;
|
||||
}
|
||||
|
||||
@ -27,19 +27,21 @@ class HMMSegment: public SegmentBase {
|
||||
vector<string>& words) const {
|
||||
PreFilter pre_filter(symbols_, sentence);
|
||||
PreFilter::Range range;
|
||||
vector<Unicode> uwords;
|
||||
uwords.reserve(sentence.size());
|
||||
vector<unicode::WordRange> wrs;
|
||||
wrs.reserve(sentence.size()/2);
|
||||
while (pre_filter.HasNext()) {
|
||||
range = pre_filter.Next();
|
||||
Cut(range.begin, range.end, uwords);
|
||||
Cut(range.begin, range.end, wrs);
|
||||
}
|
||||
TransCode::Encode(uwords, words);
|
||||
words.clear();
|
||||
words.reserve(wrs.size());
|
||||
unicode::GetStringsFromWordRanges(wrs, words);
|
||||
}
|
||||
void Cut(Unicode::const_iterator begin, Unicode::const_iterator end, vector<Unicode>& res) const {
|
||||
Unicode::const_iterator left = begin;
|
||||
Unicode::const_iterator right = begin;
|
||||
void Cut(unicode::RuneStrArray::const_iterator begin, unicode::RuneStrArray::const_iterator end, vector<unicode::WordRange>& res) const {
|
||||
unicode::RuneStrArray::const_iterator left = begin;
|
||||
unicode::RuneStrArray::const_iterator right = begin;
|
||||
while (right != end) {
|
||||
if (*right < 0x80) {
|
||||
if (right->rune < 0x80) {
|
||||
if (left != right) {
|
||||
InternalCut(left, right, res);
|
||||
}
|
||||
@ -55,7 +57,8 @@ class HMMSegment: public SegmentBase {
|
||||
}
|
||||
right ++;
|
||||
} while (false);
|
||||
res.push_back(Unicode(left, right));
|
||||
unicode::WordRange wr = {left, right - 1};
|
||||
res.push_back(wr);
|
||||
left = right;
|
||||
} else {
|
||||
right++;
|
||||
@ -67,15 +70,15 @@ class HMMSegment: public SegmentBase {
|
||||
}
|
||||
private:
|
||||
// sequential letters rule
|
||||
Unicode::const_iterator SequentialLetterRule(Unicode::const_iterator begin, Unicode::const_iterator end) const {
|
||||
Rune x = *begin;
|
||||
unicode::RuneStrArray::const_iterator SequentialLetterRule(unicode::RuneStrArray::const_iterator begin, unicode::RuneStrArray::const_iterator end) const {
|
||||
Rune x = begin->rune;
|
||||
if (('a' <= x && x <= 'z') || ('A' <= x && x <= 'Z')) {
|
||||
begin ++;
|
||||
} else {
|
||||
return begin;
|
||||
}
|
||||
while (begin != end) {
|
||||
x = *begin;
|
||||
x = begin->rune;
|
||||
if (('a' <= x && x <= 'z') || ('A' <= x && x <= 'Z') || ('0' <= x && x <= '9')) {
|
||||
begin ++;
|
||||
} else {
|
||||
@ -85,15 +88,15 @@ class HMMSegment: public SegmentBase {
|
||||
return begin;
|
||||
}
|
||||
//
|
||||
Unicode::const_iterator NumbersRule(Unicode::const_iterator begin, Unicode::const_iterator end) const {
|
||||
Rune x = *begin;
|
||||
unicode::RuneStrArray::const_iterator NumbersRule(unicode::RuneStrArray::const_iterator begin, unicode::RuneStrArray::const_iterator end) const {
|
||||
Rune x = begin->rune;
|
||||
if ('0' <= x && x <= '9') {
|
||||
begin ++;
|
||||
} else {
|
||||
return begin;
|
||||
}
|
||||
while (begin != end) {
|
||||
x = *begin;
|
||||
x = begin->rune;
|
||||
if ( ('0' <= x && x <= '9') || x == '.') {
|
||||
begin++;
|
||||
} else {
|
||||
@ -102,23 +105,24 @@ class HMMSegment: public SegmentBase {
|
||||
}
|
||||
return begin;
|
||||
}
|
||||
void InternalCut(Unicode::const_iterator begin, Unicode::const_iterator end, vector<Unicode>& res) const {
|
||||
void InternalCut(unicode::RuneStrArray::const_iterator begin, unicode::RuneStrArray::const_iterator end, vector<unicode::WordRange>& res) const {
|
||||
vector<size_t> status;
|
||||
Viterbi(begin, end, status);
|
||||
|
||||
Unicode::const_iterator left = begin;
|
||||
Unicode::const_iterator right;
|
||||
unicode::RuneStrArray::const_iterator left = begin;
|
||||
unicode::RuneStrArray::const_iterator right;
|
||||
for (size_t i = 0; i < status.size(); i++) {
|
||||
if (status[i] % 2) { //if (HMMModel::E == status[i] || HMMModel::S == status[i])
|
||||
right = begin + i + 1;
|
||||
res.push_back(Unicode(left, right));
|
||||
unicode::WordRange wr = {left, right - 1};
|
||||
res.push_back(wr);
|
||||
left = right;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void Viterbi(Unicode::const_iterator begin,
|
||||
Unicode::const_iterator end,
|
||||
void Viterbi(unicode::RuneStrArray::const_iterator begin,
|
||||
unicode::RuneStrArray::const_iterator end,
|
||||
vector<size_t>& status) const {
|
||||
size_t Y = HMMModel::STATUS_SUM;
|
||||
size_t X = end - begin;
|
||||
@ -132,7 +136,7 @@ class HMMSegment: public SegmentBase {
|
||||
|
||||
//start
|
||||
for (size_t y = 0; y < Y; y++) {
|
||||
weight[0 + y * X] = model_->startProb[y] + model_->GetEmitProb(model_->emitProbVec[y], *begin, MIN_DOUBLE);
|
||||
weight[0 + y * X] = model_->startProb[y] + model_->GetEmitProb(model_->emitProbVec[y], begin->rune, MIN_DOUBLE);
|
||||
path[0 + y * X] = -1;
|
||||
}
|
||||
|
||||
@ -143,7 +147,7 @@ class HMMSegment: public SegmentBase {
|
||||
now = x + y*X;
|
||||
weight[now] = MIN_DOUBLE;
|
||||
path[now] = HMMModel::E; // warning
|
||||
emitProb = model_->GetEmitProb(model_->emitProbVec[y], *(begin+x), MIN_DOUBLE);
|
||||
emitProb = model_->GetEmitProb(model_->emitProbVec[y], (begin+x)->rune, MIN_DOUBLE);
|
||||
for (size_t preY = 0; preY < Y; preY++) {
|
||||
old = x - 1 + preY * X;
|
||||
tmp = weight[old] + model_->transProb[preY][y] + emitProb;
|
||||
|
||||
@ -3,7 +3,7 @@
|
||||
|
||||
#include "QuerySegment.hpp"
|
||||
#include "PosTagger.hpp"
|
||||
#include "LevelSegment.hpp"
|
||||
//#include "LevelSegment.hpp"
|
||||
|
||||
namespace cppjieba {
|
||||
|
||||
@ -17,7 +17,7 @@ class Jieba {
|
||||
mix_seg_(&dict_trie_, &model_),
|
||||
full_seg_(&dict_trie_),
|
||||
query_seg_(&dict_trie_, &model_),
|
||||
level_seg_(&dict_trie_),
|
||||
//level_seg_(&dict_trie_),
|
||||
pos_tagger_(&dict_trie_, &model_) {
|
||||
}
|
||||
~Jieba() {
|
||||
@ -41,26 +41,26 @@ class Jieba {
|
||||
void CutHMM(const string& sentence, vector<string>& words) const {
|
||||
hmm_seg_.Cut(sentence, words);
|
||||
}
|
||||
void CutLevel(const string& sentence, vector<string>& words) const {
|
||||
level_seg_.Cut(sentence, words);
|
||||
}
|
||||
void CutLevel(const string& sentence, vector<pair<string, size_t> >& words) const {
|
||||
level_seg_.Cut(sentence, words);
|
||||
}
|
||||
//void CutLevel(const string& sentence, vector<string>& words) const {
|
||||
// level_seg_.Cut(sentence, words);
|
||||
//}
|
||||
//void CutLevel(const string& sentence, vector<pair<string, size_t> >& words) const {
|
||||
// level_seg_.Cut(sentence, words);
|
||||
//}
|
||||
void CutSmall(const string& sentence, vector<string>& words, size_t max_word_len) const {
|
||||
mp_seg_.Cut(sentence, words, max_word_len);
|
||||
}
|
||||
static void Locate(const vector<string>& words, vector<LocWord>& loc_words) {
|
||||
loc_words.resize(words.size());
|
||||
size_t begin = 0;
|
||||
for (size_t i = 0; i < words.size(); i++) {
|
||||
size_t len = TransCode::Decode(words[i]).size();
|
||||
loc_words[i].word = words[i];
|
||||
loc_words[i].begin = begin;
|
||||
loc_words[i].end = loc_words[i].begin + len;
|
||||
begin = loc_words[i].end;
|
||||
}
|
||||
}
|
||||
//static void Locate(const vector<string>& words, vector<LocWord>& loc_words) {
|
||||
// loc_words.resize(words.size());
|
||||
// size_t begin = 0;
|
||||
// for (size_t i = 0; i < words.size(); i++) {
|
||||
// size_t len = TransCode::Decode(words[i]).size();
|
||||
// loc_words[i].word = words[i];
|
||||
// loc_words[i].begin = begin;
|
||||
// loc_words[i].end = loc_words[i].begin + len;
|
||||
// begin = loc_words[i].end;
|
||||
// }
|
||||
//}
|
||||
|
||||
void Tag(const string& sentence, vector<pair<string, string> >& words) const {
|
||||
pos_tagger_.Tag(sentence, words);
|
||||
@ -89,7 +89,7 @@ class Jieba {
|
||||
MixSegment mix_seg_;
|
||||
FullSegment full_seg_;
|
||||
QuerySegment query_seg_;
|
||||
LevelSegment level_seg_;
|
||||
//LevelSegment level_seg_;
|
||||
|
||||
PosTagger pos_tagger_;
|
||||
|
||||
|
||||
@ -69,7 +69,7 @@ class KeywordExtractor {
|
||||
for (size_t i = 0; i < words.size(); ++i) {
|
||||
size_t t = offset;
|
||||
offset += words[i].size();
|
||||
if (IsSingleWord(words[i]) || stopWords_.find(words[i]) != stopWords_.end()) {
|
||||
if (unicode::IsSingleWord(words[i]) || stopWords_.find(words[i]) != stopWords_.end()) {
|
||||
continue;
|
||||
}
|
||||
wordmap[words[i]].offsets.push_back(t);
|
||||
@ -136,14 +136,6 @@ class KeywordExtractor {
|
||||
assert(stopWords_.size());
|
||||
}
|
||||
|
||||
bool IsSingleWord(const string& str) const {
|
||||
Unicode unicode;
|
||||
TransCode::Decode(str, unicode);
|
||||
if (unicode.size() == 1)
|
||||
return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
static bool Compare(const Word& lhs, const Word& rhs) {
|
||||
return lhs.weight > rhs.weight;
|
||||
}
|
||||
|
||||
@ -17,9 +17,9 @@ class LevelSegment: public SegmentBase{
|
||||
~LevelSegment() {
|
||||
}
|
||||
|
||||
void Cut(Unicode::const_iterator begin,
|
||||
Unicode::const_iterator end,
|
||||
vector<pair<Unicode, size_t> >& res) const {
|
||||
void Cut(unicode::RuneStrArray::const_iterator begin,
|
||||
unicode::RuneStrArray::const_iterator end,
|
||||
vector<pair<WordRange, size_t> >& res) const {
|
||||
res.clear();
|
||||
vector<Unicode> words;
|
||||
vector<Unicode> smallerWords;
|
||||
@ -49,9 +49,9 @@ class LevelSegment: public SegmentBase{
|
||||
void Cut(const string& sentence,
|
||||
vector<pair<string, size_t> >& words) const {
|
||||
words.clear();
|
||||
Unicode unicode;
|
||||
TransCode::Decode(sentence, unicode);
|
||||
vector<pair<Unicode, size_t> > unicodeWords;
|
||||
RuneStrArray unicode;
|
||||
unicode::DecodeRunesInString(sentence, unicode);
|
||||
vector<pair<WordRange, size_t> > unicodeWords;
|
||||
Cut(unicode.begin(), unicode.end(), unicodeWords);
|
||||
words.resize(unicodeWords.size());
|
||||
for (size_t i = 0; i < words.size(); i++) {
|
||||
|
||||
@ -30,17 +30,19 @@ class MPSegment: public SegmentBase {
|
||||
size_t max_word_len = MAX_WORD_LENGTH) const {
|
||||
PreFilter pre_filter(symbols_, sentence);
|
||||
PreFilter::Range range;
|
||||
vector<Unicode> uwords;
|
||||
uwords.reserve(sentence.size());
|
||||
vector<unicode::WordRange> wrs;
|
||||
wrs.reserve(sentence.size()/2);
|
||||
while (pre_filter.HasNext()) {
|
||||
range = pre_filter.Next();
|
||||
Cut(range.begin, range.end, uwords, max_word_len);
|
||||
Cut(range.begin, range.end, wrs, max_word_len);
|
||||
}
|
||||
TransCode::Encode(uwords, words);
|
||||
words.clear();
|
||||
words.reserve(wrs.size());
|
||||
unicode::GetStringsFromWordRanges(wrs, words);
|
||||
}
|
||||
void Cut(Unicode::const_iterator begin,
|
||||
Unicode::const_iterator end,
|
||||
vector<Unicode>& words,
|
||||
void Cut(unicode::RuneStrArray::const_iterator begin,
|
||||
unicode::RuneStrArray::const_iterator end,
|
||||
vector<unicode::WordRange>& words,
|
||||
size_t max_word_len = MAX_WORD_LENGTH) const {
|
||||
vector<Dag> dags;
|
||||
dictTrie_->Find(begin,
|
||||
@ -48,7 +50,7 @@ class MPSegment: public SegmentBase {
|
||||
dags,
|
||||
max_word_len);
|
||||
CalcDP(dags);
|
||||
CutByDag(dags, words);
|
||||
CutByDag(begin, end, dags, words);
|
||||
}
|
||||
|
||||
const DictTrie* GetDictTrie() const {
|
||||
@ -88,16 +90,21 @@ class MPSegment: public SegmentBase {
|
||||
}
|
||||
}
|
||||
}
|
||||
void CutByDag(const vector<Dag>& dags,
|
||||
vector<Unicode>& words) const {
|
||||
void CutByDag(unicode::RuneStrArray::const_iterator begin,
|
||||
unicode::RuneStrArray::const_iterator end,
|
||||
const vector<Dag>& dags,
|
||||
vector<unicode::WordRange>& words) const {
|
||||
size_t i = 0;
|
||||
while (i < dags.size()) {
|
||||
const DictUnit* p = dags[i].pInfo;
|
||||
if (p) {
|
||||
words.push_back(p->word);
|
||||
assert(p->word.size() >= 1);
|
||||
unicode::WordRange wr = {begin + i, begin + i + p->word.size() - 1};
|
||||
words.push_back(wr);
|
||||
i += p->word.size();
|
||||
} else { //single chinese word
|
||||
words.push_back(Unicode(1, dags[i].rune));
|
||||
unicode::WordRange wr = {begin + i, begin + i};
|
||||
words.push_back(wr);
|
||||
i++;
|
||||
}
|
||||
}
|
||||
|
||||
@ -23,52 +23,52 @@ class MixSegment: public SegmentBase {
|
||||
void Cut(const string& sentence, vector<string>& words, bool hmm = true) const {
|
||||
PreFilter pre_filter(symbols_, sentence);
|
||||
PreFilter::Range range;
|
||||
vector<Unicode> uwords;
|
||||
uwords.reserve(sentence.size());
|
||||
vector<unicode::WordRange> wrs;
|
||||
wrs.reserve(sentence.size() / 2);
|
||||
while (pre_filter.HasNext()) {
|
||||
range = pre_filter.Next();
|
||||
Cut(range.begin, range.end, uwords, hmm);
|
||||
Cut(range.begin, range.end, wrs, hmm);
|
||||
}
|
||||
TransCode::Encode(uwords, words);
|
||||
words.clear();
|
||||
words.reserve(wrs.size());
|
||||
unicode::GetStringsFromWordRanges(wrs, words);
|
||||
}
|
||||
|
||||
void Cut(Unicode::const_iterator begin, Unicode::const_iterator end, vector<Unicode>& res, bool hmm) const {
|
||||
void Cut(unicode::RuneStrArray::const_iterator begin, unicode::RuneStrArray::const_iterator end, vector<unicode::WordRange>& res, bool hmm) const {
|
||||
if (!hmm) {
|
||||
mpSeg_.Cut(begin, end, res);
|
||||
return;
|
||||
}
|
||||
vector<Unicode> words;
|
||||
vector<unicode::WordRange> words;
|
||||
assert(end >= begin);
|
||||
words.reserve(end - begin);
|
||||
mpSeg_.Cut(begin, end, words);
|
||||
|
||||
vector<Unicode> hmmRes;
|
||||
vector<unicode::WordRange> hmmRes;
|
||||
hmmRes.reserve(end - begin);
|
||||
Unicode piece;
|
||||
piece.reserve(end - begin);
|
||||
for (size_t i = 0, j = 0; i < words.size(); i++) {
|
||||
for (size_t i = 0; i < words.size(); i++) {
|
||||
//if mp Get a word, it's ok, put it into result
|
||||
if (1 != words[i].size() || (words[i].size() == 1 && mpSeg_.IsUserDictSingleChineseWord(words[i][0]))) {
|
||||
if (words[i].left != words[i].right || (words[i].left == words[i].right && mpSeg_.IsUserDictSingleChineseWord(words[i].left->rune))) {
|
||||
res.push_back(words[i]);
|
||||
continue;
|
||||
}
|
||||
|
||||
// if mp Get a single one and it is not in userdict, collect it in sequence
|
||||
j = i;
|
||||
while (j < words.size() && 1 == words[j].size() && !mpSeg_.IsUserDictSingleChineseWord(words[j][0])) {
|
||||
piece.push_back(words[j][0]);
|
||||
size_t j = i;
|
||||
while (j < words.size() && words[j].left == words[j].right && !mpSeg_.IsUserDictSingleChineseWord(words[j].left->rune)) {
|
||||
j++;
|
||||
}
|
||||
|
||||
// Cut the sequence with hmm
|
||||
hmmSeg_.Cut(piece.begin(), piece.end(), hmmRes);
|
||||
|
||||
assert(j - 1 >= i);
|
||||
// TODO
|
||||
hmmSeg_.Cut(words[i].left, words[j - 1].left + 1, hmmRes);
|
||||
//put hmm result to result
|
||||
for (size_t k = 0; k < hmmRes.size(); k++) {
|
||||
res.push_back(hmmRes[k]);
|
||||
}
|
||||
|
||||
//clear tmp vars
|
||||
piece.clear();
|
||||
hmmRes.clear();
|
||||
|
||||
//let i jump over this piece
|
||||
|
||||
@ -30,17 +30,17 @@ class PosTagger {
|
||||
segment_.Cut(src, CutRes);
|
||||
|
||||
const DictUnit *tmp = NULL;
|
||||
Unicode unico;
|
||||
unicode::RuneStrArray runes;
|
||||
const DictTrie * dict = segment_.GetDictTrie();
|
||||
assert(dict != NULL);
|
||||
for (vector<string>::iterator itr = CutRes.begin(); itr != CutRes.end(); ++itr) {
|
||||
if (!TransCode::Decode(*itr, unico)) {
|
||||
if (!unicode::DecodeRunesInString(*itr, runes)) {
|
||||
XLOG(ERROR) << "Decode failed.";
|
||||
return false;
|
||||
}
|
||||
tmp = dict->Find(unico.begin(), unico.end());
|
||||
tmp = dict->Find(runes.begin(), runes.end());
|
||||
if (tmp == NULL || tmp->tag.empty()) {
|
||||
res.push_back(make_pair(*itr, SpecialRule(unico)));
|
||||
res.push_back(make_pair(*itr, SpecialRule(runes)));
|
||||
} else {
|
||||
res.push_back(make_pair(*itr, tmp->tag));
|
||||
}
|
||||
@ -48,13 +48,13 @@ class PosTagger {
|
||||
return !res.empty();
|
||||
}
|
||||
private:
|
||||
const char* SpecialRule(const Unicode& unicode) const {
|
||||
const char* SpecialRule(const unicode::RuneStrArray& unicode) const {
|
||||
size_t m = 0;
|
||||
size_t eng = 0;
|
||||
for (size_t i = 0; i < unicode.size() && eng < unicode.size() / 2; i++) {
|
||||
if (unicode[i] < 0x80) {
|
||||
if (unicode[i].rune < 0x80) {
|
||||
eng ++;
|
||||
if ('0' <= unicode[i] && unicode[i] <= '9') {
|
||||
if ('0' <= unicode[i].rune && unicode[i].rune <= '9') {
|
||||
m++;
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,32 +1,22 @@
|
||||
#ifndef CPPJIEBA_PRE_FILTER_H
|
||||
#define CPPJIEBA_PRE_FILTER_H
|
||||
|
||||
#include "TransCode.hpp"
|
||||
#include "Trie.hpp"
|
||||
|
||||
namespace cppjieba {
|
||||
|
||||
//class PreFilterIterator {
|
||||
// public:
|
||||
// PreFilterIterator() {
|
||||
// }
|
||||
// ~PreFilterIterator() {
|
||||
// }
|
||||
//
|
||||
// private:
|
||||
// const unordered_set<Rune>& specialSymbols_;
|
||||
//}; // PreFilterIterator
|
||||
|
||||
class PreFilter {
|
||||
public:
|
||||
//TODO use WordRange instead of Range
|
||||
struct Range {
|
||||
Unicode::const_iterator begin;
|
||||
Unicode::const_iterator end;
|
||||
unicode::RuneStrArray::const_iterator begin;
|
||||
unicode::RuneStrArray::const_iterator end;
|
||||
}; // struct Range
|
||||
|
||||
PreFilter(const unordered_set<Rune>& symbols,
|
||||
const string& sentence)
|
||||
: symbols_(symbols) {
|
||||
TransCode::Decode(sentence, sentence_);
|
||||
unicode::DecodeRunesInString(sentence, sentence_);
|
||||
cursor_ = sentence_.begin();
|
||||
}
|
||||
~PreFilter() {
|
||||
@ -38,7 +28,7 @@ class PreFilter {
|
||||
Range range;
|
||||
range.begin = cursor_;
|
||||
while (cursor_ != sentence_.end()) {
|
||||
if (IsIn(symbols_, *cursor_)) {
|
||||
if (IsIn(symbols_, cursor_->rune)) {
|
||||
if (range.begin == cursor_) {
|
||||
cursor_ ++;
|
||||
}
|
||||
@ -51,8 +41,8 @@ class PreFilter {
|
||||
return range;
|
||||
}
|
||||
private:
|
||||
Unicode::const_iterator cursor_;
|
||||
Unicode sentence_;
|
||||
unicode::RuneStrArray::const_iterator cursor_;
|
||||
unicode::RuneStrArray sentence_;
|
||||
const unordered_set<Rune>& symbols_;
|
||||
}; // class PreFilter
|
||||
|
||||
|
||||
@ -9,7 +9,7 @@
|
||||
#include "SegmentBase.hpp"
|
||||
#include "FullSegment.hpp"
|
||||
#include "MixSegment.hpp"
|
||||
#include "TransCode.hpp"
|
||||
#include "Unicode.hpp"
|
||||
#include "DictTrie.hpp"
|
||||
|
||||
namespace cppjieba {
|
||||
@ -29,25 +29,27 @@ class QuerySegment: public SegmentBase {
|
||||
void Cut(const string& sentence, vector<string>& words, bool hmm = true) const {
|
||||
PreFilter pre_filter(symbols_, sentence);
|
||||
PreFilter::Range range;
|
||||
vector<Unicode> uwords;
|
||||
uwords.reserve(sentence.size());
|
||||
vector<unicode::WordRange> wrs;
|
||||
wrs.reserve(sentence.size()/2);
|
||||
while (pre_filter.HasNext()) {
|
||||
range = pre_filter.Next();
|
||||
Cut(range.begin, range.end, uwords, hmm);
|
||||
Cut(range.begin, range.end, wrs, hmm);
|
||||
}
|
||||
TransCode::Encode(uwords, words);
|
||||
words.clear();
|
||||
words.reserve(wrs.size());
|
||||
unicode::GetStringsFromWordRanges(wrs, words);
|
||||
}
|
||||
void Cut(Unicode::const_iterator begin, Unicode::const_iterator end, vector<Unicode>& res, bool hmm) const {
|
||||
void Cut(unicode::RuneStrArray::const_iterator begin, unicode::RuneStrArray::const_iterator end, vector<unicode::WordRange>& res, bool hmm) const {
|
||||
//use mix Cut first
|
||||
vector<Unicode> mixRes;
|
||||
vector<unicode::WordRange> mixRes;
|
||||
mixSeg_.Cut(begin, end, mixRes, hmm);
|
||||
|
||||
vector<Unicode> fullRes;
|
||||
for (vector<Unicode>::const_iterator mixResItr = mixRes.begin(); mixResItr != mixRes.end(); mixResItr++) {
|
||||
vector<unicode::WordRange> fullRes;
|
||||
for (vector<unicode::WordRange>::const_iterator mixResItr = mixRes.begin(); mixResItr != mixRes.end(); mixResItr++) {
|
||||
// if it's too long, Cut with fullSeg_, put fullRes in res
|
||||
if (mixResItr->size() > maxWordLen_ && !IsAllAscii(*mixResItr)) {
|
||||
fullSeg_.Cut(mixResItr->begin(), mixResItr->end(), fullRes);
|
||||
for (vector<Unicode>::const_iterator fullResItr = fullRes.begin(); fullResItr != fullRes.end(); fullResItr++) {
|
||||
if (mixResItr->Length() > maxWordLen_ && !mixResItr->IsAllAscii()) {
|
||||
fullSeg_.Cut(mixResItr->left, mixResItr->right + 1, fullRes);
|
||||
for (vector<unicode::WordRange>::const_iterator fullResItr = fullRes.begin(); fullResItr != fullRes.end(); fullResItr++) {
|
||||
res.push_back(*fullResItr);
|
||||
}
|
||||
|
||||
|
||||
@ -1,70 +0,0 @@
|
||||
/************************************
|
||||
* file enc : utf-8
|
||||
* author : wuyanyi09@gmail.com
|
||||
************************************/
|
||||
#ifndef CPPJIEBA_TRANSCODE_H
|
||||
#define CPPJIEBA_TRANSCODE_H
|
||||
|
||||
|
||||
#include "limonp/StringUtil.hpp"
|
||||
#include "limonp/LocalVector.hpp"
|
||||
|
||||
namespace cppjieba {
|
||||
|
||||
using namespace limonp;
|
||||
|
||||
typedef uint32_t Rune;
|
||||
typedef limonp::LocalVector<Rune> Unicode;
|
||||
|
||||
namespace TransCode {
|
||||
inline bool Decode(const string& str, Unicode& res) {
|
||||
#ifdef CPPJIEBA_GBK
|
||||
return gbkTrans(str, res);
|
||||
#else
|
||||
return Utf8ToUnicode32(str, res);
|
||||
#endif
|
||||
}
|
||||
|
||||
inline void Encode(Unicode::const_iterator begin, Unicode::const_iterator end, string& res) {
|
||||
#ifdef CPPJIEBA_GBK
|
||||
gbkTrans(begin, end, res);
|
||||
#else
|
||||
Unicode32ToUtf8(begin, end, res);
|
||||
#endif
|
||||
}
|
||||
|
||||
inline void Encode(const Unicode& uni, string& res) {
|
||||
Encode(uni.begin(), uni.end(), res);
|
||||
}
|
||||
|
||||
// compiler is expected to optimized this function to avoid return value copy
|
||||
inline string Encode(Unicode::const_iterator begin, Unicode::const_iterator end) {
|
||||
string res;
|
||||
res.reserve(end - begin);
|
||||
Encode(begin, end, res);
|
||||
return res;
|
||||
}
|
||||
|
||||
inline string Encode(const Unicode& unicode) {
|
||||
return Encode(unicode.begin(), unicode.end());
|
||||
}
|
||||
|
||||
// compiler is expected to optimized this function to avoid return value copy
|
||||
inline Unicode Decode(const string& str) {
|
||||
Unicode unicode;
|
||||
unicode.reserve(str.size());
|
||||
Decode(str, unicode);
|
||||
return unicode;
|
||||
}
|
||||
|
||||
inline void Encode(const vector<Unicode>& input, vector<string>& output) {
|
||||
output.resize(input.size());
|
||||
for (size_t i = 0; i < output.size(); i++) {
|
||||
Encode(input[i], output[i]);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace TransCode
|
||||
} // namespace cppjieba
|
||||
|
||||
#endif
|
||||
@ -4,36 +4,41 @@
|
||||
#include <vector>
|
||||
#include <queue>
|
||||
#include "limonp/StdExtension.hpp"
|
||||
#include "Trie.hpp"
|
||||
#include "Unicode.hpp"
|
||||
|
||||
namespace cppjieba {
|
||||
|
||||
using namespace std;
|
||||
using unicode::Rune;
|
||||
using unicode::RuneStr;
|
||||
using unicode::Unicode;
|
||||
using unicode::WordRange;
|
||||
|
||||
const size_t MAX_WORD_LENGTH = 512;
|
||||
|
||||
struct DictUnit {
|
||||
Unicode word;
|
||||
unicode::Unicode word;
|
||||
double weight;
|
||||
string tag;
|
||||
};
|
||||
}; // struct DictUnit
|
||||
|
||||
// for debugging
|
||||
inline ostream & operator << (ostream& os, const DictUnit& unit) {
|
||||
string s;
|
||||
s << unit.word;
|
||||
return os << StringFormat("%s %s %.3lf", s.c_str(), unit.tag.c_str(), unit.weight);
|
||||
}
|
||||
// inline ostream & operator << (ostream& os, const DictUnit& unit) {
|
||||
// string s;
|
||||
// s << unit.word;
|
||||
// return os << StringFormat("%s %s %.3lf", s.c_str(), unit.tag.c_str(), unit.weight);
|
||||
// }
|
||||
|
||||
struct Dag {
|
||||
Rune rune;
|
||||
LocalVector<pair<size_t, const DictUnit*> > nexts;
|
||||
RuneStr runestr;
|
||||
// [offset, nexts.first]
|
||||
limonp::LocalVector<pair<size_t, const DictUnit*> > nexts;
|
||||
const DictUnit * pInfo;
|
||||
double weight;
|
||||
size_t nextPos;
|
||||
Dag():rune(0), pInfo(NULL), weight(0.0), nextPos(0) {
|
||||
size_t nextPos; // TODO
|
||||
Dag():runestr(), pInfo(NULL), weight(0.0), nextPos(0) {
|
||||
}
|
||||
};
|
||||
}; // struct Dag
|
||||
|
||||
typedef Rune TrieKey;
|
||||
|
||||
@ -57,18 +62,18 @@ class Trie {
|
||||
DeleteNode(root_);
|
||||
}
|
||||
|
||||
const DictUnit* Find(Unicode::const_iterator begin, Unicode::const_iterator end) const {
|
||||
const DictUnit* Find(unicode::RuneStrArray::const_iterator begin, unicode::RuneStrArray::const_iterator end) const {
|
||||
if (begin == end) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
const TrieNode* ptNode = root_;
|
||||
TrieNode::NextMap::const_iterator citer;
|
||||
for (Unicode::const_iterator it = begin; it != end; it++) {
|
||||
for (unicode::RuneStrArray::const_iterator it = begin; it != end; it++) {
|
||||
if (NULL == ptNode->next) {
|
||||
return NULL;
|
||||
}
|
||||
citer = ptNode->next->find(*it);
|
||||
citer = ptNode->next->find(it->rune);
|
||||
if (ptNode->next->end() == citer) {
|
||||
return NULL;
|
||||
}
|
||||
@ -77,8 +82,8 @@ class Trie {
|
||||
return ptNode->ptValue;
|
||||
}
|
||||
|
||||
void Find(Unicode::const_iterator begin,
|
||||
Unicode::const_iterator end,
|
||||
void Find(unicode::RuneStrArray::const_iterator begin,
|
||||
unicode::RuneStrArray::const_iterator end,
|
||||
vector<struct Dag>&res,
|
||||
size_t max_word_len = MAX_WORD_LENGTH) const {
|
||||
assert(root_ != NULL);
|
||||
@ -87,10 +92,9 @@ class Trie {
|
||||
const TrieNode *ptNode = NULL;
|
||||
TrieNode::NextMap::const_iterator citer;
|
||||
for (size_t i = 0; i < size_t(end - begin); i++) {
|
||||
Rune rune = *(begin + i);
|
||||
res[i].rune = rune;
|
||||
res[i].runestr = *(begin + i);
|
||||
|
||||
if (root_->next != NULL && root_->next->end() != (citer = root_->next->find(rune))) {
|
||||
if (root_->next != NULL && root_->next->end() != (citer = root_->next->find(res[i].runestr.rune))) {
|
||||
ptNode = citer->second;
|
||||
} else {
|
||||
ptNode = NULL;
|
||||
@ -105,7 +109,7 @@ class Trie {
|
||||
if (ptNode == NULL || ptNode->next == NULL) {
|
||||
break;
|
||||
}
|
||||
citer = ptNode->next->find(*(begin + j));
|
||||
citer = ptNode->next->find((begin + j)->rune);
|
||||
if (ptNode->next->end() == citer) {
|
||||
break;
|
||||
}
|
||||
|
||||
182
include/cppjieba/Unicode.hpp
Normal file
182
include/cppjieba/Unicode.hpp
Normal file
@ -0,0 +1,182 @@
|
||||
#ifndef CPPJIEBA_UNICODE_H
|
||||
#define CPPJIEBA_UNICODE_H
|
||||
|
||||
#include <stdint.h>
|
||||
#include <stdlib.h>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include "limonp/LocalVector.hpp"
|
||||
|
||||
namespace cppjieba {
|
||||
namespace unicode {
|
||||
|
||||
typedef uint32_t Rune;
|
||||
|
||||
struct RuneStr {
|
||||
Rune rune;
|
||||
const char* str;
|
||||
uint32_t len;
|
||||
RuneStr(): rune(0), str(NULL), len(0) {
|
||||
}
|
||||
RuneStr(Rune r, const char* s, uint32_t l)
|
||||
: rune(r), str(s), len(l) {
|
||||
}
|
||||
}; // struct RuneStr
|
||||
|
||||
|
||||
typedef limonp::LocalVector<Rune> Unicode;
|
||||
typedef limonp::LocalVector<struct RuneStr> RuneStrArray;
|
||||
|
||||
// [left, right]
|
||||
struct WordRange {
|
||||
RuneStrArray::const_iterator left;
|
||||
RuneStrArray::const_iterator right;
|
||||
size_t Length() const {
|
||||
return right - left + 1;
|
||||
}
|
||||
bool IsAllAscii() const {
|
||||
for (RuneStrArray::const_iterator iter = left; iter <= right; ++iter) {
|
||||
if (iter->rune >= 0x80) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}; // struct WordRange
|
||||
|
||||
//struct RuneWordStr {
|
||||
// Unicode word;
|
||||
// const char* str;
|
||||
// size_t len;
|
||||
//}; // struct RuneWordStr
|
||||
|
||||
struct RuneStrLite {
|
||||
uint32_t rune;
|
||||
uint32_t len;
|
||||
}; // struct RuneStrLite
|
||||
|
||||
inline RuneStrLite DecodeRuneInString(const char* str, size_t len) {
|
||||
RuneStrLite rp = {0, 0};
|
||||
if (str == NULL || len == 0) {
|
||||
return rp;
|
||||
}
|
||||
if (!(str[0] & 0x80)) { // 0xxxxxxx
|
||||
// 7bit, total 7bit
|
||||
rp.rune = (uint8_t)(str[0]) & 0x7f;
|
||||
rp.len = 1;
|
||||
} else if ((uint8_t)str[0] <= 0xdf && 1 < len) {
|
||||
// 110xxxxxx
|
||||
// 5bit, total 5bit
|
||||
rp.rune = (uint8_t)(str[0]) & 0x1f;
|
||||
|
||||
// 6bit, total 11bit
|
||||
rp.rune <<= 6;
|
||||
rp.rune |= (uint8_t)(str[1]) & 0x3f;
|
||||
rp.len = 2;
|
||||
} else if((uint8_t)str[0] <= 0xef && 2 < len) { // 1110xxxxxx
|
||||
// 4bit, total 4bit
|
||||
rp.rune = (uint8_t)(str[0]) & 0x0f;
|
||||
|
||||
// 6bit, total 10bit
|
||||
rp.rune <<= 6;
|
||||
rp.rune |= (uint8_t)(str[1]) & 0x3f;
|
||||
|
||||
// 6bit, total 16bit
|
||||
rp.rune <<= 6;
|
||||
rp.rune |= (uint8_t)(str[2]) & 0x3f;
|
||||
|
||||
rp.len = 3;
|
||||
} else if((uint8_t)str[0] <= 0xf7 && 3 < len) { // 11110xxxx
|
||||
// 3bit, total 3bit
|
||||
rp.rune = (uint8_t)(str[0]) & 0x07;
|
||||
|
||||
// 6bit, total 9bit
|
||||
rp.rune <<= 6;
|
||||
rp.rune |= (uint8_t)(str[1]) & 0x3f;
|
||||
|
||||
// 6bit, total 15bit
|
||||
rp.rune <<= 6;
|
||||
rp.rune |= (uint8_t)(str[2]) & 0x3f;
|
||||
|
||||
// 6bit, total 21bit
|
||||
rp.rune <<= 6;
|
||||
rp.rune |= (uint8_t)(str[3]) & 0x3f;
|
||||
|
||||
rp.len = 4;
|
||||
} else {
|
||||
rp.rune = 0;
|
||||
rp.len = 0;
|
||||
}
|
||||
return rp;
|
||||
}
|
||||
|
||||
inline bool DecodeRunesInString(const char* s, size_t len, RuneStrArray& runes) {
|
||||
runes.clear();
|
||||
runes.reserve(len / 2);
|
||||
for (size_t i = 0; i < len;) {
|
||||
RuneStrLite rp = DecodeRuneInString(s + i, len - i);
|
||||
if (rp.len == 0) {
|
||||
return false;
|
||||
}
|
||||
RuneStr x = {rp.rune, s + i, rp.len};
|
||||
runes.push_back(x);
|
||||
i += rp.len;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
inline bool DecodeRunesInString(const std::string& s, RuneStrArray& runes) {
|
||||
return DecodeRunesInString(s.c_str(), s.size(), runes);
|
||||
}
|
||||
|
||||
inline bool DecodeRunesInString(const char* s, size_t len, Unicode& unicode) {
|
||||
unicode.clear();
|
||||
RuneStrArray runes;
|
||||
if (!DecodeRunesInString(s, len, runes)) {
|
||||
return false;
|
||||
}
|
||||
unicode.reserve(runes.size());
|
||||
for (size_t i = 0; i < runes.size(); i++) {
|
||||
unicode.push_back(runes[i].rune);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
inline bool IsSingleWord(const std::string& str) {
|
||||
RuneStrLite rp = DecodeRuneInString(str.c_str(), str.size());
|
||||
return rp.len == str.size();
|
||||
}
|
||||
|
||||
inline bool DecodeRunesInString(const std::string& s, Unicode& unicode) {
|
||||
return DecodeRunesInString(s.c_str(), s.size(), unicode);
|
||||
}
|
||||
|
||||
inline Unicode DecodeRunesInString(const std::string& s) {
|
||||
Unicode result;
|
||||
DecodeRunesInString(s, result);
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
//[left, right]
|
||||
inline std::string GetStringFromRunes(unicode::RuneStrArray::const_iterator left, unicode::RuneStrArray::const_iterator right) {
|
||||
assert(right->str >= left->str);
|
||||
return std::string(left->str, right->str - left->str + right->len);
|
||||
}
|
||||
|
||||
inline void GetStringsFromWordRanges(const std::vector<WordRange>& wrs, std::vector<std::string>& words) {
|
||||
for (size_t i = 0; i < wrs.size(); i++) {
|
||||
words.push_back(GetStringFromRunes(wrs[i].left, wrs[i].right));
|
||||
}
|
||||
}
|
||||
|
||||
inline std::vector<std::string> GetStringsFromWordRanges(const std::vector<WordRange>& wrs) {
|
||||
std::vector<std::string> result;
|
||||
GetStringsFromWordRanges(wrs, result);
|
||||
return result;
|
||||
}
|
||||
|
||||
} // namespace unicode
|
||||
} // namespace cppjieba
|
||||
|
||||
#endif // CPPJIEBA_UNICODE_H
|
||||
@ -40,16 +40,16 @@ int main(int argc, char** argv) {
|
||||
jieba.Cut("男默女泪", words);
|
||||
cout << limonp::Join(words.begin(), words.end(), "/") << endl;
|
||||
|
||||
cout << "[demo] Locate Words" << endl;
|
||||
vector<cppjieba::Jieba::LocWord> loc_words;
|
||||
jieba.Cut("南京市长江大桥", words, true);
|
||||
cppjieba::Jieba::Locate(words, loc_words);
|
||||
for (size_t i = 0; i < loc_words.size(); i++) {
|
||||
cout << loc_words[i].word
|
||||
<< ", " << loc_words[i].begin
|
||||
<< ", " << loc_words[i].end
|
||||
<< endl;
|
||||
}
|
||||
//cout << "[demo] Locate Words" << endl;
|
||||
//vector<cppjieba::Jieba::LocWord> loc_words;
|
||||
//jieba.Cut("南京市长江大桥", words, true);
|
||||
//cppjieba::Jieba::Locate(words, loc_words);
|
||||
//for (size_t i = 0; i < loc_words.size(); i++) {
|
||||
// cout << loc_words[i].word
|
||||
// << ", " << loc_words[i].begin
|
||||
// << ", " << loc_words[i].end
|
||||
// << endl;
|
||||
//}
|
||||
|
||||
cout << "[demo] TAGGING" << endl;
|
||||
vector<pair<string, string> > tagres;
|
||||
|
||||
@ -37,25 +37,25 @@ TEST(JiebaTest, Test1) {
|
||||
result << words;
|
||||
ASSERT_EQ("[\"他\", \"来到\", \"了\", \"网易\", \"杭研\", \"大厦\"]", result);
|
||||
|
||||
jieba.CutLevel("南京市长江大桥", words);
|
||||
result << words;
|
||||
ASSERT_EQ("[\"南京市\", \"长江大桥\", \"南京\", \"长江\", \"大桥\"]", result);
|
||||
//jieba.CutLevel("南京市长江大桥", words);
|
||||
//result << words;
|
||||
//ASSERT_EQ("[\"南京市\", \"长江大桥\", \"南京\", \"长江\", \"大桥\"]", result);
|
||||
|
||||
vector<pair<string, size_t> > word_levels;
|
||||
jieba.CutLevel("南京市长江大桥", word_levels);
|
||||
result << word_levels;
|
||||
ASSERT_EQ("[\"南京市:0\", \"长江大桥:0\", \"南京:1\", \"长江:1\", \"大桥:1\"]", result);
|
||||
//vector<pair<string, size_t> > word_levels;
|
||||
//jieba.CutLevel("南京市长江大桥", word_levels);
|
||||
//result << word_levels;
|
||||
//ASSERT_EQ("[\"南京市:0\", \"长江大桥:0\", \"南京:1\", \"长江:1\", \"大桥:1\"]", result);
|
||||
|
||||
vector<Jieba::LocWord> loc_words;
|
||||
jieba.Cut("南京市长江大桥", words);
|
||||
jieba.Locate(words, loc_words);
|
||||
ASSERT_EQ(loc_words.size(), 2u);
|
||||
ASSERT_EQ(loc_words[0].word, "南京市");
|
||||
ASSERT_EQ(loc_words[0].begin, 0u);
|
||||
ASSERT_EQ(loc_words[0].end, 3u);
|
||||
ASSERT_EQ(loc_words[1].word, "长江大桥");
|
||||
ASSERT_EQ(loc_words[1].begin, 3u);
|
||||
ASSERT_EQ(loc_words[1].end, 7u);
|
||||
//vector<Jieba::LocWord> loc_words;
|
||||
//jieba.Cut("南京市长江大桥", words);
|
||||
//jieba.Locate(words, loc_words);
|
||||
//ASSERT_EQ(loc_words.size(), 2u);
|
||||
//ASSERT_EQ(loc_words[0].word, "南京市");
|
||||
//ASSERT_EQ(loc_words[0].begin, 0u);
|
||||
//ASSERT_EQ(loc_words[0].end, 3u);
|
||||
//ASSERT_EQ(loc_words[1].word, "长江大桥");
|
||||
//ASSERT_EQ(loc_words[1].begin, 3u);
|
||||
//ASSERT_EQ(loc_words[1].end, 7u);
|
||||
|
||||
//vector<pair<string, string> > tagres;
|
||||
//jieba.Tag("iPhone6手机的最大特点是很容易弯曲。", tagres);
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
#include "gtest/gtest.h"
|
||||
#include "cppjieba/PreFilter.hpp"
|
||||
#include "limonp/StringUtil.hpp"
|
||||
|
||||
using namespace cppjieba;
|
||||
|
||||
@ -11,32 +12,32 @@ TEST(PreFilterTest, Test1) {
|
||||
string res;
|
||||
|
||||
{
|
||||
PreFilter filter(symbol, "你好，美丽的，世界");
|
||||
string s = "你好，美丽的，世界";
|
||||
PreFilter filter(symbol, s);
|
||||
expected = "你好/，/美丽的/，/世界";
|
||||
ASSERT_TRUE(filter.HasNext());
|
||||
vector<string> words;
|
||||
while (filter.HasNext()) {
|
||||
PreFilter::Range range;
|
||||
range = filter.Next();
|
||||
words.push_back(TransCode::Encode(range.begin, range.end));
|
||||
words.push_back(unicode::GetStringFromRunes(range.begin, range.end - 1));
|
||||
}
|
||||
res = Join(words.begin(), words.end(), "/");
|
||||
res = limonp::Join(words.begin(), words.end(), "/");
|
||||
ASSERT_EQ(res, expected);
|
||||
}
|
||||
|
||||
{
|
||||
PreFilter filter(symbol, "我来自北京邮电大学。。。学号123456，用AK47");
|
||||
string s = "我来自北京邮电大学。。。学号123456，用AK47";
|
||||
PreFilter filter(symbol, s);
|
||||
expected = "我来自北京邮电大学/。/。/。/学号123456/，/用AK47";
|
||||
ASSERT_TRUE(filter.HasNext());
|
||||
vector<string> words;
|
||||
while (filter.HasNext()) {
|
||||
PreFilter::Range range;
|
||||
range = filter.Next();
|
||||
words.push_back(TransCode::Encode(range.begin, range.end));
|
||||
}
|
||||
res = Join(words.begin(), words.end(), "/");
|
||||
for (size_t i = 0; i < words.size(); i++) {
|
||||
words.push_back(unicode::GetStringFromRunes(range.begin, range.end - 1));
|
||||
}
|
||||
res = limonp::Join(words.begin(), words.end(), "/");
|
||||
ASSERT_EQ(res, expected);
|
||||
}
|
||||
}
|
||||
|
||||
@ -4,7 +4,7 @@
|
||||
#include "cppjieba/HMMSegment.hpp"
|
||||
#include "cppjieba/FullSegment.hpp"
|
||||
#include "cppjieba/QuerySegment.hpp"
|
||||
#include "cppjieba/LevelSegment.hpp"
|
||||
//#include "cppjieba/LevelSegment.hpp"
|
||||
#include "gtest/gtest.h"
|
||||
|
||||
using namespace cppjieba;
|
||||
@ -238,6 +238,7 @@ TEST(QuerySegment, Test2) {
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
TEST(LevelSegmentTest, Test0) {
|
||||
string s;
|
||||
LevelSegment segment("../test/testdata/extra_dict/jieba.dict.small.utf8");
|
||||
@ -249,6 +250,7 @@ TEST(LevelSegmentTest, Test0) {
|
||||
segment.Cut("南京市长江大桥", res);
|
||||
ASSERT_EQ("[\"南京市\", \"长江大桥\", \"南京\", \"长江\", \"大桥\"]", s << res);
|
||||
}
|
||||
*/
|
||||
|
||||
TEST(MPSegmentTest, Unicode32) {
|
||||
string s("天气很好，🙋 我们去郊游。");
|
||||
|
||||
@ -15,7 +15,7 @@ TEST(TrieTest, Empty) {
|
||||
TEST(TrieTest, Construct) {
|
||||
vector<Unicode> keys;
|
||||
vector<const DictUnit*> values;
|
||||
keys.push_back(TransCode::Decode("你"));
|
||||
keys.push_back(unicode::DecodeRunesInString("你"));
|
||||
values.push_back((const DictUnit*)(NULL));
|
||||
Trie trie(keys, values);
|
||||
}
|
||||
@ -31,27 +31,34 @@ TEST(DictTrieTest, Test1) {
|
||||
DictTrie trie(DICT_FILE);
|
||||
ASSERT_LT(trie.GetMinWeight() + 15.6479, 0.001);
|
||||
string word("来到");
|
||||
Unicode uni;
|
||||
ASSERT_TRUE(TransCode::Decode(word, uni));
|
||||
DictUnit nodeInfo;
|
||||
nodeInfo.word = uni;
|
||||
nodeInfo.tag = "v";
|
||||
nodeInfo.weight = -8.87033;
|
||||
s1 << nodeInfo;
|
||||
s2 << (*trie.Find(uni.begin(), uni.end()));
|
||||
cppjieba::unicode::RuneStrArray uni;
|
||||
ASSERT_TRUE(unicode::DecodeRunesInString(word, uni));
|
||||
//DictUnit nodeInfo;
|
||||
//nodeInfo.word = uni;
|
||||
//nodeInfo.tag = "v";
|
||||
//nodeInfo.weight = -8.87033;
|
||||
//s1 << nodeInfo;
|
||||
//s2 << (*trie.Find(uni.begin(), uni.end()));
|
||||
const DictUnit* du = trie.Find(uni.begin(), uni.end());
|
||||
ASSERT_TRUE(du != NULL);
|
||||
ASSERT_EQ(2u, du->word.size());
|
||||
ASSERT_EQ(26469u, du->word[0]);
|
||||
ASSERT_EQ(21040u, du->word[1]);
|
||||
ASSERT_EQ("v", du->tag);
|
||||
ASSERT_NEAR(-8.870, du->weight, 0.001);
|
||||
|
||||
EXPECT_EQ("[\"26469\", \"21040\"] v -8.870", s2);
|
||||
//EXPECT_EQ("[\"26469\", \"21040\"] v -8.870", s2);
|
||||
word = "清华大学";
|
||||
LocalVector<pair<size_t, const DictUnit*> > res;
|
||||
const char * words[] = {"清", "清华", "清华大学"};
|
||||
for (size_t i = 0; i < sizeof(words)/sizeof(words[0]); i++) {
|
||||
ASSERT_TRUE(TransCode::Decode(words[i], uni));
|
||||
ASSERT_TRUE(unicode::DecodeRunesInString(words[i], uni));
|
||||
res.push_back(make_pair(uni.size() - 1, trie.Find(uni.begin(), uni.end())));
|
||||
//resMap[uni.size() - 1] = trie.Find(uni.begin(), uni.end());
|
||||
}
|
||||
vector<pair<size_t, const DictUnit*> > vec;
|
||||
vector<struct Dag> dags;
|
||||
ASSERT_TRUE(TransCode::Decode(word, uni));
|
||||
ASSERT_TRUE(unicode::DecodeRunesInString(word, uni));
|
||||
trie.Find(uni.begin(), uni.end(), dags);
|
||||
ASSERT_EQ(dags.size(), uni.size());
|
||||
ASSERT_NE(dags.size(), 0u);
|
||||
@ -64,25 +71,21 @@ TEST(DictTrieTest, Test1) {
|
||||
TEST(DictTrieTest, UserDict) {
|
||||
DictTrie trie(DICT_FILE, "../test/testdata/userdict.utf8");
|
||||
string word = "云计算";
|
||||
Unicode unicode;
|
||||
ASSERT_TRUE(TransCode::Decode(word, unicode));
|
||||
cppjieba::unicode::RuneStrArray unicode;
|
||||
ASSERT_TRUE(unicode::DecodeRunesInString(word, unicode));
|
||||
const DictUnit * unit = trie.Find(unicode.begin(), unicode.end());
|
||||
ASSERT_TRUE(unit);
|
||||
string res ;
|
||||
res << *unit;
|
||||
ASSERT_EQ("[\"20113\", \"35745\", \"31639\"] -14.100", res);
|
||||
ASSERT_NEAR(unit->weight, -14.100, 0.001);
|
||||
}
|
||||
|
||||
TEST(DictTrieTest, UserDictWithMaxWeight) {
|
||||
DictTrie trie(DICT_FILE, "../test/testdata/userdict.utf8", DictTrie::WordWeightMax);
|
||||
string word = "云计算";
|
||||
Unicode unicode;
|
||||
ASSERT_TRUE(TransCode::Decode(word, unicode));
|
||||
cppjieba::unicode::RuneStrArray unicode;
|
||||
ASSERT_TRUE(unicode::DecodeRunesInString(word, unicode));
|
||||
const DictUnit * unit = trie.Find(unicode.begin(), unicode.end());
|
||||
ASSERT_TRUE(unit);
|
||||
string res ;
|
||||
res << *unit;
|
||||
ASSERT_EQ("[\"20113\", \"35745\", \"31639\"] -2.975", res);
|
||||
ASSERT_NEAR(unit->weight, -2.975, 0.001);
|
||||
}
|
||||
|
||||
TEST(DictTrieTest, Dag) {
|
||||
@ -90,8 +93,8 @@ TEST(DictTrieTest, Dag) {
|
||||
|
||||
{
|
||||
string word = "清华大学";
|
||||
Unicode unicode;
|
||||
ASSERT_TRUE(TransCode::Decode(word, unicode));
|
||||
cppjieba::unicode::RuneStrArray unicode;
|
||||
ASSERT_TRUE(unicode::DecodeRunesInString(word, unicode));
|
||||
vector<struct Dag> res;
|
||||
trie.Find(unicode.begin(), unicode.end(), res);
|
||||
|
||||
@ -104,8 +107,8 @@ TEST(DictTrieTest, Dag) {
|
||||
|
||||
{
|
||||
string word = "北京邮电大学";
|
||||
Unicode unicode;
|
||||
ASSERT_TRUE(TransCode::Decode(word, unicode));
|
||||
cppjieba::unicode::RuneStrArray unicode;
|
||||
ASSERT_TRUE(unicode::DecodeRunesInString(word, unicode));
|
||||
vector<struct Dag> res;
|
||||
trie.Find(unicode.begin(), unicode.end(), res);
|
||||
|
||||
@ -118,8 +121,8 @@ TEST(DictTrieTest, Dag) {
|
||||
|
||||
{
|
||||
string word = "长江大桥";
|
||||
Unicode unicode;
|
||||
ASSERT_TRUE(TransCode::Decode(word, unicode));
|
||||
cppjieba::unicode::RuneStrArray unicode;
|
||||
ASSERT_TRUE(unicode::DecodeRunesInString(word, unicode));
|
||||
vector<struct Dag> res;
|
||||
trie.Find(unicode.begin(), unicode.end(), res);
|
||||
|
||||
@ -132,8 +135,8 @@ TEST(DictTrieTest, Dag) {
|
||||
|
||||
{
|
||||
string word = "长江大桥";
|
||||
Unicode unicode;
|
||||
ASSERT_TRUE(TransCode::Decode(word, unicode));
|
||||
cppjieba::unicode::RuneStrArray unicode;
|
||||
ASSERT_TRUE(unicode::DecodeRunesInString(word, unicode));
|
||||
vector<struct Dag> res;
|
||||
trie.Find(unicode.begin(), unicode.end(), res, 3);
|
||||
|
||||
@ -146,8 +149,8 @@ TEST(DictTrieTest, Dag) {
|
||||
|
||||
{
|
||||
string word = "长江大桥";
|
||||
Unicode unicode;
|
||||
ASSERT_TRUE(TransCode::Decode(word, unicode));
|
||||
cppjieba::unicode::RuneStrArray unicode;
|
||||
ASSERT_TRUE(unicode::DecodeRunesInString(word, unicode));
|
||||
vector<struct Dag> res;
|
||||
trie.Find(unicode.begin(), unicode.end(), res, 4);
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user