third_codes/cppjieba
1
0
Fork 0
mirror of https://github.com/yanyiwu/cppjieba.git synced 2025-07-18 00:00:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

big change: add RuneStr for the position of word in string

Browse Source
This commit is contained in:
yanyiwu 2016-04-17 17:30:05 +08:00
parent abcc0af034
commit 339e3ca772
20 changed files with 423 additions and 300 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
include/cppjieba/DictTrie.hpp
View File

@ -10,7 +10,7 @@
#include <limits> #include <limits>
#include "limonp/StringUtil.hpp" #include "limonp/StringUtil.hpp"
#include "limonp/Logging.hpp" #include "limonp/Logging.hpp"
#include "TransCode.hpp" #include "Unicode.hpp"
#include "Trie.hpp" #include "Trie.hpp"
namespace cppjieba { namespace cppjieba {
@ -48,12 +48,12 @@ class DictTrie {
return true; return true;
} }
const DictUnit* Find(Unicode::const_iterator begin, Unicode::const_iterator end) const { const DictUnit* Find(unicode::RuneStrArray::const_iterator begin, unicode::RuneStrArray::const_iterator end) const {
return trie_->Find(begin, end); return trie_->Find(begin, end);
} }
void Find(Unicode::const_iterator begin, void Find(unicode::RuneStrArray::const_iterator begin,
Unicode::const_iterator end, unicode::RuneStrArray::const_iterator end,
vector<struct Dag>&res, vector<struct Dag>&res,
size_t max_word_len = MAX_WORD_LENGTH) const { size_t max_word_len = MAX_WORD_LENGTH) const {
trie_->Find(begin, end, res, max_word_len); trie_->Find(begin, end, res, max_word_len);
@ -124,7 +124,7 @@ class DictTrie {
const string& word, const string& word,
double weight, double weight,
const string& tag) { const string& tag) {
if (!TransCode::Decode(word, node_info.word)) { if (!unicode::DecodeRunesInString(word, node_info.word)) {
XLOG(ERROR) << "Decode " << word << " failed."; XLOG(ERROR) << "Decode " << word << " failed.";
return false; return false;
} }

26
include/cppjieba/FullSegment.hpp
View File

@ -7,7 +7,7 @@
#include "limonp/Logging.hpp" #include "limonp/Logging.hpp"
#include "DictTrie.hpp" #include "DictTrie.hpp"
#include "SegmentBase.hpp" #include "SegmentBase.hpp"
#include "TransCode.hpp" #include "Unicode.hpp"
namespace cppjieba { namespace cppjieba {
class FullSegment: public SegmentBase { class FullSegment: public SegmentBase {
@ -29,17 +29,19 @@ class FullSegment: public SegmentBase {
vector<string>& words) const { vector<string>& words) const {
PreFilter pre_filter(symbols_, sentence); PreFilter pre_filter(symbols_, sentence);
PreFilter::Range range; PreFilter::Range range;
vector<Unicode> uwords; vector<unicode::WordRange> wrs;
uwords.reserve(sentence.size()); wrs.reserve(sentence.size()/2);
while (pre_filter.HasNext()) { while (pre_filter.HasNext()) {
range = pre_filter.Next(); range = pre_filter.Next();
Cut(range.begin, range.end, uwords); Cut(range.begin, range.end, wrs);
} }
TransCode::Encode(uwords, words); words.clear();
words.reserve(wrs.size());
unicode::GetStringsFromWordRanges(wrs, words);
} }
void Cut(Unicode::const_iterator begin, void Cut(unicode::RuneStrArray::const_iterator begin,
Unicode::const_iterator end, unicode::RuneStrArray::const_iterator end,
vector<Unicode>& res) const { vector<unicode::WordRange>& res) const {
//resut of searching in trie tree //resut of searching in trie tree
LocalVector<pair<size_t, const DictUnit*> > tRes; LocalVector<pair<size_t, const DictUnit*> > tRes;
@ -56,15 +58,19 @@ class FullSegment: public SegmentBase {
dictTrie_->Find(begin, end, dags); dictTrie_->Find(begin, end, dags);
for (size_t i = 0; i < dags.size(); i++) { for (size_t i = 0; i < dags.size(); i++) {
for (size_t j = 0; j < dags[i].nexts.size(); j++) { for (size_t j = 0; j < dags[i].nexts.size(); j++) {
size_t nextoffset = dags[i].nexts[j].first;
assert(nextoffset < dags.size());
const DictUnit* du = dags[i].nexts[j].second; const DictUnit* du = dags[i].nexts[j].second;
if (du == NULL) { if (du == NULL) {
if (dags[i].nexts.size() == 1 && maxIdx <= uIdx) { if (dags[i].nexts.size() == 1 && maxIdx <= uIdx) {
res.push_back(Unicode(1, dags[i].rune)); unicode::WordRange wr = {begin + i, begin + nextoffset};
res.push_back(wr);
} }
} else { } else {
wordLen = du->word.size(); wordLen = du->word.size();
if (wordLen >= 2 || (dags[i].nexts.size() == 1 && maxIdx <= uIdx)) { if (wordLen >= 2 || (dags[i].nexts.size() == 1 && maxIdx <= uIdx)) {
res.push_back(du->word); unicode::WordRange wr = {begin + i, begin + nextoffset};
res.push_back(wr);
} }
} }
maxIdx = uIdx + wordLen > maxIdx ? uIdx + wordLen : maxIdx; maxIdx = uIdx + wordLen > maxIdx ? uIdx + wordLen : maxIdx;

2
include/cppjieba/HMMModel.hpp
View File

@ -105,7 +105,7 @@ struct HMMModel {
XLOG(ERROR) << "emitProb illegal."; XLOG(ERROR) << "emitProb illegal.";
return false; return false;
} }
if (!TransCode::Decode(tmp2[0], unicode) || unicode.size() != 1) { if (!unicode::DecodeRunesInString(tmp2[0], unicode) || unicode.size() != 1) {
XLOG(ERROR) << "TransCode failed."; XLOG(ERROR) << "TransCode failed.";
return false; return false;
} }

50
include/cppjieba/HMMSegment.hpp
View File

@ -27,19 +27,21 @@ class HMMSegment: public SegmentBase {
vector<string>& words) const { vector<string>& words) const {
PreFilter pre_filter(symbols_, sentence); PreFilter pre_filter(symbols_, sentence);
PreFilter::Range range; PreFilter::Range range;
vector<Unicode> uwords; vector<unicode::WordRange> wrs;
uwords.reserve(sentence.size()); wrs.reserve(sentence.size()/2);
while (pre_filter.HasNext()) { while (pre_filter.HasNext()) {
range = pre_filter.Next(); range = pre_filter.Next();
Cut(range.begin, range.end, uwords); Cut(range.begin, range.end, wrs);
} }
TransCode::Encode(uwords, words); words.clear();
words.reserve(wrs.size());
unicode::GetStringsFromWordRanges(wrs, words);
} }
void Cut(Unicode::const_iterator begin, Unicode::const_iterator end, vector<Unicode>& res) const { void Cut(unicode::RuneStrArray::const_iterator begin, unicode::RuneStrArray::const_iterator end, vector<unicode::WordRange>& res) const {
Unicode::const_iterator left = begin; unicode::RuneStrArray::const_iterator left = begin;
Unicode::const_iterator right = begin; unicode::RuneStrArray::const_iterator right = begin;
while (right != end) { while (right != end) {
if (*right < 0x80) { if (right->rune < 0x80) {
if (left != right) { if (left != right) {
InternalCut(left, right, res); InternalCut(left, right, res);
} }
@ -55,7 +57,8 @@ class HMMSegment: public SegmentBase {
} }
right ++; right ++;
} while (false); } while (false);
res.push_back(Unicode(left, right)); unicode::WordRange wr = {left, right - 1};
res.push_back(wr);
left = right; left = right;
} else { } else {
right++; right++;
@ -67,15 +70,15 @@ class HMMSegment: public SegmentBase {
} }
private: private:
// sequential letters rule // sequential letters rule
Unicode::const_iterator SequentialLetterRule(Unicode::const_iterator begin, Unicode::const_iterator end) const { unicode::RuneStrArray::const_iterator SequentialLetterRule(unicode::RuneStrArray::const_iterator begin, unicode::RuneStrArray::const_iterator end) const {
Rune x = *begin; Rune x = begin->rune;
if (('a' <= x && x <= 'z') || ('A' <= x && x <= 'Z')) { if (('a' <= x && x <= 'z') || ('A' <= x && x <= 'Z')) {
begin ++; begin ++;
} else { } else {
return begin; return begin;
} }
while (begin != end) { while (begin != end) {
x = *begin; x = begin->rune;
if (('a' <= x && x <= 'z') || ('A' <= x && x <= 'Z') || ('0' <= x && x <= '9')) { if (('a' <= x && x <= 'z') || ('A' <= x && x <= 'Z') || ('0' <= x && x <= '9')) {
begin ++; begin ++;
} else { } else {
@ -85,15 +88,15 @@ class HMMSegment: public SegmentBase {
return begin; return begin;
} }
// //
Unicode::const_iterator NumbersRule(Unicode::const_iterator begin, Unicode::const_iterator end) const { unicode::RuneStrArray::const_iterator NumbersRule(unicode::RuneStrArray::const_iterator begin, unicode::RuneStrArray::const_iterator end) const {
Rune x = *begin; Rune x = begin->rune;
if ('0' <= x && x <= '9') { if ('0' <= x && x <= '9') {
begin ++; begin ++;
} else { } else {
return begin; return begin;
} }
while (begin != end) { while (begin != end) {
x = *begin; x = begin->rune;
if ( ('0' <= x && x <= '9') || x == '.') { if ( ('0' <= x && x <= '9') || x == '.') {
begin++; begin++;
} else { } else {
@ -102,23 +105,24 @@ class HMMSegment: public SegmentBase {
} }
return begin; return begin;
} }
void InternalCut(Unicode::const_iterator begin, Unicode::const_iterator end, vector<Unicode>& res) const { void InternalCut(unicode::RuneStrArray::const_iterator begin, unicode::RuneStrArray::const_iterator end, vector<unicode::WordRange>& res) const {
vector<size_t> status; vector<size_t> status;
Viterbi(begin, end, status); Viterbi(begin, end, status);
Unicode::const_iterator left = begin; unicode::RuneStrArray::const_iterator left = begin;
Unicode::const_iterator right; unicode::RuneStrArray::const_iterator right;
for (size_t i = 0; i < status.size(); i++) { for (size_t i = 0; i < status.size(); i++) {
if (status[i] % 2) { //if (HMMModel::E == status[i] || HMMModel::S == status[i]) if (status[i] % 2) { //if (HMMModel::E == status[i] || HMMModel::S == status[i])
right = begin + i + 1; right = begin + i + 1;
res.push_back(Unicode(left, right)); unicode::WordRange wr = {left, right - 1};
res.push_back(wr);
left = right; left = right;
} }
} }
} }
void Viterbi(Unicode::const_iterator begin, void Viterbi(unicode::RuneStrArray::const_iterator begin,
Unicode::const_iterator end, unicode::RuneStrArray::const_iterator end,
vector<size_t>& status) const { vector<size_t>& status) const {
size_t Y = HMMModel::STATUS_SUM; size_t Y = HMMModel::STATUS_SUM;
size_t X = end - begin; size_t X = end - begin;
@ -132,7 +136,7 @@ class HMMSegment: public SegmentBase {
//start //start
for (size_t y = 0; y < Y; y++) { for (size_t y = 0; y < Y; y++) {
weight[0 + y * X] = model_->startProb[y] + model_->GetEmitProb(model_->emitProbVec[y], *begin, MIN_DOUBLE); weight[0 + y * X] = model_->startProb[y] + model_->GetEmitProb(model_->emitProbVec[y], begin->rune, MIN_DOUBLE);
path[0 + y * X] = -1; path[0 + y * X] = -1;
} }
@ -143,7 +147,7 @@ class HMMSegment: public SegmentBase {
now = x + y*X; now = x + y*X;
weight[now] = MIN_DOUBLE; weight[now] = MIN_DOUBLE;
path[now] = HMMModel::E; // warning path[now] = HMMModel::E; // warning
emitProb = model_->GetEmitProb(model_->emitProbVec[y], *(begin+x), MIN_DOUBLE); emitProb = model_->GetEmitProb(model_->emitProbVec[y], (begin+x)->rune, MIN_DOUBLE);
for (size_t preY = 0; preY < Y; preY++) { for (size_t preY = 0; preY < Y; preY++) {
old = x - 1 + preY * X; old = x - 1 + preY * X;
tmp = weight[old] + model_->transProb[preY][y] + emitProb; tmp = weight[old] + model_->transProb[preY][y] + emitProb;

40
include/cppjieba/Jieba.hpp
View File

@ -3,7 +3,7 @@
#include "QuerySegment.hpp" #include "QuerySegment.hpp"
#include "PosTagger.hpp" #include "PosTagger.hpp"
#include "LevelSegment.hpp" //#include "LevelSegment.hpp"
namespace cppjieba { namespace cppjieba {
@ -17,7 +17,7 @@ class Jieba {
mix_seg_(&dict_trie_, &model_), mix_seg_(&dict_trie_, &model_),
full_seg_(&dict_trie_), full_seg_(&dict_trie_),
query_seg_(&dict_trie_, &model_), query_seg_(&dict_trie_, &model_),
level_seg_(&dict_trie_), //level_seg_(&dict_trie_),
pos_tagger_(&dict_trie_, &model_) { pos_tagger_(&dict_trie_, &model_) {
} }
~Jieba() { ~Jieba() {
@ -41,26 +41,26 @@ class Jieba {
void CutHMM(const string& sentence, vector<string>& words) const { void CutHMM(const string& sentence, vector<string>& words) const {
hmm_seg_.Cut(sentence, words); hmm_seg_.Cut(sentence, words);
} }
void CutLevel(const string& sentence, vector<string>& words) const { //void CutLevel(const string& sentence, vector<string>& words) const {
level_seg_.Cut(sentence, words); // level_seg_.Cut(sentence, words);
} //}
void CutLevel(const string& sentence, vector<pair<string, size_t> >& words) const { //void CutLevel(const string& sentence, vector<pair<string, size_t> >& words) const {
level_seg_.Cut(sentence, words); // level_seg_.Cut(sentence, words);
} //}
void CutSmall(const string& sentence, vector<string>& words, size_t max_word_len) const { void CutSmall(const string& sentence, vector<string>& words, size_t max_word_len) const {
mp_seg_.Cut(sentence, words, max_word_len); mp_seg_.Cut(sentence, words, max_word_len);
} }
static void Locate(const vector<string>& words, vector<LocWord>& loc_words) { //static void Locate(const vector<string>& words, vector<LocWord>& loc_words) {
loc_words.resize(words.size()); // loc_words.resize(words.size());
size_t begin = 0; // size_t begin = 0;
for (size_t i = 0; i < words.size(); i++) { // for (size_t i = 0; i < words.size(); i++) {
size_t len = TransCode::Decode(words[i]).size(); // size_t len = TransCode::Decode(words[i]).size();
loc_words[i].word = words[i]; // loc_words[i].word = words[i];
loc_words[i].begin = begin; // loc_words[i].begin = begin;
loc_words[i].end = loc_words[i].begin + len; // loc_words[i].end = loc_words[i].begin + len;
begin = loc_words[i].end; // begin = loc_words[i].end;
} // }
} //}
void Tag(const string& sentence, vector<pair<string, string> >& words) const { void Tag(const string& sentence, vector<pair<string, string> >& words) const {
pos_tagger_.Tag(sentence, words); pos_tagger_.Tag(sentence, words);
@ -89,7 +89,7 @@ class Jieba {
MixSegment mix_seg_; MixSegment mix_seg_;
FullSegment full_seg_; FullSegment full_seg_;
QuerySegment query_seg_; QuerySegment query_seg_;
LevelSegment level_seg_; //LevelSegment level_seg_;
PosTagger pos_tagger_; PosTagger pos_tagger_;

10
include/cppjieba/KeywordExtractor.hpp
View File

@ -69,7 +69,7 @@ class KeywordExtractor {
for (size_t i = 0; i < words.size(); ++i) { for (size_t i = 0; i < words.size(); ++i) {
size_t t = offset; size_t t = offset;
offset += words[i].size(); offset += words[i].size();
if (IsSingleWord(words[i]) || stopWords_.find(words[i]) != stopWords_.end()) { if (unicode::IsSingleWord(words[i]) || stopWords_.find(words[i]) != stopWords_.end()) {
continue; continue;
} }
wordmap[words[i]].offsets.push_back(t); wordmap[words[i]].offsets.push_back(t);
@ -136,14 +136,6 @@ class KeywordExtractor {
assert(stopWords_.size()); assert(stopWords_.size());
} }
bool IsSingleWord(const string& str) const {
Unicode unicode;
TransCode::Decode(str, unicode);
if (unicode.size() == 1)
return true;
return false;
}
static bool Compare(const Word& lhs, const Word& rhs) { static bool Compare(const Word& lhs, const Word& rhs) {
return lhs.weight > rhs.weight; return lhs.weight > rhs.weight;
} }

12
include/cppjieba/LevelSegment.hpp
View File

@ -17,9 +17,9 @@ class LevelSegment: public SegmentBase{
~LevelSegment() { ~LevelSegment() {
} }
void Cut(Unicode::const_iterator begin, void Cut(unicode::RuneStrArray::const_iterator begin,
Unicode::const_iterator end, unicode::RuneStrArray::const_iterator end,
vector<pair<Unicode, size_t> >& res) const { vector<pair<WordRange, size_t> >& res) const {
res.clear(); res.clear();
vector<Unicode> words; vector<Unicode> words;
vector<Unicode> smallerWords; vector<Unicode> smallerWords;
@ -49,9 +49,9 @@ class LevelSegment: public SegmentBase{
void Cut(const string& sentence, void Cut(const string& sentence,
vector<pair<string, size_t> >& words) const { vector<pair<string, size_t> >& words) const {
words.clear(); words.clear();
Unicode unicode; RuneStrArray unicode;
TransCode::Decode(sentence, unicode); unicode::DecodeRunesInString(sentence, unicode);
vector<pair<Unicode, size_t> > unicodeWords; vector<pair<WordRange, size_t> > unicodeWords;
Cut(unicode.begin(), unicode.end(), unicodeWords); Cut(unicode.begin(), unicode.end(), unicodeWords);
words.resize(unicodeWords.size()); words.resize(unicodeWords.size());
for (size_t i = 0; i < words.size(); i++) { for (size_t i = 0; i < words.size(); i++) {

31
include/cppjieba/MPSegment.hpp
View File

@ -30,17 +30,19 @@ class MPSegment: public SegmentBase {
size_t max_word_len = MAX_WORD_LENGTH) const { size_t max_word_len = MAX_WORD_LENGTH) const {
PreFilter pre_filter(symbols_, sentence); PreFilter pre_filter(symbols_, sentence);
PreFilter::Range range; PreFilter::Range range;
vector<Unicode> uwords; vector<unicode::WordRange> wrs;
uwords.reserve(sentence.size()); wrs.reserve(sentence.size()/2);
while (pre_filter.HasNext()) { while (pre_filter.HasNext()) {
range = pre_filter.Next(); range = pre_filter.Next();
Cut(range.begin, range.end, uwords, max_word_len); Cut(range.begin, range.end, wrs, max_word_len);
} }
TransCode::Encode(uwords, words); words.clear();
words.reserve(wrs.size());
unicode::GetStringsFromWordRanges(wrs, words);
} }
void Cut(Unicode::const_iterator begin, void Cut(unicode::RuneStrArray::const_iterator begin,
Unicode::const_iterator end, unicode::RuneStrArray::const_iterator end,
vector<Unicode>& words, vector<unicode::WordRange>& words,
size_t max_word_len = MAX_WORD_LENGTH) const { size_t max_word_len = MAX_WORD_LENGTH) const {
vector<Dag> dags; vector<Dag> dags;
dictTrie_->Find(begin, dictTrie_->Find(begin,
@ -48,7 +50,7 @@ class MPSegment: public SegmentBase {
dags, dags,
max_word_len); max_word_len);
CalcDP(dags); CalcDP(dags);
CutByDag(dags, words); CutByDag(begin, end, dags, words);
} }
const DictTrie* GetDictTrie() const { const DictTrie* GetDictTrie() const {
@ -88,16 +90,21 @@ class MPSegment: public SegmentBase {
} }
} }
} }
void CutByDag(const vector<Dag>& dags, void CutByDag(unicode::RuneStrArray::const_iterator begin,
vector<Unicode>& words) const { unicode::RuneStrArray::const_iterator end,
const vector<Dag>& dags,
vector<unicode::WordRange>& words) const {
size_t i = 0; size_t i = 0;
while (i < dags.size()) { while (i < dags.size()) {
const DictUnit* p = dags[i].pInfo; const DictUnit* p = dags[i].pInfo;
if (p) { if (p) {
words.push_back(p->word); assert(p->word.size() >= 1);
unicode::WordRange wr = {begin + i, begin + i + p->word.size() - 1};
words.push_back(wr);
i += p->word.size(); i += p->word.size();
} else { //single chinese word } else { //single chinese word
words.push_back(Unicode(1, dags[i].rune)); unicode::WordRange wr = {begin + i, begin + i};
words.push_back(wr);
i++; i++;
} }
} }

34
include/cppjieba/MixSegment.hpp
View File

@ -23,52 +23,52 @@ class MixSegment: public SegmentBase {
void Cut(const string& sentence, vector<string>& words, bool hmm = true) const { void Cut(const string& sentence, vector<string>& words, bool hmm = true) const {
PreFilter pre_filter(symbols_, sentence); PreFilter pre_filter(symbols_, sentence);
PreFilter::Range range; PreFilter::Range range;
vector<Unicode> uwords; vector<unicode::WordRange> wrs;
uwords.reserve(sentence.size()); wrs.reserve(sentence.size() / 2);
while (pre_filter.HasNext()) { while (pre_filter.HasNext()) {
range = pre_filter.Next(); range = pre_filter.Next();
Cut(range.begin, range.end, uwords, hmm); Cut(range.begin, range.end, wrs, hmm);
} }
TransCode::Encode(uwords, words); words.clear();
words.reserve(wrs.size());
unicode::GetStringsFromWordRanges(wrs, words);
} }
void Cut(Unicode::const_iterator begin, Unicode::const_iterator end, vector<Unicode>& res, bool hmm) const { void Cut(unicode::RuneStrArray::const_iterator begin, unicode::RuneStrArray::const_iterator end, vector<unicode::WordRange>& res, bool hmm) const {
if (!hmm) { if (!hmm) {
mpSeg_.Cut(begin, end, res); mpSeg_.Cut(begin, end, res);
return; return;
} }
vector<Unicode> words; vector<unicode::WordRange> words;
assert(end >= begin);
words.reserve(end - begin); words.reserve(end - begin);
mpSeg_.Cut(begin, end, words); mpSeg_.Cut(begin, end, words);
vector<Unicode> hmmRes; vector<unicode::WordRange> hmmRes;
hmmRes.reserve(end - begin); hmmRes.reserve(end - begin);
Unicode piece; for (size_t i = 0; i < words.size(); i++) {
piece.reserve(end - begin);
for (size_t i = 0, j = 0; i < words.size(); i++) {
//if mp Get a word, it's ok, put it into result //if mp Get a word, it's ok, put it into result
if (1 != words[i].size() || (words[i].size() == 1 && mpSeg_.IsUserDictSingleChineseWord(words[i][0]))) { if (words[i].left != words[i].right || (words[i].left == words[i].right && mpSeg_.IsUserDictSingleChineseWord(words[i].left->rune))) {
res.push_back(words[i]); res.push_back(words[i]);
continue; continue;
} }
// if mp Get a single one and it is not in userdict, collect it in sequence // if mp Get a single one and it is not in userdict, collect it in sequence
j = i; size_t j = i;
while (j < words.size() && 1 == words[j].size() && !mpSeg_.IsUserDictSingleChineseWord(words[j][0])) { while (j < words.size() && words[j].left == words[j].right && !mpSeg_.IsUserDictSingleChineseWord(words[j].left->rune)) {
piece.push_back(words[j][0]);
j++; j++;
} }
// Cut the sequence with hmm // Cut the sequence with hmm
hmmSeg_.Cut(piece.begin(), piece.end(), hmmRes); assert(j - 1 >= i);
// TODO
hmmSeg_.Cut(words[i].left, words[j - 1].left + 1, hmmRes);
//put hmm result to result //put hmm result to result
for (size_t k = 0; k < hmmRes.size(); k++) { for (size_t k = 0; k < hmmRes.size(); k++) {
res.push_back(hmmRes[k]); res.push_back(hmmRes[k]);
} }
//clear tmp vars //clear tmp vars
piece.clear();
hmmRes.clear(); hmmRes.clear();
//let i jump over this piece //let i jump over this piece

14
include/cppjieba/PosTagger.hpp
View File

@ -30,17 +30,17 @@ class PosTagger {
segment_.Cut(src, CutRes); segment_.Cut(src, CutRes);
const DictUnit *tmp = NULL; const DictUnit *tmp = NULL;
Unicode unico; unicode::RuneStrArray runes;
const DictTrie * dict = segment_.GetDictTrie(); const DictTrie * dict = segment_.GetDictTrie();
assert(dict != NULL); assert(dict != NULL);
for (vector<string>::iterator itr = CutRes.begin(); itr != CutRes.end(); ++itr) { for (vector<string>::iterator itr = CutRes.begin(); itr != CutRes.end(); ++itr) {
if (!TransCode::Decode(*itr, unico)) { if (!unicode::DecodeRunesInString(*itr, runes)) {
XLOG(ERROR) << "Decode failed."; XLOG(ERROR) << "Decode failed.";
return false; return false;
} }
tmp = dict->Find(unico.begin(), unico.end()); tmp = dict->Find(runes.begin(), runes.end());
if (tmp == NULL || tmp->tag.empty()) { if (tmp == NULL || tmp->tag.empty()) {
res.push_back(make_pair(*itr, SpecialRule(unico))); res.push_back(make_pair(*itr, SpecialRule(runes)));
} else { } else {
res.push_back(make_pair(*itr, tmp->tag)); res.push_back(make_pair(*itr, tmp->tag));
} }
@ -48,13 +48,13 @@ class PosTagger {
return !res.empty(); return !res.empty();
} }
private: private:
const char* SpecialRule(const Unicode& unicode) const { const char* SpecialRule(const unicode::RuneStrArray& unicode) const {
size_t m = 0; size_t m = 0;
size_t eng = 0; size_t eng = 0;
for (size_t i = 0; i < unicode.size() && eng < unicode.size() / 2; i++) { for (size_t i = 0; i < unicode.size() && eng < unicode.size() / 2; i++) {
if (unicode[i] < 0x80) { if (unicode[i].rune < 0x80) {
eng ++; eng ++;
if ('0' <= unicode[i] && unicode[i] <= '9') { if ('0' <= unicode[i].rune && unicode[i].rune <= '9') {
m++; m++;
} }
} }

26
include/cppjieba/PreFilter.hpp
View File

@ -1,32 +1,22 @@
#ifndef CPPJIEBA_PRE_FILTER_H #ifndef CPPJIEBA_PRE_FILTER_H
#define CPPJIEBA_PRE_FILTER_H #define CPPJIEBA_PRE_FILTER_H
#include "TransCode.hpp" #include "Trie.hpp"
namespace cppjieba { namespace cppjieba {
//class PreFilterIterator {
// public:
// PreFilterIterator() {
// }
// ~PreFilterIterator() {
// }
//
// private:
// const unordered_set<Rune>& specialSymbols_;
//}; // PreFilterIterator
class PreFilter { class PreFilter {
public: public:
//TODO use WordRange instead of Range
struct Range { struct Range {
Unicode::const_iterator begin; unicode::RuneStrArray::const_iterator begin;
Unicode::const_iterator end; unicode::RuneStrArray::const_iterator end;
}; // struct Range }; // struct Range
PreFilter(const unordered_set<Rune>& symbols, PreFilter(const unordered_set<Rune>& symbols,
const string& sentence) const string& sentence)
: symbols_(symbols) { : symbols_(symbols) {
TransCode::Decode(sentence, sentence_); unicode::DecodeRunesInString(sentence, sentence_);
cursor_ = sentence_.begin(); cursor_ = sentence_.begin();
} }
~PreFilter() { ~PreFilter() {
@ -38,7 +28,7 @@ class PreFilter {
Range range; Range range;
range.begin = cursor_; range.begin = cursor_;
while (cursor_ != sentence_.end()) { while (cursor_ != sentence_.end()) {
if (IsIn(symbols_, *cursor_)) { if (IsIn(symbols_, cursor_->rune)) {
if (range.begin == cursor_) { if (range.begin == cursor_) {
cursor_ ++; cursor_ ++;
} }
@ -51,8 +41,8 @@ class PreFilter {
return range; return range;
} }
private: private:
Unicode::const_iterator cursor_; unicode::RuneStrArray::const_iterator cursor_;
Unicode sentence_; unicode::RuneStrArray sentence_;
const unordered_set<Rune>& symbols_; const unordered_set<Rune>& symbols_;
}; // class PreFilter }; // class PreFilter

26
include/cppjieba/QuerySegment.hpp
View File

@ -9,7 +9,7 @@
#include "SegmentBase.hpp" #include "SegmentBase.hpp"
#include "FullSegment.hpp" #include "FullSegment.hpp"
#include "MixSegment.hpp" #include "MixSegment.hpp"
#include "TransCode.hpp" #include "Unicode.hpp"
#include "DictTrie.hpp" #include "DictTrie.hpp"
namespace cppjieba { namespace cppjieba {
@ -29,25 +29,27 @@ class QuerySegment: public SegmentBase {
void Cut(const string& sentence, vector<string>& words, bool hmm = true) const { void Cut(const string& sentence, vector<string>& words, bool hmm = true) const {
PreFilter pre_filter(symbols_, sentence); PreFilter pre_filter(symbols_, sentence);
PreFilter::Range range; PreFilter::Range range;
vector<Unicode> uwords; vector<unicode::WordRange> wrs;
uwords.reserve(sentence.size()); wrs.reserve(sentence.size()/2);
while (pre_filter.HasNext()) { while (pre_filter.HasNext()) {
range = pre_filter.Next(); range = pre_filter.Next();
Cut(range.begin, range.end, uwords, hmm); Cut(range.begin, range.end, wrs, hmm);
} }
TransCode::Encode(uwords, words); words.clear();
words.reserve(wrs.size());
unicode::GetStringsFromWordRanges(wrs, words);
} }
void Cut(Unicode::const_iterator begin, Unicode::const_iterator end, vector<Unicode>& res, bool hmm) const { void Cut(unicode::RuneStrArray::const_iterator begin, unicode::RuneStrArray::const_iterator end, vector<unicode::WordRange>& res, bool hmm) const {
//use mix Cut first //use mix Cut first
vector<Unicode> mixRes; vector<unicode::WordRange> mixRes;
mixSeg_.Cut(begin, end, mixRes, hmm); mixSeg_.Cut(begin, end, mixRes, hmm);
vector<Unicode> fullRes; vector<unicode::WordRange> fullRes;
for (vector<Unicode>::const_iterator mixResItr = mixRes.begin(); mixResItr != mixRes.end(); mixResItr++) { for (vector<unicode::WordRange>::const_iterator mixResItr = mixRes.begin(); mixResItr != mixRes.end(); mixResItr++) {
// if it's too long, Cut with fullSeg_, put fullRes in res // if it's too long, Cut with fullSeg_, put fullRes in res
if (mixResItr->size() > maxWordLen_ && !IsAllAscii(*mixResItr)) { if (mixResItr->Length() > maxWordLen_ && !mixResItr->IsAllAscii()) {
fullSeg_.Cut(mixResItr->begin(), mixResItr->end(), fullRes); fullSeg_.Cut(mixResItr->left, mixResItr->right + 1, fullRes);
for (vector<Unicode>::const_iterator fullResItr = fullRes.begin(); fullResItr != fullRes.end(); fullResItr++) { for (vector<unicode::WordRange>::const_iterator fullResItr = fullRes.begin(); fullResItr != fullRes.end(); fullResItr++) {
res.push_back(*fullResItr); res.push_back(*fullResItr);
} }

70
include/cppjieba/TransCode.hpp
View File

@ -1,70 +0,0 @@
/************************************
* file enc : utf-8
* author : wuyanyi09@gmail.com
************************************/
#ifndef CPPJIEBA_TRANSCODE_H
#define CPPJIEBA_TRANSCODE_H
#include "limonp/StringUtil.hpp"
#include "limonp/LocalVector.hpp"
namespace cppjieba {
using namespace limonp;
typedef uint32_t Rune;
typedef limonp::LocalVector<Rune> Unicode;
namespace TransCode {
inline bool Decode(const string& str, Unicode& res) {
#ifdef CPPJIEBA_GBK
return gbkTrans(str, res);
#else
return Utf8ToUnicode32(str, res);
#endif
}
inline void Encode(Unicode::const_iterator begin, Unicode::const_iterator end, string& res) {
#ifdef CPPJIEBA_GBK
gbkTrans(begin, end, res);
#else
Unicode32ToUtf8(begin, end, res);
#endif
}
inline void Encode(const Unicode& uni, string& res) {
Encode(uni.begin(), uni.end(), res);
}
// compiler is expected to optimized this function to avoid return value copy
inline string Encode(Unicode::const_iterator begin, Unicode::const_iterator end) {
string res;
res.reserve(end - begin);
Encode(begin, end, res);
return res;
}
inline string Encode(const Unicode& unicode) {
return Encode(unicode.begin(), unicode.end());
}
// compiler is expected to optimized this function to avoid return value copy
inline Unicode Decode(const string& str) {
Unicode unicode;
unicode.reserve(str.size());
Decode(str, unicode);
return unicode;
}
inline void Encode(const vector<Unicode>& input, vector<string>& output) {
output.resize(input.size());
for (size_t i = 0; i < output.size(); i++) {
Encode(input[i], output[i]);
}
}
} // namespace TransCode
} // namespace cppjieba
#endif

48
include/cppjieba/Trie.hpp
View File

@ -4,36 +4,41 @@
#include <vector> #include <vector>
#include <queue> #include <queue>
#include "limonp/StdExtension.hpp" #include "limonp/StdExtension.hpp"
#include "Trie.hpp" #include "Unicode.hpp"
namespace cppjieba { namespace cppjieba {
using namespace std; using namespace std;
using unicode::Rune;
using unicode::RuneStr;
using unicode::Unicode;
using unicode::WordRange;
const size_t MAX_WORD_LENGTH = 512; const size_t MAX_WORD_LENGTH = 512;
struct DictUnit { struct DictUnit {
Unicode word; unicode::Unicode word;
double weight; double weight;
string tag; string tag;
}; }; // struct DictUnit
// for debugging // for debugging
inline ostream & operator << (ostream& os, const DictUnit& unit) { // inline ostream & operator << (ostream& os, const DictUnit& unit) {
string s; // string s;
s << unit.word; // s << unit.word;
return os << StringFormat("%s %s %.3lf", s.c_str(), unit.tag.c_str(), unit.weight); // return os << StringFormat("%s %s %.3lf", s.c_str(), unit.tag.c_str(), unit.weight);
} // }
struct Dag { struct Dag {
Rune rune; RuneStr runestr;
LocalVector<pair<size_t, const DictUnit*> > nexts; // [offset, nexts.first]
limonp::LocalVector<pair<size_t, const DictUnit*> > nexts;
const DictUnit * pInfo; const DictUnit * pInfo;
double weight; double weight;
size_t nextPos; size_t nextPos; // TODO
Dag():rune(0), pInfo(NULL), weight(0.0), nextPos(0) { Dag():runestr(), pInfo(NULL), weight(0.0), nextPos(0) {
} }
}; }; // struct Dag
typedef Rune TrieKey; typedef Rune TrieKey;
@ -57,18 +62,18 @@ class Trie {
DeleteNode(root_); DeleteNode(root_);
} }
const DictUnit* Find(Unicode::const_iterator begin, Unicode::const_iterator end) const { const DictUnit* Find(unicode::RuneStrArray::const_iterator begin, unicode::RuneStrArray::const_iterator end) const {
if (begin == end) { if (begin == end) {
return NULL; return NULL;
} }
const TrieNode* ptNode = root_; const TrieNode* ptNode = root_;
TrieNode::NextMap::const_iterator citer; TrieNode::NextMap::const_iterator citer;
for (Unicode::const_iterator it = begin; it != end; it++) { for (unicode::RuneStrArray::const_iterator it = begin; it != end; it++) {
if (NULL == ptNode->next) { if (NULL == ptNode->next) {
return NULL; return NULL;
} }
citer = ptNode->next->find(*it); citer = ptNode->next->find(it->rune);
if (ptNode->next->end() == citer) { if (ptNode->next->end() == citer) {
return NULL; return NULL;
} }
@ -77,8 +82,8 @@ class Trie {
return ptNode->ptValue; return ptNode->ptValue;
} }
void Find(Unicode::const_iterator begin, void Find(unicode::RuneStrArray::const_iterator begin,
Unicode::const_iterator end, unicode::RuneStrArray::const_iterator end,
vector<struct Dag>&res, vector<struct Dag>&res,
size_t max_word_len = MAX_WORD_LENGTH) const { size_t max_word_len = MAX_WORD_LENGTH) const {
assert(root_ != NULL); assert(root_ != NULL);
@ -87,10 +92,9 @@ class Trie {
const TrieNode *ptNode = NULL; const TrieNode *ptNode = NULL;
TrieNode::NextMap::const_iterator citer; TrieNode::NextMap::const_iterator citer;
for (size_t i = 0; i < size_t(end - begin); i++) { for (size_t i = 0; i < size_t(end - begin); i++) {
Rune rune = *(begin + i); res[i].runestr = *(begin + i);
res[i].rune = rune;
if (root_->next != NULL && root_->next->end() != (citer = root_->next->find(rune))) { if (root_->next != NULL && root_->next->end() != (citer = root_->next->find(res[i].runestr.rune))) {
ptNode = citer->second; ptNode = citer->second;
} else { } else {
ptNode = NULL; ptNode = NULL;
@ -105,7 +109,7 @@ class Trie {
if (ptNode == NULL || ptNode->next == NULL) { if (ptNode == NULL || ptNode->next == NULL) {
break; break;
} }
citer = ptNode->next->find(*(begin + j)); citer = ptNode->next->find((begin + j)->rune);
if (ptNode->next->end() == citer) { if (ptNode->next->end() == citer) {
break; break;
} }

182
include/cppjieba/Unicode.hpp Normal file
View File

@ -0,0 +1,182 @@
#ifndef CPPJIEBA_UNICODE_H
#define CPPJIEBA_UNICODE_H
#include <stdint.h>
#include <stdlib.h>
#include <string>
#include <vector>
#include "limonp/LocalVector.hpp"
namespace cppjieba {
namespace unicode {
typedef uint32_t Rune;
struct RuneStr {
Rune rune;
const char* str;
uint32_t len;
RuneStr(): rune(0), str(NULL), len(0) {
}
RuneStr(Rune r, const char* s, uint32_t l)
: rune(r), str(s), len(l) {
}
}; // struct RuneStr
typedef limonp::LocalVector<Rune> Unicode;
typedef limonp::LocalVector<struct RuneStr> RuneStrArray;
// [left, right]
struct WordRange {
RuneStrArray::const_iterator left;
RuneStrArray::const_iterator right;
size_t Length() const {
return right - left + 1;
}
bool IsAllAscii() const {
for (RuneStrArray::const_iterator iter = left; iter <= right; ++iter) {
if (iter->rune >= 0x80) {
return false;
}
}
return true;
}
}; // struct WordRange
//struct RuneWordStr {
// Unicode word;
// const char* str;
// size_t len;
//}; // struct RuneWordStr
struct RuneStrLite {
uint32_t rune;
uint32_t len;
}; // struct RuneStrLite
inline RuneStrLite DecodeRuneInString(const char* str, size_t len) {
RuneStrLite rp = {0, 0};
if (str == NULL || len == 0) {
return rp;
}
if (!(str[0] & 0x80)) { // 0xxxxxxx
// 7bit, total 7bit
rp.rune = (uint8_t)(str[0]) & 0x7f;
rp.len = 1;
} else if ((uint8_t)str[0] <= 0xdf && 1 < len) {
// 110xxxxxx
// 5bit, total 5bit
rp.rune = (uint8_t)(str[0]) & 0x1f;
// 6bit, total 11bit
rp.rune <<= 6;
rp.rune |= (uint8_t)(str[1]) & 0x3f;
rp.len = 2;
} else if((uint8_t)str[0] <= 0xef && 2 < len) { // 1110xxxxxx
// 4bit, total 4bit
rp.rune = (uint8_t)(str[0]) & 0x0f;
// 6bit, total 10bit
rp.rune <<= 6;
rp.rune |= (uint8_t)(str[1]) & 0x3f;
// 6bit, total 16bit
rp.rune <<= 6;
rp.rune |= (uint8_t)(str[2]) & 0x3f;
rp.len = 3;
} else if((uint8_t)str[0] <= 0xf7 && 3 < len) { // 11110xxxx
// 3bit, total 3bit
rp.rune = (uint8_t)(str[0]) & 0x07;
// 6bit, total 9bit
rp.rune <<= 6;
rp.rune |= (uint8_t)(str[1]) & 0x3f;
// 6bit, total 15bit
rp.rune <<= 6;
rp.rune |= (uint8_t)(str[2]) & 0x3f;
// 6bit, total 21bit
rp.rune <<= 6;
rp.rune |= (uint8_t)(str[3]) & 0x3f;
rp.len = 4;
} else {
rp.rune = 0;
rp.len = 0;
}
return rp;
}
inline bool DecodeRunesInString(const char* s, size_t len, RuneStrArray& runes) {
runes.clear();
runes.reserve(len / 2);
for (size_t i = 0; i < len;) {
RuneStrLite rp = DecodeRuneInString(s + i, len - i);
if (rp.len == 0) {
return false;
}
RuneStr x = {rp.rune, s + i, rp.len};
runes.push_back(x);
i += rp.len;
}
return true;
}
inline bool DecodeRunesInString(const std::string& s, RuneStrArray& runes) {
return DecodeRunesInString(s.c_str(), s.size(), runes);
}
inline bool DecodeRunesInString(const char* s, size_t len, Unicode& unicode) {
unicode.clear();
RuneStrArray runes;
if (!DecodeRunesInString(s, len, runes)) {
return false;
}
unicode.reserve(runes.size());
for (size_t i = 0; i < runes.size(); i++) {
unicode.push_back(runes[i].rune);
}
return true;
}
inline bool IsSingleWord(const std::string& str) {
RuneStrLite rp = DecodeRuneInString(str.c_str(), str.size());
return rp.len == str.size();
}
inline bool DecodeRunesInString(const std::string& s, Unicode& unicode) {
return DecodeRunesInString(s.c_str(), s.size(), unicode);
}
inline Unicode DecodeRunesInString(const std::string& s) {
Unicode result;
DecodeRunesInString(s, result);
return result;
}
//[left, right]
inline std::string GetStringFromRunes(unicode::RuneStrArray::const_iterator left, unicode::RuneStrArray::const_iterator right) {
assert(right->str >= left->str);
return std::string(left->str, right->str - left->str + right->len);
}
inline void GetStringsFromWordRanges(const std::vector<WordRange>& wrs, std::vector<std::string>& words) {
for (size_t i = 0; i < wrs.size(); i++) {
words.push_back(GetStringFromRunes(wrs[i].left, wrs[i].right));
}
}
inline std::vector<std::string> GetStringsFromWordRanges(const std::vector<WordRange>& wrs) {
std::vector<std::string> result;
GetStringsFromWordRanges(wrs, result);
return result;
}
} // namespace unicode
} // namespace cppjieba
#endif // CPPJIEBA_UNICODE_H

20
test/demo.cpp
View File

@ -40,16 +40,16 @@ int main(int argc, char** argv) {
jieba.Cut("男默女泪", words); jieba.Cut("男默女泪", words);
cout << limonp::Join(words.begin(), words.end(), "/") << endl; cout << limonp::Join(words.begin(), words.end(), "/") << endl;
cout << "[demo] Locate Words" << endl; //cout << "[demo] Locate Words" << endl;
vector<cppjieba::Jieba::LocWord> loc_words; //vector<cppjieba::Jieba::LocWord> loc_words;
jieba.Cut("南京市长江大桥", words, true); //jieba.Cut("南京市长江大桥", words, true);
cppjieba::Jieba::Locate(words, loc_words); //cppjieba::Jieba::Locate(words, loc_words);
for (size_t i = 0; i < loc_words.size(); i++) { //for (size_t i = 0; i < loc_words.size(); i++) {
cout << loc_words[i].word // cout << loc_words[i].word
<< ", " << loc_words[i].begin // << ", " << loc_words[i].begin
<< ", " << loc_words[i].end // << ", " << loc_words[i].end
<< endl; // << endl;
} //}
cout << "[demo] TAGGING" << endl; cout << "[demo] TAGGING" << endl;
vector<pair<string, string> > tagres; vector<pair<string, string> > tagres;

34
test/unittest/jieba_test.cpp
View File

@ -37,25 +37,25 @@ TEST(JiebaTest, Test1) {
result << words; result << words;
ASSERT_EQ("[\"他\", \"来到\", \"了\", \"网易\", \"杭研\", \"大厦\"]", result); ASSERT_EQ("[\"他\", \"来到\", \"了\", \"网易\", \"杭研\", \"大厦\"]", result);
jieba.CutLevel("南京市长江大桥", words); //jieba.CutLevel("南京市长江大桥", words);
result << words; //result << words;
ASSERT_EQ("[\"南京市\", \"长江大桥\", \"南京\", \"长江\", \"大桥\"]", result); //ASSERT_EQ("[\"南京市\", \"长江大桥\", \"南京\", \"长江\", \"大桥\"]", result);
vector<pair<string, size_t> > word_levels; //vector<pair<string, size_t> > word_levels;
jieba.CutLevel("南京市长江大桥", word_levels); //jieba.CutLevel("南京市长江大桥", word_levels);
result << word_levels; //result << word_levels;
ASSERT_EQ("[\"南京市:0\", \"长江大桥:0\", \"南京:1\", \"长江:1\", \"大桥:1\"]", result); //ASSERT_EQ("[\"南京市:0\", \"长江大桥:0\", \"南京:1\", \"长江:1\", \"大桥:1\"]", result);
vector<Jieba::LocWord> loc_words; //vector<Jieba::LocWord> loc_words;
jieba.Cut("南京市长江大桥", words); //jieba.Cut("南京市长江大桥", words);
jieba.Locate(words, loc_words); //jieba.Locate(words, loc_words);
ASSERT_EQ(loc_words.size(), 2u); //ASSERT_EQ(loc_words.size(), 2u);
ASSERT_EQ(loc_words[0].word, "南京市"); //ASSERT_EQ(loc_words[0].word, "南京市");
ASSERT_EQ(loc_words[0].begin, 0u); //ASSERT_EQ(loc_words[0].begin, 0u);
ASSERT_EQ(loc_words[0].end, 3u); //ASSERT_EQ(loc_words[0].end, 3u);
ASSERT_EQ(loc_words[1].word, "长江大桥"); //ASSERT_EQ(loc_words[1].word, "长江大桥");
ASSERT_EQ(loc_words[1].begin, 3u); //ASSERT_EQ(loc_words[1].begin, 3u);
ASSERT_EQ(loc_words[1].end, 7u); //ASSERT_EQ(loc_words[1].end, 7u);
//vector<pair<string, string> > tagres; //vector<pair<string, string> > tagres;
//jieba.Tag("iPhone6手机的最大特点是很容易弯曲。", tagres); //jieba.Tag("iPhone6手机的最大特点是很容易弯曲。", tagres);

17
test/unittest/pre_filter_test.cpp
View File

@ -1,5 +1,6 @@
#include "gtest/gtest.h" #include "gtest/gtest.h"
#include "cppjieba/PreFilter.hpp" #include "cppjieba/PreFilter.hpp"
#include "limonp/StringUtil.hpp"
using namespace cppjieba; using namespace cppjieba;
@ -11,32 +12,32 @@ TEST(PreFilterTest, Test1) {
string res; string res;
{ {
PreFilter filter(symbol, "你好，美丽的，世界"); string s = "你好，美丽的，世界";
PreFilter filter(symbol, s);
expected = "你好/，/美丽的/，/世界"; expected = "你好/，/美丽的/，/世界";
ASSERT_TRUE(filter.HasNext()); ASSERT_TRUE(filter.HasNext());
vector<string> words; vector<string> words;
while (filter.HasNext()) { while (filter.HasNext()) {
PreFilter::Range range; PreFilter::Range range;
range = filter.Next(); range = filter.Next();
words.push_back(TransCode::Encode(range.begin, range.end)); words.push_back(unicode::GetStringFromRunes(range.begin, range.end - 1));
} }
res = Join(words.begin(), words.end(), "/"); res = limonp::Join(words.begin(), words.end(), "/");
ASSERT_EQ(res, expected); ASSERT_EQ(res, expected);
} }
{ {
PreFilter filter(symbol, "我来自北京邮电大学。。。学号123456，用AK47"); string s = "我来自北京邮电大学。。。学号123456，用AK47";
PreFilter filter(symbol, s);
expected = "我来自北京邮电大学/。/。/。/学号123456/，/用AK47"; expected = "我来自北京邮电大学/。/。/。/学号123456/，/用AK47";
ASSERT_TRUE(filter.HasNext()); ASSERT_TRUE(filter.HasNext());
vector<string> words; vector<string> words;
while (filter.HasNext()) { while (filter.HasNext()) {
PreFilter::Range range; PreFilter::Range range;
range = filter.Next(); range = filter.Next();
words.push_back(TransCode::Encode(range.begin, range.end)); words.push_back(unicode::GetStringFromRunes(range.begin, range.end - 1));
}
res = Join(words.begin(), words.end(), "/");
for (size_t i = 0; i < words.size(); i++) {
} }
res = limonp::Join(words.begin(), words.end(), "/");
ASSERT_EQ(res, expected); ASSERT_EQ(res, expected);
} }
} }

4
test/unittest/segments_test.cpp
View File

@ -4,7 +4,7 @@
#include "cppjieba/HMMSegment.hpp" #include "cppjieba/HMMSegment.hpp"
#include "cppjieba/FullSegment.hpp" #include "cppjieba/FullSegment.hpp"
#include "cppjieba/QuerySegment.hpp" #include "cppjieba/QuerySegment.hpp"
#include "cppjieba/LevelSegment.hpp" //#include "cppjieba/LevelSegment.hpp"
#include "gtest/gtest.h" #include "gtest/gtest.h"
using namespace cppjieba; using namespace cppjieba;
@ -238,6 +238,7 @@ TEST(QuerySegment, Test2) {
} }
} }
/*
TEST(LevelSegmentTest, Test0) { TEST(LevelSegmentTest, Test0) {
string s; string s;
LevelSegment segment("../test/testdata/extra_dict/jieba.dict.small.utf8"); LevelSegment segment("../test/testdata/extra_dict/jieba.dict.small.utf8");
@ -249,6 +250,7 @@ TEST(LevelSegmentTest, Test0) {
segment.Cut("南京市长江大桥", res); segment.Cut("南京市长江大桥", res);
ASSERT_EQ("[\"南京市\", \"长江大桥\", \"南京\", \"长江\", \"大桥\"]", s << res); ASSERT_EQ("[\"南京市\", \"长江大桥\", \"南京\", \"长江\", \"大桥\"]", s << res);
} }
*/
TEST(MPSegmentTest, Unicode32) { TEST(MPSegmentTest, Unicode32) {
string s("天气很好，🙋 我们去郊游。"); string s("天气很好，🙋 我们去郊游。");

67
test/unittest/trie_test.cpp
View File

@ -15,7 +15,7 @@ TEST(TrieTest, Empty) {
TEST(TrieTest, Construct) { TEST(TrieTest, Construct) {
vector<Unicode> keys; vector<Unicode> keys;
vector<const DictUnit*> values; vector<const DictUnit*> values;
keys.push_back(TransCode::Decode("你")); keys.push_back(unicode::DecodeRunesInString("你"));
values.push_back((const DictUnit*)(NULL)); values.push_back((const DictUnit*)(NULL));
Trie trie(keys, values); Trie trie(keys, values);
} }
@ -31,27 +31,34 @@ TEST(DictTrieTest, Test1) {
DictTrie trie(DICT_FILE); DictTrie trie(DICT_FILE);
ASSERT_LT(trie.GetMinWeight() + 15.6479, 0.001); ASSERT_LT(trie.GetMinWeight() + 15.6479, 0.001);
string word("来到"); string word("来到");
Unicode uni; cppjieba::unicode::RuneStrArray uni;
ASSERT_TRUE(TransCode::Decode(word, uni)); ASSERT_TRUE(unicode::DecodeRunesInString(word, uni));
DictUnit nodeInfo; //DictUnit nodeInfo;
nodeInfo.word = uni; //nodeInfo.word = uni;
nodeInfo.tag = "v"; //nodeInfo.tag = "v";
nodeInfo.weight = -8.87033; //nodeInfo.weight = -8.87033;
s1 << nodeInfo; //s1 << nodeInfo;
s2 << (*trie.Find(uni.begin(), uni.end())); //s2 << (*trie.Find(uni.begin(), uni.end()));
const DictUnit* du = trie.Find(uni.begin(), uni.end());
ASSERT_TRUE(du != NULL);
ASSERT_EQ(2u, du->word.size());
ASSERT_EQ(26469u, du->word[0]);
ASSERT_EQ(21040u, du->word[1]);
ASSERT_EQ("v", du->tag);
ASSERT_NEAR(-8.870, du->weight, 0.001);
EXPECT_EQ("[\"26469\", \"21040\"] v -8.870", s2); //EXPECT_EQ("[\"26469\", \"21040\"] v -8.870", s2);
word = "清华大学"; word = "清华大学";
LocalVector<pair<size_t, const DictUnit*> > res; LocalVector<pair<size_t, const DictUnit*> > res;
const char * words[] = {"清", "清华", "清华大学"}; const char * words[] = {"清", "清华", "清华大学"};
for (size_t i = 0; i < sizeof(words)/sizeof(words[0]); i++) { for (size_t i = 0; i < sizeof(words)/sizeof(words[0]); i++) {
ASSERT_TRUE(TransCode::Decode(words[i], uni)); ASSERT_TRUE(unicode::DecodeRunesInString(words[i], uni));
res.push_back(make_pair(uni.size() - 1, trie.Find(uni.begin(), uni.end()))); res.push_back(make_pair(uni.size() - 1, trie.Find(uni.begin(), uni.end())));
//resMap[uni.size() - 1] = trie.Find(uni.begin(), uni.end()); //resMap[uni.size() - 1] = trie.Find(uni.begin(), uni.end());
} }
vector<pair<size_t, const DictUnit*> > vec; vector<pair<size_t, const DictUnit*> > vec;
vector<struct Dag> dags; vector<struct Dag> dags;
ASSERT_TRUE(TransCode::Decode(word, uni)); ASSERT_TRUE(unicode::DecodeRunesInString(word, uni));
trie.Find(uni.begin(), uni.end(), dags); trie.Find(uni.begin(), uni.end(), dags);
ASSERT_EQ(dags.size(), uni.size()); ASSERT_EQ(dags.size(), uni.size());
ASSERT_NE(dags.size(), 0u); ASSERT_NE(dags.size(), 0u);
@ -64,25 +71,21 @@ TEST(DictTrieTest, Test1) {
TEST(DictTrieTest, UserDict) { TEST(DictTrieTest, UserDict) {
DictTrie trie(DICT_FILE, "../test/testdata/userdict.utf8"); DictTrie trie(DICT_FILE, "../test/testdata/userdict.utf8");
string word = "云计算"; string word = "云计算";
Unicode unicode; cppjieba::unicode::RuneStrArray unicode;
ASSERT_TRUE(TransCode::Decode(word, unicode)); ASSERT_TRUE(unicode::DecodeRunesInString(word, unicode));
const DictUnit * unit = trie.Find(unicode.begin(), unicode.end()); const DictUnit * unit = trie.Find(unicode.begin(), unicode.end());
ASSERT_TRUE(unit); ASSERT_TRUE(unit);
string res ; ASSERT_NEAR(unit->weight, -14.100, 0.001);
res << *unit;
ASSERT_EQ("[\"20113\", \"35745\", \"31639\"] -14.100", res);
} }
TEST(DictTrieTest, UserDictWithMaxWeight) { TEST(DictTrieTest, UserDictWithMaxWeight) {
DictTrie trie(DICT_FILE, "../test/testdata/userdict.utf8", DictTrie::WordWeightMax); DictTrie trie(DICT_FILE, "../test/testdata/userdict.utf8", DictTrie::WordWeightMax);
string word = "云计算"; string word = "云计算";
Unicode unicode; cppjieba::unicode::RuneStrArray unicode;
ASSERT_TRUE(TransCode::Decode(word, unicode)); ASSERT_TRUE(unicode::DecodeRunesInString(word, unicode));
const DictUnit * unit = trie.Find(unicode.begin(), unicode.end()); const DictUnit * unit = trie.Find(unicode.begin(), unicode.end());
ASSERT_TRUE(unit); ASSERT_TRUE(unit);
string res ; ASSERT_NEAR(unit->weight, -2.975, 0.001);
res << *unit;
ASSERT_EQ("[\"20113\", \"35745\", \"31639\"] -2.975", res);
} }
TEST(DictTrieTest, Dag) { TEST(DictTrieTest, Dag) {
@ -90,8 +93,8 @@ TEST(DictTrieTest, Dag) {
{ {
string word = "清华大学"; string word = "清华大学";
Unicode unicode; cppjieba::unicode::RuneStrArray unicode;
ASSERT_TRUE(TransCode::Decode(word, unicode)); ASSERT_TRUE(unicode::DecodeRunesInString(word, unicode));
vector<struct Dag> res; vector<struct Dag> res;
trie.Find(unicode.begin(), unicode.end(), res); trie.Find(unicode.begin(), unicode.end(), res);
@ -104,8 +107,8 @@ TEST(DictTrieTest, Dag) {
{ {
string word = "北京邮电大学"; string word = "北京邮电大学";
Unicode unicode; cppjieba::unicode::RuneStrArray unicode;
ASSERT_TRUE(TransCode::Decode(word, unicode)); ASSERT_TRUE(unicode::DecodeRunesInString(word, unicode));
vector<struct Dag> res; vector<struct Dag> res;
trie.Find(unicode.begin(), unicode.end(), res); trie.Find(unicode.begin(), unicode.end(), res);
@ -118,8 +121,8 @@ TEST(DictTrieTest, Dag) {
{ {
string word = "长江大桥"; string word = "长江大桥";
Unicode unicode; cppjieba::unicode::RuneStrArray unicode;
ASSERT_TRUE(TransCode::Decode(word, unicode)); ASSERT_TRUE(unicode::DecodeRunesInString(word, unicode));
vector<struct Dag> res; vector<struct Dag> res;
trie.Find(unicode.begin(), unicode.end(), res); trie.Find(unicode.begin(), unicode.end(), res);
@ -132,8 +135,8 @@ TEST(DictTrieTest, Dag) {
{ {
string word = "长江大桥"; string word = "长江大桥";
Unicode unicode; cppjieba::unicode::RuneStrArray unicode;
ASSERT_TRUE(TransCode::Decode(word, unicode)); ASSERT_TRUE(unicode::DecodeRunesInString(word, unicode));
vector<struct Dag> res; vector<struct Dag> res;
trie.Find(unicode.begin(), unicode.end(), res, 3); trie.Find(unicode.begin(), unicode.end(), res, 3);
@ -146,8 +149,8 @@ TEST(DictTrieTest, Dag) {
{ {
string word = "长江大桥"; string word = "长江大桥";
Unicode unicode; cppjieba::unicode::RuneStrArray unicode;
ASSERT_TRUE(TransCode::Decode(word, unicode)); ASSERT_TRUE(unicode::DecodeRunesInString(word, unicode));
vector<struct Dag> res; vector<struct Dag> res;
trie.Find(unicode.begin(), unicode.end(), res, 4); trie.Find(unicode.begin(), unicode.end(), res, 4);