third_codes/cppjieba
1
0
Fork 0
mirror of https://github.com/yanyiwu/cppjieba.git synced 2025-07-18 00:00:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

astyle --style=google --indent=spaces=2

Browse Source
This commit is contained in:
yanyiwu 2015-05-06 17:53:20 +08:00
parent b70875f412
commit bb32234654
21 changed files with 1727 additions and 2071 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
server/server.cpp
View File

@ -12,29 +12,25 @@
using namespace Husky;
using namespace CppJieba;
class ReqHandler: public IRequestHandler
{
class ReqHandler: public IRequestHandler {
public:
ReqHandler(const string& dictPath, const string& modelPath, const string& userDictPath): _segment(dictPath, modelPath, userDictPath) {};
virtual ~ReqHandler() {};
public:
virtual bool do_GET(const HttpReqInfo& httpReq, string& strSnd) const
{
virtual bool do_GET(const HttpReqInfo& httpReq, string& strSnd) const {
string sentence, tmp;
vector<string> words;
httpReq.GET("key", tmp);
URLDecode(tmp, sentence);
_segment.cut(sentence, words);
if(httpReq.GET("format", tmp) && tmp == "simple")
{
if(httpReq.GET("format", tmp) && tmp == "simple") {
join(words.begin(), words.end(), strSnd, " ");
return true;
}
strSnd << words;
return true;
}
virtual bool do_POST(const HttpReqInfo& httpReq, string& strSnd) const
{
virtual bool do_POST(const HttpReqInfo& httpReq, string& strSnd) const {
vector<string> words;
_segment.cut(httpReq.getBody(), words);
strSnd << words;
@ -44,15 +40,12 @@ class ReqHandler: public IRequestHandler
MixSegment _segment;
};
bool run(int argc, char** argv)
{
if(argc < 2)
{
bool run(int argc, char** argv) {
if(argc < 2) {
return false;
}
Config conf(argv[1]);
if(!conf)
{
if(!conf) {
return false;
}
int port = 0;
@ -66,8 +59,7 @@ bool run(int argc, char** argv)
LIMONP_CHECK(conf.get("queue_max_size", queueMaxSize));
LIMONP_CHECK(conf.get("dict_path", dictPath));
LIMONP_CHECK(conf.get("model_path", modelPath));
if(!conf.get("user_dict_path", userDictPath)) //optional
{
if(!conf.get("user_dict_path", userDictPath)) { //optional
userDictPath = "";
}
@ -79,10 +71,8 @@ bool run(int argc, char** argv)
}
int main(int argc, char* argv[])
{
if(!run(argc, argv))
{
int main(int argc, char* argv[]) {
if(!run(argc, argv)) {
printf("usage: %s <config_file>\n", argv[0]);
return EXIT_FAILURE;
}

106
src/DictTrie.hpp
View File

@ -15,48 +15,39 @@
namespace CppJieba
{
namespace CppJieba {
using namespace Limonp;
const double MIN_DOUBLE = -3.14e+100;
const double MAX_DOUBLE = 3.14e+100;
const size_t DICT_COLUMN_NUM = 3;
const char* const UNKNOWN_TAG = "";
class DictTrie
{
class DictTrie {
public:
DictTrie()
{
DictTrie() {
_trie = NULL;
_minWeight = MAX_DOUBLE;
}
DictTrie(const string& dictPath, const string& userDictPath = "")
{
DictTrie(const string& dictPath, const string& userDictPath = "") {
new (this) DictTrie();
init(dictPath, userDictPath);
}
~DictTrie()
{
if(_trie)
{
~DictTrie() {
if(_trie) {
delete _trie;
}
}
bool init(const string& dictPath, const string& userDictPath = "")
{
if(_trie != NULL)
{
bool init(const string& dictPath, const string& userDictPath = "") {
if(_trie != NULL) {
LogFatal("trie already initted");
}
_loadDict(dictPath);
_calculateWeight(_nodeInfos);
_minWeight = _findMinWeight(_nodeInfos);
if(userDictPath.size())
{
if(userDictPath.size()) {
double maxWeight = _findMaxWeight(_nodeInfos);
_loadUserDict(userDictPath, maxWeight, UNKNOWN_TAG);
}
@ -66,37 +57,33 @@ namespace CppJieba
return true;
}
const DictUnit* find(Unicode::const_iterator begin, Unicode::const_iterator end) const
{
const DictUnit* find(Unicode::const_iterator begin, Unicode::const_iterator end) const {
return _trie->find(begin, end);
}
bool find(Unicode::const_iterator begin, Unicode::const_iterator end, DagType& dag, size_t offset = 0) const
{
bool find(Unicode::const_iterator begin, Unicode::const_iterator end, DagType& dag, size_t offset = 0) const {
return _trie->find(begin, end, dag, offset);
}
void find(
Unicode::const_iterator begin,
Unicode::const_iterator end,
vector<SegmentChar>& res
) const
{
) const {
_trie->find(begin, end, res);
}
bool isUserDictSingleChineseWord(const Unicode::value_type& word) const
{
bool isUserDictSingleChineseWord(const Unicode::value_type& word) const {
return isIn(_userDictSingleChineseWord, word);
}
double getMinWeight() const {return _minWeight;};
double getMinWeight() const {
return _minWeight;
};
private:
Trie * _createTrie(const vector<DictUnit>& dictUnits)
{
Trie * _createTrie(const vector<DictUnit>& dictUnits) {
assert(dictUnits.size());
vector<Unicode> words;
vector<const DictUnit*> valuePointers;
for(size_t i = 0 ; i < dictUnits.size(); i ++)
{
for(size_t i = 0 ; i < dictUnits.size(); i ++) {
words.push_back(dictUnits[i].word);
valuePointers.push_back(&dictUnits[i]);
}
@ -104,32 +91,26 @@ namespace CppJieba
Trie * trie = new Trie(words, valuePointers);
return trie;
}
void _loadUserDict(const string& filePath, double defaultWeight, const string& defaultTag)
{
void _loadUserDict(const string& filePath, double defaultWeight, const string& defaultTag) {
ifstream ifs(filePath.c_str());
if(!ifs.is_open())
{
if(!ifs.is_open()) {
LogFatal("file %s open failed.", filePath.c_str());
}
string line;
DictUnit nodeInfo;
vector<string> buf;
size_t lineno;
for(lineno = 0; getline(ifs, line); lineno++)
{
for(lineno = 0; getline(ifs, line); lineno++) {
buf.clear();
split(line, buf, " ");
if(buf.size() < 1)
{
if(buf.size() < 1) {
LogFatal("split [%s] result illegal", line.c_str());
}
if(!TransCode::decode(buf[0], nodeInfo.word))
{
if(!TransCode::decode(buf[0], nodeInfo.word)) {
LogError("line[%u:%s] illegal.", lineno, line.c_str());
continue;
}
if(nodeInfo.word.size() == 1)
{
if(nodeInfo.word.size() == 1) {
_userDictSingleChineseWord.insert(nodeInfo.word[0]);
}
nodeInfo.weight = defaultWeight;
@ -138,27 +119,22 @@ namespace CppJieba
}
LogInfo("load userdict[%s] ok. lines[%u]", filePath.c_str(), lineno);
}
void _loadDict(const string& filePath)
{
void _loadDict(const string& filePath) {
ifstream ifs(filePath.c_str());
if(!ifs.is_open())
{
if(!ifs.is_open()) {
LogFatal("file %s open failed.", filePath.c_str());
}
string line;
vector<string> buf;
DictUnit nodeInfo;
for(size_t lineno = 0; getline(ifs, line); lineno++)
{
for(size_t lineno = 0; getline(ifs, line); lineno++) {
split(line, buf, " ");
if(buf.size() != DICT_COLUMN_NUM)
{
if(buf.size() != DICT_COLUMN_NUM) {
LogFatal("split result illegal, line: %s, result size: %u", line.c_str(), buf.size());
}
if(!TransCode::decode(buf[0], nodeInfo.word))
{
if(!TransCode::decode(buf[0], nodeInfo.word)) {
LogError("line[%u:%s] illegal.", lineno, line.c_str());
continue;
}
@ -168,43 +144,35 @@ namespace CppJieba
_nodeInfos.push_back(nodeInfo);
}
}
double _findMinWeight(const vector<DictUnit>& nodeInfos) const
{
double _findMinWeight(const vector<DictUnit>& nodeInfos) const {
double ret = MAX_DOUBLE;
for(size_t i = 0; i < nodeInfos.size(); i++)
{
for(size_t i = 0; i < nodeInfos.size(); i++) {
ret = min(nodeInfos[i].weight, ret);
}
return ret;
}
double _findMaxWeight(const vector<DictUnit>& nodeInfos) const
{
double _findMaxWeight(const vector<DictUnit>& nodeInfos) const {
double ret = MIN_DOUBLE;
for(size_t i = 0; i < nodeInfos.size(); i++)
{
for(size_t i = 0; i < nodeInfos.size(); i++) {
ret = max(nodeInfos[i].weight, ret);
}
return ret;
}
void _calculateWeight(vector<DictUnit>& nodeInfos) const
{
void _calculateWeight(vector<DictUnit>& nodeInfos) const {
double sum = 0.0;
for(size_t i = 0; i < nodeInfos.size(); i++)
{
for(size_t i = 0; i < nodeInfos.size(); i++) {
sum += nodeInfos[i].weight;
}
assert(sum);
for(size_t i = 0; i < nodeInfos.size(); i++)
{
for(size_t i = 0; i < nodeInfos.size(); i++) {
DictUnit& nodeInfo = nodeInfos[i];
assert(nodeInfo.weight);
nodeInfo.weight = log(double(nodeInfo.weight)/double(sum));
}
}
void _shrink(vector<DictUnit>& units) const
{
void _shrink(vector<DictUnit>& units) const {
vector<DictUnit>(units.begin(), units.end()).swap(units);
}

68
src/FullSegment.hpp
View File

@ -10,43 +10,34 @@
#include "SegmentBase.hpp"
#include "TransCode.hpp"
namespace CppJieba
{
class FullSegment: public SegmentBase
{
namespace CppJieba {
class FullSegment: public SegmentBase {
public:
FullSegment()
{
FullSegment() {
_dictTrie = NULL;
_isBorrowed = false;
}
explicit FullSegment(const string& dictPath)
{
explicit FullSegment(const string& dictPath) {
_dictTrie = NULL;
init(dictPath);
}
explicit FullSegment(const DictTrie* dictTrie)
{
explicit FullSegment(const DictTrie* dictTrie) {
_dictTrie = NULL;
init(dictTrie);
}
virtual ~FullSegment()
{
if(_dictTrie && ! _isBorrowed)
{
virtual ~FullSegment() {
if(_dictTrie && ! _isBorrowed) {
delete _dictTrie;
}
};
bool init(const string& dictPath)
{
bool init(const string& dictPath) {
assert(_dictTrie == NULL);
_dictTrie = new DictTrie(dictPath);
_isBorrowed = false;
return true;
}
bool init(const DictTrie* dictTrie)
{
bool init(const DictTrie* dictTrie) {
assert(_dictTrie == NULL);
assert(dictTrie);
_dictTrie = dictTrie;
@ -55,11 +46,9 @@ namespace CppJieba
}
using SegmentBase::cut;
bool cut(Unicode::const_iterator begin, Unicode::const_iterator end, vector<Unicode>& res) const
{
bool cut(Unicode::const_iterator begin, Unicode::const_iterator end, vector<Unicode>& res) const {
assert(_dictTrie);
if (begin >= end)
{
if (begin >= end) {
LogError("begin >= end");
return false;
}
@ -75,27 +64,21 @@ namespace CppJieba
//tmp variables
int wordLen = 0;
for (Unicode::const_iterator uItr = begin; uItr != end; uItr++)
{
for (Unicode::const_iterator uItr = begin; uItr != end; uItr++) {
//find word start from uItr
if (_dictTrie->find(uItr, end, tRes, 0))
{
if (_dictTrie->find(uItr, end, tRes, 0)) {
for(DagType::const_iterator itr = tRes.begin(); itr != tRes.end(); itr++)
//for (vector<pair<size_t, const DictUnit*> >::const_iterator itr = tRes.begin(); itr != tRes.end(); itr++)
{
wordLen = itr->second->word.size();
if (wordLen >= 2 || (tRes.size() == 1 && maxIdx <= uIdx))
{
if (wordLen >= 2 || (tRes.size() == 1 && maxIdx <= uIdx)) {
res.push_back(itr->second->word);
}
maxIdx = uIdx+wordLen > maxIdx ? uIdx+wordLen : maxIdx;
}
tRes.clear();
}
else // not found word start from uItr
{
if (maxIdx <= uIdx) // never exist in prev results
{
} else { // not found word start from uItr
if (maxIdx <= uIdx) { // never exist in prev results
//put itr itself in res
res.push_back(Unicode(1, *uItr));
@ -109,31 +92,24 @@ namespace CppJieba
return true;
}
bool cut(Unicode::const_iterator begin, Unicode::const_iterator end, vector<string>& res) const
{
bool cut(Unicode::const_iterator begin, Unicode::const_iterator end, vector<string>& res) const {
assert(_dictTrie);
if (begin >= end)
{
if (begin >= end) {
LogError("begin >= end");
return false;
}
vector<Unicode> uRes;
if (!cut(begin, end, uRes))
{
if (!cut(begin, end, uRes)) {
LogError("get unicode cut result error.");
return false;
}
string tmp;
for (vector<Unicode>::const_iterator uItr = uRes.begin(); uItr != uRes.end(); uItr++)
{
if (TransCode::encode(*uItr, tmp))
{
for (vector<Unicode>::const_iterator uItr = uRes.begin(); uItr != uRes.end(); uItr++) {
if (TransCode::encode(*uItr, tmp)) {
res.push_back(tmp);
}
else
{
} else {
LogError("encode failed.");
}
}

204
src/HMMSegment.hpp
View File

@ -12,12 +12,10 @@
#include "SegmentBase.hpp"
#include "DictTrie.hpp"
namespace CppJieba
{
namespace CppJieba {
using namespace Limonp;
typedef unordered_map<uint16_t, double> EmitProbMap;
class HMMSegment: public SegmentBase
{
class HMMSegment: public SegmentBase {
public:
/*
* STATUS:
@ -27,14 +25,12 @@ namespace CppJieba
public:
HMMSegment() {}
explicit HMMSegment(const string& filePath)
{
explicit HMMSegment(const string& filePath) {
LIMONP_CHECK(init(filePath));
}
virtual ~HMMSegment() {}
public:
bool init(const string& filePath)
{
bool init(const string& filePath) {
memset(_startProb, 0, sizeof(_startProb));
memset(_transProb, 0, sizeof(_transProb));
_statMap[0] = 'B';
@ -52,65 +48,51 @@ namespace CppJieba
public:
using SegmentBase::cut;
public:
bool cut(Unicode::const_iterator begin, Unicode::const_iterator end, vector<Unicode>& res)const
{
bool cut(Unicode::const_iterator begin, Unicode::const_iterator end, vector<Unicode>& res)const {
Unicode::const_iterator left = begin;
Unicode::const_iterator right = begin;
while(right != end)
{
if(*right < 0x80)
{
if(left != right && !_cut(left, right, res))
{
while(right != end) {
if(*right < 0x80) {
if(left != right && !_cut(left, right, res)) {
return false;
}
left = right;
do {
right = _sequentialLetterRule(left, end);
if(right != left)
{
if(right != left) {
break;
}
right = _numbersRule(left, end);
if(right != left)
{
if(right != left) {
break;
}
right ++;
} while(false);
res.push_back(Unicode(left, right));
left = right;
}
else
{
} else {
right++;
}
}
if(left != right && !_cut(left, right, res))
{
if(left != right && !_cut(left, right, res)) {
return false;
}
return true;
}
public:
virtual bool cut(Unicode::const_iterator begin, Unicode::const_iterator end, vector<string>& res)const
{
if(begin == end)
{
virtual bool cut(Unicode::const_iterator begin, Unicode::const_iterator end, vector<string>& res)const {
if(begin == end) {
return false;
}
vector<Unicode> words;
words.reserve(end - begin);
if(!cut(begin, end, words))
{
if(!cut(begin, end, words)) {
return false;
}
size_t offset = res.size();
res.resize(res.size() + words.size());
for(size_t i = 0; i < words.size(); i++)
{
if(!TransCode::encode(words[i], res[offset + i]))
{
for(size_t i = 0; i < words.size(); i++) {
if(!TransCode::encode(words[i], res[offset + i])) {
LogError("encode failed.");
}
}
@ -118,72 +100,52 @@ namespace CppJieba
}
private:
// sequential letters rule
Unicode::const_iterator _sequentialLetterRule(Unicode::const_iterator begin, Unicode::const_iterator end) const
{
Unicode::const_iterator _sequentialLetterRule(Unicode::const_iterator begin, Unicode::const_iterator end) const {
Unicode::value_type x = *begin;
if (('a' <= x && x <= 'z') || ('A' <= x && x <= 'Z'))
{
if (('a' <= x && x <= 'z') || ('A' <= x && x <= 'Z')) {
begin ++;
}
else
{
} else {
return begin;
}
while(begin != end)
{
while(begin != end) {
x = *begin;
if(('a' <= x && x <= 'z') || ('A' <= x && x <= 'Z') || ('0' <= x && x <= '9'))
{
if(('a' <= x && x <= 'z') || ('A' <= x && x <= 'Z') || ('0' <= x && x <= '9')) {
begin ++;
}
else
{
} else {
break;
}
}
return begin;
}
//
Unicode::const_iterator _numbersRule(Unicode::const_iterator begin, Unicode::const_iterator end) const
{
Unicode::const_iterator _numbersRule(Unicode::const_iterator begin, Unicode::const_iterator end) const {
Unicode::value_type x = *begin;
if('0' <= x && x <= '9')
{
if('0' <= x && x <= '9') {
begin ++;
}
else
{
} else {
return begin;
}
while(begin != end)
{
while(begin != end) {
x = *begin;
if( ('0' <= x && x <= '9') || x == '.')
{
if( ('0' <= x && x <= '9') || x == '.') {
begin++;
}
else
{
} else {
break;
}
}
return begin;
}
bool _cut(Unicode::const_iterator begin, Unicode::const_iterator end, vector<Unicode>& res) const
{
bool _cut(Unicode::const_iterator begin, Unicode::const_iterator end, vector<Unicode>& res) const {
vector<size_t> status;
if(!_viterbi(begin, end, status))
{
if(!_viterbi(begin, end, status)) {
LogError("_viterbi failed.");
return false;
}
Unicode::const_iterator left = begin;
Unicode::const_iterator right;
for(size_t i = 0; i < status.size(); i++)
{
if(status[i] % 2) //if(E == status[i] || S == status[i])
{
for(size_t i = 0; i < status.size(); i++) {
if(status[i] % 2) { //if(E == status[i] || S == status[i])
right = begin + i + 1;
res.push_back(Unicode(left, right));
left = right;
@ -192,10 +154,8 @@ namespace CppJieba
return true;
}
bool _viterbi(Unicode::const_iterator begin, Unicode::const_iterator end, vector<size_t>& status)const
{
if(begin == end)
{
bool _viterbi(Unicode::const_iterator begin, Unicode::const_iterator end, vector<size_t>& status)const {
if(begin == end) {
return false;
}
@ -210,8 +170,7 @@ namespace CppJieba
vector<double> weight(XYSize);
//start
for(size_t y = 0; y < Y; y++)
{
for(size_t y = 0; y < Y; y++) {
weight[0 + y * X] = _startProb[y] + _getEmitProb(_emitProbVec[y], *begin, MIN_DOUBLE);
path[0 + y * X] = -1;
}
@ -219,20 +178,16 @@ namespace CppJieba
double emitProb;
for(size_t x = 1; x < X; x++)
{
for(size_t y = 0; y < Y; y++)
{
for(size_t x = 1; x < X; x++) {
for(size_t y = 0; y < Y; y++) {
now = x + y*X;
weight[now] = MIN_DOUBLE;
path[now] = E; // warning
emitProb = _getEmitProb(_emitProbVec[y], *(begin+x), MIN_DOUBLE);
for(size_t preY = 0; preY < Y; preY++)
{
for(size_t preY = 0; preY < Y; preY++) {
old = x - 1 + preY * X;
tmp = weight[old] + _transProb[preY][y] + emitProb;
if(tmp > weight[now])
{
if(tmp > weight[now]) {
weight[now] = tmp;
path[now] = preY;
}
@ -243,127 +198,102 @@ namespace CppJieba
endE = weight[X-1+E*X];
endS = weight[X-1+S*X];
stat = 0;
if(endE >= endS)
{
if(endE >= endS) {
stat = E;
}
else
{
} else {
stat = S;
}
status.resize(X);
for(int x = X -1 ; x >= 0; x--)
{
for(int x = X -1 ; x >= 0; x--) {
status[x] = stat;
stat = path[x + stat*X];
}
return true;
}
bool _loadModel(const char* const filePath)
{
bool _loadModel(const char* const filePath) {
ifstream ifile(filePath);
string line;
vector<string> tmp;
vector<string> tmp2;
//load _startProb
if(!_getLine(ifile, line))
{
if(!_getLine(ifile, line)) {
return false;
}
split(line, tmp, " ");
if(tmp.size() != STATUS_SUM)
{
if(tmp.size() != STATUS_SUM) {
LogError("start_p illegal");
return false;
}
for(size_t j = 0; j< tmp.size(); j++)
{
for(size_t j = 0; j< tmp.size(); j++) {
_startProb[j] = atof(tmp[j].c_str());
}
//load _transProb
for(size_t i = 0; i < STATUS_SUM; i++)
{
if(!_getLine(ifile, line))
{
for(size_t i = 0; i < STATUS_SUM; i++) {
if(!_getLine(ifile, line)) {
return false;
}
split(line, tmp, " ");
if(tmp.size() != STATUS_SUM)
{
if(tmp.size() != STATUS_SUM) {
LogError("trans_p illegal");
return false;
}
for(size_t j =0; j < STATUS_SUM; j++)
{
for(size_t j =0; j < STATUS_SUM; j++) {
_transProb[i][j] = atof(tmp[j].c_str());
}
}
//load _emitProbB
if(!_getLine(ifile, line) || !_loadEmitProb(line, _emitProbB))
{
if(!_getLine(ifile, line) || !_loadEmitProb(line, _emitProbB)) {
return false;
}
//load _emitProbE
if(!_getLine(ifile, line) || !_loadEmitProb(line, _emitProbE))
{
if(!_getLine(ifile, line) || !_loadEmitProb(line, _emitProbE)) {
return false;
}
//load _emitProbM
if(!_getLine(ifile, line) || !_loadEmitProb(line, _emitProbM))
{
if(!_getLine(ifile, line) || !_loadEmitProb(line, _emitProbM)) {
return false;
}
//load _emitProbS
if(!_getLine(ifile, line) || !_loadEmitProb(line, _emitProbS))
{
if(!_getLine(ifile, line) || !_loadEmitProb(line, _emitProbS)) {
return false;
}
return true;
}
bool _getLine(ifstream& ifile, string& line)
{
while(getline(ifile, line))
{
bool _getLine(ifstream& ifile, string& line) {
while(getline(ifile, line)) {
trim(line);
if(line.empty())
{
if(line.empty()) {
continue;
}
if(startsWith(line, "#"))
{
if(startsWith(line, "#")) {
continue;
}
return true;
}
return false;
}
bool _loadEmitProb(const string& line, EmitProbMap& mp)
{
if(line.empty())
{
bool _loadEmitProb(const string& line, EmitProbMap& mp) {
if(line.empty()) {
return false;
}
vector<string> tmp, tmp2;
Unicode unicode;
split(line, tmp, ",");
for(size_t i = 0; i < tmp.size(); i++)
{
for(size_t i = 0; i < tmp.size(); i++) {
split(tmp[i], tmp2, ":");
if(2 != tmp2.size())
{
if(2 != tmp2.size()) {
LogError("_emitProb illegal.");
return false;
}
if(!TransCode::decode(tmp2[0], unicode) || unicode.size() != 1)
{
if(!TransCode::decode(tmp2[0], unicode) || unicode.size() != 1) {
LogError("TransCode failed.");
return false;
}
@ -371,11 +301,9 @@ namespace CppJieba
}
return true;
}
double _getEmitProb(const EmitProbMap* ptMp, uint16_t key, double defVal)const
{
double _getEmitProb(const EmitProbMap* ptMp, uint16_t key, double defVal)const {
EmitProbMap::const_iterator cit = ptMp->find(key);
if(cit == ptMp->end())
{
if(cit == ptMp->end()) {
return defVal;
}
return cit->second;

6
src/ISegment.hpp
View File

@ -2,10 +2,8 @@
#define CPPJIEBA_SEGMENTINTERFACE_H
namespace CppJieba
{
class ISegment
{
namespace CppJieba {
class ISegment {
public:
virtual ~ISegment() {};
virtual bool cut(Unicode::const_iterator begin , Unicode::const_iterator end, vector<string>& res) const = 0;

76
src/KeywordExtractor.hpp
View File

@ -5,76 +5,60 @@
#include <cmath>
#include <set>
namespace CppJieba
{
namespace CppJieba {
using namespace Limonp;
/*utf8*/
class KeywordExtractor
{
class KeywordExtractor {
public:
KeywordExtractor() {};
KeywordExtractor(const string& dictPath, const string& hmmFilePath, const string& idfPath, const string& stopWordPath, const string& userDict = "")
{
KeywordExtractor(const string& dictPath, const string& hmmFilePath, const string& idfPath, const string& stopWordPath, const string& userDict = "") {
init(dictPath, hmmFilePath, idfPath, stopWordPath, userDict);
};
~KeywordExtractor() {};
void init(const string& dictPath, const string& hmmFilePath, const string& idfPath, const string& stopWordPath, const string& userDict = "")
{
void init(const string& dictPath, const string& hmmFilePath, const string& idfPath, const string& stopWordPath, const string& userDict = "") {
_loadIdfDict(idfPath);
_loadStopWordDict(stopWordPath);
LIMONP_CHECK(_segment.init(dictPath, hmmFilePath, userDict));
};
bool extract(const string& str, vector<string>& keywords, size_t topN) const
{
bool extract(const string& str, vector<string>& keywords, size_t topN) const {
vector<pair<string, double> > topWords;
if(!extract(str, topWords, topN))
{
if(!extract(str, topWords, topN)) {
return false;
}
for(size_t i = 0; i < topWords.size(); i++)
{
for(size_t i = 0; i < topWords.size(); i++) {
keywords.push_back(topWords[i].first);
}
return true;
}
bool extract(const string& str, vector<pair<string, double> >& keywords, size_t topN) const
{
bool extract(const string& str, vector<pair<string, double> >& keywords, size_t topN) const {
vector<string> words;
if(!_segment.cut(str, words))
{
if(!_segment.cut(str, words)) {
LogError("segment cut(%s) failed.", str.c_str());
return false;
}
map<string, double> wordmap;
for(vector<string>::iterator iter = words.begin(); iter != words.end(); iter++)
{
if(_isSingleWord(*iter))
{
for(vector<string>::iterator iter = words.begin(); iter != words.end(); iter++) {
if(_isSingleWord(*iter)) {
continue;
}
wordmap[*iter] += 1.0;
}
for(map<string, double>::iterator itr = wordmap.begin(); itr != wordmap.end(); )
{
if(_stopWords.end() != _stopWords.find(itr->first))
{
for(map<string, double>::iterator itr = wordmap.begin(); itr != wordmap.end(); ) {
if(_stopWords.end() != _stopWords.find(itr->first)) {
wordmap.erase(itr++);
continue;
}
unordered_map<string, double>::const_iterator cit = _idfMap.find(itr->first);
if(cit != _idfMap.end())
{
if(cit != _idfMap.end()) {
itr->second *= cit->second;
}
else
{
} else {
itr->second *= _idfAverage;
}
itr ++;
@ -88,11 +72,9 @@ namespace CppJieba
return true;
}
private:
void _loadIdfDict(const string& idfPath)
{
void _loadIdfDict(const string& idfPath) {
ifstream ifs(idfPath.c_str());
if(!ifs.is_open())
{
if(!ifs.is_open()) {
LogFatal("open %s failed.", idfPath.c_str());
}
string line ;
@ -100,16 +82,13 @@ namespace CppJieba
double idf = 0.0;
double idfSum = 0.0;
size_t lineno = 0;
for(;getline(ifs, line); lineno++)
{
for(; getline(ifs, line); lineno++) {
buf.clear();
if(line.empty())
{
if(line.empty()) {
LogError("line[%d] empty. skipped.", lineno);
continue;
}
if(!split(line, buf, " ") || buf.size() != 2)
{
if(!split(line, buf, " ") || buf.size() != 2) {
LogError("line %d [%s] illegal. skipped.", lineno, line.c_str());
continue;
}
@ -123,23 +102,19 @@ namespace CppJieba
_idfAverage = idfSum / lineno;
assert(_idfAverage > 0.0);
}
void _loadStopWordDict(const string& filePath)
{
void _loadStopWordDict(const string& filePath) {
ifstream ifs(filePath.c_str());
if(!ifs.is_open())
{
if(!ifs.is_open()) {
LogFatal("open %s failed.", filePath.c_str());
}
string line ;
while(getline(ifs, line))
{
while(getline(ifs, line)) {
_stopWords.insert(line);
}
assert(_stopWords.size());
}
bool _isSingleWord(const string& str) const
{
bool _isSingleWord(const string& str) const {
Unicode unicode;
TransCode::decode(str, unicode);
if(unicode.size() == 1)
@ -147,8 +122,7 @@ namespace CppJieba
return false;
}
static bool _cmp(const pair<string, double>& lhs, const pair<string, double>& rhs)
{
static bool _cmp(const pair<string, double>& lhs, const pair<string, double>& rhs) {
return lhs.second > rhs.second;
}

74
src/MPSegment.hpp
View File

@ -9,51 +9,41 @@
#include "ISegment.hpp"
#include "SegmentBase.hpp"
namespace CppJieba
{
namespace CppJieba {
class MPSegment: public SegmentBase
{
class MPSegment: public SegmentBase {
public:
MPSegment() {};
MPSegment(const string& dictPath, const string& userDictPath = "")
{
MPSegment(const string& dictPath, const string& userDictPath = "") {
LIMONP_CHECK(init(dictPath, userDictPath));
};
virtual ~MPSegment() {};
bool init(const string& dictPath, const string& userDictPath = "")
{
bool init(const string& dictPath, const string& userDictPath = "") {
LIMONP_CHECK(_dictTrie.init(dictPath, userDictPath));
LogInfo("MPSegment init(%s) ok", dictPath.c_str());
return true;
}
bool isUserDictSingleChineseWord(const Unicode::value_type & value) const
{
bool isUserDictSingleChineseWord(const Unicode::value_type & value) const {
return _dictTrie.isUserDictSingleChineseWord(value);
}
using SegmentBase::cut;
virtual bool cut(Unicode::const_iterator begin, Unicode::const_iterator end, vector<string>& res)const
{
if(begin == end)
{
virtual bool cut(Unicode::const_iterator begin, Unicode::const_iterator end, vector<string>& res)const {
if(begin == end) {
return false;
}
vector<Unicode> words;
words.reserve(end - begin);
if(!cut(begin, end, words))
{
if(!cut(begin, end, words)) {
return false;
}
size_t offset = res.size();
res.resize(res.size() + words.size());
for(size_t i = 0; i < words.size(); i++)
{
if(!TransCode::encode(words[i], res[i + offset]))
{
for(size_t i = 0; i < words.size(); i++) {
if(!TransCode::encode(words[i], res[i + offset])) {
LogError("encode failed.");
res[i + offset].clear();
}
@ -61,10 +51,8 @@ namespace CppJieba
return true;
}
bool cut(Unicode::const_iterator begin , Unicode::const_iterator end, vector<Unicode>& res) const
{
if(end == begin)
{
bool cut(Unicode::const_iterator begin , Unicode::const_iterator end, vector<Unicode>& res) const {
if(end == begin) {
return false;
}
vector<SegmentChar> segmentChars;
@ -77,62 +65,48 @@ namespace CppJieba
return true;
}
const DictTrie* getDictTrie() const
{
const DictTrie* getDictTrie() const {
return &_dictTrie;
}
private:
void _calcDP(vector<SegmentChar>& segmentChars) const
{
void _calcDP(vector<SegmentChar>& segmentChars) const {
size_t nextPos;
const DictUnit* p;
double val;
for(ssize_t i = segmentChars.size() - 1; i >= 0; i--)
{
for(ssize_t i = segmentChars.size() - 1; i >= 0; i--) {
segmentChars[i].pInfo = NULL;
segmentChars[i].weight = MIN_DOUBLE;
assert(!segmentChars[i].dag.empty());
for(DagType::const_iterator it = segmentChars[i].dag.begin(); it != segmentChars[i].dag.end(); it++)
{
for(DagType::const_iterator it = segmentChars[i].dag.begin(); it != segmentChars[i].dag.end(); it++) {
nextPos = it->first;
p = it->second;
val = 0.0;
if(nextPos + 1 < segmentChars.size())
{
if(nextPos + 1 < segmentChars.size()) {
val += segmentChars[nextPos + 1].weight;
}
if(p)
{
if(p) {
val += p->weight;
}
else
{
} else {
val += _dictTrie.getMinWeight();
}
if(val > segmentChars[i].weight)
{
if(val > segmentChars[i].weight) {
segmentChars[i].pInfo = p;
segmentChars[i].weight = val;
}
}
}
}
void _cut(const vector<SegmentChar>& segmentChars, vector<Unicode>& res) const
{
void _cut(const vector<SegmentChar>& segmentChars, vector<Unicode>& res) const {
size_t i = 0;
while(i < segmentChars.size())
{
while(i < segmentChars.size()) {
const DictUnit* p = segmentChars[i].pInfo;
if(p)
{
if(p) {
res.push_back(p->word);
i += p->word.size();
}
else//single chinese word
{
} else { //single chinese word
res.push_back(Unicode(1, segmentChars[i].uniCh));
i++;
}

57
src/MixSegment.hpp
View File

@ -6,35 +6,27 @@
#include "HMMSegment.hpp"
#include "Limonp/StringUtil.hpp"
namespace CppJieba
{
class MixSegment: public SegmentBase
{
namespace CppJieba {
class MixSegment: public SegmentBase {
public:
MixSegment()
{
MixSegment() {
}
MixSegment(const string& mpSegDict, const string& hmmSegDict, const string& userDict = "")
{
MixSegment(const string& mpSegDict, const string& hmmSegDict, const string& userDict = "") {
LIMONP_CHECK(init(mpSegDict, hmmSegDict, userDict));
}
virtual ~MixSegment()
{
virtual ~MixSegment() {
}
bool init(const string& mpSegDict, const string& hmmSegDict, const string& userDict = "")
{
bool init(const string& mpSegDict, const string& hmmSegDict, const string& userDict = "") {
LIMONP_CHECK(_mpSeg.init(mpSegDict, userDict));
LIMONP_CHECK(_hmmSeg.init(hmmSegDict));
LogInfo("MixSegment init(%s, %s)", mpSegDict.c_str(), hmmSegDict.c_str());
return true;
}
using SegmentBase::cut;
virtual bool cut(Unicode::const_iterator begin, Unicode::const_iterator end, vector<Unicode>& res) const
{
virtual bool cut(Unicode::const_iterator begin, Unicode::const_iterator end, vector<Unicode>& res) const {
vector<Unicode> words;
words.reserve(end - begin);
if(!_mpSeg.cut(begin, end, words))
{
if(!_mpSeg.cut(begin, end, words)) {
LogError("mpSeg cutDAG failed.");
return false;
}
@ -43,33 +35,28 @@ namespace CppJieba
hmmRes.reserve(end - begin);
Unicode piece;
piece.reserve(end - begin);
for (size_t i = 0, j = 0; i < words.size(); i++)
{
for (size_t i = 0, j = 0; i < words.size(); i++) {
//if mp get a word, it's ok, put it into result
if (1 != words[i].size() || (words[i].size() == 1 && _mpSeg.isUserDictSingleChineseWord(words[i][0])))
{
if (1 != words[i].size() || (words[i].size() == 1 && _mpSeg.isUserDictSingleChineseWord(words[i][0]))) {
res.push_back(words[i]);
continue;
}
// if mp get a single one and it is not in userdict, collect it in sequence
j = i;
while (j < words.size() && 1 == words[j].size() && !_mpSeg.isUserDictSingleChineseWord(words[j][0]))
{
while (j < words.size() && 1 == words[j].size() && !_mpSeg.isUserDictSingleChineseWord(words[j][0])) {
piece.push_back(words[j][0]);
j++;
}
// cut the sequence with hmm
if (!_hmmSeg.cut(piece.begin(), piece.end(), hmmRes))
{
if (!_hmmSeg.cut(piece.begin(), piece.end(), hmmRes)) {
LogError("_hmmSeg cut failed.");
return false;
}
//put hmm result to result
for (size_t k = 0; k < hmmRes.size(); k++)
{
for (size_t k = 0; k < hmmRes.size(); k++) {
res.push_back(hmmRes[k]);
}
@ -83,34 +70,28 @@ namespace CppJieba
return true;
}
virtual bool cut(Unicode::const_iterator begin, Unicode::const_iterator end, vector<string>& res)const
{
if(begin == end)
{
virtual bool cut(Unicode::const_iterator begin, Unicode::const_iterator end, vector<string>& res)const {
if(begin == end) {
return false;
}
vector<Unicode> uRes;
uRes.reserve(end - begin);
if (!cut(begin, end, uRes))
{
if (!cut(begin, end, uRes)) {
return false;
}
size_t offset = res.size();
res.resize(res.size() + uRes.size());
for(size_t i = 0; i < uRes.size(); i ++, offset++)
{
if(!TransCode::encode(uRes[i], res[offset]))
{
for(size_t i = 0; i < uRes.size(); i ++, offset++) {
if(!TransCode::encode(uRes[i], res[offset])) {
LogError("encode failed.");
}
}
return true;
}
const DictTrie* getDictTrie() const
{
const DictTrie* getDictTrie() const {
return _mpSeg.getDictTrie();
}
private:

55
src/PosTagger.hpp
View File

@ -5,97 +5,78 @@
#include "Limonp/StringUtil.hpp"
#include "DictTrie.hpp"
namespace CppJieba
{
namespace CppJieba {
using namespace Limonp;
static const char* const POS_M = "m";
static const char* const POS_ENG = "eng";
static const char* const POS_X = "x";
class PosTagger
{
class PosTagger {
public:
PosTagger()
{
PosTagger() {
}
PosTagger(
const string& dictPath,
const string& hmmFilePath,
const string& userDictPath = ""
)
{
) {
init(dictPath, hmmFilePath, userDictPath);
}
~PosTagger()
{
~PosTagger() {
}
void init(
const string& dictPath,
const string& hmmFilePath,
const string& userDictPath = ""
)
{
) {
LIMONP_CHECK(_segment.init(dictPath, hmmFilePath, userDictPath));
_dictTrie = _segment.getDictTrie();
LIMONP_CHECK(_dictTrie);
};
bool tag(const string& src, vector<pair<string, string> >& res) const
{
bool tag(const string& src, vector<pair<string, string> >& res) const {
vector<string> cutRes;
if (!_segment.cut(src, cutRes))
{
if (!_segment.cut(src, cutRes)) {
LogError("_mixSegment cut failed");
return false;
}
const DictUnit *tmp = NULL;
Unicode unico;
for (vector<string>::iterator itr = cutRes.begin(); itr != cutRes.end(); ++itr)
{
if (!TransCode::decode(*itr, unico))
{
for (vector<string>::iterator itr = cutRes.begin(); itr != cutRes.end(); ++itr) {
if (!TransCode::decode(*itr, unico)) {
LogError("decode failed.");
return false;
}
tmp = _dictTrie->find(unico.begin(), unico.end());
if(tmp == NULL || tmp->tag.empty())
{
if(tmp == NULL || tmp->tag.empty()) {
res.push_back(make_pair(*itr, _specialRule(unico)));
}
else
{
} else {
res.push_back(make_pair(*itr, tmp->tag));
}
}
return !res.empty();
}
private:
const char* _specialRule(const Unicode& unicode) const
{
const char* _specialRule(const Unicode& unicode) const {
size_t m = 0;
size_t eng = 0;
for(size_t i = 0; i < unicode.size() && eng < unicode.size() / 2; i++)
{
if(unicode[i] < 0x80)
{
for(size_t i = 0; i < unicode.size() && eng < unicode.size() / 2; i++) {
if(unicode[i] < 0x80) {
eng ++;
if('0' <= unicode[i] && unicode[i] <= '9')
{
if('0' <= unicode[i] && unicode[i] <= '9') {
m++;
}
}
}
// ascii char is not found
if(eng == 0)
{
if(eng == 0) {
return POS_X;
}
// all the ascii is number char
if(m == eng)
{
if(m == eng) {
return POS_M;
}
// the ascii chars contain english letter

56
src/QuerySegment.hpp
View File

@ -13,19 +13,15 @@
#include "TransCode.hpp"
#include "DictTrie.hpp"
namespace CppJieba
{
class QuerySegment: public SegmentBase
{
namespace CppJieba {
class QuerySegment: public SegmentBase {
public:
QuerySegment() {};
QuerySegment(const string& dict, const string& model, size_t maxWordLen, const string& userDict = "")
{
QuerySegment(const string& dict, const string& model, size_t maxWordLen, const string& userDict = "") {
init(dict, model, maxWordLen, userDict);
};
virtual ~QuerySegment() {};
bool init(const string& dict, const string& model, size_t maxWordLen, const string& userDict = "")
{
bool init(const string& dict, const string& model, size_t maxWordLen, const string& userDict = "") {
LIMONP_CHECK(_mixSeg.init(dict, model, userDict));
LIMONP_CHECK(_fullSeg.init(_mixSeg.getDictTrie()));
assert(maxWordLen);
@ -33,42 +29,33 @@ namespace CppJieba
return true;
}
using SegmentBase::cut;
bool cut(Unicode::const_iterator begin, Unicode::const_iterator end, vector<Unicode>& res) const
{
if (begin >= end)
{
bool cut(Unicode::const_iterator begin, Unicode::const_iterator end, vector<Unicode>& res) const {
if (begin >= end) {
LogError("begin >= end");
return false;
}
//use mix cut first
vector<Unicode> mixRes;
if (!_mixSeg.cut(begin, end, mixRes))
{
if (!_mixSeg.cut(begin, end, mixRes)) {
LogError("_mixSeg cut failed.");
return false;
}
vector<Unicode> fullRes;
for (vector<Unicode>::const_iterator mixResItr = mixRes.begin(); mixResItr != mixRes.end(); mixResItr++)
{
for (vector<Unicode>::const_iterator mixResItr = mixRes.begin(); mixResItr != mixRes.end(); mixResItr++) {
// if it's too long, cut with _fullSeg, put fullRes in res
if (mixResItr->size() > _maxWordLen)
{
if (_fullSeg.cut(mixResItr->begin(), mixResItr->end(), fullRes))
{
for (vector<Unicode>::const_iterator fullResItr = fullRes.begin(); fullResItr != fullRes.end(); fullResItr++)
{
if (mixResItr->size() > _maxWordLen) {
if (_fullSeg.cut(mixResItr->begin(), mixResItr->end(), fullRes)) {
for (vector<Unicode>::const_iterator fullResItr = fullRes.begin(); fullResItr != fullRes.end(); fullResItr++) {
res.push_back(*fullResItr);
}
//clear tmp res
fullRes.clear();
}
}
else // just use the mix result
{
} else { // just use the mix result
res.push_back(*mixResItr);
}
}
@ -77,30 +64,23 @@ namespace CppJieba
}
bool cut(Unicode::const_iterator begin, Unicode::const_iterator end, vector<string>& res) const
{
if (begin >= end)
{
bool cut(Unicode::const_iterator begin, Unicode::const_iterator end, vector<string>& res) const {
if (begin >= end) {
LogError("begin >= end");
return false;
}
vector<Unicode> uRes;
if (!cut(begin, end, uRes))
{
if (!cut(begin, end, uRes)) {
LogError("get unicode cut result error.");
return false;
}
string tmp;
for (vector<Unicode>::const_iterator uItr = uRes.begin(); uItr != uRes.end(); uItr++)
{
if (TransCode::encode(*uItr, tmp))
{
for (vector<Unicode>::const_iterator uItr = uRes.begin(); uItr != uRes.end(); uItr++) {
if (TransCode::encode(*uItr, tmp)) {
res.push_back(tmp);
}
else
{
} else {
LogError("encode failed.");
}
}

31
src/SegmentBase.hpp
View File

@ -9,8 +9,7 @@
#include <cassert>
namespace CppJieba
{
namespace CppJieba {
using namespace Limonp;
//const char* const SPECIAL_CHARS = " \t\n";
@ -20,15 +19,15 @@ namespace CppJieba
const UnicodeValueType SPECIAL_SYMBOL[] = {32u, 9u, 10u};
#endif
class SegmentBase: public ISegment, public NonCopyable
{
class SegmentBase: public ISegment, public NonCopyable {
public:
SegmentBase(){_loadSpecialSymbols();};
SegmentBase() {
_loadSpecialSymbols();
};
virtual ~SegmentBase() {};
public:
virtual bool cut(Unicode::const_iterator begin, Unicode::const_iterator end, vector<string>& res) const = 0;
virtual bool cut(const string& str, vector<string>& res) const
{
virtual bool cut(const string& str, vector<string>& res) const {
res.clear();
Unicode unicode;
@ -39,12 +38,9 @@ namespace CppJieba
Unicode::const_iterator left = unicode.begin();
Unicode::const_iterator right;
for(right = unicode.begin(); right != unicode.end(); right++)
{
if(isIn(_specialSymbols, *right))
{
if(left != right)
{
for(right = unicode.begin(); right != unicode.end(); right++) {
if(isIn(_specialSymbols, *right)) {
if(left != right) {
cut(left, right, res);
}
res.resize(res.size() + 1);
@ -52,19 +48,16 @@ namespace CppJieba
left = right + 1;
}
}
if(left != right)
{
if(left != right) {
cut(left, right, res);
}
return true;
}
private:
void _loadSpecialSymbols()
{
void _loadSpecialSymbols() {
size_t size = sizeof(SPECIAL_SYMBOL)/sizeof(*SPECIAL_SYMBOL);
for(size_t i = 0; i < size; i ++)
{
for(size_t i = 0; i < size; i ++) {
_specialSymbols.insert(SPECIAL_SYMBOL[i]);
}
assert(_specialSymbols.size());

21
src/TransCode.hpp
View File

@ -9,16 +9,13 @@
#include "Limonp/StringUtil.hpp"
#include "Limonp/LocalVector.hpp"
namespace CppJieba
{
namespace CppJieba {
using namespace Limonp;
typedef uint16_t UnicodeValueType;
typedef Limonp::LocalVector<UnicodeValueType> Unicode;
namespace TransCode
{
inline bool decode(const string& str, Unicode& res)
{
namespace TransCode {
inline bool decode(const string& str, Unicode& res) {
#ifdef CPPJIEBA_GBK
return gbkTrans(str, res);
#else
@ -26,8 +23,7 @@ namespace CppJieba
#endif
}
inline bool encode(Unicode::const_iterator begin, Unicode::const_iterator end, string& res)
{
inline bool encode(Unicode::const_iterator begin, Unicode::const_iterator end, string& res) {
#ifdef CPPJIEBA_GBK
return gbkTrans(begin, end, res);
#else
@ -35,14 +31,12 @@ namespace CppJieba
#endif
}
inline bool encode(const Unicode& uni, string& res)
{
inline bool encode(const Unicode& uni, string& res) {
return encode(uni.begin(), uni.end(), res);
}
// compiler is expected to optimized this function to avoid return value copy
inline string encode(Unicode::const_iterator begin, Unicode::const_iterator end)
{
inline string encode(Unicode::const_iterator begin, Unicode::const_iterator end) {
string res;
res.reserve(end - begin);
encode(begin, end, res);
@ -50,8 +44,7 @@ namespace CppJieba
}
// compiler is expected to optimized this function to avoid return value copy
inline Unicode decode(const string& str)
{
inline Unicode decode(const string& str) {
Unicode unicode;
unicode.reserve(str.size());
decode(str, unicode);

155
src/Trie.hpp
View File

@ -5,20 +5,17 @@
#include <vector>
#include <queue>
namespace CppJieba
{
namespace CppJieba {
using namespace std;
struct DictUnit
{
struct DictUnit {
Unicode word;
double weight;
string tag;
};
// for debugging
inline ostream & operator << (ostream& os, const DictUnit& unit)
{
inline ostream & operator << (ostream& os, const DictUnit& unit) {
string s;
s << unit.word;
return os << string_format("%s %s %.3lf", s.c_str(), unit.tag.c_str(), unit.weight);
@ -26,35 +23,30 @@ namespace CppJieba
typedef LocalVector<std::pair<size_t, const DictUnit*> > DagType;
struct SegmentChar
{
struct SegmentChar {
uint16_t uniCh;
DagType dag;
const DictUnit * pInfo;
double weight;
size_t nextPos;
SegmentChar():uniCh(0), pInfo(NULL), weight(0.0), nextPos(0)
{}
~SegmentChar()
{}
SegmentChar():uniCh(0), pInfo(NULL), weight(0.0), nextPos(0) {
}
~SegmentChar() {
}
};
typedef Unicode::value_type TrieKey;
class TrieNode
{
class TrieNode {
public:
TrieNode(): fail(NULL), next(NULL), ptValue(NULL)
{}
const TrieNode * findNext(TrieKey key) const
{
if(next == NULL)
{
TrieNode(): fail(NULL), next(NULL), ptValue(NULL) {
}
const TrieNode * findNext(TrieKey key) const {
if(next == NULL) {
return NULL;
}
NextMap::const_iterator iter = next->find(key);
if(iter == next->end())
{
if(iter == next->end()) {
return NULL;
}
return iter->second;
@ -66,32 +58,26 @@ namespace CppJieba
const DictUnit * ptValue;
};
class Trie
{
class Trie {
public:
Trie(const vector<Unicode>& keys, const vector<const DictUnit*> & valuePointers)
{
Trie(const vector<Unicode>& keys, const vector<const DictUnit*> & valuePointers) {
_root = new TrieNode;
_createTrie(keys, valuePointers);
_build();// build automation
}
~Trie()
{
if(_root)
{
~Trie() {
if(_root) {
_deleteNode(_root);
}
}
public:
const DictUnit* find(Unicode::const_iterator begin, Unicode::const_iterator end) const
{
const DictUnit* find(Unicode::const_iterator begin, Unicode::const_iterator end) const {
TrieNode::NextMap::const_iterator citer;
const TrieNode* ptNode = _root;
for(Unicode::const_iterator it = begin; it != end; it++)
{// build automation
for(Unicode::const_iterator it = begin; it != end; it++) {
// build automation
assert(ptNode);
if(NULL == ptNode->next || ptNode->next->end() == (citer = ptNode->next->find(*it)))
{
if(NULL == ptNode->next || ptNode->next->end() == (citer = ptNode->next->find(*it))) {
return NULL;
}
ptNode = citer->second;
@ -103,14 +89,12 @@ namespace CppJieba
Unicode::const_iterator begin,
Unicode::const_iterator end,
vector<struct SegmentChar>& res
) const
{
) const {
res.resize(end - begin);
const TrieNode * now = _root;
const TrieNode* node;
// compiler will complain warnings if only "i < end - begin" .
for (size_t i = 0; i < size_t(end - begin); i++)
{
for (size_t i = 0; i < size_t(end - begin); i++) {
Unicode::value_type ch = *(begin + i);
res[i].uniCh = ch;
assert(res[i].dag.empty());
@ -118,40 +102,29 @@ namespace CppJieba
bool flag = false;
// rollback
while( now != _root )
{
while( now != _root ) {
node = now->findNext(ch);
if (node != NULL)
{
if (node != NULL) {
flag = true;
break;
}
else
{
} else {
now = now->fail;
}
}
if(!flag)
{
if(!flag) {
node = now->findNext(ch);
}
if(node == NULL)
{
if(node == NULL) {
now = _root;
}
else
{
} else {
now = node;
const TrieNode * temp = now;
while(temp != _root)
{
if (temp->ptValue)
{
while(temp != _root) {
if (temp->ptValue) {
size_t pos = i - temp->ptValue->word.size() + 1;
res[pos].dag.push_back(pair<vector<Unicode >::size_type, const DictUnit* >(i, temp->ptValue));
if(pos == i)
{
if(pos == i) {
res[pos].dag[0].second = temp->ptValue;
}
}
@ -165,26 +138,19 @@ namespace CppJieba
Unicode::const_iterator begin,
Unicode::const_iterator end,
DagType & res,
size_t offset = 0) const
{
size_t offset = 0) const {
const TrieNode * ptNode = _root;
TrieNode::NextMap::const_iterator citer;
for(Unicode::const_iterator itr = begin; itr != end ; itr++)
{
for(Unicode::const_iterator itr = begin; itr != end ; itr++) {
assert(ptNode);
if(NULL == ptNode->next || ptNode->next->end() == (citer = ptNode->next->find(*itr)))
{
if(NULL == ptNode->next || ptNode->next->end() == (citer = ptNode->next->find(*itr))) {
break;
}
ptNode = citer->second;
if(ptNode->ptValue)
{
if(itr == begin && res.size() == 1) // first singleword
{
if(ptNode->ptValue) {
if(itr == begin && res.size() == 1) { // first singleword
res[0].second = ptNode->ptValue;
}
else
{
} else {
res.push_back(pair<vector<Unicode >::size_type, const DictUnit* >(itr - begin + offset, ptNode->ptValue));
}
}
@ -192,8 +158,7 @@ namespace CppJieba
return !res.empty();
}
private:
void _build()
{
void _build() {
queue<TrieNode*> que;
assert(_root->ptValue == NULL);
assert(_root->next);
@ -213,8 +178,7 @@ namespace CppJieba
for(TrieNode::NextMap::iterator iter = now->next->begin(); iter != now->next->end(); iter++) {
back = now->fail;
while(back != NULL) {
if(back->next && (backiter = back->next->find(iter->first)) != back->next->end())
{
if(back->next && (backiter = back->next->find(iter->first)) != back->next->end()) {
iter->second->fail = backiter->second;
break;
}
@ -227,59 +191,46 @@ namespace CppJieba
}
}
}
void _createTrie(const vector<Unicode>& keys, const vector<const DictUnit*> & valuePointers)
{
if(valuePointers.empty() || keys.empty())
{
void _createTrie(const vector<Unicode>& keys, const vector<const DictUnit*> & valuePointers) {
if(valuePointers.empty() || keys.empty()) {
return;
}
assert(keys.size() == valuePointers.size());
for(size_t i = 0; i < keys.size(); i++)
{
for(size_t i = 0; i < keys.size(); i++) {
_insertNode(keys[i], valuePointers[i]);
}
}
void _insertNode(const Unicode& key, const DictUnit* ptValue)
{
void _insertNode(const Unicode& key, const DictUnit* ptValue) {
TrieNode* ptNode = _root;
TrieNode::NextMap::const_iterator kmIter;
for(Unicode::const_iterator citer = key.begin(); citer != key.end(); citer++)
{
if(NULL == ptNode->next)
{
for(Unicode::const_iterator citer = key.begin(); citer != key.end(); citer++) {
if(NULL == ptNode->next) {
ptNode->next = new TrieNode::NextMap;
}
kmIter = ptNode->next->find(*citer);
if(ptNode->next->end() == kmIter)
{
if(ptNode->next->end() == kmIter) {
TrieNode * nextNode = new TrieNode;
nextNode->next = NULL;
nextNode->ptValue = NULL;
(*ptNode->next)[*citer] = nextNode;
ptNode = nextNode;
}
else
{
} else {
ptNode = kmIter->second;
}
}
ptNode->ptValue = ptValue;
}
void _deleteNode(TrieNode* node)
{
if(!node)
{
void _deleteNode(TrieNode* node) {
if(!node) {
return;
}
if(node->next)
{
if(node->next) {
TrieNode::NextMap::iterator it;
for(it = node->next->begin(); it != node->next->end(); it++)
{
for(it = node->next->begin(); it != node->next->end(); it++) {
_deleteNode(it->second);
}
delete node->next;

3
test/keyword_demo.cpp
View File

@ -1,8 +1,7 @@
#include "../src/KeywordExtractor.hpp"
using namespace CppJieba;
int main(int argc, char ** argv)
{
int main(int argc, char ** argv) {
KeywordExtractor extractor("../dict/jieba.dict.utf8", "../dict/hmm_model.utf8", "../dict/idf.utf8", "../dict/stop_words.utf8");
//KeywordExtractor extractor("../dict/jieba.dict.utf8", "../dict/hmm_model.utf8", "../dict/idf.utf8", "../dict/stop_words.utf8", "../dict/user.dict.utf8");
string s("我是拖拉机学院手扶拖拉机专业的。不用多久，我就会升职加薪，当上CEO，走上人生巅峰。");

15
test/load_test.cpp
View File

@ -9,8 +9,7 @@
using namespace CppJieba;
void cut(size_t times = 50)
{
void cut(size_t times = 50) {
MixSegment seg("../dict/jieba.dict.utf8", "../dict/hmm_model.utf8");
vector<string> res;
string doc;
@ -18,8 +17,7 @@ void cut(size_t times = 50)
assert(ifs);
doc << ifs;
long beginTime = clock();
for(size_t i = 0; i < times; i ++)
{
for(size_t i = 0; i < times; i ++) {
printf("process [%3.0lf %%]\r", 100.0*(i+1)/times);
fflush(stdout);
res.clear();
@ -30,8 +28,7 @@ void cut(size_t times = 50)
ColorPrintln(GREEN, "cut: [%.3lf seconds]time consumed.", double(endTime - beginTime)/CLOCKS_PER_SEC);
}
void extract(size_t times = 400)
{
void extract(size_t times = 400) {
KeywordExtractor extractor("../dict/jieba.dict.utf8", "../dict/hmm_model.utf8", "../dict/idf.utf8", "../dict/stop_words.utf8");
vector<string> words;
string doc;
@ -39,8 +36,7 @@ void extract(size_t times = 400)
assert(ifs);
doc << ifs;
long beginTime = clock();
for(size_t i = 0; i < times; i ++)
{
for(size_t i = 0; i < times; i ++) {
printf("process [%3.0lf %%]\r", 100.0*(i+1)/times);
fflush(stdout);
words.clear();
@ -51,8 +47,7 @@ void extract(size_t times = 400)
ColorPrintln(GREEN, "extract: [%.3lf seconds]time consumed.", double(endTime - beginTime)/CLOCKS_PER_SEC);
}
int main(int argc, char ** argv)
{
int main(int argc, char ** argv) {
cut();
extract();
return EXIT_SUCCESS;

12
test/segment_demo.cpp
View File

@ -14,16 +14,13 @@ const char * const JIEBA_DICT_FILE = "../dict/jieba.dict.utf8";
const char * const HMM_DICT_FILE = "../dict/hmm_model.utf8";
const char * const USER_DICT_FILE = "../dict/user.dict.utf8";
void cut(const ISegment& seg, const char * const filePath)
{
void cut(const ISegment& seg, const char * const filePath) {
ifstream ifile(filePath);
vector<string> words;
string line;
string res;
while(getline(ifile, line))
{
if(!line.empty())
{
while(getline(ifile, line)) {
if(!line.empty()) {
words.clear();
seg.cut(line, words);
join(words.begin(), words.end(), res, "/");
@ -33,8 +30,7 @@ void cut(const ISegment& seg, const char * const filePath)
}
int main(int argc, char ** argv)
{
int main(int argc, char ** argv) {
{
printf("\e[32m%s\e[0m\n", "[demo] MPSegment"); // colorful
MPSegment seg(JIEBA_DICT_FILE);

3
test/tagging_demo.cpp
View File

@ -1,8 +1,7 @@
#include "../src/PosTagger.hpp"
using namespace CppJieba;
int main(int argc, char ** argv)
{
int main(int argc, char ** argv) {
PosTagger tagger("../dict/jieba.dict.utf8", "../dict/hmm_model.utf8", "../dict/user.dict.utf8");
string s("我是蓝翔技工拖拉机学院手扶拖拉机专业的。不用多久，我就会升职加薪，当上总经理，出任CEO，迎娶白富美，走上人生巅峰。");
vector<pair<string, string> > res;

6
test/unittest/TKeywordExtractor.cpp
View File

@ -5,8 +5,7 @@ using namespace CppJieba;
TEST(KeywordExtractorTest, Test1)
{
TEST(KeywordExtractorTest, Test1) {
KeywordExtractor extractor("../dict/extra_dict/jieba.dict.small.utf8", "../dict/hmm_model.utf8", "../dict/idf.utf8", "../dict/stop_words.utf8");
{
@ -30,8 +29,7 @@ TEST(KeywordExtractorTest, Test1)
}
}
TEST(KeywordExtractorTest, Test2)
{
TEST(KeywordExtractorTest, Test2) {
KeywordExtractor extractor("../dict/extra_dict/jieba.dict.small.utf8", "../dict/hmm_model.utf8", "../dict/idf.utf8", "../dict/stop_words.utf8", "../test/testdata/userdict.utf8");
{

6
test/unittest/TPosTagger.cpp
View File

@ -12,8 +12,7 @@ static const char * const QUERY_TEST3 = "iPhone6手机的最大特点是很容
static const char * const ANS_TEST3 = "[\"iPhone6:eng\", \"手机:n\", \"的:uj\", \"最大:a\", \"特点:n\", \"是:v\", \"很:zg\", \"容易:a\", \"弯曲:v\", \"。:x\"]";
//static const char * const ANS_TEST3 = "";
TEST(PosTaggerTest, Test)
{
TEST(PosTaggerTest, Test) {
PosTagger tagger("../dict/jieba.dict.utf8", "../dict/hmm_model.utf8");
{
vector<pair<string, string> > res;
@ -23,8 +22,7 @@ TEST(PosTaggerTest, Test)
ASSERT_TRUE(s == ANS_TEST1);
}
}
TEST(PosTagger, TestUserDict)
{
TEST(PosTagger, TestUserDict) {
PosTagger tagger("../dict/jieba.dict.utf8", "../dict/hmm_model.utf8", "../test/testdata/userdict.utf8");
{
vector<pair<string, string> > res;

33
test/unittest/TSegments.cpp
View File

@ -9,8 +9,7 @@
using namespace CppJieba;
TEST(MixSegmentTest, Test1)
{
TEST(MixSegmentTest, Test1) {
MixSegment segment("../dict/jieba.dict.utf8", "../dict/hmm_model.utf8");;
const char* str = "我来自北京邮电大学。。。学号123456，用AK47";
const char* res[] = {"我", "来自", "北京邮电大学", "。","。","。", "学号", "123456","，","用","AK47"};
@ -23,8 +22,7 @@ TEST(MixSegmentTest, Test1)
ASSERT_EQ(words, vector<string>(res2, res2 + sizeof(res2)/sizeof(res2[0])));
}
TEST(MixSegmentTest, NoUserDict)
{
TEST(MixSegmentTest, NoUserDict) {
MixSegment segment("../dict/extra_dict/jieba.dict.small.utf8", "../dict/hmm_model.utf8");
const char* str = "令狐冲是云计算方面的专家";
vector<string> words;
@ -33,8 +31,7 @@ TEST(MixSegmentTest, NoUserDict)
ASSERT_EQ("[\"令狐冲\", \"是\", \"云\", \"计算\", \"方面\", \"的\", \"专家\"]", res << words);
}
TEST(MixSegmentTest, UserDict)
{
TEST(MixSegmentTest, UserDict) {
MixSegment segment("../dict/extra_dict/jieba.dict.small.utf8", "../dict/hmm_model.utf8", "../dict/user.dict.utf8");
{
const char* str = "令狐冲是云计算方面的专家";
@ -60,8 +57,7 @@ TEST(MixSegmentTest, UserDict)
ASSERT_EQ("[\"IBM\", \",\", \"3.14\"]", res);
}
}
TEST(MixSegmentTest, UserDict2)
{
TEST(MixSegmentTest, UserDict2) {
MixSegment segment("../dict/extra_dict/jieba.dict.small.utf8", "../dict/hmm_model.utf8", "../test/testdata/userdict.utf8");
{
const char* str = "令狐冲是云计算方面的专家";
@ -88,8 +84,7 @@ TEST(MixSegmentTest, UserDict2)
}
}
TEST(MPSegmentTest, Test1)
{
TEST(MPSegmentTest, Test1) {
MPSegment segment("../dict/jieba.dict.utf8");;
const char* str = "我来自北京邮电大学。";
const char* res[] = {"我", "来自", "北京邮电大学", "。"};
@ -106,8 +101,7 @@ TEST(MPSegmentTest, Test1)
}
}
TEST(MPSegmentTest, Test2)
{
TEST(MPSegmentTest, Test2) {
MPSegment segment("../dict/extra_dict/jieba.dict.small.utf8");
string line;
ifstream ifs("../test/testdata/review.100");
@ -121,8 +115,7 @@ TEST(MPSegmentTest, Test2)
}
string res;
while(getline(ifs, line))
{
while(getline(ifs, line)) {
res += line;
res += '\n';
@ -137,8 +130,7 @@ TEST(MPSegmentTest, Test2)
ofs << res;
}
TEST(HMMSegmentTest, Test1)
{
TEST(HMMSegmentTest, Test1) {
HMMSegment segment("../dict/hmm_model.utf8");;
{
const char* str = "我来自北京邮电大学。。。学号123456";
@ -157,8 +149,7 @@ TEST(HMMSegmentTest, Test1)
}
}
TEST(FullSegment, Test1)
{
TEST(FullSegment, Test1) {
FullSegment segment("../dict/extra_dict/jieba.dict.small.utf8");
const char* str = "我来自北京邮电大学";
vector<string> words;
@ -170,8 +161,7 @@ TEST(FullSegment, Test1)
ASSERT_EQ(s, "[\"我\", \"来自\", \"北京\", \"北京邮电大学\", \"邮电\", \"电大\", \"大学\"]");
}
TEST(QuerySegment, Test1)
{
TEST(QuerySegment, Test1) {
QuerySegment segment("../dict/extra_dict/jieba.dict.small.utf8", "../dict/hmm_model.utf8", 3);
const char* str = "小明硕士毕业于中国科学院计算所，后在日本京都大学深造";
vector<string> words;
@ -185,8 +175,7 @@ TEST(QuerySegment, Test1)
}
TEST(QuerySegment, Test2)
{
TEST(QuerySegment, Test2) {
QuerySegment segment("../dict/extra_dict/jieba.dict.small.utf8", "../dict/hmm_model.utf8", 3, "../test/testdata/userdict.utf8");
{

15
test/unittest/TTrie.cpp
View File

@ -6,8 +6,7 @@ using namespace CppJieba;
static const char* const DICT_FILE = "../dict/extra_dict/jieba.dict.small.utf8";
TEST(DictTrieTest, NewAndDelete)
{
TEST(DictTrieTest, NewAndDelete) {
DictTrie * trie;
trie = new DictTrie(DICT_FILE);
delete trie;
@ -15,8 +14,7 @@ TEST(DictTrieTest, NewAndDelete)
delete trie;
}
TEST(DictTrieTest, Test1)
{
TEST(DictTrieTest, Test1) {
string s1, s2;
DictTrie trie;
@ -38,8 +36,7 @@ TEST(DictTrieTest, Test1)
//vector<pair<size_t, const DictUnit* > resMap;
LocalVector<pair<size_t, const DictUnit*> > res2;
const char * words[] = {"清", "清华", "清华大学"};
for(size_t i = 0; i < sizeof(words)/sizeof(words[0]); i++)
{
for(size_t i = 0; i < sizeof(words)/sizeof(words[0]); i++) {
ASSERT_TRUE(TransCode::decode(words[i], uni));
res.push_back(make_pair(uni.size() - 1, trie.find(uni.begin(), uni.end())));
//resMap[uni.size() - 1] = trie.find(uni.begin(), uni.end());
@ -55,8 +52,7 @@ TEST(DictTrieTest, Test1)
ASSERT_EQ(s1, s2);
}
TEST(DictTrieTest, UserDict)
{
TEST(DictTrieTest, UserDict) {
DictTrie trie(DICT_FILE, "../test/testdata/userdict.utf8");
string word = "云计算";
Unicode unicode;
@ -68,8 +64,7 @@ TEST(DictTrieTest, UserDict)
ASSERT_EQ("[\"20113\", \"35745\", \"31639\"] -2.975", res);
}
TEST(DictTrieTest, automation)
{
TEST(DictTrieTest, automation) {
DictTrie trie(DICT_FILE, "../test/testdata/userdict.utf8");
//string word = "yasherhs";
string word = "abcderf";