third_codes/cppjieba
1
0
Fork 0
mirror of https://github.com/yanyiwu/cppjieba.git synced 2025-07-18 00:00:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

将 HMMSegment 里面关于模型文件的数据独立成 HMMModel

Browse Source
This commit is contained in:
yanyiwu 2015-06-04 17:52:18 +08:00
parent d3b34b73c6
commit b99d0698f0
6 changed files with 186 additions and 197 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

142
src/HMMModel.hpp Normal file
View File

@ -0,0 +1,142 @@
#ifndef CPPJIEBA_HMMMODEL_H
#define CPPJIEBA_HMMMODEL_H
#include "Limonp/StringUtil.hpp"
namespace CppJieba {
using namespace Limonp;
typedef unordered_map<uint16_t, double> EmitProbMap;
struct HMMModel {
/*
* STATUS:
* 0: HMMModel::B, 1: HMMModel::E, 2: HMMModel::M, 3:HMMModel::S
* */
enum {B = 0, E = 1, M = 2, S = 3, STATUS_SUM = 4};
HMMModel(const string& modelPath) {
memset(startProb, 0, sizeof(startProb));
memset(transProb, 0, sizeof(transProb));
statMap[0] = 'B';
statMap[1] = 'E';
statMap[2] = 'M';
statMap[3] = 'S';
emitProbVec.push_back(&emitProbB);
emitProbVec.push_back(&emitProbE);
emitProbVec.push_back(&emitProbM);
emitProbVec.push_back(&emitProbS);
loadModel(modelPath);
}
~HMMModel() {
}
void loadModel(const string& filePath) {
ifstream ifile(filePath.c_str());
if(!ifile.is_open()) {
LogFatal("open %s failed.", filePath.c_str());
}
string line;
vector<string> tmp;
vector<string> tmp2;
//load startProb
if(!getLine(ifile, line)) {
LogFatal("load startProb");
}
split(line, tmp, " ");
if(tmp.size() != STATUS_SUM) {
LogFatal("start_p illegal");
}
for(size_t j = 0; j< tmp.size(); j++) {
startProb[j] = atof(tmp[j].c_str());
}
//load transProb
for(size_t i = 0; i < STATUS_SUM; i++) {
if(!getLine(ifile, line)) {
LogFatal("load transProb failed.");
}
split(line, tmp, " ");
if(tmp.size() != STATUS_SUM) {
LogFatal("trans_p illegal");
}
for(size_t j =0; j < STATUS_SUM; j++) {
transProb[i][j] = atof(tmp[j].c_str());
}
}
//load emitProbB
if(!getLine(ifile, line) || !loadEmitProb(line, emitProbB)) {
LogFatal("load emitProbB failed.");
}
//load emitProbE
if(!getLine(ifile, line) || !loadEmitProb(line, emitProbE)) {
LogFatal("load emitProbE failed.");
}
//load emitProbM
if(!getLine(ifile, line) || !loadEmitProb(line, emitProbM)) {
LogFatal("load emitProbM failed.");
}
//load emitProbS
if(!getLine(ifile, line) || !loadEmitProb(line, emitProbS)) {
LogFatal("load emitProbS failed.");
}
}
double getEmitProb(const EmitProbMap* ptMp, uint16_t key,
double defVal)const {
EmitProbMap::const_iterator cit = ptMp->find(key);
if(cit == ptMp->end()) {
return defVal;
}
return cit->second;
}
bool getLine(ifstream& ifile, string& line) {
while(getline(ifile, line)) {
trim(line);
if(line.empty()) {
continue;
}
if(startsWith(line, "#")) {
continue;
}
return true;
}
return false;
}
bool loadEmitProb(const string& line, EmitProbMap& mp) {
if(line.empty()) {
return false;
}
vector<string> tmp, tmp2;
Unicode unicode;
split(line, tmp, ",");
for(size_t i = 0; i < tmp.size(); i++) {
split(tmp[i], tmp2, ":");
if(2 != tmp2.size()) {
LogError("emitProb illegal.");
return false;
}
if(!TransCode::decode(tmp2[0], unicode) || unicode.size() != 1) {
LogError("TransCode failed.");
return false;
}
mp[unicode[0]] = atof(tmp2[1].c_str());
}
return true;
}
char statMap[STATUS_SUM];
double startProb[STATUS_SUM];
double transProb[STATUS_SUM][STATUS_SUM];
EmitProbMap emitProbB;
EmitProbMap emitProbE;
EmitProbMap emitProbM;
EmitProbMap emitProbS;
vector<EmitProbMap* > emitProbVec;
}; // struct HMMModel
} // namespace CppJieba
#endif

172
src/HMMSegment.hpp
View File

@ -5,47 +5,18 @@
#include <fstream>
#include <memory.h>
#include <cassert>
#include "Limonp/StringUtil.hpp"
#include "TransCode.hpp"
#include "ISegment.hpp"
#include "HMMModel.hpp"
#include "SegmentBase.hpp"
#include "DictTrie.hpp"
namespace CppJieba {
using namespace Limonp;
typedef unordered_map<uint16_t, double> EmitProbMap;
class HMMSegment: public SegmentBase {
public:
/*
* STATUS:
* 0:B, 1:E, 2:M, 3:S
* */
enum {B = 0, E = 1, M = 2, S = 3, STATUS_SUM = 4};
public:
HMMSegment() {}
explicit HMMSegment(const string& filePath) {
init(filePath);
explicit HMMSegment(const string& filePath): model_(filePath) {
}
virtual ~HMMSegment() {}
public:
void init(const string& filePath) {
memset(startProb_, 0, sizeof(startProb_));
memset(transProb_, 0, sizeof(transProb_));
statMap_[0] = 'B';
statMap_[1] = 'E';
statMap_[2] = 'M';
statMap_[3] = 'S';
emitProbVec_.push_back(&emitProbB_);
emitProbVec_.push_back(&emitProbE_);
emitProbVec_.push_back(&emitProbM_);
emitProbVec_.push_back(&emitProbS_);
loadModel_(filePath.c_str());
LogInfo("HMMSegment init(%s) ok.", filePath.c_str());
}
public:
using SegmentBase::cut;
public:
bool cut(Unicode::const_iterator begin, Unicode::const_iterator end, vector<Unicode>& res)const {
Unicode::const_iterator left = begin;
Unicode::const_iterator right = begin;
@ -77,7 +48,6 @@ class HMMSegment: public SegmentBase {
}
return true;
}
public:
virtual bool cut(Unicode::const_iterator begin, Unicode::const_iterator end, vector<string>& res)const {
if(begin == end) {
return false;
@ -141,7 +111,7 @@ class HMMSegment: public SegmentBase {
Unicode::const_iterator left = begin;
Unicode::const_iterator right;
for(size_t i = 0; i < status.size(); i++) {
if(status[i] % 2) { //if(E == status[i] || S == status[i])
if(status[i] % 2) { //if(HMMModel::E == status[i] || HMMModel::S == status[i])
right = begin + i + 1;
res.push_back(Unicode(left, right));
left = right;
@ -150,12 +120,13 @@ class HMMSegment: public SegmentBase {
return true;
}
bool viterbi_(Unicode::const_iterator begin, Unicode::const_iterator end, vector<size_t>& status)const {
bool viterbi_(Unicode::const_iterator begin, Unicode::const_iterator end,
vector<size_t>& status) const {
if(begin == end) {
return false;
}
size_t Y = STATUS_SUM;
size_t Y = HMMModel::STATUS_SUM;
size_t X = end - begin;
size_t XYSize = X * Y;
@ -167,22 +138,21 @@ class HMMSegment: public SegmentBase {
//start
for(size_t y = 0; y < Y; y++) {
weight[0 + y * X] = startProb_[y] + getEmitProb_(emitProbVec_[y], *begin, MIN_DOUBLE);
weight[0 + y * X] = model_.startProb[y] + model_.getEmitProb(model_.emitProbVec[y], *begin, MIN_DOUBLE);
path[0 + y * X] = -1;
}
double emitProb;
for(size_t x = 1; x < X; x++) {
for(size_t y = 0; y < Y; y++) {
now = x + y*X;
weight[now] = MIN_DOUBLE;
path[now] = E; // warning
emitProb = getEmitProb_(emitProbVec_[y], *(begin+x), MIN_DOUBLE);
path[now] = HMMModel::E; // warning
emitProb = model_.getEmitProb(model_.emitProbVec[y], *(begin+x), MIN_DOUBLE);
for(size_t preY = 0; preY < Y; preY++) {
old = x - 1 + preY * X;
tmp = weight[old] + transProb_[preY][y] + emitProb;
tmp = weight[old] + model_.transProb[preY][y] + emitProb;
if(tmp > weight[now]) {
weight[now] = tmp;
path[now] = preY;
@ -191,13 +161,13 @@ class HMMSegment: public SegmentBase {
}
}
endE = weight[X-1+E*X];
endS = weight[X-1+S*X];
endE = weight[X-1+HMMModel::E*X];
endS = weight[X-1+HMMModel::S*X];
stat = 0;
if(endE >= endS) {
stat = E;
stat = HMMModel::E;
} else {
stat = S;
stat = HMMModel::S;
}
status.resize(X);
@ -208,114 +178,10 @@ class HMMSegment: public SegmentBase {
return true;
}
void loadModel_(const char* const filePath) {
ifstream ifile(filePath);
if(!ifile.is_open()) {
LogFatal("open %s failed.", filePath);
}
string line;
vector<string> tmp;
vector<string> tmp2;
//load startProb_
if(!getLine_(ifile, line)) {
LogFatal("load startProb_");
}
split(line, tmp, " ");
if(tmp.size() != STATUS_SUM) {
LogFatal("start_p illegal");
}
for(size_t j = 0; j< tmp.size(); j++) {
startProb_[j] = atof(tmp[j].c_str());
}
//load transProb_
for(size_t i = 0; i < STATUS_SUM; i++) {
if(!getLine_(ifile, line)) {
LogFatal("load transProb_ failed.");
}
split(line, tmp, " ");
if(tmp.size() != STATUS_SUM) {
LogFatal("trans_p illegal");
}
for(size_t j =0; j < STATUS_SUM; j++) {
transProb_[i][j] = atof(tmp[j].c_str());
}
}
HMMModel model_;
}; // class HMMSegment
//load emitProbB_
if(!getLine_(ifile, line) || !loadEmitProb_(line, emitProbB_)) {
LogFatal("load emitProbB_ failed.");
}
//load emitProbE_
if(!getLine_(ifile, line) || !loadEmitProb_(line, emitProbE_)) {
LogFatal("load emitProbE_ failed.");
}
//load emitProbM_
if(!getLine_(ifile, line) || !loadEmitProb_(line, emitProbM_)) {
LogFatal("load emitProbM_ failed.");
}
//load emitProbS_
if(!getLine_(ifile, line) || !loadEmitProb_(line, emitProbS_)) {
LogFatal("load emitProbS_ failed.");
}
}
bool getLine_(ifstream& ifile, string& line) {
while(getline(ifile, line)) {
trim(line);
if(line.empty()) {
continue;
}
if(startsWith(line, "#")) {
continue;
}
return true;
}
return false;
}
bool loadEmitProb_(const string& line, EmitProbMap& mp) {
if(line.empty()) {
return false;
}
vector<string> tmp, tmp2;
Unicode unicode;
split(line, tmp, ",");
for(size_t i = 0; i < tmp.size(); i++) {
split(tmp[i], tmp2, ":");
if(2 != tmp2.size()) {
LogError("emitProb_ illegal.");
return false;
}
if(!TransCode::decode(tmp2[0], unicode) || unicode.size() != 1) {
LogError("TransCode failed.");
return false;
}
mp[unicode[0]] = atof(tmp2[1].c_str());
}
return true;
}
double getEmitProb_(const EmitProbMap* ptMp, uint16_t key, double defVal)const {
EmitProbMap::const_iterator cit = ptMp->find(key);
if(cit == ptMp->end()) {
return defVal;
}
return cit->second;
}
private:
char statMap_[STATUS_SUM];
double startProb_[STATUS_SUM];
double transProb_[STATUS_SUM][STATUS_SUM];
EmitProbMap emitProbB_;
EmitProbMap emitProbE_;
EmitProbMap emitProbM_;
EmitProbMap emitProbS_;
vector<EmitProbMap* > emitProbVec_;
};
}
} // namespace CppJieba
#endif

18
src/KeywordExtractor.hpp
View File

@ -11,17 +11,17 @@ using namespace Limonp;
/*utf8*/
class KeywordExtractor {
public:
KeywordExtractor() {};
KeywordExtractor(const string& dictPath, const string& hmmFilePath, const string& idfPath, const string& stopWordPath, const string& userDict = "") {
init(dictPath, hmmFilePath, idfPath, stopWordPath, userDict);
};
~KeywordExtractor() {};
void init(const string& dictPath, const string& hmmFilePath, const string& idfPath, const string& stopWordPath, const string& userDict = "") {
KeywordExtractor(const string& dictPath,
const string& hmmFilePath,
const string& idfPath,
const string& stopWordPath,
const string& userDict = "")
: segment_(dictPath, hmmFilePath, userDict) {
loadIdfDict_(idfPath);
loadStopWordDict_(stopWordPath);
segment_.init(dictPath, hmmFilePath, userDict);
};
}
~KeywordExtractor() {
}
bool extract(const string& str, vector<string>& keywords, size_t topN) const {
vector<pair<string, double> > topWords;

14
src/MixSegment.hpp
View File

@ -9,18 +9,14 @@
namespace CppJieba {
class MixSegment: public SegmentBase {
public:
MixSegment() {
}
MixSegment(const string& mpSegDict, const string& hmmSegDict, const string& userDict = "") {
init(mpSegDict, hmmSegDict, userDict);
MixSegment(const string& mpSegDict, const string& hmmSegDict,
const string& userDict = "")
: mpSeg_(mpSegDict, userDict),
hmmSeg_(hmmSegDict) {
LogInfo("MixSegment init %s, %s", mpSegDict.c_str(), hmmSegDict.c_str());
}
virtual ~MixSegment() {
}
void init(const string& mpSegDict, const string& hmmSegDict, const string& userDict = "") {
mpSeg_.init(mpSegDict, userDict);
hmmSeg_.init(hmmSegDict);
LogInfo("MixSegment init(%s, %s)", mpSegDict.c_str(), hmmSegDict.c_str());
}
using SegmentBase::cut;
virtual bool cut(Unicode::const_iterator begin, Unicode::const_iterator end, vector<Unicode>& res) const {
vector<Unicode> words;

22
src/PosTagger.hpp
View File

@ -14,27 +14,15 @@ static const char* const POS_X = "x";
class PosTagger {
public:
PosTagger() {
}
PosTagger(
const string& dictPath,
PosTagger(const string& dictPath,
const string& hmmFilePath,
const string& userDictPath = ""
) {
init(dictPath, hmmFilePath, userDictPath);
const string& userDictPath = "")
: segment_(dictPath, hmmFilePath, userDictPath) {
dictTrie_ = segment_.getDictTrie();
LIMONP_CHECK(dictTrie_);
}
~PosTagger() {
}
void init(
const string& dictPath,
const string& hmmFilePath,
const string& userDictPath = ""
) {
segment_.init(dictPath, hmmFilePath, userDictPath);
dictTrie_ = segment_.getDictTrie();
LIMONP_CHECK(dictTrie_);
};
bool tag(const string& src, vector<pair<string, string> >& res) const {
vector<string> cutRes;

15
src/QuerySegment.hpp
View File

@ -16,17 +16,14 @@
namespace CppJieba {
class QuerySegment: public SegmentBase {
public:
QuerySegment() {};
QuerySegment(const string& dict, const string& model, size_t maxWordLen = 4, const string& userDict = "") {
init(dict, model, maxWordLen, userDict);
};
virtual ~QuerySegment() {};
void init(const string& dict, const string& model, size_t maxWordLen, const string& userDict = "") {
mixSeg_.init(dict, model, userDict);
fullSeg_.init(mixSeg_.getDictTrie());
QuerySegment(const string& dict, const string& model, size_t maxWordLen = 4,
const string& userDict = "")
: mixSeg_(dict, model, userDict),
fullSeg_(mixSeg_.getDictTrie()) {
assert(maxWordLen);
maxWordLen_ = maxWordLen;
}
};
virtual ~QuerySegment() {};
using SegmentBase::cut;
bool cut(Unicode::const_iterator begin, Unicode::const_iterator end, vector<Unicode>& res) const {
//use mix cut first