third_codes/cppjieba
1
0
Fork 0
mirror of https://github.com/yanyiwu/cppjieba.git synced 2025-07-18 00:00:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

merge dev

Browse Source
This commit is contained in:
wyy 2014-04-11 13:07:51 +08:00
commit 0ad0d74d19
8 changed files with 332 additions and 251 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

179
src/DictTrie.hpp Normal file
View File

@ -0,0 +1,179 @@
#ifndef CPPJIEBA_DICT_TRIE_HPP
#define CPPJIEBA_DICT_TRIE_HPP
#include <iostream>
#include <fstream>
#include <map>
#include <cstring>
#include <stdint.h>
#include <cmath>
#include <limits>
#include "Limonp/str_functs.hpp"
#include "Limonp/logger.hpp"
#include "Limonp/InitOnOff.hpp"
#include "TransCode.hpp"
#include "Trie.hpp"
namespace CppJieba
{
using namespace Limonp;
const double MIN_DOUBLE = -3.14e+100;
const double MAX_DOUBLE = 3.14e+100;
const size_t DICT_COLUMN_NUM = 3;
struct DictUnit
{
Unicode word;
size_t freq;
string tag;
double logFreq; //logFreq = log(freq/sum(freq));
};
inline ostream & operator << (ostream& os, const DictUnit& unit)
{
string s;
s << unit.word;
return os << string_format("%s %u %s %.3lf", s.c_str(), unit.freq, unit.tag.c_str(), unit.logFreq);
}
typedef map<size_t, const DictUnit*> DagType;
class DictTrie: public InitOnOff
{
public:
typedef Trie<Unicode::value_type, DictUnit> TrieType;
private:
vector<DictUnit> _nodeInfos;
TrieType * _trie;
size_t _freqSum;
double _minLogFreq;
public:
DictTrie()
{
_trie = NULL;
_freqSum = 0;
_minLogFreq = MAX_DOUBLE;
_setInitFlag(false);
}
DictTrie(const string& filePath)
{
new (this) DictTrie();
_setInitFlag(init(filePath));
}
~DictTrie()
{
if(_trie)
{
delete _trie;
}
}
private:
public:
bool init(const string& filePath)
{
assert(!_getInitFlag());
_loadDict(filePath, _nodeInfos);
_freqSum = _calculateFreqSum(_nodeInfos);
assert(_freqSum);
_minLogFreq = _calculateLogFreqAndGetMinValue(_nodeInfos, _freqSum);
_trie = _creatTrie(_nodeInfos);
return _setInitFlag(_trie);
}
public:
const DictUnit* find(Unicode::const_iterator begin, Unicode::const_iterator end) const
{
return _trie->find(begin, end);
}
bool find(Unicode::const_iterator begin, Unicode::const_iterator end, DagType& dag, size_t offset = 0) const
{
return _trie->find(begin, end, dag, offset);
}
public:
double getMinLogFreq() const {return _minLogFreq;};
private:
TrieType * _creatTrie(const vector<DictUnit>& dictUnits)
{
if(dictUnits.empty())
{
return NULL;
}
vector<Unicode> words;
vector<const DictUnit*> valuePointers;
for(size_t i = 0 ; i < dictUnits.size(); i ++)
{
words.push_back(dictUnits[i].word);
valuePointers.push_back(&dictUnits[i]);
}
TrieType * trie = new TrieType(words, valuePointers);
return trie;
}
void _loadDict(const string& filePath, vector<DictUnit>& nodeInfos) const
{
ifstream ifs(filePath.c_str());
if(!ifs)
{
LogFatal("open %s failed.", filePath.c_str());
exit(1);
}
string line;
vector<string> buf;
nodeInfos.clear();
DictUnit nodeInfo;
for(size_t lineno = 0 ; getline(ifs, line); lineno++)
{
split(line, buf, " ");
assert(buf.size() == DICT_COLUMN_NUM);
if(!TransCode::decode(buf[0], nodeInfo.word))
{
LogError("line[%u:%s] illegal.", lineno, line.c_str());
continue;
}
nodeInfo.freq = atoi(buf[1].c_str());
nodeInfo.tag = buf[2];
nodeInfos.push_back(nodeInfo);
}
}
size_t _calculateFreqSum(const vector<DictUnit>& nodeInfos) const
{
size_t freqSum = 0;
for(size_t i = 0; i < nodeInfos.size(); i++)
{
freqSum += nodeInfos[i].freq;
}
return freqSum;
}
double _calculateLogFreqAndGetMinValue(vector<DictUnit>& nodeInfos, size_t freqSum) const
{
assert(freqSum);
double minLogFreq = MAX_DOUBLE;
for(size_t i = 0; i < nodeInfos.size(); i++)
{
DictUnit& nodeInfo = nodeInfos[i];
assert(nodeInfo.freq);
nodeInfo.logFreq = log(double(nodeInfo.freq)/double(freqSum));
if(minLogFreq > nodeInfo.logFreq)
{
minLogFreq = nodeInfo.logFreq;
}
}
return minLogFreq;
}
};
}
#endif

12
src/FullSegment.hpp
View File

@ -5,7 +5,7 @@
#include <set>
#include <cassert>
#include "Limonp/logger.hpp"
#include "Trie.hpp"
#include "DictTrie.hpp"
#include "ISegment.hpp"
#include "SegmentBase.hpp"
#include "TransCode.hpp"
@ -15,7 +15,7 @@ namespace CppJieba
class FullSegment: public SegmentBase
{
private:
Trie _trie;
DictTrie _dictTrie;
public:
FullSegment(){_setInitFlag(false);};
@ -29,8 +29,8 @@ namespace CppJieba
LogError("already inited before now.");
return false;
}
_trie.init(dictPath.c_str());
assert(_trie);
_dictTrie.init(dictPath.c_str());
assert(_dictTrie);
return _setInitFlag(true);
}
@ -61,10 +61,10 @@ namespace CppJieba
for (Unicode::const_iterator uItr = begin; uItr != end; uItr++)
{
//find word start from uItr
if (_trie.find(uItr, end, tRes, 0))
if (_dictTrie.find(uItr, end, tRes, 0))
{
for(DagType::const_iterator itr = tRes.begin(); itr != tRes.end(); itr++)
//for (vector<pair<size_t, const TrieNodeInfo*> >::const_iterator itr = tRes.begin(); itr != tRes.end(); itr++)
//for (vector<pair<size_t, const DictUnit*> >::const_iterator itr = tRes.begin(); itr != tRes.end(); itr++)
{
wordLen = itr->second->word.size();
if (wordLen >= 2 || (tRes.size() == 1 && maxIdx <= uIdx))

2
src/HMMSegment.hpp
View File

@ -10,7 +10,7 @@
#include "TransCode.hpp"
#include "ISegment.hpp"
#include "SegmentBase.hpp"
#include "Trie.hpp"
#include "DictTrie.hpp"
namespace CppJieba
{

20
src/MPSegment.hpp
View File

@ -9,8 +9,8 @@
#include <set>
#include <cassert>
#include "Limonp/logger.hpp"
#include "Trie.hpp"
#include "Trie.hpp"
#include "DictTrie.hpp"
#include "DictTrie.hpp"
#include "ISegment.hpp"
#include "SegmentBase.hpp"
@ -21,7 +21,7 @@ namespace CppJieba
{
uint16_t uniCh;
DagType dag;
const TrieNodeInfo * pInfo;
const DictUnit * pInfo;
double weight;
SegmentChar():uniCh(0), pInfo(NULL), weight(0.0)
@ -32,7 +32,7 @@ namespace CppJieba
class MPSegment: public SegmentBase
{
protected:
Trie _trie;
DictTrie _dictTrie;
public:
MPSegment(){_setInitFlag(false);};
@ -49,8 +49,8 @@ namespace CppJieba
LogError("already inited before now.");
return false;
}
_trie.init(dictPath);
assert(_trie);
_dictTrie.init(dictPath);
assert(_dictTrie);
LogInfo("MPSegment init(%s) ok", dictPath.c_str());
return _setInitFlag(true);
}
@ -124,7 +124,7 @@ namespace CppJieba
schar.uniCh = *it;
offset = it - begin;
schar.dag.clear();
_trie.find(it, end, schar.dag, offset);
_dictTrie.find(it, end, schar.dag, offset);
if(!isIn(schar.dag, offset))
{
schar.dag[offset] = NULL;
@ -142,7 +142,7 @@ namespace CppJieba
}
size_t nextPos;
const TrieNodeInfo* p;
const DictUnit* p;
double val;
for(int i = segContext.size() - 1; i >= 0; i--)
@ -165,7 +165,7 @@ namespace CppJieba
}
else
{
val += _trie.getMinLogFreq();
val += _dictTrie.getMinLogFreq();
}
if(val > segContext[i].weight)
{
@ -182,7 +182,7 @@ namespace CppJieba
size_t i = 0;
while(i < segContext.size())
{
const TrieNodeInfo* p = segContext[i].pInfo;
const DictUnit* p = segContext[i].pInfo;
if(p)
{
res.push_back(p->word);

12
src/PosTagger.hpp
View File

@ -3,7 +3,7 @@
#include "MixSegment.hpp"
#include "Limonp/str_functs.hpp"
#include "Trie.hpp"
#include "DictTrie.hpp"
namespace CppJieba
{
@ -13,7 +13,7 @@ namespace CppJieba
{
private:
MixSegment _segment;
Trie _trie;
DictTrie _dictTrie;
public:
PosTagger(){_setInitFlag(false);};
@ -27,8 +27,8 @@ namespace CppJieba
{
assert(!_getInitFlag());
_trie.init(dictPath);
assert(_trie);
_dictTrie.init(dictPath);
assert(_dictTrie);
return _setInitFlag(_segment.init(dictPath, hmmFilePath));
};
@ -42,7 +42,7 @@ namespace CppJieba
return false;
}
const TrieNodeInfo *tmp = NULL;
const DictUnit *tmp = NULL;
Unicode unico;
for (vector<string>::iterator itr = cutRes.begin(); itr != cutRes.end(); ++itr)
{
@ -51,7 +51,7 @@ namespace CppJieba
LogError("decode failed.");
return false;
}
tmp = _trie.find(unico.begin(), unico.end());
tmp = _dictTrie.find(unico.begin(), unico.end());
res.push_back(make_pair(*itr, tmp == NULL ? "x" : tmp->tag));
}
tmp = NULL;

4
src/QuerySegment.hpp
View File

@ -5,13 +5,13 @@
#include <set>
#include <cassert>
#include "Limonp/logger.hpp"
#include "Trie.hpp"
#include "DictTrie.hpp"
#include "ISegment.hpp"
#include "SegmentBase.hpp"
#include "FullSegment.hpp"
#include "MixSegment.hpp"
#include "TransCode.hpp"
#include "Trie.hpp"
#include "DictTrie.hpp"
namespace CppJieba
{

326
src/Trie.hpp
View File

@ -1,241 +1,143 @@
/************************************
* file enc : ASCII
* author : wuyanyi09@gmail.com
************************************/
#ifndef CPPJIEBA_TRIE_H
#define CPPJIEBA_TRIE_H
#include <iostream>
#include <fstream>
#include <map>
#include <cstring>
#include <stdint.h>
#include <cmath>
#include <limits>
#include "Limonp/str_functs.hpp"
#include "Limonp/logger.hpp"
#include "Limonp/InitOnOff.hpp"
#include "TransCode.hpp"
#ifndef CPPJIEBA_TRIE_HPP
#define CPPJIEBA_TRIE_HPP
#include "Limonp/std_outbound.hpp"
#include <vector>
namespace CppJieba
{
using namespace Limonp;
const double MIN_DOUBLE = -3.14e+100;
const double MAX_DOUBLE = 3.14e+100;
const size_t DICT_COLUMN_NUM = 3;
typedef map<uint16_t, struct TrieNode*> TrieNodeMap;
struct TrieNodeInfo;
struct TrieNode
{
TrieNodeMap hmap;
const TrieNodeInfo * ptTrieNodeInfo;
TrieNode(): ptTrieNodeInfo(NULL)
{}
};
using namespace std;
template <class KeyType, class ValueType>
class TrieNode
{
public:
typedef map<KeyType, TrieNode<KeyType, ValueType>* > KeyMapType;
public:
KeyMapType * ptKeyMap;
const ValueType * ptValue;
};
struct TrieNodeInfo
{
Unicode word;
size_t freq;
string tag;
double logFreq; //logFreq = log(freq/sum(freq));
};
inline ostream& operator << (ostream& os, const TrieNodeInfo & nodeInfo)
{
return os << nodeInfo.word << ":" << nodeInfo.freq << ":" << nodeInfo.tag << ":" << nodeInfo.logFreq ;
}
typedef map<size_t, const TrieNodeInfo*> DagType;
class Trie: public InitOnOff
{
private:
TrieNode* _root;
vector<TrieNodeInfo> _nodeInfos;
int64_t _freqSum;
double _minLogFreq;
public:
Trie()
{
_root = new TrieNode;
_freqSum = 0;
_minLogFreq = MAX_DOUBLE;
_setInitFlag(false);
}
Trie(const string& filePath)
{
new (this) Trie();
_setInitFlag(init(filePath));
}
~Trie()
{
_deleteNode(_root);
}
private:
public:
bool init(const string& filePath)
{
assert(!_getInitFlag());
_loadDict(filePath, _nodeInfos);
_createTrie(_nodeInfos, _root);
_freqSum = _calculateFreqSum(_nodeInfos);
assert(_freqSum);
_minLogFreq = _calculateLogFreqAndGetMinValue(_nodeInfos, _freqSum);
return _setInitFlag(true);
}
public:
const TrieNodeInfo* find(Unicode::const_iterator begin, Unicode::const_iterator end)const
{
TrieNodeMap::const_iterator citer;
const TrieNode* p = _root;
for(Unicode::const_iterator it = begin; it != end; it++)
template <class KeyType, class ValueType>
class Trie
{
public:
typedef TrieNode<KeyType, ValueType> TrieNodeType;
private:
TrieNodeType* _root;
public:
Trie(const vector<vector<KeyType> >& keys, const vector<const ValueType* >& valuePointers)
{
citer = p->hmap.find(*it);
if(p->hmap.end() == citer)
{
return NULL;
}
p = citer->second;
_root = new TrieNodeType;
_root->ptKeyMap = NULL;
_root->ptValue = NULL;
_createTrie(keys, valuePointers);
}
return p->ptTrieNodeInfo;
}
bool find(Unicode::const_iterator begin, Unicode::const_iterator end, DagType & res, size_t offset = 0) const
{
const TrieNode* p = _root;
TrieNodeMap::const_iterator citer;
for (Unicode::const_iterator itr = begin; itr != end; itr++)
~Trie()
{
citer = p->hmap.find(*itr);
if(p->hmap.end() == citer)
if(_root)
{
break;
}
p = citer->second;
if(p->ptTrieNodeInfo)
{
res[itr - begin + offset] = p->ptTrieNodeInfo;
_deleteNode(_root);
}
}
return !res.empty();
}
public:
double getMinLogFreq() const {return _minLogFreq;};
private:
void _insertNode(const TrieNodeInfo& nodeInfo, TrieNode* ptNode) const
{
const Unicode& unico = nodeInfo.word;
TrieNodeMap::const_iterator citer;
for(size_t i = 0; i < unico.size(); i++)
public:
const ValueType* find(typename vector<KeyType>::const_iterator begin, typename vector<KeyType>::const_iterator end) const
{
uint16_t cu = unico[i];
assert(ptNode);
citer = ptNode->hmap.find(cu);
if(ptNode->hmap.end() == citer)
{
TrieNode * next = new TrieNode;
ptNode->hmap[cu] = next;
ptNode = next;
}
else
typename TrieNodeType::KeyMapType::const_iterator citer;
const TrieNodeType* ptNode = _root;
for(typename vector<KeyType>::const_iterator it = begin; it != end; it++)
{
assert(ptNode);
if(NULL == ptNode->ptKeyMap || ptNode->ptKeyMap->end() == (citer = ptNode->ptKeyMap->find(*it)))
{
return NULL;
}
ptNode = citer->second;
}
return ptNode->ptValue;
}
ptNode->ptTrieNodeInfo = &nodeInfo;
}
private:
void _loadDict(const string& filePath, vector<TrieNodeInfo>& nodeInfos) const
{
ifstream ifs(filePath.c_str());
if(!ifs)
bool find(typename vector<KeyType>::const_iterator begin, typename vector<KeyType> ::const_iterator end, map<typename vector<KeyType>::size_type, const ValueType* >& ordererMap, size_t offset = 0) const
{
LogFatal("open %s failed.", filePath.c_str());
exit(1);
}
string line;
vector<string> buf;
nodeInfos.clear();
TrieNodeInfo nodeInfo;
for(size_t lineno = 0 ; getline(ifs, line); lineno++)
{
split(line, buf, " ");
assert(buf.size() == DICT_COLUMN_NUM);
if(!TransCode::decode(buf[0], nodeInfo.word))
const TrieNodeType * ptNode = _root;
typename TrieNodeType::KeyMapType::const_iterator citer;
ordererMap.clear();
for(typename vector<KeyType>::const_iterator itr = begin; itr != end ; itr++)
{
LogError("line[%u:%s] illegal.", lineno, line.c_str());
continue;
assert(ptNode);
if(NULL == ptNode->ptKeyMap || ptNode->ptKeyMap->end() == (citer = ptNode->ptKeyMap->find(*itr)))
{
break;
}
ptNode = citer->second;
if(ptNode->ptValue)
{
ordererMap[itr - begin + offset] = ptNode->ptValue;
}
}
nodeInfo.freq = atoi(buf[1].c_str());
nodeInfo.tag = buf[2];
nodeInfos.push_back(nodeInfo);
return ordererMap.size();
}
}
bool _createTrie(const vector<TrieNodeInfo>& nodeInfos, TrieNode * ptNode)
{
for(size_t i = 0; i < _nodeInfos.size(); i++)
private:
void _createTrie(const vector<vector<KeyType> >& keys, const vector<const ValueType*>& valuePointers)
{
_insertNode(_nodeInfos[i], ptNode);
}
return true;
}
size_t _calculateFreqSum(const vector<TrieNodeInfo>& nodeInfos) const
{
size_t freqSum = 0;
for(size_t i = 0; i < nodeInfos.size(); i++)
{
freqSum += nodeInfos[i].freq;
}
return freqSum;
}
double _calculateLogFreqAndGetMinValue(vector<TrieNodeInfo>& nodeInfos, size_t freqSum) const
{
assert(freqSum);
double minLogFreq = MAX_DOUBLE;
for(size_t i = 0; i < nodeInfos.size(); i++)
{
TrieNodeInfo& nodeInfo = nodeInfos[i];
assert(nodeInfo.freq);
nodeInfo.logFreq = log(double(nodeInfo.freq)/double(freqSum));
if(minLogFreq > nodeInfo.logFreq)
if(valuePointers.empty() || keys.empty())
{
minLogFreq = nodeInfo.logFreq;
return;
}
assert(keys.size() == valuePointers.size());
for(size_t i = 0; i < keys.size(); i++)
{
_insertNode(keys[i], valuePointers[i]);
}
}
return minLogFreq;
}
void _deleteNode(TrieNode* node)
{
if(!node)
private:
void _insertNode(const vector<KeyType>& key, const ValueType* ptValue)
{
return;
}
for(TrieNodeMap::iterator it = node->hmap.begin(); it != node->hmap.end(); it++)
{
TrieNode* next = it->second;
_deleteNode(next);
}
delete node;
}
TrieNodeType* ptNode = _root;
};
typename TrieNodeType::KeyMapType::const_iterator kmIter;
for(typename vector<KeyType>::const_iterator citer = key.begin(); citer != key.end(); citer++)
{
if(NULL == ptNode->ptKeyMap)
{
ptNode->ptKeyMap = new typename TrieNodeType::KeyMapType;
}
kmIter = ptNode->ptKeyMap->find(*citer);
if(ptNode->ptKeyMap->end() == kmIter)
{
TrieNodeType * nextNode = new TrieNodeType;
nextNode->ptKeyMap = NULL;
nextNode->ptValue = NULL;
(*ptNode->ptKeyMap)[*citer] = nextNode;
ptNode = nextNode;
}
else
{
ptNode = kmIter->second;
}
}
ptNode->ptValue = ptValue;
}
void _deleteNode(TrieNodeType* node)
{
if(!node)
{
return;
}
if(node->ptKeyMap)
{
typename TrieNodeType::KeyMapType::iterator it;
for(it = node->ptKeyMap->begin(); it != node->ptKeyMap->end(); it++)
{
_deleteNode(it->second);
}
delete node->ptKeyMap;
}
delete node;
}
};
}
#endif

28
test/unittest/TTrie.cpp
View File

@ -1,30 +1,30 @@
#include "src/Trie.hpp"
#include "src/DictTrie.hpp"
#include "gtest/gtest.h"
using namespace CppJieba;
static const char* const DICT_FILE = "../dict/extra_dict/jieba.dict.small.utf8";
TEST(TrieTest, NewAndDelete)
TEST(DictTrieTest, NewAndDelete)
{
Trie * trie;
trie = new Trie(DICT_FILE);
DictTrie * trie;
trie = new DictTrie(DICT_FILE);
delete trie;
trie = new Trie();
trie = new DictTrie();
delete trie;
}
TEST(TrieTest, Test1)
TEST(DictTrieTest, Test1)
{
string s1, s2;
Trie trie;
DictTrie trie;
ASSERT_TRUE(trie.init(DICT_FILE));
ASSERT_LT(trie.getMinLogFreq() + 15.6479, 0.001);
string word("来到");
Unicode uni;
ASSERT_TRUE(TransCode::decode(word, uni));
TrieNodeInfo nodeInfo;
DictUnit nodeInfo;
nodeInfo.word = uni;
nodeInfo.freq = 8779;
nodeInfo.tag = "v";
@ -32,11 +32,11 @@ TEST(TrieTest, Test1)
s1 << nodeInfo;
s2 << (*trie.find(uni.begin(), uni.end()));
EXPECT_EQ("[\"26469\", \"21040\"]:8779:v:-8.87033", s2);
EXPECT_EQ("[\"26469\", \"21040\"] 8779 v -8.870", s2);
word = "清华大学";
vector<pair<size_t, const TrieNodeInfo*> > res;
map<size_t, const TrieNodeInfo* > resMap;
map<size_t, const TrieNodeInfo* > mp;
vector<pair<size_t, const DictUnit*> > res;
map<size_t, const DictUnit* > resMap;
map<size_t, const DictUnit* > mp;
const char * words[] = {"清", "清华", "清华大学"};
for(size_t i = 0; i < sizeof(words)/sizeof(words[0]); i++)
{
@ -44,10 +44,10 @@ TEST(TrieTest, Test1)
res.push_back(make_pair(uni.size() - 1, trie.find(uni.begin(), uni.end())));
resMap[uni.size() - 1] = trie.find(uni.begin(), uni.end());
}
//TrieNodeInfo
//DictUnit
//res.push_back(make_pair(0, ))
vector<pair<size_t, const TrieNodeInfo*> > vec;
vector<pair<size_t, const DictUnit*> > vec;
ASSERT_TRUE(TransCode::decode(word, uni));
//print(uni);
ASSERT_TRUE(trie.find(uni.begin(), uni.end(), mp, 0));