mirror of
https://github.com/yanyiwu/cppjieba.git
synced 2025-07-18 00:00:12 +08:00
Merge pull request #10 from aholic/dev
change QuerySegment algorithm | add TrieManager | add md5 for file
This commit is contained in:
Wu Yanyi
commit
8c9907b27a
55
README.md
55
README.md
@ -80,10 +80,13 @@ Output:
|
||||
```
|
||||
我来到北京清华大学
|
||||
我来/到/北京/清华大学
|
||||
|
||||
他来到了网易杭研大厦
|
||||
他来/到/了/网易/杭/研大厦
|
||||
|
||||
小明硕士毕业于中国科学院计算所，后在日本京都大学深造
|
||||
小明/硕士/毕业于/中国/科学院/计算所/，/后/在/日/本/京/都/大/学/深/造
|
||||
|
||||
我来自北京邮电大学。。。学号091111xx。。。
|
||||
我来/自北京/邮电大学/。。。/学号/091111xx/。。。
|
||||
```
|
||||
@ -94,23 +97,60 @@ Output:
|
||||
```
|
||||
我来到北京清华大学
|
||||
我/来到/北京/清华大学
|
||||
|
||||
他来到了网易杭研大厦
|
||||
他/来到/了/网易/杭研/大厦
|
||||
杭研
|
||||
杭研
|
||||
|
||||
小明硕士毕业于中国科学院计算所，后在日本京都大学深造
|
||||
小明/硕士/毕业/于/中国科学院/计算所/，/后/在/日本京都大学/深造
|
||||
|
||||
我来自北京邮电大学。。。学号091111xx。。。
|
||||
我/来自/北京邮电大学/。。。/学号/091111xx/。。。
|
||||
```
|
||||
|
||||
### FullSegment's demo
|
||||
|
||||
Output:
|
||||
```
|
||||
我来到北京清华大学
|
||||
我/来到/北京/清华/清华大学/华大/大学
|
||||
|
||||
他来到了网易杭研大厦
|
||||
他/来到/了/网易/杭/研/大厦
|
||||
|
||||
小明硕士毕业于中国科学院计算所，后在日本京都大学深造
|
||||
小/明/硕士/毕业/于/中国/中国科学院/科学/科学院/学院/计算/计算所/，/后/在/日本/日本京都大学/京都/京都大学/大学/深造
|
||||
|
||||
我来自北京邮电大学。。。 学号 091111xx。。。
|
||||
我/来自/北京/北京邮电/北京邮电大学/邮电/邮电大学/电大/大学/。/。/。/ /学号/ 091111xx/。/。/。
|
||||
```
|
||||
|
||||
### QuerySegment's demo
|
||||
|
||||
Output:
|
||||
```
|
||||
我来到北京清华大学
|
||||
我/来到/北京/清华/清华大学/华大/大学
|
||||
|
||||
他来到了网易杭研大厦
|
||||
他/来到/了/网易/杭研/大厦
|
||||
|
||||
小明硕士毕业于中国科学院计算所，后在日本京都大学深造
|
||||
小明/硕士/毕业/于/中国/中国科学院/科学/科学院/学院/计算所/，/后/在/中国/中国科学院/科学/科学院/学院/日本/日本京都大学/京都/京都大学/大学/深造
|
||||
|
||||
我来自北京邮电大学。。。 学号 091111xx。。。
|
||||
我/来自/北京/北京邮电/北京邮电大学/邮电/邮电大学/电大/大学/。/。/。/ /学号/ 091111xx/。/。/。
|
||||
```
|
||||
|
||||
### 效果分析
|
||||
|
||||
以上依次是MP,HMM,Mix三种方法的效果。
|
||||
|
||||
可以看出效果最好的是Mix，也就是融合MP和HMM的切词算法。即可以准确切出词典已有的词，又可以切出像"杭研"这样的未登录词。
|
||||
|
||||
Full方法切出所有字典里的词语。
|
||||
|
||||
Query方法先使用Mix方法切词，对于切出来的较长的词再使用Full方法。
|
||||
|
||||
## 模块详解
|
||||
|
||||
@ -120,6 +160,10 @@ Output:
|
||||
|
||||
核心目录，包含主要源代码。
|
||||
|
||||
#### TrieManager模块
|
||||
TrieManager.hpp 提供一个单例TrieManager，负责管理trie树。
|
||||
通过该单例获取trie树时，会先判断是否已经由该字典文件生成了一颗trie树，如果已有则返回已有的trie树，否则重新创建一颗trie树返回。
|
||||
|
||||
#### Trie树
|
||||
Trie.hpp 负责载入词典的trie树，主要供Segment模块使用。
|
||||
|
||||
@ -133,6 +177,9 @@ HMMSegment.hpp
|
||||
HMM模型由dicts/下面的`hmm_model.utf8`提供。
|
||||
分词算法即viterbi算法。
|
||||
|
||||
FullSegment.hpp
|
||||
枚举句子中所有可能成词的情况，找出字典里存在的即可。
|
||||
|
||||
#### TransCode模块
|
||||
|
||||
TransCode.cpp/TransCode.h 负责转换编码类型，将utf8和gbk转换成`uint16_t`类型，也负责逆转换。
|
||||
@ -154,8 +201,8 @@ TransCode.cpp/TransCode.h 负责转换编码类型，将utf8和gbk转换成`uint
|
||||
|
||||
### MixSegment
|
||||
|
||||
分词速度大概是 62M / 54sec = 1.15M/sec
|
||||
测试环境: `Intel(R) Xeon(R) CPU E5506 @ 2.13GHz`
|
||||
分词速度大概是 = 2M/sec
|
||||
测试环境: `Intel(R) Xeon(R) CPU E5506 @ 2.13GHz` 电脑下开的ubuntu虚拟机
|
||||
|
||||
|
||||
## 联系客服
|
||||
|
||||
@ -9,13 +9,14 @@
|
||||
#include "ISegment.hpp"
|
||||
#include "SegmentBase.hpp"
|
||||
#include "TransCode.hpp"
|
||||
#include "TrieManager.hpp"
|
||||
|
||||
namespace CppJieba
|
||||
{
|
||||
class FullSegment: public SegmentBase
|
||||
{
|
||||
private:
|
||||
Trie _trie;
|
||||
Trie* _trie;
|
||||
const string _dictPath;
|
||||
|
||||
public:
|
||||
@ -24,36 +25,25 @@ namespace CppJieba
|
||||
public:
|
||||
bool init()
|
||||
{
|
||||
#ifndef NO_CODING_LOG
|
||||
if(_getInitFlag())
|
||||
{
|
||||
LogError("already inited before now.");
|
||||
return false;
|
||||
}
|
||||
#endif
|
||||
if(!_trie.init())
|
||||
_trie = TrieManager::getInstance().getTrie(_dictPath.c_str());
|
||||
if (NULL == _trie)
|
||||
{
|
||||
LogError("_trie.init failed.");
|
||||
LogError("get NULL pointor from getTrie(\"%s\")", _dictPath.c_str());
|
||||
return false;
|
||||
}
|
||||
LogInfo("_trie.loadDict(%s) start...", _dictPath.c_str());
|
||||
if(!_trie.loadDict(_dictPath.c_str()))
|
||||
{
|
||||
LogError("_trie.loadDict faield.");
|
||||
return false;
|
||||
}
|
||||
LogInfo("_trie.loadDict end.");
|
||||
return _setInitFlag(true);
|
||||
}
|
||||
bool dispose()
|
||||
{
|
||||
#ifndef NO_CODING_LOG
|
||||
if(!_getInitFlag())
|
||||
{
|
||||
return true;
|
||||
}
|
||||
#endif
|
||||
_trie.dispose();
|
||||
_setInitFlag(false);
|
||||
return true;
|
||||
}
|
||||
@ -65,18 +55,12 @@ namespace CppJieba
|
||||
bool cut(Unicode::const_iterator begin, Unicode::const_iterator end, vector<Unicode>& res) const
|
||||
{
|
||||
assert(_getInitFlag());
|
||||
#ifndef NO_CODING_LOG
|
||||
//if (!_getInitFlag())
|
||||
//{
|
||||
// LogError("not inited.");
|
||||
// return false;
|
||||
//}
|
||||
if (begin >= end)
|
||||
{
|
||||
LogError("begin >= end");
|
||||
return false;
|
||||
}
|
||||
#endif
|
||||
|
||||
//resut of searching in trie tree
|
||||
vector<pair<uint, const TrieNodeInfo*> > tRes;
|
||||
|
||||
@ -91,7 +75,7 @@ namespace CppJieba
|
||||
for (Unicode::const_iterator uItr = begin; uItr != end; uItr++)
|
||||
{
|
||||
//find word start from uItr
|
||||
if (_trie.find(uItr, end, tRes))
|
||||
if (_trie->find(uItr, end, tRes))
|
||||
{
|
||||
for (vector<pair<uint, const TrieNodeInfo*> >::const_iterator itr = tRes.begin(); itr != tRes.end(); itr++)
|
||||
{
|
||||
@ -123,26 +107,21 @@ namespace CppJieba
|
||||
|
||||
bool cut(Unicode::const_iterator begin, Unicode::const_iterator end, vector<string>& res) const
|
||||
{
|
||||
#ifndef NO_CODING_LOG
|
||||
if (!_getInitFlag())
|
||||
assert(_getInitFlag());
|
||||
if (begin >= end)
|
||||
{
|
||||
LogError("not inited.");
|
||||
LogError("begin >= end");
|
||||
return false;
|
||||
}
|
||||
if (begin > end)
|
||||
{
|
||||
LogError("begin > end");
|
||||
return false;
|
||||
}
|
||||
#endif
|
||||
|
||||
vector<Unicode> uRes;
|
||||
if (!cut(begin, end, uRes))
|
||||
{
|
||||
LogError("get unicode cut result error.");
|
||||
return false;
|
||||
}
|
||||
string tmp;
|
||||
|
||||
string tmp;
|
||||
for (vector<Unicode>::const_iterator uItr = uRes.begin(); uItr != uRes.end(); uItr++)
|
||||
{
|
||||
if (TransCode::encode(*uItr, tmp))
|
||||
|
||||
435
src/Limonp/md5.hpp
Normal file
435
src/Limonp/md5.hpp
Normal file
@ -0,0 +1,435 @@
|
||||
#ifndef __MD5_H__
|
||||
#define __MD5_H__
|
||||
|
||||
// Copyright (C) 1991-2, RSA Data Security, Inc. Created 1991. All
|
||||
// rights reserved.
|
||||
|
||||
// License to copy and use this software is granted provided that it
|
||||
// is identified as the "RSA Data Security, Inc. MD5 Message-Digest
|
||||
// Algorithm" in all material mentioning or referencing this software
|
||||
// or this function.
|
||||
//
|
||||
// License is also granted to make and use derivative works provided
|
||||
// that such works are identified as "derived from the RSA Data
|
||||
// Security, Inc. MD5 Message-Digest Algorithm" in all material
|
||||
// mentioning or referencing the derived work.
|
||||
//
|
||||
// RSA Data Security, Inc. makes no representations concerning either
|
||||
// the merchantability of this software or the suitability of this
|
||||
// software for any particular purpose. It is provided "as is"
|
||||
// without express or implied warranty of any kind.
|
||||
//
|
||||
// These notices must be retained in any copies of any part of this
|
||||
// documentation and/or software.
|
||||
|
||||
|
||||
|
||||
// The original md5 implementation avoids external libraries.
|
||||
// This version has dependency on stdio.h for file input and
|
||||
// string.h for memcpy.
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
#include <iostream>
|
||||
|
||||
namespace Limonp
|
||||
{
|
||||
|
||||
#pragma region MD5 defines
|
||||
// Constants for MD5Transform routine.
|
||||
#define S11 7
|
||||
#define S12 12
|
||||
#define S13 17
|
||||
#define S14 22
|
||||
#define S21 5
|
||||
#define S22 9
|
||||
#define S23 14
|
||||
#define S24 20
|
||||
#define S31 4
|
||||
#define S32 11
|
||||
#define S33 16
|
||||
#define S34 23
|
||||
#define S41 6
|
||||
#define S42 10
|
||||
#define S43 15
|
||||
#define S44 21
|
||||
|
||||
|
||||
// F, G, H and I are basic MD5 functions.
|
||||
#define F(x, y, z) (((x) & (y)) | ((~x) & (z)))
|
||||
#define G(x, y, z) (((x) & (z)) | ((y) & (~z)))
|
||||
#define H(x, y, z) ((x) ^ (y) ^ (z))
|
||||
#define I(x, y, z) ((y) ^ ((x) | (~z)))
|
||||
|
||||
// ROTATE_LEFT rotates x left n bits.
|
||||
#define ROTATE_LEFT(x, n) (((x) << (n)) | ((x) >> (32-(n))))
|
||||
|
||||
// FF, GG, HH, and II transformations for rounds 1, 2, 3, and 4.
|
||||
// Rotation is separate from addition to prevent recomputation.
|
||||
#define FF(a, b, c, d, x, s, ac) { \
|
||||
(a) += F ((b), (c), (d)) + (x) + (UINT4)(ac); \
|
||||
(a) = ROTATE_LEFT ((a), (s)); \
|
||||
(a) += (b); \
|
||||
}
|
||||
#define GG(a, b, c, d, x, s, ac) { \
|
||||
(a) += G ((b), (c), (d)) + (x) + (UINT4)(ac); \
|
||||
(a) = ROTATE_LEFT ((a), (s)); \
|
||||
(a) += (b); \
|
||||
}
|
||||
#define HH(a, b, c, d, x, s, ac) { \
|
||||
(a) += H ((b), (c), (d)) + (x) + (UINT4)(ac); \
|
||||
(a) = ROTATE_LEFT ((a), (s)); \
|
||||
(a) += (b); \
|
||||
}
|
||||
#define II(a, b, c, d, x, s, ac) { \
|
||||
(a) += I ((b), (c), (d)) + (x) + (UINT4)(ac); \
|
||||
(a) = ROTATE_LEFT ((a), (s)); \
|
||||
(a) += (b); \
|
||||
}
|
||||
#pragma endregion
|
||||
|
||||
|
||||
typedef unsigned char BYTE ;
|
||||
|
||||
// POINTER defines a generic pointer type
|
||||
typedef unsigned char *POINTER;
|
||||
|
||||
// UINT2 defines a two byte word
|
||||
typedef unsigned short int UINT2;
|
||||
|
||||
// UINT4 defines a four byte word
|
||||
typedef unsigned long int UINT4;
|
||||
|
||||
static unsigned char PADDING[64] = {
|
||||
0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
|
||||
};
|
||||
// convenient object that wraps
|
||||
// the C-functions for use in C++ only
|
||||
class MD5
|
||||
{
|
||||
private:
|
||||
struct __context_t {
|
||||
UINT4 state[4]; /* state (ABCD) */
|
||||
UINT4 count[2]; /* number of bits, modulo 2^64 (lsb first) */
|
||||
unsigned char buffer[64]; /* input buffer */
|
||||
} context ;
|
||||
|
||||
#pragma region static helper functions
|
||||
// The core of the MD5 algorithm is here.
|
||||
// MD5 basic transformation. Transforms state based on block.
|
||||
static void MD5Transform( UINT4 state[4], unsigned char block[64] )
|
||||
{
|
||||
UINT4 a = state[0], b = state[1], c = state[2], d = state[3], x[16];
|
||||
|
||||
Decode (x, block, 64);
|
||||
|
||||
/* Round 1 */
|
||||
FF (a, b, c, d, x[ 0], S11, 0xd76aa478); /* 1 */
|
||||
FF (d, a, b, c, x[ 1], S12, 0xe8c7b756); /* 2 */
|
||||
FF (c, d, a, b, x[ 2], S13, 0x242070db); /* 3 */
|
||||
FF (b, c, d, a, x[ 3], S14, 0xc1bdceee); /* 4 */
|
||||
FF (a, b, c, d, x[ 4], S11, 0xf57c0faf); /* 5 */
|
||||
FF (d, a, b, c, x[ 5], S12, 0x4787c62a); /* 6 */
|
||||
FF (c, d, a, b, x[ 6], S13, 0xa8304613); /* 7 */
|
||||
FF (b, c, d, a, x[ 7], S14, 0xfd469501); /* 8 */
|
||||
FF (a, b, c, d, x[ 8], S11, 0x698098d8); /* 9 */
|
||||
FF (d, a, b, c, x[ 9], S12, 0x8b44f7af); /* 10 */
|
||||
FF (c, d, a, b, x[10], S13, 0xffff5bb1); /* 11 */
|
||||
FF (b, c, d, a, x[11], S14, 0x895cd7be); /* 12 */
|
||||
FF (a, b, c, d, x[12], S11, 0x6b901122); /* 13 */
|
||||
FF (d, a, b, c, x[13], S12, 0xfd987193); /* 14 */
|
||||
FF (c, d, a, b, x[14], S13, 0xa679438e); /* 15 */
|
||||
FF (b, c, d, a, x[15], S14, 0x49b40821); /* 16 */
|
||||
|
||||
/* Round 2 */
|
||||
GG (a, b, c, d, x[ 1], S21, 0xf61e2562); /* 17 */
|
||||
GG (d, a, b, c, x[ 6], S22, 0xc040b340); /* 18 */
|
||||
GG (c, d, a, b, x[11], S23, 0x265e5a51); /* 19 */
|
||||
GG (b, c, d, a, x[ 0], S24, 0xe9b6c7aa); /* 20 */
|
||||
GG (a, b, c, d, x[ 5], S21, 0xd62f105d); /* 21 */
|
||||
GG (d, a, b, c, x[10], S22, 0x2441453); /* 22 */
|
||||
GG (c, d, a, b, x[15], S23, 0xd8a1e681); /* 23 */
|
||||
GG (b, c, d, a, x[ 4], S24, 0xe7d3fbc8); /* 24 */
|
||||
GG (a, b, c, d, x[ 9], S21, 0x21e1cde6); /* 25 */
|
||||
GG (d, a, b, c, x[14], S22, 0xc33707d6); /* 26 */
|
||||
GG (c, d, a, b, x[ 3], S23, 0xf4d50d87); /* 27 */
|
||||
GG (b, c, d, a, x[ 8], S24, 0x455a14ed); /* 28 */
|
||||
GG (a, b, c, d, x[13], S21, 0xa9e3e905); /* 29 */
|
||||
GG (d, a, b, c, x[ 2], S22, 0xfcefa3f8); /* 30 */
|
||||
GG (c, d, a, b, x[ 7], S23, 0x676f02d9); /* 31 */
|
||||
GG (b, c, d, a, x[12], S24, 0x8d2a4c8a); /* 32 */
|
||||
|
||||
/* Round 3 */
|
||||
HH (a, b, c, d, x[ 5], S31, 0xfffa3942); /* 33 */
|
||||
HH (d, a, b, c, x[ 8], S32, 0x8771f681); /* 34 */
|
||||
HH (c, d, a, b, x[11], S33, 0x6d9d6122); /* 35 */
|
||||
HH (b, c, d, a, x[14], S34, 0xfde5380c); /* 36 */
|
||||
HH (a, b, c, d, x[ 1], S31, 0xa4beea44); /* 37 */
|
||||
HH (d, a, b, c, x[ 4], S32, 0x4bdecfa9); /* 38 */
|
||||
HH (c, d, a, b, x[ 7], S33, 0xf6bb4b60); /* 39 */
|
||||
HH (b, c, d, a, x[10], S34, 0xbebfbc70); /* 40 */
|
||||
HH (a, b, c, d, x[13], S31, 0x289b7ec6); /* 41 */
|
||||
HH (d, a, b, c, x[ 0], S32, 0xeaa127fa); /* 42 */
|
||||
HH (c, d, a, b, x[ 3], S33, 0xd4ef3085); /* 43 */
|
||||
HH (b, c, d, a, x[ 6], S34, 0x4881d05); /* 44 */
|
||||
HH (a, b, c, d, x[ 9], S31, 0xd9d4d039); /* 45 */
|
||||
HH (d, a, b, c, x[12], S32, 0xe6db99e5); /* 46 */
|
||||
HH (c, d, a, b, x[15], S33, 0x1fa27cf8); /* 47 */
|
||||
HH (b, c, d, a, x[ 2], S34, 0xc4ac5665); /* 48 */
|
||||
|
||||
/* Round 4 */
|
||||
II (a, b, c, d, x[ 0], S41, 0xf4292244); /* 49 */
|
||||
II (d, a, b, c, x[ 7], S42, 0x432aff97); /* 50 */
|
||||
II (c, d, a, b, x[14], S43, 0xab9423a7); /* 51 */
|
||||
II (b, c, d, a, x[ 5], S44, 0xfc93a039); /* 52 */
|
||||
II (a, b, c, d, x[12], S41, 0x655b59c3); /* 53 */
|
||||
II (d, a, b, c, x[ 3], S42, 0x8f0ccc92); /* 54 */
|
||||
II (c, d, a, b, x[10], S43, 0xffeff47d); /* 55 */
|
||||
II (b, c, d, a, x[ 1], S44, 0x85845dd1); /* 56 */
|
||||
II (a, b, c, d, x[ 8], S41, 0x6fa87e4f); /* 57 */
|
||||
II (d, a, b, c, x[15], S42, 0xfe2ce6e0); /* 58 */
|
||||
II (c, d, a, b, x[ 6], S43, 0xa3014314); /* 59 */
|
||||
II (b, c, d, a, x[13], S44, 0x4e0811a1); /* 60 */
|
||||
II (a, b, c, d, x[ 4], S41, 0xf7537e82); /* 61 */
|
||||
II (d, a, b, c, x[11], S42, 0xbd3af235); /* 62 */
|
||||
II (c, d, a, b, x[ 2], S43, 0x2ad7d2bb); /* 63 */
|
||||
II (b, c, d, a, x[ 9], S44, 0xeb86d391); /* 64 */
|
||||
|
||||
state[0] += a;
|
||||
state[1] += b;
|
||||
state[2] += c;
|
||||
state[3] += d;
|
||||
|
||||
// Zeroize sensitive information.
|
||||
memset((POINTER)x, 0, sizeof (x));
|
||||
}
|
||||
|
||||
// Encodes input (UINT4) into output (unsigned char). Assumes len is
|
||||
// a multiple of 4.
|
||||
static void Encode( unsigned char *output, UINT4 *input, unsigned int len )
|
||||
{
|
||||
unsigned int i, j;
|
||||
|
||||
for (i = 0, j = 0; j < len; i++, j += 4) {
|
||||
output[j] = (unsigned char)(input[i] & 0xff);
|
||||
output[j+1] = (unsigned char)((input[i] >> 8) & 0xff);
|
||||
output[j+2] = (unsigned char)((input[i] >> 16) & 0xff);
|
||||
output[j+3] = (unsigned char)((input[i] >> 24) & 0xff);
|
||||
}
|
||||
}
|
||||
|
||||
// Decodes input (unsigned char) into output (UINT4). Assumes len is
|
||||
// a multiple of 4.
|
||||
static void Decode( UINT4 *output, unsigned char *input, unsigned int len )
|
||||
{
|
||||
unsigned int i, j;
|
||||
|
||||
for (i = 0, j = 0; j < len; i++, j += 4)
|
||||
output[i] = ((UINT4)input[j]) | (((UINT4)input[j+1]) << 8) |
|
||||
(((UINT4)input[j+2]) << 16) | (((UINT4)input[j+3]) << 24);
|
||||
}
|
||||
#pragma endregion
|
||||
|
||||
|
||||
public:
|
||||
// MAIN FUNCTIONS
|
||||
MD5()
|
||||
{
|
||||
Init() ;
|
||||
}
|
||||
|
||||
// MD5 initialization. Begins an MD5 operation, writing a new context.
|
||||
void Init()
|
||||
{
|
||||
context.count[0] = context.count[1] = 0;
|
||||
|
||||
// Load magic initialization constants.
|
||||
context.state[0] = 0x67452301;
|
||||
context.state[1] = 0xefcdab89;
|
||||
context.state[2] = 0x98badcfe;
|
||||
context.state[3] = 0x10325476;
|
||||
}
|
||||
|
||||
// MD5 block update operation. Continues an MD5 message-digest
|
||||
// operation, processing another message block, and updating the
|
||||
// context.
|
||||
void Update(
|
||||
unsigned char *input, // input block
|
||||
unsigned int inputLen ) // length of input block
|
||||
{
|
||||
unsigned int i, index, partLen;
|
||||
|
||||
// Compute number of bytes mod 64
|
||||
index = (unsigned int)((context.count[0] >> 3) & 0x3F);
|
||||
|
||||
// Update number of bits
|
||||
if ((context.count[0] += ((UINT4)inputLen << 3))
|
||||
< ((UINT4)inputLen << 3))
|
||||
context.count[1]++;
|
||||
context.count[1] += ((UINT4)inputLen >> 29);
|
||||
|
||||
partLen = 64 - index;
|
||||
|
||||
// Transform as many times as possible.
|
||||
if (inputLen >= partLen) {
|
||||
memcpy((POINTER)&context.buffer[index], (POINTER)input, partLen);
|
||||
MD5Transform (context.state, context.buffer);
|
||||
|
||||
for (i = partLen; i + 63 < inputLen; i += 64)
|
||||
MD5Transform (context.state, &input[i]);
|
||||
|
||||
index = 0;
|
||||
}
|
||||
else
|
||||
i = 0;
|
||||
|
||||
/* Buffer remaining input */
|
||||
memcpy((POINTER)&context.buffer[index], (POINTER)&input[i], inputLen-i);
|
||||
}
|
||||
|
||||
// MD5 finalization. Ends an MD5 message-digest operation, writing the
|
||||
// the message digest and zeroizing the context.
|
||||
// Writes to digestRaw
|
||||
void Final()
|
||||
{
|
||||
unsigned char bits[8];
|
||||
unsigned int index, padLen;
|
||||
|
||||
// Save number of bits
|
||||
Encode( bits, context.count, 8 );
|
||||
|
||||
// Pad out to 56 mod 64.
|
||||
index = (unsigned int)((context.count[0] >> 3) & 0x3f);
|
||||
padLen = (index < 56) ? (56 - index) : (120 - index);
|
||||
Update( PADDING, padLen );
|
||||
|
||||
// Append length (before padding)
|
||||
Update( bits, 8 );
|
||||
|
||||
// Store state in digest
|
||||
Encode( digestRaw, context.state, 16);
|
||||
|
||||
// Zeroize sensitive information.
|
||||
memset((POINTER)&context, 0, sizeof (context));
|
||||
|
||||
writeToString() ;
|
||||
}
|
||||
|
||||
/// Buffer must be 32+1 (nul) = 33 chars long at least
|
||||
void writeToString()
|
||||
{
|
||||
int pos ;
|
||||
|
||||
for( pos = 0 ; pos < 16 ; pos++ )
|
||||
sprintf( digestChars+(pos*2), "%02x", digestRaw[pos] ) ;
|
||||
}
|
||||
|
||||
|
||||
public:
|
||||
// an MD5 digest is a 16-byte number (32 hex digits)
|
||||
BYTE digestRaw[ 16 ] ;
|
||||
|
||||
// This version of the digest is actually
|
||||
// a "printf'd" version of the digest.
|
||||
char digestChars[ 33 ] ;
|
||||
|
||||
/// Load a file from disk and digest it
|
||||
// Digests a file and returns the result.
|
||||
const char* digestFile( const char *filename )
|
||||
{
|
||||
if (NULL == filename || strcmp(filename, "") == 0)
|
||||
return NULL;
|
||||
|
||||
Init() ;
|
||||
|
||||
FILE *file;
|
||||
|
||||
int len;
|
||||
unsigned char buffer[1024] ;
|
||||
|
||||
if((file = fopen (filename, "rb")) == NULL)
|
||||
{
|
||||
return NULL;
|
||||
}
|
||||
else
|
||||
{
|
||||
while( len = fread( buffer, 1, 1024, file ) )
|
||||
Update( buffer, len ) ;
|
||||
Final();
|
||||
|
||||
fclose( file );
|
||||
}
|
||||
|
||||
return digestChars ;
|
||||
}
|
||||
|
||||
/// Digests a byte-array already in memory
|
||||
const char* digestMemory( BYTE *memchunk, int len )
|
||||
{
|
||||
if (NULL == memchunk)
|
||||
return NULL;
|
||||
|
||||
Init() ;
|
||||
Update( memchunk, len ) ;
|
||||
Final() ;
|
||||
|
||||
return digestChars ;
|
||||
}
|
||||
|
||||
// Digests a string and prints the result.
|
||||
const char* digestString(const char *string )
|
||||
{
|
||||
if (string == NULL)
|
||||
return NULL;
|
||||
|
||||
Init() ;
|
||||
Update( (unsigned char*)string, strlen(string) ) ;
|
||||
Final() ;
|
||||
|
||||
return digestChars ;
|
||||
}
|
||||
};
|
||||
|
||||
inline bool md5String(const char* str, std::string& res)
|
||||
{
|
||||
if (NULL == str)
|
||||
{
|
||||
res = "";
|
||||
return false;
|
||||
}
|
||||
|
||||
MD5 md5;
|
||||
const char *pRes = md5.digestString(str);
|
||||
if (NULL == pRes)
|
||||
{
|
||||
res = "";
|
||||
return false;
|
||||
}
|
||||
|
||||
res = pRes;
|
||||
return true;
|
||||
}
|
||||
|
||||
inline bool md5File(const char* filepath, std::string& res)
|
||||
{
|
||||
if (NULL == filepath || strcmp(filepath, "") == 0)
|
||||
{
|
||||
res = "";
|
||||
return false;
|
||||
}
|
||||
|
||||
MD5 md5;
|
||||
const char *pRes = md5.digestFile(filepath);
|
||||
|
||||
if (NULL == pRes)
|
||||
{
|
||||
res = "";
|
||||
return false;
|
||||
}
|
||||
|
||||
res = pRes;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
@ -10,6 +10,7 @@
|
||||
#include <cassert>
|
||||
#include "Limonp/logger.hpp"
|
||||
#include "Trie.hpp"
|
||||
#include "TrieManager.hpp"
|
||||
#include "ISegment.hpp"
|
||||
#include "SegmentBase.hpp"
|
||||
|
||||
@ -32,7 +33,7 @@ namespace CppJieba
|
||||
class MPSegment: public SegmentBase
|
||||
{
|
||||
private:
|
||||
Trie _trie;
|
||||
Trie* _trie;
|
||||
private:
|
||||
const string _dictPath;
|
||||
|
||||
@ -47,18 +48,12 @@ namespace CppJieba
|
||||
LogError("already inited before now.");
|
||||
return false;
|
||||
}
|
||||
if(!_trie.init())
|
||||
_trie = TrieManager::getInstance().getTrie(_dictPath.c_str());
|
||||
if (_trie == NULL)
|
||||
{
|
||||
LogError("_trie.init failed.");
|
||||
LogError("get a NULL pointor form getTrie(\"%s\").", _dictPath.c_str());
|
||||
return false;
|
||||
}
|
||||
LogInfo("_trie.loadDict(%s) start...", _dictPath.c_str());
|
||||
if(!_trie.loadDict(_dictPath.c_str()))
|
||||
{
|
||||
LogError("_trie.loadDict faield.");
|
||||
return false;
|
||||
}
|
||||
LogInfo("_trie.loadDict end.");
|
||||
return _setInitFlag(true);
|
||||
}
|
||||
virtual bool dispose()
|
||||
@ -67,18 +62,12 @@ namespace CppJieba
|
||||
{
|
||||
return true;
|
||||
}
|
||||
_trie.dispose();
|
||||
_setInitFlag(false);
|
||||
return true;
|
||||
}
|
||||
public:
|
||||
virtual bool cut(Unicode::const_iterator begin, Unicode::const_iterator end, vector<string>& res)const
|
||||
{
|
||||
//if(!_getInitFlag())
|
||||
//{
|
||||
// LogError("not inited.");
|
||||
// return false;
|
||||
//}
|
||||
assert(_getInitFlag());
|
||||
|
||||
vector<TrieNodeInfo> segWordInfos;
|
||||
@ -145,7 +134,7 @@ namespace CppJieba
|
||||
{
|
||||
SegmentChar schar(*it);
|
||||
uint i = it - begin;
|
||||
_trie.find(it, end, i, schar.dag);
|
||||
_trie->find(it, end, i, schar.dag);
|
||||
//DagType::iterator dagIter;
|
||||
if(schar.dag.end() == schar.dag.find(i))
|
||||
{
|
||||
@ -183,7 +172,7 @@ namespace CppJieba
|
||||
}
|
||||
else
|
||||
{
|
||||
val += _trie.getMinLogFreq();
|
||||
val += _trie->getMinLogFreq();
|
||||
}
|
||||
if(val > segContext[i].weight)
|
||||
{
|
||||
@ -211,7 +200,7 @@ namespace CppJieba
|
||||
TrieNodeInfo nodeInfo;
|
||||
nodeInfo.word.push_back(segContext[i].uniCh);
|
||||
nodeInfo.freq = 0;
|
||||
nodeInfo.logFreq = _trie.getMinLogFreq();
|
||||
nodeInfo.logFreq = _trie->getMinLogFreq();
|
||||
res.push_back(nodeInfo);
|
||||
i++;
|
||||
}
|
||||
|
||||
@ -55,70 +55,93 @@ namespace CppJieba
|
||||
public:
|
||||
using SegmentBase::cut;
|
||||
public:
|
||||
virtual bool cut(Unicode::const_iterator begin, Unicode::const_iterator end, vector<string>& res)const
|
||||
virtual bool cut(Unicode::const_iterator begin, Unicode::const_iterator end, vector<Unicode>& res) const
|
||||
{
|
||||
//if(!_getInitFlag())
|
||||
//{
|
||||
// LogError("not inited.");
|
||||
// return false;
|
||||
//}
|
||||
assert(_getInitFlag());
|
||||
if(begin == end)
|
||||
if(begin >= end)
|
||||
{
|
||||
LogError("begin >= end");
|
||||
return false;
|
||||
}
|
||||
|
||||
vector<TrieNodeInfo> infos;
|
||||
if(!_mpSeg.cut(begin, end, infos))
|
||||
{
|
||||
LogError("mpSeg cutDAG failed.");
|
||||
return false;
|
||||
}
|
||||
Unicode unico;
|
||||
|
||||
vector<Unicode> hmmRes;
|
||||
Unicode piece;
|
||||
for (uint i = 0, j = 0; i < infos.size(); i++)
|
||||
{
|
||||
//if mp get a word, it's ok, put it into result
|
||||
if (1 != infos[i].word.size())
|
||||
{
|
||||
res.push_back(infos[i].word);
|
||||
continue;
|
||||
}
|
||||
|
||||
// if mp get a single one, collect it in sequence
|
||||
j = i;
|
||||
while (j < infos.size() && infos[j].word.size() == 1)
|
||||
{
|
||||
piece.push_back(infos[j].word[0]);
|
||||
j++;
|
||||
}
|
||||
|
||||
// cut the sequence with hmm
|
||||
if (!_hmmSeg.cut(piece.begin(), piece.end(), hmmRes))
|
||||
{
|
||||
LogError("_hmmSeg cut failed.");
|
||||
return false;
|
||||
}
|
||||
|
||||
//put hmm result to return
|
||||
for (uint k = 0; k < hmmRes.size(); k++)
|
||||
{
|
||||
res.push_back(hmmRes[k]);
|
||||
}
|
||||
|
||||
//clear tmp vars
|
||||
piece.clear();
|
||||
hmmRes.clear();
|
||||
|
||||
//let i jump over this piece
|
||||
i = j - 1;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
virtual bool cut(Unicode::const_iterator begin, Unicode::const_iterator end, vector<string>& res)const
|
||||
{
|
||||
assert(_getInitFlag());
|
||||
if(begin >= end)
|
||||
{
|
||||
LogError("begin >= end");
|
||||
return false;
|
||||
}
|
||||
|
||||
vector<Unicode> uRes;
|
||||
if (!cut(begin, end, uRes))
|
||||
{
|
||||
LogError("get unicode cut result error.");
|
||||
return false;
|
||||
}
|
||||
|
||||
string tmp;
|
||||
for(uint i= 0; i < infos.size(); i++)
|
||||
for (vector<Unicode>::const_iterator uItr = uRes.begin(); uItr != uRes.end(); uItr++)
|
||||
{
|
||||
TransCode::encode(infos[i].word,tmp);
|
||||
if(1 == infos[i].word.size())
|
||||
if (TransCode::encode(*uItr, tmp))
|
||||
{
|
||||
unico.push_back(infos[i].word[0]);
|
||||
res.push_back(tmp);
|
||||
}
|
||||
else
|
||||
{
|
||||
if(!unico.empty())
|
||||
{
|
||||
hmmRes.clear();
|
||||
if(!_hmmSeg.cut(unico.begin(), unico.end(), hmmRes))
|
||||
{
|
||||
LogError("_hmmSeg cut failed.");
|
||||
return false;
|
||||
}
|
||||
for(uint j = 0; j < hmmRes.size(); j++)
|
||||
{
|
||||
TransCode::encode(hmmRes[j], tmp);
|
||||
res.push_back(tmp);
|
||||
LogError("encode failed.");
|
||||
}
|
||||
}
|
||||
unico.clear();
|
||||
TransCode::encode(infos[i].word, tmp);
|
||||
res.push_back(tmp);
|
||||
}
|
||||
}
|
||||
if(!unico.empty())
|
||||
{
|
||||
hmmRes.clear();
|
||||
if(!_hmmSeg.cut(unico.begin(), unico.end(), hmmRes))
|
||||
{
|
||||
LogError("_hmmSeg cut failed.");
|
||||
return false;
|
||||
}
|
||||
for(uint j = 0; j < hmmRes.size(); j++)
|
||||
{
|
||||
TransCode::encode(hmmRes[j], tmp);
|
||||
res.push_back(tmp);
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
@ -8,34 +8,33 @@
|
||||
#include "Trie.hpp"
|
||||
#include "ISegment.hpp"
|
||||
#include "SegmentBase.hpp"
|
||||
#include "HMMSegment.hpp"
|
||||
#include "FullSegment.hpp"
|
||||
#include "MixSegment.hpp"
|
||||
#include "TransCode.hpp"
|
||||
#include "TrieManager.hpp"
|
||||
|
||||
namespace CppJieba
|
||||
{
|
||||
class QuerySegment: public SegmentBase
|
||||
{
|
||||
private:
|
||||
HMMSegment _hmmSeg;
|
||||
MixSegment _mixSeg;
|
||||
FullSegment _fullSeg;
|
||||
int _maxWordLen;
|
||||
|
||||
public:
|
||||
QuerySegment(const char* fullSegDict, const char* hmmSegDict, int maxWordLen): _hmmSeg(hmmSegDict), _fullSeg(fullSegDict), _maxWordLen(maxWordLen){};
|
||||
QuerySegment(const char* dict, const char* model, int maxWordLen): _mixSeg(dict, model), _fullSeg(dict), _maxWordLen(maxWordLen){};
|
||||
virtual ~QuerySegment(){dispose();};
|
||||
public:
|
||||
bool init()
|
||||
{
|
||||
#ifndef NO_CODING_LOG
|
||||
if (_getInitFlag())
|
||||
{
|
||||
LogError("inited.");
|
||||
}
|
||||
#endif
|
||||
if (!_hmmSeg.init())
|
||||
if (!_mixSeg.init())
|
||||
{
|
||||
LogError("_hmmSeg init");
|
||||
LogError("_mixSeg init");
|
||||
return false;
|
||||
}
|
||||
if (!_fullSeg.init())
|
||||
@ -47,14 +46,12 @@ namespace CppJieba
|
||||
}
|
||||
bool dispose()
|
||||
{
|
||||
#ifndef NO_CODING_LOG
|
||||
if(!_getInitFlag())
|
||||
{
|
||||
return true;
|
||||
}
|
||||
#endif
|
||||
_fullSeg.dispose();
|
||||
_hmmSeg.dispose();
|
||||
_mixSeg.dispose();
|
||||
_setInitFlag(false);
|
||||
return true;
|
||||
}
|
||||
@ -66,34 +63,28 @@ namespace CppJieba
|
||||
bool cut(Unicode::const_iterator begin, Unicode::const_iterator end, vector<Unicode>& res) const
|
||||
{
|
||||
assert(_getInitFlag());
|
||||
#ifndef NO_CODING_LOG
|
||||
//if (!_getInitFlag())
|
||||
//{
|
||||
// LogError("not inited.");
|
||||
// return false;
|
||||
//}
|
||||
if (begin >= end)
|
||||
{
|
||||
LogError("begin >= end");
|
||||
return false;
|
||||
}
|
||||
#endif
|
||||
//use hmm cut first
|
||||
vector<Unicode> hmmRes;
|
||||
if (!_hmmSeg.cut(begin, end, hmmRes))
|
||||
|
||||
//use mix cut first
|
||||
vector<Unicode> mixRes;
|
||||
if (!_mixSeg.cut(begin, end, mixRes))
|
||||
{
|
||||
LogError("_hmmSeg cut failed.");
|
||||
LogError("_mixSeg cut failed.");
|
||||
return false;
|
||||
}
|
||||
|
||||
vector<Unicode> fullRes;
|
||||
for (vector<Unicode>::const_iterator hmmResItr = hmmRes.begin(); hmmResItr != hmmRes.end(); hmmResItr++)
|
||||
for (vector<Unicode>::const_iterator mixResItr = mixRes.begin(); mixResItr != mixRes.end(); mixResItr++)
|
||||
{
|
||||
|
||||
// if it's too long, cut with _fullSeg, put fullRes in res
|
||||
if (hmmResItr->size() > _maxWordLen)
|
||||
if (mixResItr->size() > _maxWordLen)
|
||||
{
|
||||
if (_fullSeg.cut(hmmResItr->begin(), hmmResItr->end(), fullRes))
|
||||
if (_fullSeg.cut(mixResItr->begin(), mixResItr->end(), fullRes))
|
||||
{
|
||||
for (vector<Unicode>::const_iterator fullResItr = fullRes.begin(); fullResItr != fullRes.end(); fullResItr++)
|
||||
{
|
||||
@ -101,9 +92,9 @@ namespace CppJieba
|
||||
}
|
||||
}
|
||||
}
|
||||
else // just use the hmm result
|
||||
else // just use the mix result
|
||||
{
|
||||
res.push_back(*hmmResItr);
|
||||
res.push_back(*mixResItr);
|
||||
}
|
||||
}
|
||||
|
||||
@ -113,18 +104,13 @@ namespace CppJieba
|
||||
|
||||
bool cut(Unicode::const_iterator begin, Unicode::const_iterator end, vector<string>& res) const
|
||||
{
|
||||
#ifndef NO_CODING_LOG
|
||||
if (!_getInitFlag())
|
||||
assert(_getInitFlag());
|
||||
if (begin >= end)
|
||||
{
|
||||
LogError("not inited.");
|
||||
LogError("begin >= end");
|
||||
return false;
|
||||
}
|
||||
if (begin > end)
|
||||
{
|
||||
LogError("begin > end");
|
||||
return false;
|
||||
}
|
||||
#endif
|
||||
|
||||
vector<Unicode> uRes;
|
||||
if (!cut(begin, end, uRes))
|
||||
{
|
||||
|
||||
77
src/TrieManager.hpp
Normal file
77
src/TrieManager.hpp
Normal file
@ -0,0 +1,77 @@
|
||||
#ifndef CPPJIEBA_TRIEMANAGER_H
|
||||
#define CPPJIEBA_TRIEMANAGER_H
|
||||
|
||||
#include "Trie.hpp"
|
||||
#include "Limonp/md5.hpp"
|
||||
#include "Limonp/logger.hpp"
|
||||
|
||||
namespace CppJieba
|
||||
{
|
||||
using namespace Limonp;
|
||||
class TrieManager
|
||||
{
|
||||
private:
|
||||
unordered_map<std::string, Trie*> _tries;
|
||||
TrieManager(){};
|
||||
TrieManager(TrieManager& tm){};
|
||||
public:
|
||||
Trie* getTrie(const char* dictpath)
|
||||
{
|
||||
string md5;
|
||||
if (!md5File(dictpath, md5))
|
||||
{
|
||||
LogError("error when getting md5 for file '%s'", dictpath);
|
||||
return NULL;
|
||||
}
|
||||
else
|
||||
{
|
||||
LogInfo("md5 for file '%s': %s", dictpath, md5.c_str());
|
||||
if (_tries.find(md5) == _tries.end())
|
||||
{
|
||||
LogInfo("create a new trie for md5: '%s'", md5.c_str());
|
||||
Trie* trie = NULL;
|
||||
try
|
||||
{
|
||||
trie = new Trie();
|
||||
}
|
||||
catch (const bad_alloc& e)
|
||||
{
|
||||
LogError("error when new a trie for file '%s'", dictpath);
|
||||
return NULL;
|
||||
}
|
||||
if (NULL == trie)
|
||||
return NULL;
|
||||
|
||||
if (!trie->init())
|
||||
{
|
||||
LogError("trie init error for file '%s'", dictpath);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
LogInfo("trie->loadDict(%s) start...", dictpath);
|
||||
if (!trie->loadDict(dictpath))
|
||||
{
|
||||
LogError("trie->loadDict(%s) failed...", dictpath);
|
||||
return NULL;
|
||||
}
|
||||
LogInfo("trie->loadDict end...");
|
||||
|
||||
_tries[md5.c_str()] = trie;
|
||||
return trie;
|
||||
}
|
||||
else
|
||||
{
|
||||
LogInfo("find a exits trie for md5: '%s'", md5.c_str());
|
||||
return _tries[md5.c_str()];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static TrieManager& getInstance()
|
||||
{
|
||||
static TrieManager _this;
|
||||
return _this;
|
||||
}
|
||||
};
|
||||
}
|
||||
#endif
|
||||
@ -43,12 +43,13 @@ int main(int argc, char ** argv)
|
||||
<<"\t--algorithm\tSupported methods are [cutDAG, cutHMM, cutFull, cutQuery, cutMix] for now. \n\t\t\tIf not specified, the default is cutMix\n"
|
||||
<<"\t--dictpath\tsee example\n"
|
||||
<<"\t--modelpath\tsee example\n"
|
||||
<<"\t--maxlen\tspecify the granularity of cut used in cutQuery, If not specified, the default is 3\n"
|
||||
<<"\t--maxlen\tspecify the granularity of cut used in cutQuery. \n\t\t\tIf not specified, the default is 3\n"
|
||||
<<"example:\n"
|
||||
<<"\t"<<argv[0]<<" ../test/testlines.utf8 --dictpath ../dicts/jieba.dict.utf8 --algorithm cutDAG\n"
|
||||
<<"\t"<<argv[0]<<" ../test/testlines.utf8 --modelpath ../dicts/hmm_model.utf8 --algorithm cutHMM\n"
|
||||
<<"\t"<<argv[0]<<" ../test/testlines.utf8 --dictpath ../dicts/jieba.dict.utf8 --modelpath ../dicts/hmm_model.utf8 --algorithm cutMix\n"
|
||||
<<"\t"<<argv[0]<<" ../test/testlines.utf8 --dictpath ../dicts/jieba.dict.utf8 --modelpath ../dicts/hmm_model.utf8 --algorithm cutQuery --maxlen 3\n"
|
||||
<<"\t"<<argv[0]<<" ../test/testdata/testlines.utf8 --dictpath ../dicts/jieba.dict.utf8 --algorithm cutDAG\n"
|
||||
<<"\t"<<argv[0]<<" ../test/testdata/testlines.utf8 --dictpath ../dicts/jieba.dict.utf8 --algorithm cutFull\n"
|
||||
<<"\t"<<argv[0]<<" ../test/testdata/testlines.utf8 --modelpath ../dicts/hmm_model.utf8 --algorithm cutHMM\n"
|
||||
<<"\t"<<argv[0]<<" ../test/testdata/testlines.utf8 --dictpath ../dicts/jieba.dict.utf8 --modelpath ../dicts/hmm_model.utf8 --algorithm cutMix\n"
|
||||
<<"\t"<<argv[0]<<" ../test/testdata/testlines.utf8 --dictpath ../dicts/jieba.dict.utf8 --modelpath ../dicts/hmm_model.utf8 --algorithm cutQuery --maxlen 3\n"
|
||||
<<endl;
|
||||
|
||||
return EXIT_FAILURE;
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user