third_codes/cppjieba
1
0
Fork 0
mirror of https://github.com/yanyiwu/cppjieba.git synced 2025-07-18 00:00:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

update husky and limonp

Browse Source
This commit is contained in:
wyy 2013-12-23 23:58:54 -08:00
parent 24a15cd128
commit 3eb0470c2f
13 changed files with 127 additions and 422 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/CMakeLists.txt
View File

@ -1,6 +1,8 @@
SET(EXECUTABLE_OUTPUT_PATH ${PROJECT_BINARY_DIR}/bin) SET(EXECUTABLE_OUTPUT_PATH ${PROJECT_BINARY_DIR}/bin)
SET(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/lib) SET(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/lib)
INCLUDE_DIRECTORIES(${PROJECT_SOURCE_DIR}/src)
ADD_EXECUTABLE(cjsegment segment.cpp) ADD_EXECUTABLE(cjsegment segment.cpp)
ADD_EXECUTABLE(cjserver server.cpp) ADD_EXECUTABLE(cjserver server.cpp)
TARGET_LINK_LIBRARIES(cjserver pthread) TARGET_LINK_LIBRARIES(cjserver pthread)

10
src/HMMSegment.hpp
View File

@ -215,7 +215,7 @@ namespace CppJieba
{ {
return false; return false;
} }
splitStr(line, tmp, " "); split(line, tmp, " ");
if(tmp.size() != STATUS_SUM) if(tmp.size() != STATUS_SUM)
{ {
LogError("start_p illegal"); LogError("start_p illegal");
@ -234,7 +234,7 @@ namespace CppJieba
{ {
return false; return false;
} }
splitStr(line, tmp, " "); split(line, tmp, " ");
if(tmp.size() != STATUS_SUM) if(tmp.size() != STATUS_SUM)
{ {
LogError("trans_p illegal"); LogError("trans_p illegal");
@ -284,7 +284,7 @@ namespace CppJieba
{ {
continue; continue;
} }
if(strStartsWith(line, "#")) if(startsWith(line, "#"))
{ {
continue; continue;
} }
@ -300,10 +300,10 @@ namespace CppJieba
} }
vector<string> tmp, tmp2; vector<string> tmp, tmp2;
uint16_t unico = 0; uint16_t unico = 0;
splitStr(line, tmp, ","); split(line, tmp, ",");
for(uint i = 0; i < tmp.size(); i++) for(uint i = 0; i < tmp.size(); i++)
{ {
splitStr(tmp[i], tmp2, ":"); split(tmp[i], tmp2, ":");
if(2 != tmp2.size()) if(2 != tmp2.size())
{ {
LogError("_emitProb illegal."); LogError("_emitProb illegal.");

4
src/Husky/HttpReqInfo.hpp
View File

@ -3,7 +3,7 @@
#include <iostream> #include <iostream>
#include <string> #include <string>
#include "../Limonp/logger.hpp" #include "Limonp/logger.hpp"
namespace Husky namespace Husky
{ {
@ -88,7 +88,7 @@ namespace Husky
} }
string firstline(headerStr, lpos, rpos - lpos); string firstline(headerStr, lpos, rpos - lpos);
trim(firstline); trim(firstline);
if(!splitStr(firstline, buf, " ") || 3 != buf.size()) if(!split(firstline, buf, " ") || 3 != buf.size())
{ {
LogFatal("parse header first line failed."); LogFatal("parse header first line failed.");
return false; return false;

302
src/Husky/ServerFrame.hpp
View File

@ -1,302 +0,0 @@
#ifndef HUSKY_SERVERFRAME_H
#define HUSKY_SERVERFRAME_H
#include <stdio.h>
#include <string.h>
#include <cassert>
#include <sys/socket.h>
#include <sys/types.h>
#include <arpa/inet.h>
#include <stdlib.h>
#include <pthread.h>
#include <string.h>
#include <errno.h>
#include <unistd.h>
#include <vector>
#include "ThreadManager.hpp"
#include "HttpReqInfo.hpp"
#define INVALID_SOCKET -1
#define SOCKET_ERROR -1
#define closesocket close
#define RECV_BUFFER 10240
#define LISEN_QUEUR_LEN 1024
namespace Husky
{
using namespace Limonp;
typedef int SOCKET;
const struct timeval SOCKET_TIMEOUT = {2, 0};
const char* const RESPONSE_FORMAT = "HTTP/1.1 200 OK\r\nConnection: close\r\nServer: FrameServer/1.0.0\r\nContent-Type: text/json; charset=%s\r\nContent-Length: %d\r\n\r\n";
const char* const RESPONSE_CHARSET_UTF8 = "UTF-8";
const char* const RESPONSE_CHARSET_GB2312 = "GB2312";
const char* const CLIENT_IP_K = "CLIENT_IP";
typedef unsigned short u_short;
typedef unsigned int u_int;
class IRequestHandler
{
public:
virtual ~IRequestHandler(){};
public:
virtual bool do_GET(const HttpReqInfo& httpReq, string& res) = 0;
};
struct SPara
{
SOCKET hSock;
IRequestHandler * pHandler;
pthread_mutex_t * ppmAccept;
bool * pShutdown;
};
class ServerFrame//: public IWorkHandler
{
private:
pthread_mutex_t m_pmAccept;
bool m_bShutdown;
public:
ServerFrame(unsigned nPort, unsigned nThreadCount, IRequestHandler* pHandler)
{
m_bShutdown = false;
m_nLsnPort = nPort;
m_nThreadCount = nThreadCount;
m_pHandler = pHandler;
assert(pHandler);
pthread_mutex_init(&m_pmAccept,NULL);
};
virtual ~ServerFrame(){pthread_mutex_destroy(&m_pmAccept);};
virtual bool init()
{
if (!BindToLocalHost(m_lsnSock,m_nLsnPort))
{
LogFatal("BindToLocalHost failed.");
return false;
}
LogInfo("init ok {port:%d, threadNum:%d}", m_nLsnPort, m_nThreadCount);
return true;
}
virtual bool dispose()
{
m_bShutdown=true;
if (SOCKET_ERROR==closesocket(m_lsnSock))
{
LogError("error [%s]", strerror(errno));
return false;
}
int sockfd;
struct sockaddr_in dest;
if ((sockfd = socket(AF_INET, SOCK_STREAM, 0)) < 0)
{
LogError("error [%s]", strerror(errno));
return false;
}
bzero(&dest, sizeof(dest));
dest.sin_family = AF_INET;
dest.sin_port = htons(m_nLsnPort);
if (inet_aton("127.0.0.1", (struct in_addr *) &dest.sin_addr.s_addr) == 0)
{
LogError("error [%s]", strerror(errno));
return false;
}
if (connect(sockfd, (struct sockaddr *) &dest, sizeof(dest)) < 0)
{
LogError("error [%s]", strerror(errno));
}
close(sockfd);
return true;
}
virtual bool run()
{
if(SOCKET_ERROR==listen(m_lsnSock,LISEN_QUEUR_LEN))
{
LogError("error [%s]", strerror(errno));
return false;
}
ThreadManager thrMngr;
int i;
SPara para;
para.hSock=m_lsnSock;
para.pHandler=m_pHandler;
para.ppmAccept=&m_pmAccept;
para.pShutdown=&m_bShutdown;
for (i=0;i<m_nThreadCount;i++)
{
if (0!=thrMngr.CreateThread(ServerThread, &para))
{
break;
}
}
LogDebug("expect thread count %d, real count %d",m_nThreadCount,i);
if(i==0)
{
LogError("error [%s]", strerror(errno));
return false;
}
LogInfo("server start to run.........");
if (thrMngr.WaitMultipleThread()!=0)
{
return false;
}
return true;
}
protected:
bool BindToLocalHost(SOCKET &sock,u_short nPort)
{
sock=socket(AF_INET,SOCK_STREAM,0);
if(INVALID_SOCKET==sock)
{
LogError("error [%s]", strerror(errno));
return false;
}
int nRet = 1;
if(SOCKET_ERROR==setsockopt(m_lsnSock, SOL_SOCKET, SO_REUSEADDR, (char*)&nRet, sizeof(nRet)))
{
LogError("error [%s]", strerror(errno));
}
struct sockaddr_in addrSock;
addrSock.sin_family=AF_INET;
addrSock.sin_port=htons(nPort);
addrSock.sin_addr.s_addr=htonl(INADDR_ANY);
int retval;
retval = ::bind(sock,(sockaddr*)&addrSock,sizeof(sockaddr));
if(SOCKET_ERROR==retval)
{
LogError("error [%s]", strerror(errno));
closesocket(sock);
return false;
}
return true;
}
static void * ServerThread(void * lpParameter )
{
SPara *pPara=(SPara*)lpParameter;
SOCKET hSockLsn=pPara->hSock;
IRequestHandler *pHandler=pPara->pHandler;
bool* pShutdown = pPara->pShutdown;
pthread_mutex_t* ppmAccept = pPara->ppmAccept;
int nRetCode;
linger lng;
char chRecvBuf[RECV_BUFFER];
SOCKET hClientSock;
string strHttpResp;
sockaddr_in clientaddr;
socklen_t nSize = sizeof(clientaddr);
while(!(*pShutdown))
{
HttpReqInfo httpReq;
pthread_mutex_lock(ppmAccept);
hClientSock=accept(hSockLsn,(sockaddr *)&clientaddr, &nSize);
pthread_mutex_unlock(ppmAccept);
if(hClientSock==SOCKET_ERROR)
{
if(!(*pShutdown))
LogError("error [%s]", strerror(errno));
continue;
}
httpReq[CLIENT_IP_K] = inet_ntoa(clientaddr.sin_addr);// inet_ntoa is not thread safety at some version
lng.l_linger=1;
lng.l_onoff=1;
if(SOCKET_ERROR==setsockopt(hClientSock,SOL_SOCKET,SO_LINGER,(char*)&lng,sizeof(lng)))
{
LogError("error [%s]", strerror(errno));
}
if(SOCKET_ERROR==setsockopt(hClientSock,SOL_SOCKET,SO_RCVTIMEO,(char*)&SOCKET_TIMEOUT,sizeof(SOCKET_TIMEOUT)))
{
LogError("error [%s]", strerror(errno));
}
if(SOCKET_ERROR==setsockopt(hClientSock,SOL_SOCKET,SO_SNDTIMEO,(char*)&SOCKET_TIMEOUT,sizeof(SOCKET_TIMEOUT)))
{
LogError("error [%s]", strerror(errno));
}
string strRec;
string strSnd;
memset(chRecvBuf,0,sizeof(chRecvBuf));
nRetCode = recv(hClientSock, chRecvBuf, RECV_BUFFER, 0);
strRec = chRecvBuf;
#ifdef HUKSY_DEBUG
LogDebug("request[%s]", strRec.c_str());
#endif
if(SOCKET_ERROR==nRetCode)
{
LogDebug("error [%s]", strerror(errno));
closesocket(hClientSock);
continue;
}
if(0==nRetCode)
{
LogDebug("connection has been gracefully closed");
closesocket(hClientSock);
continue;
}
httpReq.load(strRec);
pHandler->do_GET(httpReq, strSnd);
char chHttpHeader[2048];
sprintf(chHttpHeader, RESPONSE_FORMAT, RESPONSE_CHARSET_UTF8, int(strSnd.length()));
strHttpResp=chHttpHeader;
strHttpResp+=strSnd;
#ifdef HUKSY_DEBUG
LogDebug("response'body [%s]", strSnd.c_str());
#endif
if (SOCKET_ERROR==send(hClientSock,strHttpResp.c_str(),strHttpResp.length(),0))
{
LogError("error [%s]", strerror(errno));
}
closesocket(hClientSock);
}
return 0;
}
private:
u_short m_nLsnPort;
u_short m_nThreadCount;
SOCKET m_lsnSock;
IRequestHandler *m_pHandler;
//static bool m_bShutdown;
//static pthread_mutex_t m_pmAccept;
};
}
#endif

45
src/Husky/ThreadManager.hpp
View File

@ -5,8 +5,6 @@
#include <vector> #include <vector>
#include <map> #include <map>
#define INFINITE 0
namespace Husky namespace Husky
{ {
using namespace std; using namespace std;
@ -15,47 +13,43 @@ namespace Husky
{ {
private: private:
typedef int HANDLE; typedef int HANDLE;
typedef int DWORD;
typedef void *(* PThreadFunc)(void* param); typedef void *(* PThreadFunc)(void* param);
public: public:
ThreadManager(){;} ThreadManager(){;}
~ThreadManager(){} ~ThreadManager(){}
unsigned int HandleCount(){return m_vecHandle.size();} size_t HandleCount(){return _handles.size();}
void clear() void clear()
{ {
m_vecHandle.clear(); _handles.clear();
} }
HANDLE CreateThread( PThreadFunc pFunc,void *pPara) HANDLE CreateThread( PThreadFunc pFunc,void *pPara)
{ {
pthread_t pt; pthread_t pt;
int nErrorCode=pthread_create(&pt,NULL,pFunc,pPara); int nErrorCode = pthread_create(&pt,NULL,pFunc,pPara);
if(nErrorCode!=0) if(nErrorCode != 0)
return nErrorCode; return nErrorCode;
m_vecHandle.push_back(pt); //加入线程列表 为WaitForMultipleObjects准备 _handles.push_back(pt);
return nErrorCode; return nErrorCode;
} }
//hThread (thread handler) : 为0时为默认最后一个加入管理器的线程句柄 int Wait(HANDLE hThread = 0)
//dwMilliseconds等待时间 : 单位毫秒，默认值无穷时间
//return value : -1句柄无效，其他值 WaitForSingleObject函数的返回值
DWORD Wait(HANDLE hThread=0,DWORD dwMilliseconds=INFINITE )
{ {
if( hThread==0)//最后一个加入的线程 if( hThread == 0)//the last handle
{ {
if(!m_vecHandle.empty()) if(!_handles.empty())
{ {
return pthread_join(m_vecHandle.back(),NULL); return pthread_join(_handles.back(),NULL);
} }
else else
return -1; return -1;
} }
else else
{ {
if (find(m_vecHandle.begin(),m_vecHandle.end(),hThread)==m_vecHandle.end())//不存在此句柄 if (find(_handles.begin(),_handles.end(),hThread) == _handles.end())
{ {
return -1; return -1;
} }
@ -65,31 +59,26 @@ namespace Husky
} }
int WaitMultipleThread()
//等待所有线程执行完毕
//bWaitAll是否所有线程 : 默认值1等待所有线程,0有任何线程结束，此函数返回
//dwMilliseconds : 单位毫秒，默认值无穷时间
//return value : -1没有任何句柄，其他值 WaitForMultipleObjects函数的返回值
DWORD WaitMultipleThread( bool bWaitAll=1,DWORD dwMilliseconds=INFINITE)
{ {
if (m_vecHandle.empty()) if (_handles.empty())
return -1; return -1;
int nErrorcode; int nErrorcode;
for (uint i=0;i<m_vecHandle.size();++i) for (uint i = 0; i < _handles.size(); i++)
{ {
nErrorcode=pthread_join(m_vecHandle[i], NULL); nErrorcode = pthread_join(_handles[i], NULL);
if (nErrorcode!=0) if (nErrorcode != 0)
return nErrorcode; return nErrorcode;
} }
return 0; return 0;
} }
private: private:
vector<pthread_t> m_vecHandle; vector<pthread_t> _handles;
private: private:
ThreadManager(const ThreadManager&){;}// copy forbidden ThreadManager(const ThreadManager&){;}// copy forbidden
void operator=(const ThreadManager &){}// copy forbidden void operator = (const ThreadManager &){}// copy forbidden
}; };
} }

29
src/KeywordExtractor.hpp
View File

@ -7,10 +7,20 @@ namespace CppJieba
{ {
using namespace Limonp; using namespace Limonp;
class KeywordExtractor struct KeyWordInfo
{
uint freq;
double weight;
};
class KeywordExtractor//: public MPSegment
{ {
private: private:
MPSegment _segment; MPSegment _segment;
private:
unordered_map<string, uint> _wordIndex;
vector<KeyWordInfo> _words;
protected: protected:
bool _isInited; bool _isInited;
bool _getInitFlag()const{return _isInited;}; bool _getInitFlag()const{return _isInited;};
@ -22,7 +32,22 @@ namespace CppJieba
explicit KeywordExtractor(const string& dictPath){_setInitFlag(init(dictPath));}; explicit KeywordExtractor(const string& dictPath){_setInitFlag(init(dictPath));};
~KeywordExtractor(){}; ~KeywordExtractor(){};
public: public:
bool init(const string& dictPath){return _setInitFlag(_segment.init(dictPath));}; bool init(const string& dictPath)
{
ifstream ifs(dictPath.c_str());
if(!ifs)
{
LogError("open %s failed.", dictPath.c_str());
return false;
}
string line ;
vector<string> buf;
for(uint lineno = 0; getline(ifs, line); lineno++)
{
buf.clear();
}
return _setInitFlag(_segment.init(dictPath));
};
public: public:
bool extract(const string& str, vector<string>& keywords, uint topN) bool extract(const string& str, vector<string>& keywords, uint topN)
{ {

4
src/Limonp/ArgvContext.hpp
View File

@ -22,9 +22,9 @@ namespace Limonp
for(int i = 0; i < argc; i++) for(int i = 0; i < argc; i++)
{ {
if(strStartsWith(argv[i], "-")) if(startsWith(argv[i], "-"))
{ {
if(i + 1 < argc && !strStartsWith(argv[i + 1], "-")) if(i + 1 < argc && !startsWith(argv[i + 1], "-"))
{ {
_mpss[argv[i]] = argv[i+1]; _mpss[argv[i]] = argv[i+1];
i++; i++;

19
src/Limonp/Config.hpp
View File

@ -18,7 +18,17 @@ namespace Limonp
class Config class Config
{ {
public: public:
bool loadFile(const char * const filePath) Config(const char * const filePath)
{
_loadFile(filePath);
}
public:
operator bool ()
{
return !_map.empty();
}
private:
bool _loadFile(const char * const filePath)
{ {
ifstream ifs(filePath); ifstream ifs(filePath);
if(!ifs) if(!ifs)
@ -33,12 +43,12 @@ namespace Limonp
{ {
lineno ++; lineno ++;
trim(line); trim(line);
if(line.empty() || strStartsWith(line, "#")) if(line.empty() || startsWith(line, "#"))
{ {
continue; continue;
} }
vecBuf.clear(); vecBuf.clear();
if(!splitStr(line, vecBuf, "=") || 2 != vecBuf.size()) if(!split(line, vecBuf, "=") || 2 != vecBuf.size())
{ {
LogFatal("line[%d:%s] is illegal.", lineno, line.c_str()); LogFatal("line[%d:%s] is illegal.", lineno, line.c_str());
return false; return false;
@ -57,6 +67,7 @@ namespace Limonp
ifs.close(); ifs.close();
return true; return true;
} }
public:
bool get(const string& key, string& value) const bool get(const string& key, string& value) const
{ {
map<string, string>::const_iterator it = _map.find(key); map<string, string>::const_iterator it = _map.find(key);
@ -73,7 +84,7 @@ namespace Limonp
friend ostream& operator << (ostream& os, const Config& config); friend ostream& operator << (ostream& os, const Config& config);
}; };
ostream& operator << (ostream& os, const Config& config) inline ostream& operator << (ostream& os, const Config& config)
{ {
return os << config._map; return os << config._map;
} }

14
src/Limonp/logger.hpp
View File

@ -11,6 +11,7 @@
#include <string> #include <string>
#include <stdio.h> #include <stdio.h>
#include <stdarg.h> #include <stdarg.h>
#include <cassert>
#include "io_functs.hpp" #include "io_functs.hpp"
#include "str_functs.hpp" #include "str_functs.hpp"
@ -23,6 +24,7 @@
#define LogFatal(fmt, ...) Logger::LoggingF(LL_FATAL, FILE_BASENAME, __LINE__, fmt, ## __VA_ARGS__) #define LogFatal(fmt, ...) Logger::LoggingF(LL_FATAL, FILE_BASENAME, __LINE__, fmt, ## __VA_ARGS__)
namespace Limonp namespace Limonp
{ {
using namespace std; using namespace std;
@ -36,16 +38,11 @@ namespace Limonp
public: public:
static bool Logging(uint level, const string& msg, const char* fileName, int lineNo) static bool Logging(uint level, const string& msg, const char* fileName, int lineNo)
{ {
if(level > LL_FATAL) assert(level <= LL_FATAL);
{
cerr<<"level's value is out of range"<<endl;
return false;
}
char buf[CSTR_BUFFER_SIZE]; char buf[CSTR_BUFFER_SIZE];
time_t timeNow; time_t timeNow;
time(&timeNow); time(&timeNow);
size_t ret = strftime(buf, sizeof(buf), LOG_TIME_FORMAT, localtime(&timeNow)); if(!strftime(buf, sizeof(buf), LOG_TIME_FORMAT, localtime(&timeNow)))
if(0 == ret)
{ {
fprintf(stderr, "stftime failed.\n"); fprintf(stderr, "stftime failed.\n");
return false; return false;
@ -55,6 +52,9 @@ namespace Limonp
} }
static bool LoggingF(uint level, const char* fileName, int lineNo, const string& fmt, ...) static bool LoggingF(uint level, const char* fileName, int lineNo, const string& fmt, ...)
{ {
#ifdef LOGGER_LEVEL
if(level < LOGGER_LEVEL) return true;
#endif
int size = 256; int size = 256;
string msg; string msg;
va_list ap; va_list ap;

80
src/Limonp/str_functs.hpp
View File

@ -72,28 +72,6 @@ namespace Limonp
} }
} }
//inline bool joinStr(const vector<string>& src, string& dest, const string& connectorStr)
//{
// if(src.empty())
// {
// return false;
// }
// for(uint i = 0; i < src.size() - 1; i++)
// {
// dest += src[i];
// dest += connectorStr;
// }
// dest += src[src.size() - 1];
// return true;
//}
//inline string joinStr(const vector<string>& source, const string& connector)
//{
// string res;
// joinStr(source, res, connector);
// return res;
//}
template<class T> template<class T>
void join(T begin, T end, string& res, const string& connector) void join(T begin, T end, string& res, const string& connector)
{ {
@ -122,7 +100,7 @@ namespace Limonp
inline bool splitStr(const string& src, vector<string>& res, const string& pattern) inline bool split(const string& src, vector<string>& res, const string& pattern)
{ {
if(src.empty()) if(src.empty())
{ {
@ -181,20 +159,9 @@ namespace Limonp
} }
inline uint16_t twocharToUint16(char high, char low)
{
return (((uint16_t(high) & 0x00ff ) << 8) | (uint16_t(low) & 0x00ff));
}
inline pair<char, char> uint16ToChar2(uint16_t in)
{
pair<char, char> res;
res.first = (in>>8) & 0x00ff; //high
res.second = (in) & 0x00ff; //low
return res;
}
inline bool strStartsWith(const string& str, const string& prefix) inline bool startsWith(const string& str, const string& prefix)
{ {
//return str.substr(0, prefix.size()) == prefix; //return str.substr(0, prefix.size()) == prefix;
if(prefix.length() > str.length()) if(prefix.length() > str.length())
@ -204,7 +171,7 @@ namespace Limonp
return 0 == str.compare(0, prefix.length(), prefix); return 0 == str.compare(0, prefix.length(), prefix);
} }
inline bool strEndsWith(const string& str, const string& suffix) inline bool endsWith(const string& str, const string& suffix)
{ {
if(suffix.length() > str.length()) if(suffix.length() > str.length())
{ {
@ -218,13 +185,19 @@ namespace Limonp
return str.find(ch) != string::npos; return str.find(ch) != string::npos;
} }
inline uint16_t twocharToUint16(char high, char low)
{
return (((uint16_t(high) & 0x00ff ) << 8) | (uint16_t(low) & 0x00ff));
}
inline bool utf8ToUnicode(const char * const str, uint len, vector<uint16_t>& vec) inline bool utf8ToUnicode(const char * const str, uint len, vector<uint16_t>& vec)
{ {
char ch1, ch2;
if(!str) if(!str)
{ {
return false; return false;
} }
char ch1, ch2;
uint16_t tmp;
vec.clear(); vec.clear();
for(uint i = 0;i < len;) for(uint i = 0;i < len;)
{ {
@ -237,14 +210,16 @@ namespace Limonp
{ {
ch1 = (str[i] >> 2) & 0x07; ch1 = (str[i] >> 2) & 0x07;
ch2 = (str[i+1] & 0x3f) | ((str[i] & 0x03) << 6 ); ch2 = (str[i+1] & 0x3f) | ((str[i] & 0x03) << 6 );
vec.push_back(twocharToUint16(ch1, ch2)); tmp = (((uint16_t(ch1) & 0x00ff ) << 8) | (uint16_t(ch2) & 0x00ff));
vec.push_back(tmp);
i += 2; i += 2;
} }
else if((unsigned char)str[i] <= 0xef && i + 2 < len) else if((unsigned char)str[i] <= 0xef && i + 2 < len)
{ {
ch1 = (str[i] << 4) | ((str[i+1] >> 2) & 0x0f ); ch1 = (str[i] << 4) | ((str[i+1] >> 2) & 0x0f );
ch2 = ((str[i+1]<<6) & 0xc0) | (str[i+2] & 0x3f); ch2 = ((str[i+1]<<6) & 0xc0) | (str[i+2] & 0x3f);
vec.push_back(twocharToUint16(ch1, ch2)); tmp = (((uint16_t(ch1) & 0x00ff ) << 8) | (uint16_t(ch2) & 0x00ff));
vec.push_back(tmp);
i += 3; i += 3;
} }
else else
@ -310,7 +285,8 @@ namespace Limonp
{ {
if(i + 1 < len) //&& (str[i+1] & 0x80)) if(i + 1 < len) //&& (str[i+1] & 0x80))
{ {
vec.push_back(twocharToUint16(str[i], str[i + 1])); uint16_t tmp = (((uint16_t(str[i]) & 0x00ff ) << 8) | (uint16_t(str[i+1]) & 0x00ff));
vec.push_back(tmp);
i += 2; i += 2;
} }
else else
@ -321,11 +297,20 @@ namespace Limonp
} }
return true; return true;
} }
inline bool gbkTrans(const string& str, vector<uint16_t>& vec) inline bool gbkTrans(const string& str, vector<uint16_t>& vec)
{ {
return gbkTrans(str.c_str(), str.size(), vec); return gbkTrans(str.c_str(), str.size(), vec);
} }
//inline pair<char, char> uint16ToChar2(uint16_t in)
//{
// pair<char, char> res;
// res.first = (in>>8) & 0x00ff; //high
// res.second = (in) & 0x00ff; //low
// return res;
//}
inline bool gbkTrans(vector<uint16_t>::const_iterator begin, vector<uint16_t>::const_iterator end, string& res) inline bool gbkTrans(vector<uint16_t>::const_iterator begin, vector<uint16_t>::const_iterator end, string& res)
{ {
if(begin >= end) if(begin >= end)
@ -333,18 +318,21 @@ namespace Limonp
return false; return false;
} }
res.clear(); res.clear();
pair<char, char> pa; //pair<char, char> pa;
char first, second;
while(begin != end) while(begin != end)
{ {
pa = uint16ToChar2(*begin); //pa = uint16ToChar2(*begin);
if(pa.first & 0x80) first = ((*begin)>>8) & 0x00ff;
second = (*begin) & 0x00ff;
if(first & 0x80)
{ {
res += pa.first; res += first;
res += pa.second; res += second;
} }
else else
{ {
res += pa.second; res += second;
} }
begin++; begin++;
} }

2
src/MPSegment.hpp
View File

@ -32,7 +32,7 @@ namespace CppJieba
class MPSegment: public SegmentBase class MPSegment: public SegmentBase
{ {
private: protected:
Trie* _trie; Trie* _trie;
public: public:

30
src/Trie.hpp
View File

@ -130,26 +130,13 @@ namespace CppJieba
} }
bool loadDict(const char * const filePath) bool loadDict(const char * const filePath)
{ {
if(!_getInitFlag()) assert(_getInitFlag());
{ if(!_trieInsert(filePath))
LogError("not initted.");
return false;
}
if(!checkFileExist(filePath))
{
LogError("cann't find fiel[%s].",filePath);
return false;
}
bool res = false;
res = _trieInsert(filePath);
if(!res)
{ {
LogError("_trieInsert failed."); LogError("_trieInsert failed.");
return false; return false;
} }
res = _countWeight(); if(!_countWeight())
if(!res)
{ {
LogError("_countWeight failed."); LogError("_countWeight failed.");
return false; return false;
@ -339,15 +326,20 @@ namespace CppJieba
private: private:
bool _trieInsert(const char * const filePath) bool _trieInsert(const char * const filePath)
{ {
ifstream ifile(filePath); ifstream ifs(filePath);
if(!ifs)
{
LogError("open %s failed.", filePath);
return false;
}
string line; string line;
vector<string> vecBuf; vector<string> vecBuf;
TrieNodeInfo nodeInfo; TrieNodeInfo nodeInfo;
while(getline(ifile, line)) while(getline(ifs, line))
{ {
vecBuf.clear(); vecBuf.clear();
splitStr(line, vecBuf, " "); split(line, vecBuf, " ");
if(3 < vecBuf.size()) if(3 < vecBuf.size())
{ {
LogError("line[%s] illegal.", line.c_str()); LogError("line[%s] illegal.", line.c_str());

8
src/server.cpp
View File

@ -4,7 +4,7 @@
#include <ctype.h> #include <ctype.h>
#include <string.h> #include <string.h>
#include "Limonp/Config.hpp" #include "Limonp/Config.hpp"
#include "Husky/ServerFrame.hpp" #include "Husky/HuskyServer.hpp"
#include "MPSegment.hpp" #include "MPSegment.hpp"
#include "HMMSegment.hpp" #include "HMMSegment.hpp"
#include "MixSegment.hpp" #include "MixSegment.hpp"
@ -38,8 +38,8 @@ bool run(int argc, char** argv)
{ {
return false; return false;
} }
Config conf; Config conf(argv[1]);
if(!conf.loadFile(argv[1])) if(!conf)
{ {
return false; return false;
} }
@ -90,7 +90,7 @@ bool run(int argc, char** argv)
} }
ReqHandler reqHandler(dictPath, modelPath); ReqHandler reqHandler(dictPath, modelPath);
ServerFrame sf(port, threadNum, &reqHandler); HuskyServer sf(port, threadNum, &reqHandler);
return sf.init() && sf.run(); return sf.init() && sf.run();
} }