mirror of
https://github.com/yanyiwu/cppjieba.git
synced 2025-07-18 00:00:12 +08:00
update husky and limonp
This commit is contained in:
parent
24a15cd128
commit
3eb0470c2f
@ -1,6 +1,8 @@
|
||||
SET(EXECUTABLE_OUTPUT_PATH ${PROJECT_BINARY_DIR}/bin)
|
||||
SET(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/lib)
|
||||
|
||||
INCLUDE_DIRECTORIES(${PROJECT_SOURCE_DIR}/src)
|
||||
|
||||
ADD_EXECUTABLE(cjsegment segment.cpp)
|
||||
ADD_EXECUTABLE(cjserver server.cpp)
|
||||
TARGET_LINK_LIBRARIES(cjserver pthread)
|
||||
|
@ -215,7 +215,7 @@ namespace CppJieba
|
||||
{
|
||||
return false;
|
||||
}
|
||||
splitStr(line, tmp, " ");
|
||||
split(line, tmp, " ");
|
||||
if(tmp.size() != STATUS_SUM)
|
||||
{
|
||||
LogError("start_p illegal");
|
||||
@ -234,7 +234,7 @@ namespace CppJieba
|
||||
{
|
||||
return false;
|
||||
}
|
||||
splitStr(line, tmp, " ");
|
||||
split(line, tmp, " ");
|
||||
if(tmp.size() != STATUS_SUM)
|
||||
{
|
||||
LogError("trans_p illegal");
|
||||
@ -284,7 +284,7 @@ namespace CppJieba
|
||||
{
|
||||
continue;
|
||||
}
|
||||
if(strStartsWith(line, "#"))
|
||||
if(startsWith(line, "#"))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
@ -300,10 +300,10 @@ namespace CppJieba
|
||||
}
|
||||
vector<string> tmp, tmp2;
|
||||
uint16_t unico = 0;
|
||||
splitStr(line, tmp, ",");
|
||||
split(line, tmp, ",");
|
||||
for(uint i = 0; i < tmp.size(); i++)
|
||||
{
|
||||
splitStr(tmp[i], tmp2, ":");
|
||||
split(tmp[i], tmp2, ":");
|
||||
if(2 != tmp2.size())
|
||||
{
|
||||
LogError("_emitProb illegal.");
|
||||
|
@ -3,7 +3,7 @@
|
||||
|
||||
#include <iostream>
|
||||
#include <string>
|
||||
#include "../Limonp/logger.hpp"
|
||||
#include "Limonp/logger.hpp"
|
||||
|
||||
namespace Husky
|
||||
{
|
||||
@ -88,7 +88,7 @@ namespace Husky
|
||||
}
|
||||
string firstline(headerStr, lpos, rpos - lpos);
|
||||
trim(firstline);
|
||||
if(!splitStr(firstline, buf, " ") || 3 != buf.size())
|
||||
if(!split(firstline, buf, " ") || 3 != buf.size())
|
||||
{
|
||||
LogFatal("parse header first line failed.");
|
||||
return false;
|
||||
|
@ -1,302 +0,0 @@
|
||||
#ifndef HUSKY_SERVERFRAME_H
|
||||
#define HUSKY_SERVERFRAME_H
|
||||
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
|
||||
#include <cassert>
|
||||
#include <sys/socket.h>
|
||||
#include <sys/types.h>
|
||||
#include <arpa/inet.h>
|
||||
#include <stdlib.h>
|
||||
#include <pthread.h>
|
||||
#include <string.h>
|
||||
#include <errno.h>
|
||||
#include <unistd.h>
|
||||
#include <vector>
|
||||
#include "ThreadManager.hpp"
|
||||
#include "HttpReqInfo.hpp"
|
||||
|
||||
#define INVALID_SOCKET -1
|
||||
#define SOCKET_ERROR -1
|
||||
#define closesocket close
|
||||
#define RECV_BUFFER 10240
|
||||
#define LISEN_QUEUR_LEN 1024
|
||||
|
||||
|
||||
namespace Husky
|
||||
{
|
||||
using namespace Limonp;
|
||||
typedef int SOCKET;
|
||||
const struct timeval SOCKET_TIMEOUT = {2, 0};
|
||||
const char* const RESPONSE_FORMAT = "HTTP/1.1 200 OK\r\nConnection: close\r\nServer: FrameServer/1.0.0\r\nContent-Type: text/json; charset=%s\r\nContent-Length: %d\r\n\r\n";
|
||||
const char* const RESPONSE_CHARSET_UTF8 = "UTF-8";
|
||||
const char* const RESPONSE_CHARSET_GB2312 = "GB2312";
|
||||
const char* const CLIENT_IP_K = "CLIENT_IP";
|
||||
typedef unsigned short u_short;
|
||||
typedef unsigned int u_int;
|
||||
|
||||
class IRequestHandler
|
||||
{
|
||||
public:
|
||||
virtual ~IRequestHandler(){};
|
||||
public:
|
||||
virtual bool do_GET(const HttpReqInfo& httpReq, string& res) = 0;
|
||||
|
||||
};
|
||||
|
||||
struct SPara
|
||||
{
|
||||
SOCKET hSock;
|
||||
IRequestHandler * pHandler;
|
||||
pthread_mutex_t * ppmAccept;
|
||||
bool * pShutdown;
|
||||
};
|
||||
|
||||
class ServerFrame//: public IWorkHandler
|
||||
{
|
||||
private:
|
||||
pthread_mutex_t m_pmAccept;
|
||||
bool m_bShutdown;
|
||||
public:
|
||||
ServerFrame(unsigned nPort, unsigned nThreadCount, IRequestHandler* pHandler)
|
||||
{
|
||||
m_bShutdown = false;
|
||||
m_nLsnPort = nPort;
|
||||
m_nThreadCount = nThreadCount;
|
||||
m_pHandler = pHandler;
|
||||
assert(pHandler);
|
||||
pthread_mutex_init(&m_pmAccept,NULL);
|
||||
};
|
||||
virtual ~ServerFrame(){pthread_mutex_destroy(&m_pmAccept);};
|
||||
virtual bool init()
|
||||
{
|
||||
|
||||
if (!BindToLocalHost(m_lsnSock,m_nLsnPort))
|
||||
{
|
||||
LogFatal("BindToLocalHost failed.");
|
||||
return false;
|
||||
}
|
||||
LogInfo("init ok {port:%d, threadNum:%d}", m_nLsnPort, m_nThreadCount);
|
||||
|
||||
return true;
|
||||
}
|
||||
virtual bool dispose()
|
||||
{
|
||||
m_bShutdown=true;
|
||||
if (SOCKET_ERROR==closesocket(m_lsnSock))
|
||||
{
|
||||
LogError("error [%s]", strerror(errno));
|
||||
return false;
|
||||
}
|
||||
|
||||
int sockfd;
|
||||
struct sockaddr_in dest;
|
||||
|
||||
if ((sockfd = socket(AF_INET, SOCK_STREAM, 0)) < 0)
|
||||
{
|
||||
LogError("error [%s]", strerror(errno));
|
||||
return false;
|
||||
}
|
||||
|
||||
bzero(&dest, sizeof(dest));
|
||||
dest.sin_family = AF_INET;
|
||||
dest.sin_port = htons(m_nLsnPort);
|
||||
if (inet_aton("127.0.0.1", (struct in_addr *) &dest.sin_addr.s_addr) == 0)
|
||||
{
|
||||
LogError("error [%s]", strerror(errno));
|
||||
return false;
|
||||
}
|
||||
|
||||
if (connect(sockfd, (struct sockaddr *) &dest, sizeof(dest)) < 0)
|
||||
{
|
||||
LogError("error [%s]", strerror(errno));
|
||||
}
|
||||
close(sockfd);
|
||||
return true;
|
||||
}
|
||||
virtual bool run()
|
||||
{
|
||||
if(SOCKET_ERROR==listen(m_lsnSock,LISEN_QUEUR_LEN))
|
||||
{
|
||||
LogError("error [%s]", strerror(errno));
|
||||
return false;
|
||||
}
|
||||
ThreadManager thrMngr;
|
||||
int i;
|
||||
SPara para;
|
||||
para.hSock=m_lsnSock;
|
||||
para.pHandler=m_pHandler;
|
||||
para.ppmAccept=&m_pmAccept;
|
||||
para.pShutdown=&m_bShutdown;
|
||||
for (i=0;i<m_nThreadCount;i++)
|
||||
{
|
||||
if (0!=thrMngr.CreateThread(ServerThread, ¶))
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
LogDebug("expect thread count %d, real count %d",m_nThreadCount,i);
|
||||
if(i==0)
|
||||
{
|
||||
LogError("error [%s]", strerror(errno));
|
||||
return false;
|
||||
}
|
||||
|
||||
LogInfo("server start to run.........");
|
||||
|
||||
if (thrMngr.WaitMultipleThread()!=0)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
|
||||
}
|
||||
|
||||
protected:
|
||||
|
||||
bool BindToLocalHost(SOCKET &sock,u_short nPort)
|
||||
{
|
||||
sock=socket(AF_INET,SOCK_STREAM,0);
|
||||
if(INVALID_SOCKET==sock)
|
||||
{
|
||||
LogError("error [%s]", strerror(errno));
|
||||
return false;
|
||||
}
|
||||
|
||||
int nRet = 1;
|
||||
if(SOCKET_ERROR==setsockopt(m_lsnSock, SOL_SOCKET, SO_REUSEADDR, (char*)&nRet, sizeof(nRet)))
|
||||
{
|
||||
LogError("error [%s]", strerror(errno));
|
||||
}
|
||||
|
||||
struct sockaddr_in addrSock;
|
||||
addrSock.sin_family=AF_INET;
|
||||
addrSock.sin_port=htons(nPort);
|
||||
addrSock.sin_addr.s_addr=htonl(INADDR_ANY);
|
||||
int retval;
|
||||
retval = ::bind(sock,(sockaddr*)&addrSock,sizeof(sockaddr));
|
||||
if(SOCKET_ERROR==retval)
|
||||
{
|
||||
LogError("error [%s]", strerror(errno));
|
||||
closesocket(sock);
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
|
||||
}
|
||||
|
||||
static void * ServerThread(void * lpParameter )
|
||||
{
|
||||
SPara *pPara=(SPara*)lpParameter;
|
||||
SOCKET hSockLsn=pPara->hSock;
|
||||
IRequestHandler *pHandler=pPara->pHandler;
|
||||
bool* pShutdown = pPara->pShutdown;
|
||||
pthread_mutex_t* ppmAccept = pPara->ppmAccept;
|
||||
|
||||
int nRetCode;
|
||||
linger lng;
|
||||
char chRecvBuf[RECV_BUFFER];
|
||||
|
||||
SOCKET hClientSock;
|
||||
string strHttpResp;
|
||||
|
||||
sockaddr_in clientaddr;
|
||||
socklen_t nSize = sizeof(clientaddr);
|
||||
while(!(*pShutdown))
|
||||
{
|
||||
HttpReqInfo httpReq;
|
||||
pthread_mutex_lock(ppmAccept);
|
||||
hClientSock=accept(hSockLsn,(sockaddr *)&clientaddr, &nSize);
|
||||
pthread_mutex_unlock(ppmAccept);
|
||||
|
||||
if(hClientSock==SOCKET_ERROR)
|
||||
{
|
||||
if(!(*pShutdown))
|
||||
LogError("error [%s]", strerror(errno));
|
||||
continue;
|
||||
}
|
||||
|
||||
httpReq[CLIENT_IP_K] = inet_ntoa(clientaddr.sin_addr);// inet_ntoa is not thread safety at some version
|
||||
|
||||
lng.l_linger=1;
|
||||
lng.l_onoff=1;
|
||||
if(SOCKET_ERROR==setsockopt(hClientSock,SOL_SOCKET,SO_LINGER,(char*)&lng,sizeof(lng)))
|
||||
{
|
||||
LogError("error [%s]", strerror(errno));
|
||||
}
|
||||
|
||||
if(SOCKET_ERROR==setsockopt(hClientSock,SOL_SOCKET,SO_RCVTIMEO,(char*)&SOCKET_TIMEOUT,sizeof(SOCKET_TIMEOUT)))
|
||||
{
|
||||
LogError("error [%s]", strerror(errno));
|
||||
}
|
||||
|
||||
if(SOCKET_ERROR==setsockopt(hClientSock,SOL_SOCKET,SO_SNDTIMEO,(char*)&SOCKET_TIMEOUT,sizeof(SOCKET_TIMEOUT)))
|
||||
{
|
||||
LogError("error [%s]", strerror(errno));
|
||||
}
|
||||
|
||||
|
||||
string strRec;
|
||||
string strSnd;
|
||||
memset(chRecvBuf,0,sizeof(chRecvBuf));
|
||||
nRetCode = recv(hClientSock, chRecvBuf, RECV_BUFFER, 0);
|
||||
strRec = chRecvBuf;
|
||||
|
||||
#ifdef HUKSY_DEBUG
|
||||
LogDebug("request[%s]", strRec.c_str());
|
||||
#endif
|
||||
|
||||
if(SOCKET_ERROR==nRetCode)
|
||||
{
|
||||
LogDebug("error [%s]", strerror(errno));
|
||||
closesocket(hClientSock);
|
||||
continue;
|
||||
}
|
||||
if(0==nRetCode)
|
||||
{
|
||||
LogDebug("connection has been gracefully closed");
|
||||
closesocket(hClientSock);
|
||||
continue;
|
||||
}
|
||||
httpReq.load(strRec);
|
||||
|
||||
pHandler->do_GET(httpReq, strSnd);
|
||||
|
||||
char chHttpHeader[2048];
|
||||
|
||||
sprintf(chHttpHeader, RESPONSE_FORMAT, RESPONSE_CHARSET_UTF8, int(strSnd.length()));
|
||||
|
||||
strHttpResp=chHttpHeader;
|
||||
strHttpResp+=strSnd;
|
||||
#ifdef HUKSY_DEBUG
|
||||
LogDebug("response'body [%s]", strSnd.c_str());
|
||||
#endif
|
||||
|
||||
if (SOCKET_ERROR==send(hClientSock,strHttpResp.c_str(),strHttpResp.length(),0))
|
||||
{
|
||||
LogError("error [%s]", strerror(errno));
|
||||
}
|
||||
|
||||
|
||||
closesocket(hClientSock);
|
||||
}
|
||||
|
||||
return 0;
|
||||
|
||||
}
|
||||
|
||||
private:
|
||||
u_short m_nLsnPort;
|
||||
u_short m_nThreadCount;
|
||||
SOCKET m_lsnSock;
|
||||
IRequestHandler *m_pHandler;
|
||||
//static bool m_bShutdown;
|
||||
//static pthread_mutex_t m_pmAccept;
|
||||
|
||||
};
|
||||
|
||||
|
||||
}
|
||||
#endif
|
@ -5,8 +5,6 @@
|
||||
#include <vector>
|
||||
#include <map>
|
||||
|
||||
#define INFINITE 0
|
||||
|
||||
namespace Husky
|
||||
{
|
||||
using namespace std;
|
||||
@ -15,47 +13,43 @@ namespace Husky
|
||||
{
|
||||
private:
|
||||
typedef int HANDLE;
|
||||
typedef int DWORD;
|
||||
typedef void *(* PThreadFunc)(void* param);
|
||||
public:
|
||||
ThreadManager(){;}
|
||||
~ThreadManager(){}
|
||||
|
||||
unsigned int HandleCount(){return m_vecHandle.size();}
|
||||
size_t HandleCount(){return _handles.size();}
|
||||
|
||||
void clear()
|
||||
{
|
||||
m_vecHandle.clear();
|
||||
_handles.clear();
|
||||
}
|
||||
|
||||
HANDLE CreateThread( PThreadFunc pFunc,void *pPara)
|
||||
{
|
||||
pthread_t pt;
|
||||
int nErrorCode=pthread_create(&pt,NULL,pFunc,pPara);
|
||||
if(nErrorCode!=0)
|
||||
int nErrorCode = pthread_create(&pt,NULL,pFunc,pPara);
|
||||
if(nErrorCode != 0)
|
||||
return nErrorCode;
|
||||
m_vecHandle.push_back(pt); //加入线程列表 为WaitForMultipleObjects准备
|
||||
_handles.push_back(pt);
|
||||
return nErrorCode;
|
||||
|
||||
}
|
||||
|
||||
//hThread (thread handler) : 为0时为默认最后一个加入管理器的线程句柄
|
||||
//dwMilliseconds等待时间 : 单位毫秒,默认值无穷时间
|
||||
//return value : -1句柄无效,其他值 WaitForSingleObject函数的返回值
|
||||
DWORD Wait(HANDLE hThread=0,DWORD dwMilliseconds=INFINITE )
|
||||
int Wait(HANDLE hThread = 0)
|
||||
{
|
||||
if( hThread==0)//最后一个加入的线程
|
||||
if( hThread == 0)//the last handle
|
||||
{
|
||||
if(!m_vecHandle.empty())
|
||||
if(!_handles.empty())
|
||||
{
|
||||
return pthread_join(m_vecHandle.back(),NULL);
|
||||
return pthread_join(_handles.back(),NULL);
|
||||
}
|
||||
else
|
||||
return -1;
|
||||
}
|
||||
else
|
||||
{
|
||||
if (find(m_vecHandle.begin(),m_vecHandle.end(),hThread)==m_vecHandle.end())//不存在此句柄
|
||||
if (find(_handles.begin(),_handles.end(),hThread) == _handles.end())
|
||||
{
|
||||
return -1;
|
||||
}
|
||||
@ -65,31 +59,26 @@ namespace Husky
|
||||
|
||||
}
|
||||
|
||||
|
||||
//等待所有线程执行完毕
|
||||
//bWaitAll是否所有线程 : 默认值1等待所有线程,0有任何线程结束,此函数返回
|
||||
//dwMilliseconds : 单位毫秒,默认值无穷时间
|
||||
//return value : -1没有任何句柄,其他值 WaitForMultipleObjects函数的返回值
|
||||
DWORD WaitMultipleThread( bool bWaitAll=1,DWORD dwMilliseconds=INFINITE)
|
||||
int WaitMultipleThread()
|
||||
{
|
||||
if (m_vecHandle.empty())
|
||||
if (_handles.empty())
|
||||
return -1;
|
||||
int nErrorcode;
|
||||
for (uint i=0;i<m_vecHandle.size();++i)
|
||||
for (uint i = 0; i < _handles.size(); i++)
|
||||
{
|
||||
nErrorcode=pthread_join(m_vecHandle[i], NULL);
|
||||
if (nErrorcode!=0)
|
||||
nErrorcode = pthread_join(_handles[i], NULL);
|
||||
if (nErrorcode != 0)
|
||||
return nErrorcode;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
private:
|
||||
vector<pthread_t> m_vecHandle;
|
||||
vector<pthread_t> _handles;
|
||||
|
||||
private:
|
||||
ThreadManager(const ThreadManager&){;}// copy forbidden
|
||||
void operator=(const ThreadManager &){}// copy forbidden
|
||||
void operator = (const ThreadManager &){}// copy forbidden
|
||||
};
|
||||
}
|
||||
|
||||
|
@ -7,10 +7,20 @@ namespace CppJieba
|
||||
{
|
||||
using namespace Limonp;
|
||||
|
||||
class KeywordExtractor
|
||||
struct KeyWordInfo
|
||||
{
|
||||
|
||||
uint freq;
|
||||
double weight;
|
||||
};
|
||||
|
||||
class KeywordExtractor//: public MPSegment
|
||||
{
|
||||
private:
|
||||
MPSegment _segment;
|
||||
private:
|
||||
unordered_map<string, uint> _wordIndex;
|
||||
vector<KeyWordInfo> _words;
|
||||
protected:
|
||||
bool _isInited;
|
||||
bool _getInitFlag()const{return _isInited;};
|
||||
@ -22,7 +32,22 @@ namespace CppJieba
|
||||
explicit KeywordExtractor(const string& dictPath){_setInitFlag(init(dictPath));};
|
||||
~KeywordExtractor(){};
|
||||
public:
|
||||
bool init(const string& dictPath){return _setInitFlag(_segment.init(dictPath));};
|
||||
bool init(const string& dictPath)
|
||||
{
|
||||
ifstream ifs(dictPath.c_str());
|
||||
if(!ifs)
|
||||
{
|
||||
LogError("open %s failed.", dictPath.c_str());
|
||||
return false;
|
||||
}
|
||||
string line ;
|
||||
vector<string> buf;
|
||||
for(uint lineno = 0; getline(ifs, line); lineno++)
|
||||
{
|
||||
buf.clear();
|
||||
}
|
||||
return _setInitFlag(_segment.init(dictPath));
|
||||
};
|
||||
public:
|
||||
bool extract(const string& str, vector<string>& keywords, uint topN)
|
||||
{
|
||||
|
@ -22,9 +22,9 @@ namespace Limonp
|
||||
|
||||
for(int i = 0; i < argc; i++)
|
||||
{
|
||||
if(strStartsWith(argv[i], "-"))
|
||||
if(startsWith(argv[i], "-"))
|
||||
{
|
||||
if(i + 1 < argc && !strStartsWith(argv[i + 1], "-"))
|
||||
if(i + 1 < argc && !startsWith(argv[i + 1], "-"))
|
||||
{
|
||||
_mpss[argv[i]] = argv[i+1];
|
||||
i++;
|
||||
|
@ -18,7 +18,17 @@ namespace Limonp
|
||||
class Config
|
||||
{
|
||||
public:
|
||||
bool loadFile(const char * const filePath)
|
||||
Config(const char * const filePath)
|
||||
{
|
||||
_loadFile(filePath);
|
||||
}
|
||||
public:
|
||||
operator bool ()
|
||||
{
|
||||
return !_map.empty();
|
||||
}
|
||||
private:
|
||||
bool _loadFile(const char * const filePath)
|
||||
{
|
||||
ifstream ifs(filePath);
|
||||
if(!ifs)
|
||||
@ -33,12 +43,12 @@ namespace Limonp
|
||||
{
|
||||
lineno ++;
|
||||
trim(line);
|
||||
if(line.empty() || strStartsWith(line, "#"))
|
||||
if(line.empty() || startsWith(line, "#"))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
vecBuf.clear();
|
||||
if(!splitStr(line, vecBuf, "=") || 2 != vecBuf.size())
|
||||
if(!split(line, vecBuf, "=") || 2 != vecBuf.size())
|
||||
{
|
||||
LogFatal("line[%d:%s] is illegal.", lineno, line.c_str());
|
||||
return false;
|
||||
@ -57,6 +67,7 @@ namespace Limonp
|
||||
ifs.close();
|
||||
return true;
|
||||
}
|
||||
public:
|
||||
bool get(const string& key, string& value) const
|
||||
{
|
||||
map<string, string>::const_iterator it = _map.find(key);
|
||||
@ -73,7 +84,7 @@ namespace Limonp
|
||||
friend ostream& operator << (ostream& os, const Config& config);
|
||||
};
|
||||
|
||||
ostream& operator << (ostream& os, const Config& config)
|
||||
inline ostream& operator << (ostream& os, const Config& config)
|
||||
{
|
||||
return os << config._map;
|
||||
}
|
||||
|
@ -11,6 +11,7 @@
|
||||
#include <string>
|
||||
#include <stdio.h>
|
||||
#include <stdarg.h>
|
||||
#include <cassert>
|
||||
#include "io_functs.hpp"
|
||||
#include "str_functs.hpp"
|
||||
|
||||
@ -23,6 +24,7 @@
|
||||
#define LogFatal(fmt, ...) Logger::LoggingF(LL_FATAL, FILE_BASENAME, __LINE__, fmt, ## __VA_ARGS__)
|
||||
|
||||
|
||||
|
||||
namespace Limonp
|
||||
{
|
||||
using namespace std;
|
||||
@ -36,16 +38,11 @@ namespace Limonp
|
||||
public:
|
||||
static bool Logging(uint level, const string& msg, const char* fileName, int lineNo)
|
||||
{
|
||||
if(level > LL_FATAL)
|
||||
{
|
||||
cerr<<"level's value is out of range"<<endl;
|
||||
return false;
|
||||
}
|
||||
assert(level <= LL_FATAL);
|
||||
char buf[CSTR_BUFFER_SIZE];
|
||||
time_t timeNow;
|
||||
time(&timeNow);
|
||||
size_t ret = strftime(buf, sizeof(buf), LOG_TIME_FORMAT, localtime(&timeNow));
|
||||
if(0 == ret)
|
||||
if(!strftime(buf, sizeof(buf), LOG_TIME_FORMAT, localtime(&timeNow)))
|
||||
{
|
||||
fprintf(stderr, "stftime failed.\n");
|
||||
return false;
|
||||
@ -55,6 +52,9 @@ namespace Limonp
|
||||
}
|
||||
static bool LoggingF(uint level, const char* fileName, int lineNo, const string& fmt, ...)
|
||||
{
|
||||
#ifdef LOGGER_LEVEL
|
||||
if(level < LOGGER_LEVEL) return true;
|
||||
#endif
|
||||
int size = 256;
|
||||
string msg;
|
||||
va_list ap;
|
||||
|
@ -72,28 +72,6 @@ namespace Limonp
|
||||
}
|
||||
}
|
||||
|
||||
//inline bool joinStr(const vector<string>& src, string& dest, const string& connectorStr)
|
||||
//{
|
||||
// if(src.empty())
|
||||
// {
|
||||
// return false;
|
||||
// }
|
||||
// for(uint i = 0; i < src.size() - 1; i++)
|
||||
// {
|
||||
// dest += src[i];
|
||||
// dest += connectorStr;
|
||||
// }
|
||||
// dest += src[src.size() - 1];
|
||||
// return true;
|
||||
//}
|
||||
|
||||
//inline string joinStr(const vector<string>& source, const string& connector)
|
||||
//{
|
||||
// string res;
|
||||
// joinStr(source, res, connector);
|
||||
// return res;
|
||||
//}
|
||||
|
||||
template<class T>
|
||||
void join(T begin, T end, string& res, const string& connector)
|
||||
{
|
||||
@ -122,7 +100,7 @@ namespace Limonp
|
||||
|
||||
|
||||
|
||||
inline bool splitStr(const string& src, vector<string>& res, const string& pattern)
|
||||
inline bool split(const string& src, vector<string>& res, const string& pattern)
|
||||
{
|
||||
if(src.empty())
|
||||
{
|
||||
@ -181,20 +159,9 @@ namespace Limonp
|
||||
}
|
||||
|
||||
|
||||
inline uint16_t twocharToUint16(char high, char low)
|
||||
{
|
||||
return (((uint16_t(high) & 0x00ff ) << 8) | (uint16_t(low) & 0x00ff));
|
||||
}
|
||||
|
||||
inline pair<char, char> uint16ToChar2(uint16_t in)
|
||||
{
|
||||
pair<char, char> res;
|
||||
res.first = (in>>8) & 0x00ff; //high
|
||||
res.second = (in) & 0x00ff; //low
|
||||
return res;
|
||||
}
|
||||
|
||||
inline bool strStartsWith(const string& str, const string& prefix)
|
||||
inline bool startsWith(const string& str, const string& prefix)
|
||||
{
|
||||
//return str.substr(0, prefix.size()) == prefix;
|
||||
if(prefix.length() > str.length())
|
||||
@ -204,7 +171,7 @@ namespace Limonp
|
||||
return 0 == str.compare(0, prefix.length(), prefix);
|
||||
}
|
||||
|
||||
inline bool strEndsWith(const string& str, const string& suffix)
|
||||
inline bool endsWith(const string& str, const string& suffix)
|
||||
{
|
||||
if(suffix.length() > str.length())
|
||||
{
|
||||
@ -218,13 +185,19 @@ namespace Limonp
|
||||
return str.find(ch) != string::npos;
|
||||
}
|
||||
|
||||
inline uint16_t twocharToUint16(char high, char low)
|
||||
{
|
||||
return (((uint16_t(high) & 0x00ff ) << 8) | (uint16_t(low) & 0x00ff));
|
||||
}
|
||||
|
||||
inline bool utf8ToUnicode(const char * const str, uint len, vector<uint16_t>& vec)
|
||||
{
|
||||
char ch1, ch2;
|
||||
if(!str)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
char ch1, ch2;
|
||||
uint16_t tmp;
|
||||
vec.clear();
|
||||
for(uint i = 0;i < len;)
|
||||
{
|
||||
@ -237,14 +210,16 @@ namespace Limonp
|
||||
{
|
||||
ch1 = (str[i] >> 2) & 0x07;
|
||||
ch2 = (str[i+1] & 0x3f) | ((str[i] & 0x03) << 6 );
|
||||
vec.push_back(twocharToUint16(ch1, ch2));
|
||||
tmp = (((uint16_t(ch1) & 0x00ff ) << 8) | (uint16_t(ch2) & 0x00ff));
|
||||
vec.push_back(tmp);
|
||||
i += 2;
|
||||
}
|
||||
else if((unsigned char)str[i] <= 0xef && i + 2 < len)
|
||||
{
|
||||
ch1 = (str[i] << 4) | ((str[i+1] >> 2) & 0x0f );
|
||||
ch2 = ((str[i+1]<<6) & 0xc0) | (str[i+2] & 0x3f);
|
||||
vec.push_back(twocharToUint16(ch1, ch2));
|
||||
tmp = (((uint16_t(ch1) & 0x00ff ) << 8) | (uint16_t(ch2) & 0x00ff));
|
||||
vec.push_back(tmp);
|
||||
i += 3;
|
||||
}
|
||||
else
|
||||
@ -310,7 +285,8 @@ namespace Limonp
|
||||
{
|
||||
if(i + 1 < len) //&& (str[i+1] & 0x80))
|
||||
{
|
||||
vec.push_back(twocharToUint16(str[i], str[i + 1]));
|
||||
uint16_t tmp = (((uint16_t(str[i]) & 0x00ff ) << 8) | (uint16_t(str[i+1]) & 0x00ff));
|
||||
vec.push_back(tmp);
|
||||
i += 2;
|
||||
}
|
||||
else
|
||||
@ -321,11 +297,20 @@ namespace Limonp
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
inline bool gbkTrans(const string& str, vector<uint16_t>& vec)
|
||||
{
|
||||
return gbkTrans(str.c_str(), str.size(), vec);
|
||||
}
|
||||
|
||||
//inline pair<char, char> uint16ToChar2(uint16_t in)
|
||||
//{
|
||||
// pair<char, char> res;
|
||||
// res.first = (in>>8) & 0x00ff; //high
|
||||
// res.second = (in) & 0x00ff; //low
|
||||
// return res;
|
||||
//}
|
||||
|
||||
inline bool gbkTrans(vector<uint16_t>::const_iterator begin, vector<uint16_t>::const_iterator end, string& res)
|
||||
{
|
||||
if(begin >= end)
|
||||
@ -333,18 +318,21 @@ namespace Limonp
|
||||
return false;
|
||||
}
|
||||
res.clear();
|
||||
pair<char, char> pa;
|
||||
//pair<char, char> pa;
|
||||
char first, second;
|
||||
while(begin != end)
|
||||
{
|
||||
pa = uint16ToChar2(*begin);
|
||||
if(pa.first & 0x80)
|
||||
//pa = uint16ToChar2(*begin);
|
||||
first = ((*begin)>>8) & 0x00ff;
|
||||
second = (*begin) & 0x00ff;
|
||||
if(first & 0x80)
|
||||
{
|
||||
res += pa.first;
|
||||
res += pa.second;
|
||||
res += first;
|
||||
res += second;
|
||||
}
|
||||
else
|
||||
{
|
||||
res += pa.second;
|
||||
res += second;
|
||||
}
|
||||
begin++;
|
||||
}
|
||||
|
@ -32,7 +32,7 @@ namespace CppJieba
|
||||
|
||||
class MPSegment: public SegmentBase
|
||||
{
|
||||
private:
|
||||
protected:
|
||||
Trie* _trie;
|
||||
|
||||
public:
|
||||
|
30
src/Trie.hpp
30
src/Trie.hpp
@ -130,26 +130,13 @@ namespace CppJieba
|
||||
}
|
||||
bool loadDict(const char * const filePath)
|
||||
{
|
||||
if(!_getInitFlag())
|
||||
{
|
||||
LogError("not initted.");
|
||||
return false;
|
||||
}
|
||||
|
||||
if(!checkFileExist(filePath))
|
||||
{
|
||||
LogError("cann't find fiel[%s].",filePath);
|
||||
return false;
|
||||
}
|
||||
bool res = false;
|
||||
res = _trieInsert(filePath);
|
||||
if(!res)
|
||||
assert(_getInitFlag());
|
||||
if(!_trieInsert(filePath))
|
||||
{
|
||||
LogError("_trieInsert failed.");
|
||||
return false;
|
||||
}
|
||||
res = _countWeight();
|
||||
if(!res)
|
||||
if(!_countWeight())
|
||||
{
|
||||
LogError("_countWeight failed.");
|
||||
return false;
|
||||
@ -339,15 +326,20 @@ namespace CppJieba
|
||||
private:
|
||||
bool _trieInsert(const char * const filePath)
|
||||
{
|
||||
ifstream ifile(filePath);
|
||||
ifstream ifs(filePath);
|
||||
if(!ifs)
|
||||
{
|
||||
LogError("open %s failed.", filePath);
|
||||
return false;
|
||||
}
|
||||
string line;
|
||||
vector<string> vecBuf;
|
||||
|
||||
TrieNodeInfo nodeInfo;
|
||||
while(getline(ifile, line))
|
||||
while(getline(ifs, line))
|
||||
{
|
||||
vecBuf.clear();
|
||||
splitStr(line, vecBuf, " ");
|
||||
split(line, vecBuf, " ");
|
||||
if(3 < vecBuf.size())
|
||||
{
|
||||
LogError("line[%s] illegal.", line.c_str());
|
||||
|
@ -4,7 +4,7 @@
|
||||
#include <ctype.h>
|
||||
#include <string.h>
|
||||
#include "Limonp/Config.hpp"
|
||||
#include "Husky/ServerFrame.hpp"
|
||||
#include "Husky/HuskyServer.hpp"
|
||||
#include "MPSegment.hpp"
|
||||
#include "HMMSegment.hpp"
|
||||
#include "MixSegment.hpp"
|
||||
@ -38,8 +38,8 @@ bool run(int argc, char** argv)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
Config conf;
|
||||
if(!conf.loadFile(argv[1]))
|
||||
Config conf(argv[1]);
|
||||
if(!conf)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
@ -90,7 +90,7 @@ bool run(int argc, char** argv)
|
||||
}
|
||||
|
||||
ReqHandler reqHandler(dictPath, modelPath);
|
||||
ServerFrame sf(port, threadNum, &reqHandler);
|
||||
HuskyServer sf(port, threadNum, &reqHandler);
|
||||
return sf.init() && sf.run();
|
||||
}
|
||||
|
||||
|
Loading…
x
Reference in New Issue
Block a user