From d2d6868b75d0be79cc3473201a182ab1b1d8874f Mon Sep 17 00:00:00 2001 From: wyy Date: Fri, 21 Mar 2014 11:18:34 +0800 Subject: [PATCH 1/4] merge some testfile into one testfile to reduce compiler cost --- ChangeLog.md | 4 + test/unittest/CMakeLists.txt | 2 +- test/unittest/TFullSegment.cpp | 18 ----- test/unittest/THMMSegment.cpp | 19 ----- test/unittest/TMPSegment.cpp | 46 ----------- test/unittest/TMixSegment.cpp | 16 ---- test/unittest/TQuerySegment.cpp | 20 ----- test/unittest/TSegmentBase.cpp | 36 --------- test/unittest/TSegments.cpp | 135 ++++++++++++++++++++++++++++++++ 9 files changed, 140 insertions(+), 156 deletions(-) delete mode 100644 test/unittest/TFullSegment.cpp delete mode 100644 test/unittest/THMMSegment.cpp delete mode 100644 test/unittest/TMPSegment.cpp delete mode 100644 test/unittest/TMixSegment.cpp delete mode 100644 test/unittest/TQuerySegment.cpp delete mode 100644 test/unittest/TSegmentBase.cpp create mode 100644 test/unittest/TSegments.cpp diff --git a/ChangeLog.md b/ChangeLog.md index 8e313d2..f1a6fb0 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,3 +1,7 @@ +## v2.3.5 + +1. 修改一些测试用例的文件，减少测试时编译的时间。 + ## v2.3.4 1. 修改了设计上的问题，删除了`TrieManager`这个类，以避免造成一些可能的隐患。 diff --git a/test/unittest/CMakeLists.txt b/test/unittest/CMakeLists.txt index ac3b930..79457cd 100644 --- a/test/unittest/CMakeLists.txt +++ b/test/unittest/CMakeLists.txt @@ -6,7 +6,7 @@ SET(GTEST_ROOT_DIR gtest-1.6.0) ADD_DEFINITIONS(-DLOGGER_LEVEL=LL_WARN) INCLUDE_DIRECTORIES(${GTEST_ROOT_DIR} ${GTEST_ROOT_DIR}/include ${PROJECT_SOURCE_DIR}) ADD_LIBRARY(gtest STATIC ${GTEST_ROOT_DIR}/src/gtest-all.cc) -ADD_EXECUTABLE(test.run gtest_main.cpp TKeywordExtractor.cpp TMPSegment.cpp TTrie.cpp TFullSegment.cpp TQuerySegment.cpp THMMSegment.cpp TMixSegment.cpp TSegmentBase.cpp) +ADD_EXECUTABLE(test.run gtest_main.cpp TKeywordExtractor.cpp TTrie.cpp TSegments.cpp ) TARGET_LINK_LIBRARIES(gtest pthread) TARGET_LINK_LIBRARIES(test.run gtest pthread) diff --git a/test/unittest/TFullSegment.cpp b/test/unittest/TFullSegment.cpp deleted file mode 100644 index 2c99e21..0000000 --- a/test/unittest/TFullSegment.cpp +++ /dev/null @@ -1,18 +0,0 @@ -#include "src/FullSegment.hpp" -#include "gtest/gtest.h" - -using namespace CppJieba; - -TEST(FullSegment, Test1) -{ - FullSegment segment("../dict/extra_dict/jieba.dict.small.utf8"); - const char* str = "我来自北京邮电大学。。。 学号 123456"; - vector words; - - ASSERT_EQ(segment.cut(str, words), true); - - string s; - s << words; - ASSERT_EQ(s, "[\"我\", \"来自\", \"北京\", \"北京邮电大学\", \"邮电\", \"电大\", \"大学\", \"。\", \"。\", \"。\", \" \", \"学\", \"号\", \" 123456\"]"); -} - diff --git a/test/unittest/THMMSegment.cpp b/test/unittest/THMMSegment.cpp deleted file mode 100644 index fc7c04c..0000000 --- a/test/unittest/THMMSegment.cpp +++ /dev/null @@ -1,19 +0,0 @@ -#include "src/HMMSegment.hpp" -#include "gtest/gtest.h" - -using namespace CppJieba; - -TEST(HMMSegmentTest, Test1) -{ - HMMSegment segment("../dict/hmm_model.utf8");; - const char* str = "我来自北京邮电大学。。。 学号 123456"; - const char* res[] = {"我来", "自北京", "邮电大学", "。", "。", "。", " ", "学号", " 123456"}; - //string s; - //vector buf(res, res + sizeof(res)/sizeof(res[0])); - vector words; - ASSERT_TRUE(segment); - ASSERT_TRUE(segment.cut(str, words)); - //print(words); - EXPECT_EQ(words, vector(res, res + sizeof(res)/sizeof(res[0]))); -} - diff --git a/test/unittest/TMPSegment.cpp b/test/unittest/TMPSegment.cpp deleted file mode 100644 index 3ca7268..0000000 --- a/test/unittest/TMPSegment.cpp +++ /dev/null @@ -1,46 +0,0 @@ -#include "src/MPSegment.hpp" -#include "src/Limonp/io_functs.hpp" -#include "gtest/gtest.h" - -using namespace CppJieba; -using namespace Limonp; - -TEST(MPSegmentTest, Test1) -{ - MPSegment segment("../dict/extra_dict/jieba.dict.small.utf8");; - const char* str = "我来自北京邮电大学。。。 学号 123456"; - const char* res[] = {"我", "来自", "北京邮电大学", "。","。","。"," ","学","号", " 123456"}; - vector words; - ASSERT_TRUE(segment); - ASSERT_TRUE(segment.cut(str, words)); - //print(words); - EXPECT_EQ(words, vector(res, res + sizeof(res)/sizeof(res[0]))); -} - -TEST(MPSegmentTest, Test2) -{ - MPSegment segment("../dict/extra_dict/jieba.dict.small.utf8"); - string line; - ifstream ifs("../test/testdata/review.100"); - vector words; - - string eRes; - loadFile2Str("../test/testdata/review.100.res", eRes); - string res; - - while(getline(ifs, line)) - { - res += line; - res += '\n'; - - words.clear(); - segment.cut(line, words); - string s; - s << words; - res += s; - res += '\n'; - } - WriteStr2File("../test/testdata/review.100.res", res.c_str(), "w"); - //ASSERT_EQ(res, eRes); - -} diff --git a/test/unittest/TMixSegment.cpp b/test/unittest/TMixSegment.cpp deleted file mode 100644 index fe904a4..0000000 --- a/test/unittest/TMixSegment.cpp +++ /dev/null @@ -1,16 +0,0 @@ -#include "src/MixSegment.hpp" -#include "gtest/gtest.h" - -using namespace CppJieba; - -TEST(MixSegmentTest, Test1) -{ - MixSegment segment("../dict/extra_dict/jieba.dict.small.utf8", "../dict/hmm_model.utf8");; - const char* str = "我来自北京邮电大学。。。 学号 123456"; - const char* res[] = {"我", "来自", "北京邮电大学", "。","。","。"," ","学号", " 123456"}; - vector words; - ASSERT_TRUE(segment); - ASSERT_TRUE(segment.cut(str, words)); - EXPECT_EQ(words, vector(res, res + sizeof(res)/sizeof(res[0]))); -} - diff --git a/test/unittest/TQuerySegment.cpp b/test/unittest/TQuerySegment.cpp deleted file mode 100644 index 7b76561..0000000 --- a/test/unittest/TQuerySegment.cpp +++ /dev/null @@ -1,20 +0,0 @@ -#include "src/QuerySegment.hpp" -#include "gtest/gtest.h" - -using namespace CppJieba; - -TEST(QuerySegment, Test1) -{ - QuerySegment segment("../dict/extra_dict/jieba.dict.small.utf8", "../dict/hmm_model.utf8", 3); - const char* str = "小明硕士毕业于中国科学院计算所，后在日本京都大学深造"; - vector words; - - ASSERT_TRUE(segment.cut(str, words)); - - string s1, s2; - s1 << words; - s2 = "[\"小明\", \"硕士\", \"毕业\", \"于\", \"中国\", \"中国科学院\", \"科学\", \"科学院\", \"学院\", \"计算所\", \"，\", \"后\", \"在\", \"日本\", \"京都\", \"京都大学\", \"大学\", \"深造\"]"; - ASSERT_EQ(s1, s2); - -} - diff --git a/test/unittest/TSegmentBase.cpp b/test/unittest/TSegmentBase.cpp deleted file mode 100644 index 6ee708e..0000000 --- a/test/unittest/TSegmentBase.cpp +++ /dev/null @@ -1,36 +0,0 @@ -#include "src/SegmentBase.hpp" -#include "gtest/gtest.h" - -using namespace CppJieba; - -TEST(SegmentBaseTest, Test1) -{ - const char* str = "heheh你好...hh"; - string s; - vector buf; - buf.push_back("heheh"); - buf.push_back("你好"); - buf.push_back("...hh"); - vector res; - size_t size = strlen(str); - size_t offset = 0; - while(offset < size) - { - size_t len = 0; - const char* t = str + offset; - SegmentBase::filterAscii(t, size - offset, len); - s.assign(t, len); - res.push_back(s); - //cout< buf; + buf.push_back("heheh"); + buf.push_back("你好"); + buf.push_back("...hh"); + vector res; + size_t size = strlen(str); + size_t offset = 0; + while(offset < size) + { + size_t len = 0; + const char* t = str + offset; + SegmentBase::filterAscii(t, size - offset, len); + s.assign(t, len); + res.push_back(s); + //cout< words; + ASSERT_TRUE(segment); + ASSERT_TRUE(segment.cut(str, words)); + EXPECT_EQ(words, vector(res, res + sizeof(res)/sizeof(res[0]))); +} + +TEST(MPSegmentTest, Test1) +{ + MPSegment segment("../dict/extra_dict/jieba.dict.small.utf8");; + const char* str = "我来自北京邮电大学。。。 学号 123456"; + const char* res[] = {"我", "来自", "北京邮电大学", "。","。","。"," ","学","号", " 123456"}; + vector words; + ASSERT_TRUE(segment); + ASSERT_TRUE(segment.cut(str, words)); + //print(words); + EXPECT_EQ(words, vector(res, res + sizeof(res)/sizeof(res[0]))); +} + +TEST(MPSegmentTest, Test2) +{ + MPSegment segment("../dict/extra_dict/jieba.dict.small.utf8"); + string line; + ifstream ifs("../test/testdata/review.100"); + vector words; + + string eRes; + loadFile2Str("../test/testdata/review.100.res", eRes); + string res; + + while(getline(ifs, line)) + { + res += line; + res += '\n'; + + words.clear(); + segment.cut(line, words); + string s; + s << words; + res += s; + res += '\n'; + } + WriteStr2File("../test/testdata/review.100.res", res.c_str(), "w"); + //ASSERT_EQ(res, eRes); + +} +TEST(HMMSegmentTest, Test1) +{ + HMMSegment segment("../dict/hmm_model.utf8");; + const char* str = "我来自北京邮电大学。。。 学号 123456"; + const char* res[] = {"我来", "自北京", "邮电大学", "。", "。", "。", " ", "学号", " 123456"}; + //string s; + //vector buf(res, res + sizeof(res)/sizeof(res[0])); + vector words; + ASSERT_TRUE(segment); + ASSERT_TRUE(segment.cut(str, words)); + //print(words); + EXPECT_EQ(words, vector(res, res + sizeof(res)/sizeof(res[0]))); +} + +TEST(FullSegment, Test1) +{ + FullSegment segment("../dict/extra_dict/jieba.dict.small.utf8"); + const char* str = "我来自北京邮电大学。。。 学号 123456"; + vector words; + + ASSERT_EQ(segment.cut(str, words), true); + + string s; + s << words; + ASSERT_EQ(s, "[\"我\", \"来自\", \"北京\", \"北京邮电大学\", \"邮电\", \"电大\", \"大学\", \"。\", \"。\", \"。\", \" \", \"学\", \"号\", \" 123456\"]"); +} + +TEST(QuerySegment, Test1) +{ + QuerySegment segment("../dict/extra_dict/jieba.dict.small.utf8", "../dict/hmm_model.utf8", 3); + const char* str = "小明硕士毕业于中国科学院计算所，后在日本京都大学深造"; + vector words; + + ASSERT_TRUE(segment.cut(str, words)); + + string s1, s2; + s1 << words; + s2 = "[\"小明\", \"硕士\", \"毕业\", \"于\", \"中国\", \"中国科学院\", \"科学\", \"科学院\", \"学院\", \"计算所\", \"，\", \"后\", \"在\", \"日本\", \"京都\", \"京都大学\", \"大学\", \"深造\"]"; + ASSERT_EQ(s1, s2); + +} + From 2b72d1cf9140d3ae0cfc469d5e5660c7aca52e8d Mon Sep 17 00:00:00 2001 From: wyy Date: Fri, 21 Mar 2014 11:55:40 +0800 Subject: [PATCH 2/4] calc time consumed in load_test --- test/load_test.cpp | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/test/load_test.cpp b/test/load_test.cpp index 9129a99..1b3a618 100644 --- a/test/load_test.cpp +++ b/test/load_test.cpp @@ -1,4 +1,5 @@ #include +#include #include #include "../src/Limonp/ArgvContext.hpp" #include "../src/Limonp/io_functs.hpp" @@ -30,14 +31,15 @@ void cut(const ISegment * seg, const char * const filePath, size_t times = 10) int main(int argc, char ** argv) { + MixSegment seg("../dict/jieba.dict.utf8", "../dict/hmm_model.utf8"); + if(!seg) { - MixSegment seg("../dict/jieba.dict.utf8", "../dict/hmm_model.utf8"); - if(!seg) - { - cout<<"seg init failed."< Date: Thu, 27 Mar 2014 11:44:41 +0800 Subject: [PATCH 3/4] update limonp && husky to adapter less version g++ --- CMakeLists.txt | 4 +- conf/server.conf | 2 - src/Husky/EpollServer.hpp | 281 +++++++++++++++++++++++++++++++++ src/Husky/HttpReqInfo.hpp | 33 ++-- src/Husky/HuskyServer.hpp | 300 ------------------------------------ src/Husky/ThreadManager.hpp | 87 ----------- src/KeywordExtractor.hpp | 3 +- src/Limonp/ArgvContext.hpp | 1 - src/Limonp/logger.hpp | 19 +-- src/Limonp/map_functs.hpp | 46 ------ src/Limonp/std_outbound.hpp | 24 ++- src/Limonp/str_functs.hpp | 1 - src/server.cpp | 15 +- 13 files changed, 331 insertions(+), 485 deletions(-) create mode 100644 src/Husky/EpollServer.hpp delete mode 100644 src/Husky/HuskyServer.hpp delete mode 100644 src/Husky/ThreadManager.hpp delete mode 100644 src/Limonp/map_functs.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 64a308e..79247e2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,11 +1,11 @@ PROJECT(CPPJIEBA) -CMAKE_MINIMUM_REQUIRED (VERSION 2.8) +CMAKE_MINIMUM_REQUIRED (VERSION 2.6) if (CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) set (CMAKE_INSTALL_PREFIX "/usr" CACHE PATH "default install path" FORCE ) endif() -ADD_DEFINITIONS(-std=c++0x -O3 -Wall -g) +ADD_DEFINITIONS(-O3 -Wall -g) IF (DEFINED ENC) ADD_DEFINITIONS(-DCPPJIEBA_${ENC}) ENDIF() diff --git a/conf/server.conf b/conf/server.conf index c0bc4c6..964a389 100644 --- a/conf/server.conf +++ b/conf/server.conf @@ -3,8 +3,6 @@ #socket listen port port=11200 -#number of thread -thread_num=4 #deamonize daemonize=true diff --git a/src/Husky/EpollServer.hpp b/src/Husky/EpollServer.hpp new file mode 100644 index 0000000..77a47ce --- /dev/null +++ b/src/Husky/EpollServer.hpp @@ -0,0 +1,281 @@ +#ifndef HUSKY_EPOLLSERVER_H +#define HUSKY_EPOLLSERVER_H + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "HttpReqInfo.hpp" + + + +namespace Husky +{ + using namespace Limonp; + + const char* const HTTP_FORMAT = "HTTP/1.1 200 OK\r\nConnection: close\r\nServer: HuskyServer/1.0.0\r\nContent-Type: text/json; charset=%s\r\nContent-Length: %d\r\n\r\n%s"; + const char* const CHARSET_UTF8 = "UTF-8"; + const char* const CLIENT_IP_K = "CLIENT_IP"; + + const struct linger LNG = {1, 1}; + const struct timeval SOCKET_TIMEOUT = {5, 0}; + + class IRequestHandler + { + public: + virtual ~IRequestHandler(){}; + public: + virtual bool do_GET(const HttpReqInfo& httpReq, string& res) const = 0; + }; + + class EpollServer + { + private: + static const size_t LISTEN_QUEUE_LEN = 1024; + static const size_t RECV_BUFFER_SIZE = 1024 * 8; + static const int MAXEPOLLSIZE = 512; + + private: + const IRequestHandler* _reqHandler; + int _host_socket; + int _epoll_fd; + bool _isShutDown; + int _epollSize; + unordered_map _sockIpMap; + private: + bool _isInited; + bool _getInitFlag() const {return _isInited;} + bool _setInitFlag(bool flag) {return _isInited = flag;} + public: + explicit EpollServer(uint port, const IRequestHandler* pHandler): _reqHandler(pHandler), _host_socket(-1), _isShutDown(false), _epollSize(0) + { + assert(_reqHandler); + _setInitFlag(_init_epoll(port)); + }; + ~EpollServer(){};// unfinished; + public: + operator bool() const + { + return _getInitFlag(); + } + public: + bool start() + { + //int clientSock; + sockaddr_in clientaddr; + socklen_t nSize = sizeof(clientaddr); + //char recvBuf[RECV_BUFFER_SIZE]; + struct epoll_event events[MAXEPOLLSIZE]; + int nfds, clientSock; + + while(!_isShutDown) + { + if(-1 == (nfds = epoll_wait(_epoll_fd, events, _epollSize, -1))) + { + LogFatal(strerror(errno)); + return false; + } + + //LogDebug("epoll_wait return event sum[%d]", nfds); + + for(int i = 0; i < nfds; i++) + { + if(events[i].data.fd == _host_socket) /*new connect coming.*/ + { + if(-1 == (clientSock = accept(_host_socket, (struct sockaddr*) &clientaddr, &nSize))) + { + LogError(strerror(errno)); + continue; + } + if(!_epoll_add(clientSock, EPOLLIN | EPOLLET)) + { + LogError("_epoll_add(%d, EPOLLIN | EPOLLET)", clientSock); + _closesocket(clientSock); + continue; + } + + //LogInfo("connecting from: %d:%d， client socket: %d\n", inet_ntoa(clientaddr.sin_addr), ntohs(clientaddr.sin_port), clientSock); + + /* inet_ntoa is not thread safety at some version */ + //_sockIpMap[clientSock] = inet_ntoa(clientaddr.sin_addr); + + } + else /*client socket data to be received*/ + { + _response(events[i].data.fd); + + /*close socket will case it to be removed from epoll automatically*/ + _closesocket(events[i].data.fd); + } + } + + } + return true; + } + void stop() + { + _isShutDown = true; + if(-1 == close(_host_socket)) + { + LogError(strerror(errno)); + return; + } + int sockfd; + struct sockaddr_in dest; + if((sockfd = socket(AF_INET, SOCK_STREAM, 0)) < 0) + { + LogError(strerror(errno)); + return; + } + bzero(&dest, sizeof(dest)); + dest.sin_family = AF_INET; + dest.sin_port = htons(_host_socket); + if(0 == inet_aton("127.0.0.1", (struct in_addr *) &dest.sin_addr.s_addr)) + { + LogError(strerror(errno)); + return; + } + if(connect(sockfd, (struct sockaddr *) &dest, sizeof(dest)) < 0) + { + LogError(strerror(errno)); + } + _closesocket(sockfd); + } + private: + bool _epoll_add(int sockfd, uint32_t events) + { + if (!_setNonBLock(sockfd)) + { + LogError(strerror(errno)); + return false; + } + struct epoll_event ev; + ev.data.fd = sockfd; + ev.events = events; + if(epoll_ctl(_epoll_fd, EPOLL_CTL_ADD, sockfd, &ev) < 0) + { + LogError("insert socket '%d' into epoll failed: %s", sockfd, strerror(errno)); + return false; + } + _epollSize ++; + return true; + } + bool _response(int sockfd) const + { + if(-1 == setsockopt(sockfd, SOL_SOCKET, SO_LINGER, (const char*)&LNG, sizeof(LNG))) + { + LogError(strerror(errno)); + return false; + } + if(-1 == setsockopt(sockfd, SOL_SOCKET, SO_RCVTIMEO, (const char*)&SOCKET_TIMEOUT, sizeof(SOCKET_TIMEOUT))) + { + LogError(strerror(errno)); + return false; + } + if(-1 == setsockopt(sockfd, SOL_SOCKET, SO_SNDTIMEO, (const char*)&SOCKET_TIMEOUT, sizeof(SOCKET_TIMEOUT))) + { + LogError(strerror(errno)); + return false; + } + + string strRec, strSnd, strRetByHandler; + strRec.resize(RECV_BUFFER_SIZE); + int nRetCode = recv(sockfd, (char*)strRec.c_str(), strRec.size(), 0); + if(-1 == nRetCode) + { + LogDebug(strerror(errno)); + return false; + } + if(0 == nRetCode) + { + LogDebug("client socket closed gracefully."); + return false; + } + + HttpReqInfo httpReq(strRec); + if(!_reqHandler->do_GET(httpReq, strRetByHandler)) + { + LogError("do_GET failed."); + return false; + } + string_format(strSnd, HTTP_FORMAT, CHARSET_UTF8, strRetByHandler.length(), strRetByHandler.c_str()); + if(-1 == send(sockfd, strSnd.c_str(), strSnd.length(), 0)) + { + LogError(strerror(errno)); + return false; + } + LogInfo("{response:%s, epollsize:%d}", strRetByHandler.c_str(), _epollSize); + return true; + } + bool _init_epoll(uint port) + { + _host_socket = socket(AF_INET, SOCK_STREAM, 0); + if(-1 == _host_socket) + { + LogError(strerror(errno)); + return false; + } + + int nRet = 1; + if(-1 == setsockopt(_host_socket, SOL_SOCKET, SO_REUSEADDR, (char*)&nRet, sizeof(nRet))) + { + LogError(strerror(errno)); + return false; + } + + struct sockaddr_in addrSock; + addrSock.sin_family = AF_INET; + addrSock.sin_port = htons(port); + addrSock.sin_addr.s_addr = htonl(INADDR_ANY); + if(-1 == ::bind(_host_socket, (sockaddr*)&addrSock, sizeof(sockaddr))) + { + LogError(strerror(errno)); + _closesocket(_host_socket); + return false; + } + if(-1 == listen(_host_socket, LISTEN_QUEUE_LEN)) + { + LogError(strerror(errno)); + return false; + } + + if(-1 == (_epoll_fd = epoll_create(MAXEPOLLSIZE))) + { + LogError(strerror(errno)); + return false; + } + if(!_epoll_add(_host_socket, EPOLLIN)) + { + LogError("_epoll_add(%d, EPOLLIN) failed.", _host_socket); + return false; + } + LogInfo("create socket listening port[%u], epoll{size:%d} init ok", port, _epollSize); + return true; + } + void _closesocket(int sockfd) + { + if(-1 == close(sockfd)) + { + LogError(strerror(errno)); + return; + } + _epollSize--; + } + static bool _setNonBLock(int sockfd) + { + return -1 != fcntl(sockfd, F_SETFL, fcntl(sockfd, F_GETFD, 0)|O_NONBLOCK); + } + }; +} +#endif diff --git a/src/Husky/HttpReqInfo.hpp b/src/Husky/HttpReqInfo.hpp index 2d52b73..7585a28 100644 --- a/src/Husky/HttpReqInfo.hpp +++ b/src/Husky/HttpReqInfo.hpp @@ -4,6 +4,7 @@ #include #include #include "Limonp/logger.hpp" +#include "Limonp/str_functs.hpp" namespace Husky { @@ -76,22 +77,22 @@ namespace Husky class HttpReqInfo { public: - bool load(const string& headerStr) + HttpReqInfo(const string& headerStr) { size_t lpos = 0, rpos = 0; vector buf; rpos = headerStr.find("\n", lpos); if(string::npos == rpos) { - LogFatal("headerStr illegal."); - return false; + LogError("headerStr illegal."); + return; } string firstline(headerStr, lpos, rpos - lpos); trim(firstline); if(!split(firstline, buf, " ") || 3 != buf.size()) { - LogFatal("parse header first line failed."); - return false; + LogError("parse header first line failed."); + return; } _headerMap[KEY_METHOD] = trim(buf[0]); _headerMap[KEY_PATH] = trim(buf[1]); @@ -103,12 +104,11 @@ namespace Husky _parseUrl(firstline, _methodGetMap); } - lpos = rpos + 1; if(lpos >= headerStr.size()) { - LogFatal("headerStr illegal"); - return false; + LogError("headerStr illegal"); + return; } //message header begin while(lpos < headerStr.size() && string::npos != (rpos = headerStr.find('\n', lpos)) && rpos > lpos) @@ -125,8 +125,8 @@ namespace Husky trim(v); if(k.empty()||v.empty()) { - LogFatal("headerStr illegal."); - return false; + LogError("headerStr illegal."); + return; } upper(k); _headerMap[k] = v; @@ -136,7 +136,6 @@ namespace Husky //body begin - return true; } public: string& operator[] (const string& key) @@ -156,15 +155,15 @@ namespace Husky return _find(_methodPostMap, argKey, res); } private: - HashMap _headerMap; - HashMap _methodGetMap; - HashMap _methodPostMap; + std::unordered_map _headerMap; + std::unordered_map _methodGetMap; + std::unordered_map _methodPostMap; //public: friend ostream& operator<<(ostream& os, const HttpReqInfo& obj); private: - bool _find(const HashMap& mp, const string& key, string& res)const + bool _find(const std::unordered_map& mp, const string& key, string& res)const { - HashMap::const_iterator it = mp.find(key); + std::unordered_map::const_iterator it = mp.find(key); if(it == mp.end()) { return false; @@ -173,7 +172,7 @@ namespace Husky return true; } private: - bool _parseUrl(const string& url, HashMap& mp) + bool _parseUrl(const string& url, std::unordered_map& mp) { if(url.empty()) { diff --git a/src/Husky/HuskyServer.hpp b/src/Husky/HuskyServer.hpp deleted file mode 100644 index 9df6f81..0000000 --- a/src/Husky/HuskyServer.hpp +++ /dev/null @@ -1,300 +0,0 @@ -#ifndef HUSKY_SERVERFRAME_H -#define HUSKY_SERVERFRAME_H - -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "ThreadManager.hpp" -#include "HttpReqInfo.hpp" - -#define INVALID_SOCKET -1 -#define SOCKET_ERROR -1 -#define closesocket close -#define RECV_BUFFER 10240 -#define LISEN_QUEUR_LEN 1024 - - -namespace Husky -{ - using namespace Limonp; - typedef int SOCKET; - const struct timeval SOCKET_TIMEOUT = {2, 0}; - const char* const RESPONSE_FORMAT = "HTTP/1.1 200 OK\r\nConnection: close\r\nServer: HuskyServer/1.0.0\r\nContent-Type: text/json; charset=%s\r\nContent-Length: %d\r\n\r\n"; - const char* const RESPONSE_CHARSET_UTF8 = "UTF-8"; - const char* const RESPONSE_CHARSET_GB2312 = "GB2312"; - const char* const CLIENT_IP_K = "CLIENT_IP"; - typedef unsigned short u_short; - typedef unsigned int u_int; - - class IRequestHandler - { - public: - virtual ~IRequestHandler(){}; - public: - virtual bool do_GET(const HttpReqInfo& httpReq, string& res) = 0; - - }; - - struct SPara - { - SOCKET hSock; - IRequestHandler * pHandler; - pthread_mutex_t * ppmAccept; - bool * pShutdown; - }; - - class HuskyServer - { - private: - pthread_mutex_t m_pmAccept; - bool m_bShutdown; - public: - explicit HuskyServer(unsigned nPort, unsigned nThreadCount, IRequestHandler* pHandler) - { - m_bShutdown = false; - m_nLsnPort = nPort; - m_nThreadCount = nThreadCount; - m_pHandler = pHandler; - assert(pHandler); - pthread_mutex_init(&m_pmAccept,NULL); - }; - virtual ~HuskyServer(){pthread_mutex_destroy(&m_pmAccept);}; - virtual bool init() - { - - if (!BindToLocalHost(m_lsnSock,m_nLsnPort)) - { - LogFatal("BindToLocalHost failed."); - return false; - } - LogInfo("init ok {port:%d, threadNum:%d}", m_nLsnPort, m_nThreadCount); - - return true; - } - virtual bool dispose() - { - m_bShutdown=true; - if (SOCKET_ERROR==closesocket(m_lsnSock)) - { - LogError("error [%s]", strerror(errno)); - return false; - } - - int sockfd; - struct sockaddr_in dest; - - if ((sockfd = socket(AF_INET, SOCK_STREAM, 0)) < 0) - { - LogError("error [%s]", strerror(errno)); - return false; - } - - bzero(&dest, sizeof(dest)); - dest.sin_family = AF_INET; - dest.sin_port = htons(m_nLsnPort); - if (inet_aton("127.0.0.1", (struct in_addr *) &dest.sin_addr.s_addr) == 0) - { - LogError("error [%s]", strerror(errno)); - return false; - } - - if (connect(sockfd, (struct sockaddr *) &dest, sizeof(dest)) < 0) - { - LogError("error [%s]", strerror(errno)); - } - close(sockfd); - return true; - } - virtual bool run() - { - if(SOCKET_ERROR==listen(m_lsnSock,LISEN_QUEUR_LEN)) - { - LogError("error [%s]", strerror(errno)); - return false; - } - ThreadManager thrMngr; - int i; - SPara para; - para.hSock=m_lsnSock; - para.pHandler=m_pHandler; - para.ppmAccept=&m_pmAccept; - para.pShutdown=&m_bShutdown; - for (i=0;ihSock; - IRequestHandler *pHandler=pPara->pHandler; - bool* pShutdown = pPara->pShutdown; - pthread_mutex_t* ppmAccept = pPara->ppmAccept; - - int nRetCode; - linger lng; - char chRecvBuf[RECV_BUFFER]; - - SOCKET hClientSock; - string strHttpResp; - - sockaddr_in clientaddr; - socklen_t nSize = sizeof(clientaddr); - while(!(*pShutdown)) - { - HttpReqInfo httpReq; - pthread_mutex_lock(ppmAccept); - hClientSock=accept(hSockLsn,(sockaddr *)&clientaddr, &nSize); - pthread_mutex_unlock(ppmAccept); - - if(hClientSock==SOCKET_ERROR) - { - if(!(*pShutdown)) - LogError("error [%s]", strerror(errno)); - continue; - } - - httpReq[CLIENT_IP_K] = inet_ntoa(clientaddr.sin_addr);// inet_ntoa is not thread safety at some version - - lng.l_linger=1; - lng.l_onoff=1; - if(SOCKET_ERROR==setsockopt(hClientSock,SOL_SOCKET,SO_LINGER,(char*)&lng,sizeof(lng))) - { - LogError("error [%s]", strerror(errno)); - } - - if(SOCKET_ERROR==setsockopt(hClientSock,SOL_SOCKET,SO_RCVTIMEO,(char*)&SOCKET_TIMEOUT,sizeof(SOCKET_TIMEOUT))) - { - LogError("error [%s]", strerror(errno)); - } - - if(SOCKET_ERROR==setsockopt(hClientSock,SOL_SOCKET,SO_SNDTIMEO,(char*)&SOCKET_TIMEOUT,sizeof(SOCKET_TIMEOUT))) - { - LogError("error [%s]", strerror(errno)); - } - - - string strRec; - string strSnd; - memset(chRecvBuf,0,sizeof(chRecvBuf)); - nRetCode = recv(hClientSock, chRecvBuf, RECV_BUFFER, 0); - strRec = chRecvBuf; - -#ifdef HUKSY_DEBUG - LogDebug("request[%s]", strRec.c_str()); -#endif - - if(SOCKET_ERROR==nRetCode) - { - LogDebug("error [%s]", strerror(errno)); - closesocket(hClientSock); - continue; - } - if(0==nRetCode) - { - LogDebug("connection has been gracefully closed"); - closesocket(hClientSock); - continue; - } - httpReq.load(strRec); - - pHandler->do_GET(httpReq, strSnd); - - char chHttpHeader[2048]; - - sprintf(chHttpHeader, RESPONSE_FORMAT, RESPONSE_CHARSET_UTF8, int(strSnd.length())); - - strHttpResp=chHttpHeader; - strHttpResp+=strSnd; -#ifdef HUKSY_DEBUG - LogDebug("response'body [%s]", strSnd.c_str()); -#endif - - if (SOCKET_ERROR==send(hClientSock,strHttpResp.c_str(),strHttpResp.length(),0)) - { - LogError("error [%s]", strerror(errno)); - } - - - closesocket(hClientSock); - } - - return 0; - - } - - private: - u_short m_nLsnPort; - u_short m_nThreadCount; - SOCKET m_lsnSock; - IRequestHandler *m_pHandler; - - }; - - -} -#endif diff --git a/src/Husky/ThreadManager.hpp b/src/Husky/ThreadManager.hpp deleted file mode 100644 index e9bf5f7..0000000 --- a/src/Husky/ThreadManager.hpp +++ /dev/null @@ -1,87 +0,0 @@ -#ifndef HUSKY_THREAD_MANAGER_H -#define HUSKY_THREAD_MANAGER_H -#include -#include -#include -#include - -namespace Husky -{ - using namespace std; - - class ThreadManager - { - private: - typedef pthread_t HANDLE; - typedef void *(* PThreadFunc)(void* param); - public: - ThreadManager(){;} - ~ThreadManager(){} - - size_t HandleCount(){return _handles.size();} - - void clear() - { - _handles.clear(); - } - - int CreateThread( PThreadFunc pFunc,void *pPara) - { - pthread_t pt; - int nErrorCode = pthread_create(&pt,NULL,pFunc,pPara); - if(nErrorCode != 0) - return nErrorCode; - _handles.push_back(pt); - return nErrorCode; - - } - - int Wait(HANDLE hThread = 0) - { - if( hThread == 0)//the last handle - { - if(!_handles.empty()) - { - return pthread_join(_handles.back(),NULL); - } - else - return -1; - } - else - { - if (find(_handles.begin(),_handles.end(),hThread) == _handles.end()) - { - return -1; - } - - return pthread_join(hThread, NULL); - } - - } - - int WaitMultipleThread() - { - if (_handles.empty()) - return -1; - int nErrorcode; - for (uint i = 0; i < _handles.size(); i++) - { - nErrorcode = pthread_join(_handles[i], NULL); - if (nErrorcode != 0) - return nErrorcode; - } - return 0; - } - - private: - vector _handles; - - private: - ThreadManager(const ThreadManager&){;}// copy forbidden - void operator = (const ThreadManager &){}// copy forbidden - }; -} - - - -#endif diff --git a/src/KeywordExtractor.hpp b/src/KeywordExtractor.hpp index 4bb1fee..cfb4645 100644 --- a/src/KeywordExtractor.hpp +++ b/src/KeywordExtractor.hpp @@ -3,7 +3,8 @@ #include "MixSegment.hpp" #include -#include +#include + #define MIN(X,Y) ((X) < (Y) ? (X) : (Y)) namespace CppJieba diff --git a/src/Limonp/ArgvContext.hpp b/src/Limonp/ArgvContext.hpp index dba3997..371af7b 100644 --- a/src/Limonp/ArgvContext.hpp +++ b/src/Limonp/ArgvContext.hpp @@ -9,7 +9,6 @@ #include #include #include "str_functs.hpp" -#include "map_functs.hpp" namespace Limonp { diff --git a/src/Limonp/logger.hpp b/src/Limonp/logger.hpp index aa63199..1da9519 100644 --- a/src/Limonp/logger.hpp +++ b/src/Limonp/logger.hpp @@ -26,7 +26,7 @@ namespace Limonp { using namespace std; - enum {LL_DEBUG = 0, LL_INFO = 1, LL_WARN = 2, LL_ERROR = 3, LL_FATAL = 4, LEVEL_ARRAY_SIZE = 5, CSTR_BUFFER_SIZE = 1024}; + enum {LL_DEBUG = 0, LL_INFO = 1, LL_WARN = 2, LL_ERROR = 3, LL_FATAL = 4, LEVEL_ARRAY_SIZE = 5, CSTR_BUFFER_SIZE = 32}; static const char * LOG_LEVEL_ARRAY[LEVEL_ARRAY_SIZE]= {"DEBUG","INFO","WARN","ERROR","FATAL"}; static const char * LOG_FORMAT = "%s %s:%d %s %s\n"; static const char * LOG_TIME_FORMAT = "%Y-%m-%d %H:%M:%S"; @@ -34,24 +34,19 @@ namespace Limonp class Logger { public: - static bool Logging(size_t level, const string& msg, const char* fileName, int lineNo) + static void Logging(size_t level, const string& msg, const char* fileName, int lineno) { assert(level <= LL_FATAL); char buf[CSTR_BUFFER_SIZE]; time_t timeNow; time(&timeNow); - if(!strftime(buf, sizeof(buf), LOG_TIME_FORMAT, localtime(&timeNow))) - { - fprintf(stderr, "stftime failed.\n"); - return false; - } - fprintf(stderr, LOG_FORMAT, buf, fileName, lineNo,LOG_LEVEL_ARRAY[level], msg.c_str()); - return true; + strftime(buf, sizeof(buf), LOG_TIME_FORMAT, localtime(&timeNow)); + fprintf(stderr, LOG_FORMAT, buf, fileName, lineno,LOG_LEVEL_ARRAY[level], msg.c_str()); } - static bool LoggingF(size_t level, const char* fileName, int lineNo, const string& fmt, ...) + static void LoggingF(size_t level, const char* fileName, int lineno, const string& fmt, ...) { #ifdef LOGGER_LEVEL - if(level < LOGGER_LEVEL) return true; + if(level < LOGGER_LEVEL) return; #endif int size = 256; string msg; @@ -70,7 +65,7 @@ namespace Limonp else size *= 2; } - return Logging(level, msg, fileName, lineNo); + Logging(level, msg, fileName, lineno); } }; } diff --git a/src/Limonp/map_functs.hpp b/src/Limonp/map_functs.hpp deleted file mode 100644 index d42c0c0..0000000 --- a/src/Limonp/map_functs.hpp +++ /dev/null @@ -1,46 +0,0 @@ -/************************************ - * file enc : ascii - * author : wuyanyi09@gmail.com - ************************************/ - - -#ifndef LIMONP_MAP_FUNCTS_H -#define LIMONP_MAP_FUNCTS_H - -#include -#include -#include -#include -#include - -#include -#define HashMap std::unordered_map - -namespace Limonp -{ - using namespace std; - - template - vT getMap(const map& mp, const kT & key, const vT & defaultVal) - { - typename map::const_iterator it; - it = mp.find(key); - if(mp.end() == it) - { - return defaultVal; - } - return it->second; - } - - template - void map2Vec(const map& mp, vector > & res) - { - typename map::const_iterator it = mp.begin(); - for(; it != mp.end(); it++) - { - res.push_back(*it); - } - } -} - -#endif diff --git a/src/Limonp/std_outbound.hpp b/src/Limonp/std_outbound.hpp index efb6e15..5131133 100644 --- a/src/Limonp/std_outbound.hpp +++ b/src/Limonp/std_outbound.hpp @@ -1,15 +1,29 @@ #ifndef LIMONP_STD_OUTBOUND_H #define LIMONP_STD_OUTBOUND_H -#include "map_functs.hpp" #include + +#if(__cplusplus == 201103L) +#include +#include +#else +#include +#include +namespace std +{ + using std::tr1::unordered_map; + using std::tr1::unordered_set; +} + +#endif + #include #include #include +#include namespace std { - using namespace std; template ostream& operator << (ostream& os, const vector& vec) { @@ -18,7 +32,7 @@ namespace std return os << "[]"; } os<<"[\""< - ostream& operator << (ostream& os, const HashMap& mp) + ostream& operator << (ostream& os, const std::unordered_map& mp) { if(mp.empty()) { return os << "{}"; } os<<'{'; - typename HashMap::const_iterator it = mp.begin(); + typename std::unordered_map::const_iterator it = mp.begin(); os<<*it; it++; while(it != mp.end()) diff --git a/src/Limonp/str_functs.hpp b/src/Limonp/str_functs.hpp index 12b57d7..221b8d0 100644 --- a/src/Limonp/str_functs.hpp +++ b/src/Limonp/str_functs.hpp @@ -22,7 +22,6 @@ #include #include #include "std_outbound.hpp" -#include "map_functs.hpp" #define print(x) cout<< #x": " << x < #include "Limonp/Config.hpp" #include "Limonp/io_functs.hpp" -#include "Husky/HuskyServer.hpp" +#include "Husky/EpollServer.hpp" #include "MPSegment.hpp" #include "HMMSegment.hpp" #include "MixSegment.hpp" @@ -19,7 +19,7 @@ class ReqHandler: public IRequestHandler ReqHandler(const string& dictPath, const string& modelPath): _segment(dictPath, modelPath){}; virtual ~ReqHandler(){}; public: - virtual bool do_GET(const HttpReqInfo& httpReq, string& strSnd) + virtual bool do_GET(const HttpReqInfo& httpReq, string& strSnd) const { string sentence, tmp; vector words; @@ -45,7 +45,6 @@ bool run(int argc, char** argv) return false; } unsigned int port = 0; - unsigned int threadNum = 0; string dictPath; string modelPath; string val; @@ -55,12 +54,6 @@ bool run(int argc, char** argv) return false; } port = atoi(val.c_str()); - if(!conf.get("thread_num", val)) - { - LogFatal("conf get thread_num failed."); - return false; - } - threadNum = atoi(val.c_str()); if(!conf.get("dict_path", dictPath)) { @@ -92,8 +85,8 @@ bool run(int argc, char** argv) } ReqHandler reqHandler(dictPath, modelPath); - HuskyServer sf(port, threadNum, &reqHandler); - return sf.init() && sf.run(); + EpollServer sf(port, &reqHandler); + return sf.start(); } int main(int argc, char* argv[]) From abe4be255fbeac8f25cc1e69b302555a2258208c Mon Sep 17 00:00:00 2001 From: wyy Date: Thu, 27 Mar 2014 01:41:05 -0700 Subject: [PATCH 4/4] modify some stuff to adapter lower version cmake & g++ --- ChangeLog.md | 2 +- README.md | 4 ++-- test/CMakeLists.txt | 2 +- test/unittest/CMakeLists.txt | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ChangeLog.md b/ChangeLog.md index 2d9ce3f..4263582 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,6 +1,6 @@ ## v2.3.5 is coming -1. 适配低级版本的`g++`，已在`g++ 4.4.7`上测试通过。 +1. 适配更低级版本的`g++`和`cmake`，已在`g++ 4.1.2`和`cmake 2.6`上测试通过。 2. 修改一些测试用例的文件，减少测试时编译的时间。 ## v2.3.4 diff --git a/README.md b/README.md index f99b4e4..7fe6dfc 100644 --- a/README.md +++ b/README.md @@ -16,8 +16,8 @@ ### 依赖 -* g++ (version >= 4.4.7 recommended); -* cmake (version >= 2.8 recommended); +* g++ (version >= 4.1 recommended); +* cmake (version >= 2.6 recommended); ### 下载和安装 diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 98ee947..5a8aaed 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -1,4 +1,4 @@ -SET(EXECUTABLE_OUTPUT_PATH ${PROJECT_BINARY_DIR}/test) +SET(EXECUTABLE_OUTPUT_PATH ${PROJECT_BINARY_DIR}) ADD_EXECUTABLE(segment.demo segment.cpp) ADD_EXECUTABLE(keyword.demo keyword_demo.cpp) diff --git a/test/unittest/CMakeLists.txt b/test/unittest/CMakeLists.txt index 79457cd..ebf0322 100644 --- a/test/unittest/CMakeLists.txt +++ b/test/unittest/CMakeLists.txt @@ -1,5 +1,5 @@ -SET(EXECUTABLE_OUTPUT_PATH ${PROJECT_BINARY_DIR}/test) -SET(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/test/lib) +SET(EXECUTABLE_OUTPUT_PATH ${PROJECT_BINARY_DIR}) +SET(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/lib) SET(GTEST_ROOT_DIR gtest-1.6.0)