Merge pull request #7 from aholic/master

add QuerySegment

src/FullSegment.hpp

@@ -1,5 +1,5 @@
 #ifndef CPPJIEBA_FULLSEGMENT_H
-#define CPPJIEBAi_FULLSEGMENT_H
+#define CPPJIEBA_FULLSEGMENT_H
 
 #include <algorithm>
 #include <set>
@@ -23,11 +23,13 @@ namespace CppJieba
     public:
         bool init()
         {
+#ifndef NO_CODING_LOG
             if(_getInitFlag())
             {
                 LogError("already inited before now.");
                 return false;
             }
+#endif
             if(!_trie.init())
             {
                 LogError("_trie.init failed.");
@@ -44,22 +46,35 @@ namespace CppJieba
         }
         bool dispose()
         {
+#ifndef NO_CODING_LOG
             if(!_getInitFlag())
             {
                 return true;
             }
+#endif
             _trie.dispose();
             _setInitFlag(false);
             return true;
         }
 
-        bool cut(const string& str, vector<string>& res) const 
-        {
-            return SegmentBase::cut(str, res);
-        }
+    public:
+        using SegmentBase::cut;
 
-        bool cut(Unicode::const_iterator begin, Unicode::const_iterator end, vector<string>& res) const
+    public:
+        bool cut(Unicode::const_iterator begin, Unicode::const_iterator end, vector<Unicode>& res) const
         {
+#ifndef NO_CODING_LOG
+            if (!_getInitFlag())
+            {
+                LogError("not inited.");
+                return false;
+            }
+            if (begin > end)
+            {
+                LogError("begin > end");
+                return false;
+            }
+#endif
             //resut of searching in trie tree
             vector<pair<uint, const TrieNodeInfo*> > tRes;
 
@@ -71,10 +86,9 @@ namespace CppJieba
 
             //tmp variables
             int wordLen = 0;
-            string tmp;
             for (Unicode::const_iterator uItr = begin; uItr != end; uItr++)
             {
-                //find word start from itr
+                //find word start from uItr
                 if (_trie.find(uItr, end, tRes))
                 {
                     for (vector<pair<uint, const TrieNodeInfo*> >::const_iterator itr = tRes.begin(); itr != tRes.end(); itr++)
@@ -82,32 +96,64 @@ namespace CppJieba
                         wordLen = itr->second->word.size();
                         if (wordLen >= 2 || tRes.size() == 1 && maxIdx <= uIdx)
                         {
-                            if (TransCode::encode(itr->second->word, tmp))
-                                res.push_back(tmp);
-                            else
-                                LogError("encode failed.");
-                            tmp.clear();
+                            res.push_back(itr->second->word);
                         }
                         maxIdx = uIdx+wordLen > maxIdx ? uIdx+wordLen : maxIdx;
                     }
                     tRes.clear();
                 }
-                else // not found word start from itr
+                else // not found word start from uItr
                 {
                     if (maxIdx <= uIdx) // never exist in prev results
                     {
                         //put itr itself in res
-                        Unicode uTmp(1, *uItr);
-                        if (TransCode::encode(uTmp, tmp))
-                        {
-                            res.push_back(tmp);
-                        }
-                        tmp.clear();
+                        res.push_back(Unicode(1, *uItr));
+
+                        //mark it exits
                         ++maxIdx;
                     }
                 }
                 ++uIdx;
             }
+
+            return true;
+        }
+
+        bool cut(Unicode::const_iterator begin, Unicode::const_iterator end, vector<string>& res) const
+        {
+#ifndef NO_CODING_LOG
+            if (!_getInitFlag())
+            {
+                LogError("not inited.");
+                return false;
+            }
+            if (begin > end)
+            {
+                LogError("begin > end");
+                return false;
+            }
+#endif
+            vector<Unicode> uRes;
+            if (!cut(begin, end, uRes))
+            {
+                LogError("get unicode cut result error.");
+                return false;
+            }
+            string tmp;
+
+            for (vector<Unicode>::const_iterator uItr = uRes.begin(); uItr != uRes.end(); uItr++)
+            {
+                if (TransCode::encode(*uItr, tmp))
+                {
+                    res.push_back(tmp);
+                }
+                else
+                {
+                    LogError("encode failed.");
+                }
+            }
+
+            return true;
         }
     };
 }

src/QuerySegment.hpp

@@ -0,0 +1,151 @@
+#ifndef CPPJIEBA_QUERYSEGMENT_H
+#define CPPJIEBA_QUERYSEGMENT_H
+
+#include <algorithm>
+#include <set>
+#include "Limonp/logger.hpp"
+#include "Trie.hpp"
+#include "ISegment.hpp"
+#include "SegmentBase.hpp"
+#include "HMMSegment.hpp"
+#include "FullSegment.hpp"
+#include "TransCode.hpp"
+
+namespace CppJieba
+{
+    class QuerySegment: public SegmentBase
+    {
+    private:
+        HMMSegment _hmmSeg;
+        FullSegment _fullSeg;
+        int _maxWordLen;
+
+    public:
+        QuerySegment(const char* fullSegDict, const char* hmmSegDict, int maxWordLen): _hmmSeg(hmmSegDict), _fullSeg(fullSegDict), _maxWordLen(maxWordLen){};
+        virtual ~QuerySegment(){dispose();};
+    public:
+        bool init()
+        {
+#ifndef NO_CODING_LOG
+            if (_getInitFlag())
+            {
+                LogError("inited.");
+            }
+#endif
+            if (!_hmmSeg.init())
+            {
+                LogError("_hmmSeg init");
+                return false;
+            }
+            if (!_fullSeg.init())
+            {
+                LogError("_fullSeg init");
+                return false;
+            }
+            return _setInitFlag(true);
+        }
+        bool dispose()
+        {
+#ifndef NO_CODING_LOG
+            if(!_getInitFlag())
+            {
+                return true;
+            }
+#endif
+            _fullSeg.dispose();
+            _hmmSeg.dispose();
+            _setInitFlag(false);
+            return true;
+        }
+
+    public:
+        using SegmentBase::cut;
+
+    public:
+        bool cut(Unicode::const_iterator begin, Unicode::const_iterator end, vector<Unicode>& res) const
+        {
+#ifndef NO_CODING_LOG
+            if (!_getInitFlag())
+            {
+                LogError("not inited.");
+                return false;
+            }
+            if (begin > end)
+            {
+                LogError("begin > end");
+                return false;
+            }
+#endif
+            //use hmm cut first
+            vector<Unicode> hmmRes;
+            if (!_hmmSeg.cut(begin, end, hmmRes))
+            {
+                LogError("_hmmSeg cut failed.");
+                return false;
+            }
+
+            vector<Unicode> fullRes;
+            for (vector<Unicode>::const_iterator hmmResItr = hmmRes.begin(); hmmResItr != hmmRes.end(); hmmResItr++)
+            {
+                
+                // if it's too long, cut with _fullSeg, put fullRes in res
+                if (hmmResItr->size() > _maxWordLen)
+                {
+                    if (_fullSeg.cut(hmmResItr->begin(), hmmResItr->end(), fullRes))
+                    {
+                       for (vector<Unicode>::const_iterator fullResItr = fullRes.begin(); fullResItr != fullRes.end(); fullResItr++)
+                       {
+                           res.push_back(*fullResItr);
+                       }
+                    }
+                }
+                else // just use the hmm result
+                {
+                    res.push_back(*hmmResItr);
+                }
+            }
+
+            return true;
+        }
+
+
+        bool cut(Unicode::const_iterator begin, Unicode::const_iterator end, vector<string>& res) const
+        {
+#ifndef NO_CODING_LOG
+            if (!_getInitFlag())
+            {
+                LogError("not inited.");
+                return false;
+            }
+            if (begin > end)
+            {
+                LogError("begin > end");
+                return false;
+            }
+#endif
+            vector<Unicode> uRes;
+            if (!cut(begin, end, uRes))
+            {
+                LogError("get unicode cut result error.");
+                return false;
+            }
+
+            string tmp;
+            for (vector<Unicode>::const_iterator uItr = uRes.begin(); uItr != uRes.end(); uItr++)
+            {
+                if (TransCode::encode(*uItr, tmp))
+                {
+                    res.push_back(tmp);
+                }
+                else
+                {
+                    LogError("encode failed.");
+                }
+            }
+
+            return true;
+        }
+    };
+}
+
+#endif

src/segment.cpp

@@ -1,10 +1,13 @@
 #include <iostream>
 #include <fstream>
+#include <cstdlib>
+#include <cstdio>
 #include "Limonp/ArgvContext.hpp"
 #include "MPSegment.hpp"
 #include "HMMSegment.hpp"
 #include "MixSegment.hpp"
 #include "FullSegment.hpp"
+#include "QuerySegment.hpp"
 
 using namespace CppJieba;
 
@@ -37,13 +40,15 @@ int main(int argc, char ** argv)
     {
         cout<<"usage: \n\t"<<argv[0]<<" [options] <filename>\n"
             <<"options:\n"
-            <<"\t--algorithm\tSupported methods are [cutDAG, cutHMM, cutFull, cutMix] for now. \n\t\t\tIf not specified, the default is cutMix\n"
+            <<"\t--algorithm\tSupported methods are [cutDAG, cutHMM, cutFull, cutQuery, cutMix] for now. \n\t\t\tIf not specified, the default is cutMix\n"
             <<"\t--dictpath\tsee example\n"
             <<"\t--modelpath\tsee example\n"
+            <<"\t--maxlen\tspecify the granularity of cut used in cutQuery, If not specified, the default is 3\n"
             <<"example:\n"
             <<"\t"<<argv[0]<<" ../test/testlines.utf8 --dictpath ../dicts/jieba.dict.utf8 --algorithm cutDAG\n"
             <<"\t"<<argv[0]<<" ../test/testlines.utf8 --modelpath ../dicts/hmm_model.utf8 --algorithm cutHMM\n"
             <<"\t"<<argv[0]<<" ../test/testlines.utf8 --dictpath ../dicts/jieba.dict.utf8 --modelpath ../dicts/hmm_model.utf8 --algorithm cutMix\n"
+            <<"\t"<<argv[0]<<" ../test/testlines.utf8 --dictpath ../dicts/jieba.dict.utf8 --modelpath ../dicts/hmm_model.utf8 --algorithm cutQuery --maxlen 3\n"
             <<endl;
         
         return EXIT_FAILURE;
@@ -52,6 +57,7 @@ int main(int argc, char ** argv)
     string dictPath = arg["--dictpath"];
     string modelPath = arg["--modelpath"];
     string algorithm = arg["--algorithm"];
+    int maxLen = atoi(arg["--maxlen"] == "" ? arg["--maxlen"].c_str() : "3");
 
     if("cutHMM" == algorithm)
     {
@@ -86,6 +92,17 @@ int main(int argc, char ** argv)
         cut(&seg, arg[1].c_str());
         seg.dispose();
     }
+    else if ("cutQuery" == algorithm)
+    {
+        QuerySegment seg(dictPath.c_str(), modelPath.c_str(), maxLen);
+        if (!seg.init())
+        {
+            cout << "seg init failed" << endl;
+            return false;
+        }
+        cut(&seg, arg[1].c_str());
+        seg.dispose();
+    }
     else 
     {
         MixSegment seg(dictPath.c_str(), modelPath.c_str());