third_codes/cppjieba
1
0
Fork 0
mirror of https://github.com/yanyiwu/cppjieba.git synced 2025-07-18 00:00:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

using idf.utf8 in keywordExtractor

Browse Source
This commit is contained in:
wyy 2013-12-24 06:55:27 -08:00
parent 9229fec6ca
commit 62b83a36a0
3 changed files with 258843 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

258826
dict/idf.utf8 Normal file
View File

File diff suppressed because it is too large Load Diff

49
src/KeywordExtractor.hpp
View File

@ -11,13 +11,12 @@ namespace CppJieba
struct KeyWordInfo struct KeyWordInfo
{ {
string word; string word;
uint freq;
double idf; double idf;
}; };
inline ostream& operator << (ostream& os, const KeyWordInfo & keyword) inline ostream& operator << (ostream& os, const KeyWordInfo & keyword)
{ {
return os << keyword.word << "," << keyword.freq << "," << keyword.idf; return os << keyword.word << "," << keyword.idf;
} }
class KeywordExtractor class KeywordExtractor
@ -25,9 +24,7 @@ namespace CppJieba
private: private:
MPSegment _segment; MPSegment _segment;
private: private:
unordered_map<string, const KeyWordInfo* > _wordIndex; unordered_map<string, double> _idfMap;
vector<KeyWordInfo> _wordinfos;
size_t _totalFreq;
protected: protected:
bool _isInited; bool _isInited;
bool _getInitFlag()const{return _isInited;}; bool _getInitFlag()const{return _isInited;};
@ -36,22 +33,19 @@ namespace CppJieba
operator bool(){return _getInitFlag();}; operator bool(){return _getInitFlag();};
public: public:
KeywordExtractor(){_setInitFlag(false);}; KeywordExtractor(){_setInitFlag(false);};
explicit KeywordExtractor(const string& dictPath){_setInitFlag(init(dictPath));}; explicit KeywordExtractor(const string& dictPath, const string& idfPath){_setInitFlag(init(dictPath, idfPath));};
~KeywordExtractor(){}; ~KeywordExtractor(){};
public: public:
bool init(const string& dictPath) bool init(const string& dictPath, const string& idfPath)
{ {
ifstream ifs(dictPath.c_str()); ifstream ifs(idfPath.c_str());
if(!ifs) if(!ifs)
{ {
LogError("open %s failed.", dictPath.c_str()); LogError("open %s failed.", idfPath.c_str());
return false; return false;
} }
_totalFreq = 0;
int tfreq;
string line ; string line ;
vector<string> buf; vector<string> buf;
KeyWordInfo keywordInfo;
for(uint lineno = 0; getline(ifs, line); lineno++) for(uint lineno = 0; getline(ifs, line); lineno++)
{ {
buf.clear(); buf.clear();
@ -60,33 +54,12 @@ namespace CppJieba
LogError("line[%d] empty. skipped.", lineno); LogError("line[%d] empty. skipped.", lineno);
continue; continue;
} }
if(!split(line, buf, " ") || buf.size() != 3) if(!split(line, buf, " ") || buf.size() != 2)
{ {
LogError("line %d [%s] illegal. skipped.", lineno, line.c_str()); LogError("line %d [%s] illegal. skipped.", lineno, line.c_str());
continue; continue;
} }
keywordInfo.word = buf[0]; _idfMap[buf[0]] = atof(buf[1].c_str());
tfreq= atoi(buf[1].c_str());
if(tfreq <= 0)
{
LogError("line %d [%s] illegal. skipped.", lineno, line.c_str());
continue;
}
keywordInfo.freq = tfreq;
_totalFreq += tfreq;
_wordinfos.push_back(keywordInfo);
}
// calculate idf & make index.
for(uint i = 0; i < _wordinfos.size(); i++)
{
if(_wordinfos[i].freq <= 0)
{
LogFatal("freq value is not positive.");
return false;
}
_wordinfos[i].idf = -log(_wordinfos[i].freq);
_wordIndex[_wordinfos[i].word] = &(_wordinfos[i]);
} }
return _setInitFlag(_segment.init(dictPath)); return _setInitFlag(_segment.init(dictPath));
}; };
@ -110,10 +83,10 @@ namespace CppJieba
for(unordered_map<string, double>::iterator itr = wordmap.begin(); itr != wordmap.end();) for(unordered_map<string, double>::iterator itr = wordmap.begin(); itr != wordmap.end();)
{ {
unordered_map<string, const KeyWordInfo*>::const_iterator cit = _wordIndex.find(itr->first); unordered_map<string, double>::const_iterator cit = _idfMap.find(itr->first);
if(cit != _wordIndex.end()) if(cit != _idfMap.end())
{ {
itr->second *= cit->second->idf; itr->second *= cit->second;
itr ++; itr ++;
} }
else else

13
test/unittest/TKeywordExtractor.cpp
View File

@ -5,7 +5,7 @@ using namespace CppJieba;
TEST(KeywordExtractorTest, Test1) TEST(KeywordExtractorTest, Test1)
{ {
KeywordExtractor extractor("../dict/jieba.dict.utf8"); KeywordExtractor extractor("../dict/jieba.dict.utf8", "../dict/idf.utf8");
const char* str = "我来自北京邮电大学。。。 学号 123456"; const char* str = "我来自北京邮电大学。。。 学号 123456";
const char* res[] = {"北京邮电大学", "来自"}; const char* res[] = {"北京邮电大学", "来自"};
vector<string> words; vector<string> words;
@ -16,9 +16,9 @@ TEST(KeywordExtractorTest, Test1)
TEST(KeywordExtractorTest, Test2) TEST(KeywordExtractorTest, Test2)
{ {
KeywordExtractor extractor("../dict/jieba.dict.utf8"); KeywordExtractor extractor("../dict/jieba.dict.utf8", "../dict/idf.utf8");
const char* str = "我来自北京邮电大学。。。 学号 123456"; const char* str = "我来自北京邮电大学。。。 学号 123456";
const char* res[] = {"北京邮电大学", "来自", "学", "号", "我"}; const char* res[] = {"北京邮电大学", "来自"};
vector<string> words; vector<string> words;
ASSERT_TRUE(extractor); ASSERT_TRUE(extractor);
ASSERT_TRUE(extractor.extract(str, words, 9)); ASSERT_TRUE(extractor.extract(str, words, 9));
@ -31,10 +31,9 @@ TEST(KeywordExtractorTest, Test3)
ifstream ifs("../test/testdata/weicheng.utf8"); ifstream ifs("../test/testdata/weicheng.utf8");
ASSERT_TRUE(ifs); ASSERT_TRUE(ifs);
string str((istreambuf_iterator<char>(ifs)), (istreambuf_iterator<char>())); string str((istreambuf_iterator<char>(ifs)), (istreambuf_iterator<char>()));
KeywordExtractor extractor("../dict/jieba.dict.utf8"); KeywordExtractor extractor("../dict/jieba.dict.utf8", "../dict/idf.utf8");
vector<string> keywords; vector<string> keywords;
string res; const char* res[] = {"小姐", "孙小姐", "方鸿渐", "自己", "没有"};
extractor.extract(str, keywords, 5); extractor.extract(str, keywords, 5);
res << keywords; ASSERT_EQ(keywords, vector<string>(res, res + sizeof(res)/sizeof(res[0])));
ASSERT_EQ("[\"第三性\", \"多愁多病\", \"记挂着\", \"揭去\", \"贫血症\"]", res);
} }