mirror of
https://github.com/yanyiwu/cppjieba.git
synced 2025-07-18 00:00:12 +08:00
42a93a4b98
Updated multiple files to replace instances of DecodeRunesInString with DecodeUTF8RunesInString, ensuring proper handling of UTF-8 encoded strings. This change enhances the robustness of string decoding across the cppjieba library, including updates in DictTrie, HMMModel, PosTagger, PreFilter, SegmentBase, and Unicode files. Additionally, corresponding unit tests have been modified to reflect these changes.
47 lines
1012 B
C++
47 lines
1012 B
C++
#ifndef CPPJIEBA_SEGMENTBASE_H
|
|
#define CPPJIEBA_SEGMENTBASE_H
|
|
|
|
#include "limonp/Logging.hpp"
|
|
#include "PreFilter.hpp"
|
|
#include <cassert>
|
|
|
|
|
|
namespace cppjieba {
|
|
|
|
const char* const SPECIAL_SEPARATORS = " \t\n\xEF\xBC\x8C\xE3\x80\x82";
|
|
|
|
using namespace limonp;
|
|
|
|
class SegmentBase {
|
|
public:
|
|
SegmentBase() {
|
|
XCHECK(ResetSeparators(SPECIAL_SEPARATORS));
|
|
}
|
|
virtual ~SegmentBase() {
|
|
}
|
|
|
|
virtual void Cut(const string& sentence, vector<string>& words) const = 0;
|
|
|
|
bool ResetSeparators(const string& s) {
|
|
symbols_.clear();
|
|
RuneStrArray runes;
|
|
if (!DecodeUTF8RunesInString(s, runes)) {
|
|
XLOG(ERROR) << "decode " << s << " failed";
|
|
return false;
|
|
}
|
|
for (size_t i = 0; i < runes.size(); i++) {
|
|
if (!symbols_.insert(runes[i].rune).second) {
|
|
XLOG(ERROR) << s.substr(runes[i].offset, runes[i].len) << " already exists";
|
|
return false;
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
protected:
|
|
unordered_set<Rune> symbols_;
|
|
}; // class SegmentBase
|
|
|
|
} // cppjieba
|
|
|
|
#endif
|