third_codes/cppjieba
1
0
Fork 0
mirror of https://github.com/yanyiwu/cppjieba.git synced 2025-07-18 00:00:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

add localvector

Browse Source
This commit is contained in:
wyy 2014-06-05 00:48:49 +08:00
parent 014bea02ba
commit 76dd93051e
5 changed files with 197 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/DictTrie.hpp
View File

@ -44,7 +44,7 @@ namespace CppJieba
class DictTrie: public InitOnOff
{
public:
typedef Trie<Unicode::value_type, DictUnit> TrieType;
typedef Trie<Unicode::value_type, DictUnit, Unicode, vector<Unicode>, vector<const DictUnit*> > TrieType;
private:
vector<DictUnit> _nodeInfos;
TrieType * _trie;

173
src/Limonp/LocalVector.hpp Normal file
View File

@ -0,0 +1,173 @@
#ifndef LIMONP_LOCAL_VECTOR_HPP
#define LIMONP_LOCAL_VECTOR_HPP
#include <iostream>
#include <stdlib.h>
#include <assert.h>
#include <string.h>
namespace Limonp
{
using namespace std;
const size_t LOCAL_VECTOR_BUFFER_SIZE = 32;
template <class T>
class LocalVector
{
public:
typedef const T* const_iterator ;
typedef T value_type;
typedef size_t size_type;
private:
T _buffer[LOCAL_VECTOR_BUFFER_SIZE];
T * _ptr;
size_t _size;
size_t _capacity;
public:
LocalVector()
{
_init();
};
LocalVector(const LocalVector<T>& vec)
{
_init();
*this = vec;
}
LocalVector(const_iterator begin, const_iterator end) // TODO: make it faster
{
_init();
while(begin != end)
{
push_back(*begin++);
}
}
LocalVector(size_t size, const T& t) // TODO: make it faster
{
_init();
while(size--)
{
push_back(t);
}
}
~LocalVector()
{
if(_ptr != _buffer)
{
free(_ptr);
}
};
public:
LocalVector<T>& operator = (const LocalVector<T>& vec)
{
clear();
_size = vec.size();
_capacity = vec.capacity();
if(vec._buffer == vec._ptr)
{
memcpy(_buffer, vec._buffer, sizeof(T) * _size);
_ptr = _buffer;
}
else
{
_ptr = (T*) malloc(vec.capacity() * sizeof(T));
assert(_ptr);
memcpy(_ptr, vec._ptr, vec.size() * sizeof(T));
}
return *this;
}
private:
void _init()
{
_ptr = _buffer;
_size = 0;
_capacity = LOCAL_VECTOR_BUFFER_SIZE;
}
public:
T& operator [] (size_t i)
{
return _ptr[i];
}
const T& operator [] (size_t i) const
{
return _ptr[i];
}
void push_back(const T& t)
{
if(!full())
{
_ptr[_size++] = t;
return ;
}
assert(_capacity);
reserve(_capacity * 2);
_ptr[_size ++ ] = t;
}
void reserve(size_t size)
{
if(size <= _capacity)
{
return;
}
T * next = (T*)malloc(sizeof(T) * size);
assert(next);
T * old = _ptr;
_ptr = next;
memcpy(_ptr, old, sizeof(T) * _capacity);
_capacity = size;
if(old != _buffer)
{
free(old);
}
}
bool full() const
{
return size() == capacity();
}
bool empty() const
{
return 0 == size();
}
size_t size() const
{
return _size;
}
size_t capacity() const
{
return _capacity;
}
const_iterator begin() const
{
return _ptr;
}
const_iterator end() const
{
return _ptr + _size;
}
void clear()
{
if(_ptr != _buffer)
{
free(_ptr);
}
_init();
}
};
template <class T>
ostream & operator << (ostream& os, const LocalVector<T>& vec)
{
if(vec.empty())
{
return os << "[]";
}
os<<"[\""<<vec[0];
for(size_t i = 1; i < vec.size(); i++)
{
os<<"\", \""<<vec[i];
}
os<<"\"]";
return os;
}
}
#endif

18
src/Limonp/StringUtil.hpp
View File

@ -208,7 +208,8 @@ namespace Limonp
return (((uint16_t(high) & 0x00ff ) << 8) | (uint16_t(low) & 0x00ff));
}
inline bool utf8ToUnicode(const char * const str, size_t len, vector<uint16_t>& vec)
template <class Uint16Container>
bool utf8ToUnicode(const char * const str, size_t len, Uint16Container& vec)
{
if(!str)
{
@ -247,12 +248,14 @@ namespace Limonp
}
return true;
}
inline bool utf8ToUnicode(const string& str, vector<uint16_t>& vec)
template <class Uint16Container>
bool utf8ToUnicode(const string& str, Uint16Container& vec)
{
return utf8ToUnicode(str.c_str(), str.size(), vec);
}
inline bool unicodeToUtf8(vector<uint16_t>::const_iterator begin, vector<uint16_t>::const_iterator end, string& res)
template <class Uint16ContainerConIter>
bool unicodeToUtf8(Uint16ContainerConIter begin, Uint16ContainerConIter end, string& res)
{
if(begin >= end)
{
@ -284,7 +287,8 @@ namespace Limonp
}
inline bool gbkTrans(const char* const str, size_t len, vector<uint16_t>& vec)
template <class Uint16Container>
bool gbkTrans(const char* const str, size_t len, Uint16Container& vec)
{
vec.clear();
if(!str)
@ -316,12 +320,14 @@ namespace Limonp
return true;
}
inline bool gbkTrans(const string& str, vector<uint16_t>& vec)
template <class Uint16Container>
bool gbkTrans(const string& str, Uint16Container& vec)
{
return gbkTrans(str.c_str(), str.size(), vec);
}
inline bool gbkTrans(vector<uint16_t>::const_iterator begin, vector<uint16_t>::const_iterator end, string& res)
template <class Uint16ContainerConIter>
bool gbkTrans(Uint16ContainerConIter begin, Uint16ContainerConIter end, string& res)
{
if(begin >= end)
{

3
src/TransCode.hpp
View File

@ -7,13 +7,14 @@
#include "Limonp/StringUtil.hpp"
#include "Limonp/LocalVector.hpp"
namespace CppJieba
{
using namespace Limonp;
typedef uint16_t UnicodeValueType;
typedef std::vector<UnicodeValueType> Unicode;
typedef Limonp::LocalVector<UnicodeValueType> Unicode;
namespace TransCode
{
inline bool decode(const string& str, Unicode& res)

18
src/Trie.hpp
View File

@ -17,7 +17,7 @@ namespace CppJieba
const ValueType * ptValue;
};
template <class KeyType, class ValueType>
template <class KeyType, class ValueType, class KeyContainerType = vector<KeyType>, class KeysContainerType = vector<KeyContainerType>, class ValueContainerType = vector<const ValueType* > >
class Trie
{
public:
@ -25,7 +25,7 @@ namespace CppJieba
private:
TrieNodeType* _root;
public:
Trie(const vector<vector<KeyType> >& keys, const vector<const ValueType* >& valuePointers)
Trie(const KeysContainerType& keys, const ValueContainerType& valuePointers)
{
_root = new TrieNodeType;
_root->ptKeyMap = NULL;
@ -41,11 +41,11 @@ namespace CppJieba
}
}
public:
const ValueType* find(typename vector<KeyType>::const_iterator begin, typename vector<KeyType>::const_iterator end) const
const ValueType* find(typename KeyContainerType::const_iterator begin, typename KeyContainerType::const_iterator end) const
{
typename TrieNodeType::KeyMapType::const_iterator citer;
const TrieNodeType* ptNode = _root;
for(typename vector<KeyType>::const_iterator it = begin; it != end; it++)
for(typename KeyContainerType::const_iterator it = begin; it != end; it++)
{
assert(ptNode);
if(NULL == ptNode->ptKeyMap || ptNode->ptKeyMap->end() == (citer = ptNode->ptKeyMap->find(*it)))
@ -56,12 +56,12 @@ namespace CppJieba
}
return ptNode->ptValue;
}
bool find(typename vector<KeyType>::const_iterator begin, typename vector<KeyType> ::const_iterator end, map<typename vector<KeyType>::size_type, const ValueType* >& ordererMap, size_t offset = 0) const
bool find(typename KeyContainerType::const_iterator begin, typename KeyContainerType::const_iterator end, map<typename KeyContainerType::size_type, const ValueType* >& ordererMap, size_t offset = 0) const
{
const TrieNodeType * ptNode = _root;
typename TrieNodeType::KeyMapType::const_iterator citer;
ordererMap.clear();
for(typename vector<KeyType>::const_iterator itr = begin; itr != end ; itr++)
for(typename KeyContainerType::const_iterator itr = begin; itr != end ; itr++)
{
assert(ptNode);
if(NULL == ptNode->ptKeyMap || ptNode->ptKeyMap->end() == (citer = ptNode->ptKeyMap->find(*itr)))
@ -77,7 +77,7 @@ namespace CppJieba
return ordererMap.size();
}
private:
void _createTrie(const vector<vector<KeyType> >& keys, const vector<const ValueType*>& valuePointers)
void _createTrie(const KeysContainerType& keys, const ValueContainerType& valuePointers)
{
if(valuePointers.empty() || keys.empty())
{
@ -91,13 +91,13 @@ namespace CppJieba
}
}
private:
void _insertNode(const vector<KeyType>& key, const ValueType* ptValue)
void _insertNode(const KeyContainerType& key, const ValueType* ptValue)
{
TrieNodeType* ptNode = _root;
typename TrieNodeType::KeyMapType::const_iterator kmIter;
for(typename vector<KeyType>::const_iterator citer = key.begin(); citer != key.end(); citer++)
for(typename KeyContainerType::const_iterator citer = key.begin(); citer != key.end(); citer++)
{
if(NULL == ptNode->ptKeyMap)
{