mirror of
https://github.com/fxsjy/jieba.git
synced 2025-07-10 00:01:33 +08:00
94840a734c
API changes: * class jieba.Tokenizer, jieba.posseg.POSTokenizer * class jieba.analyse.TFIDF, jieba.analyse.TextRank * global functions are mapped to jieba.(posseg.)dt, the default (POS)Tokenizer * multiprocessing only works with jieba.(posseg.)dt * new lcut, lcut_for_search functions that returns a list * jieba.analyse.textrank now returns 20 items by default Tests: * added test_lock.py to test multithread locking * demo.py now contains most of the examples in README
38 lines
1.4 KiB
Python
38 lines
1.4 KiB
Python
# encoding=utf-8
|
|
from __future__ import unicode_literals
|
|
from whoosh.analysis import RegexAnalyzer, LowercaseFilter, StopFilter, StemFilter
|
|
from whoosh.analysis import Tokenizer, Token
|
|
from whoosh.lang.porter import stem
|
|
|
|
import jieba
|
|
import re
|
|
|
|
STOP_WORDS = frozenset(('a', 'an', 'and', 'are', 'as', 'at', 'be', 'by', 'can',
|
|
'for', 'from', 'have', 'if', 'in', 'is', 'it', 'may',
|
|
'not', 'of', 'on', 'or', 'tbd', 'that', 'the', 'this',
|
|
'to', 'us', 'we', 'when', 'will', 'with', 'yet',
|
|
'you', 'your', '的', '了', '和'))
|
|
|
|
accepted_chars = re.compile(r"[\u4E00-\u9FA5]+")
|
|
|
|
|
|
class ChineseTokenizer(Tokenizer):
|
|
|
|
def __call__(self, text, **kargs):
|
|
words = jieba.tokenize(text, mode="search")
|
|
token = Token()
|
|
for (w, start_pos, stop_pos) in words:
|
|
if not accepted_chars.match(w) and len(w) <= 1:
|
|
continue
|
|
token.original = token.text = w
|
|
token.pos = start_pos
|
|
token.startchar = start_pos
|
|
token.endchar = stop_pos
|
|
yield token
|
|
|
|
|
|
def ChineseAnalyzer(stoplist=STOP_WORDS, minsize=1, stemfn=stem, cachesize=50000):
|
|
return (ChineseTokenizer() | LowercaseFilter() |
|
|
StopFilter(stoplist=stoplist, minsize=minsize) |
|
|
StemFilter(stemfn=stemfn, ignore=None, cachesize=cachesize))
|