third_codes/jieba
1
0
Fork 0
mirror of https://github.com/fxsjy/jieba.git synced 2025-07-10 00:01:33 +08:00
Code Issues Packages Projects Releases Wiki Activity

don't seprate CRLF

Browse Source
This commit is contained in:
fxsjy 2013-06-22 21:56:39 +08:00
parent c015f4e297
commit a9f53e9c85
2 changed files with 13 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
jieba/__init__.py
View File

@ -221,7 +221,9 @@ def cut(sentence,cut_all=False):
else: else:
tmp = re_skip.split(blk) tmp = re_skip.split(blk)
for x in tmp: for x in tmp:
if not cut_all: if re_skip.match(x):
yield x
elif not cut_all:
for xx in x: for xx in x:
yield xx yield xx
else: else:

17
jieba/posseg/__init__.py
View File

@ -141,13 +141,16 @@ def __cut_internal(sentence):
else: else:
tmp = re_skip.split(blk) tmp = re_skip.split(blk)
for x in tmp: for x in tmp:
for xx in x: if re_skip.match(x):
if re_num.match(xx): yield pair(x,'')
yield pair(xx,'m') else:
elif re_eng.match(x): for xx in x:
yield pair(xx,'eng') if re_num.match(xx):
else: yield pair(xx,'m')
yield pair(xx,'x') elif re_eng.match(x):
yield pair(xx,'eng')
else:
yield pair(xx,'x')
def __lcut_internal(sentence): def __lcut_internal(sentence):
return list(__cut_internal(sentence)) return list(__cut_internal(sentence))