third_codes/jieba
1
0
Fork 0
mirror of https://github.com/fxsjy/jieba.git synced 2025-07-10 00:01:33 +08:00
Code Issues Packages Projects Releases Wiki Activity

modify test_file.py; use less memory

Browse Source
This commit is contained in:
fxsjy 2013-07-29 10:17:39 +08:00
parent ed1fa64e27
commit b77645b3aa
3 changed files with 6 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
jieba/__init__.py
View File

@ -297,7 +297,7 @@ def __lcut_for_search(sentence):
@require_initialized
def enable_parallel(processnum):
def enable_parallel(processnum=None):
global pool,cut,cut_for_search
if os.name=='nt':
raise Exception("jieba: parallel mode only supports posix system")

5
test/parallel/test_file.py
View File

@ -9,14 +9,13 @@ jieba.enable_parallel()
url = sys.argv[1]
content = open(url,"rb").read()
t1 = time.time()
words = list(jieba.cut(content))
words = "/ ".join(jieba.cut(content))
t2 = time.time()
tm_cost = t2-t1
log_f = open("1.log","wb")
for w in words:
print >> log_f, w.encode("utf-8"), "/" ,
log_f.write(words.encode('utf-8'))
print 'speed' , len(content)/tm_cost, " bytes/second"

6
test/test_file.py
View File

@ -8,14 +8,14 @@ jieba.initialize()
url = sys.argv[1]
content = open(url,"rb").read()
t1 = time.time()
words = list(jieba.cut(content))
words = "/ ".join(jieba.cut(content))
t2 = time.time()
tm_cost = t2-t1
log_f = open("1.log","wb")
for w in words:
print >> log_f, w.encode("utf-8"), "/" ,
log_f.write(words.encode('utf-8'))
print 'cost',tm_cost
print 'speed' , len(content)/tm_cost, " bytes/second"