third_codes/jieba
1
0
Fork 0
mirror of https://github.com/fxsjy/jieba.git synced 2025-07-10 00:01:33 +08:00
Code Issues Packages Projects Releases Wiki Activity

Separate cal and IO process.

Browse Source
This commit is contained in:
Richard Wong 2013-07-03 15:03:45 +08:00
parent e1c1d46324
commit 3246236133
1 changed files with 15 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
jieba/__init__.py
View File

@ -264,15 +264,21 @@ def load_userdict(f):
if line_no==1: if line_no==1:
word = word.replace(u'\ufeff',u"") #remove bom flag if it exists word = word.replace(u'\ufeff',u"") #remove bom flag if it exists
if len(tup)==3: if len(tup)==3:
user_word_tag_tab[word]=tup[2].strip() add_word(word, freq, tup[2])
freq = float(freq) add_word(word, freq)
FREQ[word] = log(freq / total)
p = trie def add_word(word, freq, tag=None):
for c in word: global FREQ, trie, total, user_word_tag_tab
if not c in p: freq = float(freq)
p[c] ={} FREQ[word] = log(freq / total)
p = p[c] if tag is not None:
p['']='' #ending flag user_word_tag_tab[word] = tag.strip()
p = trie
for c in word:
if not c in p:
p[c] = {}
p = p[c]
p[''] = '' # ending flag
__ref_cut = cut __ref_cut = cut
__ref_cut_for_search = cut_for_search __ref_cut_for_search = cut_for_search