third_codes/jieba
1
0
Fork 0
mirror of https://github.com/fxsjy/jieba.git synced 2025-07-10 00:01:33 +08:00
Code Issues Packages Projects Releases Wiki Activity

speed up of viterbi

Browse Source
This commit is contained in:
fxsjy 2013-06-07 14:41:55 +08:00
parent 0087a4e7e3
commit f2d6abf063
2 changed files with 14638 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
jieba/finalseg/__init__.py
View File

@ -13,7 +13,7 @@ def load_model(f_name):
prob_start = load_model("prob_start.py") prob_start = load_model("prob_start.py")
prob_trans = load_model("prob_trans.py") prob_trans = load_model("prob_trans.py")
prob_emit = load_model("prob_emit.py") prob_emit = load_model("prob_emit.py")
char_states = load_model("char_states.py")
def viterbi(obs, states, start_p, trans_p, emit_p): def viterbi(obs, states, start_p, trans_p, emit_p):
@ -25,13 +25,15 @@ def viterbi(obs, states, start_p, trans_p, emit_p):
for t in range(1,len(obs)): for t in range(1,len(obs)):
V.append({}) V.append({})
newpath = {} newpath = {}
for y in states: cur_states = char_states.get(obs[t],states)
(prob,state ) = max([(V[t-1][y0] + trans_p[y0].get(y,MIN_FLOAT) + emit_p[y].get(obs[t],MIN_FLOAT) ,y0) for y0 in states ]) prev_states = V[t-1].keys()
for y in cur_states:
(prob,state ) = max([(V[t-1][y0] + trans_p[y0].get(y,MIN_FLOAT) + emit_p[y].get(obs[t],MIN_FLOAT) ,y0) for y0 in prev_states ])
V[t][y] =prob V[t][y] =prob
newpath[y] = path[state] + [y] newpath[y] = path[state] + [y]
path = newpath path = newpath
(prob, state) = max([(V[len(obs) - 1][y], y) for y in ('E','S')]) (prob, state) = max([(V[len(obs) - 1].get(y,MIN_FLOAT), y) for y in ('E','S')])
return (prob, path[state]) return (prob, path[state])

14632
jieba/finalseg/char_states.py Normal file
View File

File diff suppressed because it is too large Load Diff