third_codes/jieba
1
0
Fork 0
mirror of https://github.com/fxsjy/jieba.git synced 2025-07-10 00:01:33 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #240 from sing1ee/master

Browse Source 
build stable sort for graph iteration
This commit is contained in:
Sun Junyi 2015-02-16 20:28:22 +08:00
commit 4e05cde07e
1 changed files with 9 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
jieba/analyse/textrank.py
View File

@ -7,6 +7,7 @@ import collections
from operator import itemgetter from operator import itemgetter
import jieba.posseg as pseg import jieba.posseg as pseg
class UndirectWeightedGraph: class UndirectWeightedGraph:
d = 0.85 d = 0.85
@ -22,13 +23,15 @@ class UndirectWeightedGraph:
ws = collections.defaultdict(float) ws = collections.defaultdict(float)
outSum = collections.defaultdict(float) outSum = collections.defaultdict(float)
giter = list(self.graph.items()) # these two lines for build stable iteration
giter.sort()
wsdef = 1.0 / (len(self.graph) or 1.0) wsdef = 1.0 / (len(self.graph) or 1.0)
for n, out in self.graph.items(): for n, out in giter:
ws[n] = wsdef ws[n] = wsdef
outSum[n] = sum((e[2] for e in out), 0.0) outSum[n] = sum((e[2] for e in out), 0.0)
for x in xrange(10): # 10 iters for x in range(10): # 10 iters
for n, inedges in self.graph.items(): for n, inedges in giter:
s = 0 s = 0
for e in inedges: for e in inedges:
s += e[2] / outSum[e[1]] * ws[e[1]] s += e[2] / outSum[e[1]] * ws[e[1]]
@ -36,7 +39,7 @@ class UndirectWeightedGraph:
(min_rank, max_rank) = (sys.float_info[0], sys.float_info[3]) (min_rank, max_rank) = (sys.float_info[0], sys.float_info[3])
for w in itervalues(ws): for _, w in ws.items():
if w < min_rank: if w < min_rank:
min_rank = w min_rank = w
elif w > max_rank: elif w > max_rank:
@ -64,9 +67,9 @@ def textrank(sentence, topK=10, withWeight=False, allowPOS=['ns', 'n', 'vn', 'v'
cm = collections.defaultdict(int) cm = collections.defaultdict(int)
span = 5 span = 5
words = list(pseg.cut(sentence)) words = list(pseg.cut(sentence))
for i in xrange(len(words)): for i in range(len(words)):
if words[i].flag in pos_filt: if words[i].flag in pos_filt:
for j in xrange(i + 1, i + span): for j in range(i + 1, i + span):
if j >= len(words): if j >= len(words):
break break
if words[j].flag not in pos_filt: if words[j].flag not in pos_filt:
@ -75,7 +78,6 @@ def textrank(sentence, topK=10, withWeight=False, allowPOS=['ns', 'n', 'vn', 'v'
for terms, w in cm.items(): for terms, w in cm.items():
g.addEdge(terms[0], terms[1], w) g.addEdge(terms[0], terms[1], w)
nodes_rank = g.rank() nodes_rank = g.rank()
if withWeight: if withWeight:
tags = sorted(nodes_rank.items(), key=itemgetter(1), reverse=True) tags = sorted(nodes_rank.items(), key=itemgetter(1), reverse=True)