third_codes/jieba
1
0
Fork 0
mirror of https://github.com/fxsjy/jieba.git synced 2025-07-10 00:01:33 +08:00
Code Issues Packages Projects Releases Wiki Activity

add a withFlag param in textrank

Browse Source
This commit is contained in:
jerryday 2015-10-30 15:40:41 +08:00
parent 26e339f8f7
commit 4f8ca83661
2 changed files with 12 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
jieba/analyse/textrank.py
View File

@ -44,7 +44,7 @@ class UndirectWeightedGraph:
for w in itervalues(ws): for w in itervalues(ws):
if w < min_rank: if w < min_rank:
min_rank = w min_rank = w
elif w > max_rank: if w > max_rank:
max_rank = w max_rank = w
for n, w in ws.items(): for n, w in ws.items():
@ -66,7 +66,7 @@ class TextRank(KeywordExtractor):
return (wp.flag in self.pos_filt and len(wp.word.strip()) >= 2 return (wp.flag in self.pos_filt and len(wp.word.strip()) >= 2
and wp.word.lower() not in self.stop_words) and wp.word.lower() not in self.stop_words)
def textrank(self, sentence, topK=20, withWeight=False, allowPOS=('ns', 'n', 'vn', 'v')): def textrank(self, sentence, topK=20, withWeight=False, allowPOS=('ns', 'n', 'vn', 'v'), withFlag=False):
""" """
Extract keywords from sentence using TextRank algorithm. Extract keywords from sentence using TextRank algorithm.
Parameter: Parameter:
@ -75,6 +75,8 @@ class TextRank(KeywordExtractor):
if False, return a list of words. if False, return a list of words.
- allowPOS: the allowed POS list eg. ['ns', 'n', 'vn', 'v']. - allowPOS: the allowed POS list eg. ['ns', 'n', 'vn', 'v'].
if the POS of w is not in this list, it will be filtered. if the POS of w is not in this list, it will be filtered.
- withFlag: if True, return a list of pair(word, weight) like posseg.cut
if False, return a list of words
""" """
self.pos_filt = frozenset(allowPOS) self.pos_filt = frozenset(allowPOS)
g = UndirectWeightedGraph() g = UndirectWeightedGraph()
@ -87,7 +89,10 @@ class TextRank(KeywordExtractor):
break break
if not self.pairfilter(words[j]): if not self.pairfilter(words[j]):
continue continue
cm[(wp.word, words[j].word)] += 1 if allowPOS and withFlag:
cm[(wp, words[j])] += 1
else:
cm[(wp.word, words[j].word)] += 1
for terms, w in cm.items(): for terms, w in cm.items():
g.addEdge(terms[0], terms[1], w) g.addEdge(terms[0], terms[1], w)
@ -96,6 +101,7 @@ class TextRank(KeywordExtractor):
tags = sorted(nodes_rank.items(), key=itemgetter(1), reverse=True) tags = sorted(nodes_rank.items(), key=itemgetter(1), reverse=True)
else: else:
tags = sorted(nodes_rank, key=nodes_rank.__getitem__, reverse=True) tags = sorted(nodes_rank, key=nodes_rank.__getitem__, reverse=True)
if topK: if topK:
return tags[:topK] return tags[:topK]
else: else:

3
jieba/posseg/__init__.py
View File

@ -81,6 +81,9 @@ class pair(object):
def __iter__(self): def __iter__(self):
return iter((self.word, self.flag)) return iter((self.word, self.flag))
def __lt__(self, other):
return self.word < other.word
def encode(self, arg): def encode(self, arg):
return self.__unicode__().encode(arg) return self.__unicode__().encode(arg)