third_codes/jieba
1
0
Fork 0
mirror of https://github.com/fxsjy/jieba.git synced 2025-07-10 00:01:33 +08:00
Code Issues Packages Projects Releases Wiki Activity

add a withFlag param to extract_tags

Browse Source
This commit is contained in:
jerryday 2015-10-30 11:02:34 +08:00
parent b6f1ce773e
commit 26e339f8f7
1 changed files with 9 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
jieba/analyse/tfidf.py
View File

@ -72,7 +72,7 @@ class TFIDF(KeywordExtractor):
self.idf_loader.set_new_path(new_abs_path) self.idf_loader.set_new_path(new_abs_path)
self.idf_freq, self.median_idf = self.idf_loader.get_idf() self.idf_freq, self.median_idf = self.idf_loader.get_idf()
def extract_tags(self, sentence, topK=20, withWeight=False, allowPOS=()): def extract_tags(self, sentence, topK=20, withWeight=False, allowPOS=(), withFlag=False):
""" """
Extract keywords from sentence using TF-IDF algorithm. Extract keywords from sentence using TF-IDF algorithm.
Parameter: Parameter:
@ -81,6 +81,9 @@ class TFIDF(KeywordExtractor):
if False, return a list of words. if False, return a list of words.
- allowPOS: the allowed POS list eg. ['ns', 'n', 'vn', 'v','nr']. - allowPOS: the allowed POS list eg. ['ns', 'n', 'vn', 'v','nr'].
if the POS of w is not in this list,it will be filtered. if the POS of w is not in this list,it will be filtered.
- withFlag: only work with allowPOS is not empty.
if True, return a list of pair(word, weight) like posseg.cut
if False, return a list of words
""" """
if allowPOS: if allowPOS:
allowPOS = frozenset(allowPOS) allowPOS = frozenset(allowPOS)
@ -92,14 +95,16 @@ class TFIDF(KeywordExtractor):
if allowPOS: if allowPOS:
if w.flag not in allowPOS: if w.flag not in allowPOS:
continue continue
else: elif not withFlag:
w = w.word w = w.word
if len(w.strip()) < 2 or w.lower() in self.stop_words: wc = w.word if allowPOS and withFlag else w
if len(wc.strip()) < 2 or wc.lower() in self.stop_words:
continue continue
freq[w] = freq.get(w, 0.0) + 1.0 freq[w] = freq.get(w, 0.0) + 1.0
total = sum(freq.values()) total = sum(freq.values())
for k in freq: for k in freq:
freq[k] *= self.idf_freq.get(k, self.median_idf) / total kw = k.word if allowPOS and withFlag else k
freq[k] *= self.idf_freq.get(kw, self.median_idf) / total
if withWeight: if withWeight:
tags = sorted(freq.items(), key=itemgetter(1), reverse=True) tags = sorted(freq.items(), key=itemgetter(1), reverse=True)