Update README.md update paddle link. (#817)

* paddle_null_point_fix

* add core expception note

* delete yield

* modify test paddle for supporting enable_paddle()

Changelog

@@ -1,3 +1,11 @@
+2019-1-20: version 0.42.1
+1. 修复setup.py在python2.7版本无法工作的问题 (issue #809)
+
+2019-1-13: version 0.42
+1. 修复paddle模式空字符串coredump问题 @JesseyXujin
+2. 修复cut_all模式切分丢字问题 @fxsjy
+3. paddle安装检测优化 @vissssa
+
 2019-1-8: version 0.41
 1. 开启paddle模式更友好
 2. 修复cut_all模式不支持中英混合词的bug

README.md

@@ -13,7 +13,7 @@ jieba
     * 精确模式，试图将句子最精确地切开，适合文本分析；
     * 全模式，把句子中所有的可以成词的词语都扫描出来, 速度非常快，但是不能解决歧义；
     * 搜索引擎模式，在精确模式的基础上，对长词再次切分，提高召回率，适合用于搜索引擎分词。
-    * paddle模式，利用PaddlePaddle深度学习框架，训练序列标注（双向GRU）网络模型实现分词。同时支持词性标注。paddle模式使用需安装paddlepaddle-tiny，`pip install paddlepaddle-tiny==1.6.1`。目前paddle模式支持jieba v0.40及以上版本。jieba v0.40以下版本，请升级jieba，`pip install jieba --upgrade` 。[PaddlePaddle官网](www.paddlepaddle.org.cn)
+    * paddle模式，利用PaddlePaddle深度学习框架，训练序列标注（双向GRU）网络模型实现分词。同时支持词性标注。paddle模式使用需安装paddlepaddle-tiny，`pip install paddlepaddle-tiny==1.6.1`。目前paddle模式支持jieba v0.40及以上版本。jieba v0.40以下版本，请升级jieba，`pip install jieba --upgrade` 。[PaddlePaddle官网](https://www.paddlepaddle.org.cn/)
 * 支持繁体分词
 * 支持自定义词典
 * MIT 授权协议

jieba/__init__.py

@@ -1,26 +1,24 @@
 from __future__ import absolute_import, unicode_literals
-__version__ = '0.41'
+
+__version__ = '0.42.1'
 __license__ = 'MIT'
 
-import re
-import os
-import sys
-import time
-import logging
 import marshal
+import re
 import tempfile
 import threading
-from math import log
+import time
 from hashlib import md5
-from ._compat import *
+from math import log
+
 from . import finalseg
+from ._compat import *
 
 if os.name == 'nt':
     from shutil import move as _replace_file
 else:
     _replace_file = os.rename
 
-
 _get_abs_path = lambda path: os.path.normpath(os.path.join(os.getcwd(), path))
 
 DEFAULT_DICT = None
@@ -47,10 +45,11 @@ re_han_default = re.compile("([\u4E00-\u9FD5a-zA-Z0-9+#&\._%\-]+)", re.U)
 
 re_skip_default = re.compile("(\r\n|\s)", re.U)
 
+
 def setLogLevel(log_level):
-    global logger
     default_logger.setLevel(log_level)
 
+
 class Tokenizer(object):
 
     def __init__(self, dictionary=DEFAULT_DICT):
@@ -69,7 +68,8 @@ class Tokenizer(object):
     def __repr__(self):
         return '<Tokenizer dictionary=%r>' % self.dictionary
 
-    def gen_pfdict(self, f):
+    @staticmethod
+    def gen_pfdict(f):
         lfreq = {}
         ltotal = 0
         f_name = resolve_filename(f)
@@ -128,7 +128,7 @@ class Tokenizer(object):
 
             load_from_cache_fail = True
             if os.path.isfile(cache_file) and (abs_path == DEFAULT_DICT or
-                os.path.getmtime(cache_file) > os.path.getmtime(abs_path)):
+                                               os.path.getmtime(cache_file) > os.path.getmtime(abs_path)):
                 default_logger.debug(
                     "Loading model from cache %s" % cache_file)
                 try:
@@ -201,25 +201,26 @@ class Tokenizer(object):
         eng_scan = 0
         eng_buf = u''
         for k, L in iteritems(dag):
-            if eng_scan==1 and not re_eng.match(sentence[k]):
+            if eng_scan == 1 and not re_eng.match(sentence[k]):
                 eng_scan = 0
                 yield eng_buf
             if len(L) == 1 and k > old_j:
-                if re_eng.match(sentence[k]):
+                word = sentence[k:L[0] + 1]
+                if re_eng.match(word):
                     if eng_scan == 0:
                         eng_scan = 1
-                        eng_buf = sentence[k]
+                        eng_buf = word
                     else:
-                        eng_buf += sentence[k]
+                        eng_buf += word
                 if eng_scan == 0:
-                    yield sentence[k:L[0] + 1]
+                    yield word
                 old_j = L[0]
             else:
                 for j in L:
                     if j > k:
                         yield sentence[k:j + 1]
                         old_j = j
-        if eng_scan==1:
+        if eng_scan == 1:
             yield eng_buf
 
     def __cut_DAG_NO_HMM(self, sentence):
@@ -285,8 +286,8 @@ class Tokenizer(object):
                 for elem in buf:
                     yield elem
 
-    def cut(self, sentence, cut_all = False, HMM = True,use_paddle = False):
-        '''
+    def cut(self, sentence, cut_all=False, HMM=True, use_paddle=False):
+        """
         The main function that segments an entire sentence that contains
         Chinese characters into separated words.
 
@@ -294,14 +295,12 @@ class Tokenizer(object):
             - sentence: The str(unicode) to be segmented.
             - cut_all: Model type. True for full pattern, False for accurate pattern.
             - HMM: Whether to use the Hidden Markov Model.
-        '''
-        is_paddle_installed = False
-        if use_paddle == True:
-            is_paddle_installed = check_paddle_install()
+        """
+        is_paddle_installed = check_paddle_install['is_paddle_installed']
         sentence = strdecode(sentence)
-        if use_paddle == True and is_paddle_installed == True:
-            if sentence is None or sentence == "" or sentence == u"":
-                yield sentence
+        if use_paddle and is_paddle_installed:
+            # if sentence is null, it will raise core exception in paddle.
+            if sentence is None or len(sentence) == 0:
                 return
             import jieba.lac_small.predict as predict
             results = predict.get_sent(sentence)

jieba/_compat.py

@@ -1,49 +1,55 @@
 # -*- coding: utf-8 -*-
+import logging
 import os
 import sys
-import logging
 
 log_console = logging.StreamHandler(sys.stderr)
 default_logger = logging.getLogger(__name__)
 default_logger.setLevel(logging.DEBUG)
 
+
 def setLogLevel(log_level):
-    global logger
     default_logger.setLevel(log_level)
 
+
+check_paddle_install = {'is_paddle_installed': False}
+
 try:
     import pkg_resources
+
     get_module_res = lambda *res: pkg_resources.resource_stream(__name__,
                                                                 os.path.join(*res))
 except ImportError:
     get_module_res = lambda *res: open(os.path.normpath(os.path.join(
-                            os.getcwd(), os.path.dirname(__file__), *res)), 'rb')
+        os.getcwd(), os.path.dirname(__file__), *res)), 'rb')
 
 
 def enable_paddle():
-    import_paddle_check = False
     try:
         import paddle
     except ImportError:
         default_logger.debug("Installing paddle-tiny, please wait a minute......")
         os.system("pip install paddlepaddle-tiny")
-    try:
-        import paddle
-    except ImportError:
-        default_logger.debug("Import paddle error, please use command to install: pip install paddlepaddle-tiny==1.6.1."
-                             "Now, back to jieba basic cut......")
+        try:
+            import paddle
+        except ImportError:
+            default_logger.debug(
+                "Import paddle error, please use command to install: pip install paddlepaddle-tiny==1.6.1."
+                "Now, back to jieba basic cut......")
     if paddle.__version__ < '1.6.1':
         default_logger.debug("Find your own paddle version doesn't satisfy the minimum requirement (1.6.1), "
                              "please install paddle tiny by 'pip install --upgrade paddlepaddle-tiny', "
-                             "or upgrade paddle full version by 'pip install --upgrade paddlepaddle (-gpu for GPU version)' ")
+                             "or upgrade paddle full version by "
+                             "'pip install --upgrade paddlepaddle (-gpu for GPU version)' ")
     else:
         try:
             import jieba.lac_small.predict as predict
-            import_paddle_check = True
             default_logger.debug("Paddle enabled successfully......")
+            check_paddle_install['is_paddle_installed'] = True
         except ImportError:
             default_logger.debug("Import error, cannot find paddle.fluid and jieba.lac_small.predict module. "
-                             "Now, back to jieba basic cut......")
+                                 "Now, back to jieba basic cut......")
+
 
 PY2 = sys.version_info[0] == 2
 
@@ -66,6 +72,7 @@ else:
     itervalues = lambda d: iter(d.values())
     iteritems = lambda d: iter(d.items())
 
+
 def strdecode(sentence):
     if not isinstance(sentence, text_type):
         try:
@@ -74,25 +81,9 @@ def strdecode(sentence):
             sentence = sentence.decode('gbk', 'ignore')
     return sentence
 
+
 def resolve_filename(f):
     try:
         return f.name
     except AttributeError:
         return repr(f)
-
-
-def check_paddle_install():
-    is_paddle_installed =  False
-    try:
-        import paddle
-        if paddle.__version__ >= '1.6.1':
-            is_paddle_installed = True
-        else:
-            is_paddle_installed = False
-            default_logger.debug("Check the paddle version is not correct, the current version is "+ paddle.__version__+","
-            "please use command to install paddle: pip uninstall paddlepaddle(-gpu), "
-            "pip install paddlepaddle-tiny==1.6.1. Now, back to jieba basic cut......")
-    except ImportError:
-        default_logger.debug("Import paddle error, back to jieba basic cut......")
-        is_paddle_installed = False
-    return is_paddle_installed

jieba/posseg/__init__.py

@@ -1,11 +1,11 @@
 from __future__ import absolute_import, unicode_literals
-import os
-import re
-import sys
-import jieba
+
 import pickle
-from .._compat import *
+import re
+
+import jieba
 from .viterbi import viterbi
+from .._compat import *
 
 PROB_START_P = "prob_start.p"
 PROB_TRANS_P = "prob_trans.p"
@@ -252,6 +252,7 @@ class POSTokenizer(object):
     def lcut(self, *args, **kwargs):
         return list(self.cut(*args, **kwargs))
 
+
 # default Tokenizer instance
 
 dt = POSTokenizer(jieba.dt)
@@ -276,19 +277,17 @@ def cut(sentence, HMM=True, use_paddle=False):
     Note that this only works using dt, custom POSTokenizer
     instances are not supported.
     """
-    is_paddle_installed = False
-    if use_paddle == True:
-        is_paddle_installed = check_paddle_install()
-    if use_paddle==True and is_paddle_installed == True:
+    is_paddle_installed = check_paddle_install['is_paddle_installed']
+    if use_paddle and is_paddle_installed:
+        # if sentence is null, it will raise core exception in paddle.
         if sentence is None or sentence == "" or sentence == u"":
-            yield pair(None, None)
             return
         import jieba.lac_small.predict as predict
-        sents,tags = predict.get_result(strdecode(sentence))
-        for i,sent in enumerate(sents):
+        sents, tags = predict.get_result(strdecode(sentence))
+        for i, sent in enumerate(sents):
             if sent is None or tags[i] is None:
                 continue
-            yield pair(sent,tags[i])
+            yield pair(sent, tags[i])
         return
     global dt
     if jieba.pool is None:

setup.py

@@ -43,7 +43,7 @@ GitHub: https://github.com/fxsjy/jieba
 """
 
 setup(name='jieba',
-      version='0.41',
+      version='0.42.1',
       description='Chinese Words Segmentation Utilities',
       long_description=LONGDOC,
       author='Sun, Junyi',
@@ -71,5 +71,5 @@ setup(name='jieba',
       keywords='NLP,tokenizing,Chinese word segementation',
       packages=['jieba'],
       package_dir={'jieba':'jieba'},
-      package_data={'jieba':['*.*','finalseg/*','analyse/*','posseg/*', 'lac_small/*','lac_small/model_baseline/*']}
+      package_data={'jieba':['*.*','finalseg/*','analyse/*','posseg/*', 'lac_small/*.py','lac_small/*.dic', 'lac_small/model_baseline/*']}
 )

test/test_cutall.py

@@ -98,3 +98,4 @@ if __name__ == "__main__":
     cuttest('你认识那个和主席握手的的哥吗？他开一辆黑色的士。')
     jieba.add_word('超敏C反应蛋白')
     cuttest('超敏C反应蛋白是什么, java好学吗?,小潘老板都学Python')
+    cuttest('steel健身爆发力运动兴奋补充剂')

test/test_paddle.py

@@ -2,7 +2,7 @@
 import sys
 sys.path.append("../")
 import jieba
-
+jieba.enable_paddle()
 
 def cuttest(test_sent):
     result = jieba.cut(test_sent, use_paddle=True)

test/test_paddle_postag.py

@@ -2,7 +2,8 @@
 import sys
 sys.path.append("../")
 import jieba.posseg as pseg
-
+import jieba
+jieba.enable_paddle()
 
 def cuttest(test_sent):
     result = pseg.cut(test_sent, use_paddle=True)