diff --git a/README.md b/README.md index 3b15d29..906e450 100644 --- a/README.md +++ b/README.md @@ -49,13 +49,14 @@ $ conda install -c conda-forge eland ### Compatibility -- Supports Python 3.8+ and Pandas 1.5 +- Supports Python 3.8, 3.9, 3.10 and Pandas 1.5 - Supports Elasticsearch clusters that are 7.11+, recommended 8.3 or later for all features to work. If you are using the NLP with PyTorch feature make sure your Eland minor version matches the minor version of your Elasticsearch cluster. For all other features it is sufficient for the major versions to match. -- You need to use PyTorch `1.11.0` or earlier to import an NLP model. - Run `pip install torch==1.11` to install the aproppriate version of PyTorch. +- You need to use PyTorch `1.13.1` or earlier to import an NLP model. + Run `pip install torch==1.13.1` to install the aproppriate version of PyTorch. + ### Prerequisites diff --git a/bin/eland_import_hub_model b/bin/eland_import_hub_model index bef0770..85c74e7 100755 --- a/bin/eland_import_hub_model +++ b/bin/eland_import_hub_model @@ -31,6 +31,7 @@ import sys import tempfile import textwrap +import torch from elastic_transport.client_utils import DEFAULT from elasticsearch import AuthenticationException, Elasticsearch @@ -150,14 +151,29 @@ def get_es_client(cli_args): es_args['basic_auth'] = (cli_args.es_username, cli_args.es_password) es_client = Elasticsearch(**es_args) - es_info = es_client.info() - logger.info(f"Connected to cluster named '{es_info['cluster_name']}' (version: {es_info['version']['number']})") - return es_client except AuthenticationException as e: logger.error(e) exit(1) +def check_cluster_version(es_client): + es_info = es_client.info() + logger.info(f"Connected to cluster named '{es_info['cluster_name']}' (version: {es_info['version']['number']})") + + major_version = int(es_info['version']['number'].split(".")[0]) + minor_version = int(es_info['version']['number'].split(".")[1]) + + # NLP models added in 8 + if major_version < 8: + logger.error(f"Elasticsearch version {major_version} does not support NLP models. Please upgrade Elasticsearch to the latest version") + exit(1) + + # PyTorch was upgraded to version 1.13.1 in 8.7. + # and is incompatible with earlier versions + if major_version == 8 and minor_version < 7: + logger.error(f"Eland uses PyTorch version {torch.__version__} which is incompatible with Elasticsearch versions prior to 8.7. Please upgrade Elasticsearch to at least version 8.7") + exit(1) + if __name__ == "__main__": # Configure logging @@ -188,6 +204,7 @@ if __name__ == "__main__": # Connect to ES logger.info("Establishing connection to Elasticsearch") es = get_es_client(args) + check_cluster_version(es) # Trace and save model, then upload it from temp file with tempfile.TemporaryDirectory() as tmp_dir: diff --git a/noxfile.py b/noxfile.py index 06bc74f..acd5b6a 100644 --- a/noxfile.py +++ b/noxfile.py @@ -119,8 +119,8 @@ def test(session, pandas_version: str): "--nbval", ) - # PyTorch doesn't support Python 3.10 yet - if session.python == "3.10": + # PyTorch doesn't support Python 3.11 yet + if session.python == "3.11": pytest_args += ("--ignore=eland/ml/pytorch",) session.run( *pytest_args, diff --git a/requirements-dev.txt b/requirements-dev.txt index 3366ebb..4135f25 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -14,13 +14,13 @@ scikit-learn>=0.22.1,<2 xgboost>=0.90,<2 lightgbm>=2,<4 -# PyTorch doesn't support Python 3.10 yet (pytorch/pytorch#66424) +# PyTorch doesn't support Python 3.11 yet (pytorch/pytorch#86566) -# Elasticsearch uses v1.11.0 of PyTorch -torch>=1.11.0,<1.12.0; python_version<'3.10' -# Versions known to be compatible with torch 1.11 -sentence-transformers>=2.1.0,<=2.2.2; python_version<'3.10' -transformers[torch]>=4.12.0,<=4.20.1; python_version<'3.10' +# Elasticsearch uses v1.13.1 of PyTorch +torch>=1.13.1,<2.0; python_version<'3.11' +# Versions known to be compatible with PyTorch 1.13.1 +sentence-transformers>=2.1.0,<=2.2.2; python_version<'3.11' +transformers[torch]>=4.12.0,<=4.27.4; python_version<'3.11' # # Testing diff --git a/setup.py b/setup.py index 6644491..7894754 100644 --- a/setup.py +++ b/setup.py @@ -59,9 +59,9 @@ extras = { "scikit-learn": ["scikit-learn>=0.22.1,<2"], "lightgbm": ["lightgbm>=2,<4"], "pytorch": [ - "torch>=1.11.0,<1.12.0", + "torch>=1.13.1,<2.0", "sentence-transformers>=2.1.0,<=2.2.2", - "transformers[torch]>=4.12.0,<=4.20.1", + "transformers[torch]>=4.12.0,<=4.27.4", ], } extras["all"] = list({dep for deps in extras.values() for dep in deps}) diff --git a/tests/ml/pytorch/test_pytorch_model_upload_pytest.py b/tests/ml/pytorch/test_pytorch_model_upload_pytest.py index 2b0e519..5ee70c8 100644 --- a/tests/ml/pytorch/test_pytorch_model_upload_pytest.py +++ b/tests/ml/pytorch/test_pytorch_model_upload_pytest.py @@ -38,8 +38,8 @@ from tests import ES_TEST_CLIENT, ES_VERSION pytestmark = [ pytest.mark.skipif( - ES_VERSION < (8, 0, 0), - reason="This test requires at least Elasticsearch version 8.0.0", + ES_VERSION < (8, 7, 0), + reason="Eland uses Pytorch 1.13.1, versions of Elasticsearch prior to 8.7.0 are incompatible with PyTorch 1.13.1", ), pytest.mark.skipif( not HAS_SKLEARN, reason="This test requires 'scikit-learn' package to run"