Upgrade torch to 1.13.1 and check the cluster version before uploading a NLP model. (#522)

PyTorch models traced in version 1.13 of PyTorch cannot be evaluated in 
version 1.9 or earlier. With this upgrade Eland becomes incompatible with
pre 8.7 Elasticsearch and will refuse to upload a model to the cluster. 
In this scenario either upgrade Elasticsearch or use an earlier version of Eland.

README.md

@@ -49,13 +49,14 @@ $ conda install -c conda-forge eland
 
 ### Compatibility
 
-- Supports Python 3.8+ and Pandas 1.5
+- Supports Python 3.8, 3.9, 3.10 and Pandas 1.5
 - Supports Elasticsearch clusters that are 7.11+, recommended 8.3 or later for all features to work.
   If you are using the NLP with PyTorch feature make sure your Eland minor version matches the minor 
   version of your Elasticsearch cluster. For all other features it is sufficient for the major versions
   to match.
-- You need to use PyTorch `1.11.0` or earlier to import an NLP model. 
-  Run `pip install torch==1.11` to install the aproppriate version of PyTorch.
+- You need to use PyTorch `1.13.1` or earlier to import an NLP model. 
+  Run `pip install torch==1.13.1` to install the aproppriate version of PyTorch.
+  
 
 ### Prerequisites
 

bin/eland_import_hub_model

@@ -31,6 +31,7 @@ import sys
 import tempfile
 import textwrap
 
+import torch
 from elastic_transport.client_utils import DEFAULT
 from elasticsearch import AuthenticationException, Elasticsearch
 
@@ -150,14 +151,29 @@ def get_es_client(cli_args):
             es_args['basic_auth'] = (cli_args.es_username, cli_args.es_password)
 
         es_client = Elasticsearch(**es_args)
-        es_info = es_client.info()
-        logger.info(f"Connected to cluster named '{es_info['cluster_name']}' (version: {es_info['version']['number']})")
-
         return es_client
     except AuthenticationException as e:
         logger.error(e)
         exit(1)
 
+def check_cluster_version(es_client): 
+    es_info = es_client.info()
+    logger.info(f"Connected to cluster named '{es_info['cluster_name']}' (version: {es_info['version']['number']})")
+
+    major_version = int(es_info['version']['number'].split(".")[0])
+    minor_version = int(es_info['version']['number'].split(".")[1])
+
+    # NLP models added in 8
+    if major_version < 8:
+        logger.error(f"Elasticsearch version {major_version} does not support NLP models. Please upgrade Elasticsearch to the latest version")
+        exit(1)
+
+    # PyTorch was upgraded to version 1.13.1 in 8.7.
+    # and is incompatible with earlier versions
+    if major_version == 8 and minor_version < 7:
+        logger.error(f"Eland uses PyTorch version {torch.__version__} which is incompatible with Elasticsearch versions prior to 8.7. Please upgrade Elasticsearch to at least version 8.7")
+        exit(1)
+
 
 if __name__ == "__main__":
     # Configure logging
@@ -188,6 +204,7 @@ if __name__ == "__main__":
     # Connect to ES
     logger.info("Establishing connection to Elasticsearch")
     es = get_es_client(args)
+    check_cluster_version(es)
 
     # Trace and save model, then upload it from temp file
     with tempfile.TemporaryDirectory() as tmp_dir:

noxfile.py

@@ -119,8 +119,8 @@ def test(session, pandas_version: str):
         "--nbval",
     )
 
-    # PyTorch doesn't support Python 3.10 yet
-    if session.python == "3.10":
+    # PyTorch doesn't support Python 3.11 yet
+    if session.python == "3.11":
         pytest_args += ("--ignore=eland/ml/pytorch",)
     session.run(
         *pytest_args,

requirements-dev.txt

@@ -14,13 +14,13 @@ scikit-learn>=0.22.1,<2
 xgboost>=0.90,<2
 lightgbm>=2,<4
 
-# PyTorch doesn't support Python 3.10 yet (pytorch/pytorch#66424)
+# PyTorch doesn't support Python 3.11 yet (pytorch/pytorch#86566)
 
-# Elasticsearch uses v1.11.0 of PyTorch
-torch>=1.11.0,<1.12.0; python_version<'3.10'
-# Versions known to be compatible with torch 1.11
-sentence-transformers>=2.1.0,<=2.2.2; python_version<'3.10'
-transformers[torch]>=4.12.0,<=4.20.1; python_version<'3.10'
+# Elasticsearch uses v1.13.1 of PyTorch
+torch>=1.13.1,<2.0; python_version<'3.11'
+# Versions known to be compatible with PyTorch 1.13.1
+sentence-transformers>=2.1.0,<=2.2.2; python_version<'3.11'
+transformers[torch]>=4.12.0,<=4.27.4; python_version<'3.11'
 
 #
 # Testing

setup.py

@@ -59,9 +59,9 @@ extras = {
     "scikit-learn": ["scikit-learn>=0.22.1,<2"],
     "lightgbm": ["lightgbm>=2,<4"],
     "pytorch": [
-        "torch>=1.11.0,<1.12.0",
+        "torch>=1.13.1,<2.0",
         "sentence-transformers>=2.1.0,<=2.2.2",
-        "transformers[torch]>=4.12.0,<=4.20.1",
+        "transformers[torch]>=4.12.0,<=4.27.4",
     ],
 }
 extras["all"] = list({dep for deps in extras.values() for dep in deps})

tests/ml/pytorch/test_pytorch_model_upload_pytest.py

@@ -38,8 +38,8 @@ from tests import ES_TEST_CLIENT, ES_VERSION
 
 pytestmark = [
     pytest.mark.skipif(
-        ES_VERSION < (8, 0, 0),
-        reason="This test requires at least Elasticsearch version 8.0.0",
+        ES_VERSION < (8, 7, 0),
+        reason="Eland uses Pytorch 1.13.1, versions of Elasticsearch prior to 8.7.0 are incompatible with PyTorch 1.13.1",
     ),
     pytest.mark.skipif(
         not HAS_SKLEARN, reason="This test requires 'scikit-learn' package to run"