diff --git a/eland/tests/__init__.py b/eland/tests/__init__.py index 50e8930..b79b8c2 100644 --- a/eland/tests/__init__.py +++ b/eland/tests/__init__.py @@ -1,5 +1,6 @@ import os +from elasticsearch import Elasticsearch import pandas as pd ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -7,6 +8,17 @@ ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) # Define test files and indices ELASTICSEARCH_HOST = os.environ.get('ELASTICSEARCH_HOST') or 'localhost' +# Define client to use in tests +TEST_SUITE = os.environ.get('TEST_SUITE') +if TEST_SUITE == 'xpack': + print('Running xpack tests requires SSL. Setting up SSL enabled client') + certpath = os.path.join(os.path.dirname(__file__), '../../.ci/certs/ca.crt') + print(certpath) + ES_TEST_CLIENT = Elasticsearch(ELASTICSEARCH_HOST, use_ssl=True, verify_certs=True, ca_certs=certpath) +else: + ES_TEST_CLIENT = Elasticsearch(ELASTICSEARCH_HOST) + + FLIGHTS_INDEX_NAME = 'flights' FLIGHTS_MAPPING = {"mappings": { "properties": { diff --git a/eland/tests/common.py b/eland/tests/common.py index 6549d73..f4245bc 100644 --- a/eland/tests/common.py +++ b/eland/tests/common.py @@ -8,7 +8,7 @@ import eland as ed ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) # Create pandas and eland data frames -from eland.tests import ELASTICSEARCH_HOST +from eland.tests import ES_TEST_CLIENT from eland.tests import FLIGHTS_DF_FILE_NAME, FLIGHTS_INDEX_NAME, \ FLIGHTS_SMALL_INDEX_NAME, \ ECOMMERCE_DF_FILE_NAME, ECOMMERCE_INDEX_NAME @@ -17,10 +17,10 @@ _pd_flights = pd.read_json(FLIGHTS_DF_FILE_NAME).sort_index() _pd_flights['timestamp'] = \ pd.to_datetime(_pd_flights['timestamp']) _pd_flights.index = _pd_flights.index.map(str) # make index 'object' not int -_ed_flights = ed.read_es(ELASTICSEARCH_HOST, FLIGHTS_INDEX_NAME) +_ed_flights = ed.read_es(ES_TEST_CLIENT, FLIGHTS_INDEX_NAME) _pd_flights_small = _pd_flights.head(48) -_ed_flights_small = ed.read_es(ELASTICSEARCH_HOST, FLIGHTS_SMALL_INDEX_NAME) +_ed_flights_small = ed.read_es(ES_TEST_CLIENT, FLIGHTS_SMALL_INDEX_NAME) _pd_ecommerce = pd.read_json(ECOMMERCE_DF_FILE_NAME).sort_index() _pd_ecommerce['order_date'] = \ @@ -30,7 +30,7 @@ _pd_ecommerce['products.created_on'] = \ _pd_ecommerce.insert(2, 'customer_birth_date', None) _pd_ecommerce.index = _pd_ecommerce.index.map(str) # make index 'object' not int _pd_ecommerce['customer_birth_date'].astype('datetime64') -_ed_ecommerce = ed.read_es(ELASTICSEARCH_HOST, ECOMMERCE_INDEX_NAME) +_ed_ecommerce = ed.read_es(ES_TEST_CLIENT, ECOMMERCE_INDEX_NAME) class TestData: diff --git a/eland/tests/dataframe/test_datetime_pytest.py b/eland/tests/dataframe/test_datetime_pytest.py index 77dc2b8..20a59af 100644 --- a/eland/tests/dataframe/test_datetime_pytest.py +++ b/eland/tests/dataframe/test_datetime_pytest.py @@ -4,7 +4,7 @@ import numpy as np import pandas as pd import eland as ed -from eland.tests.common import ELASTICSEARCH_HOST +from eland.tests.common import ES_TEST_CLIENT from eland.tests.common import TestData from eland.tests.common import assert_pandas_eland_frame_equal @@ -37,7 +37,7 @@ class TestDataFrameDateTime(TestData): # Now create index index_name = 'eland_test_generate_es_mappings' - ed_df = ed.pandas_to_eland(df, ELASTICSEARCH_HOST, index_name, if_exists="replace", refresh=True) + ed_df = ed.pandas_to_eland(df, ES_TEST_CLIENT, index_name, if_exists="replace", refresh=True) ed_df_head = ed_df.head() assert_pandas_eland_frame_equal(df, ed_df_head) diff --git a/eland/tests/dataframe/test_init_pytest.py b/eland/tests/dataframe/test_init_pytest.py index e3f71d2..ee810c0 100644 --- a/eland/tests/dataframe/test_init_pytest.py +++ b/eland/tests/dataframe/test_init_pytest.py @@ -3,7 +3,7 @@ import pytest import eland as ed -from eland.tests import ELASTICSEARCH_HOST +from eland.tests import ES_TEST_CLIENT from eland.tests import FLIGHTS_INDEX_NAME @@ -16,15 +16,15 @@ class TestDataFrameInit: # Construct invalid DataFrame (throws) with pytest.raises(ValueError): - df = ed.DataFrame(client=ELASTICSEARCH_HOST) + df = ed.DataFrame(client=ES_TEST_CLIENT) # Construct invalid DataFrame (throws) with pytest.raises(ValueError): df = ed.DataFrame(index_pattern=FLIGHTS_INDEX_NAME) # Good constructors - df0 = ed.DataFrame(ELASTICSEARCH_HOST, FLIGHTS_INDEX_NAME) - df1 = ed.DataFrame(client=ELASTICSEARCH_HOST, index_pattern=FLIGHTS_INDEX_NAME) + df0 = ed.DataFrame(ES_TEST_CLIENT, FLIGHTS_INDEX_NAME) + df1 = ed.DataFrame(client=ES_TEST_CLIENT, index_pattern=FLIGHTS_INDEX_NAME) - qc = ed.ElandQueryCompiler(client=ELASTICSEARCH_HOST, index_pattern=FLIGHTS_INDEX_NAME) + qc = ed.ElandQueryCompiler(client=ES_TEST_CLIENT, index_pattern=FLIGHTS_INDEX_NAME) df2 = ed.DataFrame(query_compiler=qc) diff --git a/eland/tests/dataframe/test_query_pytest.py b/eland/tests/dataframe/test_query_pytest.py index d2add40..35f7f44 100644 --- a/eland/tests/dataframe/test_query_pytest.py +++ b/eland/tests/dataframe/test_query_pytest.py @@ -3,7 +3,7 @@ import pandas as pd import eland as ed -from eland.tests.common import ELASTICSEARCH_HOST +from eland.tests.common import ES_TEST_CLIENT from eland.tests.common import TestData from eland.tests.common import assert_pandas_eland_frame_equal @@ -19,7 +19,7 @@ class TestDataFrameQuery(TestData): # Now create index index_name = 'eland_test_query' - ed_df = ed.pandas_to_eland(pd_df, ELASTICSEARCH_HOST, index_name, if_exists="replace", refresh=True) + ed_df = ed.pandas_to_eland(pd_df, ES_TEST_CLIENT, index_name, if_exists="replace", refresh=True) assert_pandas_eland_frame_equal(pd_df, ed_df) diff --git a/eland/tests/dataframe/test_to_csv_pytest.py b/eland/tests/dataframe/test_to_csv_pytest.py index fd2ae5e..ed0affd 100644 --- a/eland/tests/dataframe/test_to_csv_pytest.py +++ b/eland/tests/dataframe/test_to_csv_pytest.py @@ -8,7 +8,7 @@ from elasticsearch import Elasticsearch from pandas.util.testing import assert_frame_equal import eland as ed -from eland.tests import ELASTICSEARCH_HOST +from eland.tests import ES_TEST_CLIENT from eland.tests import FLIGHTS_INDEX_NAME from eland.tests.common import ROOT_DIR from eland.tests.common import TestData @@ -52,9 +52,8 @@ class TestDataFrameToCSV(TestData): now_millis = int(round(time.time() * 1000)) test_index = FLIGHTS_INDEX_NAME + '.' + str(now_millis) - es = Elasticsearch(ELASTICSEARCH_HOST) - ed_flights_from_csv = ed.read_csv(results_file, es, test_index, index_col=0, es_refresh=True, + ed_flights_from_csv = ed.read_csv(results_file, ES_TEST_CLIENT, test_index, index_col=0, es_refresh=True, es_geo_points=['OriginLocation', 'DestLocation'], converters={ 'DestLocation': lambda x: ast.literal_eval(x), diff --git a/eland/tests/dataframe/test_utils_pytest.py b/eland/tests/dataframe/test_utils_pytest.py index 992dcc7..b0fd97c 100644 --- a/eland/tests/dataframe/test_utils_pytest.py +++ b/eland/tests/dataframe/test_utils_pytest.py @@ -4,7 +4,7 @@ import numpy as np import pandas as pd import eland as ed -from eland.tests.common import ELASTICSEARCH_HOST, assert_pandas_eland_frame_equal +from eland.tests.common import ES_TEST_CLIENT, assert_pandas_eland_frame_equal from eland.tests.common import TestData @@ -36,7 +36,7 @@ class TestDataFrameUtils(TestData): # Now create index index_name = 'eland_test_generate_es_mappings' - ed_df = ed.pandas_to_eland(df, ELASTICSEARCH_HOST, index_name, if_exists="replace", refresh=True) + ed_df = ed.pandas_to_eland(df, ES_TEST_CLIENT, index_name, if_exists="replace", refresh=True) ed_df_head = ed_df.head() assert_pandas_eland_frame_equal(df, ed_df_head) diff --git a/eland/tests/series/test_head_tail_pytest.py b/eland/tests/series/test_head_tail_pytest.py index 473a0cf..ee449f4 100644 --- a/eland/tests/series/test_head_tail_pytest.py +++ b/eland/tests/series/test_head_tail_pytest.py @@ -1,6 +1,6 @@ # File called _pytest for PyCharm compatability import eland as ed -from eland.tests import ELASTICSEARCH_HOST +from eland.tests import ES_TEST_CLIENT from eland.tests import FLIGHTS_INDEX_NAME from eland.tests.common import TestData from eland.tests.common import assert_pandas_eland_series_equal @@ -10,7 +10,7 @@ class TestSeriesHeadTail(TestData): def test_head_tail(self): pd_s = self.pd_flights()['Carrier'] - ed_s = ed.Series(ELASTICSEARCH_HOST, FLIGHTS_INDEX_NAME, 'Carrier') + ed_s = ed.Series(ES_TEST_CLIENT, FLIGHTS_INDEX_NAME, 'Carrier') pd_s_head = pd_s.head(10) ed_s_head = ed_s.head(10) diff --git a/eland/tests/series/test_name_pytest.py b/eland/tests/series/test_name_pytest.py index f395ae3..c3e72f3 100644 --- a/eland/tests/series/test_name_pytest.py +++ b/eland/tests/series/test_name_pytest.py @@ -1,6 +1,6 @@ # File called _pytest for PyCharm compatability import eland as ed -from eland.tests import ELASTICSEARCH_HOST +from eland.tests import ES_TEST_CLIENT from eland.tests import FLIGHTS_INDEX_NAME from eland.tests.common import TestData from eland.tests.common import assert_pandas_eland_series_equal @@ -11,7 +11,7 @@ class TestSeriesName(TestData): def test_name(self): # deep copy pandas DataFrame as .name alters this reference frame pd_series = self.pd_flights()['Carrier'].copy(deep=True) - ed_series = ed.Series(ELASTICSEARCH_HOST, FLIGHTS_INDEX_NAME, 'Carrier') + ed_series = ed.Series(ES_TEST_CLIENT, FLIGHTS_INDEX_NAME, 'Carrier') assert_pandas_eland_series_equal(pd_series, ed_series) assert ed_series.name == pd_series.name diff --git a/eland/tests/series/test_rename_pytest.py b/eland/tests/series/test_rename_pytest.py index 96cc19f..0f93fb3 100644 --- a/eland/tests/series/test_rename_pytest.py +++ b/eland/tests/series/test_rename_pytest.py @@ -1,6 +1,6 @@ # File called _pytest for PyCharm compatability import eland as ed -from eland.tests import ELASTICSEARCH_HOST +from eland.tests import ES_TEST_CLIENT from eland.tests import FLIGHTS_INDEX_NAME from eland.tests.common import TestData from eland.tests.common import assert_pandas_eland_series_equal @@ -10,7 +10,7 @@ class TestSeriesRename(TestData): def test_rename(self): pd_carrier = self.pd_flights()['Carrier'] - ed_carrier = ed.Series(ELASTICSEARCH_HOST, FLIGHTS_INDEX_NAME, 'Carrier') + ed_carrier = ed.Series(ES_TEST_CLIENT, FLIGHTS_INDEX_NAME, 'Carrier') assert_pandas_eland_series_equal(pd_carrier, ed_carrier) diff --git a/eland/tests/series/test_repr_pytest.py b/eland/tests/series/test_repr_pytest.py index 6b3dec0..8d5bed4 100644 --- a/eland/tests/series/test_repr_pytest.py +++ b/eland/tests/series/test_repr_pytest.py @@ -1,6 +1,6 @@ # File called _pytest for PyCharm compatability import eland as ed -from eland.tests import ELASTICSEARCH_HOST +from eland.tests import ES_TEST_CLIENT from eland.tests import FLIGHTS_INDEX_NAME from eland.tests.common import TestData @@ -9,7 +9,7 @@ class TestSeriesRepr(TestData): def test_repr_flights_carrier(self): pd_s = self.pd_flights()['Carrier'] - ed_s = ed.Series(ELASTICSEARCH_HOST, FLIGHTS_INDEX_NAME, 'Carrier') + ed_s = ed.Series(ES_TEST_CLIENT, FLIGHTS_INDEX_NAME, 'Carrier') pd_repr = repr(pd_s) ed_repr = repr(ed_s) @@ -18,7 +18,7 @@ class TestSeriesRepr(TestData): def test_repr_flights_carrier_5(self): pd_s = self.pd_flights()['Carrier'].head(5) - ed_s = ed.Series(ELASTICSEARCH_HOST, FLIGHTS_INDEX_NAME, 'Carrier').head(5) + ed_s = ed.Series(ES_TEST_CLIENT, FLIGHTS_INDEX_NAME, 'Carrier').head(5) pd_repr = repr(pd_s) ed_repr = repr(ed_s) diff --git a/eland/tests/setup_tests.py b/eland/tests/setup_tests.py index 43f78b0..3c7c616 100644 --- a/eland/tests/setup_tests.py +++ b/eland/tests/setup_tests.py @@ -78,7 +78,15 @@ def _setup_test_nested(es): if __name__ == '__main__': # Create connection to Elasticsearch - use defaults print('Connecting to ES', ELASTICSEARCH_HOST) - es = Elasticsearch(ELASTICSEARCH_HOST) + TEST_SUITE = os.environ.get('TEST_SUITE') + if TEST_SUITE == 'xpack': + print('Running xpack tests requires SSL. Setting up SSL enabled client') + certpath = os.path.join(os.path.dirname(__file__), '../../.ci/certs/ca.crt') + print(certpath) + es = Elasticsearch(ELASTICSEARCH_HOST, use_ssl=True, verify_certs=True, ca_certs=certpath) + else: + es = Elasticsearch(ELASTICSEARCH_HOST) + _setup_data(es) _setup_test_mappings(es)