third_codes/eland
1
0
Fork 0
mirror of https://github.com/elastic/eland.git synced 2025-07-11 00:02:14 +08:00
Code Issues Packages Projects Releases Wiki Activity

Refactors eland tests to accept pre-configured client

Browse Source
This commit is contained in:
Winterflower 2019-12-02 20:23:40 +01:00
parent 9c15e237ca
commit 33674088ca
12 changed files with 47 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
eland/tests/__init__.py
View File

@ -1,5 +1,6 @@
import os
from elasticsearch import Elasticsearch
import pandas as pd
ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
@ -7,6 +8,17 @@ ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
# Define test files and indices
ELASTICSEARCH_HOST = os.environ.get('ELASTICSEARCH_HOST') or 'localhost'
# Define client to use in tests
TEST_SUITE = os.environ.get('TEST_SUITE')
if TEST_SUITE == 'xpack':
print('Running xpack tests requires SSL. Setting up SSL enabled client')
certpath = os.path.join(os.path.dirname(__file__), '../../.ci/certs/ca.crt')
print(certpath)
ES_TEST_CLIENT = Elasticsearch(ELASTICSEARCH_HOST, use_ssl=True, verify_certs=True, ca_certs=certpath)
else:
ES_TEST_CLIENT = Elasticsearch(ELASTICSEARCH_HOST)
FLIGHTS_INDEX_NAME = 'flights'
FLIGHTS_MAPPING = {"mappings": {
"properties": {

8
eland/tests/common.py
View File

@ -8,7 +8,7 @@ import eland as ed
ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
# Create pandas and eland data frames
from eland.tests import ELASTICSEARCH_HOST
from eland.tests import ES_TEST_CLIENT
from eland.tests import FLIGHTS_DF_FILE_NAME, FLIGHTS_INDEX_NAME, \
FLIGHTS_SMALL_INDEX_NAME, \
ECOMMERCE_DF_FILE_NAME, ECOMMERCE_INDEX_NAME
@ -17,10 +17,10 @@ _pd_flights = pd.read_json(FLIGHTS_DF_FILE_NAME).sort_index()
_pd_flights['timestamp'] = \
pd.to_datetime(_pd_flights['timestamp'])
_pd_flights.index = _pd_flights.index.map(str) # make index 'object' not int
_ed_flights = ed.read_es(ELASTICSEARCH_HOST, FLIGHTS_INDEX_NAME)
_ed_flights = ed.read_es(ES_TEST_CLIENT, FLIGHTS_INDEX_NAME)
_pd_flights_small = _pd_flights.head(48)
_ed_flights_small = ed.read_es(ELASTICSEARCH_HOST, FLIGHTS_SMALL_INDEX_NAME)
_ed_flights_small = ed.read_es(ES_TEST_CLIENT, FLIGHTS_SMALL_INDEX_NAME)
_pd_ecommerce = pd.read_json(ECOMMERCE_DF_FILE_NAME).sort_index()
_pd_ecommerce['order_date'] = \
@ -30,7 +30,7 @@ _pd_ecommerce['products.created_on'] = \
_pd_ecommerce.insert(2, 'customer_birth_date', None)
_pd_ecommerce.index = _pd_ecommerce.index.map(str) # make index 'object' not int
_pd_ecommerce['customer_birth_date'].astype('datetime64')
_ed_ecommerce = ed.read_es(ELASTICSEARCH_HOST, ECOMMERCE_INDEX_NAME)
_ed_ecommerce = ed.read_es(ES_TEST_CLIENT, ECOMMERCE_INDEX_NAME)
class TestData:

4
eland/tests/dataframe/test_datetime_pytest.py
View File

@ -4,7 +4,7 @@ import numpy as np
import pandas as pd
import eland as ed
from eland.tests.common import ELASTICSEARCH_HOST
from eland.tests.common import ES_TEST_CLIENT
from eland.tests.common import TestData
from eland.tests.common import assert_pandas_eland_frame_equal
@ -37,7 +37,7 @@ class TestDataFrameDateTime(TestData):
# Now create index
index_name = 'eland_test_generate_es_mappings'
ed_df = ed.pandas_to_eland(df, ELASTICSEARCH_HOST, index_name, if_exists="replace", refresh=True)
ed_df = ed.pandas_to_eland(df, ES_TEST_CLIENT, index_name, if_exists="replace", refresh=True)
ed_df_head = ed_df.head()
assert_pandas_eland_frame_equal(df, ed_df_head)

10
eland/tests/dataframe/test_init_pytest.py
View File

@ -3,7 +3,7 @@
import pytest
import eland as ed
from eland.tests import ELASTICSEARCH_HOST
from eland.tests import ES_TEST_CLIENT
from eland.tests import FLIGHTS_INDEX_NAME
@ -16,15 +16,15 @@ class TestDataFrameInit:
# Construct invalid DataFrame (throws)
with pytest.raises(ValueError):
df = ed.DataFrame(client=ELASTICSEARCH_HOST)
df = ed.DataFrame(client=ES_TEST_CLIENT)
# Construct invalid DataFrame (throws)
with pytest.raises(ValueError):
df = ed.DataFrame(index_pattern=FLIGHTS_INDEX_NAME)
# Good constructors
df0 = ed.DataFrame(ELASTICSEARCH_HOST, FLIGHTS_INDEX_NAME)
df1 = ed.DataFrame(client=ELASTICSEARCH_HOST, index_pattern=FLIGHTS_INDEX_NAME)
df0 = ed.DataFrame(ES_TEST_CLIENT, FLIGHTS_INDEX_NAME)
df1 = ed.DataFrame(client=ES_TEST_CLIENT, index_pattern=FLIGHTS_INDEX_NAME)
qc = ed.ElandQueryCompiler(client=ELASTICSEARCH_HOST, index_pattern=FLIGHTS_INDEX_NAME)
qc = ed.ElandQueryCompiler(client=ES_TEST_CLIENT, index_pattern=FLIGHTS_INDEX_NAME)
df2 = ed.DataFrame(query_compiler=qc)

4
eland/tests/dataframe/test_query_pytest.py
View File

@ -3,7 +3,7 @@
import pandas as pd
import eland as ed
from eland.tests.common import ELASTICSEARCH_HOST
from eland.tests.common import ES_TEST_CLIENT
from eland.tests.common import TestData
from eland.tests.common import assert_pandas_eland_frame_equal
@ -19,7 +19,7 @@ class TestDataFrameQuery(TestData):
# Now create index
index_name = 'eland_test_query'
ed_df = ed.pandas_to_eland(pd_df, ELASTICSEARCH_HOST, index_name, if_exists="replace", refresh=True)
ed_df = ed.pandas_to_eland(pd_df, ES_TEST_CLIENT, index_name, if_exists="replace", refresh=True)
assert_pandas_eland_frame_equal(pd_df, ed_df)

5
eland/tests/dataframe/test_to_csv_pytest.py
View File

@ -8,7 +8,7 @@ from elasticsearch import Elasticsearch
from pandas.util.testing import assert_frame_equal
import eland as ed
from eland.tests import ELASTICSEARCH_HOST
from eland.tests import ES_TEST_CLIENT
from eland.tests import FLIGHTS_INDEX_NAME
from eland.tests.common import ROOT_DIR
from eland.tests.common import TestData
@ -52,9 +52,8 @@ class TestDataFrameToCSV(TestData):
now_millis = int(round(time.time() * 1000))
test_index = FLIGHTS_INDEX_NAME + '.' + str(now_millis)
es = Elasticsearch(ELASTICSEARCH_HOST)
ed_flights_from_csv = ed.read_csv(results_file, es, test_index, index_col=0, es_refresh=True,
ed_flights_from_csv = ed.read_csv(results_file, ES_TEST_CLIENT, test_index, index_col=0, es_refresh=True,
es_geo_points=['OriginLocation', 'DestLocation'],
converters={
'DestLocation': lambda x: ast.literal_eval(x),

4
eland/tests/dataframe/test_utils_pytest.py
View File

@ -4,7 +4,7 @@ import numpy as np
import pandas as pd
import eland as ed
from eland.tests.common import ELASTICSEARCH_HOST, assert_pandas_eland_frame_equal
from eland.tests.common import ES_TEST_CLIENT, assert_pandas_eland_frame_equal
from eland.tests.common import TestData
@ -36,7 +36,7 @@ class TestDataFrameUtils(TestData):
# Now create index
index_name = 'eland_test_generate_es_mappings'
ed_df = ed.pandas_to_eland(df, ELASTICSEARCH_HOST, index_name, if_exists="replace", refresh=True)
ed_df = ed.pandas_to_eland(df, ES_TEST_CLIENT, index_name, if_exists="replace", refresh=True)
ed_df_head = ed_df.head()
assert_pandas_eland_frame_equal(df, ed_df_head)

4
eland/tests/series/test_head_tail_pytest.py
View File

@ -1,6 +1,6 @@
# File called _pytest for PyCharm compatability
import eland as ed
from eland.tests import ELASTICSEARCH_HOST
from eland.tests import ES_TEST_CLIENT
from eland.tests import FLIGHTS_INDEX_NAME
from eland.tests.common import TestData
from eland.tests.common import assert_pandas_eland_series_equal
@ -10,7 +10,7 @@ class TestSeriesHeadTail(TestData):
def test_head_tail(self):
pd_s = self.pd_flights()['Carrier']
ed_s = ed.Series(ELASTICSEARCH_HOST, FLIGHTS_INDEX_NAME, 'Carrier')
ed_s = ed.Series(ES_TEST_CLIENT, FLIGHTS_INDEX_NAME, 'Carrier')
pd_s_head = pd_s.head(10)
ed_s_head = ed_s.head(10)

4
eland/tests/series/test_name_pytest.py
View File

@ -1,6 +1,6 @@
# File called _pytest for PyCharm compatability
import eland as ed
from eland.tests import ELASTICSEARCH_HOST
from eland.tests import ES_TEST_CLIENT
from eland.tests import FLIGHTS_INDEX_NAME
from eland.tests.common import TestData
from eland.tests.common import assert_pandas_eland_series_equal
@ -11,7 +11,7 @@ class TestSeriesName(TestData):
def test_name(self):
# deep copy pandas DataFrame as .name alters this reference frame
pd_series = self.pd_flights()['Carrier'].copy(deep=True)
ed_series = ed.Series(ELASTICSEARCH_HOST, FLIGHTS_INDEX_NAME, 'Carrier')
ed_series = ed.Series(ES_TEST_CLIENT, FLIGHTS_INDEX_NAME, 'Carrier')
assert_pandas_eland_series_equal(pd_series, ed_series)
assert ed_series.name == pd_series.name

4
eland/tests/series/test_rename_pytest.py
View File

@ -1,6 +1,6 @@
# File called _pytest for PyCharm compatability
import eland as ed
from eland.tests import ELASTICSEARCH_HOST
from eland.tests import ES_TEST_CLIENT
from eland.tests import FLIGHTS_INDEX_NAME
from eland.tests.common import TestData
from eland.tests.common import assert_pandas_eland_series_equal
@ -10,7 +10,7 @@ class TestSeriesRename(TestData):
def test_rename(self):
pd_carrier = self.pd_flights()['Carrier']
ed_carrier = ed.Series(ELASTICSEARCH_HOST, FLIGHTS_INDEX_NAME, 'Carrier')
ed_carrier = ed.Series(ES_TEST_CLIENT, FLIGHTS_INDEX_NAME, 'Carrier')
assert_pandas_eland_series_equal(pd_carrier, ed_carrier)

6
eland/tests/series/test_repr_pytest.py
View File

@ -1,6 +1,6 @@
# File called _pytest for PyCharm compatability
import eland as ed
from eland.tests import ELASTICSEARCH_HOST
from eland.tests import ES_TEST_CLIENT
from eland.tests import FLIGHTS_INDEX_NAME
from eland.tests.common import TestData
@ -9,7 +9,7 @@ class TestSeriesRepr(TestData):
def test_repr_flights_carrier(self):
pd_s = self.pd_flights()['Carrier']
ed_s = ed.Series(ELASTICSEARCH_HOST, FLIGHTS_INDEX_NAME, 'Carrier')
ed_s = ed.Series(ES_TEST_CLIENT, FLIGHTS_INDEX_NAME, 'Carrier')
pd_repr = repr(pd_s)
ed_repr = repr(ed_s)
@ -18,7 +18,7 @@ class TestSeriesRepr(TestData):
def test_repr_flights_carrier_5(self):
pd_s = self.pd_flights()['Carrier'].head(5)
ed_s = ed.Series(ELASTICSEARCH_HOST, FLIGHTS_INDEX_NAME, 'Carrier').head(5)
ed_s = ed.Series(ES_TEST_CLIENT, FLIGHTS_INDEX_NAME, 'Carrier').head(5)
pd_repr = repr(pd_s)
ed_repr = repr(ed_s)

10
eland/tests/setup_tests.py
View File

@ -78,7 +78,15 @@ def _setup_test_nested(es):
if __name__ == '__main__':
# Create connection to Elasticsearch - use defaults
print('Connecting to ES', ELASTICSEARCH_HOST)
es = Elasticsearch(ELASTICSEARCH_HOST)
TEST_SUITE = os.environ.get('TEST_SUITE')
if TEST_SUITE == 'xpack':
print('Running xpack tests requires SSL. Setting up SSL enabled client')
certpath = os.path.join(os.path.dirname(__file__), '../../.ci/certs/ca.crt')
print(certpath)
es = Elasticsearch(ELASTICSEARCH_HOST, use_ssl=True, verify_certs=True, ca_certs=certpath)
else:
es = Elasticsearch(ELASTICSEARCH_HOST)
_setup_data(es)
_setup_test_mappings(es)