third_codes/eland
1
0
Fork 0
mirror of https://github.com/elastic/eland.git synced 2025-07-11 00:02:14 +08:00
Code Issues Packages Projects Releases Wiki Activity

Refactors eland tests to accept pre-configured client

Browse Source
This commit is contained in:
Winterflower 2019-12-02 20:23:40 +01:00
parent 9c15e237ca
commit 33674088ca
12 changed files with 47 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
eland/tests/__init__.py
View File

@ -1,5 +1,6 @@
import os import os
from elasticsearch import Elasticsearch
import pandas as pd import pandas as pd
ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
@ -7,6 +8,17 @@ ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
# Define test files and indices # Define test files and indices
ELASTICSEARCH_HOST = os.environ.get('ELASTICSEARCH_HOST') or 'localhost' ELASTICSEARCH_HOST = os.environ.get('ELASTICSEARCH_HOST') or 'localhost'
# Define client to use in tests
TEST_SUITE = os.environ.get('TEST_SUITE')
if TEST_SUITE == 'xpack':
print('Running xpack tests requires SSL. Setting up SSL enabled client')
certpath = os.path.join(os.path.dirname(__file__), '../../.ci/certs/ca.crt')
print(certpath)
ES_TEST_CLIENT = Elasticsearch(ELASTICSEARCH_HOST, use_ssl=True, verify_certs=True, ca_certs=certpath)
else:
ES_TEST_CLIENT = Elasticsearch(ELASTICSEARCH_HOST)
FLIGHTS_INDEX_NAME = 'flights' FLIGHTS_INDEX_NAME = 'flights'
FLIGHTS_MAPPING = {"mappings": { FLIGHTS_MAPPING = {"mappings": {
"properties": { "properties": {

8
eland/tests/common.py
View File

@ -8,7 +8,7 @@ import eland as ed
ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
# Create pandas and eland data frames # Create pandas and eland data frames
from eland.tests import ELASTICSEARCH_HOST from eland.tests import ES_TEST_CLIENT
from eland.tests import FLIGHTS_DF_FILE_NAME, FLIGHTS_INDEX_NAME, \ from eland.tests import FLIGHTS_DF_FILE_NAME, FLIGHTS_INDEX_NAME, \
FLIGHTS_SMALL_INDEX_NAME, \ FLIGHTS_SMALL_INDEX_NAME, \
ECOMMERCE_DF_FILE_NAME, ECOMMERCE_INDEX_NAME ECOMMERCE_DF_FILE_NAME, ECOMMERCE_INDEX_NAME
@ -17,10 +17,10 @@ _pd_flights = pd.read_json(FLIGHTS_DF_FILE_NAME).sort_index()
_pd_flights['timestamp'] = \ _pd_flights['timestamp'] = \
pd.to_datetime(_pd_flights['timestamp']) pd.to_datetime(_pd_flights['timestamp'])
_pd_flights.index = _pd_flights.index.map(str) # make index 'object' not int _pd_flights.index = _pd_flights.index.map(str) # make index 'object' not int
_ed_flights = ed.read_es(ELASTICSEARCH_HOST, FLIGHTS_INDEX_NAME) _ed_flights = ed.read_es(ES_TEST_CLIENT, FLIGHTS_INDEX_NAME)
_pd_flights_small = _pd_flights.head(48) _pd_flights_small = _pd_flights.head(48)
_ed_flights_small = ed.read_es(ELASTICSEARCH_HOST, FLIGHTS_SMALL_INDEX_NAME) _ed_flights_small = ed.read_es(ES_TEST_CLIENT, FLIGHTS_SMALL_INDEX_NAME)
_pd_ecommerce = pd.read_json(ECOMMERCE_DF_FILE_NAME).sort_index() _pd_ecommerce = pd.read_json(ECOMMERCE_DF_FILE_NAME).sort_index()
_pd_ecommerce['order_date'] = \ _pd_ecommerce['order_date'] = \
@ -30,7 +30,7 @@ _pd_ecommerce['products.created_on'] = \
_pd_ecommerce.insert(2, 'customer_birth_date', None) _pd_ecommerce.insert(2, 'customer_birth_date', None)
_pd_ecommerce.index = _pd_ecommerce.index.map(str) # make index 'object' not int _pd_ecommerce.index = _pd_ecommerce.index.map(str) # make index 'object' not int
_pd_ecommerce['customer_birth_date'].astype('datetime64') _pd_ecommerce['customer_birth_date'].astype('datetime64')
_ed_ecommerce = ed.read_es(ELASTICSEARCH_HOST, ECOMMERCE_INDEX_NAME) _ed_ecommerce = ed.read_es(ES_TEST_CLIENT, ECOMMERCE_INDEX_NAME)
class TestData: class TestData:

4
eland/tests/dataframe/test_datetime_pytest.py
View File

@ -4,7 +4,7 @@ import numpy as np
import pandas as pd import pandas as pd
import eland as ed import eland as ed
from eland.tests.common import ELASTICSEARCH_HOST from eland.tests.common import ES_TEST_CLIENT
from eland.tests.common import TestData from eland.tests.common import TestData
from eland.tests.common import assert_pandas_eland_frame_equal from eland.tests.common import assert_pandas_eland_frame_equal
@ -37,7 +37,7 @@ class TestDataFrameDateTime(TestData):
# Now create index # Now create index
index_name = 'eland_test_generate_es_mappings' index_name = 'eland_test_generate_es_mappings'
ed_df = ed.pandas_to_eland(df, ELASTICSEARCH_HOST, index_name, if_exists="replace", refresh=True) ed_df = ed.pandas_to_eland(df, ES_TEST_CLIENT, index_name, if_exists="replace", refresh=True)
ed_df_head = ed_df.head() ed_df_head = ed_df.head()
assert_pandas_eland_frame_equal(df, ed_df_head) assert_pandas_eland_frame_equal(df, ed_df_head)

10
eland/tests/dataframe/test_init_pytest.py
View File

@ -3,7 +3,7 @@
import pytest import pytest
import eland as ed import eland as ed
from eland.tests import ELASTICSEARCH_HOST from eland.tests import ES_TEST_CLIENT
from eland.tests import FLIGHTS_INDEX_NAME from eland.tests import FLIGHTS_INDEX_NAME
@ -16,15 +16,15 @@ class TestDataFrameInit:
# Construct invalid DataFrame (throws) # Construct invalid DataFrame (throws)
with pytest.raises(ValueError): with pytest.raises(ValueError):
df = ed.DataFrame(client=ELASTICSEARCH_HOST) df = ed.DataFrame(client=ES_TEST_CLIENT)
# Construct invalid DataFrame (throws) # Construct invalid DataFrame (throws)
with pytest.raises(ValueError): with pytest.raises(ValueError):
df = ed.DataFrame(index_pattern=FLIGHTS_INDEX_NAME) df = ed.DataFrame(index_pattern=FLIGHTS_INDEX_NAME)
# Good constructors # Good constructors
df0 = ed.DataFrame(ELASTICSEARCH_HOST, FLIGHTS_INDEX_NAME) df0 = ed.DataFrame(ES_TEST_CLIENT, FLIGHTS_INDEX_NAME)
df1 = ed.DataFrame(client=ELASTICSEARCH_HOST, index_pattern=FLIGHTS_INDEX_NAME) df1 = ed.DataFrame(client=ES_TEST_CLIENT, index_pattern=FLIGHTS_INDEX_NAME)
qc = ed.ElandQueryCompiler(client=ELASTICSEARCH_HOST, index_pattern=FLIGHTS_INDEX_NAME) qc = ed.ElandQueryCompiler(client=ES_TEST_CLIENT, index_pattern=FLIGHTS_INDEX_NAME)
df2 = ed.DataFrame(query_compiler=qc) df2 = ed.DataFrame(query_compiler=qc)

4
eland/tests/dataframe/test_query_pytest.py
View File

@ -3,7 +3,7 @@
import pandas as pd import pandas as pd
import eland as ed import eland as ed
from eland.tests.common import ELASTICSEARCH_HOST from eland.tests.common import ES_TEST_CLIENT
from eland.tests.common import TestData from eland.tests.common import TestData
from eland.tests.common import assert_pandas_eland_frame_equal from eland.tests.common import assert_pandas_eland_frame_equal
@ -19,7 +19,7 @@ class TestDataFrameQuery(TestData):
# Now create index # Now create index
index_name = 'eland_test_query' index_name = 'eland_test_query'
ed_df = ed.pandas_to_eland(pd_df, ELASTICSEARCH_HOST, index_name, if_exists="replace", refresh=True) ed_df = ed.pandas_to_eland(pd_df, ES_TEST_CLIENT, index_name, if_exists="replace", refresh=True)
assert_pandas_eland_frame_equal(pd_df, ed_df) assert_pandas_eland_frame_equal(pd_df, ed_df)

5
eland/tests/dataframe/test_to_csv_pytest.py
View File

@ -8,7 +8,7 @@ from elasticsearch import Elasticsearch
from pandas.util.testing import assert_frame_equal from pandas.util.testing import assert_frame_equal
import eland as ed import eland as ed
from eland.tests import ELASTICSEARCH_HOST from eland.tests import ES_TEST_CLIENT
from eland.tests import FLIGHTS_INDEX_NAME from eland.tests import FLIGHTS_INDEX_NAME
from eland.tests.common import ROOT_DIR from eland.tests.common import ROOT_DIR
from eland.tests.common import TestData from eland.tests.common import TestData
@ -52,9 +52,8 @@ class TestDataFrameToCSV(TestData):
now_millis = int(round(time.time() * 1000)) now_millis = int(round(time.time() * 1000))
test_index = FLIGHTS_INDEX_NAME + '.' + str(now_millis) test_index = FLIGHTS_INDEX_NAME + '.' + str(now_millis)
es = Elasticsearch(ELASTICSEARCH_HOST)
ed_flights_from_csv = ed.read_csv(results_file, es, test_index, index_col=0, es_refresh=True, ed_flights_from_csv = ed.read_csv(results_file, ES_TEST_CLIENT, test_index, index_col=0, es_refresh=True,
es_geo_points=['OriginLocation', 'DestLocation'], es_geo_points=['OriginLocation', 'DestLocation'],
converters={ converters={
'DestLocation': lambda x: ast.literal_eval(x), 'DestLocation': lambda x: ast.literal_eval(x),

4
eland/tests/dataframe/test_utils_pytest.py
View File

@ -4,7 +4,7 @@ import numpy as np
import pandas as pd import pandas as pd
import eland as ed import eland as ed
from eland.tests.common import ELASTICSEARCH_HOST, assert_pandas_eland_frame_equal from eland.tests.common import ES_TEST_CLIENT, assert_pandas_eland_frame_equal
from eland.tests.common import TestData from eland.tests.common import TestData
@ -36,7 +36,7 @@ class TestDataFrameUtils(TestData):
# Now create index # Now create index
index_name = 'eland_test_generate_es_mappings' index_name = 'eland_test_generate_es_mappings'
ed_df = ed.pandas_to_eland(df, ELASTICSEARCH_HOST, index_name, if_exists="replace", refresh=True) ed_df = ed.pandas_to_eland(df, ES_TEST_CLIENT, index_name, if_exists="replace", refresh=True)
ed_df_head = ed_df.head() ed_df_head = ed_df.head()
assert_pandas_eland_frame_equal(df, ed_df_head) assert_pandas_eland_frame_equal(df, ed_df_head)

4
eland/tests/series/test_head_tail_pytest.py
View File

@ -1,6 +1,6 @@
# File called _pytest for PyCharm compatability # File called _pytest for PyCharm compatability
import eland as ed import eland as ed
from eland.tests import ELASTICSEARCH_HOST from eland.tests import ES_TEST_CLIENT
from eland.tests import FLIGHTS_INDEX_NAME from eland.tests import FLIGHTS_INDEX_NAME
from eland.tests.common import TestData from eland.tests.common import TestData
from eland.tests.common import assert_pandas_eland_series_equal from eland.tests.common import assert_pandas_eland_series_equal
@ -10,7 +10,7 @@ class TestSeriesHeadTail(TestData):
def test_head_tail(self): def test_head_tail(self):
pd_s = self.pd_flights()['Carrier'] pd_s = self.pd_flights()['Carrier']
ed_s = ed.Series(ELASTICSEARCH_HOST, FLIGHTS_INDEX_NAME, 'Carrier') ed_s = ed.Series(ES_TEST_CLIENT, FLIGHTS_INDEX_NAME, 'Carrier')
pd_s_head = pd_s.head(10) pd_s_head = pd_s.head(10)
ed_s_head = ed_s.head(10) ed_s_head = ed_s.head(10)

4
eland/tests/series/test_name_pytest.py
View File

@ -1,6 +1,6 @@
# File called _pytest for PyCharm compatability # File called _pytest for PyCharm compatability
import eland as ed import eland as ed
from eland.tests import ELASTICSEARCH_HOST from eland.tests import ES_TEST_CLIENT
from eland.tests import FLIGHTS_INDEX_NAME from eland.tests import FLIGHTS_INDEX_NAME
from eland.tests.common import TestData from eland.tests.common import TestData
from eland.tests.common import assert_pandas_eland_series_equal from eland.tests.common import assert_pandas_eland_series_equal
@ -11,7 +11,7 @@ class TestSeriesName(TestData):
def test_name(self): def test_name(self):
# deep copy pandas DataFrame as .name alters this reference frame # deep copy pandas DataFrame as .name alters this reference frame
pd_series = self.pd_flights()['Carrier'].copy(deep=True) pd_series = self.pd_flights()['Carrier'].copy(deep=True)
ed_series = ed.Series(ELASTICSEARCH_HOST, FLIGHTS_INDEX_NAME, 'Carrier') ed_series = ed.Series(ES_TEST_CLIENT, FLIGHTS_INDEX_NAME, 'Carrier')
assert_pandas_eland_series_equal(pd_series, ed_series) assert_pandas_eland_series_equal(pd_series, ed_series)
assert ed_series.name == pd_series.name assert ed_series.name == pd_series.name

4
eland/tests/series/test_rename_pytest.py
View File

@ -1,6 +1,6 @@
# File called _pytest for PyCharm compatability # File called _pytest for PyCharm compatability
import eland as ed import eland as ed
from eland.tests import ELASTICSEARCH_HOST from eland.tests import ES_TEST_CLIENT
from eland.tests import FLIGHTS_INDEX_NAME from eland.tests import FLIGHTS_INDEX_NAME
from eland.tests.common import TestData from eland.tests.common import TestData
from eland.tests.common import assert_pandas_eland_series_equal from eland.tests.common import assert_pandas_eland_series_equal
@ -10,7 +10,7 @@ class TestSeriesRename(TestData):
def test_rename(self): def test_rename(self):
pd_carrier = self.pd_flights()['Carrier'] pd_carrier = self.pd_flights()['Carrier']
ed_carrier = ed.Series(ELASTICSEARCH_HOST, FLIGHTS_INDEX_NAME, 'Carrier') ed_carrier = ed.Series(ES_TEST_CLIENT, FLIGHTS_INDEX_NAME, 'Carrier')
assert_pandas_eland_series_equal(pd_carrier, ed_carrier) assert_pandas_eland_series_equal(pd_carrier, ed_carrier)

6
eland/tests/series/test_repr_pytest.py
View File

@ -1,6 +1,6 @@
# File called _pytest for PyCharm compatability # File called _pytest for PyCharm compatability
import eland as ed import eland as ed
from eland.tests import ELASTICSEARCH_HOST from eland.tests import ES_TEST_CLIENT
from eland.tests import FLIGHTS_INDEX_NAME from eland.tests import FLIGHTS_INDEX_NAME
from eland.tests.common import TestData from eland.tests.common import TestData
@ -9,7 +9,7 @@ class TestSeriesRepr(TestData):
def test_repr_flights_carrier(self): def test_repr_flights_carrier(self):
pd_s = self.pd_flights()['Carrier'] pd_s = self.pd_flights()['Carrier']
ed_s = ed.Series(ELASTICSEARCH_HOST, FLIGHTS_INDEX_NAME, 'Carrier') ed_s = ed.Series(ES_TEST_CLIENT, FLIGHTS_INDEX_NAME, 'Carrier')
pd_repr = repr(pd_s) pd_repr = repr(pd_s)
ed_repr = repr(ed_s) ed_repr = repr(ed_s)
@ -18,7 +18,7 @@ class TestSeriesRepr(TestData):
def test_repr_flights_carrier_5(self): def test_repr_flights_carrier_5(self):
pd_s = self.pd_flights()['Carrier'].head(5) pd_s = self.pd_flights()['Carrier'].head(5)
ed_s = ed.Series(ELASTICSEARCH_HOST, FLIGHTS_INDEX_NAME, 'Carrier').head(5) ed_s = ed.Series(ES_TEST_CLIENT, FLIGHTS_INDEX_NAME, 'Carrier').head(5)
pd_repr = repr(pd_s) pd_repr = repr(pd_s)
ed_repr = repr(ed_s) ed_repr = repr(ed_s)

8
eland/tests/setup_tests.py
View File

@ -78,8 +78,16 @@ def _setup_test_nested(es):
if __name__ == '__main__': if __name__ == '__main__':
# Create connection to Elasticsearch - use defaults # Create connection to Elasticsearch - use defaults
print('Connecting to ES', ELASTICSEARCH_HOST) print('Connecting to ES', ELASTICSEARCH_HOST)
TEST_SUITE = os.environ.get('TEST_SUITE')
if TEST_SUITE == 'xpack':
print('Running xpack tests requires SSL. Setting up SSL enabled client')
certpath = os.path.join(os.path.dirname(__file__), '../../.ci/certs/ca.crt')
print(certpath)
es = Elasticsearch(ELASTICSEARCH_HOST, use_ssl=True, verify_certs=True, ca_certs=certpath)
else:
es = Elasticsearch(ELASTICSEARCH_HOST) es = Elasticsearch(ELASTICSEARCH_HOST)
_setup_data(es) _setup_data(es)
_setup_test_mappings(es) _setup_test_mappings(es)
_setup_test_nested(es) _setup_test_nested(es)