Fix bugs with field mapping and lint issue (#346)

* Fix bugs with field mapping:

1. If no permission to call _mapping, return readable error
2. If index is wildcard, fix issues with user warnings

* Fixing lint issues

* Removing trailing backslashes in doc

* Remove pandas/matplotlib deprecation warning

This warning is due to a conflict between
pandas/matplotlib that may be resolved in a later
version. For now, ignore the warning so CI works.
This commit is contained in:
Stephen Dodson 2021-03-30 14:49:54 +00:00 committed by GitHub
parent 985afe74e0
commit 1040160451
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 189 additions and 2691 deletions

View File

@ -197,13 +197,13 @@ currently using a minimum version of PyCharm 2019.2.4.
* To run the automatic formatter and check for lint issues run
``` bash
> nox -s format`
> nox -s format
```
* To test specific versions of Python run
``` bash
> nox -s test-3.8`
> nox -s test-3.8
```
### Documentation

View File

@ -202,6 +202,11 @@ class FieldMappings:
)
get_mapping = client.indices.get_mapping(index=index_pattern)
if not get_mapping: # dict is empty
raise ValueError(
f"Can not get mapping for {index_pattern} "
f"check indexes exist and client has permission to get mapping."
)
# Get all fields (including all nested) and then all field_caps
all_fields = FieldMappings._extract_fields_from_mapping(get_mapping)
@ -307,7 +312,10 @@ class FieldMappings:
if field_type == "date" and "format" in x:
date_format = x["format"]
# If there is a conflicting type, warn - first values added wins
if field_name in fields and fields[field_name] != field_type:
if field_name in fields and fields[field_name] != (
field_type,
date_format,
):
warnings.warn(
f"Field {field_name} has conflicting types "
f"{fields[field_name]} != {field_type}",
@ -403,14 +411,14 @@ class FieldMappings:
if "non_aggregatable_indices" in vv:
warnings.warn(
"Field {} has conflicting aggregatable fields across indexes {}",
format(field, vv["non_aggregatable_indices"]),
f"Field {field} has conflicting aggregatable fields across indexes "
f"{str(vv['non_aggregatable_indices'])}",
UserWarning,
)
if "non_searchable_indices" in vv:
warnings.warn(
"Field {} has conflicting searchable fields across indexes {}",
format(field, vv["non_searchable_indices"]),
f"Field {field} has conflicting searchable fields across indexes "
f"{str(vv['non_searchable_indices'])}",
UserWarning,
)

View File

@ -65,19 +65,15 @@ except ImportError:
try:
from .xgboost import _MODEL_TRANSFORMERS as _XGBOOST_MODEL_TRANSFORMERS
from .xgboost import (
XGBClassifier,
XGBoostClassifierTransformer,
XGBoostForestTransformer,
XGBoostRegressorTransformer,
XGBRegressor,
)
__all__ += [
"XGBoostClassifierTransformer",
"XGBClassifier",
"XGBoostForestTransformer",
"XGBoostRegressorTransformer",
"XGBRegressor",
]
_MODEL_TRANSFORMERS.update(_XGBOOST_MODEL_TRANSFORMERS)
except ImportError:
@ -86,16 +82,12 @@ except ImportError:
try:
from .lightgbm import _MODEL_TRANSFORMERS as _LIGHTGBM_MODEL_TRANSFORMERS
from .lightgbm import (
LGBMClassifier,
LGBMClassifierTransformer,
LGBMForestTransformer,
LGBMRegressor,
LGBMRegressorTransformer,
)
__all__ += [
"LGBMRegressor",
"LGBMClassifier",
"LGBMForestTransformer",
"LGBMRegressorTransformer",
"LGBMClassifierTransformer",

File diff suppressed because one or more lines are too long

View File

@ -18,9 +18,7 @@
"outputs": [
{
"data": {
"text/plain": [
"False"
]
"text/plain": "False"
},
"execution_count": 2,
"metadata": {},
@ -42,8 +40,8 @@
"name": "stdout",
"output_type": "stream",
"text": [
"2020-10-28 22:01:46.397163: read 10000 rows\n",
"2020-10-28 22:01:47.100938: read 13059 rows\n"
"2021-03-30 11:57:39.116425: read 10000 rows\n",
"2021-03-30 11:57:39.522722: read 13059 rows\n"
]
}
],
@ -59,9 +57,7 @@
"outputs": [
{
"data": {
"text/plain": [
"pandas.core.frame.DataFrame"
]
"text/plain": "pandas.core.frame.DataFrame"
},
"execution_count": 4,
"metadata": {},
@ -79,146 +75,8 @@
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "3e3e6e7371be43aabd4f9a2bb62ed737",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"HBox(children=(HTML(value='Progress'), FloatProgress(value=0.0, max=2.0), HTML(value='')))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>account length</th>\n",
" <th>area code</th>\n",
" <th>churn</th>\n",
" <th>customer service calls</th>\n",
" <th>international plan</th>\n",
" <th>number vmail messages</th>\n",
" <th>phone number</th>\n",
" <th>state</th>\n",
" <th>total day calls</th>\n",
" <th>total day charge</th>\n",
" <th>...</th>\n",
" <th>total eve calls</th>\n",
" <th>total eve charge</th>\n",
" <th>total eve minutes</th>\n",
" <th>total intl calls</th>\n",
" <th>total intl charge</th>\n",
" <th>total intl minutes</th>\n",
" <th>total night calls</th>\n",
" <th>total night charge</th>\n",
" <th>total night minutes</th>\n",
" <th>voice mail plan</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>128</td>\n",
" <td>415</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>no</td>\n",
" <td>25</td>\n",
" <td>382-4657</td>\n",
" <td>KS</td>\n",
" <td>110</td>\n",
" <td>45.07</td>\n",
" <td>...</td>\n",
" <td>99</td>\n",
" <td>16.78</td>\n",
" <td>197.4</td>\n",
" <td>3</td>\n",
" <td>2.7</td>\n",
" <td>10.0</td>\n",
" <td>91</td>\n",
" <td>11.01</td>\n",
" <td>244.7</td>\n",
" <td>yes</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>107</td>\n",
" <td>415</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>no</td>\n",
" <td>26</td>\n",
" <td>371-7191</td>\n",
" <td>OH</td>\n",
" <td>123</td>\n",
" <td>27.47</td>\n",
" <td>...</td>\n",
" <td>103</td>\n",
" <td>16.62</td>\n",
" <td>195.5</td>\n",
" <td>3</td>\n",
" <td>3.7</td>\n",
" <td>13.7</td>\n",
" <td>103</td>\n",
" <td>11.45</td>\n",
" <td>254.4</td>\n",
" <td>yes</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>\n",
"<p>2 rows × 21 columns</p>"
],
"text/plain": [
" account length area code churn customer service calls \\\n",
"0 128 415 0 1 \n",
"1 107 415 0 1 \n",
"\n",
" international plan number vmail messages phone number state \\\n",
"0 no 25 382-4657 KS \n",
"1 no 26 371-7191 OH \n",
"\n",
" total day calls total day charge ... total eve calls total eve charge \\\n",
"0 110 45.07 ... 99 16.78 \n",
"1 123 27.47 ... 103 16.62 \n",
"\n",
" total eve minutes total intl calls total intl charge total intl minutes \\\n",
"0 197.4 3 2.7 10.0 \n",
"1 195.5 3 3.7 13.7 \n",
"\n",
" total night calls total night charge total night minutes voice mail plan \n",
"0 91 11.01 244.7 yes \n",
"1 103 11.45 254.4 yes \n",
"\n",
"[2 rows x 21 columns]"
]
"text/plain": " account length area code churn customer service calls \\\n0 128 415 0 1 \n1 107 415 0 1 \n\n international plan number vmail messages phone number state \\\n0 no 25 382-4657 KS \n1 no 26 371-7191 OH \n\n total day calls total day charge ... total eve calls total eve charge \\\n0 110 45.07 ... 99 16.78 \n1 123 27.47 ... 103 16.62 \n\n total eve minutes total intl calls total intl charge total intl minutes \\\n0 197.4 3 2.7 10.0 \n1 195.5 3 3.7 13.7 \n\n total night calls total night charge total night minutes voice mail plan \n0 91 11.01 244.7 yes \n1 103 11.45 254.4 yes \n\n[2 rows x 21 columns]",
"text/html": "<div>\n<style scoped>\n .dataframe tbody tr th:only-of-type {\n vertical-align: middle;\n }\n\n .dataframe tbody tr th {\n vertical-align: top;\n }\n\n .dataframe thead th {\n text-align: right;\n }\n</style>\n<table border=\"1\" class=\"dataframe\">\n <thead>\n <tr style=\"text-align: right;\">\n <th></th>\n <th>account length</th>\n <th>area code</th>\n <th>churn</th>\n <th>customer service calls</th>\n <th>international plan</th>\n <th>number vmail messages</th>\n <th>phone number</th>\n <th>state</th>\n <th>total day calls</th>\n <th>total day charge</th>\n <th>...</th>\n <th>total eve calls</th>\n <th>total eve charge</th>\n <th>total eve minutes</th>\n <th>total intl calls</th>\n <th>total intl charge</th>\n <th>total intl minutes</th>\n <th>total night calls</th>\n <th>total night charge</th>\n <th>total night minutes</th>\n <th>voice mail plan</th>\n </tr>\n </thead>\n <tbody>\n <tr>\n <th>0</th>\n <td>128</td>\n <td>415</td>\n <td>0</td>\n <td>1</td>\n <td>no</td>\n <td>25</td>\n <td>382-4657</td>\n <td>KS</td>\n <td>110</td>\n <td>45.07</td>\n <td>...</td>\n <td>99</td>\n <td>16.78</td>\n <td>197.4</td>\n <td>3</td>\n <td>2.7</td>\n <td>10.0</td>\n <td>91</td>\n <td>11.01</td>\n <td>244.7</td>\n <td>yes</td>\n </tr>\n <tr>\n <th>1</th>\n <td>107</td>\n <td>415</td>\n <td>0</td>\n <td>1</td>\n <td>no</td>\n <td>26</td>\n <td>371-7191</td>\n <td>OH</td>\n <td>123</td>\n <td>27.47</td>\n <td>...</td>\n <td>103</td>\n <td>16.62</td>\n <td>195.5</td>\n <td>3</td>\n <td>3.7</td>\n <td>13.7</td>\n <td>103</td>\n <td>11.45</td>\n <td>254.4</td>\n <td>yes</td>\n </tr>\n </tbody>\n</table>\n</div>\n<p>2 rows × 21 columns</p>"
},
"execution_count": 5,
"metadata": {},
@ -237,38 +95,7 @@
"outputs": [
{
"data": {
"text/plain": [
"{'took': 0,\n",
" 'timed_out': False,\n",
" '_shards': {'total': 1, 'successful': 1, 'skipped': 0, 'failed': 0},\n",
" 'hits': {'total': {'value': 2, 'relation': 'eq'},\n",
" 'max_score': 1.0,\n",
" 'hits': [{'_index': 'churn',\n",
" '_type': '_doc',\n",
" '_id': '0',\n",
" '_score': 1.0,\n",
" '_source': {'state': 'KS',\n",
" 'account length': 128,\n",
" 'area code': 415,\n",
" 'phone number': '382-4657',\n",
" 'international plan': 'no',\n",
" 'voice mail plan': 'yes',\n",
" 'number vmail messages': 25,\n",
" 'total day minutes': 265.1,\n",
" 'total day calls': 110,\n",
" 'total day charge': 45.07,\n",
" 'total eve minutes': 197.4,\n",
" 'total eve calls': 99,\n",
" 'total eve charge': 16.78,\n",
" 'total night minutes': 244.7,\n",
" 'total night calls': 91,\n",
" 'total night charge': 11.01,\n",
" 'total intl minutes': 10.0,\n",
" 'total intl calls': 3,\n",
" 'total intl charge': 2.7,\n",
" 'customer service calls': 1,\n",
" 'churn': 0}}]}}"
]
"text/plain": "{'took': 0,\n 'timed_out': False,\n '_shards': {'total': 1, 'successful': 1, 'skipped': 0, 'failed': 0},\n 'hits': {'total': {'value': 2, 'relation': 'eq'},\n 'max_score': 1.0,\n 'hits': [{'_index': 'churn',\n '_id': '0',\n '_score': 1.0,\n '_source': {'state': 'KS',\n 'account length': 128,\n 'area code': 415,\n 'phone number': '382-4657',\n 'international plan': 'no',\n 'voice mail plan': 'yes',\n 'number vmail messages': 25,\n 'total day minutes': 265.1,\n 'total day calls': 110,\n 'total day charge': 45.07,\n 'total eve minutes': 197.4,\n 'total eve calls': 99,\n 'total eve charge': 16.78,\n 'total night minutes': 244.7,\n 'total night calls': 91,\n 'total night charge': 11.01,\n 'total intl minutes': 10.0,\n 'total intl calls': 3,\n 'total intl charge': 2.7,\n 'customer service calls': 1,\n 'churn': 0}}]}}"
},
"execution_count": 6,
"metadata": {},
@ -287,9 +114,7 @@
"outputs": [
{
"data": {
"text/plain": [
"{'acknowledged': True}"
]
"text/plain": "{'acknowledged': True}"
},
"execution_count": 7,
"metadata": {},

View File

@ -32,13 +32,7 @@
"outputs": [
{
"data": {
"text/plain": [
"AvgTicketPrice 640.387\n",
"Cancelled False\n",
"dayOfWeek 3\n",
"timestamp 2018-01-22 00:05:00.494117676\n",
"dtype: object"
]
"text/plain": "AvgTicketPrice 640.387285\nCancelled False\ndayOfWeek 3\ntimestamp 2018-01-21 23:43:19.256498944\ndtype: object"
},
"execution_count": 3,
"metadata": {},
@ -57,12 +51,7 @@
"outputs": [
{
"data": {
"text/plain": [
"AvgTicketPrice 640.387285\n",
"Cancelled 0.000000\n",
"dayOfWeek 3.000000\n",
"dtype: float64"
]
"text/plain": "AvgTicketPrice 640.387285\nCancelled 0.000000\ndayOfWeek 3.000000\ndtype: float64"
},
"execution_count": 4,
"metadata": {},
@ -81,14 +70,7 @@
"outputs": [
{
"data": {
"text/plain": [
"AvgTicketPrice 640.387\n",
"Cancelled False\n",
"dayOfWeek 3\n",
"timestamp 2018-01-22 00:05:00.494117676\n",
"DestCountry NaN\n",
"dtype: object"
]
"text/plain": "AvgTicketPrice 640.387285\nCancelled False\ndayOfWeek 3\ntimestamp 2018-01-21 23:43:19.256498944\nDestCountry NaN\ndtype: object"
},
"execution_count": 5,
"metadata": {},
@ -107,11 +89,7 @@
"outputs": [
{
"data": {
"text/plain": [
"AvgTicketPrice 213.430365\n",
"dayOfWeek 2.000000\n",
"dtype: float64"
]
"text/plain": "AvgTicketPrice 213.430365\ndayOfWeek 2.000000\ndtype: float64"
},
"execution_count": 6,
"metadata": {},
@ -130,11 +108,7 @@
"outputs": [
{
"data": {
"text/plain": [
"AvgTicketPrice 213.430365\n",
"dayOfWeek 2.000000\n",
"dtype: float64"
]
"text/plain": "AvgTicketPrice 213.430365\ndayOfWeek 2.000000\ndtype: float64"
},
"execution_count": 7,
"metadata": {},
@ -153,14 +127,7 @@
"outputs": [
{
"data": {
"text/plain": [
"AvgTicketPrice 213.43\n",
"Cancelled NaN\n",
"dayOfWeek 2\n",
"timestamp NaT\n",
"DestCountry NaN\n",
"dtype: object"
]
"text/plain": "AvgTicketPrice 213.430365\nCancelled NaN\ndayOfWeek 2.0\ntimestamp NaT\nDestCountry NaN\ndtype: object"
},
"execution_count": 8,
"metadata": {},

File diff suppressed because one or more lines are too long