diff --git a/example/Online Retail Analysis.ipynb b/example/Online Retail Analysis.ipynb new file mode 100644 index 0000000..1cdb001 --- /dev/null +++ b/example/Online Retail Analysis.ipynb @@ -0,0 +1,1105 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 97, + "metadata": {}, + "outputs": [], + "source": [ + "import eland as ed\n", + "import numpy as np" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Exploratory Data Analysis with eland" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Getting Started\n", + "\n", + "To get started, let's explore the attributes of the `online-retail` index. First, we'll instantiate the data frame by pointing the constructor to a particular instance in our local elasticsearch cluster. \n", + "\n", + "The `online-retail` index was created by running `python load_data.py` from the `examples` directory." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "df = ed.read_es(\"http://localhost:9200\", \"online-retail\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Here we see that the `\"_id\"` field was used to index our data frame. " + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'_id'" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.index.index_field" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Next, we can check which field from elasticsearch are available to our eland data frame. `columns` is available as a parameter when instantiating the data frame which allows one to choose only a subset of fields from your index to be included in the data frame. Since we didn't set this parameter, we have access to all fields." + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Index(['country', 'customer_id', 'description', 'invoice_date', 'invoice_no',\n", + " 'quantity', 'stock_code', 'unit_price'],\n", + " dtype='object')" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.columns" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now, let's see the data types of our fields. Running `df.dtypes`, we can see that elasticsearch field types are mapped to pandas field types." + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "country object\n", + "customer_id object\n", + "description object\n", + "invoice_date datetime64[ns]\n", + "invoice_no object\n", + "quantity int64\n", + "stock_code object\n", + "unit_price float64\n", + "dtype: object" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.dtypes" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We also offer a `.info_es()` data frame method that shows all info about the underlying index. It also contains information about operations being passed from data frame methods to elasticsearch. More on this later." + ] + }, + { + "cell_type": "code", + "execution_count": 70, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "index_pattern: online-retail\n", + "Index:\n", + "\tindex_field: _id\n", + "\tis_source_field: False\n", + "Mappings:\n", + "\tcapabilities: _source es_dtype pd_dtype searchable aggregatable\n", + "country True keyword object True True\n", + "customer_id True keyword object True True\n", + "description True keyword object True True\n", + "invoice_date True date datetime64[ns] True True\n", + "invoice_no True keyword object True True\n", + "quantity True integer int64 True True\n", + "stock_code True keyword object True True\n", + "unit_price True float float64 True True\n", + "Operations:\n", + "\ttasks: []\n", + "\tsize: None\n", + "\tsort_params: None\n", + "\tcolumns: None\n", + "\tpost_processing: []\n", + "\n" + ] + } + ], + "source": [ + "print(df.info_es())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Selecting and Indexing Data\n", + "\n", + "Now that we understand how to create a data frame and get access to it's underlying attributes, let's see how we can select subsets of our data." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### head and tail\n", + "\n", + "much like pandas, eland data frames offer `.head(n)` and `.tail(n)` methods that return the first and last n rows, respectively." + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
countrycustomer_iddescriptioninvoice_dateinvoice_noquantitystock_codeunit_price
wXcVa24BUkfJ5hz0pRsLUnited Kingdom17850WHITE HANGING HEART T-LIGHT HOLDER2010-12-01 08:26:00536365685123A2.55
wncVa24BUkfJ5hz0pRsLUnited Kingdom17850WHITE METAL LANTERN2010-12-01 08:26:005363656710533.39
\n", + "
\n", + "

2 rows x 8 columns

" + ], + "text/plain": [ + " country customer_id \\\n", + "wXcVa24BUkfJ5hz0pRsL United Kingdom 17850 \n", + "wncVa24BUkfJ5hz0pRsL United Kingdom 17850 \n", + "\n", + " description invoice_date \\\n", + "wXcVa24BUkfJ5hz0pRsL WHITE HANGING HEART T-LIGHT HOLDER 2010-12-01 08:26:00 \n", + "wncVa24BUkfJ5hz0pRsL WHITE METAL LANTERN 2010-12-01 08:26:00 \n", + "\n", + " invoice_no quantity stock_code unit_price \n", + "wXcVa24BUkfJ5hz0pRsL 536365 6 85123A 2.55 \n", + "wncVa24BUkfJ5hz0pRsL 536365 6 71053 3.39 \n", + "\n", + "[2 rows x 8 columns]" + ] + }, + "execution_count": 44, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.head(2)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
countrycustomer_iddescriptioninvoice_dateinvoice_noquantitystock_codeunit_price
vXgVa24BUkfJ5hz0txvjUnited KingdomMULTICOLOUR HONEYCOMB FAN2011-01-20 18:08:005416961212091.63
vngVa24BUkfJ5hz0txvjUnited KingdomPACK OF 72 RETROSPOT CAKE CASES2011-01-20 18:08:005416961212121.25
\n", + "
\n", + "

2 rows x 8 columns

" + ], + "text/plain": [ + " country customer_id \\\n", + "vXgVa24BUkfJ5hz0txvj United Kingdom \n", + "vngVa24BUkfJ5hz0txvj United Kingdom \n", + "\n", + " description invoice_date \\\n", + "vXgVa24BUkfJ5hz0txvj MULTICOLOUR HONEYCOMB FAN 2011-01-20 18:08:00 \n", + "vngVa24BUkfJ5hz0txvj PACK OF 72 RETROSPOT CAKE CASES 2011-01-20 18:08:00 \n", + "\n", + " invoice_no quantity stock_code unit_price \n", + "vXgVa24BUkfJ5hz0txvj 541696 1 21209 1.63 \n", + "vngVa24BUkfJ5hz0txvj 541696 1 21212 1.25 \n", + "\n", + "[2 rows x 8 columns]" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.tail(2)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### selecting columns\n", + "\n", + "you can also pass a list of columns to select columns from the data frame in a specified order." + ] + }, + { + "cell_type": "code", + "execution_count": 56, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
countryinvoice_date
wXcVa24BUkfJ5hz0pRsLUnited Kingdom2010-12-01 08:26:00
wncVa24BUkfJ5hz0pRsLUnited Kingdom2010-12-01 08:26:00
w3cVa24BUkfJ5hz0pRsLUnited Kingdom2010-12-01 08:26:00
xHcVa24BUkfJ5hz0pRsLUnited Kingdom2010-12-01 08:26:00
xXcVa24BUkfJ5hz0pRsLUnited Kingdom2010-12-01 08:26:00
\n", + "
\n", + "

5 rows x 2 columns

" + ], + "text/plain": [ + " country invoice_date\n", + "wXcVa24BUkfJ5hz0pRsL United Kingdom 2010-12-01 08:26:00\n", + "wncVa24BUkfJ5hz0pRsL United Kingdom 2010-12-01 08:26:00\n", + "w3cVa24BUkfJ5hz0pRsL United Kingdom 2010-12-01 08:26:00\n", + "xHcVa24BUkfJ5hz0pRsL United Kingdom 2010-12-01 08:26:00\n", + "xXcVa24BUkfJ5hz0pRsL United Kingdom 2010-12-01 08:26:00\n", + "\n", + "[5 rows x 2 columns]" + ] + }, + "execution_count": 56, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df[['country', 'invoice_date']].head(5)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Boolean Indexing\n", + "\n", + "we also allow you to filter the data frame using boolean indexing. Under the hood, a boolean index maps to a `terms` query that is then passed to elasticsearch to filter the index." + ] + }, + { + "cell_type": "code", + "execution_count": 111, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'term': {'country': 'Germany'}}\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
countrycustomer_iddescriptioninvoice_dateinvoice_noquantitystock_codeunit_price
FncVa24BUkfJ5hz0pSBJGermany12662SET OF 6 T-LIGHTS SANTA2010-12-01 13:04:005365276228092.95
F3cVa24BUkfJ5hz0pSBJGermany12662ROTATING SILVER ANGELS T-LIGHT HLDR2010-12-01 13:04:005365276843472.55
GHcVa24BUkfJ5hz0pSBJGermany12662MULTI COLOUR SILVER T-LIGHT HOLDER2010-12-01 13:04:0053652712849450.85
GXcVa24BUkfJ5hz0pSBJGermany126625 HOOK HANGER MAGIC TOADSTOOL2010-12-01 13:04:0053652712222421.65
GncVa24BUkfJ5hz0pSBJGermany126623 HOOK HANGER MAGIC GARDEN2010-12-01 13:04:0053652712222441.95
\n", + "
\n", + "

5 rows x 8 columns

" + ], + "text/plain": [ + " country customer_id \\\n", + "FncVa24BUkfJ5hz0pSBJ Germany 12662 \n", + "F3cVa24BUkfJ5hz0pSBJ Germany 12662 \n", + "GHcVa24BUkfJ5hz0pSBJ Germany 12662 \n", + "GXcVa24BUkfJ5hz0pSBJ Germany 12662 \n", + "GncVa24BUkfJ5hz0pSBJ Germany 12662 \n", + "\n", + " description invoice_date \\\n", + "FncVa24BUkfJ5hz0pSBJ SET OF 6 T-LIGHTS SANTA 2010-12-01 13:04:00 \n", + "F3cVa24BUkfJ5hz0pSBJ ROTATING SILVER ANGELS T-LIGHT HLDR 2010-12-01 13:04:00 \n", + "GHcVa24BUkfJ5hz0pSBJ MULTI COLOUR SILVER T-LIGHT HOLDER 2010-12-01 13:04:00 \n", + "GXcVa24BUkfJ5hz0pSBJ 5 HOOK HANGER MAGIC TOADSTOOL 2010-12-01 13:04:00 \n", + "GncVa24BUkfJ5hz0pSBJ 3 HOOK HANGER MAGIC GARDEN 2010-12-01 13:04:00 \n", + "\n", + " invoice_no quantity stock_code unit_price \n", + "FncVa24BUkfJ5hz0pSBJ 536527 6 22809 2.95 \n", + "F3cVa24BUkfJ5hz0pSBJ 536527 6 84347 2.55 \n", + "GHcVa24BUkfJ5hz0pSBJ 536527 12 84945 0.85 \n", + "GXcVa24BUkfJ5hz0pSBJ 536527 12 22242 1.65 \n", + "GncVa24BUkfJ5hz0pSBJ 536527 12 22244 1.95 \n", + "\n", + "[5 rows x 8 columns]" + ] + }, + "execution_count": 111, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# the construction of a boolean vector maps directly to an elasticsearch query\n", + "print(df['country']=='Germany')\n", + "df[(df['country']=='Germany')].head(5)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# add isin example" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can also combine boolean vectors to further filter the data frame." + ] + }, + { + "cell_type": "code", + "execution_count": 115, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
countrycustomer_iddescriptioninvoice_dateinvoice_noquantitystock_codeunit_price
5XcVa24BUkfJ5hz0q3MqGermany12471FUNKY DIVA PEN2010-12-10 09:35:0053817496227410.85
7XcVa24BUkfJ5hz0q3MqGermany12471LIPSTICK PEN RED2010-12-10 09:35:00538174100224190.36
FHcVa24BUkfJ5hz0s-K9Germany12500PACK OF 6 BIRDY GIFT TAGS2011-01-10 09:48:00540553144225851.06
XncVa24BUkfJ5hz0s-K9Germany12524BOX OF 24 COCKTAIL PARASOLS2011-01-10 10:35:00540562100846920.42
\n", + "
\n", + "

4 rows x 8 columns

" + ], + "text/plain": [ + " country customer_id description \\\n", + "5XcVa24BUkfJ5hz0q3Mq Germany 12471 FUNKY DIVA PEN \n", + "7XcVa24BUkfJ5hz0q3Mq Germany 12471 LIPSTICK PEN RED \n", + "FHcVa24BUkfJ5hz0s-K9 Germany 12500 PACK OF 6 BIRDY GIFT TAGS \n", + "XncVa24BUkfJ5hz0s-K9 Germany 12524 BOX OF 24 COCKTAIL PARASOLS \n", + "\n", + " invoice_date invoice_no quantity stock_code \\\n", + "5XcVa24BUkfJ5hz0q3Mq 2010-12-10 09:35:00 538174 96 22741 \n", + "7XcVa24BUkfJ5hz0q3Mq 2010-12-10 09:35:00 538174 100 22419 \n", + "FHcVa24BUkfJ5hz0s-K9 2011-01-10 09:48:00 540553 144 22585 \n", + "XncVa24BUkfJ5hz0s-K9 2011-01-10 10:35:00 540562 100 84692 \n", + "\n", + " unit_price \n", + "5XcVa24BUkfJ5hz0q3Mq 0.85 \n", + "7XcVa24BUkfJ5hz0q3Mq 0.36 \n", + "FHcVa24BUkfJ5hz0s-K9 1.06 \n", + "XncVa24BUkfJ5hz0s-K9 0.42 \n", + "\n", + "[4 rows x 8 columns]" + ] + }, + "execution_count": 115, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df[(df['country']=='Germany') & (df['quantity']>90)]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Using this example, let see how eland translates this boolean filter to an elasticsearch `bool` query." + ] + }, + { + "cell_type": "code", + "execution_count": 74, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "index_pattern: online-retail\n", + "Index:\n", + "\tindex_field: _id\n", + "\tis_source_field: False\n", + "Mappings:\n", + "\tcapabilities: _source es_dtype pd_dtype searchable aggregatable\n", + "country True keyword object True True\n", + "customer_id True keyword object True True\n", + "description True keyword object True True\n", + "invoice_date True date datetime64[ns] True True\n", + "invoice_no True keyword object True True\n", + "quantity True integer int64 True True\n", + "stock_code True keyword object True True\n", + "unit_price True float float64 True True\n", + "Operations:\n", + "\ttasks: [('boolean_filter', {'bool': {'must': [{'term': {'country': 'Germany'}}, {'range': {'quantity': {'gt': 90}}}]}})]\n", + "\tsize: None\n", + "\tsort_params: None\n", + "\tcolumns: None\n", + "\tpost_processing: []\n", + "\n" + ] + } + ], + "source": [ + "print(df[(df['country']=='Germany') & (df['quantity']>90)].info_es())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Aggregation and Descriptive Statistics\n", + "\n", + "Let's begin to ask some questions of our data and use eland to get the answers." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**How many different countries are there?**" + ] + }, + { + "cell_type": "code", + "execution_count": 76, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "country 24\n", + "dtype: int64" + ] + }, + "execution_count": 76, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df['country'].nunique()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**What is the total sum of products ordered?**" + ] + }, + { + "cell_type": "code", + "execution_count": 80, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "quantity 548076.0\n", + "dtype: float64" + ] + }, + "execution_count": 80, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df['quantity'].sum()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Show me the sum, mean, min, and max of the qunatity and unit_price fields**" + ] + }, + { + "cell_type": "code", + "execution_count": 93, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
quantityunit_price
sum548076.000000383761.569666
mean8.3632315.855916
max74215.00000016888.019531
min-74215.0000000.000000
\n", + "
" + ], + "text/plain": [ + " quantity unit_price\n", + "sum 548076.000000 383761.569666\n", + "mean 8.363231 5.855916\n", + "max 74215.000000 16888.019531\n", + "min -74215.000000 0.000000" + ] + }, + "execution_count": 93, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df[['quantity','unit_price']].agg(['sum', 'mean', 'max', 'min'])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Give me descriptive statistics for the entire data frame**" + ] + }, + { + "cell_type": "code", + "execution_count": 119, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
quantityunit_price
count65534.00000065534.000000
mean8.3632315.855916
std413.694481145.755942
min-74215.0000000.000000
25%1.0000001.250000
50%2.0000002.510000
75%8.0000004.234706
max74215.00000016888.019531
\n", + "
" + ], + "text/plain": [ + " quantity unit_price\n", + "count 65534.000000 65534.000000\n", + "mean 8.363231 5.855916\n", + "std 413.694481 145.755942\n", + "min -74215.000000 0.000000\n", + "25% 1.000000 1.250000\n", + "50% 2.000000 2.510000\n", + "75% 8.000000 4.234706\n", + "max 74215.000000 16888.019531" + ] + }, + "execution_count": 119, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.describe()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Show me a histogram of numeric columns**" + ] + }, + { + "cell_type": "code", + "execution_count": 110, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([[,\n", + " ]],\n", + " dtype=object)" + ] + }, + "execution_count": 110, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtsAAAEICAYAAACOMji0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3df7RfdX3n++dLfijVKqD2DCXMhDtk2iKMqLlAl733nsEKAb0N07GKpRKtmrbC0t7LnRFsb1WULlxz0QqjdFJJCR1q5FJtshRKU+XcjmsVhCg/BHSIGEsyASrhh5EpNvi+f3w/B76Gk+ScnLPPj53nY63vOt/ve3/2/n7e2ef7Pe/s/dmfnapCkiRJ0sx73lx3QJIkSeori21JkiSpIxbbkiRJUkcstiVJkqSOWGxLkiRJHbHYliRJkjpisS1NQpIbkqyY635I0v4qyR8n+b9n6b3uTjI6G++l/ovzbEs/KcmHgGOq6jd2s/ztwLuq6pdms1+SpIFWCP+Xqlo0132R9sYj25IkSUCSA+e6D+ofi23Na0leleTrSX6Q5HNJ1ib5aJK3J/nqLm0ryTHt+RuSfCPJE0keaEerx9stbm1XJPn7JN9P8ntt2TLgA8BbkuxIckeLjyV5V5JfAP4Y+MW2/LEk/3OSh5IcMPQevzq+riRpYPh7ur2+KslH2/PRJFuSnJ/k4STbkrxj17ZJXgjcAPxs+x7ekeRn9/CeH0pyXfsb8oP2N+WVQ8s3J3l/kjuBHyY5sMV+uS0/IMkHknynrb8xyVFt2c8n2ZBke5JvJ3nzjP+jacGz2Na8leRg4C+BPwMOB/5f4N9NcvUfAucAhwJvAH4nyZm7tPkl4OeA1wF/kOQXquqvgD8EPldVL6qqVw6vUFX3Ar8N/F1bfmhV3Qo8Apw61PRtwNWTz1aSBPwz4CXAkcA7gU8lOWy4QVX9EDgd+O/te/hFVfXf97Ld5Qz+hhwO/Dnwl0kOGlr+VgZ/Kw6tqp27rPt/tuVnAC8GfhN4shX9G9r2fgY4C/h0kmOnmLN6zmJb89nJwEHAH1XVP1XVdcCtk1mxqsaq6q6q+nFV3Ql8Fvjfdmn24ar6H1V1B3AH8MrnbGjy1gC/AZDkcOA0Bl/AkqTJ+yfgovadfz2wg8FBkenaWFXXVdU/AR8HXsDgb8y4y6rqgar6HxOs+y7g96vq2zVwR1U9ArwR2FxVf1pVO6vqG8BfAL82A/1Vjzg2SfPZzwJb6yev4v3eZFZMchJwCXAccDDwfAZHNYY9OPT8SeBF+95V/gtwbzvS8Wbgv1bVtmlsT5L2R4/scmR5ut/N4x4Yf1JVP06yhcHfmOcsn8BRwHcmiP8L4KQkjw3FDmRwNlZ6hke2NZ9tA45MkqHYP28/fwj81HgwyT/bZd0/B9YDR1XVSxiMsw6Ts7cpep6zvKq2An8H/CqDISR+2UrScz3J0Hc3g2Ej+2KqU6kdNf4kyfOARcDw0JM9be8B4F/uJv7/teGE448XVdXvTLFv6jmLbc1nfwfsBN6b5KAkvwqc2JbdAbwiyQlJXgB8aJd1fxrYXlX/mORE4Nen8L4PAYvbF/Luli9qY8qHXQ38B+B44PNTeD9J2l/cDvx6u+hwGc8d3jdZDwEvTfKSSbZ/Tbtw/UDgd4GngJsnue5ngI8kWZKBf53kpcAXgX+V5G3tb9RB7YL5X5hyNuo1i23NW1X1IwZHit8ObAfeQitiq+q/ARcBfwPcB3x1l9XfA1yU5AfAHwDXTuGtx4ebPJLk6xMs/wpwN/Bgku8Pxb/A4LTiF6rqySm8nyTtL94H/O/AY8DZDC6Cn7Kq+haDa3Hub7NC7XY2kmYdg78hjzI4+/irbfz2ZHycwd+QvwaeAK4EDqmqHzC4MP4sBkfJHwQ+xmDYovQMb2qjBSXJVcCWqvr9ue7LRJJ8B/itqvqbue6LJGnvNyqTuuaRbWmGJPl3DMb9fWWu+yJJkuYHZyORZkCSMeBY4G1V9eM57o4k7VeS3AD8LxMs+sPZ7ou0K4eRSJIkSR1xGIkkSZLUkQU7jORlL3tZLV68eK678Ywf/vCHvPCFL5zrbnSir7n1NS/ob27zMa+NGzd+v6pePtf9mAlJDgBuY3AzqTcmORpYC7wU2MhgmNSPkjyfwVSXrwEeAd5SVZvbNi5kcJvtp4H3VtWNLb4M+CRwAPCZqrpkb/3Zl+/5+fg7MpPMb+Hqc27Q7/ym+z2/YIvtxYsXc9ttt811N54xNjbG6OjoXHejE33Nra95QX9zm495JZnUXU0XiPcB9wIvbq8/BnyiqtYm+WMGRfQV7eejVXVMkrNau7ckOZbBNGivYHB3vr9J8q/atj4FvB7YAtyaZH1V3bOnzuzL9/x8/B2ZSea3cPU5N+h3ftP9nncYiSSJJIuANzC4gQftzq2nANe1JmuAM9vz5e01bfnrWvvlwNqqeqqqvgtsYnAjqhOBTVV1f5s/f21rK0m9Z7EtSQL4IwZ3QB2fTeelwGNVtbO93gIc2Z4fyeBW1bTlj7f2z8R3WWd3cUnqvQU7jESSNDOSvBF4uKo2Jhmd476sBFYCjIyMMDY2NqX1d+zYMeV1FhLzW7j6nBv0P7/psNiWJL0W+JUkZwAvYDBm+5PAoUkObEevFwFbW/utwFHAliQHAi9hcKHkeHzc8Dq7i/+EqloFrAJYunRpTXUMaJ/HjYL5LWR9zg36n990OIxEkvZzVXVhVS2qqsUMLnD8SlWdDdwEvKk1WwGsa8/Xt9e05V+pwU0b1gNnJXl+m8lkCfA14FZgSZKjkxzc3mP9LKQmSXPOI9uSpN15P7A2yUeBbwBXtviVwJ8l2QRsZ1A8U1V3J7kWuAfYCZxbVU8DJDkPuJHB1H+rq+ruWc1EkuaIxbYk6RlVNQaMtef3M5hJZNc2/wj82m7Wvxi4eIL49cD1M9hVSVoQHEYiSZIkdcRiW5IkSeqIw0i037pr6+O8/YIv7bHN5kveMEu9kTQT/FxLmm88si1JkiR1xGJbkiRJ6ojFtiRJktQRi21JkiSpI3sttpO8IMnXktyR5O4kH27xq5J8N8nt7XFCiyfJZUk2JbkzyauHtrUiyX3tsWIo/pokd7V1LkuSLpKVJEmSZtNkZiN5CjilqnYkOQj4apIb2rJ/X1XX7dL+dAa36F0CnARcAZyU5HDgg8BSoICNSdZX1aOtzbuBWxjc9GAZcAOSJEnSArbXI9s1sKO9PKg9ag+rLAeubuvdDBya5AjgNGBDVW1vBfYGYFlb9uKqurmqCrgaOHMaOUmSJEnzwqTm2U5yALAROAb4VFXdkuR3gIuT/AHwZeCCqnoKOBJ4YGj1LS22p/iWCeIT9WMlsBJgZGSEsbGxyXR/VuzYsWNe9Wcm9TW3kUPg/ON37rHNQs27r/usr3lJkvprUsV2VT0NnJDkUOALSY4DLgQeBA4GVgHvBy7qqqOtH6vae7F06dIaHR3t8u2mZGxsjPnUn5nU19wuv2Ydl96154/A5rNHZ6czM6yv+6yveUmS+mtKs5FU1WPATcCyqtrWhoo8BfwpcGJrthU4ami1RS22p/iiCeKSJEnSgjaZ2Uhe3o5ok+QQ4PXAt9pYa9rMIWcC32yrrAfOabOSnAw8XlXbgBuBU5McluQw4FTgxrbsiSQnt22dA6yb2TQlSZKk2TeZYSRHAGvauO3nAddW1ReTfCXJy4EAtwO/3dpfD5wBbAKeBN4BUFXbk3wEuLW1u6iqtrfn7wGuAg5hMAuJM5FIkiRpwdtrsV1VdwKvmiB+ym7aF3DubpatBlZPEL8NOG5vfZEkSZIWEu8gKUmSJHXEYluSJEnqiMW2JIkkL0jytSR3JLk7yYdb/Kok301ye3uc0OJJclmSTUnuTPLqoW2tSHJfe6wYir8myV1tncvaRfGS1GuTmmdbktR7TwGnVNWOJAcBX00yfrH6v6+q63ZpfzqwpD1OAq4ATkpyOPBBYCmDuw1vTLK+3Tn4CuDdwC0MLqZfhhfES+o5j2xLkmj3TdjRXh7UHrWHVZYDV7f1bgYObVPCngZsqKrtrcDeACxry15cVTe3C+mvZjBtrCT1mke2JUkAtCleNwLHAJ+qqluS/A5wcZI/AL4MXNBuZnYk8MDQ6ltabE/xLRPEd+3DSmAlwMjICGNjY1PKYeQQOP/4nXtsM9Vtzic7duxY0P3fmz7n1+fcoP/5TYfFtiQJgKp6Gjih3cjsC0mOAy4EHgQOBlYB7wcu6rAPq9r7sHTp0hodHZ3S+pdfs45L79rzn7bNZ09tm/PJ2NgYU/03WUj6nF+fc4P+5zcdDiORJP2EqnoMuAlYVlXb2lCRp4A/BU5szbYCRw2ttqjF9hRfNEFcknrNYluSRJKXtyPaJDkEeD3wrTbWmjZzyJnAN9sq64Fz2qwkJwOPV9U24Ebg1CSHJTkMOBW4sS17IsnJbVvnAOtmM0dJmgsOI5EkARwBrGnjtp8HXFtVX0zylSQvBwLcDvx2a389cAawCXgSeAdAVW1P8hHg1tbuoqra3p6/B7gKOITBLCTORCKp9yy2JUlU1Z3AqyaIn7Kb9gWcu5tlq4HVE8RvA46bXk8laWFxGIkkSZLUEYttSZIkqSMW25IkSVJHLLYlSZKkjlhsS5IkSR2x2JYkSZI6YrEtSZIkdcRiW5IkSeqIxbYkSZLUkb0W20lekORrSe5IcneSD7f40UluSbIpyeeSHNziz2+vN7Xli4e2dWGLfzvJaUPxZS22KckFM5+mJEmSNPsmc2T7KeCUqnolcAKwLMnJwMeAT1TVMcCjwDtb+3cCj7b4J1o7khwLnAW8AlgGfDrJAUkOAD4FnA4cC7y1tZUkSZIWtL0W2zWwo708qD0KOAW4rsXXAGe258vba9ry1yVJi6+tqqeq6rvAJuDE9thUVfdX1Y+Ata2tJEmStKAdOJlG7ejzRuAYBkehvwM8VlU7W5MtwJHt+ZHAAwBVtTPJ48BLW/zmoc0Or/PALvGTdtOPlcBKgJGREcbGxibT/VmxY8eOedWfmdTX3EYOgfOP37nHNgs1777us77mJUnqr0kV21X1NHBCkkOBLwA/32mvdt+PVcAqgKVLl9bo6OhcdGNCY2NjzKf+zKS+5nb5Neu49K49fwQ2nz06O52ZYX3dZ33NS5LUX1OajaSqHgNuAn4RODTJeKWyCNjanm8FjgJoy18CPDIc32Wd3cUlSZKkBW0ys5G8vB3RJskhwOuBexkU3W9qzVYA69rz9e01bflXqqpa/Kw2W8nRwBLga8CtwJI2u8nBDC6iXD8TyUmSJElzaTLDSI4A1rRx288Drq2qLya5B1ib5KPAN4ArW/srgT9LsgnYzqB4pqruTnItcA+wEzi3DU8hyXnAjcABwOqqunvGMpQkSZLmyF6L7aq6E3jVBPH7Gcwksmv8H4Ff2822LgYuniB+PXD9JPorSZIkLRjeQVKS5A3MJKkjFtuSJPAGZpLUCYttSZI3MJOkjkxqnm1JUv/NhxuYTffmZX2+WRX0/8ZOfc6vz7lB//ObDottSRIwP25gNt2bl/X5ZlXQ/xs79Tm/PucG/c9vOhxGIkn6Cd7ATJJmjsW2JMkbmElSRxxGIkkCb2AmSZ2w2JYkeQMzSeqIw0gkSZKkjlhsS5IkSR2x2JYkSZI6YrEtSZIkdcRiW5IkSeqIxbYkSZLUEYttSZIkqSMW25IkSVJHLLYlSZKkjlhsS5IkSR3Za7Gd5KgkNyW5J8ndSd7X4h9KsjXJ7e1xxtA6FybZlOTbSU4bii9rsU1JLhiKH53klhb/XJKDZzpRSZIkabZN5sj2TuD8qjoWOBk4N8mxbdknquqE9rgeoC07C3gFsAz4dJIDkhwAfAo4HTgWeOvQdj7WtnUM8CjwzhnKT5IkSZozey22q2pbVX29Pf8BcC9w5B5WWQ6sraqnquq7wCbgxPbYVFX3V9WPgLXA8iQBTgGua+uvAc7c14QkSZKk+eLAqTROshh4FXAL8FrgvCTnALcxOPr9KINC/Oah1bbwbHH+wC7xk4CXAo9V1c4J2u/6/iuBlQAjIyOMjY1Npfud2rFjx7zqz0zqa24jh8D5x+/cY5uFmndf91lf85Ik9deki+0kLwL+AvjdqnoiyRXAR4BqPy8FfrOTXjZVtQpYBbB06dIaHR3t8u2mZGxsjPnUn5nU19wuv2Ydl96154/A5rNHZ6czM6yv+6yveUmS+mtSxXaSgxgU2tdU1ecBquqhoeV/AnyxvdwKHDW0+qIWYzfxR4BDkxzYjm4Pt5ckSZIWrMnMRhLgSuDeqvr4UPyIoWb/Fvhme74eOCvJ85McDSwBvgbcCixpM48czOAiyvVVVcBNwJva+iuAddNLS5IkSZp7kzmy/VrgbcBdSW5vsQ8wmE3kBAbDSDYDvwVQVXcnuRa4h8FMJudW1dMASc4DbgQOAFZX1d1te+8H1ib5KPANBsW9JEmStKDttdiuqq8CmWDR9XtY52Lg4gni10+0XlXdz2C2EknSLEtyFHA1MMLgAMqqqvpkkg8B7wb+oTX9wNA0rxcymKb1aeC9VXVjiy8DPsngoMpnquqSFj+awSxULwU2Am9rM1NJUq95B0lJkvdTkKSOWGxL0n7O+ylIUnemNM+2JKnfFvr9FPo8fz70f675PufX59yg//lNh8W2JAnox/0U+jx/PvR/rvk+59fn3KD/+U2HxbYkyfspSFJHHLMtSfs576cgSd3xyLYkyfspSFJHLLYlaT/n/RQkqTsOI5EkSZI6YrEtSZIkdcRiW5IkSeqIxbYkSZLUEYttSZIkqSMW25IkSVJHLLYlSZKkjlhsS5IkSR2x2JYkSZI6YrEtSZIkdcRiW5IkSerIXovtJEcluSnJPUnuTvK+Fj88yYYk97Wfh7V4klyWZFOSO5O8emhbK1r7+5KsGIq/JsldbZ3LkqSLZCVJkqTZNJkj2zuB86vqWOBk4NwkxwIXAF+uqiXAl9trgNOBJe2xErgCBsU58EHgJOBE4IPjBXpr8+6h9ZZNPzVJkiRpbu212K6qbVX19fb8B8C9wJHAcmBNa7YGOLM9Xw5cXQM3A4cmOQI4DdhQVdur6lFgA7CsLXtxVd1cVQVcPbQtSZIkacE6cCqNkywGXgXcAoxU1ba26EFgpD0/EnhgaLUtLban+JYJ4hO9/0oGR8sZGRlhbGxsKt3v1I4dO+ZVf2ZSX3MbOQTOP37nHtss1Lz7us/6mpckqb8mXWwneRHwF8DvVtUTw8Oqq6qSVAf9+wlVtQpYBbB06dIaHR3t+i0nbWxsjPnUn5nU19wuv2Ydl96154/A5rNHZ6czM6yv+6yveUmS+mtSs5EkOYhBoX1NVX2+hR9qQ0BoPx9u8a3AUUOrL2qxPcUXTRCXJEmSFrTJzEYS4Erg3qr6+NCi9cD4jCIrgHVD8XParCQnA4+34SY3AqcmOaxdGHkqcGNb9kSSk9t7nTO0LUmSJGnBmsyR7dcCbwNOSXJ7e5wBXAK8Psl9wC+31wDXA/cDm4A/Ad4DUFXbgY8At7bHRS1Ga/OZts53gBtmIDdJ0iQ5zaskdWOvY7ar6qvA7r4QXzdB+wLO3c22VgOrJ4jfBhy3t75IkjozPs3r15P8NLAxyQbg7Qymeb0kyQUMpnl9Pz85zetJDKZwPWlomtelQLXtrG+zUI1P83oLgwMzy/DgiqSe8w6SkiSneZWkjkxp6j9JUv/N5TSv053itc9TekL/p7/sc359zg36n990WGxLkp4x19O8TneK1z5P6Qn9n/6yz/n1OTfof37T4TASSRLgNK+S1AWLbUmS07xKUkccRiJJgmeneb0rye0t9gEG07pem+SdwPeAN7dl1wNnMJiy9UngHTCY5jXJ+DSv8NxpXq8CDmEwC4kzkUjqPYttSZLTvEpSRxxGIkmSJHXEYluSJEnqiMW2JEmS1BGLbUmSJKkjFtuSJElSRyy2JUmSpI5YbEuSJEkdsdiWJEmSOmKxLUmSJHXEYluSJEnqiMW2JEmS1BGLbUmSJKkjFtuSJElSR/ZabCdZneThJN8cin0oydYkt7fHGUPLLkyyKcm3k5w2FF/WYpuSXDAUPzrJLS3+uSQHz2SCkiRJ0lyZzJHtq4BlE8Q/UVUntMf1AEmOBc4CXtHW+XSSA5IcAHwKOB04FnhrawvwsbatY4BHgXdOJyFJkiRpvjhwbw2q6m+TLJ7k9pYDa6vqKeC7STYBJ7Zlm6rqfoAka4HlSe4FTgF+vbVZA3wIuGKyCUiSNBWLL/jSXttsvuQNs9ATSfuDvRbbe3BeknOA24Dzq+pR4Ejg5qE2W1oM4IFd4icBLwUeq6qdE7R/jiQrgZUAIyMjjI2NTaP7M2vHjh3zqj8zqa+5jRwC5x+/c49tFmrefd1nfc1LktRf+1psXwF8BKj281LgN2eqU7tTVauAVQBLly6t0dHRrt9y0sbGxphP/ZlJfc3t8mvWcelde/4IbD57dHY6M8P6us/6mtd8kGQ18Ebg4ao6rsU+BLwb+IfW7ANDwwYvZDDs72ngvVV1Y4svAz4JHAB8pqouafGjgbUMDrJsBN5WVT+anewkae7s02wkVfVQVT1dVT8G/oRnh4psBY4aarqoxXYXfwQ4NMmBu8QlSbPrKrw+R5Jm3D4V20mOGHr5b4HxmUrWA2cleX47irEE+BpwK7CkzTxyMIMv6fVVVcBNwJva+iuAdfvSJ0nSvquqvwW2T7L5M9fnVNV3gfHrc06kXZ/TjlqPX58TBtfnXNfWXwOcOaMJSNI8tddhJEk+C4wCL0uyBfggMJrkBAbDSDYDvwVQVXcnuRa4B9gJnFtVT7ftnAfcyODU4uqquru9xfuBtUk+CnwDuHLGspMkTdesXp8z3WtzJnMtxmTM12sD+n7dQp/z63Nu0P/8pmMys5G8dYLwbgviqroYuHiC+PXA9RPE7+fZYSiSpPlj1q/Pme61OZO5FmMy5uv1Gn2/bqHP+fU5N+h/ftMx/W8kSVIvVdVD48+T/AnwxfZyd9fhsJv4M9fntKPbXp8jab/h7dolSRPy+hxJmj6PbEuSvD5HkjpisS1J8vocSeqIw0gkSZKkjlhsS5IkSR2x2JYkSZI6YrEtSZIkdcRiW5IkSeqIxbYkSZLUEYttSZIkqSMW25IkSVJHLLYlSZKkjlhsS5IkSR2x2JYkSZI6YrEtSZIkdcRiW5IkSeqIxbYkSZLUEYttSZIkqSN7LbaTrE7ycJJvDsUOT7IhyX3t52EtniSXJdmU5M4krx5aZ0Vrf1+SFUPx1yS5q61zWZLMdJKSJEnSXDhwEm2uAv4TcPVQ7ALgy1V1SZIL2uv3A6cDS9rjJOAK4KQkhwMfBJYCBWxMsr6qHm1t3g3cAlwPLANumH5q0vQtvuBLe22z+ZI3zEJPJEnSQrTXI9tV9bfA9l3Cy4E17fka4Myh+NU1cDNwaJIjgNOADVW1vRXYG4BlbdmLq+rmqioGBf2ZSJIkST0wmSPbExmpqm3t+YPASHt+JPDAULstLban+JYJ4hNKshJYCTAyMsLY2Ng+dn/m7dixY171Zyb1NbeRQ+D843dOezvz8d+mr/usr3nNB0lWA28EHq6q41rscOBzwGJgM/Dmqnq0Dff7JHAG8CTw9qr6eltnBfD7bbMfrao1Lf4aBmdKD2FwFvN97SCLJPXavhbbz6iqSjIrX5hVtQpYBbB06dIaHR2djbedlLGxMeZTf2ZSX3O7/Jp1XHrXtD8CbD57dPqdmWF93Wd9zWueuAqHDErSjNvX2UgeakNAaD8fbvGtwFFD7Ra12J7iiyaIS5JmkUMGJakb+3pYbz2wArik/Vw3FD8vyVoGRzser6ptSW4E/nB81hLgVODCqtqe5IkkJzM42nEOcPk+9kmSNLNmfcjgdIcL9nl4GPR/KFWf8+tzbtD//KZjr8V2ks8Co8DLkmxhcIrwEuDaJO8Evge8uTW/nsEYvk0MxvG9A6AV1R8Bbm3tLqqq8SMo7+HZcXw34GlFSZp3ZmvI4HSHC/Z5eBj0fyhVn/Prc27Q//ymY6/fSFX11t0set0EbQs4dzfbWQ2sniB+G3Dc3vohSZp1DyU5op2hnOyQwdFd4mM4ZFDSfsw7SEqSdmd8yCA8d8jgOe1GZifThgwCNwKnJjmsDRs8FbixLXsiycltJpNzhrYlSb02/XNtkqQFzyGDktQNi21JkkMGJakjDiORJEmSOmKxLUmSJHXEYluSJEnqiMW2JEmS1BGLbUmSJKkjFtuSJElSR5z6T5oFiy/40l7bbL7kDbPQE0mSNJs8si1JkiR1xGJbkiRJ6ojFtiRJktQRi21JkiSpIxbbkiRJUkcstiVJkqSOWGxLkiRJHbHYliRJkjpisS1JkiR1ZFrFdpLNSe5KcnuS21rs8CQbktzXfh7W4klyWZJNSe5M8uqh7axo7e9LsmJ6KUmSJEnzw0wc2f43VXVCVS1try8AvlxVS4Avt9cApwNL2mMlcAUMinPgg8BJwInAB8cLdEmSJGkh62IYyXJgTXu+BjhzKH51DdwMHJrkCOA0YENVba+qR4ENwLIO+iVJ2geexZSkfXfgNNcv4K+TFPCfq2oVMFJV29ryB4GR9vxI4IGhdbe02O7iz5FkJYOj4oyMjDA2NjbN7s+cHTt2zKv+zKS+5jZyCJx//M5pb2cy/zaTeZ+Z/Dfu6z7ra14LxL+pqu8PvR4/i3lJkgva6/fzk2cxT2JwFvOkobOYSxn87diYZH07yCJJvTXdYvuXqmprkp8BNiT51vDCqqpWiM+IVsyvAli6dGmNjo7O1KanbWxsjPnUn5nU19wuv2Ydl9413Y8AbD57dK9t3n7Bl2ZkO5PV133W17wWqOXAaHu+BhhjUGw/cxYTuDnJ+FnMUdpZTIAk42cxPzu73Zak2TWtSqOqtrafDyf5AoMx1w8lOaKqtrUv2Idb863AUUOrL2qxrTz7hT0eH5tOvyRJM2rWzmJO9wzmTJ2xuvyadXttc/yRL5n2+0xV38/u9Dm/PucG/c9vOva52E7yQuB5VfWD9vxU4CJgPbACuKT9HP/GWg+cl2Qtg1OLj7eC/EbgD4cuijwVuHBf+yVJmnGzdhZzumcwZ+qM1WTM5Nmoyer72Y4b2PEAAAftSURBVJ0+59fn3KD/+U3HdL6RRoAvJBnfzp9X1V8luRW4Nsk7ge8Bb27trwfOADYBTwLvAKiq7Uk+Atza2l00fppRkjT3PIspSftun4vtqrofeOUE8UeA100QL+Dc3WxrNbB6X/siSeqGZzElaXpm51ybJGmh8iymJE2DxbYkabc8iylJ09PFTW0kSZIkYbEtSZIkdcZiW5IkSeqIxbYkSZLUES+QlPYziydz6/hL3jALPZEkqf88si1JkiR1xGJbkiRJ6ojDSKRpmsywDEmStH/yyLYkSZLUEYttSZIkqSMW25IkSVJHLLYlSZKkjniBpCRJ+8A56yVNhke2JUmSpI54ZFuSpI549FuSR7YlSZKkjlhsS5IkSR1xGIl6aTKnbs8/fhY6MgV767OnmqfH0/mar/zsS/02b4rtJMuATwIHAJ+pqkvmuEvSgjP+R/v843fy9gVwG3kL4P2P3/WS9jfzothOcgDwKeD1wBbg1iTrq+qeue2ZNH9MpjCdTR6N01T5XS9pfzQvim3gRGBTVd0PkGQtsBzwC7hnZupI5nwrPPtmPv37Dvdld0fsLewXDL/r98FMfQY8kyTNjVTVXPeBJG8CllXVu9rrtwEnVdV5u7RbCaxsL38O+PasdnTPXgZ8f6470ZG+5tbXvKC/uc3HvP5FVb18rjuxEEzmu34Gvufn4+/ITDK/havPuUG/8/u5qvrpfV15vhzZnpSqWgWsmut+TCTJbVW1dK770YW+5tbXvKC/ufU1Lz1rut/zff8dMb+Fq8+5Qb/zS3LbdNafL1P/bQWOGnq9qMUkSf3hd72k/c58KbZvBZYkOTrJwcBZwPo57pMkaWb5XS9pvzMvhpFU1c4k5wE3MpgOanVV3T3H3ZqqeTm8ZYb0Nbe+5gX9za2vee0XZum7vu+/I+a3cPU5N+h3ftPKbV5cIClJkiT10XwZRiJJkiT1jsW2JEmS1BGL7RmS5PwkleRl7XWSXJZkU5I7k7x6rvs4FUn+Y5Jvtb5/IcmhQ8subHl9O8lpc9nPfZVkWev/piQXzHV/9lWSo5LclOSeJHcneV+LH55kQ5L72s/D5rqv+yrJAUm+keSL7fXRSW5p++5z7UI7qTefa9g/PtvQ7893kkOTXNf+lt6b5Bf7sv+S/B/t9/KbST6b5AULed8lWZ3k4STfHIpNuK/2pb6z2J4BSY4CTgX+fih8OrCkPVYCV8xB16ZjA3BcVf1r4L8BFwIkOZbBDAKvAJYBn87gFswLRp69ZfTpwLHAW1teC9FO4PyqOhY4GTi35XIB8OWqWgJ8ub1eqN4H3Dv0+mPAJ6rqGOBR4J1z0ivNKz37XMP+8dmGfn++Pwn8VVX9PPBKBnku+P2X5EjgvcDSqjqOwcXOZ7Gw991VDGqaYbvbV1Ou7yy2Z8YngP8ADF9tuhy4ugZuBg5NcsSc9G4fVNVfV9XO9vJmBvPhwiCvtVX1VFV9F9jE4BbMC8kzt4yuqh8B47eMXnCqaltVfb09/wGDL/MjGeSzpjVbA5w5Nz2cniSLgDcAn2mvA5wCXNeaLNjcNON687mG/n+2od+f7yQvAf5X4EqAqvpRVT1Gf/bfgcAhSQ4EfgrYxgLed1X1t8D2XcK721dTru8stqcpyXJga1XdscuiI4EHhl5vabGF6DeBG9rzPuTVhxyeI8li4FXALcBIVW1rix4ERuaoW9P1Rwz+I/vj9vqlwGND/xHsxb7TjOjl5xp6+9mGfn++jwb+AfjTNkzmM0leSA/2X1VtBf4fBmfztwGPAxvpz74bt7t9NeXvGovtSUjyN21c0q6P5cAHgD+Y6z7ui73kNd7m9xiczrxm7nqqvUnyIuAvgN+tqieGl9Vgfs8FN8dnkjcCD1fVxrnuizRX+vjZhv3i830g8Grgiqp6FfBDdhkyslD3Xxu7vJzBfyh+Fnghzx2C0SvT3Vfz4qY2811V/fJE8STHM/hlu2Nw9otFwNeTnMgCuC3x7vIal+TtwBuB19WzE7LP+7wmoQ85PCPJQQz+GF9TVZ9v4YeSHFFV29rprYfnrof77LXAryQ5A3gB8GIGYyAPTXJgO4KyoPedZlSvPtfQ68829P/zvQXYUlW3tNfXMSi2+7D/fhn4blX9A0CSzzPYn33Zd+N2t6+m/F3jke1pqKq7qupnqmpxVS1m8OF6dVU9yOAWxOe0q1ZPBh4fOh0x7yVZxuD03q9U1ZNDi9YDZyV5fpKjGVwg8LW56OM09OaW0W2M45XAvVX18aFF64EV7fkKYN1s9226qurCqlrUPltnAV+pqrOBm4A3tWYLMjd1ojefa+j3Zxv6//ludcADSX6uhV4H3EM/9t/fAycn+an2ezqeWy/23ZDd7asp13feQXIGJdnM4Orc77dfwP/E4NTKk8A7quq2uezfVCTZBDwfeKSFbq6q327Lfo/BOO6dDE5t3jDxVuavdjTlj3j2ltEXz3GX9kmSXwL+K3AXz457/ACDsZ3XAv8c+B7w5qra9eKPBSPJKPB/VdUbk/xPDC5+Oxz4BvAbVfXUXPZP80NfPtew/3y2ob+f7yQnMLj482DgfuAdDA5yLvj9l+TDwFsY1AHfAN7FYNzygtx3ST4LjAIvAx4CPgj8JRPsq32p7yy2JUmSpI44jESSJEnqiMW2JEmS1BGLbUmSJKkjFtuSJElSRyy2JUmSpI5YbEuSJEkdsdiWJEmSOvL/A8g5NWsg5r5GAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "df[(df['quantity']>-50) & \n", + " (df['quantity']<50) & \n", + " (df['unit_price']>0) & \n", + " (df['unit_price']<100)].select_dtypes(include=[np.number]).hist(figsize=[12,4], bins=30)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/example/README.md b/example/README.md new file mode 100644 index 0000000..32b5824 --- /dev/null +++ b/example/README.md @@ -0,0 +1,17 @@ +# Example Walkthrough for eland + +This example demonstrate the functionality of `eland` through a walkthrough of a simple analysis of the [Online Retail Dataset](https://archive.ics.uci.edu/ml/datasets/online+retail). + +To run this example, make sure that you have an elasticsearch cluster running on port 9200 and please install any additional dependencies in addition to `eland`: + +``` +pip install -r requirements-example.txt +``` + +Once these requirements are satisfied, load the data using the provided script: + +``` +python load.py +``` + +This will create an index called `online-retail` with a mapping defined in `load.py`. \ No newline at end of file diff --git a/example/load_data.py b/example/load_data.py new file mode 100644 index 0000000..4b93d91 --- /dev/null +++ b/example/load_data.py @@ -0,0 +1,131 @@ +import argparse +import csv + +from elasticsearch import Elasticsearch, helpers +from elasticsearch.exceptions import TransportError + +def create_index(es, index): + mapping = { + "mappings": { + "properties": { + "invoice_no": {"type": "keyword"}, + "stock_code": {"type": "keyword"}, + "description": {"type": "keyword"}, + "quantity": {"type": "integer"}, + "invoice_date": {"type": "date", "format": "MM/dd/yyyy HH:mm"}, + "unit_price": {"type": "float"}, + "customer_id": {"type": "keyword"}, + "country": {"type": "keyword"} + } + } + } + + # create an empty index + try: + es.indices.create(index=index, body=mapping) + except TransportError as e: + # ignore already existing index + if e.error == "resource_already_exists_exception": + pass + else: + raise + +def parse_date(date): + """ + we need to convert dates to conform to the mapping in the following way: + months: one or two digit ints -> MM + days: one or two digit ints -> dd + years: two digit ints -> yyyy + times: {H}H:mm -> HH:mm + """ + + date = date.split("/") + + month = date[0] if len(date[0]) == 2 else "0{}".format(date[0]) + + day = date[1] if len(date[1]) == 2 else "0{}".format(date[1]) + + year = date[2].split(" ")[0] + year = "20{}".format(year) + + time = date[2].split(" ")[1] + time = time if len(time) == 5 else "0{}".format(time) + + date = "{}/{}/{} {}".format(month, day, year, time) + + return date + +def parse_line(line): + """ + creates the document to be indexed + """ + obj = { + "invoice_no": line[0], + "stock_code": line[1], + "description": line[2], + "quantity": line[3], + "invoice_date": parse_date(line[4]), + "unit_price": line[5], + "customer_id": line[6], + "country": line[7].replace("\n", "") + } + + return obj + +def load_data(es): + """ + generate one document per line of online-retail.csv + read file line by line to avoid loading all data into memory + """ + + create_index(es, "online-retail") + + header = True + with open("data/online-retail.csv", "r") as f: + reader = csv.reader(f, quotechar='"', delimiter=',', quoting=csv.QUOTE_ALL) + for line in reader: + if header: + header=False + continue + doc = parse_line(line) + + yield doc + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "-H", + "--host", + action="store", + default="localhost:9200", + help="The elasticsearch host you wish to connect to. (Default: localhost:9200)" + ) + + args = parser.parse_args() + + # create the elasticsearch client, pointing to the host parameter + es = Elasticsearch(args.host) + index='online-retail' + + # load data from online retail csv in data directory + stream = load_data(es) + for ok, result in helpers.streaming_bulk( + es, + actions=stream, + index=index, + chunk_size=1000 + ): + action, result = result.popitem() + doc_id = "/{}/doc/{}".format(index, result['_id']) + + if not ok: + print("Failed to {} document {} {}".format(action, doc_id, result)) + else: + print(doc_id) + + # make docs available for searches + es.indices.refresh(index=index) + + # notify user of number of documents indexed + print(es.count(index=index)["count"], "documents in index") diff --git a/example/requirements-example.txt b/example/requirements-example.txt new file mode 100644 index 0000000..dc9beca --- /dev/null +++ b/example/requirements-example.txt @@ -0,0 +1,80 @@ +alabaster==0.7.12 +appnope==0.1.0 +atomicwrites==1.3.0 +attrs==19.3.0 +Babel==2.7.0 +backcall==0.1.0 +bleach==3.1.0 +certifi==2019.9.11 +chardet==3.0.4 +cycler==0.10.0 +decorator==4.4.1 +defusedxml==0.6.0 +docutils==0.15.2 +eland==0.1 +elasticsearch==7.1.0 +entrypoints==0.3 +idna==2.8 +imagesize==1.1.0 +importlib-metadata==0.23 +ipykernel==5.1.3 +ipython==7.9.0 +ipython-genutils==0.2.0 +ipywidgets==7.5.1 +jedi==0.15.1 +Jinja2==2.10.3 +jsonschema==3.1.1 +jupyter==1.0.0 +jupyter-client==5.3.4 +jupyter-console==6.0.0 +jupyter-core==4.6.1 +kiwisolver==1.1.0 +MarkupSafe==1.1.1 +matplotlib==3.1.1 +mistune==0.8.4 +more-itertools==7.2.0 +nbconvert==5.6.1 +nbformat==4.4.0 +notebook==6.0.2 +numpy==1.17.4 +numpydoc==0.8.0 +packaging==19.2 +pandas==0.25.1 +pandocfilters==1.4.2 +parso==0.5.1 +pexpect==4.7.0 +pickleshare==0.7.5 +pluggy==0.13.0 +prometheus-client==0.7.1 +prompt-toolkit==2.0.10 +ptyprocess==0.6.0 +py==1.8.0 +Pygments==2.4.2 +pyparsing==2.4.5 +pyrsistent==0.15.5 +pytest==5.2.2 +python-dateutil==2.8.1 +pytz==2019.3 +pyzmq==18.1.1 +qtconsole==4.5.5 +requests==2.22.0 +Send2Trash==1.5.0 +six==1.13.0 +snowballstemmer==2.0.0 +Sphinx==2.2.1 +sphinx-rtd-theme==0.4.3 +sphinxcontrib-applehelp==1.0.1 +sphinxcontrib-devhelp==1.0.1 +sphinxcontrib-htmlhelp==1.0.2 +sphinxcontrib-jsmath==1.0.1 +sphinxcontrib-qthelp==1.0.2 +sphinxcontrib-serializinghtml==1.1.3 +terminado==0.8.3 +testpath==0.4.4 +tornado==6.0.3 +traitlets==4.3.3 +urllib3==1.25.7 +wcwidth==0.1.7 +webencodings==0.5.1 +widgetsnbextension==3.5.1 +zipp==0.6.0