diff --git a/.buildkite/Dockerfile b/.buildkite/Dockerfile new file mode 100644 index 0000000..0cdd359 --- /dev/null +++ b/.buildkite/Dockerfile @@ -0,0 +1,7 @@ +ARG PYTHON_VERSION=3.9 +FROM python:${PYTHON_VERSION} + +WORKDIR /code/eland +RUN python -m pip install nox + +COPY . . diff --git a/.buildkite/build-documentation.sh b/.buildkite/build-documentation.sh new file mode 100755 index 0000000..f1e908c --- /dev/null +++ b/.buildkite/build-documentation.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +sudo apt-get update +sudo apt-get install -y pandoc python3 python3-pip +python3 -m pip install nox +/opt/buildkite-agent/.local/bin/nox -s docs + +# I couldn't make this work, for some reason pandoc is not found in the docker container repository: +# docker build --file .buildkite/Dockerfile --tag elastic/eland --build-arg PYTHON_VERSION=${PYTHON_VERSION} . +# docker run \ +# --name doc_build \ +# --rm \ +# elastic/eland \ +# apt-get update && \ +# sudo apt-get install --yes pandoc && \ +# nox -s docs diff --git a/.buildkite/lint-code.sh b/.buildkite/lint-code.sh new file mode 100755 index 0000000..311a4c0 --- /dev/null +++ b/.buildkite/lint-code.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +docker build --file .buildkite/Dockerfile --tag elastic/eland --build-arg PYTHON_VERSION=${PYTHON_VERSION} . +docker run \ + --name linter \ + --rm \ + elastic/eland \ + nox -s lint diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml new file mode 100644 index 0000000..7f268e4 --- /dev/null +++ b/.buildkite/pipeline.yml @@ -0,0 +1,33 @@ +steps: + - label: ":terminal: Lint code" + env: + PYTHON_VERSION: 3 + agents: + provider: "gcp" + commands: + - ./.buildkite/lint-code.sh + - label: ":books: Build documentation" + env: + PYTHON_VERSION: 3.9-bookworm + agents: + provider: "gcp" + commands: + - ./.buildkite/build-documentation.sh + - label: "Eland :python: {{ matrix.python }} :elasticsearch: {{ matrix.stack }}" + agents: + provider: "gcp" + env: + PYTHON_VERSION: "{{ matrix.python }}" + PANDAS_VERSION: '1.5.0' + TEST_SUITE: "xpack" + ELASTICSEARCH_VERSION: "{{ matrix.stack }}" + matrix: + setup: + python: + - '3.10' + - '3.9' + - '3.8' + stack: + - '8.7-SNAPSHOT' + - '8.8-SNAPSHOT' + command: ./.buildkite/run-tests diff --git a/.buildkite/run-elasticsearch.sh b/.buildkite/run-elasticsearch.sh new file mode 100755 index 0000000..97028ee --- /dev/null +++ b/.buildkite/run-elasticsearch.sh @@ -0,0 +1,185 @@ +#!/usr/bin/env bash +# +# Launch one or more Elasticsearch nodes via the Docker image, +# to form a cluster suitable for running the REST API tests. +# +# Export the ELASTICSEARCH_VERSION variable, eg. 'elasticsearch:8.0.0-SNAPSHOT'. + +# Version 1.0 +# - Initial version of the run-elasticsearch.sh script + + +if [[ -z "$ELASTICSEARCH_VERSION" ]]; then + echo -e "\033[31;1mERROR:\033[0m Required environment variable [ELASTICSEARCH_VERSION] not set\033[0m" + exit 1 +fi + +set -euxo pipefail + +SCRIPT_PATH=$(dirname $(realpath -s $0)) + +moniker=$(echo "$ELASTICSEARCH_VERSION" | tr -C "[:alnum:]" '-') +suffix=rest-test + +NODE_NAME=${NODE_NAME-${moniker}node1} +MASTER_NODE_NAME=${MASTER_NODE_NAME-${NODE_NAME}} +CLUSTER_NAME=${CLUSTER_NAME-${moniker}${suffix}} +HTTP_PORT=${HTTP_PORT-9200} + +ELASTIC_PASSWORD=${ELASTIC_PASSWORD-changeme} + +DETACH=${DETACH-false} +CLEANUP=${CLEANUP-false} + +volume_name=${NODE_NAME}-${suffix}-data +network_default=${moniker}${suffix} +NETWORK_NAME=${NETWORK_NAME-"$network_default"} + +set +x + +# Set vm.max_map_count kernel setting to 262144 if we're in CI +if [[ "$BUILDKITE" == "true" ]]; then + sudo sysctl -w vm.max_map_count=262144 +fi + +function cleanup_volume { + if [[ "$(docker volume ls -q -f name=$1)" ]]; then + echo -e "\033[34;1mINFO:\033[0m Removing volume $1\033[0m" + (docker volume rm "$1") || true + fi +} +function container_running { + if [[ "$(docker ps -q -f name=$1)" ]]; then + return 0; + else return 1; + fi +} +function cleanup_node { + if container_running "$1"; then + echo -e "\033[34;1mINFO:\033[0m Removing container $1\033[0m" + (docker container rm --force --volumes "$1") || true + cleanup_volume "$1-${suffix}-data" + fi +} +function cleanup_network { + if [[ "$(docker network ls -q -f name=$1)" ]]; then + echo -e "\033[34;1mINFO:\033[0m Removing network $1\033[0m" + (docker network rm "$1") || true + fi +} + +function cleanup { + if [[ "$DETACH" != "true" ]] || [[ "$1" == "1" ]]; then + echo -e "\033[34;1mINFO:\033[0m clean the node and volume on startup (1) OR on exit if not detached\033[0m" + cleanup_node "$NODE_NAME" + fi + if [[ "$DETACH" != "true" ]]; then + echo -e "\033[34;1mINFO:\033[0m clean the network if not detached (start and exit)\033[0m" + cleanup_network "$NETWORK_NAME" + fi +}; +trap "cleanup 0" EXIT + +if [[ "$CLEANUP" == "true" ]]; then + trap - EXIT + if [[ -z "$(docker network ls -q -f name=${NETWORK_NAME})" ]]; then + echo -e "\033[34;1mINFO:\033[0m $NETWORK_NAME is already deleted\033[0m" + exit 0 + fi + containers=$(docker network inspect -f '{{ range $key, $value := .Containers }}{{ printf "%s\n" .Name}}{{ end }}' ${NETWORK_NAME}) + while read -r container; do + cleanup_node "$container" + done <<< "$containers" + cleanup_network "$NETWORK_NAME" + echo -e "\033[32;1mSUCCESS:\033[0m Cleaned up and exiting\033[0m" + exit 0 +fi + +echo -e "\033[34;1mINFO:\033[0m Making sure previous run leftover infrastructure is removed \033[0m" +cleanup 1 + +echo -e "\033[34;1mINFO:\033[0m Creating network $NETWORK_NAME if it does not exist already \033[0m" +docker network inspect "$NETWORK_NAME" > /dev/null 2>&1 || docker network create "$NETWORK_NAME" + +environment=($(cat <<-END + --env node.name=$NODE_NAME + --env cluster.name=$CLUSTER_NAME + --env cluster.initial_master_nodes=$MASTER_NODE_NAME + --env discovery.seed_hosts=$MASTER_NODE_NAME + --env cluster.routing.allocation.disk.threshold_enabled=false + --env bootstrap.memory_lock=true + --env node.attr.testattr=test + --env path.repo=/tmp + --env repositories.url.allowed_urls=http://snapshot.test* + --env ELASTIC_PASSWORD=$ELASTIC_PASSWORD + --env xpack.license.self_generated.type=trial + --env xpack.security.enabled=false + --env xpack.security.http.ssl.enabled=false + --env xpack.security.transport.ssl.enabled=false + --env xpack.ml.max_machine_memory_percent=90 +END +)) + +volumes=($(cat <<-END + --volume $volume_name:/usr/share/elasticsearch/data +END +)) + +url="http://elastic:$ELASTIC_PASSWORD@$NODE_NAME" + +# Pull the container, retry on failures up to 5 times with +# short delays between each attempt. Fixes most transient network errors. +docker_pull_attempts=0 +until [ "$docker_pull_attempts" -ge 5 ] +do + docker pull docker.elastic.co/elasticsearch/"$ELASTICSEARCH_VERSION" && break + docker_pull_attempts=$((docker_pull_attempts+1)) + sleep 10 +done + +echo -e "\033[34;1mINFO:\033[0m Starting container $NODE_NAME \033[0m" +set -x +docker run \ + --name "$NODE_NAME" \ + --network "$NETWORK_NAME" \ + --env ES_JAVA_OPTS=-"Xms2g -Xmx2g" \ + "${environment[@]}" \ + "${volumes[@]}" \ + --publish "$HTTP_PORT":9200 \ + --ulimit nofile=65536:65536 \ + --ulimit memlock=-1:-1 \ + --detach="$DETACH" \ + --health-cmd="curl --insecure --fail $url:9200/_cluster/health || exit 1" \ + --health-interval=2s \ + --health-retries=20 \ + --health-timeout=2s \ + --rm \ + docker.elastic.co/elasticsearch/"$ELASTICSEARCH_VERSION"; +set +x + +if [[ "$DETACH" == "true" ]]; then + until ! container_running "$NODE_NAME" || (container_running "$NODE_NAME" && [[ "$(docker inspect -f "{{.State.Health.Status}}" ${NODE_NAME})" != "starting" ]]); do + echo "" + docker inspect -f "{{range .State.Health.Log}}{{.Output}}{{end}}" ${NODE_NAME} + echo -e "\033[34;1mINFO:\033[0m waiting for node $NODE_NAME to be up\033[0m" + sleep 2; + done; + + # Always show logs if the container is running, this is very useful both on CI as well as while developing + if container_running $NODE_NAME; then + docker logs $NODE_NAME + fi + + if ! container_running $NODE_NAME || [[ "$(docker inspect -f "{{.State.Health.Status}}" ${NODE_NAME})" != "healthy" ]]; then + cleanup 1 + echo + echo -e "\033[31;1mERROR:\033[0m Failed to start ${ELASTICSEARCH_VERSION} in detached mode beyond health checks\033[0m" + echo -e "\033[31;1mERROR:\033[0m dumped the docker log before shutting the node down\033[0m" + exit 1 + else + echo + echo -e "\033[32;1mSUCCESS:\033[0m Detached and healthy: ${NODE_NAME} on docker network: ${NETWORK_NAME}\033[0m" + echo -e "\033[32;1mSUCCESS:\033[0m Running on: ${url/$NODE_NAME/localhost}:${HTTP_PORT}\033[0m" + exit 0 + fi +fi diff --git a/.buildkite/run-repository.sh b/.buildkite/run-repository.sh new file mode 100755 index 0000000..9b2934a --- /dev/null +++ b/.buildkite/run-repository.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# +# Called by entry point `run-test` use this script to add your repository specific test commands +# +# Once called Elasticsearch is up and running and the following parameters are available to this script + +# ELASTICSEARCH_VERSION -- version e.g Major.Minor.Patch(-Prelease) +# ELASTICSEARCH_CONTAINER -- the docker moniker as a reference to know which docker image distribution is used +# ELASTICSEARCH_URL -- The url at which elasticsearch is reachable +# NETWORK_NAME -- The docker network name +# NODE_NAME -- The docker container name also used as Elasticsearch node name + +# When run in CI the test-matrix is used to define additional variables + +# TEST_SUITE -- `xpack` +# + +PYTHON_VERSION=${PYTHON_VERSION-3.8} +echo -e "\033[34;1mINFO:\033[0m URL ${ELASTICSEARCH_URL}\033[0m" +echo -e "\033[34;1mINFO:\033[0m VERSION ${ELASTICSEARCH_VERSION}\033[0m" +echo -e "\033[34;1mINFO:\033[0m CONTAINER ${ELASTICSEARCH_CONTAINER}\033[0m" +echo -e "\033[34;1mINFO:\033[0m TEST_SUITE ${TEST_SUITE}\033[0m" +echo -e "\033[34;1mINFO:\033[0m PYTHON_VERSION ${PYTHON_VERSION}\033[0m" +echo -e "\033[34;1mINFO:\033[0m PANDAS_VERSION ${PANDAS_VERSION}\033[0m" + +echo -e "\033[1m>>>>> Build [elastic/eland container] >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m" + +docker build --file .buildkite/Dockerfile --tag elastic/eland --build-arg PYTHON_VERSION=${PYTHON_VERSION} . + +echo -e "\033[1m>>>>> Run [elastic/eland container] >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m" + +docker run \ + --network=${NETWORK_NAME} \ + --env "ELASTICSEARCH_HOST=${ELASTICSEARCH_URL}" \ + --env "TEST_SUITE=${TEST_SUITE}" \ + --name eland-test-runner \ + --rm \ + elastic/eland \ + nox -s "test-${PYTHON_VERSION}(pandas_version='${PANDAS_VERSION}')" diff --git a/.buildkite/run-tests b/.buildkite/run-tests new file mode 100755 index 0000000..286a3cf --- /dev/null +++ b/.buildkite/run-tests @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# +# Version 1.0 +# - Moved to .ci folder and seperated out `run-repository.sh` + +if [[ -z $ELASTICSEARCH_VERSION ]]; then + echo -e "\033[31;1mERROR:\033[0m Required environment variable [ELASTICSEARCH_VERSION] not set\033[0m" + exit 1 +fi +set -euxo pipefail + +TEST_SUITE=${TEST_SUITE-xpack} +NODE_NAME=localhost +PANDAS_VERSION=${PANDAS_VERSION-1.5.0} + +elasticsearch_image=elasticsearch +elasticsearch_url=http://elastic:changeme@${NODE_NAME}:9200 +if [[ $TEST_SUITE != "xpack" ]]; then + elasticsearch_image=elasticsearch-${TEST_SUITE} + elasticsearch_url=http://${NODE_NAME}:9200 +fi + +function cleanup { + status=$? + set +x + ELASTICSEARCH_VERSION=${elasticsearch_image}:${ELASTICSEARCH_VERSION} \ + NODE_NAME=${NODE_NAME} \ + NETWORK_NAME=elasticsearch \ + CLEANUP=true \ + bash ./.buildkite/run-elasticsearch.sh + # Report status and exit + if [[ "$status" == "0" ]]; then + echo -e "\n\033[32;1mSUCCESS run-tests\033[0m" + exit 0 + else + echo -e "\n\033[31;1mFAILURE during run-tests\033[0m" + exit ${status} + fi +} +trap cleanup EXIT + +echo "--- :elasticsearch: Starting Elasticsearch" + +ELASTICSEARCH_VERSION=${elasticsearch_image}:${ELASTICSEARCH_VERSION} \ + NODE_NAME=${NODE_NAME} \ + NETWORK_NAME=host \ + DETACH=true \ + bash .buildkite/run-elasticsearch.sh + +echo "+++ :python: Run tests" + +ELASTICSEARCH_CONTAINER=${elasticsearch_image}:${ELASTICSEARCH_VERSION} \ + NETWORK_NAME=host \ + NODE_NAME=${NODE_NAME} \ + ELASTICSEARCH_URL=${elasticsearch_url} \ + TEST_SUITE=${TEST_SUITE} \ + PANDAS_VERSION=${PANDAS_VERSION} \ + bash .buildkite/run-repository.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index 2c65a5b..0000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,38 +0,0 @@ -name: CI - -on: [push, pull_request] - -defaults: - run: - shell: bash - -jobs: - lint: - runs-on: ubuntu-latest - steps: - - name: Checkout Repository - uses: actions/checkout@v2 - - name: Set up Python 3 - uses: actions/setup-python@v2 - with: - python-version: 3 - - name: Install dependencies - run: python3 -m pip install nox - - name: Lint the code - run: nox -s lint - - docs: - runs-on: ubuntu-latest - steps: - - name: Checkout Repository - uses: actions/checkout@v2 - - name: Set up Python 3 - uses: actions/setup-python@v2 - with: - python-version: 3 - - name: Install dependencies - run: | - sudo apt-get install --yes pandoc - python3 -m pip install nox - - name: Build documentation - run: nox -s docs