Add Buildkite configuration (#515)

* [CI] Adds Buildkite configuration * Removes GitHub Actions * Moves lint and docs tasks to Buildkite
2025-07-11 00:02:14 +08:00 · 2023-07-17 14:08:41 +01:00 · 2023-07-17 14:08:41 +01:00 · 2134c71ab4
commit 2134c71ab4
parent b5bcba713d
8 changed files with 344 additions and 38 deletions
--- a/.buildkite/Dockerfile
+++ b/.buildkite/Dockerfile
@ -0,0 +1,7 @@
+ARG PYTHON_VERSION=3.9
+FROM python:${PYTHON_VERSION}
+
+WORKDIR /code/eland
+RUN python -m pip install nox
+
+COPY . .
--- a/.buildkite/build-documentation.sh
+++ b/.buildkite/build-documentation.sh
@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+sudo apt-get update
+sudo apt-get install -y pandoc python3 python3-pip
+python3 -m pip install nox
+/opt/buildkite-agent/.local/bin/nox -s docs
+
+# I couldn't make this work, for some reason pandoc is not found in the docker container repository:
+# docker build --file .buildkite/Dockerfile --tag elastic/eland --build-arg PYTHON_VERSION=${PYTHON_VERSION} .
+# docker run \
+#        --name doc_build \
+#        --rm \
+#        elastic/eland \
+#        apt-get update && \
+#        sudo apt-get install --yes pandoc && \
+#        nox -s docs
--- a/.buildkite/lint-code.sh
+++ b/.buildkite/lint-code.sh
@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+docker build --file .buildkite/Dockerfile --tag elastic/eland --build-arg PYTHON_VERSION=${PYTHON_VERSION} .
+docker run \
+       --name linter \
+       --rm \
+       elastic/eland \
+       nox -s lint
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@ -0,0 +1,33 @@
+steps:
+  - label: ":terminal: Lint code"
+    env:
+      PYTHON_VERSION: 3
+    agents:
+      provider: "gcp"
+    commands:
+      - ./.buildkite/lint-code.sh
+  - label: ":books: Build documentation"
+    env:
+      PYTHON_VERSION: 3.9-bookworm
+    agents:
+      provider: "gcp"
+    commands:
+      - ./.buildkite/build-documentation.sh
+  - label: "Eland :python: {{ matrix.python }} :elasticsearch: {{ matrix.stack }}"
+    agents:
+      provider: "gcp"
+    env:
+      PYTHON_VERSION: "{{ matrix.python }}"
+      PANDAS_VERSION: '1.5.0'
+      TEST_SUITE: "xpack"
+      ELASTICSEARCH_VERSION: "{{ matrix.stack }}"
+    matrix:
+      setup:
+        python:
+          - '3.10'
+          - '3.9'
+          - '3.8'
+        stack:
+          - '8.7-SNAPSHOT'
+          - '8.8-SNAPSHOT'
+    command: ./.buildkite/run-tests
--- a/.buildkite/run-elasticsearch.sh
+++ b/.buildkite/run-elasticsearch.sh
@ -0,0 +1,185 @@
+#!/usr/bin/env bash
+#
+# Launch one or more Elasticsearch nodes via the Docker image,
+# to form a cluster suitable for running the REST API tests.
+#
+# Export the ELASTICSEARCH_VERSION variable, eg. 'elasticsearch:8.0.0-SNAPSHOT'.
+
+# Version 1.0
+# - Initial version of the run-elasticsearch.sh script
+
+
+if [[ -z "$ELASTICSEARCH_VERSION" ]]; then
+  echo -e "\033[31;1mERROR:\033[0m Required environment variable [ELASTICSEARCH_VERSION] not set\033[0m"
+  exit 1
+fi
+
+set -euxo pipefail
+
+SCRIPT_PATH=$(dirname $(realpath -s $0))
+
+moniker=$(echo "$ELASTICSEARCH_VERSION" | tr -C "[:alnum:]" '-')
+suffix=rest-test
+
+NODE_NAME=${NODE_NAME-${moniker}node1}
+MASTER_NODE_NAME=${MASTER_NODE_NAME-${NODE_NAME}}
+CLUSTER_NAME=${CLUSTER_NAME-${moniker}${suffix}}
+HTTP_PORT=${HTTP_PORT-9200}
+
+ELASTIC_PASSWORD=${ELASTIC_PASSWORD-changeme}
+
+DETACH=${DETACH-false}
+CLEANUP=${CLEANUP-false}
+
+volume_name=${NODE_NAME}-${suffix}-data
+network_default=${moniker}${suffix}
+NETWORK_NAME=${NETWORK_NAME-"$network_default"}
+
+set +x
+
+# Set vm.max_map_count kernel setting to 262144 if we're in CI
+if [[ "$BUILDKITE" == "true" ]]; then
+  sudo sysctl -w vm.max_map_count=262144
+fi
+
+function cleanup_volume {
+  if [[ "$(docker volume ls -q -f name=$1)" ]]; then
+    echo -e "\033[34;1mINFO:\033[0m Removing volume $1\033[0m"
+    (docker volume rm "$1") || true
+  fi
+}
+function container_running {
+  if [[ "$(docker ps -q -f name=$1)" ]]; then
+    return 0;
+    else return 1;
+  fi
+}
+function cleanup_node {
+  if container_running "$1"; then
+    echo -e "\033[34;1mINFO:\033[0m Removing container $1\033[0m"
+    (docker container rm --force --volumes "$1") || true
+    cleanup_volume "$1-${suffix}-data"
+  fi
+}
+function cleanup_network {
+  if [[ "$(docker network ls -q -f name=$1)" ]]; then
+    echo -e "\033[34;1mINFO:\033[0m Removing network $1\033[0m"
+    (docker network rm "$1") || true
+  fi
+}
+
+function cleanup {
+  if [[ "$DETACH" != "true" ]] || [[ "$1" == "1" ]]; then
+    echo -e "\033[34;1mINFO:\033[0m clean the node and volume on startup (1) OR on exit if not detached\033[0m"
+    cleanup_node "$NODE_NAME"
+  fi
+  if [[ "$DETACH" != "true" ]]; then
+    echo -e "\033[34;1mINFO:\033[0m clean the network if not detached (start and exit)\033[0m"
+    cleanup_network "$NETWORK_NAME"
+  fi
+};
+trap "cleanup 0" EXIT
+
+if [[ "$CLEANUP" == "true" ]]; then
+  trap - EXIT
+  if [[ -z "$(docker network ls -q -f name=${NETWORK_NAME})" ]]; then
+    echo -e "\033[34;1mINFO:\033[0m $NETWORK_NAME is already deleted\033[0m"
+    exit 0
+  fi
+  containers=$(docker network inspect -f '{{ range $key, $value := .Containers }}{{ printf "%s\n" .Name}}{{ end }}' ${NETWORK_NAME})
+  while read -r container; do
+    cleanup_node "$container"
+  done <<< "$containers"
+  cleanup_network "$NETWORK_NAME"
+  echo -e "\033[32;1mSUCCESS:\033[0m Cleaned up and exiting\033[0m"
+  exit 0
+fi
+
+echo -e "\033[34;1mINFO:\033[0m Making sure previous run leftover infrastructure is removed \033[0m"
+cleanup 1
+
+echo -e "\033[34;1mINFO:\033[0m Creating network $NETWORK_NAME if it does not exist already \033[0m"
+docker network inspect "$NETWORK_NAME" > /dev/null 2>&1 || docker network create "$NETWORK_NAME"
+
+environment=($(cat <<-END
+  --env node.name=$NODE_NAME
+  --env cluster.name=$CLUSTER_NAME
+  --env cluster.initial_master_nodes=$MASTER_NODE_NAME
+  --env discovery.seed_hosts=$MASTER_NODE_NAME
+  --env cluster.routing.allocation.disk.threshold_enabled=false
+  --env bootstrap.memory_lock=true
+  --env node.attr.testattr=test
+  --env path.repo=/tmp
+  --env repositories.url.allowed_urls=http://snapshot.test*
+  --env ELASTIC_PASSWORD=$ELASTIC_PASSWORD
+  --env xpack.license.self_generated.type=trial
+  --env xpack.security.enabled=false
+  --env xpack.security.http.ssl.enabled=false
+  --env xpack.security.transport.ssl.enabled=false
+  --env xpack.ml.max_machine_memory_percent=90
+END
+))
+
+volumes=($(cat <<-END
+  --volume $volume_name:/usr/share/elasticsearch/data
+END
+))
+
+url="http://elastic:$ELASTIC_PASSWORD@$NODE_NAME"
+
+# Pull the container, retry on failures up to 5 times with
+# short delays between each attempt. Fixes most transient network errors.
+docker_pull_attempts=0
+until [ "$docker_pull_attempts" -ge 5 ]
+do
+   docker pull docker.elastic.co/elasticsearch/"$ELASTICSEARCH_VERSION" && break
+   docker_pull_attempts=$((docker_pull_attempts+1))
+   sleep 10
+done
+
+echo -e "\033[34;1mINFO:\033[0m Starting container $NODE_NAME \033[0m"
+set -x
+docker run \
+  --name "$NODE_NAME" \
+  --network "$NETWORK_NAME" \
+  --env ES_JAVA_OPTS=-"Xms2g -Xmx2g" \
+  "${environment[@]}" \
+  "${volumes[@]}" \
+  --publish "$HTTP_PORT":9200 \
+  --ulimit nofile=65536:65536 \
+  --ulimit memlock=-1:-1 \
+  --detach="$DETACH" \
+  --health-cmd="curl --insecure --fail $url:9200/_cluster/health || exit 1" \
+  --health-interval=2s \
+  --health-retries=20 \
+  --health-timeout=2s \
+  --rm \
+  docker.elastic.co/elasticsearch/"$ELASTICSEARCH_VERSION";
+set +x
+
+if [[ "$DETACH" == "true" ]]; then
+  until ! container_running "$NODE_NAME" || (container_running "$NODE_NAME" && [[ "$(docker inspect -f "{{.State.Health.Status}}" ${NODE_NAME})" != "starting" ]]); do
+    echo ""
+    docker inspect -f "{{range .State.Health.Log}}{{.Output}}{{end}}" ${NODE_NAME}
+    echo -e "\033[34;1mINFO:\033[0m waiting for node $NODE_NAME to be up\033[0m"
+    sleep 2;
+  done;
+
+  # Always show logs if the container is running, this is very useful both on CI as well as while developing
+  if container_running $NODE_NAME; then
+    docker logs $NODE_NAME
+  fi
+
+  if ! container_running $NODE_NAME || [[ "$(docker inspect -f "{{.State.Health.Status}}" ${NODE_NAME})" != "healthy" ]]; then
+    cleanup 1
+    echo
+    echo -e "\033[31;1mERROR:\033[0m Failed to start ${ELASTICSEARCH_VERSION} in detached mode beyond health checks\033[0m"
+    echo -e "\033[31;1mERROR:\033[0m dumped the docker log before shutting the node down\033[0m"
+    exit 1
+  else
+    echo
+    echo -e "\033[32;1mSUCCESS:\033[0m Detached and healthy: ${NODE_NAME} on docker network: ${NETWORK_NAME}\033[0m"
+    echo -e "\033[32;1mSUCCESS:\033[0m Running on: ${url/$NODE_NAME/localhost}:${HTTP_PORT}\033[0m"
+    exit 0
+  fi
+fi
--- a/.buildkite/run-repository.sh
+++ b/.buildkite/run-repository.sh
@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+#
+# Called by entry point `run-test` use this script to add your repository specific test commands
+#
+# Once called Elasticsearch is up and running and the following parameters are available to this script
+
+# ELASTICSEARCH_VERSION -- version e.g Major.Minor.Patch(-Prelease)
+# ELASTICSEARCH_CONTAINER -- the docker moniker as a reference to know which docker image distribution is used
+# ELASTICSEARCH_URL -- The url at which elasticsearch is reachable
+# NETWORK_NAME -- The docker network name
+# NODE_NAME -- The docker container name also used as Elasticsearch node name
+
+# When run in CI the test-matrix is used to define additional variables
+
+# TEST_SUITE -- `xpack`
+#
+
+PYTHON_VERSION=${PYTHON_VERSION-3.8}
+echo -e "\033[34;1mINFO:\033[0m URL ${ELASTICSEARCH_URL}\033[0m"
+echo -e "\033[34;1mINFO:\033[0m VERSION ${ELASTICSEARCH_VERSION}\033[0m"
+echo -e "\033[34;1mINFO:\033[0m CONTAINER ${ELASTICSEARCH_CONTAINER}\033[0m"
+echo -e "\033[34;1mINFO:\033[0m TEST_SUITE ${TEST_SUITE}\033[0m"
+echo -e "\033[34;1mINFO:\033[0m PYTHON_VERSION ${PYTHON_VERSION}\033[0m"
+echo -e "\033[34;1mINFO:\033[0m PANDAS_VERSION ${PANDAS_VERSION}\033[0m"
+
+echo -e "\033[1m>>>>> Build [elastic/eland container] >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m"
+
+docker build --file .buildkite/Dockerfile --tag elastic/eland --build-arg PYTHON_VERSION=${PYTHON_VERSION} .
+
+echo -e "\033[1m>>>>> Run [elastic/eland container] >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m"
+
+docker run \
+  --network=${NETWORK_NAME} \
+  --env "ELASTICSEARCH_HOST=${ELASTICSEARCH_URL}" \
+  --env "TEST_SUITE=${TEST_SUITE}" \
+  --name eland-test-runner \
+  --rm \
+  elastic/eland \
+  nox -s "test-${PYTHON_VERSION}(pandas_version='${PANDAS_VERSION}')"
--- a/.buildkite/run-tests
+++ b/.buildkite/run-tests
@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+#
+# Version 1.0
+# - Moved to .ci folder and seperated out `run-repository.sh`
+
+if [[ -z $ELASTICSEARCH_VERSION ]]; then
+  echo -e "\033[31;1mERROR:\033[0m Required environment variable [ELASTICSEARCH_VERSION] not set\033[0m"
+  exit 1
+fi
+set -euxo pipefail
+
+TEST_SUITE=${TEST_SUITE-xpack}
+NODE_NAME=localhost
+PANDAS_VERSION=${PANDAS_VERSION-1.5.0}
+
+elasticsearch_image=elasticsearch
+elasticsearch_url=http://elastic:changeme@${NODE_NAME}:9200
+if [[ $TEST_SUITE != "xpack" ]]; then
+  elasticsearch_image=elasticsearch-${TEST_SUITE}
+  elasticsearch_url=http://${NODE_NAME}:9200
+fi
+
+function cleanup {
+  status=$?
+  set +x
+  ELASTICSEARCH_VERSION=${elasticsearch_image}:${ELASTICSEARCH_VERSION} \
+    NODE_NAME=${NODE_NAME} \
+    NETWORK_NAME=elasticsearch \
+    CLEANUP=true \
+    bash ./.buildkite/run-elasticsearch.sh
+  # Report status and exit
+  if [[ "$status" == "0" ]]; then
+    echo -e "\n\033[32;1mSUCCESS run-tests\033[0m"
+    exit 0
+  else
+    echo -e "\n\033[31;1mFAILURE during run-tests\033[0m"
+    exit ${status}
+  fi
+}
+trap cleanup EXIT
+
+echo "--- :elasticsearch: Starting Elasticsearch"
+
+ELASTICSEARCH_VERSION=${elasticsearch_image}:${ELASTICSEARCH_VERSION} \
+  NODE_NAME=${NODE_NAME} \
+  NETWORK_NAME=host \
+  DETACH=true \
+  bash .buildkite/run-elasticsearch.sh
+
+echo "+++ :python: Run tests"
+
+ELASTICSEARCH_CONTAINER=${elasticsearch_image}:${ELASTICSEARCH_VERSION} \
+  NETWORK_NAME=host \
+  NODE_NAME=${NODE_NAME} \
+  ELASTICSEARCH_URL=${elasticsearch_url} \
+  TEST_SUITE=${TEST_SUITE} \
+  PANDAS_VERSION=${PANDAS_VERSION} \
+  bash .buildkite/run-repository.sh
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@ -1,38 +0,0 @@
-name: CI
-
-on: [push, pull_request]
-
-defaults:
-  run:
-    shell: bash
-
-jobs:
-  lint:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout Repository
-        uses: actions/checkout@v2
-      - name: Set up Python 3
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3
-      - name: Install dependencies
-        run: python3 -m pip install nox
-      - name: Lint the code
-        run: nox -s lint
-
-  docs:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout Repository
-        uses: actions/checkout@v2
-      - name: Set up Python 3
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3
-      - name: Install dependencies
-        run: |
-            sudo apt-get install --yes pandoc
-            python3 -m pip install nox
-      - name: Build documentation
-        run: nox -s docs