From d39c67342351ae87da347848633b31b2472d72ec Mon Sep 17 00:00:00 2001 From: Fernando Briano Date: Tue, 18 Jul 2023 14:27:06 +0100 Subject: [PATCH] [Backport][7.x][CI] Migrates CI to Buildkite (#560) --- {.ci => .buildkite}/Dockerfile | 0 .buildkite/build-documentation.sh | 5 ++ .buildkite/lint-code.sh | 7 ++ .buildkite/pipeline.yml | 38 +++++++++++ {.ci => .buildkite}/run-elasticsearch.sh | 5 ++ {.ci => .buildkite}/run-repository.sh | 2 +- {.ci => .buildkite}/run-tests | 6 +- .ci/jobs/defaults.yml | 82 ------------------------ .ci/jobs/elastic+eland+main.yml | 14 ---- .ci/jobs/elastic+eland+pull-request.yml | 19 ------ .ci/test-matrix.yml | 21 ------ .github/workflows/ci.yml | 38 ----------- 12 files changed, 59 insertions(+), 178 deletions(-) rename {.ci => .buildkite}/Dockerfile (100%) create mode 100644 .buildkite/build-documentation.sh create mode 100644 .buildkite/lint-code.sh create mode 100644 .buildkite/pipeline.yml rename {.ci => .buildkite}/run-elasticsearch.sh (97%) rename {.ci => .buildkite}/run-repository.sh (93%) rename {.ci => .buildkite}/run-tests (93%) delete mode 100755 .ci/jobs/defaults.yml delete mode 100755 .ci/jobs/elastic+eland+main.yml delete mode 100644 .ci/jobs/elastic+eland+pull-request.yml delete mode 100755 .ci/test-matrix.yml delete mode 100644 .github/workflows/ci.yml diff --git a/.ci/Dockerfile b/.buildkite/Dockerfile similarity index 100% rename from .ci/Dockerfile rename to .buildkite/Dockerfile diff --git a/.buildkite/build-documentation.sh b/.buildkite/build-documentation.sh new file mode 100644 index 0000000..4849c65 --- /dev/null +++ b/.buildkite/build-documentation.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +sudo apt-get update +sudo apt-get install -y pandoc python3 python3-pip +python3 -m pip install nox +/opt/buildkite-agent/.local/bin/nox -s docs diff --git a/.buildkite/lint-code.sh b/.buildkite/lint-code.sh new file mode 100644 index 0000000..311a4c0 --- /dev/null +++ b/.buildkite/lint-code.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +docker build --file .buildkite/Dockerfile --tag elastic/eland --build-arg PYTHON_VERSION=${PYTHON_VERSION} . +docker run \ + --name linter \ + --rm \ + elastic/eland \ + nox -s lint diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml new file mode 100644 index 0000000..1299859 --- /dev/null +++ b/.buildkite/pipeline.yml @@ -0,0 +1,38 @@ +steps: + - label: ":terminal: Lint code" + env: + PYTHON_VERSION: 3 + agents: + provider: "gcp" + commands: + - ./.buildkite/lint-code.sh + - label: ":books: Build documentation" + env: + PYTHON_VERSION: 3.9-bookworm + agents: + provider: "gcp" + commands: + - ./.buildkite/build-documentation.sh + - label: "Eland :python: {{ matrix.python }} :elasticsearch: {{ matrix.stack }}" + agents: + provider: "gcp" + env: + PYTHON_VERSION: "{{ matrix.python }}" + PANDAS_VERSION: "{{ matrix.pandas }}" + TEST_SUITE: "xpack" + ELASTICSEARCH_VERSION: "{{ matrix.stack }}" + matrix: + setup: + python: + - '3.10' + - '3.9' + - '3.8' + pandas: + - '1.2.0' + - '1.3.0' + stack: + - '7.x-SNAPSHOT' + - '7.11-SNAPSHOT' + - '7.14-SNAPSHOT' + - '8.0-SNAPSHOT' + command: ./.buildkite/run-tests diff --git a/.ci/run-elasticsearch.sh b/.buildkite/run-elasticsearch.sh similarity index 97% rename from .ci/run-elasticsearch.sh rename to .buildkite/run-elasticsearch.sh index 7428b2f..17c76ae 100755 --- a/.ci/run-elasticsearch.sh +++ b/.buildkite/run-elasticsearch.sh @@ -37,6 +37,11 @@ NETWORK_NAME=${NETWORK_NAME-"$network_default"} set +x +# Set vm.max_map_count kernel setting to 262144 if we're in CI +if [[ "$BUILDKITE" == "true" ]]; then + sudo sysctl -w vm.max_map_count=262144 +fi + function cleanup_volume { if [[ "$(docker volume ls -q -f name=$1)" ]]; then echo -e "\033[34;1mINFO:\033[0m Removing volume $1\033[0m" diff --git a/.ci/run-repository.sh b/.buildkite/run-repository.sh similarity index 93% rename from .ci/run-repository.sh rename to .buildkite/run-repository.sh index d068992..69c81a4 100755 --- a/.ci/run-repository.sh +++ b/.buildkite/run-repository.sh @@ -25,7 +25,7 @@ echo -e "\033[34;1mINFO:\033[0m PANDAS_VERSION ${PANDAS_VERSION}\033[0m" echo -e "\033[1m>>>>> Build [elastic/eland container] >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m" -docker build --file .ci/Dockerfile --tag elastic/eland --build-arg PYTHON_VERSION=${PYTHON_VERSION} . +docker build --file .buildkite/Dockerfile --tag elastic/eland --build-arg PYTHON_VERSION=${PYTHON_VERSION} . echo -e "\033[1m>>>>> Run [elastic/eland container] >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m" diff --git a/.ci/run-tests b/.buildkite/run-tests similarity index 93% rename from .ci/run-tests rename to .buildkite/run-tests index a72fe7d..41a001a 100755 --- a/.ci/run-tests +++ b/.buildkite/run-tests @@ -29,7 +29,7 @@ function cleanup { NODE_NAME=${NODE_NAME} \ NETWORK_NAME=elasticsearch \ CLEANUP=true \ - bash ./.ci/run-elasticsearch.sh + bash ./.buildkite/run-elasticsearch.sh # Report status and exit if [[ "$status" == "0" ]]; then echo -e "\n\033[32;1mSUCCESS run-tests\033[0m" @@ -47,7 +47,7 @@ ELASTICSEARCH_VERSION=${elasticsearch_image}:${ELASTICSEARCH_VERSION} \ NODE_NAME=${NODE_NAME} \ NETWORK_NAME=host \ DETACH=true \ - bash .ci/run-elasticsearch.sh + bash .buildkite/run-elasticsearch.sh echo -e "\033[1m>>>>> Repository specific tests >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m" @@ -57,5 +57,5 @@ ELASTICSEARCH_CONTAINER=${elasticsearch_image}:${ELASTICSEARCH_VERSION} \ ELASTICSEARCH_URL=${elasticsearch_url} \ TEST_SUITE=${TEST_SUITE} \ PANDAS_VERSION=${PANDAS_VERSION} \ - bash .ci/run-repository.sh + bash .buildkite/run-repository.sh diff --git a/.ci/jobs/defaults.yml b/.ci/jobs/defaults.yml deleted file mode 100755 index dafc8d5..0000000 --- a/.ci/jobs/defaults.yml +++ /dev/null @@ -1,82 +0,0 @@ ---- - -##### GLOBAL METADATA - -- meta: - cluster: clients-ci - -##### JOB DEFAULTS - -- job: - project-type: matrix - logrotate: - daysToKeep: 30 - numToKeep: 100 - parameters: - - string: - name: branch_specifier - default: refs/heads/main - description: the Git branch specifier to build (<branchName>, <tagName>, - <commitId>, etc.) - properties: - - github: - url: https://github.com/elastic/eland - - inject: - properties-content: HOME=$JENKINS_HOME - concurrent: true - node: flyweight - scm: - - git: - name: origin - credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba - reference-repo: /var/lib/jenkins/.git-references/eland.git - branches: - - ${branch_specifier} - url: git@github.com:elastic/eland.git - basedir: '' - wipe-workspace: 'True' - triggers: - - github - axes: - - axis: - type: slave - name: label - values: - - linux - - axis: - type: yaml - filename: .ci/test-matrix.yml - name: ELASTICSEARCH_VERSION - - axis: - type: yaml - filename: .ci/test-matrix.yml - name: PYTHON_VERSION - - axis: - type: yaml - filename: .ci/test-matrix.yml - name: PANDAS_VERSION - - axis: - type: yaml - filename: .ci/test-matrix.yml - name: TEST_SUITE - yaml-strategy: - exclude-key: exclude - filename: .ci/test-matrix.yml - wrappers: - - ansicolor - - timeout: - type: absolute - timeout: 120 - fail: true - - timestamps - - workspace-cleanup - builders: - - shell: |- - #!/usr/local/bin/runbld - .ci/run-tests - publishers: - - email: - recipients: build-lang-clients@elastic.co - - junit: - results: "build/output/*-junit.xml" - allow-empty-results: true diff --git a/.ci/jobs/elastic+eland+main.yml b/.ci/jobs/elastic+eland+main.yml deleted file mode 100755 index 009e80d..0000000 --- a/.ci/jobs/elastic+eland+main.yml +++ /dev/null @@ -1,14 +0,0 @@ ---- -- job: - name: elastic+eland+main - display-name: 'elastic / eland # main' - description: Eland is a data science client with a Pandas-like interface - junit_results: "*-junit.xml" - parameters: - - string: - name: branch_specifier - default: refs/heads/main - description: The Git branch specifier to build - triggers: - - github - - timed: '@daily' diff --git a/.ci/jobs/elastic+eland+pull-request.yml b/.ci/jobs/elastic+eland+pull-request.yml deleted file mode 100644 index 0249070..0000000 --- a/.ci/jobs/elastic+eland+pull-request.yml +++ /dev/null @@ -1,19 +0,0 @@ ---- -- job: - name: elastic+eland+pull-request - display-name: 'elastic / eland # pull-request' - description: Testing of eland pull requests. - scm: - - git: - branches: - - ${ghprbActualCommit} - refspec: +refs/pull/*:refs/remotes/origin/pr/* - triggers: - - github-pull-request: - org-list: - - elastic - allow-whitelist-orgs-as-admins: true - github-hooks: true - status-context: clients-ci - cancel-builds-on-update: true - publishers: [] \ No newline at end of file diff --git a/.ci/test-matrix.yml b/.ci/test-matrix.yml deleted file mode 100755 index cb7f6a0..0000000 --- a/.ci/test-matrix.yml +++ /dev/null @@ -1,21 +0,0 @@ ---- - -ELASTICSEARCH_VERSION: - - 8.0.0-SNAPSHOT - - 7.x-SNAPSHOT - - 7.14-SNAPSHOT - - 7.11-SNAPSHOT - -PANDAS_VERSION: - - 1.2.0 - - 1.3.0 - -PYTHON_VERSION: - - 3.9 - - 3.8 - - 3.7 - -TEST_SUITE: - - xpack - -exclude: ~ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index 2c65a5b..0000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,38 +0,0 @@ -name: CI - -on: [push, pull_request] - -defaults: - run: - shell: bash - -jobs: - lint: - runs-on: ubuntu-latest - steps: - - name: Checkout Repository - uses: actions/checkout@v2 - - name: Set up Python 3 - uses: actions/setup-python@v2 - with: - python-version: 3 - - name: Install dependencies - run: python3 -m pip install nox - - name: Lint the code - run: nox -s lint - - docs: - runs-on: ubuntu-latest - steps: - - name: Checkout Repository - uses: actions/checkout@v2 - - name: Set up Python 3 - uses: actions/setup-python@v2 - with: - python-version: 3 - - name: Install dependencies - run: | - sudo apt-get install --yes pandoc - python3 -m pip install nox - - name: Build documentation - run: nox -s docs