[Backport][7.x][CI] Migrates CI to Buildkite (#560)

.buildkite/build-documentation.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+sudo apt-get update
+sudo apt-get install -y pandoc python3 python3-pip
+python3 -m pip install nox
+/opt/buildkite-agent/.local/bin/nox -s docs

.buildkite/lint-code.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+docker build --file .buildkite/Dockerfile --tag elastic/eland --build-arg PYTHON_VERSION=${PYTHON_VERSION} .
+docker run \
+       --name linter \
+       --rm \
+       elastic/eland \
+       nox -s lint

.buildkite/pipeline.yml

@@ -0,0 +1,38 @@
+steps:
+  - label: ":terminal: Lint code"
+    env:
+      PYTHON_VERSION: 3
+    agents:
+      provider: "gcp"
+    commands:
+      - ./.buildkite/lint-code.sh
+  - label: ":books: Build documentation"
+    env:
+      PYTHON_VERSION: 3.9-bookworm
+    agents:
+      provider: "gcp"
+    commands:
+      - ./.buildkite/build-documentation.sh
+  - label: "Eland :python: {{ matrix.python }} :elasticsearch: {{ matrix.stack }}"
+    agents:
+      provider: "gcp"
+    env:
+      PYTHON_VERSION: "{{ matrix.python }}"
+      PANDAS_VERSION: "{{ matrix.pandas }}"
+      TEST_SUITE: "xpack"
+      ELASTICSEARCH_VERSION: "{{ matrix.stack }}"
+    matrix:
+      setup:
+        python:
+          - '3.10'
+          - '3.9'
+          - '3.8'
+        pandas:
+          - '1.2.0'
+          - '1.3.0'
+        stack:
+          - '7.x-SNAPSHOT'
+          - '7.11-SNAPSHOT'
+          - '7.14-SNAPSHOT'
+          - '8.0-SNAPSHOT'
+    command: ./.buildkite/run-tests

.buildkite/run-elasticsearch.sh

@@ -37,6 +37,11 @@ NETWORK_NAME=${NETWORK_NAME-"$network_default"}
 
 set +x
 
+# Set vm.max_map_count kernel setting to 262144 if we're in CI
+if [[ "$BUILDKITE" == "true" ]]; then
+  sudo sysctl -w vm.max_map_count=262144
+fi
+
 function cleanup_volume {
   if [[ "$(docker volume ls -q -f name=$1)" ]]; then
     echo -e "\033[34;1mINFO:\033[0m Removing volume $1\033[0m"

.buildkite/run-repository.sh

@@ -25,7 +25,7 @@ echo -e "\033[34;1mINFO:\033[0m PANDAS_VERSION ${PANDAS_VERSION}\033[0m"
 
 echo -e "\033[1m>>>>> Build [elastic/eland container] >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m"
 
-docker build --file .ci/Dockerfile --tag elastic/eland --build-arg PYTHON_VERSION=${PYTHON_VERSION} .
+docker build --file .buildkite/Dockerfile --tag elastic/eland --build-arg PYTHON_VERSION=${PYTHON_VERSION} .
 
 echo -e "\033[1m>>>>> Run [elastic/eland container] >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m"
 

.buildkite/run-tests

@@ -29,7 +29,7 @@ function cleanup {
     NODE_NAME=${NODE_NAME} \
     NETWORK_NAME=elasticsearch \
     CLEANUP=true \
-    bash ./.ci/run-elasticsearch.sh
+    bash ./.buildkite/run-elasticsearch.sh
   # Report status and exit
   if [[ "$status" == "0" ]]; then
     echo -e "\n\033[32;1mSUCCESS run-tests\033[0m"
@@ -47,7 +47,7 @@ ELASTICSEARCH_VERSION=${elasticsearch_image}:${ELASTICSEARCH_VERSION} \
   NODE_NAME=${NODE_NAME} \
   NETWORK_NAME=host \
   DETACH=true \
-  bash .ci/run-elasticsearch.sh
+  bash .buildkite/run-elasticsearch.sh
 
 echo -e "\033[1m>>>>> Repository specific tests >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m"
 
@@ -57,5 +57,5 @@ ELASTICSEARCH_CONTAINER=${elasticsearch_image}:${ELASTICSEARCH_VERSION} \
   ELASTICSEARCH_URL=${elasticsearch_url} \
   TEST_SUITE=${TEST_SUITE} \
   PANDAS_VERSION=${PANDAS_VERSION} \
-  bash .ci/run-repository.sh
+  bash .buildkite/run-repository.sh
 

.ci/jobs/defaults.yml

@@ -1,82 +0,0 @@
----
-
-##### GLOBAL METADATA
-
-- meta:
-    cluster: clients-ci
-
-##### JOB DEFAULTS
-
-- job:
-    project-type: matrix
-    logrotate:
-      daysToKeep: 30
-      numToKeep: 100
-    parameters:
-    - string:
-        name: branch_specifier
-        default: refs/heads/main
-        description: the Git branch specifier to build (<branchName>, <tagName>,
-          <commitId>, etc.)
-    properties:
-    - github:
-        url: https://github.com/elastic/eland
-    - inject:
-        properties-content: HOME=$JENKINS_HOME
-    concurrent: true
-    node: flyweight
-    scm:
-    - git:
-        name: origin
-        credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba
-        reference-repo: /var/lib/jenkins/.git-references/eland.git
-        branches:
-        - ${branch_specifier}
-        url: git@github.com:elastic/eland.git
-        basedir: ''
-        wipe-workspace: 'True'
-    triggers:
-    - github
-    axes:
-    - axis:
-        type: slave
-        name: label
-        values:
-        - linux
-    - axis:
-        type: yaml
-        filename: .ci/test-matrix.yml
-        name: ELASTICSEARCH_VERSION
-    - axis:
-        type: yaml
-        filename: .ci/test-matrix.yml
-        name: PYTHON_VERSION
-    - axis:
-        type: yaml
-        filename: .ci/test-matrix.yml
-        name: PANDAS_VERSION
-    - axis:
-        type: yaml
-        filename: .ci/test-matrix.yml
-        name: TEST_SUITE
-    yaml-strategy:
-      exclude-key: exclude
-      filename: .ci/test-matrix.yml
-    wrappers:
-      - ansicolor
-      - timeout:
-          type: absolute
-          timeout: 120
-          fail: true
-      - timestamps
-      - workspace-cleanup
-    builders:
-      - shell: |-
-          #!/usr/local/bin/runbld
-          .ci/run-tests
-    publishers:
-      - email:
-          recipients: build-lang-clients@elastic.co
-      - junit:
-          results: "build/output/*-junit.xml"
-          allow-empty-results: true

.ci/jobs/elastic+eland+main.yml

@@ -1,14 +0,0 @@
----
-- job:
-    name: elastic+eland+main
-    display-name: 'elastic / eland # main'
-    description: Eland is a data science client with a Pandas-like interface
-    junit_results: "*-junit.xml"
-    parameters:
-      - string:
-          name: branch_specifier
-          default: refs/heads/main
-          description: The Git branch specifier to build
-    triggers:
-      - github
-      - timed: '@daily'

.ci/jobs/elastic+eland+pull-request.yml

@@ -1,19 +0,0 @@
----
-- job:
-    name: elastic+eland+pull-request
-    display-name: 'elastic / eland # pull-request'
-    description: Testing of eland pull requests.
-    scm:
-    - git:
-        branches:
-        - ${ghprbActualCommit}
-        refspec: +refs/pull/*:refs/remotes/origin/pr/*
-    triggers:
-    - github-pull-request:
-        org-list:
-        - elastic
-        allow-whitelist-orgs-as-admins: true
-        github-hooks: true
-        status-context: clients-ci
-        cancel-builds-on-update: true
-    publishers: []

.ci/test-matrix.yml

@@ -1,21 +0,0 @@
----
-
-ELASTICSEARCH_VERSION:
-  - 8.0.0-SNAPSHOT
-  - 7.x-SNAPSHOT
-  - 7.14-SNAPSHOT
-  - 7.11-SNAPSHOT
-
-PANDAS_VERSION:
-  - 1.2.0
-  - 1.3.0
-
-PYTHON_VERSION:
-  - 3.9
-  - 3.8
-  - 3.7
-
-TEST_SUITE:
-  - xpack
-
-exclude: ~

.github/workflows/ci.yml

@@ -1,38 +0,0 @@
-name: CI
-
-on: [push, pull_request]
-
-defaults:
-  run:
-    shell: bash
-
-jobs:
-  lint:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout Repository
-        uses: actions/checkout@v2
-      - name: Set up Python 3
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3
-      - name: Install dependencies
-        run: python3 -m pip install nox
-      - name: Lint the code
-        run: nox -s lint
-
-  docs:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout Repository
-        uses: actions/checkout@v2
-      - name: Set up Python 3
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3
-      - name: Install dependencies
-        run: |
-            sudo apt-get install --yes pandoc
-            python3 -m pip install nox
-      - name: Build documentation
-        run: nox -s docs