diff --git a/.ci/Dockerfile b/.ci/Dockerfile deleted file mode 100644 index 0cdd359..0000000 --- a/.ci/Dockerfile +++ /dev/null @@ -1,7 +0,0 @@ -ARG PYTHON_VERSION=3.9 -FROM python:${PYTHON_VERSION} - -WORKDIR /code/eland -RUN python -m pip install nox - -COPY . . diff --git a/.ci/jobs/defaults.yml b/.ci/jobs/defaults.yml deleted file mode 100755 index dafc8d5..0000000 --- a/.ci/jobs/defaults.yml +++ /dev/null @@ -1,82 +0,0 @@ ---- - -##### GLOBAL METADATA - -- meta: - cluster: clients-ci - -##### JOB DEFAULTS - -- job: - project-type: matrix - logrotate: - daysToKeep: 30 - numToKeep: 100 - parameters: - - string: - name: branch_specifier - default: refs/heads/main - description: the Git branch specifier to build (<branchName>, <tagName>, - <commitId>, etc.) - properties: - - github: - url: https://github.com/elastic/eland - - inject: - properties-content: HOME=$JENKINS_HOME - concurrent: true - node: flyweight - scm: - - git: - name: origin - credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba - reference-repo: /var/lib/jenkins/.git-references/eland.git - branches: - - ${branch_specifier} - url: git@github.com:elastic/eland.git - basedir: '' - wipe-workspace: 'True' - triggers: - - github - axes: - - axis: - type: slave - name: label - values: - - linux - - axis: - type: yaml - filename: .ci/test-matrix.yml - name: ELASTICSEARCH_VERSION - - axis: - type: yaml - filename: .ci/test-matrix.yml - name: PYTHON_VERSION - - axis: - type: yaml - filename: .ci/test-matrix.yml - name: PANDAS_VERSION - - axis: - type: yaml - filename: .ci/test-matrix.yml - name: TEST_SUITE - yaml-strategy: - exclude-key: exclude - filename: .ci/test-matrix.yml - wrappers: - - ansicolor - - timeout: - type: absolute - timeout: 120 - fail: true - - timestamps - - workspace-cleanup - builders: - - shell: |- - #!/usr/local/bin/runbld - .ci/run-tests - publishers: - - email: - recipients: build-lang-clients@elastic.co - - junit: - results: "build/output/*-junit.xml" - allow-empty-results: true diff --git a/.ci/jobs/elastic+eland+7.x.yml b/.ci/jobs/elastic+eland+7.x.yml deleted file mode 100755 index 2dff0e7..0000000 --- a/.ci/jobs/elastic+eland+7.x.yml +++ /dev/null @@ -1,14 +0,0 @@ ---- -- job: - name: elastic+eland+7.x - display-name: 'elastic / eland # 7.x' - description: Eland is a data science client with a Pandas-like interface - junit_results: "*-junit.xml" - parameters: - - string: - name: branch_specifier - default: refs/heads/7.x - description: The Git branch specifier to build - triggers: - - github - - timed: '@daily' diff --git a/.ci/jobs/elastic+eland+main.yml b/.ci/jobs/elastic+eland+main.yml deleted file mode 100755 index 009e80d..0000000 --- a/.ci/jobs/elastic+eland+main.yml +++ /dev/null @@ -1,14 +0,0 @@ ---- -- job: - name: elastic+eland+main - display-name: 'elastic / eland # main' - description: Eland is a data science client with a Pandas-like interface - junit_results: "*-junit.xml" - parameters: - - string: - name: branch_specifier - default: refs/heads/main - description: The Git branch specifier to build - triggers: - - github - - timed: '@daily' diff --git a/.ci/jobs/elastic+eland+pull-request.yml b/.ci/jobs/elastic+eland+pull-request.yml deleted file mode 100644 index 0249070..0000000 --- a/.ci/jobs/elastic+eland+pull-request.yml +++ /dev/null @@ -1,19 +0,0 @@ ---- -- job: - name: elastic+eland+pull-request - display-name: 'elastic / eland # pull-request' - description: Testing of eland pull requests. - scm: - - git: - branches: - - ${ghprbActualCommit} - refspec: +refs/pull/*:refs/remotes/origin/pr/* - triggers: - - github-pull-request: - org-list: - - elastic - allow-whitelist-orgs-as-admins: true - github-hooks: true - status-context: clients-ci - cancel-builds-on-update: true - publishers: [] \ No newline at end of file diff --git a/.ci/run-elasticsearch.sh b/.ci/run-elasticsearch.sh deleted file mode 100755 index abcde11..0000000 --- a/.ci/run-elasticsearch.sh +++ /dev/null @@ -1,189 +0,0 @@ -#!/usr/bin/env bash -# -# Launch one or more Elasticsearch nodes via the Docker image, -# to form a cluster suitable for running the REST API tests. -# -# Export the ELASTICSEARCH_VERSION variable, eg. 'elasticsearch:8.0.0-SNAPSHOT'. - -# Version 1.0 -# - Initial version of the run-elasticsearch.sh script - - -if [[ -z "$ELASTICSEARCH_VERSION" ]]; then - echo -e "\033[31;1mERROR:\033[0m Required environment variable [ELASTICSEARCH_VERSION] not set\033[0m" - exit 1 -fi - -set -euxo pipefail - -SCRIPT_PATH=$(dirname $(realpath -s $0)) - -moniker=$(echo "$ELASTICSEARCH_VERSION" | tr -C "[:alnum:]" '-') -suffix=rest-test - -NODE_NAME=${NODE_NAME-${moniker}node1} -MASTER_NODE_NAME=${MASTER_NODE_NAME-${NODE_NAME}} -CLUSTER_NAME=${CLUSTER_NAME-${moniker}${suffix}} -HTTP_PORT=${HTTP_PORT-9200} - -ELASTIC_PASSWORD=${ELASTIC_PASSWORD-changeme} - -DETACH=${DETACH-false} -CLEANUP=${CLEANUP-false} - -volume_name=${NODE_NAME}-${suffix}-data -network_default=${moniker}${suffix} -NETWORK_NAME=${NETWORK_NAME-"$network_default"} - -set +x - -function cleanup_volume { - if [[ "$(docker volume ls -q -f name=$1)" ]]; then - echo -e "\033[34;1mINFO:\033[0m Removing volume $1\033[0m" - (docker volume rm "$1") || true - fi -} -function container_running { - if [[ "$(docker ps -q -f name=$1)" ]]; then - return 0; - else return 1; - fi -} -function cleanup_node { - if container_running "$1"; then - echo -e "\033[34;1mINFO:\033[0m Removing container $1\033[0m" - (docker container rm --force --volumes "$1") || true - cleanup_volume "$1-${suffix}-data" - fi -} -function cleanup_network { - if [[ "$(docker network ls -q -f name=$1)" ]]; then - echo -e "\033[34;1mINFO:\033[0m Removing network $1\033[0m" - (docker network rm "$1") || true - fi -} - -function cleanup { - if [[ "$DETACH" != "true" ]] || [[ "$1" == "1" ]]; then - echo -e "\033[34;1mINFO:\033[0m clean the node and volume on startup (1) OR on exit if not detached\033[0m" - cleanup_node "$NODE_NAME" - fi - if [[ "$DETACH" != "true" ]]; then - echo -e "\033[34;1mINFO:\033[0m clean the network if not detached (start and exit)\033[0m" - cleanup_network "$NETWORK_NAME" - fi -}; -trap "cleanup 0" EXIT - -if [[ "$CLEANUP" == "true" ]]; then - trap - EXIT - if [[ -z "$(docker network ls -q -f name=${NETWORK_NAME})" ]]; then - echo -e "\033[34;1mINFO:\033[0m $NETWORK_NAME is already deleted\033[0m" - exit 0 - fi - containers=$(docker network inspect -f '{{ range $key, $value := .Containers }}{{ printf "%s\n" .Name}}{{ end }}' ${NETWORK_NAME}) - while read -r container; do - cleanup_node "$container" - done <<< "$containers" - cleanup_network "$NETWORK_NAME" - echo -e "\033[32;1mSUCCESS:\033[0m Cleaned up and exiting\033[0m" - exit 0 -fi - -echo -e "\033[34;1mINFO:\033[0m Making sure previous run leftover infrastructure is removed \033[0m" -cleanup 1 - -echo -e "\033[34;1mINFO:\033[0m Creating network $NETWORK_NAME if it does not exist already \033[0m" -docker network inspect "$NETWORK_NAME" > /dev/null 2>&1 || docker network create "$NETWORK_NAME" - -environment=($(cat <<-END - --env node.name=$NODE_NAME - --env cluster.name=$CLUSTER_NAME - --env cluster.initial_master_nodes=$MASTER_NODE_NAME - --env discovery.seed_hosts=$MASTER_NODE_NAME - --env cluster.routing.allocation.disk.threshold_enabled=false - --env bootstrap.memory_lock=true - --env node.attr.testattr=test - --env path.repo=/tmp - --env repositories.url.allowed_urls=http://snapshot.test* -END -)) - -volumes=($(cat <<-END - --volume $volume_name:/usr/share/elasticsearch/data -END -)) - -if [[ "$ELASTICSEARCH_VERSION" != *oss* ]]; then - environment+=($(cat <<-END - --env ELASTIC_PASSWORD=$ELASTIC_PASSWORD - --env xpack.license.self_generated.type=trial - --env xpack.security.enabled=false - --env xpack.security.http.ssl.enabled=false - --env xpack.security.transport.ssl.enabled=false - --env xpack.ml.max_machine_memory_percent=90 -END -)) -fi - -url="http://$NODE_NAME" -if [[ "$ELASTICSEARCH_VERSION" != *oss* ]]; then - url="http://elastic:$ELASTIC_PASSWORD@$NODE_NAME" -fi - -# Pull the container, retry on failures up to 5 times with -# short delays between each attempt. Fixes most transient network errors. -docker_pull_attempts=0 -until [ "$docker_pull_attempts" -ge 5 ] -do - docker pull docker.elastic.co/elasticsearch/"$ELASTICSEARCH_VERSION" && break - docker_pull_attempts=$((docker_pull_attempts+1)) - sleep 10 -done - -echo -e "\033[34;1mINFO:\033[0m Starting container $NODE_NAME \033[0m" -set -x -docker run \ - --name "$NODE_NAME" \ - --network "$NETWORK_NAME" \ - --env ES_JAVA_OPTS=-"Xms2g -Xmx2g" \ - "${environment[@]}" \ - "${volumes[@]}" \ - --publish "$HTTP_PORT":9200 \ - --ulimit nofile=65536:65536 \ - --ulimit memlock=-1:-1 \ - --detach="$DETACH" \ - --health-cmd="curl --insecure --fail $url:9200/_cluster/health || exit 1" \ - --health-interval=2s \ - --health-retries=20 \ - --health-timeout=2s \ - --rm \ - docker.elastic.co/elasticsearch/"$ELASTICSEARCH_VERSION"; -set +x - -if [[ "$DETACH" == "true" ]]; then - until ! container_running "$NODE_NAME" || (container_running "$NODE_NAME" && [[ "$(docker inspect -f "{{.State.Health.Status}}" ${NODE_NAME})" != "starting" ]]); do - echo "" - docker inspect -f "{{range .State.Health.Log}}{{.Output}}{{end}}" ${NODE_NAME} - echo -e "\033[34;1mINFO:\033[0m waiting for node $NODE_NAME to be up\033[0m" - sleep 2; - done; - - # Always show logs if the container is running, this is very useful both on CI as well as while developing - if container_running $NODE_NAME; then - docker logs $NODE_NAME - fi - - if ! container_running $NODE_NAME || [[ "$(docker inspect -f "{{.State.Health.Status}}" ${NODE_NAME})" != "healthy" ]]; then - cleanup 1 - echo - echo -e "\033[31;1mERROR:\033[0m Failed to start ${ELASTICSEARCH_VERSION} in detached mode beyond health checks\033[0m" - echo -e "\033[31;1mERROR:\033[0m dumped the docker log before shutting the node down\033[0m" - exit 1 - else - echo - echo -e "\033[32;1mSUCCESS:\033[0m Detached and healthy: ${NODE_NAME} on docker network: ${NETWORK_NAME}\033[0m" - echo -e "\033[32;1mSUCCESS:\033[0m Running on: ${url/$NODE_NAME/localhost}:${HTTP_PORT}\033[0m" - exit 0 - fi -fi diff --git a/.ci/run-repository.sh b/.ci/run-repository.sh deleted file mode 100755 index d068992..0000000 --- a/.ci/run-repository.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash -# -# Called by entry point `run-test` use this script to add your repository specific test commands -# -# Once called Elasticsearch is up and running and the following parameters are available to this script - -# ELASTICSEARCH_VERSION -- version e.g Major.Minor.Patch(-Prelease) -# ELASTICSEARCH_CONTAINER -- the docker moniker as a reference to know which docker image distribution is used -# ELASTICSEARCH_URL -- The url at which elasticsearch is reachable -# NETWORK_NAME -- The docker network name -# NODE_NAME -- The docker container name also used as Elasticsearch node name - -# When run in CI the test-matrix is used to define additional variables - -# TEST_SUITE -- either `oss` or `xpack`, defaults to `oss` in `run-tests` -# - -PYTHON_VERSION=${PYTHON_VERSION-3.8} -echo -e "\033[34;1mINFO:\033[0m URL ${ELASTICSEARCH_URL}\033[0m" -echo -e "\033[34;1mINFO:\033[0m VERSION ${ELASTICSEARCH_VERSION}\033[0m" -echo -e "\033[34;1mINFO:\033[0m CONTAINER ${ELASTICSEARCH_CONTAINER}\033[0m" -echo -e "\033[34;1mINFO:\033[0m TEST_SUITE ${TEST_SUITE}\033[0m" -echo -e "\033[34;1mINFO:\033[0m PYTHON_VERSION ${PYTHON_VERSION}\033[0m" -echo -e "\033[34;1mINFO:\033[0m PANDAS_VERSION ${PANDAS_VERSION}\033[0m" - -echo -e "\033[1m>>>>> Build [elastic/eland container] >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m" - -docker build --file .ci/Dockerfile --tag elastic/eland --build-arg PYTHON_VERSION=${PYTHON_VERSION} . - -echo -e "\033[1m>>>>> Run [elastic/eland container] >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m" - -docker run \ - --network=${NETWORK_NAME} \ - --env "ELASTICSEARCH_HOST=${ELASTICSEARCH_URL}" \ - --env "TEST_SUITE=${TEST_SUITE}" \ - --name eland-test-runner \ - --rm \ - elastic/eland \ - nox -s "test-${PYTHON_VERSION}(pandas_version='${PANDAS_VERSION}')" diff --git a/.ci/run-tests b/.ci/run-tests deleted file mode 100755 index 2567314..0000000 --- a/.ci/run-tests +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env bash -# -# Version 1.0 -# - Moved to .ci folder and seperated out `run-repository.sh` - -if [[ -z $ELASTICSEARCH_VERSION ]]; then - echo -e "\033[31;1mERROR:\033[0m Required environment variable [ELASTICSEARCH_VERSION] not set\033[0m" - exit 1 -fi -set -euxo pipefail - - -TEST_SUITE=${TEST_SUITE-xpack} -NODE_NAME=localhost -PANDAS_VERSION=${PANDAS_VERSION-1.5.0} - - -elasticsearch_image=elasticsearch -elasticsearch_url=http://elastic:changeme@${NODE_NAME}:9200 -if [[ $TEST_SUITE != "xpack" ]]; then - elasticsearch_image=elasticsearch-${TEST_SUITE} - elasticsearch_url=http://${NODE_NAME}:9200 -fi - -function cleanup { - status=$? - set +x - ELASTICSEARCH_VERSION=${elasticsearch_image}:${ELASTICSEARCH_VERSION} \ - NODE_NAME=${NODE_NAME} \ - NETWORK_NAME=elasticsearch \ - CLEANUP=true \ - bash ./.ci/run-elasticsearch.sh - # Report status and exit - if [[ "$status" == "0" ]]; then - echo -e "\n\033[32;1mSUCCESS run-tests\033[0m" - exit 0 - else - echo -e "\n\033[31;1mFAILURE during run-tests\033[0m" - exit ${status} - fi -} -trap cleanup EXIT - -echo -e "\033[1m>>>>> Start [$ELASTICSEARCH_VERSION container] >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m" - -ELASTICSEARCH_VERSION=${elasticsearch_image}:${ELASTICSEARCH_VERSION} \ - NODE_NAME=${NODE_NAME} \ - NETWORK_NAME=host \ - DETACH=true \ - bash .ci/run-elasticsearch.sh - -echo -e "\033[1m>>>>> Repository specific tests >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m" - -ELASTICSEARCH_CONTAINER=${elasticsearch_image}:${ELASTICSEARCH_VERSION} \ - NETWORK_NAME=host \ - NODE_NAME=${NODE_NAME} \ - ELASTICSEARCH_URL=${elasticsearch_url} \ - TEST_SUITE=${TEST_SUITE} \ - PANDAS_VERSION=${PANDAS_VERSION} \ - bash .ci/run-repository.sh - diff --git a/.ci/test-matrix.yml b/.ci/test-matrix.yml deleted file mode 100644 index 5b4ed7a..0000000 --- a/.ci/test-matrix.yml +++ /dev/null @@ -1,19 +0,0 @@ ---- - -ELASTICSEARCH_VERSION: - - '8.6.0-SNAPSHOT' - - '8.7.0-SNAPSHOT' - - '8.8.0-SNAPSHOT' - -PANDAS_VERSION: - - '1.5.0' - -PYTHON_VERSION: - - '3.10' - - '3.9' - - '3.8' - -TEST_SUITE: - - xpack - -exclude: ~