From fc7c214ae6f2991ded3f41f64632df08d990129b Mon Sep 17 00:00:00 2001 From: Winterflower Date: Thu, 21 Nov 2019 13:53:27 +0100 Subject: [PATCH 01/17] Initial skeleton for .ci folder --- .ci/certs/ca.crt | 20 ++ .ci/certs/ca.pem | 20 ++ .ci/certs/testnode.crt | 19 ++ .ci/certs/testnode.key | 27 +++ .ci/jobs/defaults.yml | 74 +++++++ ...elastic+clients-flight-recorder+master.yml | 14 ++ .ci/run-elasticsearch.sh | 197 ++++++++++++++++++ .ci/run-tests | 55 +++++ .ci/test-matrix.yml | 11 + 9 files changed, 437 insertions(+) create mode 100755 .ci/certs/ca.crt create mode 100644 .ci/certs/ca.pem create mode 100755 .ci/certs/testnode.crt create mode 100755 .ci/certs/testnode.key create mode 100755 .ci/jobs/defaults.yml create mode 100755 .ci/jobs/elastic+clients-flight-recorder+master.yml create mode 100755 .ci/run-elasticsearch.sh create mode 100755 .ci/run-tests create mode 100755 .ci/test-matrix.yml diff --git a/.ci/certs/ca.crt b/.ci/certs/ca.crt new file mode 100755 index 0000000..6402874 --- /dev/null +++ b/.ci/certs/ca.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDSTCCAjGgAwIBAgIUIwN+0zglsexRKwE1RGHvlCcmrdwwDQYJKoZIhvcNAQEL +BQAwNDEyMDAGA1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5l +cmF0ZWQgQ0EwHhcNMTkwMjEzMDcyMjQwWhcNMjIwMjEyMDcyMjQwWjA0MTIwMAYD +VQQDEylFbGFzdGljIENlcnRpZmljYXRlIFRvb2wgQXV0b2dlbmVyYXRlZCBDQTCC +ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBANILs0JO0e7x29zeVx21qalK +XKdX+AMlGJPH75wWO/Jq6YHtxt1wYIg762krOBXfG6JsFSOIwIv5VrzGGRGjSPt9 +OXQyXrDDiQvsBT3rpzLNdDs7KMl2tZswwv7w9ujgud0cYnS1MOpn81rfPc73DvMg +xuhplofDx6fn3++PjVRU2FNiIVWyEoaxRjCeGPMBubKZYaYbQA6vYM4Z+ByG727B +AyAER3t7xmvYti/EoO2hv2HQk5zgcj/Oq3AJKhnt8LH8fnfm3TnYNM1htvXqhN05 +vsvhvm2PHfnA5qLlSr/3W0aI/U/PqfsFDCgyRV097sMIaKkmavb0Ue7aQ7lgtp0C +AwEAAaNTMFEwHQYDVR0OBBYEFDRKlCMowWR1rwxE0d1lTEQe5O71MB8GA1UdIwQY +MBaAFDRKlCMowWR1rwxE0d1lTEQe5O71MA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI +hvcNAQELBQADggEBAKbCJ95EBpeuvF70KEt6QU70k/SH1NRvM9YzKryV0D975Jvu +HOSm9HgSTULeAUFZIa4oYyf3QUfVoI+2T/aQrfXA3gfrJWsHURkyNmiHOFAbYHqi +xA6i249G2GTEjc1+le/M2N2CcDKAmurW6vSGK4upXQbPd6KmnhHREX74zkWjnOa+ ++tibbSSOCT4Tmja2DbBxAPuivU9IB1g/hIUmbYQqKffQrBJA0658tz6w63a/Q7xN +pCvvbSgiMZ6qcVIcJkBT2IooYie+ax45pQECHthgIUcQAzfmIfqlU0Qfl8rDgAmn +0c1o6HQjKGU2aVGgSRuaaiHaSZjbPIZVS51sOoI= +-----END CERTIFICATE----- diff --git a/.ci/certs/ca.pem b/.ci/certs/ca.pem new file mode 100644 index 0000000..6402874 --- /dev/null +++ b/.ci/certs/ca.pem @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDSTCCAjGgAwIBAgIUIwN+0zglsexRKwE1RGHvlCcmrdwwDQYJKoZIhvcNAQEL +BQAwNDEyMDAGA1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5l +cmF0ZWQgQ0EwHhcNMTkwMjEzMDcyMjQwWhcNMjIwMjEyMDcyMjQwWjA0MTIwMAYD +VQQDEylFbGFzdGljIENlcnRpZmljYXRlIFRvb2wgQXV0b2dlbmVyYXRlZCBDQTCC +ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBANILs0JO0e7x29zeVx21qalK +XKdX+AMlGJPH75wWO/Jq6YHtxt1wYIg762krOBXfG6JsFSOIwIv5VrzGGRGjSPt9 +OXQyXrDDiQvsBT3rpzLNdDs7KMl2tZswwv7w9ujgud0cYnS1MOpn81rfPc73DvMg +xuhplofDx6fn3++PjVRU2FNiIVWyEoaxRjCeGPMBubKZYaYbQA6vYM4Z+ByG727B +AyAER3t7xmvYti/EoO2hv2HQk5zgcj/Oq3AJKhnt8LH8fnfm3TnYNM1htvXqhN05 +vsvhvm2PHfnA5qLlSr/3W0aI/U/PqfsFDCgyRV097sMIaKkmavb0Ue7aQ7lgtp0C +AwEAAaNTMFEwHQYDVR0OBBYEFDRKlCMowWR1rwxE0d1lTEQe5O71MB8GA1UdIwQY +MBaAFDRKlCMowWR1rwxE0d1lTEQe5O71MA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI +hvcNAQELBQADggEBAKbCJ95EBpeuvF70KEt6QU70k/SH1NRvM9YzKryV0D975Jvu +HOSm9HgSTULeAUFZIa4oYyf3QUfVoI+2T/aQrfXA3gfrJWsHURkyNmiHOFAbYHqi +xA6i249G2GTEjc1+le/M2N2CcDKAmurW6vSGK4upXQbPd6KmnhHREX74zkWjnOa+ ++tibbSSOCT4Tmja2DbBxAPuivU9IB1g/hIUmbYQqKffQrBJA0658tz6w63a/Q7xN +pCvvbSgiMZ6qcVIcJkBT2IooYie+ax45pQECHthgIUcQAzfmIfqlU0Qfl8rDgAmn +0c1o6HQjKGU2aVGgSRuaaiHaSZjbPIZVS51sOoI= +-----END CERTIFICATE----- diff --git a/.ci/certs/testnode.crt b/.ci/certs/testnode.crt new file mode 100755 index 0000000..ff3bcb3 --- /dev/null +++ b/.ci/certs/testnode.crt @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDIjCCAgqgAwIBAgIUI4QU6jA1dYSCbdIA6oAb2TBEluowDQYJKoZIhvcNAQEL +BQAwNDEyMDAGA1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5l +cmF0ZWQgQ0EwHhcNMTkwMjEzMDcyMzEzWhcNMjIwMjEyMDcyMzEzWjATMREwDwYD +VQQDEwhpbnN0YW5jZTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAJeT +yOy6EAScZxrULKjHePciiz38grivCrhFFV+dThaRCcl3DhDzb9Eny5q5iEw3WvLQ +Rqmf01jncNIhaocTt66VqveXaMubbE8O0LcG6e4kpFO+JtnVF8JTARTc+ux/1uD6 +hO1VG/HItM7WQrQxh4hfB2u1AX2YQtoqEtXXEC+UHWfl4QzuzXjBnKCkO/L9/6Tf +yNFQWXxKnIiTs8Xm9sEhhSCBJPlLTQu+MX4vR2Uwj5XZmflDUr+ZTenl9qYxL6b3 +SWhh/qEl4GAj1+tS7ZZOxE0237mUh3IIFYSWSaMm8K2m/BYHkLNWL5B1dMic0lsv +osSoYrQuCef4HQMCitsCAwEAAaNNMEswHQYDVR0OBBYEFFMg4l1GLW8lYbwASY+r +YeWYRzIiMB8GA1UdIwQYMBaAFDRKlCMowWR1rwxE0d1lTEQe5O71MAkGA1UdEwQC +MAAwDQYJKoZIhvcNAQELBQADggEBAEQrgh1xALpumQTzsjxFRGque/vlKTgRs5Kh +xtgapr6wjIbdq7dagee+4yNOKzS5lGVXCgwrJlHESv9qY0uumT/33vK2uduJ7NAd +fR2ZzyBnhMX+mkYhmGrGYCTUMUIwOIQYa4Evis4W+LHmCIDG03l7gLHfdIBe9VMO +pDZum8f6ng0MM49s8/rXODNYKw8kFyUhnfChqMi/2yggb1uUIfKlJJIchkgYjE13 +zuC+fjo029Pq1jeMIdxugLf/7I/8NiW1Yj9aCXevUXG1qzHFEuKAinBXYOZO/vWS +LaEqOhwrzNynwgGpYAr7Rfgv4AflltYIIav4PZT03P7fbyAAf8s= +-----END CERTIFICATE----- diff --git a/.ci/certs/testnode.key b/.ci/certs/testnode.key new file mode 100755 index 0000000..c35b4bc --- /dev/null +++ b/.ci/certs/testnode.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpQIBAAKCAQEAl5PI7LoQBJxnGtQsqMd49yKLPfyCuK8KuEUVX51OFpEJyXcO +EPNv0SfLmrmITDda8tBGqZ/TWOdw0iFqhxO3rpWq95doy5tsTw7Qtwbp7iSkU74m +2dUXwlMBFNz67H/W4PqE7VUb8ci0ztZCtDGHiF8Ha7UBfZhC2ioS1dcQL5QdZ+Xh +DO7NeMGcoKQ78v3/pN/I0VBZfEqciJOzxeb2wSGFIIEk+UtNC74xfi9HZTCPldmZ ++UNSv5lN6eX2pjEvpvdJaGH+oSXgYCPX61Ltlk7ETTbfuZSHcggVhJZJoybwrab8 +FgeQs1YvkHV0yJzSWy+ixKhitC4J5/gdAwKK2wIDAQABAoIBAQCRFTJna/xy/WUu +59FLR4qAOj8++JgCwACpue4oU7/vl6nffSYokWoAr2+RzG4qTX2vFi3cpA8+dGCn +sLZvTi8tWzKGxBTZdg2oakzaMzLr74SeZ052iCGyrZJGbvF6Ny7srr1XEXSq6+os +ZCb6pMHOhO7saBdiKMAsY8MdjTl/33AduuE6ztqv+L92xTr2g4QlbT1KvWlEgppU +k4Gy7zdETkPBTSH/17ZwyGJoJICIAhbL4IpmOM4dPIg8nFkVPPpy6p0z4uGjtgnK +nreZ2EKMzCafBaHn7A77gpi0OrQdl6pe0fsGqv/323YjCJPbwwl5TsoNq44DzwiX +3M7XiVJxAoGBAOCne56vdN4uZmCgLVGT2JSUNVPOu4bfjrxWH6cslzrPT2Zhp3lO +M4axZ3gmcervV252YEZXntXDHHCSfrECllRN1WFD63XmyQ/CkhuvZkkeRHfzL1TE +EdqHOTqs4sRETZ7+RITFC81DZQkWWOKeyXMjyPBqd7RnThQHijB1c8Y5AoGBAKy6 +CVKBx+zz5crVD0tz4UhOmz1wRNN0CL0l+FXRuFSgbzMIvwpfiqe25crgeLHe2M2/ +TogdWbjZ2nUZQTzoRsSkQ6cKHpj+G/gWurp/UcHHXFVwgLSPF7c3KHDtiYq7Vqw0 +bvmhM03LI6+ZIPRV7hLBr7WP7UmpAiREMF7tTnmzAoGBAIkx3w3WywFQxtblmyeB +qbd7F2IaE23XoxyjX+tBEQ4qQqwcoSE0v8TXHIBEwjceeX+NLVhn9ClJYVniLRq+ +oL3VVqVyzB4RleJZCc98e3PV1yyFx/b1Uo3pHOsXX9lKeTjKwV9v0rhFGzPEgP3M +yOvXA8TG0FnM6OLUg/D6GX0JAoGAMuHS4TVOGeV3ahr9mHKYiN5vKNgrzka+VEod +L9rJ/FQOrfADpyCiDen5I5ygsXU+VM3oanyK88NpcVlxOGoMft0M+OYoQVWKE7lO +ZKYhBX6fGqQ7pfUJPXXIOgwfmni5fZ0sm+j63g3bg10OsiumKGxaQJgXhL1+3gQg +Y7ZwibUCgYEAlZoFFvkMLjpOSaHk1z5ZZnt19X0QUIultBwkumSqMPm+Ks7+uDrx +thGUCoz4ecr/ci4bIUY7mB+zfAbqnBOMxreJqCRbAIuRypo1IlWkTp8DywoDOfMW +NfzjVmzJ7EJu44nGmVAi1jw4Pbseivvi1ujMCoPgaE8I1uSh144bwN8= +-----END RSA PRIVATE KEY----- diff --git a/.ci/jobs/defaults.yml b/.ci/jobs/defaults.yml new file mode 100755 index 0000000..707244a --- /dev/null +++ b/.ci/jobs/defaults.yml @@ -0,0 +1,74 @@ +--- + +##### GLOBAL METADATA + +- meta: + cluster: clients-ci + +##### JOB DEFAULTS + +- job: + project-type: matrix + logrotate: + daysToKeep: 30 + numToKeep: 100 + parameters: + - string: + name: branch_specifier + default: refs/heads/master + description: the Git branch specifier to build (<branchName>, <tagName>, + <commitId>, etc.) + properties: + - github: + url: https://github.com/elastic/clients-flight-recorder/ + - inject: + properties-content: HOME=$JENKINS_HOME + concurrent: true + node: flyweight + scm: + - git: + name: origin + credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba + reference-repo: /var/lib/jenkins/.git-references/clients-flight-recorder.git + branches: + - ${branch_specifier} + url: git@github.com:elastic/clients-flight-recorder.git + basedir: '' + wipe-workspace: 'True' + triggers: + - github + axes: + - axis: + type: slave + name: label + values: + - linux + - axis: + type: yaml + filename: .ci/test-matrix.yml + name: ELASTICSEARCH_VERSION + - axis: + type: yaml + filename: .ci/test-matrix.yml + name: TEST_SUITE + yaml-strategy: + exclude-key: exclude + filename: .ci/test-matrix.yml + wrappers: + - ansicolor + - timeout: + type: absolute + timeout: 120 + fail: true + - timestamps + - workspace-cleanup + builders: + - shell: |- + #!/usr/local/bin/runbld + .ci/run-tests + publishers: + - email: + recipients: infra-root+build@elastic.co + - junit: + results: "build/output/*-junit.xml" + allow-empty-results: true diff --git a/.ci/jobs/elastic+clients-flight-recorder+master.yml b/.ci/jobs/elastic+clients-flight-recorder+master.yml new file mode 100755 index 0000000..53900c9 --- /dev/null +++ b/.ci/jobs/elastic+clients-flight-recorder+master.yml @@ -0,0 +1,14 @@ +--- +- job: + name: elastic+clients-flight-recorder+master + display-name: 'elastic / clients-flight-recorder # master' + description: Flight recorder captures normalized request responses to be notified of state changes + junit_results: "*-junit.xml" + parameters: + - string: + name: branch_specifier + default: refs/heads/master + description: The Git branch specifier to build + triggers: + - github + - timed: '@daily' diff --git a/.ci/run-elasticsearch.sh b/.ci/run-elasticsearch.sh new file mode 100755 index 0000000..68daeb4 --- /dev/null +++ b/.ci/run-elasticsearch.sh @@ -0,0 +1,197 @@ +#!/usr/bin/env bash +# +# Launch one or more Elasticsearch nodes via the Docker image, +# to form a cluster suitable for running the REST API tests. +# +# Export the ELASTICSEARCH_VERSION variable, eg. 'elasticsearch:8.0.0-SNAPSHOT'. + +if [[ -z "$ELASTICSEARCH_VERSION" ]]; then + echo -e "\033[31;1mERROR:\033[0m Required environment variable [ELASTICSEARCH_VERSION] not set\033[0m" + exit 1 +fi + +set -euxo pipefail + +SCRIPT_PATH=$(dirname $(realpath -s $0)) + +moniker=$(echo "$ELASTICSEARCH_VERSION" | tr -C "[:alnum:]" '-') +suffix=rest-test + +NODE_NAME=${NODE_NAME-${moniker}node1} +MASTER_NODE_NAME=${MASTER_NODE_NAME-${NODE_NAME}} +CLUSTER_NAME=${CLUSTER_NAME-${moniker}${suffix}} +HTTP_PORT=${HTTP_PORT-9200} + +ELASTIC_PASSWORD=${ELASTIC_PASSWORD-changeme} +SSL_CERT=${SSL_CERT-"${SCRIPT_PATH}/certs/testnode.crt"} +SSL_KEY=${SSL_KEY-"${SCRIPT_PATH}/certs/testnode.key"} +SSL_CA=${SSL_CA-"${SCRIPT_PATH}/certs/ca.crt"} +SSL_CA_PEM=${SSL_CA-"${SCRIPT_PATH}/certs/ca.pem"} + +DETACH=${DETACH-false} +CLEANUP=${CLEANUP-false} + +volume_name=${NODE_NAME}-${suffix}-data +network_default=${moniker}${suffix} +NETWORK_NAME=${NETWORK_NAME-"$network_default"} + +set +x + +function cleanup_volume { + if [[ "$(docker volume ls -q -f name=$1)" ]]; then + echo -e "\033[34;1mINFO:\033[0m Removing volume $1\033[0m" + (docker volume rm "$1") || true + fi +} +function container_running { + if [[ "$(docker ps -q -f name=$1)" ]]; then + return 0; + else return 1; + fi +} +function cleanup_node { + if container_running "$1"; then + echo -e "\033[34;1mINFO:\033[0m Removing container $1\033[0m" + (docker container rm --force --volumes "$1") || true + cleanup_volume "$1-${suffix}-data" + fi +} +function cleanup_network { + if [[ "$(docker network ls -q -f name=$1)" ]]; then + echo -e "\033[34;1mINFO:\033[0m Removing network $1\033[0m" + (docker network rm "$1") || true + fi +} + +function cleanup { + if [[ "$DETACH" != "true" ]] || [[ "$1" == "1" ]]; then + echo -e "\033[34;1mINFO:\033[0m clean the node and volume on startup (1) OR on exit if not detached\033[0m" + cleanup_node "$NODE_NAME" + fi + if [[ "$DETACH" != "true" ]]; then + echo -e "\033[34;1mINFO:\033[0m clean the network if not detached (start and exit)\033[0m" + cleanup_network "$NETWORK_NAME" + fi +}; +trap "cleanup 0" EXIT + +if [[ "$CLEANUP" == "true" ]]; then + trap - EXIT + if [[ -z "$(docker network ls -q -f name=${NETWORK_NAME})" ]]; then + echo -e "\033[34;1mINFO:\033[0m $NETWORK_NAME is already deleted\033[0m" + exit 0 + fi + containers=$(docker network inspect -f '{{ range $key, $value := .Containers }}{{ printf "%s\n" .Name}}{{ end }}' ${NETWORK_NAME}) + while read -r container; do + cleanup_node "$container" + done <<< "$containers" + cleanup_network "$NETWORK_NAME" + echo -e "\033[32;1mSUCCESS:\033[0m Cleaned up and exiting\033[0m" + exit 0 +fi + +echo -e "\033[34;1mINFO:\033[0m Making sure previous run leftover infrastructure is removed \033[0m" +cleanup 1 + +echo -e "\033[34;1mINFO:\033[0m Creating network $NETWORK_NAME if it does not exist already \033[0m" +docker network inspect "$NETWORK_NAME" > /dev/null 2>&1 || docker network create "$NETWORK_NAME" + +environment=($(cat <<-END + --env node.name=$NODE_NAME + --env cluster.name=$CLUSTER_NAME + --env cluster.initial_master_nodes=$MASTER_NODE_NAME + --env discovery.seed_hosts=$MASTER_NODE_NAME + --env cluster.routing.allocation.disk.threshold_enabled=false + --env bootstrap.memory_lock=true + --env node.attr.testattr=test + --env path.repo=/tmp + --env repositories.url.allowed_urls=http://snapshot.test* +END +)) + +volumes=($(cat <<-END + --volume $volume_name:/usr/share/elasticsearch/data +END +)) + +if [[ "$ELASTICSEARCH_VERSION" != *oss* ]]; then + environment+=($(cat <<-END + --env ELASTIC_PASSWORD=$ELASTIC_PASSWORD + --env xpack.license.self_generated.type=trial + --env xpack.security.enabled=true + --env xpack.security.http.ssl.enabled=true + --env xpack.security.http.ssl.verification_mode=certificate + --env xpack.security.http.ssl.key=certs/testnode.key + --env xpack.security.http.ssl.certificate=certs/testnode.crt + --env xpack.security.http.ssl.certificate_authorities=certs/ca.crt + --env xpack.security.transport.ssl.enabled=true + --env xpack.security.transport.ssl.key=certs/testnode.key + --env xpack.security.transport.ssl.certificate=certs/testnode.crt + --env xpack.security.transport.ssl.certificate_authorities=certs/ca.crt +END +)) + volumes+=($(cat <<-END + --volume $SSL_CERT:/usr/share/elasticsearch/config/certs/testnode.crt + --volume $SSL_KEY:/usr/share/elasticsearch/config/certs/testnode.key + --volume $SSL_CA:/usr/share/elasticsearch/config/certs/ca.crt + --volume $SSL_CA_PEM:/usr/share/elasticsearch/config/certs/ca.pem +END +)) +fi + +url="http://$NODE_NAME" +if [[ "$ELASTICSEARCH_VERSION" != *oss* ]]; then + url="https://elastic:$ELASTIC_PASSWORD@$NODE_NAME" +fi + +cert_validation_flags="--insecure" +if [[ "$NODE_NAME" == "instance" ]]; then + cert_validation_flags="--cacert /usr/share/elasticsearch/config/certs/ca.pem --resolve ${NODE_NAME}:443:127.0.0.1" +fi + +echo -e "\033[34;1mINFO:\033[0m Starting container $NODE_NAME \033[0m" +set -x +docker run \ + --name "$NODE_NAME" \ + --network "$NETWORK_NAME" \ + --env ES_JAVA_OPTS=-"Xms1g -Xmx1g" \ + "${environment[@]}" \ + "${volumes[@]}" \ + --publish "$HTTP_PORT":9200 \ + --ulimit nofile=65536:65536 \ + --ulimit memlock=-1:-1 \ + --detach="$DETACH" \ + --health-cmd="curl $cert_validation_flags --fail $url:9200/_cluster/health || exit 1" \ + --health-interval=2s \ + --health-retries=20 \ + --health-timeout=2s \ + --rm \ + docker.elastic.co/elasticsearch/"$ELASTICSEARCH_VERSION"; +set +x + +if [[ "$DETACH" == "true" ]]; then + until ! container_running "$NODE_NAME" || (container_running "$NODE_NAME" && [[ "$(docker inspect -f "{{.State.Health.Status}}" ${NODE_NAME})" != "starting" ]]); do + echo "" + docker inspect -f "{{range .State.Health.Log}}{{.Output}}{{end}}" ${NODE_NAME} + echo -e "\033[34;1mINFO:\033[0m waiting for node $NODE_NAME to be up\033[0m" + sleep 2; + done; + + # Always show logs if the container is running, this is very useful both on CI as well as while developing + if container_running "$NODE_NAME"; then + docker logs $NODE_NAME + fi + + if ! container_running "$NODE_NAME" || [[ "$(docker inspect -f "{{.State.Health.Status}}" ${NODE_NAME})" != "healthy" ]]; then + cleanup 1 + echo + echo -e "\033[31;1mERROR:\033[0m Failed to start ${ELASTICSEARCH_VERSION} in detached mode beyond health checks\033[0m" + echo -e "\033[31;1mERROR:\033[0m dumped the docker log before shutting the node down\033[0m" + exit 1 + else + echo + echo -e "\033[32;1mSUCCESS:\033[0m Detached and healthy: ${NODE_NAME} on docker network: ${NETWORK_NAME}\033[0m" + echo -e "\033[32;1mSUCCESS:\033[0m Running on: ${url/$NODE_NAME/localhost}:${HTTP_PORT}\033[0m" + exit 0 + fi +fi diff --git a/.ci/run-tests b/.ci/run-tests new file mode 100755 index 0000000..48a7d09 --- /dev/null +++ b/.ci/run-tests @@ -0,0 +1,55 @@ +#!/usr/bin/env bash + +if [[ -z $ELASTICSEARCH_VERSION ]]; then + echo -e "\033[31;1mERROR:\033[0m Required environment variable [ELASTICSEARCH_VERSION] not set\033[0m" + exit 1 +fi +set -euxo pipefail + + +TEST_SUITE=${TEST_SUITE-oss} +NODE_NAME=instance + + +repo=$(pwd) + +elasticsearch_image=elasticsearch +elasticsearch_url=https://elastic:changeme@${NODE_NAME}:9200 +if [[ $TEST_SUITE != "xpack" ]]; then + elasticsearch_image=elasticsearch-${TEST_SUITE} + elasticsearch_url=http://${NODE_NAME}:9200 +fi + +function cleanup { + status=$? + set +x + ELASTICSEARCH_VERSION=${elasticsearch_image}:${ELASTICSEARCH_VERSION} \ + NODE_NAME=${NODE_NAME} \ + NETWORK_NAME=elasticsearch \ + CLEANUP=true \ + bash ./.ci/run-elasticsearch.sh + # Report status and exit + if [[ "$status" == "0" ]]; then + echo -e "\n\033[32;1mSUCCESS run-tests\033[0m" + exit 0 + else + echo -e "\n\033[31;1mFAILURE during run-tests\033[0m" + exit ${status} + fi +} +trap cleanup EXIT + +echo -e "\033[1m>>>>> Start [$ELASTICSEARCH_VERSION container] >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m" + +ELASTICSEARCH_VERSION=${elasticsearch_image}:${ELASTICSEARCH_VERSION} \ + NODE_NAME=${NODE_NAME} \ + NETWORK_NAME=elasticsearch \ + DETACH=true \ + bash .ci/run-elasticsearch.sh + +echo -e "\033[1m>>>>> YOUR STEPS HERE >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m" + +## Add your steps here +## Use ${elasticsearch_url} to talk to elasticsearch +## declare your matrix variables here as well e.g DOTNET_VERSION=${DOTNET_VERSION-3.0.100} + diff --git a/.ci/test-matrix.yml b/.ci/test-matrix.yml new file mode 100755 index 0000000..570b68b --- /dev/null +++ b/.ci/test-matrix.yml @@ -0,0 +1,11 @@ +--- + +ELASTICSEARCH_VERSION: + - 8.0.0-SNAPSHOT + +TEST_SUITE: + - oss + - xpack + + +exclude: ~ From 031e914dbe663d0b2a03fab43b064222d6e786c2 Mon Sep 17 00:00:00 2001 From: Winterflower Date: Thu, 21 Nov 2019 17:12:45 +0100 Subject: [PATCH 02/17] Renaming configuration items to be eland specific --- .ci/jobs/defaults.yml | 4 ++-- ...ts-flight-recorder+master.yml => elastic+eland+master.yml} | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) rename .ci/jobs/{elastic+clients-flight-recorder+master.yml => elastic+eland+master.yml} (76%) diff --git a/.ci/jobs/defaults.yml b/.ci/jobs/defaults.yml index 707244a..afd3af8 100755 --- a/.ci/jobs/defaults.yml +++ b/.ci/jobs/defaults.yml @@ -20,7 +20,7 @@ <commitId>, etc.) properties: - github: - url: https://github.com/elastic/clients-flight-recorder/ + url: https://github.com/elastic/eland - inject: properties-content: HOME=$JENKINS_HOME concurrent: true @@ -32,7 +32,7 @@ reference-repo: /var/lib/jenkins/.git-references/clients-flight-recorder.git branches: - ${branch_specifier} - url: git@github.com:elastic/clients-flight-recorder.git + url: git@github.com:elastic/eland.git basedir: '' wipe-workspace: 'True' triggers: diff --git a/.ci/jobs/elastic+clients-flight-recorder+master.yml b/.ci/jobs/elastic+eland+master.yml similarity index 76% rename from .ci/jobs/elastic+clients-flight-recorder+master.yml rename to .ci/jobs/elastic+eland+master.yml index 53900c9..f849fba 100755 --- a/.ci/jobs/elastic+clients-flight-recorder+master.yml +++ b/.ci/jobs/elastic+eland+master.yml @@ -1,7 +1,7 @@ --- - job: - name: elastic+clients-flight-recorder+master - display-name: 'elastic / clients-flight-recorder # master' + name: elastic+eland+master + display-name: 'elastic / eland # master' description: Flight recorder captures normalized request responses to be notified of state changes junit_results: "*-junit.xml" parameters: From c4d269ccdb4fc0432f729ef1bd1f6d145bbdb2ce Mon Sep 17 00:00:00 2001 From: Winterflower Date: Fri, 22 Nov 2019 14:42:58 +0100 Subject: [PATCH 03/17] Customising CI setup for eland --- .ci/jobs/defaults.yml | 2 +- .ci/jobs/elastic+eland+master.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/jobs/defaults.yml b/.ci/jobs/defaults.yml index afd3af8..2344b56 100755 --- a/.ci/jobs/defaults.yml +++ b/.ci/jobs/defaults.yml @@ -29,7 +29,7 @@ - git: name: origin credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba - reference-repo: /var/lib/jenkins/.git-references/clients-flight-recorder.git + reference-repo: /var/lib/jenkins/.git-references/eland.git branches: - ${branch_specifier} url: git@github.com:elastic/eland.git diff --git a/.ci/jobs/elastic+eland+master.yml b/.ci/jobs/elastic+eland+master.yml index f849fba..3d28635 100755 --- a/.ci/jobs/elastic+eland+master.yml +++ b/.ci/jobs/elastic+eland+master.yml @@ -2,7 +2,7 @@ - job: name: elastic+eland+master display-name: 'elastic / eland # master' - description: Flight recorder captures normalized request responses to be notified of state changes + description: Eland is a data science client with a Pandas-like interface junit_results: "*-junit.xml" parameters: - string: From ad118fcbad8824f9de3e89f97047784416554401 Mon Sep 17 00:00:00 2001 From: Winterflower Date: Mon, 25 Nov 2019 18:14:20 +0100 Subject: [PATCH 04/17] Adding 7.5 snapshot in test matrix --- .ci/test-matrix.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.ci/test-matrix.yml b/.ci/test-matrix.yml index 570b68b..e1e63cf 100755 --- a/.ci/test-matrix.yml +++ b/.ci/test-matrix.yml @@ -2,6 +2,7 @@ ELASTICSEARCH_VERSION: - 8.0.0-SNAPSHOT + - 7.5-SNAPSHOT TEST_SUITE: - oss From d0aa84a8940e0d42eaa57de63346b57e45e6af2a Mon Sep 17 00:00:00 2001 From: Winterflower Date: Mon, 25 Nov 2019 19:15:37 +0100 Subject: [PATCH 05/17] First attempt at suitable Dockerfile and config for building Docker image for eland --- .ci/Dockerfile | 6 ++++++ .ci/run-tests | 4 ++++ .dockerignore | 3 +++ 3 files changed, 13 insertions(+) create mode 100644 .ci/Dockerfile create mode 100644 .dockerignore diff --git a/.ci/Dockerfile b/.ci/Dockerfile new file mode 100644 index 0000000..91c76c6 --- /dev/null +++ b/.ci/Dockerfile @@ -0,0 +1,6 @@ +ARG PYTHON_VERSION=3 +FROM python:${PYTHON_VERSION} + +WORKDIR /code/eland +COPY . . +RUN pip install requirements-dev.txt \ No newline at end of file diff --git a/.ci/run-tests b/.ci/run-tests index 48a7d09..d4ee6ac 100755 --- a/.ci/run-tests +++ b/.ci/run-tests @@ -53,3 +53,7 @@ echo -e "\033[1m>>>>> YOUR STEPS HERE >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m" ## Use ${elasticsearch_url} to talk to elasticsearch ## declare your matrix variables here as well e.g DOTNET_VERSION=${DOTNET_VERSION-3.0.100} + +echo -e "\033[1m>>>>> Build [elastic/eland container] >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m" + +docker build --file .ci/DockerFile --tag elastic/eland . \ No newline at end of file diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..eb6d255 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,3 @@ +docs/* +example/* +.git From 12236078a8467e7cd04ae824ed3534da7b4aecbd Mon Sep 17 00:00:00 2001 From: Winterflower Date: Mon, 25 Nov 2019 19:50:19 +0100 Subject: [PATCH 06/17] Adding -r flag to pip install command and fixing name of Dockerfile in run-tests --- .ci/Dockerfile | 2 +- .ci/run-tests | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/Dockerfile b/.ci/Dockerfile index 91c76c6..127e433 100644 --- a/.ci/Dockerfile +++ b/.ci/Dockerfile @@ -3,4 +3,4 @@ FROM python:${PYTHON_VERSION} WORKDIR /code/eland COPY . . -RUN pip install requirements-dev.txt \ No newline at end of file +RUN pip install -r requirements-dev.txt \ No newline at end of file diff --git a/.ci/run-tests b/.ci/run-tests index d4ee6ac..3ac2f69 100755 --- a/.ci/run-tests +++ b/.ci/run-tests @@ -56,4 +56,4 @@ echo -e "\033[1m>>>>> YOUR STEPS HERE >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m" echo -e "\033[1m>>>>> Build [elastic/eland container] >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m" -docker build --file .ci/DockerFile --tag elastic/eland . \ No newline at end of file +docker build --file .ci/Dockerfile --tag elastic/eland . \ No newline at end of file From c5b479b4f918cff11b01f02d3b4c81082bf72e6c Mon Sep 17 00:00:00 2001 From: Winterflower Date: Mon, 25 Nov 2019 20:41:58 +0100 Subject: [PATCH 07/17] Adding line to read ES hostname from docker env var --- .ci/run-tests | 14 +++++++++++++- eland/tests/__init__.py | 2 +- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/.ci/run-tests b/.ci/run-tests index 3ac2f69..6834a45 100755 --- a/.ci/run-tests +++ b/.ci/run-tests @@ -56,4 +56,16 @@ echo -e "\033[1m>>>>> YOUR STEPS HERE >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m" echo -e "\033[1m>>>>> Build [elastic/eland container] >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m" -docker build --file .ci/Dockerfile --tag elastic/eland . \ No newline at end of file +docker build --file .ci/Dockerfile --tag elastic/eland . + +echo -e "\033[1m>>>>> Run [elastic/eland container] >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m" + +ELASTICSEARCH_HOST=${elasticsearch_url} + +docker run \ + --network=elasticsearch \ + --env "ELASTICSEARCH_HOST" \ + --name eland-test-runner \ + --rm \ + elastic/eland \ + ./run_build.sh \ No newline at end of file diff --git a/eland/tests/__init__.py b/eland/tests/__init__.py index f5dfb16..3ba2af5 100644 --- a/eland/tests/__init__.py +++ b/eland/tests/__init__.py @@ -5,7 +5,7 @@ import pandas as pd ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) # Define test files and indices -ELASTICSEARCH_HOST = 'localhost' # TODO externalise this +ELASTICSEARCH_HOST = os.environ.get('ELASTICSEARCH_HOST') or 'localhost' # TODO externalise this FLIGHTS_INDEX_NAME = 'flights' FLIGHTS_MAPPING = {"mappings": { From 1c9b8f1298e83c8d38942b54de28730f0ae85757 Mon Sep 17 00:00:00 2001 From: Winterflower Date: Mon, 25 Nov 2019 21:07:37 +0100 Subject: [PATCH 08/17] Adding run_build script --- run_build.sh | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 run_build.sh diff --git a/run_build.sh b/run_build.sh new file mode 100644 index 0000000..c967422 --- /dev/null +++ b/run_build.sh @@ -0,0 +1,4 @@ +# Run the Eland build + +python -m eland.tests.setup_tests +pytest \ No newline at end of file From 80aec42d10f86d3f96b2c21d54364acc8f8bf361 Mon Sep 17 00:00:00 2001 From: Winterflower Date: Tue, 26 Nov 2019 13:56:41 +0100 Subject: [PATCH 09/17] Add shebang header to run_tests --- run_build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run_build.sh b/run_build.sh index c967422..c63a89b 100644 --- a/run_build.sh +++ b/run_build.sh @@ -1,4 +1,4 @@ -# Run the Eland build +#!/usr/bin/env bash python -m eland.tests.setup_tests pytest \ No newline at end of file From cc8a27398a30c9aabe96205d0070e5a0747a2558 Mon Sep 17 00:00:00 2001 From: Winterflower Date: Tue, 26 Nov 2019 13:57:32 +0100 Subject: [PATCH 10/17] Adding newline to end of file --- .ci/Dockerfile | 2 +- .ci/run-tests | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.ci/Dockerfile b/.ci/Dockerfile index 127e433..9620cb7 100644 --- a/.ci/Dockerfile +++ b/.ci/Dockerfile @@ -3,4 +3,4 @@ FROM python:${PYTHON_VERSION} WORKDIR /code/eland COPY . . -RUN pip install -r requirements-dev.txt \ No newline at end of file +RUN pip install -r requirements-dev.txt diff --git a/.ci/run-tests b/.ci/run-tests index 6834a45..eb8c8f4 100755 --- a/.ci/run-tests +++ b/.ci/run-tests @@ -68,4 +68,5 @@ docker run \ --name eland-test-runner \ --rm \ elastic/eland \ - ./run_build.sh \ No newline at end of file + ./run_build.sh + \ No newline at end of file From ba1c9328237bf10b4229c40c37b2a54f700c227c Mon Sep 17 00:00:00 2001 From: Winterflower Date: Tue, 26 Nov 2019 16:28:24 +0100 Subject: [PATCH 11/17] Making run_build script an executable during container build --- .ci/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/.ci/Dockerfile b/.ci/Dockerfile index 9620cb7..d5aca11 100644 --- a/.ci/Dockerfile +++ b/.ci/Dockerfile @@ -3,4 +3,5 @@ FROM python:${PYTHON_VERSION} WORKDIR /code/eland COPY . . +RUN chmod +x ./run_build.sh RUN pip install -r requirements-dev.txt From ce477021c198ef1364b5e2125847a1afa31a930d Mon Sep 17 00:00:00 2001 From: Winterflower Date: Wed, 27 Nov 2019 21:03:03 +0100 Subject: [PATCH 12/17] Strips hostname from extra http if present in string to prevent failures in low level Python socket module --- eland/tests/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eland/tests/__init__.py b/eland/tests/__init__.py index 3ba2af5..631842f 100644 --- a/eland/tests/__init__.py +++ b/eland/tests/__init__.py @@ -5,7 +5,8 @@ import pandas as pd ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) # Define test files and indices -ELASTICSEARCH_HOST = os.environ.get('ELASTICSEARCH_HOST') or 'localhost' # TODO externalise this +ELASTICSEARCH_HOST = os.environ.get('ELASTICSEARCH_HOST') or 'localhost' +ELASTICSEARCH_HOST = ELASTICSEARCH_HOST.lstrip('http://') FLIGHTS_INDEX_NAME = 'flights' FLIGHTS_MAPPING = {"mappings": { From 83d0c3de38cd7615acf0edd82dd0775a4e4525e4 Mon Sep 17 00:00:00 2001 From: Winterflower Date: Wed, 27 Nov 2019 21:03:47 +0100 Subject: [PATCH 13/17] Adds helpful statement to know which ES instance you are connecting to --- eland/tests/setup_tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/eland/tests/setup_tests.py b/eland/tests/setup_tests.py index afeafe2..b507acc 100644 --- a/eland/tests/setup_tests.py +++ b/eland/tests/setup_tests.py @@ -66,6 +66,7 @@ def _setup_test_nested(es): if __name__ == '__main__': # Create connection to Elasticsearch - use defaults + print('Connecting to ES', ELASTICSEARCH_HOST) es = Elasticsearch(ELASTICSEARCH_HOST) _setup_data(es) From 9ab411984a65a9e5e55448868ef666ab34e3ae5d Mon Sep 17 00:00:00 2001 From: Winterflower Date: Wed, 27 Nov 2019 21:05:02 +0100 Subject: [PATCH 14/17] Adds extra step to Dockerfile build to take advantage of layer caching and avoid repulling dependencies in requirements.txt if they don't change --- .ci/Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.ci/Dockerfile b/.ci/Dockerfile index d5aca11..c966d2b 100644 --- a/.ci/Dockerfile +++ b/.ci/Dockerfile @@ -2,6 +2,9 @@ ARG PYTHON_VERSION=3 FROM python:${PYTHON_VERSION} WORKDIR /code/eland +COPY requirements-dev.txt . +RUN pip install -r requirements-dev.txt + COPY . . RUN chmod +x ./run_build.sh -RUN pip install -r requirements-dev.txt + From 78b4a2291b9ac9a68d6dba883816d8087b47ecaf Mon Sep 17 00:00:00 2001 From: Winterflower Date: Wed, 27 Nov 2019 21:29:26 +0100 Subject: [PATCH 15/17] Drops default Python version of Dockerfile to 3.7, because Pandas 0.25.1 is not compatible with Python 3.8 and Pandas 0.25.2 is breaking some eland tests --- .ci/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/Dockerfile b/.ci/Dockerfile index c966d2b..0fc268a 100644 --- a/.ci/Dockerfile +++ b/.ci/Dockerfile @@ -1,4 +1,4 @@ -ARG PYTHON_VERSION=3 +ARG PYTHON_VERSION=3.7 FROM python:${PYTHON_VERSION} WORKDIR /code/eland From 8a9e83d0e55d4c06986d93682deda6cdf6b0517a Mon Sep 17 00:00:00 2001 From: Winterflower Date: Thu, 28 Nov 2019 13:50:05 +0100 Subject: [PATCH 16/17] Removes line that sets run_build.sh as executable and adds Python versions to test_matrix.yml --- .ci/Dockerfile | 2 +- .ci/test-matrix.yml | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.ci/Dockerfile b/.ci/Dockerfile index 0fc268a..e397271 100644 --- a/.ci/Dockerfile +++ b/.ci/Dockerfile @@ -6,5 +6,5 @@ COPY requirements-dev.txt . RUN pip install -r requirements-dev.txt COPY . . -RUN chmod +x ./run_build.sh + diff --git a/.ci/test-matrix.yml b/.ci/test-matrix.yml index e1e63cf..c1917cd 100755 --- a/.ci/test-matrix.yml +++ b/.ci/test-matrix.yml @@ -8,5 +8,10 @@ TEST_SUITE: - oss - xpack +PYTHON_VERSION: + - 3.7 + - 3.6 + - 3.5.3 + exclude: ~ From beaf71920192f366bc0b67a875f1f29cfb91cfa9 Mon Sep 17 00:00:00 2001 From: Winterflower Date: Fri, 29 Nov 2019 08:58:31 +0100 Subject: [PATCH 17/17] Making run_build.sh executable --- run_build.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 run_build.sh diff --git a/run_build.sh b/run_build.sh old mode 100644 new mode 100755