diff --git a/.circleci/config.yml b/.circleci/config.yml index 9d497c0..ed2bf8a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,11 +1,35 @@ version: 2 jobs: bats-unit-test: - machine: true + docker: + # This image is built from test/docker/Test.dockerfile + - image: hashicorpdev/vault-helm-test:0.1.0 steps: - checkout - - run: make test-image - - run: make test-unit + - run: bats ./test/unit -t + acceptance: + docker: + # This image is build from test/docker/Test.dockerfile + - image: hashicorpdev/vault-helm-test:0.1.0 + + steps: + - checkout + - run: + name: terraform init & apply + command: | + echo -e "${GOOGLE_APP_CREDS}" | base64 -d > vault-helm-test.json + export GOOGLE_CREDENTIALS=vault-helm-test.json + make provision-cluster + - run: + name: Run acceptance tests + command: bats ./test/acceptance -t + + - run: + name: terraform destroy + command: | + export GOOGLE_CREDENTIALS=vault-helm-test.json + make destroy-cluster + when: always update-helm-charts-index: docker: - image: circleci/golang:latest @@ -24,6 +48,12 @@ workflows: build_and_test: jobs: - bats-unit-test + - acceptance: + requires: + - bats-unit-test + filters: + branches: + only: master update-helm-charts-index: jobs: - update-helm-charts-index: diff --git a/Makefile b/Makefile index 4698fb9..8c9bf7f 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,8 @@ TEST_IMAGE?=vault-helm-test +GOOGLE_CREDENTIALS?=vault-helm-test.json +CLOUDSDK_CORE_PROJECT?=vault-helm-dev-246514 +# set to run a single test - e.g acceptance/server-ha-enterprise-dr.bats +ACCEPTANCE_TESTS?=acceptance test-image: @docker build --rm -t '$(TEST_IMAGE)' -f $(CURDIR)/test/docker/Test.dockerfile $(CURDIR) @@ -6,12 +10,56 @@ test-image: test-unit: @docker run -it -v ${PWD}:/helm-test vault-helm-test bats /helm-test/test/unit -test-acceptance: - @docker run -it -v ${PWD}:/helm-test vault-helm-test bats /helm-test/test/acceptance - test-bats: test-unit test-acceptance test: test-image test-bats +# run acceptance tests on GKE +# set google project/credential vars above +test-acceptance: + @docker run -it -v ${PWD}:/helm-test \ + -e GOOGLE_CREDENTIALS=${GOOGLE_CREDENTIALS} \ + -e CLOUDSDK_CORE_PROJECT=${CLOUDSDK_CORE_PROJECT} \ + -e KUBECONFIG=/helm-test/.kube/config \ + -w /helm-test \ + $(TEST_IMAGE) \ + make acceptance + +# destroy GKE cluster using terraform +test-destroy: + @docker run -it -v ${PWD}:/helm-test \ + -e GOOGLE_CREDENTIALS=${GOOGLE_CREDENTIALS} \ + -e CLOUDSDK_CORE_PROJECT=${CLOUDSDK_CORE_PROJECT} \ + -w /helm-test \ + $(TEST_IMAGE) \ + make destroy-cluster + +# provision GKE cluster using terraform +test-provision: + @docker run -it -v ${PWD}:/helm-test \ + -e GOOGLE_CREDENTIALS=${GOOGLE_CREDENTIALS} \ + -e CLOUDSDK_CORE_PROJECT=${CLOUDSDK_CORE_PROJECT} \ + -e KUBECONFIG=/helm-test/.kube/config \ + -w /helm-test \ + $(TEST_IMAGE) \ + make provision-cluster + +# this target is for running the acceptance tests +# it is run in the docker container above when the test-acceptance target is invoked +acceptance: + gcloud auth activate-service-account --key-file=${GOOGLE_CREDENTIALS} + bats test/${ACCEPTANCE_TESTS} + +# this target is for provisioning the GKE cluster +# it is run in the docker container above when the test-provision target is invoked +provision-cluster: + gcloud auth activate-service-account --key-file=${GOOGLE_CREDENTIALS} + terraform init test/terraform + terraform apply -var project=${CLOUDSDK_CORE_PROJECT} -var init_cli=true -auto-approve test/terraform + +# this target is for removing the GKE cluster +# it is run in the docker container above when the test-destroy target is invoked +destroy-cluster: + terraform destroy -auto-approve .PHONY: test-docker diff --git a/test/README.md b/test/README.md new file mode 100644 index 0000000..e4ce891 --- /dev/null +++ b/test/README.md @@ -0,0 +1,10 @@ +# Running Vault Helm Acceptance tests + +The Makefile at the top level of this repo contains a few target that should help with running acceptance tests in your own GKE instance. + +* Set the GOOGLE_CREDENTIALS and CLOUDSDK_CORE_PROJECT variables at the top of the file. GOOGLE_CREDENTIALS should contain the local path to your Google Cloud Platform account credentials in JSON format. CLOUDSDK_CORE_PROJECT should be set to the ID of your GCP project. +* Run `make test-image` to create the docker image (with dependencies installed) that will be re-used in the below steps. +* Run `make test-provision` to provision the GKE cluster using terraform. +* Run `make test-acceptance` to run the acceptance tests in this already provisioned cluster. +* You can choose to only run certain tests by setting the ACCEPTANCE_TESTS variable and re-running the above target. +* Run `make test-destroy` when you have finished testing and want to tear-down and remove the cluster. \ No newline at end of file diff --git a/test/acceptance/_helpers.bash b/test/acceptance/_helpers.bash index 031daf5..466a517 100644 --- a/test/acceptance/_helpers.bash +++ b/test/acceptance/_helpers.bash @@ -65,7 +65,7 @@ wait_for_running_consul() { done echo "consul clients never became ready." - exit 1 + return 1 } # wait for a pod to be ready @@ -96,7 +96,7 @@ wait_for_running() { done echo "${POD_NAME} never became ready." - exit 1 + return 1 } wait_for_ready() { @@ -126,7 +126,7 @@ wait_for_ready() { done echo "${POD_NAME} never became ready." - exit 1 + return 1 } wait_for_complete_job() { @@ -155,5 +155,5 @@ wait_for_complete_job() { done echo "${POD_NAME} never completed." - exit 1 + return 1 } diff --git a/test/acceptance/server-ha-enterprise-perf.bats b/test/acceptance/server-ha-enterprise-perf.bats index 6543663..48f9887 100644 --- a/test/acceptance/server-ha-enterprise-perf.bats +++ b/test/acceptance/server-ha-enterprise-perf.bats @@ -35,7 +35,7 @@ load _helpers kubectl exec -ti "$(name_prefix)-east-0" -- vault operator unseal ${primary_token} wait_for_ready "$(name_prefix)-east-0" - sleep 10 + sleep 30 # Vault Unseal local pods=($(kubectl get pods --selector='app.kubernetes.io/name=vault' -o json | jq -r '.items[].metadata.name')) @@ -103,7 +103,7 @@ load _helpers kubectl exec -ti "$(name_prefix)-west-0" -- vault operator unseal ${secondary_token} wait_for_ready "$(name_prefix)-west-0" - sleep 10 + sleep 30 # Vault Unseal local pods=($(kubectl get pods --selector='app.kubernetes.io/instance=vault-west' -o json | jq -r '.items[].metadata.name')) @@ -134,7 +134,7 @@ load _helpers kubectl exec -ti "$(name_prefix)-west-0" -- vault write sys/replication/performance/secondary/enable token=${secondary_replica_token} - sleep 10 + sleep 30 local pods=($(kubectl get pods --selector='app.kubernetes.io/instance=vault-west' -o json | jq -r '.items[].metadata.name')) for pod in "${pods[@]}" diff --git a/test/docker/Test.dockerfile b/test/docker/Test.dockerfile index 003a06f..9bbe478 100644 --- a/test/docker/Test.dockerfile +++ b/test/docker/Test.dockerfile @@ -10,6 +10,7 @@ FROM alpine:latest WORKDIR /root ENV BATS_VERSION "1.1.0" +ENV TERRAFORM_VERSION "0.12.10" # base packages RUN apk update && apk add --no-cache --virtual .build-deps \ @@ -21,6 +22,7 @@ RUN apk update && apk add --no-cache --virtual .build-deps \ python \ py-pip \ git \ + make \ jq # yq @@ -31,6 +33,11 @@ RUN curl -OL https://dl.google.com/dl/cloudsdk/channels/rapid/install_google_clo bash install_google_cloud_sdk.bash --disable-prompts --install-dir='/root/' && \ ln -s /root/google-cloud-sdk/bin/gcloud /usr/local/bin/gcloud +# terraform +RUN curl -sSL https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip -o /tmp/tf.zip \ + && unzip /tmp/tf.zip \ + && ln -s /root/terraform /usr/local/bin/terraform + # kubectl RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl && \ chmod +x ./kubectl && \ diff --git a/test/terraform/main.tf b/test/terraform/main.tf index e3fc2ef..1c3f035 100644 --- a/test/terraform/main.tf +++ b/test/terraform/main.tf @@ -1,8 +1,5 @@ provider "google" { project = "${var.project}" - region = "us-central1" - - credentials = "${file("vault-helm-dev-creds.json")}" } resource "random_id" "suffix" { @@ -18,20 +15,6 @@ data "google_service_account" "gcpapi" { account_id = "${var.gcp_service_account}" } -resource "google_kms_key_ring" "keyring" { - name = "vault-helm-unseal-kr" - location = "global" -} - -resource "google_kms_crypto_key" "vault-helm-unseal-key" { - name = "vault-helm-unseal-key" - key_ring = "${google_kms_key_ring.keyring.self_link}" - - lifecycle { - prevent_destroy = true - } -} - resource "google_container_cluster" "cluster" { name = "vault-helm-dev-${random_id.suffix.dec}" project = "${var.project}"