Get acceptance tests running against GKE in CI - merges to master only. (#291)

* Get acceptance tests running against GKE in CI - merges to master only.

* Adding README.md

.circleci/config.yml

@@ -1,11 +1,35 @@
 version: 2
 jobs:
   bats-unit-test:
-    machine: true
+    docker:
+        # This image is built from test/docker/Test.dockerfile
+        - image: hashicorpdev/vault-helm-test:0.1.0
     steps:
       - checkout
-      - run: make test-image
+      - run: bats ./test/unit -t
-      - run: make test-unit
+  acceptance:
+    docker:
+        # This image is build from test/docker/Test.dockerfile
+        - image: hashicorpdev/vault-helm-test:0.1.0
+
+    steps:
+        - checkout
+        - run:
+            name: terraform init & apply
+            command: |
+                echo -e "${GOOGLE_APP_CREDS}" | base64 -d > vault-helm-test.json
+                export GOOGLE_CREDENTIALS=vault-helm-test.json
+                make provision-cluster
+        - run:
+            name: Run acceptance tests
+            command: bats ./test/acceptance -t
+
+        - run:
+            name: terraform destroy
+            command: |
+                export GOOGLE_CREDENTIALS=vault-helm-test.json
+                make destroy-cluster
+            when: always
   update-helm-charts-index:
     docker:
       - image: circleci/golang:latest
@@ -24,6 +48,12 @@ workflows:
   build_and_test:
     jobs:
       - bats-unit-test
+      - acceptance:
+            requires:
+                - bats-unit-test
+            filters:
+                branches:
+                    only: master
   update-helm-charts-index:
     jobs:
       - update-helm-charts-index:

Makefile

@@ -1,4 +1,8 @@
 TEST_IMAGE?=vault-helm-test
+GOOGLE_CREDENTIALS?=vault-helm-test.json
+CLOUDSDK_CORE_PROJECT?=vault-helm-dev-246514
+# set to run a single test - e.g acceptance/server-ha-enterprise-dr.bats
+ACCEPTANCE_TESTS?=acceptance
 
 test-image:
 	@docker build --rm -t '$(TEST_IMAGE)' -f $(CURDIR)/test/docker/Test.dockerfile $(CURDIR)
@@ -6,12 +10,56 @@ test-image:
 test-unit:
 	@docker run -it -v ${PWD}:/helm-test vault-helm-test bats /helm-test/test/unit
 
-test-acceptance:
-	@docker run -it -v ${PWD}:/helm-test vault-helm-test bats /helm-test/test/acceptance
-
 test-bats: test-unit test-acceptance
 
 test: test-image test-bats
 
+# run acceptance tests on GKE
+# set google project/credential vars above
+test-acceptance:
+	@docker run -it -v ${PWD}:/helm-test \
+	-e GOOGLE_CREDENTIALS=${GOOGLE_CREDENTIALS} \
+	-e CLOUDSDK_CORE_PROJECT=${CLOUDSDK_CORE_PROJECT} \
+	-e KUBECONFIG=/helm-test/.kube/config \
+	-w /helm-test \
+	$(TEST_IMAGE) \
+	make acceptance
+	
+# destroy GKE cluster using terraform
+test-destroy:
+	@docker run -it -v ${PWD}:/helm-test \
+	-e GOOGLE_CREDENTIALS=${GOOGLE_CREDENTIALS} \
+	-e CLOUDSDK_CORE_PROJECT=${CLOUDSDK_CORE_PROJECT} \
+	-w /helm-test \
+	$(TEST_IMAGE) \
+	make destroy-cluster
+
+# provision GKE cluster using terraform
+test-provision:
+	@docker run -it -v ${PWD}:/helm-test \
+	-e GOOGLE_CREDENTIALS=${GOOGLE_CREDENTIALS} \
+	-e CLOUDSDK_CORE_PROJECT=${CLOUDSDK_CORE_PROJECT} \
+	-e KUBECONFIG=/helm-test/.kube/config \
+	-w /helm-test \
+	$(TEST_IMAGE) \
+	make provision-cluster
+
+# this target is for running the acceptance tests
+# it is run in the docker container above when the test-acceptance target is invoked
+acceptance:
+	gcloud auth activate-service-account --key-file=${GOOGLE_CREDENTIALS}
+	bats test/${ACCEPTANCE_TESTS}
+
+# this target is for provisioning the GKE cluster
+# it is run in the docker container above when the test-provision target is invoked
+provision-cluster:
+	gcloud auth activate-service-account --key-file=${GOOGLE_CREDENTIALS}
+	terraform init test/terraform
+	terraform apply -var project=${CLOUDSDK_CORE_PROJECT} -var init_cli=true -auto-approve test/terraform
+
+# this target is for removing the GKE cluster
+# it is run in the docker container above when the test-destroy target is invoked
+destroy-cluster:
+	terraform destroy -auto-approve
 
 .PHONY: test-docker

test/README.md

@@ -0,0 +1,10 @@
+# Running Vault Helm Acceptance tests
+
+The Makefile at the top level of this repo contains a few target that should help with running acceptance tests in your own GKE instance.
+
+* Set the GOOGLE_CREDENTIALS and CLOUDSDK_CORE_PROJECT variables at the top of the file. GOOGLE_CREDENTIALS should contain the local path to your Google Cloud Platform account credentials in JSON format. CLOUDSDK_CORE_PROJECT should be set to the ID of your GCP project.
+* Run `make test-image` to create the docker image (with dependencies installed) that will be re-used in the below steps.
+* Run `make test-provision` to provision the GKE cluster using terraform.
+* Run `make test-acceptance` to run the acceptance tests in this already provisioned cluster.
+* You can choose to only run certain tests by setting the ACCEPTANCE_TESTS variable and re-running the above target.
+* Run `make test-destroy` when you have finished testing and want to tear-down and remove the cluster.

test/acceptance/_helpers.bash

@@ -65,7 +65,7 @@ wait_for_running_consul() {
     done
 
     echo "consul clients never became ready."
-    exit 1
+    return 1
 }
 
 # wait for a pod to be ready
@@ -96,7 +96,7 @@ wait_for_running() {
     done
 
     echo "${POD_NAME} never became ready."
-    exit 1
+    return 1
 }
 
 wait_for_ready() {
@@ -126,7 +126,7 @@ wait_for_ready() {
     done
 
     echo "${POD_NAME} never became ready."
-    exit 1
+    return 1
 }
 
 wait_for_complete_job() {
@@ -155,5 +155,5 @@ wait_for_complete_job() {
     done
 
     echo "${POD_NAME} never completed."
-    exit 1
+    return 1
 }

test/acceptance/server-ha-enterprise-perf.bats

@@ -35,7 +35,7 @@ load _helpers
   kubectl exec -ti "$(name_prefix)-east-0" -- vault operator unseal ${primary_token}
   wait_for_ready "$(name_prefix)-east-0"
 
-  sleep 10
+  sleep 30
 
   # Vault Unseal
   local pods=($(kubectl get pods --selector='app.kubernetes.io/name=vault' -o json | jq -r '.items[].metadata.name'))
@@ -103,7 +103,7 @@ load _helpers
   kubectl exec -ti "$(name_prefix)-west-0" -- vault operator unseal ${secondary_token}
   wait_for_ready "$(name_prefix)-west-0"
 
-  sleep 10
+  sleep 30
 
   # Vault Unseal
   local pods=($(kubectl get pods --selector='app.kubernetes.io/instance=vault-west' -o json | jq -r '.items[].metadata.name'))
@@ -134,7 +134,7 @@ load _helpers
 
   kubectl exec -ti "$(name_prefix)-west-0" -- vault write sys/replication/performance/secondary/enable token=${secondary_replica_token}
 
-  sleep 10
+  sleep 30
 
   local pods=($(kubectl get pods --selector='app.kubernetes.io/instance=vault-west' -o json | jq -r '.items[].metadata.name'))
   for pod in "${pods[@]}"

test/docker/Test.dockerfile

@@ -10,6 +10,7 @@ FROM alpine:latest
 WORKDIR /root
 
 ENV BATS_VERSION "1.1.0"
+ENV TERRAFORM_VERSION "0.12.10"
 
 # base packages
 RUN apk update && apk add --no-cache --virtual .build-deps \
@@ -21,6 +22,7 @@ RUN apk update && apk add --no-cache --virtual .build-deps \
     python \
     py-pip \
     git \
+    make \
     jq
 
 # yq
@@ -31,6 +33,11 @@ RUN curl -OL https://dl.google.com/dl/cloudsdk/channels/rapid/install_google_clo
     bash install_google_cloud_sdk.bash --disable-prompts --install-dir='/root/' && \
     ln -s /root/google-cloud-sdk/bin/gcloud /usr/local/bin/gcloud
 
+# terraform
+RUN curl -sSL https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip -o /tmp/tf.zip \
+    && unzip /tmp/tf.zip  \
+    && ln -s /root/terraform /usr/local/bin/terraform
+
 # kubectl
 RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl && \
     chmod +x ./kubectl && \

test/terraform/main.tf

@@ -1,8 +1,5 @@
 provider "google" {
   project = "${var.project}"
-  region  = "us-central1"
-
-  credentials = "${file("vault-helm-dev-creds.json")}"
 }
 
 resource "random_id" "suffix" {
@@ -18,20 +15,6 @@ data "google_service_account" "gcpapi" {
   account_id = "${var.gcp_service_account}"
 }
 
-resource "google_kms_key_ring" "keyring" {
-  name     = "vault-helm-unseal-kr"
-  location = "global"
-}
-
-resource "google_kms_crypto_key" "vault-helm-unseal-key" {
-  name            = "vault-helm-unseal-key"
-  key_ring        = "${google_kms_key_ring.keyring.self_link}"
-
-  lifecycle {
-    prevent_destroy = true
-  }
-}
-
 resource "google_container_cluster" "cluster" {
   name               = "vault-helm-dev-${random_id.suffix.dec}"
   project            = "${var.project}"