From 48eebb0b0921a0c274fad532c97288dac2b6398e Mon Sep 17 00:00:00 2001 From: Theron Voran Date: Fri, 10 Dec 2021 17:11:35 -0800 Subject: [PATCH] Run CI tests in github workflows (#657) Ports the bats unit, chart-verifier, and bats acceptance tests to use github workflows and actions. The acceptance tests run using kind, and run for multiple k8s versions, on pushes to the main branch. Adds a SKIP_CSI env check in the CSI acceptance test, set in the workflow if K8s version is less than 1.16. Adds kubeAdmConfigPatches to the kind config to allow testing the CSI provider on K8s versions prior to 1.21. Updates the Secrets Store CSI driver to 1.0.0 in tests. Makes the HA Vault tests more robust by waiting for all consul client pods to be Ready, and waits with a timeout for Vault to start responding as sealed (since the tests on GitHub runners were often failing at that point). Co-authored-by: Tom Proctor --- .github/workflows/acceptance.yaml | 34 +++++++++++++++++++ .../workflows/setup-test-tools/action.yaml | 18 ++++++++++ .github/workflows/tests.yaml | 25 ++++++++++++++ test/acceptance/_helpers.bash | 32 +++++++++-------- test/acceptance/csi.bats | 17 ++++++++-- test/acceptance/server-ha-enterprise-dr.bats | 12 +++---- .../acceptance/server-ha-enterprise-perf.bats | 12 +++---- test/acceptance/server-ha-raft.bats | 8 +++-- test/acceptance/server-ha.bats | 11 +++--- test/acceptance/server.bats | 4 +-- test/kind/config.yaml | 13 +++++++ 11 files changed, 142 insertions(+), 44 deletions(-) create mode 100644 .github/workflows/acceptance.yaml create mode 100644 .github/workflows/setup-test-tools/action.yaml create mode 100644 .github/workflows/tests.yaml diff --git a/.github/workflows/acceptance.yaml b/.github/workflows/acceptance.yaml new file mode 100644 index 0000000..644875e --- /dev/null +++ b/.github/workflows/acceptance.yaml @@ -0,0 +1,34 @@ +name: Acceptance Tests + +on: + push: + branches: + - main + workflow_dispatch: {} + +jobs: + kind: + strategy: + fail-fast: false + matrix: + kind-k8s-version: [1.14.10, 1.19.11, 1.20.7, 1.21.2, 1.22.4] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Setup test tools + uses: ./.github/workflows/setup-test-tools + + - name: Create K8s Kind Cluster + uses: helm/kind-action@v1.2.0 + with: + config: test/kind/config.yaml + node_image: kindest/node:v${{ matrix.kind-k8s-version }} + + # Skip CSI tests if K8s version < 1.16.x + - run: echo K8S_MINOR=$(kubectl version -o json | jq -r .serverVersion.minor) >> $GITHUB_ENV + - if: ${{ env.K8S_MINOR < 16 }} + run: echo "SKIP_CSI=true" >> $GITHUB_ENV + + - run: bats ./test/acceptance -t + env: + VAULT_LICENSE_CI: ${{ secrets.VAULT_LICENSE_CI }} diff --git a/.github/workflows/setup-test-tools/action.yaml b/.github/workflows/setup-test-tools/action.yaml new file mode 100644 index 0000000..3fa2854 --- /dev/null +++ b/.github/workflows/setup-test-tools/action.yaml @@ -0,0 +1,18 @@ +name: Setup common testing tools +description: Install bats and python-yq + +runs: + using: "composite" + steps: + - uses: actions/setup-node@v2 + with: + node-version: '14' + - run: npm install -g bats@${BATS_VERSION} + shell: bash + env: + BATS_VERSION: '1.5.0' + - run: bats -v + shell: bash + - uses: actions/setup-python@v2 + - run: pip install yq + shell: bash diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml new file mode 100644 index 0000000..0aba6ee --- /dev/null +++ b/.github/workflows/tests.yaml @@ -0,0 +1,25 @@ +name: Tests + +on: [push, workflow_dispatch] + +jobs: + bats-unit-tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: ./.github/workflows/setup-test-tools + - run: bats ./test/unit -t + + chart-verifier: + runs-on: ubuntu-latest + env: + CHART_VERIFIER_VERSION: '1.2.1' + steps: + - uses: actions/checkout@v2 + - name: Setup test tools + uses: ./.github/workflows/setup-test-tools + - uses: actions/setup-go@v2 + with: + go-version: '1.17.4' + - run: go install github.com/redhat-certification/chart-verifier@${CHART_VERIFIER_VERSION} + - run: bats ./test/chart -t diff --git a/test/acceptance/_helpers.bash b/test/acceptance/_helpers.bash index 466a517..db8b051 100644 --- a/test/acceptance/_helpers.bash +++ b/test/acceptance/_helpers.bash @@ -40,35 +40,37 @@ helm_install_ha() { ${BATS_TEST_DIRNAME}/../.. } -# wait for consul to be running +# wait for consul to be ready wait_for_running_consul() { + kubectl wait --for=condition=Ready --timeout=5m pod -l app=consul,component=client +} + +wait_for_sealed_vault() { + POD_NAME=$1 + check() { - # This requests the pod and checks whether the status is running - # and the ready state is true. If so, it outputs the name. Otherwise - # it outputs empty. Therefore, to check for success, check for nonzero - # string length. - kubectl get pods -l component=client -o json | \ - jq -r '.items[0] | select( - .status.phase == "Running" and - ([ .status.conditions[] | select(.type == "Ready" and .status == "True") ] | length) == 1 - ) | .metadata.name' + sealed_status=$(kubectl exec $1 -- vault status -format=json | jq -r '.sealed') + if [ "$sealed_status" == "true" ]; then + return 0 + fi + return 1 } for i in $(seq 60); do - if [ -n "$(check ${POD_NAME})" ]; then - echo "consul clients are ready." + if check ${POD_NAME}; then + echo "Vault on ${POD_NAME} is running." return fi - echo "Waiting for ${POD_NAME} to be ready..." + echo "Waiting for Vault on ${POD_NAME} to be running..." sleep 2 done - echo "consul clients never became ready." + echo "Vault on ${POD_NAME} never became running." return 1 } -# wait for a pod to be ready +# wait for a pod to be running wait_for_running() { POD_NAME=$1 diff --git a/test/acceptance/csi.bats b/test/acceptance/csi.bats index d222ca2..c7c6549 100644 --- a/test/acceptance/csi.bats +++ b/test/acceptance/csi.bats @@ -2,18 +2,27 @@ load _helpers +check_skip_csi() { + if [ ! -z ${SKIP_CSI} ]; then + skip "Skipping CSI tests" + fi +} + @test "csi: testing deployment" { + check_skip_csi + cd `chart_dir` kubectl delete namespace acceptance --ignore-not-found=true kubectl create namespace acceptance # Install Secrets Store CSI driver - CSI_DRIVER_VERSION=0.2.0 - helm install secrets-store-csi-driver https://github.com/kubernetes-sigs/secrets-store-csi-driver/blob/v${CSI_DRIVER_VERSION}/charts/secrets-store-csi-driver-${CSI_DRIVER_VERSION}.tgz?raw=true \ + CSI_DRIVER_VERSION=1.0.0 + helm install secrets-store-csi-driver https://kubernetes-sigs.github.io/secrets-store-csi-driver/charts/secrets-store-csi-driver-${CSI_DRIVER_VERSION}.tgz?raw=true \ --wait --timeout=5m \ --namespace=acceptance \ - --set linux.image.pullPolicy="IfNotPresent" + --set linux.image.pullPolicy="IfNotPresent" \ + --set syncSecret.enabled=true # Install Vault and Vault provider helm install vault \ --wait --timeout=5m \ @@ -49,6 +58,8 @@ load _helpers # Clean up teardown() { + check_skip_csi + if [[ ${CLEANUP:-true} == "true" ]] then echo "helm/pvc teardown" diff --git a/test/acceptance/server-ha-enterprise-dr.bats b/test/acceptance/server-ha-enterprise-dr.bats index ee27518..cdcfabe 100644 --- a/test/acceptance/server-ha-enterprise-dr.bats +++ b/test/acceptance/server-ha-enterprise-dr.bats @@ -15,9 +15,7 @@ load _helpers wait_for_running "$(name_prefix)-east-0" # Sealed, not initialized - local sealed_status=$(kubectl exec "$(name_prefix)-east-0" -- vault status -format=json | - jq -r '.sealed' ) - [ "${sealed_status}" == "true" ] + wait_for_sealed_vault $(name_prefix)-east-0 local init_status=$(kubectl exec "$(name_prefix)-east-0" -- vault status -format=json | jq -r '.initialized') @@ -50,7 +48,7 @@ load _helpers fi done - # Sealed, not initialized + # Unsealed, initialized local sealed_status=$(kubectl exec "$(name_prefix)-east-0" -- vault status -format=json | jq -r '.sealed' ) [ "${sealed_status}" == "false" ] @@ -84,9 +82,7 @@ load _helpers wait_for_running "$(name_prefix)-west-0" # Sealed, not initialized - local sealed_status=$(kubectl exec "$(name_prefix)-west-0" -- vault status -format=json | - jq -r '.sealed' ) - [ "${sealed_status}" == "true" ] + wait_for_sealed_vault $(name_prefix)-west-0 local init_status=$(kubectl exec "$(name_prefix)-west-0" -- vault status -format=json | jq -r '.initialized') @@ -119,7 +115,7 @@ load _helpers fi done - # Sealed, not initialized + # Unsealed, initialized local sealed_status=$(kubectl exec "$(name_prefix)-west-0" -- vault status -format=json | jq -r '.sealed' ) [ "${sealed_status}" == "false" ] diff --git a/test/acceptance/server-ha-enterprise-perf.bats b/test/acceptance/server-ha-enterprise-perf.bats index c359c1c..68830ca 100644 --- a/test/acceptance/server-ha-enterprise-perf.bats +++ b/test/acceptance/server-ha-enterprise-perf.bats @@ -15,9 +15,7 @@ load _helpers wait_for_running "$(name_prefix)-east-0" # Sealed, not initialized - local sealed_status=$(kubectl exec "$(name_prefix)-east-0" -- vault status -format=json | - jq -r '.sealed' ) - [ "${sealed_status}" == "true" ] + wait_for_sealed_vault $(name_prefix)-east-0 local init_status=$(kubectl exec "$(name_prefix)-east-0" -- vault status -format=json | jq -r '.initialized') @@ -50,7 +48,7 @@ load _helpers fi done - # Sealed, not initialized + # Unsealed, initialized local sealed_status=$(kubectl exec "$(name_prefix)-east-0" -- vault status -format=json | jq -r '.sealed' ) [ "${sealed_status}" == "false" ] @@ -84,9 +82,7 @@ load _helpers wait_for_running "$(name_prefix)-west-0" # Sealed, not initialized - local sealed_status=$(kubectl exec "$(name_prefix)-west-0" -- vault status -format=json | - jq -r '.sealed' ) - [ "${sealed_status}" == "true" ] + wait_for_sealed_vault $(name_prefix)-west-0 local init_status=$(kubectl exec "$(name_prefix)-west-0" -- vault status -format=json | jq -r '.initialized') @@ -119,7 +115,7 @@ load _helpers fi done - # Sealed, not initialized + # Unsealed, initialized local sealed_status=$(kubectl exec "$(name_prefix)-west-0" -- vault status -format=json | jq -r '.sealed' ) [ "${sealed_status}" == "false" ] diff --git a/test/acceptance/server-ha-raft.bats b/test/acceptance/server-ha-raft.bats index 9f9f3de..f06ca87 100644 --- a/test/acceptance/server-ha-raft.bats +++ b/test/acceptance/server-ha-raft.bats @@ -11,9 +11,7 @@ load _helpers wait_for_running $(name_prefix)-0 # Sealed, not initialized - local sealed_status=$(kubectl exec "$(name_prefix)-0" -- vault status -format=json | - jq -r '.sealed' ) - [ "${sealed_status}" == "true" ] + wait_for_sealed_vault $(name_prefix)-0 local init_status=$(kubectl exec "$(name_prefix)-0" -- vault status -format=json | jq -r '.initialized') @@ -112,6 +110,10 @@ setup() { teardown() { if [[ ${CLEANUP:-true} == "true" ]] then + # If the test failed, print some debug output + if [[ "$BATS_ERROR_STATUS" -ne 0 ]]; then + kubectl logs -l app.kubernetes.io/name=vault + fi helm delete vault kubectl delete --all pvc kubectl delete namespace acceptance --ignore-not-found=true diff --git a/test/acceptance/server-ha.bats b/test/acceptance/server-ha.bats index 3d62959..4180f8c 100644 --- a/test/acceptance/server-ha.bats +++ b/test/acceptance/server-ha.bats @@ -10,9 +10,7 @@ load _helpers wait_for_running $(name_prefix)-0 # Sealed, not initialized - local sealed_status=$(kubectl exec "$(name_prefix)-0" -- vault status -format=json | - jq -r '.sealed' ) - [ "${sealed_status}" == "true" ] + wait_for_sealed_vault $(name_prefix)-0 local init_status=$(kubectl exec "$(name_prefix)-0" -- vault status -format=json | jq -r '.initialized') @@ -91,7 +89,7 @@ setup() { helm install consul \ https://github.com/hashicorp/consul-helm/archive/v0.28.0.tar.gz \ - --set 'ui.enabled=false' \ + --set 'ui.enabled=false' wait_for_running_consul } @@ -100,6 +98,11 @@ setup() { teardown() { if [[ ${CLEANUP:-true} == "true" ]] then + # If the test failed, print some debug output + if [[ "$BATS_ERROR_STATUS" -ne 0 ]]; then + kubectl logs -l app=consul + kubectl logs -l app.kubernetes.io/name=vault + fi helm delete vault helm delete consul kubectl delete --all pvc diff --git a/test/acceptance/server.bats b/test/acceptance/server.bats index 84a4e7d..1e944a0 100644 --- a/test/acceptance/server.bats +++ b/test/acceptance/server.bats @@ -13,9 +13,7 @@ load _helpers wait_for_running $(name_prefix)-0 # Sealed, not initialized - local sealed_status=$(kubectl exec "$(name_prefix)-0" -- vault status -format=json | - jq -r '.sealed' ) - [ "${sealed_status}" == "true" ] + wait_for_sealed_vault $(name_prefix)-0 local init_status=$(kubectl exec "$(name_prefix)-0" -- vault status -format=json | jq -r '.initialized') diff --git a/test/kind/config.yaml b/test/kind/config.yaml index 2509664..8b18a3a 100644 --- a/test/kind/config.yaml +++ b/test/kind/config.yaml @@ -5,3 +5,16 @@ nodes: - role: worker - role: worker - role: worker +# These apiServer settings are included for running the CSI provider on K8s +# prior to 1.21 +kubeadmConfigPatches: + - | + apiVersion: kubeadm.k8s.io/v1beta2 + kind: ClusterConfiguration + metadata: + name: config + apiServer: + extraArgs: + "service-account-issuer": "https://kubernetes.default.svc.cluster.local" + "service-account-signing-key-file": "/etc/kubernetes/pki/sa.key" + "service-account-api-audiences": "https://kubernetes.default.svc.cluster.local"