openbao-helm/test/acceptance/server-ha-enterprise-dr.bats

#!/usr/bin/env bats

load _helpers

@test "server/ha-enterprise-raft: testing DR deployment" {
  cd `chart_dir`

  helm install "$(name_prefix)-east" \
    --set='server.image.repository=hashicorp/vault-enterprise' \
    --set='server.image.tag=1.9.3-ent' \
    --set='injector.enabled=false' \
    --set='server.ha.enabled=true' \
    --set='server.ha.raft.enabled=true' \
    --set='server.enterpriseLicense.secretName=vault-license' .
  wait_for_running "$(name_prefix)-east-0"

  # Sealed, not initialized
  wait_for_sealed_vault $(name_prefix)-east-0

  local init_status=$(kubectl exec "$(name_prefix)-east-0" -- vault status -format=json |
    jq -r '.initialized')
  [ "${init_status}" == "false" ]

  # Vault Init
  local init=$(kubectl exec -ti "$(name_prefix)-east-0" -- \
    vault operator init -format=json -n 1 -t 1)

  local primary_token=$(echo ${init} | jq -r '.unseal_keys_b64[0]')
  [ "${primary_token}" != "" ]

  local primary_root=$(echo ${init} | jq -r '.root_token')
  [ "${primary_root}" != "" ]

  kubectl exec -ti "$(name_prefix)-east-0" -- vault operator unseal ${primary_token}
  wait_for_ready "$(name_prefix)-east-0"

  sleep 10

  # Vault Unseal
  local pods=($(kubectl get pods --selector='app.kubernetes.io/name=vault' -o json | jq -r '.items[].metadata.name'))
  for pod in "${pods[@]}"
  do
      if [[ ${pod?} != "$(name_prefix)-east-0" ]]
      then
          kubectl exec -ti ${pod} -- vault operator raft join http://$(name_prefix)-east-0.$(name_prefix)-east-internal:8200
          kubectl exec -ti ${pod} -- vault operator unseal ${primary_token}
          wait_for_ready "${pod}"
      fi
  done

  # Unsealed, initialized
  local sealed_status=$(kubectl exec "$(name_prefix)-east-0" -- vault status -format=json |
    jq -r '.sealed' )
  [ "${sealed_status}" == "false" ]

  local init_status=$(kubectl exec "$(name_prefix)-east-0" -- vault status -format=json |
    jq -r '.initialized')
  [ "${init_status}" == "true" ]

  kubectl exec "$(name_prefix)-east-0" -- vault login ${primary_root}

  local raft_status=$(kubectl exec "$(name_prefix)-east-0" -- vault operator raft list-peers -format=json |
    jq -r '.data.config.servers | length')
  [ "${raft_status}" == "3" ]

  kubectl exec -ti $(name_prefix)-east-0 -- vault write -f sys/replication/dr/primary/enable primary_cluster_addr=https://$(name_prefix)-east-active:8201

  local secondary=$(kubectl exec -ti "$(name_prefix)-east-0" -- vault write sys/replication/dr/primary/secondary-token id=secondary -format=json)
  [ "${secondary}" != "" ]

  local secondary_replica_token=$(echo ${secondary} | jq -r '.wrap_info.token')
  [ "${secondary_replica_token}" != "" ]

  # Install vault-west
  helm install "$(name_prefix)-west" \
    --set='injector.enabled=false' \
    --set='server.image.repository=hashicorp/vault-enterprise' \
    --set='server.image.tag=1.9.3-ent' \
    --set='server.ha.enabled=true' \
    --set='server.ha.raft.enabled=true' \
    --set='server.enterpriseLicense.secretName=vault-license' .
  wait_for_running "$(name_prefix)-west-0"

  # Sealed, not initialized
  wait_for_sealed_vault $(name_prefix)-west-0

  local init_status=$(kubectl exec "$(name_prefix)-west-0" -- vault status -format=json |
    jq -r '.initialized')
  [ "${init_status}" == "false" ]

  # Vault Init
  local init=$(kubectl exec -ti "$(name_prefix)-west-0" -- \
    vault operator init -format=json -n 1 -t 1)

  local secondary_token=$(echo ${init} | jq -r '.unseal_keys_b64[0]')
  [ "${secondary_token}" != "" ]

  local secondary_root=$(echo ${init} | jq -r '.root_token')
  [ "${secondary_root}" != "" ]

  kubectl exec -ti "$(name_prefix)-west-0" -- vault operator unseal ${secondary_token}
  wait_for_ready "$(name_prefix)-west-0"

  sleep 10

  # Vault Unseal
  local pods=($(kubectl get pods --selector='app.kubernetes.io/instance=vault-west' -o json | jq -r '.items[].metadata.name'))
  for pod in "${pods[@]}"
  do
      if [[ ${pod?} != "$(name_prefix)-west-0" ]]
      then
          kubectl exec -ti ${pod} -- vault operator raft join http://$(name_prefix)-west-0.$(name_prefix)-west-internal:8200
          kubectl exec -ti ${pod} -- vault operator unseal ${secondary_token}
          wait_for_ready "${pod}"
      fi
  done

  # Unsealed, initialized
  local sealed_status=$(kubectl exec "$(name_prefix)-west-0" -- vault status -format=json |
    jq -r '.sealed' )
  [ "${sealed_status}" == "false" ]

  local init_status=$(kubectl exec "$(name_prefix)-west-0" -- vault status -format=json |
    jq -r '.initialized')
  [ "${init_status}" == "true" ]

  kubectl exec "$(name_prefix)-west-0" -- vault login ${secondary_root}

  local raft_status=$(kubectl exec "$(name_prefix)-west-0" -- vault operator raft list-peers -format=json |
    jq -r '.data.config.servers | length')
  [ "${raft_status}" == "3" ]

  kubectl exec -ti "$(name_prefix)-west-0" -- vault write sys/replication/dr/secondary/enable token=${secondary_replica_token}

  sleep 10

  local pods=($(kubectl get pods --selector='app.kubernetes.io/instance=vault-west' -o json | jq -r '.items[].metadata.name'))
  for pod in "${pods[@]}"
  do
      if [[ ${pod?} != "$(name_prefix)-west-0" ]]
      then
          kubectl delete pod "${pod?}"
          wait_for_running "${pod?}"
          kubectl exec -ti ${pod} -- vault operator unseal ${primary_token}
          wait_for_ready "${pod}"
      fi
  done
}

setup() {
  kubectl delete namespace acceptance --ignore-not-found=true
  kubectl create namespace acceptance
  kubectl config set-context --current --namespace=acceptance
  kubectl create secret generic vault-license --from-literal license=$VAULT_LICENSE_CI
}

#cleanup
teardown() {
  if [[ ${CLEANUP:-true} == "true" ]]
  then
	  helm delete vault-east
	  helm delete vault-west
	  kubectl delete --all pvc
	  kubectl delete namespace acceptance --ignore-not-found=true
  fi
}