Add Raft HA support (#229)

* Add raft support

* Add acceptance test

* Update templates/server-headless-service.yaml

Co-Authored-By: Theron Voran <tvoran@users.noreply.github.com>

* Add notes to raft configurables

Co-authored-by: Theron Voran <tvoran@users.noreply.github.com>

templates/_helpers.tpl

@@ -133,6 +133,10 @@ Set's additional environment variables based on the mode.
             - name: VAULT_DEV_ROOT_TOKEN_ID
               value: "root"
   {{ end }}
+  {{ if and (eq .mode "ha") (eq (.Values.server.ha.raft.enabled | toString) "true") }}
+            - name: VAULT_CLUSTER_ADDR
+              value: "https://$(HOSTNAME).vault-internal:8201"
+  {{ end }}
 {{- end -}}
 
 {{/*
@@ -144,7 +148,7 @@ based on the mode configured.
             - name: audit
               mountPath: /vault/audit
   {{ end }}
-  {{ if eq .mode "standalone" }}
+  {{ if or (eq .mode "standalone") (and (eq .mode "ha") (eq (.Values.server.ha.raft.enabled | toString) "true"))  }}
     {{ if eq (.Values.server.dataStorage.enabled | toString) "true" }}
             - name: data
               mountPath: /vault/data
@@ -169,7 +173,7 @@ storage might be desired by the user.
 {{- define "vault.volumeclaims" -}}
   {{- if and (ne .mode "dev") (or .Values.server.dataStorage.enabled .Values.server.auditStorage.enabled) }}
   volumeClaimTemplates:
-      {{- if and (eq (.Values.server.dataStorage.enabled | toString) "true") (eq .mode "standalone") }}
+      {{- if and (eq (.Values.server.dataStorage.enabled | toString) "true") (or (eq .mode "standalone") (eq (.Values.server.ha.raft.enabled | toString ) "true" )) }}
     - metadata:
         name: data
       spec:

templates/server-config-configmap.yaml

@@ -17,8 +17,10 @@ data:
     disable_mlock = true
   {{- if eq .mode "standalone" }}
     {{ tpl .Values.server.standalone.config . | nindent 4 | trim }}
-  {{- else if eq .mode "ha" }}
+  {{- else if and (eq .mode "ha") (eq (.Values.server.ha.raft.enabled | toString) "false") }}
     {{ tpl .Values.server.ha.config . | nindent 4 | trim }}
+  {{- else if and (eq .mode "ha") (eq (.Values.server.ha.raft.enabled | toString) "true") }}
+    {{ tpl .Values.server.ha.raft.config . | nindent 4 | trim }}
   {{ end }}
 {{- end }}
 {{- end }}

templates/server-headless-service.yaml

@@ -0,0 +1,35 @@
+{{ template "vault.mode" . }}
+{{- if ne .mode "external" }}
+{{- if and (eq (.Values.server.service.enabled | toString) "true" ) (eq (.Values.global.enabled | toString) "true") }}
+# Service for Vault cluster
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ template "vault.fullname" . }}-internal
+  namespace: {{ .Release.Namespace }}
+  labels:
+    helm.sh/chart: {{ include "vault.chart" . }}
+    app.kubernetes.io/name: {{ include "vault.name" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+  annotations:
+    service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"
+{{- if .Values.server.service.annotations }}
+{{ toYaml .Values.server.service.annotations | indent 4 }}
+{{- end }}
+spec:
+  clusterIP: None
+  publishNotReadyAddresses: true
+  ports:
+    - name: "{{ include "vault.scheme" . }}"
+      port: {{ .Values.server.service.port }}
+      targetPort: {{ .Values.server.service.targetPort }}
+    - name: internal
+      port: 8201
+      targetPort: 8201
+  selector:
+    app.kubernetes.io/name: {{ include "vault.name" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    component: server
+{{- end }}
+{{- end }}

templates/server-statefulset.yaml

@@ -12,7 +12,7 @@ metadata:
     app.kubernetes.io/instance: {{ .Release.Name }}
     app.kubernetes.io/managed-by: {{ .Release.Service }}
 spec:
-  serviceName: {{ template "vault.fullname" . }}
+  serviceName: {{ template "vault.fullname" . }}-internal
   podManagementPolicy: Parallel
   replicas: {{ template "vault.replicas" . }}
   updateStrategy:
@@ -71,11 +71,15 @@ spec:
             - name: VAULT_ADDR
               value: "{{ include "vault.scheme" . }}://127.0.0.1:8200"
             - name: VAULT_API_ADDR
-              value: "{{ include "vault.scheme" . }}://$(POD_IP):8200"
+              value: "{{ include "vault.scheme" . }}-internal://$(POD_IP):8200"
             - name: SKIP_CHOWN
               value: "true"
             - name: SKIP_SETCAP
               value: "true"
+            - name: HOSTNAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
             {{ template "vault.envs" . }}
             {{- include "vault.extraEnvironmentVars" .Values.server | nindent 12 }}
             {{- include "vault.extraSecretEnvironmentVars" .Values.server | nindent 12 }}

test/acceptance/server-ha-raft.bats

@@ -0,0 +1,121 @@
+#!/usr/bin/env bats
+
+load _helpers
+
+@test "server/ha-raft: testing deployment" {
+  cd `chart_dir`
+
+  helm install "$(name_prefix)" \
+    --set='server.ha.enabled=true' \
+    --set='server.ha.raft.enabled=true' .
+  wait_for_running $(name_prefix)-0
+
+  # Sealed, not initialized
+  local sealed_status=$(kubectl exec "$(name_prefix)-0" -- vault status -format=json |
+    jq -r '.sealed' )
+  [ "${sealed_status}" == "true" ]
+
+  local init_status=$(kubectl exec "$(name_prefix)-0" -- vault status -format=json |
+    jq -r '.initialized')
+  [ "${init_status}" == "false" ]
+
+  # Security
+  local ipc=$(kubectl get statefulset "$(name_prefix)" --output json |
+    jq -r '.spec.template.spec.containers[0].securityContext.capabilities.add[0]')
+  [ "${ipc}" == "IPC_LOCK" ]
+
+  # Replicas
+  local replicas=$(kubectl get statefulset "$(name_prefix)" --output json |
+    jq -r '.spec.replicas')
+  [ "${replicas}" == "3" ]
+
+  # Volume Mounts
+  local volumeCount=$(kubectl get statefulset "$(name_prefix)" --output json |
+    jq -r '.spec.template.spec.containers[0].volumeMounts | length')
+  [ "${volumeCount}" == "2" ]
+
+  # Volumes
+  local volumeCount=$(kubectl get statefulset "$(name_prefix)" --output json |
+    jq -r '.spec.template.spec.volumes | length')
+  [ "${volumeCount}" == "1" ]
+
+  local volume=$(kubectl get statefulset "$(name_prefix)" --output json |
+    jq -r '.spec.template.spec.volumes[0].configMap.name')
+  [ "${volume}" == "$(name_prefix)-config" ]
+
+  # Service
+  local service=$(kubectl get service "$(name_prefix)" --output json |
+    jq -r '.spec.clusterIP')
+  [ "${service}" != "None" ]
+
+  local service=$(kubectl get service "$(name_prefix)" --output json |
+    jq -r '.spec.type')
+  [ "${service}" == "ClusterIP" ]
+
+  local ports=$(kubectl get service "$(name_prefix)" --output json |
+    jq -r '.spec.ports | length')
+  [ "${ports}" == "2" ]
+
+  local ports=$(kubectl get service "$(name_prefix)" --output json |
+    jq -r '.spec.ports[0].port')
+  [ "${ports}" == "8200" ]
+
+  local ports=$(kubectl get service "$(name_prefix)" --output json |
+    jq -r '.spec.ports[1].port')
+  [ "${ports}" == "8201" ]
+
+  # Vault Init
+  local init=$(kubectl exec -ti "$(name_prefix)-0" -- \
+    vault operator init -format=json -n 1 -t 1)
+
+  local token=$(echo ${init} | jq -r '.unseal_keys_b64[0]')
+  [ "${token}" != "" ]
+  
+  local root=$(echo ${init} | jq -r '.root_token')
+  [ "${root}" != "" ]
+
+  kubectl exec -ti vault-0 -- vault operator unseal ${token}
+  wait_for_ready "$(name_prefix)-0"
+
+  sleep 5
+
+  # Vault Unseal
+  local pods=($(kubectl get pods --selector='app.kubernetes.io/name=vault' -o json | jq -r '.items[].metadata.name'))
+  for pod in "${pods[@]}"
+  do
+      if [[ ${pod?} != "$(name_prefix)-0" ]]
+      then
+          kubectl exec -ti ${pod} -- vault operator raft join http://$(name_prefix)-0.$(name_prefix)-internal:8200
+          kubectl exec -ti ${pod} -- vault operator unseal ${token}
+          wait_for_ready "${pod}"
+      fi
+  done
+
+  # Sealed, not initialized
+  local sealed_status=$(kubectl exec "$(name_prefix)-0" -- vault status -format=json |
+    jq -r '.sealed' )
+  [ "${sealed_status}" == "false" ]
+
+  local init_status=$(kubectl exec "$(name_prefix)-0" -- vault status -format=json |
+    jq -r '.initialized')
+  [ "${init_status}" == "true" ]
+
+  kubectl exec "$(name_prefix)-0" -- vault login ${root}
+
+  local raft_status=$(kubectl exec "$(name_prefix)-0" -- vault operator raft configuration -format=json | 
+    jq -r '.data.config.servers | length')
+  [ "${raft_status}" == "3" ]
+}
+
+setup() {
+  kubectl delete namespace acceptance --ignore-not-found=true
+  kubectl create namespace acceptance
+  kubectl config set-context --current --namespace=acceptance
+}
+
+#cleanup
+teardown() {
+  helm delete vault
+  kubectl delete --all pvc
+  kubectl delete namespace acceptance --ignore-not-found=true
+}

test/unit/server-configmap.bats

@@ -17,6 +17,14 @@ load _helpers
       yq 'length > 0' | tee /dev/stderr)
   [ "${actual}" = "true" ]
 
+  local actual=$(helm template \
+      --show-only templates/server-config-configmap.yaml \
+      --set 'server.ha.enabled=true' \
+      --set 'server.ha.raft.enabled=true' \
+      . | tee /dev/stderr |
+      yq 'length > 0' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+
   local actual=$(helm template \
       --show-only templates/server-config-configmap.yaml \
       --set 'server.standalone.enabled=true' \
@@ -25,6 +33,28 @@ load _helpers
   [ "${actual}" = "true" ]
 }
 
+@test "server/ConfigMap: raft config disabled by default" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      --show-only templates/server-config-configmap.yaml \
+      --set 'server.ha.enabled=true' \
+      . | tee /dev/stderr |
+      grep "raft" | yq 'length > 0' | tee /dev/stderr)
+  [ "${actual}" != "true" ]
+}
+
+@test "server/ConfigMap: raft config can be enabled" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      --show-only templates/server-config-configmap.yaml \
+      --set 'server.ha.enabled=true' \
+      --set 'server.ha.raft.enabled=true' \
+      . | tee /dev/stderr |
+      grep "raft" | yq 'length > 0' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+}
+
+
 @test "server/ConfigMap: disabled by server.dev.enabled true" {
   cd `chart_dir`
   local actual=$( (helm template \

test/unit/server-dev-statefulset.bats

@@ -249,19 +249,19 @@ load _helpers
       yq -r '.spec.template.spec.containers[0].env' | tee /dev/stderr)
 
   local actual=$(echo $object |
-     yq -r '.[7].name' | tee /dev/stderr)
+     yq -r '.[8].name' | tee /dev/stderr)
   [ "${actual}" = "FOO" ]
 
   local actual=$(echo $object |
-      yq -r '.[7].value' | tee /dev/stderr)
+      yq -r '.[8].value' | tee /dev/stderr)
   [ "${actual}" = "bar" ]
 
   local actual=$(echo $object |
-      yq -r '.[8].name' | tee /dev/stderr)
+      yq -r '.[9].name' | tee /dev/stderr)
   [ "${actual}" = "FOOBAR" ]
 
   local actual=$(echo $object |
-      yq -r '.[8].value' | tee /dev/stderr)
+      yq -r '.[9].value' | tee /dev/stderr)
   [ "${actual}" = "foobar" ]
 }
 
@@ -282,23 +282,23 @@ load _helpers
       yq -r '.spec.template.spec.containers[0].env' | tee /dev/stderr)
 
   local actual=$(echo $object |
-      yq -r '.[6].name' | tee /dev/stderr)
+      yq -r '.[7].name' | tee /dev/stderr)
   [ "${actual}" = "ENV_FOO_0" ]
   local actual=$(echo $object |
-      yq -r '.[6].valueFrom.secretKeyRef.name' | tee /dev/stderr)
+      yq -r '.[7].valueFrom.secretKeyRef.name' | tee /dev/stderr)
   [ "${actual}" = "secret_name_0" ]
   local actual=$(echo $object |
-      yq -r '.[6].valueFrom.secretKeyRef.key' | tee /dev/stderr)
+      yq -r '.[7].valueFrom.secretKeyRef.key' | tee /dev/stderr)
   [ "${actual}" = "secret_key_0" ]
 
   local actual=$(echo $object |
-      yq -r '.[7].name' | tee /dev/stderr)
+      yq -r '.[8].name' | tee /dev/stderr)
   [ "${actual}" = "ENV_FOO_1" ]
   local actual=$(echo $object |
-      yq -r '.[7].valueFrom.secretKeyRef.name' | tee /dev/stderr)
+      yq -r '.[8].valueFrom.secretKeyRef.name' | tee /dev/stderr)
   [ "${actual}" = "secret_name_1" ]
   local actual=$(echo $object |
-      yq -r '.[7].valueFrom.secretKeyRef.key' | tee /dev/stderr)
+      yq -r '.[8].valueFrom.secretKeyRef.key' | tee /dev/stderr)
   [ "${actual}" = "secret_key_1" ]
 }
 

test/unit/server-ha-statefulset.bats

@@ -349,19 +349,19 @@ load _helpers
       yq -r '.spec.template.spec.containers[0].env' | tee /dev/stderr)
 
   local actual=$(echo $object |
-     yq -r '.[6].name' | tee /dev/stderr)
+     yq -r '.[7].name' | tee /dev/stderr)
   [ "${actual}" = "FOO" ]
 
   local actual=$(echo $object |
-      yq -r '.[6].value' | tee /dev/stderr)
+      yq -r '.[7].value' | tee /dev/stderr)
   [ "${actual}" = "bar" ]
 
   local actual=$(echo $object |
-      yq -r '.[7].name' | tee /dev/stderr)
+      yq -r '.[8].name' | tee /dev/stderr)
   [ "${actual}" = "FOOBAR" ]
 
   local actual=$(echo $object |
-      yq -r '.[7].value' | tee /dev/stderr)
+      yq -r '.[8].value' | tee /dev/stderr)
   [ "${actual}" = "foobar" ]
 }
 
@@ -383,23 +383,23 @@ load _helpers
       yq -r '.spec.template.spec.containers[0].env' | tee /dev/stderr)
 
   local actual=$(echo $object |
-      yq -r '.[6].name' | tee /dev/stderr)
+      yq -r '.[7].name' | tee /dev/stderr)
   [ "${actual}" = "ENV_FOO_0" ]
   local actual=$(echo $object |
-      yq -r '.[6].valueFrom.secretKeyRef.name' | tee /dev/stderr)
+      yq -r '.[7].valueFrom.secretKeyRef.name' | tee /dev/stderr)
   [ "${actual}" = "secret_name_0" ]
   local actual=$(echo $object |
-      yq -r '.[6].valueFrom.secretKeyRef.key' | tee /dev/stderr)
+      yq -r '.[7].valueFrom.secretKeyRef.key' | tee /dev/stderr)
   [ "${actual}" = "secret_key_0" ]
 
   local actual=$(echo $object |
-      yq -r '.[7].name' | tee /dev/stderr)
+      yq -r '.[8].name' | tee /dev/stderr)
   [ "${actual}" = "ENV_FOO_1" ]
   local actual=$(echo $object |
-      yq -r '.[7].valueFrom.secretKeyRef.name' | tee /dev/stderr)
+      yq -r '.[8].valueFrom.secretKeyRef.name' | tee /dev/stderr)
   [ "${actual}" = "secret_name_1" ]
   local actual=$(echo $object |
-      yq -r '.[7].valueFrom.secretKeyRef.key' | tee /dev/stderr)
+      yq -r '.[8].valueFrom.secretKeyRef.key' | tee /dev/stderr)
   [ "${actual}" = "secret_key_1" ]
 }
 

test/unit/server-statefulset.bats

@@ -384,19 +384,19 @@ load _helpers
       yq -r '.spec.template.spec.containers[0].env' | tee /dev/stderr)
 
   local actual=$(echo $object |
-     yq -r '.[6].name' | tee /dev/stderr)
+     yq -r '.[7].name' | tee /dev/stderr)
   [ "${actual}" = "FOO" ]
 
   local actual=$(echo $object |
-      yq -r '.[6].value' | tee /dev/stderr)
+      yq -r '.[7].value' | tee /dev/stderr)
   [ "${actual}" = "bar" ]
 
   local actual=$(echo $object |
-      yq -r '.[7].name' | tee /dev/stderr)
+      yq -r '.[8].name' | tee /dev/stderr)
   [ "${actual}" = "FOOBAR" ]
 
   local actual=$(echo $object |
-      yq -r '.[7].value' | tee /dev/stderr)
+      yq -r '.[8].value' | tee /dev/stderr)
   [ "${actual}" = "foobar" ]
 
   local object=$(helm template \
@@ -407,19 +407,19 @@ load _helpers
       yq -r '.spec.template.spec.containers[0].env' | tee /dev/stderr)
 
   local actual=$(echo $object |
-     yq -r '.[6].name' | tee /dev/stderr)
+     yq -r '.[7].name' | tee /dev/stderr)
   [ "${actual}" = "FOO" ]
 
   local actual=$(echo $object |
-      yq -r '.[6].value' | tee /dev/stderr)
+      yq -r '.[7].value' | tee /dev/stderr)
   [ "${actual}" = "bar" ]
 
   local actual=$(echo $object |
-      yq -r '.[7].name' | tee /dev/stderr)
+      yq -r '.[8].name' | tee /dev/stderr)
   [ "${actual}" = "FOOBAR" ]
 
   local actual=$(echo $object |
-      yq -r '.[7].value' | tee /dev/stderr)
+      yq -r '.[8].value' | tee /dev/stderr)
   [ "${actual}" = "foobar" ]
 }
 

values.yaml

@@ -314,12 +314,35 @@ server:
   ha:
     enabled: false
     replicas: 3
+    
+    # Enables Vault's integrated Raft storage.  Unlike the typical HA modes where 
+    # Vault's persistence is external (such as Consul), enabling Raft mode will create 
+    # persistent volumes for Vault to store data.  The Vault cluster will coordinate leader 
+    # elections and failovers internally.
+    raft:
+      
+      # Enables Raft integrated storage
+      enabled: false
+      config: |
+        ui = true
+        cluster_addr = "https://POD_IP:8201"
+
+        listener "tcp" {
+          tls_disable = 1
+          address = "[::]:8200"
+          cluster_address = "[::]:8201"
+        }
+
+        storage "raft" {
+          path = "/vault/data"
+        }
 
     # config is a raw string of default configuration when using a Stateful
     # deployment. Default is to use a Consul for its HA storage backend.
     # This should be HCL.
     config: |
       ui = true
+      cluster_addr = "https://POD_IP:8201"
 
       listener "tcp" {
         tls_disable = 1