Add Raft HA support (#229)

* Add raft support * Add acceptance test * Update templates/server-headless-service.yaml Co-Authored-By: Theron Voran <tvoran@users.noreply.github.com> * Add notes to raft configurables Co-authored-by: Theron Voran <tvoran@users.noreply.github.com>
2020-03-18 15:49:14 -04:00 · 2020-03-18 15:49:14 -04:00 · 58b96dbc10
commit 58b96dbc10
parent d0f89fced8
10 changed files with 252 additions and 33 deletions
--- a/templates/_helpers.tpl
+++ b/templates/_helpers.tpl
@ -133,6 +133,10 @@ Set's additional environment variables based on the mode.
            - name: VAULT_DEV_ROOT_TOKEN_ID
              value: "root"
  {{ end }}
+  {{ if and (eq .mode "ha") (eq (.Values.server.ha.raft.enabled | toString) "true") }}
+            - name: VAULT_CLUSTER_ADDR
+              value: "https://$(HOSTNAME).vault-internal:8201"
+  {{ end }}
 {{- end -}}

 {{/*
@ -144,7 +148,7 @@ based on the mode configured.
            - name: audit
              mountPath: /vault/audit
  {{ end }}
-  {{ if eq .mode "standalone" }}
+  {{ if or (eq .mode "standalone") (and (eq .mode "ha") (eq (.Values.server.ha.raft.enabled | toString) "true"))  }}
    {{ if eq (.Values.server.dataStorage.enabled | toString) "true" }}
            - name: data
              mountPath: /vault/data
@ -169,7 +173,7 @@ storage might be desired by the user.
 {{- define "vault.volumeclaims" -}}
  {{- if and (ne .mode "dev") (or .Values.server.dataStorage.enabled .Values.server.auditStorage.enabled) }}
  volumeClaimTemplates:
-      {{- if and (eq (.Values.server.dataStorage.enabled | toString) "true") (eq .mode "standalone") }}
+      {{- if and (eq (.Values.server.dataStorage.enabled | toString) "true") (or (eq .mode "standalone") (eq (.Values.server.ha.raft.enabled | toString ) "true" )) }}
    - metadata:
        name: data
      spec:
--- a/templates/server-config-configmap.yaml
+++ b/templates/server-config-configmap.yaml
@ -17,8 +17,10 @@ data:
    disable_mlock = true
  {{- if eq .mode "standalone" }}
    {{ tpl .Values.server.standalone.config . | nindent 4 | trim }}
-  {{- else if eq .mode "ha" }}
+  {{- else if and (eq .mode "ha") (eq (.Values.server.ha.raft.enabled | toString) "false") }}
    {{ tpl .Values.server.ha.config . | nindent 4 | trim }}
+  {{- else if and (eq .mode "ha") (eq (.Values.server.ha.raft.enabled | toString) "true") }}
+    {{ tpl .Values.server.ha.raft.config . | nindent 4 | trim }}
  {{ end }}
 {{- end }}
 {{- end }}
--- a/templates/server-headless-service.yaml
+++ b/templates/server-headless-service.yaml
@ -0,0 +1,35 @@
+{{ template "vault.mode" . }}
+{{- if ne .mode "external" }}
+{{- if and (eq (.Values.server.service.enabled | toString) "true" ) (eq (.Values.global.enabled | toString) "true") }}
+# Service for Vault cluster
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ template "vault.fullname" . }}-internal
+  namespace: {{ .Release.Namespace }}
+  labels:
+    helm.sh/chart: {{ include "vault.chart" . }}
+    app.kubernetes.io/name: {{ include "vault.name" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+  annotations:
+    service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"
+{{- if .Values.server.service.annotations }}
+{{ toYaml .Values.server.service.annotations | indent 4 }}
+{{- end }}
+spec:
+  clusterIP: None
+  publishNotReadyAddresses: true
+  ports:
+    - name: "{{ include "vault.scheme" . }}"
+      port: {{ .Values.server.service.port }}
+      targetPort: {{ .Values.server.service.targetPort }}
+    - name: internal
+      port: 8201
+      targetPort: 8201
+  selector:
+    app.kubernetes.io/name: {{ include "vault.name" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    component: server
+{{- end }}
+{{- end }}
--- a/templates/server-statefulset.yaml
+++ b/templates/server-statefulset.yaml
@ -12,7 +12,7 @@ metadata:
    app.kubernetes.io/instance: {{ .Release.Name }}
    app.kubernetes.io/managed-by: {{ .Release.Service }}
 spec:
-  serviceName: {{ template "vault.fullname" . }}
+  serviceName: {{ template "vault.fullname" . }}-internal
  podManagementPolicy: Parallel
  replicas: {{ template "vault.replicas" . }}
  updateStrategy:
@ -71,11 +71,15 @@ spec:
            - name: VAULT_ADDR
              value: "{{ include "vault.scheme" . }}://127.0.0.1:8200"
            - name: VAULT_API_ADDR
-              value: "{{ include "vault.scheme" . }}://$(POD_IP):8200"
+              value: "{{ include "vault.scheme" . }}-internal://$(POD_IP):8200"
            - name: SKIP_CHOWN
              value: "true"
            - name: SKIP_SETCAP
              value: "true"
+            - name: HOSTNAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
            {{ template "vault.envs" . }}
            {{- include "vault.extraEnvironmentVars" .Values.server | nindent 12 }}
            {{- include "vault.extraSecretEnvironmentVars" .Values.server | nindent 12 }}
--- a/test/acceptance/server-ha-raft.bats
+++ b/test/acceptance/server-ha-raft.bats
@ -0,0 +1,121 @@
+#!/usr/bin/env bats
+
+load _helpers
+
+@test "server/ha-raft: testing deployment" {
+  cd `chart_dir`
+
+  helm install "$(name_prefix)" \
+    --set='server.ha.enabled=true' \
+    --set='server.ha.raft.enabled=true' .
+  wait_for_running $(name_prefix)-0
+
+  # Sealed, not initialized
+  local sealed_status=$(kubectl exec "$(name_prefix)-0" -- vault status -format=json |
+    jq -r '.sealed' )
+  [ "${sealed_status}" == "true" ]
+
+  local init_status=$(kubectl exec "$(name_prefix)-0" -- vault status -format=json |
+    jq -r '.initialized')
+  [ "${init_status}" == "false" ]
+
+  # Security
+  local ipc=$(kubectl get statefulset "$(name_prefix)" --output json |
+    jq -r '.spec.template.spec.containers[0].securityContext.capabilities.add[0]')
+  [ "${ipc}" == "IPC_LOCK" ]
+
+  # Replicas
+  local replicas=$(kubectl get statefulset "$(name_prefix)" --output json |
+    jq -r '.spec.replicas')
+  [ "${replicas}" == "3" ]
+
+  # Volume Mounts
+  local volumeCount=$(kubectl get statefulset "$(name_prefix)" --output json |
+    jq -r '.spec.template.spec.containers[0].volumeMounts | length')
+  [ "${volumeCount}" == "2" ]
+
+  # Volumes
+  local volumeCount=$(kubectl get statefulset "$(name_prefix)" --output json |
+    jq -r '.spec.template.spec.volumes | length')
+  [ "${volumeCount}" == "1" ]
+
+  local volume=$(kubectl get statefulset "$(name_prefix)" --output json |
+    jq -r '.spec.template.spec.volumes[0].configMap.name')
+  [ "${volume}" == "$(name_prefix)-config" ]
+
+  # Service
+  local service=$(kubectl get service "$(name_prefix)" --output json |
+    jq -r '.spec.clusterIP')
+  [ "${service}" != "None" ]
+
+  local service=$(kubectl get service "$(name_prefix)" --output json |
+    jq -r '.spec.type')
+  [ "${service}" == "ClusterIP" ]
+
+  local ports=$(kubectl get service "$(name_prefix)" --output json |
+    jq -r '.spec.ports | length')
+  [ "${ports}" == "2" ]
+
+  local ports=$(kubectl get service "$(name_prefix)" --output json |
+    jq -r '.spec.ports[0].port')
+  [ "${ports}" == "8200" ]
+
+  local ports=$(kubectl get service "$(name_prefix)" --output json |
+    jq -r '.spec.ports[1].port')
+  [ "${ports}" == "8201" ]
+
+  # Vault Init
+  local init=$(kubectl exec -ti "$(name_prefix)-0" -- \
+    vault operator init -format=json -n 1 -t 1)
+
+  local token=$(echo ${init} | jq -r '.unseal_keys_b64[0]')
+  [ "${token}" != "" ]
+  
+  local root=$(echo ${init} | jq -r '.root_token')
+  [ "${root}" != "" ]
+
+  kubectl exec -ti vault-0 -- vault operator unseal ${token}
+  wait_for_ready "$(name_prefix)-0"
+
+  sleep 5
+
+  # Vault Unseal
+  local pods=($(kubectl get pods --selector='app.kubernetes.io/name=vault' -o json | jq -r '.items[].metadata.name'))
+  for pod in "${pods[@]}"
+  do
+      if [[ ${pod?} != "$(name_prefix)-0" ]]
+      then
+          kubectl exec -ti ${pod} -- vault operator raft join http://$(name_prefix)-0.$(name_prefix)-internal:8200
+          kubectl exec -ti ${pod} -- vault operator unseal ${token}
+          wait_for_ready "${pod}"
+      fi
+  done
+
+  # Sealed, not initialized
+  local sealed_status=$(kubectl exec "$(name_prefix)-0" -- vault status -format=json |
+    jq -r '.sealed' )
+  [ "${sealed_status}" == "false" ]
+
+  local init_status=$(kubectl exec "$(name_prefix)-0" -- vault status -format=json |
+    jq -r '.initialized')
+  [ "${init_status}" == "true" ]
+
+  kubectl exec "$(name_prefix)-0" -- vault login ${root}
+
+  local raft_status=$(kubectl exec "$(name_prefix)-0" -- vault operator raft configuration -format=json | 
+    jq -r '.data.config.servers | length')
+  [ "${raft_status}" == "3" ]
+}
+
+setup() {
+  kubectl delete namespace acceptance --ignore-not-found=true
+  kubectl create namespace acceptance
+  kubectl config set-context --current --namespace=acceptance
+}
+
+#cleanup
+teardown() {
+  helm delete vault
+  kubectl delete --all pvc
+  kubectl delete namespace acceptance --ignore-not-found=true
+}
--- a/test/unit/server-configmap.bats
+++ b/test/unit/server-configmap.bats
@ -17,6 +17,14 @@ load _helpers
      yq 'length > 0' | tee /dev/stderr)
  [ "${actual}" = "true" ]

+  local actual=$(helm template \
+      --show-only templates/server-config-configmap.yaml \
+      --set 'server.ha.enabled=true' \
+      --set 'server.ha.raft.enabled=true' \
+      . | tee /dev/stderr |
+      yq 'length > 0' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+
  local actual=$(helm template \
      --show-only templates/server-config-configmap.yaml \
      --set 'server.standalone.enabled=true' \
@ -25,6 +33,28 @@ load _helpers
  [ "${actual}" = "true" ]
 }

+@test "server/ConfigMap: raft config disabled by default" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      --show-only templates/server-config-configmap.yaml \
+      --set 'server.ha.enabled=true' \
+      . | tee /dev/stderr |
+      grep "raft" | yq 'length > 0' | tee /dev/stderr)
+  [ "${actual}" != "true" ]
+}
+
+@test "server/ConfigMap: raft config can be enabled" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      --show-only templates/server-config-configmap.yaml \
+      --set 'server.ha.enabled=true' \
+      --set 'server.ha.raft.enabled=true' \
+      . | tee /dev/stderr |
+      grep "raft" | yq 'length > 0' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+}
+
+
@test "server/ConfigMap: disabled by server.dev.enabled true" {
  cd `chart_dir`
  local actual=$( (helm template \
--- a/test/unit/server-dev-statefulset.bats
+++ b/test/unit/server-dev-statefulset.bats
@ -249,19 +249,19 @@ load _helpers
      yq -r '.spec.template.spec.containers[0].env' | tee /dev/stderr)

  local actual=$(echo $object |
-     yq -r '.[7].name' | tee /dev/stderr)
+     yq -r '.[8].name' | tee /dev/stderr)
  [ "${actual}" = "FOO" ]

  local actual=$(echo $object |
-      yq -r '.[7].value' | tee /dev/stderr)
+      yq -r '.[8].value' | tee /dev/stderr)
  [ "${actual}" = "bar" ]

  local actual=$(echo $object |
-      yq -r '.[8].name' | tee /dev/stderr)
+      yq -r '.[9].name' | tee /dev/stderr)
  [ "${actual}" = "FOOBAR" ]

  local actual=$(echo $object |
-      yq -r '.[8].value' | tee /dev/stderr)
+      yq -r '.[9].value' | tee /dev/stderr)
  [ "${actual}" = "foobar" ]
 }

@ -282,23 +282,23 @@ load _helpers
      yq -r '.spec.template.spec.containers[0].env' | tee /dev/stderr)

  local actual=$(echo $object |
-      yq -r '.[6].name' | tee /dev/stderr)
+      yq -r '.[7].name' | tee /dev/stderr)
  [ "${actual}" = "ENV_FOO_0" ]
  local actual=$(echo $object |
-      yq -r '.[6].valueFrom.secretKeyRef.name' | tee /dev/stderr)
+      yq -r '.[7].valueFrom.secretKeyRef.name' | tee /dev/stderr)
  [ "${actual}" = "secret_name_0" ]
  local actual=$(echo $object |
-      yq -r '.[6].valueFrom.secretKeyRef.key' | tee /dev/stderr)
+      yq -r '.[7].valueFrom.secretKeyRef.key' | tee /dev/stderr)
  [ "${actual}" = "secret_key_0" ]

  local actual=$(echo $object |
-      yq -r '.[7].name' | tee /dev/stderr)
+      yq -r '.[8].name' | tee /dev/stderr)
  [ "${actual}" = "ENV_FOO_1" ]
  local actual=$(echo $object |
-      yq -r '.[7].valueFrom.secretKeyRef.name' | tee /dev/stderr)
+      yq -r '.[8].valueFrom.secretKeyRef.name' | tee /dev/stderr)
  [ "${actual}" = "secret_name_1" ]
  local actual=$(echo $object |
-      yq -r '.[7].valueFrom.secretKeyRef.key' | tee /dev/stderr)
+      yq -r '.[8].valueFrom.secretKeyRef.key' | tee /dev/stderr)
  [ "${actual}" = "secret_key_1" ]
 }

--- a/test/unit/server-ha-statefulset.bats
+++ b/test/unit/server-ha-statefulset.bats
@ -349,19 +349,19 @@ load _helpers
      yq -r '.spec.template.spec.containers[0].env' | tee /dev/stderr)

  local actual=$(echo $object |
-     yq -r '.[6].name' | tee /dev/stderr)
+     yq -r '.[7].name' | tee /dev/stderr)
  [ "${actual}" = "FOO" ]

  local actual=$(echo $object |
-      yq -r '.[6].value' | tee /dev/stderr)
+      yq -r '.[7].value' | tee /dev/stderr)
  [ "${actual}" = "bar" ]

  local actual=$(echo $object |
-      yq -r '.[7].name' | tee /dev/stderr)
+      yq -r '.[8].name' | tee /dev/stderr)
  [ "${actual}" = "FOOBAR" ]

  local actual=$(echo $object |
-      yq -r '.[7].value' | tee /dev/stderr)
+      yq -r '.[8].value' | tee /dev/stderr)
  [ "${actual}" = "foobar" ]
 }

@ -383,23 +383,23 @@ load _helpers
      yq -r '.spec.template.spec.containers[0].env' | tee /dev/stderr)

  local actual=$(echo $object |
-      yq -r '.[6].name' | tee /dev/stderr)
+      yq -r '.[7].name' | tee /dev/stderr)
  [ "${actual}" = "ENV_FOO_0" ]
  local actual=$(echo $object |
-      yq -r '.[6].valueFrom.secretKeyRef.name' | tee /dev/stderr)
+      yq -r '.[7].valueFrom.secretKeyRef.name' | tee /dev/stderr)
  [ "${actual}" = "secret_name_0" ]
  local actual=$(echo $object |
-      yq -r '.[6].valueFrom.secretKeyRef.key' | tee /dev/stderr)
+      yq -r '.[7].valueFrom.secretKeyRef.key' | tee /dev/stderr)
  [ "${actual}" = "secret_key_0" ]

  local actual=$(echo $object |
-      yq -r '.[7].name' | tee /dev/stderr)
+      yq -r '.[8].name' | tee /dev/stderr)
  [ "${actual}" = "ENV_FOO_1" ]
  local actual=$(echo $object |
-      yq -r '.[7].valueFrom.secretKeyRef.name' | tee /dev/stderr)
+      yq -r '.[8].valueFrom.secretKeyRef.name' | tee /dev/stderr)
  [ "${actual}" = "secret_name_1" ]
  local actual=$(echo $object |
-      yq -r '.[7].valueFrom.secretKeyRef.key' | tee /dev/stderr)
+      yq -r '.[8].valueFrom.secretKeyRef.key' | tee /dev/stderr)
  [ "${actual}" = "secret_key_1" ]
 }

--- a/test/unit/server-statefulset.bats
+++ b/test/unit/server-statefulset.bats
@ -384,19 +384,19 @@ load _helpers
      yq -r '.spec.template.spec.containers[0].env' | tee /dev/stderr)

  local actual=$(echo $object |
-     yq -r '.[6].name' | tee /dev/stderr)
+     yq -r '.[7].name' | tee /dev/stderr)
  [ "${actual}" = "FOO" ]

  local actual=$(echo $object |
-      yq -r '.[6].value' | tee /dev/stderr)
+      yq -r '.[7].value' | tee /dev/stderr)
  [ "${actual}" = "bar" ]

  local actual=$(echo $object |
-      yq -r '.[7].name' | tee /dev/stderr)
+      yq -r '.[8].name' | tee /dev/stderr)
  [ "${actual}" = "FOOBAR" ]

  local actual=$(echo $object |
-      yq -r '.[7].value' | tee /dev/stderr)
+      yq -r '.[8].value' | tee /dev/stderr)
  [ "${actual}" = "foobar" ]

  local object=$(helm template \
@ -407,19 +407,19 @@ load _helpers
      yq -r '.spec.template.spec.containers[0].env' | tee /dev/stderr)

  local actual=$(echo $object |
-     yq -r '.[6].name' | tee /dev/stderr)
+     yq -r '.[7].name' | tee /dev/stderr)
  [ "${actual}" = "FOO" ]

  local actual=$(echo $object |
-      yq -r '.[6].value' | tee /dev/stderr)
+      yq -r '.[7].value' | tee /dev/stderr)
  [ "${actual}" = "bar" ]

  local actual=$(echo $object |
-      yq -r '.[7].name' | tee /dev/stderr)
+      yq -r '.[8].name' | tee /dev/stderr)
  [ "${actual}" = "FOOBAR" ]

  local actual=$(echo $object |
-      yq -r '.[7].value' | tee /dev/stderr)
+      yq -r '.[8].value' | tee /dev/stderr)
  [ "${actual}" = "foobar" ]
 }

--- a/values.yaml
+++ b/values.yaml
@ -315,11 +315,34 @@ server:
    enabled: false
    replicas: 3
    
+    # Enables Vault's integrated Raft storage.  Unlike the typical HA modes where 
+    # Vault's persistence is external (such as Consul), enabling Raft mode will create 
+    # persistent volumes for Vault to store data.  The Vault cluster will coordinate leader 
+    # elections and failovers internally.
+    raft:
+      
+      # Enables Raft integrated storage
+      enabled: false
+      config: |
+        ui = true
+        cluster_addr = "https://POD_IP:8201"
+
+        listener "tcp" {
+          tls_disable = 1
+          address = "[::]:8200"
+          cluster_address = "[::]:8201"
+        }
+
+        storage "raft" {
+          path = "/vault/data"
+        }
+
    # config is a raw string of default configuration when using a Stateful
    # deployment. Default is to use a Consul for its HA storage backend.
    # This should be HCL.
    config: |
      ui = true
+      cluster_addr = "https://POD_IP:8201"

      listener "tcp" {
        tls_disable = 1