diff --git a/templates/_helpers.tpl b/templates/_helpers.tpl index f985a8c..5639142 100644 --- a/templates/_helpers.tpl +++ b/templates/_helpers.tpl @@ -133,6 +133,10 @@ Set's additional environment variables based on the mode. - name: VAULT_DEV_ROOT_TOKEN_ID value: "root" {{ end }} + {{ if and (eq .mode "ha") (eq (.Values.server.ha.raft.enabled | toString) "true") }} + - name: VAULT_CLUSTER_ADDR + value: "https://$(HOSTNAME).vault-internal:8201" + {{ end }} {{- end -}} {{/* @@ -144,7 +148,7 @@ based on the mode configured. - name: audit mountPath: /vault/audit {{ end }} - {{ if eq .mode "standalone" }} + {{ if or (eq .mode "standalone") (and (eq .mode "ha") (eq (.Values.server.ha.raft.enabled | toString) "true")) }} {{ if eq (.Values.server.dataStorage.enabled | toString) "true" }} - name: data mountPath: /vault/data @@ -169,7 +173,7 @@ storage might be desired by the user. {{- define "vault.volumeclaims" -}} {{- if and (ne .mode "dev") (or .Values.server.dataStorage.enabled .Values.server.auditStorage.enabled) }} volumeClaimTemplates: - {{- if and (eq (.Values.server.dataStorage.enabled | toString) "true") (eq .mode "standalone") }} + {{- if and (eq (.Values.server.dataStorage.enabled | toString) "true") (or (eq .mode "standalone") (eq (.Values.server.ha.raft.enabled | toString ) "true" )) }} - metadata: name: data spec: diff --git a/templates/server-config-configmap.yaml b/templates/server-config-configmap.yaml index 6748d0f..6e05850 100644 --- a/templates/server-config-configmap.yaml +++ b/templates/server-config-configmap.yaml @@ -17,8 +17,10 @@ data: disable_mlock = true {{- if eq .mode "standalone" }} {{ tpl .Values.server.standalone.config . | nindent 4 | trim }} - {{- else if eq .mode "ha" }} + {{- else if and (eq .mode "ha") (eq (.Values.server.ha.raft.enabled | toString) "false") }} {{ tpl .Values.server.ha.config . | nindent 4 | trim }} + {{- else if and (eq .mode "ha") (eq (.Values.server.ha.raft.enabled | toString) "true") }} + {{ tpl .Values.server.ha.raft.config . | nindent 4 | trim }} {{ end }} {{- end }} {{- end }} diff --git a/templates/server-headless-service.yaml b/templates/server-headless-service.yaml new file mode 100644 index 0000000..80a94a3 --- /dev/null +++ b/templates/server-headless-service.yaml @@ -0,0 +1,35 @@ +{{ template "vault.mode" . }} +{{- if ne .mode "external" }} +{{- if and (eq (.Values.server.service.enabled | toString) "true" ) (eq (.Values.global.enabled | toString) "true") }} +# Service for Vault cluster +apiVersion: v1 +kind: Service +metadata: + name: {{ template "vault.fullname" . }}-internal + namespace: {{ .Release.Namespace }} + labels: + helm.sh/chart: {{ include "vault.chart" . }} + app.kubernetes.io/name: {{ include "vault.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + annotations: + service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" +{{- if .Values.server.service.annotations }} +{{ toYaml .Values.server.service.annotations | indent 4 }} +{{- end }} +spec: + clusterIP: None + publishNotReadyAddresses: true + ports: + - name: "{{ include "vault.scheme" . }}" + port: {{ .Values.server.service.port }} + targetPort: {{ .Values.server.service.targetPort }} + - name: internal + port: 8201 + targetPort: 8201 + selector: + app.kubernetes.io/name: {{ include "vault.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + component: server +{{- end }} +{{- end }} diff --git a/templates/server-statefulset.yaml b/templates/server-statefulset.yaml index 18e0d6b..5b4752b 100644 --- a/templates/server-statefulset.yaml +++ b/templates/server-statefulset.yaml @@ -12,7 +12,7 @@ metadata: app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/managed-by: {{ .Release.Service }} spec: - serviceName: {{ template "vault.fullname" . }} + serviceName: {{ template "vault.fullname" . }}-internal podManagementPolicy: Parallel replicas: {{ template "vault.replicas" . }} updateStrategy: @@ -71,11 +71,15 @@ spec: - name: VAULT_ADDR value: "{{ include "vault.scheme" . }}://127.0.0.1:8200" - name: VAULT_API_ADDR - value: "{{ include "vault.scheme" . }}://$(POD_IP):8200" + value: "{{ include "vault.scheme" . }}-internal://$(POD_IP):8200" - name: SKIP_CHOWN value: "true" - name: SKIP_SETCAP value: "true" + - name: HOSTNAME + valueFrom: + fieldRef: + fieldPath: metadata.name {{ template "vault.envs" . }} {{- include "vault.extraEnvironmentVars" .Values.server | nindent 12 }} {{- include "vault.extraSecretEnvironmentVars" .Values.server | nindent 12 }} diff --git a/test/acceptance/server-ha-raft.bats b/test/acceptance/server-ha-raft.bats new file mode 100644 index 0000000..17951b8 --- /dev/null +++ b/test/acceptance/server-ha-raft.bats @@ -0,0 +1,121 @@ +#!/usr/bin/env bats + +load _helpers + +@test "server/ha-raft: testing deployment" { + cd `chart_dir` + + helm install "$(name_prefix)" \ + --set='server.ha.enabled=true' \ + --set='server.ha.raft.enabled=true' . + wait_for_running $(name_prefix)-0 + + # Sealed, not initialized + local sealed_status=$(kubectl exec "$(name_prefix)-0" -- vault status -format=json | + jq -r '.sealed' ) + [ "${sealed_status}" == "true" ] + + local init_status=$(kubectl exec "$(name_prefix)-0" -- vault status -format=json | + jq -r '.initialized') + [ "${init_status}" == "false" ] + + # Security + local ipc=$(kubectl get statefulset "$(name_prefix)" --output json | + jq -r '.spec.template.spec.containers[0].securityContext.capabilities.add[0]') + [ "${ipc}" == "IPC_LOCK" ] + + # Replicas + local replicas=$(kubectl get statefulset "$(name_prefix)" --output json | + jq -r '.spec.replicas') + [ "${replicas}" == "3" ] + + # Volume Mounts + local volumeCount=$(kubectl get statefulset "$(name_prefix)" --output json | + jq -r '.spec.template.spec.containers[0].volumeMounts | length') + [ "${volumeCount}" == "2" ] + + # Volumes + local volumeCount=$(kubectl get statefulset "$(name_prefix)" --output json | + jq -r '.spec.template.spec.volumes | length') + [ "${volumeCount}" == "1" ] + + local volume=$(kubectl get statefulset "$(name_prefix)" --output json | + jq -r '.spec.template.spec.volumes[0].configMap.name') + [ "${volume}" == "$(name_prefix)-config" ] + + # Service + local service=$(kubectl get service "$(name_prefix)" --output json | + jq -r '.spec.clusterIP') + [ "${service}" != "None" ] + + local service=$(kubectl get service "$(name_prefix)" --output json | + jq -r '.spec.type') + [ "${service}" == "ClusterIP" ] + + local ports=$(kubectl get service "$(name_prefix)" --output json | + jq -r '.spec.ports | length') + [ "${ports}" == "2" ] + + local ports=$(kubectl get service "$(name_prefix)" --output json | + jq -r '.spec.ports[0].port') + [ "${ports}" == "8200" ] + + local ports=$(kubectl get service "$(name_prefix)" --output json | + jq -r '.spec.ports[1].port') + [ "${ports}" == "8201" ] + + # Vault Init + local init=$(kubectl exec -ti "$(name_prefix)-0" -- \ + vault operator init -format=json -n 1 -t 1) + + local token=$(echo ${init} | jq -r '.unseal_keys_b64[0]') + [ "${token}" != "" ] + + local root=$(echo ${init} | jq -r '.root_token') + [ "${root}" != "" ] + + kubectl exec -ti vault-0 -- vault operator unseal ${token} + wait_for_ready "$(name_prefix)-0" + + sleep 5 + + # Vault Unseal + local pods=($(kubectl get pods --selector='app.kubernetes.io/name=vault' -o json | jq -r '.items[].metadata.name')) + for pod in "${pods[@]}" + do + if [[ ${pod?} != "$(name_prefix)-0" ]] + then + kubectl exec -ti ${pod} -- vault operator raft join http://$(name_prefix)-0.$(name_prefix)-internal:8200 + kubectl exec -ti ${pod} -- vault operator unseal ${token} + wait_for_ready "${pod}" + fi + done + + # Sealed, not initialized + local sealed_status=$(kubectl exec "$(name_prefix)-0" -- vault status -format=json | + jq -r '.sealed' ) + [ "${sealed_status}" == "false" ] + + local init_status=$(kubectl exec "$(name_prefix)-0" -- vault status -format=json | + jq -r '.initialized') + [ "${init_status}" == "true" ] + + kubectl exec "$(name_prefix)-0" -- vault login ${root} + + local raft_status=$(kubectl exec "$(name_prefix)-0" -- vault operator raft configuration -format=json | + jq -r '.data.config.servers | length') + [ "${raft_status}" == "3" ] +} + +setup() { + kubectl delete namespace acceptance --ignore-not-found=true + kubectl create namespace acceptance + kubectl config set-context --current --namespace=acceptance +} + +#cleanup +teardown() { + helm delete vault + kubectl delete --all pvc + kubectl delete namespace acceptance --ignore-not-found=true +} diff --git a/test/unit/server-configmap.bats b/test/unit/server-configmap.bats index 2aa8856..fe2ac12 100755 --- a/test/unit/server-configmap.bats +++ b/test/unit/server-configmap.bats @@ -17,6 +17,14 @@ load _helpers yq 'length > 0' | tee /dev/stderr) [ "${actual}" = "true" ] + local actual=$(helm template \ + --show-only templates/server-config-configmap.yaml \ + --set 'server.ha.enabled=true' \ + --set 'server.ha.raft.enabled=true' \ + . | tee /dev/stderr | + yq 'length > 0' | tee /dev/stderr) + [ "${actual}" = "true" ] + local actual=$(helm template \ --show-only templates/server-config-configmap.yaml \ --set 'server.standalone.enabled=true' \ @@ -25,6 +33,28 @@ load _helpers [ "${actual}" = "true" ] } +@test "server/ConfigMap: raft config disabled by default" { + cd `chart_dir` + local actual=$(helm template \ + --show-only templates/server-config-configmap.yaml \ + --set 'server.ha.enabled=true' \ + . | tee /dev/stderr | + grep "raft" | yq 'length > 0' | tee /dev/stderr) + [ "${actual}" != "true" ] +} + +@test "server/ConfigMap: raft config can be enabled" { + cd `chart_dir` + local actual=$(helm template \ + --show-only templates/server-config-configmap.yaml \ + --set 'server.ha.enabled=true' \ + --set 'server.ha.raft.enabled=true' \ + . | tee /dev/stderr | + grep "raft" | yq 'length > 0' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + + @test "server/ConfigMap: disabled by server.dev.enabled true" { cd `chart_dir` local actual=$( (helm template \ diff --git a/test/unit/server-dev-statefulset.bats b/test/unit/server-dev-statefulset.bats index 10a9da6..5ce3405 100755 --- a/test/unit/server-dev-statefulset.bats +++ b/test/unit/server-dev-statefulset.bats @@ -249,19 +249,19 @@ load _helpers yq -r '.spec.template.spec.containers[0].env' | tee /dev/stderr) local actual=$(echo $object | - yq -r '.[7].name' | tee /dev/stderr) + yq -r '.[8].name' | tee /dev/stderr) [ "${actual}" = "FOO" ] local actual=$(echo $object | - yq -r '.[7].value' | tee /dev/stderr) + yq -r '.[8].value' | tee /dev/stderr) [ "${actual}" = "bar" ] local actual=$(echo $object | - yq -r '.[8].name' | tee /dev/stderr) + yq -r '.[9].name' | tee /dev/stderr) [ "${actual}" = "FOOBAR" ] local actual=$(echo $object | - yq -r '.[8].value' | tee /dev/stderr) + yq -r '.[9].value' | tee /dev/stderr) [ "${actual}" = "foobar" ] } @@ -282,23 +282,23 @@ load _helpers yq -r '.spec.template.spec.containers[0].env' | tee /dev/stderr) local actual=$(echo $object | - yq -r '.[6].name' | tee /dev/stderr) + yq -r '.[7].name' | tee /dev/stderr) [ "${actual}" = "ENV_FOO_0" ] local actual=$(echo $object | - yq -r '.[6].valueFrom.secretKeyRef.name' | tee /dev/stderr) + yq -r '.[7].valueFrom.secretKeyRef.name' | tee /dev/stderr) [ "${actual}" = "secret_name_0" ] local actual=$(echo $object | - yq -r '.[6].valueFrom.secretKeyRef.key' | tee /dev/stderr) + yq -r '.[7].valueFrom.secretKeyRef.key' | tee /dev/stderr) [ "${actual}" = "secret_key_0" ] local actual=$(echo $object | - yq -r '.[7].name' | tee /dev/stderr) + yq -r '.[8].name' | tee /dev/stderr) [ "${actual}" = "ENV_FOO_1" ] local actual=$(echo $object | - yq -r '.[7].valueFrom.secretKeyRef.name' | tee /dev/stderr) + yq -r '.[8].valueFrom.secretKeyRef.name' | tee /dev/stderr) [ "${actual}" = "secret_name_1" ] local actual=$(echo $object | - yq -r '.[7].valueFrom.secretKeyRef.key' | tee /dev/stderr) + yq -r '.[8].valueFrom.secretKeyRef.key' | tee /dev/stderr) [ "${actual}" = "secret_key_1" ] } diff --git a/test/unit/server-ha-statefulset.bats b/test/unit/server-ha-statefulset.bats index 11c4e93..db2ea6b 100755 --- a/test/unit/server-ha-statefulset.bats +++ b/test/unit/server-ha-statefulset.bats @@ -349,19 +349,19 @@ load _helpers yq -r '.spec.template.spec.containers[0].env' | tee /dev/stderr) local actual=$(echo $object | - yq -r '.[6].name' | tee /dev/stderr) + yq -r '.[7].name' | tee /dev/stderr) [ "${actual}" = "FOO" ] local actual=$(echo $object | - yq -r '.[6].value' | tee /dev/stderr) + yq -r '.[7].value' | tee /dev/stderr) [ "${actual}" = "bar" ] local actual=$(echo $object | - yq -r '.[7].name' | tee /dev/stderr) + yq -r '.[8].name' | tee /dev/stderr) [ "${actual}" = "FOOBAR" ] local actual=$(echo $object | - yq -r '.[7].value' | tee /dev/stderr) + yq -r '.[8].value' | tee /dev/stderr) [ "${actual}" = "foobar" ] } @@ -383,23 +383,23 @@ load _helpers yq -r '.spec.template.spec.containers[0].env' | tee /dev/stderr) local actual=$(echo $object | - yq -r '.[6].name' | tee /dev/stderr) + yq -r '.[7].name' | tee /dev/stderr) [ "${actual}" = "ENV_FOO_0" ] local actual=$(echo $object | - yq -r '.[6].valueFrom.secretKeyRef.name' | tee /dev/stderr) + yq -r '.[7].valueFrom.secretKeyRef.name' | tee /dev/stderr) [ "${actual}" = "secret_name_0" ] local actual=$(echo $object | - yq -r '.[6].valueFrom.secretKeyRef.key' | tee /dev/stderr) + yq -r '.[7].valueFrom.secretKeyRef.key' | tee /dev/stderr) [ "${actual}" = "secret_key_0" ] local actual=$(echo $object | - yq -r '.[7].name' | tee /dev/stderr) + yq -r '.[8].name' | tee /dev/stderr) [ "${actual}" = "ENV_FOO_1" ] local actual=$(echo $object | - yq -r '.[7].valueFrom.secretKeyRef.name' | tee /dev/stderr) + yq -r '.[8].valueFrom.secretKeyRef.name' | tee /dev/stderr) [ "${actual}" = "secret_name_1" ] local actual=$(echo $object | - yq -r '.[7].valueFrom.secretKeyRef.key' | tee /dev/stderr) + yq -r '.[8].valueFrom.secretKeyRef.key' | tee /dev/stderr) [ "${actual}" = "secret_key_1" ] } diff --git a/test/unit/server-statefulset.bats b/test/unit/server-statefulset.bats index 1db272a..25d7798 100755 --- a/test/unit/server-statefulset.bats +++ b/test/unit/server-statefulset.bats @@ -384,19 +384,19 @@ load _helpers yq -r '.spec.template.spec.containers[0].env' | tee /dev/stderr) local actual=$(echo $object | - yq -r '.[6].name' | tee /dev/stderr) + yq -r '.[7].name' | tee /dev/stderr) [ "${actual}" = "FOO" ] local actual=$(echo $object | - yq -r '.[6].value' | tee /dev/stderr) + yq -r '.[7].value' | tee /dev/stderr) [ "${actual}" = "bar" ] local actual=$(echo $object | - yq -r '.[7].name' | tee /dev/stderr) + yq -r '.[8].name' | tee /dev/stderr) [ "${actual}" = "FOOBAR" ] local actual=$(echo $object | - yq -r '.[7].value' | tee /dev/stderr) + yq -r '.[8].value' | tee /dev/stderr) [ "${actual}" = "foobar" ] local object=$(helm template \ @@ -407,19 +407,19 @@ load _helpers yq -r '.spec.template.spec.containers[0].env' | tee /dev/stderr) local actual=$(echo $object | - yq -r '.[6].name' | tee /dev/stderr) + yq -r '.[7].name' | tee /dev/stderr) [ "${actual}" = "FOO" ] local actual=$(echo $object | - yq -r '.[6].value' | tee /dev/stderr) + yq -r '.[7].value' | tee /dev/stderr) [ "${actual}" = "bar" ] local actual=$(echo $object | - yq -r '.[7].name' | tee /dev/stderr) + yq -r '.[8].name' | tee /dev/stderr) [ "${actual}" = "FOOBAR" ] local actual=$(echo $object | - yq -r '.[7].value' | tee /dev/stderr) + yq -r '.[8].value' | tee /dev/stderr) [ "${actual}" = "foobar" ] } diff --git a/values.yaml b/values.yaml index e31e40f..50aa6b6 100644 --- a/values.yaml +++ b/values.yaml @@ -314,12 +314,35 @@ server: ha: enabled: false replicas: 3 + + # Enables Vault's integrated Raft storage. Unlike the typical HA modes where + # Vault's persistence is external (such as Consul), enabling Raft mode will create + # persistent volumes for Vault to store data. The Vault cluster will coordinate leader + # elections and failovers internally. + raft: + + # Enables Raft integrated storage + enabled: false + config: | + ui = true + cluster_addr = "https://POD_IP:8201" + + listener "tcp" { + tls_disable = 1 + address = "[::]:8200" + cluster_address = "[::]:8201" + } + + storage "raft" { + path = "/vault/data" + } # config is a raw string of default configuration when using a Stateful # deployment. Default is to use a Consul for its HA storage backend. # This should be HCL. config: | ui = true + cluster_addr = "https://POD_IP:8201" listener "tcp" { tls_disable = 1