From d186b6ff298b071de9217ce9d1c0ed85eb53b097 Mon Sep 17 00:00:00 2001 From: Christopher Swenson Date: Mon, 18 Mar 2024 11:03:56 -0700 Subject: [PATCH] Add annotation on config change (#1001) When updating the Vault config (and corresponding) configmap, we now generate a checksum of the config and set it as an annotation on both the configmap and the Vault StatefulSet pod template. This allows the deployer to know what pods need to be restarted to pick up the a changed config. We still recommend using the standard upgrade [method for Vault on Kubernetes](https://developer.hashicorp.com/vault/tutorials/kubernetes/kubernetes-raft-deployment-guide#upgrading-vault-on-kubernetes), i.e., using the `OnDelete` strategy for the Vault StatefulSet, so updating the config and doing a `helm upgrade` should not trigger the pods to restart, and then deleting pods one at a time, starting with the standby pods. With `kubectl` and `jq`, you can check check which pods need to be updated by first getting the value of the current configmap checksum: ```shell kubectl get pods -o json | jq -r ".items[] | select(.metadata.annotations.\"config/checksum\" != $(kubectl get configmap vault-config -o json | jq '.metadata.annotations."config/checksum"') ) | .metadata.name" ``` Fixes #748. --------- Co-authored-by: Tom Proctor --- templates/_helpers.tpl | 32 ++++++++++++++++++++++++-- templates/server-config-configmap.yaml | 24 ++++--------------- test/docker/Test.dockerfile | 6 ++++- test/unit/server-configmap.bats | 19 +++++++++++++++ test/unit/server-statefulset.bats | 28 ++++++++++++++++++++++ values.yaml | 9 +++++++- 6 files changed, 95 insertions(+), 23 deletions(-) diff --git a/templates/_helpers.tpl b/templates/_helpers.tpl index 8f77f92..7a22d04 100644 --- a/templates/_helpers.tpl +++ b/templates/_helpers.tpl @@ -457,9 +457,12 @@ Sets the injector deployment update strategy {{/* Sets extra pod annotations */}} -{{- define "vault.annotations" -}} - {{- if .Values.server.annotations }} +{{- define "vault.annotations" }} annotations: + {{- if .Values.server.includeConfigAnnotation }} + vault.hashicorp.com/config-checksum: {{ include "vault.config" . | sha256sum }} + {{- end }} + {{- if .Values.server.annotations }} {{- $tp := typeOf .Values.server.annotations }} {{- if eq $tp "string" }} {{- tpl .Values.server.annotations . | nindent 8 }} @@ -1075,3 +1078,28 @@ Supported inputs are Values.ui {{- end -}} {{- end }} {{- end -}} + +{{/* +config file from values +*/}} +{{- define "vault.config" -}} + {{- if or (eq .mode "ha") (eq .mode "standalone") }} + {{- $type := typeOf (index .Values.server .mode).config }} + {{- if eq $type "string" }} + disable_mlock = true + {{- if eq .mode "standalone" }} + {{ tpl .Values.server.standalone.config . | nindent 4 | trim }} + {{- else if and (eq .mode "ha") (eq (.Values.server.ha.raft.enabled | toString) "false") }} + {{ tpl .Values.server.ha.config . | nindent 4 | trim }} + {{- else if and (eq .mode "ha") (eq (.Values.server.ha.raft.enabled | toString) "true") }} + {{ tpl .Values.server.ha.raft.config . | nindent 4 | trim }} + {{ end }} + {{- else }} + {{- if and (eq .mode "ha") (eq (.Values.server.ha.raft.enabled | toString) "true") }} +{{ merge (dict "disable_mlock" true) (index .Values.server .mode).raft.config | toPrettyJson | indent 4 }} + {{- else }} +{{ merge (dict "disable_mlock" true) (index .Values.server .mode).config | toPrettyJson | indent 4 }} + {{- end }} + {{- end }} + {{- end }} +{{- end -}} \ No newline at end of file diff --git a/templates/server-config-configmap.yaml b/templates/server-config-configmap.yaml index 5c66057..1fed2e6 100644 --- a/templates/server-config-configmap.yaml +++ b/templates/server-config-configmap.yaml @@ -18,27 +18,13 @@ metadata: app.kubernetes.io/name: {{ include "vault.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- if .Values.server.includeConfigAnnotation }} + annotations: + vault.hashicorp.com/config-checksum: {{ include "vault.config" . | sha256sum }} +{{- end }} data: extraconfig-from-values.hcl: |- - {{- if or (eq .mode "ha") (eq .mode "standalone") }} - {{- $type := typeOf (index .Values.server .mode).config }} - {{- if eq $type "string" }} - disable_mlock = true - {{- if eq .mode "standalone" }} - {{ tpl .Values.server.standalone.config . | nindent 4 | trim }} - {{- else if and (eq .mode "ha") (eq (.Values.server.ha.raft.enabled | toString) "false") }} - {{ tpl .Values.server.ha.config . | nindent 4 | trim }} - {{- else if and (eq .mode "ha") (eq (.Values.server.ha.raft.enabled | toString) "true") }} - {{ tpl .Values.server.ha.raft.config . | nindent 4 | trim }} - {{ end }} - {{- else }} - {{- if and (eq .mode "ha") (eq (.Values.server.ha.raft.enabled | toString) "true") }} -{{ merge (dict "disable_mlock" true) (index .Values.server .mode).raft.config | toPrettyJson | indent 4 }} - {{- else }} -{{ merge (dict "disable_mlock" true) (index .Values.server .mode).config | toPrettyJson | indent 4 }} - {{- end }} - {{- end }} - {{- end }} + {{ template "vault.config" . }} {{- end }} {{- end }} {{- end }} diff --git a/test/docker/Test.dockerfile b/test/docker/Test.dockerfile index 472a97a..69baa47 100644 --- a/test/docker/Test.dockerfile +++ b/test/docker/Test.dockerfile @@ -28,7 +28,11 @@ RUN apk update && apk add --no-cache --virtual .build-deps \ jq # yq -RUN pip install yq +RUN python3 -m venv venv && \ + . venv/bin/activate && \ + pip install yq && \ + ln -s $PWD/venv/bin/yq /usr/local/bin/yq && \ + deactivate # gcloud RUN curl -OL https://dl.google.com/dl/cloudsdk/channels/rapid/install_google_cloud_sdk.bash && \ diff --git a/test/unit/server-configmap.bats b/test/unit/server-configmap.bats index eea7e70..dcb9076 100755 --- a/test/unit/server-configmap.bats +++ b/test/unit/server-configmap.bats @@ -139,3 +139,22 @@ load _helpers yq 'length > 0' | tee /dev/stderr) [ "${actual}" = "false" ] } + +@test "server/ConfigMap: config checksum annotation defaults to off" { + cd `chart_dir` + local actual=$(helm template \ + --show-only templates/server-config-configmap.yaml \ + . | tee /dev/stderr | + yq '.metadata.annotations["vault.hashicorp.com/config-checksum"] == null' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + +@test "server/ConfigMap: config checksum annotation can be enabled" { + cd `chart_dir` + local actual=$(helm template \ + --show-only templates/server-config-configmap.yaml \ + --set 'server.includeConfigAnnotation=true' \ + . | tee /dev/stderr | + yq '.metadata.annotations["vault.hashicorp.com/config-checksum"] == null' | tee /dev/stderr) + [ "${actual}" = "false" ] +} diff --git a/test/unit/server-statefulset.bats b/test/unit/server-statefulset.bats index 8acd9ee..b2b8efe 100755 --- a/test/unit/server-statefulset.bats +++ b/test/unit/server-statefulset.bats @@ -1681,6 +1681,34 @@ load _helpers [ "${actual}" = "true" ] } +@test "server/standalone-StatefulSet: config checksum annotation defaults to off" { + cd `chart_dir` + local actual=$(helm template \ + --show-only templates/server-statefulset.yaml \ + . | tee /dev/stderr | + yq '.spec.template.metadata.annotations["vault.hashicorp.com/config-checksum"] == null' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + +@test "server/standalone-StatefulSet: config checksum annotation off does not set annotations" { + cd `chart_dir` + local actual=$(helm template \ + --show-only templates/server-statefulset.yaml \ + . | tee /dev/stderr | + yq '.spec.template.metadata.annotations | length == 0' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + +@test "server/standalone-StatefulSet: config checksum annotation can be enabled" { + cd `chart_dir` + local actual=$(helm template \ + --show-only templates/server-statefulset.yaml \ + --set 'server.includeConfigAnnotation=true' \ + . | tee /dev/stderr | + yq '.spec.template.metadata.annotations["vault.hashicorp.com/config-checksum"] == null' | tee /dev/stderr) + [ "${actual}" = "false" ] +} + #-------------------------------------------------------------------- # priorityClassName diff --git a/values.yaml b/values.yaml index 17f5ca5..a57b9de 100644 --- a/values.yaml +++ b/values.yaml @@ -668,6 +668,13 @@ server: # of the annotations to apply to the server pods annotations: {} + # Add an annotation to the server configmap and the statefulset pods, + # vaultproject.io/config-checksum, that is a hash of the Vault configuration. + # This can be used together with an OnDelete deployment strategy to help + # identify which pods still need to be deleted during a deployment to pick up + # any configuration changes. + configAnnotation: false + # Enables a headless service to be used by the Vault Statefulset service: enabled: true @@ -714,7 +721,7 @@ server: # PreferDualStack: Allocates IPv4 and IPv6 cluster IPs for the Service. # RequireDualStack: Allocates Service .spec.ClusterIPs from both IPv4 and IPv6 address ranges. ipFamilyPolicy: "" - + # Sets the families that should be supported and the order in which they should be applied to ClusterIP as well. # Can be IPv4 and/or IPv6. ipFamilies: []