Add annotation on config change (#1001)

When updating the Vault config (and corresponding)
configmap, we now generate a checksum of the config
and set it as an annotation on both the configmap
and the Vault StatefulSet pod template.

This allows the deployer to know what pods need to
be restarted to pick up the a changed config.

We still recommend using the standard upgrade
[method for Vault on Kubernetes](https://developer.hashicorp.com/vault/tutorials/kubernetes/kubernetes-raft-deployment-guide#upgrading-vault-on-kubernetes),
i.e., using the `OnDelete` strategy
for the Vault StatefulSet, so updating the config
and doing a `helm upgrade` should not trigger the
pods to restart, and then deleting pods one
at a time, starting with the standby pods.

With `kubectl` and `jq`, you can check check which
pods need to be updated by first getting the value
of the current configmap checksum:

```shell
kubectl get pods -o json | jq -r ".items[] | select(.metadata.annotations.\"config/checksum\" != $(kubectl get configmap vault-config -o json | jq '.metadata.annotations."config/checksum"') ) | .metadata.name"
```

Fixes #748.

---------

Co-authored-by: Tom Proctor <tomhjp@users.noreply.github.com>

templates/_helpers.tpl

@@ -457,9 +457,12 @@ Sets the injector deployment update strategy
 {{/*
 Sets extra pod annotations
 */}}
-{{- define "vault.annotations" -}}
+{{- define "vault.annotations" }}
-  {{- if .Values.server.annotations }}
       annotations:
+  {{- if .Values.server.includeConfigAnnotation }}
+        vault.hashicorp.com/config-checksum: {{ include "vault.config" . | sha256sum }}
+  {{- end }}
+  {{- if .Values.server.annotations }}
         {{- $tp := typeOf .Values.server.annotations }}
         {{- if eq $tp "string" }}
           {{- tpl .Values.server.annotations . | nindent 8 }}
@@ -1075,3 +1078,28 @@ Supported inputs are Values.ui
 {{- end -}}
 {{- end }}
 {{- end -}}
+
+{{/*
+config file from values
+*/}}
+{{- define "vault.config" -}}
+  {{- if or (eq .mode "ha") (eq .mode "standalone") }}
+  {{- $type := typeOf (index .Values.server .mode).config }}
+  {{- if eq $type "string" }}
+    disable_mlock = true
+  {{- if eq .mode "standalone" }}
+    {{ tpl .Values.server.standalone.config . | nindent 4 | trim }}
+  {{- else if and (eq .mode "ha") (eq (.Values.server.ha.raft.enabled | toString) "false") }}
+    {{ tpl .Values.server.ha.config . | nindent 4 | trim }}
+  {{- else if and (eq .mode "ha") (eq (.Values.server.ha.raft.enabled | toString) "true") }}
+    {{ tpl .Values.server.ha.raft.config . | nindent 4 | trim }}
+  {{ end }}
+  {{- else }}
+  {{- if and (eq .mode "ha") (eq (.Values.server.ha.raft.enabled | toString) "true") }}
+{{ merge (dict "disable_mlock" true) (index .Values.server .mode).raft.config | toPrettyJson | indent 4 }}
+  {{- else }}
+{{ merge (dict "disable_mlock" true) (index .Values.server .mode).config | toPrettyJson | indent 4 }}
+  {{- end }}
+  {{- end }}
+  {{- end }}
+{{- end -}}

templates/server-config-configmap.yaml

@@ -18,27 +18,13 @@ metadata:
     app.kubernetes.io/name: {{ include "vault.name" . }}
     app.kubernetes.io/instance: {{ .Release.Name }}
     app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- if .Values.server.includeConfigAnnotation }}
+  annotations:
+    vault.hashicorp.com/config-checksum: {{ include "vault.config" . | sha256sum }}
+{{- end }}
 data:
   extraconfig-from-values.hcl: |-
-  {{- if or (eq .mode "ha") (eq .mode "standalone") }}
+    {{ template "vault.config" . }}
-  {{- $type := typeOf (index .Values.server .mode).config }}
-  {{- if eq $type "string" }}
-    disable_mlock = true
-  {{- if eq .mode "standalone" }}
-    {{ tpl .Values.server.standalone.config . | nindent 4 | trim }}
-  {{- else if and (eq .mode "ha") (eq (.Values.server.ha.raft.enabled | toString) "false") }}
-    {{ tpl .Values.server.ha.config . | nindent 4 | trim }}
-  {{- else if and (eq .mode "ha") (eq (.Values.server.ha.raft.enabled | toString) "true") }}
-    {{ tpl .Values.server.ha.raft.config . | nindent 4 | trim }}
-  {{ end }}
-  {{- else }}
-  {{- if and (eq .mode "ha") (eq (.Values.server.ha.raft.enabled | toString) "true") }}
-{{ merge (dict "disable_mlock" true) (index .Values.server .mode).raft.config | toPrettyJson | indent 4 }}
-  {{- else }}
-{{ merge (dict "disable_mlock" true) (index .Values.server .mode).config | toPrettyJson | indent 4 }}
-  {{- end }}
-  {{- end }}
-  {{- end }}
 {{- end }}
 {{- end }}
 {{- end }}

test/docker/Test.dockerfile

@@ -28,7 +28,11 @@ RUN apk update && apk add --no-cache --virtual .build-deps \
     jq
 
 # yq
-RUN pip install yq
+RUN python3 -m venv venv && \
+    . venv/bin/activate && \
+    pip install yq && \
+    ln -s $PWD/venv/bin/yq /usr/local/bin/yq && \
+    deactivate
 
 # gcloud
 RUN curl -OL https://dl.google.com/dl/cloudsdk/channels/rapid/install_google_cloud_sdk.bash && \

test/unit/server-configmap.bats

@@ -139,3 +139,22 @@ load _helpers
       yq 'length > 0' | tee /dev/stderr)
   [ "${actual}" = "false" ]
 }
+
+@test "server/ConfigMap: config checksum annotation defaults to off" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      --show-only templates/server-config-configmap.yaml \
+      . | tee /dev/stderr |
+      yq '.metadata.annotations["vault.hashicorp.com/config-checksum"] == null' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+}
+
+@test "server/ConfigMap: config checksum annotation can be enabled" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      --show-only templates/server-config-configmap.yaml \
+      --set 'server.includeConfigAnnotation=true' \
+      . | tee /dev/stderr |
+      yq '.metadata.annotations["vault.hashicorp.com/config-checksum"] == null' | tee /dev/stderr)
+  [ "${actual}" = "false" ]
+}

test/unit/server-statefulset.bats

@@ -1681,6 +1681,34 @@ load _helpers
   [ "${actual}" = "true" ]
 }
 
+@test "server/standalone-StatefulSet: config checksum annotation defaults to off" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      --show-only templates/server-statefulset.yaml \
+      . | tee /dev/stderr |
+      yq '.spec.template.metadata.annotations["vault.hashicorp.com/config-checksum"] == null' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+}
+
+@test "server/standalone-StatefulSet: config checksum annotation off does not set annotations" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      --show-only templates/server-statefulset.yaml \
+      . | tee /dev/stderr |
+      yq '.spec.template.metadata.annotations | length == 0' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+}
+
+@test "server/standalone-StatefulSet: config checksum annotation can be enabled" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      --show-only templates/server-statefulset.yaml \
+      --set 'server.includeConfigAnnotation=true' \
+      . | tee /dev/stderr |
+      yq '.spec.template.metadata.annotations["vault.hashicorp.com/config-checksum"] == null' | tee /dev/stderr)
+  [ "${actual}" = "false" ]
+}
+
 #--------------------------------------------------------------------
 # priorityClassName
 

values.yaml

@@ -668,6 +668,13 @@ server:
   # of the annotations to apply to the server pods
   annotations: {}
 
+  # Add an annotation to the server configmap and the statefulset pods,
+  # vaultproject.io/config-checksum, that is a hash of the Vault configuration.
+  # This can be used together with an OnDelete deployment strategy to help
+  # identify which pods still need to be deleted during a deployment to pick up
+  # any configuration changes.
+  configAnnotation: false
+
   # Enables a headless service to be used by the Vault Statefulset
   service:
     enabled: true
@@ -714,7 +721,7 @@ server:
     #     PreferDualStack: Allocates IPv4 and IPv6 cluster IPs for the Service.
     #     RequireDualStack: Allocates Service .spec.ClusterIPs from both IPv4 and IPv6 address ranges.
     ipFamilyPolicy: ""
-    
+
     # Sets the families that should be supported and the order in which they should be applied to ClusterIP as well.
     # Can be IPv4 and/or IPv6.
     ipFamilies: []