Add annotation on config change (#1001)

When updating the Vault config (and corresponding)
configmap, we now generate a checksum of the config
and set it as an annotation on both the configmap
and the Vault StatefulSet pod template.

This allows the deployer to know what pods need to
be restarted to pick up the a changed config.

We still recommend using the standard upgrade
[method for Vault on Kubernetes](https://developer.hashicorp.com/vault/tutorials/kubernetes/kubernetes-raft-deployment-guide#upgrading-vault-on-kubernetes),
i.e., using the `OnDelete` strategy
for the Vault StatefulSet, so updating the config
and doing a `helm upgrade` should not trigger the
pods to restart, and then deleting pods one
at a time, starting with the standby pods.

With `kubectl` and `jq`, you can check check which
pods need to be updated by first getting the value
of the current configmap checksum:

```shell
kubectl get pods -o json | jq -r ".items[] | select(.metadata.annotations.\"config/checksum\" != $(kubectl get configmap vault-config -o json | jq '.metadata.annotations."config/checksum"') ) | .metadata.name"
```

Fixes #748.

---------

Co-authored-by: Tom Proctor <tomhjp@users.noreply.github.com>
This commit is contained in:
Christopher Swenson 2024-03-18 11:03:56 -07:00 committed by GitHub
parent 6930c378d2
commit d186b6ff29
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 95 additions and 23 deletions

View file

@ -457,9 +457,12 @@ Sets the injector deployment update strategy
{{/*
Sets extra pod annotations
*/}}
{{- define "vault.annotations" -}}
{{- if .Values.server.annotations }}
{{- define "vault.annotations" }}
annotations:
{{- if .Values.server.includeConfigAnnotation }}
vault.hashicorp.com/config-checksum: {{ include "vault.config" . | sha256sum }}
{{- end }}
{{- if .Values.server.annotations }}
{{- $tp := typeOf .Values.server.annotations }}
{{- if eq $tp "string" }}
{{- tpl .Values.server.annotations . | nindent 8 }}
@ -1075,3 +1078,28 @@ Supported inputs are Values.ui
{{- end -}}
{{- end }}
{{- end -}}
{{/*
config file from values
*/}}
{{- define "vault.config" -}}
{{- if or (eq .mode "ha") (eq .mode "standalone") }}
{{- $type := typeOf (index .Values.server .mode).config }}
{{- if eq $type "string" }}
disable_mlock = true
{{- if eq .mode "standalone" }}
{{ tpl .Values.server.standalone.config . | nindent 4 | trim }}
{{- else if and (eq .mode "ha") (eq (.Values.server.ha.raft.enabled | toString) "false") }}
{{ tpl .Values.server.ha.config . | nindent 4 | trim }}
{{- else if and (eq .mode "ha") (eq (.Values.server.ha.raft.enabled | toString) "true") }}
{{ tpl .Values.server.ha.raft.config . | nindent 4 | trim }}
{{ end }}
{{- else }}
{{- if and (eq .mode "ha") (eq (.Values.server.ha.raft.enabled | toString) "true") }}
{{ merge (dict "disable_mlock" true) (index .Values.server .mode).raft.config | toPrettyJson | indent 4 }}
{{- else }}
{{ merge (dict "disable_mlock" true) (index .Values.server .mode).config | toPrettyJson | indent 4 }}
{{- end }}
{{- end }}
{{- end }}
{{- end -}}

View file

@ -18,27 +18,13 @@ metadata:
app.kubernetes.io/name: {{ include "vault.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
{{- if .Values.server.includeConfigAnnotation }}
annotations:
vault.hashicorp.com/config-checksum: {{ include "vault.config" . | sha256sum }}
{{- end }}
data:
extraconfig-from-values.hcl: |-
{{- if or (eq .mode "ha") (eq .mode "standalone") }}
{{- $type := typeOf (index .Values.server .mode).config }}
{{- if eq $type "string" }}
disable_mlock = true
{{- if eq .mode "standalone" }}
{{ tpl .Values.server.standalone.config . | nindent 4 | trim }}
{{- else if and (eq .mode "ha") (eq (.Values.server.ha.raft.enabled | toString) "false") }}
{{ tpl .Values.server.ha.config . | nindent 4 | trim }}
{{- else if and (eq .mode "ha") (eq (.Values.server.ha.raft.enabled | toString) "true") }}
{{ tpl .Values.server.ha.raft.config . | nindent 4 | trim }}
{{ end }}
{{- else }}
{{- if and (eq .mode "ha") (eq (.Values.server.ha.raft.enabled | toString) "true") }}
{{ merge (dict "disable_mlock" true) (index .Values.server .mode).raft.config | toPrettyJson | indent 4 }}
{{- else }}
{{ merge (dict "disable_mlock" true) (index .Values.server .mode).config | toPrettyJson | indent 4 }}
{{- end }}
{{- end }}
{{- end }}
{{ template "vault.config" . }}
{{- end }}
{{- end }}
{{- end }}

View file

@ -28,7 +28,11 @@ RUN apk update && apk add --no-cache --virtual .build-deps \
jq
# yq
RUN pip install yq
RUN python3 -m venv venv && \
. venv/bin/activate && \
pip install yq && \
ln -s $PWD/venv/bin/yq /usr/local/bin/yq && \
deactivate
# gcloud
RUN curl -OL https://dl.google.com/dl/cloudsdk/channels/rapid/install_google_cloud_sdk.bash && \

View file

@ -139,3 +139,22 @@ load _helpers
yq 'length > 0' | tee /dev/stderr)
[ "${actual}" = "false" ]
}
@test "server/ConfigMap: config checksum annotation defaults to off" {
cd `chart_dir`
local actual=$(helm template \
--show-only templates/server-config-configmap.yaml \
. | tee /dev/stderr |
yq '.metadata.annotations["vault.hashicorp.com/config-checksum"] == null' | tee /dev/stderr)
[ "${actual}" = "true" ]
}
@test "server/ConfigMap: config checksum annotation can be enabled" {
cd `chart_dir`
local actual=$(helm template \
--show-only templates/server-config-configmap.yaml \
--set 'server.includeConfigAnnotation=true' \
. | tee /dev/stderr |
yq '.metadata.annotations["vault.hashicorp.com/config-checksum"] == null' | tee /dev/stderr)
[ "${actual}" = "false" ]
}

View file

@ -1681,6 +1681,34 @@ load _helpers
[ "${actual}" = "true" ]
}
@test "server/standalone-StatefulSet: config checksum annotation defaults to off" {
cd `chart_dir`
local actual=$(helm template \
--show-only templates/server-statefulset.yaml \
. | tee /dev/stderr |
yq '.spec.template.metadata.annotations["vault.hashicorp.com/config-checksum"] == null' | tee /dev/stderr)
[ "${actual}" = "true" ]
}
@test "server/standalone-StatefulSet: config checksum annotation off does not set annotations" {
cd `chart_dir`
local actual=$(helm template \
--show-only templates/server-statefulset.yaml \
. | tee /dev/stderr |
yq '.spec.template.metadata.annotations | length == 0' | tee /dev/stderr)
[ "${actual}" = "true" ]
}
@test "server/standalone-StatefulSet: config checksum annotation can be enabled" {
cd `chart_dir`
local actual=$(helm template \
--show-only templates/server-statefulset.yaml \
--set 'server.includeConfigAnnotation=true' \
. | tee /dev/stderr |
yq '.spec.template.metadata.annotations["vault.hashicorp.com/config-checksum"] == null' | tee /dev/stderr)
[ "${actual}" = "false" ]
}
#--------------------------------------------------------------------
# priorityClassName

View file

@ -668,6 +668,13 @@ server:
# of the annotations to apply to the server pods
annotations: {}
# Add an annotation to the server configmap and the statefulset pods,
# vaultproject.io/config-checksum, that is a hash of the Vault configuration.
# This can be used together with an OnDelete deployment strategy to help
# identify which pods still need to be deleted during a deployment to pick up
# any configuration changes.
configAnnotation: false
# Enables a headless service to be used by the Vault Statefulset
service:
enabled: true
@ -714,7 +721,7 @@ server:
# PreferDualStack: Allocates IPv4 and IPv6 cluster IPs for the Service.
# RequireDualStack: Allocates Service .spec.ClusterIPs from both IPv4 and IPv6 address ranges.
ipFamilyPolicy: ""
# Sets the families that should be supported and the order in which they should be applied to ClusterIP as well.
# Can be IPv4 and/or IPv6.
ipFamilies: []