Support deploying multiple injector replicas with auto-TLS (#436)

templates/injector-certs-secret.yaml

@@ -0,0 +1,10 @@
+{{- if and (eq (.Values.injector.enabled | toString) "true" ) (eq (.Values.global.enabled | toString) "true") (eq (.Values.injector.leaderElector.enabled | toString) "true") (gt (.Values.injector.replicas | int) 1) }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: vault-injector-certs
+  labels:
+    app.kubernetes.io/name: {{ include "vault.name" . }}-agent-injector
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}

templates/injector-deployment.yaml

@@ -11,7 +11,7 @@ metadata:
     app.kubernetes.io/managed-by: {{ .Release.Service }}
     component: webhook
 spec:
-  replicas: 1
+  replicas: {{ .Values.injector.replicas }}
   selector:
     matchLabels:
       app.kubernetes.io/name: {{ template "vault.name" . }}-agent-injector
@@ -88,6 +88,14 @@ spec:
             - name: AGENT_INJECT_TELEMETRY_PATH
               value: "/metrics"
             {{- end }}
+            {{- if and (eq (.Values.injector.leaderElector.enabled | toString) "true") (gt (.Values.injector.replicas | int) 1) }}
+            - name: AGENT_INJECT_USE_LEADER_ELECTOR
+              value: "true"
+            - name: NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            {{- end }}
             {{- include "vault.extraEnvironmentVars" .Values.injector | nindent 12 }}
           args:
             - agent-inject
@@ -112,6 +120,35 @@ spec:
             periodSeconds: 2
             successThreshold: 1
             timeoutSeconds: 5
+        {{- if and (eq (.Values.injector.leaderElector.enabled | toString) "true") (gt (.Values.injector.replicas | int) 1) }}
+        - name: leader-elector
+          image: {{ .Values.injector.leaderElector.image.repository }}:{{ .Values.injector.leaderElector.image.tag }}
+          args:
+            - --election={{ template "vault.fullname" . }}-agent-injector-leader
+            - --election-namespace={{ .Release.Namespace }}
+            - --http=0.0.0.0:4040
+            - --ttl={{ .Values.injector.leaderElector.ttl }}
+          livenessProbe:
+            httpGet:
+              path: /
+              port: 4040
+              scheme: HTTP
+            failureThreshold: 2
+            initialDelaySeconds: 1
+            periodSeconds: 2
+            successThreshold: 1
+            timeoutSeconds: 5
+          readinessProbe:
+            httpGet:
+              path: /
+              port: 4040
+              scheme: HTTP
+            failureThreshold: 2
+            initialDelaySeconds: 2
+            periodSeconds: 2
+            successThreshold: 1
+            timeoutSeconds: 5
+        {{- end }}
 {{- if .Values.injector.certs.secretName }}
           volumeMounts:
             - name: webhook-certs

templates/injector-leader-endpoint.yaml

@@ -0,0 +1,12 @@
+{{- if and (eq (.Values.injector.enabled | toString) "true" ) (eq (.Values.global.enabled | toString) "true") (eq (.Values.injector.leaderElector.enabled | toString) "true") (gt (.Values.injector.replicas | int) 1) }}
+# This is created here so it can be cleaned up easily, since if
+# the endpoint is left around the leader won't expire for about a minute.
+apiVersion: v1
+kind: Endpoints
+metadata:
+  name: {{ template "vault.fullname" . }}-agent-injector-leader
+  labels:
+    app.kubernetes.io/name: {{ include "vault.name" . }}-agent-injector
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}

templates/injector-role.yaml

@@ -0,0 +1,19 @@
+{{- if and (eq (.Values.injector.enabled | toString) "true" ) (eq (.Values.global.enabled | toString) "true") (eq (.Values.injector.leaderElector.enabled | toString) "true") (gt (.Values.injector.replicas | int) 1) }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ template "vault.fullname" . }}-agent-injector-leader-elector-role
+  labels:
+    app.kubernetes.io/name: {{ include "vault.name" . }}-agent-injector
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+rules:
+  - apiGroups: [""]
+    resources: ["endpoints", "secrets"]
+    verbs:
+      - "create"
+      - "get"
+      - "watch"
+      - "list"
+      - "update"
+{{- end }}

templates/injector-rolebinding.yaml

@@ -0,0 +1,18 @@
+{{- if and (eq (.Values.injector.enabled | toString) "true" ) (eq (.Values.global.enabled | toString) "true") (eq (.Values.injector.leaderElector.enabled | toString) "true") (gt (.Values.injector.replicas | int) 1) }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ template "vault.fullname" . }}-agent-injector-leader-elector-binding
+  labels:
+    app.kubernetes.io/name: {{ include "vault.name" . }}-agent-injector
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ template "vault.fullname" . }}-agent-injector-leader-elector-role
+subjects:
+  - kind: ServiceAccount
+    name: {{ template "vault.fullname" . }}-agent-injector
+    namespace: {{ .Release.Namespace }}
+{{- end }}

test/acceptance/injector-leader-elector.bats

@@ -0,0 +1,46 @@
+#!/usr/bin/env bats
+
+load _helpers
+
+@test "injector: testing leader elector" {
+  cd `chart_dir`
+  
+  kubectl delete namespace acceptance --ignore-not-found=true
+  kubectl create namespace acceptance
+  kubectl config set-context --current --namespace=acceptance
+
+  helm install "$(name_prefix)" \
+    --set="injector.replicas=3" .
+  kubectl wait --for condition=Ready pod -l app.kubernetes.io/name=vault-agent-injector
+
+  pods=($(kubectl get pods -l app.kubernetes.io/name=vault-agent-injector -o json | jq -r '.items[] | .metadata.name'))
+  [ "${#pods[@]}" == 3 ]
+
+  leader="$(echo "$(kubectl exec ${pods[0]} -c sidecar-injector -- wget --quiet --output-document - localhost:4040)" | jq -r .name)"
+  # Check the leader name is valid - i.e. one of the 3 pods
+  [[ " ${pods[@]} " =~ " ${leader} " ]]
+
+  # Check every pod agrees on who the leader is
+  for pod in "${pods[@]}"
+  do
+    pod_leader="$(echo "$(kubectl exec $pod -c sidecar-injector -- wget --quiet --output-document - localhost:4040)" | jq -r .name)"
+    [ "${pod_leader}" == "${leader}" ]
+  done
+}
+
+setup() {
+  kubectl delete namespace acceptance --ignore-not-found=true
+  kubectl create namespace acceptance
+  kubectl config set-context --current --namespace=acceptance
+}
+
+# Clean up
+teardown() {
+  if [[ ${CLEANUP:-true} == "true" ]]
+  then
+      echo "helm/pvc teardown"
+      helm delete vault
+      kubectl delete --all pvc
+      kubectl delete namespace acceptance
+  fi
+}

test/unit/injector-deployment.bats

@@ -425,13 +425,13 @@ load _helpers
 #--------------------------------------------------------------------
 # affinity
 
-@test "injector/deployment: affinity not set by default" {
+@test "injector/deployment: affinity set by default" {
   cd `chart_dir`
   local actual=$(helm template \
       --show-only templates/injector-deployment.yaml  \
       . | tee /dev/stderr |
       yq '.spec.template.spec | .affinity? == null' | tee /dev/stderr)
-  [ "${actual}" = "true" ]
+  [ "${actual}" = "false" ]
 }
 
 @test "injector/deployment: affinity can be set" {

test/unit/injector-leader-elector.bats

@@ -0,0 +1,264 @@
+#!/usr/bin/env bats
+
+load _helpers
+
+#--------------------------------------------------------------------
+# Deployment
+
+@test "injector/deployment: leader elector replica count" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      --show-only templates/injector-deployment.yaml \
+      --set "injector.replicas=2" \
+      . | tee /dev/stderr |
+      yq '.spec.replicas' | tee /dev/stderr)
+  [ "${actual}" = "2" ]
+}
+
+@test "injector/deployment: leader elector - sidecar is created only when enabled" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      --show-only templates/injector-deployment.yaml \
+      . | tee /dev/stderr |
+      yq '.spec.template.spec.containers | length' | tee /dev/stderr)
+  [ "${actual}" = "1" ]
+
+  local actual=$(helm template \
+      --show-only templates/injector-deployment.yaml \
+      --set "injector.replicas=2" \
+      --set "injector.leaderElector.enabled=false" \
+      . | tee /dev/stderr |
+      yq '.spec.template.spec.containers | length' | tee /dev/stderr)
+  [ "${actual}" = "1" ]
+
+  local actual=$(helm template \
+      --show-only templates/injector-deployment.yaml \
+      --set "injector.replicas=2" \
+      . | tee /dev/stderr |
+      yq '.spec.template.spec.containers | length' | tee /dev/stderr)
+  [ "${actual}" = "2" ]
+}
+
+@test "injector/deployment: leader elector image name is configurable" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      --show-only templates/injector-deployment.yaml \
+      --set "injector.replicas=2" \
+      --set "injector.leaderElector.image.repository=SomeOtherImage" \
+      --set "injector.leaderElector.image.tag=SomeOtherTag" \
+      . | tee /dev/stderr |
+      yq -r '.spec.template.spec.containers[1].image' | tee /dev/stderr)
+  [ "${actual}" = "SomeOtherImage:SomeOtherTag" ]
+}
+
+@test "injector/deployment: leader elector configuration for sidecar-injector" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      --show-only templates/injector-deployment.yaml \
+      . | tee /dev/stderr |
+      yq -r '.spec.template.spec.containers[0].env[] | select(.name == "AGENT_INJECT_USE_LEADER_ELECTOR") | .value' | tee /dev/stderr)
+  [ "${actual}" = "" ]
+
+  local actual=$(helm template \
+      --show-only templates/injector-deployment.yaml \
+      --set "injector.replicas=2" \
+      . | tee /dev/stderr |
+      yq -r '.spec.template.spec.containers[0].env[] | select(.name == "AGENT_INJECT_USE_LEADER_ELECTOR") | .value' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+
+  local actual=$(helm template \
+      --show-only templates/injector-deployment.yaml \
+      . | tee /dev/stderr |
+      yq -r '.spec.template.spec.containers[0].env[] | select(.name == "NAMESPACE") | .valueFrom.fieldRef.fieldPath' | tee /dev/stderr)
+  [ "${actual}" = "" ]
+
+  local actual=$(helm template \
+      --show-only templates/injector-deployment.yaml \
+      --set "injector.replicas=2" \
+      . | tee /dev/stderr |
+      yq -r '.spec.template.spec.containers[0].env[] | select(.name == "NAMESPACE") | .valueFrom.fieldRef.fieldPath' | tee /dev/stderr)
+  [ "${actual}" = "metadata.namespace" ]
+}
+
+@test "injector/deployment: leader elector TTL is configurable" {
+  cd `chart_dir`
+  # Default value 60s
+  local actual=$(helm template \
+      --show-only templates/injector-deployment.yaml \
+      --set "injector.replicas=2" \
+      . | tee /dev/stderr |
+      yq -r '.spec.template.spec.containers[1].args[3]' | tee /dev/stderr)
+  [ "${actual}" = "--ttl=60s" ]
+
+  # Configured to 30s
+  local actual=$(helm template \
+      --show-only templates/injector-deployment.yaml \
+      --set "injector.replicas=2" \
+      --set "injector.leaderElector.ttl=30s" \
+      . | tee /dev/stderr |
+      yq -r '.spec.template.spec.containers[1].args[3]' | tee /dev/stderr)
+  [ "${actual}" = "--ttl=30s" ]
+}
+
+#--------------------------------------------------------------------
+# Resource creation
+
+@test "injector/certs-secret: created/skipped as appropriate" {
+  cd `chart_dir`
+  local actual=$( (helm template \
+      --show-only templates/injector-certs-secret.yaml \
+      . || echo "---") | tee /dev/stderr |
+      yq 'length > 0' | tee /dev/stderr)
+  [ "${actual}" = "false" ]
+
+  local actual=$( (helm template \
+      --show-only templates/injector-certs-secret.yaml \
+      --set "injector.replicas=2" \
+      --set "global.enabled=false" \
+      . || echo "---") | tee /dev/stderr |
+      yq 'length > 0' | tee /dev/stderr)
+  [ "${actual}" = "false" ]
+
+  local actual=$( (helm template \
+      --show-only templates/injector-certs-secret.yaml \
+      --set "injector.replicas=2" \
+      --set "injector.enabled=false" \
+      . || echo "---") | tee /dev/stderr |
+      yq 'length > 0' | tee /dev/stderr)
+  [ "${actual}" = "false" ]
+
+  local actual=$( (helm template \
+      --show-only templates/injector-certs-secret.yaml \
+      --set "injector.replicas=2" \
+      --set "injector.leaderElector.enabled=false" \
+      . || echo "---") | tee /dev/stderr |
+      yq 'length > 0' | tee /dev/stderr)
+  [ "${actual}" = "false" ]
+
+  local actual=$( (helm template \
+      --show-only templates/injector-certs-secret.yaml \
+      --set "injector.replicas=2" \
+      . || echo "---") | tee /dev/stderr |
+      yq 'length > 0' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+}
+
+@test "injector/leader-endpoint: created/skipped as appropriate" {
+  cd `chart_dir`
+  local actual=$( (helm template \
+      --show-only templates/injector-leader-endpoint.yaml \
+      . || echo "---") | tee /dev/stderr |
+      yq 'length > 0' | tee /dev/stderr)
+  [ "${actual}" = "false" ]
+
+  local actual=$( (helm template \
+      --show-only templates/injector-leader-endpoint.yaml \
+      --set "injector.replicas=2" \
+      --set "global.enabled=false" \
+      . || echo "---") | tee /dev/stderr |
+      yq 'length > 0' | tee /dev/stderr)
+  [ "${actual}" = "false" ]
+
+  local actual=$( (helm template \
+      --show-only templates/injector-leader-endpoint.yaml \
+      --set "injector.replicas=2" \
+      --set "injector.enabled=false" \
+      . || echo "---") | tee /dev/stderr |
+      yq 'length > 0' | tee /dev/stderr)
+  [ "${actual}" = "false" ]
+
+  local actual=$( (helm template \
+      --show-only templates/injector-leader-endpoint.yaml \
+      --set "injector.replicas=2" \
+      --set "injector.leaderElector.enabled=false" \
+      . || echo "---") | tee /dev/stderr |
+      yq 'length > 0' | tee /dev/stderr)
+  [ "${actual}" = "false" ]
+
+  local actual=$( (helm template \
+      --show-only templates/injector-leader-endpoint.yaml \
+      --set "injector.replicas=2" \
+      . || echo "---") | tee /dev/stderr |
+      yq 'length > 0' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+}
+
+@test "injector/role: created/skipped as appropriate" {
+  cd `chart_dir`
+  local actual=$( (helm template \
+      --show-only templates/injector-role.yaml \
+      . || echo "---") | tee /dev/stderr |
+      yq 'length > 0' | tee /dev/stderr)
+  [ "${actual}" = "false" ]
+
+  local actual=$( (helm template \
+      --show-only templates/injector-role.yaml \
+      --set "injector.replicas=2" \
+      --set "global.enabled=false" \
+      . || echo "---") | tee /dev/stderr |
+      yq 'length > 0' | tee /dev/stderr)
+  [ "${actual}" = "false" ]
+
+  local actual=$( (helm template \
+      --show-only templates/injector-role.yaml \
+      --set "injector.replicas=2" \
+      --set "injector.enabled=false" \
+      . || echo "---") | tee /dev/stderr |
+      yq 'length > 0' | tee /dev/stderr)
+  [ "${actual}" = "false" ]
+
+  local actual=$( (helm template \
+      --show-only templates/injector-role.yaml \
+      --set "injector.replicas=2" \
+      --set "injector.leaderElector.enabled=false" \
+      . || echo "---") | tee /dev/stderr |
+      yq 'length > 0' | tee /dev/stderr)
+  [ "${actual}" = "false" ]
+
+  local actual=$( (helm template \
+      --show-only templates/injector-role.yaml \
+      --set "injector.replicas=2" \
+      . || echo "---") | tee /dev/stderr |
+      yq 'length > 0' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+}
+
+@test "injector/rolebinding: created/skipped as appropriate" {
+  cd `chart_dir`
+  local actual=$( (helm template \
+      --show-only templates/injector-rolebinding.yaml \
+      . || echo "---") | tee /dev/stderr |
+      yq 'length > 0' | tee /dev/stderr)
+  [ "${actual}" = "false" ]
+
+  local actual=$( (helm template \
+      --show-only templates/injector-rolebinding.yaml \
+      --set "injector.replicas=2" \
+      --set "global.enabled=false" \
+      . || echo "---") | tee /dev/stderr |
+      yq 'length > 0' | tee /dev/stderr)
+  [ "${actual}" = "false" ]
+
+  local actual=$( (helm template \
+      --show-only templates/injector-rolebinding.yaml \
+      --set "injector.replicas=2" \
+      --set "injector.enabled=false" \
+      . || echo "---") | tee /dev/stderr |
+      yq 'length > 0' | tee /dev/stderr)
+  [ "${actual}" = "false" ]
+
+  local actual=$( (helm template \
+      --show-only templates/injector-rolebinding.yaml \
+      --set "injector.replicas=2" \
+      --set "injector.leaderElector.enabled=false" \
+      . || echo "---") | tee /dev/stderr |
+      yq 'length > 0' | tee /dev/stderr)
+  [ "${actual}" = "false" ]
+
+  local actual=$( (helm template \
+      --show-only templates/injector-rolebinding.yaml \
+      --set "injector.replicas=2" \
+      . || echo "---") | tee /dev/stderr |
+      yq 'length > 0' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+}

values.yaml

@@ -27,6 +27,17 @@ injector:
   # True if you want to enable vault agent injection.
   enabled: true
 
+  replicas: 1
+
+  # If multiple replicas are specified, by default a leader-elector side-car
+  # will be created so that only one injector attempts to create TLS certificates.
+  leaderElector:
+    enabled: true
+    image:
+      repository: "gcr.io/google_containers/leader-elector"
+      tag: "0.4"
+    ttl: 60s
+
   # If true, will enable a node exporter metrics endpoint at /metrics.
   metrics:
     enabled: false
@@ -112,7 +123,17 @@ injector:
   # Affinity Settings for injector pods
   # This should be a multi-line string matching the affinity section of a
   # PodSpec.
-  affinity: null
+  # Commenting out or setting as empty the affinity variable, will allow
+  # deployment of multiple replicas to single node services such as Minikube.
+  affinity: |
+    podAntiAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        - labelSelector:
+            matchLabels:
+              app.kubernetes.io/name: {{ template "vault.name" . }}
+              app.kubernetes.io/instance: "{{ .Release.Name }}"
+              component: injector
+          topologyKey: kubernetes.io/hostname
 
   # Toleration Settings for injector pods
   # This should be a multi-line string matching the Toleration array