diff --git a/templates/injector-certs-secret.yaml b/templates/injector-certs-secret.yaml new file mode 100644 index 0000000..aec8021 --- /dev/null +++ b/templates/injector-certs-secret.yaml @@ -0,0 +1,10 @@ +{{- if and (eq (.Values.injector.enabled | toString) "true" ) (eq (.Values.global.enabled | toString) "true") (eq (.Values.injector.leaderElector.enabled | toString) "true") (gt (.Values.injector.replicas | int) 1) }} +apiVersion: v1 +kind: Secret +metadata: + name: vault-injector-certs + labels: + app.kubernetes.io/name: {{ include "vault.name" . }}-agent-injector + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} \ No newline at end of file diff --git a/templates/injector-deployment.yaml b/templates/injector-deployment.yaml index 9240b09..a227902 100644 --- a/templates/injector-deployment.yaml +++ b/templates/injector-deployment.yaml @@ -11,7 +11,7 @@ metadata: app.kubernetes.io/managed-by: {{ .Release.Service }} component: webhook spec: - replicas: 1 + replicas: {{ .Values.injector.replicas }} selector: matchLabels: app.kubernetes.io/name: {{ template "vault.name" . }}-agent-injector @@ -88,6 +88,14 @@ spec: - name: AGENT_INJECT_TELEMETRY_PATH value: "/metrics" {{- end }} + {{- if and (eq (.Values.injector.leaderElector.enabled | toString) "true") (gt (.Values.injector.replicas | int) 1) }} + - name: AGENT_INJECT_USE_LEADER_ELECTOR + value: "true" + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + {{- end }} {{- include "vault.extraEnvironmentVars" .Values.injector | nindent 12 }} args: - agent-inject @@ -112,6 +120,35 @@ spec: periodSeconds: 2 successThreshold: 1 timeoutSeconds: 5 + {{- if and (eq (.Values.injector.leaderElector.enabled | toString) "true") (gt (.Values.injector.replicas | int) 1) }} + - name: leader-elector + image: {{ .Values.injector.leaderElector.image.repository }}:{{ .Values.injector.leaderElector.image.tag }} + args: + - --election={{ template "vault.fullname" . }}-agent-injector-leader + - --election-namespace={{ .Release.Namespace }} + - --http=0.0.0.0:4040 + - --ttl={{ .Values.injector.leaderElector.ttl }} + livenessProbe: + httpGet: + path: / + port: 4040 + scheme: HTTP + failureThreshold: 2 + initialDelaySeconds: 1 + periodSeconds: 2 + successThreshold: 1 + timeoutSeconds: 5 + readinessProbe: + httpGet: + path: / + port: 4040 + scheme: HTTP + failureThreshold: 2 + initialDelaySeconds: 2 + periodSeconds: 2 + successThreshold: 1 + timeoutSeconds: 5 + {{- end }} {{- if .Values.injector.certs.secretName }} volumeMounts: - name: webhook-certs diff --git a/templates/injector-leader-endpoint.yaml b/templates/injector-leader-endpoint.yaml new file mode 100644 index 0000000..fc4ef16 --- /dev/null +++ b/templates/injector-leader-endpoint.yaml @@ -0,0 +1,12 @@ +{{- if and (eq (.Values.injector.enabled | toString) "true" ) (eq (.Values.global.enabled | toString) "true") (eq (.Values.injector.leaderElector.enabled | toString) "true") (gt (.Values.injector.replicas | int) 1) }} +# This is created here so it can be cleaned up easily, since if +# the endpoint is left around the leader won't expire for about a minute. +apiVersion: v1 +kind: Endpoints +metadata: + name: {{ template "vault.fullname" . }}-agent-injector-leader + labels: + app.kubernetes.io/name: {{ include "vault.name" . }}-agent-injector + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} \ No newline at end of file diff --git a/templates/injector-role.yaml b/templates/injector-role.yaml new file mode 100644 index 0000000..e74524c --- /dev/null +++ b/templates/injector-role.yaml @@ -0,0 +1,19 @@ +{{- if and (eq (.Values.injector.enabled | toString) "true" ) (eq (.Values.global.enabled | toString) "true") (eq (.Values.injector.leaderElector.enabled | toString) "true") (gt (.Values.injector.replicas | int) 1) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ template "vault.fullname" . }}-agent-injector-leader-elector-role + labels: + app.kubernetes.io/name: {{ include "vault.name" . }}-agent-injector + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +rules: + - apiGroups: [""] + resources: ["endpoints", "secrets"] + verbs: + - "create" + - "get" + - "watch" + - "list" + - "update" +{{- end }} \ No newline at end of file diff --git a/templates/injector-rolebinding.yaml b/templates/injector-rolebinding.yaml new file mode 100644 index 0000000..e06d242 --- /dev/null +++ b/templates/injector-rolebinding.yaml @@ -0,0 +1,18 @@ +{{- if and (eq (.Values.injector.enabled | toString) "true" ) (eq (.Values.global.enabled | toString) "true") (eq (.Values.injector.leaderElector.enabled | toString) "true") (gt (.Values.injector.replicas | int) 1) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ template "vault.fullname" . }}-agent-injector-leader-elector-binding + labels: + app.kubernetes.io/name: {{ include "vault.name" . }}-agent-injector + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ template "vault.fullname" . }}-agent-injector-leader-elector-role +subjects: + - kind: ServiceAccount + name: {{ template "vault.fullname" . }}-agent-injector + namespace: {{ .Release.Namespace }} +{{- end }} \ No newline at end of file diff --git a/test/acceptance/injector-leader-elector.bats b/test/acceptance/injector-leader-elector.bats new file mode 100644 index 0000000..30d84b9 --- /dev/null +++ b/test/acceptance/injector-leader-elector.bats @@ -0,0 +1,46 @@ +#!/usr/bin/env bats + +load _helpers + +@test "injector: testing leader elector" { + cd `chart_dir` + + kubectl delete namespace acceptance --ignore-not-found=true + kubectl create namespace acceptance + kubectl config set-context --current --namespace=acceptance + + helm install "$(name_prefix)" \ + --set="injector.replicas=3" . + kubectl wait --for condition=Ready pod -l app.kubernetes.io/name=vault-agent-injector + + pods=($(kubectl get pods -l app.kubernetes.io/name=vault-agent-injector -o json | jq -r '.items[] | .metadata.name')) + [ "${#pods[@]}" == 3 ] + + leader="$(echo "$(kubectl exec ${pods[0]} -c sidecar-injector -- wget --quiet --output-document - localhost:4040)" | jq -r .name)" + # Check the leader name is valid - i.e. one of the 3 pods + [[ " ${pods[@]} " =~ " ${leader} " ]] + + # Check every pod agrees on who the leader is + for pod in "${pods[@]}" + do + pod_leader="$(echo "$(kubectl exec $pod -c sidecar-injector -- wget --quiet --output-document - localhost:4040)" | jq -r .name)" + [ "${pod_leader}" == "${leader}" ] + done +} + +setup() { + kubectl delete namespace acceptance --ignore-not-found=true + kubectl create namespace acceptance + kubectl config set-context --current --namespace=acceptance +} + +# Clean up +teardown() { + if [[ ${CLEANUP:-true} == "true" ]] + then + echo "helm/pvc teardown" + helm delete vault + kubectl delete --all pvc + kubectl delete namespace acceptance + fi +} \ No newline at end of file diff --git a/test/unit/injector-deployment.bats b/test/unit/injector-deployment.bats index 4c25e6a..a117480 100755 --- a/test/unit/injector-deployment.bats +++ b/test/unit/injector-deployment.bats @@ -425,13 +425,13 @@ load _helpers #-------------------------------------------------------------------- # affinity -@test "injector/deployment: affinity not set by default" { +@test "injector/deployment: affinity set by default" { cd `chart_dir` local actual=$(helm template \ --show-only templates/injector-deployment.yaml \ . | tee /dev/stderr | yq '.spec.template.spec | .affinity? == null' | tee /dev/stderr) - [ "${actual}" = "true" ] + [ "${actual}" = "false" ] } @test "injector/deployment: affinity can be set" { diff --git a/test/unit/injector-leader-elector.bats b/test/unit/injector-leader-elector.bats new file mode 100644 index 0000000..6c77d81 --- /dev/null +++ b/test/unit/injector-leader-elector.bats @@ -0,0 +1,264 @@ +#!/usr/bin/env bats + +load _helpers + +#-------------------------------------------------------------------- +# Deployment + +@test "injector/deployment: leader elector replica count" { + cd `chart_dir` + local actual=$(helm template \ + --show-only templates/injector-deployment.yaml \ + --set "injector.replicas=2" \ + . | tee /dev/stderr | + yq '.spec.replicas' | tee /dev/stderr) + [ "${actual}" = "2" ] +} + +@test "injector/deployment: leader elector - sidecar is created only when enabled" { + cd `chart_dir` + local actual=$(helm template \ + --show-only templates/injector-deployment.yaml \ + . | tee /dev/stderr | + yq '.spec.template.spec.containers | length' | tee /dev/stderr) + [ "${actual}" = "1" ] + + local actual=$(helm template \ + --show-only templates/injector-deployment.yaml \ + --set "injector.replicas=2" \ + --set "injector.leaderElector.enabled=false" \ + . | tee /dev/stderr | + yq '.spec.template.spec.containers | length' | tee /dev/stderr) + [ "${actual}" = "1" ] + + local actual=$(helm template \ + --show-only templates/injector-deployment.yaml \ + --set "injector.replicas=2" \ + . | tee /dev/stderr | + yq '.spec.template.spec.containers | length' | tee /dev/stderr) + [ "${actual}" = "2" ] +} + +@test "injector/deployment: leader elector image name is configurable" { + cd `chart_dir` + local actual=$(helm template \ + --show-only templates/injector-deployment.yaml \ + --set "injector.replicas=2" \ + --set "injector.leaderElector.image.repository=SomeOtherImage" \ + --set "injector.leaderElector.image.tag=SomeOtherTag" \ + . | tee /dev/stderr | + yq -r '.spec.template.spec.containers[1].image' | tee /dev/stderr) + [ "${actual}" = "SomeOtherImage:SomeOtherTag" ] +} + +@test "injector/deployment: leader elector configuration for sidecar-injector" { + cd `chart_dir` + local actual=$(helm template \ + --show-only templates/injector-deployment.yaml \ + . | tee /dev/stderr | + yq -r '.spec.template.spec.containers[0].env[] | select(.name == "AGENT_INJECT_USE_LEADER_ELECTOR") | .value' | tee /dev/stderr) + [ "${actual}" = "" ] + + local actual=$(helm template \ + --show-only templates/injector-deployment.yaml \ + --set "injector.replicas=2" \ + . | tee /dev/stderr | + yq -r '.spec.template.spec.containers[0].env[] | select(.name == "AGENT_INJECT_USE_LEADER_ELECTOR") | .value' | tee /dev/stderr) + [ "${actual}" = "true" ] + + local actual=$(helm template \ + --show-only templates/injector-deployment.yaml \ + . | tee /dev/stderr | + yq -r '.spec.template.spec.containers[0].env[] | select(.name == "NAMESPACE") | .valueFrom.fieldRef.fieldPath' | tee /dev/stderr) + [ "${actual}" = "" ] + + local actual=$(helm template \ + --show-only templates/injector-deployment.yaml \ + --set "injector.replicas=2" \ + . | tee /dev/stderr | + yq -r '.spec.template.spec.containers[0].env[] | select(.name == "NAMESPACE") | .valueFrom.fieldRef.fieldPath' | tee /dev/stderr) + [ "${actual}" = "metadata.namespace" ] +} + +@test "injector/deployment: leader elector TTL is configurable" { + cd `chart_dir` + # Default value 60s + local actual=$(helm template \ + --show-only templates/injector-deployment.yaml \ + --set "injector.replicas=2" \ + . | tee /dev/stderr | + yq -r '.spec.template.spec.containers[1].args[3]' | tee /dev/stderr) + [ "${actual}" = "--ttl=60s" ] + + # Configured to 30s + local actual=$(helm template \ + --show-only templates/injector-deployment.yaml \ + --set "injector.replicas=2" \ + --set "injector.leaderElector.ttl=30s" \ + . | tee /dev/stderr | + yq -r '.spec.template.spec.containers[1].args[3]' | tee /dev/stderr) + [ "${actual}" = "--ttl=30s" ] +} + +#-------------------------------------------------------------------- +# Resource creation + +@test "injector/certs-secret: created/skipped as appropriate" { + cd `chart_dir` + local actual=$( (helm template \ + --show-only templates/injector-certs-secret.yaml \ + . || echo "---") | tee /dev/stderr | + yq 'length > 0' | tee /dev/stderr) + [ "${actual}" = "false" ] + + local actual=$( (helm template \ + --show-only templates/injector-certs-secret.yaml \ + --set "injector.replicas=2" \ + --set "global.enabled=false" \ + . || echo "---") | tee /dev/stderr | + yq 'length > 0' | tee /dev/stderr) + [ "${actual}" = "false" ] + + local actual=$( (helm template \ + --show-only templates/injector-certs-secret.yaml \ + --set "injector.replicas=2" \ + --set "injector.enabled=false" \ + . || echo "---") | tee /dev/stderr | + yq 'length > 0' | tee /dev/stderr) + [ "${actual}" = "false" ] + + local actual=$( (helm template \ + --show-only templates/injector-certs-secret.yaml \ + --set "injector.replicas=2" \ + --set "injector.leaderElector.enabled=false" \ + . || echo "---") | tee /dev/stderr | + yq 'length > 0' | tee /dev/stderr) + [ "${actual}" = "false" ] + + local actual=$( (helm template \ + --show-only templates/injector-certs-secret.yaml \ + --set "injector.replicas=2" \ + . || echo "---") | tee /dev/stderr | + yq 'length > 0' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + +@test "injector/leader-endpoint: created/skipped as appropriate" { + cd `chart_dir` + local actual=$( (helm template \ + --show-only templates/injector-leader-endpoint.yaml \ + . || echo "---") | tee /dev/stderr | + yq 'length > 0' | tee /dev/stderr) + [ "${actual}" = "false" ] + + local actual=$( (helm template \ + --show-only templates/injector-leader-endpoint.yaml \ + --set "injector.replicas=2" \ + --set "global.enabled=false" \ + . || echo "---") | tee /dev/stderr | + yq 'length > 0' | tee /dev/stderr) + [ "${actual}" = "false" ] + + local actual=$( (helm template \ + --show-only templates/injector-leader-endpoint.yaml \ + --set "injector.replicas=2" \ + --set "injector.enabled=false" \ + . || echo "---") | tee /dev/stderr | + yq 'length > 0' | tee /dev/stderr) + [ "${actual}" = "false" ] + + local actual=$( (helm template \ + --show-only templates/injector-leader-endpoint.yaml \ + --set "injector.replicas=2" \ + --set "injector.leaderElector.enabled=false" \ + . || echo "---") | tee /dev/stderr | + yq 'length > 0' | tee /dev/stderr) + [ "${actual}" = "false" ] + + local actual=$( (helm template \ + --show-only templates/injector-leader-endpoint.yaml \ + --set "injector.replicas=2" \ + . || echo "---") | tee /dev/stderr | + yq 'length > 0' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + +@test "injector/role: created/skipped as appropriate" { + cd `chart_dir` + local actual=$( (helm template \ + --show-only templates/injector-role.yaml \ + . || echo "---") | tee /dev/stderr | + yq 'length > 0' | tee /dev/stderr) + [ "${actual}" = "false" ] + + local actual=$( (helm template \ + --show-only templates/injector-role.yaml \ + --set "injector.replicas=2" \ + --set "global.enabled=false" \ + . || echo "---") | tee /dev/stderr | + yq 'length > 0' | tee /dev/stderr) + [ "${actual}" = "false" ] + + local actual=$( (helm template \ + --show-only templates/injector-role.yaml \ + --set "injector.replicas=2" \ + --set "injector.enabled=false" \ + . || echo "---") | tee /dev/stderr | + yq 'length > 0' | tee /dev/stderr) + [ "${actual}" = "false" ] + + local actual=$( (helm template \ + --show-only templates/injector-role.yaml \ + --set "injector.replicas=2" \ + --set "injector.leaderElector.enabled=false" \ + . || echo "---") | tee /dev/stderr | + yq 'length > 0' | tee /dev/stderr) + [ "${actual}" = "false" ] + + local actual=$( (helm template \ + --show-only templates/injector-role.yaml \ + --set "injector.replicas=2" \ + . || echo "---") | tee /dev/stderr | + yq 'length > 0' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + +@test "injector/rolebinding: created/skipped as appropriate" { + cd `chart_dir` + local actual=$( (helm template \ + --show-only templates/injector-rolebinding.yaml \ + . || echo "---") | tee /dev/stderr | + yq 'length > 0' | tee /dev/stderr) + [ "${actual}" = "false" ] + + local actual=$( (helm template \ + --show-only templates/injector-rolebinding.yaml \ + --set "injector.replicas=2" \ + --set "global.enabled=false" \ + . || echo "---") | tee /dev/stderr | + yq 'length > 0' | tee /dev/stderr) + [ "${actual}" = "false" ] + + local actual=$( (helm template \ + --show-only templates/injector-rolebinding.yaml \ + --set "injector.replicas=2" \ + --set "injector.enabled=false" \ + . || echo "---") | tee /dev/stderr | + yq 'length > 0' | tee /dev/stderr) + [ "${actual}" = "false" ] + + local actual=$( (helm template \ + --show-only templates/injector-rolebinding.yaml \ + --set "injector.replicas=2" \ + --set "injector.leaderElector.enabled=false" \ + . || echo "---") | tee /dev/stderr | + yq 'length > 0' | tee /dev/stderr) + [ "${actual}" = "false" ] + + local actual=$( (helm template \ + --show-only templates/injector-rolebinding.yaml \ + --set "injector.replicas=2" \ + . || echo "---") | tee /dev/stderr | + yq 'length > 0' | tee /dev/stderr) + [ "${actual}" = "true" ] +} \ No newline at end of file diff --git a/values.yaml b/values.yaml index 10b89b2..4455f19 100644 --- a/values.yaml +++ b/values.yaml @@ -27,6 +27,17 @@ injector: # True if you want to enable vault agent injection. enabled: true + replicas: 1 + + # If multiple replicas are specified, by default a leader-elector side-car + # will be created so that only one injector attempts to create TLS certificates. + leaderElector: + enabled: true + image: + repository: "gcr.io/google_containers/leader-elector" + tag: "0.4" + ttl: 60s + # If true, will enable a node exporter metrics endpoint at /metrics. metrics: enabled: false @@ -112,7 +123,17 @@ injector: # Affinity Settings for injector pods # This should be a multi-line string matching the affinity section of a # PodSpec. - affinity: null + # Commenting out or setting as empty the affinity variable, will allow + # deployment of multiple replicas to single node services such as Minikube. + affinity: | + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app.kubernetes.io/name: {{ template "vault.name" . }} + app.kubernetes.io/instance: "{{ .Release.Name }}" + component: injector + topologyKey: kubernetes.io/hostname # Toleration Settings for injector pods # This should be a multi-line string matching the Toleration array