Support vault-k8s internal leader election (#568)

templates/injector-deployment.yaml

@@ -110,6 +110,10 @@ spec:
             - name: AGENT_INJECT_TEMPLATE_CONFIG_EXIT_ON_RETRY_FAILURE
               value: "{{ .Values.injector.agentDefaults.templateConfig.exitOnRetryFailure }}"
             {{- include "vault.extraEnvironmentVars" .Values.injector | nindent 12 }}
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
           args:
             - agent-inject
             - 2>&1
@@ -139,35 +143,6 @@ spec:
               mountPath: /etc/webhook/certs
               readOnly: true
 {{- end }}
-        {{- if and (eq (.Values.injector.leaderElector.enabled | toString) "true") (gt (.Values.injector.replicas | int) 1) }}
-        - name: leader-elector
-          image: {{ .Values.injector.leaderElector.image.repository }}:{{ .Values.injector.leaderElector.image.tag }}
-          args:
-            - --election={{ template "vault.fullname" . }}-agent-injector-leader
-            - --election-namespace={{ .Release.Namespace }}
-            - --http=0.0.0.0:4040
-            - --ttl={{ .Values.injector.leaderElector.ttl }}
-          livenessProbe:
-            httpGet:
-              path: /
-              port: 4040
-              scheme: HTTP
-            failureThreshold: 2
-            initialDelaySeconds: 5
-            periodSeconds: 2
-            successThreshold: 1
-            timeoutSeconds: 5
-          readinessProbe:
-            httpGet:
-              path: /
-              port: 4040
-              scheme: HTTP
-            failureThreshold: 2
-            initialDelaySeconds: 5
-            periodSeconds: 2
-            successThreshold: 1
-            timeoutSeconds: 5
-        {{- end }}
 {{- if .Values.injector.certs.secretName }}
       volumes:
         - name: webhook-certs

templates/injector-leader-endpoint.yaml

@@ -1,12 +0,0 @@
-{{- if and (eq (.Values.injector.enabled | toString) "true" ) (eq (.Values.global.enabled | toString) "true") (eq (.Values.injector.leaderElector.enabled | toString) "true") (gt (.Values.injector.replicas | int) 1) }}
-# This is created here so it can be cleaned up easily, since if
-# the endpoint is left around the leader won't expire for about a minute.
-apiVersion: v1
-kind: Endpoints
-metadata:
-  name: {{ template "vault.fullname" . }}-agent-injector-leader
-  labels:
-    app.kubernetes.io/name: {{ include "vault.name" . }}-agent-injector
-    app.kubernetes.io/instance: {{ .Release.Name }}
-    app.kubernetes.io/managed-by: {{ .Release.Service }}
-{{- end }}

templates/injector-role.yaml

@@ -9,11 +9,17 @@ metadata:
     app.kubernetes.io/managed-by: {{ .Release.Service }}
 rules:
   - apiGroups: [""]
-    resources: ["endpoints", "secrets"]
+    resources: ["secrets", "configmaps"]
     verbs:
       - "create"
       - "get"
       - "watch"
       - "list"
       - "update"
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs:
+      - "get"
+      - "patch"
+      - "delete"
 {{- end }}

test/acceptance/injector-leader-elector.bats

@@ -22,21 +22,16 @@ load _helpers
   tries=0
   until [ $tries -ge 60 ]
   do
-    leader="$(echo "$(kubectl exec ${pods[0]} -c sidecar-injector -- wget --quiet --output-document - localhost:4040)" | jq -r .name)"
-    [ -n "${leader}" ] && break
-    ((tries++))
+    owner=$(kubectl get configmaps vault-k8s-leader -o json | jq -r .metadata.ownerReferences\[0\].name)
+    leader=$(kubectl get pods $owner -o json | jq -r .metadata.name)
+    [ -n "${leader}" ] && [ "${leader}" != "null" ] && break
+    let "tries=tries+1"
     sleep .5
   done
 
   # Check the leader name is valid - i.e. one of the 3 pods
   [[ " ${pods[@]} " =~ " ${leader} " ]]
 
-  # Check every pod agrees on who the leader is
-  for pod in "${pods[@]}"
-  do
-    pod_leader="$(echo "$(kubectl exec $pod -c sidecar-injector -- wget --quiet --output-document - localhost:4040)" | jq -r .name)"
-    [ "${pod_leader}" == "${leader}" ]
-  done
 }
 
 setup() {

test/unit/injector-leader-elector.bats

@@ -5,7 +5,7 @@ load _helpers
 #--------------------------------------------------------------------
 # Deployment
 
-@test "injector/deployment: leader elector replica count" {
+@test "injector/deployment: replica count" {
   cd `chart_dir`
   local actual=$(helm template \
       --show-only templates/injector-deployment.yaml \
@@ -15,42 +15,6 @@ load _helpers
   [ "${actual}" = "2" ]
 }
 
-@test "injector/deployment: leader elector - sidecar is created only when enabled" {
-  cd `chart_dir`
-  local actual=$(helm template \
-      --show-only templates/injector-deployment.yaml \
-      . | tee /dev/stderr |
-      yq '.spec.template.spec.containers | length' | tee /dev/stderr)
-  [ "${actual}" = "1" ]
-
-  local actual=$(helm template \
-      --show-only templates/injector-deployment.yaml \
-      --set "injector.replicas=2" \
-      --set "injector.leaderElector.enabled=false" \
-      . | tee /dev/stderr |
-      yq '.spec.template.spec.containers | length' | tee /dev/stderr)
-  [ "${actual}" = "1" ]
-
-  local actual=$(helm template \
-      --show-only templates/injector-deployment.yaml \
-      --set "injector.replicas=2" \
-      . | tee /dev/stderr |
-      yq '.spec.template.spec.containers | length' | tee /dev/stderr)
-  [ "${actual}" = "2" ]
-}
-
-@test "injector/deployment: leader elector image name is configurable" {
-  cd `chart_dir`
-  local actual=$(helm template \
-      --show-only templates/injector-deployment.yaml \
-      --set "injector.replicas=2" \
-      --set "injector.leaderElector.image.repository=SomeOtherImage" \
-      --set "injector.leaderElector.image.tag=SomeOtherTag" \
-      . | tee /dev/stderr |
-      yq -r '.spec.template.spec.containers[1].image' | tee /dev/stderr)
-  [ "${actual}" = "SomeOtherImage:SomeOtherTag" ]
-}
-
 @test "injector/deployment: leader elector configuration for sidecar-injector" {
   cd `chart_dir`
   local actual=$(helm template \
@@ -80,26 +44,6 @@ load _helpers
   [ "${actual}" = "metadata.namespace" ]
 }
 
-@test "injector/deployment: leader elector TTL is configurable" {
-  cd `chart_dir`
-  # Default value 60s
-  local actual=$(helm template \
-      --show-only templates/injector-deployment.yaml \
-      --set "injector.replicas=2" \
-      . | tee /dev/stderr |
-      yq -r '.spec.template.spec.containers[1].args[3]' | tee /dev/stderr)
-  [ "${actual}" = "--ttl=60s" ]
-
-  # Configured to 30s
-  local actual=$(helm template \
-      --show-only templates/injector-deployment.yaml \
-      --set "injector.replicas=2" \
-      --set "injector.leaderElector.ttl=30s" \
-      . | tee /dev/stderr |
-      yq -r '.spec.template.spec.containers[1].args[3]' | tee /dev/stderr)
-  [ "${actual}" = "--ttl=30s" ]
-}
-
 #--------------------------------------------------------------------
 # Resource creation
 
@@ -143,46 +87,6 @@ load _helpers
   [ "${actual}" = "true" ]
 }
 
-@test "injector/leader-endpoint: created/skipped as appropriate" {
-  cd `chart_dir`
-  local actual=$( (helm template \
-      --show-only templates/injector-leader-endpoint.yaml \
-      . || echo "---") | tee /dev/stderr |
-      yq 'length > 0' | tee /dev/stderr)
-  [ "${actual}" = "false" ]
-
-  local actual=$( (helm template \
-      --show-only templates/injector-leader-endpoint.yaml \
-      --set "injector.replicas=2" \
-      --set "global.enabled=false" \
-      . || echo "---") | tee /dev/stderr |
-      yq 'length > 0' | tee /dev/stderr)
-  [ "${actual}" = "false" ]
-
-  local actual=$( (helm template \
-      --show-only templates/injector-leader-endpoint.yaml \
-      --set "injector.replicas=2" \
-      --set "injector.enabled=false" \
-      . || echo "---") | tee /dev/stderr |
-      yq 'length > 0' | tee /dev/stderr)
-  [ "${actual}" = "false" ]
-
-  local actual=$( (helm template \
-      --show-only templates/injector-leader-endpoint.yaml \
-      --set "injector.replicas=2" \
-      --set "injector.leaderElector.enabled=false" \
-      . || echo "---") | tee /dev/stderr |
-      yq 'length > 0' | tee /dev/stderr)
-  [ "${actual}" = "false" ]
-
-  local actual=$( (helm template \
-      --show-only templates/injector-leader-endpoint.yaml \
-      --set "injector.replicas=2" \
-      . || echo "---") | tee /dev/stderr |
-      yq 'length > 0' | tee /dev/stderr)
-  [ "${actual}" = "true" ]
-}
-
 @test "injector/role: created/skipped as appropriate" {
   cd `chart_dir`
   local actual=$( (helm template \

values.schema.json

@@ -281,20 +281,6 @@
                     "properties": {
                         "enabled": {
                             "type": "boolean"
-                        },
-                        "image": {
-                            "type": "object",
-                            "properties": {
-                                "repository": {
-                                    "type": "string"
-                                },
-                                "tag": {
-                                    "type": "string"
-                                }
-                            }
-                        },
-                        "ttl": {
-                            "type": "string"
                         }
                     }
                 },

values.yaml

@@ -33,14 +33,10 @@ injector:
   # Configures the port the injector should listen on
   port: 8080
 
-  # If multiple replicas are specified, by default a leader-elector side-car
-  # will be created so that only one injector attempts to create TLS certificates.
+  # If multiple replicas are specified, by default a leader will be determined
+  # so that only one injector attempts to create TLS certificates.
   leaderElector:
     enabled: true
-    image:
-      repository: "gcr.io/google_containers/leader-elector"
-      tag: "0.4"
-    ttl: 60s
 
   # If true, will enable a node exporter metrics endpoint at /metrics.
   metrics: