Statefulset Liveness Probe failing on standby nodes due to SSL, initial delay and 429 response #137 (#138)

livenessProbe

 * Set the scheme for vault.scheme to ensure that the check works if tls enabled or not

 * Allow a configurable value initialDelaySeconds rather than the set 5 seconds

 * Set the default initialDelaySeconds to 60 seconds before the probe starts to allow for vault unsealing

 * Set the path to /v1/sys/health?standbyok=true to ensure a 200 response on standbys

readinessProbe

 * Set the path comment to /v1/sys/health?standbyok=true to ensure a 200 response on standbys

 * Set the scheme for vault.scheme to ensure that the check works if tls enabled or not

 * Statefulset liveness probe path check set to /v1/sys/health?standbyok=true

 * Server Statefulset test added for livenessProbe.initialDelaySeconds

templates/server-statefulset.yaml

@@ -90,6 +90,7 @@ spec:
             httpGet:
               path: {{ .Values.server.readinessProbe.path | quote }}
               port: 8200
+              scheme: {{ include "vault.scheme" . | upper }}
             {{- else }}
             # Check status; unsealed vault servers return 0
             # The exit code reflects the seal status:
@@ -110,7 +111,8 @@ spec:
             httpGet:
               path: {{ .Values.server.livenessProbe.path | quote }}
               port: 8200
-            initialDelaySeconds: 5
+              scheme: {{ include "vault.scheme" . | upper }}
+            initialDelaySeconds: {{ .Values.server.livenessProbe.initialDelaySeconds }}
             periodSeconds: 3
             successThreshold: 1
             timeoutSeconds: 5

test/unit/server-statefulset.bats

@@ -781,5 +781,26 @@ load _helpers
       --set 'server.livenessProbe.enabled=true' \
       . | tee /dev/stderr |
       yq -r '.spec.template.spec.containers[0].livenessProbe.httpGet.path' | tee /dev/stderr)
-  [ "${actual}" = "/v1/sys/health?standbyok" ]
+  [ "${actual}" = "/v1/sys/health?standbyok=true" ]
+}
+
+@test "server/standalone-StatefulSet: livenessProbe initialDelaySeconds default" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      -x templates/server-statefulset.yaml \
+      --set 'server.livenessProbe.enabled=true' \
+      . | tee /dev/stderr |
+      yq -r '.spec.template.spec.containers[0].livenessProbe.initialDelaySeconds' | tee /dev/stderr)
+  [ "${actual}" = "60" ]
+}
+
+@test "server/standalone-StatefulSet: livenessProbe initialDelaySeconds configurable" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      -x templates/server-statefulset.yaml \
+      --set 'server.livenessProbe.enabled=true' \
+      --set 'server.livenessProbe.initialDelaySeconds=30' \
+      . | tee /dev/stderr |
+      yq -r '.spec.template.spec.containers[0].livenessProbe.initialDelaySeconds' | tee /dev/stderr)
+  [ "${actual}" = "30" ]
 }

values.yaml

@@ -64,11 +64,12 @@ server:
   readinessProbe:
     enabled: true
     # If you need to use a http path instead of the default exec
-    # path: /v1/sys/health?standbyok
+    # path: /v1/sys/health?standbyok=true
   # Used to enable a livenessProbe for the pods
   livenessProbe:
     enabled: false
-    path: /v1/sys/health?standbyok
+    path: "/v1/sys/health?standbyok=true"
+    initialDelaySeconds: 60
 
   # extraEnvironmentVars is a list of extra enviroment variables to set with the stateful set. These could be
   # used to include variables required for auto-unseal.