feat(helm): unify workload manifests (Deployment, DaemonSet); add support for StatefulSet

charts/ingress-nginx/README.md

@@ -294,7 +294,7 @@ Kubernetes: `>=1.20.0-0`
 | controller.admissionWebhooks.service.type | string | `"ClusterIP"` |  |
 | controller.affinity | object | `{}` | Affinity and anti-affinity rules for server scheduling to nodes # Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity # |
 | controller.allowSnippetAnnotations | bool | `true` | This configuration defines if Ingress Controller should allow users to set their own *-snippet annotations, otherwise this is forbidden / dropped when users add those annotations. Global snippets in ConfigMap are still respected |
-| controller.annotations | object | `{}` | Annotations to be added to the controller Deployment or DaemonSet # |
+| controller.annotations | object | `{}` | Annotations to be added to the controller Deployment or DaemonSet or StatefulSet # |
 | controller.autoscaling.annotations | object | `{}` |  |
 | controller.autoscaling.apiVersion | string | `"autoscaling/v2"` |  |
 | controller.autoscaling.behavior | object | `{}` |  |
@@ -357,8 +357,8 @@ Kubernetes: `>=1.20.0-0`
 | controller.keda.restoreToOriginalReplicaCount | bool | `false` |  |
 | controller.keda.scaledObject.annotations | object | `{}` |  |
 | controller.keda.triggers | list | `[]` |  |
-| controller.kind | string | `"Deployment"` | Use a `DaemonSet` or `Deployment` |
+| controller.kind | string | `"Deployment"` | Use a `DaemonSet` or `Deployment` or `StatefulSet`; `Both` creates a `DaemonSet` and a `Deployment` |
-| controller.labels | object | `{}` | Labels to be added to the controller Deployment or DaemonSet and other resources that do not have option to specify labels # |
+| controller.labels | object | `{}` | Labels to be added to the controller Deployment or DaemonSet or StatefulSet and other resources that do not have option to specify labels # |
 | controller.lifecycle | object | `{"preStop":{"exec":{"command":["/wait-shutdown"]}}}` | Improve connection draining when ingress controller pod is deleted using a lifecycle hook: With this new hook, we increased the default terminationGracePeriodSeconds from 30 seconds to 300, allowing the draining of connections up to five minutes. If the active connections end before that, the pod will terminate gracefully at that time. To effectively take advantage of this feature, the Configmap feature worker-shutdown-timeout new value is 240s instead of 10s. # |
 | controller.livenessProbe.failureThreshold | int | `5` |  |
 | controller.livenessProbe.httpGet.path | string | `"/healthz"` |  |
@@ -398,6 +398,7 @@ Kubernetes: `>=1.20.0-0`
 | controller.opentelemetry.image | string | `"registry.k8s.io/ingress-nginx/opentelemetry:v20230312-helm-chart-4.5.2-28-g66a760794@sha256:40f766ac4a9832f36f217bb0e98d44c8d38faeccbfe861fbc1a76af7e9ab257f"` |  |
 | controller.podAnnotations | object | `{}` | Annotations to be added to controller pods # |
 | controller.podLabels | object | `{}` | Labels to add to the pod container metadata |
+| controller.podManagementPolicy | string | `"OrderedReady"` | Pod management policy for StatefulSet # |
 | controller.podSecurityContext | object | `{}` | Security Context policies for controller pods |
 | controller.priorityClassName | string | `""` |  |
 | controller.proxySetHeaders | object | `{}` | Will add custom headers before sending traffic to backends according to https://github.com/kubernetes/ingress-nginx/tree/main/docs/examples/customization/custom-headers |
@@ -452,7 +453,8 @@ Kubernetes: `>=1.20.0-0`
 | controller.topologySpreadConstraints | list | `[]` | Topology spread constraints rely on node labels to identify the topology domain(s) that each Node is in. # Ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ # |
 | controller.udp.annotations | object | `{}` | Annotations to be added to the udp config configmap |
 | controller.udp.configMapNamespace | string | `""` | Allows customization of the udp-services-configmap; defaults to $(POD_NAMESPACE) |
-| controller.updateStrategy | object | `{}` | The update strategy to apply to the Deployment or DaemonSet # |
+| controller.updateStrategy | object | `{}` | The update strategy to apply to the Deployment or DaemonSet or StatefulSet # |
+| controller.volumeClaimTemplates | list | `[]` | Volume claim templates for StatefulSet pods # |
 | controller.watchIngressWithoutClass | bool | `false` | Process Ingress objects without ingressClass annotation/ingressClassName field Overrides value for --watch-ingress-without-class flag of the controller binary Defaults to false |
 | defaultBackend.affinity | object | `{}` |  |
 | defaultBackend.autoscaling.annotations | object | `{}` |  |

charts/ingress-nginx/ci/statefulset-default-values.yaml

@@ -0,0 +1,9 @@
+# Left blank to test default values
+controller:
+  kind: StatefulSet
+  image:
+    repository: ingress-controller/controller
+    tag: 1.0.0-dev
+    digest: null
+  service:
+    type: ClusterIP

charts/ingress-nginx/ci/statefulset-pod-management-policy.yaml

@@ -0,0 +1,10 @@
+# Left blank to test default values
+controller:
+  kind: StatefulSet
+  podManagementPolicy: Parallel
+  image:
+    repository: ingress-controller/controller
+    tag: 1.0.0-dev
+    digest: null
+  service:
+    type: ClusterIP

charts/ingress-nginx/ci/statefulset-volume-claim-templates.yaml

@@ -0,0 +1,22 @@
+# Left blank to test default values
+controller:
+  kind: StatefulSet
+  volumeClaimTemplates:
+  - metadata:
+      name: cache
+    spec:
+      storageClassName: manual
+      hostPath: "/mnt/nginx-cache"
+      accessModes:
+      - ReadWriteOnce
+      capacity:
+        storage: 1Gi
+  extraVolumeMounts:
+  - name: cache
+    mountPath: /cache
+  image:
+    repository: ingress-controller/controller
+    tag: 1.0.0-dev
+    digest: null
+  service:
+    type: ClusterIP

charts/ingress-nginx/templates/controller-daemonset.yaml

@@ -1,237 +0,0 @@
-{{- if or (eq .Values.controller.kind "DaemonSet") (eq .Values.controller.kind "Both") -}}
-{{- include  "isControllerTagValid" . -}}
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  labels:
-    {{- include "ingress-nginx.labels" . | nindent 4 }}
-    app.kubernetes.io/component: controller
-    {{- with .Values.controller.labels }}
-    {{- toYaml . | nindent 4 }}
-    {{- end }}
-  name: {{ include "ingress-nginx.controller.fullname" . }}
-  namespace: {{ .Release.Namespace }}
-  {{- if .Values.controller.annotations }}
-  annotations: {{ toYaml .Values.controller.annotations | nindent 4 }}
-  {{- end }}
-spec:
-  selector:
-    matchLabels:
-      {{- include "ingress-nginx.selectorLabels" . | nindent 6 }}
-      app.kubernetes.io/component: controller
-  revisionHistoryLimit: {{ .Values.revisionHistoryLimit }}
-  {{- if .Values.controller.updateStrategy }}
-  updateStrategy: {{ toYaml .Values.controller.updateStrategy | nindent 4 }}
-  {{- end }}
-  minReadySeconds: {{ .Values.controller.minReadySeconds }}
-  template:
-    metadata:
-    {{- if .Values.controller.podAnnotations }}
-      annotations:
-      {{- range $key, $value := .Values.controller.podAnnotations }}
-        {{ $key }}: {{ $value | quote }}
-      {{- end }}
-    {{- end }}
-      labels:
-        {{- include "ingress-nginx.labels" . | nindent 8 }}
-        app.kubernetes.io/component: controller
-        {{- with .Values.controller.labels }}
-        {{- toYaml . | nindent 8 }}
-        {{- end }}
-      {{- if .Values.controller.podLabels }}
-        {{- toYaml .Values.controller.podLabels | nindent 8 }}
-      {{- end }}
-    spec:
-    {{- if .Values.controller.dnsConfig }}
-      dnsConfig: {{ toYaml .Values.controller.dnsConfig | nindent 8 }}
-    {{- end }}
-    {{- if .Values.controller.hostname }}
-      hostname: {{ toYaml .Values.controller.hostname | nindent 8 }}
-    {{- end }}
-      dnsPolicy: {{ .Values.controller.dnsPolicy }}
-    {{- if .Values.imagePullSecrets }}
-      imagePullSecrets: {{ toYaml .Values.imagePullSecrets | nindent 8 }}
-    {{- end }}
-    {{- if .Values.controller.priorityClassName }}
-      priorityClassName: {{ .Values.controller.priorityClassName }}
-    {{- end }}
-    {{- if or .Values.controller.podSecurityContext .Values.controller.sysctls }}
-      securityContext:
-    {{- end }}
-    {{- if .Values.controller.podSecurityContext  }}
-        {{- toYaml .Values.controller.podSecurityContext | nindent 8 }}
-    {{- end }}
-    {{- if .Values.controller.sysctls }}
-        sysctls:
-    {{- range $sysctl, $value := .Values.controller.sysctls }}
-        - name: {{ $sysctl | quote }}
-          value: {{ $value | quote }}
-    {{- end }}
-    {{- end }}
-    {{- if .Values.controller.shareProcessNamespace }}
-      shareProcessNamespace: {{ .Values.controller.shareProcessNamespace }}
-    {{- end }}
-      containers:
-        - name: {{ .Values.controller.containerName }}
-          {{- with .Values.controller.image }}
-          image: "{{- if .repository -}}{{ .repository }}{{ else }}{{ .registry }}/{{ include "ingress-nginx.image" . }}{{- end -}}:{{ .tag }}{{ include "ingress-nginx.imageDigest" . }}"
-          {{- end }}
-          imagePullPolicy: {{ .Values.controller.image.pullPolicy }}
-        {{- if .Values.controller.lifecycle }}
-          lifecycle: {{ toYaml .Values.controller.lifecycle | nindent 12 }}
-        {{- end }}
-          args:
-            {{- include "ingress-nginx.params" . | nindent 12 }}
-          securityContext: {{ include "controller.containerSecurityContext" . | nindent 12 }}
-          env:
-            - name: POD_NAME
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.name
-            - name: POD_NAMESPACE
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.namespace
-          {{- if .Values.controller.enableMimalloc }}
-            - name: LD_PRELOAD
-              value: /usr/local/lib/libmimalloc.so
-          {{- end }}
-          {{- if .Values.controller.extraEnvs }}
-            {{- toYaml .Values.controller.extraEnvs | nindent 12 }}
-          {{- end }}
-          {{- if .Values.controller.startupProbe }}
-          startupProbe: {{ toYaml .Values.controller.startupProbe | nindent 12 }}
-          {{- end }}
-          {{- if .Values.controller.livenessProbe }}
-          livenessProbe: {{ toYaml .Values.controller.livenessProbe | nindent 12 }}
-          {{- end }}
-          {{- if .Values.controller.readinessProbe }}
-          readinessProbe: {{ toYaml .Values.controller.readinessProbe | nindent 12 }}
-          {{- end }}
-          ports:
-          {{- range $key, $value := .Values.controller.containerPort }}
-            - name: {{ $key }}
-              containerPort: {{ $value }}
-              protocol: TCP
-              {{- if $.Values.controller.hostPort.enabled }}
-              hostPort: {{ index $.Values.controller.hostPort.ports $key | default $value }}
-              {{- end }}
-          {{- end }}
-          {{- if .Values.controller.metrics.enabled }}
-            - name: {{ .Values.controller.metrics.portName }}
-              containerPort: {{ .Values.controller.metrics.port }}
-              protocol: TCP
-          {{- end }}
-          {{- if .Values.controller.admissionWebhooks.enabled }}
-            - name: webhook
-              containerPort: {{ .Values.controller.admissionWebhooks.port }}
-              protocol: TCP
-          {{- end }}
-          {{- range $key, $value := .Values.tcp }}
-            - name: {{ if $.Values.portNamePrefix }}{{ $.Values.portNamePrefix }}-{{ end }}{{ $key }}-tcp
-              containerPort: {{ $key }}
-              protocol: TCP
-              {{- if $.Values.controller.hostPort.enabled }}
-              hostPort: {{ $key }}
-              {{- end }}
-          {{- end }}
-          {{- range $key, $value := .Values.udp }}
-            - name: {{ if $.Values.portNamePrefix }}{{ $.Values.portNamePrefix }}-{{ end }}{{ $key }}-udp
-              containerPort: {{ $key }}
-              protocol: UDP
-              {{- if $.Values.controller.hostPort.enabled }}
-              hostPort: {{ $key }}
-              {{- end }}
-          {{- end }}
-        {{- if (or .Values.controller.customTemplate.configMapName .Values.controller.extraVolumeMounts .Values.controller.admissionWebhooks.enabled .Values.controller.extraModules) }}
-          volumeMounts:
-          {{- if .Values.controller.extraModules }}
-            - name: modules
-              mountPath: /modules_mount
-          {{- end }}
-          {{- if .Values.controller.customTemplate.configMapName }}
-            - mountPath: /etc/nginx/template
-              name: nginx-template-volume
-              readOnly: true
-          {{- end }}
-          {{- if .Values.controller.admissionWebhooks.enabled }}
-            - name: webhook-cert
-              mountPath: /usr/local/certificates/
-              readOnly: true
-          {{- end }}
-          {{- if .Values.controller.extraVolumeMounts }}
-            {{- toYaml .Values.controller.extraVolumeMounts | nindent 12 }}
-          {{- end }}
-        {{- end }}
-        {{- if .Values.controller.resources }}
-          resources: {{ toYaml .Values.controller.resources | nindent 12 }}
-        {{- end }}
-      {{- if .Values.controller.extraContainers }}
-        {{ toYaml .Values.controller.extraContainers | nindent 8 }}
-      {{- end }}
-
-
-    {{- if (or .Values.controller.extraInitContainers .Values.controller.extraModules) }}
-      initContainers:
-      {{- if .Values.controller.extraInitContainers }}
-        {{ toYaml .Values.controller.extraInitContainers | nindent 8 }}
-      {{- end }}
-      {{- if .Values.controller.extraModules }}
-        {{- range .Values.controller.extraModules }}
-          {{ $containerSecurityContext := .containerSecurityContext | default $.Values.controller.containerSecurityContext }}
-{{ include "extraModules" (dict "name" .name "image" .image "containerSecurityContext" $containerSecurityContext) | indent 8 }}
-        {{- end }}
-      {{- end }}
-      {{- if .Values.controller.opentelemetry.enabled}}
-          {{ $otelContainerSecurityContext := $.Values.controller.opentelemetry.containerSecurityContext | default $.Values.controller.containerSecurityContext }}
-          {{- include "extraModules" (dict "name" "opentelemetry" "image" .Values.controller.opentelemetry.image "containerSecurityContext" $otelContainerSecurityContext) | nindent 8}}
-      {{- end}}
-    {{- end }}
-    {{- if .Values.controller.hostNetwork }}
-      hostNetwork: {{ .Values.controller.hostNetwork }}
-    {{- end }}
-    {{- if .Values.controller.nodeSelector }}
-      nodeSelector: {{ toYaml .Values.controller.nodeSelector | nindent 8 }}
-    {{- end }}
-    {{- if .Values.controller.tolerations }}
-      tolerations: {{ toYaml .Values.controller.tolerations | nindent 8 }}
-    {{- end }}
-    {{- if .Values.controller.affinity }}
-      affinity: {{ toYaml .Values.controller.affinity | nindent 8 }}
-    {{- end }}
-    {{- if .Values.controller.topologySpreadConstraints }}
-      topologySpreadConstraints: {{ toYaml .Values.controller.topologySpreadConstraints | nindent 8 }}
-    {{- end }}
-      serviceAccountName: {{ template "ingress-nginx.serviceAccountName" . }}
-      terminationGracePeriodSeconds: {{ .Values.controller.terminationGracePeriodSeconds }}
-    {{- if (or .Values.controller.customTemplate.configMapName .Values.controller.extraVolumeMounts .Values.controller.admissionWebhooks.enabled .Values.controller.extraVolumes .Values.controller.extraModules .Values.controller.opentelemetry.enabled) }}
-      volumes:
-      {{- if (or .Values.controller.extraModules .Values.controller.opentelemetry.enabled)}}
-        - name: modules
-          emptyDir: {}
-      {{- end }}
-      {{- if .Values.controller.customTemplate.configMapName }}
-        - name: nginx-template-volume
-          configMap:
-            name: {{ .Values.controller.customTemplate.configMapName }}
-            items:
-            - key: {{ .Values.controller.customTemplate.configMapKey }}
-              path: nginx.tmpl
-      {{- end }}
-      {{- if .Values.controller.admissionWebhooks.enabled }}
-        - name: webhook-cert
-          secret:
-            secretName: {{ include "ingress-nginx.fullname" . }}-admission
-        {{- if .Values.controller.admissionWebhooks.certManager.enabled }}
-            items:
-              - key: tls.crt
-                path: cert
-              - key: tls.key
-                path: key
-        {{- end }}
-      {{- end }}
-      {{- if .Values.controller.extraVolumes }}
-        {{ toYaml .Values.controller.extraVolumes | nindent 8 }}
-      {{- end }}
-    {{- end }}
-{{- end }}

charts/ingress-nginx/templates/controller-service.yaml

@@ -1,4 +1,4 @@
-{{- if and .Values.controller.service.enabled .Values.controller.service.external.enabled -}}
+{{- if or (eq .Values.controller.kind "StatefulSet") (and .Values.controller.service.enabled .Values.controller.service.external.enabled) -}}
 apiVersion: v1
 kind: Service
 metadata:

charts/ingress-nginx/templates/controller-workload.yaml

@@ -1,7 +1,21 @@
-{{- if or (eq .Values.controller.kind "Deployment") (eq .Values.controller.kind "Both") -}}
+{{- $kinds := list }}
-{{- include  "isControllerTagValid" . -}}
+{{- if eq .Values.controller.kind "Deployment" }}
+{{- $kinds = list "Deployment" }}
+{{- else if eq .Values.controller.kind "StatefulSet" }}
+{{- $kinds = list "StatefulSet" }}
+{{- else if eq .Values.controller.kind "DaemonSet" }}
+{{- $kinds = list "DaemonSet" }}
+{{- else if eq .Values.controller.kind "Both" }}
+{{- $kinds = list "DaemonSet" "Deployment" }}
+{{- else }}
+{{- fail "Unsupported value for controller.kind" }}
+{{- end }}
+{{- include  "isControllerTagValid" . }}
+{{- range $kind := $kinds }}
+---
+{{- with $ }}
 apiVersion: apps/v1
-kind: Deployment
+kind: {{ $kind }}
 metadata:
   labels:
     {{- include "ingress-nginx.labels" . | nindent 4 }}
@@ -19,15 +33,29 @@ spec:
     matchLabels:
       {{- include "ingress-nginx.selectorLabels" . | nindent 6 }}
       app.kubernetes.io/component: controller
-  {{- if not .Values.controller.autoscaling.enabled }}
+  {{- if and (ne $kind "DaemonSet") (not .Values.controller.autoscaling.enabled) }}
   replicas: {{ .Values.controller.replicaCount }}
   {{- end }}
   revisionHistoryLimit: {{ .Values.revisionHistoryLimit }}
   {{- if .Values.controller.updateStrategy }}
-  strategy:
+  {{- if eq $kind "Deployment" }}
-    {{ toYaml .Values.controller.updateStrategy | nindent 4 }}
+  strategy: {{ toYaml .Values.controller.updateStrategy | nindent 4 }}
+  {{- else }}
+  updateStrategy: {{ toYaml .Values.controller.updateStrategy | nindent 4 }}
   {{- end }}
+  {{- end }}
+  {{- if eq $kind "StatefulSet" }}
+  serviceName: {{ include "ingress-nginx.controller.fullname" . }}
+  {{- with .Values.controller.podManagementPolicy }}
+  podManagementPolicy: {{ . | quote }}
+  {{- end }}
+  {{- with .Values.controller.volumeClaimTemplates }}
+  volumeClaimTemplates: {{ toYaml . | nindent 4 }}
+  {{- end }}
+  {{- end }}
+  {{- if and (gt (.Values.controller.minReadySeconds | int) 0) (or (ne $kind "StatefulSet") (.Capabilities.KubeVersion.Version | semverCompare ">= 1.23")) }}
   minReadySeconds: {{ .Values.controller.minReadySeconds }}
+  {{- end }}
   template:
     metadata:
     {{- if .Values.controller.podAnnotations }}
@@ -241,3 +269,4 @@ spec:
       {{- end }}
     {{- end }}
 {{- end }}
+{{- end }}

charts/ingress-nginx/values.yaml

@@ -165,20 +165,21 @@ controller:
   #         key: FOO
   #         name: secret-resource
 
-  # -- Use a `DaemonSet` or `Deployment`
+  # -- Use a `DaemonSet` or `Deployment` or `StatefulSet`; `Both` creates a `DaemonSet` and a `Deployment`
   kind: Deployment
-  # -- Annotations to be added to the controller Deployment or DaemonSet
+
+  # -- Annotations to be added to the controller Deployment or DaemonSet or StatefulSet
   ##
   annotations: {}
   #  keel.sh/pollSchedule: "@every 60m"
 
-  # -- Labels to be added to the controller Deployment or DaemonSet and other resources that do not have option to specify labels
+  # -- Labels to be added to the controller Deployment or DaemonSet or StatefulSet and other resources that do not have option to specify labels
   ##
   labels: {}
   #  keel.sh/policy: patch
   #  keel.sh/trigger: poll
 
-  # -- The update strategy to apply to the Deployment or DaemonSet
+  # -- The update strategy to apply to the Deployment or DaemonSet or StatefulSet
   ##
   updateStrategy: {}
   #  rollingUpdate:
@@ -188,6 +189,15 @@ controller:
   # -- `minReadySeconds` to avoid killing pods before we are ready
   ##
   minReadySeconds: 0
+
+  # -- Pod management policy for StatefulSet
+  ##
+  podManagementPolicy: OrderedReady
+
+  # -- Volume claim templates for StatefulSet pods
+  ##
+  volumeClaimTemplates: []
+
   # -- Node tolerations for server scheduling to nodes with taints
   ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/
   ##