diff --git a/charts/ingress-nginx/README.md b/charts/ingress-nginx/README.md index cb3848634..972eb4067 100644 --- a/charts/ingress-nginx/README.md +++ b/charts/ingress-nginx/README.md @@ -294,7 +294,7 @@ Kubernetes: `>=1.20.0-0` | controller.admissionWebhooks.service.type | string | `"ClusterIP"` | | | controller.affinity | object | `{}` | Affinity and anti-affinity rules for server scheduling to nodes # Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity # | | controller.allowSnippetAnnotations | bool | `true` | This configuration defines if Ingress Controller should allow users to set their own *-snippet annotations, otherwise this is forbidden / dropped when users add those annotations. Global snippets in ConfigMap are still respected | -| controller.annotations | object | `{}` | Annotations to be added to the controller Deployment or DaemonSet # | +| controller.annotations | object | `{}` | Annotations to be added to the controller Deployment or DaemonSet or StatefulSet # | | controller.autoscaling.annotations | object | `{}` | | | controller.autoscaling.apiVersion | string | `"autoscaling/v2"` | | | controller.autoscaling.behavior | object | `{}` | | @@ -357,8 +357,8 @@ Kubernetes: `>=1.20.0-0` | controller.keda.restoreToOriginalReplicaCount | bool | `false` | | | controller.keda.scaledObject.annotations | object | `{}` | | | controller.keda.triggers | list | `[]` | | -| controller.kind | string | `"Deployment"` | Use a `DaemonSet` or `Deployment` | -| controller.labels | object | `{}` | Labels to be added to the controller Deployment or DaemonSet and other resources that do not have option to specify labels # | +| controller.kind | string | `"Deployment"` | Use a `DaemonSet` or `Deployment` or `StatefulSet`; `Both` creates a `DaemonSet` and a `Deployment` | +| controller.labels | object | `{}` | Labels to be added to the controller Deployment or DaemonSet or StatefulSet and other resources that do not have option to specify labels # | | controller.lifecycle | object | `{"preStop":{"exec":{"command":["/wait-shutdown"]}}}` | Improve connection draining when ingress controller pod is deleted using a lifecycle hook: With this new hook, we increased the default terminationGracePeriodSeconds from 30 seconds to 300, allowing the draining of connections up to five minutes. If the active connections end before that, the pod will terminate gracefully at that time. To effectively take advantage of this feature, the Configmap feature worker-shutdown-timeout new value is 240s instead of 10s. # | | controller.livenessProbe.failureThreshold | int | `5` | | | controller.livenessProbe.httpGet.path | string | `"/healthz"` | | @@ -398,6 +398,7 @@ Kubernetes: `>=1.20.0-0` | controller.opentelemetry.image | string | `"registry.k8s.io/ingress-nginx/opentelemetry:v20230312-helm-chart-4.5.2-28-g66a760794@sha256:40f766ac4a9832f36f217bb0e98d44c8d38faeccbfe861fbc1a76af7e9ab257f"` | | | controller.podAnnotations | object | `{}` | Annotations to be added to controller pods # | | controller.podLabels | object | `{}` | Labels to add to the pod container metadata | +| controller.podManagementPolicy | string | `"OrderedReady"` | Pod management policy for StatefulSet # | | controller.podSecurityContext | object | `{}` | Security Context policies for controller pods | | controller.priorityClassName | string | `""` | | | controller.proxySetHeaders | object | `{}` | Will add custom headers before sending traffic to backends according to https://github.com/kubernetes/ingress-nginx/tree/main/docs/examples/customization/custom-headers | @@ -452,7 +453,8 @@ Kubernetes: `>=1.20.0-0` | controller.topologySpreadConstraints | list | `[]` | Topology spread constraints rely on node labels to identify the topology domain(s) that each Node is in. # Ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ # | | controller.udp.annotations | object | `{}` | Annotations to be added to the udp config configmap | | controller.udp.configMapNamespace | string | `""` | Allows customization of the udp-services-configmap; defaults to $(POD_NAMESPACE) | -| controller.updateStrategy | object | `{}` | The update strategy to apply to the Deployment or DaemonSet # | +| controller.updateStrategy | object | `{}` | The update strategy to apply to the Deployment or DaemonSet or StatefulSet # | +| controller.volumeClaimTemplates | list | `[]` | Volume claim templates for StatefulSet pods # | | controller.watchIngressWithoutClass | bool | `false` | Process Ingress objects without ingressClass annotation/ingressClassName field Overrides value for --watch-ingress-without-class flag of the controller binary Defaults to false | | defaultBackend.affinity | object | `{}` | | | defaultBackend.autoscaling.annotations | object | `{}` | | diff --git a/charts/ingress-nginx/ci/statefulset-default-values.yaml b/charts/ingress-nginx/ci/statefulset-default-values.yaml new file mode 100644 index 000000000..ba93c2b51 --- /dev/null +++ b/charts/ingress-nginx/ci/statefulset-default-values.yaml @@ -0,0 +1,9 @@ +# Left blank to test default values +controller: + kind: StatefulSet + image: + repository: ingress-controller/controller + tag: 1.0.0-dev + digest: null + service: + type: ClusterIP diff --git a/charts/ingress-nginx/ci/statefulset-pod-management-policy.yaml b/charts/ingress-nginx/ci/statefulset-pod-management-policy.yaml new file mode 100644 index 000000000..4520ac297 --- /dev/null +++ b/charts/ingress-nginx/ci/statefulset-pod-management-policy.yaml @@ -0,0 +1,10 @@ +# Left blank to test default values +controller: + kind: StatefulSet + podManagementPolicy: Parallel + image: + repository: ingress-controller/controller + tag: 1.0.0-dev + digest: null + service: + type: ClusterIP diff --git a/charts/ingress-nginx/ci/statefulset-volume-claim-templates.yaml b/charts/ingress-nginx/ci/statefulset-volume-claim-templates.yaml new file mode 100644 index 000000000..7b376dd21 --- /dev/null +++ b/charts/ingress-nginx/ci/statefulset-volume-claim-templates.yaml @@ -0,0 +1,22 @@ +# Left blank to test default values +controller: + kind: StatefulSet + volumeClaimTemplates: + - metadata: + name: cache + spec: + storageClassName: manual + hostPath: "/mnt/nginx-cache" + accessModes: + - ReadWriteOnce + capacity: + storage: 1Gi + extraVolumeMounts: + - name: cache + mountPath: /cache + image: + repository: ingress-controller/controller + tag: 1.0.0-dev + digest: null + service: + type: ClusterIP diff --git a/charts/ingress-nginx/templates/controller-daemonset.yaml b/charts/ingress-nginx/templates/controller-daemonset.yaml deleted file mode 100644 index bce21a7d6..000000000 --- a/charts/ingress-nginx/templates/controller-daemonset.yaml +++ /dev/null @@ -1,237 +0,0 @@ -{{- if or (eq .Values.controller.kind "DaemonSet") (eq .Values.controller.kind "Both") -}} -{{- include "isControllerTagValid" . -}} -apiVersion: apps/v1 -kind: DaemonSet -metadata: - labels: - {{- include "ingress-nginx.labels" . | nindent 4 }} - app.kubernetes.io/component: controller - {{- with .Values.controller.labels }} - {{- toYaml . | nindent 4 }} - {{- end }} - name: {{ include "ingress-nginx.controller.fullname" . }} - namespace: {{ .Release.Namespace }} - {{- if .Values.controller.annotations }} - annotations: {{ toYaml .Values.controller.annotations | nindent 4 }} - {{- end }} -spec: - selector: - matchLabels: - {{- include "ingress-nginx.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: controller - revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} - {{- if .Values.controller.updateStrategy }} - updateStrategy: {{ toYaml .Values.controller.updateStrategy | nindent 4 }} - {{- end }} - minReadySeconds: {{ .Values.controller.minReadySeconds }} - template: - metadata: - {{- if .Values.controller.podAnnotations }} - annotations: - {{- range $key, $value := .Values.controller.podAnnotations }} - {{ $key }}: {{ $value | quote }} - {{- end }} - {{- end }} - labels: - {{- include "ingress-nginx.labels" . | nindent 8 }} - app.kubernetes.io/component: controller - {{- with .Values.controller.labels }} - {{- toYaml . | nindent 8 }} - {{- end }} - {{- if .Values.controller.podLabels }} - {{- toYaml .Values.controller.podLabels | nindent 8 }} - {{- end }} - spec: - {{- if .Values.controller.dnsConfig }} - dnsConfig: {{ toYaml .Values.controller.dnsConfig | nindent 8 }} - {{- end }} - {{- if .Values.controller.hostname }} - hostname: {{ toYaml .Values.controller.hostname | nindent 8 }} - {{- end }} - dnsPolicy: {{ .Values.controller.dnsPolicy }} - {{- if .Values.imagePullSecrets }} - imagePullSecrets: {{ toYaml .Values.imagePullSecrets | nindent 8 }} - {{- end }} - {{- if .Values.controller.priorityClassName }} - priorityClassName: {{ .Values.controller.priorityClassName }} - {{- end }} - {{- if or .Values.controller.podSecurityContext .Values.controller.sysctls }} - securityContext: - {{- end }} - {{- if .Values.controller.podSecurityContext }} - {{- toYaml .Values.controller.podSecurityContext | nindent 8 }} - {{- end }} - {{- if .Values.controller.sysctls }} - sysctls: - {{- range $sysctl, $value := .Values.controller.sysctls }} - - name: {{ $sysctl | quote }} - value: {{ $value | quote }} - {{- end }} - {{- end }} - {{- if .Values.controller.shareProcessNamespace }} - shareProcessNamespace: {{ .Values.controller.shareProcessNamespace }} - {{- end }} - containers: - - name: {{ .Values.controller.containerName }} - {{- with .Values.controller.image }} - image: "{{- if .repository -}}{{ .repository }}{{ else }}{{ .registry }}/{{ include "ingress-nginx.image" . }}{{- end -}}:{{ .tag }}{{ include "ingress-nginx.imageDigest" . }}" - {{- end }} - imagePullPolicy: {{ .Values.controller.image.pullPolicy }} - {{- if .Values.controller.lifecycle }} - lifecycle: {{ toYaml .Values.controller.lifecycle | nindent 12 }} - {{- end }} - args: - {{- include "ingress-nginx.params" . | nindent 12 }} - securityContext: {{ include "controller.containerSecurityContext" . | nindent 12 }} - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - {{- if .Values.controller.enableMimalloc }} - - name: LD_PRELOAD - value: /usr/local/lib/libmimalloc.so - {{- end }} - {{- if .Values.controller.extraEnvs }} - {{- toYaml .Values.controller.extraEnvs | nindent 12 }} - {{- end }} - {{- if .Values.controller.startupProbe }} - startupProbe: {{ toYaml .Values.controller.startupProbe | nindent 12 }} - {{- end }} - {{- if .Values.controller.livenessProbe }} - livenessProbe: {{ toYaml .Values.controller.livenessProbe | nindent 12 }} - {{- end }} - {{- if .Values.controller.readinessProbe }} - readinessProbe: {{ toYaml .Values.controller.readinessProbe | nindent 12 }} - {{- end }} - ports: - {{- range $key, $value := .Values.controller.containerPort }} - - name: {{ $key }} - containerPort: {{ $value }} - protocol: TCP - {{- if $.Values.controller.hostPort.enabled }} - hostPort: {{ index $.Values.controller.hostPort.ports $key | default $value }} - {{- end }} - {{- end }} - {{- if .Values.controller.metrics.enabled }} - - name: {{ .Values.controller.metrics.portName }} - containerPort: {{ .Values.controller.metrics.port }} - protocol: TCP - {{- end }} - {{- if .Values.controller.admissionWebhooks.enabled }} - - name: webhook - containerPort: {{ .Values.controller.admissionWebhooks.port }} - protocol: TCP - {{- end }} - {{- range $key, $value := .Values.tcp }} - - name: {{ if $.Values.portNamePrefix }}{{ $.Values.portNamePrefix }}-{{ end }}{{ $key }}-tcp - containerPort: {{ $key }} - protocol: TCP - {{- if $.Values.controller.hostPort.enabled }} - hostPort: {{ $key }} - {{- end }} - {{- end }} - {{- range $key, $value := .Values.udp }} - - name: {{ if $.Values.portNamePrefix }}{{ $.Values.portNamePrefix }}-{{ end }}{{ $key }}-udp - containerPort: {{ $key }} - protocol: UDP - {{- if $.Values.controller.hostPort.enabled }} - hostPort: {{ $key }} - {{- end }} - {{- end }} - {{- if (or .Values.controller.customTemplate.configMapName .Values.controller.extraVolumeMounts .Values.controller.admissionWebhooks.enabled .Values.controller.extraModules) }} - volumeMounts: - {{- if .Values.controller.extraModules }} - - name: modules - mountPath: /modules_mount - {{- end }} - {{- if .Values.controller.customTemplate.configMapName }} - - mountPath: /etc/nginx/template - name: nginx-template-volume - readOnly: true - {{- end }} - {{- if .Values.controller.admissionWebhooks.enabled }} - - name: webhook-cert - mountPath: /usr/local/certificates/ - readOnly: true - {{- end }} - {{- if .Values.controller.extraVolumeMounts }} - {{- toYaml .Values.controller.extraVolumeMounts | nindent 12 }} - {{- end }} - {{- end }} - {{- if .Values.controller.resources }} - resources: {{ toYaml .Values.controller.resources | nindent 12 }} - {{- end }} - {{- if .Values.controller.extraContainers }} - {{ toYaml .Values.controller.extraContainers | nindent 8 }} - {{- end }} - - - {{- if (or .Values.controller.extraInitContainers .Values.controller.extraModules) }} - initContainers: - {{- if .Values.controller.extraInitContainers }} - {{ toYaml .Values.controller.extraInitContainers | nindent 8 }} - {{- end }} - {{- if .Values.controller.extraModules }} - {{- range .Values.controller.extraModules }} - {{ $containerSecurityContext := .containerSecurityContext | default $.Values.controller.containerSecurityContext }} -{{ include "extraModules" (dict "name" .name "image" .image "containerSecurityContext" $containerSecurityContext) | indent 8 }} - {{- end }} - {{- end }} - {{- if .Values.controller.opentelemetry.enabled}} - {{ $otelContainerSecurityContext := $.Values.controller.opentelemetry.containerSecurityContext | default $.Values.controller.containerSecurityContext }} - {{- include "extraModules" (dict "name" "opentelemetry" "image" .Values.controller.opentelemetry.image "containerSecurityContext" $otelContainerSecurityContext) | nindent 8}} - {{- end}} - {{- end }} - {{- if .Values.controller.hostNetwork }} - hostNetwork: {{ .Values.controller.hostNetwork }} - {{- end }} - {{- if .Values.controller.nodeSelector }} - nodeSelector: {{ toYaml .Values.controller.nodeSelector | nindent 8 }} - {{- end }} - {{- if .Values.controller.tolerations }} - tolerations: {{ toYaml .Values.controller.tolerations | nindent 8 }} - {{- end }} - {{- if .Values.controller.affinity }} - affinity: {{ toYaml .Values.controller.affinity | nindent 8 }} - {{- end }} - {{- if .Values.controller.topologySpreadConstraints }} - topologySpreadConstraints: {{ toYaml .Values.controller.topologySpreadConstraints | nindent 8 }} - {{- end }} - serviceAccountName: {{ template "ingress-nginx.serviceAccountName" . }} - terminationGracePeriodSeconds: {{ .Values.controller.terminationGracePeriodSeconds }} - {{- if (or .Values.controller.customTemplate.configMapName .Values.controller.extraVolumeMounts .Values.controller.admissionWebhooks.enabled .Values.controller.extraVolumes .Values.controller.extraModules .Values.controller.opentelemetry.enabled) }} - volumes: - {{- if (or .Values.controller.extraModules .Values.controller.opentelemetry.enabled)}} - - name: modules - emptyDir: {} - {{- end }} - {{- if .Values.controller.customTemplate.configMapName }} - - name: nginx-template-volume - configMap: - name: {{ .Values.controller.customTemplate.configMapName }} - items: - - key: {{ .Values.controller.customTemplate.configMapKey }} - path: nginx.tmpl - {{- end }} - {{- if .Values.controller.admissionWebhooks.enabled }} - - name: webhook-cert - secret: - secretName: {{ include "ingress-nginx.fullname" . }}-admission - {{- if .Values.controller.admissionWebhooks.certManager.enabled }} - items: - - key: tls.crt - path: cert - - key: tls.key - path: key - {{- end }} - {{- end }} - {{- if .Values.controller.extraVolumes }} - {{ toYaml .Values.controller.extraVolumes | nindent 8 }} - {{- end }} - {{- end }} -{{- end }} diff --git a/charts/ingress-nginx/templates/controller-service.yaml b/charts/ingress-nginx/templates/controller-service.yaml index 2b28196de..fe8251c73 100644 --- a/charts/ingress-nginx/templates/controller-service.yaml +++ b/charts/ingress-nginx/templates/controller-service.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.controller.service.enabled .Values.controller.service.external.enabled -}} +{{- if or (eq .Values.controller.kind "StatefulSet") (and .Values.controller.service.enabled .Values.controller.service.external.enabled) -}} apiVersion: v1 kind: Service metadata: diff --git a/charts/ingress-nginx/templates/controller-deployment.yaml b/charts/ingress-nginx/templates/controller-workload.yaml similarity index 88% rename from charts/ingress-nginx/templates/controller-deployment.yaml rename to charts/ingress-nginx/templates/controller-workload.yaml index 323d87623..54cdcfeaf 100644 --- a/charts/ingress-nginx/templates/controller-deployment.yaml +++ b/charts/ingress-nginx/templates/controller-workload.yaml @@ -1,7 +1,21 @@ -{{- if or (eq .Values.controller.kind "Deployment") (eq .Values.controller.kind "Both") -}} -{{- include "isControllerTagValid" . -}} +{{- $kinds := list }} +{{- if eq .Values.controller.kind "Deployment" }} +{{- $kinds = list "Deployment" }} +{{- else if eq .Values.controller.kind "StatefulSet" }} +{{- $kinds = list "StatefulSet" }} +{{- else if eq .Values.controller.kind "DaemonSet" }} +{{- $kinds = list "DaemonSet" }} +{{- else if eq .Values.controller.kind "Both" }} +{{- $kinds = list "DaemonSet" "Deployment" }} +{{- else }} +{{- fail "Unsupported value for controller.kind" }} +{{- end }} +{{- include "isControllerTagValid" . }} +{{- range $kind := $kinds }} +--- +{{- with $ }} apiVersion: apps/v1 -kind: Deployment +kind: {{ $kind }} metadata: labels: {{- include "ingress-nginx.labels" . | nindent 4 }} @@ -19,15 +33,29 @@ spec: matchLabels: {{- include "ingress-nginx.selectorLabels" . | nindent 6 }} app.kubernetes.io/component: controller - {{- if not .Values.controller.autoscaling.enabled }} + {{- if and (ne $kind "DaemonSet") (not .Values.controller.autoscaling.enabled) }} replicas: {{ .Values.controller.replicaCount }} {{- end }} revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} {{- if .Values.controller.updateStrategy }} - strategy: - {{ toYaml .Values.controller.updateStrategy | nindent 4 }} + {{- if eq $kind "Deployment" }} + strategy: {{ toYaml .Values.controller.updateStrategy | nindent 4 }} + {{- else }} + updateStrategy: {{ toYaml .Values.controller.updateStrategy | nindent 4 }} {{- end }} + {{- end }} + {{- if eq $kind "StatefulSet" }} + serviceName: {{ include "ingress-nginx.controller.fullname" . }} + {{- with .Values.controller.podManagementPolicy }} + podManagementPolicy: {{ . | quote }} + {{- end }} + {{- with .Values.controller.volumeClaimTemplates }} + volumeClaimTemplates: {{ toYaml . | nindent 4 }} + {{- end }} + {{- end }} + {{- if and (gt (.Values.controller.minReadySeconds | int) 0) (or (ne $kind "StatefulSet") (.Capabilities.KubeVersion.Version | semverCompare ">= 1.23")) }} minReadySeconds: {{ .Values.controller.minReadySeconds }} + {{- end }} template: metadata: {{- if .Values.controller.podAnnotations }} @@ -241,3 +269,4 @@ spec: {{- end }} {{- end }} {{- end }} +{{- end }} diff --git a/charts/ingress-nginx/values.yaml b/charts/ingress-nginx/values.yaml index c2e8cdcd7..b0a422245 100644 --- a/charts/ingress-nginx/values.yaml +++ b/charts/ingress-nginx/values.yaml @@ -165,20 +165,21 @@ controller: # key: FOO # name: secret-resource - # -- Use a `DaemonSet` or `Deployment` + # -- Use a `DaemonSet` or `Deployment` or `StatefulSet`; `Both` creates a `DaemonSet` and a `Deployment` kind: Deployment - # -- Annotations to be added to the controller Deployment or DaemonSet + + # -- Annotations to be added to the controller Deployment or DaemonSet or StatefulSet ## annotations: {} # keel.sh/pollSchedule: "@every 60m" - # -- Labels to be added to the controller Deployment or DaemonSet and other resources that do not have option to specify labels + # -- Labels to be added to the controller Deployment or DaemonSet or StatefulSet and other resources that do not have option to specify labels ## labels: {} # keel.sh/policy: patch # keel.sh/trigger: poll - # -- The update strategy to apply to the Deployment or DaemonSet + # -- The update strategy to apply to the Deployment or DaemonSet or StatefulSet ## updateStrategy: {} # rollingUpdate: @@ -188,6 +189,15 @@ controller: # -- `minReadySeconds` to avoid killing pods before we are ready ## minReadySeconds: 0 + + # -- Pod management policy for StatefulSet + ## + podManagementPolicy: OrderedReady + + # -- Volume claim templates for StatefulSet pods + ## + volumeClaimTemplates: [] + # -- Node tolerations for server scheduling to nodes with taints ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ ##