feat(helm): Optionally use cert-manager instead admission patch (#9279)

2022-12-07 13:16:38 +01:00 · 2022-12-07 13:16:38 +01:00 · d7674e4323
commit d7674e4323
parent ad4655a568
16 changed files with 147 additions and 9 deletions
--- a/charts/ingress-nginx/README.md
+++ b/charts/ingress-nginx/README.md
@ -217,6 +217,21 @@ With nginx-ingress-controller version 0.25+, the nginx ingress controller pod ex

 With nginx-ingress-controller in 0.25.* work only with kubernetes 1.14+, 0.26 fix [this issue](https://github.com/kubernetes/ingress-nginx/pull/4521)

+#### How the Chart Configures the Hooks
+A validating and configuration requires the endpoint to which the request is sent to use TLS. It is possible to set up custom certificates to do this, but in most cases, a self-signed certificate is enough. The setup of this component requires some more complex orchestration when using helm. The steps are created to be idempotent and to allow turning the feature on and off without running into helm quirks.
+
+1. A pre-install hook provisions a certificate into the same namespace using a format compatible with provisioning using end user certificates. If the certificate already exists, the hook exits.
+2. The ingress nginx controller pod is configured to use a TLS proxy container, which will load that certificate.
+3. Validating and Mutating webhook configurations are created in the cluster.
+4. A post-install hook reads the CA from the secret created by step 1 and patches the Validating and Mutating webhook configurations. This process will allow a custom CA provisioned by some other process to also be patched into the webhook configurations. The chosen failure policy is also patched into the webhook configurations
+
+#### Alternatives
+It should be possible to use [cert-manager/cert-manager](https://github.com/cert-manager/cert-manager) if a more complete solution is required.
+
+You can enable automatic self-signed TLS certificate provisioning via cert-manager by setting the `controller.admissionWebhooks.certManager.enable` value to true.
+
+Please ensure that cert-manager is correctly installed and configured.
+
 ### Helm Error When Upgrading: spec.clusterIP: Invalid value: ""

 If you are upgrading this chart from a version between 0.31.0 and 1.2.2 then you may get an error like this:
@ -240,6 +255,9 @@ Kubernetes: `>=1.20.0-0`
 | commonLabels | object | `{}` |  |
 | controller.addHeaders | object | `{}` | Will add custom headers before sending response traffic to the client according to: https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#add-headers |
 | controller.admissionWebhooks.annotations | object | `{}` |  |
+| controller.admissionWebhooks.certManager.admissionCert.duration | string | `""` |  |
+| controller.admissionWebhooks.certManager.enabled | bool | `false` |  |
+| controller.admissionWebhooks.certManager.rootCert.duration | string | `""` |  |
 | controller.admissionWebhooks.certificate | string | `"/usr/local/certificates/cert"` |  |
 | controller.admissionWebhooks.createSecretJob.resources | object | `{}` |  |
 | controller.admissionWebhooks.createSecretJob.securityContext.allowPrivilegeEscalation | bool | `false` |  |
--- a/charts/ingress-nginx/README.md.gotmpl
+++ b/charts/ingress-nginx/README.md.gotmpl
@ -216,6 +216,21 @@ With nginx-ingress-controller version 0.25+, the nginx ingress controller pod ex

 With nginx-ingress-controller in 0.25.* work only with kubernetes 1.14+, 0.26 fix [this issue](https://github.com/kubernetes/ingress-nginx/pull/4521)

+#### How the Chart Configures the Hooks
+A validating and configuration requires the endpoint to which the request is sent to use TLS. It is possible to set up custom certificates to do this, but in most cases, a self-signed certificate is enough. The setup of this component requires some more complex orchestration when using helm. The steps are created to be idempotent and to allow turning the feature on and off without running into helm quirks.
+
+1. A pre-install hook provisions a certificate into the same namespace using a format compatible with provisioning using end user certificates. If the certificate already exists, the hook exits.
+2. The ingress nginx controller pod is configured to use a TLS proxy container, which will load that certificate.
+3. Validating and Mutating webhook configurations are created in the cluster.
+4. A post-install hook reads the CA from the secret created by step 1 and patches the Validating and Mutating webhook configurations. This process will allow a custom CA provisioned by some other process to also be patched into the webhook configurations. The chosen failure policy is also patched into the webhook configurations
+
+#### Alternatives
+It should be possible to use [cert-manager/cert-manager](https://github.com/cert-manager/cert-manager) if a more complete solution is required.
+
+You can enable automatic self-signed TLS certificate provisioning via cert-manager by setting the `controller.admissionWebhooks.certManager.enable` value to true.
+
+Please ensure that cert-manager is correctly installed and configured.
+
 ### Helm Error When Upgrading: spec.clusterIP: Invalid value: ""

 If you are upgrading this chart from a version between 0.31.0 and 1.2.2 then you may get an error like this:
--- a/charts/ingress-nginx/ci/controller-admission-tls-cert-manager-values.yaml
+++ b/charts/ingress-nginx/ci/controller-admission-tls-cert-manager-values.yaml
@ -0,0 +1,6 @@
+controller:
+  admissionWebhooks:
+    certManager:
+      enabled: true
+  service:
+    type: ClusterIP
--- a/charts/ingress-nginx/templates/admission-webhooks/cert-manager.yaml
+++ b/charts/ingress-nginx/templates/admission-webhooks/cert-manager.yaml
@ -0,0 +1,63 @@
+{{- if and .Values.controller.admissionWebhooks.enabled .Values.controller.admissionWebhooks.certManager.enabled -}}
+{{- if not .Values.controller.admissionWebhooks.certManager.issuerRef -}}
+# Create a selfsigned Issuer, in order to create a root CA certificate for
+# signing webhook serving certificates
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  name: {{ include "ingress-nginx.fullname" . }}-self-signed-issuer
+  namespace: {{ .Release.Namespace }}
+spec:
+  selfSigned: {}
+---
+# Generate a CA Certificate used to sign certificates for the webhook
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: {{ include "ingress-nginx.fullname" . }}-root-cert
+  namespace: {{ .Release.Namespace }}
+spec:
+  secretName: {{ include "ingress-nginx.fullname" . }}-root-cert
+  duration: {{ .Values.controller.admissionWebhooks.certManager.rootCert.duration | default "43800h0m0s" | quote }}
+  issuerRef:
+    name: {{ include "ingress-nginx.fullname" . }}-self-signed-issuer
+  commonName: "ca.webhook.ingress-nginx"
+  isCA: true
+  subject:
+    organizations:
+      - ingress-nginx
+---
+# Create an Issuer that uses the above generated CA certificate to issue certs
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  name: {{ include "ingress-nginx.fullname" . }}-root-issuer
+  namespace: {{ .Release.Namespace }}
+spec:
+  ca:
+    secretName: {{ include "ingress-nginx.fullname" . }}-root-cert
+{{- end }}
+---
+# generate a server certificate for the apiservices to use
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: {{ include "ingress-nginx.fullname" . }}-admission
+  namespace: {{ .Release.Namespace }}
+spec:
+  secretName: {{ include "ingress-nginx.fullname" . }}-admission
+  duration: {{ .Values.controller.admissionWebhooks.certManager.admissionCert.duration | default "8760h0m0s" | quote }}
+  issuerRef:
+    {{- if .Values.controller.admissionWebhooks.certManager.issuerRef }}
+    {{- toYaml .Values.controller.admissionWebhooks.certManager.issuerRef | nindent 4 }}
+    {{- else }}
+    name: {{ include "ingress-nginx.fullname" . }}-root-issuer
+    {{- end }}
+  dnsNames:
+    - {{ include "ingress-nginx.controller.fullname" . }}-admission
+    - {{ include "ingress-nginx.controller.fullname" . }}-admission.{{ .Release.Namespace }}
+    - {{ include "ingress-nginx.controller.fullname" . }}-admission.{{ .Release.Namespace }}.svc
+  subject:
+    organizations:
+      - ingress-nginx-admission
+{{- end -}}
--- a/charts/ingress-nginx/templates/admission-webhooks/job-patch/clusterrole.yaml
+++ b/charts/ingress-nginx/templates/admission-webhooks/job-patch/clusterrole.yaml
@ -1,4 +1,4 @@
-{{- if and .Values.controller.admissionWebhooks.enabled .Values.controller.admissionWebhooks.patch.enabled -}}
+{{- if and .Values.controller.admissionWebhooks.enabled .Values.controller.admissionWebhooks.patch.enabled (not .Values.controller.admissionWebhooks.certManager.enabled) -}}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
--- a/charts/ingress-nginx/templates/admission-webhooks/job-patch/clusterrolebinding.yaml
+++ b/charts/ingress-nginx/templates/admission-webhooks/job-patch/clusterrolebinding.yaml
@ -1,4 +1,4 @@
-{{- if and .Values.controller.admissionWebhooks.enabled .Values.controller.admissionWebhooks.patch.enabled -}}
+{{- if and .Values.controller.admissionWebhooks.enabled .Values.controller.admissionWebhooks.patch.enabled (not .Values.controller.admissionWebhooks.certManager.enabled) -}}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
--- a/charts/ingress-nginx/templates/admission-webhooks/job-patch/job-createSecret.yaml
+++ b/charts/ingress-nginx/templates/admission-webhooks/job-patch/job-createSecret.yaml
@ -1,4 +1,4 @@
-{{- if and .Values.controller.admissionWebhooks.enabled .Values.controller.admissionWebhooks.patch.enabled -}}
+{{- if and .Values.controller.admissionWebhooks.enabled .Values.controller.admissionWebhooks.patch.enabled (not .Values.controller.admissionWebhooks.certManager.enabled) -}}
 apiVersion: batch/v1
 kind: Job
 metadata:
--- a/charts/ingress-nginx/templates/admission-webhooks/job-patch/job-patchWebhook.yaml
+++ b/charts/ingress-nginx/templates/admission-webhooks/job-patch/job-patchWebhook.yaml
@ -1,4 +1,4 @@
-{{- if and .Values.controller.admissionWebhooks.enabled .Values.controller.admissionWebhooks.patch.enabled -}}
+{{- if and .Values.controller.admissionWebhooks.enabled .Values.controller.admissionWebhooks.patch.enabled (not .Values.controller.admissionWebhooks.certManager.enabled) -}}
 apiVersion: batch/v1
 kind: Job
 metadata:
--- a/charts/ingress-nginx/templates/admission-webhooks/job-patch/role.yaml
+++ b/charts/ingress-nginx/templates/admission-webhooks/job-patch/role.yaml
@ -1,4 +1,4 @@
-{{- if and .Values.controller.admissionWebhooks.enabled .Values.controller.admissionWebhooks.patch.enabled -}}
+{{- if and .Values.controller.admissionWebhooks.enabled .Values.controller.admissionWebhooks.patch.enabled (not .Values.controller.admissionWebhooks.certManager.enabled) -}}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
--- a/charts/ingress-nginx/templates/admission-webhooks/job-patch/rolebinding.yaml
+++ b/charts/ingress-nginx/templates/admission-webhooks/job-patch/rolebinding.yaml
@ -1,4 +1,4 @@
-{{- if and .Values.controller.admissionWebhooks.enabled .Values.controller.admissionWebhooks.patch.enabled -}}
+{{- if and .Values.controller.admissionWebhooks.enabled .Values.controller.admissionWebhooks.patch.enabled (not .Values.controller.admissionWebhooks.certManager.enabled) -}}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
--- a/charts/ingress-nginx/templates/admission-webhooks/job-patch/serviceaccount.yaml
+++ b/charts/ingress-nginx/templates/admission-webhooks/job-patch/serviceaccount.yaml
@ -1,4 +1,4 @@
-{{- if and .Values.controller.admissionWebhooks.enabled .Values.controller.admissionWebhooks.patch.enabled -}}
+{{- if and .Values.controller.admissionWebhooks.enabled .Values.controller.admissionWebhooks.patch.enabled (not .Values.controller.admissionWebhooks.certManager.enabled) -}}
 apiVersion: v1
 kind: ServiceAccount
 metadata:
--- a/charts/ingress-nginx/templates/admission-webhooks/validating-webhook.yaml
+++ b/charts/ingress-nginx/templates/admission-webhooks/validating-webhook.yaml
@ -4,8 +4,13 @@
 apiVersion: admissionregistration.k8s.io/v1
 kind: ValidatingWebhookConfiguration
 metadata:
+  annotations:
+  {{- if .Values.controller.admissionWebhooks.certManager.enabled }}
+    certmanager.k8s.io/inject-ca-from: {{ printf "%s/%s-admission" .Release.Namespace (include "ingress-nginx.fullname" .) | quote }}
+    cert-manager.io/inject-ca-from: {{ printf "%s/%s-admission" .Release.Namespace (include "ingress-nginx.fullname" .) | quote }}
+  {{- end }}
  {{- if .Values.controller.admissionWebhooks.annotations }}
-  annotations: {{ toYaml .Values.controller.admissionWebhooks.annotations | nindent 4 }}
+    {{- toYaml .Values.controller.admissionWebhooks.annotations | nindent 4 }}
  {{- end }}
  labels:
    {{- include "ingress-nginx.labels" . | nindent 4 }}
--- a/charts/ingress-nginx/templates/controller-daemonset.yaml
+++ b/charts/ingress-nginx/templates/controller-daemonset.yaml
@ -225,6 +225,13 @@ spec:
        - name: webhook-cert
          secret:
            secretName: {{ include "ingress-nginx.fullname" . }}-admission
+        {{- if .Values.controller.admissionWebhooks.certManager.enabled }}
+            items:
+              - key: tls.crt
+                path: cert
+              - key: tls.key
+                path: key
+        {{- end }}
      {{- end }}
      {{- if .Values.controller.extraVolumes }}
        {{ toYaml .Values.controller.extraVolumes | nindent 8 }}
--- a/charts/ingress-nginx/templates/controller-deployment.yaml
+++ b/charts/ingress-nginx/templates/controller-deployment.yaml
@ -228,6 +228,13 @@ spec:
        - name: webhook-cert
          secret:
            secretName: {{ include "ingress-nginx.fullname" . }}-admission
+        {{- if .Values.controller.admissionWebhooks.certManager.enabled }}
+            items:
+              - key: tls.crt
+                path: cert
+              - key: tls.key
+                path: key
+        {{- end }}
      {{- end }}
      {{- if .Values.controller.extraVolumes }}
        {{ toYaml .Values.controller.extraVolumes | nindent 8 }}
--- a/charts/ingress-nginx/values.yaml
+++ b/charts/ingress-nginx/values.yaml
@ -677,6 +677,17 @@ controller:
        runAsUser: 2000
        fsGroup: 2000

+    # Use certmanager to generate webhook certs
+    certManager:
+      enabled: false
+      # self-signed root certificate
+      rootCert:
+        duration: ""  # default to be 5y
+      admissionCert:
+        duration: ""  # default to be 1y
+      # issuerRef:
+      #   name: "issuer"
+      #   kind: "ClusterIssuer"

  metrics:
    port: 10254
--- a/test/e2e/run-chart-test.sh
+++ b/test/e2e/run-chart-test.sh
@ -90,6 +90,12 @@ echo "[dev-env] copying docker images to cluster..."

 kind load docker-image --name="${KIND_CLUSTER_NAME}" --nodes=${KIND_WORKERS} ${REGISTRY}/controller:${TAG}

+if [ "${SKIP_CERT_MANAGER_CREATION:-false}" = "false" ]; then
+  echo "[dev-env] apply cert-manager ..."
+  kubectl apply --wait -f https://github.com/cert-manager/cert-manager/releases/download/v1.10.0/cert-manager.yaml
+  sleep 10
+fi
+
 echo "[dev-env] running helm chart e2e tests..."
 # Uses a custom chart-testing image to avoid timeouts waiting for namespace deletion.
 # The changes can be found here: https://github.com/aledbf/chart-testing/commit/41fe0ae0733d0c9a538099fb3cec522e888e3d82