diff --git a/charts/ingress-nginx/README.md b/charts/ingress-nginx/README.md index 36c98984c..0e2800d94 100644 --- a/charts/ingress-nginx/README.md +++ b/charts/ingress-nginx/README.md @@ -217,6 +217,21 @@ With nginx-ingress-controller version 0.25+, the nginx ingress controller pod ex With nginx-ingress-controller in 0.25.* work only with kubernetes 1.14+, 0.26 fix [this issue](https://github.com/kubernetes/ingress-nginx/pull/4521) +#### How the Chart Configures the Hooks +A validating and configuration requires the endpoint to which the request is sent to use TLS. It is possible to set up custom certificates to do this, but in most cases, a self-signed certificate is enough. The setup of this component requires some more complex orchestration when using helm. The steps are created to be idempotent and to allow turning the feature on and off without running into helm quirks. + +1. A pre-install hook provisions a certificate into the same namespace using a format compatible with provisioning using end user certificates. If the certificate already exists, the hook exits. +2. The ingress nginx controller pod is configured to use a TLS proxy container, which will load that certificate. +3. Validating and Mutating webhook configurations are created in the cluster. +4. A post-install hook reads the CA from the secret created by step 1 and patches the Validating and Mutating webhook configurations. This process will allow a custom CA provisioned by some other process to also be patched into the webhook configurations. The chosen failure policy is also patched into the webhook configurations + +#### Alternatives +It should be possible to use [cert-manager/cert-manager](https://github.com/cert-manager/cert-manager) if a more complete solution is required. + +You can enable automatic self-signed TLS certificate provisioning via cert-manager by setting the `controller.admissionWebhooks.certManager.enable` value to true. + +Please ensure that cert-manager is correctly installed and configured. + ### Helm Error When Upgrading: spec.clusterIP: Invalid value: "" If you are upgrading this chart from a version between 0.31.0 and 1.2.2 then you may get an error like this: @@ -240,6 +255,9 @@ Kubernetes: `>=1.20.0-0` | commonLabels | object | `{}` | | | controller.addHeaders | object | `{}` | Will add custom headers before sending response traffic to the client according to: https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#add-headers | | controller.admissionWebhooks.annotations | object | `{}` | | +| controller.admissionWebhooks.certManager.admissionCert.duration | string | `""` | | +| controller.admissionWebhooks.certManager.enabled | bool | `false` | | +| controller.admissionWebhooks.certManager.rootCert.duration | string | `""` | | | controller.admissionWebhooks.certificate | string | `"/usr/local/certificates/cert"` | | | controller.admissionWebhooks.createSecretJob.resources | object | `{}` | | | controller.admissionWebhooks.createSecretJob.securityContext.allowPrivilegeEscalation | bool | `false` | | diff --git a/charts/ingress-nginx/README.md.gotmpl b/charts/ingress-nginx/README.md.gotmpl index 9a7d7bdfd..78e4f38d4 100644 --- a/charts/ingress-nginx/README.md.gotmpl +++ b/charts/ingress-nginx/README.md.gotmpl @@ -216,6 +216,21 @@ With nginx-ingress-controller version 0.25+, the nginx ingress controller pod ex With nginx-ingress-controller in 0.25.* work only with kubernetes 1.14+, 0.26 fix [this issue](https://github.com/kubernetes/ingress-nginx/pull/4521) +#### How the Chart Configures the Hooks +A validating and configuration requires the endpoint to which the request is sent to use TLS. It is possible to set up custom certificates to do this, but in most cases, a self-signed certificate is enough. The setup of this component requires some more complex orchestration when using helm. The steps are created to be idempotent and to allow turning the feature on and off without running into helm quirks. + +1. A pre-install hook provisions a certificate into the same namespace using a format compatible with provisioning using end user certificates. If the certificate already exists, the hook exits. +2. The ingress nginx controller pod is configured to use a TLS proxy container, which will load that certificate. +3. Validating and Mutating webhook configurations are created in the cluster. +4. A post-install hook reads the CA from the secret created by step 1 and patches the Validating and Mutating webhook configurations. This process will allow a custom CA provisioned by some other process to also be patched into the webhook configurations. The chosen failure policy is also patched into the webhook configurations + +#### Alternatives +It should be possible to use [cert-manager/cert-manager](https://github.com/cert-manager/cert-manager) if a more complete solution is required. + +You can enable automatic self-signed TLS certificate provisioning via cert-manager by setting the `controller.admissionWebhooks.certManager.enable` value to true. + +Please ensure that cert-manager is correctly installed and configured. + ### Helm Error When Upgrading: spec.clusterIP: Invalid value: "" If you are upgrading this chart from a version between 0.31.0 and 1.2.2 then you may get an error like this: diff --git a/charts/ingress-nginx/ci/controller-admission-tls-cert-manager-values.yaml b/charts/ingress-nginx/ci/controller-admission-tls-cert-manager-values.yaml new file mode 100644 index 000000000..a13241cd4 --- /dev/null +++ b/charts/ingress-nginx/ci/controller-admission-tls-cert-manager-values.yaml @@ -0,0 +1,6 @@ +controller: + admissionWebhooks: + certManager: + enabled: true + service: + type: ClusterIP diff --git a/charts/ingress-nginx/templates/admission-webhooks/cert-manager.yaml b/charts/ingress-nginx/templates/admission-webhooks/cert-manager.yaml new file mode 100644 index 000000000..55fab471c --- /dev/null +++ b/charts/ingress-nginx/templates/admission-webhooks/cert-manager.yaml @@ -0,0 +1,63 @@ +{{- if and .Values.controller.admissionWebhooks.enabled .Values.controller.admissionWebhooks.certManager.enabled -}} +{{- if not .Values.controller.admissionWebhooks.certManager.issuerRef -}} +# Create a selfsigned Issuer, in order to create a root CA certificate for +# signing webhook serving certificates +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ include "ingress-nginx.fullname" . }}-self-signed-issuer + namespace: {{ .Release.Namespace }} +spec: + selfSigned: {} +--- +# Generate a CA Certificate used to sign certificates for the webhook +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ include "ingress-nginx.fullname" . }}-root-cert + namespace: {{ .Release.Namespace }} +spec: + secretName: {{ include "ingress-nginx.fullname" . }}-root-cert + duration: {{ .Values.controller.admissionWebhooks.certManager.rootCert.duration | default "43800h0m0s" | quote }} + issuerRef: + name: {{ include "ingress-nginx.fullname" . }}-self-signed-issuer + commonName: "ca.webhook.ingress-nginx" + isCA: true + subject: + organizations: + - ingress-nginx +--- +# Create an Issuer that uses the above generated CA certificate to issue certs +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ include "ingress-nginx.fullname" . }}-root-issuer + namespace: {{ .Release.Namespace }} +spec: + ca: + secretName: {{ include "ingress-nginx.fullname" . }}-root-cert +{{- end }} +--- +# generate a server certificate for the apiservices to use +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ include "ingress-nginx.fullname" . }}-admission + namespace: {{ .Release.Namespace }} +spec: + secretName: {{ include "ingress-nginx.fullname" . }}-admission + duration: {{ .Values.controller.admissionWebhooks.certManager.admissionCert.duration | default "8760h0m0s" | quote }} + issuerRef: + {{- if .Values.controller.admissionWebhooks.certManager.issuerRef }} + {{- toYaml .Values.controller.admissionWebhooks.certManager.issuerRef | nindent 4 }} + {{- else }} + name: {{ include "ingress-nginx.fullname" . }}-root-issuer + {{- end }} + dnsNames: + - {{ include "ingress-nginx.controller.fullname" . }}-admission + - {{ include "ingress-nginx.controller.fullname" . }}-admission.{{ .Release.Namespace }} + - {{ include "ingress-nginx.controller.fullname" . }}-admission.{{ .Release.Namespace }}.svc + subject: + organizations: + - ingress-nginx-admission +{{- end -}} diff --git a/charts/ingress-nginx/templates/admission-webhooks/job-patch/clusterrole.yaml b/charts/ingress-nginx/templates/admission-webhooks/job-patch/clusterrole.yaml index 5659a1f10..f9ec70974 100644 --- a/charts/ingress-nginx/templates/admission-webhooks/job-patch/clusterrole.yaml +++ b/charts/ingress-nginx/templates/admission-webhooks/job-patch/clusterrole.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.controller.admissionWebhooks.enabled .Values.controller.admissionWebhooks.patch.enabled -}} +{{- if and .Values.controller.admissionWebhooks.enabled .Values.controller.admissionWebhooks.patch.enabled (not .Values.controller.admissionWebhooks.certManager.enabled) -}} apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: diff --git a/charts/ingress-nginx/templates/admission-webhooks/job-patch/clusterrolebinding.yaml b/charts/ingress-nginx/templates/admission-webhooks/job-patch/clusterrolebinding.yaml index abf17fb9f..002abd43b 100644 --- a/charts/ingress-nginx/templates/admission-webhooks/job-patch/clusterrolebinding.yaml +++ b/charts/ingress-nginx/templates/admission-webhooks/job-patch/clusterrolebinding.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.controller.admissionWebhooks.enabled .Values.controller.admissionWebhooks.patch.enabled -}} +{{- if and .Values.controller.admissionWebhooks.enabled .Values.controller.admissionWebhooks.patch.enabled (not .Values.controller.admissionWebhooks.certManager.enabled) -}} apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: diff --git a/charts/ingress-nginx/templates/admission-webhooks/job-patch/job-createSecret.yaml b/charts/ingress-nginx/templates/admission-webhooks/job-patch/job-createSecret.yaml index de78f45bd..d93433ecd 100644 --- a/charts/ingress-nginx/templates/admission-webhooks/job-patch/job-createSecret.yaml +++ b/charts/ingress-nginx/templates/admission-webhooks/job-patch/job-createSecret.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.controller.admissionWebhooks.enabled .Values.controller.admissionWebhooks.patch.enabled -}} +{{- if and .Values.controller.admissionWebhooks.enabled .Values.controller.admissionWebhooks.patch.enabled (not .Values.controller.admissionWebhooks.certManager.enabled) -}} apiVersion: batch/v1 kind: Job metadata: diff --git a/charts/ingress-nginx/templates/admission-webhooks/job-patch/job-patchWebhook.yaml b/charts/ingress-nginx/templates/admission-webhooks/job-patch/job-patchWebhook.yaml index b670aa05a..0fa3ff9a2 100644 --- a/charts/ingress-nginx/templates/admission-webhooks/job-patch/job-patchWebhook.yaml +++ b/charts/ingress-nginx/templates/admission-webhooks/job-patch/job-patchWebhook.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.controller.admissionWebhooks.enabled .Values.controller.admissionWebhooks.patch.enabled -}} +{{- if and .Values.controller.admissionWebhooks.enabled .Values.controller.admissionWebhooks.patch.enabled (not .Values.controller.admissionWebhooks.certManager.enabled) -}} apiVersion: batch/v1 kind: Job metadata: diff --git a/charts/ingress-nginx/templates/admission-webhooks/job-patch/role.yaml b/charts/ingress-nginx/templates/admission-webhooks/job-patch/role.yaml index 795bac6b9..2aab6f4b1 100644 --- a/charts/ingress-nginx/templates/admission-webhooks/job-patch/role.yaml +++ b/charts/ingress-nginx/templates/admission-webhooks/job-patch/role.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.controller.admissionWebhooks.enabled .Values.controller.admissionWebhooks.patch.enabled -}} +{{- if and .Values.controller.admissionWebhooks.enabled .Values.controller.admissionWebhooks.patch.enabled (not .Values.controller.admissionWebhooks.certManager.enabled) -}} apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: diff --git a/charts/ingress-nginx/templates/admission-webhooks/job-patch/rolebinding.yaml b/charts/ingress-nginx/templates/admission-webhooks/job-patch/rolebinding.yaml index 698c5c864..60c3f4ff0 100644 --- a/charts/ingress-nginx/templates/admission-webhooks/job-patch/rolebinding.yaml +++ b/charts/ingress-nginx/templates/admission-webhooks/job-patch/rolebinding.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.controller.admissionWebhooks.enabled .Values.controller.admissionWebhooks.patch.enabled -}} +{{- if and .Values.controller.admissionWebhooks.enabled .Values.controller.admissionWebhooks.patch.enabled (not .Values.controller.admissionWebhooks.certManager.enabled) -}} apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: diff --git a/charts/ingress-nginx/templates/admission-webhooks/job-patch/serviceaccount.yaml b/charts/ingress-nginx/templates/admission-webhooks/job-patch/serviceaccount.yaml index eae475118..00be54ec5 100644 --- a/charts/ingress-nginx/templates/admission-webhooks/job-patch/serviceaccount.yaml +++ b/charts/ingress-nginx/templates/admission-webhooks/job-patch/serviceaccount.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.controller.admissionWebhooks.enabled .Values.controller.admissionWebhooks.patch.enabled -}} +{{- if and .Values.controller.admissionWebhooks.enabled .Values.controller.admissionWebhooks.patch.enabled (not .Values.controller.admissionWebhooks.certManager.enabled) -}} apiVersion: v1 kind: ServiceAccount metadata: diff --git a/charts/ingress-nginx/templates/admission-webhooks/validating-webhook.yaml b/charts/ingress-nginx/templates/admission-webhooks/validating-webhook.yaml index 8caffcb03..f27244dc9 100644 --- a/charts/ingress-nginx/templates/admission-webhooks/validating-webhook.yaml +++ b/charts/ingress-nginx/templates/admission-webhooks/validating-webhook.yaml @@ -4,8 +4,13 @@ apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration metadata: + annotations: + {{- if .Values.controller.admissionWebhooks.certManager.enabled }} + certmanager.k8s.io/inject-ca-from: {{ printf "%s/%s-admission" .Release.Namespace (include "ingress-nginx.fullname" .) | quote }} + cert-manager.io/inject-ca-from: {{ printf "%s/%s-admission" .Release.Namespace (include "ingress-nginx.fullname" .) | quote }} + {{- end }} {{- if .Values.controller.admissionWebhooks.annotations }} - annotations: {{ toYaml .Values.controller.admissionWebhooks.annotations | nindent 4 }} + {{- toYaml .Values.controller.admissionWebhooks.annotations | nindent 4 }} {{- end }} labels: {{- include "ingress-nginx.labels" . | nindent 4 }} diff --git a/charts/ingress-nginx/templates/controller-daemonset.yaml b/charts/ingress-nginx/templates/controller-daemonset.yaml index 802730331..e6721566b 100644 --- a/charts/ingress-nginx/templates/controller-daemonset.yaml +++ b/charts/ingress-nginx/templates/controller-daemonset.yaml @@ -225,6 +225,13 @@ spec: - name: webhook-cert secret: secretName: {{ include "ingress-nginx.fullname" . }}-admission + {{- if .Values.controller.admissionWebhooks.certManager.enabled }} + items: + - key: tls.crt + path: cert + - key: tls.key + path: key + {{- end }} {{- end }} {{- if .Values.controller.extraVolumes }} {{ toYaml .Values.controller.extraVolumes | nindent 8 }} diff --git a/charts/ingress-nginx/templates/controller-deployment.yaml b/charts/ingress-nginx/templates/controller-deployment.yaml index 5cd3c1901..9f1cf70db 100644 --- a/charts/ingress-nginx/templates/controller-deployment.yaml +++ b/charts/ingress-nginx/templates/controller-deployment.yaml @@ -228,6 +228,13 @@ spec: - name: webhook-cert secret: secretName: {{ include "ingress-nginx.fullname" . }}-admission + {{- if .Values.controller.admissionWebhooks.certManager.enabled }} + items: + - key: tls.crt + path: cert + - key: tls.key + path: key + {{- end }} {{- end }} {{- if .Values.controller.extraVolumes }} {{ toYaml .Values.controller.extraVolumes | nindent 8 }} diff --git a/charts/ingress-nginx/values.yaml b/charts/ingress-nginx/values.yaml index 6dc28c08d..161f974fe 100644 --- a/charts/ingress-nginx/values.yaml +++ b/charts/ingress-nginx/values.yaml @@ -677,6 +677,17 @@ controller: runAsUser: 2000 fsGroup: 2000 + # Use certmanager to generate webhook certs + certManager: + enabled: false + # self-signed root certificate + rootCert: + duration: "" # default to be 5y + admissionCert: + duration: "" # default to be 1y + # issuerRef: + # name: "issuer" + # kind: "ClusterIssuer" metrics: port: 10254 diff --git a/test/e2e/run-chart-test.sh b/test/e2e/run-chart-test.sh index fdb3121d8..b2e54993f 100755 --- a/test/e2e/run-chart-test.sh +++ b/test/e2e/run-chart-test.sh @@ -83,13 +83,19 @@ if [ "${SKIP_IMAGE_CREATION:-false}" = "false" ]; then echo "[dev-env] building image" make -C ${DIR}/../../ clean-image build image fi - + KIND_WORKERS=$(kind get nodes --name="${KIND_CLUSTER_NAME}" | awk '{printf (NR>1?",":"") $1}') echo "[dev-env] copying docker images to cluster..." kind load docker-image --name="${KIND_CLUSTER_NAME}" --nodes=${KIND_WORKERS} ${REGISTRY}/controller:${TAG} +if [ "${SKIP_CERT_MANAGER_CREATION:-false}" = "false" ]; then + echo "[dev-env] apply cert-manager ..." + kubectl apply --wait -f https://github.com/cert-manager/cert-manager/releases/download/v1.10.0/cert-manager.yaml + sleep 10 +fi + echo "[dev-env] running helm chart e2e tests..." # Uses a custom chart-testing image to avoid timeouts waiting for namespace deletion. # The changes can be found here: https://github.com/aledbf/chart-testing/commit/41fe0ae0733d0c9a538099fb3cec522e888e3d82