Merge 9781d2b7f3 into de1a4c463c

.github/workflows/ci.yaml

@@ -193,12 +193,18 @@ jobs:
           export TAGNGINX=$(cat images/nginx/TAG)
           make BASE_IMAGE=registry.k8s.io/ingress-nginx/nginx:${TAGNGINX} clean-image build image image-chroot
           make -C test/e2e-image image
+          cd images/custom-error-pages/rootfs && docker image build \
+              --build-arg=BASE_IMAGE=registry.k8s.io/ingress-nginx/nginx:${TAGNGINX} \
+              --build-arg GOLANG_VERSION=$(cat ../../../GOLANG_VERSION) \
+              --tag ingress-controller/custom-error-pages:1.0.0-dev . \
+              && cd ../../..
 
           echo "creating images cache..."
           docker save \
             nginx-ingress-controller:e2e \
             ingress-controller/controller:1.0.0-dev \
             ingress-controller/controller-chroot:1.0.0-dev \
+            ingress-controller/custom-error-pages:1.0.0-dev \
             | gzip > docker.tar.gz
 
       - name: cache

.github/workflows/zz-tmpl-k8s-e2e.yaml

@@ -43,6 +43,7 @@ jobs:
           SKIP_CLUSTER_CREATION: true
           SKIP_INGRESS_IMAGE_CREATION: true
           SKIP_E2E_IMAGE_CREATION: true
+          SKIP_CUSTOMERRORPAGES_IMAGE_CREATION: true
           IS_CHROOT: ${{ inputs.variation == 'CHROOT' }}
         run: |
           kind get kubeconfig > $HOME/.kube/kind-config-kind

charts/ingress-nginx/README.md

@@ -550,6 +550,7 @@ metadata:
 | defaultBackend.service.annotations | object | `{}` |  |
 | defaultBackend.service.clusterIPs | list | `[]` | Pre-defined cluster internal IP addresses of the default backend service. Take care of collisions with existing services. This value is immutable. Set once, it can not be changed without deleting and re-creating the service. Ref: https://kubernetes.io/docs/concepts/services-networking/service/#choosing-your-own-ip-address |
 | defaultBackend.service.externalIPs | list | `[]` | List of IP addresses at which the default backend service is available # Ref: https://kubernetes.io/docs/concepts/services-networking/service/#external-ips # |
+| defaultBackend.service.extraPorts | list | `[]` | Additional ports to expose for defaultBackend pods |
 | defaultBackend.service.loadBalancerSourceRanges | list | `[]` |  |
 | defaultBackend.service.servicePort | int | `80` |  |
 | defaultBackend.service.type | string | `"ClusterIP"` |  |

charts/ingress-nginx/ci/deployment-defaultbackend-values.yaml

@@ -0,0 +1,21 @@
+controller:
+  image:
+    repository: ingress-controller/controller
+    tag: 1.0.0-dev
+    digest: null
+  service:
+    type: ClusterIP
+
+defaultBackend:
+  enabled: true
+  extraEnvs:
+    - name: IS_METRICS_EXPORT
+      value: "false"
+    - name: METRICS_PORT
+      value: "8081"
+  service:
+    extraPorts:
+      - name: metrics
+        port: 8081
+        protocol: TCP
+        targetPort: 8081

charts/ingress-nginx/templates/default-backend-deployment.yaml

@@ -93,6 +93,13 @@ spec:
             - name: http
               containerPort: {{ .Values.defaultBackend.port }}
               protocol: TCP
+          {{- if .Values.defaultBackend.service.extraPorts }}
+            {{- range .Values.defaultBackend.service.extraPorts }}
+            - name: {{ .name }}
+              containerPort: {{ .targetPort }}
+              protocol: {{ .protocol }}
+            {{- end }}
+          {{- end }}
         {{- if .Values.defaultBackend.extraVolumeMounts }}
           volumeMounts: {{- toYaml .Values.defaultBackend.extraVolumeMounts | nindent 12 }}
         {{- end }}

charts/ingress-nginx/templates/default-backend-service.yaml

@@ -38,6 +38,9 @@ spec:
     {{- if semverCompare ">=1.20.0-0" .Capabilities.KubeVersion.Version }}
       appProtocol: http
     {{- end }}
+  {{- if .Values.defaultBackend.service.extraPorts }}
+    {{ toYaml .Values.defaultBackend.service.extraPorts | nindent 4 }}
+  {{- end}}
   selector:
     {{- include "ingress-nginx.selectorLabels" . | nindent 4 }}
     app.kubernetes.io/component: default-backend

charts/ingress-nginx/tests/default-backend-deployment_test.yaml

@@ -196,3 +196,18 @@ tests:
       - equal:
           path: spec.template.spec.automountServiceAccountToken
           value: false
+
+  - it: should create a Deployment with extraPorts if `defaultBackend.service.extraPorts` is set
+    set:
+      defaultBackend.enabled: true
+      defaultBackend.service.extraPorts[0]:
+        name: example
+        protocol: TCP
+        targetPort: 9999
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].ports
+          content:
+            name: example
+            protocol: TCP
+            containerPort: 9999

charts/ingress-nginx/tests/default-backend-service_test.yaml

@@ -50,3 +50,20 @@ tests:
           value:
             - 10.0.0.1
             - fd00::1
+
+  - it: should create a Service with extraPorts if `defaultBackend.service.extraPorts` is set
+    set:
+      defaultBackend.enabled: true
+      defaultBackend.service.extraPorts[0]:
+        name: example
+        port: 8888
+        protocol: TCP
+        targetPort: 65535
+    asserts:
+      - contains:
+          path: spec.ports
+          content:
+            name: example
+            port: 8888
+            protocol: TCP
+            targetPort: 65535

charts/ingress-nginx/values.yaml

@@ -1188,6 +1188,13 @@ defaultBackend:
     loadBalancerSourceRanges: []
     servicePort: 80
     type: ClusterIP
+
+    # -- Additional ports to expose for defaultBackend pods
+    extraPorts: []
+    #  - name: metrics
+    #    port: 9090
+    #    protocol: TCP
+    #    targetPort: 9090
   priorityClassName: ""
   # -- Labels to be added to the default backend resources
   labels: {}

docs/e2e-tests.md

@@ -287,6 +287,10 @@ Do not try to edit it manually.
 - [should return a self generated SSL certificate](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/defaultbackend/ssl.go#L29)
 ### [[Default Backend] change default settings](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/defaultbackend/with_hosts.go#L30)
 - [should apply the annotation to the default backend](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/defaultbackend/with_hosts.go#L38)
+### [[Default Backend] custom-error-pages](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/defaultbackend/custom_error_pages.go#L33)
+- [should export /metrics and /debug/vars by default](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/defaultbackend/custom_error_pages.go#L36)
+- [shouldn't export /metrics and /debug/vars when IS_METRICS_EXPORT is set to false](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/defaultbackend/custom_error_pages.go#L57)
+- [shouldn't export /metrics and /debug/vars when METRICS_PORT is set to a different port](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/defaultbackend/custom_error_pages.go#L80)
 ### [[Disable Leader] Routing works when leader election was disabled](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/disableleaderelection/disable_leader.go#L28)
 - [should create multiple ingress routings rules when leader election has disabled](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/disableleaderelection/disable_leader.go#L35)
 ### [[Endpointslices] long service name](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/endpointslices/longname.go#L29)

images/Makefile

@@ -42,7 +42,7 @@ export DOCKER_CLI_EXPERIMENTAL=enabled
 
 # build with buildx
 PLATFORMS?=linux/amd64,linux/arm,linux/arm64
-OUTPUT=
+OUTPUT?=
 PROGRESS=plain
 
 

images/custom-error-pages/rootfs/main.go

@@ -25,6 +25,7 @@ import (
 	"os"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
@@ -59,15 +60,26 @@ const (
 	// RequestId is a unique ID that identifies the request - same as for backend service
 	RequestId = "X-Request-ID"
 
-	// ErrFilesPathVar is the name of the environment variable indicating
+	// ErrFilesPathEnvVar is the name of the environment variable indicating
 	// the location on disk of files served by the handler.
-	ErrFilesPathVar = "ERROR_FILES_PATH"
+	ErrFilesPathEnvVar = "ERROR_FILES_PATH"
 
-	// DefaultFormatVar is the name of the environment variable indicating
+	// DefaultFormatEnvVar is the name of the environment variable indicating
 	// the default error MIME type that should be returned if either the
 	// client does not specify an Accept header, or the Accept header provided
 	// cannot be mapped to a file extension.
-	DefaultFormatVar = "DEFAULT_RESPONSE_FORMAT"
+	DefaultFormatEnvVar = "DEFAULT_RESPONSE_FORMAT"
+
+	//  IsMetricsExportEnvVar is the name of the environment variable indicating
+	// whether or not to export /metrics and /debug/vars.
+	IsMetricsExportEnvVar = "IS_METRICS_EXPORT"
+
+	// MetricsPortEnvVar is the name of the environment variable indicating
+	// the port on which to export /metrics and /debug/vars.
+	MetricsPortEnvVar = "METRICS_PORT"
+
+	// CustomErrorPagesPort is the port on which to listen to serve custom error pages.
+	CustomErrorPagesPort = "8080"
 )
 
 func init() {
@@ -76,25 +88,97 @@ func init() {
 }
 
 func main() {
+	listeners := createListeners()
+	startListeners(listeners)
+}
+
+func createListeners() []Listener {
 	errFilesPath := "/www"
-	if os.Getenv(ErrFilesPathVar) != "" {
-		errFilesPath = os.Getenv(ErrFilesPathVar)
+	if os.Getenv(ErrFilesPathEnvVar) != "" {
+		errFilesPath = os.Getenv(ErrFilesPathEnvVar)
 	}
 
 	defaultFormat := "text/html"
-	if os.Getenv(DefaultFormatVar) != "" {
-		defaultFormat = os.Getenv(DefaultFormatVar)
+	if os.Getenv(DefaultFormatEnvVar) != "" {
+		defaultFormat = os.Getenv(DefaultFormatEnvVar)
 	}
 
-	http.HandleFunc("/", errorHandler(errFilesPath, defaultFormat))
+	isExportMetrics := true
+	if os.Getenv(IsMetricsExportEnvVar) != "" {
+		val, err := strconv.ParseBool(os.Getenv(IsMetricsExportEnvVar))
+		if err == nil {
+			isExportMetrics = val
+		}
+	}
 
-	http.Handle("/metrics", promhttp.Handler())
+	metricsPort := "8080"
+	if os.Getenv(MetricsPortEnvVar) != "" {
+		metricsPort = os.Getenv(MetricsPortEnvVar)
+	}
 
-	http.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
+	var listeners []Listener
+
+	// MUST use NewServerMux when not exporting /metrics because expvar HTTP handler registers
+	// against DefaultServerMux as a consequence of importing it in client_golang/prometheus.
+	if !isExportMetrics {
+		mux := http.NewServeMux()
+		mux.HandleFunc("/", errorHandler(errFilesPath, defaultFormat))
+		mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
+			w.WriteHeader(http.StatusOK)
+		})
+		listeners = append(listeners, Listener{mux, CustomErrorPagesPort})
+		return listeners
+	}
+
+	// MUST use DefaultServerMux when exporting /metrics to the public because /debug/vars is
+	// only available with DefaultServerMux.
+	if metricsPort == CustomErrorPagesPort {
+		mux := http.DefaultServeMux
+		mux.Handle("/metrics", promhttp.Handler())
+		mux.HandleFunc("/", errorHandler(errFilesPath, defaultFormat))
+		mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
+			w.WriteHeader(http.StatusOK)
+		})
+		listeners = append(listeners, Listener{mux, CustomErrorPagesPort})
+		return listeners
+	}
+
+	// MUST use DefaultServerMux for /metrics and NewServerMux for custom error pages when you
+	// wish to expose /metrics only to internal services, because expvar HTTP handler registers
+	// against DefaultServerMux.
+	metricsMux := http.DefaultServeMux
+	metricsMux.Handle("/metrics", promhttp.Handler())
+
+	errorsMux := http.NewServeMux()
+	errorsMux.HandleFunc("/", errorHandler(errFilesPath, defaultFormat))
+	errorsMux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
 		w.WriteHeader(http.StatusOK)
 	})
 
-	http.ListenAndServe(fmt.Sprintf(":8080"), nil)
+	listeners = append(listeners, Listener{metricsMux, metricsPort}, Listener{errorsMux, CustomErrorPagesPort})
+	return listeners
+}
+
+func startListeners(listeners []Listener) {
+	var wg sync.WaitGroup
+
+	for _, listener := range listeners {
+		wg.Add(1)
+		go func(l Listener) {
+			defer wg.Done()
+			err := http.ListenAndServe(fmt.Sprintf(":%s", l.port), l.mux)
+			if err != nil {
+				log.Fatal(err)
+			}
+		}(listener)
+	}
+
+	wg.Wait()
+}
+
+type Listener struct {
+	mux  *http.ServeMux
+	port string
 }
 
 func errorHandler(path, defaultFormat string) func(http.ResponseWriter, *http.Request) {

test/e2e/CUSTOMERRORPAGES_IMAGE

@@ -0,0 +1 @@
+ingress-controller/custom-error-pages:1.0.0-dev

test/e2e/defaultbackend/custom_error_pages.go

@@ -0,0 +1,121 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package defaultbackend
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"strings"
+
+	"github.com/onsi/ginkgo/v2"
+	"github.com/stretchr/testify/assert"
+	appsv1 "k8s.io/api/apps/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"k8s.io/ingress-nginx/test/e2e/framework"
+)
+
+var _ = framework.IngressNginxDescribe("[Default Backend] custom-error-pages", func() {
+	f := framework.NewDefaultFramework("custom-error-pages")
+
+	ginkgo.It("should export /metrics and /debug/vars by default", func() {
+		tt := []struct {
+			Name   string
+			Path   string
+			Status int
+		}{
+			{"request to /metrics should return HTTP 200", "/metrics", http.StatusOK},
+			{"request to /debug/vars should return HTTP 200", "/debug/vars", http.StatusOK},
+		}
+
+		setupIngressControllerWithCustomErrorPages(f, nil)
+
+		for _, t := range tt {
+			ginkgo.By(t.Name)
+			f.HTTPTestClient().
+				GET(t.Path).
+				Expect().
+				Status(t.Status)
+		}
+	})
+
+	ginkgo.It("shouldn't export /metrics and /debug/vars when IS_METRICS_EXPORT is set to false", func() {
+		tt := []struct {
+			Name   string
+			Path   string
+			Status int
+		}{
+			{"request to /metrics should return HTTP 404", "/metrics", http.StatusNotFound},
+			{"request to /debug/vars should return HTTP 404", "/debug/vars", http.StatusNotFound},
+		}
+
+		setupIngressControllerWithCustomErrorPages(f, map[string]string{
+			"IS_METRICS_EXPORT": "false",
+		})
+
+		for _, t := range tt {
+			ginkgo.By(t.Name)
+			f.HTTPTestClient().
+				GET(t.Path).
+				Expect().
+				Status(t.Status)
+		}
+	})
+
+	ginkgo.It("shouldn't export /metrics and /debug/vars when METRICS_PORT is set to a different port", func() {
+		tt := []struct {
+			Name   string
+			Path   string
+			Status int
+		}{
+			{"request to /metrics should return HTTP 404", "/metrics", http.StatusNotFound},
+			{"request to /debug/vars should return HTTP 404", "/debug/vars", http.StatusNotFound},
+		}
+
+		setupIngressControllerWithCustomErrorPages(f, map[string]string{
+			"IS_METRICS_EXPORT": "true",
+			"METRICS_PORT":      "8081",
+		})
+
+		for _, t := range tt {
+			ginkgo.By(t.Name)
+			f.HTTPTestClient().
+				GET(t.Path).
+				Expect().
+				Status(t.Status)
+		}
+	})
+})
+
+func setupIngressControllerWithCustomErrorPages(f *framework.Framework, envVars map[string]string) {
+	f.NewCustomErrorPagesDeployment(framework.WithEnvVars(envVars))
+
+	err := f.UpdateIngressControllerDeployment(func(deployment *appsv1.Deployment) error {
+		args := deployment.Spec.Template.Spec.Containers[0].Args
+		args = append(args, fmt.Sprintf("--default-backend-service=%v/%v", f.Namespace, framework.CustomErrorPagesService))
+		deployment.Spec.Template.Spec.Containers[0].Args = args
+		_, err := f.KubeClientSet.AppsV1().Deployments(f.Namespace).Update(context.TODO(), deployment, metav1.UpdateOptions{})
+		return err
+	})
+	assert.Nil(ginkgo.GinkgoT(), err, "updating deployment")
+
+	f.WaitForNginxServer("_",
+		func(server string) bool {
+			return strings.Contains(server, `set $proxy_upstream_name "upstream-default-backend"`)
+		})
+}

test/e2e/framework/deployment.go

@@ -49,6 +49,12 @@ var HTTPBunImage = os.Getenv("HTTPBUN_IMAGE")
 // EchoImage is the default image to be used by the echo service
 const EchoImage = "registry.k8s.io/ingress-nginx/e2e-test-echo:v1.1.1@sha256:a1e0152e2eeab26e3f6fd3986f3d82b17bc7711717cae5392dcd18dd447ba6ef" //#nosec G101
 
+// CustomErrorPagesService name of the deployment for the custom-error-pages app
+const CustomErrorPagesService = "custom-error-pages"
+
+// CustomErrorPagesImage is the default image that is used to deploy custom-error-pages with the framework
+var CustomErrorPagesImage = os.Getenv("CUSTOMERRORPAGES_IMAGE")
+
 // TODO: change all Deployment functions to use these options
 // in order to reduce complexity and have a unified API across the
 // framework
@@ -58,6 +64,7 @@ type deploymentOptions struct {
 	image          string
 	replicas       int
 	svcAnnotations map[string]string
+	envVars        map[string]string
 }
 
 // WithDeploymentNamespace allows configuring the deployment's namespace
@@ -103,6 +110,74 @@ func WithImage(i string) func(*deploymentOptions) {
 	}
 }
 
+// WithEnvVars allows configuring environment variables for the deployment
+func WithEnvVars(e map[string]string) func(*deploymentOptions) {
+	return func(o *deploymentOptions) {
+		o.envVars = e
+	}
+}
+
+// NewCustomErrorPagesDeployment creates a new single replica deployment of the custom-error-pages server image in a particular namespace
+func (f *Framework) NewCustomErrorPagesDeployment(opts ...func(*deploymentOptions)) {
+	options := &deploymentOptions{
+		namespace: f.Namespace,
+		name:      CustomErrorPagesService,
+		replicas:  1,
+		image:     CustomErrorPagesImage,
+	}
+	for _, o := range opts {
+		o(options)
+	}
+
+	envVars := []corev1.EnvVar{}
+	for k, v := range options.envVars {
+		envVars = append(envVars, corev1.EnvVar{Name: k, Value: v})
+	}
+
+	f.EnsureDeployment(newDeployment(
+		options.name,
+		options.namespace,
+		options.image,
+		8080,
+		int32(options.replicas), //nolint:gosec // disable G115
+		nil, nil,
+		envVars,
+		[]corev1.VolumeMount{},
+		[]corev1.Volume{},
+		false,
+	))
+
+	f.EnsureService(&corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:        options.name,
+			Namespace:   options.namespace,
+			Annotations: options.svcAnnotations,
+		},
+		Spec: corev1.ServiceSpec{
+			Ports: []corev1.ServicePort{
+				{
+					Name:       "http",
+					Port:       8080,
+					TargetPort: intstr.FromInt(8080),
+					Protocol:   corev1.ProtocolTCP,
+				},
+			},
+			Selector: map[string]string{
+				"app": options.name,
+			},
+		},
+	})
+
+	err := WaitForEndpoints(
+		f.KubeClientSet,
+		DefaultTimeout,
+		options.name,
+		options.namespace,
+		options.replicas,
+	)
+	assert.Nil(ginkgo.GinkgoT(), err, "waiting for endpoints to become ready")
+}
+
 // NewEchoDeployment creates a new single replica deployment of the echo server image in a particular namespace
 func (f *Framework) NewEchoDeployment(opts ...func(*deploymentOptions)) {
 	options := &deploymentOptions{
@@ -120,7 +195,7 @@ func (f *Framework) NewEchoDeployment(opts ...func(*deploymentOptions)) {
 		options.namespace,
 		options.image,
 		80,
-		int32(options.replicas),
+		int32(options.replicas), //nolint:gosec // disable G115
 		nil, nil, nil,
 		[]corev1.VolumeMount{},
 		[]corev1.Volume{},
@@ -211,7 +286,7 @@ func (f *Framework) NewHttpbunDeployment(opts ...func(*deploymentOptions)) strin
 		options.namespace,
 		options.image,
 		80,
-		int32(options.replicas),
+		int32(options.replicas), //nolint:gosec // disable G115
 		nil, nil,
 		// Required to get hostname information
 		[]corev1.EnvVar{

test/e2e/run-e2e-suite.sh

@@ -52,6 +52,7 @@ fi
 BASEDIR=$(dirname "$0")
 NGINX_BASE_IMAGE=$(cat $BASEDIR/../../NGINX_BASE)
 HTTPBUN_IMAGE=$(cat $BASEDIR/HTTPBUN_IMAGE)
+CUSTOMERRORPAGES_IMAGE=$(cat $BASEDIR/CUSTOMERRORPAGES_IMAGE)
 
 echo -e "${BGREEN}Granting permissions to ingress-nginx e2e service account...${NC}"
 kubectl create serviceaccount ingress-nginx-e2e || true
@@ -82,6 +83,7 @@ kubectl run --rm \
   --env="E2E_CHECK_LEAKS=${E2E_CHECK_LEAKS}" \
   --env="NGINX_BASE_IMAGE=${NGINX_BASE_IMAGE}" \
   --env="HTTPBUN_IMAGE=${HTTPBUN_IMAGE}" \
+  --env="CUSTOMERRORPAGES_IMAGE=${CUSTOMERRORPAGES_IMAGE}" \
   --overrides='{ "apiVersion": "v1", "spec":{"serviceAccountName": "ingress-nginx-e2e"}}' \
   e2e --image=nginx-ingress-controller:e2e
 

test/e2e/run-kind-e2e.sh

@@ -52,6 +52,7 @@ export KUBECONFIG="${KUBECONFIG:-$HOME/.kube/kind-config-$KIND_CLUSTER_NAME}"
 SKIP_INGRESS_IMAGE_CREATION="${SKIP_INGRESS_IMAGE_CREATION:-false}"
 SKIP_E2E_IMAGE_CREATION="${SKIP_E2E_IMAGE_CREATION:=false}"
 SKIP_CLUSTER_CREATION="${SKIP_CLUSTER_CREATION:-false}"
+SKIP_CUSTOMERRORPAGES_IMAGE_CREATION="${SKIP_CUSTOMERRORPAGES_IMAGE_CREATION:-false}"
 
 if ! command -v kind --version &> /dev/null; then
   echo "kind is not installed. Use the package manager or visit the official site https://kind.sigs.k8s.io/"
@@ -104,6 +105,12 @@ if [ "${SKIP_E2E_IMAGE_CREATION}" = "false" ]; then
   echo "[dev-env] ..done building e2e-image"
 fi
 
+if [ "${SKIP_CUSTOMERRORPAGES_IMAGE_CREATION}" = "false" ]; then
+  echo "[dev-env] building custom-error-pages image"
+  make NAME=custom-error-pages -C "${DIR}"/../../images build
+  echo "[dev-env] .. done building custom-error-pages image"
+fi
+
 # Preload images used in e2e tests
 KIND_WORKERS=$(kind get nodes --name="${KIND_CLUSTER_NAME}" | grep worker | awk '{printf (NR>1?",":"") $1}')
 
@@ -111,5 +118,6 @@ echo "[dev-env] copying docker images to cluster..."
 
 kind load docker-image --name="${KIND_CLUSTER_NAME}" --nodes="${KIND_WORKERS}" nginx-ingress-controller:e2e
 kind load docker-image --name="${KIND_CLUSTER_NAME}" --nodes="${KIND_WORKERS}" "${REGISTRY}"/controller:"${TAG}"
+kind load docker-image --name="${KIND_CLUSTER_NAME}" --nodes="${KIND_WORKERS}" "${REGISTRY}"/custom-error-pages:"${TAG}"
 echo "[dev-env] running e2e tests..."
 make -C "${DIR}"/../../ e2e-test