From 54990cea7f6d40e6c9d35dd3b59d7fafeb3c0941 Mon Sep 17 00:00:00 2001
From: Ricardo Lopes <ricardoapl.dev@gmail.com>
Date: Wed, 14 Feb 2024 09:31:31 +0000
Subject: [PATCH 01/16] Add feature toggle for /metrics in custom error pages

Signed-off-by: Ricardo Lopes <ricardoapl.dev@gmail.com>
---
 images/custom-error-pages/rootfs/main.go | 35 +++++++++++++++++++-----
 1 file changed, 28 insertions(+), 7 deletions(-)

diff --git a/images/custom-error-pages/rootfs/main.go b/images/custom-error-pages/rootfs/main.go
index 7d3d73029..7047e5556 100644
--- a/images/custom-error-pages/rootfs/main.go
+++ b/images/custom-error-pages/rootfs/main.go
@@ -68,6 +68,10 @@ const (
 	// client does not specify an Accept header, or the Accept header provided
 	// cannot be mapped to a file extension.
 	DefaultFormatVar = "DEFAULT_RESPONSE_FORMAT"
+
+	// IsExposeSignalsVar is the name of the environment variable indicating
+	// whether or not to expose signals such as /metrics and /healthz.
+	IsExposeSignalsVar = "IS_EXPOSE_SIGNALS"
 )
 
 func init() {
@@ -86,15 +90,32 @@ func main() {
 		defaultFormat = os.Getenv(DefaultFormatVar)
 	}
 
-	http.HandleFunc("/", errorHandler(errFilesPath, defaultFormat))
+	isExposeSignals := true
+	if os.Getenv(IsExposeSignalsVar) != "" {
+		val, err := strconv.ParseBool(os.Getenv(IsExposeSignalsVar))
+		if err == nil {
+			isExposeSignals = val
+		}
+	}
 
-	http.Handle("/metrics", promhttp.Handler())
+	var mux *http.ServeMux
+	if isExposeSignals {
+		mux = http.DefaultServeMux
+		mux.Handle("/metrics", promhttp.Handler())
+		mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
+			w.WriteHeader(http.StatusOK)
+		})
+	} else {
+		// Use a new ServerMux because expvar HTTP handler registers itself against DefaultServerMux
+		// as a consequence of importing it in client_golang/prometheus.
+		mux = http.NewServeMux()
+	}
 
-	http.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
-		w.WriteHeader(http.StatusOK)
-	})
-
-	http.ListenAndServe(fmt.Sprintf(":8080"), nil)
+	mux.HandleFunc("/", errorHandler(errFilesPath, defaultFormat))
+	err := http.ListenAndServe(":8080", mux)
+	if err != nil {
+		log.Fatal(err)
+	}
 }
 
 func errorHandler(path, defaultFormat string) func(http.ResponseWriter, *http.Request) {

From b502b5fcdc83323e9aca9754a31181c6a2a3a64c Mon Sep 17 00:00:00 2001
From: Ricardo Lopes <ricardoapl.dev@gmail.com>
Date: Wed, 14 Feb 2024 10:20:02 +0000
Subject: [PATCH 02/16] Allow exposing custom error pages /metrics on different
 port

Signed-off-by: Ricardo Lopes <ricardoapl.dev@gmail.com>
---
 images/custom-error-pages/rootfs/main.go | 57 +++++++++++++++++++-----
 1 file changed, 46 insertions(+), 11 deletions(-)

diff --git a/images/custom-error-pages/rootfs/main.go b/images/custom-error-pages/rootfs/main.go
index 7047e5556..8aa9dd9de 100644
--- a/images/custom-error-pages/rootfs/main.go
+++ b/images/custom-error-pages/rootfs/main.go
@@ -72,6 +72,13 @@ const (
 	// IsExposeSignalsVar is the name of the environment variable indicating
 	// whether or not to expose signals such as /metrics and /healthz.
 	IsExposeSignalsVar = "IS_EXPOSE_SIGNALS"
+
+	// ExposeSignalsPortVar is the name of the environment variable indicating
+	// the port on which to expose signals such as /metrics and /healthz.
+	ExposeSignalsPortVar = "EXPOSE_SIGNALS_PORT"
+
+	// CustomErrorPagesPort is the port on which to listen to serve custom error pages.
+	CustomErrorPagesPort = "8080"
 )
 
 func init() {
@@ -98,23 +105,51 @@ func main() {
 		}
 	}
 
+	exposeSignalsPort := "8080"
+	if os.Getenv(ExposeSignalsPortVar) != "" {
+		exposeSignalsPort = os.Getenv(ExposeSignalsPortVar)
+	}
+
 	var mux *http.ServeMux
 	if isExposeSignals {
-		mux = http.DefaultServeMux
-		mux.Handle("/metrics", promhttp.Handler())
-		mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
-			w.WriteHeader(http.StatusOK)
-		})
+		if exposeSignalsPort == CustomErrorPagesPort {
+			mux = http.DefaultServeMux
+			mux.HandleFunc("/", errorHandler(errFilesPath, defaultFormat))
+			mux.Handle("/metrics", promhttp.Handler())
+			mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
+				w.WriteHeader(http.StatusOK)
+			})
+			err := http.ListenAndServe(fmt.Sprintf(":%s", CustomErrorPagesPort), mux)
+			if err != nil {
+				panic(err)
+			}
+		} else {
+			mux = http.NewServeMux()
+			mux.HandleFunc("/", errorHandler(errFilesPath, defaultFormat))
+			http.Handle("/metrics", promhttp.Handler())
+			http.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
+				w.WriteHeader(http.StatusOK)
+			})
+			go func() {
+				err := http.ListenAndServe(fmt.Sprintf(":%s", CustomErrorPagesPort), mux)
+				if err != nil {
+					panic(err)
+				}
+			}()
+			err := http.ListenAndServe(fmt.Sprintf(":%s", exposeSignalsPort), nil)
+			if err != nil {
+				panic(err)
+			}
+		}
 	} else {
 		// Use a new ServerMux because expvar HTTP handler registers itself against DefaultServerMux
 		// as a consequence of importing it in client_golang/prometheus.
 		mux = http.NewServeMux()
-	}
-
-	mux.HandleFunc("/", errorHandler(errFilesPath, defaultFormat))
-	err := http.ListenAndServe(":8080", mux)
-	if err != nil {
-		log.Fatal(err)
+		mux.HandleFunc("/", errorHandler(errFilesPath, defaultFormat))
+		err := http.ListenAndServe(fmt.Sprintf(":%s", CustomErrorPagesPort), mux)
+		if err != nil {
+			panic(err)
+		}
 	}
 }
 

From 20e86ed27a0815136b44fc6b450f6b5d253a7efe Mon Sep 17 00:00:00 2001
From: Ricardo Lopes <ricardoapl.dev@gmail.com>
Date: Wed, 14 Feb 2024 11:21:14 +0000
Subject: [PATCH 03/16] Refactor and rename custom error pages vars

Signed-off-by: Ricardo Lopes <ricardoapl.dev@gmail.com>
---
 images/custom-error-pages/rootfs/main.go | 53 ++++++++++--------------
 1 file changed, 22 insertions(+), 31 deletions(-)

diff --git a/images/custom-error-pages/rootfs/main.go b/images/custom-error-pages/rootfs/main.go
index 8aa9dd9de..609b9d473 100644
--- a/images/custom-error-pages/rootfs/main.go
+++ b/images/custom-error-pages/rootfs/main.go
@@ -69,13 +69,13 @@ const (
 	// cannot be mapped to a file extension.
 	DefaultFormatVar = "DEFAULT_RESPONSE_FORMAT"
 
-	// IsExposeSignalsVar is the name of the environment variable indicating
-	// whether or not to expose signals such as /metrics and /healthz.
-	IsExposeSignalsVar = "IS_EXPOSE_SIGNALS"
+	// IsExportMetricsVar is the name of the environment variable indicating
+	// whether or not to export /metrics and /healthz.
+	IsExportMetricsVar = "IS_EXPORT_METRICS"
 
-	// ExposeSignalsPortVar is the name of the environment variable indicating
-	// the port on which to expose signals such as /metrics and /healthz.
-	ExposeSignalsPortVar = "EXPOSE_SIGNALS_PORT"
+	// MetricsPortVar is the name of the environment variable indicating
+	// the port on which to export /metrics and /healthz.
+	MetricsPortVar = "METRICS_PORT"
 
 	// CustomErrorPagesPort is the port on which to listen to serve custom error pages.
 	CustomErrorPagesPort = "8080"
@@ -97,59 +97,50 @@ func main() {
 		defaultFormat = os.Getenv(DefaultFormatVar)
 	}
 
-	isExposeSignals := true
-	if os.Getenv(IsExposeSignalsVar) != "" {
-		val, err := strconv.ParseBool(os.Getenv(IsExposeSignalsVar))
+	isExportMetrics := true
+	if os.Getenv(IsExportMetricsVar) != "" {
+		val, err := strconv.ParseBool(os.Getenv(IsExportMetricsVar))
 		if err == nil {
-			isExposeSignals = val
+			isExportMetrics = val
 		}
 	}
 
-	exposeSignalsPort := "8080"
-	if os.Getenv(ExposeSignalsPortVar) != "" {
-		exposeSignalsPort = os.Getenv(ExposeSignalsPortVar)
+	metricsPort := "8080"
+	if os.Getenv(MetricsPortVar) != "" {
+		metricsPort = os.Getenv(MetricsPortVar)
 	}
 
 	var mux *http.ServeMux
-	if isExposeSignals {
-		if exposeSignalsPort == CustomErrorPagesPort {
+	if isExportMetrics {
+		if metricsPort == CustomErrorPagesPort {
 			mux = http.DefaultServeMux
-			mux.HandleFunc("/", errorHandler(errFilesPath, defaultFormat))
 			mux.Handle("/metrics", promhttp.Handler())
 			mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
 				w.WriteHeader(http.StatusOK)
 			})
-			err := http.ListenAndServe(fmt.Sprintf(":%s", CustomErrorPagesPort), mux)
-			if err != nil {
-				panic(err)
-			}
 		} else {
 			mux = http.NewServeMux()
-			mux.HandleFunc("/", errorHandler(errFilesPath, defaultFormat))
 			http.Handle("/metrics", promhttp.Handler())
 			http.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
 				w.WriteHeader(http.StatusOK)
 			})
 			go func() {
-				err := http.ListenAndServe(fmt.Sprintf(":%s", CustomErrorPagesPort), mux)
+				err := http.ListenAndServe(fmt.Sprintf(":%s", metricsPort), nil)
 				if err != nil {
 					panic(err)
 				}
 			}()
-			err := http.ListenAndServe(fmt.Sprintf(":%s", exposeSignalsPort), nil)
-			if err != nil {
-				panic(err)
-			}
 		}
 	} else {
 		// Use a new ServerMux because expvar HTTP handler registers itself against DefaultServerMux
 		// as a consequence of importing it in client_golang/prometheus.
 		mux = http.NewServeMux()
-		mux.HandleFunc("/", errorHandler(errFilesPath, defaultFormat))
-		err := http.ListenAndServe(fmt.Sprintf(":%s", CustomErrorPagesPort), mux)
-		if err != nil {
-			panic(err)
-		}
+	}
+
+	mux.HandleFunc("/", errorHandler(errFilesPath, defaultFormat))
+	err := http.ListenAndServe(fmt.Sprintf(":%s", CustomErrorPagesPort), mux)
+	if err != nil {
+		panic(err)
 	}
 }
 

From ddd1c90f5ab110b8e60580acb432f15115f1bc97 Mon Sep 17 00:00:00 2001
From: Ricardo Lopes <ricardoapl.dev@gmail.com>
Date: Wed, 14 Feb 2024 11:50:43 +0000
Subject: [PATCH 04/16] Refactor branches in custom error pages

Signed-off-by: Ricardo Lopes <ricardoapl.dev@gmail.com>
---
 images/custom-error-pages/rootfs/main.go | 23 ++++++++++++-----------
 1 file changed, 12 insertions(+), 11 deletions(-)

diff --git a/images/custom-error-pages/rootfs/main.go b/images/custom-error-pages/rootfs/main.go
index 609b9d473..5176ecf1c 100644
--- a/images/custom-error-pages/rootfs/main.go
+++ b/images/custom-error-pages/rootfs/main.go
@@ -110,22 +110,23 @@ func main() {
 		metricsPort = os.Getenv(MetricsPortVar)
 	}
 
-	var mux *http.ServeMux
+	var errorsMux, metricsMux *http.ServeMux
 	if isExportMetrics {
 		if metricsPort == CustomErrorPagesPort {
-			mux = http.DefaultServeMux
-			mux.Handle("/metrics", promhttp.Handler())
-			mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
+			errorsMux = http.DefaultServeMux
+			errorsMux.Handle("/metrics", promhttp.Handler())
+			errorsMux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
 				w.WriteHeader(http.StatusOK)
 			})
 		} else {
-			mux = http.NewServeMux()
-			http.Handle("/metrics", promhttp.Handler())
-			http.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
+			errorsMux = http.NewServeMux()
+			metricsMux = http.DefaultServeMux
+			metricsMux.Handle("/metrics", promhttp.Handler())
+			metricsMux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
 				w.WriteHeader(http.StatusOK)
 			})
 			go func() {
-				err := http.ListenAndServe(fmt.Sprintf(":%s", metricsPort), nil)
+				err := http.ListenAndServe(fmt.Sprintf(":%s", metricsPort), metricsMux)
 				if err != nil {
 					panic(err)
 				}
@@ -134,11 +135,11 @@ func main() {
 	} else {
 		// Use a new ServerMux because expvar HTTP handler registers itself against DefaultServerMux
 		// as a consequence of importing it in client_golang/prometheus.
-		mux = http.NewServeMux()
+		errorsMux = http.NewServeMux()
 	}
 
-	mux.HandleFunc("/", errorHandler(errFilesPath, defaultFormat))
-	err := http.ListenAndServe(fmt.Sprintf(":%s", CustomErrorPagesPort), mux)
+	errorsMux.HandleFunc("/", errorHandler(errFilesPath, defaultFormat))
+	err := http.ListenAndServe(fmt.Sprintf(":%s", CustomErrorPagesPort), errorsMux)
 	if err != nil {
 		panic(err)
 	}

From 14278417d1dd7dc3e849cc035d657f0c2f52c1b3 Mon Sep 17 00:00:00 2001
From: Ricardo Lopes <ricardoapl.dev@gmail.com>
Date: Wed, 14 Feb 2024 12:36:21 +0000
Subject: [PATCH 05/16] Split create and serve logic for custom error pages

Signed-off-by: Ricardo Lopes <ricardoapl.dev@gmail.com>
---
 images/custom-error-pages/rootfs/main.go | 91 ++++++++++++++++--------
 1 file changed, 61 insertions(+), 30 deletions(-)

diff --git a/images/custom-error-pages/rootfs/main.go b/images/custom-error-pages/rootfs/main.go
index 5176ecf1c..23a847927 100644
--- a/images/custom-error-pages/rootfs/main.go
+++ b/images/custom-error-pages/rootfs/main.go
@@ -25,6 +25,7 @@ import (
 	"os"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
@@ -87,6 +88,11 @@ func init() {
 }
 
 func main() {
+	listeners := createListeners()
+	startListeners(listeners)
+}
+
+func createListeners() []Listener {
 	errFilesPath := "/www"
 	if os.Getenv(ErrFilesPathVar) != "" {
 		errFilesPath = os.Getenv(ErrFilesPathVar)
@@ -110,39 +116,64 @@ func main() {
 		metricsPort = os.Getenv(MetricsPortVar)
 	}
 
-	var errorsMux, metricsMux *http.ServeMux
-	if isExportMetrics {
-		if metricsPort == CustomErrorPagesPort {
-			errorsMux = http.DefaultServeMux
-			errorsMux.Handle("/metrics", promhttp.Handler())
-			errorsMux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
-				w.WriteHeader(http.StatusOK)
-			})
-		} else {
-			errorsMux = http.NewServeMux()
-			metricsMux = http.DefaultServeMux
-			metricsMux.Handle("/metrics", promhttp.Handler())
-			metricsMux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
-				w.WriteHeader(http.StatusOK)
-			})
-			go func() {
-				err := http.ListenAndServe(fmt.Sprintf(":%s", metricsPort), metricsMux)
-				if err != nil {
-					panic(err)
-				}
-			}()
-		}
-	} else {
-		// Use a new ServerMux because expvar HTTP handler registers itself against DefaultServerMux
-		// as a consequence of importing it in client_golang/prometheus.
-		errorsMux = http.NewServeMux()
+	var listeners []Listener
+
+	// MUST use NewServerMux when not exporting /metrics because expvar HTTP handler registers
+	// against DefaultServerMux as a consequence of importing it in client_golang/prometheus.
+	if !isExportMetrics {
+		mux := http.NewServeMux()
+		mux.HandleFunc("/", errorHandler(errFilesPath, defaultFormat))
+		listeners = append(listeners, Listener{mux, CustomErrorPagesPort})
+		return listeners
 	}
 
-	errorsMux.HandleFunc("/", errorHandler(errFilesPath, defaultFormat))
-	err := http.ListenAndServe(fmt.Sprintf(":%s", CustomErrorPagesPort), errorsMux)
-	if err != nil {
-		panic(err)
+	// MUST use DefaultServerMux when exporting /metrics to the public because /debug/vars is
+	// only available with DefaultServerMux.
+	if metricsPort == CustomErrorPagesPort {
+		mux := http.DefaultServeMux
+		mux.Handle("/metrics", promhttp.Handler())
+		mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
+			w.WriteHeader(http.StatusOK)
+		})
+		mux.HandleFunc("/", errorHandler(errFilesPath, defaultFormat))
+		listeners = append(listeners, Listener{mux, CustomErrorPagesPort})
+		return listeners
 	}
+
+	// MUST use DefaultServerMux for /metrics and NewServerMux for custom error pages when you
+	// wish to expose /metrics only to internal services, because expvar HTTP handler registers
+	// against DefaultServerMux.
+	metricsMux := http.DefaultServeMux
+	metricsMux.Handle("/metrics", promhttp.Handler())
+	metricsMux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+	})
+	errorsMux := http.NewServeMux()
+	errorsMux.HandleFunc("/", errorHandler(errFilesPath, defaultFormat))
+	listeners = append(listeners, Listener{metricsMux, metricsPort}, Listener{errorsMux, CustomErrorPagesPort})
+	return listeners
+}
+
+func startListeners(listeners []Listener) {
+	var wg sync.WaitGroup
+
+	for _, listener := range listeners {
+		wg.Add(1)
+		go func(l Listener) {
+			defer wg.Done()
+			err := http.ListenAndServe(fmt.Sprintf(":%s", l.port), l.mux)
+			if err != nil {
+				log.Fatal(err)
+			}
+		}(listener)
+	}
+
+	wg.Wait()
+}
+
+type Listener struct {
+	mux  *http.ServeMux
+	port string
 }
 
 func errorHandler(path, defaultFormat string) func(http.ResponseWriter, *http.Request) {

From 4d5db854d1610b559858b359f80ff02a739c8f76 Mon Sep 17 00:00:00 2001
From: Ricardo Lopes <ricardoapl.dev@gmail.com>
Date: Wed, 14 Feb 2024 17:54:56 +0000
Subject: [PATCH 06/16] Fix custom error pages health check when not exporting
 /metrics

Signed-off-by: Ricardo Lopes <ricardoapl.dev@gmail.com>
---
 images/custom-error-pages/rootfs/main.go | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/images/custom-error-pages/rootfs/main.go b/images/custom-error-pages/rootfs/main.go
index 23a847927..c81770e7b 100644
--- a/images/custom-error-pages/rootfs/main.go
+++ b/images/custom-error-pages/rootfs/main.go
@@ -71,11 +71,11 @@ const (
 	DefaultFormatVar = "DEFAULT_RESPONSE_FORMAT"
 
 	// IsExportMetricsVar is the name of the environment variable indicating
-	// whether or not to export /metrics and /healthz.
+	// whether or not to export /metrics and /debug/vars.
 	IsExportMetricsVar = "IS_EXPORT_METRICS"
 
 	// MetricsPortVar is the name of the environment variable indicating
-	// the port on which to export /metrics and /healthz.
+	// the port on which to export /metrics and /debug/vars.
 	MetricsPortVar = "METRICS_PORT"
 
 	// CustomErrorPagesPort is the port on which to listen to serve custom error pages.
@@ -123,6 +123,9 @@ func createListeners() []Listener {
 	if !isExportMetrics {
 		mux := http.NewServeMux()
 		mux.HandleFunc("/", errorHandler(errFilesPath, defaultFormat))
+		mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
+			w.WriteHeader(http.StatusOK)
+		})
 		listeners = append(listeners, Listener{mux, CustomErrorPagesPort})
 		return listeners
 	}
@@ -132,10 +135,10 @@ func createListeners() []Listener {
 	if metricsPort == CustomErrorPagesPort {
 		mux := http.DefaultServeMux
 		mux.Handle("/metrics", promhttp.Handler())
+		mux.HandleFunc("/", errorHandler(errFilesPath, defaultFormat))
 		mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
 			w.WriteHeader(http.StatusOK)
 		})
-		mux.HandleFunc("/", errorHandler(errFilesPath, defaultFormat))
 		listeners = append(listeners, Listener{mux, CustomErrorPagesPort})
 		return listeners
 	}
@@ -145,11 +148,13 @@ func createListeners() []Listener {
 	// against DefaultServerMux.
 	metricsMux := http.DefaultServeMux
 	metricsMux.Handle("/metrics", promhttp.Handler())
-	metricsMux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
-		w.WriteHeader(http.StatusOK)
-	})
+
 	errorsMux := http.NewServeMux()
 	errorsMux.HandleFunc("/", errorHandler(errFilesPath, defaultFormat))
+	errorsMux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+	})
+
 	listeners = append(listeners, Listener{metricsMux, metricsPort}, Listener{errorsMux, CustomErrorPagesPort})
 	return listeners
 }

From afcd6b31abcac9e2766f5922d8a84fde2b1502ee Mon Sep 17 00:00:00 2001
From: Ricardo Lopes <ricardoapl.dev@gmail.com>
Date: Thu, 15 Feb 2024 13:09:47 +0000
Subject: [PATCH 07/16] Allow chart to expose extra ports for default backend

Signed-off-by: Ricardo Lopes <ricardoapl.dev@gmail.com>
---
 .../templates/default-backend-deployment.yaml              | 7 +++++++
 .../ingress-nginx/templates/default-backend-service.yaml   | 3 +++
 charts/ingress-nginx/values.yaml                           | 7 +++++++
 3 files changed, 17 insertions(+)

diff --git a/charts/ingress-nginx/templates/default-backend-deployment.yaml b/charts/ingress-nginx/templates/default-backend-deployment.yaml
index 4a17f7444..9e4a22344 100644
--- a/charts/ingress-nginx/templates/default-backend-deployment.yaml
+++ b/charts/ingress-nginx/templates/default-backend-deployment.yaml
@@ -93,6 +93,13 @@ spec:
             - name: http
               containerPort: {{ .Values.defaultBackend.port }}
               protocol: TCP
+          {{- if .Values.defaultBackend.service.extraPorts }}
+            {{- range .Values.defaultBackend.service.extraPorts }}
+            - name: {{ .name }}
+              containerPort: {{ .targetPort }}
+              protocol: {{ .protocol }}
+            {{- end }}
+          {{- end }}
         {{- if .Values.defaultBackend.extraVolumeMounts }}
           volumeMounts: {{- toYaml .Values.defaultBackend.extraVolumeMounts | nindent 12 }}
         {{- end }}
diff --git a/charts/ingress-nginx/templates/default-backend-service.yaml b/charts/ingress-nginx/templates/default-backend-service.yaml
index 5a836365b..d1f9fedd0 100644
--- a/charts/ingress-nginx/templates/default-backend-service.yaml
+++ b/charts/ingress-nginx/templates/default-backend-service.yaml
@@ -38,6 +38,9 @@ spec:
     {{- if semverCompare ">=1.20.0-0" .Capabilities.KubeVersion.Version }}
       appProtocol: http
     {{- end }}
+  {{- if .Values.defaultBackend.service.extraPorts }}
+    {{ toYaml .Values.defaultBackend.service.extraPorts | nindent 4 }}
+  {{- end}}
   selector:
     {{- include "ingress-nginx.selectorLabels" . | nindent 4 }}
     app.kubernetes.io/component: default-backend
diff --git a/charts/ingress-nginx/values.yaml b/charts/ingress-nginx/values.yaml
index 116adf7ca..0cc2db517 100644
--- a/charts/ingress-nginx/values.yaml
+++ b/charts/ingress-nginx/values.yaml
@@ -1176,6 +1176,13 @@ defaultBackend:
     loadBalancerSourceRanges: []
     servicePort: 80
     type: ClusterIP
+
+    # -- Additional ports to expose for defaultBackend pods
+    extraPorts: []
+    #  - name: metrics
+    #    port: 9090
+    #    protocol: HTTP
+    #    targetPort: 9090
   priorityClassName: ""
   # -- Labels to be added to the default backend resources
   labels: {}

From 3e2cb9ff621b30bb3ea1becf3978d9dd746680d1 Mon Sep 17 00:00:00 2001
From: Ricardo Lopes <ricardoapl.dev@gmail.com>
Date: Thu, 15 Feb 2024 14:39:00 +0000
Subject: [PATCH 08/16] Update README for the chart

Signed-off-by: Ricardo Lopes <ricardoapl.dev@gmail.com>
---
 charts/ingress-nginx/README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/charts/ingress-nginx/README.md b/charts/ingress-nginx/README.md
index c0b00e56d..83a6bed43 100644
--- a/charts/ingress-nginx/README.md
+++ b/charts/ingress-nginx/README.md
@@ -546,6 +546,7 @@ metadata:
 | defaultBackend.service.annotations | object | `{}` |  |
 | defaultBackend.service.clusterIPs | list | `[]` | Pre-defined cluster internal IP addresses of the default backend service. Take care of collisions with existing services. This value is immutable. Set once, it can not be changed without deleting and re-creating the service. Ref: https://kubernetes.io/docs/concepts/services-networking/service/#choosing-your-own-ip-address |
 | defaultBackend.service.externalIPs | list | `[]` | List of IP addresses at which the default backend service is available # Ref: https://kubernetes.io/docs/concepts/services-networking/service/#external-ips # |
+| defaultBackend.service.extraPorts | list | `[]` | Additional ports to expose for defaultBackend pods |
 | defaultBackend.service.loadBalancerSourceRanges | list | `[]` |  |
 | defaultBackend.service.servicePort | int | `80` |  |
 | defaultBackend.service.type | string | `"ClusterIP"` |  |

From 9ce143b46295c61a11e376d2ee9756373ca07a18 Mon Sep 17 00:00:00 2001
From: Ricardo Lopes <ricardoapl.dev@gmail.com>
Date: Fri, 16 Feb 2024 11:01:41 +0000
Subject: [PATCH 09/16] Fix unsupported value HTTP for protocol

Signed-off-by: Ricardo Lopes <ricardoapl.dev@gmail.com>
---
 charts/ingress-nginx/values.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/charts/ingress-nginx/values.yaml b/charts/ingress-nginx/values.yaml
index 0cc2db517..bab44e9ef 100644
--- a/charts/ingress-nginx/values.yaml
+++ b/charts/ingress-nginx/values.yaml
@@ -1181,7 +1181,7 @@ defaultBackend:
     extraPorts: []
     #  - name: metrics
     #    port: 9090
-    #    protocol: HTTP
+    #    protocol: TCP
     #    targetPort: 9090
   priorityClassName: ""
   # -- Labels to be added to the default backend resources

From 02776b42a5bd4c2f7aec1185660540a7b381157d Mon Sep 17 00:00:00 2001
From: Ricardo Lopes <ricardoapl.dev@gmail.com>
Date: Thu, 7 Mar 2024 14:52:05 +0000
Subject: [PATCH 10/16] Rename environment variables

Signed-off-by: Ricardo Lopes <ricardoapl.dev@gmail.com>
---
 images/custom-error-pages/rootfs/main.go | 32 ++++++++++++------------
 1 file changed, 16 insertions(+), 16 deletions(-)

diff --git a/images/custom-error-pages/rootfs/main.go b/images/custom-error-pages/rootfs/main.go
index c81770e7b..8a8cb6846 100644
--- a/images/custom-error-pages/rootfs/main.go
+++ b/images/custom-error-pages/rootfs/main.go
@@ -60,23 +60,23 @@ const (
 	// RequestId is a unique ID that identifies the request - same as for backend service
 	RequestId = "X-Request-ID"
 
-	// ErrFilesPathVar is the name of the environment variable indicating
+	// ErrFilesPathEnvVar is the name of the environment variable indicating
 	// the location on disk of files served by the handler.
-	ErrFilesPathVar = "ERROR_FILES_PATH"
+	ErrFilesPathEnvVar = "ERROR_FILES_PATH"
 
-	// DefaultFormatVar is the name of the environment variable indicating
+	// DefaultFormatEnvVar is the name of the environment variable indicating
 	// the default error MIME type that should be returned if either the
 	// client does not specify an Accept header, or the Accept header provided
 	// cannot be mapped to a file extension.
-	DefaultFormatVar = "DEFAULT_RESPONSE_FORMAT"
+	DefaultFormatEnvVar = "DEFAULT_RESPONSE_FORMAT"
 
-	// IsExportMetricsVar is the name of the environment variable indicating
+	//  IsMetricsExportEnvVar is the name of the environment variable indicating
 	// whether or not to export /metrics and /debug/vars.
-	IsExportMetricsVar = "IS_EXPORT_METRICS"
+	IsMetricsExportEnvVar = "IS_METRICS_EXPORT"
 
-	// MetricsPortVar is the name of the environment variable indicating
+	// MetricsPortEnvVar is the name of the environment variable indicating
 	// the port on which to export /metrics and /debug/vars.
-	MetricsPortVar = "METRICS_PORT"
+	MetricsPortEnvVar = "METRICS_PORT"
 
 	// CustomErrorPagesPort is the port on which to listen to serve custom error pages.
 	CustomErrorPagesPort = "8080"
@@ -94,26 +94,26 @@ func main() {
 
 func createListeners() []Listener {
 	errFilesPath := "/www"
-	if os.Getenv(ErrFilesPathVar) != "" {
-		errFilesPath = os.Getenv(ErrFilesPathVar)
+	if os.Getenv(ErrFilesPathEnvVar) != "" {
+		errFilesPath = os.Getenv(ErrFilesPathEnvVar)
 	}
 
 	defaultFormat := "text/html"
-	if os.Getenv(DefaultFormatVar) != "" {
-		defaultFormat = os.Getenv(DefaultFormatVar)
+	if os.Getenv(DefaultFormatEnvVar) != "" {
+		defaultFormat = os.Getenv(DefaultFormatEnvVar)
 	}
 
 	isExportMetrics := true
-	if os.Getenv(IsExportMetricsVar) != "" {
-		val, err := strconv.ParseBool(os.Getenv(IsExportMetricsVar))
+	if os.Getenv(IsMetricsExportEnvVar) != "" {
+		val, err := strconv.ParseBool(os.Getenv(IsMetricsExportEnvVar))
 		if err == nil {
 			isExportMetrics = val
 		}
 	}
 
 	metricsPort := "8080"
-	if os.Getenv(MetricsPortVar) != "" {
-		metricsPort = os.Getenv(MetricsPortVar)
+	if os.Getenv(MetricsPortEnvVar) != "" {
+		metricsPort = os.Getenv(MetricsPortEnvVar)
 	}
 
 	var listeners []Listener

From e1a319469467048d3f42c9730d31334130b0a357 Mon Sep 17 00:00:00 2001
From: Ricardo Lopes <ricardoapl.dev@gmail.com>
Date: Mon, 13 May 2024 16:33:02 +0100
Subject: [PATCH 11/16] Add unit tests for Helm chart

Signed-off-by: Ricardo Lopes <ricardoapl.dev@gmail.com>
---
 .../tests/default-backend-deployment_test.yaml  | 15 +++++++++++++++
 .../tests/default-backend-service_test.yaml     | 17 +++++++++++++++++
 2 files changed, 32 insertions(+)

diff --git a/charts/ingress-nginx/tests/default-backend-deployment_test.yaml b/charts/ingress-nginx/tests/default-backend-deployment_test.yaml
index 11d400c46..0839e9c00 100644
--- a/charts/ingress-nginx/tests/default-backend-deployment_test.yaml
+++ b/charts/ingress-nginx/tests/default-backend-deployment_test.yaml
@@ -196,3 +196,18 @@ tests:
       - equal:
           path: spec.template.spec.automountServiceAccountToken
           value: false
+
+  - it: should create a Deployment with extraPorts if `defaultBackend.service.extraPorts` is set
+    set:
+      defaultBackend.enabled: true
+      defaultBackend.service.extraPorts[0]:
+        name: example
+        protocol: TCP
+        targetPort: 9999
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].ports
+          content:
+            name: example
+            protocol: TCP
+            containerPort: 9999
diff --git a/charts/ingress-nginx/tests/default-backend-service_test.yaml b/charts/ingress-nginx/tests/default-backend-service_test.yaml
index 521d82091..2c9223d79 100644
--- a/charts/ingress-nginx/tests/default-backend-service_test.yaml
+++ b/charts/ingress-nginx/tests/default-backend-service_test.yaml
@@ -50,3 +50,20 @@ tests:
           value:
             - 10.0.0.1
             - fd00::1
+
+  - it: should create a Service with extraPorts if `defaultBackend.service.extraPorts` is set
+    set:
+      defaultBackend.enabled: true
+      defaultBackend.service.extraPorts[0]:
+        name: example
+        port: 8888
+        protocol: TCP
+        targetPort: 65535
+    asserts:
+      - contains:
+          path: spec.ports
+          content:
+            name: example
+            port: 8888
+            protocol: TCP
+            targetPort: 65535

From 9e568609a7ce6aae610fb12d24a88cf97e686bb7 Mon Sep 17 00:00:00 2001
From: Ricardo Lopes <ricardoapl.dev@gmail.com>
Date: Wed, 15 May 2024 14:52:18 +0100
Subject: [PATCH 12/16] Add e2e tests for Helm chart

Signed-off-by: Ricardo Lopes <ricardoapl.dev@gmail.com>
---
 .../ci/deployment-defaultbackend-values.yaml  | 21 +++++++++++++++++++
 1 file changed, 21 insertions(+)
 create mode 100644 charts/ingress-nginx/ci/deployment-defaultbackend-values.yaml

diff --git a/charts/ingress-nginx/ci/deployment-defaultbackend-values.yaml b/charts/ingress-nginx/ci/deployment-defaultbackend-values.yaml
new file mode 100644
index 000000000..44c06a56d
--- /dev/null
+++ b/charts/ingress-nginx/ci/deployment-defaultbackend-values.yaml
@@ -0,0 +1,21 @@
+controller:
+  image:
+    repository: ingress-controller/controller
+    tag: 1.0.0-dev
+    digest: null
+  service:
+    type: ClusterIP
+
+defaultBackend:
+  enabled: true
+  extraEnvs:
+    - name: IS_METRICS_EXPORT
+      value: "false"
+    - name: METRICS_PORT
+      value: "8081"
+  service:
+    extraPorts:
+      - name: metrics
+        port: 8081
+        protocol: TCP
+        targetPort: 8081

From ec239c69a9bf3ef93424765a2ba71c98eeabe2b8 Mon Sep 17 00:00:00 2001
From: Ricardo Lopes <ricardoapl.dev@gmail.com>
Date: Thu, 16 May 2024 12:37:08 +0100
Subject: [PATCH 13/16] Add e2e tests for custom-error-pages

Signed-off-by: Ricardo Lopes <ricardoapl.dev@gmail.com>
---
 docs/e2e-tests.md                             |   4 +
 test/e2e/CUSTOMERRORPAGES_IMAGE               |   1 +
 test/e2e/defaultbackend/custom_error_pages.go | 121 ++++++++++++++++++
 test/e2e/framework/deployment.go              |  75 +++++++++++
 test/e2e/run-e2e-suite.sh                     |   2 +
 test/e2e/run-kind-e2e.sh                      |   8 ++
 6 files changed, 211 insertions(+)
 create mode 100644 test/e2e/CUSTOMERRORPAGES_IMAGE
 create mode 100644 test/e2e/defaultbackend/custom_error_pages.go

diff --git a/docs/e2e-tests.md b/docs/e2e-tests.md
index 163a5ff5c..7fb1eb63d 100644
--- a/docs/e2e-tests.md
+++ b/docs/e2e-tests.md
@@ -283,6 +283,10 @@ Do not try to edit it manually.
 - [should return a self generated SSL certificate](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/defaultbackend/ssl.go#L29)
 ### [[Default Backend] change default settings](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/defaultbackend/with_hosts.go#L30)
 - [should apply the annotation to the default backend](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/defaultbackend/with_hosts.go#L38)
+### [[Default Backend] custom-error-pages](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/defaultbackend/custom_error_pages.go#L33)
+- [should export /metrics and /debug/vars by default](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/defaultbackend/custom_error_pages.go#L36)
+- [shouldn't export /metrics and /debug/vars when IS_METRICS_EXPORT is set to false](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/defaultbackend/custom_error_pages.go#L57)
+- [shouldn't export /metrics and /debug/vars when METRICS_PORT is set to a different port](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/defaultbackend/custom_error_pages.go#L80)
 ### [[Disable Leader] Routing works when leader election was disabled](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/disableleaderelection/disable_leader.go#L28)
 - [should create multiple ingress routings rules when leader election has disabled](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/disableleaderelection/disable_leader.go#L35)
 ### [[Endpointslices] long service name](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/endpointslices/longname.go#L29)
diff --git a/test/e2e/CUSTOMERRORPAGES_IMAGE b/test/e2e/CUSTOMERRORPAGES_IMAGE
new file mode 100644
index 000000000..865e0bb3e
--- /dev/null
+++ b/test/e2e/CUSTOMERRORPAGES_IMAGE
@@ -0,0 +1 @@
+localhost/custom-error-pages:e2e
diff --git a/test/e2e/defaultbackend/custom_error_pages.go b/test/e2e/defaultbackend/custom_error_pages.go
new file mode 100644
index 000000000..597a69b19
--- /dev/null
+++ b/test/e2e/defaultbackend/custom_error_pages.go
@@ -0,0 +1,121 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package defaultbackend
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"strings"
+
+	"github.com/onsi/ginkgo/v2"
+	"github.com/stretchr/testify/assert"
+	appsv1 "k8s.io/api/apps/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"k8s.io/ingress-nginx/test/e2e/framework"
+)
+
+var _ = framework.IngressNginxDescribe("[Default Backend] custom-error-pages", func() {
+	f := framework.NewDefaultFramework("custom-error-pages")
+
+	ginkgo.It("should export /metrics and /debug/vars by default", func() {
+		tt := []struct {
+			Name   string
+			Path   string
+			Status int
+		}{
+			{"request to /metrics should return HTTP 200", "/metrics", http.StatusOK},
+			{"request to /debug/vars should return HTTP 200", "/debug/vars", http.StatusOK},
+		}
+
+		setupIngressControllerWithCustomErrorPages(f, nil)
+
+		for _, t := range tt {
+			ginkgo.By(t.Name)
+			f.HTTPTestClient().
+				GET(t.Path).
+				Expect().
+				Status(t.Status)
+		}
+	})
+
+	ginkgo.It("shouldn't export /metrics and /debug/vars when IS_METRICS_EXPORT is set to false", func() {
+		tt := []struct {
+			Name   string
+			Path   string
+			Status int
+		}{
+			{"request to /metrics should return HTTP 404", "/metrics", http.StatusNotFound},
+			{"request to /debug/vars should return HTTP 404", "/debug/vars", http.StatusNotFound},
+		}
+
+		setupIngressControllerWithCustomErrorPages(f, map[string]string{
+			"IS_METRICS_EXPORT": "false",
+		})
+
+		for _, t := range tt {
+			ginkgo.By(t.Name)
+			f.HTTPTestClient().
+				GET(t.Path).
+				Expect().
+				Status(t.Status)
+		}
+	})
+
+	ginkgo.It("shouldn't export /metrics and /debug/vars when METRICS_PORT is set to a different port", func() {
+		tt := []struct {
+			Name   string
+			Path   string
+			Status int
+		}{
+			{"request to /metrics should return HTTP 404", "/metrics", http.StatusNotFound},
+			{"request to /debug/vars should return HTTP 404", "/debug/vars", http.StatusNotFound},
+		}
+
+		setupIngressControllerWithCustomErrorPages(f, map[string]string{
+			"IS_METRICS_EXPORT": "true",
+			"METRICS_PORT":      "8081",
+		})
+
+		for _, t := range tt {
+			ginkgo.By(t.Name)
+			f.HTTPTestClient().
+				GET(t.Path).
+				Expect().
+				Status(t.Status)
+		}
+	})
+})
+
+func setupIngressControllerWithCustomErrorPages(f *framework.Framework, envVars map[string]string) {
+	f.NewCustomErrorPagesDeployment(framework.WithEnvVars(envVars))
+
+	err := f.UpdateIngressControllerDeployment(func(deployment *appsv1.Deployment) error {
+		args := deployment.Spec.Template.Spec.Containers[0].Args
+		args = append(args, fmt.Sprintf("--default-backend-service=%v/%v", f.Namespace, framework.CustomErrorPagesService))
+		deployment.Spec.Template.Spec.Containers[0].Args = args
+		_, err := f.KubeClientSet.AppsV1().Deployments(f.Namespace).Update(context.TODO(), deployment, metav1.UpdateOptions{})
+		return err
+	})
+	assert.Nil(ginkgo.GinkgoT(), err, "updating deployment")
+
+	f.WaitForNginxServer("_",
+		func(server string) bool {
+			return strings.Contains(server, `set $proxy_upstream_name "upstream-default-backend"`)
+		})
+}
diff --git a/test/e2e/framework/deployment.go b/test/e2e/framework/deployment.go
index d6e59f18a..148552f3e 100644
--- a/test/e2e/framework/deployment.go
+++ b/test/e2e/framework/deployment.go
@@ -49,6 +49,12 @@ var HTTPBunImage = os.Getenv("HTTPBUN_IMAGE")
 // EchoImage is the default image to be used by the echo service
 const EchoImage = "registry.k8s.io/ingress-nginx/e2e-test-echo:v1.0.1@sha256:1cec65aa768720290d05d65ab1c297ca46b39930e56bc9488259f9114fcd30e2" //#nosec G101
 
+// CustomErrorPagesService name of the deployment for the custom-error-pages app
+const CustomErrorPagesService = "custom-error-pages"
+
+// CustomErrorPagesImage is the default image that is used to deploy custom-error-pages with the framework
+var CustomErrorPagesImage = os.Getenv("CUSTOMERRORPAGES_IMAGE")
+
 // TODO: change all Deployment functions to use these options
 // in order to reduce complexity and have a unified API across the
 // framework
@@ -58,6 +64,7 @@ type deploymentOptions struct {
 	image          string
 	replicas       int
 	svcAnnotations map[string]string
+	envVars        map[string]string
 }
 
 // WithDeploymentNamespace allows configuring the deployment's namespace
@@ -103,6 +110,74 @@ func WithImage(i string) func(*deploymentOptions) {
 	}
 }
 
+// WithEnvVars allows configuring environment variables for the deployment
+func WithEnvVars(e map[string]string) func(*deploymentOptions) {
+	return func(o *deploymentOptions) {
+		o.envVars = e
+	}
+}
+
+// NewCustomErrorPagesDeployment creates a new single replica deployment of the custom-error-pages server image in a particular namespace
+func (f *Framework) NewCustomErrorPagesDeployment(opts ...func(*deploymentOptions)) {
+	options := &deploymentOptions{
+		namespace: f.Namespace,
+		name:      CustomErrorPagesService,
+		replicas:  1,
+		image:     CustomErrorPagesImage,
+	}
+	for _, o := range opts {
+		o(options)
+	}
+
+	envVars := []corev1.EnvVar{}
+	for k, v := range options.envVars {
+		envVars = append(envVars, corev1.EnvVar{Name: k, Value: v})
+	}
+
+	f.EnsureDeployment(newDeployment(
+		options.name,
+		options.namespace,
+		options.image,
+		8080,
+		int32(options.replicas),
+		nil, nil,
+		envVars,
+		[]corev1.VolumeMount{},
+		[]corev1.Volume{},
+		false,
+	))
+
+	f.EnsureService(&corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:        options.name,
+			Namespace:   options.namespace,
+			Annotations: options.svcAnnotations,
+		},
+		Spec: corev1.ServiceSpec{
+			Ports: []corev1.ServicePort{
+				{
+					Name:       "http",
+					Port:       8080,
+					TargetPort: intstr.FromInt(8080),
+					Protocol:   corev1.ProtocolTCP,
+				},
+			},
+			Selector: map[string]string{
+				"app": options.name,
+			},
+		},
+	})
+
+	err := WaitForEndpoints(
+		f.KubeClientSet,
+		DefaultTimeout,
+		options.name,
+		options.namespace,
+		options.replicas,
+	)
+	assert.Nil(ginkgo.GinkgoT(), err, "waiting for endpoints to become ready")
+}
+
 // NewEchoDeployment creates a new single replica deployment of the echo server image in a particular namespace
 func (f *Framework) NewEchoDeployment(opts ...func(*deploymentOptions)) {
 	options := &deploymentOptions{
diff --git a/test/e2e/run-e2e-suite.sh b/test/e2e/run-e2e-suite.sh
index 909368e96..9629889f1 100755
--- a/test/e2e/run-e2e-suite.sh
+++ b/test/e2e/run-e2e-suite.sh
@@ -52,6 +52,7 @@ fi
 BASEDIR=$(dirname "$0")
 NGINX_BASE_IMAGE=$(cat $BASEDIR/../../NGINX_BASE)
 HTTPBUN_IMAGE=$(cat $BASEDIR/HTTPBUN_IMAGE)
+CUSTOMERRORPAGES_IMAGE=$(cat $BASEDIR/CUSTOMERRORPAGES_IMAGE)
 
 echo -e "${BGREEN}Granting permissions to ingress-nginx e2e service account...${NC}"
 kubectl create serviceaccount ingress-nginx-e2e || true
@@ -82,6 +83,7 @@ kubectl run --rm \
   --env="E2E_CHECK_LEAKS=${E2E_CHECK_LEAKS}" \
   --env="NGINX_BASE_IMAGE=${NGINX_BASE_IMAGE}" \
   --env="HTTPBUN_IMAGE=${HTTPBUN_IMAGE}" \
+  --env="CUSTOMERRORPAGES_IMAGE=${CUSTOMERRORPAGES_IMAGE}" \
   --overrides='{ "apiVersion": "v1", "spec":{"serviceAccountName": "ingress-nginx-e2e"}}' \
   e2e --image=nginx-ingress-controller:e2e
 
diff --git a/test/e2e/run-kind-e2e.sh b/test/e2e/run-kind-e2e.sh
index 01b909f30..2e09872e2 100755
--- a/test/e2e/run-kind-e2e.sh
+++ b/test/e2e/run-kind-e2e.sh
@@ -52,6 +52,7 @@ export KUBECONFIG="${KUBECONFIG:-$HOME/.kube/kind-config-$KIND_CLUSTER_NAME}"
 SKIP_INGRESS_IMAGE_CREATION="${SKIP_INGRESS_IMAGE_CREATION:-false}"
 SKIP_E2E_IMAGE_CREATION="${SKIP_E2E_IMAGE_CREATION:=false}"
 SKIP_CLUSTER_CREATION="${SKIP_CLUSTER_CREATION:-false}"
+SKIP_CUSTOMERRORPAGES_IMAGE_CREATION="${SKIP_CUSTOMERRORPAGES_IMAGE_CREATION:-false}"
 
 if ! command -v kind --version &> /dev/null; then
   echo "kind is not installed. Use the package manager or visit the official site https://kind.sigs.k8s.io/"
@@ -104,6 +105,12 @@ if [ "${SKIP_E2E_IMAGE_CREATION}" = "false" ]; then
   echo "[dev-env] ..done building e2e-image"
 fi
 
+if [ "${SKIP_CUSTOMERRORPAGES_IMAGE_CREATION}" = "false" ]; then
+  echo "[dev-env] building custom-error-pages image"
+  REGISTRY=localhost NAME=custom-error-pages TAG=e2e make -C "${DIR}"/../../images build
+  echo "[dev-env] .. done building custom-error-pages image"
+fi
+
 # Preload images used in e2e tests
 KIND_WORKERS=$(kind get nodes --name="${KIND_CLUSTER_NAME}" | grep worker | awk '{printf (NR>1?",":"") $1}')
 
@@ -111,5 +118,6 @@ echo "[dev-env] copying docker images to cluster..."
 
 kind load docker-image --name="${KIND_CLUSTER_NAME}" --nodes="${KIND_WORKERS}" nginx-ingress-controller:e2e
 kind load docker-image --name="${KIND_CLUSTER_NAME}" --nodes="${KIND_WORKERS}" "${REGISTRY}"/controller:"${TAG}"
+kind load docker-image --name="${KIND_CLUSTER_NAME}" --nodes="${KIND_WORKERS}" "localhost/custom-error-pages:e2e"
 echo "[dev-env] running e2e tests..."
 make -C "${DIR}"/../../ e2e-test

From e425baf7348fc46dea772b17c3bec4e1e33b33cb Mon Sep 17 00:00:00 2001
From: Ricardo Lopes <ricardoapl.dev@gmail.com>
Date: Fri, 17 May 2024 16:59:52 +0100
Subject: [PATCH 14/16] Load images from ci build cache

Signed-off-by: Ricardo Lopes <ricardoapl.dev@gmail.com>
---
 .github/workflows/ci.yaml              | 6 ++++++
 .github/workflows/zz-tmpl-k8s-e2e.yaml | 2 ++
 images/Makefile                        | 8 ++++----
 test/e2e/CUSTOMERRORPAGES_IMAGE        | 2 +-
 test/e2e/run-kind-e2e.sh               | 4 ++--
 5 files changed, 15 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 6d0532105..1d6188517 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -193,12 +193,18 @@ jobs:
           export TAGNGINX=$(cat images/nginx/TAG)
           make BASE_IMAGE=registry.k8s.io/ingress-nginx/nginx:${TAGNGINX} clean-image build image image-chroot
           make -C test/e2e-image image
+          cd images/custom-error-pages/rootfs && docker image build \
+              --build-arg=BASE_IMAGE=registry.k8s.io/ingress-nginx/nginx-1.25:${TAGNGINX} \
+              --build-arg GOLANG_VERSION=$(cat ../../../GOLANG_VERSION) \
+              --tag ingress-controller/custom-error-pages:1.0.0-dev . \
+              && cd ../../..
 
           echo "creating images cache..."
           docker save \
             nginx-ingress-controller:e2e \
             ingress-controller/controller:1.0.0-dev \
             ingress-controller/controller-chroot:1.0.0-dev \
+            ingress-controller/custom-error-pages:1.0.0-dev \
             | gzip > docker.tar.gz
 
       - name: cache
diff --git a/.github/workflows/zz-tmpl-k8s-e2e.yaml b/.github/workflows/zz-tmpl-k8s-e2e.yaml
index 996b673f9..3eaab8c82 100644
--- a/.github/workflows/zz-tmpl-k8s-e2e.yaml
+++ b/.github/workflows/zz-tmpl-k8s-e2e.yaml
@@ -43,6 +43,8 @@ jobs:
           SKIP_CLUSTER_CREATION: true
           SKIP_INGRESS_IMAGE_CREATION: true
           SKIP_E2E_IMAGE_CREATION: true
+          SKIP_CUSTOMERRORPAGES_IMAGE_CREATION: true
+          ENABLE_VALIDATIONS: ${{ inputs.variation == 'VALIDATIONS' }}
           IS_CHROOT: ${{ inputs.variation == 'CHROOT' }}
         run: |
           kind get kubeconfig > $HOME/.kube/kind-config-kind
diff --git a/images/Makefile b/images/Makefile
index 31560168d..ca7b52803 100644
--- a/images/Makefile
+++ b/images/Makefile
@@ -41,9 +41,9 @@ EXTRAARGS ?= $(shell cat $(NAME)/EXTRAARGS)
 export DOCKER_CLI_EXPERIMENTAL=enabled
 
 # build with buildx
-PLATFORMS?=linux/amd64,linux/arm,linux/arm64
-OUTPUT=
-PROGRESS=plain
+BUILDX_PLATFORMS ?= linux/amd64,linux/arm,linux/arm64,linux/s390x
+OUTPUT ?=
+PROGRESS = plain
 
 
 precheck:
@@ -58,7 +58,7 @@ build: precheck ensure-buildx
 		--label=org.opencontainers.image.description="Ingress NGINX $(NAME) image" \
 		--build-arg BASE_IMAGE=$(BASE_IMAGE) \
 		--build-arg GOLANG_VERSION=$(GO_VERSION) \
-		--platform=${PLATFORMS} $(OUTPUT) \
+		--platform=${BUILDX_PLATFORMS} $(OUTPUT) \
 		--progress=$(PROGRESS) \
 		--pull $(EXTRAARGS) \
 		-t $(IMAGE):$(TAG) $(NAME)/rootfs
diff --git a/test/e2e/CUSTOMERRORPAGES_IMAGE b/test/e2e/CUSTOMERRORPAGES_IMAGE
index 865e0bb3e..cd3f9faa2 100644
--- a/test/e2e/CUSTOMERRORPAGES_IMAGE
+++ b/test/e2e/CUSTOMERRORPAGES_IMAGE
@@ -1 +1 @@
-localhost/custom-error-pages:e2e
+ingress-controller/custom-error-pages:1.0.0-dev
diff --git a/test/e2e/run-kind-e2e.sh b/test/e2e/run-kind-e2e.sh
index 2e09872e2..c507d8658 100755
--- a/test/e2e/run-kind-e2e.sh
+++ b/test/e2e/run-kind-e2e.sh
@@ -107,7 +107,7 @@ fi
 
 if [ "${SKIP_CUSTOMERRORPAGES_IMAGE_CREATION}" = "false" ]; then
   echo "[dev-env] building custom-error-pages image"
-  REGISTRY=localhost NAME=custom-error-pages TAG=e2e make -C "${DIR}"/../../images build
+  make NAME=custom-error-pages -C "${DIR}"/../../images build
   echo "[dev-env] .. done building custom-error-pages image"
 fi
 
@@ -118,6 +118,6 @@ echo "[dev-env] copying docker images to cluster..."
 
 kind load docker-image --name="${KIND_CLUSTER_NAME}" --nodes="${KIND_WORKERS}" nginx-ingress-controller:e2e
 kind load docker-image --name="${KIND_CLUSTER_NAME}" --nodes="${KIND_WORKERS}" "${REGISTRY}"/controller:"${TAG}"
-kind load docker-image --name="${KIND_CLUSTER_NAME}" --nodes="${KIND_WORKERS}" "localhost/custom-error-pages:e2e"
+kind load docker-image --name="${KIND_CLUSTER_NAME}" --nodes="${KIND_WORKERS}" "${REGISTRY}"/custom-error-pages:"${TAG}"
 echo "[dev-env] running e2e tests..."
 make -C "${DIR}"/../../ e2e-test

From 942aeefaa9f6a53bac03619294d54446b319b78c Mon Sep 17 00:00:00 2001
From: Ricardo Lopes <ricardoapl.dev@gmail.com>
Date: Wed, 4 Dec 2024 15:44:54 +0000
Subject: [PATCH 15/16] Ignore G115 safe conversion int -> int32

Signed-off-by: Ricardo Lopes <ricardoapl.dev@gmail.com>
---
 test/e2e/framework/deployment.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/test/e2e/framework/deployment.go b/test/e2e/framework/deployment.go
index 148552f3e..63cdbc561 100644
--- a/test/e2e/framework/deployment.go
+++ b/test/e2e/framework/deployment.go
@@ -139,7 +139,7 @@ func (f *Framework) NewCustomErrorPagesDeployment(opts ...func(*deploymentOption
 		options.namespace,
 		options.image,
 		8080,
-		int32(options.replicas),
+		int32(options.replicas), //nolint:gosec // disable G115
 		nil, nil,
 		envVars,
 		[]corev1.VolumeMount{},
@@ -195,7 +195,7 @@ func (f *Framework) NewEchoDeployment(opts ...func(*deploymentOptions)) {
 		options.namespace,
 		options.image,
 		80,
-		int32(options.replicas),
+		int32(options.replicas), //nolint:gosec // disable G115
 		nil, nil, nil,
 		[]corev1.VolumeMount{},
 		[]corev1.Volume{},
@@ -286,7 +286,7 @@ func (f *Framework) NewHttpbunDeployment(opts ...func(*deploymentOptions)) strin
 		options.namespace,
 		options.image,
 		80,
-		int32(options.replicas),
+		int32(options.replicas), //nolint:gosec // disable G115
 		nil, nil,
 		// Required to get hostname information
 		[]corev1.EnvVar{

From 9781d2b7f354ad8de6c2f84e328a5a88eb2ac9b6 Mon Sep 17 00:00:00 2001
From: Ricardo Lopes <ricardoapl.dev@gmail.com>
Date: Mon, 16 Dec 2024 09:58:46 +0000
Subject: [PATCH 16/16] Fix re-adding stuff on rebase

Signed-off-by: Ricardo Lopes <ricardoapl.dev@gmail.com>
---
 .github/workflows/ci.yaml              | 2 +-
 .github/workflows/zz-tmpl-k8s-e2e.yaml | 1 -
 images/Makefile                        | 8 ++++----
 3 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 1d6188517..343f3342d 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -194,7 +194,7 @@ jobs:
           make BASE_IMAGE=registry.k8s.io/ingress-nginx/nginx:${TAGNGINX} clean-image build image image-chroot
           make -C test/e2e-image image
           cd images/custom-error-pages/rootfs && docker image build \
-              --build-arg=BASE_IMAGE=registry.k8s.io/ingress-nginx/nginx-1.25:${TAGNGINX} \
+              --build-arg=BASE_IMAGE=registry.k8s.io/ingress-nginx/nginx:${TAGNGINX} \
               --build-arg GOLANG_VERSION=$(cat ../../../GOLANG_VERSION) \
               --tag ingress-controller/custom-error-pages:1.0.0-dev . \
               && cd ../../..
diff --git a/.github/workflows/zz-tmpl-k8s-e2e.yaml b/.github/workflows/zz-tmpl-k8s-e2e.yaml
index 3eaab8c82..eb2424519 100644
--- a/.github/workflows/zz-tmpl-k8s-e2e.yaml
+++ b/.github/workflows/zz-tmpl-k8s-e2e.yaml
@@ -44,7 +44,6 @@ jobs:
           SKIP_INGRESS_IMAGE_CREATION: true
           SKIP_E2E_IMAGE_CREATION: true
           SKIP_CUSTOMERRORPAGES_IMAGE_CREATION: true
-          ENABLE_VALIDATIONS: ${{ inputs.variation == 'VALIDATIONS' }}
           IS_CHROOT: ${{ inputs.variation == 'CHROOT' }}
         run: |
           kind get kubeconfig > $HOME/.kube/kind-config-kind
diff --git a/images/Makefile b/images/Makefile
index ca7b52803..6500638b3 100644
--- a/images/Makefile
+++ b/images/Makefile
@@ -41,9 +41,9 @@ EXTRAARGS ?= $(shell cat $(NAME)/EXTRAARGS)
 export DOCKER_CLI_EXPERIMENTAL=enabled
 
 # build with buildx
-BUILDX_PLATFORMS ?= linux/amd64,linux/arm,linux/arm64,linux/s390x
-OUTPUT ?=
-PROGRESS = plain
+PLATFORMS?=linux/amd64,linux/arm,linux/arm64
+OUTPUT?=
+PROGRESS=plain
 
 
 precheck:
@@ -58,7 +58,7 @@ build: precheck ensure-buildx
 		--label=org.opencontainers.image.description="Ingress NGINX $(NAME) image" \
 		--build-arg BASE_IMAGE=$(BASE_IMAGE) \
 		--build-arg GOLANG_VERSION=$(GO_VERSION) \
-		--platform=${BUILDX_PLATFORMS} $(OUTPUT) \
+		--platform=${PLATFORMS} $(OUTPUT) \
 		--progress=$(PROGRESS) \
 		--pull $(EXTRAARGS) \
 		-t $(IMAGE):$(TAG) $(NAME)/rootfs