From fdf53cba81706d2000f1c29290435c3940e5a522 Mon Sep 17 00:00:00 2001
From: Rafael da Fonseca <rafael.fonseca@wildlifestudios.com>
Date: Fri, 26 Jan 2024 13:48:36 +0000
Subject: [PATCH] feat: Add support for h2c communications

---
 charts/ingress-nginx/README.md                |  4 +-
 charts/ingress-nginx/README.md.gotmpl         |  1 +
 charts/ingress-nginx/templates/NOTES.txt      |  6 ++
 .../controller-service-internal.yaml          | 12 ++++
 .../templates/controller-service.yaml         | 12 ++++
 charts/ingress-nginx/values.yaml              | 10 ++++
 images/nginx-1.25/rootfs/Dockerfile           |  2 +-
 images/nginx/rootfs/Dockerfile                |  2 +-
 internal/ingress/controller/config/config.go  |  1 +
 internal/ingress/controller/controller.go     |  1 +
 .../ingress/controller/template/template.go   | 57 +++++++++++++++++++
 pkg/flags/flags.go                            |  6 ++
 rootfs/Dockerfile-chroot                      |  2 +-
 rootfs/etc/nginx/template/nginx.tmpl          |  2 +
 14 files changed, 114 insertions(+), 4 deletions(-)

diff --git a/charts/ingress-nginx/README.md b/charts/ingress-nginx/README.md
index 7c351e4c8..3351c4886 100644
--- a/charts/ingress-nginx/README.md
+++ b/charts/ingress-nginx/README.md
@@ -289,7 +289,7 @@ As of version `1.26.0` of this chart, by simply not providing any clusterIP valu
 | controller.configAnnotations | object | `{}` | Annotations to be added to the controller config configuration configmap. |
 | controller.configMapNamespace | string | `""` | Allows customization of the configmap / nginx-configmap namespace; defaults to $(POD_NAMESPACE) |
 | controller.containerName | string | `"controller"` | Configures the controller container name |
-| controller.containerPort | object | `{"http":80,"https":443}` | Configures the ports that the nginx-controller listens on |
+| controller.containerPort | object | `{"http":80,"h2c":81,"https":443}` | Configures the ports that the nginx-controller listens on |
 | controller.containerSecurityContext | object | `{}` | Security context for controller containers |
 | controller.customTemplate.configMapKey | string | `""` |  |
 | controller.customTemplate.configMapName | string | `""` |  |
@@ -313,6 +313,7 @@ As of version `1.26.0` of this chart, by simply not providing any clusterIP valu
 | controller.hostNetwork | bool | `false` | Required for use with CNI based kubernetes installations (such as ones set up by kubeadm), since CNI and hostport don't mix yet. Can be deprecated once https://github.com/kubernetes/kubernetes/issues/23920 is merged |
 | controller.hostPort.enabled | bool | `false` | Enable 'hostPort' or not |
 | controller.hostPort.ports.http | int | `80` | 'hostPort' http port |
+| controller.hostPort.ports.h2c | int | `81` | 'hostPort' h2c port |
 | controller.hostPort.ports.https | int | `443` | 'hostPort' https port |
 | controller.hostname | object | `{}` | Optionally customize the pod hostname. |
 | controller.image.allowPrivilegeEscalation | bool | `false` |  |
@@ -458,6 +459,7 @@ As of version `1.26.0` of this chart, by simply not providing any clusterIP valu
 | controller.service.nodePorts.tcp | object | `{}` | Node port mapping for external TCP listeners. If left empty, the service controller allocates them from the configured node port range. Example: tcp:   8080: 30080 |
 | controller.service.nodePorts.udp | object | `{}` | Node port mapping for external UDP listeners. If left empty, the service controller allocates them from the configured node port range. Example: udp:   53: 30053 |
 | controller.service.ports.http | int | `80` | Port the external HTTP listener is published with. |
+| controller.service.ports.h2c | int | `81` | Port the external HTTP listener is published with. |
 | controller.service.ports.https | int | `443` | Port the external HTTPS listener is published with. |
 | controller.service.sessionAffinity | string | `""` | Session affinity of the external controller service. Must be either "None" or "ClientIP" if set. Defaults to "None". Ref: https://kubernetes.io/docs/reference/networking/virtual-ips/#session-affinity |
 | controller.service.targetPorts.http | string | `"http"` | Port of the ingress controller the external HTTP listener is mapped to. |
diff --git a/charts/ingress-nginx/README.md.gotmpl b/charts/ingress-nginx/README.md.gotmpl
index 17b029bbf..5cba67c56 100644
--- a/charts/ingress-nginx/README.md.gotmpl
+++ b/charts/ingress-nginx/README.md.gotmpl
@@ -109,6 +109,7 @@ controller:
   service:
     targetPorts:
       http: http
+      h2c: h2c
       https: http
     annotations:
       service.beta.kubernetes.io/aws-load-balancer-ssl-cert: arn:aws:acm:XX-XXXX-X:XXXXXXXXX:certificate/XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXX
diff --git a/charts/ingress-nginx/templates/NOTES.txt b/charts/ingress-nginx/templates/NOTES.txt
index f4923007e..c90b91009 100644
--- a/charts/ingress-nginx/templates/NOTES.txt
+++ b/charts/ingress-nginx/templates/NOTES.txt
@@ -8,6 +8,11 @@ Get the application URL by running these commands:
 {{- else }}
   export HTTP_NODE_PORT=$(kubectl get service --namespace {{ include "ingress-nginx.namespace" . }} {{ include "ingress-nginx.controller.fullname" . }} --output jsonpath="{.spec.ports[0].nodePort}")
 {{- end }}
+{{- if (not (empty .Values.controller.service.nodePorts.h2c)) }}
+  export H2C_NODE_PORT={{ .Values.controller.service.nodePorts.h2c }}
+{{- else }}
+  export H2C_NODE_PORT=$(kubectl get service --namespace {{ include "ingress-nginx.namespace" . }} {{ include "ingress-nginx.controller.fullname" . }} --output jsonpath="{.spec.ports[0].nodePort}")
+{{- end }}
 {{- if (not (empty .Values.controller.service.nodePorts.https)) }}
   export HTTPS_NODE_PORT={{ .Values.controller.service.nodePorts.https }}
 {{- else }}
@@ -16,6 +21,7 @@ Get the application URL by running these commands:
   export NODE_IP="$(kubectl get nodes --output jsonpath="{.items[0].status.addresses[1].address}")"
 
   echo "Visit http://${NODE_IP}:${HTTP_NODE_PORT} to access your application via HTTP."
+  echo "Visit http://${NODE_IP}:${H2C_NODE_PORT} to access your application via h2c."
   echo "Visit https://${NODE_IP}:${HTTPS_NODE_PORT} to access your application via HTTPS."
 {{- else if contains "LoadBalancer" .Values.controller.service.type }}
 It may take a few minutes for the load balancer IP to be available.
diff --git a/charts/ingress-nginx/templates/controller-service-internal.yaml b/charts/ingress-nginx/templates/controller-service-internal.yaml
index 950eb8f05..479ec2d02 100644
--- a/charts/ingress-nginx/templates/controller-service-internal.yaml
+++ b/charts/ingress-nginx/templates/controller-service-internal.yaml
@@ -65,6 +65,18 @@ spec:
       nodePort: {{ .Values.controller.service.internal.nodePorts.http }}
     {{- end }}
   {{- end }}
+  {{- if .Values.controller.service.enableH2c }}
+    - name: h2c
+      port: {{ .Values.controller.service.internal.ports.h2c | default .Values.controller.service.ports.h2c }}
+      protocol: TCP
+      targetPort: {{ .Values.controller.service.internal.targetPorts.h2c | default .Values.controller.service.targetPorts.h2c }}
+    {{- if and (semverCompare ">=1.20" .Capabilities.KubeVersion.Version) (.Values.controller.service.internal.appProtocol) }}
+      appProtocol: kubernetes.io/h2c
+    {{- end }}
+    {{- if (and $setNodePorts (not (empty .Values.controller.service.internal.nodePorts.h2c))) }}
+      nodePort: {{ .Values.controller.service.internal.nodePorts.h2c }}
+    {{- end }}
+  {{- end }}
   {{- if .Values.controller.service.enableHttps }}
     - name: https
       port: {{ .Values.controller.service.internal.ports.https | default .Values.controller.service.ports.https }}
diff --git a/charts/ingress-nginx/templates/controller-service.yaml b/charts/ingress-nginx/templates/controller-service.yaml
index 74f608536..69e0bd752 100644
--- a/charts/ingress-nginx/templates/controller-service.yaml
+++ b/charts/ingress-nginx/templates/controller-service.yaml
@@ -65,6 +65,18 @@ spec:
       nodePort: {{ .Values.controller.service.nodePorts.http }}
     {{- end }}
   {{- end }}
+  {{- if .Values.controller.service.enableH2c }}
+    - name: h2c
+      port: {{ .Values.controller.service.ports.h2c }}
+      protocol: TCP
+      targetPort: {{ .Values.controller.service.targetPorts.h2c }}
+    {{- if and (semverCompare ">=1.20" .Capabilities.KubeVersion.Version) (.Values.controller.service.appProtocol) }}
+      appProtocol: kubernetes.io/h2c
+    {{- end }}
+    {{- if (and $setNodePorts (not (empty .Values.controller.service.nodePorts.h2c))) }}
+      nodePort: {{ .Values.controller.service.nodePorts.h2c }}
+    {{- end }}
+  {{- end }}
   {{- if .Values.controller.service.enableHttps }}
     - name: https
       port: {{ .Values.controller.service.ports.https }}
diff --git a/charts/ingress-nginx/values.yaml b/charts/ingress-nginx/values.yaml
index e010a2dbb..d0cc3010b 100644
--- a/charts/ingress-nginx/values.yaml
+++ b/charts/ingress-nginx/values.yaml
@@ -44,6 +44,7 @@ controller:
   # -- Configures the ports that the nginx-controller listens on
   containerPort:
     http: 80
+    h2c: 81
     https: 443
   # -- Will add custom configuration options to Nginx https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/
   config: {}
@@ -100,6 +101,8 @@ controller:
     ports:
       # -- 'hostPort' http port
       http: 80
+      # -- 'h2cPort' h2c port
+      h2c: 81
       # -- 'hostPort' https port
       https: 443
   # NetworkPolicy for controller component.
@@ -495,11 +498,15 @@ controller:
       - IPv4
     # -- Enable the HTTP listener on both controller services or not.
     enableHttp: true
+    # -- Enable the H2C listener on both controller services or not.
+    enableH2c: true
     # -- Enable the HTTPS listener on both controller services or not.
     enableHttps: true
     ports:
       # -- Port the external HTTP listener is published with.
       http: 80
+      # -- Port the external H2C listener is published with.
+      h2c: 81
       # -- Port the external HTTPS listener is published with.
       https: 443
     targetPorts:
@@ -577,6 +584,9 @@ controller:
       # -- Port the internal HTTP listener is published with.
       # Defaults to the value of `controller.service.ports.http`.
       # http: 80
+      # -- Port the internal H2C listener is published with.
+      # Defaults to the value of `controller.service.ports.h2c`.
+      # h2c: 81
       # -- Port the internal HTTPS listener is published with.
       # Defaults to the value of `controller.service.ports.https`.
       # https: 443
diff --git a/images/nginx-1.25/rootfs/Dockerfile b/images/nginx-1.25/rootfs/Dockerfile
index 8996547e6..cd2a9af9c 100644
--- a/images/nginx-1.25/rootfs/Dockerfile
+++ b/images/nginx-1.25/rootfs/Dockerfile
@@ -66,6 +66,6 @@ RUN apk update \
   chown -R www-data.www-data ${dir}; \
   done'
 
-EXPOSE 80 443
+EXPOSE 80 81 443
 
 CMD ["nginx", "-g", "daemon off;"]
diff --git a/images/nginx/rootfs/Dockerfile b/images/nginx/rootfs/Dockerfile
index daf44f52a..8b332ec7b 100644
--- a/images/nginx/rootfs/Dockerfile
+++ b/images/nginx/rootfs/Dockerfile
@@ -68,6 +68,6 @@ RUN apk update \
   chown -R www-data.www-data ${dir}; \
   done'
 
-EXPOSE 80 443
+EXPOSE 80 81 443
 
 CMD ["nginx", "-g", "daemon off;"]
diff --git a/internal/ingress/controller/config/config.go b/internal/ingress/controller/config/config.go
index bad82b8b0..b86582c3d 100644
--- a/internal/ingress/controller/config/config.go
+++ b/internal/ingress/controller/config/config.go
@@ -954,6 +954,7 @@ type TemplateConfig struct {
 // NGINX Ingress controller
 type ListenPorts struct {
 	HTTP     int `json:"HTTP"`
+	H2C      int `json:"H2C"`
 	HTTPS    int `json:"HTTPS"`
 	Health   int `json:"Health"`
 	Default  int `json:"Default"`
diff --git a/internal/ingress/controller/controller.go b/internal/ingress/controller/controller.go
index cb8d3712c..5239be1de 100644
--- a/internal/ingress/controller/controller.go
+++ b/internal/ingress/controller/controller.go
@@ -458,6 +458,7 @@ func (n *NGINXController) getStreamServices(configmapName string, proto apiv1.Pr
 
 	rp := []int{
 		n.cfg.ListenPorts.HTTP,
+		n.cfg.ListenPorts.H2C,
 		n.cfg.ListenPorts.HTTPS,
 		n.cfg.ListenPorts.SSLProxy,
 		n.cfg.ListenPorts.Health,
diff --git a/internal/ingress/controller/template/template.go b/internal/ingress/controller/template/template.go
index 7410ce6e0..70f020e34 100644
--- a/internal/ingress/controller/template/template.go
+++ b/internal/ingress/controller/template/template.go
@@ -275,6 +275,7 @@ var funcMap = text_template.FuncMap{
 	"buildCustomErrorLocationsPerServer": buildCustomErrorLocationsPerServer,
 	"shouldLoadModSecurityModule":        shouldLoadModSecurityModule,
 	"buildHTTPListener":                  buildHTTPListener,
+	"buildH2CListener":                   buildH2CListener,
 	"buildHTTPSListener":                 buildHTTPSListener,
 	"buildOpentelemetryForLocation":      buildOpentelemetryForLocation,
 	"shouldLoadOpentelemetryModule":      shouldLoadOpentelemetryModule,
@@ -1398,6 +1399,44 @@ func buildHTTPListener(t, s interface{}) string {
 	return strings.Join(out, "\n")
 }
 
+func buildH2CListener(t, s interface{}) string {
+	var out []string
+
+	tc, ok := t.(config.TemplateConfig)
+	if !ok {
+		klog.Errorf("expected a 'config.TemplateConfig' type but %T was returned", t)
+		return ""
+	}
+
+	hostname, ok := s.(string)
+	if !ok {
+		klog.Errorf("expected a 'string' type but %T was returned", s)
+		return ""
+	}
+
+	addrV4 := []string{""}
+	if len(tc.Cfg.BindAddressIpv4) > 0 {
+		addrV4 = tc.Cfg.BindAddressIpv4
+	}
+
+	co := commonListenOptions(&tc, hostname)
+
+	out = append(out, h2cListener(addrV4, co, &tc)...)
+
+	if !tc.IsIPV6Enabled {
+		return strings.Join(out, "\n")
+	}
+
+	addrV6 := []string{"[::]"}
+	if len(tc.Cfg.BindAddressIpv6) > 0 {
+		addrV6 = tc.Cfg.BindAddressIpv6
+	}
+
+	out = append(out, h2cListener(addrV6, co, &tc)...)
+
+	return strings.Join(out, "\n")
+}
+
 func buildHTTPSListener(t, s interface{}) string {
 	var out []string
 
@@ -1478,6 +1517,24 @@ func httpListener(addresses []string, co string, tc *config.TemplateConfig) []st
 	return out
 }
 
+func h2cListener(addresses []string, co string, tc *config.TemplateConfig) []string {
+	out := make([]string, 0)
+	for _, address := range addresses {
+		lo := []string{"listen"}
+
+		if address == "" {
+			lo = append(lo, fmt.Sprintf("%v", tc.ListenPorts.H2C))
+		} else {
+			lo = append(lo, fmt.Sprintf("%v:%v", address, tc.ListenPorts.H2C))
+		}
+
+		lo = append(lo, co, " http2;")
+		out = append(out, strings.Join(lo, " "))
+	}
+
+	return out
+}
+
 func httpsListener(addresses []string, co string, tc *config.TemplateConfig) []string {
 	out := make([]string, 0)
 	for _, address := range addresses {
diff --git a/pkg/flags/flags.go b/pkg/flags/flags.go
index d3bc4ee86..f2ea13bf6 100644
--- a/pkg/flags/flags.go
+++ b/pkg/flags/flags.go
@@ -181,6 +181,7 @@ Requires the update-status parameter.`)
 		monitorMaxBatchSize  = flags.Int("monitor-max-batch-size", 10000, "Max batch size of NGINX metrics.")
 
 		httpPort  = flags.Int("http-port", 80, `Port to use for servicing HTTP traffic.`)
+		h2cPort   = flags.Int("h2c-port", 81, `Port to use for servicing H2c traffic.`)
 		httpsPort = flags.Int("https-port", 443, `Port to use for servicing HTTPS traffic.`)
 
 		sslProxyPort  = flags.Int("ssl-passthrough-proxy-port", 442, `Port to use internally for SSL Passthrough.`)
@@ -259,6 +260,10 @@ https://blog.maxmind.com/2019/12/18/significant-changes-to-accessing-and-using-g
 		return false, nil, fmt.Errorf("port %v is already in use. Please check the flag --http-port", *httpPort)
 	}
 
+	if !ing_net.IsPortAvailable(*h2cPort) {
+		return false, nil, fmt.Errorf("port %v is already in use. Please check the flag --h2c-port", *h2cPort)
+	}
+
 	if !ing_net.IsPortAvailable(*httpsPort) {
 		return false, nil, fmt.Errorf("port %v is already in use. Please check the flag --https-port", *httpsPort)
 	}
@@ -357,6 +362,7 @@ https://blog.maxmind.com/2019/12/18/significant-changes-to-accessing-and-using-g
 			Default:  *defServerPort,
 			Health:   *healthzPort,
 			HTTP:     *httpPort,
+			H2C:      *h2cPort,
 			HTTPS:    *httpsPort,
 			SSLProxy: *sslProxyPort,
 		},
diff --git a/rootfs/Dockerfile-chroot b/rootfs/Dockerfile-chroot
index a210aa7bf..2dfafb9cc 100644
--- a/rootfs/Dockerfile-chroot
+++ b/rootfs/Dockerfile-chroot
@@ -118,7 +118,7 @@ RUN mkdir -p /chroot/modules_mount \
 
 USER www-data
 
-EXPOSE 80 443
+EXPOSE 80 81 443
 
 ENTRYPOINT ["/usr/bin/dumb-init", "--"]
 
diff --git a/rootfs/etc/nginx/template/nginx.tmpl b/rootfs/etc/nginx/template/nginx.tmpl
index 94dc12412..548b120a4 100644
--- a/rootfs/etc/nginx/template/nginx.tmpl
+++ b/rootfs/etc/nginx/template/nginx.tmpl
@@ -587,6 +587,7 @@ http {
         server_name {{ $redirect.From }};
 
         {{ buildHTTPListener  $all $redirect.From }}
+        {{ buildH2CListener  $all $redirect.From }}
         {{ buildHTTPSListener $all $redirect.From }}
 
         ssl_certificate_by_lua_block {
@@ -975,6 +976,7 @@ stream {
         {{ $server := .Second }}
 
         {{ buildHTTPListener  $all $server.Hostname }}
+        {{ buildH2CListener  $all $server.Hostname }}
         {{ buildHTTPSListener $all $server.Hostname }}
 
         set $proxy_upstream_name "-";