Add sslpassthrough tests (#9457)

build/build.sh

@@ -16,10 +16,10 @@
 
 GO_BUILD_CMD="go build"
 
-if [ -n "$DEBUG" ]; then
+#if [ -n "$DEBUG" ]; then
-	set -x
+#	set -x
-	GO_BUILD_CMD="go build -v"
+#	GO_BUILD_CMD="go build -v"
-fi
+#fi
 
 set -o errexit
 set -o nounset

build/run-in-docker.sh

@@ -65,23 +65,21 @@ fi
 
 USER=${USER:-nobody}
 
-echo "..printing env & other vars to stdout"
+#echo "..printing env & other vars to stdout"
-echo "HOSTNAME=`hostname`"
+#echo "HOSTNAME=`hostname`"
-uname -a
+#uname -a
-env
+#env
-echo "DIND_ENABLED=$DOCKER_IN_DOCKER_ENABLED"
+#echo "DIND_ENABLED=$DOCKER_IN_DOCKER_ENABLED"
-echo "done..printing env & other vars to stdout"
+#echo "done..printing env & other vars to stdout"
 
 if [[ "$DOCKER_IN_DOCKER_ENABLED" == "true" ]]; then
   echo "..reached DIND check TRUE block, inside run-in-docker.sh"
   echo "FLAGS=$FLAGS"
-  go env
+  #go env
-  set -x
   go install -mod=mod github.com/onsi/ginkgo/v2/ginkgo@v2.6.1
   find / -type f -name ginkgo 2>/dev/null
   which ginkgo
   /bin/bash -c "${FLAGS}"
-  set +x
 else
   echo "Reached DIND check ELSE block, inside run-in-docker.sh"
   docker run                                            \

pkg/tcpproxy/tcp.go

@@ -80,6 +80,7 @@ func (p *TCPProxy) Handle(conn net.Conn) {
 	}
 
 	hostPort := net.JoinHostPort(proxy.IP, fmt.Sprintf("%v", proxy.Port))
+	klog.V(4).InfoS("passing to", "hostport", hostPort)
 	clientConn, err := net.Dial("tcp", hostPort)
 	if err != nil {
 		klog.V(4).ErrorS(err, "error dialing proxy", "ip", proxy.IP, "port", proxy.Port, "hostname", proxy.Hostname)
@@ -99,7 +100,7 @@ func (p *TCPProxy) Handle(conn net.Conn) {
 		}
 		proxyProtocolHeader := fmt.Sprintf("PROXY %s %s %s %d %d\r\n", protocol, remoteAddr.IP.String(), localAddr.IP.String(), remoteAddr.Port, localAddr.Port)
 		klog.V(4).InfoS("Writing Proxy Protocol", "header", proxyProtocolHeader)
-		_, err = fmt.Fprintf(clientConn, proxyProtocolHeader)
+		_, err = fmt.Fprint(clientConn, proxyProtocolHeader)
 	}
 	if err != nil {
 		klog.ErrorS(err, "Error writing Proxy Protocol header")
@@ -126,8 +127,5 @@ func pipe(client, server net.Conn) {
 	go doCopy(server, client, cancel)
 	go doCopy(client, server, cancel)
 
-	select {
+	<-cancel
-	case <-cancel:
-		return
-	}
 }

test/e2e/framework/deployment.go

@@ -85,9 +85,10 @@ func (f *Framework) NewEchoDeployment(opts ...func(*deploymentOptions)) {
 	}
 
 	deployment := newDeployment(options.name, options.namespace, "registry.k8s.io/ingress-nginx/e2e-test-echo@sha256:778ac6d1188c8de8ecabeddd3c37b72c8adc8c712bad2bd7a81fb23a3514934c", 80, int32(options.replicas),
-		nil,
+		nil, nil, nil,
 		[]corev1.VolumeMount{},
 		[]corev1.Volume{},
+		true,
 	)
 
 	f.EnsureDeployment(deployment)
@@ -183,7 +184,7 @@ func (f *Framework) NGINXDeployment(name string, cfg string, waitendpoint bool)
 	assert.Nil(ginkgo.GinkgoT(), err, "creating configmap")
 
 	deployment := newDeployment(name, f.Namespace, f.GetNginxBaseImage(), 80, 1,
-		nil,
+		nil, nil, nil,
 		[]corev1.VolumeMount{
 			{
 				Name:      name,
@@ -203,7 +204,7 @@ func (f *Framework) NGINXDeployment(name string, cfg string, waitendpoint bool)
 					},
 				},
 			},
-		},
+		}, true,
 	)
 
 	f.EnsureDeployment(deployment)
@@ -334,8 +335,8 @@ func (f *Framework) NewGRPCBinDeployment() {
 	assert.Nil(ginkgo.GinkgoT(), err, "waiting for endpoints to become ready")
 }
 
-func newDeployment(name, namespace, image string, port int32, replicas int32, command []string,
+func newDeployment(name, namespace, image string, port int32, replicas int32, command []string, args []string, env []corev1.EnvVar,
-	volumeMounts []corev1.VolumeMount, volumes []corev1.Volume) *appsv1.Deployment {
+	volumeMounts []corev1.VolumeMount, volumes []corev1.Volume, setProbe bool) *appsv1.Deployment {
 	probe := &corev1.Probe{
 		InitialDelaySeconds: 2,
 		PeriodSeconds:       1,
@@ -381,8 +382,6 @@ func newDeployment(name, namespace, image string, port int32, replicas int32, co
 									ContainerPort: port,
 								},
 							},
-							ReadinessProbe: probe,
-							LivenessProbe:  probe,
 							VolumeMounts: volumeMounts,
 						},
 					},
@@ -392,10 +391,20 @@ func newDeployment(name, namespace, image string, port int32, replicas int32, co
 		},
 	}
 
+	if setProbe {
+		d.Spec.Template.Spec.Containers[0].ReadinessProbe = probe
+		d.Spec.Template.Spec.Containers[0].LivenessProbe = probe
+	}
 	if len(command) > 0 {
 		d.Spec.Template.Spec.Containers[0].Command = command
 	}
 
+	if len(args) > 0 {
+		d.Spec.Template.Spec.Containers[0].Args = args
+	}
+	if len(env) > 0 {
+		d.Spec.Template.Spec.Containers[0].Env = env
+	}
 	return d
 }
 
@@ -404,9 +413,13 @@ func (f *Framework) NewHttpbinDeployment() {
 	f.NewDeployment(HTTPBinService, "registry.k8s.io/ingress-nginx/e2e-test-httpbin@sha256:c6372ef57a775b95f18e19d4c735a9819f2e7bb4641e5e3f27287d831dfeb7e8", 80, 1)
 }
 
-// NewDeployment creates a new deployment in a particular namespace.
 func (f *Framework) NewDeployment(name, image string, port int32, replicas int32) {
-	deployment := newDeployment(name, f.Namespace, image, port, replicas, nil, nil, nil)
+	f.NewDeploymentWithOpts(name, image, port, replicas, nil, nil, nil, nil, nil, true)
+}
+
+// NewDeployment creates a new deployment in a particular namespace.
+func (f *Framework) NewDeploymentWithOpts(name, image string, port int32, replicas int32, command []string, args []string, env []corev1.EnvVar, volumeMounts []corev1.VolumeMount, volumes []corev1.Volume, setProbe bool) {
+	deployment := newDeployment(name, f.Namespace, image, port, replicas, command, args, env, volumeMounts, volumes, setProbe)
 
 	f.EnsureDeployment(deployment)
 

test/e2e/framework/httpexpect/request.go

@@ -17,8 +17,10 @@ limitations under the License.
 package httpexpect
 
 import (
+	"context"
 	"fmt"
 	"io"
+	"net"
 	"net/http"
 	"net/url"
 	"path"
@@ -71,6 +73,33 @@ func (h *HTTPRequest) DoRequest(method, rpath string) *HTTPRequest {
 	return h
 }
 
+// ForceResolve forces the test resolver to point to a specific endpoint
+func (h *HTTPRequest) ForceResolve(ip string, port uint16) *HTTPRequest {
+	addr := net.ParseIP(ip)
+	if addr == nil {
+		h.chain.fail(fmt.Sprintf("invalid ip address: %s", ip))
+		return h
+	}
+	dialer := &net.Dialer{
+		Timeout:   h.client.Timeout,
+		KeepAlive: h.client.Timeout,
+		DualStack: true,
+	}
+	resolveAddr := fmt.Sprintf("%s:%d", ip, int(port))
+
+	oldTransport, ok := h.client.Transport.(*http.Transport)
+	if !ok {
+		h.chain.fail("invalid old transport address")
+		return h
+	}
+	newTransport := oldTransport.Clone()
+	newTransport.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) {
+		return dialer.DialContext(ctx, network, resolveAddr)
+	}
+	h.client.Transport = newTransport
+	return h
+}
+
 // Expect executes the request and returns an HTTP response.
 func (h *HTTPRequest) Expect() *HTTPResponse {
 	if h.query != nil {

test/e2e/settings/ssl_passthrough.go

@@ -0,0 +1,165 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package settings
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"net/http"
+	"strings"
+
+	"github.com/onsi/ginkgo/v2"
+	"github.com/stretchr/testify/assert"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"k8s.io/ingress-nginx/test/e2e/framework"
+)
+
+var _ = framework.IngressNginxDescribe("[Flag] enable-ssl-passthrough", func() {
+	f := framework.NewDefaultFramework("ssl-passthrough")
+
+	ginkgo.BeforeEach(func() {
+		err := f.UpdateIngressControllerDeployment(func(deployment *appsv1.Deployment) error {
+			args := deployment.Spec.Template.Spec.Containers[0].Args
+			args = append(args, "--enable-ssl-passthrough")
+			deployment.Spec.Template.Spec.Containers[0].Args = args
+			_, err := f.KubeClientSet.AppsV1().Deployments(f.Namespace).Update(context.TODO(), deployment, metav1.UpdateOptions{})
+			return err
+		})
+		assert.Nil(ginkgo.GinkgoT(), err, "updating ingress controller deployment flags")
+
+		f.WaitForNginxServer("_",
+			func(server string) bool {
+				return strings.Contains(server, "listen 442")
+			})
+	})
+
+	ginkgo.Describe("With enable-ssl-passthrough enabled", func() {
+		ginkgo.It("should enable ssl-passthrough-proxy-port on a different port", func() {
+
+			err := f.UpdateIngressControllerDeployment(func(deployment *appsv1.Deployment) error {
+				args := deployment.Spec.Template.Spec.Containers[0].Args
+				args = append(args, "--ssl-passthrough-proxy-port=1442")
+				deployment.Spec.Template.Spec.Containers[0].Args = args
+				_, err := f.KubeClientSet.AppsV1().Deployments(f.Namespace).Update(context.TODO(), deployment, metav1.UpdateOptions{})
+				return err
+			})
+			assert.Nil(ginkgo.GinkgoT(), err, "updating ingress controller deployment flags")
+
+			f.WaitForNginxServer("_",
+				func(server string) bool {
+					return strings.Contains(server, "listen 1442")
+				})
+
+			f.HTTPTestClient().
+				GET("/").
+				WithHeader("Host", "something").
+				Expect().
+				Status(http.StatusNotFound)
+		})
+
+		ginkgo.It("should pass unknown traffic to default backend and handle known traffic", func() {
+
+			host := "testpassthrough.com"
+			echoName := "echopass"
+
+			/* Even with enable-ssl-passthrough enabled, only annotated ingresses may receive the trafic */
+			annotations := map[string]string{
+				"nginx.ingress.kubernetes.io/ssl-passthrough": "true",
+			}
+
+			ingressDef := framework.NewSingleIngressWithTLS(host, "/", host, []string{host}, f.Namespace, echoName, 80, annotations)
+			tlsConfig, err := framework.CreateIngressTLSSecret(f.KubeClientSet,
+				ingressDef.Spec.TLS[0].Hosts,
+				ingressDef.Spec.TLS[0].SecretName,
+				ingressDef.Namespace)
+
+			volumeMount := []corev1.VolumeMount{
+				{
+					Name:      "certs",
+					ReadOnly:  true,
+					MountPath: "/certs",
+				},
+			}
+			volume := []corev1.Volume{
+				{
+					Name: "certs",
+					VolumeSource: corev1.VolumeSource{
+						Secret: &corev1.SecretVolumeSource{
+							SecretName: ingressDef.Spec.TLS[0].SecretName,
+						},
+					},
+				},
+			}
+			envs := []corev1.EnvVar{
+				{
+					Name:  "HTTPBUN_SSL_CERT",
+					Value: "/certs/tls.crt",
+				},
+				{
+					Name:  "HTTPBUN_SSL_KEY",
+					Value: "/certs/tls.key",
+				},
+			}
+			f.NewDeploymentWithOpts("echopass", "ghcr.io/sharat87/httpbun:latest", 80, 1, nil, nil, envs, volumeMount, volume, false)
+
+			f.EnsureIngress(ingressDef)
+
+			assert.Nil(ginkgo.GinkgoT(), err)
+			framework.WaitForTLS(f.GetURL(framework.HTTPS), tlsConfig)
+
+			f.WaitForNginxServer(host,
+				func(server string) bool {
+					return strings.Contains(server, "listen 442")
+				})
+
+			/* This one should not receive traffic as it does not contain passthrough annotation */
+			hostBad := "noannotationnopassthrough.com"
+			ingBad := f.EnsureIngress(framework.NewSingleIngressWithTLS(hostBad, "/", hostBad, []string{hostBad}, f.Namespace, echoName, 80, nil))
+			tlsConfigBad, err := framework.CreateIngressTLSSecret(f.KubeClientSet,
+				ingBad.Spec.TLS[0].Hosts,
+				ingBad.Spec.TLS[0].SecretName,
+				ingBad.Namespace)
+			assert.Nil(ginkgo.GinkgoT(), err)
+			framework.WaitForTLS(f.GetURL(framework.HTTPS), tlsConfigBad)
+
+			f.WaitForNginxServer(hostBad,
+				func(server string) bool {
+					return strings.Contains(server, "listen 442")
+				})
+
+			f.HTTPTestClientWithTLSConfig(&tls.Config{ServerName: host, InsecureSkipVerify: true}).
+				GET("/").
+				WithURL(fmt.Sprintf("https://%s:443", host)).
+				ForceResolve(f.GetNginxIP(), 443).
+				Expect().
+				Status(http.StatusOK)
+
+			f.HTTPTestClientWithTLSConfig(&tls.Config{ServerName: hostBad, InsecureSkipVerify: true}).
+				GET("/").
+				WithURL(fmt.Sprintf("https://%s:443", hostBad)).
+				ForceResolve(f.GetNginxIP(), 443).
+				Expect().
+				Status(http.StatusNotFound)
+
+		})
+	})
+})