Move X-Forwarded-Port variable to the location context

Resolves issue #4038 where the X-Forwarded-Port header would be set to the value of the https listening port if all of the following settings were satisfied:
- The ingress controller was started with a non-default HTTPS port set with the `--https-port` argument
- An ingress is created having:
  - the `nginx.ingress.kubernetes.io/auth-url` annotation set
  - TLS enabled

This commit solves this issue by moving the setting of the `pass_server_port` variable from the server, one level down to the location context.

rootfs/etc/nginx/template/nginx.tmpl

@@ -817,10 +817,6 @@ stream {
         {{ end }}
         {{ end }}
         set $proxy_upstream_name "-";
-        set $pass_access_scheme $scheme;
-        set $pass_server_port $server_port;
-        set $best_http_host $http_host;
-        set $pass_port $pass_server_port;
 
         {{/* Listen on {{ $all.ListenPorts.SSLProxy }} because port {{ $all.ListenPorts.HTTPS }} is used in the TLS sni server */}}
         {{/* This listener must always have proxy_protocol enabled, because the SNI listener forwards on source IP info in it. */}}
@@ -1109,6 +1105,10 @@ stream {
             set $balancer_ewma_score -1;
             set $proxy_upstream_name    "{{ buildUpstreamName $location }}";
             set $proxy_host             $proxy_upstream_name;
+            set $pass_access_scheme $scheme;
+            set $pass_server_port $server_port;
+            set $best_http_host $http_host;
+            set $pass_port $pass_server_port;
 
             set $proxy_alternative_upstream_name "";
 

test/e2e-image/Dockerfile

@@ -18,6 +18,7 @@ COPY e2e.sh             /e2e.sh
 COPY cloud-generic      /cloud-generic
 COPY cluster-wide       /cluster-wide
 COPY overlay            /overlay
+COPY namespace-overlays /namespace-overlays
 RUN sed -E -i 's|^- .*deploy/cloud-generic$|- ../cloud-generic|' /overlay/kustomization.yaml
 COPY wait-for-nginx.sh  /
 COPY e2e.test           /

test/e2e-image/namespace-overlays/forwarded-port-headers/deployment-patch.yaml

@@ -0,0 +1,12 @@
+- op: replace
+  path: /spec/template/spec/containers/0/ports/0/containerPort
+  value: 1080
+- op: replace
+  path: /spec/template/spec/containers/0/ports/1/containerPort
+  value: 1443
+- op: add
+  path: /spec/template/spec/containers/0/args/-
+  value: --http-port=1080
+- op: add
+  path: /spec/template/spec/containers/0/args/-
+  value: --https-port=1443

test/e2e-image/namespace-overlays/forwarded-port-headers/kustomization.yaml

@@ -0,0 +1,16 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+patchesJson6902:
+  - target:
+      group: apps
+      version: v1
+      kind: Deployment
+      name: nginx-ingress-controller
+    path: deployment-patch.yaml
+  - target:
+      version: v1
+      kind: Service
+      name: ingress-nginx
+    path: service-patch.yaml
+bases:
+- ../../overlay

test/e2e-image/namespace-overlays/forwarded-port-headers/service-patch.yaml

@@ -0,0 +1,6 @@
+- op: replace
+  path: /spec/ports/0/targetPort
+  value: 1080
+- op: replace
+  path: /spec/ports/1/targetPort
+  value: 1443

test/e2e/framework/exec.go

@@ -88,9 +88,9 @@ func (f *Framework) ExecCommand(pod *corev1.Pod, command string) (string, error)
 }
 
 // NewIngressController deploys a new NGINX Ingress controller in a namespace
-func (f *Framework) NewIngressController(namespace string) error {
+func (f *Framework) NewIngressController(namespace string, namespaceOverlay string) error {
 	// Creates an nginx deployment
-	cmd := exec.Command("./wait-for-nginx.sh", namespace)
+	cmd := exec.Command("./wait-for-nginx.sh", namespace, namespaceOverlay)
 	out, err := cmd.CombinedOutput()
 	if err != nil {
 		return fmt.Errorf("Unexpected error waiting for ingress controller deployment: %v.\nLogs:\n%v", err, string(out))

test/e2e/framework/framework.go

@@ -103,7 +103,7 @@ func (f *Framework) BeforeEach() {
 	f.Namespace = ingressNamespace
 
 	By("Starting new ingress controller")
-	err = f.NewIngressController(f.Namespace)
+	err = f.NewIngressController(f.Namespace, f.BaseName)
 	Expect(err).NotTo(HaveOccurred())
 
 	err = WaitForPodsReady(f.KubeClientSet, DefaultTimeout, 1, f.Namespace, metav1.ListOptions{

test/e2e/settings/listen_nondefault_ports.go

@@ -0,0 +1,150 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package settings
+
+import (
+	"fmt"
+	"net/http"
+	"strings"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/parnurzeal/gorequest"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"k8s.io/ingress-nginx/test/e2e/framework"
+)
+
+var _ = framework.IngressNginxDescribe("Listen on nondefault ports", func() {
+
+	host := "forwarded-headers"
+
+	f := framework.NewDefaultFramework("forwarded-port-headers")
+
+	BeforeEach(func() {
+		f.NewEchoDeployment()
+
+		f.WaitForNginxServer("_",
+			func(server string) bool {
+				return strings.Contains(server, "listen 1443")
+			})
+	})
+
+	Context("with a plain HTTP ingress", func() {
+		It("should set X-Forwarded-Port headers accordingly when listening on a non-default HTTP port", func() {
+
+			ing := framework.NewSingleIngress(host, "/", host, f.Namespace, "http-svc", 80, nil)
+			f.EnsureIngress(ing)
+
+			f.WaitForNginxServer(host,
+				func(server string) bool {
+					return strings.Contains(server, "server_name forwarded-headers")
+				})
+
+			resp, body, errs := gorequest.New().
+				Get(f.GetURL(framework.HTTP)).
+				Set("Host", host).
+				End()
+
+			Expect(errs).Should(BeEmpty())
+			Expect(resp.StatusCode).Should(Equal(http.StatusOK))
+			Expect(body).Should(ContainSubstring(fmt.Sprintf("x-forwarded-port=%d", 1080)))
+		})
+	})
+
+	Context("with a TLS enabled ingress", func() {
+
+		It("should set X-Forwarded-Port header to 443", func() {
+
+			ing := framework.NewSingleIngressWithTLS(host, "/", host, []string{host}, f.Namespace, "http-svc", 80, nil)
+			f.EnsureIngress(ing)
+
+			tlsConfig, err := framework.CreateIngressTLSSecret(f.KubeClientSet,
+				ing.Spec.TLS[0].Hosts,
+				ing.Spec.TLS[0].SecretName,
+				ing.Namespace)
+			Expect(err).NotTo(HaveOccurred())
+
+			framework.WaitForTLS(f.GetURL(framework.HTTPS), tlsConfig)
+
+			f.WaitForNginxServer(host,
+				func(server string) bool {
+					return strings.Contains(server, "server_name forwarded-headers")
+				})
+
+			resp, body, errs := gorequest.New().
+				Get(f.GetURL(framework.HTTPS)).
+				TLSClientConfig(tlsConfig).
+				Set("Host", host).
+				End()
+			Expect(errs).Should(BeEmpty())
+			Expect(resp.StatusCode).Should(Equal(http.StatusOK))
+			Expect(body).Should(ContainSubstring(fmt.Sprintf("x-forwarded-port=443")))
+		})
+
+		Context("when external authentication is configured", func() {
+
+			It("should set the X-Forwarded-Port header to 443", func() {
+
+				f.NewHttpbinDeployment()
+
+				var httpbinIP string
+
+				err := framework.WaitForEndpoints(f.KubeClientSet, framework.DefaultTimeout, "httpbin", f.Namespace, 1)
+				Expect(err).NotTo(HaveOccurred())
+
+				e, err := f.KubeClientSet.CoreV1().Endpoints(f.Namespace).Get("httpbin", metav1.GetOptions{})
+				Expect(err).NotTo(HaveOccurred())
+
+				httpbinIP = e.Subsets[0].Addresses[0].IP
+
+				annotations := map[string]string{
+					"nginx.ingress.kubernetes.io/auth-url":    fmt.Sprintf("http://%s/basic-auth/user/password", httpbinIP),
+					"nginx.ingress.kubernetes.io/auth-signin": "http://$host/auth/start",
+				}
+
+				ing := framework.NewSingleIngressWithTLS(host, "/", host, []string{host}, f.Namespace, "http-svc", 80, &annotations)
+
+				f.EnsureIngress(ing)
+				tlsConfig, err := framework.CreateIngressTLSSecret(f.KubeClientSet,
+					ing.Spec.TLS[0].Hosts,
+					ing.Spec.TLS[0].SecretName,
+					ing.Namespace)
+				Expect(err).NotTo(HaveOccurred())
+				framework.WaitForTLS(f.GetURL(framework.HTTPS), tlsConfig)
+				f.WaitForNginxServer(host,
+					func(server string) bool {
+						return strings.Contains(server, "server_name forwarded-headers")
+					})
+				resp, body, errs := gorequest.New().
+					Get(f.GetURL(framework.HTTPS)).
+					TLSClientConfig(tlsConfig).
+					Set("Host", host).
+					SetBasicAuth("user", "password").
+					End()
+				Expect(errs).Should(BeEmpty())
+				Expect(resp.StatusCode).Should(Equal(http.StatusOK))
+				Expect(body).Should(ContainSubstring(fmt.Sprintf("x-forwarded-port=443")))
+
+			})
+
+		})
+
+	})
+
+})

test/e2e/wait-for-nginx.sh

@@ -22,6 +22,7 @@ fi
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
 export NAMESPACE=$1
+export NAMESPACE_OVERLAY=$2
 
 echo "deploying NGINX Ingress controller in namespace $NAMESPACE"
 
@@ -60,6 +61,21 @@ bases:
 - ../cluster-wide-$NAMESPACE
 EOF
 
+# Use the namespace overlay if it was requested
+if [[ ! -z "$NAMESPACE_OVERLAY" && -d "$DIR/namespace-overlays/$NAMESPACE_OVERLAY" ]]; then
+    echo "Namespace overlay $NAMESPACE_OVERLAY is being used for namespace $NAMESPACE"
+    OVERLAY="$DIR/namespace-overlays/$NAMESPACE"
+    mkdir "$OVERLAY"
+    cat << EOF > "$OVERLAY/kustomization.yaml"
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: $NAMESPACE
+bases:
+- ../../namespace-overlays/$NAMESPACE_OVERLAY
+- ../../cluster-wide-$NAMESPACE
+EOF
+fi
+
 kubectl apply --kustomize "$OVERLAY"
 
 # wait for the deployment and fail if there is an error before starting the execution of any test