Add e2e Tests for AuthTLS

Adds E2E tests for the following annotations:

- auth-tls-secret
- auth-tls-verify-depth
- auth-tls-verify-client
- auth-tls-error-page
- auth-tls-pass-certificate-to-upstream

internal/ingress/resolver/main.go

@@ -18,7 +18,6 @@ package resolver
 
 import (
 	apiv1 "k8s.io/api/core/v1"
-
 	"k8s.io/ingress-nginx/internal/ingress/defaults"
 )
 
@@ -27,15 +26,18 @@ type Resolver interface {
 	// GetDefaultBackend returns the backend that must be used as default
 	GetDefaultBackend() defaults.Backend
 
-	// GetSecret searches for secrets contenating the namespace and name using a the character /
+	// GetSecret searches for secrets containing the namespace and name using a the character /
 	GetSecret(string) (*apiv1.Secret, error)
 
 	// GetAuthCertificate resolves a given secret name into an SSL certificate.
 	// The secret must contain 3 keys named:
+
 	//   ca.crt: contains the certificate chain used for authentication
+	//   tls.crt: contains the server certificate
+	//   tls.key: contains the server key
 	GetAuthCertificate(string) (*AuthSSLCert, error)
 
-	// GetService searches for services contenating the namespace and name using a the character /
+	// GetService searches for services containing the namespace and name using a the character /
 	GetService(string) (*apiv1.Service, error)
 }
 

test/e2e/annotations/authtls.go

@@ -0,0 +1,208 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package annotations
+
+import (
+	"crypto/tls"
+	"fmt"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/parnurzeal/gorequest"
+	"k8s.io/ingress-nginx/test/e2e/framework"
+	"net/http"
+	"strings"
+)
+
+var _ = framework.IngressNginxDescribe("Annotations - AuthTLS", func() {
+	f := framework.NewDefaultFramework("authtls")
+
+	BeforeEach(func() {
+		err := f.NewEchoDeploymentWithReplicas(2)
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	AfterEach(func() {
+	})
+
+	It("should set valid auth-tls-secret", func() {
+		host := "authtls.foo.com"
+		nameSpace := f.IngressController.Namespace
+
+		clientConfig, err := framework.CreateIngressMASecret(
+			f.KubeClientSet,
+			host,
+			host,
+			nameSpace)
+		Expect(err).ToNot(HaveOccurred())
+
+		annotations := map[string]string{
+			"nginx.ingress.kubernetes.io/auth-tls-secret": nameSpace + "/" + host,
+		}
+
+		ing := framework.NewSingleIngressWithTLS(host, "/", host, nameSpace, "http-svc", 80, &annotations)
+		_, err = f.EnsureIngress(ing)
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(ing).NotTo(BeNil())
+
+		// Since we can use the same certificate-chain for tls as well as mutual-auth, we will check all values
+		sslCertDirective := fmt.Sprintf("ssl_certificate /etc/ingress-controller/ssl/%s-%s.pem;", nameSpace, host)
+		sslKeyDirective := fmt.Sprintf("ssl_certificate_key /etc/ingress-controller/ssl/%s-%s.pem;", nameSpace, host)
+		sslClientCertDirective := fmt.Sprintf("ssl_client_certificate /etc/ingress-controller/ssl/%s-%s.pem;", nameSpace, host)
+
+		sslVerify := "ssl_verify_client on;"
+		sslVerifyDepth := "ssl_verify_depth 1;"
+
+		err = f.WaitForNginxServer(host,
+			func(server string) bool {
+				return strings.Contains(server, sslCertDirective) && strings.Contains(server, sslKeyDirective) && strings.Contains(server, sslClientCertDirective) && strings.Contains(server, sslVerify) && strings.Contains(server, sslVerifyDepth)
+			})
+		Expect(err).NotTo(HaveOccurred())
+
+		// Send Request without Client Certs
+		req := gorequest.New()
+		uri := "/"
+		resp, _, errs := req.
+			Get(f.IngressController.HTTPSURL+uri).
+			TLSClientConfig(&tls.Config{ServerName: host, InsecureSkipVerify: true}).
+			Set("Host", host).
+			End()
+		Expect(len(errs)).Should(BeNumerically("==", 0))
+		Expect(resp.StatusCode).Should(Equal(http.StatusBadRequest))
+
+		// Send Request Passing the Client Certs
+		resp, _, errs = req.
+			Get(f.IngressController.HTTPSURL+uri).
+			TLSClientConfig(clientConfig).
+			Set("Host", host).
+			End()
+		Expect(len(errs)).Should(BeNumerically("==", 0))
+		Expect(resp.StatusCode).Should(Equal(http.StatusOK))
+	})
+
+	It("should set valid auth-tls-secret, sslVerify to off, and sslVerifyDepth to 2", func() {
+		host := "authtls.foo.com"
+		nameSpace := f.IngressController.Namespace
+
+		_, err := framework.CreateIngressMASecret(
+			f.KubeClientSet,
+			host,
+			host,
+			nameSpace)
+		Expect(err).ToNot(HaveOccurred())
+
+		annotations := map[string]string{
+			"nginx.ingress.kubernetes.io/auth-tls-secret":        nameSpace + "/" + host,
+			"nginx.ingress.kubernetes.io/auth-tls-verify-client": "off",
+			"nginx.ingress.kubernetes.io/auth-tls-verify-depth":  "2",
+		}
+
+		ing := framework.NewSingleIngressWithTLS(host, "/", host, nameSpace, "http-svc", 80, &annotations)
+		_, err = f.EnsureIngress(ing)
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(ing).NotTo(BeNil())
+
+		// Since we can use the same certificate-chain for tls as well as mutual-auth, we will check all values
+		sslCertDirective := fmt.Sprintf("ssl_certificate /etc/ingress-controller/ssl/%s-%s.pem;", nameSpace, host)
+		sslKeyDirective := fmt.Sprintf("ssl_certificate_key /etc/ingress-controller/ssl/%s-%s.pem;", nameSpace, host)
+		sslClientCertDirective := fmt.Sprintf("ssl_client_certificate /etc/ingress-controller/ssl/%s-%s.pem;", nameSpace, host)
+
+		sslVerify := "ssl_verify_client off;"
+		sslVerifyDepth := "ssl_verify_depth 2;"
+
+		err = f.WaitForNginxServer(host,
+			func(server string) bool {
+				return strings.Contains(server, sslCertDirective) && strings.Contains(server, sslKeyDirective) && strings.Contains(server, sslClientCertDirective) && strings.Contains(server, sslVerify) && strings.Contains(server, sslVerifyDepth)
+			})
+		Expect(err).NotTo(HaveOccurred())
+
+		// Send Request without Client Certs
+		req := gorequest.New()
+		uri := "/"
+		resp, _, errs := req.
+			Get(f.IngressController.HTTPSURL+uri).
+			TLSClientConfig(&tls.Config{ServerName: host, InsecureSkipVerify: true}).
+			Set("Host", host).
+			End()
+		Expect(len(errs)).Should(BeNumerically("==", 0))
+		Expect(resp.StatusCode).Should(Equal(http.StatusOK))
+	})
+
+	It("should set valid auth-tls-secret, pass certificate to upstream, and error page", func() {
+		host := "authtls.foo.com"
+		nameSpace := f.IngressController.Namespace
+
+		errorPath := "/error"
+
+		clientConfig, err := framework.CreateIngressMASecret(
+			f.KubeClientSet,
+			host,
+			host,
+			nameSpace)
+		Expect(err).ToNot(HaveOccurred())
+
+		annotations := map[string]string{
+			"nginx.ingress.kubernetes.io/auth-tls-secret":                       nameSpace + "/" + host,
+			"nginx.ingress.kubernetes.io/auth-tls-error-page":                   f.IngressController.HTTPURL + errorPath,
+			"nginx.ingress.kubernetes.io/auth-tls-pass-certificate-to-upstream": "true",
+		}
+
+		ing := framework.NewSingleIngressWithTLS(host, "/", host, nameSpace, "http-svc", 80, &annotations)
+		_, err = f.EnsureIngress(ing)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(ing).NotTo(BeNil())
+
+		// Since we can use the same certificate-chain for tls as well as mutual-auth, we will check all values
+		sslCertDirective := fmt.Sprintf("ssl_certificate /etc/ingress-controller/ssl/%s-%s.pem;", nameSpace, host)
+		sslKeyDirective := fmt.Sprintf("ssl_certificate_key /etc/ingress-controller/ssl/%s-%s.pem;", nameSpace, host)
+		sslClientCertDirective := fmt.Sprintf("ssl_client_certificate /etc/ingress-controller/ssl/%s-%s.pem;", nameSpace, host)
+
+		sslVerify := "ssl_verify_client on;"
+		sslVerifyDepth := "ssl_verify_depth 1;"
+		sslErrorPage := fmt.Sprintf("error_page 495 496 = %s;", f.IngressController.HTTPURL+errorPath)
+		sslUpstreamClientCert := "proxy_set_header ssl-client-cert $ssl_client_escaped_cert;"
+
+		err = f.WaitForNginxServer(host,
+			func(server string) bool {
+				return strings.Contains(server, sslCertDirective) && strings.Contains(server, sslKeyDirective) && strings.Contains(server, sslClientCertDirective) && strings.Contains(server, sslVerify) && strings.Contains(server, sslVerifyDepth) && strings.Contains(server, sslErrorPage) && strings.Contains(server, sslUpstreamClientCert)
+			})
+		Expect(err).NotTo(HaveOccurred())
+
+		// Send Request without Client Certs
+		req := gorequest.New()
+		uri := "/"
+		resp, _, errs := req.
+			Get(f.IngressController.HTTPSURL+uri).
+			TLSClientConfig(&tls.Config{ServerName: host, InsecureSkipVerify: true}).
+			Set("Host", host).
+			RedirectPolicy(noRedirectPolicyFunc).
+			End()
+		Expect(len(errs)).Should(BeNumerically("==", 0))
+		Expect(resp.StatusCode).Should(Equal(http.StatusFound))
+		Expect(resp.Header.Get("Location")).Should(Equal(f.IngressController.HTTPURL + errorPath))
+
+		// Send Request Passing the Client Certs
+		resp, _, errs = req.
+			Get(f.IngressController.HTTPSURL+uri).
+			TLSClientConfig(clientConfig).
+			Set("Host", host).
+			End()
+		Expect(len(errs)).Should(BeNumerically("==", 0))
+		Expect(resp.StatusCode).Should(Equal(http.StatusOK))
+	})
+})

test/e2e/framework/framework.go

@@ -263,6 +263,7 @@ func (f *Framework) matchNginxConditions(name string, matcher func(cfg string) b
 				glog.Infof("nginx.conf:\n%v", o)
 			}
 
+			// passes the nginx config to the passed function
 			if matcher(strings.Join(strings.Fields(o), " ")) {
 				match = true
 			}

test/e2e/framework/ssl.go

@@ -51,21 +51,24 @@ func CreateIngressTLSSecret(client kubernetes.Interface, hosts []string, secretN
 		return nil, fmt.Errorf("require a non-empty host for client hello")
 	}
 
-	var k, c bytes.Buffer
+	var serverKey, serverCert bytes.Buffer
+	var data map[string][]byte
 	host := strings.Join(hosts, ",")
-	if err := generateRSACert(host, true, &k, &c); err != nil {
+
+	if err := generateRSACert(host, true, &serverKey, &serverCert); err != nil {
 		return nil, err
 	}
 
-	cert, key := c.Bytes(), k.Bytes()
+	data = map[string][]byte{
+		v1.TLSCertKey:       serverCert.Bytes(),
+		v1.TLSPrivateKeyKey: serverKey.Bytes(),
+	}
+
 	newSecret := &v1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: secretName,
 		},
-		Data: map[string][]byte{
+		Data: data,
-			v1.TLSCertKey:       cert,
-			v1.TLSPrivateKeyKey: key,
-		},
 	}
 
 	var apierr error
@@ -81,7 +84,59 @@ func CreateIngressTLSSecret(client kubernetes.Interface, hosts []string, secretN
 	}
 
 	serverName := hosts[0]
-	return tlsConfig(serverName, cert)
+	return tlsConfig(serverName, serverCert.Bytes())
+}
+
+// CreateIngressMASecret creates or updates a Secret containing a Mutual Auth
+// certificate-chain for the given Ingress and returns a TLS configuration suitable
+// for HTTP clients to use against that particular Ingress.
+func CreateIngressMASecret(client kubernetes.Interface, host string, secretName, namespace string) (*tls.Config, error) {
+	if len(host) == 0 {
+		return nil, fmt.Errorf("requires a non-empty host")
+	}
+
+	var caCert, serverKey, serverCert, clientKey, clientCert bytes.Buffer
+	var data map[string][]byte
+
+	if err := generateRSAMutualAuthCerts(host, &caCert, &serverKey, &serverCert, &clientKey, &clientCert); err != nil {
+		return nil, err
+	}
+
+	data = map[string][]byte{
+		v1.TLSCertKey:       serverCert.Bytes(),
+		v1.TLSPrivateKeyKey: serverKey.Bytes(),
+		"ca.crt":            caCert.Bytes(),
+	}
+
+	newSecret := &v1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: secretName,
+		},
+		Data: data,
+	}
+
+	var apierr error
+	curSecret, err := client.CoreV1().Secrets(namespace).Get(secretName, metav1.GetOptions{})
+	if err == nil && curSecret != nil {
+		curSecret.Data = newSecret.Data
+		_, apierr = client.CoreV1().Secrets(namespace).Update(curSecret)
+	} else {
+		_, apierr = client.CoreV1().Secrets(namespace).Create(newSecret)
+	}
+	if apierr != nil {
+		return nil, apierr
+	}
+
+	clientPair, err := tls.X509KeyPair(clientCert.Bytes(), clientKey.Bytes())
+	if err != nil {
+		return nil, err
+	}
+
+	return &tls.Config{
+		ServerName:         host,
+		Certificates:       []tls.Certificate{clientPair},
+		InsecureSkipVerify: true,
+	}, nil
 }
 
 // WaitForTLS waits until the TLS handshake with a given server completes successfully.
@@ -141,8 +196,125 @@ func generateRSACert(host string, isCA bool, keyOut, certOut io.Writer) error {
 		return fmt.Errorf("failed creating cert: %v", err)
 	}
 	if err := pem.Encode(keyOut, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)}); err != nil {
-		return fmt.Errorf("failed creating keay: %v", err)
+		return fmt.Errorf("failed creating key: %v", err)
 	}
+
+	return nil
+}
+
+// generateRSAMutualAuthCerts generates a complete basic self-signed certificate-chain (ca, server, client) using a
+// key-length of rsaBits, valid for validFor time.
+func generateRSAMutualAuthCerts(host string, caCertOut, serverKeyOut, serverCertOut, clientKeyOut, clientCertOut io.Writer) error {
+	notBefore := time.Now()
+	notAfter := notBefore.Add(validFor)
+
+	serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)
+	serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
+	if err != nil {
+		return fmt.Errorf("failed to generate serial number: %s", err)
+	}
+
+	// Generate the CA key and CA cert
+	caKey, err := rsa.GenerateKey(rand.Reader, rsaBits)
+	if err != nil {
+		return fmt.Errorf("failed to generate key: %v", err)
+	}
+
+	caTemplate := x509.Certificate{
+		Subject: pkix.Name{
+			CommonName:   host + "-ca",
+			Organization: []string{"Acme Co"},
+		},
+		SerialNumber: serialNumber,
+		NotBefore:    notBefore,
+		NotAfter:     notAfter,
+
+		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
+		BasicConstraintsValid: true,
+	}
+
+	caTemplate.IsCA = true
+	caTemplate.KeyUsage |= x509.KeyUsageCertSign
+
+	caBytes, err := x509.CreateCertificate(rand.Reader, &caTemplate, &caTemplate, &caKey.PublicKey, caKey)
+	if err != nil {
+		return fmt.Errorf("failed to create certificate: %s", err)
+	}
+	if err := pem.Encode(caCertOut, &pem.Block{Type: "CERTIFICATE", Bytes: caBytes}); err != nil {
+		return fmt.Errorf("failed creating cert: %v", err)
+	}
+
+	// Generate the Server Key and CSR for the server
+	serverKey, err := rsa.GenerateKey(rand.Reader, rsaBits)
+	if err != nil {
+		return fmt.Errorf("failed to generate key: %v", err)
+	}
+
+	// Create the server cert and sign with the csr
+	serialNumber, err = rand.Int(rand.Reader, serialNumberLimit)
+	if err != nil {
+		return fmt.Errorf("failed to generate serial number: %s", err)
+	}
+
+	serverTemplate := x509.Certificate{
+		SerialNumber: serialNumber,
+		Subject: pkix.Name{
+			CommonName:   host,
+			Organization: []string{"Acme Co"},
+		},
+		NotBefore: notBefore,
+		NotAfter:  notAfter,
+
+		KeyUsage:    x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
+		ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
+	}
+
+	serverBytes, err := x509.CreateCertificate(rand.Reader, &serverTemplate, &caTemplate, &serverKey.PublicKey, caKey)
+	if err != nil {
+		return fmt.Errorf("failed to create certificate: %s", err)
+	}
+	if err := pem.Encode(serverCertOut, &pem.Block{Type: "CERTIFICATE", Bytes: serverBytes}); err != nil {
+		return fmt.Errorf("failed creating cert: %v", err)
+	}
+	if err := pem.Encode(serverKeyOut, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(serverKey)}); err != nil {
+		return fmt.Errorf("failed creating key: %v", err)
+	}
+
+	// Create the client key and certificate
+	clientKey, err := rsa.GenerateKey(rand.Reader, rsaBits)
+	if err != nil {
+		return fmt.Errorf("failed to generate key: %v", err)
+	}
+
+	serialNumber, err = rand.Int(rand.Reader, serialNumberLimit)
+	if err != nil {
+		return fmt.Errorf("failed to generate serial number: %s", err)
+	}
+	clientTemplate := x509.Certificate{
+		SerialNumber: serialNumber,
+		Subject: pkix.Name{
+			CommonName:   host + "-client",
+			Organization: []string{"Acme Co"},
+		},
+		NotBefore: notBefore,
+		NotAfter:  notAfter,
+
+		KeyUsage:    x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
+		ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
+	}
+
+	clientBytes, err := x509.CreateCertificate(rand.Reader, &clientTemplate, &caTemplate, &clientKey.PublicKey, caKey)
+	if err != nil {
+		return fmt.Errorf("failed to create certificate: %s", err)
+	}
+	if err := pem.Encode(clientCertOut, &pem.Block{Type: "CERTIFICATE", Bytes: clientBytes}); err != nil {
+		return fmt.Errorf("failed creating cert: %v", err)
+	}
+	if err := pem.Encode(clientKeyOut, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(clientKey)}); err != nil {
+		return fmt.Errorf("failed creating key: %v", err)
+	}
+
 	return nil
 }