From b503c6bdeba8e9a1f11affacd26a5fa5bd1d6fe6 Mon Sep 17 00:00:00 2001 From: Fernando Diaz Date: Fri, 5 Oct 2018 22:35:56 -0500 Subject: [PATCH] Add e2e Tests for AuthTLS Adds E2E tests for the following annotations: - auth-tls-secret - auth-tls-verify-depth - auth-tls-verify-client - auth-tls-error-page - auth-tls-pass-certificate-to-upstream --- internal/ingress/resolver/main.go | 8 +- test/e2e/annotations/authtls.go | 208 ++++++++++++++++++++++++++++++ test/e2e/framework/framework.go | 1 + test/e2e/framework/ssl.go | 190 +++++++++++++++++++++++++-- 4 files changed, 395 insertions(+), 12 deletions(-) create mode 100644 test/e2e/annotations/authtls.go diff --git a/internal/ingress/resolver/main.go b/internal/ingress/resolver/main.go index 1e103b13a..11af30d55 100644 --- a/internal/ingress/resolver/main.go +++ b/internal/ingress/resolver/main.go @@ -18,7 +18,6 @@ package resolver import ( apiv1 "k8s.io/api/core/v1" - "k8s.io/ingress-nginx/internal/ingress/defaults" ) @@ -27,15 +26,18 @@ type Resolver interface { // GetDefaultBackend returns the backend that must be used as default GetDefaultBackend() defaults.Backend - // GetSecret searches for secrets contenating the namespace and name using a the character / + // GetSecret searches for secrets containing the namespace and name using a the character / GetSecret(string) (*apiv1.Secret, error) // GetAuthCertificate resolves a given secret name into an SSL certificate. // The secret must contain 3 keys named: + // ca.crt: contains the certificate chain used for authentication + // tls.crt: contains the server certificate + // tls.key: contains the server key GetAuthCertificate(string) (*AuthSSLCert, error) - // GetService searches for services contenating the namespace and name using a the character / + // GetService searches for services containing the namespace and name using a the character / GetService(string) (*apiv1.Service, error) } diff --git a/test/e2e/annotations/authtls.go b/test/e2e/annotations/authtls.go new file mode 100644 index 000000000..73ab53ca0 --- /dev/null +++ b/test/e2e/annotations/authtls.go @@ -0,0 +1,208 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package annotations + +import ( + "crypto/tls" + "fmt" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/parnurzeal/gorequest" + "k8s.io/ingress-nginx/test/e2e/framework" + "net/http" + "strings" +) + +var _ = framework.IngressNginxDescribe("Annotations - AuthTLS", func() { + f := framework.NewDefaultFramework("authtls") + + BeforeEach(func() { + err := f.NewEchoDeploymentWithReplicas(2) + Expect(err).NotTo(HaveOccurred()) + }) + + AfterEach(func() { + }) + + It("should set valid auth-tls-secret", func() { + host := "authtls.foo.com" + nameSpace := f.IngressController.Namespace + + clientConfig, err := framework.CreateIngressMASecret( + f.KubeClientSet, + host, + host, + nameSpace) + Expect(err).ToNot(HaveOccurred()) + + annotations := map[string]string{ + "nginx.ingress.kubernetes.io/auth-tls-secret": nameSpace + "/" + host, + } + + ing := framework.NewSingleIngressWithTLS(host, "/", host, nameSpace, "http-svc", 80, &annotations) + _, err = f.EnsureIngress(ing) + + Expect(err).NotTo(HaveOccurred()) + Expect(ing).NotTo(BeNil()) + + // Since we can use the same certificate-chain for tls as well as mutual-auth, we will check all values + sslCertDirective := fmt.Sprintf("ssl_certificate /etc/ingress-controller/ssl/%s-%s.pem;", nameSpace, host) + sslKeyDirective := fmt.Sprintf("ssl_certificate_key /etc/ingress-controller/ssl/%s-%s.pem;", nameSpace, host) + sslClientCertDirective := fmt.Sprintf("ssl_client_certificate /etc/ingress-controller/ssl/%s-%s.pem;", nameSpace, host) + + sslVerify := "ssl_verify_client on;" + sslVerifyDepth := "ssl_verify_depth 1;" + + err = f.WaitForNginxServer(host, + func(server string) bool { + return strings.Contains(server, sslCertDirective) && strings.Contains(server, sslKeyDirective) && strings.Contains(server, sslClientCertDirective) && strings.Contains(server, sslVerify) && strings.Contains(server, sslVerifyDepth) + }) + Expect(err).NotTo(HaveOccurred()) + + // Send Request without Client Certs + req := gorequest.New() + uri := "/" + resp, _, errs := req. + Get(f.IngressController.HTTPSURL+uri). + TLSClientConfig(&tls.Config{ServerName: host, InsecureSkipVerify: true}). + Set("Host", host). + End() + Expect(len(errs)).Should(BeNumerically("==", 0)) + Expect(resp.StatusCode).Should(Equal(http.StatusBadRequest)) + + // Send Request Passing the Client Certs + resp, _, errs = req. + Get(f.IngressController.HTTPSURL+uri). + TLSClientConfig(clientConfig). + Set("Host", host). + End() + Expect(len(errs)).Should(BeNumerically("==", 0)) + Expect(resp.StatusCode).Should(Equal(http.StatusOK)) + }) + + It("should set valid auth-tls-secret, sslVerify to off, and sslVerifyDepth to 2", func() { + host := "authtls.foo.com" + nameSpace := f.IngressController.Namespace + + _, err := framework.CreateIngressMASecret( + f.KubeClientSet, + host, + host, + nameSpace) + Expect(err).ToNot(HaveOccurred()) + + annotations := map[string]string{ + "nginx.ingress.kubernetes.io/auth-tls-secret": nameSpace + "/" + host, + "nginx.ingress.kubernetes.io/auth-tls-verify-client": "off", + "nginx.ingress.kubernetes.io/auth-tls-verify-depth": "2", + } + + ing := framework.NewSingleIngressWithTLS(host, "/", host, nameSpace, "http-svc", 80, &annotations) + _, err = f.EnsureIngress(ing) + + Expect(err).NotTo(HaveOccurred()) + Expect(ing).NotTo(BeNil()) + + // Since we can use the same certificate-chain for tls as well as mutual-auth, we will check all values + sslCertDirective := fmt.Sprintf("ssl_certificate /etc/ingress-controller/ssl/%s-%s.pem;", nameSpace, host) + sslKeyDirective := fmt.Sprintf("ssl_certificate_key /etc/ingress-controller/ssl/%s-%s.pem;", nameSpace, host) + sslClientCertDirective := fmt.Sprintf("ssl_client_certificate /etc/ingress-controller/ssl/%s-%s.pem;", nameSpace, host) + + sslVerify := "ssl_verify_client off;" + sslVerifyDepth := "ssl_verify_depth 2;" + + err = f.WaitForNginxServer(host, + func(server string) bool { + return strings.Contains(server, sslCertDirective) && strings.Contains(server, sslKeyDirective) && strings.Contains(server, sslClientCertDirective) && strings.Contains(server, sslVerify) && strings.Contains(server, sslVerifyDepth) + }) + Expect(err).NotTo(HaveOccurred()) + + // Send Request without Client Certs + req := gorequest.New() + uri := "/" + resp, _, errs := req. + Get(f.IngressController.HTTPSURL+uri). + TLSClientConfig(&tls.Config{ServerName: host, InsecureSkipVerify: true}). + Set("Host", host). + End() + Expect(len(errs)).Should(BeNumerically("==", 0)) + Expect(resp.StatusCode).Should(Equal(http.StatusOK)) + }) + + It("should set valid auth-tls-secret, pass certificate to upstream, and error page", func() { + host := "authtls.foo.com" + nameSpace := f.IngressController.Namespace + + errorPath := "/error" + + clientConfig, err := framework.CreateIngressMASecret( + f.KubeClientSet, + host, + host, + nameSpace) + Expect(err).ToNot(HaveOccurred()) + + annotations := map[string]string{ + "nginx.ingress.kubernetes.io/auth-tls-secret": nameSpace + "/" + host, + "nginx.ingress.kubernetes.io/auth-tls-error-page": f.IngressController.HTTPURL + errorPath, + "nginx.ingress.kubernetes.io/auth-tls-pass-certificate-to-upstream": "true", + } + + ing := framework.NewSingleIngressWithTLS(host, "/", host, nameSpace, "http-svc", 80, &annotations) + _, err = f.EnsureIngress(ing) + Expect(err).NotTo(HaveOccurred()) + Expect(ing).NotTo(BeNil()) + + // Since we can use the same certificate-chain for tls as well as mutual-auth, we will check all values + sslCertDirective := fmt.Sprintf("ssl_certificate /etc/ingress-controller/ssl/%s-%s.pem;", nameSpace, host) + sslKeyDirective := fmt.Sprintf("ssl_certificate_key /etc/ingress-controller/ssl/%s-%s.pem;", nameSpace, host) + sslClientCertDirective := fmt.Sprintf("ssl_client_certificate /etc/ingress-controller/ssl/%s-%s.pem;", nameSpace, host) + + sslVerify := "ssl_verify_client on;" + sslVerifyDepth := "ssl_verify_depth 1;" + sslErrorPage := fmt.Sprintf("error_page 495 496 = %s;", f.IngressController.HTTPURL+errorPath) + sslUpstreamClientCert := "proxy_set_header ssl-client-cert $ssl_client_escaped_cert;" + + err = f.WaitForNginxServer(host, + func(server string) bool { + return strings.Contains(server, sslCertDirective) && strings.Contains(server, sslKeyDirective) && strings.Contains(server, sslClientCertDirective) && strings.Contains(server, sslVerify) && strings.Contains(server, sslVerifyDepth) && strings.Contains(server, sslErrorPage) && strings.Contains(server, sslUpstreamClientCert) + }) + Expect(err).NotTo(HaveOccurred()) + + // Send Request without Client Certs + req := gorequest.New() + uri := "/" + resp, _, errs := req. + Get(f.IngressController.HTTPSURL+uri). + TLSClientConfig(&tls.Config{ServerName: host, InsecureSkipVerify: true}). + Set("Host", host). + RedirectPolicy(noRedirectPolicyFunc). + End() + Expect(len(errs)).Should(BeNumerically("==", 0)) + Expect(resp.StatusCode).Should(Equal(http.StatusFound)) + Expect(resp.Header.Get("Location")).Should(Equal(f.IngressController.HTTPURL + errorPath)) + + // Send Request Passing the Client Certs + resp, _, errs = req. + Get(f.IngressController.HTTPSURL+uri). + TLSClientConfig(clientConfig). + Set("Host", host). + End() + Expect(len(errs)).Should(BeNumerically("==", 0)) + Expect(resp.StatusCode).Should(Equal(http.StatusOK)) + }) +}) diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go index df0ef6402..10234a719 100644 --- a/test/e2e/framework/framework.go +++ b/test/e2e/framework/framework.go @@ -263,6 +263,7 @@ func (f *Framework) matchNginxConditions(name string, matcher func(cfg string) b glog.Infof("nginx.conf:\n%v", o) } + // passes the nginx config to the passed function if matcher(strings.Join(strings.Fields(o), " ")) { match = true } diff --git a/test/e2e/framework/ssl.go b/test/e2e/framework/ssl.go index f8f1a700f..9b6055e2c 100644 --- a/test/e2e/framework/ssl.go +++ b/test/e2e/framework/ssl.go @@ -51,21 +51,24 @@ func CreateIngressTLSSecret(client kubernetes.Interface, hosts []string, secretN return nil, fmt.Errorf("require a non-empty host for client hello") } - var k, c bytes.Buffer + var serverKey, serverCert bytes.Buffer + var data map[string][]byte host := strings.Join(hosts, ",") - if err := generateRSACert(host, true, &k, &c); err != nil { + + if err := generateRSACert(host, true, &serverKey, &serverCert); err != nil { return nil, err } - cert, key := c.Bytes(), k.Bytes() + data = map[string][]byte{ + v1.TLSCertKey: serverCert.Bytes(), + v1.TLSPrivateKeyKey: serverKey.Bytes(), + } + newSecret := &v1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: secretName, }, - Data: map[string][]byte{ - v1.TLSCertKey: cert, - v1.TLSPrivateKeyKey: key, - }, + Data: data, } var apierr error @@ -81,7 +84,59 @@ func CreateIngressTLSSecret(client kubernetes.Interface, hosts []string, secretN } serverName := hosts[0] - return tlsConfig(serverName, cert) + return tlsConfig(serverName, serverCert.Bytes()) +} + +// CreateIngressMASecret creates or updates a Secret containing a Mutual Auth +// certificate-chain for the given Ingress and returns a TLS configuration suitable +// for HTTP clients to use against that particular Ingress. +func CreateIngressMASecret(client kubernetes.Interface, host string, secretName, namespace string) (*tls.Config, error) { + if len(host) == 0 { + return nil, fmt.Errorf("requires a non-empty host") + } + + var caCert, serverKey, serverCert, clientKey, clientCert bytes.Buffer + var data map[string][]byte + + if err := generateRSAMutualAuthCerts(host, &caCert, &serverKey, &serverCert, &clientKey, &clientCert); err != nil { + return nil, err + } + + data = map[string][]byte{ + v1.TLSCertKey: serverCert.Bytes(), + v1.TLSPrivateKeyKey: serverKey.Bytes(), + "ca.crt": caCert.Bytes(), + } + + newSecret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + }, + Data: data, + } + + var apierr error + curSecret, err := client.CoreV1().Secrets(namespace).Get(secretName, metav1.GetOptions{}) + if err == nil && curSecret != nil { + curSecret.Data = newSecret.Data + _, apierr = client.CoreV1().Secrets(namespace).Update(curSecret) + } else { + _, apierr = client.CoreV1().Secrets(namespace).Create(newSecret) + } + if apierr != nil { + return nil, apierr + } + + clientPair, err := tls.X509KeyPair(clientCert.Bytes(), clientKey.Bytes()) + if err != nil { + return nil, err + } + + return &tls.Config{ + ServerName: host, + Certificates: []tls.Certificate{clientPair}, + InsecureSkipVerify: true, + }, nil } // WaitForTLS waits until the TLS handshake with a given server completes successfully. @@ -141,8 +196,125 @@ func generateRSACert(host string, isCA bool, keyOut, certOut io.Writer) error { return fmt.Errorf("failed creating cert: %v", err) } if err := pem.Encode(keyOut, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)}); err != nil { - return fmt.Errorf("failed creating keay: %v", err) + return fmt.Errorf("failed creating key: %v", err) } + + return nil +} + +// generateRSAMutualAuthCerts generates a complete basic self-signed certificate-chain (ca, server, client) using a +// key-length of rsaBits, valid for validFor time. +func generateRSAMutualAuthCerts(host string, caCertOut, serverKeyOut, serverCertOut, clientKeyOut, clientCertOut io.Writer) error { + notBefore := time.Now() + notAfter := notBefore.Add(validFor) + + serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) + serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) + if err != nil { + return fmt.Errorf("failed to generate serial number: %s", err) + } + + // Generate the CA key and CA cert + caKey, err := rsa.GenerateKey(rand.Reader, rsaBits) + if err != nil { + return fmt.Errorf("failed to generate key: %v", err) + } + + caTemplate := x509.Certificate{ + Subject: pkix.Name{ + CommonName: host + "-ca", + Organization: []string{"Acme Co"}, + }, + SerialNumber: serialNumber, + NotBefore: notBefore, + NotAfter: notAfter, + + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, + BasicConstraintsValid: true, + } + + caTemplate.IsCA = true + caTemplate.KeyUsage |= x509.KeyUsageCertSign + + caBytes, err := x509.CreateCertificate(rand.Reader, &caTemplate, &caTemplate, &caKey.PublicKey, caKey) + if err != nil { + return fmt.Errorf("failed to create certificate: %s", err) + } + if err := pem.Encode(caCertOut, &pem.Block{Type: "CERTIFICATE", Bytes: caBytes}); err != nil { + return fmt.Errorf("failed creating cert: %v", err) + } + + // Generate the Server Key and CSR for the server + serverKey, err := rsa.GenerateKey(rand.Reader, rsaBits) + if err != nil { + return fmt.Errorf("failed to generate key: %v", err) + } + + // Create the server cert and sign with the csr + serialNumber, err = rand.Int(rand.Reader, serialNumberLimit) + if err != nil { + return fmt.Errorf("failed to generate serial number: %s", err) + } + + serverTemplate := x509.Certificate{ + SerialNumber: serialNumber, + Subject: pkix.Name{ + CommonName: host, + Organization: []string{"Acme Co"}, + }, + NotBefore: notBefore, + NotAfter: notAfter, + + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, + } + + serverBytes, err := x509.CreateCertificate(rand.Reader, &serverTemplate, &caTemplate, &serverKey.PublicKey, caKey) + if err != nil { + return fmt.Errorf("failed to create certificate: %s", err) + } + if err := pem.Encode(serverCertOut, &pem.Block{Type: "CERTIFICATE", Bytes: serverBytes}); err != nil { + return fmt.Errorf("failed creating cert: %v", err) + } + if err := pem.Encode(serverKeyOut, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(serverKey)}); err != nil { + return fmt.Errorf("failed creating key: %v", err) + } + + // Create the client key and certificate + clientKey, err := rsa.GenerateKey(rand.Reader, rsaBits) + if err != nil { + return fmt.Errorf("failed to generate key: %v", err) + } + + serialNumber, err = rand.Int(rand.Reader, serialNumberLimit) + if err != nil { + return fmt.Errorf("failed to generate serial number: %s", err) + } + clientTemplate := x509.Certificate{ + SerialNumber: serialNumber, + Subject: pkix.Name{ + CommonName: host + "-client", + Organization: []string{"Acme Co"}, + }, + NotBefore: notBefore, + NotAfter: notAfter, + + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, + } + + clientBytes, err := x509.CreateCertificate(rand.Reader, &clientTemplate, &caTemplate, &clientKey.PublicKey, caKey) + if err != nil { + return fmt.Errorf("failed to create certificate: %s", err) + } + if err := pem.Encode(clientCertOut, &pem.Block{Type: "CERTIFICATE", Bytes: clientBytes}); err != nil { + return fmt.Errorf("failed creating cert: %v", err) + } + if err := pem.Encode(clientKeyOut, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(clientKey)}); err != nil { + return fmt.Errorf("failed creating key: %v", err) + } + return nil }