From a858c549d9e95fd11148cb42cce6f840791499ca Mon Sep 17 00:00:00 2001 From: Manuel de Brito Fontes Date: Sun, 12 Nov 2017 13:52:55 -0300 Subject: [PATCH] Add e2e tests for auth annotation --- rootfs/etc/nginx/template/nginx.tmpl | 3 + test/e2e/annotations/alias.go | 19 +- test/e2e/annotations/auth.go | 306 ++++++++++++++++++++++++++- test/e2e/e2e.go | 2 +- test/e2e/framework/echo.go | 6 +- test/e2e/framework/exec.go | 53 +++++ test/e2e/framework/framework.go | 41 +++- test/e2e/framework/util.go | 14 +- 8 files changed, 416 insertions(+), 28 deletions(-) create mode 100644 test/e2e/framework/exec.go diff --git a/rootfs/etc/nginx/template/nginx.tmpl b/rootfs/etc/nginx/template/nginx.tmpl index 1a0fd3421..c90781d2a 100644 --- a/rootfs/etc/nginx/template/nginx.tmpl +++ b/rootfs/etc/nginx/template/nginx.tmpl @@ -393,6 +393,7 @@ http { {{ range $index, $server := $servers }} + ## start server {{ $server.Hostname }} server { server_name {{ $server.Hostname }} {{ $server.Alias }}; {{ template "SERVER" serverConfig $all $server }} @@ -404,6 +405,8 @@ http { {{ template "CUSTOM_ERRORS" $all }} } + ## end server {{ $server.Hostname }} + {{ end }} # default server, used for NGINX healthcheck and access to nginx stats diff --git a/test/e2e/annotations/alias.go b/test/e2e/annotations/alias.go index c0fb4b5d0..07b6f7936 100644 --- a/test/e2e/annotations/alias.go +++ b/test/e2e/annotations/alias.go @@ -19,6 +19,7 @@ package annotations import ( "fmt" "net/http" + "strings" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -75,7 +76,11 @@ var _ = framework.IngressNginxDescribe("Annotations - Alias", func() { Expect(err).NotTo(HaveOccurred()) Expect(ing).NotTo(BeNil()) - err = f.WaitForNginxServer(host) + err = f.WaitForNginxServer(host, + func(server string) bool { + return strings.Contains(server, "server_name foo") && + !strings.Contains(server, "return 503") + }) Expect(err).NotTo(HaveOccurred()) resp, body, errs := gorequest.New(). @@ -98,19 +103,19 @@ var _ = framework.IngressNginxDescribe("Annotations - Alias", func() { }) It("should return status code 200 for host 'foo' and 'bar'", func() { - host := "bar" + host := "foo" ing, err := f.EnsureIngress(&v1beta1.Ingress{ ObjectMeta: metav1.ObjectMeta{ Name: host, Namespace: f.Namespace.Name, Annotations: map[string]string{ - "nginx.ingress.kubernetes.io/server-alias": host, + "nginx.ingress.kubernetes.io/server-alias": "bar", }, }, Spec: v1beta1.IngressSpec{ Rules: []v1beta1.IngressRule{ { - Host: "foo", + Host: host, IngressRuleValue: v1beta1.IngressRuleValue{ HTTP: &v1beta1.HTTPIngressRuleValue{ Paths: []v1beta1.HTTPIngressPath{ @@ -132,7 +137,11 @@ var _ = framework.IngressNginxDescribe("Annotations - Alias", func() { Expect(err).NotTo(HaveOccurred()) Expect(ing).NotTo(BeNil()) - err = f.WaitForNginxServer(host) + err = f.WaitForNginxServer(host, + func(server string) bool { + return strings.Contains(server, "server_name foo") && + !strings.Contains(server, "return 503") + }) Expect(err).NotTo(HaveOccurred()) hosts := []string{"foo", "bar"} diff --git a/test/e2e/annotations/auth.go b/test/e2e/annotations/auth.go index 1ee7e1546..cffe270ed 100644 --- a/test/e2e/annotations/auth.go +++ b/test/e2e/annotations/auth.go @@ -1,15 +1,303 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package annotations -// Tests: -// No auth -// Basic +import ( + "fmt" + "net/http" + "os/exec" + "strings" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/parnurzeal/gorequest" + + corev1 "k8s.io/api/core/v1" + v1beta1 "k8s.io/api/extensions/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + + "k8s.io/ingress-nginx/test/e2e/framework" +) + +var _ = framework.IngressNginxDescribe("Annotations - Alias", func() { + f := framework.NewDefaultFramework("alias") + + BeforeEach(func() { + err := f.NewEchoDeployment() + Expect(err).NotTo(HaveOccurred()) + }) + + AfterEach(func() { + }) + + It("should return status code 200 when no authentication is configured", func() { + host := "auth" + + ing, err := f.EnsureIngress(buildIngress(host, f.Namespace.Name)) + Expect(err).NotTo(HaveOccurred()) + Expect(ing).NotTo(BeNil()) + + err = f.WaitForNginxServer(host, + func(server string) bool { + return strings.Contains(server, "server_name auth") && + !strings.Contains(server, "return 503") + }) + Expect(err).NotTo(HaveOccurred()) + + resp, body, errs := gorequest.New(). + Get(f.NginxHTTPURL). + Set("Host", host). + End() + + Expect(len(errs)).Should(BeNumerically("==", 0)) + Expect(resp.StatusCode).Should(Equal(http.StatusOK)) + Expect(body).Should(ContainSubstring(fmt.Sprintf("host=%v", host))) + }) + + It("should return status code 503 when authentication is configured with an invalid secret", func() { + host := "auth" + + bi := buildIngress(host, f.Namespace.Name) + bi.Annotations["nginx.ingress.kubernetes.io/auth-type"] = "basic" + bi.Annotations["nginx.ingress.kubernetes.io/auth-secret"] = "something" + bi.Annotations["nginx.ingress.kubernetes.io/auth-realm"] = "test auth" + + ing, err := f.EnsureIngress(bi) + Expect(err).NotTo(HaveOccurred()) + Expect(ing).NotTo(BeNil()) + + err = f.WaitForNginxServer(host, + func(server string) bool { + return strings.Contains(server, "server_name auth") && + strings.Contains(server, "return 503") + }) + Expect(err).NotTo(HaveOccurred()) + + resp, body, errs := gorequest.New(). + Get(f.NginxHTTPURL). + Set("Host", host). + End() + + Expect(len(errs)).Should(BeNumerically("==", 0)) + Expect(resp.StatusCode).Should(Equal(http.StatusServiceUnavailable)) + Expect(body).Should(ContainSubstring("503 Service Temporarily Unavailable")) + }) + + It("should return status code 401 when authentication is configured but Authorization header is not configured", func() { + host := "auth" + + s, err := f.EnsureSecret(buildSecret("foo", "bar", "test", f.Namespace.Name)) + Expect(err).NotTo(HaveOccurred()) + Expect(s).NotTo(BeNil()) + Expect(s.ObjectMeta).NotTo(BeNil()) + + bi := buildIngress(host, f.Namespace.Name) + bi.Annotations["nginx.ingress.kubernetes.io/auth-type"] = "basic" + bi.Annotations["nginx.ingress.kubernetes.io/auth-secret"] = s.Name + bi.Annotations["nginx.ingress.kubernetes.io/auth-realm"] = "test auth" + + ing, err := f.EnsureIngress(bi) + Expect(err).NotTo(HaveOccurred()) + Expect(ing).NotTo(BeNil()) + + err = f.WaitForNginxServer(host, + func(server string) bool { + return strings.Contains(server, "server_name auth") && + !strings.Contains(server, "return 503") + }) + Expect(err).NotTo(HaveOccurred()) + + resp, body, errs := gorequest.New(). + Get(f.NginxHTTPURL). + Set("Host", host). + End() + + Expect(len(errs)).Should(BeNumerically("==", 0)) + Expect(resp.StatusCode).Should(Equal(http.StatusUnauthorized)) + Expect(body).Should(ContainSubstring("401 Authorization Required")) + }) + + It("should return status code 401 when authentication is configured and Authorization header is sent with invalid credentials", func() { + host := "auth" + + s, err := f.EnsureSecret(buildSecret("foo", "bar", "test", f.Namespace.Name)) + Expect(err).NotTo(HaveOccurred()) + Expect(s).NotTo(BeNil()) + Expect(s.ObjectMeta).NotTo(BeNil()) + + bi := buildIngress(host, f.Namespace.Name) + bi.Annotations["nginx.ingress.kubernetes.io/auth-type"] = "basic" + bi.Annotations["nginx.ingress.kubernetes.io/auth-secret"] = s.Name + bi.Annotations["nginx.ingress.kubernetes.io/auth-realm"] = "test auth" + + ing, err := f.EnsureIngress(bi) + Expect(err).NotTo(HaveOccurred()) + Expect(ing).NotTo(BeNil()) + + err = f.WaitForNginxServer(host, + func(server string) bool { + return strings.Contains(server, "server_name auth") && + !strings.Contains(server, "return 503") + }) + Expect(err).NotTo(HaveOccurred()) + + resp, body, errs := gorequest.New(). + Get(f.NginxHTTPURL). + Set("Host", host). + SetBasicAuth("user", "pass"). + End() + + Expect(len(errs)).Should(BeNumerically("==", 0)) + Expect(resp.StatusCode).Should(Equal(http.StatusUnauthorized)) + Expect(body).Should(ContainSubstring("401 Authorization Required")) + }) + + It("should return status code 200 when authentication is configured and Authorization header is sent", func() { + host := "auth" + + s, err := f.EnsureSecret(buildSecret("foo", "bar", "test", f.Namespace.Name)) + Expect(err).NotTo(HaveOccurred()) + Expect(s).NotTo(BeNil()) + Expect(s.ObjectMeta).NotTo(BeNil()) + + bi := buildIngress(host, f.Namespace.Name) + bi.Annotations["nginx.ingress.kubernetes.io/auth-type"] = "basic" + bi.Annotations["nginx.ingress.kubernetes.io/auth-secret"] = s.Name + bi.Annotations["nginx.ingress.kubernetes.io/auth-realm"] = "test auth" + + ing, err := f.EnsureIngress(bi) + Expect(err).NotTo(HaveOccurred()) + Expect(ing).NotTo(BeNil()) + + err = f.WaitForNginxServer(host, + func(server string) bool { + return strings.Contains(server, "server_name auth") && + !strings.Contains(server, "return 503") + }) + Expect(err).NotTo(HaveOccurred()) + + resp, _, errs := gorequest.New(). + Get(f.NginxHTTPURL). + Set("Host", host). + SetBasicAuth("foo", "bar"). + End() + + Expect(len(errs)).Should(BeNumerically("==", 0)) + Expect(resp.StatusCode).Should(Equal(http.StatusOK)) + }) + + It("should return status code 500 when authentication is configured with invalid content and Authorization header is sent", func() { + host := "auth" + + s, err := f.EnsureSecret( + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: f.Namespace.Name, + }, + Data: map[string][]byte{ + // invalid content + "auth": []byte("foo:"), + }, + Type: corev1.SecretTypeOpaque, + }, + ) + Expect(err).NotTo(HaveOccurred()) + Expect(s).NotTo(BeNil()) + Expect(s.ObjectMeta).NotTo(BeNil()) + + bi := buildIngress(host, f.Namespace.Name) + bi.Annotations["nginx.ingress.kubernetes.io/auth-type"] = "basic" + bi.Annotations["nginx.ingress.kubernetes.io/auth-secret"] = s.Name + bi.Annotations["nginx.ingress.kubernetes.io/auth-realm"] = "test auth" + + ing, err := f.EnsureIngress(bi) + Expect(err).NotTo(HaveOccurred()) + Expect(ing).NotTo(BeNil()) + + err = f.WaitForNginxServer(host, + func(server string) bool { + return strings.Contains(server, "server_name auth") && + !strings.Contains(server, "return 503") + }) + Expect(err).NotTo(HaveOccurred()) + + resp, _, errs := gorequest.New(). + Get(f.NginxHTTPURL). + Set("Host", host). + SetBasicAuth("foo", "bar"). + End() + + Expect(len(errs)).Should(BeNumerically("==", 0)) + Expect(resp.StatusCode).Should(Equal(http.StatusInternalServerError)) + }) +}) + +// TODO: test Digest Auth // 401 // Realm name // Auth ok // Auth error -// Digest -// 401 -// Realm name -// Auth ok -// Auth error -// Check return 403 if there's an error retrieving the secret + +func buildIngress(host, namespace string) *v1beta1.Ingress { + return &v1beta1.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Name: host, + Namespace: namespace, + Annotations: map[string]string{}, + }, + Spec: v1beta1.IngressSpec{ + Rules: []v1beta1.IngressRule{ + { + Host: host, + IngressRuleValue: v1beta1.IngressRuleValue{ + HTTP: &v1beta1.HTTPIngressRuleValue{ + Paths: []v1beta1.HTTPIngressPath{ + { + Path: "/", + Backend: v1beta1.IngressBackend{ + ServiceName: "http-svc", + ServicePort: intstr.FromInt(80), + }, + }, + }, + }, + }, + }, + }, + }, + } +} + +func buildSecret(username, password, name, namespace string) *corev1.Secret { + out, err := exec.Command("openssl", "passwd", "-crypt", password).CombinedOutput() + encpass := fmt.Sprintf("%v:%s\n", username, out) + Expect(err).NotTo(HaveOccurred()) + + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + DeletionGracePeriodSeconds: framework.NewInt64(1), + }, + Data: map[string][]byte{ + "auth": []byte(encpass), + }, + Type: corev1.SecretTypeOpaque, + } +} diff --git a/test/e2e/e2e.go b/test/e2e/e2e.go index 4f0601687..0982635c9 100644 --- a/test/e2e/e2e.go +++ b/test/e2e/e2e.go @@ -43,6 +43,6 @@ func RunE2ETests(t *testing.T) { config.GinkgoConfig.SkipString = `\[Flaky\]|\[Feature:.+\]` } - glog.Infof("Starting e2e run %q on Ginkgo node %d", framework.RunId, config.GinkgoConfig.ParallelNode) + glog.Infof("Starting e2e run %q on Ginkgo node %d", framework.RunID, config.GinkgoConfig.ParallelNode) ginkgo.RunSpecs(t, "nginx-ingress-controller e2e suite") } diff --git a/test/e2e/framework/echo.go b/test/e2e/framework/echo.go index 7d9cdc966..1868bd4d2 100644 --- a/test/e2e/framework/echo.go +++ b/test/e2e/framework/echo.go @@ -33,9 +33,8 @@ import ( func (f *Framework) NewEchoDeployment() error { deployment := &extensions.Deployment{ ObjectMeta: metav1.ObjectMeta{ - Name: "http-svc", - Namespace: f.Namespace.Name, - DeletionGracePeriodSeconds: NewInt64(5), + Name: "http-svc", + Namespace: f.Namespace.Name, }, Spec: extensions.DeploymentSpec{ Replicas: NewInt32(1), @@ -51,6 +50,7 @@ func (f *Framework) NewEchoDeployment() error { }, }, Spec: corev1.PodSpec{ + TerminationGracePeriodSeconds: NewInt64(1), Containers: []corev1.Container{ { Name: "http-svc", diff --git a/test/e2e/framework/exec.go b/test/e2e/framework/exec.go new file mode 100644 index 000000000..aecc0170b --- /dev/null +++ b/test/e2e/framework/exec.go @@ -0,0 +1,53 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +import ( + "bytes" + "fmt" + "os/exec" + + "k8s.io/api/core/v1" +) + +// ExecCommand executes a command inside a the first container in a running pod +func (f *Framework) ExecCommand(pod *v1.Pod, command string) (string, error) { + var ( + execOut bytes.Buffer + execErr bytes.Buffer + ) + + if len(pod.Spec.Containers) != 1 { + return "", fmt.Errorf("could not determine which container to use") + } + + args := fmt.Sprintf("kubectl exec -n %v %v -- %v", pod.Namespace, pod.Name, command) + cmd := exec.Command("/bin/bash", "-c", args) + cmd.Stdout = &execOut + cmd.Stderr = &execErr + + err := cmd.Run() + if err != nil { + return "", fmt.Errorf("could not execute: %v", err) + } + + if execErr.Len() > 0 { + return "", fmt.Errorf("stderr: %v", execErr.String()) + } + + return execOut.String(), nil +} diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go index d26c7b07d..aed434df6 100644 --- a/test/e2e/framework/framework.go +++ b/test/e2e/framework/framework.go @@ -22,7 +22,9 @@ import ( "k8s.io/api/core/v1" apiextcs "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" + restclient "k8s.io/client-go/rest" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -51,6 +53,7 @@ type Framework struct { // A Kubernetes and Service Catalog client KubeClientSet kubernetes.Interface + KubeConfig *restclient.Config APIExtensionsClientSet apiextcs.Interface // Namespace in which all test resources should reside @@ -86,6 +89,7 @@ func (f *Framework) BeforeEach() { kubeConfig, err := LoadConfig(TestContext.KubeConfig, TestContext.KubeContext) Expect(err).NotTo(HaveOccurred()) + f.KubeConfig = kubeConfig f.KubeClientSet, err = kubernetes.NewForConfig(kubeConfig) Expect(err).NotTo(HaveOccurred()) @@ -161,7 +165,40 @@ func (f *Framework) GetNginxURL(scheme RequestScheme) (string, error) { return fmt.Sprintf("%v://%v:%v", scheme, ip, port), nil } -func (f *Framework) WaitForNginxServer(name string) error { +// WaitForNginxServer waits until the nginx configuration contains a particular server section +func (f *Framework) WaitForNginxServer(name string, matcher func(cfg string) bool) error { + // initial wait to allow the update of the ingress controller time.Sleep(5 * time.Second) - return nil + return wait.PollImmediate(Poll, time.Minute*2, f.matchNginxConditions(name, matcher)) +} + +func (f *Framework) matchNginxConditions(name string, matcher func(cfg string) bool) wait.ConditionFunc { + return func() (bool, error) { + l, err := f.KubeClientSet.CoreV1().Pods("ingress-nginx").List(metav1.ListOptions{ + LabelSelector: "app=ingress-nginx", + }) + if err != nil { + return false, err + } + + if len(l.Items) == 0 { + return false, fmt.Errorf("no nginx ingress controller pod is running") + } + + if len(l.Items) != 1 { + return false, fmt.Errorf("unexpected number of nginx ingress controller pod is running (%v)", len(l.Items)) + } + + cmd := fmt.Sprintf("cat /etc/nginx/nginx.conf | awk '/## start server %v/,/## end server %v/'", name, name) + o, err := f.ExecCommand(&l.Items[0], cmd) + if err != nil { + return false, err + } + + if matcher(o) { + return true, nil + } + + return false, nil + } } diff --git a/test/e2e/framework/util.go b/test/e2e/framework/util.go index bad785de1..3d9a378a3 100644 --- a/test/e2e/framework/util.go +++ b/test/e2e/framework/util.go @@ -92,8 +92,8 @@ func LoadConfig(config, context string) (*rest.Config, error) { return clientcmd.NewDefaultClientConfig(*c, &clientcmd.ConfigOverrides{}).ClientConfig() } -// RunId unique identifier of the e2e run -var RunId = uuid.NewUUID() +// RunID unique identifier of the e2e run +var RunID = uuid.NewUUID() func CreateKubeNamespace(baseName string, c kubernetes.Interface) (*v1.Namespace, error) { ns := &v1.Namespace{ @@ -119,12 +119,9 @@ func CreateKubeNamespace(baseName string, c kubernetes.Interface) (*v1.Namespace return got, nil } +// DeleteKubeNamespace deletes a namespace and all the objects inside func DeleteKubeNamespace(c kubernetes.Interface, namespace string) error { - deletePolicy := metav1.DeletePropagationForeground - return c.Core().Namespaces().Delete(namespace, &metav1.DeleteOptions{ - GracePeriodSeconds: NewInt64(0), - PropagationPolicy: &deletePolicy, - }) + return c.Core().Namespaces().Delete(namespace, metav1.NewDeleteOptions(0)) } func ExpectNoError(err error, explain ...interface{}) { @@ -135,7 +132,7 @@ func ExpectNoError(err error, explain ...interface{}) { } func WaitForKubeNamespaceNotExist(c kubernetes.Interface, namespace string) error { - return wait.PollImmediate(Poll, time.Minute*1, namespaceNotExist(c, namespace)) + return wait.PollImmediate(Poll, time.Minute*2, namespaceNotExist(c, namespace)) } func namespaceNotExist(c kubernetes.Interface, namespace string) wait.ConditionFunc { @@ -151,6 +148,7 @@ func namespaceNotExist(c kubernetes.Interface, namespace string) wait.ConditionF } } +// WaitForNoPodsInNamespace waits until there are no pods running in a namespace func WaitForNoPodsInNamespace(c kubernetes.Interface, namespace string) error { return wait.PollImmediate(Poll, time.Minute*2, noPodsInNamespace(c, namespace)) }