diff --git a/.travis.yml b/.travis.yml index 7cabc9d5e..c8a4410a8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -54,7 +54,6 @@ jobs: - if ! go get github.com/jteeuwen/go-bindata/...; then github.com/jteeuwen/go-bindata/...;fi - make e2e-image - test/e2e/up.sh - - test/e2e/wait-for-nginx.sh script: - make e2e-test # split builds to avoid job timeouts diff --git a/Gopkg.lock b/Gopkg.lock index c96994361..738495c5d 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -865,6 +865,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "37155c2e5c2212237cff4f2cc220127e9aff6205e2b7cd05af11c42d4d0062ea" + inputs-digest = "262bd1cf8d4735c8d09a6f6d83fed25ad9e1478443a7f6368210e7c3fbb58977" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Makefile b/Makefile index d901f2463..920b65ee6 100644 --- a/Makefile +++ b/Makefile @@ -159,7 +159,7 @@ e2e-image: sub-container-amd64 .PHONY: e2e-test e2e-test: @go test -o e2e-tests -c ./test/e2e - @KUBECONFIG=${HOME}/.kube/config ./e2e-tests -test.parallel 1 + @KUBECONFIG=${HOME}/.kube/config ./e2e-tests -alsologtostderr -test.v -logtostderr -ginkgo.trace .PHONY: cover cover: diff --git a/internal/ingress/controller/store/store_test.go b/internal/ingress/controller/store/store_test.go index 3549197db..c0c9eb6b1 100644 --- a/internal/ingress/controller/store/store_test.go +++ b/internal/ingress/controller/store/store_test.go @@ -24,7 +24,6 @@ import ( "time" "github.com/eapache/channels" - apiv1 "k8s.io/api/core/v1" "k8s.io/api/extensions/v1beta1" extensions "k8s.io/api/extensions/v1beta1" k8sErrors "k8s.io/apimachinery/pkg/api/errors" @@ -67,10 +66,10 @@ func TestStore(t *testing.T) { fs := newFS(t) storer := New(true, - ns.Name, - fmt.Sprintf("%v/config", ns.Name), - fmt.Sprintf("%v/tcp", ns.Name), - fmt.Sprintf("%v/udp", ns.Name), + ns, + fmt.Sprintf("%v/config", ns), + fmt.Sprintf("%v/tcp", ns), + fmt.Sprintf("%v/udp", ns), "", 10*time.Minute, clientSet, @@ -79,7 +78,7 @@ func TestStore(t *testing.T) { storer.Run(stopCh) - key := fmt.Sprintf("%v/anything", ns.Name) + key := fmt.Sprintf("%v/anything", ns) ing, err := storer.GetIngress(key) if err == nil { t.Errorf("expected an error but none returned") @@ -154,10 +153,10 @@ func TestStore(t *testing.T) { fs := newFS(t) storer := New(true, - ns.Name, - fmt.Sprintf("%v/config", ns.Name), - fmt.Sprintf("%v/tcp", ns.Name), - fmt.Sprintf("%v/udp", ns.Name), + ns, + fmt.Sprintf("%v/config", ns), + fmt.Sprintf("%v/tcp", ns), + fmt.Sprintf("%v/udp", ns), "", 10*time.Minute, clientSet, @@ -169,7 +168,7 @@ func TestStore(t *testing.T) { ing, err := ensureIngress(&v1beta1.Ingress{ ObjectMeta: metav1.ObjectMeta{ Name: "dummy", - Namespace: ns.Name, + Namespace: ns, }, Spec: v1beta1.IngressSpec{ Rules: []v1beta1.IngressRule{ @@ -200,7 +199,7 @@ func TestStore(t *testing.T) { _, err = ensureIngress(&v1beta1.Ingress{ ObjectMeta: metav1.ObjectMeta{ Name: "custom-class", - Namespace: ns.Name, + Namespace: ns, Annotations: map[string]string{ "kubernetes.io/ingress.class": "something", }, @@ -295,10 +294,10 @@ func TestStore(t *testing.T) { fs := newFS(t) storer := New(true, - ns.Name, - fmt.Sprintf("%v/config", ns.Name), - fmt.Sprintf("%v/tcp", ns.Name), - fmt.Sprintf("%v/udp", ns.Name), + ns, + fmt.Sprintf("%v/config", ns), + fmt.Sprintf("%v/tcp", ns), + fmt.Sprintf("%v/udp", ns), "", 10*time.Minute, clientSet, @@ -308,12 +307,12 @@ func TestStore(t *testing.T) { storer.Run(stopCh) secretName := "not-referenced" - _, _, _, err = framework.CreateIngressTLSSecret(clientSet, []string{"foo"}, secretName, ns.Name) + _, _, _, err = framework.CreateIngressTLSSecret(clientSet, []string{"foo"}, secretName, ns) if err != nil { t.Errorf("unexpected error creating secret: %v", err) } - err = framework.WaitForSecretInNamespace(clientSet, ns.Name, secretName) + err = framework.WaitForSecretInNamespace(clientSet, ns, secretName) if err != nil { t.Errorf("unexpected error waiting for secret: %v", err) } @@ -328,7 +327,7 @@ func TestStore(t *testing.T) { t.Errorf("expected 0 events of type Delete but %v occurred", del) } - err = clientSet.CoreV1().Secrets(ns.Name).Delete(secretName, &metav1.DeleteOptions{}) + err = clientSet.CoreV1().Secrets(ns).Delete(secretName, &metav1.DeleteOptions{}) if err != nil { t.Errorf("unexpected error deleting secret: %v", err) } @@ -384,10 +383,10 @@ func TestStore(t *testing.T) { fs := newFS(t) storer := New(true, - ns.Name, - fmt.Sprintf("%v/config", ns.Name), - fmt.Sprintf("%v/tcp", ns.Name), - fmt.Sprintf("%v/udp", ns.Name), + ns, + fmt.Sprintf("%v/config", ns), + fmt.Sprintf("%v/tcp", ns), + fmt.Sprintf("%v/udp", ns), "", 10*time.Minute, clientSet, @@ -402,7 +401,7 @@ func TestStore(t *testing.T) { _, err := ensureIngress(&v1beta1.Ingress{ ObjectMeta: metav1.ObjectMeta{ Name: ingressName, - Namespace: ns.Name, + Namespace: ns, }, Spec: v1beta1.IngressSpec{ TLS: []v1beta1.IngressTLS{ @@ -420,17 +419,17 @@ func TestStore(t *testing.T) { t.Errorf("unexpected error creating ingress: %v", err) } - err = framework.WaitForIngressInNamespace(clientSet, ns.Name, ingressName) + err = framework.WaitForIngressInNamespace(clientSet, ns, ingressName) if err != nil { t.Errorf("unexpected error waiting for secret: %v", err) } - _, _, _, err = framework.CreateIngressTLSSecret(clientSet, []string{"foo"}, secretName, ns.Name) + _, _, _, err = framework.CreateIngressTLSSecret(clientSet, []string{"foo"}, secretName, ns) if err != nil { t.Errorf("unexpected error creating secret: %v", err) } - err = framework.WaitForSecretInNamespace(clientSet, ns.Name, secretName) + err = framework.WaitForSecretInNamespace(clientSet, ns, secretName) if err != nil { t.Errorf("unexpected error waiting for secret: %v", err) } @@ -446,7 +445,7 @@ func TestStore(t *testing.T) { t.Errorf("expected 1 events of type Update but %v occurred", upd) } - err = clientSet.CoreV1().Secrets(ns.Name).Delete(secretName, &metav1.DeleteOptions{}) + err = clientSet.CoreV1().Secrets(ns).Delete(secretName, &metav1.DeleteOptions{}) if err != nil { t.Errorf("unexpected error deleting secret: %v", err) } @@ -496,10 +495,10 @@ func TestStore(t *testing.T) { fs := newFS(t) storer := New(true, - ns.Name, - fmt.Sprintf("%v/config", ns.Name), - fmt.Sprintf("%v/tcp", ns.Name), - fmt.Sprintf("%v/udp", ns.Name), + ns, + fmt.Sprintf("%v/config", ns), + fmt.Sprintf("%v/tcp", ns), + fmt.Sprintf("%v/udp", ns), "", 10*time.Minute, clientSet, @@ -514,7 +513,7 @@ func TestStore(t *testing.T) { _, err := ensureIngress(&v1beta1.Ingress{ ObjectMeta: metav1.ObjectMeta{ Name: name, - Namespace: ns.Name, + Namespace: ns, }, Spec: v1beta1.IngressSpec{ TLS: []v1beta1.IngressTLS{ @@ -547,7 +546,7 @@ func TestStore(t *testing.T) { t.Errorf("unexpected error creating ingress: %v", err) } - err = framework.WaitForIngressInNamespace(clientSet, ns.Name, name) + err = framework.WaitForIngressInNamespace(clientSet, ns, name) if err != nil { t.Errorf("unexpected error waiting for ingress: %v", err) } @@ -568,26 +567,26 @@ func TestStore(t *testing.T) { t.Errorf("expected 0 events of type Delete but %v occurred", del) } - _, _, _, err = framework.CreateIngressTLSSecret(clientSet, secretHosts, name, ns.Name) + _, _, _, err = framework.CreateIngressTLSSecret(clientSet, secretHosts, name, ns) if err != nil { t.Errorf("unexpected error creating secret: %v", err) } t.Run("should exists a secret in the local store and filesystem", func(t *testing.T) { - err := framework.WaitForSecretInNamespace(clientSet, ns.Name, name) + err := framework.WaitForSecretInNamespace(clientSet, ns, name) if err != nil { t.Errorf("unexpected error waiting for secret: %v", err) } time.Sleep(5 * time.Second) - pemFile := fmt.Sprintf("%v/%v-%v.pem", file.DefaultSSLDirectory, ns.Name, name) + pemFile := fmt.Sprintf("%v/%v-%v.pem", file.DefaultSSLDirectory, ns, name) err = framework.WaitForFileInFS(pemFile, fs) if err != nil { t.Errorf("unexpected error waiting for file to exist on the file system: %v", err) } - secretName := fmt.Sprintf("%v/%v", ns.Name, name) + secretName := fmt.Sprintf("%v/%v", ns, name) sslCert, err := storer.GetLocalSSLCert(secretName) if err != nil { t.Errorf("unexpected error reading local secret %v: %v", secretName, err) @@ -615,24 +614,24 @@ func TestStore(t *testing.T) { // check invalid secret (missing ca) } -func createNamespace(clientSet *kubernetes.Clientset, t *testing.T) *apiv1.Namespace { +func createNamespace(clientSet *kubernetes.Clientset, t *testing.T) string { t.Log("creating temporal namespace") ns, err := framework.CreateKubeNamespace("store-test", clientSet) if err != nil { t.Errorf("unexpected error creating ingress client: %v", err) } - t.Logf("temporal namespace %v created", ns.Name) + t.Logf("temporal namespace %v created", ns) return ns } -func deleteNamespace(ns *apiv1.Namespace, clientSet *kubernetes.Clientset, t *testing.T) { - t.Logf("deleting temporal namespace %v created", ns.Name) - err := framework.DeleteKubeNamespace(clientSet, ns.Name) +func deleteNamespace(ns string, clientSet *kubernetes.Clientset, t *testing.T) { + t.Logf("deleting temporal namespace %v created", ns) + err := framework.DeleteKubeNamespace(clientSet, ns) if err != nil { t.Errorf("unexpected error creating ingress client: %v", err) } - t.Logf("temporal namespace %v deleted", ns.Name) + t.Logf("temporal namespace %v deleted", ns) } func ensureIngress(ingress *extensions.Ingress, clientSet *kubernetes.Clientset) (*extensions.Ingress, error) { diff --git a/test/e2e/annotations/affinity.go b/test/e2e/annotations/affinity.go index a14edc190..30d31031c 100644 --- a/test/e2e/annotations/affinity.go +++ b/test/e2e/annotations/affinity.go @@ -49,7 +49,7 @@ var _ = framework.IngressNginxDescribe("Annotations - Affinity", func() { ing, err := f.EnsureIngress(&v1beta1.Ingress{ ObjectMeta: metav1.ObjectMeta{ Name: host, - Namespace: f.Namespace.Name, + Namespace: f.IngressController.Namespace, Annotations: map[string]string{ "nginx.ingress.kubernetes.io/affinity": "cookie", "nginx.ingress.kubernetes.io/session-cookie-name": "SERVERID", @@ -81,12 +81,12 @@ var _ = framework.IngressNginxDescribe("Annotations - Affinity", func() { err = f.WaitForNginxServer(host, func(server string) bool { - return strings.Contains(server, "proxy_pass http://sticky-"+f.Namespace.Name+"-http-svc-80;") + return strings.Contains(server, "proxy_pass http://sticky-"+f.IngressController.Namespace+"-http-svc-80;") }) Expect(err).NotTo(HaveOccurred()) resp, _, errs := gorequest.New(). - Get(f.NginxHTTPURL). + Get(f.IngressController.HTTPURL). Set("Host", host). End() @@ -101,7 +101,7 @@ var _ = framework.IngressNginxDescribe("Annotations - Affinity", func() { ing, err := f.EnsureIngress(&v1beta1.Ingress{ ObjectMeta: metav1.ObjectMeta{ Name: host, - Namespace: f.Namespace.Name, + Namespace: f.IngressController.Namespace, Annotations: map[string]string{ "nginx.ingress.kubernetes.io/affinity": "cookie", "nginx.ingress.kubernetes.io/session-cookie-name": "SERVERID", @@ -135,12 +135,12 @@ var _ = framework.IngressNginxDescribe("Annotations - Affinity", func() { err = f.WaitForNginxServer(host, func(server string) bool { - return strings.Contains(server, "proxy_pass http://sticky-"+f.Namespace.Name+"-http-svc-80;") + return strings.Contains(server, "proxy_pass http://sticky-"+f.IngressController.Namespace+"-http-svc-80;") }) Expect(err).NotTo(HaveOccurred()) resp, body, errs := gorequest.New(). - Get(f.NginxHTTPURL). + Get(f.IngressController.HTTPURL). Set("Host", host). End() diff --git a/test/e2e/annotations/alias.go b/test/e2e/annotations/alias.go index b7fcc9c1e..dc8196fb6 100644 --- a/test/e2e/annotations/alias.go +++ b/test/e2e/annotations/alias.go @@ -48,7 +48,7 @@ var _ = framework.IngressNginxDescribe("Annotations - Alias", func() { ing, err := f.EnsureIngress(&v1beta1.Ingress{ ObjectMeta: metav1.ObjectMeta{ Name: host, - Namespace: f.Namespace.Name, + Namespace: f.IngressController.Namespace, }, Spec: v1beta1.IngressSpec{ Rules: []v1beta1.IngressRule{ @@ -83,7 +83,7 @@ var _ = framework.IngressNginxDescribe("Annotations - Alias", func() { Expect(err).NotTo(HaveOccurred()) resp, body, errs := gorequest.New(). - Get(f.NginxHTTPURL). + Get(f.IngressController.HTTPURL). Set("Host", host). End() @@ -92,7 +92,7 @@ var _ = framework.IngressNginxDescribe("Annotations - Alias", func() { Expect(body).Should(ContainSubstring(fmt.Sprintf("host=%v", host))) resp, body, errs = gorequest.New(). - Get(f.NginxHTTPURL). + Get(f.IngressController.HTTPURL). Set("Host", "bar"). End() @@ -106,7 +106,7 @@ var _ = framework.IngressNginxDescribe("Annotations - Alias", func() { ing, err := f.EnsureIngress(&v1beta1.Ingress{ ObjectMeta: metav1.ObjectMeta{ Name: host, - Namespace: f.Namespace.Name, + Namespace: f.IngressController.Namespace, Annotations: map[string]string{ "nginx.ingress.kubernetes.io/server-alias": "bar", }, @@ -146,7 +146,7 @@ var _ = framework.IngressNginxDescribe("Annotations - Alias", func() { hosts := []string{"foo", "bar"} for _, host := range hosts { resp, body, errs := gorequest.New(). - Get(f.NginxHTTPURL). + Get(f.IngressController.HTTPURL). Set("Host", host). End() diff --git a/test/e2e/annotations/auth.go b/test/e2e/annotations/auth.go index dfc64e4b6..0309b37be 100644 --- a/test/e2e/annotations/auth.go +++ b/test/e2e/annotations/auth.go @@ -26,9 +26,7 @@ import ( "github.com/parnurzeal/gorequest" corev1 "k8s.io/api/core/v1" - v1beta1 "k8s.io/api/extensions/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/ingress-nginx/test/e2e/framework" ) @@ -47,9 +45,9 @@ var _ = framework.IngressNginxDescribe("Annotations - Alias", func() { It("should return status code 200 when no authentication is configured", func() { host := "auth" - ing, err := f.EnsureIngress(buildIngress(host, f.Namespace.Name)) + bi, err := f.EnsureIngress(framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, nil)) Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) + Expect(bi).NotTo(BeNil()) err = f.WaitForNginxServer(host, func(server string) bool { @@ -59,7 +57,7 @@ var _ = framework.IngressNginxDescribe("Annotations - Alias", func() { Expect(err).NotTo(HaveOccurred()) resp, body, errs := gorequest.New(). - Get(f.NginxHTTPURL). + Get(f.IngressController.HTTPURL). Set("Host", host). End() @@ -71,7 +69,10 @@ var _ = framework.IngressNginxDescribe("Annotations - Alias", func() { It("should return status code 503 when authentication is configured with an invalid secret", func() { host := "auth" - bi := buildIngress(host, f.Namespace.Name) + bi, err := f.EnsureIngress(framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, nil)) + Expect(err).NotTo(HaveOccurred()) + Expect(bi).NotTo(BeNil()) + bi.Annotations["nginx.ingress.kubernetes.io/auth-type"] = "basic" bi.Annotations["nginx.ingress.kubernetes.io/auth-secret"] = "something" bi.Annotations["nginx.ingress.kubernetes.io/auth-realm"] = "test auth" @@ -88,7 +89,7 @@ var _ = framework.IngressNginxDescribe("Annotations - Alias", func() { Expect(err).NotTo(HaveOccurred()) resp, body, errs := gorequest.New(). - Get(f.NginxHTTPURL). + Get(f.IngressController.HTTPURL). Set("Host", host). End() @@ -100,12 +101,15 @@ var _ = framework.IngressNginxDescribe("Annotations - Alias", func() { It("should return status code 401 when authentication is configured but Authorization header is not configured", func() { host := "auth" - s, err := f.EnsureSecret(buildSecret("foo", "bar", "test", f.Namespace.Name)) + s, err := f.EnsureSecret(buildSecret("foo", "bar", "test", f.IngressController.Namespace)) Expect(err).NotTo(HaveOccurred()) Expect(s).NotTo(BeNil()) Expect(s.ObjectMeta).NotTo(BeNil()) - bi := buildIngress(host, f.Namespace.Name) + bi, err := f.EnsureIngress(framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, nil)) + Expect(err).NotTo(HaveOccurred()) + Expect(bi).NotTo(BeNil()) + bi.Annotations["nginx.ingress.kubernetes.io/auth-type"] = "basic" bi.Annotations["nginx.ingress.kubernetes.io/auth-secret"] = s.Name bi.Annotations["nginx.ingress.kubernetes.io/auth-realm"] = "test auth" @@ -122,7 +126,7 @@ var _ = framework.IngressNginxDescribe("Annotations - Alias", func() { Expect(err).NotTo(HaveOccurred()) resp, body, errs := gorequest.New(). - Get(f.NginxHTTPURL). + Get(f.IngressController.HTTPURL). Set("Host", host). End() @@ -134,12 +138,15 @@ var _ = framework.IngressNginxDescribe("Annotations - Alias", func() { It("should return status code 401 when authentication is configured and Authorization header is sent with invalid credentials", func() { host := "auth" - s, err := f.EnsureSecret(buildSecret("foo", "bar", "test", f.Namespace.Name)) + s, err := f.EnsureSecret(buildSecret("foo", "bar", "test", f.IngressController.Namespace)) Expect(err).NotTo(HaveOccurred()) Expect(s).NotTo(BeNil()) Expect(s.ObjectMeta).NotTo(BeNil()) - bi := buildIngress(host, f.Namespace.Name) + bi, err := f.EnsureIngress(framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, nil)) + Expect(err).NotTo(HaveOccurred()) + Expect(bi).NotTo(BeNil()) + bi.Annotations["nginx.ingress.kubernetes.io/auth-type"] = "basic" bi.Annotations["nginx.ingress.kubernetes.io/auth-secret"] = s.Name bi.Annotations["nginx.ingress.kubernetes.io/auth-realm"] = "test auth" @@ -156,7 +163,7 @@ var _ = framework.IngressNginxDescribe("Annotations - Alias", func() { Expect(err).NotTo(HaveOccurred()) resp, body, errs := gorequest.New(). - Get(f.NginxHTTPURL). + Get(f.IngressController.HTTPURL). Set("Host", host). SetBasicAuth("user", "pass"). End() @@ -169,12 +176,15 @@ var _ = framework.IngressNginxDescribe("Annotations - Alias", func() { It("should return status code 200 when authentication is configured and Authorization header is sent", func() { host := "auth" - s, err := f.EnsureSecret(buildSecret("foo", "bar", "test", f.Namespace.Name)) + s, err := f.EnsureSecret(buildSecret("foo", "bar", "test", f.IngressController.Namespace)) Expect(err).NotTo(HaveOccurred()) Expect(s).NotTo(BeNil()) Expect(s.ObjectMeta).NotTo(BeNil()) - bi := buildIngress(host, f.Namespace.Name) + bi, err := f.EnsureIngress(framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, nil)) + Expect(err).NotTo(HaveOccurred()) + Expect(bi).NotTo(BeNil()) + bi.Annotations["nginx.ingress.kubernetes.io/auth-type"] = "basic" bi.Annotations["nginx.ingress.kubernetes.io/auth-secret"] = s.Name bi.Annotations["nginx.ingress.kubernetes.io/auth-realm"] = "test auth" @@ -191,7 +201,7 @@ var _ = framework.IngressNginxDescribe("Annotations - Alias", func() { Expect(err).NotTo(HaveOccurred()) resp, _, errs := gorequest.New(). - Get(f.NginxHTTPURL). + Get(f.IngressController.HTTPURL). Set("Host", host). SetBasicAuth("foo", "bar"). End() @@ -207,7 +217,7 @@ var _ = framework.IngressNginxDescribe("Annotations - Alias", func() { &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: "test", - Namespace: f.Namespace.Name, + Namespace: f.IngressController.Namespace, }, Data: map[string][]byte{ // invalid content @@ -220,7 +230,10 @@ var _ = framework.IngressNginxDescribe("Annotations - Alias", func() { Expect(s).NotTo(BeNil()) Expect(s.ObjectMeta).NotTo(BeNil()) - bi := buildIngress(host, f.Namespace.Name) + bi, err := f.EnsureIngress(framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, nil)) + Expect(err).NotTo(HaveOccurred()) + Expect(bi).NotTo(BeNil()) + bi.Annotations["nginx.ingress.kubernetes.io/auth-type"] = "basic" bi.Annotations["nginx.ingress.kubernetes.io/auth-secret"] = s.Name bi.Annotations["nginx.ingress.kubernetes.io/auth-realm"] = "test auth" @@ -237,7 +250,7 @@ var _ = framework.IngressNginxDescribe("Annotations - Alias", func() { Expect(err).NotTo(HaveOccurred()) resp, _, errs := gorequest.New(). - Get(f.NginxHTTPURL). + Get(f.IngressController.HTTPURL). Set("Host", host). SetBasicAuth("foo", "bar"). End() @@ -253,36 +266,6 @@ var _ = framework.IngressNginxDescribe("Annotations - Alias", func() { // Auth ok // Auth error -func buildIngress(host, namespace string) *v1beta1.Ingress { - return &v1beta1.Ingress{ - ObjectMeta: metav1.ObjectMeta{ - Name: host, - Namespace: namespace, - Annotations: map[string]string{}, - }, - Spec: v1beta1.IngressSpec{ - Rules: []v1beta1.IngressRule{ - { - Host: host, - IngressRuleValue: v1beta1.IngressRuleValue{ - HTTP: &v1beta1.HTTPIngressRuleValue{ - Paths: []v1beta1.HTTPIngressPath{ - { - Path: "/", - Backend: v1beta1.IngressBackend{ - ServiceName: "http-svc", - ServicePort: intstr.FromInt(80), - }, - }, - }, - }, - }, - }, - }, - }, - } -} - func buildSecret(username, password, name, namespace string) *corev1.Secret { out, err := exec.Command("openssl", "passwd", "-crypt", password).CombinedOutput() encpass := fmt.Sprintf("%v:%s\n", username, out) diff --git a/test/e2e/annotations/luarestywaf.go b/test/e2e/annotations/luarestywaf.go index 8d0111ab7..f43994f8a 100644 --- a/test/e2e/annotations/luarestywaf.go +++ b/test/e2e/annotations/luarestywaf.go @@ -25,10 +25,6 @@ import ( . "github.com/onsi/gomega" "github.com/parnurzeal/gorequest" - v1beta1 "k8s.io/api/extensions/v1beta1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/ingress-nginx/test/e2e/framework" ) @@ -45,7 +41,7 @@ var _ = framework.IngressNginxDescribe("Annotations - lua-resty-waf", func() { host := "foo" createIngress(f, host, map[string]string{"nginx.ingress.kubernetes.io/lua-resty-waf": "active"}) - url := fmt.Sprintf("%s?msg=XSS", f.NginxHTTPURL) + url := fmt.Sprintf("%s?msg=XSS", f.IngressController.HTTPURL) resp, _, errs := gorequest.New(). Get(url). Set("Host", host). @@ -60,7 +56,7 @@ var _ = framework.IngressNginxDescribe("Annotations - lua-resty-waf", func() { "nginx.ingress.kubernetes.io/lua-resty-waf": "active", "nginx.ingress.kubernetes.io/lua-resty-waf-ignore-rulesets": "41000_sqli, 42000_xss"}) - url := fmt.Sprintf("%s?msg=XSS", f.NginxHTTPURL) + url := fmt.Sprintf("%s?msg=XSS", f.IngressController.HTTPURL) resp, _, errs := gorequest.New(). Get(url). Set("Host", host). @@ -88,7 +84,7 @@ var _ = framework.IngressNginxDescribe("Annotations - lua-resty-waf", func() { ]=]`, }) - url := fmt.Sprintf("%s?msg=my-message", f.NginxHTTPURL) + url := fmt.Sprintf("%s?msg=my-message", f.IngressController.HTTPURL) resp, _, errs := gorequest.New(). Get(url). Set("Host", host). @@ -97,7 +93,7 @@ var _ = framework.IngressNginxDescribe("Annotations - lua-resty-waf", func() { Expect(len(errs)).Should(Equal(0)) Expect(resp.StatusCode).Should(Equal(http.StatusOK)) - url = fmt.Sprintf("%s?msg=my-foo-message", f.NginxHTTPURL) + url = fmt.Sprintf("%s?msg=my-foo-message", f.IngressController.HTTPURL) resp, _, errs = gorequest.New(). Get(url). Set("Host", host). @@ -112,7 +108,7 @@ var _ = framework.IngressNginxDescribe("Annotations - lua-resty-waf", func() { host := "foo" createIngress(f, host, map[string]string{}) - url := fmt.Sprintf("%s?msg=XSS", f.NginxHTTPURL) + url := fmt.Sprintf("%s?msg=XSS", f.IngressController.HTTPURL) resp, _, errs := gorequest.New(). Get(url). Set("Host", host). @@ -125,7 +121,7 @@ var _ = framework.IngressNginxDescribe("Annotations - lua-resty-waf", func() { host := "foo" createIngress(f, host, map[string]string{"nginx.ingress.kubernetes.io/lua-resty-waf": "simulate"}) - url := fmt.Sprintf("%s?msg=XSS", f.NginxHTTPURL) + url := fmt.Sprintf("%s?msg=XSS", f.IngressController.HTTPURL) resp, _, errs := gorequest.New(). Get(url). Set("Host", host). @@ -143,33 +139,7 @@ var _ = framework.IngressNginxDescribe("Annotations - lua-resty-waf", func() { }) func createIngress(f *framework.Framework, host string, annotations map[string]string) { - ing, err := f.EnsureIngress(&v1beta1.Ingress{ - ObjectMeta: metav1.ObjectMeta{ - Name: host, - Namespace: f.Namespace.Name, - Annotations: annotations, - }, - Spec: v1beta1.IngressSpec{ - Rules: []v1beta1.IngressRule{ - { - Host: host, - IngressRuleValue: v1beta1.IngressRuleValue{ - HTTP: &v1beta1.HTTPIngressRuleValue{ - Paths: []v1beta1.HTTPIngressPath{ - { - Path: "/", - Backend: v1beta1.IngressBackend{ - ServiceName: "http-svc", - ServicePort: intstr.FromInt(80), - }, - }, - }, - }, - }, - }, - }, - }, - }) + ing, err := f.EnsureIngress(framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, &annotations)) Expect(err).NotTo(HaveOccurred()) Expect(ing).NotTo(BeNil()) @@ -181,7 +151,7 @@ func createIngress(f *framework.Framework, host string, annotations map[string]s Expect(err).NotTo(HaveOccurred()) resp, body, errs := gorequest.New(). - Get(f.NginxHTTPURL). + Get(f.IngressController.HTTPURL). Set("Host", host). End() diff --git a/test/e2e/defaultbackend/default_backend.go b/test/e2e/defaultbackend/default_backend.go index 4e552630d..f957bbfe1 100644 --- a/test/e2e/defaultbackend/default_backend.go +++ b/test/e2e/defaultbackend/default_backend.go @@ -76,9 +76,9 @@ var _ = framework.IngressNginxDescribe("Default backend", func() { switch test.Scheme { case framework.HTTP: - cm = request.CustomMethod(test.Method, f.NginxHTTPURL) + cm = request.CustomMethod(test.Method, f.IngressController.HTTPURL) case framework.HTTPS: - cm = request.CustomMethod(test.Method, f.NginxHTTPSURL) + cm = request.CustomMethod(test.Method, f.IngressController.HTTPSURL) // the default backend uses a self generated certificate cm.Transport = &http.Transport{ TLSClientConfig: &tls.Config{ diff --git a/test/e2e/defaultbackend/ssl.go b/test/e2e/defaultbackend/ssl.go index 49511d491..7dda5ce31 100644 --- a/test/e2e/defaultbackend/ssl.go +++ b/test/e2e/defaultbackend/ssl.go @@ -38,7 +38,7 @@ var _ = framework.IngressNginxDescribe("Default backend - SSL", func() { It("should return a self generated SSL certificate", func() { By("checking SSL Certificate using the NGINX IP address") resp, _, errs := gorequest.New(). - Post(f.NginxHTTPSURL). + Post(f.IngressController.HTTPSURL). TLSClientConfig(&tls.Config{ // the default backend uses a self generated certificate InsecureSkipVerify: true, @@ -53,7 +53,7 @@ var _ = framework.IngressNginxDescribe("Default backend - SSL", func() { By("checking SSL Certificate using the NGINX catch all server") resp, _, errs = gorequest.New(). - Post(f.NginxHTTPSURL). + Post(f.IngressController.HTTPSURL). TLSClientConfig(&tls.Config{ // the default backend uses a self generated certificate InsecureSkipVerify: true, diff --git a/test/e2e/framework/echo.go b/test/e2e/framework/echo.go index 2a825f031..9e0727713 100644 --- a/test/e2e/framework/echo.go +++ b/test/e2e/framework/echo.go @@ -40,7 +40,7 @@ func (f *Framework) NewEchoDeploymentWithReplicas(replicas int32) error { deployment := &extensions.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: "http-svc", - Namespace: f.Namespace.Name, + Namespace: f.IngressController.Namespace, }, Spec: extensions.DeploymentSpec{ Replicas: NewInt32(replicas), @@ -84,7 +84,7 @@ func (f *Framework) NewEchoDeploymentWithReplicas(replicas int32) error { return fmt.Errorf("unexpected error creating deployement for echoserver") } - err = f.WaitForPodsReady(10*time.Second, int(replicas), metav1.ListOptions{ + err = WaitForPodsReady(f.KubeClientSet, 5*time.Minute, int(replicas), f.IngressController.Namespace, metav1.ListOptions{ LabelSelector: fields.SelectorFromSet(fields.Set(d.Spec.Template.ObjectMeta.Labels)).String(), }) if err != nil { @@ -94,7 +94,7 @@ func (f *Framework) NewEchoDeploymentWithReplicas(replicas int32) error { service := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "http-svc", - Namespace: f.Namespace.Name, + Namespace: f.IngressController.Namespace, }, Spec: corev1.ServiceSpec{ Ports: []corev1.ServicePort{ diff --git a/test/e2e/framework/exec.go b/test/e2e/framework/exec.go index 54a191ae4..d587f3533 100644 --- a/test/e2e/framework/exec.go +++ b/test/e2e/framework/exec.go @@ -47,3 +47,26 @@ func (f *Framework) ExecCommand(pod *v1.Pod, command string) (string, error) { return execOut.String(), nil } + +// NewIngressController deploys a new NGINX Ingress controller in a namespace +func (f *Framework) NewIngressController(namespace string) error { + var ( + execOut bytes.Buffer + execErr bytes.Buffer + ) + + cmd := exec.Command("test/e2e/wait-for-nginx.sh", namespace) + cmd.Stdout = &execOut + cmd.Stderr = &execErr + + err := cmd.Run() + if err != nil { + return fmt.Errorf("could not execute: %v", err) + } + + if execErr.Len() > 0 { + return fmt.Errorf("stderr: %v", execErr.String()) + } + + return nil +} diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go index 2af16d1df..f7cf443f0 100644 --- a/test/e2e/framework/framework.go +++ b/test/e2e/framework/framework.go @@ -21,9 +21,11 @@ import ( appsv1beta1 "k8s.io/api/apps/v1beta1" "k8s.io/api/core/v1" + extensions "k8s.io/api/extensions/v1beta1" apiextcs "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" restclient "k8s.io/client-go/rest" @@ -53,27 +55,26 @@ type Framework struct { KubeConfig *restclient.Config APIExtensionsClientSet apiextcs.Interface - // Namespace in which all test resources should reside - Namespace *v1.Namespace - // To make sure that this framework cleans up after itself, no matter what, // we install a Cleanup action before each test and clear it after. If we // should abort, the AfterSuite hook should run all Cleanup actions. cleanupHandle CleanupActionHandle - // Map to hold the default nginx-configuration configmap data before a test run - DefaultNginxConfigMapData map[string]string + IngressController *ingressController +} - NginxHTTPURL string - NginxHTTPSURL string +type ingressController struct { + HTTPURL string + HTTPSURL string + + Namespace string } // NewDefaultFramework makes a new framework and sets up a BeforeEach/AfterEach for // you (you can write additional before/after each functions). func NewDefaultFramework(baseName string) *Framework { f := &Framework{ - BaseName: baseName, - DefaultNginxConfigMapData: nil, + BaseName: baseName, } BeforeEach(f.BeforeEach) @@ -95,43 +96,48 @@ func (f *Framework) BeforeEach() { Expect(err).NotTo(HaveOccurred()) By("Building a namespace api object") - f.Namespace, err = CreateKubeNamespace(f.BaseName, f.KubeClientSet) + ingressNamespace, err := CreateKubeNamespace(f.BaseName, f.KubeClientSet) + Expect(err).NotTo(HaveOccurred()) + + f.IngressController = &ingressController{ + Namespace: ingressNamespace, + } + + By("Starting new ingress controller") + err = f.NewIngressController(f.IngressController.Namespace) + Expect(err).NotTo(HaveOccurred()) + + err = WaitForPodsReady(f.KubeClientSet, 5*time.Minute, 1, f.IngressController.Namespace, metav1.ListOptions{ + LabelSelector: "app=ingress-nginx", + }) Expect(err).NotTo(HaveOccurred()) By("Building NGINX HTTP URL") - f.NginxHTTPURL, err = f.GetNginxURL(HTTP) + HTTPURL, err := f.GetNginxURL(HTTP) Expect(err).NotTo(HaveOccurred()) + f.IngressController.HTTPURL = HTTPURL + By("Building NGINX HTTPS URL") - f.NginxHTTPSURL, err = f.GetNginxURL(HTTPS) + HTTPSURL, err := f.GetNginxURL(HTTPS) Expect(err).NotTo(HaveOccurred()) - By("Persist nginx-configuration configMap Data") - if f.DefaultNginxConfigMapData == nil { - f.DefaultNginxConfigMapData, err = f.GetNginxConfigMapData() - Expect(err).NotTo(HaveOccurred()) - Expect(f.DefaultNginxConfigMapData).NotTo(BeNil()) - } + f.IngressController.HTTPSURL = HTTPSURL + + // we wait for any change in the informers and SSL certificate generation + time.Sleep(5 * time.Second) } // AfterEach deletes the namespace, after reading its events. func (f *Framework) AfterEach() { RemoveCleanupAction(f.cleanupHandle) - By("Deleting test namespace") - err := DeleteKubeNamespace(f.KubeClientSet, f.Namespace.Name) - Expect(err).NotTo(HaveOccurred()) - By("Waiting for test namespace to no longer exist") - err = WaitForNoPodsInNamespace(f.KubeClientSet, f.Namespace.Name) - Expect(err).NotTo(HaveOccurred()) - - By("Reset nginx-configuration configMap to default state") - err = f.SetNginxConfigMapData(f.DefaultNginxConfigMapData) + err := DeleteKubeNamespace(f.KubeClientSet, f.IngressController.Namespace) Expect(err).NotTo(HaveOccurred()) } -// IngressNginxDescribe wrapper function for ginkgo describe. Adds namespacing. +// IngressNginxDescribe wrapper function for ginkgo describe. Adds namespacing. func IngressNginxDescribe(text string, body func()) bool { return Describe("[nginx-ingress] "+text, body) } @@ -148,7 +154,10 @@ func (f *Framework) GetNginxIP() (string, error) { // GetNginxPort returns the number of TCP port where NGINX is running func (f *Framework) GetNginxPort(name string) (int, error) { - s, err := f.KubeClientSet.CoreV1().Services("ingress-nginx").Get("ingress-nginx", metav1.GetOptions{}) + s, err := f.KubeClientSet. + CoreV1(). + Services(f.IngressController.Namespace). + Get("ingress-nginx", metav1.GetOptions{}) if err != nil { return -1, err } @@ -179,45 +188,37 @@ func (f *Framework) GetNginxURL(scheme RequestScheme) (string, error) { // WaitForNginxServer waits until the nginx configuration contains a particular server section func (f *Framework) WaitForNginxServer(name string, matcher func(cfg string) bool) error { - // initial wait to allow the update of the ingress controller - time.Sleep(5 * time.Second) - return wait.PollImmediate(Poll, time.Minute*2, f.matchNginxConditions(name, matcher)) + return wait.Poll(Poll, time.Minute*5, f.matchNginxConditions(name, matcher)) } // WaitForNginxConfiguration waits until the nginx configuration contains a particular configuration func (f *Framework) WaitForNginxConfiguration(matcher func(cfg string) bool) error { - // initial wait to allow the update of the ingress controller - time.Sleep(5 * time.Second) - return wait.PollImmediate(Poll, time.Minute*2, f.matchNginxConditions("", matcher)) + return wait.Poll(Poll, time.Minute*5, f.matchNginxConditions("", matcher)) } // NginxLogs returns the logs of the nginx ingress controller pod running func (f *Framework) NginxLogs() (string, error) { - l, err := f.KubeClientSet.CoreV1().Pods("ingress-nginx").List(metav1.ListOptions{ + l, err := f.KubeClientSet.CoreV1().Pods(f.IngressController.Namespace).List(metav1.ListOptions{ LabelSelector: "app=ingress-nginx", }) if err != nil { return "", err } - if len(l.Items) == 0 { - return "", fmt.Errorf("no nginx ingress controller pod is running") - } - for _, pod := range l.Items { - if strings.HasPrefix(pod.GetName(), "nginx-ingress-controller") && - len(pod.Status.ContainerStatuses) > 0 && - pod.Status.ContainerStatuses[0].State.Running != nil { - return f.Logs(&pod) + if strings.HasPrefix(pod.GetName(), "nginx-ingress-controller") { + if isRunning, err := podRunningReady(&pod); err == nil && isRunning { + return f.Logs(&pod) + } } } - return "", fmt.Errorf("no nginx ingress controller pod is running") + return "", fmt.Errorf("no nginx ingress controller pod is running (logs)") } func (f *Framework) matchNginxConditions(name string, matcher func(cfg string) bool) wait.ConditionFunc { return func() (bool, error) { - l, err := f.KubeClientSet.CoreV1().Pods("ingress-nginx").List(metav1.ListOptions{ + l, err := f.KubeClientSet.CoreV1().Pods(f.IngressController.Namespace).List(metav1.ListOptions{ LabelSelector: "app=ingress-nginx", }) if err != nil { @@ -225,7 +226,7 @@ func (f *Framework) matchNginxConditions(name string, matcher func(cfg string) b } if len(l.Items) == 0 { - return false, fmt.Errorf("no nginx ingress controller pod is running") + return false, nil } var cmd string @@ -239,17 +240,15 @@ func (f *Framework) matchNginxConditions(name string, matcher func(cfg string) b Loop: for _, p := range l.Items { if strings.HasPrefix(p.GetName(), "nginx-ingress-controller") { - for _, cs := range p.Status.ContainerStatuses { - if cs.State.Running != nil && cs.Name == "nginx-ingress-controller" { - pod = &p - break Loop - } + if isRunning, err := podRunningReady(&p); err == nil && isRunning { + pod = &p + break Loop } } } if pod == nil { - return false, fmt.Errorf("no nginx ingress controller pod is running") + return false, nil } o, err := f.ExecCommand(pod, cmd) @@ -259,17 +258,23 @@ func (f *Framework) matchNginxConditions(name string, matcher func(cfg string) b var match bool errs := InterceptGomegaFailures(func() { + if glog.V(10) && len(o) > 0 { + glog.Infof("nginx.conf:\n%v", o) + } + if matcher(o) { match = true } }) - glog.V(2).Infof("Errors waiting for conditions: %v", errs) - if match { return true, nil } + if len(errs) > 0 { + glog.V(2).Infof("Errors waiting for conditions: %v", errs) + } + return false, nil } } @@ -279,7 +284,10 @@ func (f *Framework) getNginxConfigMap() (*v1.ConfigMap, error) { return nil, fmt.Errorf("KubeClientSet not initialized") } - config, err := f.KubeClientSet.CoreV1().ConfigMaps("ingress-nginx").Get("nginx-configuration", metav1.GetOptions{}) + config, err := f.KubeClientSet. + CoreV1(). + ConfigMaps(f.IngressController.Namespace). + Get("nginx-configuration", metav1.GetOptions{}) if err != nil { return nil, err } @@ -314,7 +322,10 @@ func (f *Framework) SetNginxConfigMapData(cmData map[string]string) error { config.Data = cmData - _, err = f.KubeClientSet.CoreV1().ConfigMaps("ingress-nginx").Update(config) + _, err = f.KubeClientSet. + CoreV1(). + ConfigMaps(f.IngressController.Namespace). + Update(config) if err != nil { return err } @@ -343,16 +354,75 @@ func UpdateDeployment(kubeClientSet kubernetes.Interface, namespace string, name return err } - if err = updateFunc(deployment); err != nil { - return err + if updateFunc != nil { + if err := updateFunc(deployment); err != nil { + return err + } } - err = WaitForPodsReady(kubeClientSet, 60*time.Second, replicas, namespace, metav1.ListOptions{ + if *deployment.Spec.Replicas != int32(replicas) { + glog.Infof("updating replica count from %v to %v...", *deployment.Spec.Replicas, replicas) + deployment, err := kubeClientSet.AppsV1beta1().Deployments(namespace).Get(name, metav1.GetOptions{}) + if err != nil { + return err + } + + deployment.Spec.Replicas = NewInt32(int32(replicas)) + _, err = kubeClientSet.AppsV1beta1().Deployments(namespace).Update(deployment) + if err != nil { + return errors.Wrapf(err, "scaling the number of replicas to %v", replicas) + } + } + + err = WaitForPodsReady(kubeClientSet, 5*time.Minute, replicas, namespace, metav1.ListOptions{ LabelSelector: fields.SelectorFromSet(fields.Set(deployment.Spec.Template.ObjectMeta.Labels)).String(), }) if err != nil { - return errors.Wrap(err, "failed to wait for nginx-ingress-controller pods to restart") + return errors.Wrapf(err, "failed to wait for nginx-ingress-controller replica count to be %v", replicas) } return nil } + +// NewSingleIngress creates a simple ingress rule +func NewSingleIngress(name, path, host, ns string, annotations *map[string]string) *extensions.Ingress { + if annotations == nil { + annotations = &map[string]string{} + } + + ing := &extensions.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns, + Annotations: *annotations, + }, + Spec: extensions.IngressSpec{ + TLS: []extensions.IngressTLS{ + { + Hosts: []string{host}, + SecretName: host, + }, + }, + Rules: []extensions.IngressRule{ + { + Host: host, + IngressRuleValue: extensions.IngressRuleValue{ + HTTP: &extensions.HTTPIngressRuleValue{ + Paths: []extensions.HTTPIngressPath{ + { + Path: path, + Backend: extensions.IngressBackend{ + ServiceName: "http-svc", + ServicePort: intstr.FromInt(80), + }, + }, + }, + }, + }, + }, + }, + }, + } + + return ing +} diff --git a/test/e2e/framework/k8s.go b/test/e2e/framework/k8s.go index 198bce78d..db21765ed 100644 --- a/test/e2e/framework/k8s.go +++ b/test/e2e/framework/k8s.go @@ -17,6 +17,7 @@ limitations under the License. package framework import ( + "fmt" "time" api "k8s.io/api/core/v1" @@ -26,6 +27,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" + podutil "k8s.io/kubernetes/pkg/api/v1/pod" ) // EnsureSecret creates a Secret object or returns it if it already exists. @@ -45,10 +47,19 @@ func (f *Framework) EnsureIngress(ingress *extensions.Ingress) (*extensions.Ingr s, err := f.KubeClientSet.ExtensionsV1beta1().Ingresses(ingress.Namespace).Update(ingress) if err != nil { if k8sErrors.IsNotFound(err) { - return f.KubeClientSet.ExtensionsV1beta1().Ingresses(ingress.Namespace).Create(ingress) + s, err = f.KubeClientSet.ExtensionsV1beta1().Ingresses(ingress.Namespace).Create(ingress) + if err != nil { + return nil, err + } + } else { + return nil, err } - return nil, err } + + if s.Annotations == nil { + s.Annotations = make(map[string]string) + } + return s, nil } @@ -76,14 +87,9 @@ func (f *Framework) EnsureDeployment(deployment *extensions.Deployment) (*extens return d, nil } -// WaitForPodsReady waits for a given amount of time until a group of Pods is running in the framework's namespace. -func (f *Framework) WaitForPodsReady(timeout time.Duration, expectedReplicas int, opts metav1.ListOptions) error { - return WaitForPodsReady(f.KubeClientSet, timeout, expectedReplicas, f.Namespace.Name, opts) -} - // WaitForPodsReady waits for a given amount of time until a group of Pods is running in the given namespace. func WaitForPodsReady(kubeClientSet kubernetes.Interface, timeout time.Duration, expectedReplicas int, namespace string, opts metav1.ListOptions) error { - return wait.Poll(time.Second, timeout, func() (bool, error) { + return wait.Poll(2*time.Second, timeout, func() (bool, error) { pl, err := kubeClientSet.CoreV1().Pods(namespace).List(opts) if err != nil { return false, err @@ -91,10 +97,9 @@ func WaitForPodsReady(kubeClientSet kubernetes.Interface, timeout time.Duration, r := 0 for _, p := range pl.Items { - if p.Status.Phase != core.PodRunning { - continue + if isRunning, _ := podRunningReady(&p); isRunning { + r++ } - r++ } if r == expectedReplicas { @@ -104,3 +109,19 @@ func WaitForPodsReady(kubeClientSet kubernetes.Interface, timeout time.Duration, return false, nil }) } + +// podRunningReady checks whether pod p's phase is running and it has a ready +// condition of status true. +func podRunningReady(p *core.Pod) (bool, error) { + // Check the phase is running. + if p.Status.Phase != core.PodRunning { + return false, fmt.Errorf("want pod '%s' on '%s' to be '%v' but was '%v'", + p.ObjectMeta.Name, p.Spec.NodeName, core.PodRunning, p.Status.Phase) + } + // Check the ready condition is true. + if !podutil.IsPodReady(p) { + return false, fmt.Errorf("pod '%s' on '%s' didn't have condition {%v %v}; conditions: %v", + p.ObjectMeta.Name, p.Spec.NodeName, core.PodReady, core.ConditionTrue, p.Status.Conditions) + } + return true, nil +} diff --git a/test/e2e/framework/test_context.go b/test/e2e/framework/test_context.go index 537424c5f..17ba4ef7e 100644 --- a/test/e2e/framework/test_context.go +++ b/test/e2e/framework/test_context.go @@ -46,6 +46,10 @@ func RegisterCommonFlags() { // Randomize specs as well as suites config.GinkgoConfig.RandomizeAllSpecs = true + // Default SlowSpecThreshold is 5 seconds. + // Too low for the kind of operations we need to tests + config.DefaultReporterConfig.SlowSpecThreshold = 20 + flag.StringVar(&TestContext.KubeHost, "kubernetes-host", "http://127.0.0.1:8080", "The kubernetes host, or apiserver, to connect to") flag.StringVar(&TestContext.KubeConfig, "kubernetes-config", os.Getenv(clientcmd.RecommendedConfigPathEnvVar), "Path to config containing embedded authinfo for kubernetes. Default value is from environment variable "+clientcmd.RecommendedConfigPathEnvVar) flag.StringVar(&TestContext.KubeContext, "kubernetes-context", "", "config context to use for kubernetes. If unset, will use value from 'current-context'") diff --git a/test/e2e/framework/util.go b/test/e2e/framework/util.go index 63606d1a2..28c1fbce9 100644 --- a/test/e2e/framework/util.go +++ b/test/e2e/framework/util.go @@ -100,7 +100,7 @@ func LoadConfig(config, context string) (*rest.Config, error) { var RunID = uuid.NewUUID() // CreateKubeNamespace creates a new namespace in the cluster -func CreateKubeNamespace(baseName string, c kubernetes.Interface) (*v1.Namespace, error) { +func CreateKubeNamespace(baseName string, c kubernetes.Interface) (string, error) { ts := time.Now().UnixNano() ns := &v1.Namespace{ ObjectMeta: metav1.ObjectMeta{ @@ -120,9 +120,9 @@ func CreateKubeNamespace(baseName string, c kubernetes.Interface) (*v1.Namespace return true, nil }) if err != nil { - return nil, err + return "", err } - return got, nil + return got.Name, nil } // DeleteKubeNamespace deletes a namespace and all the objects inside @@ -140,7 +140,7 @@ func ExpectNoError(err error, explain ...interface{}) { // WaitForKubeNamespaceNotExist waits until a namespaces is not present in the cluster func WaitForKubeNamespaceNotExist(c kubernetes.Interface, namespace string) error { - return wait.PollImmediate(Poll, time.Minute*2, namespaceNotExist(c, namespace)) + return wait.PollImmediate(Poll, time.Minute*5, namespaceNotExist(c, namespace)) } func namespaceNotExist(c kubernetes.Interface, namespace string) wait.ConditionFunc { @@ -158,7 +158,7 @@ func namespaceNotExist(c kubernetes.Interface, namespace string) wait.ConditionF // WaitForNoPodsInNamespace waits until there are no pods running in a namespace func WaitForNoPodsInNamespace(c kubernetes.Interface, namespace string) error { - return wait.PollImmediate(Poll, time.Minute*2, noPodsInNamespace(c, namespace)) + return wait.PollImmediate(Poll, time.Minute*5, noPodsInNamespace(c, namespace)) } func noPodsInNamespace(c kubernetes.Interface, namespace string) wait.ConditionFunc { diff --git a/test/e2e/lua/dynamic_configuration.go b/test/e2e/lua/dynamic_configuration.go index 15c964cfe..99b9226d6 100644 --- a/test/e2e/lua/dynamic_configuration.go +++ b/test/e2e/lua/dynamic_configuration.go @@ -41,7 +41,7 @@ var _ = framework.IngressNginxDescribe("Dynamic Configuration", func() { f := framework.NewDefaultFramework("dynamic-configuration") BeforeEach(func() { - err := enableDynamicConfiguration(f.KubeClientSet) + err := enableDynamicConfiguration(f.IngressController.Namespace, f.KubeClientSet) Expect(err).NotTo(HaveOccurred()) err = f.NewEchoDeploymentWithReplicas(1) @@ -52,6 +52,8 @@ var _ = framework.IngressNginxDescribe("Dynamic Configuration", func() { Expect(err).NotTo(HaveOccurred()) Expect(ing).NotTo(BeNil()) + time.Sleep(5 * time.Second) + err = f.WaitForNginxServer(host, func(server string) bool { return strings.Contains(server, "proxy_pass http://upstream_balancer;") @@ -59,7 +61,7 @@ var _ = framework.IngressNginxDescribe("Dynamic Configuration", func() { Expect(err).NotTo(HaveOccurred()) resp, _, errs := gorequest.New(). - Get(f.NginxHTTPURL). + Get(f.IngressController.HTTPURL). Set("Host", host). End() Expect(len(errs)).Should(BeNumerically("==", 0)) @@ -72,29 +74,21 @@ var _ = framework.IngressNginxDescribe("Dynamic Configuration", func() { }) AfterEach(func() { - err := disableDynamicConfiguration(f.KubeClientSet) - Expect(err).NotTo(HaveOccurred()) }) Context("when only backends change", func() { It("should handle endpoints only changes", func() { resp, _, errs := gorequest.New(). - Get(fmt.Sprintf("%s?id=endpoints_only_changes", f.NginxHTTPURL)). + Get(fmt.Sprintf("%s?id=endpoints_only_changes", f.IngressController.HTTPURL)). Set("Host", "foo.com"). End() Expect(len(errs)).Should(BeNumerically("==", 0)) Expect(resp.StatusCode).Should(Equal(http.StatusOK)) replicas := 2 - err := framework.UpdateDeployment(f.KubeClientSet, f.Namespace.Name, "http-svc", replicas, - func(deployment *appsv1beta1.Deployment) error { - deployment.Spec.Replicas = framework.NewInt32(int32(replicas)) - _, err := f.KubeClientSet.AppsV1beta1().Deployments(f.Namespace.Name).Update(deployment) - return err - }) + err := framework.UpdateDeployment(f.KubeClientSet, f.IngressController.Namespace, "http-svc", replicas, nil) Expect(err).NotTo(HaveOccurred()) - time.Sleep(5 * time.Second) log, err := f.NginxLogs() Expect(err).ToNot(HaveOccurred()) Expect(log).ToNot(BeEmpty()) @@ -113,18 +107,18 @@ var _ = framework.IngressNginxDescribe("Dynamic Configuration", func() { }) It("should handle annotation changes", func() { - ingress, err := f.KubeClientSet.ExtensionsV1beta1().Ingresses(f.Namespace.Name).Get("foo.com", metav1.GetOptions{}) + ingress, err := f.KubeClientSet.ExtensionsV1beta1().Ingresses(f.IngressController.Namespace).Get("foo.com", metav1.GetOptions{}) Expect(err).ToNot(HaveOccurred()) ingress.ObjectMeta.Annotations["nginx.ingress.kubernetes.io/load-balance"] = "round_robin" - _, err = f.KubeClientSet.ExtensionsV1beta1().Ingresses(f.Namespace.Name).Update(ingress) + _, err = f.KubeClientSet.ExtensionsV1beta1().Ingresses(f.IngressController.Namespace).Update(ingress) Expect(err).ToNot(HaveOccurred()) time.Sleep(5 * time.Second) log, err := f.NginxLogs() Expect(err).ToNot(HaveOccurred()) Expect(log).ToNot(BeEmpty()) - index := strings.Index(log, fmt.Sprintf("reason: 'UPDATE' Ingress %s/foo.com", f.Namespace.Name)) + index := strings.Index(log, fmt.Sprintf("reason: 'UPDATE' Ingress %s/foo.com", f.IngressController.Namespace)) restOfLogs := log[index:] By("POSTing new backends to Lua endpoint") @@ -140,7 +134,7 @@ var _ = framework.IngressNginxDescribe("Dynamic Configuration", func() { }) It("should handle a non backend update", func() { - ingress, err := f.KubeClientSet.ExtensionsV1beta1().Ingresses(f.Namespace.Name).Get("foo.com", metav1.GetOptions{}) + ingress, err := f.KubeClientSet.ExtensionsV1beta1().Ingresses(f.IngressController.Namespace).Get("foo.com", metav1.GetOptions{}) Expect(err).ToNot(HaveOccurred()) ingress.Spec.TLS = []v1beta1.IngressTLS{ @@ -156,7 +150,7 @@ var _ = framework.IngressNginxDescribe("Dynamic Configuration", func() { ingress.Namespace) Expect(err).ToNot(HaveOccurred()) - _, err = f.KubeClientSet.ExtensionsV1beta1().Ingresses(f.Namespace.Name).Update(ingress) + _, err = f.KubeClientSet.ExtensionsV1beta1().Ingresses(f.IngressController.Namespace).Update(ingress) Expect(err).ToNot(HaveOccurred()) time.Sleep(5 * time.Second) @@ -191,31 +185,26 @@ var _ = framework.IngressNginxDescribe("Dynamic Configuration", func() { cookieName := "STICKYSESSION" By("Updating affinity annotation on ingress") - ingress, err := f.KubeClientSet.ExtensionsV1beta1().Ingresses(f.Namespace.Name).Get("foo.com", metav1.GetOptions{}) + ingress, err := f.KubeClientSet.ExtensionsV1beta1().Ingresses(f.IngressController.Namespace).Get("foo.com", metav1.GetOptions{}) Expect(err).ToNot(HaveOccurred()) ingress.ObjectMeta.Annotations = map[string]string{ "nginx.ingress.kubernetes.io/affinity": "cookie", "nginx.ingress.kubernetes.io/session-cookie-name": cookieName, } - _, err = f.KubeClientSet.ExtensionsV1beta1().Ingresses(f.Namespace.Name).Update(ingress) + _, err = f.KubeClientSet.ExtensionsV1beta1().Ingresses(f.IngressController.Namespace).Update(ingress) Expect(err).ToNot(HaveOccurred()) time.Sleep(5 * time.Second) By("Increasing the number of service replicas") - replicas := 2 - err = framework.UpdateDeployment(f.KubeClientSet, f.Namespace.Name, "http-svc", replicas, - func(deployment *appsv1beta1.Deployment) error { - deployment.Spec.Replicas = framework.NewInt32(int32(replicas)) - _, err := f.KubeClientSet.AppsV1beta1().Deployments(f.Namespace.Name).Update(deployment) - return err - }) + err = framework.UpdateDeployment(f.KubeClientSet, f.IngressController.Namespace, "http-svc", 2, nil) Expect(err).NotTo(HaveOccurred()) + time.Sleep(5 * time.Second) By("Making a first request") host := "foo.com" resp, _, errs := gorequest.New(). - Get(f.NginxHTTPURL). + Get(f.IngressController.HTTPURL). Set("Host", host). End() Expect(len(errs)).Should(BeNumerically("==", 0)) @@ -227,7 +216,7 @@ var _ = framework.IngressNginxDescribe("Dynamic Configuration", func() { By("Making a second request with the previous session cookie") resp, _, errs = gorequest.New(). - Get(f.NginxHTTPURL). + Get(f.IngressController.HTTPURL). AddCookie(sessionCookie). Set("Host", host). End() @@ -236,7 +225,7 @@ var _ = framework.IngressNginxDescribe("Dynamic Configuration", func() { By("Making a third request with no cookie") resp, _, errs = gorequest.New(). - Get(f.NginxHTTPURL). + Get(f.IngressController.HTTPURL). Set("Host", host). End() @@ -248,7 +237,7 @@ var _ = framework.IngressNginxDescribe("Dynamic Configuration", func() { Expect(log).ToNot(BeEmpty()) By("Checking that upstreams are sticky when session cookie is used") - index := strings.Index(log, fmt.Sprintf("reason: 'UPDATE' Ingress %s/foo.com", f.Namespace.Name)) + index := strings.Index(log, fmt.Sprintf("reason: 'UPDATE' Ingress %s/foo.com", f.IngressController.Namespace)) reqLogs := log[index:] re := regexp.MustCompile(`\d{1,3}(?:\.\d{1,3}){3}(?::\d{1,5})`) upstreams := re.FindAllString(reqLogs, -1) @@ -259,7 +248,7 @@ var _ = framework.IngressNginxDescribe("Dynamic Configuration", func() { It("should NOT use sticky sessions when a default backend and no ingress rules configured", func() { By("Updating affinity annotation and rules on ingress") - ingress, err := f.KubeClientSet.ExtensionsV1beta1().Ingresses(f.Namespace.Name).Get("foo.com", metav1.GetOptions{}) + ingress, err := f.KubeClientSet.ExtensionsV1beta1().Ingresses(f.IngressController.Namespace).Get("foo.com", metav1.GetOptions{}) Expect(err).ToNot(HaveOccurred()) ingress.Spec = v1beta1.IngressSpec{ Backend: &v1beta1.IngressBackend{ @@ -270,14 +259,14 @@ var _ = framework.IngressNginxDescribe("Dynamic Configuration", func() { ingress.ObjectMeta.Annotations = map[string]string{ "nginx.ingress.kubernetes.io/affinity": "cookie", } - _, err = f.KubeClientSet.ExtensionsV1beta1().Ingresses(f.Namespace.Name).Update(ingress) + _, err = f.KubeClientSet.ExtensionsV1beta1().Ingresses(f.IngressController.Namespace).Update(ingress) Expect(err).ToNot(HaveOccurred()) time.Sleep(5 * time.Second) By("Making a request") host := "foo.com" resp, _, errs := gorequest.New(). - Get(f.NginxHTTPURL). + Get(f.IngressController.HTTPURL). Set("Host", host). End() Expect(len(errs)).Should(BeNumerically("==", 0)) @@ -290,71 +279,27 @@ var _ = framework.IngressNginxDescribe("Dynamic Configuration", func() { }) }) -func enableDynamicConfiguration(kubeClientSet kubernetes.Interface) error { - return framework.UpdateDeployment(kubeClientSet, "ingress-nginx", "nginx-ingress-controller", 1, +func enableDynamicConfiguration(namespace string, kubeClientSet kubernetes.Interface) error { + return framework.UpdateDeployment(kubeClientSet, namespace, "nginx-ingress-controller", 1, func(deployment *appsv1beta1.Deployment) error { args := deployment.Spec.Template.Spec.Containers[0].Args args = append(args, "--enable-dynamic-configuration") deployment.Spec.Template.Spec.Containers[0].Args = args - _, err := kubeClientSet.AppsV1beta1().Deployments("ingress-nginx").Update(deployment) + _, err := kubeClientSet.AppsV1beta1().Deployments(namespace).Update(deployment) if err != nil { return err } - time.Sleep(15 * time.Second) - return nil - }) -} -func disableDynamicConfiguration(kubeClientSet kubernetes.Interface) error { - return framework.UpdateDeployment(kubeClientSet, "ingress-nginx", "nginx-ingress-controller", 1, - func(deployment *appsv1beta1.Deployment) error { - args := deployment.Spec.Template.Spec.Containers[0].Args - var newArgs []string - for _, arg := range args { - if arg != "--enable-dynamic-configuration" { - newArgs = append(newArgs, arg) - } - } - deployment.Spec.Template.Spec.Containers[0].Args = newArgs - _, err := kubeClientSet.AppsV1beta1().Deployments("ingress-nginx").Update(deployment) - if err != nil { - return err - } - time.Sleep(15 * time.Second) + time.Sleep(5 * time.Second) + return nil }) } func ensureIngress(f *framework.Framework, host string) (*extensions.Ingress, error) { - return f.EnsureIngress(&v1beta1.Ingress{ - ObjectMeta: metav1.ObjectMeta{ - Name: host, - Namespace: f.Namespace.Name, - Annotations: map[string]string{ - "nginx.ingress.kubernetes.io/load-balance": "ewma", - }, - }, - Spec: v1beta1.IngressSpec{ - Rules: []v1beta1.IngressRule{ - { - Host: host, - IngressRuleValue: v1beta1.IngressRuleValue{ - HTTP: &v1beta1.HTTPIngressRuleValue{ - Paths: []v1beta1.HTTPIngressPath{ - { - Path: "/", - Backend: v1beta1.IngressBackend{ - ServiceName: "http-svc", - ServicePort: intstr.FromInt(80), - }, - }, - }, - }, - }, - }, - }, - }, - }) + return f.EnsureIngress(framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, &map[string]string{ + "nginx.ingress.kubernetes.io/load-balance": "ewma", + })) } func getCookie(name string, cookies []*http.Cookie) (*http.Cookie, error) { diff --git a/test/e2e/settings/no_auth_locations.go b/test/e2e/settings/no_auth_locations.go index ae801a840..61779db34 100644 --- a/test/e2e/settings/no_auth_locations.go +++ b/test/e2e/settings/no_auth_locations.go @@ -46,7 +46,7 @@ var _ = framework.IngressNginxDescribe("No Auth locations", func() { err := f.NewEchoDeployment() Expect(err).NotTo(HaveOccurred()) - s, err := f.EnsureSecret(buildSecret(username, password, secretName, f.Namespace.Name)) + s, err := f.EnsureSecret(buildSecret(username, password, secretName, f.IngressController.Namespace)) Expect(err).NotTo(HaveOccurred()) Expect(s).NotTo(BeNil()) Expect(s.ObjectMeta).NotTo(BeNil()) @@ -54,7 +54,7 @@ var _ = framework.IngressNginxDescribe("No Auth locations", func() { err = f.UpdateNginxConfigMapData(setting, noAuthPath) Expect(err).NotTo(HaveOccurred()) - bi := buildBasicAuthIngressWithSecondPath(host, f.Namespace.Name, s.Name, noAuthPath) + bi := buildBasicAuthIngressWithSecondPath(host, f.IngressController.Namespace, s.Name, noAuthPath) ing, err := f.EnsureIngress(bi) Expect(err).NotTo(HaveOccurred()) Expect(ing).NotTo(BeNil()) @@ -71,7 +71,7 @@ var _ = framework.IngressNginxDescribe("No Auth locations", func() { Expect(err).NotTo(HaveOccurred()) resp, body, errs := gorequest.New(). - Get(f.NginxHTTPURL). + Get(f.IngressController.HTTPURL). Set("Host", host). End() @@ -88,7 +88,7 @@ var _ = framework.IngressNginxDescribe("No Auth locations", func() { Expect(err).NotTo(HaveOccurred()) resp, _, errs := gorequest.New(). - Get(f.NginxHTTPURL). + Get(f.IngressController.HTTPURL). Set("Host", host). SetBasicAuth(username, password). End() @@ -105,7 +105,7 @@ var _ = framework.IngressNginxDescribe("No Auth locations", func() { Expect(err).NotTo(HaveOccurred()) resp, _, errs := gorequest.New(). - Get(fmt.Sprintf("%s/noauth", f.NginxHTTPURL)). + Get(fmt.Sprintf("%s/noauth", f.IngressController.HTTPURL)). Set("Host", host). End() diff --git a/test/e2e/settings/proxy_protocol.go b/test/e2e/settings/proxy_protocol.go index 94ce2b4b5..645ee1b53 100644 --- a/test/e2e/settings/proxy_protocol.go +++ b/test/e2e/settings/proxy_protocol.go @@ -25,9 +25,6 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "k8s.io/api/extensions/v1beta1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/ingress-nginx/test/e2e/framework" ) @@ -53,34 +50,7 @@ var _ = framework.IngressNginxDescribe("Proxy Protocol", func() { err := f.UpdateNginxConfigMapData(setting, "true") Expect(err).NotTo(HaveOccurred()) - ing, err := f.EnsureIngress(&v1beta1.Ingress{ - ObjectMeta: metav1.ObjectMeta{ - Name: host, - Namespace: f.Namespace.Name, - Annotations: map[string]string{}, - }, - Spec: v1beta1.IngressSpec{ - Rules: []v1beta1.IngressRule{ - { - Host: host, - IngressRuleValue: v1beta1.IngressRuleValue{ - HTTP: &v1beta1.HTTPIngressRuleValue{ - Paths: []v1beta1.HTTPIngressPath{ - { - Path: "/", - Backend: v1beta1.IngressBackend{ - ServiceName: "http-svc", - ServicePort: intstr.FromInt(80), - }, - }, - }, - }, - }, - }, - }, - }, - }) - + ing, err := f.EnsureIngress(framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, nil)) Expect(err).NotTo(HaveOccurred()) Expect(ing).NotTo(BeNil()) diff --git a/test/e2e/settings/server_tokens.go b/test/e2e/settings/server_tokens.go index c5420ffff..3ff80196f 100644 --- a/test/e2e/settings/server_tokens.go +++ b/test/e2e/settings/server_tokens.go @@ -44,34 +44,7 @@ var _ = framework.IngressNginxDescribe("Server Tokens", func() { err := f.UpdateNginxConfigMapData(serverTokens, "false") Expect(err).NotTo(HaveOccurred()) - ing, err := f.EnsureIngress(&v1beta1.Ingress{ - ObjectMeta: metav1.ObjectMeta{ - Name: serverTokens, - Namespace: f.Namespace.Name, - Annotations: map[string]string{}, - }, - Spec: v1beta1.IngressSpec{ - Rules: []v1beta1.IngressRule{ - { - Host: serverTokens, - IngressRuleValue: v1beta1.IngressRuleValue{ - HTTP: &v1beta1.HTTPIngressRuleValue{ - Paths: []v1beta1.HTTPIngressPath{ - { - Path: "/", - Backend: v1beta1.IngressBackend{ - ServiceName: "http-svc", - ServicePort: intstr.FromInt(80), - }, - }, - }, - }, - }, - }, - }, - }, - }) - + ing, err := f.EnsureIngress(framework.NewSingleIngress(serverTokens, "/", serverTokens, f.IngressController.Namespace, nil)) Expect(err).NotTo(HaveOccurred()) Expect(ing).NotTo(BeNil()) @@ -90,7 +63,7 @@ var _ = framework.IngressNginxDescribe("Server Tokens", func() { ing, err := f.EnsureIngress(&v1beta1.Ingress{ ObjectMeta: metav1.ObjectMeta{ Name: serverTokens, - Namespace: f.Namespace.Name, + Namespace: f.IngressController.Namespace, Annotations: map[string]string{}, }, Spec: v1beta1.IngressSpec{ diff --git a/test/e2e/ssl/secret_update.go b/test/e2e/ssl/secret_update.go index 27e28081a..1f744a645 100644 --- a/test/e2e/ssl/secret_update.go +++ b/test/e2e/ssl/secret_update.go @@ -25,9 +25,7 @@ import ( . "github.com/onsi/gomega" "k8s.io/api/core/v1" - "k8s.io/api/extensions/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/ingress-nginx/test/e2e/framework" ) @@ -48,7 +46,7 @@ var _ = framework.IngressNginxDescribe("SSL", func() { dummySecret, err := f.EnsureSecret(&v1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: "dummy", - Namespace: f.Namespace.Name, + Namespace: f.IngressController.Namespace, }, Data: map[string][]byte{ "key": []byte("value"), @@ -56,39 +54,7 @@ var _ = framework.IngressNginxDescribe("SSL", func() { }) Expect(err).NotTo(HaveOccurred()) - ing, err := f.EnsureIngress(&v1beta1.Ingress{ - ObjectMeta: metav1.ObjectMeta{ - Name: host, - Namespace: f.Namespace.Name, - }, - Spec: v1beta1.IngressSpec{ - TLS: []v1beta1.IngressTLS{ - { - Hosts: []string{host}, - SecretName: host, - }, - }, - Rules: []v1beta1.IngressRule{ - { - Host: host, - IngressRuleValue: v1beta1.IngressRuleValue{ - HTTP: &v1beta1.HTTPIngressRuleValue{ - Paths: []v1beta1.HTTPIngressPath{ - { - Path: "/", - Backend: v1beta1.IngressBackend{ - ServiceName: "http-svc", - ServicePort: intstr.FromInt(80), - }, - }, - }, - }, - }, - }, - }, - }, - }) - + ing, err := f.EnsureIngress(framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, nil)) Expect(err).ToNot(HaveOccurred()) Expect(ing).ToNot(BeNil()) @@ -109,12 +75,12 @@ var _ = framework.IngressNginxDescribe("SSL", func() { Expect(err).ToNot(HaveOccurred()) Expect(log).ToNot(BeEmpty()) - Expect(log).ToNot(ContainSubstring(fmt.Sprintf("starting syncing of secret %v/dummy", f.Namespace.Name))) - time.Sleep(30 * time.Second) + Expect(log).ToNot(ContainSubstring(fmt.Sprintf("starting syncing of secret %v/dummy", f.IngressController.Namespace))) + time.Sleep(5 * time.Second) dummySecret.Data["some-key"] = []byte("some value") - f.KubeClientSet.CoreV1().Secrets(f.Namespace.Name).Update(dummySecret) - time.Sleep(30 * time.Second) - Expect(log).ToNot(ContainSubstring(fmt.Sprintf("starting syncing of secret %v/dummy", f.Namespace.Name))) - Expect(log).ToNot(ContainSubstring(fmt.Sprintf("error obtaining PEM from secret %v/dummy", f.Namespace.Name))) + f.KubeClientSet.CoreV1().Secrets(f.IngressController.Namespace).Update(dummySecret) + time.Sleep(5 * time.Second) + Expect(log).ToNot(ContainSubstring(fmt.Sprintf("starting syncing of secret %v/dummy", f.IngressController.Namespace))) + Expect(log).ToNot(ContainSubstring(fmt.Sprintf("error obtaining PEM from secret %v/dummy", f.IngressController.Namespace))) }) }) diff --git a/test/e2e/up.sh b/test/e2e/up.sh index b4f5f9a39..5c5dfaf2f 100755 --- a/test/e2e/up.sh +++ b/test/e2e/up.sh @@ -26,8 +26,8 @@ curl -Lo minikube https://storage.googleapis.com/minikube/releases/v0.25.2/minik sudo mv minikube /usr/local/bin/ echo "starting minikube..." -# Using sync-frequency=5s helps to speed up the tests (during the cleanup of resources inside a namespace) -sudo minikube start --vm-driver=none --kubernetes-version=$KUBERNETES_VERSION --extra-config=kubelet.sync-frequency=5s +# Using a lower value for sync-frequency to speed up the tests (during the cleanup of resources inside a namespace) +sudo minikube start --vm-driver=none --kubernetes-version=$KUBERNETES_VERSION --extra-config=kubelet.sync-frequency=1s minikube update-context diff --git a/test/e2e/wait-for-nginx.sh b/test/e2e/wait-for-nginx.sh index fc309c159..84cf78cbf 100755 --- a/test/e2e/wait-for-nginx.sh +++ b/test/e2e/wait-for-nginx.sh @@ -14,48 +14,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -export JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -echo "deploying NGINX Ingress controller" -cat deploy/namespace.yaml | kubectl apply -f - -cat deploy/default-backend.yaml | kubectl apply -f - -cat deploy/configmap.yaml | kubectl apply -f - -cat deploy/tcp-services-configmap.yaml | kubectl apply -f - -cat deploy/udp-services-configmap.yaml | kubectl apply -f - -cat deploy/without-rbac.yaml | kubectl apply -f - -cat deploy/provider/baremetal/service-nodeport.yaml | kubectl apply -f - +NAMESPACE=$1 -echo "updating image..." -kubectl set image \ - deployments \ - --namespace ingress-nginx \ - --selector app=ingress-nginx \ - nginx-ingress-controller=quay.io/kubernetes-ingress-controller/nginx-ingress-controller:test +echo "deploying NGINX Ingress controller in namespace $NAMESPACE" -sleep 5 - -echo "waiting NGINX ingress pod..." - -function waitForPod() { - until kubectl get pods -n ingress-nginx -l app=ingress-nginx -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; - do - sleep 1; - done -} - -export -f waitForPod - -timeout 30s bash -c waitForPod - -if kubectl get pods -n ingress-nginx -l app=ingress-nginx -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; -then - echo "Kubernetes deployments started" -else - echo "Kubernetes deployments with issues:" - kubectl get pods -n ingress-nginx - - echo "Reason:" - kubectl describe pods -n ingress-nginx - kubectl logs -n ingress-nginx -l app=ingress-nginx - exit 1 -fi +cat $DIR/../manifests/ingress-controller/default-backend.yaml | kubectl create --namespace=$NAMESPACE -f - +cat $DIR/../manifests/ingress-controller/configmap.yaml | kubectl create --namespace=$NAMESPACE -f - +cat $DIR/../manifests/ingress-controller/tcp-services-configmap.yaml | kubectl create --namespace=$NAMESPACE -f - +cat $DIR/../manifests/ingress-controller/udp-services-configmap.yaml | kubectl create --namespace=$NAMESPACE -f - +cat $DIR/../manifests/ingress-controller/with-rbac.yaml | kubectl create --namespace=$NAMESPACE -f - +cat $DIR/../manifests/ingress-controller/service-nodeport.yaml | kubectl create --namespace=$NAMESPACE -f - diff --git a/test/manifests/ingress-controller/configmap.yaml b/test/manifests/ingress-controller/configmap.yaml new file mode 100644 index 000000000..7e9a95052 --- /dev/null +++ b/test/manifests/ingress-controller/configmap.yaml @@ -0,0 +1,6 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: nginx-configuration + labels: + app: ingress-nginx diff --git a/test/manifests/ingress-controller/default-backend.yaml b/test/manifests/ingress-controller/default-backend.yaml new file mode 100644 index 000000000..d4245e3ef --- /dev/null +++ b/test/manifests/ingress-controller/default-backend.yaml @@ -0,0 +1,53 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: default-http-backend + labels: + app: default-http-backend +spec: + replicas: 1 + selector: + matchLabels: + app: default-http-backend + template: + metadata: + labels: + app: default-http-backend + spec: + terminationGracePeriodSeconds: 60 + containers: + - name: default-http-backend + # Any image is permissible as long as: + # 1. It serves a 404 page at / + # 2. It serves 200 on a /healthz endpoint + image: gcr.io/google_containers/defaultbackend:1.4 + livenessProbe: + httpGet: + path: /healthz + port: 8080 + scheme: HTTP + initialDelaySeconds: 30 + timeoutSeconds: 5 + ports: + - containerPort: 8080 + resources: + limits: + cpu: 10m + memory: 20Mi + requests: + cpu: 10m + memory: 20Mi +--- + +apiVersion: v1 +kind: Service +metadata: + name: default-http-backend + labels: + app: default-http-backend +spec: + ports: + - port: 80 + targetPort: 8080 + selector: + app: default-http-backend diff --git a/test/manifests/ingress-controller/rbac.yaml b/test/manifests/ingress-controller/rbac.yaml new file mode 100644 index 000000000..16e131347 --- /dev/null +++ b/test/manifests/ingress-controller/rbac.yaml @@ -0,0 +1,130 @@ +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: nginx-ingress-serviceaccount + +--- + +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: nginx-ingress-clusterrole +rules: + - apiGroups: + - "" + resources: + - configmaps + - endpoints + - nodes + - pods + - secrets + verbs: + - list + - watch + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch + - apiGroups: + - "extensions" + resources: + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - apiGroups: + - "extensions" + resources: + - ingresses/status + verbs: + - update + +--- + +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: Role +metadata: + name: nginx-ingress-role +rules: + - apiGroups: + - "" + resources: + - configmaps + - pods + - secrets + - namespaces + verbs: + - get + - apiGroups: + - "" + resources: + - configmaps + resourceNames: + # Defaults to "-" + # Here: "-" + # This has to be adapted if you change either parameter + # when launching the nginx-ingress-controller. + - "ingress-controller-leader-nginx" + verbs: + - get + - update + - apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - apiGroups: + - "" + resources: + - endpoints + verbs: + - get + +--- + +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: RoleBinding +metadata: + name: nginx-ingress-role-nisa-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: nginx-ingress-role +subjects: + - kind: ServiceAccount + name: nginx-ingress-serviceaccount + +--- + +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: nginx-ingress-clusterrole-nisa-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: nginx-ingress-clusterrole +subjects: + - kind: ServiceAccount + name: nginx-ingress-serviceaccount diff --git a/test/manifests/ingress-controller/service-nodeport.yaml b/test/manifests/ingress-controller/service-nodeport.yaml new file mode 100644 index 000000000..e165fcaa9 --- /dev/null +++ b/test/manifests/ingress-controller/service-nodeport.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: ingress-nginx +spec: + type: NodePort + ports: + - name: http + port: 80 + targetPort: 80 + protocol: TCP + - name: https + port: 443 + targetPort: 443 + protocol: TCP + selector: + app: ingress-nginx diff --git a/test/manifests/ingress-controller/tcp-services-configmap.yaml b/test/manifests/ingress-controller/tcp-services-configmap.yaml new file mode 100644 index 000000000..f78b66128 --- /dev/null +++ b/test/manifests/ingress-controller/tcp-services-configmap.yaml @@ -0,0 +1,4 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: tcp-services diff --git a/test/manifests/ingress-controller/udp-services-configmap.yaml b/test/manifests/ingress-controller/udp-services-configmap.yaml new file mode 100644 index 000000000..1740e7ff1 --- /dev/null +++ b/test/manifests/ingress-controller/udp-services-configmap.yaml @@ -0,0 +1,4 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: udp-services diff --git a/test/manifests/ingress-controller/with-rbac.yaml b/test/manifests/ingress-controller/with-rbac.yaml new file mode 100644 index 000000000..0110b3c38 --- /dev/null +++ b/test/manifests/ingress-controller/with-rbac.yaml @@ -0,0 +1,59 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: nginx-ingress-controller +spec: + replicas: 1 + selector: + matchLabels: + app: ingress-nginx + template: + metadata: + labels: + app: ingress-nginx + spec: + terminationGracePeriodSeconds: 0 + #serviceAccountName: nginx-ingress-serviceaccount + containers: + - name: nginx-ingress-controller + image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.13.0 + args: + - /nginx-ingress-controller + - --default-backend-service=$(POD_NAMESPACE)/default-http-backend + - --configmap=$(POD_NAMESPACE)/nginx-configuration + - --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services + - --udp-services-configmap=$(POD_NAMESPACE)/udp-services + - --annotations-prefix=nginx.ingress.kubernetes.io + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + ports: + - name: http + containerPort: 80 + - name: https + containerPort: 443 + livenessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + readinessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1