From 8a0dec9dd115c3e31eeda5cf73834d5078da306a Mon Sep 17 00:00:00 2001 From: Manuel Alejandro de Brito Fontes Date: Thu, 29 Nov 2018 10:53:48 -0300 Subject: [PATCH 1/3] Add probes to deployments used in e2e tests --- test/e2e/annotations/canary.go | 2 +- test/e2e/framework/deployment.go | 28 +++++++++++++++---- test/e2e/framework/framework.go | 39 ++++---------------------- test/e2e/framework/k8s.go | 47 ++++++++++++++++++++++++++++++++ 4 files changed, 76 insertions(+), 40 deletions(-) diff --git a/test/e2e/annotations/canary.go b/test/e2e/annotations/canary.go index 6201eadb2..4c2d5e056 100644 --- a/test/e2e/annotations/canary.go +++ b/test/e2e/annotations/canary.go @@ -40,7 +40,7 @@ var _ = framework.IngressNginxDescribe("Annotations - canary", func() { f.NewEchoDeployment() // Deployment for canary backend - f.NewDeployment("http-svc-canary", "gcr.io/kubernetes-e2e-test-images/echoserver:2.1", 8080, 1) + f.NewDeployment("http-svc-canary", "gcr.io/kubernetes-e2e-test-images/echoserver:2.2", 8080, 1) }) Context("when canaried by header", func() { diff --git a/test/e2e/framework/deployment.go b/test/e2e/framework/deployment.go index a01e3d333..13e426e7e 100644 --- a/test/e2e/framework/deployment.go +++ b/test/e2e/framework/deployment.go @@ -17,8 +17,6 @@ limitations under the License. package framework import ( - "time" - . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" @@ -36,7 +34,7 @@ func (f *Framework) NewEchoDeployment() { // NewEchoDeploymentWithReplicas creates a new deployment of the echoserver image in a particular namespace. Number of // replicas is configurable func (f *Framework) NewEchoDeploymentWithReplicas(replicas int32) { - f.NewDeployment("http-svc", "gcr.io/kubernetes-e2e-test-images/echoserver:2.1", 8080, replicas) + f.NewDeployment("http-svc", "gcr.io/kubernetes-e2e-test-images/echoserver:2.2", 8080, replicas) } // NewHttpbinDeployment creates a new single replica deployment of the httpbin image in a particular namespace. @@ -46,6 +44,19 @@ func (f *Framework) NewHttpbinDeployment() { // NewDeployment creates a new deployment in a particular namespace. func (f *Framework) NewDeployment(name, image string, port int32, replicas int32) { + probe := &corev1.Probe{ + InitialDelaySeconds: 5, + PeriodSeconds: 10, + SuccessThreshold: 1, + TimeoutSeconds: 1, + Handler: corev1.Handler{ + HTTPGet: &corev1.HTTPGetAction{ + Port: intstr.FromString("http"), + Path: "/", + }, + }, + } + deployment := &extensions.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -77,6 +88,8 @@ func (f *Framework) NewDeployment(name, image string, port int32, replicas int32 ContainerPort: port, }, }, + ReadinessProbe: probe, + LivenessProbe: probe, }, }, }, @@ -88,10 +101,10 @@ func (f *Framework) NewDeployment(name, image string, port int32, replicas int32 Expect(err).NotTo(HaveOccurred(), "failed to create a deployment") Expect(d).NotTo(BeNil(), "expected a deployement but none returned") - err = WaitForPodsReady(f.KubeClientSet, 5*time.Minute, int(replicas), f.IngressController.Namespace, metav1.ListOptions{ + err = WaitForPodsReady(f.KubeClientSet, defaultTimeout, int(replicas), f.IngressController.Namespace, metav1.ListOptions{ LabelSelector: fields.SelectorFromSet(fields.Set(d.Spec.Template.ObjectMeta.Labels)).String(), }) - Expect(err).NotTo(HaveOccurred(), "failed to wait for to become ready") + Expect(err).NotTo(HaveOccurred(), "failed to wait for pods to become ready") service := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ @@ -104,7 +117,7 @@ func (f *Framework) NewDeployment(name, image string, port int32, replicas int32 Name: "http", Port: 80, TargetPort: intstr.FromInt(int(port)), - Protocol: "TCP", + Protocol: corev1.ProtocolTCP, }, }, Selector: map[string]string{ @@ -115,4 +128,7 @@ func (f *Framework) NewDeployment(name, image string, port int32, replicas int32 s := f.EnsureService(service) Expect(s).NotTo(BeNil(), "expected a service but none returned") + + err = WaitForEndpoints(f.KubeClientSet, defaultTimeout, name, f.IngressController.Namespace) + Expect(err).NotTo(HaveOccurred(), "failed to wait for endpoints to become ready") } diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go index d2ec36035..7ca0eee56 100644 --- a/test/e2e/framework/framework.go +++ b/test/e2e/framework/framework.go @@ -112,7 +112,7 @@ func (f *Framework) BeforeEach() { err = f.NewIngressController(f.IngressController.Namespace) Expect(err).NotTo(HaveOccurred()) - err = WaitForPodsReady(f.KubeClientSet, 5*time.Minute, 1, f.IngressController.Namespace, metav1.ListOptions{ + err = WaitForPodsReady(f.KubeClientSet, defaultTimeout, 1, f.IngressController.Namespace, metav1.ListOptions{ LabelSelector: "app.kubernetes.io/name=ingress-nginx", }) Expect(err).NotTo(HaveOccurred()) @@ -197,19 +197,13 @@ func (f *Framework) WaitForNginxConfiguration(matcher func(cfg string) bool) { } func nginxLogs(client kubernetes.Interface, namespace string) (string, error) { - l, err := client.CoreV1().Pods(namespace).List(metav1.ListOptions{ - LabelSelector: "app.kubernetes.io/name=ingress-nginx", - }) + pod, err := getIngressNGINXPod(namespace, client) if err != nil { return "", err } - for _, pod := range l.Items { - if strings.HasPrefix(pod.GetName(), "nginx-ingress-controller") { - if isRunning, err := podRunningReady(&pod); err == nil && isRunning { - return Logs(&pod) - } - } + if isRunning, err := podRunningReady(pod); err == nil && isRunning { + return Logs(pod) } return "", fmt.Errorf("no nginx ingress controller pod is running (logs)") @@ -222,17 +216,11 @@ func (f *Framework) NginxLogs() (string, error) { func (f *Framework) matchNginxConditions(name string, matcher func(cfg string) bool) wait.ConditionFunc { return func() (bool, error) { - l, err := f.KubeClientSet.CoreV1().Pods(f.IngressController.Namespace).List(metav1.ListOptions{ - LabelSelector: "app.kubernetes.io/name=ingress-nginx", - }) + pod, err := getIngressNGINXPod(f.IngressController.Namespace, f.KubeClientSet) if err != nil { return false, err } - if len(l.Items) == 0 { - return false, nil - } - var cmd string if name == "" { cmd = fmt.Sprintf("cat /etc/nginx/nginx.conf") @@ -240,21 +228,6 @@ func (f *Framework) matchNginxConditions(name string, matcher func(cfg string) b cmd = fmt.Sprintf("cat /etc/nginx/nginx.conf | awk '/## start server %v/,/## end server %v/'", name, name) } - var pod *v1.Pod - - for _, p := range l.Items { - if strings.HasPrefix(p.GetName(), "nginx-ingress-controller") { - if isRunning, err := podRunningReady(&p); err == nil && isRunning { - pod = &p - break - } - } - } - - if pod == nil { - return false, nil - } - o, err := f.ExecCommand(pod, cmd) if err != nil { return false, err @@ -369,7 +342,7 @@ func UpdateDeployment(kubeClientSet kubernetes.Interface, namespace string, name } } - err = WaitForPodsReady(kubeClientSet, 5*time.Minute, replicas, namespace, metav1.ListOptions{ + err = WaitForPodsReady(kubeClientSet, defaultTimeout, replicas, namespace, metav1.ListOptions{ LabelSelector: fields.SelectorFromSet(fields.Set(deployment.Spec.Template.ObjectMeta.Labels)).String(), }) if err != nil { diff --git a/test/e2e/framework/k8s.go b/test/e2e/framework/k8s.go index 5a40fe100..39812969d 100644 --- a/test/e2e/framework/k8s.go +++ b/test/e2e/framework/k8s.go @@ -18,6 +18,7 @@ package framework import ( "fmt" + "strings" "time" . "github.com/onsi/gomega" @@ -141,6 +142,22 @@ func WaitForPodsReady(kubeClientSet kubernetes.Interface, timeout time.Duration, }) } +// WaitForEndpoints waits for a given amount of time until an endpoint contains. +func WaitForEndpoints(kubeClientSet kubernetes.Interface, timeout time.Duration, name, ns string) error { + return wait.Poll(2*time.Second, timeout, func() (bool, error) { + endpoint, err := kubeClientSet.CoreV1().Endpoints(ns).Get(name, metav1.GetOptions{}) + if k8sErrors.IsNotFound(err) { + return false, err + } + Expect(err).NotTo(HaveOccurred()) + if len(endpoint.Subsets) == 0 || len(endpoint.Subsets[0].Addresses) == 0 { + return false, err + } + + return true, nil + }) +} + // podRunningReady checks whether pod p's phase is running and it has a ready // condition of status true. func podRunningReady(p *core.Pod) (bool, error) { @@ -156,3 +173,33 @@ func podRunningReady(p *core.Pod) (bool, error) { } return true, nil } + +func getIngressNGINXPod(ns string, kubeClientSet kubernetes.Interface) (*core.Pod, error) { + l, err := kubeClientSet.CoreV1().Pods(ns).List(metav1.ListOptions{ + LabelSelector: "app.kubernetes.io/name=ingress-nginx", + }) + if err != nil { + return nil, err + } + + if len(l.Items) == 0 { + return nil, fmt.Errorf("There is no ingress-nginx pods running in namespace %v", ns) + } + + var pod *core.Pod + + for _, p := range l.Items { + if strings.HasPrefix(p.GetName(), "nginx-ingress-controller") { + if isRunning, err := podRunningReady(&p); err == nil && isRunning { + pod = &p + break + } + } + } + + if pod == nil { + return nil, fmt.Errorf("There is no ingress-nginx pods running in namespace %v", ns) + } + + return pod, nil +} From e192bbaf14538d5f88a53ce5d2d116997c485692 Mon Sep 17 00:00:00 2001 From: Manuel Alejandro de Brito Fontes Date: Thu, 29 Nov 2018 12:15:37 -0300 Subject: [PATCH 2/3] Wait for endpoints instead of pods --- test/e2e/annotations/proxy.go | 31 +++++++++++++++---------------- test/e2e/framework/deployment.go | 6 ------ 2 files changed, 15 insertions(+), 22 deletions(-) diff --git a/test/e2e/annotations/proxy.go b/test/e2e/annotations/proxy.go index 83f8df1b9..b7ce91d38 100644 --- a/test/e2e/annotations/proxy.go +++ b/test/e2e/annotations/proxy.go @@ -27,6 +27,7 @@ import ( var _ = framework.IngressNginxDescribe("Annotations - Proxy", func() { f := framework.NewDefaultFramework("proxy") + host := "proxy.foo.com" BeforeEach(func() { f.NewEchoDeploymentWithReplicas(2) @@ -36,7 +37,6 @@ var _ = framework.IngressNginxDescribe("Annotations - Proxy", func() { }) It("should set proxy_redirect to off", func() { - host := "proxy.foo.com" annotations := map[string]string{ "nginx.ingress.kubernetes.io/proxy-redirect-from": "off", "nginx.ingress.kubernetes.io/proxy-redirect-to": "goodbye.com", @@ -52,7 +52,6 @@ var _ = framework.IngressNginxDescribe("Annotations - Proxy", func() { }) It("should set proxy_redirect to default", func() { - host := "proxy.foo.com" annotations := map[string]string{ "nginx.ingress.kubernetes.io/proxy-redirect-from": "default", "nginx.ingress.kubernetes.io/proxy-redirect-to": "goodbye.com", @@ -68,7 +67,6 @@ var _ = framework.IngressNginxDescribe("Annotations - Proxy", func() { }) It("should set proxy_redirect to hello.com goodbye.com", func() { - host := "proxy.foo.com" annotations := map[string]string{ "nginx.ingress.kubernetes.io/proxy-redirect-from": "hello.com", "nginx.ingress.kubernetes.io/proxy-redirect-to": "goodbye.com", @@ -84,7 +82,6 @@ var _ = framework.IngressNginxDescribe("Annotations - Proxy", func() { }) It("should set proxy client-max-body-size to 8m", func() { - host := "proxy.foo.com" annotations := map[string]string{ "nginx.ingress.kubernetes.io/proxy-body-size": "8m", } @@ -99,7 +96,6 @@ var _ = framework.IngressNginxDescribe("Annotations - Proxy", func() { }) It("should not set proxy client-max-body-size to incorrect value", func() { - host := "proxy.foo.com" annotations := map[string]string{ "nginx.ingress.kubernetes.io/proxy-body-size": "15r", } @@ -114,7 +110,6 @@ var _ = framework.IngressNginxDescribe("Annotations - Proxy", func() { }) It("should set valid proxy timeouts", func() { - host := "proxy.foo.com" annotations := map[string]string{ "nginx.ingress.kubernetes.io/proxy-connect-timeout": "50", "nginx.ingress.kubernetes.io/proxy-send-timeout": "20", @@ -126,12 +121,13 @@ var _ = framework.IngressNginxDescribe("Annotations - Proxy", func() { f.WaitForNginxServer(host, func(server string) bool { - return strings.Contains(server, "proxy_connect_timeout 50s;") && strings.Contains(server, "proxy_send_timeout 20s;") && strings.Contains(server, "proxy_read_timeout 20s;") + return strings.Contains(server, "proxy_connect_timeout 50s;") && + strings.Contains(server, "proxy_send_timeout 20s;") && + strings.Contains(server, "proxy_read_timeout 20s;") }) }) It("should not set invalid proxy timeouts", func() { - host := "proxy.foo.com" annotations := map[string]string{ "nginx.ingress.kubernetes.io/proxy-connect-timeout": "50k", "nginx.ingress.kubernetes.io/proxy-send-timeout": "20k", @@ -143,12 +139,13 @@ var _ = framework.IngressNginxDescribe("Annotations - Proxy", func() { f.WaitForNginxServer(host, func(server string) bool { - return !strings.Contains(server, "proxy_connect_timeout 50ks;") && !strings.Contains(server, "proxy_send_timeout 20ks;") && !strings.Contains(server, "proxy_read_timeout 60s;") + return !strings.Contains(server, "proxy_connect_timeout 50ks;") && + !strings.Contains(server, "proxy_send_timeout 20ks;") && + !strings.Contains(server, "proxy_read_timeout 60s;") }) }) It("should turn on proxy-buffering", func() { - host := "proxy.foo.com" annotations := map[string]string{ "nginx.ingress.kubernetes.io/proxy-buffering": "on", "nginx.ingress.kubernetes.io/proxy-buffer-size": "8k", @@ -159,12 +156,14 @@ var _ = framework.IngressNginxDescribe("Annotations - Proxy", func() { f.WaitForNginxServer(host, func(server string) bool { - return strings.Contains(server, "proxy_buffering on;") && strings.Contains(server, "proxy_buffer_size 8k;") && strings.Contains(server, "proxy_buffers 4 8k;") && strings.Contains(server, "proxy_request_buffering on;") + return strings.Contains(server, "proxy_buffering on;") && + strings.Contains(server, "proxy_buffer_size 8k;") && + strings.Contains(server, "proxy_buffers 4 8k;") && + strings.Contains(server, "proxy_request_buffering on;") }) }) It("should turn off proxy-request-buffering", func() { - host := "proxy.foo.com" annotations := map[string]string{ "nginx.ingress.kubernetes.io/proxy-request-buffering": "off", } @@ -179,7 +178,6 @@ var _ = framework.IngressNginxDescribe("Annotations - Proxy", func() { }) It("should build proxy next upstream", func() { - host := "proxy.foo.com" annotations := map[string]string{ "nginx.ingress.kubernetes.io/proxy-next-upstream": "error timeout http_502", "nginx.ingress.kubernetes.io/proxy-next-upstream-tries": "5", @@ -190,12 +188,12 @@ var _ = framework.IngressNginxDescribe("Annotations - Proxy", func() { f.WaitForNginxServer(host, func(server string) bool { - return strings.Contains(server, "proxy_next_upstream error timeout http_502;") && strings.Contains(server, "proxy_next_upstream_tries 5;") + return strings.Contains(server, "proxy_next_upstream error timeout http_502;") && + strings.Contains(server, "proxy_next_upstream_tries 5;") }) }) It("should setup proxy cookies", func() { - host := "proxy.foo.com" annotations := map[string]string{ "nginx.ingress.kubernetes.io/proxy-cookie-domain": "localhost example.org", "nginx.ingress.kubernetes.io/proxy-cookie-path": "/one/ /", @@ -206,7 +204,8 @@ var _ = framework.IngressNginxDescribe("Annotations - Proxy", func() { f.WaitForNginxServer(host, func(server string) bool { - return strings.Contains(server, "proxy_cookie_domain localhost example.org;") && strings.Contains(server, "proxy_cookie_path /one/ /;") + return strings.Contains(server, "proxy_cookie_domain localhost example.org;") && + strings.Contains(server, "proxy_cookie_path /one/ /;") }) }) }) diff --git a/test/e2e/framework/deployment.go b/test/e2e/framework/deployment.go index 13e426e7e..f44932207 100644 --- a/test/e2e/framework/deployment.go +++ b/test/e2e/framework/deployment.go @@ -22,7 +22,6 @@ import ( corev1 "k8s.io/api/core/v1" extensions "k8s.io/api/extensions/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/util/intstr" ) @@ -101,11 +100,6 @@ func (f *Framework) NewDeployment(name, image string, port int32, replicas int32 Expect(err).NotTo(HaveOccurred(), "failed to create a deployment") Expect(d).NotTo(BeNil(), "expected a deployement but none returned") - err = WaitForPodsReady(f.KubeClientSet, defaultTimeout, int(replicas), f.IngressController.Namespace, metav1.ListOptions{ - LabelSelector: fields.SelectorFromSet(fields.Set(d.Spec.Template.ObjectMeta.Labels)).String(), - }) - Expect(err).NotTo(HaveOccurred(), "failed to wait for pods to become ready") - service := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: name, From 430045d249faae5fa666238b6026e5a909279b7e Mon Sep 17 00:00:00 2001 From: Manuel Alejandro de Brito Fontes Date: Thu, 29 Nov 2018 13:05:49 -0300 Subject: [PATCH 3/3] Fix error in e2e test proxy setting check --- test/e2e/annotations/proxy.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/annotations/proxy.go b/test/e2e/annotations/proxy.go index b7ce91d38..f6169f4a3 100644 --- a/test/e2e/annotations/proxy.go +++ b/test/e2e/annotations/proxy.go @@ -141,7 +141,7 @@ var _ = framework.IngressNginxDescribe("Annotations - Proxy", func() { func(server string) bool { return !strings.Contains(server, "proxy_connect_timeout 50ks;") && !strings.Contains(server, "proxy_send_timeout 20ks;") && - !strings.Contains(server, "proxy_read_timeout 60s;") + !strings.Contains(server, "proxy_read_timeout 20ks;") }) })