Merge pull request #3488 from aledbf/wait-endpoints

Add probes to deployments used in e2e tests

test/e2e/annotations/canary.go

@@ -40,7 +40,7 @@ var _ = framework.IngressNginxDescribe("Annotations - canary", func() {
 		f.NewEchoDeployment()
 
 		// Deployment for canary backend
-		f.NewDeployment("http-svc-canary", "gcr.io/kubernetes-e2e-test-images/echoserver:2.1", 8080, 1)
+		f.NewDeployment("http-svc-canary", "gcr.io/kubernetes-e2e-test-images/echoserver:2.2", 8080, 1)
 	})
 
 	Context("when canaried by header", func() {

test/e2e/annotations/proxy.go

@@ -27,6 +27,7 @@ import (
 
 var _ = framework.IngressNginxDescribe("Annotations - Proxy", func() {
 	f := framework.NewDefaultFramework("proxy")
+	host := "proxy.foo.com"
 
 	BeforeEach(func() {
 		f.NewEchoDeploymentWithReplicas(2)
@@ -36,7 +37,6 @@ var _ = framework.IngressNginxDescribe("Annotations - Proxy", func() {
 	})
 
 	It("should set proxy_redirect to off", func() {
-		host := "proxy.foo.com"
 		annotations := map[string]string{
 			"nginx.ingress.kubernetes.io/proxy-redirect-from": "off",
 			"nginx.ingress.kubernetes.io/proxy-redirect-to":   "goodbye.com",
@@ -52,7 +52,6 @@ var _ = framework.IngressNginxDescribe("Annotations - Proxy", func() {
 	})
 
 	It("should set proxy_redirect to default", func() {
-		host := "proxy.foo.com"
 		annotations := map[string]string{
 			"nginx.ingress.kubernetes.io/proxy-redirect-from": "default",
 			"nginx.ingress.kubernetes.io/proxy-redirect-to":   "goodbye.com",
@@ -68,7 +67,6 @@ var _ = framework.IngressNginxDescribe("Annotations - Proxy", func() {
 	})
 
 	It("should set proxy_redirect to hello.com goodbye.com", func() {
-		host := "proxy.foo.com"
 		annotations := map[string]string{
 			"nginx.ingress.kubernetes.io/proxy-redirect-from": "hello.com",
 			"nginx.ingress.kubernetes.io/proxy-redirect-to":   "goodbye.com",
@@ -84,7 +82,6 @@ var _ = framework.IngressNginxDescribe("Annotations - Proxy", func() {
 	})
 
 	It("should set proxy client-max-body-size to 8m", func() {
-		host := "proxy.foo.com"
 		annotations := map[string]string{
 			"nginx.ingress.kubernetes.io/proxy-body-size": "8m",
 		}
@@ -99,7 +96,6 @@ var _ = framework.IngressNginxDescribe("Annotations - Proxy", func() {
 	})
 
 	It("should not set proxy client-max-body-size to incorrect value", func() {
-		host := "proxy.foo.com"
 		annotations := map[string]string{
 			"nginx.ingress.kubernetes.io/proxy-body-size": "15r",
 		}
@@ -114,7 +110,6 @@ var _ = framework.IngressNginxDescribe("Annotations - Proxy", func() {
 	})
 
 	It("should set valid proxy timeouts", func() {
-		host := "proxy.foo.com"
 		annotations := map[string]string{
 			"nginx.ingress.kubernetes.io/proxy-connect-timeout": "50",
 			"nginx.ingress.kubernetes.io/proxy-send-timeout":    "20",
@@ -126,12 +121,13 @@ var _ = framework.IngressNginxDescribe("Annotations - Proxy", func() {
 
 		f.WaitForNginxServer(host,
 			func(server string) bool {
-				return strings.Contains(server, "proxy_connect_timeout 50s;") && strings.Contains(server, "proxy_send_timeout 20s;") && strings.Contains(server, "proxy_read_timeout 20s;")
+				return strings.Contains(server, "proxy_connect_timeout 50s;") &&
+					strings.Contains(server, "proxy_send_timeout 20s;") &&
+					strings.Contains(server, "proxy_read_timeout 20s;")
 			})
 	})
 
 	It("should not set invalid proxy timeouts", func() {
-		host := "proxy.foo.com"
 		annotations := map[string]string{
 			"nginx.ingress.kubernetes.io/proxy-connect-timeout": "50k",
 			"nginx.ingress.kubernetes.io/proxy-send-timeout":    "20k",
@@ -143,12 +139,13 @@ var _ = framework.IngressNginxDescribe("Annotations - Proxy", func() {
 
 		f.WaitForNginxServer(host,
 			func(server string) bool {
-				return !strings.Contains(server, "proxy_connect_timeout 50ks;") && !strings.Contains(server, "proxy_send_timeout 20ks;") && !strings.Contains(server, "proxy_read_timeout 60s;")
+				return !strings.Contains(server, "proxy_connect_timeout 50ks;") &&
+					!strings.Contains(server, "proxy_send_timeout 20ks;") &&
+					!strings.Contains(server, "proxy_read_timeout 20ks;")
 			})
 	})
 
 	It("should turn on proxy-buffering", func() {
-		host := "proxy.foo.com"
 		annotations := map[string]string{
 			"nginx.ingress.kubernetes.io/proxy-buffering":   "on",
 			"nginx.ingress.kubernetes.io/proxy-buffer-size": "8k",
@@ -159,12 +156,14 @@ var _ = framework.IngressNginxDescribe("Annotations - Proxy", func() {
 
 		f.WaitForNginxServer(host,
 			func(server string) bool {
-				return strings.Contains(server, "proxy_buffering on;") && strings.Contains(server, "proxy_buffer_size 8k;") && strings.Contains(server, "proxy_buffers 4 8k;") && strings.Contains(server, "proxy_request_buffering on;")
+				return strings.Contains(server, "proxy_buffering on;") &&
+					strings.Contains(server, "proxy_buffer_size 8k;") &&
+					strings.Contains(server, "proxy_buffers 4 8k;") &&
+					strings.Contains(server, "proxy_request_buffering on;")
 			})
 	})
 
 	It("should turn off proxy-request-buffering", func() {
-		host := "proxy.foo.com"
 		annotations := map[string]string{
 			"nginx.ingress.kubernetes.io/proxy-request-buffering": "off",
 		}
@@ -179,7 +178,6 @@ var _ = framework.IngressNginxDescribe("Annotations - Proxy", func() {
 	})
 
 	It("should build proxy next upstream", func() {
-		host := "proxy.foo.com"
 		annotations := map[string]string{
 			"nginx.ingress.kubernetes.io/proxy-next-upstream":       "error timeout http_502",
 			"nginx.ingress.kubernetes.io/proxy-next-upstream-tries": "5",
@@ -190,12 +188,12 @@ var _ = framework.IngressNginxDescribe("Annotations - Proxy", func() {
 
 		f.WaitForNginxServer(host,
 			func(server string) bool {
-				return strings.Contains(server, "proxy_next_upstream error timeout http_502;") && strings.Contains(server, "proxy_next_upstream_tries 5;")
+				return strings.Contains(server, "proxy_next_upstream error timeout http_502;") &&
+					strings.Contains(server, "proxy_next_upstream_tries 5;")
 			})
 	})
 
 	It("should setup proxy cookies", func() {
-		host := "proxy.foo.com"
 		annotations := map[string]string{
 			"nginx.ingress.kubernetes.io/proxy-cookie-domain": "localhost example.org",
 			"nginx.ingress.kubernetes.io/proxy-cookie-path":   "/one/ /",
@@ -206,7 +204,8 @@ var _ = framework.IngressNginxDescribe("Annotations - Proxy", func() {
 
 		f.WaitForNginxServer(host,
 			func(server string) bool {
-				return strings.Contains(server, "proxy_cookie_domain localhost example.org;") && strings.Contains(server, "proxy_cookie_path /one/ /;")
+				return strings.Contains(server, "proxy_cookie_domain localhost example.org;") &&
+					strings.Contains(server, "proxy_cookie_path /one/ /;")
 			})
 	})
 })

test/e2e/framework/deployment.go

@@ -17,14 +17,11 @@ limitations under the License.
 package framework
 
 import (
-	"time"
-
 	. "github.com/onsi/gomega"
 
 	corev1 "k8s.io/api/core/v1"
 	extensions "k8s.io/api/extensions/v1beta1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/apimachinery/pkg/util/intstr"
 )
 
@@ -36,7 +33,7 @@ func (f *Framework) NewEchoDeployment() {
 // NewEchoDeploymentWithReplicas creates a new deployment of the echoserver image in a particular namespace. Number of
 // replicas is configurable
 func (f *Framework) NewEchoDeploymentWithReplicas(replicas int32) {
-	f.NewDeployment("http-svc", "gcr.io/kubernetes-e2e-test-images/echoserver:2.1", 8080, replicas)
+	f.NewDeployment("http-svc", "gcr.io/kubernetes-e2e-test-images/echoserver:2.2", 8080, replicas)
 }
 
 // NewHttpbinDeployment creates a new single replica deployment of the httpbin image in a particular namespace.
@@ -46,6 +43,19 @@ func (f *Framework) NewHttpbinDeployment() {
 
 // NewDeployment creates a new deployment in a particular namespace.
 func (f *Framework) NewDeployment(name, image string, port int32, replicas int32) {
+	probe := &corev1.Probe{
+		InitialDelaySeconds: 5,
+		PeriodSeconds:       10,
+		SuccessThreshold:    1,
+		TimeoutSeconds:      1,
+		Handler: corev1.Handler{
+			HTTPGet: &corev1.HTTPGetAction{
+				Port: intstr.FromString("http"),
+				Path: "/",
+			},
+		},
+	}
+
 	deployment := &extensions.Deployment{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      name,
@@ -77,6 +87,8 @@ func (f *Framework) NewDeployment(name, image string, port int32, replicas int32
 									ContainerPort: port,
 								},
 							},
+							ReadinessProbe: probe,
+							LivenessProbe:  probe,
 						},
 					},
 				},
@@ -88,11 +100,6 @@ func (f *Framework) NewDeployment(name, image string, port int32, replicas int32
 	Expect(err).NotTo(HaveOccurred(), "failed to create a deployment")
 	Expect(d).NotTo(BeNil(), "expected a deployement but none returned")
 
-	err = WaitForPodsReady(f.KubeClientSet, 5*time.Minute, int(replicas), f.IngressController.Namespace, metav1.ListOptions{
-		LabelSelector: fields.SelectorFromSet(fields.Set(d.Spec.Template.ObjectMeta.Labels)).String(),
-	})
-	Expect(err).NotTo(HaveOccurred(), "failed to wait for to become ready")
-
 	service := &corev1.Service{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      name,
@@ -104,7 +111,7 @@ func (f *Framework) NewDeployment(name, image string, port int32, replicas int32
 					Name:       "http",
 					Port:       80,
 					TargetPort: intstr.FromInt(int(port)),
-					Protocol:   "TCP",
+					Protocol:   corev1.ProtocolTCP,
 				},
 			},
 			Selector: map[string]string{
@@ -115,4 +122,7 @@ func (f *Framework) NewDeployment(name, image string, port int32, replicas int32
 
 	s := f.EnsureService(service)
 	Expect(s).NotTo(BeNil(), "expected a service but none returned")
+
+	err = WaitForEndpoints(f.KubeClientSet, defaultTimeout, name, f.IngressController.Namespace)
+	Expect(err).NotTo(HaveOccurred(), "failed to wait for endpoints to become ready")
 }

test/e2e/framework/framework.go

@@ -112,7 +112,7 @@ func (f *Framework) BeforeEach() {
 	err = f.NewIngressController(f.IngressController.Namespace)
 	Expect(err).NotTo(HaveOccurred())
 
-	err = WaitForPodsReady(f.KubeClientSet, 5*time.Minute, 1, f.IngressController.Namespace, metav1.ListOptions{
+	err = WaitForPodsReady(f.KubeClientSet, defaultTimeout, 1, f.IngressController.Namespace, metav1.ListOptions{
 		LabelSelector: "app.kubernetes.io/name=ingress-nginx",
 	})
 	Expect(err).NotTo(HaveOccurred())
@@ -197,19 +197,13 @@ func (f *Framework) WaitForNginxConfiguration(matcher func(cfg string) bool) {
 }
 
 func nginxLogs(client kubernetes.Interface, namespace string) (string, error) {
-	l, err := client.CoreV1().Pods(namespace).List(metav1.ListOptions{
+	pod, err := getIngressNGINXPod(namespace, client)
-		LabelSelector: "app.kubernetes.io/name=ingress-nginx",
-	})
 	if err != nil {
 		return "", err
 	}
 
-	for _, pod := range l.Items {
+	if isRunning, err := podRunningReady(pod); err == nil && isRunning {
-		if strings.HasPrefix(pod.GetName(), "nginx-ingress-controller") {
+		return Logs(pod)
-			if isRunning, err := podRunningReady(&pod); err == nil && isRunning {
-				return Logs(&pod)
-			}
-		}
 	}
 
 	return "", fmt.Errorf("no nginx ingress controller pod is running (logs)")
@@ -222,17 +216,11 @@ func (f *Framework) NginxLogs() (string, error) {
 
 func (f *Framework) matchNginxConditions(name string, matcher func(cfg string) bool) wait.ConditionFunc {
 	return func() (bool, error) {
-		l, err := f.KubeClientSet.CoreV1().Pods(f.IngressController.Namespace).List(metav1.ListOptions{
+		pod, err := getIngressNGINXPod(f.IngressController.Namespace, f.KubeClientSet)
-			LabelSelector: "app.kubernetes.io/name=ingress-nginx",
-		})
 		if err != nil {
 			return false, err
 		}
 
-		if len(l.Items) == 0 {
-			return false, nil
-		}
-
 		var cmd string
 		if name == "" {
 			cmd = fmt.Sprintf("cat /etc/nginx/nginx.conf")
@@ -240,21 +228,6 @@ func (f *Framework) matchNginxConditions(name string, matcher func(cfg string) b
 			cmd = fmt.Sprintf("cat /etc/nginx/nginx.conf | awk '/## start server %v/,/## end server %v/'", name, name)
 		}
 
-		var pod *v1.Pod
-
-		for _, p := range l.Items {
-			if strings.HasPrefix(p.GetName(), "nginx-ingress-controller") {
-				if isRunning, err := podRunningReady(&p); err == nil && isRunning {
-					pod = &p
-					break
-				}
-			}
-		}
-
-		if pod == nil {
-			return false, nil
-		}
-
 		o, err := f.ExecCommand(pod, cmd)
 		if err != nil {
 			return false, err
@@ -369,7 +342,7 @@ func UpdateDeployment(kubeClientSet kubernetes.Interface, namespace string, name
 		}
 	}
 
-	err = WaitForPodsReady(kubeClientSet, 5*time.Minute, replicas, namespace, metav1.ListOptions{
+	err = WaitForPodsReady(kubeClientSet, defaultTimeout, replicas, namespace, metav1.ListOptions{
 		LabelSelector: fields.SelectorFromSet(fields.Set(deployment.Spec.Template.ObjectMeta.Labels)).String(),
 	})
 	if err != nil {

test/e2e/framework/k8s.go

@@ -18,6 +18,7 @@ package framework
 
 import (
 	"fmt"
+	"strings"
 	"time"
 
 	. "github.com/onsi/gomega"
@@ -141,6 +142,22 @@ func WaitForPodsReady(kubeClientSet kubernetes.Interface, timeout time.Duration,
 	})
 }
 
+// WaitForEndpoints waits for a given amount of time until an endpoint contains.
+func WaitForEndpoints(kubeClientSet kubernetes.Interface, timeout time.Duration, name, ns string) error {
+	return wait.Poll(2*time.Second, timeout, func() (bool, error) {
+		endpoint, err := kubeClientSet.CoreV1().Endpoints(ns).Get(name, metav1.GetOptions{})
+		if k8sErrors.IsNotFound(err) {
+			return false, err
+		}
+		Expect(err).NotTo(HaveOccurred())
+		if len(endpoint.Subsets) == 0 || len(endpoint.Subsets[0].Addresses) == 0 {
+			return false, err
+		}
+
+		return true, nil
+	})
+}
+
 // podRunningReady checks whether pod p's phase is running and it has a ready
 // condition of status true.
 func podRunningReady(p *core.Pod) (bool, error) {
@@ -156,3 +173,33 @@ func podRunningReady(p *core.Pod) (bool, error) {
 	}
 	return true, nil
 }
+
+func getIngressNGINXPod(ns string, kubeClientSet kubernetes.Interface) (*core.Pod, error) {
+	l, err := kubeClientSet.CoreV1().Pods(ns).List(metav1.ListOptions{
+		LabelSelector: "app.kubernetes.io/name=ingress-nginx",
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	if len(l.Items) == 0 {
+		return nil, fmt.Errorf("There is no ingress-nginx pods running in namespace %v", ns)
+	}
+
+	var pod *core.Pod
+
+	for _, p := range l.Items {
+		if strings.HasPrefix(p.GetName(), "nginx-ingress-controller") {
+			if isRunning, err := podRunningReady(&p); err == nil && isRunning {
+				pod = &p
+				break
+			}
+		}
+	}
+
+	if pod == nil {
+		return nil, fmt.Errorf("There is no ingress-nginx pods running in namespace %v", ns)
+	}
+
+	return pod, nil
+}