diff --git a/test/e2e/annotations/canary.go b/test/e2e/annotations/canary.go index 6201eadb2..4c2d5e056 100644 --- a/test/e2e/annotations/canary.go +++ b/test/e2e/annotations/canary.go @@ -40,7 +40,7 @@ var _ = framework.IngressNginxDescribe("Annotations - canary", func() { f.NewEchoDeployment() // Deployment for canary backend - f.NewDeployment("http-svc-canary", "gcr.io/kubernetes-e2e-test-images/echoserver:2.1", 8080, 1) + f.NewDeployment("http-svc-canary", "gcr.io/kubernetes-e2e-test-images/echoserver:2.2", 8080, 1) }) Context("when canaried by header", func() { diff --git a/test/e2e/framework/deployment.go b/test/e2e/framework/deployment.go index a01e3d333..13e426e7e 100644 --- a/test/e2e/framework/deployment.go +++ b/test/e2e/framework/deployment.go @@ -17,8 +17,6 @@ limitations under the License. package framework import ( - "time" - . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" @@ -36,7 +34,7 @@ func (f *Framework) NewEchoDeployment() { // NewEchoDeploymentWithReplicas creates a new deployment of the echoserver image in a particular namespace. Number of // replicas is configurable func (f *Framework) NewEchoDeploymentWithReplicas(replicas int32) { - f.NewDeployment("http-svc", "gcr.io/kubernetes-e2e-test-images/echoserver:2.1", 8080, replicas) + f.NewDeployment("http-svc", "gcr.io/kubernetes-e2e-test-images/echoserver:2.2", 8080, replicas) } // NewHttpbinDeployment creates a new single replica deployment of the httpbin image in a particular namespace. @@ -46,6 +44,19 @@ func (f *Framework) NewHttpbinDeployment() { // NewDeployment creates a new deployment in a particular namespace. func (f *Framework) NewDeployment(name, image string, port int32, replicas int32) { + probe := &corev1.Probe{ + InitialDelaySeconds: 5, + PeriodSeconds: 10, + SuccessThreshold: 1, + TimeoutSeconds: 1, + Handler: corev1.Handler{ + HTTPGet: &corev1.HTTPGetAction{ + Port: intstr.FromString("http"), + Path: "/", + }, + }, + } + deployment := &extensions.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -77,6 +88,8 @@ func (f *Framework) NewDeployment(name, image string, port int32, replicas int32 ContainerPort: port, }, }, + ReadinessProbe: probe, + LivenessProbe: probe, }, }, }, @@ -88,10 +101,10 @@ func (f *Framework) NewDeployment(name, image string, port int32, replicas int32 Expect(err).NotTo(HaveOccurred(), "failed to create a deployment") Expect(d).NotTo(BeNil(), "expected a deployement but none returned") - err = WaitForPodsReady(f.KubeClientSet, 5*time.Minute, int(replicas), f.IngressController.Namespace, metav1.ListOptions{ + err = WaitForPodsReady(f.KubeClientSet, defaultTimeout, int(replicas), f.IngressController.Namespace, metav1.ListOptions{ LabelSelector: fields.SelectorFromSet(fields.Set(d.Spec.Template.ObjectMeta.Labels)).String(), }) - Expect(err).NotTo(HaveOccurred(), "failed to wait for to become ready") + Expect(err).NotTo(HaveOccurred(), "failed to wait for pods to become ready") service := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ @@ -104,7 +117,7 @@ func (f *Framework) NewDeployment(name, image string, port int32, replicas int32 Name: "http", Port: 80, TargetPort: intstr.FromInt(int(port)), - Protocol: "TCP", + Protocol: corev1.ProtocolTCP, }, }, Selector: map[string]string{ @@ -115,4 +128,7 @@ func (f *Framework) NewDeployment(name, image string, port int32, replicas int32 s := f.EnsureService(service) Expect(s).NotTo(BeNil(), "expected a service but none returned") + + err = WaitForEndpoints(f.KubeClientSet, defaultTimeout, name, f.IngressController.Namespace) + Expect(err).NotTo(HaveOccurred(), "failed to wait for endpoints to become ready") } diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go index d2ec36035..7ca0eee56 100644 --- a/test/e2e/framework/framework.go +++ b/test/e2e/framework/framework.go @@ -112,7 +112,7 @@ func (f *Framework) BeforeEach() { err = f.NewIngressController(f.IngressController.Namespace) Expect(err).NotTo(HaveOccurred()) - err = WaitForPodsReady(f.KubeClientSet, 5*time.Minute, 1, f.IngressController.Namespace, metav1.ListOptions{ + err = WaitForPodsReady(f.KubeClientSet, defaultTimeout, 1, f.IngressController.Namespace, metav1.ListOptions{ LabelSelector: "app.kubernetes.io/name=ingress-nginx", }) Expect(err).NotTo(HaveOccurred()) @@ -197,19 +197,13 @@ func (f *Framework) WaitForNginxConfiguration(matcher func(cfg string) bool) { } func nginxLogs(client kubernetes.Interface, namespace string) (string, error) { - l, err := client.CoreV1().Pods(namespace).List(metav1.ListOptions{ - LabelSelector: "app.kubernetes.io/name=ingress-nginx", - }) + pod, err := getIngressNGINXPod(namespace, client) if err != nil { return "", err } - for _, pod := range l.Items { - if strings.HasPrefix(pod.GetName(), "nginx-ingress-controller") { - if isRunning, err := podRunningReady(&pod); err == nil && isRunning { - return Logs(&pod) - } - } + if isRunning, err := podRunningReady(pod); err == nil && isRunning { + return Logs(pod) } return "", fmt.Errorf("no nginx ingress controller pod is running (logs)") @@ -222,17 +216,11 @@ func (f *Framework) NginxLogs() (string, error) { func (f *Framework) matchNginxConditions(name string, matcher func(cfg string) bool) wait.ConditionFunc { return func() (bool, error) { - l, err := f.KubeClientSet.CoreV1().Pods(f.IngressController.Namespace).List(metav1.ListOptions{ - LabelSelector: "app.kubernetes.io/name=ingress-nginx", - }) + pod, err := getIngressNGINXPod(f.IngressController.Namespace, f.KubeClientSet) if err != nil { return false, err } - if len(l.Items) == 0 { - return false, nil - } - var cmd string if name == "" { cmd = fmt.Sprintf("cat /etc/nginx/nginx.conf") @@ -240,21 +228,6 @@ func (f *Framework) matchNginxConditions(name string, matcher func(cfg string) b cmd = fmt.Sprintf("cat /etc/nginx/nginx.conf | awk '/## start server %v/,/## end server %v/'", name, name) } - var pod *v1.Pod - - for _, p := range l.Items { - if strings.HasPrefix(p.GetName(), "nginx-ingress-controller") { - if isRunning, err := podRunningReady(&p); err == nil && isRunning { - pod = &p - break - } - } - } - - if pod == nil { - return false, nil - } - o, err := f.ExecCommand(pod, cmd) if err != nil { return false, err @@ -369,7 +342,7 @@ func UpdateDeployment(kubeClientSet kubernetes.Interface, namespace string, name } } - err = WaitForPodsReady(kubeClientSet, 5*time.Minute, replicas, namespace, metav1.ListOptions{ + err = WaitForPodsReady(kubeClientSet, defaultTimeout, replicas, namespace, metav1.ListOptions{ LabelSelector: fields.SelectorFromSet(fields.Set(deployment.Spec.Template.ObjectMeta.Labels)).String(), }) if err != nil { diff --git a/test/e2e/framework/k8s.go b/test/e2e/framework/k8s.go index 5a40fe100..39812969d 100644 --- a/test/e2e/framework/k8s.go +++ b/test/e2e/framework/k8s.go @@ -18,6 +18,7 @@ package framework import ( "fmt" + "strings" "time" . "github.com/onsi/gomega" @@ -141,6 +142,22 @@ func WaitForPodsReady(kubeClientSet kubernetes.Interface, timeout time.Duration, }) } +// WaitForEndpoints waits for a given amount of time until an endpoint contains. +func WaitForEndpoints(kubeClientSet kubernetes.Interface, timeout time.Duration, name, ns string) error { + return wait.Poll(2*time.Second, timeout, func() (bool, error) { + endpoint, err := kubeClientSet.CoreV1().Endpoints(ns).Get(name, metav1.GetOptions{}) + if k8sErrors.IsNotFound(err) { + return false, err + } + Expect(err).NotTo(HaveOccurred()) + if len(endpoint.Subsets) == 0 || len(endpoint.Subsets[0].Addresses) == 0 { + return false, err + } + + return true, nil + }) +} + // podRunningReady checks whether pod p's phase is running and it has a ready // condition of status true. func podRunningReady(p *core.Pod) (bool, error) { @@ -156,3 +173,33 @@ func podRunningReady(p *core.Pod) (bool, error) { } return true, nil } + +func getIngressNGINXPod(ns string, kubeClientSet kubernetes.Interface) (*core.Pod, error) { + l, err := kubeClientSet.CoreV1().Pods(ns).List(metav1.ListOptions{ + LabelSelector: "app.kubernetes.io/name=ingress-nginx", + }) + if err != nil { + return nil, err + } + + if len(l.Items) == 0 { + return nil, fmt.Errorf("There is no ingress-nginx pods running in namespace %v", ns) + } + + var pod *core.Pod + + for _, p := range l.Items { + if strings.HasPrefix(p.GetName(), "nginx-ingress-controller") { + if isRunning, err := podRunningReady(&p); err == nil && isRunning { + pod = &p + break + } + } + } + + if pod == nil { + return nil, fmt.Errorf("There is no ingress-nginx pods running in namespace %v", ns) + } + + return pod, nil +}