From 0e19740ee2e40fa691dc8cceae5de80550003822 Mon Sep 17 00:00:00 2001 From: Manuel Alejandro de Brito Fontes Date: Mon, 29 Jun 2020 18:28:52 -0400 Subject: [PATCH] Update e2e configuration --- Makefile | 4 +- test/e2e/defaultbackend/ssl.go | 2 + test/e2e/framework/deployment.go | 2 +- test/e2e/framework/framework.go | 17 ++++---- test/e2e/framework/k8s.go | 57 ++++++++++++++++----------- test/e2e/framework/util.go | 18 ++++----- test/e2e/gracefulshutdown/shutdown.go | 4 -- test/e2e/kind.yaml | 3 -- 8 files changed, 58 insertions(+), 49 deletions(-) diff --git a/Makefile b/Makefile index ab0199c8d..856d34660 100644 --- a/Makefile +++ b/Makefile @@ -33,7 +33,7 @@ TAG ?= 0.33.0 # Allow limiting the scope of the e2e tests. By default run everything FOCUS ?= .* # number of parallel test -E2E_NODES ?= 6 +E2E_NODES ?= 10 # run e2e test suite with tests that check for memory leaks? (default is false) E2E_CHECK_LEAKS ?= @@ -50,7 +50,7 @@ endif REGISTRY ?= quay.io/kubernetes-ingress-controller -BASE_IMAGE ?= quay.io/kubernetes-ingress-controller/nginx:e3c49c52f4b74fe47ad65d6f3266a02e8b6b622f +BASE_IMAGE ?= gcr.io/k8s-staging-ingress-nginx/nginx@sha256:35da1d3e00f5e763e59cb59159bf88ba0f0b6e8835885ac9d8b63029a478dba7 GOARCH=$(ARCH) diff --git a/test/e2e/defaultbackend/ssl.go b/test/e2e/defaultbackend/ssl.go index 30a6f9688..f453dd2d3 100644 --- a/test/e2e/defaultbackend/ssl.go +++ b/test/e2e/defaultbackend/ssl.go @@ -28,6 +28,8 @@ var _ = framework.IngressNginxDescribe("[Default Backend] SSL", func() { ginkgo.It("should return a self generated SSL certificate", func() { ginkgo.By("checking SSL Certificate using the NGINX IP address") + framework.Sleep() + resp := f.HTTPTestClient(). GET("/"). WithURL(f.GetURL(framework.HTTPS)). diff --git a/test/e2e/framework/deployment.go b/test/e2e/framework/deployment.go index 134e61f28..8f77ca536 100644 --- a/test/e2e/framework/deployment.go +++ b/test/e2e/framework/deployment.go @@ -389,7 +389,7 @@ func (f *Framework) DeleteDeployment(name string) error { }) assert.Nil(ginkgo.GinkgoT(), err, "deleting deployment") - return WaitForPodsDeleted(f.KubeClientSet, 2*time.Minute, f.Namespace, metav1.ListOptions{ + return waitForPodsDeleted(f.KubeClientSet, 2*time.Minute, f.Namespace, metav1.ListOptions{ LabelSelector: labelSelectorToString(d.Spec.Selector.MatchLabels), }) } diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go index cf1ef6b06..b33e7ccb3 100644 --- a/test/e2e/framework/framework.go +++ b/test/e2e/framework/framework.go @@ -203,20 +203,20 @@ func (f *Framework) GetURL(scheme RequestScheme) string { // WaitForNginxServer waits until the nginx configuration contains a particular server section func (f *Framework) WaitForNginxServer(name string, matcher func(cfg string) bool) { - err := wait.Poll(Poll, DefaultTimeout, f.matchNginxConditions(name, matcher)) + err := wait.PollImmediate(Poll, DefaultTimeout, f.matchNginxConditions(name, matcher)) assert.Nil(ginkgo.GinkgoT(), err, "waiting for nginx server condition/s") Sleep() } // WaitForNginxConfiguration waits until the nginx configuration contains a particular configuration func (f *Framework) WaitForNginxConfiguration(matcher func(cfg string) bool) { - err := wait.Poll(Poll, DefaultTimeout, f.matchNginxConditions("", matcher)) + err := wait.PollImmediate(Poll, DefaultTimeout, f.matchNginxConditions("", matcher)) assert.Nil(ginkgo.GinkgoT(), err, "waiting for nginx server condition/s") } // WaitForNginxCustomConfiguration waits until the nginx configuration given part (from, to) contains a particular configuration func (f *Framework) WaitForNginxCustomConfiguration(from string, to string, matcher func(cfg string) bool) { - err := wait.Poll(Poll, DefaultTimeout, f.matchNginxCustomConditions(from, to, matcher)) + err := wait.PollImmediate(Poll, DefaultTimeout, f.matchNginxCustomConditions(from, to, matcher)) assert.Nil(ginkgo.GinkgoT(), err, "waiting for nginx server condition/s") } @@ -364,7 +364,6 @@ func (f *Framework) UpdateNginxConfigMapData(key string, value string) { assert.Nil(ginkgo.GinkgoT(), err, "updating configuration configmap") } - Sleep(1) f.waitForReload(fn) } @@ -377,7 +376,7 @@ func (f *Framework) waitForReload(fn func()) { err := wait.Poll(Poll, DefaultTimeout, func() (bool, error) { // most of the cases reload the ingress controller // in cases where the value is not modified we could wait forever - if count > 4 { + if count > 3 { return true, nil } @@ -390,8 +389,12 @@ func (f *Framework) waitForReload(fn func()) { func (f *Framework) getReloadCount() int { ip := f.GetNginxPodIP() + mf, err := f.GetMetric("nginx_ingress_controller_success", ip) - assert.Nil(ginkgo.GinkgoT(), err) + if err != nil { + return 0 + } + assert.NotNil(ginkgo.GinkgoT(), mf) rc0, err := extractReloadCount(mf) @@ -422,7 +425,7 @@ func (f *Framework) DeleteNGINXPod(grace int64) { err = f.KubeClientSet.CoreV1().Pods(ns).Delete(context.TODO(), pod.GetName(), *metav1.NewDeleteOptions(grace)) assert.Nil(ginkgo.GinkgoT(), err, "deleting ingress nginx pod") - err = wait.Poll(Poll, DefaultTimeout, func() (bool, error) { + err = wait.PollImmediate(Poll, DefaultTimeout, func() (bool, error) { pod, err := GetIngressNGINXPod(ns, f.KubeClientSet) if err != nil || pod == nil { return false, nil diff --git a/test/e2e/framework/k8s.go b/test/e2e/framework/k8s.go index f247fe2d5..95c8635e6 100644 --- a/test/e2e/framework/k8s.go +++ b/test/e2e/framework/k8s.go @@ -129,7 +129,7 @@ func (f *Framework) EnsureDeployment(deployment *appsv1.Deployment) *appsv1.Depl // waitForPodsReady waits for a given amount of time until a group of Pods is running in the given namespace. func waitForPodsReady(kubeClientSet kubernetes.Interface, timeout time.Duration, expectedReplicas int, namespace string, opts metav1.ListOptions) error { - return wait.Poll(Poll, timeout, func() (bool, error) { + return wait.PollImmediate(Poll, timeout, func() (bool, error) { pl, err := kubeClientSet.CoreV1().Pods(namespace).List(context.TODO(), opts) if err != nil { return false, nil @@ -150,9 +150,9 @@ func waitForPodsReady(kubeClientSet kubernetes.Interface, timeout time.Duration, }) } -// WaitForPodsDeleted waits for a given amount of time until a group of Pods are deleted in the given namespace. -func WaitForPodsDeleted(kubeClientSet kubernetes.Interface, timeout time.Duration, namespace string, opts metav1.ListOptions) error { - return wait.Poll(Poll, timeout, func() (bool, error) { +// waitForPodsDeleted waits for a given amount of time until a group of Pods are deleted in the given namespace. +func waitForPodsDeleted(kubeClientSet kubernetes.Interface, timeout time.Duration, namespace string, opts metav1.ListOptions) error { + return wait.PollImmediate(Poll, timeout, func() (bool, error) { pl, err := kubeClientSet.CoreV1().Pods(namespace).List(context.TODO(), opts) if err != nil { return false, nil @@ -171,7 +171,7 @@ func WaitForEndpoints(kubeClientSet kubernetes.Interface, timeout time.Duration, return nil } - return wait.Poll(Poll, timeout, func() (bool, error) { + return wait.PollImmediate(Poll, timeout, func() (bool, error) { endpoint, err := kubeClientSet.CoreV1().Endpoints(ns).Get(context.TODO(), name, metav1.GetOptions{}) if k8sErrors.IsNotFound(err) { return false, nil @@ -214,29 +214,40 @@ func podRunningReady(p *core.Pod) (bool, error) { // GetIngressNGINXPod returns the ingress controller running pod func GetIngressNGINXPod(ns string, kubeClientSet kubernetes.Interface) (*core.Pod, error) { - err := waitForPodsReady(kubeClientSet, DefaultTimeout, 1, ns, metav1.ListOptions{ - LabelSelector: "app.kubernetes.io/name=ingress-nginx", + var pod *core.Pod + err := wait.Poll(Poll, DefaultTimeout, func() (bool, error) { + l, err := kubeClientSet.CoreV1().Pods(ns).List(context.TODO(), metav1.ListOptions{ + LabelSelector: "app.kubernetes.io/name=ingress-nginx", + }) + if err != nil { + return false, nil + } + + for _, p := range l.Items { + if strings.HasPrefix(p.GetName(), "nginx-ingress-controller") { + isRunning, err := podRunningReady(&p) + if err != nil { + continue + } + + if isRunning { + pod = &p + return true, nil + } + } + } + + return false, nil }) if err != nil { + if err == wait.ErrWaitTimeout { + return nil, fmt.Errorf("timeout waiting at least one ingress-nginx pod running in namespace %v", ns) + } + return nil, err } - l, err := kubeClientSet.CoreV1().Pods(ns).List(context.TODO(), metav1.ListOptions{ - LabelSelector: "app.kubernetes.io/name=ingress-nginx", - }) - if err != nil { - return nil, nil - } - - for _, p := range l.Items { - if strings.HasPrefix(p.GetName(), "nginx-ingress-controller") { - if isRunning, err := podRunningReady(&p); err == nil && isRunning { - return &p, nil - } - } - } - - return nil, fmt.Errorf("there is no ingress-nginx pods running in namespace %v", ns) + return pod, nil } func createDeploymentWithRetries(c kubernetes.Interface, namespace string, obj *appsv1.Deployment) error { diff --git a/test/e2e/framework/util.go b/test/e2e/framework/util.go index a15a25028..1e65cca3d 100644 --- a/test/e2e/framework/util.go +++ b/test/e2e/framework/util.go @@ -35,7 +35,7 @@ import ( const ( // Poll how often to poll for conditions - Poll = 2 * time.Second + Poll = 1 * time.Second // DefaultTimeout time to wait for operations to complete DefaultTimeout = 180 * time.Second @@ -93,7 +93,7 @@ func CreateKubeNamespace(baseName string, c kubernetes.Interface) (string, error var got *corev1.Namespace var err error - err = wait.Poll(Poll, DefaultTimeout, func() (bool, error) { + err = wait.PollImmediate(Poll, DefaultTimeout, func() (bool, error) { got, err = c.CoreV1().Namespaces().Create(context.TODO(), ns, metav1.CreateOptions{}) if err != nil { Logf("Unexpected error while creating namespace: %v", err) @@ -119,7 +119,7 @@ func deleteKubeNamespace(c kubernetes.Interface, namespace string) error { // WaitForKubeNamespaceNotExist waits until a namespaces is not present in the cluster func WaitForKubeNamespaceNotExist(c kubernetes.Interface, namespace string) error { - return wait.Poll(Poll, DefaultTimeout, namespaceNotExist(c, namespace)) + return wait.PollImmediate(Poll, DefaultTimeout, namespaceNotExist(c, namespace)) } func namespaceNotExist(c kubernetes.Interface, namespace string) wait.ConditionFunc { @@ -137,7 +137,7 @@ func namespaceNotExist(c kubernetes.Interface, namespace string) wait.ConditionF // WaitForNoPodsInNamespace waits until there are no pods running in a namespace func WaitForNoPodsInNamespace(c kubernetes.Interface, namespace string) error { - return wait.Poll(Poll, DefaultTimeout, noPodsInNamespace(c, namespace)) + return wait.PollImmediate(Poll, DefaultTimeout, noPodsInNamespace(c, namespace)) } func noPodsInNamespace(c kubernetes.Interface, namespace string) wait.ConditionFunc { @@ -167,12 +167,12 @@ func WaitForPodRunningInNamespace(c kubernetes.Interface, pod *corev1.Pod) error } func waitTimeoutForPodRunningInNamespace(c kubernetes.Interface, podName, namespace string, timeout time.Duration) error { - return wait.Poll(Poll, DefaultTimeout, podRunning(c, podName, namespace)) + return wait.PollImmediate(Poll, DefaultTimeout, podRunning(c, podName, namespace)) } // WaitForSecretInNamespace waits a default amount of time for the specified secret is present in a particular namespace func WaitForSecretInNamespace(c kubernetes.Interface, namespace, name string) error { - return wait.Poll(Poll, DefaultTimeout, secretInNamespace(c, namespace, name)) + return wait.PollImmediate(Poll, DefaultTimeout, secretInNamespace(c, namespace, name)) } func secretInNamespace(c kubernetes.Interface, namespace, name string) wait.ConditionFunc { @@ -194,7 +194,7 @@ func secretInNamespace(c kubernetes.Interface, namespace, name string) wait.Cond // WaitForFileInFS waits a default amount of time for the specified file is present in the filesystem func WaitForFileInFS(file string) error { - return wait.Poll(Poll, DefaultTimeout, fileInFS(file)) + return wait.PollImmediate(Poll, DefaultTimeout, fileInFS(file)) } func fileInFS(file string) wait.ConditionFunc { @@ -218,7 +218,7 @@ func fileInFS(file string) wait.ConditionFunc { // WaitForNoIngressInNamespace waits until there is no ingress object in a particular namespace func WaitForNoIngressInNamespace(c kubernetes.Interface, namespace, name string) error { - return wait.Poll(Poll, DefaultTimeout, noIngressInNamespace(c, namespace, name)) + return wait.PollImmediate(Poll, DefaultTimeout, noIngressInNamespace(c, namespace, name)) } func noIngressInNamespace(c kubernetes.Interface, namespace, name string) wait.ConditionFunc { @@ -240,7 +240,7 @@ func noIngressInNamespace(c kubernetes.Interface, namespace, name string) wait.C // WaitForIngressInNamespace waits until a particular ingress object exists namespace func WaitForIngressInNamespace(c kubernetes.Interface, namespace, name string) error { - return wait.Poll(Poll, DefaultTimeout, ingressInNamespace(c, namespace, name)) + return wait.PollImmediate(Poll, DefaultTimeout, ingressInNamespace(c, namespace, name)) } func ingressInNamespace(c kubernetes.Interface, namespace, name string) wait.ConditionFunc { diff --git a/test/e2e/gracefulshutdown/shutdown.go b/test/e2e/gracefulshutdown/shutdown.go index 37b849ce5..e1f41a2cc 100644 --- a/test/e2e/gracefulshutdown/shutdown.go +++ b/test/e2e/gracefulshutdown/shutdown.go @@ -100,8 +100,6 @@ var _ = framework.IngressNginxDescribe("[Shutdown] ingress controller", func() { c <- resp.StatusCode }(host, result) - framework.Sleep() - f.ScaleDeploymentToZero("nginx-ingress-controller") assert.Equal(ginkgo.GinkgoT(), <-result, http.StatusOK, "expecting a valid response from HTTP request") @@ -145,8 +143,6 @@ var _ = framework.IngressNginxDescribe("[Shutdown] ingress controller", func() { c <- resp.StatusCode }(host, result) - framework.Sleep() - f.ScaleDeploymentToZero("nginx-ingress-controller") assert.Equal(ginkgo.GinkgoT(), <-result, http.StatusOK, "expecting a valid response from HTTP request") diff --git a/test/e2e/kind.yaml b/test/e2e/kind.yaml index a0bf30765..ffb5dd22b 100644 --- a/test/e2e/kind.yaml +++ b/test/e2e/kind.yaml @@ -9,6 +9,3 @@ kubeadmConfigPatches: kind: ClusterConfiguration metadata: name: config - controllerManager: - extraArgs: - namespace-sync-period: 10s