From c2550930b15f2a20a0aceec2656fccdbcebdd38a Mon Sep 17 00:00:00 2001 From: Manuel Alejandro de Brito Fontes Date: Thu, 12 Dec 2019 23:07:50 -0300 Subject: [PATCH] Fix e2e test flakes --- test/e2e/framework/deployment.go | 14 ++--- test/e2e/framework/fastcgi_helloserver.go | 5 +- test/e2e/framework/grpc_fortune_teller.go | 5 +- test/e2e/framework/influxdb.go | 4 +- test/e2e/framework/k8s.go | 65 +++++++++++++++-------- test/e2e/settings/ingress_class.go | 4 ++ 6 files changed, 57 insertions(+), 40 deletions(-) diff --git a/test/e2e/framework/deployment.go b/test/e2e/framework/deployment.go index c1c8c02c0..564e26f80 100644 --- a/test/e2e/framework/deployment.go +++ b/test/e2e/framework/deployment.go @@ -187,8 +187,7 @@ Request Body: }, ) - d, err := f.EnsureDeployment(deployment) - Expect(err).NotTo(HaveOccurred(), "failed to create a deployment") + d := f.EnsureDeployment(deployment) Expect(d).NotTo(BeNil(), "expected a deployment but none returned") service := &corev1.Service{ @@ -273,8 +272,7 @@ server { }, ) - d, err := f.EnsureDeployment(deployment) - Expect(err).NotTo(HaveOccurred(), "failed to create a deployment") + d := f.EnsureDeployment(deployment) Expect(d).NotTo(BeNil(), "expected a deployment but none returned") service := &corev1.Service{ @@ -377,8 +375,7 @@ func (f *Framework) NewHttpbinDeployment() { func (f *Framework) NewDeployment(name, image string, port int32, replicas int32) { deployment := newDeployment(name, f.Namespace, image, port, replicas, nil, nil, nil) - d, err := f.EnsureDeployment(deployment) - Expect(err).NotTo(HaveOccurred(), "failed to create a deployment") + d := f.EnsureDeployment(deployment) Expect(d).NotTo(BeNil(), "expected a deployment but none returned") service := &corev1.Service{ @@ -404,7 +401,7 @@ func (f *Framework) NewDeployment(name, image string, port int32, replicas int32 s := f.EnsureService(service) Expect(s).NotTo(BeNil(), "expected a service but none returned") - err = WaitForEndpoints(f.KubeClientSet, DefaultTimeout, name, f.Namespace, int(replicas)) + err := WaitForEndpoints(f.KubeClientSet, DefaultTimeout, name, f.Namespace, int(replicas)) Expect(err).NotTo(HaveOccurred(), "failed to wait for endpoints to become ready") } @@ -427,7 +424,6 @@ func (f *Framework) ScaleDeploymentToZero(name string) { d.Spec.Replicas = NewInt32(0) - d, err = f.EnsureDeployment(d) - Expect(err).NotTo(HaveOccurred(), "waiting deployment scale to 0") + d = f.EnsureDeployment(d) Expect(d).NotTo(BeNil(), "expected a deployment but none returned") } diff --git a/test/e2e/framework/fastcgi_helloserver.go b/test/e2e/framework/fastcgi_helloserver.go index 5c1b0b31a..dc7388186 100644 --- a/test/e2e/framework/fastcgi_helloserver.go +++ b/test/e2e/framework/fastcgi_helloserver.go @@ -73,11 +73,10 @@ func (f *Framework) NewNewFastCGIHelloServerDeploymentWithReplicas(replicas int3 }, } - d, err := f.EnsureDeployment(deployment) - Expect(err).NotTo(HaveOccurred()) + d := f.EnsureDeployment(deployment) Expect(d).NotTo(BeNil(), "expected a fastcgi-helloserver deployment") - err = WaitForPodsReady(f.KubeClientSet, DefaultTimeout, int(replicas), f.Namespace, metav1.ListOptions{ + err := WaitForPodsReady(f.KubeClientSet, DefaultTimeout, int(replicas), f.Namespace, metav1.ListOptions{ LabelSelector: fields.SelectorFromSet(fields.Set(d.Spec.Template.ObjectMeta.Labels)).String(), }) Expect(err).NotTo(HaveOccurred(), "failed to wait for to become ready") diff --git a/test/e2e/framework/grpc_fortune_teller.go b/test/e2e/framework/grpc_fortune_teller.go index 7cab3e026..4bbeee65b 100644 --- a/test/e2e/framework/grpc_fortune_teller.go +++ b/test/e2e/framework/grpc_fortune_teller.go @@ -73,11 +73,10 @@ func (f *Framework) NewNewGRPCFortuneTellerDeploymentWithReplicas(replicas int32 }, } - d, err := f.EnsureDeployment(deployment) - Expect(err).NotTo(HaveOccurred()) + d := f.EnsureDeployment(deployment) Expect(d).NotTo(BeNil(), "expected a fortune-teller deployment") - err = WaitForPodsReady(f.KubeClientSet, DefaultTimeout, int(replicas), f.Namespace, metav1.ListOptions{ + err := WaitForPodsReady(f.KubeClientSet, DefaultTimeout, int(replicas), f.Namespace, metav1.ListOptions{ LabelSelector: fields.SelectorFromSet(fields.Set(d.Spec.Template.ObjectMeta.Labels)).String(), }) Expect(err).NotTo(HaveOccurred(), "failed to wait for to become ready") diff --git a/test/e2e/framework/influxdb.go b/test/e2e/framework/influxdb.go index c2ca8d7f0..c38bf9c4a 100644 --- a/test/e2e/framework/influxdb.go +++ b/test/e2e/framework/influxdb.go @@ -135,9 +135,7 @@ func (f *Framework) NewInfluxDBDeployment() { }, } - d, err := f.EnsureDeployment(deployment) - Expect(err).NotTo(HaveOccurred(), "failed to create an Influxdb deployment") - + d := f.EnsureDeployment(deployment) Expect(d).NotTo(BeNil(), "unexpected error creating deployment for influxdb") err = WaitForPodsReady(f.KubeClientSet, DefaultTimeout, 1, f.Namespace, metav1.ListOptions{ diff --git a/test/e2e/framework/k8s.go b/test/e2e/framework/k8s.go index 760ef2d6a..6d06df966 100644 --- a/test/e2e/framework/k8s.go +++ b/test/e2e/framework/k8s.go @@ -31,6 +31,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/util/retry" podutil "k8s.io/kubernetes/pkg/api/v1/pod" ) @@ -69,18 +70,24 @@ func (f *Framework) EnsureConfigMap(configMap *api.ConfigMap) (*api.ConfigMap, e // EnsureIngress creates an Ingress object or returns it if it already exists. func (f *Framework) EnsureIngress(ingress *networking.Ingress) *networking.Ingress { - ing, err := f.KubeClientSet.NetworkingV1beta1().Ingresses(ingress.Namespace).Update(ingress) + ing, err := f.KubeClientSet.NetworkingV1beta1().Ingresses(ingress.Namespace).Create(ingress) if err != nil { - if k8sErrors.IsNotFound(err) { - ing, err = f.KubeClientSet.NetworkingV1beta1().Ingresses(ingress.Namespace).Create(ingress) - Expect(err).NotTo(HaveOccurred(), "unexpected error creating ingress") - return ing - } + if k8sErrors.IsAlreadyExists(err) { + err = retry.RetryOnConflict(retry.DefaultRetry, func() error { + var err error + ing, err = f.KubeClientSet.NetworkingV1beta1().Ingresses(ingress.Namespace).Update(ingress) + if err != nil { + return err + } - Expect(err).NotTo(HaveOccurred()) + return nil + }) + + Expect(err).NotTo(HaveOccurred()) + } } - Expect(ing).NotTo(BeNil()) + Expect(ing).NotTo(BeNil(), "expected an ingress but none returned") if ing.Annotations == nil { ing.Annotations = make(map[string]string) @@ -91,33 +98,47 @@ func (f *Framework) EnsureIngress(ingress *networking.Ingress) *networking.Ingre // EnsureService creates a Service object or returns it if it already exists. func (f *Framework) EnsureService(service *core.Service) *core.Service { - s, err := f.KubeClientSet.CoreV1().Services(service.Namespace).Update(service) + s, err := f.KubeClientSet.CoreV1().Services(service.Namespace).Create(service) if err != nil { - if k8sErrors.IsNotFound(err) { - s, err := f.KubeClientSet.CoreV1().Services(service.Namespace).Create(service) - Expect(err).NotTo(HaveOccurred(), "unexpected error creating service") - return s + if k8sErrors.IsAlreadyExists(err) { + err = retry.RetryOnConflict(retry.DefaultRetry, func() error { + var err error + s, err = f.KubeClientSet.CoreV1().Services(service.Namespace).Update(service) + if err != nil { + return err + } + return nil + }) + + Expect(err).NotTo(HaveOccurred()) } - - Expect(err).NotTo(HaveOccurred()) } Expect(s).NotTo(BeNil(), "expected a service but none returned") - return s } // EnsureDeployment creates a Deployment object or returns it if it already exists. -func (f *Framework) EnsureDeployment(deployment *appsv1.Deployment) (*appsv1.Deployment, error) { - d, err := f.KubeClientSet.AppsV1().Deployments(deployment.Namespace).Update(deployment) +func (f *Framework) EnsureDeployment(deployment *appsv1.Deployment) *appsv1.Deployment { + d, err := f.KubeClientSet.AppsV1().Deployments(deployment.Namespace).Create(deployment) if err != nil { - if k8sErrors.IsNotFound(err) { - return f.KubeClientSet.AppsV1().Deployments(deployment.Namespace).Create(deployment) + if k8sErrors.IsAlreadyExists(err) { + err = retry.RetryOnConflict(retry.DefaultRetry, func() error { + d, err = f.KubeClientSet.AppsV1().Deployments(deployment.Namespace).Update(deployment) + if err != nil { + return err + } + + return nil + }) + + Expect(err).NotTo(HaveOccurred()) } - return nil, err } - return d, nil + + Expect(d).NotTo(BeNil(), "expected a deployment but none returned") + return d } // WaitForPodsReady waits for a given amount of time until a group of Pods is running in the given namespace. diff --git a/test/e2e/settings/ingress_class.go b/test/e2e/settings/ingress_class.go index be95b2657..0108113e9 100644 --- a/test/e2e/settings/ingress_class.go +++ b/test/e2e/settings/ingress_class.go @@ -24,6 +24,7 @@ import ( . "github.com/onsi/gomega" "github.com/parnurzeal/gorequest" appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/ingress-nginx/test/e2e/framework" ) @@ -141,6 +142,9 @@ var _ = framework.IngressNginxDescribe("Ingress class", func() { Expect(errs).To(BeNil()) Expect(resp.StatusCode).Should(Equal(http.StatusOK)) + ing, err := f.KubeClientSet.NetworkingV1beta1().Ingresses(f.Namespace).Get(host, metav1.GetOptions{}) + Expect(err).To(BeNil()) + delete(ing.Annotations, "kubernetes.io/ingress.class") f.EnsureIngress(ing)