From 74d73ff076e674454c0e2333f170a9daf580928b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Herv=C3=A9=20Werner?= Date: Fri, 20 Jan 2023 17:26:46 +0100 Subject: [PATCH] Add E2E test to verify that changes to one or more configmap trigger an update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Hervé Werner --- internal/ingress/controller/store/store.go | 1 - test/e2e/annotations/fastcgi.go | 5 +- test/e2e/framework/framework.go | 2 +- test/e2e/framework/influxdb.go | 6 +- test/e2e/framework/k8s.go | 41 ++++--- test/e2e/settings/configmap_change.go | 4 + test/e2e/settings/ocsp/ocsp.go | 4 +- test/e2e/tcpudp/tcp.go | 129 ++++++++++++--------- 8 files changed, 107 insertions(+), 85 deletions(-) diff --git a/internal/ingress/controller/store/store.go b/internal/ingress/controller/store/store.go index e0f6cfb54..d1a0219ba 100644 --- a/internal/ingress/controller/store/store.go +++ b/internal/ingress/controller/store/store.go @@ -702,7 +702,6 @@ func New( }, } - // TODO: add e2e test to verify that changes to one or more configmap trigger an update changeTriggerUpdate := func(name string) bool { return name == configmap || name == tcp || name == udp } diff --git a/test/e2e/annotations/fastcgi.go b/test/e2e/annotations/fastcgi.go index 0dc6ae4a4..572eca548 100644 --- a/test/e2e/annotations/fastcgi.go +++ b/test/e2e/annotations/fastcgi.go @@ -21,7 +21,6 @@ import ( "strings" "github.com/onsi/ginkgo/v2" - "github.com/stretchr/testify/assert" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -81,9 +80,7 @@ var _ = framework.DescribeAnnotation("backend-protocol - FastCGI", func() { }, } - cm, err := f.EnsureConfigMap(configuration) - assert.Nil(ginkgo.GinkgoT(), err, "creating configmap") - assert.NotNil(ginkgo.GinkgoT(), cm, "expected a configmap but none returned") + f.EnsureConfigMap(configuration) host := "fastcgi-params-configmap" diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go index 5053150c0..cac6dfd20 100644 --- a/test/e2e/framework/framework.go +++ b/test/e2e/framework/framework.go @@ -391,7 +391,7 @@ func (f *Framework) UpdateNginxConfigMapData(key string, value string) { } // WaitForReload calls the passed function and -// asser it has caused at least 1 reload. +// asserts it has caused at least 1 reload. func (f *Framework) WaitForReload(fn func()) { initialReloadCount := getReloadCount(f.pod, f.Namespace, f.KubeClientSet) diff --git a/test/e2e/framework/influxdb.go b/test/e2e/framework/influxdb.go index c3c0e3421..43a5702e6 100644 --- a/test/e2e/framework/influxdb.go +++ b/test/e2e/framework/influxdb.go @@ -68,9 +68,7 @@ func (f *Framework) NewInfluxDBDeployment() { }, } - cm, err := f.EnsureConfigMap(configuration) - assert.Nil(ginkgo.GinkgoT(), err, "creating an Influxdb deployment") - assert.NotNil(ginkgo.GinkgoT(), cm, "expected a configmap but none returned") + f.EnsureConfigMap(configuration) deployment := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ @@ -136,7 +134,7 @@ func (f *Framework) NewInfluxDBDeployment() { d := f.EnsureDeployment(deployment) - err = waitForPodsReady(f.KubeClientSet, DefaultTimeout, 1, f.Namespace, metav1.ListOptions{ + err := waitForPodsReady(f.KubeClientSet, DefaultTimeout, 1, f.Namespace, metav1.ListOptions{ LabelSelector: fields.SelectorFromSet(fields.Set(d.Spec.Template.ObjectMeta.Labels)).String(), }) assert.Nil(ginkgo.GinkgoT(), err, "waiting for influxdb pod to become ready") diff --git a/test/e2e/framework/k8s.go b/test/e2e/framework/k8s.go index ef9c522d6..fc3e59b08 100644 --- a/test/e2e/framework/k8s.go +++ b/test/e2e/framework/k8s.go @@ -25,9 +25,7 @@ import ( "github.com/onsi/ginkgo/v2" "github.com/stretchr/testify/assert" appsv1 "k8s.io/api/apps/v1" - api "k8s.io/api/core/v1" core "k8s.io/api/core/v1" - v1 "k8s.io/api/core/v1" networking "k8s.io/api/networking/v1" k8sErrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -36,8 +34,8 @@ import ( "k8s.io/client-go/kubernetes" ) -// EnsureSecret creates a Secret object or returns it if it already exists. -func (f *Framework) EnsureSecret(secret *api.Secret) *api.Secret { +// EnsureSecret creates a Secret object or returns it. +func (f *Framework) EnsureSecret(secret *core.Secret) *core.Secret { err := createSecretWithRetries(f.KubeClientSet, secret.Namespace, secret) assert.Nil(ginkgo.GinkgoT(), err, "creating secret") @@ -48,17 +46,30 @@ func (f *Framework) EnsureSecret(secret *api.Secret) *api.Secret { return s } -// EnsureConfigMap creates a ConfigMap object or returns it if it already exists. -func (f *Framework) EnsureConfigMap(configMap *api.ConfigMap) (*api.ConfigMap, error) { - cm, err := f.KubeClientSet.CoreV1().ConfigMaps(configMap.Namespace).Create(context.TODO(), configMap, metav1.CreateOptions{}) - if err != nil { - if k8sErrors.IsAlreadyExists(err) { - return f.KubeClientSet.CoreV1().ConfigMaps(configMap.Namespace).Update(context.TODO(), configMap, metav1.UpdateOptions{}) - } - return nil, err +// GetConfigMap gets a ConfigMap object from the given namespace, name and returns it, throws error if it does not exist. +func (f *Framework) GetConfigMap(namespace string, name string) *core.ConfigMap { + cm, err := f.KubeClientSet.CoreV1().ConfigMaps(namespace).Get(context.TODO(), name, metav1.GetOptions{}) + assert.Nil(ginkgo.GinkgoT(), err, "getting configmap") + assert.NotNil(ginkgo.GinkgoT(), cm, "expected a configmap but none returned") + return cm +} + +// EnsureConfigMap creates or updates an existing ConfigMap object or returns it. +func (f *Framework) EnsureConfigMap(configMap *core.ConfigMap) *core.ConfigMap { + cm := configMap.DeepCopy() + // Clean out ResourceVersion field if present + if cm.ObjectMeta.ResourceVersion != "" { + cm.ObjectMeta.ResourceVersion = "" } - return cm, nil + res, err := f.KubeClientSet.CoreV1().ConfigMaps(configMap.Namespace).Create(context.TODO(), cm, metav1.CreateOptions{}) + if k8sErrors.IsAlreadyExists(err) { + res, err = f.KubeClientSet.CoreV1().ConfigMaps(configMap.Namespace).Update(context.TODO(), cm, metav1.UpdateOptions{}) + } + assert.Nil(ginkgo.GinkgoT(), err, "updating configmap") + assert.NotNil(ginkgo.GinkgoT(), res, "updating configmap") + + return res } // GetIngress gets an Ingress object from the given namespace, name and returns it, throws error if it does not exists. @@ -293,7 +304,7 @@ func createDeploymentWithRetries(c kubernetes.Interface, namespace string, obj * return retryWithExponentialBackOff(createFunc) } -func createSecretWithRetries(c kubernetes.Interface, namespace string, obj *v1.Secret) error { +func createSecretWithRetries(c kubernetes.Interface, namespace string, obj *core.Secret) error { if obj == nil { return fmt.Errorf("Object provided to create is empty") } @@ -313,7 +324,7 @@ func createSecretWithRetries(c kubernetes.Interface, namespace string, obj *v1.S return retryWithExponentialBackOff(createFunc) } -func createServiceWithRetries(c kubernetes.Interface, namespace string, obj *v1.Service) error { +func createServiceWithRetries(c kubernetes.Interface, namespace string, obj *core.Service) error { if obj == nil { return fmt.Errorf("Object provided to create is empty") } diff --git a/test/e2e/settings/configmap_change.go b/test/e2e/settings/configmap_change.go index be3ab0b11..3e37b62cd 100644 --- a/test/e2e/settings/configmap_change.go +++ b/test/e2e/settings/configmap_change.go @@ -73,5 +73,9 @@ var _ = framework.DescribeSetting("Configmap change", func() { return strings.ContainsAny(cfg, "error_log /var/log/nginx/error.log debug;") }) assert.NotEqual(ginkgo.GinkgoT(), checksum, newChecksum) + + logs, err := f.NginxLogs() + assert.Nil(ginkgo.GinkgoT(), err, "obtaining nginx logs") + assert.Contains(ginkgo.GinkgoT(), logs, "Backend successfully reloaded") }) }) diff --git a/test/e2e/settings/ocsp/ocsp.go b/test/e2e/settings/ocsp/ocsp.go index ea137aeb1..adf4351e7 100644 --- a/test/e2e/settings/ocsp/ocsp.go +++ b/test/e2e/settings/ocsp/ocsp.go @@ -85,7 +85,7 @@ var _ = framework.DescribeSetting("OCSP", func() { cfsslDB, err := os.ReadFile("empty.db") assert.Nil(ginkgo.GinkgoT(), err) - cmap, err := f.EnsureConfigMap(&corev1.ConfigMap{ + f.EnsureConfigMap(&corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: "ocspserve", Namespace: f.Namespace, @@ -95,8 +95,6 @@ var _ = framework.DescribeSetting("OCSP", func() { "db-config.json": []byte(`{"driver":"sqlite3","data_source":"/data/empty.db"}`), }, }) - assert.Nil(ginkgo.GinkgoT(), err) - assert.NotNil(ginkgo.GinkgoT(), cmap) d, s := ocspserveDeployment(f.Namespace) f.EnsureDeployment(d) diff --git a/test/e2e/tcpudp/tcp.go b/test/e2e/tcpudp/tcp.go index b62134a03..16a633b63 100644 --- a/test/e2e/tcpudp/tcp.go +++ b/test/e2e/tcpudp/tcp.go @@ -21,6 +21,7 @@ import ( "fmt" "net" "net/http" + "regexp" "strings" "time" @@ -36,58 +37,39 @@ import ( var _ = framework.IngressNginxDescribe("[TCP] tcp-services", func() { f := framework.NewDefaultFramework("tcp") + var ip string + + ginkgo.BeforeEach(func() { + ip = f.GetNginxIP() + }) ginkgo.It("should expose a TCP service", func() { f.NewEchoDeployment() - config, err := f.KubeClientSet. - CoreV1(). - ConfigMaps(f.Namespace). - Get(context.TODO(), "tcp-services", metav1.GetOptions{}) - assert.Nil(ginkgo.GinkgoT(), err, "unexpected error obtaining tcp-services configmap") - assert.NotNil(ginkgo.GinkgoT(), config, "expected a configmap but none returned") - - if config.Data == nil { - config.Data = map[string]string{} + cm := f.GetConfigMap(f.Namespace, "tcp-services") + cm.Data = map[string]string{ + "8080": fmt.Sprintf("%v/%v:80", f.Namespace, framework.EchoService), } + f.EnsureConfigMap(cm) - config.Data["8080"] = fmt.Sprintf("%v/%v:80", f.Namespace, framework.EchoService) - - _, err = f.KubeClientSet. - CoreV1(). - ConfigMaps(f.Namespace). - Update(context.TODO(), config, metav1.UpdateOptions{}) - assert.Nil(ginkgo.GinkgoT(), err, "unexpected error updating configmap") - - svc, err := f.KubeClientSet. - CoreV1(). - Services(f.Namespace). - Get(context.TODO(), "nginx-ingress-controller", metav1.GetOptions{}) - assert.Nil(ginkgo.GinkgoT(), err, "unexpected error obtaining ingress-nginx service") - assert.NotNil(ginkgo.GinkgoT(), svc, "expected a service but none returned") - + svc := f.GetService(f.Namespace, "nginx-ingress-controller") svc.Spec.Ports = append(svc.Spec.Ports, corev1.ServicePort{ Name: framework.EchoService, Port: 8080, TargetPort: intstr.FromInt(8080), }) - _, err = f.KubeClientSet. + _, err := f.KubeClientSet. CoreV1(). Services(f.Namespace). Update(context.TODO(), svc, metav1.UpdateOptions{}) assert.Nil(ginkgo.GinkgoT(), err, "unexpected error updating service") - // wait for update and nginx reload and new endpoint is available - framework.Sleep() - f.WaitForNginxConfiguration( func(cfg string) bool { return strings.Contains(cfg, fmt.Sprintf(`ngx.var.proxy_upstream_name="tcp-%v-%v-80"`, f.Namespace, framework.EchoService)) }) - ip := f.GetNginxIP() - f.HTTPTestClient(). GET("/"). WithURL(fmt.Sprintf("http://%v:8080", ip)). @@ -122,44 +104,25 @@ var _ = framework.IngressNginxDescribe("[TCP] tcp-services", func() { } f.EnsureService(externalService) - // Expose the `external name` port on the `ingress-nginx` service - svc, err := f.KubeClientSet. - CoreV1(). - Services(f.Namespace). - Get(context.TODO(), "nginx-ingress-controller", metav1.GetOptions{}) - assert.Nil(ginkgo.GinkgoT(), err, "unexpected error obtaining ingress-nginx service") - assert.NotNil(ginkgo.GinkgoT(), svc, "expected a service but none returned") - + // Expose the `external name` port on the `ingress-nginx-controller` service + svc := f.GetService(f.Namespace, "nginx-ingress-controller") svc.Spec.Ports = append(svc.Spec.Ports, corev1.ServicePort{ Name: "dns-svc", Port: 5353, TargetPort: intstr.FromInt(5353), }) - _, err = f.KubeClientSet. + _, err := f.KubeClientSet. CoreV1(). Services(f.Namespace). Update(context.TODO(), svc, metav1.UpdateOptions{}) assert.Nil(ginkgo.GinkgoT(), err, "unexpected error updating service") // Update the TCP configmap to link port 5353 to the DNS external name service - config, err := f.KubeClientSet. - CoreV1(). - ConfigMaps(f.Namespace). - Get(context.TODO(), "tcp-services", metav1.GetOptions{}) - assert.Nil(ginkgo.GinkgoT(), err, "unexpected error obtaining tcp-services configmap") - assert.NotNil(ginkgo.GinkgoT(), config, "expected a configmap but none returned") - - if config.Data == nil { - config.Data = map[string]string{} + config := f.GetConfigMap(f.Namespace, "tcp-services") + config.Data = map[string]string{ + "5353": fmt.Sprintf("%v/dns-external-name-svc:5353", f.Namespace), } - - config.Data["5353"] = fmt.Sprintf("%v/dns-external-name-svc:5353", f.Namespace) - - _, err = f.KubeClientSet. - CoreV1(). - ConfigMaps(f.Namespace). - Update(context.TODO(), config, metav1.UpdateOptions{}) - assert.Nil(ginkgo.GinkgoT(), err, "unexpected error updating configmap") + f.EnsureConfigMap(config) // Validate that the generated nginx config contains the expected `proxy_upstream_name` value f.WaitForNginxConfiguration( @@ -168,7 +131,6 @@ var _ = framework.IngressNginxDescribe("[TCP] tcp-services", func() { }) // Execute the test. Use the `external name` service to resolve a domain name. - ip := f.GetNginxIP() resolver := net.Resolver{ PreferGo: true, Dial: func(ctx context.Context, network, address string) (net.Conn, error) { @@ -203,4 +165,57 @@ var _ = framework.IngressNginxDescribe("[TCP] tcp-services", func() { assert.Nil(ginkgo.GinkgoT(), err, "unexpected error from DNS resolver") assert.Contains(ginkgo.GinkgoT(), ips, "8.8.4.4") }) + + ginkgo.It("should reload after an update in the configuration", func() { + + ginkgo.By("setting up a first deployment") + f.NewEchoDeployment(framework.WithDeploymentName("first-service")) + + cm := f.GetConfigMap(f.Namespace, "tcp-services") + cm.Data = map[string]string{ + "8080": fmt.Sprintf("%v/first-service:80", f.Namespace), + } + f.EnsureConfigMap(cm) + + checksumRegex := regexp.MustCompile(`Configuration checksum:\s+(\d+)`) + checksum := "" + + f.WaitForNginxConfiguration( + func(cfg string) bool { + // before returning, extract the current checksum + match := checksumRegex.FindStringSubmatch(cfg) + if len(match) > 0 { + checksum = match[1] + } + + return strings.Contains(cfg, fmt.Sprintf(`ngx.var.proxy_upstream_name="tcp-%v-first-service-80"`, + f.Namespace)) + }) + assert.NotEmpty(ginkgo.GinkgoT(), checksum) + + ginkgo.By("updating the tcp service to a second deployment") + f.NewEchoDeployment(framework.WithDeploymentName("second-service")) + + cm = f.GetConfigMap(f.Namespace, "tcp-services") + cm.Data["8080"] = fmt.Sprintf("%v/second-service:80", f.Namespace) + f.EnsureConfigMap(cm) + + newChecksum := "" + f.WaitForNginxConfiguration( + func(cfg string) bool { + match := checksumRegex.FindStringSubmatch(cfg) + if len(match) > 0 { + newChecksum = match[1] + } + + return strings.Contains(cfg, fmt.Sprintf(`ngx.var.proxy_upstream_name="tcp-%v-second-service-80"`, + f.Namespace)) + }) + assert.NotEqual(ginkgo.GinkgoT(), checksum, newChecksum) + + logs, err := f.NginxLogs() + assert.Nil(ginkgo.GinkgoT(), err, "obtaining nginx logs") + assert.Contains(ginkgo.GinkgoT(), logs, "Backend successfully reloaded") + }) + })