Add E2E test to verify that changes to one or more configmap trigger an update

Signed-off-by: Hervé Werner <dud225@hotmail.com>

internal/ingress/controller/store/store.go

@@ -702,7 +702,6 @@ func New(
 		},
 	}
 
-	// TODO: add e2e test to verify that changes to one or more configmap trigger an update
 	changeTriggerUpdate := func(name string) bool {
 		return name == configmap || name == tcp || name == udp
 	}

test/e2e/annotations/fastcgi.go

@@ -21,7 +21,6 @@ import (
 	"strings"
 
 	"github.com/onsi/ginkgo/v2"
-	"github.com/stretchr/testify/assert"
 	corev1 "k8s.io/api/core/v1"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -81,9 +80,7 @@ var _ = framework.DescribeAnnotation("backend-protocol - FastCGI", func() {
 			},
 		}
 
-		cm, err := f.EnsureConfigMap(configuration)
+		f.EnsureConfigMap(configuration)
-		assert.Nil(ginkgo.GinkgoT(), err, "creating configmap")
-		assert.NotNil(ginkgo.GinkgoT(), cm, "expected a configmap but none returned")
 
 		host := "fastcgi-params-configmap"
 

test/e2e/framework/framework.go

@@ -391,7 +391,7 @@ func (f *Framework) UpdateNginxConfigMapData(key string, value string) {
 }
 
 // WaitForReload calls the passed function and
-// asser it has caused at least 1 reload.
+// asserts it has caused at least 1 reload.
 func (f *Framework) WaitForReload(fn func()) {
 	initialReloadCount := getReloadCount(f.pod, f.Namespace, f.KubeClientSet)
 

test/e2e/framework/influxdb.go

@@ -68,9 +68,7 @@ func (f *Framework) NewInfluxDBDeployment() {
 		},
 	}
 
-	cm, err := f.EnsureConfigMap(configuration)
+	f.EnsureConfigMap(configuration)
-	assert.Nil(ginkgo.GinkgoT(), err, "creating an Influxdb deployment")
-	assert.NotNil(ginkgo.GinkgoT(), cm, "expected a configmap but none returned")
 
 	deployment := &appsv1.Deployment{
 		ObjectMeta: metav1.ObjectMeta{
@@ -136,7 +134,7 @@ func (f *Framework) NewInfluxDBDeployment() {
 
 	d := f.EnsureDeployment(deployment)
 
-	err = waitForPodsReady(f.KubeClientSet, DefaultTimeout, 1, f.Namespace, metav1.ListOptions{
+	err := waitForPodsReady(f.KubeClientSet, DefaultTimeout, 1, f.Namespace, metav1.ListOptions{
 		LabelSelector: fields.SelectorFromSet(fields.Set(d.Spec.Template.ObjectMeta.Labels)).String(),
 	})
 	assert.Nil(ginkgo.GinkgoT(), err, "waiting for influxdb pod to become ready")

test/e2e/framework/k8s.go

@@ -25,9 +25,7 @@ import (
 	"github.com/onsi/ginkgo/v2"
 	"github.com/stretchr/testify/assert"
 	appsv1 "k8s.io/api/apps/v1"
-	api "k8s.io/api/core/v1"
 	core "k8s.io/api/core/v1"
-	v1 "k8s.io/api/core/v1"
 	networking "k8s.io/api/networking/v1"
 	k8sErrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -36,8 +34,8 @@ import (
 	"k8s.io/client-go/kubernetes"
 )
 
-// EnsureSecret creates a Secret object or returns it if it already exists.
+// EnsureSecret creates a Secret object or returns it.
-func (f *Framework) EnsureSecret(secret *api.Secret) *api.Secret {
+func (f *Framework) EnsureSecret(secret *core.Secret) *core.Secret {
 	err := createSecretWithRetries(f.KubeClientSet, secret.Namespace, secret)
 	assert.Nil(ginkgo.GinkgoT(), err, "creating secret")
 
@@ -48,17 +46,30 @@ func (f *Framework) EnsureSecret(secret *api.Secret) *api.Secret {
 	return s
 }
 
-// EnsureConfigMap creates a ConfigMap object or returns it if it already exists.
+// GetConfigMap gets a ConfigMap object from the given namespace, name and returns it, throws error if it does not exist.
-func (f *Framework) EnsureConfigMap(configMap *api.ConfigMap) (*api.ConfigMap, error) {
+func (f *Framework) GetConfigMap(namespace string, name string) *core.ConfigMap {
-	cm, err := f.KubeClientSet.CoreV1().ConfigMaps(configMap.Namespace).Create(context.TODO(), configMap, metav1.CreateOptions{})
+	cm, err := f.KubeClientSet.CoreV1().ConfigMaps(namespace).Get(context.TODO(), name, metav1.GetOptions{})
-	if err != nil {
+	assert.Nil(ginkgo.GinkgoT(), err, "getting configmap")
-		if k8sErrors.IsAlreadyExists(err) {
+	assert.NotNil(ginkgo.GinkgoT(), cm, "expected a configmap but none returned")
-			return f.KubeClientSet.CoreV1().ConfigMaps(configMap.Namespace).Update(context.TODO(), configMap, metav1.UpdateOptions{})
+	return cm
-		}
+}
-		return nil, err
+
+// EnsureConfigMap creates or updates an existing ConfigMap object or returns it.
+func (f *Framework) EnsureConfigMap(configMap *core.ConfigMap) *core.ConfigMap {
+	cm := configMap.DeepCopy()
+	// Clean out ResourceVersion field if present
+	if cm.ObjectMeta.ResourceVersion != "" {
+		cm.ObjectMeta.ResourceVersion = ""
 	}
 
-	return cm, nil
+	res, err := f.KubeClientSet.CoreV1().ConfigMaps(configMap.Namespace).Create(context.TODO(), cm, metav1.CreateOptions{})
+	if k8sErrors.IsAlreadyExists(err) {
+		res, err = f.KubeClientSet.CoreV1().ConfigMaps(configMap.Namespace).Update(context.TODO(), cm, metav1.UpdateOptions{})
+	}
+	assert.Nil(ginkgo.GinkgoT(), err, "updating configmap")
+	assert.NotNil(ginkgo.GinkgoT(), res, "updating configmap")
+
+	return res
 }
 
 // GetIngress gets an Ingress object from the given namespace, name and returns it, throws error if it does not exists.
@@ -293,7 +304,7 @@ func createDeploymentWithRetries(c kubernetes.Interface, namespace string, obj *
 	return retryWithExponentialBackOff(createFunc)
 }
 
-func createSecretWithRetries(c kubernetes.Interface, namespace string, obj *v1.Secret) error {
+func createSecretWithRetries(c kubernetes.Interface, namespace string, obj *core.Secret) error {
 	if obj == nil {
 		return fmt.Errorf("Object provided to create is empty")
 	}
@@ -313,7 +324,7 @@ func createSecretWithRetries(c kubernetes.Interface, namespace string, obj *v1.S
 	return retryWithExponentialBackOff(createFunc)
 }
 
-func createServiceWithRetries(c kubernetes.Interface, namespace string, obj *v1.Service) error {
+func createServiceWithRetries(c kubernetes.Interface, namespace string, obj *core.Service) error {
 	if obj == nil {
 		return fmt.Errorf("Object provided to create is empty")
 	}

test/e2e/settings/configmap_change.go

@@ -73,5 +73,9 @@ var _ = framework.DescribeSetting("Configmap change", func() {
 				return strings.ContainsAny(cfg, "error_log  /var/log/nginx/error.log debug;")
 			})
 		assert.NotEqual(ginkgo.GinkgoT(), checksum, newChecksum)
+
+		logs, err := f.NginxLogs()
+		assert.Nil(ginkgo.GinkgoT(), err, "obtaining nginx logs")
+		assert.Contains(ginkgo.GinkgoT(), logs, "Backend successfully reloaded")
 	})
 })

test/e2e/settings/ocsp/ocsp.go

@@ -85,7 +85,7 @@ var _ = framework.DescribeSetting("OCSP", func() {
 		cfsslDB, err := os.ReadFile("empty.db")
 		assert.Nil(ginkgo.GinkgoT(), err)
 
-		cmap, err := f.EnsureConfigMap(&corev1.ConfigMap{
+		f.EnsureConfigMap(&corev1.ConfigMap{
 			ObjectMeta: metav1.ObjectMeta{
 				Name:      "ocspserve",
 				Namespace: f.Namespace,
@@ -95,8 +95,6 @@ var _ = framework.DescribeSetting("OCSP", func() {
 				"db-config.json": []byte(`{"driver":"sqlite3","data_source":"/data/empty.db"}`),
 			},
 		})
-		assert.Nil(ginkgo.GinkgoT(), err)
-		assert.NotNil(ginkgo.GinkgoT(), cmap)
 
 		d, s := ocspserveDeployment(f.Namespace)
 		f.EnsureDeployment(d)

test/e2e/tcpudp/tcp.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"net"
 	"net/http"
+	"regexp"
 	"strings"
 	"time"
 
@@ -36,58 +37,39 @@ import (
 
 var _ = framework.IngressNginxDescribe("[TCP] tcp-services", func() {
 	f := framework.NewDefaultFramework("tcp")
+	var ip string
+
+	ginkgo.BeforeEach(func() {
+		ip = f.GetNginxIP()
+	})
 
 	ginkgo.It("should expose a TCP service", func() {
 		f.NewEchoDeployment()
 
-		config, err := f.KubeClientSet.
+		cm := f.GetConfigMap(f.Namespace, "tcp-services")
-			CoreV1().
+		cm.Data = map[string]string{
-			ConfigMaps(f.Namespace).
+			"8080": fmt.Sprintf("%v/%v:80", f.Namespace, framework.EchoService),
-			Get(context.TODO(), "tcp-services", metav1.GetOptions{})
-		assert.Nil(ginkgo.GinkgoT(), err, "unexpected error obtaining tcp-services configmap")
-		assert.NotNil(ginkgo.GinkgoT(), config, "expected a configmap but none returned")
-
-		if config.Data == nil {
-			config.Data = map[string]string{}
 		}
+		f.EnsureConfigMap(cm)
 
-		config.Data["8080"] = fmt.Sprintf("%v/%v:80", f.Namespace, framework.EchoService)
+		svc := f.GetService(f.Namespace, "nginx-ingress-controller")
-
-		_, err = f.KubeClientSet.
-			CoreV1().
-			ConfigMaps(f.Namespace).
-			Update(context.TODO(), config, metav1.UpdateOptions{})
-		assert.Nil(ginkgo.GinkgoT(), err, "unexpected error updating configmap")
-
-		svc, err := f.KubeClientSet.
-			CoreV1().
-			Services(f.Namespace).
-			Get(context.TODO(), "nginx-ingress-controller", metav1.GetOptions{})
-		assert.Nil(ginkgo.GinkgoT(), err, "unexpected error obtaining ingress-nginx service")
-		assert.NotNil(ginkgo.GinkgoT(), svc, "expected a service but none returned")
-
 		svc.Spec.Ports = append(svc.Spec.Ports, corev1.ServicePort{
 			Name:       framework.EchoService,
 			Port:       8080,
 			TargetPort: intstr.FromInt(8080),
 		})
-		_, err = f.KubeClientSet.
+		_, err := f.KubeClientSet.
 			CoreV1().
 			Services(f.Namespace).
 			Update(context.TODO(), svc, metav1.UpdateOptions{})
 		assert.Nil(ginkgo.GinkgoT(), err, "unexpected error updating service")
 
-		// wait for update and nginx reload and new endpoint is available
-		framework.Sleep()
-
 		f.WaitForNginxConfiguration(
 			func(cfg string) bool {
 				return strings.Contains(cfg, fmt.Sprintf(`ngx.var.proxy_upstream_name="tcp-%v-%v-80"`,
 					f.Namespace, framework.EchoService))
 			})
 
-		ip := f.GetNginxIP()
-
 		f.HTTPTestClient().
 			GET("/").
 			WithURL(fmt.Sprintf("http://%v:8080", ip)).
@@ -122,44 +104,25 @@ var _ = framework.IngressNginxDescribe("[TCP] tcp-services", func() {
 		}
 		f.EnsureService(externalService)
 
-		// Expose the `external name` port on the `ingress-nginx` service
+		// Expose the `external name` port on the `ingress-nginx-controller` service
-		svc, err := f.KubeClientSet.
+		svc := f.GetService(f.Namespace, "nginx-ingress-controller")
-			CoreV1().
-			Services(f.Namespace).
-			Get(context.TODO(), "nginx-ingress-controller", metav1.GetOptions{})
-		assert.Nil(ginkgo.GinkgoT(), err, "unexpected error obtaining ingress-nginx service")
-		assert.NotNil(ginkgo.GinkgoT(), svc, "expected a service but none returned")
-
 		svc.Spec.Ports = append(svc.Spec.Ports, corev1.ServicePort{
 			Name:       "dns-svc",
 			Port:       5353,
 			TargetPort: intstr.FromInt(5353),
 		})
-		_, err = f.KubeClientSet.
+		_, err := f.KubeClientSet.
 			CoreV1().
 			Services(f.Namespace).
 			Update(context.TODO(), svc, metav1.UpdateOptions{})
 		assert.Nil(ginkgo.GinkgoT(), err, "unexpected error updating service")
 
 		// Update the TCP configmap to link port 5353 to the DNS external name service
-		config, err := f.KubeClientSet.
+		config := f.GetConfigMap(f.Namespace, "tcp-services")
-			CoreV1().
+		config.Data = map[string]string{
-			ConfigMaps(f.Namespace).
+			"5353": fmt.Sprintf("%v/dns-external-name-svc:5353", f.Namespace),
-			Get(context.TODO(), "tcp-services", metav1.GetOptions{})
-		assert.Nil(ginkgo.GinkgoT(), err, "unexpected error obtaining tcp-services configmap")
-		assert.NotNil(ginkgo.GinkgoT(), config, "expected a configmap but none returned")
-
-		if config.Data == nil {
-			config.Data = map[string]string{}
 		}
-
+		f.EnsureConfigMap(config)
-		config.Data["5353"] = fmt.Sprintf("%v/dns-external-name-svc:5353", f.Namespace)
-
-		_, err = f.KubeClientSet.
-			CoreV1().
-			ConfigMaps(f.Namespace).
-			Update(context.TODO(), config, metav1.UpdateOptions{})
-		assert.Nil(ginkgo.GinkgoT(), err, "unexpected error updating configmap")
 
 		// Validate that the generated nginx config contains the expected `proxy_upstream_name` value
 		f.WaitForNginxConfiguration(
@@ -168,7 +131,6 @@ var _ = framework.IngressNginxDescribe("[TCP] tcp-services", func() {
 			})
 
 		// Execute the test. Use the `external name` service to resolve a domain name.
-		ip := f.GetNginxIP()
 		resolver := net.Resolver{
 			PreferGo: true,
 			Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
@@ -203,4 +165,57 @@ var _ = framework.IngressNginxDescribe("[TCP] tcp-services", func() {
 		assert.Nil(ginkgo.GinkgoT(), err, "unexpected error from DNS resolver")
 		assert.Contains(ginkgo.GinkgoT(), ips, "8.8.4.4")
 	})
+
+	ginkgo.It("should reload after an update in the configuration", func() {
+
+		ginkgo.By("setting up a first deployment")
+		f.NewEchoDeployment(framework.WithDeploymentName("first-service"))
+
+		cm := f.GetConfigMap(f.Namespace, "tcp-services")
+		cm.Data = map[string]string{
+			"8080": fmt.Sprintf("%v/first-service:80", f.Namespace),
+		}
+		f.EnsureConfigMap(cm)
+
+		checksumRegex := regexp.MustCompile(`Configuration checksum:\s+(\d+)`)
+		checksum := ""
+
+		f.WaitForNginxConfiguration(
+			func(cfg string) bool {
+				// before returning, extract the current checksum
+				match := checksumRegex.FindStringSubmatch(cfg)
+				if len(match) > 0 {
+					checksum = match[1]
+				}
+
+				return strings.Contains(cfg, fmt.Sprintf(`ngx.var.proxy_upstream_name="tcp-%v-first-service-80"`,
+					f.Namespace))
+			})
+		assert.NotEmpty(ginkgo.GinkgoT(), checksum)
+
+		ginkgo.By("updating the tcp service to a second deployment")
+		f.NewEchoDeployment(framework.WithDeploymentName("second-service"))
+
+		cm = f.GetConfigMap(f.Namespace, "tcp-services")
+		cm.Data["8080"] = fmt.Sprintf("%v/second-service:80", f.Namespace)
+		f.EnsureConfigMap(cm)
+
+		newChecksum := ""
+		f.WaitForNginxConfiguration(
+			func(cfg string) bool {
+				match := checksumRegex.FindStringSubmatch(cfg)
+				if len(match) > 0 {
+					newChecksum = match[1]
+				}
+
+				return strings.Contains(cfg, fmt.Sprintf(`ngx.var.proxy_upstream_name="tcp-%v-second-service-80"`,
+					f.Namespace))
+			})
+		assert.NotEqual(ginkgo.GinkgoT(), checksum, newChecksum)
+
+		logs, err := f.NginxLogs()
+		assert.Nil(ginkgo.GinkgoT(), err, "obtaining nginx logs")
+		assert.Contains(ginkgo.GinkgoT(), logs, "Backend successfully reloaded")
+	})
+
 })