DevFW-CICD/ingress-nginx-helm
16
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add E2E test to verify that changes to one or more configmap trigger an update

Browse source 
Signed-off-by: Hervé Werner <dud225@hotmail.com>
This commit is contained in:
Hervé Werner 2023-01-20 17:26:46 +01:00
parent ad81bf5a30
commit 74d73ff076
8 changed files with 107 additions and 85 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
internal/ingress/controller/store/store.go
View file

@ -702,7 +702,6 @@ func New(
},
}
// TODO: add e2e test to verify that changes to one or more configmap trigger an update
changeTriggerUpdate := func(name string) bool {
return name == configmap || name == tcp || name == udp
}

5
test/e2e/annotations/fastcgi.go
View file

@ -21,7 +21,6 @@ import (
"strings"
"github.com/onsi/ginkgo/v2"
"github.com/stretchr/testify/assert"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@ -81,9 +80,7 @@ var _ = framework.DescribeAnnotation("backend-protocol - FastCGI", func() {
},
}
cm, err := f.EnsureConfigMap(configuration)
assert.Nil(ginkgo.GinkgoT(), err, "creating configmap")
assert.NotNil(ginkgo.GinkgoT(), cm, "expected a configmap but none returned")
f.EnsureConfigMap(configuration)
host := "fastcgi-params-configmap"

2
test/e2e/framework/framework.go
View file

@ -391,7 +391,7 @@ func (f *Framework) UpdateNginxConfigMapData(key string, value string) {
}
// WaitForReload calls the passed function and
// asser it has caused at least 1 reload.
// asserts it has caused at least 1 reload.
func (f *Framework) WaitForReload(fn func()) {
initialReloadCount := getReloadCount(f.pod, f.Namespace, f.KubeClientSet)

6
test/e2e/framework/influxdb.go
View file

@ -68,9 +68,7 @@ func (f *Framework) NewInfluxDBDeployment() {
},
}
cm, err := f.EnsureConfigMap(configuration)
assert.Nil(ginkgo.GinkgoT(), err, "creating an Influxdb deployment")
assert.NotNil(ginkgo.GinkgoT(), cm, "expected a configmap but none returned")
f.EnsureConfigMap(configuration)
deployment := &appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
@ -136,7 +134,7 @@ func (f *Framework) NewInfluxDBDeployment() {
d := f.EnsureDeployment(deployment)
err = waitForPodsReady(f.KubeClientSet, DefaultTimeout, 1, f.Namespace, metav1.ListOptions{
err := waitForPodsReady(f.KubeClientSet, DefaultTimeout, 1, f.Namespace, metav1.ListOptions{
LabelSelector: fields.SelectorFromSet(fields.Set(d.Spec.Template.ObjectMeta.Labels)).String(),
})
assert.Nil(ginkgo.GinkgoT(), err, "waiting for influxdb pod to become ready")

41
test/e2e/framework/k8s.go
View file

@ -25,9 +25,7 @@ import (
"github.com/onsi/ginkgo/v2"
"github.com/stretchr/testify/assert"
appsv1 "k8s.io/api/apps/v1"
api "k8s.io/api/core/v1"
core "k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
networking "k8s.io/api/networking/v1"
k8sErrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@ -36,8 +34,8 @@ import (
"k8s.io/client-go/kubernetes"
)
// EnsureSecret creates a Secret object or returns it if it already exists.
func (f *Framework) EnsureSecret(secret *api.Secret) *api.Secret {
// EnsureSecret creates a Secret object or returns it.
func (f *Framework) EnsureSecret(secret *core.Secret) *core.Secret {
err := createSecretWithRetries(f.KubeClientSet, secret.Namespace, secret)
assert.Nil(ginkgo.GinkgoT(), err, "creating secret")
@ -48,17 +46,30 @@ func (f *Framework) EnsureSecret(secret *api.Secret) *api.Secret {
return s
}
// EnsureConfigMap creates a ConfigMap object or returns it if it already exists.
func (f *Framework) EnsureConfigMap(configMap *api.ConfigMap) (*api.ConfigMap, error) {
cm, err := f.KubeClientSet.CoreV1().ConfigMaps(configMap.Namespace).Create(context.TODO(), configMap, metav1.CreateOptions{})
if err != nil {
if k8sErrors.IsAlreadyExists(err) {
return f.KubeClientSet.CoreV1().ConfigMaps(configMap.Namespace).Update(context.TODO(), configMap, metav1.UpdateOptions{})
}
return nil, err
// GetConfigMap gets a ConfigMap object from the given namespace, name and returns it, throws error if it does not exist.
func (f *Framework) GetConfigMap(namespace string, name string) *core.ConfigMap {
cm, err := f.KubeClientSet.CoreV1().ConfigMaps(namespace).Get(context.TODO(), name, metav1.GetOptions{})
assert.Nil(ginkgo.GinkgoT(), err, "getting configmap")
assert.NotNil(ginkgo.GinkgoT(), cm, "expected a configmap but none returned")
return cm
}
// EnsureConfigMap creates or updates an existing ConfigMap object or returns it.
func (f *Framework) EnsureConfigMap(configMap *core.ConfigMap) *core.ConfigMap {
cm := configMap.DeepCopy()
// Clean out ResourceVersion field if present
if cm.ObjectMeta.ResourceVersion != "" {
cm.ObjectMeta.ResourceVersion = ""
}
return cm, nil
res, err := f.KubeClientSet.CoreV1().ConfigMaps(configMap.Namespace).Create(context.TODO(), cm, metav1.CreateOptions{})
if k8sErrors.IsAlreadyExists(err) {
res, err = f.KubeClientSet.CoreV1().ConfigMaps(configMap.Namespace).Update(context.TODO(), cm, metav1.UpdateOptions{})
}
assert.Nil(ginkgo.GinkgoT(), err, "updating configmap")
assert.NotNil(ginkgo.GinkgoT(), res, "updating configmap")
return res
}
// GetIngress gets an Ingress object from the given namespace, name and returns it, throws error if it does not exists.
@ -293,7 +304,7 @@ func createDeploymentWithRetries(c kubernetes.Interface, namespace string, obj *
return retryWithExponentialBackOff(createFunc)
}
func createSecretWithRetries(c kubernetes.Interface, namespace string, obj *v1.Secret) error {
func createSecretWithRetries(c kubernetes.Interface, namespace string, obj *core.Secret) error {
if obj == nil {
return fmt.Errorf("Object provided to create is empty")
}
@ -313,7 +324,7 @@ func createSecretWithRetries(c kubernetes.Interface, namespace string, obj *v1.S
return retryWithExponentialBackOff(createFunc)
}
func createServiceWithRetries(c kubernetes.Interface, namespace string, obj *v1.Service) error {
func createServiceWithRetries(c kubernetes.Interface, namespace string, obj *core.Service) error {
if obj == nil {
return fmt.Errorf("Object provided to create is empty")
}

4
test/e2e/settings/configmap_change.go
View file

@ -73,5 +73,9 @@ var _ = framework.DescribeSetting("Configmap change", func() {
return strings.ContainsAny(cfg, "error_log /var/log/nginx/error.log debug;")
})
assert.NotEqual(ginkgo.GinkgoT(), checksum, newChecksum)
logs, err := f.NginxLogs()
assert.Nil(ginkgo.GinkgoT(), err, "obtaining nginx logs")
assert.Contains(ginkgo.GinkgoT(), logs, "Backend successfully reloaded")
})
})

4
test/e2e/settings/ocsp/ocsp.go
View file

@ -85,7 +85,7 @@ var _ = framework.DescribeSetting("OCSP", func() {
cfsslDB, err := os.ReadFile("empty.db")
assert.Nil(ginkgo.GinkgoT(), err)
cmap, err := f.EnsureConfigMap(&corev1.ConfigMap{
f.EnsureConfigMap(&corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: "ocspserve",
Namespace: f.Namespace,
@ -95,8 +95,6 @@ var _ = framework.DescribeSetting("OCSP", func() {
"db-config.json": []byte(`{"driver":"sqlite3","data_source":"/data/empty.db"}`),
},
})
assert.Nil(ginkgo.GinkgoT(), err)
assert.NotNil(ginkgo.GinkgoT(), cmap)
d, s := ocspserveDeployment(f.Namespace)
f.EnsureDeployment(d)

129
test/e2e/tcpudp/tcp.go
View file

@ -21,6 +21,7 @@ import (
"fmt"
"net"
"net/http"
"regexp"
"strings"
"time"
@ -36,58 +37,39 @@ import (
var _ = framework.IngressNginxDescribe("[TCP] tcp-services", func() {
f := framework.NewDefaultFramework("tcp")
var ip string
ginkgo.BeforeEach(func() {
ip = f.GetNginxIP()
})
ginkgo.It("should expose a TCP service", func() {
f.NewEchoDeployment()
config, err := f.KubeClientSet.
CoreV1().
ConfigMaps(f.Namespace).
Get(context.TODO(), "tcp-services", metav1.GetOptions{})
assert.Nil(ginkgo.GinkgoT(), err, "unexpected error obtaining tcp-services configmap")
assert.NotNil(ginkgo.GinkgoT(), config, "expected a configmap but none returned")
if config.Data == nil {
config.Data = map[string]string{}
cm := f.GetConfigMap(f.Namespace, "tcp-services")
cm.Data = map[string]string{
"8080": fmt.Sprintf("%v/%v:80", f.Namespace, framework.EchoService),
}
f.EnsureConfigMap(cm)
config.Data["8080"] = fmt.Sprintf("%v/%v:80", f.Namespace, framework.EchoService)
_, err = f.KubeClientSet.
CoreV1().
ConfigMaps(f.Namespace).
Update(context.TODO(), config, metav1.UpdateOptions{})
assert.Nil(ginkgo.GinkgoT(), err, "unexpected error updating configmap")
svc, err := f.KubeClientSet.
CoreV1().
Services(f.Namespace).
Get(context.TODO(), "nginx-ingress-controller", metav1.GetOptions{})
assert.Nil(ginkgo.GinkgoT(), err, "unexpected error obtaining ingress-nginx service")
assert.NotNil(ginkgo.GinkgoT(), svc, "expected a service but none returned")
svc := f.GetService(f.Namespace, "nginx-ingress-controller")
svc.Spec.Ports = append(svc.Spec.Ports, corev1.ServicePort{
Name: framework.EchoService,
Port: 8080,
TargetPort: intstr.FromInt(8080),
})
_, err = f.KubeClientSet.
_, err := f.KubeClientSet.
CoreV1().
Services(f.Namespace).
Update(context.TODO(), svc, metav1.UpdateOptions{})
assert.Nil(ginkgo.GinkgoT(), err, "unexpected error updating service")
// wait for update and nginx reload and new endpoint is available
framework.Sleep()
f.WaitForNginxConfiguration(
func(cfg string) bool {
return strings.Contains(cfg, fmt.Sprintf(`ngx.var.proxy_upstream_name="tcp-%v-%v-80"`,
f.Namespace, framework.EchoService))
})
ip := f.GetNginxIP()
f.HTTPTestClient().
GET("/").
WithURL(fmt.Sprintf("http://%v:8080", ip)).
@ -122,44 +104,25 @@ var _ = framework.IngressNginxDescribe("[TCP] tcp-services", func() {
}
f.EnsureService(externalService)
// Expose the `external name` port on the `ingress-nginx` service
svc, err := f.KubeClientSet.
CoreV1().
Services(f.Namespace).
Get(context.TODO(), "nginx-ingress-controller", metav1.GetOptions{})
assert.Nil(ginkgo.GinkgoT(), err, "unexpected error obtaining ingress-nginx service")
assert.NotNil(ginkgo.GinkgoT(), svc, "expected a service but none returned")
// Expose the `external name` port on the `ingress-nginx-controller` service
svc := f.GetService(f.Namespace, "nginx-ingress-controller")
svc.Spec.Ports = append(svc.Spec.Ports, corev1.ServicePort{
Name: "dns-svc",
Port: 5353,
TargetPort: intstr.FromInt(5353),
})
_, err = f.KubeClientSet.
_, err := f.KubeClientSet.
CoreV1().
Services(f.Namespace).
Update(context.TODO(), svc, metav1.UpdateOptions{})
assert.Nil(ginkgo.GinkgoT(), err, "unexpected error updating service")
// Update the TCP configmap to link port 5353 to the DNS external name service
config, err := f.KubeClientSet.
CoreV1().
ConfigMaps(f.Namespace).
Get(context.TODO(), "tcp-services", metav1.GetOptions{})
assert.Nil(ginkgo.GinkgoT(), err, "unexpected error obtaining tcp-services configmap")
assert.NotNil(ginkgo.GinkgoT(), config, "expected a configmap but none returned")
if config.Data == nil {
config.Data = map[string]string{}
config := f.GetConfigMap(f.Namespace, "tcp-services")
config.Data = map[string]string{
"5353": fmt.Sprintf("%v/dns-external-name-svc:5353", f.Namespace),
}
config.Data["5353"] = fmt.Sprintf("%v/dns-external-name-svc:5353", f.Namespace)
_, err = f.KubeClientSet.
CoreV1().
ConfigMaps(f.Namespace).
Update(context.TODO(), config, metav1.UpdateOptions{})
assert.Nil(ginkgo.GinkgoT(), err, "unexpected error updating configmap")
f.EnsureConfigMap(config)
// Validate that the generated nginx config contains the expected `proxy_upstream_name` value
f.WaitForNginxConfiguration(
@ -168,7 +131,6 @@ var _ = framework.IngressNginxDescribe("[TCP] tcp-services", func() {
})
// Execute the test. Use the `external name` service to resolve a domain name.
ip := f.GetNginxIP()
resolver := net.Resolver{
PreferGo: true,
Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
@ -203,4 +165,57 @@ var _ = framework.IngressNginxDescribe("[TCP] tcp-services", func() {
assert.Nil(ginkgo.GinkgoT(), err, "unexpected error from DNS resolver")
assert.Contains(ginkgo.GinkgoT(), ips, "8.8.4.4")
})
ginkgo.It("should reload after an update in the configuration", func() {
ginkgo.By("setting up a first deployment")
f.NewEchoDeployment(framework.WithDeploymentName("first-service"))
cm := f.GetConfigMap(f.Namespace, "tcp-services")
cm.Data = map[string]string{
"8080": fmt.Sprintf("%v/first-service:80", f.Namespace),
}
f.EnsureConfigMap(cm)
checksumRegex := regexp.MustCompile(`Configuration checksum:\s+(\d+)`)
checksum := ""
f.WaitForNginxConfiguration(
func(cfg string) bool {
// before returning, extract the current checksum
match := checksumRegex.FindStringSubmatch(cfg)
if len(match) > 0 {
checksum = match[1]
}
return strings.Contains(cfg, fmt.Sprintf(`ngx.var.proxy_upstream_name="tcp-%v-first-service-80"`,
f.Namespace))
})
assert.NotEmpty(ginkgo.GinkgoT(), checksum)
ginkgo.By("updating the tcp service to a second deployment")
f.NewEchoDeployment(framework.WithDeploymentName("second-service"))
cm = f.GetConfigMap(f.Namespace, "tcp-services")
cm.Data["8080"] = fmt.Sprintf("%v/second-service:80", f.Namespace)
f.EnsureConfigMap(cm)
newChecksum := ""
f.WaitForNginxConfiguration(
func(cfg string) bool {
match := checksumRegex.FindStringSubmatch(cfg)
if len(match) > 0 {
newChecksum = match[1]
}
return strings.Contains(cfg, fmt.Sprintf(`ngx.var.proxy_upstream_name="tcp-%v-second-service-80"`,
f.Namespace))
})
assert.NotEqual(ginkgo.GinkgoT(), checksum, newChecksum)
logs, err := f.NginxLogs()
assert.Nil(ginkgo.GinkgoT(), err, "obtaining nginx logs")
assert.Contains(ginkgo.GinkgoT(), logs, "Backend successfully reloaded")
})
})