From 60bf6ba6429feb4a5d8b485d1bcee87187fbc368 Mon Sep 17 00:00:00 2001 From: Brendan Kamp Date: Mon, 12 Jun 2023 12:25:49 +0200 Subject: [PATCH] chore: move httpbun to be part of framework (#9955) Signed-off-by: Spazzy --- test/e2e/HTTPBUN_IMAGE | 1 + test/e2e/annotations/auth.go | 53 +----- test/e2e/annotations/grpc.go | 17 +- test/e2e/annotations/satisfy.go | 17 +- test/e2e/framework/deployment.go | 111 +++++++++---- test/e2e/framework/framework.go | 29 +++- test/e2e/run-e2e-suite.sh | 2 + .../servicebackend/service_externalname.go | 153 +++++++++++------- test/e2e/settings/brotli.go | 9 +- .../settings/disable_service_external_name.go | 16 +- test/e2e/settings/global_external_auth.go | 10 +- test/e2e/settings/listen_nondefault_ports.go | 19 +-- test/e2e/settings/ssl_passthrough.go | 32 +++- 13 files changed, 265 insertions(+), 204 deletions(-) create mode 100644 test/e2e/HTTPBUN_IMAGE diff --git a/test/e2e/HTTPBUN_IMAGE b/test/e2e/HTTPBUN_IMAGE new file mode 100644 index 000000000..2d95865c7 --- /dev/null +++ b/test/e2e/HTTPBUN_IMAGE @@ -0,0 +1 @@ +registry.k8s.io/ingress-nginx/e2e-test-httpbun:v20230505-v0.0.1 diff --git a/test/e2e/annotations/auth.go b/test/e2e/annotations/auth.go index 4ca034825..8011186a1 100644 --- a/test/e2e/annotations/auth.go +++ b/test/e2e/annotations/auth.go @@ -23,7 +23,6 @@ import ( "net/url" "regexp" "strings" - "time" "golang.org/x/crypto/bcrypt" @@ -38,7 +37,7 @@ import ( ) var _ = framework.DescribeAnnotation("auth-*", func() { - f := framework.NewDefaultFramework("auth") + f := framework.NewDefaultFramework("auth", framework.WithHTTPBunEnabled()) ginkgo.BeforeEach(func() { f.NewEchoDeployment() @@ -390,10 +389,10 @@ http { assert.GreaterOrEqual(ginkgo.GinkgoT(), len(e.Subsets), 1, "expected at least one endpoint") assert.GreaterOrEqual(ginkgo.GinkgoT(), len(e.Subsets[0].Addresses), 1, "expected at least one address ready in the endpoint") - httpbunIP := e.Subsets[0].Addresses[0].IP + nginxIP := e.Subsets[0].Addresses[0].IP annotations = map[string]string{ - "nginx.ingress.kubernetes.io/auth-url": fmt.Sprintf("http://%s/cookies/set/alma/armud", httpbunIP), + "nginx.ingress.kubernetes.io/auth-url": fmt.Sprintf("http://%s/cookies/set/alma/armud", nginxIP), "nginx.ingress.kubernetes.io/auth-signin": "http://$host/auth/start", } @@ -457,21 +456,8 @@ http { var ing *networking.Ingress ginkgo.BeforeEach(func() { - f.NewHttpbunDeployment() - - err := framework.WaitForEndpoints(f.KubeClientSet, framework.DefaultTimeout, framework.HTTPBunService, f.Namespace, 1) - assert.Nil(ginkgo.GinkgoT(), err) - - e, err := f.KubeClientSet.CoreV1().Endpoints(f.Namespace).Get(context.TODO(), framework.HTTPBunService, metav1.GetOptions{}) - assert.Nil(ginkgo.GinkgoT(), err) - - assert.GreaterOrEqual(ginkgo.GinkgoT(), len(e.Subsets), 1, "expected at least one endpoint") - assert.GreaterOrEqual(ginkgo.GinkgoT(), len(e.Subsets[0].Addresses), 1, "expected at least one address ready in the endpoint") - - httpbunIP := e.Subsets[0].Addresses[0].IP - annotations = map[string]string{ - "nginx.ingress.kubernetes.io/auth-url": fmt.Sprintf("http://%s/basic-auth/user/password", httpbunIP), + "nginx.ingress.kubernetes.io/auth-url": fmt.Sprintf("http://%s/basic-auth/user/password", f.HTTPBunIP), "nginx.ingress.kubernetes.io/auth-signin": "http://$host/auth/start", } @@ -650,20 +636,8 @@ http { var ing *networking.Ingress ginkgo.BeforeEach(func() { - f.NewHttpbunDeployment() - - var httpbunIP string - - err := framework.WaitForEndpoints(f.KubeClientSet, framework.DefaultTimeout, framework.HTTPBunService, f.Namespace, 1) - assert.Nil(ginkgo.GinkgoT(), err) - - e, err := f.KubeClientSet.CoreV1().Endpoints(f.Namespace).Get(context.TODO(), framework.HTTPBunService, metav1.GetOptions{}) - assert.Nil(ginkgo.GinkgoT(), err) - - httpbunIP = e.Subsets[0].Addresses[0].IP - annotations = map[string]string{ - "nginx.ingress.kubernetes.io/auth-url": fmt.Sprintf("http://%s/basic-auth/user/password", httpbunIP), + "nginx.ingress.kubernetes.io/auth-url": fmt.Sprintf("http://%s/basic-auth/user/password", f.HTTPBunIP), "nginx.ingress.kubernetes.io/auth-signin": "http://$host/auth/start", "nginx.ingress.kubernetes.io/auth-signin-redirect-param": "orig", } @@ -729,23 +703,8 @@ http { barPath := "/bar" ginkgo.BeforeEach(func() { - f.NewHttpbunDeployment() - - err := framework.WaitForEndpoints(f.KubeClientSet, framework.DefaultTimeout, framework.HTTPBunService, f.Namespace, 1) - assert.Nil(ginkgo.GinkgoT(), err) - - framework.Sleep(1 * time.Second) - - e, err := f.KubeClientSet.CoreV1().Endpoints(f.Namespace).Get(context.TODO(), framework.HTTPBunService, metav1.GetOptions{}) - assert.Nil(ginkgo.GinkgoT(), err) - - assert.GreaterOrEqual(ginkgo.GinkgoT(), len(e.Subsets), 1, "expected at least one endpoint") - assert.GreaterOrEqual(ginkgo.GinkgoT(), len(e.Subsets[0].Addresses), 1, "expected at least one address ready in the endpoint") - - httpbunIP := e.Subsets[0].Addresses[0].IP - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/auth-url": fmt.Sprintf("http://%s/basic-auth/user/password", httpbunIP), + "nginx.ingress.kubernetes.io/auth-url": fmt.Sprintf("http://%s/basic-auth/user/password", f.HTTPBunIP), "nginx.ingress.kubernetes.io/auth-signin": "http://$host/auth/start", "nginx.ingress.kubernetes.io/auth-cache-key": "fixed", "nginx.ingress.kubernetes.io/auth-cache-duration": "200 201 401 30m", diff --git a/test/e2e/annotations/grpc.go b/test/e2e/annotations/grpc.go index 2bdac553b..c8e530ead 100644 --- a/test/e2e/annotations/grpc.go +++ b/test/e2e/annotations/grpc.go @@ -37,7 +37,7 @@ import ( ) var _ = framework.DescribeAnnotation("backend-protocol - GRPC", func() { - f := framework.NewDefaultFramework("grpc") + f := framework.NewDefaultFramework("grpc", framework.WithHTTPBunEnabled()) ginkgo.It("should use grpc_pass in the configuration file", func() { f.NewGRPCFortuneTellerDeployment() @@ -124,8 +124,6 @@ var _ = framework.DescribeAnnotation("backend-protocol - GRPC", func() { ginkgo.It("authorization metadata should be overwritten by external auth response headers", func() { f.NewGRPCBinDeployment() - f.NewHttpbunDeployment() - host := "echo" svc := &corev1.Service{ @@ -148,19 +146,8 @@ var _ = framework.DescribeAnnotation("backend-protocol - GRPC", func() { } f.EnsureService(svc) - err := framework.WaitForEndpoints(f.KubeClientSet, framework.DefaultTimeout, framework.HTTPBunService, f.Namespace, 1) - assert.Nil(ginkgo.GinkgoT(), err) - - e, err := f.KubeClientSet.CoreV1().Endpoints(f.Namespace).Get(context.TODO(), framework.HTTPBunService, metav1.GetOptions{}) - assert.Nil(ginkgo.GinkgoT(), err) - - assert.GreaterOrEqual(ginkgo.GinkgoT(), len(e.Subsets), 1, "expected at least one endpoint") - assert.GreaterOrEqual(ginkgo.GinkgoT(), len(e.Subsets[0].Addresses), 1, "expected at least one address ready in the endpoint") - - httpbunIP := e.Subsets[0].Addresses[0].IP - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/auth-url": fmt.Sprintf("http://%s/response-headers?authorization=foo", httpbunIP), + "nginx.ingress.kubernetes.io/auth-url": fmt.Sprintf("http://%s/response-headers?authorization=foo", f.HTTPBunIP), "nginx.ingress.kubernetes.io/auth-response-headers": "Authorization", "nginx.ingress.kubernetes.io/backend-protocol": "GRPC", } diff --git a/test/e2e/annotations/satisfy.go b/test/e2e/annotations/satisfy.go index 758ad21a4..6ba6db33e 100644 --- a/test/e2e/annotations/satisfy.go +++ b/test/e2e/annotations/satisfy.go @@ -17,7 +17,6 @@ limitations under the License. package annotations import ( - "context" "fmt" "net/http" "net/url" @@ -27,13 +26,12 @@ import ( "github.com/stretchr/testify/assert" networking "k8s.io/api/networking/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/ingress-nginx/test/e2e/framework" ) var _ = framework.DescribeAnnotation("satisfy", func() { - f := framework.NewDefaultFramework("satisfy") + f := framework.NewDefaultFramework("satisfy", framework.WithHTTPBunEnabled()) ginkgo.BeforeEach(func() { f.NewEchoDeployment() @@ -84,17 +82,6 @@ var _ = framework.DescribeAnnotation("satisfy", func() { ginkgo.It("should allow multiple auth with satisfy any", func() { host := "auth" - // setup external auth - f.NewHttpbunDeployment() - - err := framework.WaitForEndpoints(f.KubeClientSet, framework.DefaultTimeout, framework.HTTPBunService, f.Namespace, 1) - assert.Nil(ginkgo.GinkgoT(), err) - - e, err := f.KubeClientSet.CoreV1().Endpoints(f.Namespace).Get(context.TODO(), framework.HTTPBunService, metav1.GetOptions{}) - assert.Nil(ginkgo.GinkgoT(), err) - - httpbunIP := e.Subsets[0].Addresses[0].IP - // create basic auth secret at ingress s := f.EnsureSecret(buildSecret("uname", "pwd", "basic-secret", f.Namespace)) @@ -105,7 +92,7 @@ var _ = framework.DescribeAnnotation("satisfy", func() { "nginx.ingress.kubernetes.io/auth-realm": "test basic auth", // annotations for external auth - "nginx.ingress.kubernetes.io/auth-url": fmt.Sprintf("http://%s/basic-auth/user/password", httpbunIP), + "nginx.ingress.kubernetes.io/auth-url": fmt.Sprintf("http://%s/basic-auth/user/password", f.HTTPBunIP), "nginx.ingress.kubernetes.io/auth-signin": "http://$host/auth/start", // set satisfy any diff --git a/test/e2e/framework/deployment.go b/test/e2e/framework/deployment.go index 565b8f4ac..d4e20fbf5 100644 --- a/test/e2e/framework/deployment.go +++ b/test/e2e/framework/deployment.go @@ -43,12 +43,28 @@ const HTTPBunService = "httpbun" // NipService name of external service using nip.io const NIPService = "external-nip" +// HTTPBunImage is the default image that is used to deploy HTTPBun with the framwork +var HTTPBunImage = os.Getenv("HTTPBUN_IMAGE") + +// EchoImage is the default image to be used by the echo service +const EchoImage = "registry.k8s.io/ingress-nginx/e2e-test-echo@sha256:4938d1d91a2b7d19454460a8c1b010b89f6ff92d2987fd889ac3e8fc3b70d91a" + +// TODO: change all Deployment functions to use these options +// in order to reduce complexity and have a unified API accross the +// framework type deploymentOptions struct { - namespace string name string - replicas int - svcAnnotations map[string]string + namespace string image string + port int32 + replicas int + command []string + args []string + env []corev1.EnvVar + volumeMounts []corev1.VolumeMount + volumes []corev1.Volume + svcAnnotations map[string]string + setProbe bool } // WithDeploymentNamespace allows configuring the deployment's namespace @@ -100,22 +116,25 @@ func (f *Framework) NewEchoDeployment(opts ...func(*deploymentOptions)) { namespace: f.Namespace, name: EchoService, replicas: 1, - image: "registry.k8s.io/ingress-nginx/e2e-test-echo@sha256:6fc5aa2994c86575975bb20a5203651207029a0d28e3f491d8a127d08baadab4", + image: EchoImage, } for _, o := range opts { o(options) } - deployment := newDeployment(options.name, options.namespace, options.image, 80, int32(options.replicas), + f.EnsureDeployment(newDeployment( + options.name, + options.namespace, + options.image, + 80, + int32(options.replicas), nil, nil, nil, []corev1.VolumeMount{}, []corev1.Volume{}, true, - ) + )) - f.EnsureDeployment(deployment) - - service := &corev1.Service{ + f.EnsureService(&corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: options.name, Namespace: options.namespace, @@ -134,11 +153,15 @@ func (f *Framework) NewEchoDeployment(opts ...func(*deploymentOptions)) { "app": options.name, }, }, - } + }) - f.EnsureService(service) - - err := WaitForEndpoints(f.KubeClientSet, DefaultTimeout, options.name, options.namespace, options.replicas) + err := WaitForEndpoints( + f.KubeClientSet, + DefaultTimeout, + options.name, + options.namespace, + options.replicas, + ) assert.Nil(ginkgo.GinkgoT(), err, "waiting for endpoints to become ready") } @@ -147,6 +170,12 @@ func BuildNIPHost(ip string) string { return fmt.Sprintf("%s.nip.io", ip) } +// GetNipHost used to generate a nip host for external DNS resolving +// for the instance deployed by the framework +func (f *Framework) GetNIPHost() string { + return BuildNIPHost(f.HTTPBunIP) +} + // BuildNIPExternalNameService used to generate a service pointing to nip.io to // help resolve to an IP address func BuildNIPExternalNameService(f *Framework, ip, portName string) *corev1.Service { @@ -177,22 +206,27 @@ func (f *Framework) NewHttpbunDeployment(opts ...func(*deploymentOptions)) strin namespace: f.Namespace, name: HTTPBunService, replicas: 1, - image: "registry.k8s.io/ingress-nginx/e2e-test-httpbun:v20230505-v0.0.1", + image: HTTPBunImage, } for _, o := range opts { o(options) } - deployment := newDeployment(options.name, options.namespace, options.image, 80, int32(options.replicas), + // Create the HTTPBun Deployment + f.EnsureDeployment(newDeployment( + options.name, + options.namespace, + options.image, + 80, + int32(options.replicas), nil, nil, nil, []corev1.VolumeMount{}, []corev1.Volume{}, true, - ) + )) - f.EnsureDeployment(deployment) - - service := &corev1.Service{ + // Create a service pointing to deployment + f.EnsureService(&corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: options.name, Namespace: options.namespace, @@ -211,14 +245,26 @@ func (f *Framework) NewHttpbunDeployment(opts ...func(*deploymentOptions)) strin "app": options.name, }, }, - } + }) - s := f.EnsureService(service) - - err := WaitForEndpoints(f.KubeClientSet, DefaultTimeout, options.name, options.namespace, options.replicas) + // Wait for deployment to become available + err := WaitForEndpoints( + f.KubeClientSet, + DefaultTimeout, + options.name, + options.namespace, + options.replicas, + ) assert.Nil(ginkgo.GinkgoT(), err, "waiting for endpoints to become ready") - return s.Spec.ClusterIPs[0] + // Get cluster ip for HTTPBun to be used in tests + e, err := f.KubeClientSet. + CoreV1(). + Endpoints(f.Namespace). + Get(context.TODO(), HTTPBunService, metav1.GetOptions{}) + assert.Nil(ginkgo.GinkgoT(), err, "failed to get httpbun endpoint") + + return e.Subsets[0].Addresses[0].IP } // NewSlowEchoDeployment creates a new deployment of the slow echo server image in a particular namespace. @@ -276,13 +322,16 @@ func (f *Framework) NGINXDeployment(name string, cfg string, waitendpoint bool) "nginx.conf": cfg, } - _, err := f.KubeClientSet.CoreV1().ConfigMaps(f.Namespace).Create(context.TODO(), &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: f.Namespace, - }, - Data: cfgMap, - }, metav1.CreateOptions{}) + _, err := f.KubeClientSet. + CoreV1(). + ConfigMaps(f.Namespace). + Create(context.TODO(), &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: f.Namespace, + }, + Data: cfgMap, + }, metav1.CreateOptions{}) assert.Nil(ginkgo.GinkgoT(), err, "creating configmap") deployment := newDeployment(name, f.Namespace, f.GetNginxBaseImage(), 80, 1, diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go index a3d5ea760..69f6dae78 100644 --- a/test/e2e/framework/framework.go +++ b/test/e2e/framework/framework.go @@ -67,16 +67,32 @@ type Framework struct { IngressClass string pod *v1.Pod + // We use httpbun as a service that we route to in our tests through + // the ingress controller. We add it as part of the framework as it + // is used extensively + HTTPBunIP string + HTTPBunEnabled bool +} + +// WithHTTPBunEnabled deploys an instance of HTTPBun for the specific test +func WithHTTPBunEnabled() func(*Framework) { + return func(f *Framework) { + f.HTTPBunEnabled = true + } } // NewDefaultFramework makes a new framework and sets up a BeforeEach/AfterEach for // you (you can write additional before/after each functions). -func NewDefaultFramework(baseName string) *Framework { +func NewDefaultFramework(baseName string, opts ...func(*Framework)) *Framework { defer ginkgo.GinkgoRecover() f := &Framework{ BaseName: baseName, } + // set framework options + for _, o := range opts { + o(f) + } ginkgo.BeforeEach(f.BeforeEach) ginkgo.AfterEach(f.AfterEach) @@ -86,12 +102,16 @@ func NewDefaultFramework(baseName string) *Framework { // NewSimpleFramework makes a new framework that allows the usage of a namespace // for arbitraty tests. -func NewSimpleFramework(baseName string) *Framework { +func NewSimpleFramework(baseName string, opts ...func(*Framework)) *Framework { defer ginkgo.GinkgoRecover() f := &Framework{ BaseName: baseName, } + // set framework options + for _, o := range opts { + o(f) + } ginkgo.BeforeEach(f.CreateEnvironment) ginkgo.AfterEach(f.DestroyEnvironment) @@ -140,6 +160,11 @@ func (f *Framework) BeforeEach() { assert.Nil(ginkgo.GinkgoT(), err, "updating ingress controller pod information") f.WaitForNginxListening(80) + + // If HTTPBun is enabled deploy an instance to the namespace + if f.HTTPBunEnabled { + f.HTTPBunIP = f.NewHttpbunDeployment() + } } // AfterEach deletes the namespace, after reading its events. diff --git a/test/e2e/run-e2e-suite.sh b/test/e2e/run-e2e-suite.sh index a3bf589cd..b56312afd 100755 --- a/test/e2e/run-e2e-suite.sh +++ b/test/e2e/run-e2e-suite.sh @@ -51,6 +51,7 @@ fi BASEDIR=$(dirname "$0") NGINX_BASE_IMAGE=$(cat $BASEDIR/../../NGINX_BASE) +HTTPBUN_IMAGE=$(cat $BASEDIR/HTTPBUN_IMAGE) echo -e "${BGREEN}Granting permissions to ingress-nginx e2e service account...${NC}" kubectl create serviceaccount ingress-nginx-e2e || true @@ -79,6 +80,7 @@ kubectl run --rm \ --env="IS_CHROOT=${IS_CHROOT:-false}"\ --env="E2E_CHECK_LEAKS=${E2E_CHECK_LEAKS}" \ --env="NGINX_BASE_IMAGE=${NGINX_BASE_IMAGE}" \ + --env="HTTPBUN_IMAGE=${HTTPBUN_IMAGE}" \ --overrides='{ "apiVersion": "v1", "spec":{"serviceAccountName": "ingress-nginx-e2e"}}' \ e2e --image=nginx-ingress-controller:e2e diff --git a/test/e2e/servicebackend/service_externalname.go b/test/e2e/servicebackend/service_externalname.go index 2fd6cd080..89ae77b10 100644 --- a/test/e2e/servicebackend/service_externalname.go +++ b/test/e2e/servicebackend/service_externalname.go @@ -35,7 +35,7 @@ import ( ) var _ = framework.IngressNginxDescribe("[Service] Type ExternalName", func() { - f := framework.NewDefaultFramework("type-externalname") + f := framework.NewDefaultFramework("type-externalname", framework.WithHTTPBunEnabled()) ginkgo.It("works with external name set to incomplete fqdn", func() { f.NewEchoDeployment() @@ -43,7 +43,7 @@ var _ = framework.IngressNginxDescribe("[Service] Type ExternalName", func() { svc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: framework.HTTPBunService, + Name: framework.NIPService, Namespace: f.Namespace, }, Spec: corev1.ServiceSpec{ @@ -51,10 +51,15 @@ var _ = framework.IngressNginxDescribe("[Service] Type ExternalName", func() { Type: corev1.ServiceTypeExternalName, }, } - f.EnsureService(svc) - ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.HTTPBunService, 80, nil) + ing := framework.NewSingleIngress(host, + "/", + host, + f.Namespace, + framework.NIPService, + 80, + nil) f.EnsureIngress(ing) f.WaitForNginxServer(host, @@ -70,10 +75,6 @@ var _ = framework.IngressNginxDescribe("[Service] Type ExternalName", func() { }) ginkgo.It("should return 200 for service type=ExternalName without a port defined", func() { - // This is a workaround so we only depend on a self hosted instance of - // httpbun - ip := f.NewHttpbunDeployment() - host := "echo" svc := &corev1.Service{ @@ -82,17 +83,23 @@ var _ = framework.IngressNginxDescribe("[Service] Type ExternalName", func() { Namespace: f.Namespace, }, Spec: corev1.ServiceSpec{ - ExternalName: framework.BuildNIPHost(ip), + ExternalName: f.GetNIPHost(), Type: corev1.ServiceTypeExternalName, }, } - f.EnsureService(svc) annotations := map[string]string{ - "nginx.ingress.kubernetes.io/upstream-vhost": framework.BuildNIPHost(ip), + "nginx.ingress.kubernetes.io/upstream-vhost": f.GetNIPHost(), } - ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.HTTPBunService, 80, annotations) + + ing := framework.NewSingleIngress(host, + "/", + host, + f.Namespace, + framework.HTTPBunService, + 80, + annotations) f.EnsureIngress(ing) f.WaitForNginxServer(host, @@ -108,19 +115,21 @@ var _ = framework.IngressNginxDescribe("[Service] Type ExternalName", func() { }) ginkgo.It("should return 200 for service type=ExternalName with a port defined", func() { - // This is a workaround so we only depend on a self hosted instance of - // httpbun - ip := f.NewHttpbunDeployment() - host := "echo" - svc := framework.BuildNIPExternalNameService(f, ip, host) + svc := framework.BuildNIPExternalNameService(f, f.HTTPBunIP, host) f.EnsureService(svc) annotations := map[string]string{ - "nginx.ingress.kubernetes.io/upstream-vhost": framework.BuildNIPHost(ip), + "nginx.ingress.kubernetes.io/upstream-vhost": f.GetNIPHost(), } - ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.HTTPBunService, 80, annotations) + ing := framework.NewSingleIngress(host, + "/", + host, + f.Namespace, + framework.HTTPBunService, + 80, + annotations) f.EnsureIngress(ing) f.WaitForNginxServer(host, @@ -140,7 +149,7 @@ var _ = framework.IngressNginxDescribe("[Service] Type ExternalName", func() { svc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: framework.HTTPBunService, + Name: framework.NIPService, Namespace: f.Namespace, }, Spec: corev1.ServiceSpec{ @@ -148,10 +157,15 @@ var _ = framework.IngressNginxDescribe("[Service] Type ExternalName", func() { Type: corev1.ServiceTypeExternalName, }, } - f.EnsureService(svc) - ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.HTTPBunService, 80, nil) + ing := framework.NewSingleIngress(host, + "/", + host, + f.Namespace, + framework.NIPService, + 80, + nil) f.EnsureIngress(ing) f.WaitForNginxServer(host, @@ -167,19 +181,22 @@ var _ = framework.IngressNginxDescribe("[Service] Type ExternalName", func() { }) ginkgo.It("should return 200 for service type=ExternalName using a port name", func() { - // This is a workaround so we only depend on a self hosted instance of - // httpbun - ip := f.NewHttpbunDeployment() - host := "echo" - svc := framework.BuildNIPExternalNameService(f, ip, host) + svc := framework.BuildNIPExternalNameService(f, f.HTTPBunIP, host) f.EnsureService(svc) annotations := map[string]string{ - "nginx.ingress.kubernetes.io/upstream-vhost": framework.BuildNIPHost(ip), + "nginx.ingress.kubernetes.io/upstream-vhost": f.GetNIPHost(), } - ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.HTTPBunService, 80, annotations) + ing := framework.NewSingleIngress(host, + "/", + host, + f.Namespace, + framework.HTTPBunService, + 80, + annotations) + namedBackend := networking.IngressBackend{ Service: &networking.IngressServiceBackend{ Name: framework.NIPService, @@ -188,6 +205,7 @@ var _ = framework.IngressNginxDescribe("[Service] Type ExternalName", func() { }, }, } + ing.Spec.Rules[0].HTTP.Paths[0].Backend = namedBackend f.EnsureIngress(ing) @@ -204,10 +222,6 @@ var _ = framework.IngressNginxDescribe("[Service] Type ExternalName", func() { }) ginkgo.It("should return 200 for service type=ExternalName using FQDN with trailing dot", func() { - // This is a workaround so we only depend on a self hosted instance of - // httpbun - ip := f.NewHttpbunDeployment() - host := "echo" svc := &corev1.Service{ @@ -216,14 +230,19 @@ var _ = framework.IngressNginxDescribe("[Service] Type ExternalName", func() { Namespace: f.Namespace, }, Spec: corev1.ServiceSpec{ - ExternalName: framework.BuildNIPHost(ip), + ExternalName: f.GetNIPHost(), Type: corev1.ServiceTypeExternalName, }, } - f.EnsureService(svc) - ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.HTTPBunService, 80, nil) + ing := framework.NewSingleIngress(host, + "/", + host, + f.Namespace, + framework.HTTPBunService, + 80, + nil) f.EnsureIngress(ing) f.WaitForNginxServer(host, @@ -239,20 +258,23 @@ var _ = framework.IngressNginxDescribe("[Service] Type ExternalName", func() { }) ginkgo.It("should update the external name after a service update", func() { - // This is a workaround so we only depend on a self hosted instance of - // httpbun - ip := f.NewHttpbunDeployment() - host := "echo" - svc := framework.BuildNIPExternalNameService(f, ip, host) + svc := framework.BuildNIPExternalNameService(f, f.HTTPBunIP, host) f.EnsureService(svc) annotations := map[string]string{ - "nginx.ingress.kubernetes.io/upstream-vhost": framework.BuildNIPHost(ip), + "nginx.ingress.kubernetes.io/upstream-vhost": f.GetNIPHost(), } - ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.HTTPBunService, 80, annotations) + ing := framework.NewSingleIngress(host, + "/", + host, + f.Namespace, + framework.HTTPBunService, + 80, + annotations) + namedBackend := networking.IngressBackend{ Service: &networking.IngressServiceBackend{ Name: framework.NIPService, @@ -279,14 +301,20 @@ var _ = framework.IngressNginxDescribe("[Service] Type ExternalName", func() { assert.Contains(ginkgo.GinkgoT(), body, `"X-Forwarded-Host": "echo"`) - svc, err := f.KubeClientSet.CoreV1().Services(f.Namespace).Get(context.TODO(), framework.NIPService, metav1.GetOptions{}) + svc, err := f.KubeClientSet. + CoreV1(). + Services(f.Namespace). + Get(context.TODO(), framework.NIPService, metav1.GetOptions{}) assert.Nil(ginkgo.GinkgoT(), err, "unexpected error obtaining external service") - ip = f.NewHttpbunDeployment(framework.WithDeploymentName("eu-server")) - + //Deploy a new instance to switch routing to + ip := f.NewHttpbunDeployment(framework.WithDeploymentName("eu-server")) svc.Spec.ExternalName = framework.BuildNIPHost(ip) - _, err = f.KubeClientSet.CoreV1().Services(f.Namespace).Update(context.Background(), svc, metav1.UpdateOptions{}) + _, err = f.KubeClientSet. + CoreV1(). + Services(f.Namespace). + Update(context.Background(), svc, metav1.UpdateOptions{}) assert.Nil(ginkgo.GinkgoT(), err, "unexpected error updating external service") framework.Sleep() @@ -302,21 +330,31 @@ var _ = framework.IngressNginxDescribe("[Service] Type ExternalName", func() { assert.Contains(ginkgo.GinkgoT(), body, `"X-Forwarded-Host": "echo"`) ginkgo.By("checking the service is updated to use new host") - curlCmd := fmt.Sprintf("curl --fail --silent http://localhost:%v/configuration/backends", nginx.StatusPort) + curlCmd := fmt.Sprintf( + "curl --fail --silent http://localhost:%v/configuration/backends", + nginx.StatusPort, + ) + output, err := f.ExecIngressPod(curlCmd) assert.Nil(ginkgo.GinkgoT(), err) - assert.Contains(ginkgo.GinkgoT(), output, fmt.Sprintf("{\"address\":\"%s\"", framework.BuildNIPHost(ip))) + assert.Contains( + ginkgo.GinkgoT(), + output, + fmt.Sprintf("{\"address\":\"%s\"", framework.BuildNIPHost(ip)), + ) }) ginkgo.It("should sync ingress on external name service addition/deletion", func() { - // This is a workaround so we only depend on a self hosted instance of - // httpbun - ip := f.NewHttpbunDeployment() - host := "echo" // Create the Ingress first - ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.NIPService, 80, nil) + ing := framework.NewSingleIngress(host, + "/", + host, + f.Namespace, + framework.NIPService, + 80, + nil) f.EnsureIngress(ing) f.WaitForNginxServer(host, @@ -332,7 +370,7 @@ var _ = framework.IngressNginxDescribe("[Service] Type ExternalName", func() { Status(http.StatusServiceUnavailable) // Now create the service - svc := framework.BuildNIPExternalNameService(f, ip, host) + svc := framework.BuildNIPExternalNameService(f, f.HTTPBunIP, host) f.EnsureService(svc) framework.Sleep() @@ -345,7 +383,10 @@ var _ = framework.IngressNginxDescribe("[Service] Type ExternalName", func() { Status(http.StatusOK) // And back to 503 after deleting the service - err := f.KubeClientSet.CoreV1().Services(f.Namespace).Delete(context.TODO(), framework.NIPService, metav1.DeleteOptions{}) + err := f.KubeClientSet. + CoreV1(). + Services(f.Namespace). + Delete(context.TODO(), framework.NIPService, metav1.DeleteOptions{}) assert.Nil(ginkgo.GinkgoT(), err, "unexpected error deleting external service") framework.Sleep() diff --git a/test/e2e/settings/brotli.go b/test/e2e/settings/brotli.go index a13678f66..aacaddec5 100644 --- a/test/e2e/settings/brotli.go +++ b/test/e2e/settings/brotli.go @@ -28,14 +28,13 @@ import ( ) var _ = framework.IngressNginxDescribe("brotli", func() { - f := framework.NewDefaultFramework("brotli") + f := framework.NewDefaultFramework( + "brotli", + framework.WithHTTPBunEnabled(), + ) host := "brotli" - ginkgo.BeforeEach(func() { - f.NewHttpbunDeployment() - }) - ginkgo.It("should only compress responses that meet the `brotli-min-length` condition", func() { brotliMinLength := 24 contentEncoding := "application/octet-stream" diff --git a/test/e2e/settings/disable_service_external_name.go b/test/e2e/settings/disable_service_external_name.go index 7f03e5355..4ecf69e81 100644 --- a/test/e2e/settings/disable_service_external_name.go +++ b/test/e2e/settings/disable_service_external_name.go @@ -33,7 +33,10 @@ import ( ) var _ = framework.IngressNginxDescribe("[Flag] disable-service-external-name", func() { - f := framework.NewDefaultFramework("disabled-service-external-name") + f := framework.NewDefaultFramework( + "disabled-service-external-name", + framework.WithHTTPBunEnabled(), + ) ginkgo.BeforeEach(func() { f.NewEchoDeployment(framework.WithDeploymentReplicas(2)) @@ -54,21 +57,18 @@ var _ = framework.IngressNginxDescribe("[Flag] disable-service-external-name", f externalhost := "echo-external-svc.com" - ip := f.NewHttpbunDeployment() - svc := framework.BuildNIPExternalNameService(f, ip, "echo") - f.EnsureService(svc) + f.EnsureService(framework.BuildNIPExternalNameService(f, f.HTTPBunIP, "echo")) - svcexternal := &corev1.Service{ + f.EnsureService(&corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "external", Namespace: f.Namespace, }, Spec: corev1.ServiceSpec{ - ExternalName: framework.BuildNIPHost(ip), + ExternalName: f.GetNIPHost(), Type: corev1.ServiceTypeExternalName, }, - } - f.EnsureService(svcexternal) + }) ingexternal := framework.NewSingleIngress(externalhost, "/", externalhost, f.Namespace, "external", 80, nil) f.EnsureIngress(ingexternal) diff --git a/test/e2e/settings/global_external_auth.go b/test/e2e/settings/global_external_auth.go index c5964299f..cc98099ae 100644 --- a/test/e2e/settings/global_external_auth.go +++ b/test/e2e/settings/global_external_auth.go @@ -32,7 +32,10 @@ import ( ) var _ = framework.DescribeSetting("[Security] global-auth-url", func() { - f := framework.NewDefaultFramework("global-external-auth") + f := framework.NewDefaultFramework( + "global-external-auth", + framework.WithHTTPBunEnabled(), + ) host := "global-external-auth" @@ -50,7 +53,6 @@ var _ = framework.DescribeSetting("[Security] global-auth-url", func() { ginkgo.BeforeEach(func() { f.NewEchoDeployment() - f.NewHttpbunDeployment() }) ginkgo.Context("when global external authentication is configured", func() { @@ -307,9 +309,9 @@ http { assert.GreaterOrEqual(ginkgo.GinkgoT(), len(e.Subsets), 1, "expected at least one endpoint") assert.GreaterOrEqual(ginkgo.GinkgoT(), len(e.Subsets[0].Addresses), 1, "expected at least one address ready in the endpoint") - httpbunIP := e.Subsets[0].Addresses[0].IP + nginxIP := e.Subsets[0].Addresses[0].IP - f.UpdateNginxConfigMapData(globalExternalAuthURLSetting, fmt.Sprintf("http://%s/cookies/set/alma/armud", httpbunIP)) + f.UpdateNginxConfigMapData(globalExternalAuthURLSetting, fmt.Sprintf("http://%s/cookies/set/alma/armud", nginxIP)) ing1 = framework.NewSingleIngress(host, "/", host, f.Namespace, "http-cookie-with-error", 80, nil) f.EnsureIngress(ing1) diff --git a/test/e2e/settings/listen_nondefault_ports.go b/test/e2e/settings/listen_nondefault_ports.go index e682cef06..7e3b11b21 100644 --- a/test/e2e/settings/listen_nondefault_ports.go +++ b/test/e2e/settings/listen_nondefault_ports.go @@ -17,14 +17,12 @@ limitations under the License. package settings import ( - "context" "fmt" "net/http" "strings" "github.com/onsi/ginkgo/v2" "github.com/stretchr/testify/assert" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/ingress-nginx/test/e2e/framework" ) @@ -33,7 +31,7 @@ var _ = framework.IngressNginxDescribe("[Flag] custom HTTP and HTTPS ports", fun host := "forwarded-headers" - f := framework.NewDefaultFramework("forwarded-port-headers") + f := framework.NewDefaultFramework("forwarded-port-headers", framework.WithHTTPBunEnabled()) ginkgo.BeforeEach(func() { f.NewEchoDeployment() @@ -98,21 +96,8 @@ var _ = framework.IngressNginxDescribe("[Flag] custom HTTP and HTTPS ports", fun ginkgo.Context("when external authentication is configured", func() { ginkgo.It("should set the X-Forwarded-Port header to 443", func() { - f.NewHttpbunDeployment() - - err := framework.WaitForEndpoints(f.KubeClientSet, framework.DefaultTimeout, framework.HTTPBunService, f.Namespace, 1) - assert.Nil(ginkgo.GinkgoT(), err) - - e, err := f.KubeClientSet.CoreV1().Endpoints(f.Namespace).Get(context.TODO(), framework.HTTPBunService, metav1.GetOptions{}) - assert.Nil(ginkgo.GinkgoT(), err) - - assert.GreaterOrEqual(ginkgo.GinkgoT(), len(e.Subsets), 1, "expected at least one endpoint") - assert.GreaterOrEqual(ginkgo.GinkgoT(), len(e.Subsets[0].Addresses), 1, "expected at least one address ready in the endpoint") - - httpbunIP := e.Subsets[0].Addresses[0].IP - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/auth-url": fmt.Sprintf("http://%s/basic-auth/user/password", httpbunIP), + "nginx.ingress.kubernetes.io/auth-url": fmt.Sprintf("http://%s/basic-auth/user/password", f.HTTPBunIP), "nginx.ingress.kubernetes.io/auth-signin": "http://$host/auth/start", } diff --git a/test/e2e/settings/ssl_passthrough.go b/test/e2e/settings/ssl_passthrough.go index a906a2d11..f0859f878 100644 --- a/test/e2e/settings/ssl_passthrough.go +++ b/test/e2e/settings/ssl_passthrough.go @@ -34,7 +34,7 @@ import ( ) var _ = framework.IngressNginxDescribe("[Flag] enable-ssl-passthrough", func() { - f := framework.NewDefaultFramework("ssl-passthrough") + f := framework.NewDefaultFramework("ssl-passthrough", framework.WithHTTPBunEnabled()) ginkgo.BeforeEach(func() { err := f.UpdateIngressControllerDeployment(func(deployment *appsv1.Deployment) error { @@ -86,7 +86,14 @@ var _ = framework.IngressNginxDescribe("[Flag] enable-ssl-passthrough", func() { "nginx.ingress.kubernetes.io/ssl-passthrough": "true", } - ingressDef := framework.NewSingleIngressWithTLS(host, "/", host, []string{host}, f.Namespace, echoName, 80, annotations) + ingressDef := framework.NewSingleIngressWithTLS(host, + "/", + host, + []string{host}, + f.Namespace, + echoName, + 80, + annotations) tlsConfig, err := framework.CreateIngressTLSSecret(f.KubeClientSet, ingressDef.Spec.TLS[0].Hosts, ingressDef.Spec.TLS[0].SecretName, @@ -119,7 +126,17 @@ var _ = framework.IngressNginxDescribe("[Flag] enable-ssl-passthrough", func() { Value: "/certs/tls.key", }, } - f.NewDeploymentWithOpts("echopass", "ghcr.io/sharat87/httpbun:latest", 80, 1, nil, nil, envs, volumeMount, volume, false) + + f.NewDeploymentWithOpts("echopass", + framework.HTTPBunImage, + 80, + 1, + nil, + nil, + envs, + volumeMount, + volume, + false) f.EnsureIngress(ingressDef) @@ -133,7 +150,14 @@ var _ = framework.IngressNginxDescribe("[Flag] enable-ssl-passthrough", func() { /* This one should not receive traffic as it does not contain passthrough annotation */ hostBad := "noannotationnopassthrough.com" - ingBad := f.EnsureIngress(framework.NewSingleIngressWithTLS(hostBad, "/", hostBad, []string{hostBad}, f.Namespace, echoName, 80, nil)) + ingBad := f.EnsureIngress(framework.NewSingleIngressWithTLS(hostBad, + "/", + hostBad, + []string{hostBad}, + f.Namespace, + echoName, + 80, + nil)) tlsConfigBad, err := framework.CreateIngressTLSSecret(f.KubeClientSet, ingBad.Spec.TLS[0].Hosts, ingBad.Spec.TLS[0].SecretName,