From ec239c69a9bf3ef93424765a2ba71c98eeabe2b8 Mon Sep 17 00:00:00 2001 From: Ricardo Lopes Date: Thu, 16 May 2024 12:37:08 +0100 Subject: [PATCH] Add e2e tests for custom-error-pages Signed-off-by: Ricardo Lopes --- docs/e2e-tests.md | 4 + test/e2e/CUSTOMERRORPAGES_IMAGE | 1 + test/e2e/defaultbackend/custom_error_pages.go | 121 ++++++++++++++++++ test/e2e/framework/deployment.go | 75 +++++++++++ test/e2e/run-e2e-suite.sh | 2 + test/e2e/run-kind-e2e.sh | 8 ++ 6 files changed, 211 insertions(+) create mode 100644 test/e2e/CUSTOMERRORPAGES_IMAGE create mode 100644 test/e2e/defaultbackend/custom_error_pages.go diff --git a/docs/e2e-tests.md b/docs/e2e-tests.md index 163a5ff5c..7fb1eb63d 100644 --- a/docs/e2e-tests.md +++ b/docs/e2e-tests.md @@ -283,6 +283,10 @@ Do not try to edit it manually. - [should return a self generated SSL certificate](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/defaultbackend/ssl.go#L29) ### [[Default Backend] change default settings](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/defaultbackend/with_hosts.go#L30) - [should apply the annotation to the default backend](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/defaultbackend/with_hosts.go#L38) +### [[Default Backend] custom-error-pages](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/defaultbackend/custom_error_pages.go#L33) +- [should export /metrics and /debug/vars by default](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/defaultbackend/custom_error_pages.go#L36) +- [shouldn't export /metrics and /debug/vars when IS_METRICS_EXPORT is set to false](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/defaultbackend/custom_error_pages.go#L57) +- [shouldn't export /metrics and /debug/vars when METRICS_PORT is set to a different port](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/defaultbackend/custom_error_pages.go#L80) ### [[Disable Leader] Routing works when leader election was disabled](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/disableleaderelection/disable_leader.go#L28) - [should create multiple ingress routings rules when leader election has disabled](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/disableleaderelection/disable_leader.go#L35) ### [[Endpointslices] long service name](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/endpointslices/longname.go#L29) diff --git a/test/e2e/CUSTOMERRORPAGES_IMAGE b/test/e2e/CUSTOMERRORPAGES_IMAGE new file mode 100644 index 000000000..865e0bb3e --- /dev/null +++ b/test/e2e/CUSTOMERRORPAGES_IMAGE @@ -0,0 +1 @@ +localhost/custom-error-pages:e2e diff --git a/test/e2e/defaultbackend/custom_error_pages.go b/test/e2e/defaultbackend/custom_error_pages.go new file mode 100644 index 000000000..597a69b19 --- /dev/null +++ b/test/e2e/defaultbackend/custom_error_pages.go @@ -0,0 +1,121 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package defaultbackend + +import ( + "context" + "fmt" + "net/http" + "strings" + + "github.com/onsi/ginkgo/v2" + "github.com/stretchr/testify/assert" + appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "k8s.io/ingress-nginx/test/e2e/framework" +) + +var _ = framework.IngressNginxDescribe("[Default Backend] custom-error-pages", func() { + f := framework.NewDefaultFramework("custom-error-pages") + + ginkgo.It("should export /metrics and /debug/vars by default", func() { + tt := []struct { + Name string + Path string + Status int + }{ + {"request to /metrics should return HTTP 200", "/metrics", http.StatusOK}, + {"request to /debug/vars should return HTTP 200", "/debug/vars", http.StatusOK}, + } + + setupIngressControllerWithCustomErrorPages(f, nil) + + for _, t := range tt { + ginkgo.By(t.Name) + f.HTTPTestClient(). + GET(t.Path). + Expect(). + Status(t.Status) + } + }) + + ginkgo.It("shouldn't export /metrics and /debug/vars when IS_METRICS_EXPORT is set to false", func() { + tt := []struct { + Name string + Path string + Status int + }{ + {"request to /metrics should return HTTP 404", "/metrics", http.StatusNotFound}, + {"request to /debug/vars should return HTTP 404", "/debug/vars", http.StatusNotFound}, + } + + setupIngressControllerWithCustomErrorPages(f, map[string]string{ + "IS_METRICS_EXPORT": "false", + }) + + for _, t := range tt { + ginkgo.By(t.Name) + f.HTTPTestClient(). + GET(t.Path). + Expect(). + Status(t.Status) + } + }) + + ginkgo.It("shouldn't export /metrics and /debug/vars when METRICS_PORT is set to a different port", func() { + tt := []struct { + Name string + Path string + Status int + }{ + {"request to /metrics should return HTTP 404", "/metrics", http.StatusNotFound}, + {"request to /debug/vars should return HTTP 404", "/debug/vars", http.StatusNotFound}, + } + + setupIngressControllerWithCustomErrorPages(f, map[string]string{ + "IS_METRICS_EXPORT": "true", + "METRICS_PORT": "8081", + }) + + for _, t := range tt { + ginkgo.By(t.Name) + f.HTTPTestClient(). + GET(t.Path). + Expect(). + Status(t.Status) + } + }) +}) + +func setupIngressControllerWithCustomErrorPages(f *framework.Framework, envVars map[string]string) { + f.NewCustomErrorPagesDeployment(framework.WithEnvVars(envVars)) + + err := f.UpdateIngressControllerDeployment(func(deployment *appsv1.Deployment) error { + args := deployment.Spec.Template.Spec.Containers[0].Args + args = append(args, fmt.Sprintf("--default-backend-service=%v/%v", f.Namespace, framework.CustomErrorPagesService)) + deployment.Spec.Template.Spec.Containers[0].Args = args + _, err := f.KubeClientSet.AppsV1().Deployments(f.Namespace).Update(context.TODO(), deployment, metav1.UpdateOptions{}) + return err + }) + assert.Nil(ginkgo.GinkgoT(), err, "updating deployment") + + f.WaitForNginxServer("_", + func(server string) bool { + return strings.Contains(server, `set $proxy_upstream_name "upstream-default-backend"`) + }) +} diff --git a/test/e2e/framework/deployment.go b/test/e2e/framework/deployment.go index d6e59f18a..148552f3e 100644 --- a/test/e2e/framework/deployment.go +++ b/test/e2e/framework/deployment.go @@ -49,6 +49,12 @@ var HTTPBunImage = os.Getenv("HTTPBUN_IMAGE") // EchoImage is the default image to be used by the echo service const EchoImage = "registry.k8s.io/ingress-nginx/e2e-test-echo:v1.0.1@sha256:1cec65aa768720290d05d65ab1c297ca46b39930e56bc9488259f9114fcd30e2" //#nosec G101 +// CustomErrorPagesService name of the deployment for the custom-error-pages app +const CustomErrorPagesService = "custom-error-pages" + +// CustomErrorPagesImage is the default image that is used to deploy custom-error-pages with the framework +var CustomErrorPagesImage = os.Getenv("CUSTOMERRORPAGES_IMAGE") + // TODO: change all Deployment functions to use these options // in order to reduce complexity and have a unified API across the // framework @@ -58,6 +64,7 @@ type deploymentOptions struct { image string replicas int svcAnnotations map[string]string + envVars map[string]string } // WithDeploymentNamespace allows configuring the deployment's namespace @@ -103,6 +110,74 @@ func WithImage(i string) func(*deploymentOptions) { } } +// WithEnvVars allows configuring environment variables for the deployment +func WithEnvVars(e map[string]string) func(*deploymentOptions) { + return func(o *deploymentOptions) { + o.envVars = e + } +} + +// NewCustomErrorPagesDeployment creates a new single replica deployment of the custom-error-pages server image in a particular namespace +func (f *Framework) NewCustomErrorPagesDeployment(opts ...func(*deploymentOptions)) { + options := &deploymentOptions{ + namespace: f.Namespace, + name: CustomErrorPagesService, + replicas: 1, + image: CustomErrorPagesImage, + } + for _, o := range opts { + o(options) + } + + envVars := []corev1.EnvVar{} + for k, v := range options.envVars { + envVars = append(envVars, corev1.EnvVar{Name: k, Value: v}) + } + + f.EnsureDeployment(newDeployment( + options.name, + options.namespace, + options.image, + 8080, + int32(options.replicas), + nil, nil, + envVars, + []corev1.VolumeMount{}, + []corev1.Volume{}, + false, + )) + + f.EnsureService(&corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: options.name, + Namespace: options.namespace, + Annotations: options.svcAnnotations, + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: "http", + Port: 8080, + TargetPort: intstr.FromInt(8080), + Protocol: corev1.ProtocolTCP, + }, + }, + Selector: map[string]string{ + "app": options.name, + }, + }, + }) + + err := WaitForEndpoints( + f.KubeClientSet, + DefaultTimeout, + options.name, + options.namespace, + options.replicas, + ) + assert.Nil(ginkgo.GinkgoT(), err, "waiting for endpoints to become ready") +} + // NewEchoDeployment creates a new single replica deployment of the echo server image in a particular namespace func (f *Framework) NewEchoDeployment(opts ...func(*deploymentOptions)) { options := &deploymentOptions{ diff --git a/test/e2e/run-e2e-suite.sh b/test/e2e/run-e2e-suite.sh index 909368e96..9629889f1 100755 --- a/test/e2e/run-e2e-suite.sh +++ b/test/e2e/run-e2e-suite.sh @@ -52,6 +52,7 @@ fi BASEDIR=$(dirname "$0") NGINX_BASE_IMAGE=$(cat $BASEDIR/../../NGINX_BASE) HTTPBUN_IMAGE=$(cat $BASEDIR/HTTPBUN_IMAGE) +CUSTOMERRORPAGES_IMAGE=$(cat $BASEDIR/CUSTOMERRORPAGES_IMAGE) echo -e "${BGREEN}Granting permissions to ingress-nginx e2e service account...${NC}" kubectl create serviceaccount ingress-nginx-e2e || true @@ -82,6 +83,7 @@ kubectl run --rm \ --env="E2E_CHECK_LEAKS=${E2E_CHECK_LEAKS}" \ --env="NGINX_BASE_IMAGE=${NGINX_BASE_IMAGE}" \ --env="HTTPBUN_IMAGE=${HTTPBUN_IMAGE}" \ + --env="CUSTOMERRORPAGES_IMAGE=${CUSTOMERRORPAGES_IMAGE}" \ --overrides='{ "apiVersion": "v1", "spec":{"serviceAccountName": "ingress-nginx-e2e"}}' \ e2e --image=nginx-ingress-controller:e2e diff --git a/test/e2e/run-kind-e2e.sh b/test/e2e/run-kind-e2e.sh index 01b909f30..2e09872e2 100755 --- a/test/e2e/run-kind-e2e.sh +++ b/test/e2e/run-kind-e2e.sh @@ -52,6 +52,7 @@ export KUBECONFIG="${KUBECONFIG:-$HOME/.kube/kind-config-$KIND_CLUSTER_NAME}" SKIP_INGRESS_IMAGE_CREATION="${SKIP_INGRESS_IMAGE_CREATION:-false}" SKIP_E2E_IMAGE_CREATION="${SKIP_E2E_IMAGE_CREATION:=false}" SKIP_CLUSTER_CREATION="${SKIP_CLUSTER_CREATION:-false}" +SKIP_CUSTOMERRORPAGES_IMAGE_CREATION="${SKIP_CUSTOMERRORPAGES_IMAGE_CREATION:-false}" if ! command -v kind --version &> /dev/null; then echo "kind is not installed. Use the package manager or visit the official site https://kind.sigs.k8s.io/" @@ -104,6 +105,12 @@ if [ "${SKIP_E2E_IMAGE_CREATION}" = "false" ]; then echo "[dev-env] ..done building e2e-image" fi +if [ "${SKIP_CUSTOMERRORPAGES_IMAGE_CREATION}" = "false" ]; then + echo "[dev-env] building custom-error-pages image" + REGISTRY=localhost NAME=custom-error-pages TAG=e2e make -C "${DIR}"/../../images build + echo "[dev-env] .. done building custom-error-pages image" +fi + # Preload images used in e2e tests KIND_WORKERS=$(kind get nodes --name="${KIND_CLUSTER_NAME}" | grep worker | awk '{printf (NR>1?",":"") $1}') @@ -111,5 +118,6 @@ echo "[dev-env] copying docker images to cluster..." kind load docker-image --name="${KIND_CLUSTER_NAME}" --nodes="${KIND_WORKERS}" nginx-ingress-controller:e2e kind load docker-image --name="${KIND_CLUSTER_NAME}" --nodes="${KIND_WORKERS}" "${REGISTRY}"/controller:"${TAG}" +kind load docker-image --name="${KIND_CLUSTER_NAME}" --nodes="${KIND_WORKERS}" "localhost/custom-error-pages:e2e" echo "[dev-env] running e2e tests..." make -C "${DIR}"/../../ e2e-test