From 0a0d4aa88e6c061037237f095ca89e724c5325a6 Mon Sep 17 00:00:00 2001 From: Ricardo Katz Date: Mon, 6 May 2024 01:03:33 +0000 Subject: [PATCH] First pass of partial e2e --- NGINX_BASE | 2 +- build/dev-env.sh | 2 +- .../templates/controller-deployment.yaml | 41 +++++++------------ cmd/dataplane/main.go | 3 +- images/test-runner/Makefile | 4 +- internal/ingress/controller/checker.go | 10 ++--- internal/ingress/controller/nginx.go | 9 +++- test/e2e-image/e2e.sh | 1 - test/e2e/framework/exec.go | 2 +- test/e2e/framework/framework.go | 9 ++-- test/e2e/run-chart-test.sh | 2 +- test/e2e/run-kind-e2e.sh | 2 +- test/e2e/wait-for-nginx.sh | 3 +- 13 files changed, 42 insertions(+), 48 deletions(-) diff --git a/NGINX_BASE b/NGINX_BASE index 4d2c4917f..88411cb82 100644 --- a/NGINX_BASE +++ b/NGINX_BASE @@ -1 +1 @@ -registry.k8s.io/ingress-nginx/nginx-1.25:v0.0.6@sha256:b3e027ab191eb9461a9bcf25092eabb1d547cba164992dbd722c1aa2b4a936ee +rpkatz/nginx-split:v1 \ No newline at end of file diff --git a/build/dev-env.sh b/build/dev-env.sh index 93ed3de40..699c98030 100755 --- a/build/dev-env.sh +++ b/build/dev-env.sh @@ -64,7 +64,7 @@ echo "[dev-env] building image" make build image docker tag "${REGISTRY}/controller:${TAG}" "${DEV_IMAGE}" -export K8S_VERSION=${K8S_VERSION:-v1.26.3@sha256:61b92f38dff6ccc29969e7aa154d34e38b89443af1a2c14e6cfbd2df6419c66f} +export K8S_VERSION=${K8S_VERSION:-v1.29.2@sha256:51a1434a5397193442f0be2a297b488b6c919ce8a3931be0ce822606ea5ca245} KIND_CLUSTER_NAME="ingress-nginx-dev" diff --git a/charts/ingress-nginx/templates/controller-deployment.yaml b/charts/ingress-nginx/templates/controller-deployment.yaml index 7c276c5bb..7791fbeda 100644 --- a/charts/ingress-nginx/templates/controller-deployment.yaml +++ b/charts/ingress-nginx/templates/controller-deployment.yaml @@ -78,19 +78,7 @@ spec: shareProcessNamespace: {{ .Values.controller.shareProcessNamespace }} {{- end }} containers: - - args: - - /nginx-ingress-dataplane - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: LD_PRELOAD - value: /usr/local/lib/libmimalloc.so + - name: "dataplane" image: ingress-controller/dataplane:1.0.0-dev imagePullPolicy: IfNotPresent lifecycle: @@ -98,7 +86,20 @@ spec: exec: command: - /wait-shutdown - name: "dataplane" + args: + - /nginx-ingress-dataplane + securityContext: {{ include "ingress-nginx.controller.containerSecurityContext" . | nindent 12 }} + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: LD_PRELOAD + value: /usr/local/lib/libmimalloc.so ports: - containerPort: 80 hostPort: 80 @@ -112,18 +113,6 @@ spec: requests: cpu: 100m memory: 90Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - add: - - NET_BIND_SERVICE - drop: - - ALL - readOnlyRootFilesystem: false - runAsNonRoot: true - runAsUser: 101 - seccompProfile: - type: RuntimeDefault volumeMounts: - mountPath: /etc/ingress-controller name: ingress-controller diff --git a/cmd/dataplane/main.go b/cmd/dataplane/main.go index b58337d98..d0ca037c0 100644 --- a/cmd/dataplane/main.go +++ b/cmd/dataplane/main.go @@ -49,7 +49,8 @@ func main() { })) //mc := metric.NewDummyCollector() - go metrics.RegisterProfiler(nginx.ProfilerAddress, nginx.ProfilerPort) + // TODO: Enable metric collector for DP again + //go metrics.RegisterProfiler(nginx.ProfilerAddress, nginx.ProfilerPort) mux := http.NewServeMux() metrics.RegisterHealthz(nginx.HealthPath, mux) diff --git a/images/test-runner/Makefile b/images/test-runner/Makefile index ac0757ab2..c7117b5b0 100644 --- a/images/test-runner/Makefile +++ b/images/test-runner/Makefile @@ -58,7 +58,7 @@ image: --build-arg CHART_TESTING_VERSION=3.8.0 \ --build-arg YAML_LINT_VERSION=1.33.0 \ --build-arg YAMALE_VERSION=4.0.4 \ - --build-arg HELM_VERSION=3.11.2 \ + --build-arg HELM_VERSION=3.14.3 \ --build-arg GINKGO_VERSION=2.17.1 \ --build-arg GOLINT_VERSION=latest \ -t ${IMAGE}:${TAG} rootfs @@ -79,7 +79,7 @@ build: ensure-buildx --build-arg CHART_TESTING_VERSION=3.8.0 \ --build-arg YAML_LINT_VERSION=1.33.0 \ --build-arg YAMALE_VERSION=4.0.4 \ - --build-arg HELM_VERSION=3.11.2 \ + --build-arg HELM_VERSION=3.14.3 \ --build-arg GINKGO_VERSION=2.17.1 \ --build-arg GOLINT_VERSION=latest \ -t ${IMAGE}:${TAG} rootfs diff --git a/internal/ingress/controller/checker.go b/internal/ingress/controller/checker.go index d1bf19ddf..50a1ff89c 100644 --- a/internal/ingress/controller/checker.go +++ b/internal/ingress/controller/checker.go @@ -19,11 +19,6 @@ package controller import ( "fmt" "net/http" - "os" - "strconv" - "strings" - - "github.com/ncabatoff/process-exporter/proc" "k8s.io/ingress-nginx/internal/nginx" ) @@ -40,7 +35,8 @@ func (n *NGINXController) Check(_ *http.Request) error { } // check the nginx master process is running - fs, err := proc.NewFS("/proc", false) + // TODO: Move this checker to dataplane! + /*fs, err := proc.NewFS("/proc", false) if err != nil { return fmt.Errorf("reading /proc directory: %w", err) } @@ -58,7 +54,7 @@ func (n *NGINXController) Check(_ *http.Request) error { _, err = fs.Proc(pid) if err != nil { return fmt.Errorf("checking for NGINX process with PID %v: %w", pid, err) - } + }*/ statusCode, _, err := nginx.NewGetStatusRequest("/is-dynamic-lb-initialized") if err != nil { diff --git a/internal/ingress/controller/nginx.go b/internal/ingress/controller/nginx.go index 5c32f7d44..f21facc8d 100644 --- a/internal/ingress/controller/nginx.go +++ b/internal/ingress/controller/nginx.go @@ -40,6 +40,7 @@ import ( "github.com/eapache/channels" apiv1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes/scheme" v1core "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/tools/record" @@ -191,7 +192,13 @@ func NewNGINXController(config *Configuration, mc metric.Collector) *NGINXContro // everything. // It should be guaranteed by the dataplane that this file is changed just once // NGINX finishes starting - err = retry.OnError(retry.DefaultBackoff, func(err error) bool { + readinessBackoff := wait.Backoff{ + Steps: 5, + Duration: 3 * time.Millisecond, + Factor: 5.0, + Jitter: 0.1, + } + err = retry.OnError(readinessBackoff, func(err error) bool { return true }, func() error { _, errStat := os.Stat(nginxdataplane.ReadyFile) diff --git a/test/e2e-image/e2e.sh b/test/e2e-image/e2e.sh index f8ecd5337..73f4a2ef1 100755 --- a/test/e2e-image/e2e.sh +++ b/test/e2e-image/e2e.sh @@ -24,7 +24,6 @@ E2E_CHECK_LEAKS=${E2E_CHECK_LEAKS:-""} reportFile="report-e2e-test-suite.xml" ginkgo_args=( - "--fail-fast" "--flake-attempts=2" "--junit-report=${reportFile}" "--nodes=${E2E_NODES}" diff --git a/test/e2e/framework/exec.go b/test/e2e/framework/exec.go index 580a8f58e..e0ef4cf04 100644 --- a/test/e2e/framework/exec.go +++ b/test/e2e/framework/exec.go @@ -67,7 +67,7 @@ func (f *Framework) ExecCommand(pod *corev1.Pod, command string) (string, error) ) //nolint:gosec // Ignore G204 error - cmd := exec.Command("/bin/bash", "-c", fmt.Sprintf("%v exec --namespace %s %s --container controller -- %s", KubectlPath, pod.Namespace, pod.Name, command)) + cmd := exec.Command("/bin/bash", "-c", fmt.Sprintf("%v exec --namespace %s %s --container dataplane -- %s", KubectlPath, pod.Namespace, pod.Name, command)) cmd.Stdout = &execOut cmd.Stderr = &execErr diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go index b71d84baa..ac5b3eac5 100644 --- a/test/e2e/framework/framework.go +++ b/test/e2e/framework/framework.go @@ -183,7 +183,8 @@ func (f *Framework) AfterEach() { return } - cmd := "cat /etc/nginx/nginx.conf" + // TODO: This file location should be a const + cmd := "cat /etc/nginx/conf/nginx.conf" o, err := f.ExecCommand(f.pod, cmd) if err != nil { Logf("Unexpected error obtaining nginx.conf file: %v", err) @@ -303,9 +304,9 @@ func (f *Framework) matchNginxConditions(name string, matcher func(cfg string) b return func() (bool, error) { var cmd string if name == "" { - cmd = "cat /etc/nginx/nginx.conf" + cmd = "cat /etc/nginx/conf/nginx.conf" } else { - cmd = fmt.Sprintf("cat /etc/nginx/nginx.conf | awk '/## start server %v/,/## end server %v/'", name, name) + cmd = fmt.Sprintf("cat /etc/nginx/conf/nginx.conf | awk '/## start server %v/,/## end server %v/'", name, name) } o, err := f.ExecCommand(f.pod, cmd) @@ -328,7 +329,7 @@ func (f *Framework) matchNginxConditions(name string, matcher func(cfg string) b func (f *Framework) matchNginxCustomConditions(from, to string, matcher func(cfg string) bool) wait.ConditionFunc { return func() (bool, error) { - cmd := fmt.Sprintf("cat /etc/nginx/nginx.conf| awk '/%v/,/%v/'", from, to) + cmd := fmt.Sprintf("cat /etc/nginx/conf/nginx.conf| awk '/%v/,/%v/'", from, to) o, err := f.ExecCommand(f.pod, cmd) if err != nil { diff --git a/test/e2e/run-chart-test.sh b/test/e2e/run-chart-test.sh index a453bd1b4..dadb5195d 100755 --- a/test/e2e/run-chart-test.sh +++ b/test/e2e/run-chart-test.sh @@ -62,7 +62,7 @@ export KUBECONFIG="${KUBECONFIG:-$HOME/.kube/kind-config-$KIND_CLUSTER_NAME}" if [ "${SKIP_CLUSTER_CREATION:-false}" = "false" ]; then echo "[dev-env] creating Kubernetes cluster with kind" - export K8S_VERSION=${K8S_VERSION:-v1.26.3@sha256:61b92f38dff6ccc29969e7aa154d34e38b89443af1a2c14e6cfbd2df6419c66f} + export K8S_VERSION=${K8S_VERSION:-v1.29.2@sha256:51a1434a5397193442f0be2a297b488b6c919ce8a3931be0ce822606ea5ca245} kind create cluster \ --verbosity=${KIND_LOG_LEVEL} \ diff --git a/test/e2e/run-kind-e2e.sh b/test/e2e/run-kind-e2e.sh index e41f5c28b..06ba93e98 100755 --- a/test/e2e/run-kind-e2e.sh +++ b/test/e2e/run-kind-e2e.sh @@ -64,7 +64,7 @@ echo "Running e2e with nginx base image ${NGINX_BASE_IMAGE}" if [ "${SKIP_CLUSTER_CREATION}" = "false" ]; then echo "[dev-env] creating Kubernetes cluster with kind" - export K8S_VERSION=${K8S_VERSION:-v1.26.3@sha256:61b92f38dff6ccc29969e7aa154d34e38b89443af1a2c14e6cfbd2df6419c66f} + export K8S_VERSION=${K8S_VERSION:-v1.29.2@sha256:51a1434a5397193442f0be2a297b488b6c919ce8a3931be0ce822606ea5ca245} # delete the cluster if it exists if kind get clusters | grep "${KIND_CLUSTER_NAME}"; then diff --git a/test/e2e/wait-for-nginx.sh b/test/e2e/wait-for-nginx.sh index ac0584962..a6e62dd43 100755 --- a/test/e2e/wait-for-nginx.sh +++ b/test/e2e/wait-for-nginx.sh @@ -34,7 +34,8 @@ function on_exit { test $error_code == 0 && return; echo "Obtaining ingress controller pod logs..." - kubectl logs -l app.kubernetes.io/name=ingress-nginx -n $NAMESPACE + kubectl logs -l app.kubernetes.io/name=ingress-nginx -n $NAMESPACE -c controller + kubectl logs -l app.kubernetes.io/name=ingress-nginx -n $NAMESPACE -c dataplane } trap on_exit EXIT