DevFW-CICD/ingress-nginx-helm
16
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

First pass of partial e2e

Browse source
This commit is contained in:
Ricardo Katz 2024-05-06 01:03:33 +00:00
parent 74e1959628
commit 0a0d4aa88e
13 changed files with 42 additions and 48 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
NGINX_BASE
View file

@ -1 +1 @@
registry.k8s.io/ingress-nginx/nginx-1.25:v0.0.6@sha256:b3e027ab191eb9461a9bcf25092eabb1d547cba164992dbd722c1aa2b4a936ee
rpkatz/nginx-split:v1

2
build/dev-env.sh
View file

@ -64,7 +64,7 @@ echo "[dev-env] building image"
make build image
docker tag "${REGISTRY}/controller:${TAG}" "${DEV_IMAGE}"
export K8S_VERSION=${K8S_VERSION:-v1.26.3@sha256:61b92f38dff6ccc29969e7aa154d34e38b89443af1a2c14e6cfbd2df6419c66f}
export K8S_VERSION=${K8S_VERSION:-v1.29.2@sha256:51a1434a5397193442f0be2a297b488b6c919ce8a3931be0ce822606ea5ca245}
KIND_CLUSTER_NAME="ingress-nginx-dev"

41
charts/ingress-nginx/templates/controller-deployment.yaml
View file

@ -78,19 +78,7 @@ spec:
shareProcessNamespace: {{ .Values.controller.shareProcessNamespace }}
{{- end }}
containers:
- args:
- /nginx-ingress-dataplane
env:
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: LD_PRELOAD
value: /usr/local/lib/libmimalloc.so
- name: "dataplane"
image: ingress-controller/dataplane:1.0.0-dev
imagePullPolicy: IfNotPresent
lifecycle:
@ -98,7 +86,20 @@ spec:
exec:
command:
- /wait-shutdown
name: "dataplane"
args:
- /nginx-ingress-dataplane
securityContext: {{ include "ingress-nginx.controller.containerSecurityContext" . | nindent 12 }}
env:
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: LD_PRELOAD
value: /usr/local/lib/libmimalloc.so
ports:
- containerPort: 80
hostPort: 80
@ -112,18 +113,6 @@ spec:
requests:
cpu: 100m
memory: 90Mi
securityContext:
allowPrivilegeEscalation: false
capabilities:
add:
- NET_BIND_SERVICE
drop:
- ALL
readOnlyRootFilesystem: false
runAsNonRoot: true
runAsUser: 101
seccompProfile:
type: RuntimeDefault
volumeMounts:
- mountPath: /etc/ingress-controller
name: ingress-controller

3
cmd/dataplane/main.go
View file

@ -49,7 +49,8 @@ func main() {
}))
//mc := metric.NewDummyCollector()
go metrics.RegisterProfiler(nginx.ProfilerAddress, nginx.ProfilerPort)
// TODO: Enable metric collector for DP again
//go metrics.RegisterProfiler(nginx.ProfilerAddress, nginx.ProfilerPort)
mux := http.NewServeMux()
metrics.RegisterHealthz(nginx.HealthPath, mux)

4
images/test-runner/Makefile
View file

@ -58,7 +58,7 @@ image:
--build-arg CHART_TESTING_VERSION=3.8.0 \
--build-arg YAML_LINT_VERSION=1.33.0 \
--build-arg YAMALE_VERSION=4.0.4 \
--build-arg HELM_VERSION=3.11.2 \
--build-arg HELM_VERSION=3.14.3 \
--build-arg GINKGO_VERSION=2.17.1 \
--build-arg GOLINT_VERSION=latest \
-t ${IMAGE}:${TAG} rootfs
@ -79,7 +79,7 @@ build: ensure-buildx
--build-arg CHART_TESTING_VERSION=3.8.0 \
--build-arg YAML_LINT_VERSION=1.33.0 \
--build-arg YAMALE_VERSION=4.0.4 \
--build-arg HELM_VERSION=3.11.2 \
--build-arg HELM_VERSION=3.14.3 \
--build-arg GINKGO_VERSION=2.17.1 \
--build-arg GOLINT_VERSION=latest \
-t ${IMAGE}:${TAG} rootfs

10
internal/ingress/controller/checker.go
View file

@ -19,11 +19,6 @@ package controller
import (
"fmt"
"net/http"
"os"
"strconv"
"strings"
"github.com/ncabatoff/process-exporter/proc"
"k8s.io/ingress-nginx/internal/nginx"
)
@ -40,7 +35,8 @@ func (n *NGINXController) Check(_ *http.Request) error {
}
// check the nginx master process is running
fs, err := proc.NewFS("/proc", false)
// TODO: Move this checker to dataplane!
/*fs, err := proc.NewFS("/proc", false)
if err != nil {
return fmt.Errorf("reading /proc directory: %w", err)
}
@ -58,7 +54,7 @@ func (n *NGINXController) Check(_ *http.Request) error {
_, err = fs.Proc(pid)
if err != nil {
return fmt.Errorf("checking for NGINX process with PID %v: %w", pid, err)
}
}*/
statusCode, _, err := nginx.NewGetStatusRequest("/is-dynamic-lb-initialized")
if err != nil {

9
internal/ingress/controller/nginx.go
View file

@ -40,6 +40,7 @@ import (
"github.com/eapache/channels"
apiv1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/kubernetes/scheme"
v1core "k8s.io/client-go/kubernetes/typed/core/v1"
"k8s.io/client-go/tools/record"
@ -191,7 +192,13 @@ func NewNGINXController(config *Configuration, mc metric.Collector) *NGINXContro
// everything.
// It should be guaranteed by the dataplane that this file is changed just once
// NGINX finishes starting
err = retry.OnError(retry.DefaultBackoff, func(err error) bool {
readinessBackoff := wait.Backoff{
Steps: 5,
Duration: 3 * time.Millisecond,
Factor: 5.0,
Jitter: 0.1,
}
err = retry.OnError(readinessBackoff, func(err error) bool {
return true
}, func() error {
_, errStat := os.Stat(nginxdataplane.ReadyFile)

1
test/e2e-image/e2e.sh
View file

@ -24,7 +24,6 @@ E2E_CHECK_LEAKS=${E2E_CHECK_LEAKS:-""}
reportFile="report-e2e-test-suite.xml"
ginkgo_args=(
"--fail-fast"
"--flake-attempts=2"
"--junit-report=${reportFile}"
"--nodes=${E2E_NODES}"

2
test/e2e/framework/exec.go
View file

@ -67,7 +67,7 @@ func (f *Framework) ExecCommand(pod *corev1.Pod, command string) (string, error)
)
//nolint:gosec // Ignore G204 error
cmd := exec.Command("/bin/bash", "-c", fmt.Sprintf("%v exec --namespace %s %s --container controller -- %s", KubectlPath, pod.Namespace, pod.Name, command))
cmd := exec.Command("/bin/bash", "-c", fmt.Sprintf("%v exec --namespace %s %s --container dataplane -- %s", KubectlPath, pod.Namespace, pod.Name, command))
cmd.Stdout = &execOut
cmd.Stderr = &execErr

9
test/e2e/framework/framework.go
View file

@ -183,7 +183,8 @@ func (f *Framework) AfterEach() {
return
}
cmd := "cat /etc/nginx/nginx.conf"
// TODO: This file location should be a const
cmd := "cat /etc/nginx/conf/nginx.conf"
o, err := f.ExecCommand(f.pod, cmd)
if err != nil {
Logf("Unexpected error obtaining nginx.conf file: %v", err)
@ -303,9 +304,9 @@ func (f *Framework) matchNginxConditions(name string, matcher func(cfg string) b
return func() (bool, error) {
var cmd string
if name == "" {
cmd = "cat /etc/nginx/nginx.conf"
cmd = "cat /etc/nginx/conf/nginx.conf"
} else {
cmd = fmt.Sprintf("cat /etc/nginx/nginx.conf | awk '/## start server %v/,/## end server %v/'", name, name)
cmd = fmt.Sprintf("cat /etc/nginx/conf/nginx.conf | awk '/## start server %v/,/## end server %v/'", name, name)
}
o, err := f.ExecCommand(f.pod, cmd)
@ -328,7 +329,7 @@ func (f *Framework) matchNginxConditions(name string, matcher func(cfg string) b
func (f *Framework) matchNginxCustomConditions(from, to string, matcher func(cfg string) bool) wait.ConditionFunc {
return func() (bool, error) {
cmd := fmt.Sprintf("cat /etc/nginx/nginx.conf| awk '/%v/,/%v/'", from, to)
cmd := fmt.Sprintf("cat /etc/nginx/conf/nginx.conf| awk '/%v/,/%v/'", from, to)
o, err := f.ExecCommand(f.pod, cmd)
if err != nil {

2
test/e2e/run-chart-test.sh
View file

@ -62,7 +62,7 @@ export KUBECONFIG="${KUBECONFIG:-$HOME/.kube/kind-config-$KIND_CLUSTER_NAME}"
if [ "${SKIP_CLUSTER_CREATION:-false}" = "false" ]; then
echo "[dev-env] creating Kubernetes cluster with kind"
export K8S_VERSION=${K8S_VERSION:-v1.26.3@sha256:61b92f38dff6ccc29969e7aa154d34e38b89443af1a2c14e6cfbd2df6419c66f}
export K8S_VERSION=${K8S_VERSION:-v1.29.2@sha256:51a1434a5397193442f0be2a297b488b6c919ce8a3931be0ce822606ea5ca245}
kind create cluster \
--verbosity=${KIND_LOG_LEVEL} \

2
test/e2e/run-kind-e2e.sh
View file

@ -64,7 +64,7 @@ echo "Running e2e with nginx base image ${NGINX_BASE_IMAGE}"
if [ "${SKIP_CLUSTER_CREATION}" = "false" ]; then
echo "[dev-env] creating Kubernetes cluster with kind"
export K8S_VERSION=${K8S_VERSION:-v1.26.3@sha256:61b92f38dff6ccc29969e7aa154d34e38b89443af1a2c14e6cfbd2df6419c66f}
export K8S_VERSION=${K8S_VERSION:-v1.29.2@sha256:51a1434a5397193442f0be2a297b488b6c919ce8a3931be0ce822606ea5ca245}
# delete the cluster if it exists
if kind get clusters | grep "${KIND_CLUSTER_NAME}"; then

3
test/e2e/wait-for-nginx.sh
View file

@ -34,7 +34,8 @@ function on_exit {
test $error_code == 0 && return;
echo "Obtaining ingress controller pod logs..."
kubectl logs -l app.kubernetes.io/name=ingress-nginx -n $NAMESPACE
kubectl logs -l app.kubernetes.io/name=ingress-nginx -n $NAMESPACE -c controller
kubectl logs -l app.kubernetes.io/name=ingress-nginx -n $NAMESPACE -c dataplane
}
trap on_exit EXIT