Jail/chroot nginx process inside controller container (#8337)

* Initial work on chrooting nginx process

* More improvements in chroot

* Fix charts and some file locations

* Fix symlink on non chrooted container

* fix psp test

* Add e2e tests to chroot image

* Fix logger

* Add internal logger in controller

* Fix overlay for chrooted tests

* Fix tests

* fix boilerplates

* Fix unittest to point to the right pid

* Fix PR review

.github/workflows/ci.yaml

@@ -100,13 +100,14 @@ jobs:
           REGISTRY: ingress-controller
         run: |
           echo "building images..."
-          make clean-image build image
+          make clean-image build image image-chroot
           make -C test/e2e-image image
 
           echo "creating images cache..."
           docker save \
             nginx-ingress-controller:e2e \
             ingress-controller/controller:1.0.0-dev \
+            ingress-controller/controller-chroot:1.0.0-dev \
             | pigz > docker.tar.gz
 
       - name: cache
@@ -250,6 +251,65 @@ jobs:
           kind get kubeconfig > $HOME/.kube/kind-config-kind
           make kind-e2e-test
 
+  kubernetes-chroot:
+    name: Kubernetes chroot
+    runs-on: ubuntu-latest
+    needs:
+      - changes
+      - build
+    if: |
+      (needs.changes.outputs.go == 'true')
+
+    strategy:
+      matrix:
+        k8s: [v1.21.10, v1.22.7, v1.23.4]
+
+    steps:
+
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: cache
+        uses: actions/download-artifact@v2
+        with:
+          name: docker.tar.gz
+
+      - name: Create Kubernetes ${{ matrix.k8s }} cluster
+        id: kind
+        uses: engineerd/setup-kind@v0.5.0
+        with:
+          version: v0.12.0
+          config: test/e2e/kind.yaml
+          image: kindest/node:${{ matrix.k8s }}
+
+      - uses: geekyeggo/delete-artifact@v1
+        with:
+          name: docker.tar.gz
+          failOnError: false
+
+      - name: Prepare cluster for testing
+        id: local-path
+        run: |
+          kubectl version
+          echo
+          echo "installing helm 3..."
+          curl -sSL https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash
+
+      - name: Load images from cache
+        run: |
+          echo "loading docker images..."
+          pigz -dc docker.tar.gz | docker load
+
+      - name: Run e2e tests
+        env:
+          KIND_CLUSTER_NAME: kind
+          SKIP_CLUSTER_CREATION: true
+          SKIP_IMAGE_CREATION: true
+          IS_CHROOT: true
+        run: |
+          kind get kubeconfig > $HOME/.kube/kind-config-kind
+          make kind-e2e-test
+
   test-image-build:
     permissions:
       contents: read  # for dorny/paths-filter to fetch a list of changed files

Makefile

@@ -75,11 +75,30 @@ image: clean-image ## Build image for a particular arch.
 		--build-arg BUILD_ID="$(BUILD_ID)" \
 		-t $(REGISTRY)/controller:$(TAG) rootfs
 
+.PHONY: image-chroot
+image-chroot: clean-chroot-image ## Build image for a particular arch.
+	echo "Building docker image ($(ARCH))..."
+	@docker build \
+		--no-cache \
+		--build-arg BASE_IMAGE="$(BASE_IMAGE)" \
+		--build-arg VERSION="$(TAG)" \
+		--build-arg TARGETARCH="$(ARCH)" \
+		--build-arg COMMIT_SHA="$(COMMIT_SHA)" \
+		--build-arg BUILD_ID="$(BUILD_ID)" \
+		-t $(REGISTRY)/controller-chroot:$(TAG) rootfs -f rootfs/Dockerfile.chroot
+
 .PHONY: clean-image
 clean-image: ## Removes local image
 	echo "removing old image $(REGISTRY)/controller:$(TAG)"
 	@docker rmi -f $(REGISTRY)/controller:$(TAG) || true
 
+
+.PHONY: clean-chroot-image
+clean-chroot-image: ## Removes local image
+	echo "removing old image $(REGISTRY)/controller-chroot:$(TAG)"
+	@docker rmi -f $(REGISTRY)/controller-chroot:$(TAG) || true
+
+
 .PHONY: build
 build:  ## Build ingress controller, debug tool and pre-stop hook.
 	@build/run-in-docker.sh \
@@ -221,3 +240,14 @@ release: ensure-buildx clean
 		--build-arg COMMIT_SHA="$(COMMIT_SHA)" \
 		--build-arg BUILD_ID="$(BUILD_ID)" \
 		-t $(REGISTRY)/controller:$(TAG) rootfs
+	
+	@docker buildx build \
+		--no-cache \
+		--push \
+		--progress plain \
+		--platform $(subst $(SPACE),$(COMMA),$(PLATFORMS)) \
+		--build-arg BASE_IMAGE="$(BASE_IMAGE)" \
+		--build-arg VERSION="$(TAG)" \
+		--build-arg COMMIT_SHA="$(COMMIT_SHA)" \
+		--build-arg BUILD_ID="$(BUILD_ID)" \
+		-t $(REGISTRY)/controller-chroot:$(TAG) rootfs -f rootfs/Dockerfile.chroot

build/build.sh

@@ -69,3 +69,4 @@ go build \
     -X ${PKG}/version.COMMIT=${COMMIT_SHA} \
     -X ${PKG}/version.REPO=${REPO_INFO}" \
   -o "${TARGETS_DIR}/wait-shutdown" "${PKG}/cmd/waitshutdown"
+

build/test.sh

@@ -23,6 +23,7 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
+mkdir -p /tmp/nginx
 if [ -z "${PKG}" ]; then
   echo "PKG must be set"
   exit 1

charts/ingress-nginx/README.md

@@ -306,6 +306,7 @@ Kubernetes: `>=1.19.0-0`
 | controller.hostPort.ports.https | int | `443` | 'hostPort' https port |
 | controller.hostname | object | `{}` | Optionally customize the pod hostname. |
 | controller.image.allowPrivilegeEscalation | bool | `true` |  |
+| controller.image.chroot | bool | `false` |  |
 | controller.image.digest | string | `"sha256:31f47c1e202b39fadecf822a9b76370bd4baed199a005b3e7d4d1455f4fd3fe2"` |  |
 | controller.image.image | string | `"ingress-nginx/controller"` |  |
 | controller.image.pullPolicy | string | `"IfNotPresent"` |  |

charts/ingress-nginx/templates/_helpers.tpl

@@ -43,11 +43,40 @@ capabilities:
   - ALL
   add:
   - NET_BIND_SERVICE
+  {{- if .Values.controller.image.chroot }}
+  - SYS_CHROOT
+  {{- end }}
 runAsUser: {{ .Values.controller.image.runAsUser }}
 allowPrivilegeEscalation: {{ .Values.controller.image.allowPrivilegeEscalation }}
 {{- end }}
 {{- end -}}
 
+{{/*
+Get specific image
+*/}}
+{{- define "ingress-nginx.image" -}}
+{{- if .chroot -}}
+{{- printf "%s-chroot" .image -}}
+{{- else -}}
+{{- printf "%s" .image -}}
+{{- end }}
+{{- end -}}
+
+{{/*
+Get specific image digest
+*/}}
+{{- define "ingress-nginx.imageDigest" -}}
+{{- if .chroot -}}
+{{- if .digestChroot -}}
+{{- printf "@%s" .digestChroot -}}
+{{- end }}
+{{- else -}}
+{{ if .digest -}}
+{{- printf "@%s" .digest -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
 {{/*
 Create a default fully qualified controller name.
 We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).

charts/ingress-nginx/templates/controller-daemonset.yaml

@@ -74,7 +74,7 @@ spec:
       containers:
         - name: {{ .Values.controller.containerName }}
           {{- with .Values.controller.image }}
-          image: "{{- if .repository -}}{{ .repository }}{{ else }}{{ .registry }}/{{ .image }}{{- end -}}:{{ .tag }}{{- if (.digest) -}} @{{.digest}} {{- end -}}"
+          image: "{{- if .repository -}}{{ .repository }}{{ else }}{{ .registry }}/{{ include "ingress-nginx.image" . }}{{- end -}}:{{ .tag }}{{ include "ingress-nginx.imageDigest" . }}"
           {{- end }}
           imagePullPolicy: {{ .Values.controller.image.pullPolicy }}
         {{- if .Values.controller.lifecycle }}

charts/ingress-nginx/templates/controller-deployment.yaml

@@ -78,7 +78,7 @@ spec:
       containers:
         - name: {{ .Values.controller.containerName }}
           {{- with .Values.controller.image }}
-          image: "{{- if .repository -}}{{ .repository }}{{ else }}{{ .registry }}/{{ .image }}{{- end -}}:{{ .tag }}{{- if (.digest) -}} @{{.digest}} {{- end -}}"
+          image: "{{- if .repository -}}{{ .repository }}{{ else }}{{ .registry }}/{{ include "ingress-nginx.image" . }}{{- end -}}:{{ .tag }}{{ include "ingress-nginx.imageDigest" . }}"
           {{- end }}
           imagePullPolicy: {{ .Values.controller.image.pullPolicy }}
         {{- if .Values.controller.lifecycle }}

charts/ingress-nginx/values.yaml

@@ -16,6 +16,8 @@ commonLabels: {}
 controller:
   name: controller
   image:
+    ## Keep false as default for now!
+    chroot: false
     registry: k8s.gcr.io
     image: ingress-nginx/controller
     ## for backwards compatibility consider setting the full image url via the repository value below
@@ -23,6 +25,7 @@ controller:
     ## repository:
     tag: "v1.1.3"
     digest: sha256:31f47c1e202b39fadecf822a9b76370bd4baed199a005b3e7d4d1455f4fd3fe2
+    # digestChroot: "" # TODO: Fill when we have it
     pullPolicy: IfNotPresent
     # www-data -> uid 101
     runAsUser: 101

cmd/nginx/flags.go

@@ -192,6 +192,8 @@ Takes the form "<host>:port". If not provided, no admission controller is starte
 		statusPort = flags.Int("status-port", 10246, `Port to use for the lua HTTP endpoint configuration.`)
 		streamPort = flags.Int("stream-port", 10247, "Port to use for the lua TCP/UDP endpoint configuration.")
 
+		internalLoggerAddress = flags.String("internal-logger-address", "127.0.0.1:11514", "Address to be used when binding internal syslogger")
+
 		profilerPort = flags.Int("profiler-port", 10245, "Port to use for expose the ingress controller Go profiler when it is enabled.")
 
 		statusUpdateInterval = flags.Int("status-update-interval", status.UpdateInterval, "Time interval in seconds in which the status should check if an update is required. Default is 60 seconds")
@@ -344,6 +346,7 @@ https://blog.maxmind.com/2019/12/18/significant-changes-to-accessing-and-using-g
 		ValidationWebhook:         *validationWebhook,
 		ValidationWebhookCertPath: *validationWebhookCert,
 		ValidationWebhookKeyPath:  *validationWebhookKey,
+		InternalLoggerAddress:     *internalLoggerAddress,
 	}
 
 	if *apiserverHost != "" {

cmd/nginx/logger.go

@@ -0,0 +1,51 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+
+	"k8s.io/klog/v2"
+
+	"gopkg.in/mcuadros/go-syslog.v2"
+)
+
+func logger(address string) {
+	channel := make(syslog.LogPartsChannel)
+	handler := syslog.NewChannelHandler(channel)
+
+	server := syslog.NewServer()
+
+	server.SetFormat(syslog.RFC3164)
+	server.SetHandler(handler)
+	if err := server.ListenUDP(address); err != nil {
+		klog.Fatalf("failed bind internal syslog: %w", err)
+	}
+
+	if err := server.Boot(); err != nil {
+		klog.Fatalf("failed to boot internal syslog: %w", err)
+	}
+	klog.Infof("Is Chrooted, starting logger")
+
+	for logParts := range channel {
+		fmt.Printf("%s\n", logParts["content"])
+	}
+
+	server.Wait()
+	klog.Infof("Stopping logger")
+
+}

cmd/nginx/main.go

@@ -152,6 +152,13 @@ func main() {
 	registerHealthz(nginx.HealthPath, ngx, mux)
 	registerMetrics(reg, mux)
 
+	_, errExists := os.Stat("/chroot")
+	if errExists == nil {
+		conf.IsChroot = true
+		go logger(conf.InternalLoggerAddress)
+
+	}
+
 	go startHTTPServer(conf.HealthCheckHost, conf.ListenPorts.Health, mux)
 	go ngx.Start()
 

go.mod

@@ -130,6 +130,7 @@ require (
 	google.golang.org/protobuf v1.27.1 // indirect
 	gopkg.in/go-playground/assert.v1 v1.2.1 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/mcuadros/go-syslog.v2 v2.3.0 // indirect
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect

go.sum

@@ -1214,6 +1214,8 @@ gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/mcuadros/go-syslog.v2 v2.3.0 h1:kcsiS+WsTKyIEPABJBJtoG0KkOS6yzvJ+/eZlhD79kk=
+gopkg.in/mcuadros/go-syslog.v2 v2.3.0/go.mod h1:l5LPIyOOyIdQquNg+oU6Z3524YwrcqEm0aKH+5zpt2U=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
 gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=

internal/ingress/controller/checker_test.go

@@ -76,7 +76,7 @@ func TestNginxCheck(t *testing.T) {
 			})
 
 			// create pid file
-			os.MkdirAll("/tmp", file.ReadWriteByUser)
+			os.MkdirAll("/tmp/nginx", file.ReadWriteByUser)
 			pidFile, err := os.Create(nginx.PID)
 			if err != nil {
 				t.Fatalf("unexpected error: %v", err)

internal/ingress/controller/config/config.go

@@ -958,12 +958,11 @@ type TemplateConfig struct {
 	EnableMetrics            bool
 	MaxmindEditionFiles      *[]string
 	MonitorMaxBatchSize      int
-
-	PID            string
-	StatusPath     string
-	StatusPort     int
-	StreamPort     int
-	StreamSnippets []string
+	PID                      string
+	StatusPath               string
+	StatusPort               int
+	StreamPort               int
+	StreamSnippets           []string
 }
 
 // ListenPorts describe the ports required to run the

internal/ingress/controller/controller.go

@@ -120,6 +120,9 @@ type Configuration struct {
 
 	PostShutdownGracePeriod int
 	ShutdownGracePeriod     int
+
+	InternalLoggerAddress string
+	IsChroot              bool
 }
 
 // GetPublishService returns the Service used to set the load-balancer status of Ingresses.

internal/ingress/controller/nginx.go

@@ -575,6 +575,15 @@ func (n NGINXController) generateTemplate(cfg ngx_config.Configuration, ingressC
 
 	cfg.DefaultSSLCertificate = n.getDefaultSSLCertificate()
 
+	if n.cfg.IsChroot {
+		if cfg.AccessLogPath == "/var/log/nginx/access.log" {
+			cfg.AccessLogPath = fmt.Sprintf("syslog:server=%s", n.cfg.InternalLoggerAddress)
+		}
+		if cfg.ErrorLogPath == "/var/log/nginx/error.log" {
+			cfg.ErrorLogPath = fmt.Sprintf("syslog:server=%s", n.cfg.InternalLoggerAddress)
+		}
+	}
+
 	tc := ngx_config.TemplateConfig{
 		ProxySetHeaders:          setHeaders,
 		AddHeaders:               addHeaders,
@@ -614,7 +623,8 @@ func (n NGINXController) testTemplate(cfg []byte) error {
 	if len(cfg) == 0 {
 		return fmt.Errorf("invalid NGINX configuration (empty)")
 	}
-	tmpfile, err := os.CreateTemp("", tempNginxPattern)
+	tmpDir := os.TempDir() + "/nginx"
+	tmpfile, err := os.CreateTemp(tmpDir, tempNginxPattern)
 	if err != nil {
 		return err
 	}

internal/ingress/controller/process/nginx.go

@@ -20,7 +20,7 @@ import (
 	"os/exec"
 	"syscall"
 
-	"k8s.io/klog/v2"
+	klog "k8s.io/klog/v2"
 )
 
 // IsRespawnIfRequired checks if error type is exec.ExitError or not

internal/ingress/controller/util.go

@@ -29,7 +29,7 @@ import (
 	networking "k8s.io/api/networking/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/ingress-nginx/internal/ingress"
-	"k8s.io/klog/v2"
+	klog "k8s.io/klog/v2"
 )
 
 // newUpstream creates an upstream without servers.
@@ -98,7 +98,7 @@ func rlimitMaxNumFiles() int {
 }
 
 const (
-	defBinary = "/usr/local/nginx/sbin/nginx"
+	defBinary = "/usr/bin/nginx"
 	cfgPath   = "/etc/nginx/nginx.conf"
 )
 

internal/ingress/metric/collectors/socket.go

@@ -111,7 +111,7 @@ var defObjectives = map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}
 // NewSocketCollector creates a new SocketCollector instance using
 // the ingress watch namespace and class used by the controller
 func NewSocketCollector(pod, namespace, class string, metricsPerHost bool, buckets HistogramBuckets) (*SocketCollector, error) {
-	socket := "/tmp/prometheus-nginx.socket"
+	socket := "/tmp/nginx/prometheus-nginx.socket"
 	// unix sockets must be unlink()ed before being used
 	_ = syscall.Unlink(socket)
 

internal/nginx/main.go

@@ -28,7 +28,7 @@ import (
 	"time"
 
 	ps "github.com/mitchellh/go-ps"
-	"k8s.io/klog/v2"
+	klog "k8s.io/klog/v2"
 )
 
 // TODO: Check https://github.com/kubernetes/kubernetes/blob/master/pkg/master/ports/ports.go for ports already being used
@@ -40,7 +40,7 @@ var ProfilerPort = 10245
 var TemplatePath = "/etc/nginx/template/nginx.tmpl"
 
 // PID defines the location of the pid file used by NGINX
-var PID = "/tmp/nginx.pid"
+var PID = "/tmp/nginx/nginx.pid"
 
 // StatusPort port used by NGINX for the status server
 var StatusPort = 10246

rootfs/Dockerfile

@@ -54,6 +54,7 @@ RUN bash -xeu -c ' \
     /etc/ingress-controller/auth \
     /var/log \
     /var/log/nginx \
+    /tmp/nginx \
   ); \
   for dir in "${writeDirs[@]}"; do \
     mkdir -p ${dir}; \
@@ -67,7 +68,8 @@ RUN apk add --no-cache libcap \
   && setcap -v cap_net_bind_service=+ep /usr/local/nginx/sbin/nginx \
   && setcap    cap_net_bind_service=+ep /usr/bin/dumb-init \
   && setcap -v cap_net_bind_service=+ep /usr/bin/dumb-init \
-  && apk del libcap
+  && apk del libcap \
+  && ln -sf /usr/local/nginx/sbin/nginx /usr/bin/nginx
 
 USER www-data
 

rootfs/Dockerfile.chroot

@@ -0,0 +1,112 @@
+# Copyright 2022 The Kubernetes Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+
+ARG BASE_IMAGE
+
+FROM ${BASE_IMAGE} as chroot
+
+# This intermediary image will be used only to copy all the required files to the chroot
+# TODO: Simplify in a future to a single Dockerfile
+COPY chroot.sh /chroot.sh
+RUN apk update \
+  && apk upgrade \
+  && /chroot.sh
+
+FROM alpine:3.14.2
+
+ARG TARGETARCH
+ARG VERSION
+ARG COMMIT_SHA
+ARG BUILD_ID=UNSET
+
+LABEL org.opencontainers.image.title="NGINX Ingress Controller for Kubernetes"
+LABEL org.opencontainers.image.documentation="https://kubernetes.github.io/ingress-nginx/"
+LABEL org.opencontainers.image.source="https://github.com/kubernetes/ingress-nginx"
+LABEL org.opencontainers.image.vendor="The Kubernetes Authors"
+LABEL org.opencontainers.image.licenses="Apache-2.0"
+LABEL org.opencontainers.image.version="${VERSION}"
+LABEL org.opencontainers.image.revision="${COMMIT_SHA}"
+
+LABEL build_id="${BUILD_ID}"
+
+# This will be injected in the chroot. Don't change :)
+ENV LUA_PATH="/usr/local/share/luajit-2.1.0-beta3/?.lua;/usr/local/share/lua/5.1/?.lua;/usr/local/lib/lua/?.lua;;"
+ENV LUA_CPATH="/usr/local/lib/lua/?/?.so;/usr/local/lib/lua/?.so;;"
+ENV PATH=$PATH:/usr/local/luajit/bin:/usr/local/nginx/sbin:/usr/local/nginx/bin
+
+RUN apk update \
+  && apk upgrade \
+  && apk add -U --no-cache \
+     bash \
+     curl \
+     openssl \
+     ca-certificates \
+     dumb-init \
+     tzdata \
+     diffutils \
+     util-linux \
+  && ln -s /usr/local/nginx/sbin/nginx /sbin/nginx \
+  && adduser -S -D -H -u 101 -h /usr/local/nginx \
+    -s /sbin/nologin -G www-data -g www-data www-data 
+
+COPY --from=chroot /chroot /chroot
+
+COPY --chown=www-data:www-data etc /chroot/etc
+
+COPY --chown=www-data:www-data bin/${TARGETARCH}/dbg /
+COPY --chown=www-data:www-data bin/${TARGETARCH}/nginx-ingress-controller /
+COPY --chown=www-data:www-data bin/${TARGETARCH}/wait-shutdown /
+COPY --chown=www-data:www-data nginx-chroot-wrapper.sh /usr/bin/nginx
+
+WORKDIR /chroot/etc/nginx
+
+# Fix permission during the build to avoid issues at runtime
+# with volumes (custom templates)
+RUN bash -xeu -c ' \
+  writeDirs=( \
+    /var/log \
+  ); \
+  for dir in "${writeDirs[@]}"; do \
+    mkdir -p ${dir}; \
+    chown -R www-data.www-data ${dir}; \
+  done'
+
+RUN apk add --no-cache libcap \
+  && setcap    cap_sys_chroot,cap_net_bind_service=+ep /nginx-ingress-controller \
+  && setcap -v cap_sys_chroot,cap_net_bind_service=+ep /nginx-ingress-controller \
+  && setcap    cap_sys_chroot,cap_net_bind_service=+ep /usr/bin/unshare \
+  && setcap -v cap_sys_chroot,cap_net_bind_service=+ep /usr/bin/unshare \
+  && setcap    cap_net_bind_service=+ep /chroot/usr/local/nginx/sbin/nginx \
+  && setcap -v cap_net_bind_service=+ep /chroot/usr/local/nginx/sbin/nginx \
+  && setcap    cap_sys_chroot,cap_net_bind_service=+ep /usr/bin/dumb-init \
+  && setcap -v cap_sys_chroot,cap_net_bind_service=+ep /usr/bin/dumb-init \
+  && apk del libcap
+
+RUN  ln -sf /chroot/etc/nginx /etc/nginx \
+  && ln -sf /chroot/tmp/nginx /tmp/nginx \
+  && ln -sf /chroot/etc/ingress-controller /etc/ingress-controller \
+  && ln -sf /chroot/var/log/nginx /var/log/nginx \
+  && touch /chroot/var/log/nginx/access.log \
+  && chown www-data:www-data /chroot/var/log/nginx/access.log \
+  && echo "" > /chroot/etc/resolv.conf \
+  && chown -R www-data.www-data /chroot/var/log/nginx /chroot/etc/resolv.conf
+
+USER www-data
+
+EXPOSE 80 443
+
+ENTRYPOINT ["/usr/bin/dumb-init", "--"]
+
+CMD ["/nginx-ingress-controller"]
+ 

rootfs/chroot.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+
+# Copyright 2022 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -x
+writeDirs=( \
+  /chroot/etc/nginx \
+  /chroot/usr/local/ \
+  /chroot/etc/ingress-controller \
+  /chroot/etc/ingress-controller/ssl \
+  /chroot/etc/ingress-controller/auth \
+  /chroot/opt/modsecurity/var/log \
+  /chroot/opt/modsecurity/var/upload \
+  /chroot/opt/modsecurity/var/audit \
+  /chroot/var/log/audit \
+  /chroot/var/lib/nginx \
+  /chroot/var/log/nginx \
+  /chroot/var/lib/nginx/body \
+  /chroot/var/lib/nginx/fastcgi \
+  /chroot/var/lib/nginx/proxy \
+  /chroot/var/lib/nginx/scgi \
+  /chroot/var/lib/nginx/uwsgi \
+  /chroot/tmp/nginx
+);
+
+for dir in "${writeDirs[@]}"; do
+  mkdir -p ${dir};
+  chown -R www-data.www-data ${dir};
+done
+
+mkdir -p  /chroot/lib /chroot/proc /chroot/usr /chroot/bin /chroot/dev /chroot/run 
+cp /etc/passwd /etc/group /chroot/etc/
+cp -a /usr/* /chroot/usr/
+mv /var/log/nginx /chroot/var/log/
+cp -a /etc/nginx/* /chroot/etc/nginx/
+cp /lib/ld-musl-* /lib/libcrypto* /lib/libssl* /lib/libz* /chroot/lib/
+mknod -m 0666 /chroot/dev/null c 1 3
+mknod -m 0666 /chroot/dev/random c 1 8
+mknod -m 0666 /chroot/dev/urandom c 1 9
+mknod -m 0666 /chroot/dev/full c 1 7
+mknod -m 0666 /chroot/dev/ptmx c 5 2
+mknod -m 0666 /chroot/dev/zero c 1 5
+mknod -m 0666 /chroot/dev/tty c 5 0

rootfs/etc/nginx/lua/monitor.lua

@@ -26,7 +26,7 @@ local _M = {}
 
 local function send(payload)
   local s = assert(socket())
-  assert(s:connect("unix:/tmp/prometheus-nginx.socket"))
+  assert(s:connect("unix:/tmp/nginx/prometheus-nginx.socket"))
   assert(s:send(payload))
   assert(s:close())
 end

rootfs/etc/nginx/lua/test/monitor_test.lua

@@ -148,7 +148,7 @@ describe("Monitor", function()
         },
       })
 
-      assert.stub(tcp_mock.connect).was_called_with(tcp_mock, "unix:/tmp/prometheus-nginx.socket")
+      assert.stub(tcp_mock.connect).was_called_with(tcp_mock, "unix:/tmp/nginx/prometheus-nginx.socket")
       assert.stub(tcp_mock.send).was_called_with(tcp_mock, expected_payload)
       assert.stub(tcp_mock.close).was_called_with(tcp_mock)
     end)

rootfs/etc/nginx/nginx.conf

@@ -1,5 +1,7 @@
 # A very simple nginx configuration file that forces nginx to start.
-pid /tmp/nginx.pid;
+pid /tmp/nginx/nginx.pid;
+
+error_log stderr;
 
 events {}
 http {}

rootfs/etc/nginx/template/nginx.tmpl

@@ -285,10 +285,10 @@ http {
     keepalive_timeout  {{ $cfg.KeepAlive }}s;
     keepalive_requests {{ $cfg.KeepAliveRequests }};
 
-    client_body_temp_path           /tmp/client-body;
-    fastcgi_temp_path               /tmp/fastcgi-temp;
-    proxy_temp_path                 /tmp/proxy-temp;
-    ajp_temp_path                   /tmp/ajp-temp;
+    client_body_temp_path           /tmp/nginx/client-body;
+    fastcgi_temp_path               /tmp/nginx/fastcgi-temp;
+    proxy_temp_path                 /tmp/nginx/proxy-temp;
+    ajp_temp_path                   /tmp/nginx/ajp-temp;
 
     client_header_buffer_size       {{ $cfg.ClientHeaderBufferSize }};
     client_header_timeout           {{ $cfg.ClientHeaderTimeout }}s;
@@ -536,7 +536,7 @@ http {
     {{ end }}
 
     # Cache for internal auth checks
-    proxy_cache_path /tmp/nginx-cache-auth levels=1:2 keys_zone=auth_cache:10m max_size=128m inactive=30m use_temp_path=off;
+    proxy_cache_path /tmp/nginx/nginx-cache-auth levels=1:2 keys_zone=auth_cache:10m max_size=128m inactive=30m use_temp_path=off;
 
     # Global filters
     {{ range $ip := $cfg.BlockCIDRs }}deny {{ trimSpace $ip }};
@@ -773,8 +773,8 @@ stream {
     access_log {{ or $cfg.StreamAccessLogPath $cfg.AccessLogPath }} log_stream {{ $cfg.AccessLogParams }};
     {{ end }}
 
-    error_log  {{ $cfg.ErrorLogPath }} {{ $cfg.ErrorLogLevel }};
    
+    error_log  {{ $cfg.ErrorLogPath }} {{ $cfg.ErrorLogLevel }};
     {{ if $cfg.EnableRealIp }}
     {{ range $trusted_ip := $cfg.ProxyRealIPCIDR }}
     set_real_ip_from    {{ $trusted_ip }};

rootfs/nginx-chroot-wrapper.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+# Copyright 2022 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+cat /etc/resolv.conf > /chroot/etc/resolv.conf
+unshare  -S 101 -R /chroot nginx "$@"

test/data/cleanConf.expected.conf

@@ -1,7 +1,7 @@
 # Configuration checksum: 
 
 # setup custom paths that do not require root access
-pid /tmp/nginx.pid;
+pid /tmp/nginx/nginx.pid;
 
 daemon off;
 

test/data/cleanConf.src.conf

@@ -1,7 +1,7 @@
 # Configuration checksum: 
 
 # setup custom paths that do not require root access
-pid /tmp/nginx.pid;
+pid /tmp/nginx/nginx.pid;
 
 
 

test/e2e-image/namespace-overlays/admission/values.yaml

@@ -3,6 +3,7 @@ fullnameOverride: nginx-ingress
 controller:
   image:
     repository: ingress-controller/controller
+    chroot: true
     tag: 1.0.0-dev
     digest:
   containerPort:

test/e2e-image/namespace-overlays/custom-health-check-path/values.yaml

@@ -3,6 +3,7 @@ fullnameOverride: nginx-ingress
 controller:
   image:
     repository: ingress-controller/controller
+    chroot: true
     tag: 1.0.0-dev
     digest:
   extraArgs:

test/e2e-image/namespace-overlays/forwarded-port-headers/values.yaml

@@ -3,6 +3,7 @@ fullnameOverride: nginx-ingress
 controller:
   image:
     repository: ingress-controller/controller
+    chroot: true
     tag: 1.0.0-dev
     digest:
   containerPort:

test/e2e-image/namespace-overlays/namespace-selector/values.yaml

@@ -3,6 +3,7 @@ fullnameOverride: nginx-ingress
 controller:
   image:
     repository: ingress-controller/controller
+    chroot: true
     tag: 1.0.0-dev
     digest:
   containerPort:

test/e2e/framework/logs.go

@@ -18,12 +18,15 @@ package framework
 
 import (
 	"context"
+	"time"
 
 	"k8s.io/client-go/kubernetes"
 )
 
 // Logs returns the log entries of a given Pod.
 func Logs(client kubernetes.Interface, namespace, podName string) (string, error) {
+	// Logs from jails take a bigger time to get shipped due to the need of tailing them
+	Sleep(3 * time.Second)
 	logs, err := client.CoreV1().RESTClient().Get().
 		Resource("pods").
 		Namespace(namespace).

test/e2e/run.sh

@@ -58,7 +58,7 @@ export KUBECONFIG="${KUBECONFIG:-$HOME/.kube/kind-config-$KIND_CLUSTER_NAME}"
 if [ "${SKIP_CLUSTER_CREATION:-false}" = "false" ]; then
   echo "[dev-env] creating Kubernetes cluster with kind"
 
-  export K8S_VERSION=${K8S_VERSION:-v1.21.1@sha256:69860bda5563ac81e3c0057d654b5253219618a22ec3a346306239bba8cfa1a6}
+  export K8S_VERSION=${K8S_VERSION:-v1.21.10@sha256:84709f09756ba4f863769bdcabe5edafc2ada72d3c8c44d6515fc581b66b029c}
 
   kind create cluster \
     --verbosity=${KIND_LOG_LEVEL} \
@@ -77,7 +77,7 @@ if [ "${SKIP_IMAGE_CREATION:-false}" = "false" ]; then
   fi
 
   echo "[dev-env] building image"
-  make -C ${DIR}/../../ clean-image build image
+  make -C ${DIR}/../../ clean-image build image image-chroot
   make -C ${DIR}/../e2e-image image
 fi
 
@@ -87,6 +87,11 @@ KIND_WORKERS=$(kind get nodes --name="${KIND_CLUSTER_NAME}" | grep worker | awk
 echo "[dev-env] copying docker images to cluster..."
 
 kind load docker-image --name="${KIND_CLUSTER_NAME}" --nodes=${KIND_WORKERS} nginx-ingress-controller:e2e
+
+if [ "${IS_CHROOT:-false}" = "true" ]; then
+   docker tag ${REGISTRY}/controller-chroot:${TAG} ${REGISTRY}/controller:${TAG}
+fi
+
 kind load docker-image --name="${KIND_CLUSTER_NAME}" --nodes=${KIND_WORKERS} ${REGISTRY}/controller:${TAG}
 
 echo "[dev-env] running e2e tests..."

test/e2e/settings/access_log.go

@@ -31,17 +31,19 @@ var _ = framework.DescribeSetting("access-log", func() {
 		ginkgo.It("use the default configuration", func() {
 			f.WaitForNginxConfiguration(
 				func(cfg string) bool {
-					return strings.Contains(cfg, "access_log /var/log/nginx/access.log upstreaminfo") &&
-						strings.Contains(cfg, "access_log /var/log/nginx/access.log log_stream")
+					return (strings.Contains(cfg, "access_log /var/log/nginx/access.log upstreaminfo") &&
+						strings.Contains(cfg, "access_log /var/log/nginx/access.log log_stream")) ||
+						(strings.Contains(cfg, "access_log syslog:server=127.0.0.1:11514 upstreaminfo") &&
+							strings.Contains(cfg, "access_log syslog:server=127.0.0.1:11514 log_stream"))
 				})
 		})
 
 		ginkgo.It("use the specified configuration", func() {
-			f.UpdateNginxConfigMapData("access-log-path", "/tmp/access.log")
+			f.UpdateNginxConfigMapData("access-log-path", "/tmp/nginx/access.log")
 			f.WaitForNginxConfiguration(
 				func(cfg string) bool {
-					return strings.Contains(cfg, "access_log /tmp/access.log upstreaminfo") &&
-						strings.Contains(cfg, "access_log /tmp/access.log log_stream")
+					return strings.Contains(cfg, "access_log /tmp/nginx/access.log upstreaminfo") &&
+						strings.Contains(cfg, "access_log /tmp/nginx/access.log log_stream")
 				})
 		})
 	})
@@ -49,11 +51,12 @@ var _ = framework.DescribeSetting("access-log", func() {
 	ginkgo.Context("http-access-log-path", func() {
 
 		ginkgo.It("use the specified configuration", func() {
-			f.UpdateNginxConfigMapData("http-access-log-path", "/tmp/http-access.log")
+			f.UpdateNginxConfigMapData("http-access-log-path", "/tmp/nginx/http-access.log")
 			f.WaitForNginxConfiguration(
 				func(cfg string) bool {
-					return strings.Contains(cfg, "access_log /tmp/http-access.log upstreaminfo") &&
-						strings.Contains(cfg, "access_log /var/log/nginx/access.log log_stream")
+					return strings.Contains(cfg, "access_log /tmp/nginx/http-access.log upstreaminfo") &&
+						(strings.Contains(cfg, "access_log /var/log/nginx/access.log log_stream") ||
+							strings.Contains(cfg, "access_log syslog:server=127.0.0.1:11514 log_stream"))
 				})
 		})
 	})
@@ -61,11 +64,12 @@ var _ = framework.DescribeSetting("access-log", func() {
 	ginkgo.Context("stream-access-log-path", func() {
 
 		ginkgo.It("use the specified configuration", func() {
-			f.UpdateNginxConfigMapData("stream-access-log-path", "/tmp/stream-access.log")
+			f.UpdateNginxConfigMapData("stream-access-log-path", "/tmp/nginx/stream-access.log")
 			f.WaitForNginxConfiguration(
 				func(cfg string) bool {
-					return strings.Contains(cfg, "access_log /tmp/stream-access.log log_stream") &&
-						strings.Contains(cfg, "access_log /var/log/nginx/access.log upstreaminfo")
+					return strings.Contains(cfg, "access_log /tmp/nginx/stream-access.log log_stream") &&
+						(strings.Contains(cfg, "access_log /var/log/nginx/access.log upstreaminfo") ||
+							strings.Contains(cfg, "access_log syslog:server=127.0.0.1:11514 upstreaminfo"))
 				})
 		})
 	})
@@ -74,13 +78,13 @@ var _ = framework.DescribeSetting("access-log", func() {
 
 		ginkgo.It("use the specified configuration", func() {
 			f.SetNginxConfigMapData(map[string]string{
-				"http-access-log-path":   "/tmp/http-access.log",
-				"stream-access-log-path": "/tmp/stream-access.log",
+				"http-access-log-path":   "/tmp/nginx/http-access.log",
+				"stream-access-log-path": "/tmp/nginx/stream-access.log",
 			})
 			f.WaitForNginxConfiguration(
 				func(cfg string) bool {
-					return strings.Contains(cfg, "access_log /tmp/http-access.log upstreaminfo") &&
-						strings.Contains(cfg, "access_log /tmp/stream-access.log log_stream")
+					return strings.Contains(cfg, "access_log /tmp/nginx/http-access.log upstreaminfo") &&
+						strings.Contains(cfg, "access_log /tmp/nginx/stream-access.log log_stream")
 				})
 		})
 	})

test/e2e/settings/pod_security_policy_volumes.go

@@ -82,10 +82,10 @@ var _ = framework.IngressNginxDescribe("[Security] Pod Security Policies with vo
 
 			deployment.Spec.Template.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{
 				{
-					Name: "ssl", MountPath: "/etc/ingress-controller",
+					Name: "ssl", MountPath: "/etc/my-amazing-ssl",
 				},
 				{
-					Name: "tmp", MountPath: "/tmp",
+					Name: "tmp", MountPath: "/my-other-tmp",
 				},
 			}
 

test/e2e/wait-for-nginx.sh

@@ -59,6 +59,7 @@ fullnameOverride: nginx-ingress
 controller:
   image:
     repository: ingress-controller/controller
+    chroot: true
     tag: 1.0.0-dev
     digest:
   scope: