Create NGINX dataplane handler

cmd/dataplane/Dockerfile

@@ -0,0 +1,90 @@
+# Copyright 2024 The Kubernetes Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+## This is a temporary file!!
+
+ARG BASE_IMAGE
+
+FROM ${BASE_IMAGE}
+
+ARG TARGETARCH
+ARG VERSION
+ARG COMMIT_SHA
+ARG BUILD_ID=UNSET
+
+LABEL org.opencontainers.image.title="NGINX Ingress Controller for Kubernetes - Dataplane"
+LABEL org.opencontainers.image.documentation="https://kubernetes.github.io/ingress-nginx/"
+LABEL org.opencontainers.image.source="https://github.com/kubernetes/ingress-nginx"
+LABEL org.opencontainers.image.vendor="The Kubernetes Authors"
+LABEL org.opencontainers.image.licenses="Apache-2.0"
+LABEL org.opencontainers.image.version="${VERSION}"
+LABEL org.opencontainers.image.revision="${COMMIT_SHA}"
+
+LABEL build_id="${BUILD_ID}"
+
+WORKDIR  /etc/nginx
+
+RUN apk update \
+  && apk upgrade \
+  && apk add --no-cache \
+    diffutils \
+  && rm -rf /var/cache/apk/*
+
+COPY --chown=www-data:www-data etc /etc
+
+# COPY --chown=www-data:www-data bin/${TARGETARCH}/dbg /
+COPY --chown=www-data:www-data bin/${TARGETARCH}/nginx-ingress-dataplane /
+# COPY --chown=www-data:www-data bin/${TARGETARCH}/wait-shutdown /
+
+# Fix permission during the build to avoid issues at runtime
+# with volumes (custom templates)
+RUN bash -xeu -c ' \
+  writeDirs=( \
+    /etc/ingress-controller/ssl \
+    /etc/ingress-controller/auth \
+    /etc/ingress-controller/geoip \
+    /etc/ingress-controller/telemetry \
+    /etc/nginx/conf \
+    /var/log \
+    /var/log/nginx \
+    /tmp/nginx \
+  ); \
+  for dir in "${writeDirs[@]}"; do \
+    mkdir -p ${dir}; \
+    chown -R www-data.www-data ${dir}; \
+  done' \
+  # LD_LIBRARY_PATH does not work so below is needed for  opentelemetry/other modules
+  # Put libs of newer modules under `/modules_mount/<other>/lib` and add that path below
+  # Could get complicated arch specific paths become a need
+  && echo "/lib:/usr/lib:/usr/local/lib:/modules_mount/etc/nginx/modules/otel" > /etc/ld-musl-x86_64.path
+  
+
+RUN apk add --no-cache libcap \
+  && setcap    cap_net_bind_service=+ep /nginx-ingress-controller \
+  && setcap -v cap_net_bind_service=+ep /nginx-ingress-controller \
+  && setcap    cap_net_bind_service=+ep /usr/local/nginx/sbin/nginx \
+  && setcap -v cap_net_bind_service=+ep /usr/local/nginx/sbin/nginx \
+  && setcap    cap_net_bind_service=+ep /usr/bin/dumb-init \
+  && setcap -v cap_net_bind_service=+ep /usr/bin/dumb-init \
+  && apk del libcap \
+  && ln -sf /usr/local/nginx/sbin/nginx /usr/bin/nginx
+
+USER www-data
+
+# Create symlinks to redirect nginx logs to stdout and stderr docker log collector
+RUN  ln -sf /dev/stdout /var/log/nginx/access.log \
+  && ln -sf /dev/stderr /var/log/nginx/error.log
+
+ENTRYPOINT ["/usr/bin/dumb-init", "--"]
+CMD ["/nginx-ingress-dataplane"]

cmd/dataplane/main.go

@@ -17,59 +17,72 @@ limitations under the License.
 package main
 
 import (
-	"fmt"
 	"net/http"
 	"os"
+	"os/signal"
+	"syscall"
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/collectors"
 
 	"k8s.io/klog/v2"
 
-	"k8s.io/ingress-nginx/internal/ingress/controller"
+	dataplanenginx "k8s.io/ingress-nginx/cmd/dataplane/pkg/nginx"
-	"k8s.io/ingress-nginx/internal/ingress/metric"
 	"k8s.io/ingress-nginx/internal/nginx"
 	"k8s.io/ingress-nginx/pkg/metrics"
-	"k8s.io/ingress-nginx/pkg/util/process"
-	"k8s.io/ingress-nginx/version"
 )
 
 func main() {
 	klog.InitFlags(nil)
 
-	fmt.Println(version.String())
+	//fmt.Println(version.String())
-	var err error
+	//var err error
 
 	reg := prometheus.NewRegistry()
 
 	reg.MustRegister(collectors.NewGoCollector())
+	// TODO: Below is supported just on Linux, do not register if OS is not Linux
 	reg.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{
 		PidFn:        func() (int, error) { return os.Getpid(), nil },
 		ReportErrors: true,
 	}))
 
-	mc := metric.NewDummyCollector()
+	//mc := metric.NewDummyCollector()
+	go metrics.RegisterProfiler(nginx.ProfilerAddress, nginx.ProfilerPort)
 	
-	// Pass the ValidationWebhook status to determine if we need to start the collector
-	// for the admissionWebhook
-	// TODO: Dataplane does not contain validation webhook so the MetricCollector should not receive
-	// this as an argument
-	mc.Start(conf.ValidationWebhook)
-
-	if conf.EnableProfiling {
-		go metrics.RegisterProfiler(nginx.ProfilerAddress, nginx.ProfilerPort)
-	}
-
-	ngx := controller.NewNGINXController(conf, mc)
-
 	mux := http.NewServeMux()
 	metrics.RegisterHealthz(nginx.HealthPath, mux)
 	metrics.RegisterMetrics(reg, mux)
 
-	go metrics.StartHTTPServer(conf.HealthCheckHost, conf.ListenPorts.Health, mux)
+	errCh := make(chan error)
-	go ngx.Start()
+	stopCh := make(chan bool)
+
+	signalChan := make(chan os.Signal, 1)
+	signal.Notify(signalChan, syscall.SIGTERM)
+	// TODO: Turn delay configurable
+	n := dataplanenginx.NewNGINXExecutor(mux, 10, errCh, stopCh)
+	//go executor.Start()
+	go metrics.StartHTTPServer("127.0.0.1", 12345, mux)
+
+	// TODO: deal with OS signals
+	select {
+	case err := <- errCh:
+		klog.ErrorS(err, "error executing NGINX")
+		os.Exit(1)
+	
+	case <- stopCh:
+		klog.Warning("received request to stop")
+		os.Exit(0)
+
+	case <- signalChan:
+		klog.InfoS("Received SIGTERM, shutting down")
+		exitCode := 0
+		if err := n.Stop(); err != nil {
+			klog.Warningf("Error during sigterm shutdown: %v", err)
+			exitCode = 1
+		}
+		klog.InfoS("Exiting", "code", exitCode)
+		os.Exit(exitCode)
+	}
 
-	process.HandleSigterm(ngx, conf.PostShutdownGracePeriod, func(code int) {
-		os.Exit(code)
-	})
 }

cmd/dataplane/pkg/nginx/nginx.go

@@ -0,0 +1,99 @@
+package nginx
+
+import (
+	"fmt"
+	"net/http"
+
+	nginxdataplane "k8s.io/ingress-nginx/internal/dataplane/nginx"
+	"k8s.io/klog/v2"
+)
+
+type nginxExecutor struct {
+	cmd nginxdataplane.NginxExecutor
+	errch chan error
+	stopch chan bool
+	stopdelay int
+}
+
+func NewNGINXExecutor(mux *http.ServeMux, stopdelay int, errch chan error, stopch chan bool) *nginxExecutor {
+	n := &nginxExecutor{
+		cmd: nginxdataplane.NewNginxCommand(),
+		stopdelay: stopdelay,
+		errch: errch,
+		stopch: stopch,
+	}
+	registerDataplaneHandler(n, mux)
+	return n
+}
+
+func (n *nginxExecutor) Start() {
+	n.cmd.Start(n.errch)
+}
+
+func (n *nginxExecutor) Stop() error {
+	return n.cmd.Stop()
+}
+
+func (n *nginxExecutor) handleStop(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodGet {
+		w.WriteHeader(http.StatusForbidden)
+		return
+	}
+	if err := n.Stop(); err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		n.errch <- fmt.Errorf("error stopping: %w", err)
+		return
+	}
+	w.WriteHeader(http.StatusOK)
+	n.stopch <- true
+}
+
+func (n *nginxExecutor) handleReload(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodGet {
+		w.WriteHeader(http.StatusForbidden)
+		return
+	}
+	o, err := n.cmd.Reload()
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		w.Write([]byte(err.Error() + "\n"))
+		w.Write(o)
+		return
+	}
+	w.WriteHeader(http.StatusOK)
+}
+
+func (n *nginxExecutor) handleTest(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodPost {
+		w.WriteHeader(http.StatusForbidden)
+		return
+	}
+	if err := r.ParseForm(); err != nil {
+		w.WriteHeader(http.StatusForbidden)
+		klog.ErrorS(err, "error parsing request", "handler", "test")
+		return
+	}
+
+	testFile := r.FormValue("testfile")
+	if testFile == "" {
+		w.WriteHeader(http.StatusForbidden)
+		klog.ErrorS(fmt.Errorf("testfile parameter not found"), "error parsing request", "handler", "test")
+		return
+	}
+
+	o, err := n.cmd.Test(testFile)
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		w.Write([]byte(err.Error() + "\n"))
+		w.Write(o)
+		klog.ErrorS(err, "error testing file", "output", string(o), "handler", "test")
+		return
+	}
+	w.WriteHeader(http.StatusOK)
+}
+
+func registerDataplaneHandler(n *nginxExecutor, mux *http.ServeMux) {
+	mux.HandleFunc("/stop", n.handleStop)
+	mux.HandleFunc("/reload", n.handleReload)
+	mux.HandleFunc("/test", n.handleTest)
+}