diff --git a/cmd/dataplane/Dockerfile b/cmd/dataplane/Dockerfile new file mode 100644 index 000000000..c6b8576fa --- /dev/null +++ b/cmd/dataplane/Dockerfile @@ -0,0 +1,90 @@ +# Copyright 2024 The Kubernetes Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## This is a temporary file!! + +ARG BASE_IMAGE + +FROM ${BASE_IMAGE} + +ARG TARGETARCH +ARG VERSION +ARG COMMIT_SHA +ARG BUILD_ID=UNSET + +LABEL org.opencontainers.image.title="NGINX Ingress Controller for Kubernetes - Dataplane" +LABEL org.opencontainers.image.documentation="https://kubernetes.github.io/ingress-nginx/" +LABEL org.opencontainers.image.source="https://github.com/kubernetes/ingress-nginx" +LABEL org.opencontainers.image.vendor="The Kubernetes Authors" +LABEL org.opencontainers.image.licenses="Apache-2.0" +LABEL org.opencontainers.image.version="${VERSION}" +LABEL org.opencontainers.image.revision="${COMMIT_SHA}" + +LABEL build_id="${BUILD_ID}" + +WORKDIR /etc/nginx + +RUN apk update \ + && apk upgrade \ + && apk add --no-cache \ + diffutils \ + && rm -rf /var/cache/apk/* + +COPY --chown=www-data:www-data etc /etc + +# COPY --chown=www-data:www-data bin/${TARGETARCH}/dbg / +COPY --chown=www-data:www-data bin/${TARGETARCH}/nginx-ingress-dataplane / +# COPY --chown=www-data:www-data bin/${TARGETARCH}/wait-shutdown / + +# Fix permission during the build to avoid issues at runtime +# with volumes (custom templates) +RUN bash -xeu -c ' \ + writeDirs=( \ + /etc/ingress-controller/ssl \ + /etc/ingress-controller/auth \ + /etc/ingress-controller/geoip \ + /etc/ingress-controller/telemetry \ + /etc/nginx/conf \ + /var/log \ + /var/log/nginx \ + /tmp/nginx \ + ); \ + for dir in "${writeDirs[@]}"; do \ + mkdir -p ${dir}; \ + chown -R www-data.www-data ${dir}; \ + done' \ + # LD_LIBRARY_PATH does not work so below is needed for opentelemetry/other modules + # Put libs of newer modules under `/modules_mount//lib` and add that path below + # Could get complicated arch specific paths become a need + && echo "/lib:/usr/lib:/usr/local/lib:/modules_mount/etc/nginx/modules/otel" > /etc/ld-musl-x86_64.path + + +RUN apk add --no-cache libcap \ + && setcap cap_net_bind_service=+ep /nginx-ingress-controller \ + && setcap -v cap_net_bind_service=+ep /nginx-ingress-controller \ + && setcap cap_net_bind_service=+ep /usr/local/nginx/sbin/nginx \ + && setcap -v cap_net_bind_service=+ep /usr/local/nginx/sbin/nginx \ + && setcap cap_net_bind_service=+ep /usr/bin/dumb-init \ + && setcap -v cap_net_bind_service=+ep /usr/bin/dumb-init \ + && apk del libcap \ + && ln -sf /usr/local/nginx/sbin/nginx /usr/bin/nginx + +USER www-data + +# Create symlinks to redirect nginx logs to stdout and stderr docker log collector +RUN ln -sf /dev/stdout /var/log/nginx/access.log \ + && ln -sf /dev/stderr /var/log/nginx/error.log + +ENTRYPOINT ["/usr/bin/dumb-init", "--"] +CMD ["/nginx-ingress-dataplane"] diff --git a/cmd/dataplane/main.go b/cmd/dataplane/main.go index b587528d9..f11fb02ec 100644 --- a/cmd/dataplane/main.go +++ b/cmd/dataplane/main.go @@ -17,59 +17,72 @@ limitations under the License. package main import ( - "fmt" "net/http" "os" + "os/signal" + "syscall" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/collectors" "k8s.io/klog/v2" - "k8s.io/ingress-nginx/internal/ingress/controller" - "k8s.io/ingress-nginx/internal/ingress/metric" + dataplanenginx "k8s.io/ingress-nginx/cmd/dataplane/pkg/nginx" "k8s.io/ingress-nginx/internal/nginx" "k8s.io/ingress-nginx/pkg/metrics" - "k8s.io/ingress-nginx/pkg/util/process" - "k8s.io/ingress-nginx/version" ) func main() { klog.InitFlags(nil) - fmt.Println(version.String()) - var err error + //fmt.Println(version.String()) + //var err error reg := prometheus.NewRegistry() reg.MustRegister(collectors.NewGoCollector()) + // TODO: Below is supported just on Linux, do not register if OS is not Linux reg.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{ PidFn: func() (int, error) { return os.Getpid(), nil }, ReportErrors: true, })) - mc := metric.NewDummyCollector() + //mc := metric.NewDummyCollector() + go metrics.RegisterProfiler(nginx.ProfilerAddress, nginx.ProfilerPort) - // Pass the ValidationWebhook status to determine if we need to start the collector - // for the admissionWebhook - // TODO: Dataplane does not contain validation webhook so the MetricCollector should not receive - // this as an argument - mc.Start(conf.ValidationWebhook) - - if conf.EnableProfiling { - go metrics.RegisterProfiler(nginx.ProfilerAddress, nginx.ProfilerPort) - } - - ngx := controller.NewNGINXController(conf, mc) - mux := http.NewServeMux() metrics.RegisterHealthz(nginx.HealthPath, mux) metrics.RegisterMetrics(reg, mux) - go metrics.StartHTTPServer(conf.HealthCheckHost, conf.ListenPorts.Health, mux) - go ngx.Start() + errCh := make(chan error) + stopCh := make(chan bool) + + signalChan := make(chan os.Signal, 1) + signal.Notify(signalChan, syscall.SIGTERM) + // TODO: Turn delay configurable + n := dataplanenginx.NewNGINXExecutor(mux, 10, errCh, stopCh) + //go executor.Start() + go metrics.StartHTTPServer("127.0.0.1", 12345, mux) + + // TODO: deal with OS signals + select { + case err := <- errCh: + klog.ErrorS(err, "error executing NGINX") + os.Exit(1) + + case <- stopCh: + klog.Warning("received request to stop") + os.Exit(0) + + case <- signalChan: + klog.InfoS("Received SIGTERM, shutting down") + exitCode := 0 + if err := n.Stop(); err != nil { + klog.Warningf("Error during sigterm shutdown: %v", err) + exitCode = 1 + } + klog.InfoS("Exiting", "code", exitCode) + os.Exit(exitCode) + } - process.HandleSigterm(ngx, conf.PostShutdownGracePeriod, func(code int) { - os.Exit(code) - }) } diff --git a/cmd/dataplane/pkg/nginx/nginx.go b/cmd/dataplane/pkg/nginx/nginx.go new file mode 100644 index 000000000..a33830100 --- /dev/null +++ b/cmd/dataplane/pkg/nginx/nginx.go @@ -0,0 +1,99 @@ +package nginx + +import ( + "fmt" + "net/http" + + nginxdataplane "k8s.io/ingress-nginx/internal/dataplane/nginx" + "k8s.io/klog/v2" +) + +type nginxExecutor struct { + cmd nginxdataplane.NginxExecutor + errch chan error + stopch chan bool + stopdelay int +} + +func NewNGINXExecutor(mux *http.ServeMux, stopdelay int, errch chan error, stopch chan bool) *nginxExecutor { + n := &nginxExecutor{ + cmd: nginxdataplane.NewNginxCommand(), + stopdelay: stopdelay, + errch: errch, + stopch: stopch, + } + registerDataplaneHandler(n, mux) + return n +} + +func (n *nginxExecutor) Start() { + n.cmd.Start(n.errch) +} + +func (n *nginxExecutor) Stop() error { + return n.cmd.Stop() +} + +func (n *nginxExecutor) handleStop(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + w.WriteHeader(http.StatusForbidden) + return + } + if err := n.Stop(); err != nil { + w.WriteHeader(http.StatusInternalServerError) + n.errch <- fmt.Errorf("error stopping: %w", err) + return + } + w.WriteHeader(http.StatusOK) + n.stopch <- true +} + +func (n *nginxExecutor) handleReload(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + w.WriteHeader(http.StatusForbidden) + return + } + o, err := n.cmd.Reload() + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte(err.Error() + "\n")) + w.Write(o) + return + } + w.WriteHeader(http.StatusOK) +} + +func (n *nginxExecutor) handleTest(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + w.WriteHeader(http.StatusForbidden) + return + } + if err := r.ParseForm(); err != nil { + w.WriteHeader(http.StatusForbidden) + klog.ErrorS(err, "error parsing request", "handler", "test") + return + } + + testFile := r.FormValue("testfile") + if testFile == "" { + w.WriteHeader(http.StatusForbidden) + klog.ErrorS(fmt.Errorf("testfile parameter not found"), "error parsing request", "handler", "test") + return + } + + o, err := n.cmd.Test(testFile) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte(err.Error() + "\n")) + w.Write(o) + klog.ErrorS(err, "error testing file", "output", string(o), "handler", "test") + return + } + w.WriteHeader(http.StatusOK) +} + +func registerDataplaneHandler(n *nginxExecutor, mux *http.ServeMux) { + mux.HandleFunc("/stop", n.handleStop) + mux.HandleFunc("/reload", n.handleReload) + mux.HandleFunc("/test", n.handleTest) +} \ No newline at end of file