From 7678c7c54b92ba17b3cd63ce2b1247704350ac4a Mon Sep 17 00:00:00 2001 From: Ricardo Katz Date: Fri, 19 Apr 2024 20:36:07 -0300 Subject: [PATCH] It is working --- deploy/static/provider/kind/deploy.yaml | 93 +++++++++++++++++-------- internal/dataplane/nginx/nginx.go | 26 +++++-- internal/dataplane/nginx/remote.go | 4 +- internal/ingress/controller/nginx.go | 16 ++++- pkg/util/file/structure.go | 5 ++ rootfs/etc/nginx/{conf => }/nginx.conf | 0 rootfs/etc/nginx/opentracing.json | 1 - 7 files changed, 108 insertions(+), 37 deletions(-) rename rootfs/etc/nginx/{conf => }/nginx.conf (100%) delete mode 100644 rootfs/etc/nginx/opentracing.json diff --git a/deploy/static/provider/kind/deploy.yaml b/deploy/static/provider/kind/deploy.yaml index d65e2cf39..abe6faf1b 100644 --- a/deploy/static/provider/kind/deploy.yaml +++ b/deploy/static/provider/kind/deploy.yaml @@ -432,6 +432,7 @@ spec: - --watch-ingress-without-class=true - --enable-metrics=false - --publish-status-address=localhost + - --profiler-port=21524 env: - name: POD_NAME valueFrom: @@ -450,39 +451,11 @@ spec: exec: command: - /wait-shutdown - livenessProbe: - failureThreshold: 5 - httpGet: - path: /healthz - port: 10254 - scheme: HTTP - initialDelaySeconds: 10 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 1 name: controller ports: - - containerPort: 80 - hostPort: 80 - name: http - protocol: TCP - - containerPort: 443 - hostPort: 443 - name: https - protocol: TCP - containerPort: 8443 name: webhook protocol: TCP - readinessProbe: - failureThreshold: 3 - httpGet: - path: /healthz - port: 10254 - scheme: HTTP - initialDelaySeconds: 10 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 1 resources: requests: cpu: 100m @@ -503,6 +476,64 @@ spec: - mountPath: /usr/local/certificates/ name: webhook-cert readOnly: true + - mountPath: /etc/ingress-controller + name: ingress-controller + - mountPath: /etc/nginx/conf + name: nginx-conf + - args: + - /nginx-ingress-dataplane + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: LD_PRELOAD + value: /usr/local/lib/libmimalloc.so + image: gcr.io/k8s-staging-ingress-nginx/dataplane:v0.0.16 + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + exec: + command: + - /wait-shutdown + name: dataplane + ports: + - containerPort: 80 + hostPort: 80 + name: http + protocol: TCP + - containerPort: 443 + hostPort: 443 + name: https + protocol: TCP + resources: + requests: + cpu: 100m + memory: 90Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + add: + - NET_BIND_SERVICE + drop: + - ALL + readOnlyRootFilesystem: false + runAsNonRoot: true + runAsUser: 101 + seccompProfile: + type: RuntimeDefault + volumeMounts: + - mountPath: /usr/local/certificates/ + name: webhook-cert + readOnly: true + - mountPath: /etc/ingress-controller + name: ingress-controller + - mountPath: /etc/nginx/conf + name: nginx-conf dnsPolicy: ClusterFirst nodeSelector: ingress-ready: "true" @@ -520,6 +551,12 @@ spec: - name: webhook-cert secret: secretName: ingress-nginx-admission + - name: ingress-controller + emptyDir: + sizeLimit: 500Mi + - name: nginx-conf + emptyDir: + sizeLimit: 500Mi --- apiVersion: batch/v1 kind: Job diff --git a/internal/dataplane/nginx/nginx.go b/internal/dataplane/nginx/nginx.go index 320e90687..ddb592704 100644 --- a/internal/dataplane/nginx/nginx.go +++ b/internal/dataplane/nginx/nginx.go @@ -1,6 +1,8 @@ package nginx import ( + "errors" + "io/fs" "os" "os/exec" "path/filepath" @@ -12,7 +14,9 @@ import ( const ( defBinary = "/usr/bin/nginx" CfgPath = "/etc/nginx/conf/nginx.conf" + initialConf = "/etc/nginx/nginx.conf" TempDir = "/etc/ingress-controller/tempconf" + ReadyFile = TempDir + "/ready" ) // NginxExecTester defines the interface to execute @@ -45,7 +49,7 @@ func NewNginxCommand() NginxCommand { } // ExecCommand instanciates an exec.Cmd object to call nginx program -func (nc NginxCommand) execCommand(args ...string) *exec.Cmd { +func (nc NginxCommand) execCommand(start bool, args ...string) *exec.Cmd { cmdArgs := []string{} cmdArgs = append(cmdArgs, "-c", CfgPath) @@ -60,7 +64,21 @@ func (nc NginxCommand) execCommand(args ...string) *exec.Cmd { } func (nc NginxCommand) Start(errch chan error) error { - cmd := nc.execCommand() + klog.Infof("starting NGINX") + _, err := os.Stat(CfgPath) + if err != nil && errors.Is(err, fs.ErrNotExist) { + orig, err := os.ReadFile(initialConf) + if err != nil { + return err + } + if err = os.WriteFile(CfgPath, orig, 0644); err != nil { + return err + } + } + if err := os.WriteFile(ReadyFile, []byte("OK"), 0644); err != nil { + return err + } + cmd := nc.execCommand(true) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr if err := cmd.Start(); err != nil { @@ -74,12 +92,12 @@ func (nc NginxCommand) Start(errch chan error) error { } func (nc NginxCommand) Reload() ([]byte, error) { - cmd := nc.execCommand("-s", "reload") + cmd := nc.execCommand(false, "-s", "reload") return cmd.CombinedOutput() } func (nc NginxCommand) Stop() error { - cmd := nc.execCommand("-s", "quit") + cmd := nc.execCommand(false, "-s", "quit") cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr return cmd.Run() diff --git a/internal/dataplane/nginx/remote.go b/internal/dataplane/nginx/remote.go index beba5fdf0..344513270 100644 --- a/internal/dataplane/nginx/remote.go +++ b/internal/dataplane/nginx/remote.go @@ -22,7 +22,7 @@ func NewNginxRemote(host string) NginxExecutor { } func (nc NginxRemote) Start(errch chan error) error { - getStart, err := url.JoinPath(nc.host, "start") // TODO: Turn this path a constant on dataplane + /*getStart, err := url.JoinPath(nc.host, "start") // TODO: Turn this path a constant on dataplane if err != nil { return err } @@ -36,7 +36,7 @@ func (nc NginxRemote) Start(errch chan error) error { } if resp.StatusCode != http.StatusOK { return fmt.Errorf("error executing start: %s", string(body)) - } + }*/ // TODO: Add a ping/watcher to backend and populate error channel return nil diff --git a/internal/ingress/controller/nginx.go b/internal/ingress/controller/nginx.go index 1f1c0a6d3..80879f21e 100644 --- a/internal/ingress/controller/nginx.go +++ b/internal/ingress/controller/nginx.go @@ -183,7 +183,7 @@ func NewNGINXController(config *Configuration, mc metric.Collector) *NGINXContro klog.Fatalf("Error creating file watcher for %v: %v", nginx.TemplatePath, err) } - filesToWatch := []string{} + filesToWatch := []string{nginxdataplane.ReadyFile} if err := os.Mkdir("/etc/ingress-controller/geoip/", 0o755); err != nil && !os.IsExist(err) { klog.Fatalf("Error creating geoip dir: %v", err) @@ -420,7 +420,19 @@ func (n *NGINXController) Stop() error { } func (n *NGINXController) start() { - // TODO: do a better retry of start before failing + + // TODO: Start should ping the http and https ports + // First pass should wait it to be ready for X seconds, otherwise fail + // Second pass should open a goroutine and keep trying / pinging http port. + // After x retries it should state nginx is dead and restart everything + // Right now, if Dataplane dies, the dataplane container alone will be restarted + // This means the default configuration will come back again and will only be reload + // in case a full reload is requested. + // Another approach is to filewatch nginx.conf and if not controller changing it, reload + // Another problem here is: we are just re-creating the file in case it does not exists, + // so dynamic reconfiguration will not be detected. + // We need a better way for controller to detect dataplane dying and get new information + // again if err := n.command.Start(n.ngxErrCh); err != nil { n.stopCh <- struct{}{} klog.Fatalf("error starting NGINX: %s", err) diff --git a/pkg/util/file/structure.go b/pkg/util/file/structure.go index 7d4f26da9..ad9a256b0 100644 --- a/pkg/util/file/structure.go +++ b/pkg/util/file/structure.go @@ -31,11 +31,16 @@ const ( // The name of each file is -.pem. The content is the concatenated // certificate and key. DefaultSSLDirectory = "/etc/ingress-controller/ssl" + + OpenTelemetry = "/etc/ingress-controller/telemetry" + TempDir = "/etc/ingress-controller/tempconf" ) var directories = []string{ DefaultSSLDirectory, AuthDirectory, + OpenTelemetry, + TempDir, } // CreateRequiredDirectories verifies if the required directories to diff --git a/rootfs/etc/nginx/conf/nginx.conf b/rootfs/etc/nginx/nginx.conf similarity index 100% rename from rootfs/etc/nginx/conf/nginx.conf rename to rootfs/etc/nginx/nginx.conf diff --git a/rootfs/etc/nginx/opentracing.json b/rootfs/etc/nginx/opentracing.json deleted file mode 100644 index 9e26dfeeb..000000000 --- a/rootfs/etc/nginx/opentracing.json +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file