It is working

deploy/static/provider/kind/deploy.yaml

@@ -432,6 +432,7 @@ spec:
         - --watch-ingress-without-class=true
         - --enable-metrics=false
         - --publish-status-address=localhost
+        - --profiler-port=21524
         env:
         - name: POD_NAME
           valueFrom:
@@ -450,39 +451,11 @@ spec:
             exec:
               command:
               - /wait-shutdown
-        livenessProbe:
-          failureThreshold: 5
-          httpGet:
-            path: /healthz
-            port: 10254
-            scheme: HTTP
-          initialDelaySeconds: 10
-          periodSeconds: 10
-          successThreshold: 1
-          timeoutSeconds: 1
         name: controller
         ports:
-        - containerPort: 80
-          hostPort: 80
-          name: http
-          protocol: TCP
-        - containerPort: 443
-          hostPort: 443
-          name: https
-          protocol: TCP
         - containerPort: 8443
           name: webhook
           protocol: TCP
-        readinessProbe:
-          failureThreshold: 3
-          httpGet:
-            path: /healthz
-            port: 10254
-            scheme: HTTP
-          initialDelaySeconds: 10
-          periodSeconds: 10
-          successThreshold: 1
-          timeoutSeconds: 1
         resources:
           requests:
             cpu: 100m
@@ -503,6 +476,64 @@ spec:
         - mountPath: /usr/local/certificates/
           name: webhook-cert
           readOnly: true
+        - mountPath: /etc/ingress-controller
+          name: ingress-controller
+        - mountPath: /etc/nginx/conf
+          name: nginx-conf
+      - args:
+        - /nginx-ingress-dataplane
+        env:
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: POD_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        - name: LD_PRELOAD
+          value: /usr/local/lib/libmimalloc.so
+        image: gcr.io/k8s-staging-ingress-nginx/dataplane:v0.0.16
+        imagePullPolicy: IfNotPresent
+        lifecycle:
+          preStop:
+            exec:
+              command:
+              - /wait-shutdown
+        name: dataplane
+        ports:
+        - containerPort: 80
+          hostPort: 80
+          name: http
+          protocol: TCP
+        - containerPort: 443
+          hostPort: 443
+          name: https
+          protocol: TCP
+        resources:
+          requests:
+            cpu: 100m
+            memory: 90Mi
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            add:
+            - NET_BIND_SERVICE
+            drop:
+            - ALL
+          readOnlyRootFilesystem: false
+          runAsNonRoot: true
+          runAsUser: 101
+          seccompProfile:
+            type: RuntimeDefault
+        volumeMounts:
+        - mountPath: /usr/local/certificates/
+          name: webhook-cert
+          readOnly: true
+        - mountPath: /etc/ingress-controller
+          name: ingress-controller
+        - mountPath: /etc/nginx/conf
+          name: nginx-conf
       dnsPolicy: ClusterFirst
       nodeSelector:
         ingress-ready: "true"
@@ -520,6 +551,12 @@ spec:
       - name: webhook-cert
         secret:
           secretName: ingress-nginx-admission
+      - name: ingress-controller
+        emptyDir:
+          sizeLimit: 500Mi
+      - name: nginx-conf
+        emptyDir:
+          sizeLimit: 500Mi
 ---
 apiVersion: batch/v1
 kind: Job

internal/dataplane/nginx/nginx.go

@@ -1,6 +1,8 @@
 package nginx
 
 import (
+	"errors"
+	"io/fs"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -12,7 +14,9 @@ import (
 const (
 	defBinary = "/usr/bin/nginx"
 	CfgPath   = "/etc/nginx/conf/nginx.conf"
+	initialConf = "/etc/nginx/nginx.conf"
 	TempDir = "/etc/ingress-controller/tempconf"
+	ReadyFile = TempDir + "/ready"
 )
 
 // NginxExecTester defines the interface to execute
@@ -45,7 +49,7 @@ func NewNginxCommand() NginxCommand {
 }
 
 // ExecCommand instanciates an exec.Cmd object to call nginx program
-func (nc NginxCommand) execCommand(args ...string) *exec.Cmd {
+func (nc NginxCommand) execCommand(start bool, args ...string) *exec.Cmd {
 	cmdArgs := []string{}
 
 	cmdArgs = append(cmdArgs, "-c", CfgPath)
@@ -60,7 +64,21 @@ func (nc NginxCommand) execCommand(args ...string) *exec.Cmd {
 }
 
 func (nc NginxCommand) Start(errch chan error) error {
-	cmd := nc.execCommand()
+	klog.Infof("starting NGINX")
+	_, err := os.Stat(CfgPath)
+	if err != nil && errors.Is(err, fs.ErrNotExist) {
+		orig, err := os.ReadFile(initialConf)
+		if err != nil {
+			return err
+		}
+		if err = os.WriteFile(CfgPath, orig, 0644); err != nil {
+ 			return err
+    	}
+	}
+	if err := os.WriteFile(ReadyFile, []byte("OK"), 0644); err != nil {
+		return err
+	}
+	cmd := nc.execCommand(true)
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
 	if err := cmd.Start(); err != nil {
@@ -74,12 +92,12 @@ func (nc NginxCommand) Start(errch chan error) error {
 }
 
 func (nc NginxCommand) Reload() ([]byte, error) {
-	cmd := nc.execCommand("-s", "reload")
+	cmd := nc.execCommand(false, "-s", "reload")
 	return cmd.CombinedOutput()
 }
 
 func (nc NginxCommand) Stop() error {
-	cmd := nc.execCommand("-s", "quit")
+	cmd := nc.execCommand(false, "-s", "quit")
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
 	return cmd.Run()

internal/dataplane/nginx/remote.go

@@ -22,7 +22,7 @@ func NewNginxRemote(host string) NginxExecutor {
 }
 
 func (nc NginxRemote) Start(errch chan error) error {
-	getStart, err := url.JoinPath(nc.host, "start") // TODO: Turn this path a constant on dataplane
+	/*getStart, err := url.JoinPath(nc.host, "start") // TODO: Turn this path a constant on dataplane
 	if err != nil {
 		return err
 	}
@@ -36,7 +36,7 @@ func (nc NginxRemote) Start(errch chan error) error {
 	}
 	if resp.StatusCode != http.StatusOK {
 		return fmt.Errorf("error executing start: %s", string(body))
-	}
+	}*/
 
 	// TODO: Add a ping/watcher to backend and populate error channel
 	return nil

internal/ingress/controller/nginx.go

@@ -183,7 +183,7 @@ func NewNGINXController(config *Configuration, mc metric.Collector) *NGINXContro
 		klog.Fatalf("Error creating file watcher for %v: %v", nginx.TemplatePath, err)
 	}
 
-	filesToWatch := []string{}
+	filesToWatch := []string{nginxdataplane.ReadyFile}
 
 	if err := os.Mkdir("/etc/ingress-controller/geoip/", 0o755); err != nil && !os.IsExist(err) {
 		klog.Fatalf("Error creating geoip dir: %v", err)
@@ -420,7 +420,19 @@ func (n *NGINXController) Stop() error {
 }
 
 func (n *NGINXController) start() {
-	// TODO: do a better retry of start before failing 
+
+	// TODO: Start should ping the http and https ports
+	// First pass should wait it to be ready for X seconds, otherwise fail
+	// Second pass should open a goroutine and keep trying / pinging http port.
+	// After x retries it should state nginx is dead and restart everything
+	// Right now, if Dataplane dies, the dataplane container alone will be restarted
+	// This means the default configuration will come back again and will only be reload
+	// in case a full reload is requested.
+	// Another approach is to filewatch nginx.conf and if not controller changing it, reload
+	// Another problem here is: we are just re-creating the file in case it does not exists, 
+	// so dynamic reconfiguration will not be detected.
+	// We need a better way for controller to detect dataplane dying and get new information
+	// again
 	if err := n.command.Start(n.ngxErrCh); err != nil {
 		n.stopCh <- struct{}{}
 		klog.Fatalf("error starting NGINX: %s", err)

pkg/util/file/structure.go

@@ -31,11 +31,16 @@ const (
 	// The name of each file is <namespace>-<secret name>.pem. The content is the concatenated
 	// certificate and key.
 	DefaultSSLDirectory = "/etc/ingress-controller/ssl"
+
+	OpenTelemetry = "/etc/ingress-controller/telemetry"
+	TempDir = "/etc/ingress-controller/tempconf"
 )
 
 var directories = []string{
 	DefaultSSLDirectory,
 	AuthDirectory,
+	OpenTelemetry,
+	TempDir,
 }
 
 // CreateRequiredDirectories verifies if the required directories to

rootfs/etc/nginx/opentracing.json

@@ -1 +0,0 @@
-{}