diff --git a/docs/enhancements/20231001-split-containers.md b/docs/enhancements/20231001-split-containers.md new file mode 100644 index 000000000..3c2e85094 --- /dev/null +++ b/docs/enhancements/20231001-split-containers.md @@ -0,0 +1,110 @@ +# Proposal to split containers + +* All the NGINX files should live on one container + * No file other than NGINX files should exist on this container + * This includes not mounting the service account +* All the controller files should live on a different container + * Controller container should have bare minimum to work (just go program) + * ServiceAccount should be mounted just on controller + +* Inside nginx container, there should be a really small http listener just able +to start, stop and reload NGINX + +## Roadmap (what needs to be done) +* Map what needs to be done to mount the SA just on controller container +* Map all the required files for NGINX to work +* Map all the required network calls between controller and NGINX + * eg.: Dynamic lua reconfiguration +* Map problematic features that will need attention + * SSLPassthrough today happens on controller process and needs to happen on NGINX + +### Ports and endpoints on NGINX container +* Public HTTP/HTTPs port - 80 and 443 +* Lua configuration port - 10246 (HTTP) and 10247 (Stream) +* 3333 (temp) - Dataplane controller http server + * /reload - (POST) Reloads the configuration. + * "config" argument is the location of temporary file that should be used / moved to nginx.conf + * /test - (POST) Test the configuration of a given file location + * "config" argument is the location of temporary file that should be tested + +### Mounting empty SA on controller container + +```yaml +kind: Pod +apiVersion: v1 +metadata: + name: test +spec: + containers: + - name: nginx + image: nginx:latest + ports: + - containerPort: 80 + - name: othernginx + image: alpine:latest + command: ["/bin/sh"] + args: ["-c", "while true; do date; sleep 3; done"] + volumeMounts: + - mountPath: /var/run/secrets/kubernetes.io/serviceaccount + name: emptysecret + volumes: + - name: emptysecret + emptyDir: + sizeLimit: 1Mi +``` + +### Mapped folders on NGINX configuration +**WARNING** We need to be aware of inter mount containers and inode problems. If we +mount a file instead of a directory, it may take time to reflect the file value on +the target container + +* "/etc/nginx/lua/?.lua;/etc/nginx/lua/vendor/?.lua;;"; - Lua scripts +* "/var/log/nginx" - NGINX logs +* "/tmp/nginx (nginx.pid)" - NGINX pid directory / file, fcgi socket, etc +* " /etc/nginx/geoip" - GeoIP database directory - OK - /etc/ingress-controller/geoip +* /etc/nginx/mime.types - Mime types +* /etc/ingress-controller/ssl - SSL directory (fake cert, auth cert) +* /etc/ingress-controller/auth - Authentication files +* /etc/nginx/modsecurity - Modsecurity configuration +* /etc/nginx/owasp-modsecurity-crs - Modsecurity rules +* /etc/nginx/tickets.key - SSL tickets - OK - /etc/ingress-controller/tickets.key +* /etc/nginx/opentelemetry.toml - OTEL config - OK - /etc/ingress-controller/telemetry +* /etc/nginx/opentracing.json - Opentracing config - OK - /etc/ingress-controller/telemetry +* /etc/nginx/modules - NGINX modules +* /etc/nginx/fastcgi_params (maybe) - fcgi params +* /etc/nginx/template - Template, may be used by controller only + +##### List of modules +``` +ngx_http_auth_digest_module.so ngx_http_modsecurity_module.so +ngx_http_brotli_filter_module.so ngx_http_opentracing_module.so +ngx_http_brotli_static_module.so ngx_stream_geoip2_module.so +ngx_http_geoip2_module.so +``` + +##### List of files that may be removed +``` +-rw-r--r-- 1 www-data www-data 1077 Jun 23 19:44 fastcgi.conf +-rw-r--r-- 1 www-data www-data 1077 Jun 23 19:44 fastcgi.conf.default +-rw-r--r-- 1 www-data www-data 1007 Jun 23 19:44 fastcgi_params +-rw-r--r-- 1 www-data www-data 1007 Jun 23 19:44 fastcgi_params.default +drwxr-xr-x 2 www-data www-data 4096 Jun 23 19:34 geoip +-rw-r--r-- 1 www-data www-data 2837 Jun 23 19:44 koi-utf +-rw-r--r-- 1 www-data www-data 2223 Jun 23 19:44 koi-win +drwxr-xr-x 6 www-data www-data 4096 Sep 19 14:13 lua +-rw-r--r-- 1 www-data www-data 5349 Jun 23 19:44 mime.types +-rw-r--r-- 1 www-data www-data 5349 Jun 23 19:44 mime.types.default +drwxr-xr-x 2 www-data www-data 4096 Jun 23 19:44 modsecurity +drwxr-xr-x 2 www-data www-data 4096 Jun 23 19:44 modules +-rw-r--r-- 1 www-data www-data 18275 Oct 1 21:28 nginx.conf +-rw-r--r-- 1 www-data www-data 2656 Jun 23 19:44 nginx.conf.default +-rwx------ 1 www-data www-data 420 Oct 1 21:28 opentelemetry.toml +-rw-r--r-- 1 www-data www-data 2 Oct 1 21:28 opentracing.json +drwxr-xr-x 7 www-data www-data 4096 Jun 23 19:44 owasp-modsecurity-crs +-rw-r--r-- 1 www-data www-data 636 Jun 23 19:44 scgi_params +-rw-r--r-- 1 www-data www-data 636 Jun 23 19:44 scgi_params.default +drwxr-xr-x 2 www-data www-data 4096 Sep 19 14:13 template +-rw-r--r-- 1 www-data www-data 664 Jun 23 19:44 uwsgi_params +-rw-r--r-- 1 www-data www-data 664 Jun 23 19:44 uwsgi_params.default +-rw-r--r-- 1 www-data www-data 3610 Jun 23 19:44 win-utf +``` diff --git a/docs/user-guide/nginx-configuration/configmap.md b/docs/user-guide/nginx-configuration/configmap.md index bc9dfae0e..92ac39ee6 100644 --- a/docs/user-guide/nginx-configuration/configmap.md +++ b/docs/user-guide/nginx-configuration/configmap.md @@ -164,7 +164,7 @@ The following table shows a configuration option's name, type, and the default v |[enable-opentelemetry](#enable-opentelemetry)|bool|"false"|| |[opentelemetry-trust-incoming-span](#opentelemetry-trust-incoming-span)|bool|"true"|| |[opentelemetry-operation-name](#opentelemetry-operation-name)|string|""|| -|[opentelemetry-config](#/etc/nginx/opentelemetry.toml)|string|"/etc/nginx/opentelemetry.toml"|| +|[opentelemetry-config](#/etc/ingress-controller/telemetry/opentelemetry.toml)|string|"/etc/ingress-controller/telemetry/opentelemetry.toml"|| |[otlp-collector-host](#otlp-collector-host)|string|""|| |[otlp-collector-port](#otlp-collector-port)|int|4317|| |[otel-max-queuesize](#otel-max-queuesize)|int||| diff --git a/docs/user-guide/third-party-addons/opentelemetry.md b/docs/user-guide/third-party-addons/opentelemetry.md index d735fca4f..a4bd8d060 100644 --- a/docs/user-guide/third-party-addons/opentelemetry.md +++ b/docs/user-guide/third-party-addons/opentelemetry.md @@ -165,7 +165,7 @@ To install the example and collectors run: kind: ConfigMap data: enable-opentelemetry: "true" - opentelemetry-config: "/etc/nginx/opentelemetry.toml" + opentelemetry-config: "/etc/ingress-controller/telemetry/opentelemetry.toml" opentelemetry-operation-name: "HTTP $request_method $service_name $uri" opentelemetry-trust-incoming-span: "true" otlp-collector-host: "otel-coll-collector.otel.svc" diff --git a/go.work.sum b/go.work.sum index b90672997..863a10664 100644 --- a/go.work.sum +++ b/go.work.sum @@ -186,6 +186,7 @@ github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY= github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE= github.com/moby/term v0.0.0-20221205130635-1aeaba878587/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= @@ -199,6 +200,7 @@ github.com/onsi/gomega v1.27.4/go.mod h1:riYq/GJKh8hhoM01HN6Vmuy93AarCXCBGpvFDK3 github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0= github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pquerna/cachecontrol v0.1.0/go.mod h1:NrUG3Z7Rdu85UNR3vm7SOsl1nFIeSiQnrHV5K9mBcUI= github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= @@ -254,6 +256,7 @@ golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -284,6 +287,7 @@ google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:xZnkP7mR google.golang.org/genproto v0.0.0-20230706204954-ccb25ca9f130/go.mod h1:O9kGHb51iE/nOGvQaDUuadVYqovW56s5emA88lQnj6Y= google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98 h1:Z0hjGZePRE0ZBWotvtrwxFNrNE9CUAGtplaDK5NNI/g= google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98/go.mod h1:S7mY02OqCJTD0E1OiQy1F72PWFB4bZJ87cAtLPYgDR0= +google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d h1:VBu5YqKPv6XiJ199exd8Br+Aetz+o08F+PLMnwJQHAY= google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= google.golang.org/genproto/googleapis/api v0.0.0-20230706204954-ccb25ca9f130/go.mod h1:mPBs5jNgx2GuQGvFwUvVKqtn6HsUw9nP64BedgvqEsQ= google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98/go.mod h1:rsr7RhLuwsDKL7RmgDDCUc6yaGr1iqceVb5Wv6f6YvQ= diff --git a/images/kube-webhook-certgen/rootfs/go.mod b/images/kube-webhook-certgen/rootfs/go.mod index b0cf83320..b30fa5f27 100644 --- a/images/kube-webhook-certgen/rootfs/go.mod +++ b/images/kube-webhook-certgen/rootfs/go.mod @@ -26,7 +26,7 @@ require ( github.com/google/go-cmp v0.5.9 // indirect github.com/google/gofuzz v1.1.0 // indirect github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98 // indirect - github.com/google/uuid v1.3.0 // indirect + github.com/google/uuid v1.3.1 // indirect github.com/imdario/mergo v0.3.6 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect diff --git a/images/kube-webhook-certgen/rootfs/go.sum b/images/kube-webhook-certgen/rootfs/go.sum index b41bf8a06..004bd7d96 100644 --- a/images/kube-webhook-certgen/rootfs/go.sum +++ b/images/kube-webhook-certgen/rootfs/go.sum @@ -202,8 +202,8 @@ github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98/go.mod h1:czg5+yv1E0Z github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= diff --git a/internal/ingress/controller/config/config.go b/internal/ingress/controller/config/config.go index fadc924be..9bb13823f 100644 --- a/internal/ingress/controller/config/config.go +++ b/internal/ingress/controller/config/config.go @@ -987,7 +987,7 @@ func NewDefault() Configuration { BindAddressIpv6: defBindAddress, OpentracingTrustIncomingSpan: true, OpentelemetryTrustIncomingSpan: true, - OpentelemetryConfig: "/etc/nginx/opentelemetry.toml", + OpentelemetryConfig: "/etc/ingress-controller/telemetry/opentelemetry.toml", OtlpCollectorPort: "4317", OtelServiceName: "nginx", OtelSampler: "AlwaysOn", diff --git a/internal/ingress/controller/nginx.go b/internal/ingress/controller/nginx.go index 30f785586..83b38bd3a 100644 --- a/internal/ingress/controller/nginx.go +++ b/internal/ingress/controller/nginx.go @@ -22,6 +22,7 @@ import ( "encoding/json" "errors" "fmt" + "io/fs" "net" "net/http" "os" @@ -180,7 +181,11 @@ func NewNGINXController(config *Configuration, mc metric.Collector) *NGINXContro } filesToWatch := []string{} - err = filepath.Walk("/etc/nginx/geoip/", func(path string, info os.FileInfo, err error) error { + + if err := os.Mkdir("/etc/ingress-controller/geoip/", 0o755); err != nil && !os.IsExist(err) { + klog.Fatalf("Error creating geoip dir: %v", err) + } + err = filepath.WalkDir("/etc/ingress-controller/geoip/", func(path string, info fs.DirEntry, err error) error { if err != nil { return err } @@ -1107,7 +1112,7 @@ func createOpentracingCfg(cfg *ngx_config.Configuration) error { // Expand possible environment variables before writing the configuration to file. expanded := os.ExpandEnv(configData) - return os.WriteFile("/etc/nginx/opentracing.json", []byte(expanded), file.ReadWriteByUser) + return os.WriteFile("/etc/ingress-controller/telemetry/opentracing.json", []byte(expanded), file.ReadWriteByUser) } func createOpentelemetryCfg(cfg *ngx_config.Configuration) error { diff --git a/internal/ingress/controller/store/store.go b/internal/ingress/controller/store/store.go index 918dfd41a..4288785de 100644 --- a/internal/ingress/controller/store/store.go +++ b/internal/ingress/controller/store/store.go @@ -1215,7 +1215,7 @@ func (s *k8sStore) setConfig(cmap *corev1.ConfigMap) { s.backendConfig.UseGeoIP2 = false } - s.writeSSLSessionTicketKey(cmap, "/etc/nginx/tickets.key") + s.writeSSLSessionTicketKey(cmap, "/etc/ingress-controller/tickets.key") } // Run initiates the synchronization of the informers and the initial diff --git a/internal/ingress/controller/template/template.go b/internal/ingress/controller/template/template.go index 6702e08db..c2d461b21 100644 --- a/internal/ingress/controller/template/template.go +++ b/internal/ingress/controller/template/template.go @@ -1230,11 +1230,11 @@ func buildOpentracing(c, s interface{}) string { //nolint:gocritic // rewriting if-else to switch statement is not more readable if cfg.DatadogCollectorHost != "" { - buf.WriteString("opentracing_load_tracer /usr/local/lib/libdd_opentracing.so /etc/nginx/opentracing.json;") + buf.WriteString("opentracing_load_tracer /usr/local/lib/libdd_opentracing.so /etc/ingress-controller/telemetry/opentracing.json;") } else if cfg.ZipkinCollectorHost != "" { - buf.WriteString("opentracing_load_tracer /usr/local/lib/libzipkin_opentracing_plugin.so /etc/nginx/opentracing.json;") + buf.WriteString("opentracing_load_tracer /usr/local/lib/libzipkin_opentracing_plugin.so /etc/ingress-controller/telemetry/opentracing.json;") } else if cfg.JaegerCollectorHost != "" || cfg.JaegerEndpoint != "" { - buf.WriteString("opentracing_load_tracer /usr/local/lib/libjaegertracing_plugin.so /etc/nginx/opentracing.json;") + buf.WriteString("opentracing_load_tracer /usr/local/lib/libjaegertracing_plugin.so /etc/ingress-controller/telemetry/opentracing.json;") } buf.WriteString("\r\n") diff --git a/internal/ingress/controller/template/template_test.go b/internal/ingress/controller/template/template_test.go index 110967711..c4ad6cd3a 100644 --- a/internal/ingress/controller/template/template_test.go +++ b/internal/ingress/controller/template/template_test.go @@ -1663,7 +1663,7 @@ func TestBuildOpenTracing(t *testing.T) { EnableOpentracing: true, JaegerCollectorHost: "jaeger-host.com", } - expected = "opentracing_load_tracer /usr/local/lib/libjaegertracing_plugin.so /etc/nginx/opentracing.json;\r\n" + expected = "opentracing_load_tracer /usr/local/lib/libjaegertracing_plugin.so /etc/ingress-controller/telemetry/opentracing.json;\r\n" actual = buildOpentracing(cfgJaeger, []*ingress.Server{}) if expected != actual { @@ -1674,7 +1674,7 @@ func TestBuildOpenTracing(t *testing.T) { EnableOpentracing: true, ZipkinCollectorHost: "zipkin-host.com", } - expected = "opentracing_load_tracer /usr/local/lib/libzipkin_opentracing_plugin.so /etc/nginx/opentracing.json;\r\n" + expected = "opentracing_load_tracer /usr/local/lib/libzipkin_opentracing_plugin.so /etc/ingress-controller/telemetry/opentracing.json;\r\n" actual = buildOpentracing(cfgZipkin, []*ingress.Server{}) if expected != actual { @@ -1685,7 +1685,7 @@ func TestBuildOpenTracing(t *testing.T) { EnableOpentracing: true, DatadogCollectorHost: "datadog-host.com", } - expected = "opentracing_load_tracer /usr/local/lib/libdd_opentracing.so /etc/nginx/opentracing.json;\r\n" + expected = "opentracing_load_tracer /usr/local/lib/libdd_opentracing.so /etc/ingress-controller/telemetry/opentracing.json;\r\n" actual = buildOpentracing(cfgDatadog, []*ingress.Server{}) if expected != actual { @@ -1696,7 +1696,7 @@ func TestBuildOpenTracing(t *testing.T) { EnableOpentracing: true, JaegerEndpoint: "http://jaeger-collector.com:14268/api/traces", } - expected = "opentracing_load_tracer /usr/local/lib/libjaegertracing_plugin.so /etc/nginx/opentracing.json;\r\n" + expected = "opentracing_load_tracer /usr/local/lib/libjaegertracing_plugin.so /etc/ingress-controller/telemetry/opentracing.json;\r\n" actual = buildOpentracing(cfgJaegerEndpoint, []*ingress.Server{}) if expected != actual { @@ -1709,7 +1709,7 @@ func TestBuildOpenTracing(t *testing.T) { OpentracingOperationName: "my-operation-name", OpentracingLocationOperationName: "my-location-operation-name", } - expected = "opentracing_load_tracer /usr/local/lib/libdd_opentracing.so /etc/nginx/opentracing.json;\r\n" + expected = "opentracing_load_tracer /usr/local/lib/libdd_opentracing.so /etc/ingress-controller/telemetry/opentracing.json;\r\n" expected += "opentracing_operation_name \"my-operation-name\";\n" expected += "opentracing_location_operation_name \"my-location-operation-name\";\n" actual = buildOpentracing(cfgOpenTracing, []*ingress.Server{}) diff --git a/internal/nginx/maxmind.go b/internal/nginx/maxmind.go index 4719a22c1..1dc2f4d4e 100644 --- a/internal/nginx/maxmind.go +++ b/internal/nginx/maxmind.go @@ -56,7 +56,7 @@ var MaxmindRetriesTimeout = time.Second * 0 const minimumRetriesCount = 1 const ( - geoIPPath = "/etc/nginx/geoip" + geoIPPath = "/etc/ingress-controller/geoip" dbExtension = ".mmdb" maxmindURL = "https://download.maxmind.com/app/geoip_download?license_key=%v&edition_id=%v&suffix=tar.gz" diff --git a/magefiles/go.mod b/magefiles/go.mod index e846c884a..db252f7c3 100644 --- a/magefiles/go.mod +++ b/magefiles/go.mod @@ -17,7 +17,7 @@ require ( github.com/Masterminds/semver v1.5.0 // indirect github.com/cyphar/filepath-securejoin v0.2.4 // indirect github.com/dprotaso/go-yit v0.0.0-20191028211022-135eb7262960 // indirect - github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/gobwas/glob v0.2.3 // indirect github.com/golang/protobuf v1.5.3 // indirect diff --git a/magefiles/go.sum b/magefiles/go.sum index f89975c18..ddb7475da 100644 --- a/magefiles/go.sum +++ b/magefiles/go.sum @@ -14,8 +14,8 @@ github.com/dprotaso/go-yit v0.0.0-20191028211022-135eb7262960 h1:aRd8M7HJVZOqn/v github.com/dprotaso/go-yit v0.0.0-20191028211022-135eb7262960/go.mod h1:9HQzr9D/0PGwMEbC3d5AB7oi67+h4TsQqItC1GVYG58= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= -github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= @@ -113,7 +113,6 @@ golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/rootfs/Dockerfile b/rootfs/Dockerfile index 75cf3fc65..86517c6c1 100644 --- a/rootfs/Dockerfile +++ b/rootfs/Dockerfile @@ -49,9 +49,10 @@ COPY --chown=www-data:www-data bin/${TARGETARCH}/wait-shutdown / # with volumes (custom templates) RUN bash -xeu -c ' \ writeDirs=( \ - /etc/ingress-controller \ /etc/ingress-controller/ssl \ /etc/ingress-controller/auth \ + /etc/ingress-controller/geoip \ + /etc/ingress-controller/telemetry \ /var/log \ /var/log/nginx \ /tmp/nginx \ diff --git a/rootfs/chroot.sh b/rootfs/chroot.sh index 9f3cbd804..38547b69c 100755 --- a/rootfs/chroot.sh +++ b/rootfs/chroot.sh @@ -21,6 +21,8 @@ writeDirs=( \ /chroot/etc/ingress-controller \ /chroot/etc/ingress-controller/ssl \ /chroot/etc/ingress-controller/auth \ + /chroot/etc/ingress-controller/telemetry \ + /chroot/etc/ingress-controller/geoip \ /chroot/opt/modsecurity/var/log \ /chroot/opt/modsecurity/var/upload \ /chroot/opt/modsecurity/var/audit \ @@ -41,7 +43,8 @@ for dir in "${writeDirs[@]}"; do done mkdir -p /chroot/lib /chroot/proc /chroot/usr /chroot/bin /chroot/dev /chroot/run -cp /etc/passwd /etc/group /chroot/etc/ +cp /etc/passwd /etc/group /etc/hosts /chroot/etc/ cp -a /usr/* /chroot/usr/ cp -a /etc/nginx/* /chroot/etc/nginx/ +cp -a /etc/ingress-controller/* /chroot/etc/ingress-controller/ cp /lib/ld-musl-* /lib/libcrypto* /lib/libssl* /lib/libz* /chroot/lib/ diff --git a/rootfs/etc/nginx/template/nginx.tmpl b/rootfs/etc/nginx/template/nginx.tmpl index a6ebe5d2a..de0c5089b 100644 --- a/rootfs/etc/nginx/template/nginx.tmpl +++ b/rootfs/etc/nginx/template/nginx.tmpl @@ -175,7 +175,7 @@ http { {{ range $index, $file := $all.MaxmindEditionFiles }} {{ if eq $file "GeoLite2-Country.mmdb" }} - geoip2 /etc/nginx/geoip/GeoLite2-Country.mmdb { + geoip2 /etc/ingress-controller/geoip/GeoLite2-Country.mmdb { $geoip2_country_code source=$remote_addr country iso_code; $geoip2_country_name source=$remote_addr country names en; $geoip2_country_geoname_id source=$remote_addr country geoname_id; @@ -186,7 +186,7 @@ http { {{ end }} {{ if eq $file "GeoIP2-Country.mmdb" }} - geoip2 /etc/nginx/geoip/GeoIP2-Country.mmdb { + geoip2 /etc/ingress-controller/geoip/GeoIP2-Country.mmdb { $geoip2_country_code source=$remote_addr country iso_code; $geoip2_country_name source=$remote_addr country names en; $geoip2_country_geoname_id source=$remote_addr country geoname_id; @@ -197,7 +197,7 @@ http { {{ end }} {{ if eq $file "GeoLite2-City.mmdb" }} - geoip2 /etc/nginx/geoip/GeoLite2-City.mmdb { + geoip2 /etc/ingress-controller/geoip/GeoLite2-City.mmdb { $geoip2_city_country_code source=$remote_addr country iso_code; $geoip2_city_country_name source=$remote_addr country names en; $geoip2_city_country_geoname_id source=$remote_addr country geoname_id; @@ -220,7 +220,7 @@ http { {{ end }} {{ if eq $file "GeoIP2-City.mmdb" }} - geoip2 /etc/nginx/geoip/GeoIP2-City.mmdb { + geoip2 /etc/ingress-controller/geoip/GeoIP2-City.mmdb { $geoip2_city_country_code source=$remote_addr country iso_code; $geoip2_city_country_name source=$remote_addr country names en; $geoip2_city_country_geoname_id source=$remote_addr country geoname_id; @@ -243,21 +243,21 @@ http { {{ end }} {{ if eq $file "GeoLite2-ASN.mmdb" }} - geoip2 /etc/nginx/geoip/GeoLite2-ASN.mmdb { + geoip2 /etc/ingress-controller/geoip/GeoLite2-ASN.mmdb { $geoip2_asn source=$remote_addr autonomous_system_number; $geoip2_org source=$remote_addr autonomous_system_organization; } {{ end }} {{ if eq $file "GeoIP2-ASN.mmdb" }} - geoip2 /etc/nginx/geoip/GeoIP2-ASN.mmdb { + geoip2 /etc/ingress-controller/geoip/GeoIP2-ASN.mmdb { $geoip2_asn source=$remote_addr autonomous_system_number; $geoip2_org source=$remote_addr autonomous_system_organization; } {{ end }} {{ if eq $file "GeoIP2-ISP.mmdb" }} - geoip2 /etc/nginx/geoip/GeoIP2-ISP.mmdb { + geoip2 /etc/ingress-controller/geoip/GeoIP2-ISP.mmdb { $geoip2_isp source=$remote_addr isp; $geoip2_isp_org source=$remote_addr organization; $geoip2_asn source=$remote_addr default=0 autonomous_system_number; @@ -265,13 +265,13 @@ http { {{ end }} {{ if eq $file "GeoIP2-Connection-Type.mmdb" }} - geoip2 /etc/nginx/geoip/GeoIP2-Connection-Type.mmdb { + geoip2 /etc/ingress-controller/geoip/GeoIP2-Connection-Type.mmdb { $geoip2_connection_type connection_type; } {{ end }} {{ if eq $file "GeoIP2-Anonymous-IP.mmdb" }} - geoip2 /etc/nginx/geoip/GeoIP2-Anonymous-IP.mmdb { + geoip2 /etc/ingress-controller/geoip/GeoIP2-Anonymous-IP.mmdb { $geoip2_is_anon source=$remote_addr is_anonymous; $geoip2_is_anonymous source=$remote_addr default=0 is_anonymous; $geoip2_is_anonymous_vpn source=$remote_addr default=0 is_anonymous_vpn; @@ -469,7 +469,7 @@ http { ssl_session_tickets {{ if $cfg.SSLSessionTickets }}on{{ else }}off{{ end }}; {{ if not (empty $cfg.SSLSessionTicketKey ) }} - ssl_session_ticket_key /etc/nginx/tickets.key; + ssl_session_ticket_key /etc/ingress-controller/tickets.key; {{ end }} # slightly reduce the time-to-first-byte diff --git a/test/e2e/settings/geoip2.go b/test/e2e/settings/geoip2.go index e37286d5c..7da26d810 100644 --- a/test/e2e/settings/geoip2.go +++ b/test/e2e/settings/geoip2.go @@ -54,7 +54,7 @@ var _ = framework.DescribeSetting("Geoip2", func() { }) assert.Nil(ginkgo.GinkgoT(), err, "updating ingress controller deployment flags") - filename := fmt.Sprintf("/etc/nginx/geoip/%s.mmdb", edition) + filename := fmt.Sprintf("/etc/ingress-controller/geoip/%s.mmdb", edition) exec, err := f.ExecIngressPod(fmt.Sprintf(`sh -c "mkdir -p '%s' && wget -O '%s' '%s' 2>&1"`, filepath.Dir(filename), filename, testdataURL)) framework.Logf(exec) assert.Nil(ginkgo.GinkgoT(), err, fmt.Sprintln("error downloading test geoip2 db", filename)) diff --git a/test/e2e/settings/opentelemetry.go b/test/e2e/settings/opentelemetry.go index 15b5d165e..b5fc6ff4e 100644 --- a/test/e2e/settings/opentelemetry.go +++ b/test/e2e/settings/opentelemetry.go @@ -31,7 +31,7 @@ const ( opentelemetryOperationName = "opentelemetry-operation-name" opentelemetryLocationOperationName = "opentelemetry-location-operation-name" opentelemetryConfig = "opentelemetry-config" - opentelemetryConfigPath = "/etc/nginx/opentelemetry.toml" + opentelemetryConfigPath = "/etc/ingress-controller/telemetry/opentelemetry.toml" enable = "true" )