diff --git a/internal/ingress/controller/nginx.go b/internal/ingress/controller/nginx.go index 20fad5afb..6f138d754 100644 --- a/internal/ingress/controller/nginx.go +++ b/internal/ingress/controller/nginx.go @@ -52,6 +52,7 @@ import ( "k8s.io/ingress-nginx/internal/ingress/controller/process" "k8s.io/ingress-nginx/internal/ingress/controller/store" ngx_template "k8s.io/ingress-nginx/internal/ingress/controller/template" + "k8s.io/ingress-nginx/internal/ingress/controller/template/crossplane" "k8s.io/ingress-nginx/internal/ingress/metric" "k8s.io/ingress-nginx/internal/ingress/status" ing_net "k8s.io/ingress-nginx/internal/net" @@ -158,7 +159,7 @@ func NewNGINXController(config *Configuration, mc metric.Collector) *NGINXContro } onTemplateChange := func() { - template, err := ngx_template.NewTemplate(nginx.TemplatePath) + template, err := crossplane.NewTemplate() if err != nil { // this error is different from the rest because it must be clear why nginx is not working klog.ErrorS(err, "Error loading new template") @@ -170,7 +171,7 @@ func NewNGINXController(config *Configuration, mc metric.Collector) *NGINXContro n.syncQueue.EnqueueTask(task.GetDummyObject("template-change")) } - ngxTpl, err := ngx_template.NewTemplate(nginx.TemplatePath) + ngxTpl, err := crossplane.NewTemplate() if err != nil { klog.Fatalf("Invalid NGINX configuration template: %v", err) } @@ -700,7 +701,7 @@ func (n *NGINXController) OnUpdate(ingressCfg ingress.Configuration) error { err = n.testTemplate(content) if err != nil { - return err + return fmt.Errorf("err %s content %s", err, string(content)) } if klog.V(2).Enabled() { @@ -868,13 +869,14 @@ func (n *NGINXController) configureDynamically(pcfg *ingress.Configuration) erro } } - streamConfigurationChanged := !reflect.DeepEqual(n.runningConfig.TCPEndpoints, pcfg.TCPEndpoints) || !reflect.DeepEqual(n.runningConfig.UDPEndpoints, pcfg.UDPEndpoints) + // TODO: (ricardo) - Disable in case this is crossplane, we don't support stream on this mode + /*streamConfigurationChanged := !reflect.DeepEqual(n.runningConfig.TCPEndpoints, pcfg.TCPEndpoints) || !reflect.DeepEqual(n.runningConfig.UDPEndpoints, pcfg.UDPEndpoints) if streamConfigurationChanged { err := updateStreamConfiguration(pcfg.TCPEndpoints, pcfg.UDPEndpoints) if err != nil { return err } - } + }*/ serversChanged := !reflect.DeepEqual(n.runningConfig.Servers, pcfg.Servers) if serversChanged { diff --git a/internal/ingress/controller/template/crossplane/authlocation.go b/internal/ingress/controller/template/crossplane/authlocation.go new file mode 100644 index 000000000..64c15eeb6 --- /dev/null +++ b/internal/ingress/controller/template/crossplane/authlocation.go @@ -0,0 +1,251 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package crossplane + +import ( + "fmt" + "strings" + + ngx_crossplane "github.com/nginxinc/nginx-go-crossplane" + "k8s.io/ingress-nginx/internal/ingress/annotations/authreq" + "k8s.io/ingress-nginx/internal/ingress/controller/config" + "k8s.io/ingress-nginx/pkg/apis/ingress" +) + +type externalAuth struct { + URL string `json:"url"` + // Host contains the hostname defined in the URL + Host string `json:"host"` + SigninURL string `json:"signinUrl"` + SigninURLRedirectParam string `json:"signinUrlRedirectParam,omitempty"` + Method string `json:"method"` + ResponseHeaders []string `json:"responseHeaders,omitempty"` + RequestRedirect string `json:"requestRedirect"` + AuthSnippet string `json:"authSnippet"` + AuthCacheKey string `json:"authCacheKey"` + AuthCacheDuration []string `json:"authCacheDuration"` + KeepaliveConnections int `json:"keepaliveConnections"` + KeepaliveShareVars bool `json:"keepaliveShareVars"` + KeepaliveRequests int `json:"keepaliveRequests"` + KeepaliveTimeout int `json:"keepaliveTimeout"` + ProxySetHeaders map[string]string `json:"proxySetHeaders,omitempty"` + AlwaysSetCookie bool `json:"alwaysSetCookie,omitempty"` +} + +func buildExternalAuth(cfg any) *externalAuth { + switch v := cfg.(type) { + case config.GlobalExternalAuth: + return &externalAuth{ + AlwaysSetCookie: v.AlwaysSetCookie, + AuthCacheKey: v.AuthCacheKey, + AuthCacheDuration: v.AuthCacheDuration, + Method: v.Method, + Host: v.Host, + RequestRedirect: v.RequestRedirect, + ProxySetHeaders: v.ProxySetHeaders, + ResponseHeaders: v.ResponseHeaders, + URL: v.URL, + SigninURL: v.SigninURL, + SigninURLRedirectParam: v.SigninURLRedirectParam, + } + case authreq.Config: + return &externalAuth{ + AlwaysSetCookie: v.AlwaysSetCookie, + AuthCacheKey: v.AuthCacheKey, + AuthCacheDuration: v.AuthCacheDuration, + Method: v.Method, + Host: v.Host, + RequestRedirect: v.RequestRedirect, + ProxySetHeaders: v.ProxySetHeaders, + ResponseHeaders: v.ResponseHeaders, + URL: v.URL, + SigninURL: v.SigninURL, + SigninURLRedirectParam: v.SigninURLRedirectParam, + KeepaliveShareVars: v.KeepaliveShareVars, + KeepaliveConnections: v.KeepaliveConnections, + KeepaliveRequests: v.KeepaliveRequests, + KeepaliveTimeout: v.KeepaliveTimeout, + } + default: + return nil + } +} + +func (c *Template) buildAuthLocation(server *ingress.Server, + location *ingress.Location, locationConfig locationCfg) *ngx_crossplane.Directive { + locationDirectives := ngx_crossplane.Directives{ + buildDirective("internal"), + } + + if c.tplConfig.Cfg.EnableOpentelemetry || location.Opentelemetry.Enabled { + locationDirectives = append(locationDirectives, + buildDirective("opentelemetry", "on"), + buildDirective("opentelemetry_propagate"), + ) + } + + if !c.tplConfig.Cfg.EnableAuthAccessLog { + locationDirectives = append(locationDirectives, buildDirective("access_log", "off")) + } + + if locationConfig.externalAuth.AuthCacheKey != "" { + locationDirectives = append(locationDirectives, + buildDirective("set", "$tmp_cache_key", fmt.Sprintf("%s%s%s", server.Hostname, locationConfig.authPath, locationConfig.externalAuth.AuthCacheKey)), + buildDirective("set", "$cache_key", ""), + buildDirective("rewrite_by_lua_file", "/etc/nginx/lua/nginx/ngx_conf_rewrite_auth.lua"), + buildDirective("proxy_cache", "auth_cache"), + buildDirective("proxy_cache_key", "$cache_key"), + ) + for i := range locationConfig.externalAuth.AuthCacheDuration { + locationDirectives = append(locationDirectives, + buildDirective("proxy_cache_valid", strings.Split(locationConfig.externalAuth.AuthCacheDuration[i], " ")), + ) + } + } + + /* + ngx_auth_request module overrides variables in the parent request, + therefore we have to explicitly set this variable again so that when the parent request + resumes it has the correct value set for this variable so that Lua can pick backend correctly + */ + locationDirectives = append(locationDirectives, + buildDirective("set", "$proxy_upstream_name", location.Backend), + ) + + locationDirectives = append(locationDirectives, + buildDirective("proxy_pass_request_body", "off")) + + locationDirectives = append(locationDirectives, + buildDirective("proxy_ssl_server_name", "on")) + locationDirectives = append(locationDirectives, + buildDirective("proxy_pass_request_headers", "on")) + + locationDirectives = append(locationDirectives, + buildDirective("proxy_set_header", "Content-Length", "")) + locationDirectives = append(locationDirectives, + buildDirective("proxy_set_header", "X-Forwarded-Proto", "")) + locationDirectives = append(locationDirectives, + buildDirective("proxy_set_header", "X-Request-ID", "$req_id")) + locationDirectives = append(locationDirectives, + buildDirective("proxy_set_header", "Host", locationConfig.externalAuth.Host)) + locationDirectives = append(locationDirectives, + buildDirective("proxy_set_header", "X-Original-URL", "$scheme://$http_host$request_uri")) + locationDirectives = append(locationDirectives, + buildDirective("proxy_set_header", "X-Original-Method", "$request_method")) + locationDirectives = append(locationDirectives, + buildDirective("proxy_set_header", "X-Sent-From", "nginx-ingress-controller")) + locationDirectives = append(locationDirectives, + buildDirective("proxy_set_header", "X-Real-IP", "$remote_addr")) + + if locationConfig.externalAuth.Method != "" { + locationDirectives = append(locationDirectives, + buildDirective("proxy_method", locationConfig.externalAuth.Method)) + locationDirectives = append(locationDirectives, + buildDirective("proxy_set_header", "X-Original-URI", "$request_uri")) + locationDirectives = append(locationDirectives, + buildDirective("proxy_set_header", "X-Scheme", "$pass_access_scheme")) + } + + if c.tplConfig.Cfg.UseForwardedHeaders && c.tplConfig.Cfg.ComputeFullForwardedFor { + locationDirectives = append(locationDirectives, + buildDirective("proxy_set_header", "X-Forwarded-For", "$full_x_forwarded_for")) + } else { + locationDirectives = append(locationDirectives, + buildDirective("proxy_set_header", "X-Forwarded-For", "$remote_addr")) + } + + if locationConfig.externalAuth.RequestRedirect != "" { + locationDirectives = append(locationDirectives, + buildDirective("proxy_set_header", "X-Auth-Request-Redirect", locationConfig.externalAuth.RequestRedirect)) + } else { + locationDirectives = append(locationDirectives, + buildDirective("proxy_set_header", "X-Auth-Request-Redirect", "$request_uri")) + } + + if locationConfig.externalAuth.Method != "" { + locationDirectives = append(locationDirectives, + buildDirective("proxy_set_header", "X-Original-URI", "$request_uri")) + locationDirectives = append(locationDirectives, + buildDirective("proxy_set_header", "X-Scheme", "$pass_access_scheme")) + } + + if locationConfig.externalAuth.AuthCacheKey != "" { + locationDirectives = append(locationDirectives, + buildDirective("proxy_buffering", "on")) + } else { + locationDirectives = append(locationDirectives, + buildDirective("proxy_buffering", location.Proxy.ProxyBuffering)) + } + + locationDirectives = append(locationDirectives, + buildDirective("proxy_buffer_size", location.Proxy.BufferSize)) + locationDirectives = append(locationDirectives, + buildDirective("proxy_buffers", location.Proxy.BuffersNumber, location.Proxy.BufferSize)) + locationDirectives = append(locationDirectives, + buildDirective("proxy_request_buffering", location.Proxy.RequestBuffering)) + + if isValidByteSize(location.Proxy.BodySize, true) { + locationDirectives = append(locationDirectives, + buildDirective("client_max_body_size", location.Proxy.BodySize)) + } + + if isValidByteSize(location.ClientBodyBufferSize, false) { + locationDirectives = append(locationDirectives, + buildDirective("client_body_buffer_size", location.ClientBodyBufferSize)) + } + + if server.CertificateAuth.CAFileName != "" { + locationDirectives = append(locationDirectives, + buildDirective("proxy_set_header", "ssl-client-verify", "$ssl_client_verify")) + + locationDirectives = append(locationDirectives, + buildDirective("proxy_set_header", "ssl-client-subject-dn", "$ssl_client_s_dn")) + + locationDirectives = append(locationDirectives, + buildDirective("proxy_set_header", "ssl-client-issuer-dn", "$ssl_client_i_dn")) + + if server.CertificateAuth.PassCertToUpstream { + locationDirectives = append(locationDirectives, + buildDirective("proxy_set_header", "ssl-client-cert", "$ssl_client_escaped_cert")) + } + } + + for name, value := range locationConfig.externalAuth.ProxySetHeaders { + locationDirectives = append(locationDirectives, + buildDirective("proxy_set_header", name, value)) + } + + if locationConfig.applyAuthUpstream && locationConfig.applyGlobalAuth { + locationDirectives = append(locationDirectives, + buildDirective("proxy_http_version", "1.1")) + locationDirectives = append(locationDirectives, + buildDirective("proxy_set_header", "Connection", "")) + locationDirectives = append(locationDirectives, + buildDirective("set", "$target", + changeHostPort(locationConfig.externalAuth.URL, buildAuthUpstreamName(location, server.Hostname)))) + } else { + locationDirectives = append(locationDirectives, + buildDirective("proxy_http_version", location.Proxy.ProxyHTTPVersion)) + locationDirectives = append(locationDirectives, + buildDirective("set", "$target", locationConfig.externalAuth.URL)) + } + locationDirectives = append(locationDirectives, + buildDirective("proxy_pass", "$target")) + + return buildBlockDirective("location", + []string{"=", locationConfig.authPath}, locationDirectives) +} diff --git a/internal/ingress/controller/template/crossplane/cors.go b/internal/ingress/controller/template/crossplane/cors.go new file mode 100644 index 000000000..932c2489b --- /dev/null +++ b/internal/ingress/controller/template/crossplane/cors.go @@ -0,0 +1,81 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package crossplane + +import ( + "fmt" + + ngx_crossplane "github.com/nginxinc/nginx-go-crossplane" + "k8s.io/ingress-nginx/internal/ingress/annotations/cors" +) + +func buildCorsDirectives(locationcors cors.Config) ngx_crossplane.Directives { + directives := make(ngx_crossplane.Directives, 0) + if len(locationcors.CorsAllowOrigin) > 0 { + directives = append(directives, buildCorsOriginRegex(locationcors.CorsAllowOrigin)...) + + } + directives = append(directives, + buildBlockDirective("if", + []string{"$request_method", "=", "OPTIONS"}, ngx_crossplane.Directives{ + buildDirective("set", "$cors", "${cors}options"), + }, + ), + ) + + directives = append(directives, + commonCorsDirective(locationcors, false), + commonCorsDirective(locationcors, true), + ) + return directives +} + +// commonCorsDirective builds the common cors directives for a location +func commonCorsDirective(cfg cors.Config, options bool) *ngx_crossplane.Directive { + corsDir := "true" + if options { + corsDir = "trueoptions" + } + corsBlock := buildBlockDirective("if", []string{"$cors", "=", corsDir}, + ngx_crossplane.Directives{ + buildDirective("more_set_headers", "Access-Control-Allow-Origin: $http_origin"), + buildDirective("more_set_headers", fmt.Sprintf("Access-Control-Allow-Methods: %s", cfg.CorsAllowMethods)), + buildDirective("more_set_headers", fmt.Sprintf("Access-Control-Allow-Headers: %s", cfg.CorsAllowHeaders)), + buildDirective("more_set_headers", fmt.Sprintf("Access-Control-Max-Age: %d", cfg.CorsMaxAge)), + }, + ) + + if cfg.CorsAllowCredentials { + corsBlock.Block = append(corsBlock.Block, + buildDirective("more_set_headers", fmt.Sprintf("Access-Control-Allow-Credentials: %t", cfg.CorsAllowCredentials)), + ) + } + if cfg.CorsExposeHeaders != "" { + corsBlock.Block = append(corsBlock.Block, + buildDirective("more_set_headers", fmt.Sprintf("Access-Control-Expose-Headers: %s", cfg.CorsExposeHeaders)), + ) + } + + if options { + corsBlock.Block = append(corsBlock.Block, + buildDirective("more_set_headers", "Content-Type: text/plain charset=UTF-8"), + buildDirective("more_set_headers", "Content-Length: 0"), + buildDirective("return", "204"), + ) + } + return corsBlock +} diff --git a/internal/ingress/controller/template/crossplane/crossplane.go b/internal/ingress/controller/template/crossplane/crossplane.go index 1307de2f7..13796fdb8 100644 --- a/internal/ingress/controller/template/crossplane/crossplane.go +++ b/internal/ingress/controller/template/crossplane/crossplane.go @@ -18,10 +18,12 @@ package crossplane import ( "bytes" + "os" ngx_crossplane "github.com/nginxinc/nginx-go-crossplane" "k8s.io/ingress-nginx/internal/ingress/controller/config" + "k8s.io/ingress-nginx/internal/ingress/controller/template/crossplane/extramodules" ) /* @@ -41,7 +43,7 @@ type Template struct { mimeFile string } -func NewTemplate() *Template { +func NewTemplate() (*Template, error) { lua := ngx_crossplane.Lua{} return &Template{ mimeFile: "/etc/nginx/mime.types", @@ -50,7 +52,7 @@ func NewTemplate() *Template { lua.RegisterBuilder(), }, }, - } + }, nil } func (c *Template) SetMimeFile(file string) { @@ -72,5 +74,49 @@ func (c *Template) Write(conf *config.TemplateConfig) ([]byte, error) { var buf bytes.Buffer err := ngx_crossplane.Build(&buf, *c.config, &ngx_crossplane.BuildOptions{}) + if err != nil { + return nil, err + } + + lua := ngx_crossplane.Lua{} + options := ngx_crossplane.ParseOptions{ + ParseComments: true, + ErrorOnUnknownDirectives: true, + StopParsingOnError: true, + DirectiveSources: []ngx_crossplane.MatchFunc{ + ngx_crossplane.DefaultDirectivesMatchFunc, + ngx_crossplane.MatchLuaLatest, + ngx_crossplane.MatchHeadersMoreLatest, + extramodules.BrotliMatchFn, + extramodules.OpentelemetryMatchFn, + ngx_crossplane.MatchGeoip2Latest, + }, + LexOptions: ngx_crossplane.LexOptions{ + Lexers: []ngx_crossplane.RegisterLexer{lua.RegisterLexer()}, + }, + // Modules that needs to be ported: + // // https://github.com/openresty/set-misc-nginx-module?tab=readme-ov-file#set_escape_uri + IgnoreDirectives: []string{"set_escape_uri"}, + } + + tmpFile, err := os.CreateTemp("", "") + if err != nil { + return nil, err + } + defer func() { + _ = os.Remove(tmpFile.Name()) + _ = tmpFile.Close() + }() + + _, err = tmpFile.Write(buf.Bytes()) + if err != nil { + return nil, err + } + + _, err = ngx_crossplane.Parse(tmpFile.Name(), &options) + if err != nil { + return nil, err + } + return buf.Bytes(), err } diff --git a/internal/ingress/controller/template/crossplane/crossplane_internal_test.go b/internal/ingress/controller/template/crossplane/crossplane_internal_test.go index b8bc3258d..73093c1a8 100644 --- a/internal/ingress/controller/template/crossplane/crossplane_internal_test.go +++ b/internal/ingress/controller/template/crossplane/crossplane_internal_test.go @@ -41,14 +41,15 @@ func Test_Internal_buildEvents(t *testing.T) { Directive: "events", Block: ngx_crossplane.Directives{ buildDirective("worker_connections", 16384), - buildDirective("use", "epool"), + buildDirective("use", "epoll"), buildDirective("multi_accept", true), }, }, }, } - cplane := NewTemplate() + cplane, err := NewTemplate() + require.NoError(t, err) cplane.config = &c cplane.tplConfig = tplConfig cplane.buildEvents() @@ -72,7 +73,7 @@ func Test_Internal_buildEvents(t *testing.T) { Directive: "events", Block: ngx_crossplane.Directives{ buildDirective("worker_connections", 50), - buildDirective("use", "epool"), + buildDirective("use", "epoll"), buildDirective("multi_accept", false), buildDirective("debug_connection", "127.0.0.1/32"), buildDirective("debug_connection", "192.168.0.10"), @@ -81,7 +82,8 @@ func Test_Internal_buildEvents(t *testing.T) { }, } - cplane := NewTemplate() + cplane, err := NewTemplate() + require.NoError(t, err) cplane.config = &c cplane.tplConfig = tplConfig cplane.buildEvents() diff --git a/internal/ingress/controller/template/crossplane/crossplane_internal_utils_test.go b/internal/ingress/controller/template/crossplane/crossplane_internal_utils_test.go index f31701781..b5c2b6962 100644 --- a/internal/ingress/controller/template/crossplane/crossplane_internal_utils_test.go +++ b/internal/ingress/controller/template/crossplane/crossplane_internal_utils_test.go @@ -17,6 +17,7 @@ limitations under the License. package crossplane import ( + "reflect" "testing" ngx_crossplane "github.com/nginxinc/nginx-go-crossplane" @@ -67,3 +68,36 @@ func Test_Internal_buildLuaDictionaries(t *testing.T) { require.Equal(t, "lua_shared_dict", directives[1].Directive) require.Equal(t, []string{"otherdict", "1025K"}, directives[1].Args) } + +func Test_Internal_buildCorsOriginRegex(t *testing.T) { + tests := []struct { + name string + corsOrigins []string + want ngx_crossplane.Directives + }{ + { + name: "wildcard returns a single directive", + corsOrigins: []string{"*"}, + want: ngx_crossplane.Directives{ + buildDirective("set", "$http_origin", "*"), + buildDirective("set", "$cors", "true"), + }, + }, + { + name: "multiple hosts should be changed properly", + corsOrigins: []string{"*.xpto.com", " lalala.com"}, + want: ngx_crossplane.Directives{ + buildBlockDirective("if", []string{"$http_origin", "~*", "([A-Za-z0-9\\-]+\\.xpto\\.com)", "|", "(lalala\\.com)"}, + ngx_crossplane.Directives{buildDirective("set", "$cors", "true")}, + ), + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := buildCorsOriginRegex(tt.corsOrigins); !reflect.DeepEqual(got, tt.want) { + t.Errorf("buildCorsOriginRegex() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/internal/ingress/controller/template/crossplane/crossplane_test.go b/internal/ingress/controller/template/crossplane/crossplane_test.go index 08ea94276..3d3fa745d 100644 --- a/internal/ingress/controller/template/crossplane/crossplane_test.go +++ b/internal/ingress/controller/template/crossplane/crossplane_test.go @@ -24,10 +24,18 @@ import ( ngx_crossplane "github.com/nginxinc/nginx-go-crossplane" "github.com/stretchr/testify/require" + "k8s.io/ingress-nginx/internal/ingress/annotations/authreq" + "k8s.io/ingress-nginx/internal/ingress/annotations/authtls" + "k8s.io/ingress-nginx/internal/ingress/annotations/cors" + "k8s.io/ingress-nginx/internal/ingress/annotations/mirror" + "k8s.io/ingress-nginx/internal/ingress/annotations/proxy" + "k8s.io/ingress-nginx/internal/ingress/annotations/proxyssl" "k8s.io/ingress-nginx/internal/ingress/controller/config" "k8s.io/ingress-nginx/internal/ingress/controller/template/crossplane" "k8s.io/ingress-nginx/internal/ingress/controller/template/crossplane/extramodules" + "k8s.io/ingress-nginx/internal/ingress/resolver" "k8s.io/ingress-nginx/pkg/apis/ingress" + utilingress "k8s.io/ingress-nginx/pkg/util/ingress" ) const mockMimeTypes = ` @@ -38,6 +46,28 @@ types { } ` +func defaultConfig() *config.TemplateConfig { + tplConfig := &config.TemplateConfig{ + Cfg: config.NewDefault(), + } + tplConfig.ListenPorts = &config.ListenPorts{ + HTTP: 80, + HTTPS: 443, + Health: 10245, + Default: 8080, + SSLProxy: 442, + } + defaultCertificate := &ingress.SSLCert{ + PemFileName: "bla.crt", + PemCertKey: "bla.key", + } + tplConfig.StatusPort = 10246 + tplConfig.StatusPath = "/status" + tplConfig.HealthzURI = "/healthz" + tplConfig.Cfg.DefaultSSLCertificate = defaultCertificate + return tplConfig +} + var resolvers = []net.IP{net.ParseIP("::1"), net.ParseIP("192.168.20.10")} // TestTemplate should be a roundtrip test. @@ -48,21 +78,21 @@ var resolvers = []net.IP{net.ParseIP("::1"), net.ParseIP("192.168.20.10")} func TestCrossplaneTemplate(t *testing.T) { lua := ngx_crossplane.Lua{} options := ngx_crossplane.ParseOptions{ + ParseComments: true, ErrorOnUnknownDirectives: true, StopParsingOnError: true, - IgnoreDirectives: []string{"more_clear_headers", "more_set_headers"}, // TODO: Add more_set_headers DirectiveSources: []ngx_crossplane.MatchFunc{ ngx_crossplane.DefaultDirectivesMatchFunc, ngx_crossplane.MatchLuaLatest, + ngx_crossplane.MatchHeadersMoreLatest, extramodules.BrotliMatchFn, + extramodules.OpentelemetryMatchFn, + ngx_crossplane.MatchGeoip2Latest, }, LexOptions: ngx_crossplane.LexOptions{ Lexers: []ngx_crossplane.RegisterLexer{lua.RegisterLexer()}, }, - } - defaultCertificate := &ingress.SSLCert{ - PemFileName: "bla.crt", - PemCertKey: "bla.key", + IgnoreDirectives: []string{"set_escape_uri"}, } mimeFile, err := os.CreateTemp("", "") @@ -71,13 +101,11 @@ func TestCrossplaneTemplate(t *testing.T) { require.NoError(t, err) require.NoError(t, mimeFile.Close()) - tpl := crossplane.NewTemplate() + tpl, err := crossplane.NewTemplate() + require.NoError(t, err) t.Run("it should be able to marshall and unmarshall the default configuration", func(t *testing.T) { - tplConfig := &config.TemplateConfig{ - Cfg: config.NewDefault(), - } - tplConfig.Cfg.DefaultSSLCertificate = defaultCertificate + tplConfig := defaultConfig() tplConfig.Cfg.EnableBrotli = true tplConfig.Cfg.HideHeaders = []string{"x-fake-header", "x-another-fake-header"} tplConfig.Cfg.Resolver = resolvers @@ -101,11 +129,137 @@ func TestCrossplaneTemplate(t *testing.T) { require.NoError(t, err) }) - t.Run("it should set the right logging configs", func(t *testing.T) { - tplConfig := &config.TemplateConfig{ - Cfg: config.NewDefault(), + t.Run("it should be able to marshall and unmarshall with server config", func(t *testing.T) { + tplConfig := defaultConfig() + tplConfig.EnableMetrics = true + tplConfig.Cfg.EnableBrotli = true + tplConfig.Cfg.EnableOpentelemetry = true + tplConfig.Cfg.HideHeaders = []string{"x-fake-header", "x-another-fake-header"} + tplConfig.Cfg.Resolver = resolvers + tplConfig.Cfg.DisableIpv6DNS = true + tplConfig.IsIPV6Enabled = true + tplConfig.Cfg.BindAddressIpv6 = []string{"[::cabe:ca]"} + tplConfig.Cfg.BlockReferers = []string{"testlala.com"} + tplConfig.Cfg.ReusePort = true + tplConfig.BacklogSize = 5 + tplConfig.Cfg.BlockUserAgents = []string{"somebrowser"} + tplConfig.Cfg.UseForwardedHeaders = true + tplConfig.Cfg.LogFormatEscapeNone = true + tplConfig.Cfg.DisableAccessLog = true + tplConfig.Cfg.UpstreamKeepaliveConnections = 0 + tplConfig.Cfg.CustomHTTPErrors = []int{411, 412, 413} // Duplicated on purpose + tplConfig.RedirectServers = []*utilingress.Redirect{ + { + From: "www.xpto123.com", + To: "www.abcdefg.tld", + }, } - tplConfig.Cfg.DefaultSSLCertificate = defaultCertificate + tplConfig.Servers = []*ingress.Server{ + { + Hostname: "_", + }, + { + Hostname: "*.something.com", + Aliases: []string{"abc.com", "def.com"}, + Locations: []*ingress.Location{ + { + Mirror: mirror.Config{ + Source: "/mirror", + Host: "something.com", + Target: "http://www.mymirror.com", + RequestBody: "off", + }, + }, + { + DefaultBackendUpstreamName: "something", + CustomHTTPErrors: []int{403, 404, 403, 409}, // Duplicated on purpose! + }, + { + DefaultBackendUpstreamName: "otherthing", + CustomHTTPErrors: []int{403, 404, 403, 409}, // Duplicated on purpose! + }, + { + CorsConfig: cors.Config{ + CorsEnabled: true, + CorsAllowOrigin: []string{"xpto.com", "*.bla.com"}, + CorsAllowMethods: "GET,POST", + CorsAllowHeaders: "XPTO", + CorsMaxAge: 600, + CorsAllowCredentials: true, + CorsExposeHeaders: "XPTO", + }, + Backend: "somebackend", + ClientBodyBufferSize: "512k", + Proxy: proxy.Config{ + ProxyBuffering: "on", + RequestBuffering: "on", + BuffersNumber: 10, + BufferSize: "1024k", + ProxyHTTPVersion: "1.1", + NextUpstream: "10.10.10.10", + }, + ExternalAuth: authreq.Config{ + AuthCacheDuration: []string{"60s"}, + Host: "someauth.com", + URL: "http://someauth.com", + Method: "GET", + ProxySetHeaders: map[string]string{ + "someheader": "something", + }, + AuthCacheKey: "blabla", + SigninURL: "http://externallogin.tld", + }, + Path: "/xpto123", + }, + }, + }, + { + Hostname: "otherthing.com", + Aliases: []string{"abcde.com", "xpto.com"}, + CertificateAuth: authtls.Config{ + MatchCN: "CN=bla; listen xpto\"", + AuthSSLCert: resolver.AuthSSLCert{ + CAFileName: "/something/xpto.crt", + CRLFileName: "/something/xpto.crt", + }, + VerifyClient: "optional", + ValidationDepth: 2, + ErrorPage: "/xpto.html", + }, + ProxySSL: proxyssl.Config{ + AuthSSLCert: resolver.AuthSSLCert{ + CAFileName: "/something/xpto.crt", + PemFileName: "/something/mycert.crt", + }, + Ciphers: "HIGH:!aNULL:!MD5", + Protocols: "TLSv1 TLSv1.1 TLSv1.2 TLSv1.3", + Verify: "on", + VerifyDepth: 2, + ProxySSLName: "xpto.com", + ProxySSLServerName: "on", + }, + SSLCiphers: "HIGH:!aNULL:", + SSLPreferServerCiphers: "on", + }, + } + + tpl.SetMimeFile(mimeFile.Name()) + content, err := tpl.Write(tplConfig) + require.NoError(t, err) + + tmpFile, err := os.CreateTemp("", "") + require.NoError(t, err) + _, err = tmpFile.Write(content) + require.NoError(t, err) + require.NoError(t, tmpFile.Close()) + + _, err = ngx_crossplane.Parse(tmpFile.Name(), &options) + require.NoError(t, err) + require.Equal(t, "bla", string(content)) + }) + + t.Run("it should set the right logging configs", func(t *testing.T) { + tplConfig := defaultConfig() tplConfig.Cfg.DisableAccessLog = false tplConfig.Cfg.HTTPAccessLogPath = "/lalala.log" @@ -124,10 +278,7 @@ func TestCrossplaneTemplate(t *testing.T) { }) t.Run("it should be able to marshall and unmarshall the specified configuration", func(t *testing.T) { - tplConfig := &config.TemplateConfig{ - Cfg: config.NewDefault(), - } - tplConfig.Cfg.DefaultSSLCertificate = defaultCertificate + tplConfig := defaultConfig() tplConfig.Cfg.WorkerCPUAffinity = "0001 0010 0100 1000" tplConfig.Cfg.LuaSharedDicts = map[string]int{ "configuration_data": 10240, @@ -186,7 +337,9 @@ func TestCrossplaneTemplate(t *testing.T) { tplConfig.Cfg.UpstreamKeepaliveTimeout = 200 tplConfig.Cfg.UpstreamKeepaliveRequests = 15 - tpl = crossplane.NewTemplate() + tpl, err = crossplane.NewTemplate() + require.NoError(t, err) + tpl.SetMimeFile(mimeFile.Name()) content, err := tpl.Write(tplConfig) require.NoError(t, err) diff --git a/internal/ingress/controller/template/crossplane/events.go b/internal/ingress/controller/template/crossplane/events.go index fa0c599e5..ba0a76328 100644 --- a/internal/ingress/controller/template/crossplane/events.go +++ b/internal/ingress/controller/template/crossplane/events.go @@ -25,7 +25,7 @@ func (c *Template) buildEvents() { Directive: "events", Block: ngx_crossplane.Directives{ buildDirective("worker_connections", c.tplConfig.Cfg.MaxWorkerConnections), - buildDirective("use", "epool"), + buildDirective("use", "epoll"), buildDirective("multi_accept", c.tplConfig.Cfg.EnableMultiAccept), }, } diff --git a/internal/ingress/controller/template/crossplane/extramodules/README.md b/internal/ingress/controller/template/crossplane/extramodules/README.md index 6bd8a4e6b..b63277687 100644 --- a/internal/ingress/controller/template/crossplane/extramodules/README.md +++ b/internal/ingress/controller/template/crossplane/extramodules/README.md @@ -6,5 +6,5 @@ The generation of the files is done using go-crossplane generator ## Brotli ``` -go run ./cmd/generate/ -src-path=ngx_brotli/ -directive-map-name=brotliDirectives -match-func-name=BrotliMatchFn > ../ingress-crossplane/internal/ingress/controller/template/crossplane/extramodules/brotli.go +go run ./cmd/generate/ -src-path=ngx_brotli/ -directive-map-name=brotliDirectives -match-func-name=BrotliMatchFn > ../ingress-nginx/internal/ingress/controller/template/crossplane/extramodules/brotli.go ``` \ No newline at end of file diff --git a/internal/ingress/controller/template/crossplane/extramodules/opentelemetry.go b/internal/ingress/controller/template/crossplane/extramodules/opentelemetry.go new file mode 100644 index 000000000..f3a9cde9a --- /dev/null +++ b/internal/ingress/controller/template/crossplane/extramodules/opentelemetry.go @@ -0,0 +1,71 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/** + * Copyright (c) F5, Inc. + * + * This source code is licensed under the Apache License, Version 2.0 license found in the + * LICENSE file in the root directory of this source tree. + */ + +// Code generated by generator; DO NOT EDIT. +// All the definitions are extracted from the source code +// Each bit mask describes these behaviors: +// - how many arguments the directive can take +// - whether or not it is a block directive +// - whether this is a flag (takes one argument that's either "on" or "off") +// - which contexts it's allowed to be in + +package extramodules + +var opentelemetryDirectives = map[string][]uint{ + "opentelemetry": { + ngxHTTPMainConf | ngxHTTPSrvConf | ngxHTTPLocConf | ngxConfTake1, + }, + "opentelemetry_attribute": { + ngxHTTPMainConf | ngxHTTPSrvConf | ngxHTTPLocConf | ngxConfTake2, + }, + "opentelemetry_capture_headers": { + ngxHTTPMainConf | ngxHTTPSrvConf | ngxHTTPLocConf | ngxConfTake1, + }, + "opentelemetry_config": { + ngxHTTPMainConf | ngxConfTake1, + }, + "opentelemetry_ignore_paths": { + ngxHTTPMainConf | ngxHTTPSrvConf | ngxHTTPLocConf | ngxConfTake1, + }, + "opentelemetry_operation_name": { + ngxHTTPMainConf | ngxHTTPSrvConf | ngxHTTPLocConf | ngxConfTake1, + }, + "opentelemetry_propagate": { + ngxHTTPMainConf | ngxHTTPSrvConf | ngxHTTPLocConf | ngxConfNoArgs | ngxConfTake1, + }, + "opentelemetry_sensitive_header_names": { + ngxHTTPMainConf | ngxHTTPSrvConf | ngxHTTPLocConf | ngxConfTake1, + }, + "opentelemetry_sensitive_header_values": { + ngxHTTPMainConf | ngxHTTPSrvConf | ngxHTTPLocConf | ngxConfTake1, + }, + "opentelemetry_trust_incoming_spans": { + ngxHTTPMainConf | ngxHTTPSrvConf | ngxHTTPLocConf | ngxConfTake1, + }, +} + + +func OpentelemetryMatchFn(directive string) ([]uint, bool) { + m, ok := opentelemetryDirectives[directive] + return m, ok +} diff --git a/internal/ingress/controller/template/crossplane/http.go b/internal/ingress/controller/template/crossplane/http.go index dc3e43dce..5da0b248a 100644 --- a/internal/ingress/controller/template/crossplane/http.go +++ b/internal/ingress/controller/template/crossplane/http.go @@ -22,6 +22,8 @@ import ( "strings" ngx_crossplane "github.com/nginxinc/nginx-go-crossplane" + + utilingress "k8s.io/ingress-nginx/pkg/util/ingress" ) func (c *Template) initHTTPDirectives() ngx_crossplane.Directives { @@ -124,7 +126,7 @@ func (c *Template) buildHTTP() { httpBlock = append(httpBlock, buildDirective("gzip_comp_level", cfg.GzipLevel)) httpBlock = append(httpBlock, buildDirective("gzip_http_version", "1.1")) httpBlock = append(httpBlock, buildDirective("gzip_min_length", cfg.GzipMinLength)) - httpBlock = append(httpBlock, buildDirective("gzip_types", cfg.GzipTypes)) + httpBlock = append(httpBlock, buildDirective("gzip_types", strings.Split(cfg.GzipTypes, " "))) httpBlock = append(httpBlock, buildDirective("gzip_proxied", "any")) httpBlock = append(httpBlock, buildDirective("gzip_vary", "on")) @@ -140,10 +142,23 @@ func (c *Template) buildHTTP() { httpBlock = append(httpBlock, buildDirective("brotli_types", cfg.BrotliTypes)) } + if (c.tplConfig.Cfg.EnableOpentelemetry || shouldLoadOpentelemetryModule(c.tplConfig.Servers)) && + cfg.OpentelemetryOperationName != "" { + httpBlock = append(httpBlock, buildDirective("opentelemetry_operation_name", cfg.OpentelemetryOperationName)) + } + if !cfg.ShowServerTokens { httpBlock = append(httpBlock, buildDirective("more_clear_headers", "Server")) } + if cfg.UseGeoIP2 && c.tplConfig.MaxmindEditionFiles != nil && len(*c.tplConfig.MaxmindEditionFiles) > 0 { + geoipDirectives := buildGeoIPDirectives(cfg.GeoIP2AutoReloadMinutes, *c.tplConfig.MaxmindEditionFiles) + // We do this to avoid adding empty blocks + if len(geoipDirectives) > 0 { + httpBlock = append(httpBlock, geoipDirectives...) + } + } + httpBlock = append(httpBlock, buildBlockDirective( "geo", []string{"$literal_dollar"}, @@ -153,11 +168,9 @@ func (c *Template) buildHTTP() { )) if len(c.tplConfig.AddHeaders) > 0 { - additionalHeaders := make([]string, 0) for headerName, headerValue := range c.tplConfig.AddHeaders { - additionalHeaders = append(additionalHeaders, fmt.Sprintf("%s: %s", headerName, headerValue)) + httpBlock = append(httpBlock, buildDirective("more_set_headers", fmt.Sprintf("%s: %s", headerName, headerValue))) } - httpBlock = append(httpBlock, buildDirective("more_set_headers", additionalHeaders)) } escape := "" @@ -257,15 +270,6 @@ func (c *Template) buildHTTP() { httpBlock = append(httpBlock, buildDirective("proxy_pass_header", "Server")) } - if cfg.EnableBrotli { - httpBlock = append(httpBlock, - buildDirective("brotli", "on"), - buildDirective("brotli_comp_level", cfg.BrotliLevel), - buildDirective("brotli_min_length", cfg.BrotliMinLength), - buildDirective("brotli_types", strings.Split(cfg.BrotliTypes, " ")), - ) - } - for k := range cfg.HideHeaders { httpBlock = append(httpBlock, buildDirective("proxy_hide_header", cfg.HideHeaders[k])) } @@ -284,6 +288,30 @@ func (c *Template) buildHTTP() { } httpBlock = append(httpBlock, buildBlockDirective("upstream", []string{"upstream_balancer"}, blockUpstreamDirectives)) + // Adding Rate limit + for _, rl := range filterRateLimits(c.tplConfig.Servers) { + id := fmt.Sprintf("$allowlist_%s", rl.ID) + httpBlock = append(httpBlock, buildDirective("#", "Ratelimit", rl.Name)) + rlDirectives := ngx_crossplane.Directives{ + buildDirective("default", 0), + } + for _, ip := range rl.Allowlist { + rlDirectives = append(rlDirectives, buildDirective(ip, "1")) + } + mapRateLimitDirective := buildMapDirective(id, fmt.Sprintf("$limit_%s", rl.ID), ngx_crossplane.Directives{ + buildDirective("0", cfg.LimitConnZoneVariable), + buildDirective("1", ""), + }) + httpBlock = append(httpBlock, buildBlockDirective("geo", []string{"$remote_addr", id}, rlDirectives), mapRateLimitDirective) + } + + zoneRL := buildRateLimitZones(c.tplConfig.Servers) + if len(zoneRL) > 0 { + httpBlock = append(httpBlock, zoneRL...) + } + + // End of Rate limit configs + for i := range cfg.BlockCIDRs { httpBlock = append(httpBlock, buildDirective("deny", strings.TrimSpace(cfg.BlockCIDRs[i]))) } @@ -309,6 +337,65 @@ func (c *Template) buildHTTP() { fmt.Sprintf("@custom_upstream-default-backend_%d", v))) } + if redirectServers, ok := c.tplConfig.RedirectServers.([]*utilingress.Redirect); ok { + for _, server := range redirectServers { + httpBlock = append(httpBlock, buildStartServer(server.From)) + serverBlock := c.buildRedirectServer(server) + httpBlock = append(httpBlock, serverBlock) + httpBlock = append(httpBlock, buildEndServer(server.From)) + } + } + + /* + {{ range $server := $servers }} + {{ range $location := $server.Locations }} + {{ $applyGlobalAuth := shouldApplyGlobalAuth $location $all.Cfg.GlobalExternalAuth.URL }} + {{ $applyAuthUpstream := shouldApplyAuthUpstream $location $all.Cfg }} + {{ if and (eq $applyAuthUpstream true) (eq $applyGlobalAuth false) }} + ## start auth upstream {{ $server.Hostname }}{{ $location.Path }} + upstream {{ buildAuthUpstreamName $location $server.Hostname }} { + {{- $externalAuth := $location.ExternalAuth }} + server {{ extractHostPort $externalAuth.URL }}; + + keepalive {{ $externalAuth.KeepaliveConnections }}; + keepalive_requests {{ $externalAuth.KeepaliveRequests }}; + keepalive_timeout {{ $externalAuth.KeepaliveTimeout }}s; + } + ## end auth upstream {{ $server.Hostname }}{{ $location.Path }} + {{ end }} + {{ end }} + {{ end }} + */ + for _, server := range c.tplConfig.Servers { + for _, location := range server.Locations { + if shouldApplyAuthUpstream(location, cfg) && !shouldApplyGlobalAuth(location, cfg.GlobalExternalAuth.URL) { + authUpstreamBlock := buildBlockDirective("upstream", + []string{buildAuthUpstreamName(location, server.Hostname)}, ngx_crossplane.Directives{ + buildDirective("server", extractHostPort(location.ExternalAuth.URL)), + buildDirective("keepalive", location.ExternalAuth.KeepaliveConnections), + buildDirective("keepalive_requests", location.ExternalAuth.KeepaliveRequests), + buildDirective("keepalive_timeout", seconds(location.ExternalAuth.KeepaliveTimeout)), + }, + ) + httpBlock = append(httpBlock, + buildStartAuthUpstream(server.Hostname, location.Path), + authUpstreamBlock, + buildEndAuthUpstream(server.Hostname, location.Path), + ) + } + } + } + + for _, server := range c.tplConfig.Servers { + httpBlock = append(httpBlock, buildStartServer(server.Hostname)) + serverBlock := c.buildServerDirective(server) + httpBlock = append(httpBlock, serverBlock) + httpBlock = append(httpBlock, buildEndServer(server.Hostname)) + } + + httpBlock = append(httpBlock, c.buildDefaultBackend()) + httpBlock = append(httpBlock, c.buildHealthAndStatsServer()) + c.config.Parsed = append(c.config.Parsed, &ngx_crossplane.Directive{ Directive: "http", Block: httpBlock, diff --git a/internal/ingress/controller/template/crossplane/location.go b/internal/ingress/controller/template/crossplane/location.go new file mode 100644 index 000000000..af2ee433b --- /dev/null +++ b/internal/ingress/controller/template/crossplane/location.go @@ -0,0 +1,756 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package crossplane + +import ( + "fmt" + "sort" + "strconv" + "strings" + + ngx_crossplane "github.com/nginxinc/nginx-go-crossplane" + "k8s.io/apimachinery/pkg/util/sets" + + "k8s.io/ingress-nginx/internal/ingress/controller/config" + "k8s.io/ingress-nginx/pkg/apis/ingress" +) + +func buildMirrorLocationDirective(locs []*ingress.Location) ngx_crossplane.Directives { + mirrorDirectives := make(ngx_crossplane.Directives, 0) + + mapped := sets.Set[string]{} + + for _, loc := range locs { + if loc.Mirror.Source == "" || loc.Mirror.Target == "" || loc.Mirror.Host == "" { + continue + } + + if mapped.Has(loc.Mirror.Source) { + continue + } + + mapped.Insert(loc.Mirror.Source) + mirrorDirectives = append(mirrorDirectives, buildBlockDirective("location", + []string{"=", loc.Mirror.Source}, + ngx_crossplane.Directives{ + buildDirective("internal"), + buildDirective("proxy_set_header", "Host", loc.Mirror.Host), + buildDirective("proxy_pass", loc.Mirror.Target), + })) + } + return mirrorDirectives +} + +// buildCustomErrorLocationsPerServer is a utility function which will collect all +// custom error codes for all locations of a server block, deduplicates them, +// and returns a set which is unique by default-upstream and error code. It returns an array +// of errorLocations, each of which contain the upstream name and a list of +// error codes for that given upstream, so that sufficiently unique +// @custom error location blocks can be created in the template +func buildCustomErrorLocationsPerServer(server *ingress.Server, enableMetrics bool) ngx_crossplane.Directives { + type errorLocation struct { + UpstreamName string + Codes []int + } + + codesMap := make(map[string]map[int]bool) + for _, loc := range server.Locations { + backendUpstream := loc.DefaultBackendUpstreamName + + var dedupedCodes map[int]bool + if existingMap, ok := codesMap[backendUpstream]; ok { + dedupedCodes = existingMap + } else { + dedupedCodes = make(map[int]bool) + } + + for _, code := range loc.CustomHTTPErrors { + dedupedCodes[code] = true + } + codesMap[backendUpstream] = dedupedCodes + } + + errorLocations := []errorLocation{} + + for upstream, dedupedCodes := range codesMap { + codesForUpstream := []int{} + for code := range dedupedCodes { + codesForUpstream = append(codesForUpstream, code) + } + sort.Ints(codesForUpstream) + errorLocations = append(errorLocations, errorLocation{ + UpstreamName: upstream, + Codes: codesForUpstream, + }) + } + + sort.Slice(errorLocations, func(i, j int) bool { + return errorLocations[i].UpstreamName < errorLocations[j].UpstreamName + }) + + errorLocationsDirectives := make(ngx_crossplane.Directives, 0) + for i := range errorLocations { + errorLocationsDirectives = append(errorLocationsDirectives, buildCustomErrorLocation(errorLocations[i].UpstreamName, errorLocations[i].Codes, enableMetrics)...) + } + return errorLocationsDirectives + +} + +func buildCustomErrorLocation(upstreamName string, errorCodes []int, enableMetrics bool) ngx_crossplane.Directives { + directives := make(ngx_crossplane.Directives, len(errorCodes)) + for i := range errorCodes { + locationDirectives := ngx_crossplane.Directives{ + buildDirective("internal"), + buildDirective("proxy_intercept_errors", "off"), + buildDirective("proxy_set_header", "X-Code", errorCodes[i]), + buildDirective("proxy_set_header", "X-Format", "$http_accept"), + buildDirective("proxy_set_header", "X-Original-URI", "$request_uri"), + buildDirective("proxy_set_header", "X-Namespace", "$namespace"), + buildDirective("proxy_set_header", "X-Ingress-Name", "$ingress_name"), + buildDirective("proxy_set_header", "X-Service-Name", "$service_name"), + buildDirective("proxy_set_header", "X-Service-Port", "$service_port"), + buildDirective("proxy_set_header", "X-Request-ID", "$req_id"), + buildDirective("proxy_set_header", "X-Forwarded-For", "$remote_addr"), + buildDirective("proxy_set_header", "Host", "$best_http_host"), + buildDirective("set", "$proxy_upstream_name", upstreamName), + buildDirective("rewrite", "(.*)", "/", "break"), + buildDirective("proxy_pass", "http://upstream_balancer"), + } + + if enableMetrics { + locationDirectives = append(locationDirectives, buildDirective("log_by_lua_file", "/etc/nginx/lua/nginx/ngx_conf_log.lua")) + } + locationName := fmt.Sprintf("@custom_%s_%d", upstreamName, errorCodes[i]) + directives[i] = buildBlockDirective("location", []string{locationName}, locationDirectives) + } + + return directives +} + +type locationCfg struct { + pathLocation []string + authPath string + externalAuth *externalAuth + proxySetHeader string + applyGlobalAuth bool + applyAuthUpstream bool +} + +func (c *Template) buildServerLocations(server *ingress.Server, locations []*ingress.Location) ngx_crossplane.Directives { + serverLocations := make(ngx_crossplane.Directives, 0) + + cfg := c.tplConfig.Cfg + enforceRegexModifier := false + needsRewrite := func(loc *ingress.Location) bool { + return loc.Rewrite.Target != "" && + loc.Rewrite.Target != loc.Path + } + + for _, location := range locations { + if needsRewrite(location) || location.Rewrite.UseRegex { + enforceRegexModifier = true + break + } + } + + for _, location := range locations { + locationConfig := locationCfg{ + pathLocation: buildLocation(location, enforceRegexModifier), + proxySetHeader: getProxySetHeader(location), + authPath: buildAuthLocation(location, cfg.GlobalExternalAuth.URL), + applyGlobalAuth: shouldApplyGlobalAuth(location, cfg.GlobalExternalAuth.URL), + applyAuthUpstream: shouldApplyAuthUpstream(location, cfg), + externalAuth: &externalAuth{}, + } + + if location.Rewrite.AppRoot != "" { + serverLocations = append(serverLocations, + buildBlockDirective("if", []string{"$uri", "=", "/"}, + ngx_crossplane.Directives{ + buildDirective("return", "302", fmt.Sprintf("$scheme://$http_host%s", location.Rewrite.AppRoot)), + })) + } + + if locationConfig.applyGlobalAuth { + locationConfig.externalAuth = buildExternalAuth(cfg.GlobalExternalAuth) + } else { + locationConfig.externalAuth = buildExternalAuth(location.ExternalAuth) + } + if locationConfig.authPath != "" { + serverLocations = append(serverLocations, c.buildAuthLocation(server, location, locationConfig)) + } + if location.Denied == nil && locationConfig.externalAuth != nil && locationConfig.externalAuth.SigninURL != "" { + directives := ngx_crossplane.Directives{ + buildDirective("internal"), + buildDirective("add_header", "Set-Cookie", "$auth_cookie"), + } + if location.CorsConfig.CorsEnabled { + directives = append(directives, buildCorsDirectives(location.CorsConfig)...) + } + directives = append(directives, + buildDirective("return", + "302", + buildAuthSignURL(locationConfig.externalAuth.SigninURL, locationConfig.externalAuth.SigninURLRedirectParam))) + + serverLocations = append(serverLocations, buildBlockDirective("location", + []string{buildAuthSignURLLocation(location.Path, locationConfig.externalAuth.SigninURL)}, directives)) + + } + serverLocations = append(serverLocations, c.buildLocation(server, location, locationConfig)) + + } + + return serverLocations +} + +func (c *Template) buildLocation(server *ingress.Server, + location *ingress.Location, locationConfig locationCfg) *ngx_crossplane.Directive { + ing := getIngressInformation(location.Ingress, server.Hostname, location.IngressPath) + cfg := c.tplConfig + locationDirectives := ngx_crossplane.Directives{ + buildDirective("set", "$namespace", ing.Namespace), + buildDirective("set", "$ingress_name", ing.Rule), + buildDirective("set", "$service_name", ing.Service), + buildDirective("set", "$service_port", ing.ServicePort), + buildDirective("set", "$balancer_ewma_score", "-1"), + buildDirective("set", "$proxy_upstream_name", location.Backend), + buildDirective("set", "$proxy_host", "$proxy_upstream_name"), + buildDirective("set", "$pass_access_scheme", "$scheme"), + buildDirective("set", "$best_http_host", "$http_host"), + buildDirective("set", "$pass_port", "$pass_server_port"), + buildDirective("set", "$proxy_alternative_upstream_name", ""), + buildDirective("set", "$location_path", strings.ReplaceAll(ing.Path, `$`, `${literal_dollar}`)), + } + + locationDirectives = append(locationDirectives, locationConfigForLua(location, *c.tplConfig)...) + locationDirectives = append(locationDirectives, buildCertificateDirectives(location)...) + + if cfg.Cfg.UseProxyProtocol { + locationDirectives = append(locationDirectives, + buildDirective("set", "$pass_server_port", "$proxy_protocol_server_port")) + } else { + locationDirectives = append(locationDirectives, + buildDirective("set", "$pass_server_port", "$server_port")) + } + + locationDirectives = append(locationDirectives, + buildOpentelemetryForLocationDirectives(cfg.Cfg.EnableOpentelemetry, cfg.Cfg.OpentelemetryTrustIncomingSpan, location)...) + + locationDirectives = append(locationDirectives, + buildDirective("rewrite_by_lua_file", "/etc/nginx/lua/nginx/ngx_rewrite.lua"), + buildDirective("header_filter_by_lua_file", "/etc/nginx/lua/nginx/ngx_conf_srv_hdr_filter.lua"), + buildDirective("log_by_lua_file", "/etc/nginx/lua/nginx/ngx_conf_log_block.lua"), + buildDirective("rewrite_log", location.Logs.Rewrite), + // buildDirective("http2_push_preload", location.HTTP2PushPreload), // This directive is deprecated, keeping out of new crossplane + buildDirective("port_in_redirect", location.UsePortInRedirects)) + + if location.Mirror.Source != "" { + locationDirectives = append(locationDirectives, + buildDirective("mirror", location.Mirror.Source), + buildDirective("mirror_request_body", location.Mirror.RequestBody), + ) + } + + if !location.Logs.Access { + locationDirectives = append(locationDirectives, + buildDirective("access_log", "off"), + ) + } + if location.Denied != nil { + locationDirectives = append(locationDirectives, + buildDirectiveWithComment("return", fmt.Sprintf("Location denied. Reason: %s", *location.Denied), "503")) + } else { + locationDirectives = append(locationDirectives, c.buildAllowedLocation(server, location, locationConfig)...) + } + + return buildBlockDirective("location", locationConfig.pathLocation, locationDirectives) +} + +func (c *Template) buildAllowedLocation(server *ingress.Server, location *ingress.Location, locationConfig locationCfg) ngx_crossplane.Directives { + dir := make(ngx_crossplane.Directives, 0) + proxySetHeader := locationConfig.proxySetHeader + for _, ip := range location.Denylist.CIDR { + dir = append(dir, buildDirective("deny", ip)) + } + if len(location.Allowlist.CIDR) > 0 { + for _, ip := range location.Allowlist.CIDR { + dir = append(dir, buildDirective("allow", ip)) + } + dir = append(dir, buildDirective("deny", "all")) + } + + if location.CorsConfig.CorsEnabled { + dir = append(dir, buildCorsDirectives(location.CorsConfig)...) + } + // TODO: Implement the build Auth Location + if !isLocationInLocationList(location, c.tplConfig.Cfg.NoAuthLocations) { + dir = append(dir, buildAuthLocationConfig(location, locationConfig)...) + } + + dir = append(dir, buildRateLimit(location)...) + + if isValidByteSize(location.Proxy.BodySize, true) { + dir = append(dir, buildDirective("client_max_body_size", location.Proxy.BodySize)) + } + if isValidByteSize(location.ClientBodyBufferSize, false) { + dir = append(dir, buildDirective("client_body_buffer_size", location.ClientBodyBufferSize)) + } + + if location.UpstreamVhost != "" { + dir = append(dir, buildDirective(proxySetHeader, "Host", location.UpstreamVhost)) + } else { + dir = append(dir, buildDirective(proxySetHeader, "Host", "$best_http_host")) + } + + if server.CertificateAuth.CAFileName != "" { + dir = append(dir, + buildDirective(proxySetHeader, "ssl-client-verify", "$ssl_client_verify"), + buildDirective(proxySetHeader, "ssl-client-subject-dn", "$ssl_client_s_dn"), + buildDirective(proxySetHeader, "ssl-client-issuer-dn", "$ssl_client_i_dn"), + ) + + if server.CertificateAuth.PassCertToUpstream { + dir = append(dir, buildDirective(proxySetHeader, "ssl-client-cert", "$ssl_client_escaped_cert")) + } + } + + dir = append(dir, + buildDirective(proxySetHeader, "Upgrade", "$http_upgrade"), + buildDirective(proxySetHeader, "X-Request-ID", "$req_id"), + buildDirective(proxySetHeader, "X-Real-IP", "$remote_addr"), + buildDirective(proxySetHeader, "X-Forwarded-Host", "$best_http_host"), + buildDirective(proxySetHeader, "X-Forwarded-Port", "$pass_port"), + buildDirective(proxySetHeader, "X-Forwarded-Proto", "$pass_access_scheme"), + buildDirective(proxySetHeader, "X-Forwarded-Scheme", "$pass_access_scheme"), + buildDirective(proxySetHeader, "X-Real-IP", "$remote_addr"), + buildDirective(proxySetHeader, "X-Scheme", "$pass_access_scheme"), + buildDirective(proxySetHeader, "X-Original-Forwarded-For", + fmt.Sprintf("$http_%s", strings.ToLower(strings.ReplaceAll(c.tplConfig.Cfg.ForwardedForHeader, "-", "_")))), + buildDirectiveWithComment(proxySetHeader, + "mitigate HTTProxy Vulnerability - https://www.nginx.com/blog/mitigating-the-httpoxy-vulnerability-with-nginx/", "Proxy", ""), + buildDirective("proxy_connect_timeout", seconds(location.Proxy.ConnectTimeout)), + buildDirective("proxy_read_timeout", seconds(location.Proxy.ReadTimeout)), + buildDirective("proxy_send_timeout", seconds(location.Proxy.SendTimeout)), + buildDirective("proxy_buffering", location.Proxy.ProxyBuffering), + buildDirective("proxy_buffer_size", location.Proxy.BufferSize), + buildDirective("proxy_buffers", location.Proxy.BuffersNumber, location.Proxy.BufferSize), + buildDirective("proxy_request_buffering", location.Proxy.RequestBuffering), + buildDirective("proxy_http_version", location.Proxy.ProxyHTTPVersion), + buildDirective("proxy_cookie_domain", strings.Split(location.Proxy.CookieDomain, " ")), + buildDirective("proxy_cookie_path", strings.Split(location.Proxy.CookiePath, " ")), + buildDirective("proxy_next_upstream_timeout", location.Proxy.NextUpstreamTimeout), + buildDirective("proxy_next_upstream_tries", location.Proxy.NextUpstreamTries), + buildDirective("proxy_next_upstream", buildNextUpstream(location.Proxy.NextUpstream, c.tplConfig.Cfg.RetryNonIdempotent)), + ) + + if isValidByteSize(location.Proxy.ProxyMaxTempFileSize, true) { + dir = append(dir, buildDirective("proxy_max_temp_file_size", location.Proxy.ProxyMaxTempFileSize)) + } + + if c.tplConfig.Cfg.UseForwardedHeaders && c.tplConfig.Cfg.ComputeFullForwardedFor { + dir = append(dir, buildDirective(proxySetHeader, "X-Forwarded-For", "$full_x_forwarded_for")) + } else { + dir = append(dir, buildDirective(proxySetHeader, "X-Forwarded-For", "$remote_addr")) + } + + if c.tplConfig.Cfg.ProxyAddOriginalURIHeader { + dir = append(dir, buildDirective(proxySetHeader, "X-Original-URI", "$request_uri")) + } + + if location.Connection.Enabled { + dir = append(dir, buildDirective(proxySetHeader, "Connection", location.Connection.Header)) + } else { + dir = append(dir, buildDirective(proxySetHeader, "Connection", "$connection_upgrade")) + } + + for k, v := range c.tplConfig.ProxySetHeaders { + dir = append(dir, buildDirective(proxySetHeader, k, v)) + } + + for k, v := range location.CustomHeaders.Headers { + dir = append(dir, buildDirective("more_set_headers", fmt.Sprintf("%s: %s", k, strings.ReplaceAll(v, `$`, `${literal_dollar}`)))) + } + + if strings.HasPrefix(location.Backend, "custom-default-backend-") { + dir = append(dir, + buildDirective("proxy_set_header", "X-Code", "503"), + buildDirective("proxy_set_header", "X-Format", "$http_accept"), + buildDirective("proxy_set_header", "X-Namespace", "$namespace"), + buildDirective("proxy_set_header", "X-Ingress-Name", "$ingress_name"), + buildDirective("proxy_set_header", "X-Service-Name", "$service_name"), + buildDirective("proxy_set_header", "X-Service-Port", "$service_port"), + buildDirective("proxy_set_header", "X-Request-ID", "$req_id"), + ) + } + + if location.Satisfy != "" { + dir = append(dir, buildDirective("satisfy", location.Satisfy)) + } + + if len(location.CustomHTTPErrors) > 0 && !location.DisableProxyInterceptErrors { + dir = append(dir, buildDirective("proxy_intercept_errors", "on")) + } + + for _, errorcode := range location.CustomHTTPErrors { + dir = append(dir, buildDirective( + "error_page", + errorcode, "=", + fmt.Sprintf("@custom_%s_%d", location.DefaultBackendUpstreamName, errorcode)), + ) + } + + switch location.BackendProtocol { + case "GRPC", "GRPCS": + dir = append(dir, + buildDirective("grpc_connect_timeout", seconds(location.Proxy.ConnectTimeout)), + buildDirective("grpc_send_timeout", seconds(location.Proxy.SendTimeout)), + buildDirective("grpc_read_timeout", seconds(location.Proxy.ReadTimeout)), + ) + case "FCGI": + dir = append(dir, buildDirective("include", "/etc/nginx/fastcgi_params")) + if location.FastCGI.Index != "" { + dir = append(dir, buildDirective("fastcgi_index", location.FastCGI.Index)) + } + for k, v := range location.FastCGI.Params { + dir = append(dir, buildDirective("fastcgi_param", k, v)) + } + } + + if location.Redirect.URL != "" { + dir = append(dir, buildDirective("return", location.Redirect.Code, location.Redirect.URL)) + } + + dir = append(dir, buildProxyPass(c.tplConfig.Backends, location)...) + + if location.Proxy.ProxyRedirectFrom == "default" || location.Proxy.ProxyRedirectFrom == "off" { + dir = append(dir, buildDirective("proxy_redirect", location.Proxy.ProxyRedirectFrom)) + } else if location.Proxy.ProxyRedirectTo != "off" { + dir = append(dir, buildDirective("proxy_redirect", location.Proxy.ProxyRedirectFrom, location.Proxy.ProxyRedirectTo)) + } + + return dir +} + +func buildCertificateDirectives(location *ingress.Location) ngx_crossplane.Directives { + cert := make(ngx_crossplane.Directives, 0) + if location.ProxySSL.CAFileName != "" { + cert = append(cert, + buildDirectiveWithComment( + "proxy_ssl_trusted_certificate", + fmt.Sprintf("#PEM sha: %s", location.ProxySSL.CASHA), + location.ProxySSL.CAFileName, + ), + buildDirective("proxy_ssl_ciphers", location.ProxySSL.Ciphers), + buildDirective("proxy_ssl_protocols", strings.Split(location.ProxySSL.Protocols, " ")), + buildDirective("proxy_ssl_verify", location.ProxySSL.Verify), + buildDirective("proxy_ssl_verify_depth", location.ProxySSL.VerifyDepth), + ) + } + if location.ProxySSL.ProxySSLName != "" { + cert = append(cert, buildDirective("proxy_ssl_name", location.ProxySSL.ProxySSLName)) + } + if location.ProxySSL.ProxySSLServerName != "" { + cert = append(cert, buildDirective("proxy_ssl_server_name", location.ProxySSL.ProxySSLServerName)) + } + if location.ProxySSL.PemFileName != "" { + cert = append(cert, + buildDirective("proxy_ssl_certificate", location.ProxySSL.PemFileName), + buildDirective("proxy_ssl_certificate_key", location.ProxySSL.PemFileName), + ) + } + return cert +} + +type ingressInformation struct { + Namespace string + Path string + Rule string + Service string + ServicePort string + Annotations map[string]string +} + +func getIngressInformation(ing *ingress.Ingress, hostname, ingressPath string) *ingressInformation { + if ing == nil { + return &ingressInformation{} + } + + info := &ingressInformation{ + Namespace: ing.GetNamespace(), + Rule: ing.GetName(), + Annotations: ing.Annotations, + Path: ingressPath, + } + + if ingressPath == "" { + ingressPath = "/" + info.Path = "/" + } + + if ing.Spec.DefaultBackend != nil && ing.Spec.DefaultBackend.Service != nil { + info.Service = ing.Spec.DefaultBackend.Service.Name + if ing.Spec.DefaultBackend.Service.Port.Number > 0 { + info.ServicePort = strconv.Itoa(int(ing.Spec.DefaultBackend.Service.Port.Number)) + } else { + info.ServicePort = ing.Spec.DefaultBackend.Service.Port.Name + } + } + + for _, rule := range ing.Spec.Rules { + if rule.HTTP == nil { + continue + } + + if hostname != "_" && rule.Host == "" { + continue + } + + host := "_" + if rule.Host != "" { + host = rule.Host + } + + if hostname != host { + continue + } + + for _, rPath := range rule.HTTP.Paths { + if ingressPath != rPath.Path { + continue + } + + if rPath.Backend.Service == nil { + continue + } + + if info.Service != "" && rPath.Backend.Service.Name == "" { + // empty rule. Only contains a Path and PathType + return info + } + + info.Service = rPath.Backend.Service.Name + if rPath.Backend.Service.Port.Number > 0 { + info.ServicePort = strconv.Itoa(int(rPath.Backend.Service.Port.Number)) + } else { + info.ServicePort = rPath.Backend.Service.Port.Name + } + + return info + } + } + + return info +} + +func buildOpentelemetryForLocationDirectives(isOTEnabled, isOTTrustSet bool, location *ingress.Location) ngx_crossplane.Directives { + isOTEnabledInLoc := location.Opentelemetry.Enabled + isOTSetInLoc := location.Opentelemetry.Set + directives := make(ngx_crossplane.Directives, 0) + if isOTEnabled { + if isOTSetInLoc && !isOTEnabledInLoc { + return ngx_crossplane.Directives{ + buildDirective("opentelemetry", "off"), + } + } + } else if !isOTSetInLoc || !isOTEnabledInLoc { + return directives + } + + if location != nil { + directives = append(directives, + buildDirective("opentelemetry", "on"), + buildDirective("opentelemetry_propagate"), + ) + if location.Opentelemetry.OperationName != "" { + directives = append(directives, + buildDirective("opentelemetry_operation_name", location.Opentelemetry.OperationName)) + } + + if (!isOTTrustSet && !location.Opentelemetry.TrustSet) || + (location.Opentelemetry.TrustSet && !location.Opentelemetry.TrustEnabled) { + directives = append(directives, + buildDirective("opentelemetry_trust_incoming_spans", "off"), + ) + } else { + directives = append(directives, + buildDirective("opentelemetry_trust_incoming_spans", "on"), + ) + } + } + + return directives +} + +// buildRateLimit produces an array of limit_req to be used inside the Path of +// Ingress rules. The order: connections by IP first, then RPS, and RPM last. +func buildRateLimit(loc *ingress.Location) ngx_crossplane.Directives { + limits := make(ngx_crossplane.Directives, 0) + + if loc.RateLimit.Connections.Limit > 0 { + limits = append(limits, buildDirective("limit_conn", loc.RateLimit.Connections.Name, loc.RateLimit.Connections.Limit)) + } + + if loc.RateLimit.RPS.Limit > 0 { + limits = append(limits, + buildDirective( + "limit_req", + fmt.Sprintf("zone=%s", loc.RateLimit.RPS.Name), + fmt.Sprintf("burst=%d", loc.RateLimit.RPS.Burst), + "nodelay", + ), + ) + } + + if loc.RateLimit.RPM.Limit > 0 { + limits = append(limits, + buildDirective( + "limit_req", + fmt.Sprintf("zone=%s", loc.RateLimit.RPM.Name), + fmt.Sprintf("burst=%d", loc.RateLimit.RPM.Burst), + "nodelay", + ), + ) + } + + if loc.RateLimit.LimitRateAfter > 0 { + limits = append(limits, + buildDirective( + "limit_rate_after", + fmt.Sprintf("%dk", loc.RateLimit.LimitRateAfter), + ), + ) + } + + if loc.RateLimit.LimitRate > 0 { + limits = append(limits, + buildDirective( + "limit_rate", + fmt.Sprintf("%dk", loc.RateLimit.LimitRate), + ), + ) + } + + return limits +} + +// locationConfigForLua formats some location specific configuration into Lua table represented as string +func locationConfigForLua(location *ingress.Location, all config.TemplateConfig) ngx_crossplane.Directives { + /* Lua expects the following vars + force_ssl_redirect = string_to_bool(ngx.var.force_ssl_redirect), + ssl_redirect = string_to_bool(ngx.var.ssl_redirect), + force_no_ssl_redirect = string_to_bool(ngx.var.force_no_ssl_redirect), + preserve_trailing_slash = string_to_bool(ngx.var.preserve_trailing_slash), + use_port_in_redirects = string_to_bool(ngx.var.use_port_in_redirects), + */ + + return ngx_crossplane.Directives{ + buildDirective("set", "$force_ssl_redirect", strconv.FormatBool(location.Rewrite.ForceSSLRedirect)), + buildDirective("set", "$ssl_redirect", strconv.FormatBool(location.Rewrite.SSLRedirect)), + buildDirective("set", "$force_no_ssl_redirect", strconv.FormatBool(isLocationInLocationList(location, all.Cfg.NoTLSRedirectLocations))), + buildDirective("set", "$preserve_trailing_slash", strconv.FormatBool(location.Rewrite.PreserveTrailingSlash)), + buildDirective("set", "$use_port_in_redirects", strconv.FormatBool(location.UsePortInRedirects)), + } +} + +func isLocationInLocationList(loc *ingress.Location, rawLocationList string) bool { + locationList := strings.Split(rawLocationList, ",") + + for _, locationListItem := range locationList { + locationListItem = strings.Trim(locationListItem, " ") + if locationListItem == "" { + continue + } + if strings.HasPrefix(loc.Path, locationListItem) { + return true + } + } + + return false +} + +func buildAuthLocationConfig(location *ingress.Location, locationConfig locationCfg) ngx_crossplane.Directives { + directives := make(ngx_crossplane.Directives, 0) + if locationConfig.authPath != "" { + if locationConfig.applyAuthUpstream && !locationConfig.applyGlobalAuth { + directives = append(directives, buildDirective("set", "$auth_cookie", "")) + directives = append(directives, buildDirective("add_header", "Set-Cookie", "$auth_cookie")) + directives = append(directives, buildAuthResponseHeaders(locationConfig.proxySetHeader, locationConfig.externalAuth.ResponseHeaders, true)...) + if len(locationConfig.externalAuth.ResponseHeaders) > 0 { + directives = append(directives, buildDirective("set", "$auth_response_headers", strings.Join(locationConfig.externalAuth.ResponseHeaders, ","))) + } + directives = append(directives, + buildDirective("set", "$auth_path", locationConfig.authPath), + buildDirective("set", "$auth_keepalive_share_vars", strconv.FormatBool(locationConfig.externalAuth.KeepaliveShareVars)), + buildDirective("access_by_lua_file", "/etc/nginx/lua/nginx/ngx_conf_external_auth.lua"), + ) + } else { + directives = append(directives, + buildDirective("auth_request", locationConfig.authPath), + buildDirective("auth_request_set", "$auth_cookie", "$upstream_http_set_cookie"), + ) + cookieDirective := buildDirective("add_header", "Set-Cookie", "$auth_cookie") + if locationConfig.externalAuth.AlwaysSetCookie { + cookieDirective.Args = append(cookieDirective.Args, "always") + } + directives = append(directives, cookieDirective) + directives = append(directives, buildAuthResponseHeaders(locationConfig.proxySetHeader, locationConfig.externalAuth.ResponseHeaders, false)...) + } + } + + if locationConfig.externalAuth.SigninURL != "" { + directives = append(directives, + buildDirective("set_escape_uri", "$escaped_request_uri", "$request_uri"), + buildDirective("error_page", "401", "=", buildAuthSignURLLocation(location.Path, locationConfig.externalAuth.SigninURL)), + ) + } + if location.BasicDigestAuth.Secured { + var authDirective, authFileDirective string + if location.BasicDigestAuth.Type == "basic" { + authDirective, authFileDirective = "auth_basic", "auth_basic_user_file" + } else { + authDirective, authFileDirective = "auth_digest", "auth_digest_user_file" + } + + directives = append(directives, + buildDirective(authDirective, location.BasicDigestAuth.Realm), + buildDirective(authFileDirective, location.BasicDigestAuth.File), + buildDirective(locationConfig.proxySetHeader, "Authorization", ""), + ) + } + + return directives + /* + Missing this Lua script + # `auth_request` module does not support HTTP keepalives in upstream block: + # https://trac.nginx.org/nginx/ticket/1579 + access_by_lua_block { + local res = ngx.location.capture('{{ $authPath }}', { method = ngx.HTTP_GET, body = '', share_all_vars = {{ $externalAuth.KeepaliveShareVars }} }) + if res.status == ngx.HTTP_OK then + ngx.var.auth_cookie = res.header['Set-Cookie'] + {{- range $line := buildAuthUpstreamLuaHeaders $externalAuth.ResponseHeaders }} # IF 4 + {{ $line }} + {{- end }} # END IF 4 + return + end + if res.status == ngx.HTTP_UNAUTHORIZED or res.status == ngx.HTTP_FORBIDDEN then + ngx.exit(res.status) + end + ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR) + } + + */ +} diff --git a/internal/ingress/controller/template/crossplane/server.go b/internal/ingress/controller/template/crossplane/server.go new file mode 100644 index 000000000..541e156f0 --- /dev/null +++ b/internal/ingress/controller/template/crossplane/server.go @@ -0,0 +1,268 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package crossplane + +import ( + "fmt" + "strings" + + ngx_crossplane "github.com/nginxinc/nginx-go-crossplane" + + "k8s.io/ingress-nginx/pkg/apis/ingress" + utilingress "k8s.io/ingress-nginx/pkg/util/ingress" + "k8s.io/utils/ptr" +) + +func (c *Template) buildServerDirective(server *ingress.Server) *ngx_crossplane.Directive { + cfg := c.tplConfig.Cfg + serverName := buildServerName(server.Hostname) + serverBlock := ngx_crossplane.Directives{ + buildDirective("server_name", serverName, server.Aliases), + buildDirective("http2", cfg.UseHTTP2), + buildDirective("set", "$proxy_upstream_name", "-"), + buildDirective("ssl_certificate_by_lua_file", "/etc/nginx/lua/nginx/ngx_conf_certificate.lua"), + } + + serverBlock = append(serverBlock, buildListener(*c.tplConfig, server.Hostname)...) + serverBlock = append(serverBlock, c.buildBlockers()...) + + if server.Hostname == "_" { + serverBlock = append(serverBlock, buildDirective("ssl_reject_handshake", cfg.SSLRejectHandshake)) + } + + if server.CertificateAuth.MatchCN != "" { + matchCNBlock := buildBlockDirective("if", + []string{"$ssl_client_s_dn", "!~", server.CertificateAuth.MatchCN}, + ngx_crossplane.Directives{ + buildDirective("return", "403", "client certificate unauthorized"), + }) + serverBlock = append(serverBlock, matchCNBlock) + } + + if server.AuthTLSError != "" { + serverBlock = append(serverBlock, buildDirective("return", 403)) + } else { + + serverBlock = append(serverBlock, c.buildCertificateDirectives(server)...) + serverBlock = append(serverBlock, buildCustomErrorLocationsPerServer(server, c.tplConfig.EnableMetrics)...) + serverBlock = append(serverBlock, buildMirrorLocationDirective(server.Locations)...) + + // The other locations should come here! + serverBlock = append(serverBlock, c.buildServerLocations(server, server.Locations)...) + + } + + // "/healthz" location + if server.Hostname == "_" { + dirs := ngx_crossplane.Directives{ + buildDirective("access_log", "off"), + buildDirective("return", "200"), + } + if cfg.EnableOpentelemetry { + dirs = append(dirs, buildDirective("opentelemetry", "off")) + } + healthLocation := buildBlockDirective("location", + []string{c.tplConfig.HealthzURI}, dirs) + serverBlock = append(serverBlock, healthLocation) + + // "/nginx_status" location + statusLocationDirs := ngx_crossplane.Directives{} + if cfg.EnableOpentelemetry { + statusLocationDirs = append(statusLocationDirs, buildDirective("opentelemetry", "off")) + } + + for _, v := range c.tplConfig.NginxStatusIpv4Whitelist { + statusLocationDirs = append(statusLocationDirs, buildDirective("allow", v)) + } + + if c.tplConfig.IsIPV6Enabled { + for _, v := range c.tplConfig.NginxStatusIpv6Whitelist { + statusLocationDirs = append(statusLocationDirs, buildDirective("allow", v)) + } + } + statusLocationDirs = append(statusLocationDirs, + buildDirective("deny", "all"), + buildDirective("access_log", "off"), + buildDirective("stub_status", "on")) + + // End of "nginx_status" location + + serverBlock = append(serverBlock, buildBlockDirective("location", []string{"/nginx_status"}, statusLocationDirs)) + + } + + // DO NOT MOVE! THIS IS THE END DIRECTIVE OF SERVERS + serverBlock = append(serverBlock, buildCustomErrorLocation("upstream-default-backend", cfg.CustomHTTPErrors, c.tplConfig.EnableMetrics)...) + + return &ngx_crossplane.Directive{ + Directive: "server", + Block: serverBlock, + } +} + +func (c *Template) buildCertificateDirectives(server *ingress.Server) ngx_crossplane.Directives { + certDirectives := make(ngx_crossplane.Directives, 0) + + if server.CertificateAuth.CAFileName != "" { + certAuth := server.CertificateAuth + certDirectives = append(certDirectives, buildDirective("ssl_client_certificate", certAuth.CAFileName)) + certDirectives = append(certDirectives, buildDirective("ssl_verify_client", certAuth.VerifyClient)) + certDirectives = append(certDirectives, buildDirective("ssl_verify_depth", certAuth.ValidationDepth)) + if certAuth.CRLFileName != "" { + certDirectives = append(certDirectives, buildDirective("ssl_crl", certAuth.CRLFileName)) + } + if certAuth.ErrorPage != "" { + certDirectives = append(certDirectives, buildDirective("error_page", "495", "496", "=", certAuth.ErrorPage)) + } + } + + prxSSL := server.ProxySSL + if prxSSL.CAFileName != "" { + certDirectives = append(certDirectives, buildDirective("proxy_ssl_trusted_certificate", prxSSL.CAFileName)) + certDirectives = append(certDirectives, buildDirective("proxy_ssl_ciphers", prxSSL.Ciphers)) + certDirectives = append(certDirectives, buildDirective("proxy_ssl_protocols", strings.Split(prxSSL.Protocols, " "))) + certDirectives = append(certDirectives, buildDirective("proxy_ssl_verify", prxSSL.Verify)) + certDirectives = append(certDirectives, buildDirective("proxy_ssl_verify_depth", prxSSL.VerifyDepth)) + if prxSSL.ProxySSLName != "" { + certDirectives = append(certDirectives, buildDirective("proxy_ssl_name", prxSSL.ProxySSLName)) + certDirectives = append(certDirectives, buildDirective("proxy_ssl_server_name", prxSSL.ProxySSLServerName)) + } + } + if prxSSL.PemFileName != "" { + certDirectives = append(certDirectives, buildDirective("proxy_ssl_certificate", prxSSL.PemFileName)) + certDirectives = append(certDirectives, buildDirective("proxy_ssl_certificate_key", prxSSL.PemFileName)) + } + if server.SSLCiphers != "" { + certDirectives = append(certDirectives, buildDirective("ssl_ciphers", server.SSLCiphers)) + } + + if server.SSLPreferServerCiphers != "" { + certDirectives = append(certDirectives, buildDirective("ssl_prefer_server_ciphers", server.SSLPreferServerCiphers)) + } + + return certDirectives +} + +// buildRedirectServer builds the server blocks for redirections +func (c *Template) buildRedirectServer(server *utilingress.Redirect) *ngx_crossplane.Directive { + serverBlock := ngx_crossplane.Directives{ + buildDirective("server_name", server.From), + buildDirective("ssl_certificate_by_lua_file", "/etc/nginx/lua/nginx/ngx_conf_certificate.lua"), + buildDirective("set_by_lua_file", "$redirect_to", "/etc/nginx/lua/nginx/ngx_srv_redirect.lua", server.To), + } + serverBlock = append(serverBlock, buildListener(*c.tplConfig, server.From)...) + serverBlock = append(serverBlock, c.buildBlockers()...) + serverBlock = append(serverBlock, buildDirective("return", c.tplConfig.Cfg.HTTPRedirectCode, "$redirect_to")) + + return &ngx_crossplane.Directive{ + Directive: "server", + Block: serverBlock, + } +} + +// buildDefaultBackend builds the default catch all server +func (c *Template) buildDefaultBackend() *ngx_crossplane.Directive { + var reusePort *string + if c.tplConfig.Cfg.ReusePort { + reusePort = ptr.To("reuseport") + } + serverBlock := ngx_crossplane.Directives{ + buildDirective("listen", c.tplConfig.ListenPorts.Default, "default_server", reusePort, fmt.Sprintf("backlog=%d", c.tplConfig.BacklogSize)), + } + if c.tplConfig.IsIPV6Enabled { + serverBlock = append(serverBlock, buildDirective( + "listen", + fmt.Sprintf("[::]:%d", c.tplConfig.ListenPorts.Default), + "default_server", reusePort, + fmt.Sprintf("backlog=%d", c.tplConfig.BacklogSize), + )) + } + serverBlock = append(serverBlock, buildDirective("set", "$proxy_upstream_name", "internal")) + serverBlock = append(serverBlock, buildDirective("access_log", "off")) + serverBlock = append(serverBlock, buildBlockDirective("location", []string{"/"}, ngx_crossplane.Directives{ + buildDirective("return", "404"), + })) + + return &ngx_crossplane.Directive{ + Directive: "server", + Block: serverBlock, + } +} + +func (c *Template) buildHealthAndStatsServer() *ngx_crossplane.Directive { + serverBlock := ngx_crossplane.Directives{ + buildDirective("listen", fmt.Sprintf("127.0.0.1:%d", c.tplConfig.StatusPort)), + buildDirective("set", "$proxy_upstream_name", "internal"), + buildDirective("keepalive_timeout", "0"), + buildDirective("gzip", "off"), + buildDirective("access_log", "off"), + buildBlockDirective( + "location", + []string{c.tplConfig.HealthzURI}, ngx_crossplane.Directives{ + buildDirective("return", "200"), + }), + buildBlockDirective( + "location", + []string{"/is-dynamic-lb-initialized"}, ngx_crossplane.Directives{ + buildDirective("content_by_lua_file", "/etc/nginx/lua/nginx/ngx_conf_is_dynamic_lb_initialized.lua"), + }), + buildBlockDirective( + "location", + []string{c.tplConfig.StatusPath}, ngx_crossplane.Directives{ + buildDirective("stub_status", "on"), + }), + buildBlockDirective( + "location", + []string{"/configuration"}, ngx_crossplane.Directives{ + buildDirective("client_max_body_size", luaConfigurationRequestBodySize(c.tplConfig.Cfg)), + buildDirective("client_body_buffer_size", luaConfigurationRequestBodySize(c.tplConfig.Cfg)), + buildDirective("proxy_buffering", "off"), + buildDirective("content_by_lua_file", "/etc/nginx/lua/nginx/ngx_conf_configuration.lua"), + }), + buildBlockDirective( + "location", + []string{"/"}, ngx_crossplane.Directives{ + buildDirective("return", "404"), + }), + } + if c.tplConfig.Cfg.EnableOpentelemetry { + serverBlock = append(serverBlock, buildDirective("opentelemetry", "off")) + } + + return &ngx_crossplane.Directive{ + Directive: "server", + Block: serverBlock, + } +} + +func (c *Template) buildBlockers() ngx_crossplane.Directives { + blockers := make(ngx_crossplane.Directives, 0) + if len(c.tplConfig.Cfg.BlockUserAgents) > 0 { + uaDirectives := buildBlockDirective("if", []string{"$block_ua"}, ngx_crossplane.Directives{ + buildDirective("return", "403"), + }) + blockers = append(blockers, uaDirectives) + } + + if len(c.tplConfig.Cfg.BlockReferers) > 0 { + refDirectives := buildBlockDirective("if", []string{"$block_ref"}, ngx_crossplane.Directives{ + buildDirective("return", "403"), + }) + blockers = append(blockers, refDirectives) + } + return blockers +} diff --git a/internal/ingress/controller/template/crossplane/testdata/nginx-new.tmpl b/internal/ingress/controller/template/crossplane/testdata/nginx-new.tmpl index 64e5a0c3f..547fd12da 100644 --- a/internal/ingress/controller/template/crossplane/testdata/nginx-new.tmpl +++ b/internal/ingress/controller/template/crossplane/testdata/nginx-new.tmpl @@ -86,113 +86,113 @@ http { {{ range $index, $file := $all.MaxmindEditionFiles }} {{ if eq $file "GeoLite2-Country.mmdb" }} - geoip2 /etc/ingress-controller/geoip/GeoLite2-Country.mmdb { + geoip2 /etc/ingress-controller/geoip/GeoLite2-Country.mmdb { # OK {{ if (gt $cfg.GeoIP2AutoReloadMinutes 0) }} - auto_reload {{ $cfg.GeoIP2AutoReloadMinutes }}m; + auto_reload {{ $cfg.GeoIP2AutoReloadMinutes }}m; # OK {{ end }} - $geoip2_country_code source=$remote_addr country iso_code; - $geoip2_country_name source=$remote_addr country names en; - $geoip2_country_geoname_id source=$remote_addr country geoname_id; - $geoip2_continent_code source=$remote_addr continent code; - $geoip2_continent_name source=$remote_addr continent names en; - $geoip2_continent_geoname_id source=$remote_addr continent geoname_id; + $geoip2_country_code source=$remote_addr country iso_code; # OK + $geoip2_country_name source=$remote_addr country names en; # OK + $geoip2_country_geoname_id source=$remote_addr country geoname_id; # OK + $geoip2_continent_code source=$remote_addr continent code; # OK + $geoip2_continent_name source=$remote_addr continent names en; # OK + $geoip2_continent_geoname_id source=$remote_addr continent geoname_id; # OK } {{ end }} {{ if eq $file "GeoIP2-Country.mmdb" }} - geoip2 /etc/ingress-controller/geoip/GeoIP2-Country.mmdb { + geoip2 /etc/ingress-controller/geoip/GeoIP2-Country.mmdb { # OK {{ if (gt $cfg.GeoIP2AutoReloadMinutes 0) }} - auto_reload {{ $cfg.GeoIP2AutoReloadMinutes }}m; + auto_reload {{ $cfg.GeoIP2AutoReloadMinutes }}m; # OK {{ end }} - $geoip2_country_code source=$remote_addr country iso_code; - $geoip2_country_name source=$remote_addr country names en; - $geoip2_country_geoname_id source=$remote_addr country geoname_id; - $geoip2_continent_code source=$remote_addr continent code; - $geoip2_continent_name source=$remote_addr continent names en; - $geoip2_continent_geoname_id source=$remote_addr continent geoname_id; + $geoip2_country_code source=$remote_addr country iso_code; # OK + $geoip2_country_name source=$remote_addr country names en; # OK + $geoip2_country_geoname_id source=$remote_addr country geoname_id; # OK + $geoip2_continent_code source=$remote_addr continent code; # OK + $geoip2_continent_name source=$remote_addr continent names en; # OK + $geoip2_continent_geoname_id source=$remote_addr continent geoname_id; # OK } {{ end }} {{ if eq $file "GeoLite2-City.mmdb" }} - geoip2 /etc/ingress-controller/geoip/GeoLite2-City.mmdb { + geoip2 /etc/ingress-controller/geoip/GeoLite2-City.mmdb { # OK {{ if (gt $cfg.GeoIP2AutoReloadMinutes 0) }} - auto_reload {{ $cfg.GeoIP2AutoReloadMinutes }}m; + auto_reload {{ $cfg.GeoIP2AutoReloadMinutes }}m; # OK {{ end }} - $geoip2_city_country_code source=$remote_addr country iso_code; - $geoip2_city_country_name source=$remote_addr country names en; - $geoip2_city_country_geoname_id source=$remote_addr country geoname_id; - $geoip2_city source=$remote_addr city names en; - $geoip2_city_geoname_id source=$remote_addr city geoname_id; - $geoip2_postal_code source=$remote_addr postal code; - $geoip2_dma_code source=$remote_addr location metro_code; - $geoip2_latitude source=$remote_addr location latitude; - $geoip2_longitude source=$remote_addr location longitude; - $geoip2_time_zone source=$remote_addr location time_zone; - $geoip2_region_code source=$remote_addr subdivisions 0 iso_code; - $geoip2_region_name source=$remote_addr subdivisions 0 names en; - $geoip2_region_geoname_id source=$remote_addr subdivisions 0 geoname_id; - $geoip2_subregion_code source=$remote_addr subdivisions 1 iso_code; - $geoip2_subregion_name source=$remote_addr subdivisions 1 names en; - $geoip2_subregion_geoname_id source=$remote_addr subdivisions 1 geoname_id; - $geoip2_city_continent_code source=$remote_addr continent code; - $geoip2_city_continent_name source=$remote_addr continent names en; + $geoip2_city_country_code source=$remote_addr country iso_code; # OK + $geoip2_city_country_name source=$remote_addr country names en; # OK + $geoip2_city_country_geoname_id source=$remote_addr country geoname_id; # OK + $geoip2_city source=$remote_addr city names en; # OK + $geoip2_city_geoname_id source=$remote_addr city geoname_id; # OK + $geoip2_postal_code source=$remote_addr postal code; # OK + $geoip2_dma_code source=$remote_addr location metro_code; # OK + $geoip2_latitude source=$remote_addr location latitude; # OK + $geoip2_longitude source=$remote_addr location longitude; # OK + $geoip2_time_zone source=$remote_addr location time_zone; # OK + $geoip2_region_code source=$remote_addr subdivisions 0 iso_code; # OK + $geoip2_region_name source=$remote_addr subdivisions 0 names en; # OK + $geoip2_region_geoname_id source=$remote_addr subdivisions 0 geoname_id; # OK + $geoip2_subregion_code source=$remote_addr subdivisions 1 iso_code; # OK + $geoip2_subregion_name source=$remote_addr subdivisions 1 names en; # OK + $geoip2_subregion_geoname_id source=$remote_addr subdivisions 1 geoname_id; # OK + $geoip2_city_continent_code source=$remote_addr continent code; # OK + $geoip2_city_continent_name source=$remote_addr continent names en; # OK } {{ end }} {{ if eq $file "GeoIP2-City.mmdb" }} - geoip2 /etc/ingress-controller/geoip/GeoIP2-City.mmdb { + geoip2 /etc/ingress-controller/geoip/GeoIP2-City.mmdb { # OK {{ if (gt $cfg.GeoIP2AutoReloadMinutes 0) }} - auto_reload {{ $cfg.GeoIP2AutoReloadMinutes }}m; + auto_reload {{ $cfg.GeoIP2AutoReloadMinutes }}m; # OK {{ end }} - $geoip2_city_country_code source=$remote_addr country iso_code; - $geoip2_city_country_name source=$remote_addr country names en; - $geoip2_city_country_geoname_id source=$remote_addr country geoname_id; - $geoip2_city source=$remote_addr city names en; - $geoip2_city_geoname_id source=$remote_addr city geoname_id; - $geoip2_postal_code source=$remote_addr postal code; - $geoip2_dma_code source=$remote_addr location metro_code; - $geoip2_latitude source=$remote_addr location latitude; - $geoip2_longitude source=$remote_addr location longitude; - $geoip2_time_zone source=$remote_addr location time_zone; - $geoip2_region_code source=$remote_addr subdivisions 0 iso_code; - $geoip2_region_name source=$remote_addr subdivisions 0 names en; - $geoip2_region_geoname_id source=$remote_addr subdivisions 0 geoname_id; - $geoip2_subregion_code source=$remote_addr subdivisions 1 iso_code; - $geoip2_subregion_name source=$remote_addr subdivisions 1 names en; - $geoip2_subregion_geoname_id source=$remote_addr subdivisions 1 geoname_id; - $geoip2_city_continent_code source=$remote_addr continent code; - $geoip2_city_continent_name source=$remote_addr continent names en; + $geoip2_city_country_code source=$remote_addr country iso_code; # OK + $geoip2_city_country_name source=$remote_addr country names en; # OK + $geoip2_city_country_geoname_id source=$remote_addr country geoname_id; # OK + $geoip2_city source=$remote_addr city names en; # OK + $geoip2_city_geoname_id source=$remote_addr city geoname_id; # OK + $geoip2_postal_code source=$remote_addr postal code; # OK + $geoip2_dma_code source=$remote_addr location metro_code; # OK + $geoip2_latitude source=$remote_addr location latitude; # OK + $geoip2_longitude source=$remote_addr location longitude; # OK + $geoip2_time_zone source=$remote_addr location time_zone; # OK + $geoip2_region_code source=$remote_addr subdivisions 0 iso_code; # OK + $geoip2_region_name source=$remote_addr subdivisions 0 names en; # OK + $geoip2_region_geoname_id source=$remote_addr subdivisions 0 geoname_id; # OK + $geoip2_subregion_code source=$remote_addr subdivisions 1 iso_code; # OK + $geoip2_subregion_name source=$remote_addr subdivisions 1 names en; # OK + $geoip2_subregion_geoname_id source=$remote_addr subdivisions 1 geoname_id; # OK + $geoip2_city_continent_code source=$remote_addr continent code; # OK + $geoip2_city_continent_name source=$remote_addr continent names en; # OK } {{ end }} {{ if eq $file "GeoLite2-ASN.mmdb" }} - geoip2 /etc/ingress-controller/geoip/GeoLite2-ASN.mmdb { + geoip2 /etc/ingress-controller/geoip/GeoLite2-ASN.mmdb { # OK {{ if (gt $cfg.GeoIP2AutoReloadMinutes 0) }} - auto_reload {{ $cfg.GeoIP2AutoReloadMinutes }}m; + auto_reload {{ $cfg.GeoIP2AutoReloadMinutes }}m; # OK {{ end }} - $geoip2_asn source=$remote_addr autonomous_system_number; - $geoip2_org source=$remote_addr autonomous_system_organization; + $geoip2_asn source=$remote_addr autonomous_system_number; # OK + $geoip2_org source=$remote_addr autonomous_system_organization; # OK } {{ end }} {{ if eq $file "GeoIP2-ASN.mmdb" }} - geoip2 /etc/ingress-controller/geoip/GeoIP2-ASN.mmdb { + geoip2 /etc/ingress-controller/geoip/GeoIP2-ASN.mmdb { # OK {{ if (gt $cfg.GeoIP2AutoReloadMinutes 0) }} - auto_reload {{ $cfg.GeoIP2AutoReloadMinutes }}m; + auto_reload {{ $cfg.GeoIP2AutoReloadMinutes }}m; # OK {{ end }} - $geoip2_asn source=$remote_addr autonomous_system_number; - $geoip2_org source=$remote_addr autonomous_system_organization; + $geoip2_asn source=$remote_addr autonomous_system_number; # OK + $geoip2_org source=$remote_addr autonomous_system_organization; # OK } {{ end }} {{ if eq $file "GeoIP2-ISP.mmdb" }} - geoip2 /etc/ingress-controller/geoip/GeoIP2-ISP.mmdb { + geoip2 /etc/ingress-controller/geoip/GeoIP2-ISP.mmdb { # OK {{ if (gt $cfg.GeoIP2AutoReloadMinutes 0) }} - auto_reload {{ $cfg.GeoIP2AutoReloadMinutes }}m; + auto_reload {{ $cfg.GeoIP2AutoReloadMinutes }}m; # OK {{ end }} - $geoip2_isp source=$remote_addr isp; - $geoip2_isp_org source=$remote_addr organization; - $geoip2_asn source=$remote_addr default=0 autonomous_system_number; + $geoip2_isp source=$remote_addr isp; # OK + $geoip2_isp_org source=$remote_addr organization; # OK + $geoip2_asn source=$remote_addr default=0 autonomous_system_number; # OK } {{ end }} @@ -203,16 +203,16 @@ http { {{ end }} {{ if eq $file "GeoIP2-Anonymous-IP.mmdb" }} - geoip2 /etc/ingress-controller/geoip/GeoIP2-Anonymous-IP.mmdb { + geoip2 /etc/ingress-controller/geoip/GeoIP2-Anonymous-IP.mmdb { # OK {{ if (gt $cfg.GeoIP2AutoReloadMinutes 0) }} - auto_reload {{ $cfg.GeoIP2AutoReloadMinutes }}m; + auto_reload {{ $cfg.GeoIP2AutoReloadMinutes }}m; # OK {{ end }} - $geoip2_is_anon source=$remote_addr is_anonymous; - $geoip2_is_anonymous source=$remote_addr default=0 is_anonymous; - $geoip2_is_anonymous_vpn source=$remote_addr default=0 is_anonymous_vpn; - $geoip2_is_hosting_provider source=$remote_addr default=0 is_hosting_provider; - $geoip2_is_public_proxy source=$remote_addr default=0 is_public_proxy; - $geoip2_is_tor_exit_node source=$remote_addr default=0 is_tor_exit_node; + $geoip2_is_anon source=$remote_addr is_anonymous; # OK + $geoip2_is_anonymous source=$remote_addr default=0 is_anonymous; # OK + $geoip2_is_anonymous_vpn source=$remote_addr default=0 is_anonymous_vpn; # OK + $geoip2_is_hosting_provider source=$remote_addr default=0 is_hosting_provider; # OK + $geoip2_is_public_proxy source=$remote_addr default=0 is_public_proxy; # OK + $geoip2_is_tor_exit_node source=$remote_addr default=0 is_tor_exit_node; # OK } {{ end }} @@ -252,7 +252,7 @@ http { {{ if and (ne $cfg.HTTP2MaxHeaderSize "") (ne $cfg.HTTP2MaxFieldSize "") }} # OK http2_max_field_size {{ $cfg.HTTP2MaxFieldSize }}; # OK - http2_max_header_size {{ $cfg.HTTP2MaxHeaderSize }}; # OK + http2_max_header_size {{ $cfg.HTTP2MaxHeaderSize }}; # OK {{ end }} {{ if (gt $cfg.HTTP2MaxRequests 0) }} # OK @@ -519,28 +519,28 @@ http { {{/* Build server redirects (from/to www) */}} {{ range $redirect := .RedirectServers }} ## start server {{ $redirect.From }} - server { - server_name {{ $redirect.From }}; + server { #OK + server_name {{ $redirect.From }}; # OK - {{ buildHTTPListener $all $redirect.From }} - {{ buildHTTPSListener $all $redirect.From }} + {{ buildHTTPListener $all $redirect.From }} # OK + {{ buildHTTPSListener $all $redirect.From }} # OK - ssl_certificate_by_lua_file /etc/nginx/lua/nginx/ngx_conf_certificate.lua; + ssl_certificate_by_lua_file /etc/nginx/lua/nginx/ngx_conf_certificate.lua; # OK {{ if gt (len $cfg.BlockUserAgents) 0 }} - if ($block_ua) { - return 403; + if ($block_ua) { # OK + return 403; # OK } {{ end }} {{ if gt (len $cfg.BlockReferers) 0 }} - if ($block_ref) { - return 403; + if ($block_ref) { # OK + return 403; #OK } {{ end }} - set_by_lua_file $redirect_to /etc/nginx/lua/nginx/ngx_srv_redirect.lua {{ $redirect.To }}; + set_by_lua_file $redirect_to /etc/nginx/lua/nginx/ngx_srv_redirect.lua {{ $redirect.To }}; # OK - return {{ $all.Cfg.HTTPRedirectCode }} $redirect_to; + return {{ $all.Cfg.HTTPRedirectCode }} $redirect_to; # OK } ## end server {{ $redirect.From }} {{ end }} @@ -567,234 +567,111 @@ http { {{ range $server := $servers }} ## start server {{ $server.Hostname }} server { - server_name {{ buildServerName $server.Hostname }} {{range $server.Aliases }}{{ . }} {{ end }}; + server_name {{ buildServerName $server.Hostname }} {{range $server.Aliases }}{{ . }} {{ end }}; OK {{ if $cfg.UseHTTP2 }} - http2 on; + http2 on; OK {{ end }} {{ if gt (len $cfg.BlockUserAgents) 0 }} - if ($block_ua) { - return 403; + if ($block_ua) { OK + return 403; OK } {{ end }} {{ if gt (len $cfg.BlockReferers) 0 }} - if ($block_ref) { - return 403; + if ($block_ref) { OK + return 403; OK } {{ end }} {{ template "SERVER" serverConfig $all $server }} - {{ if not (empty $cfg.ServerSnippet) }} - # Custom code snippet configured in the configuration configmap - {{ $cfg.ServerSnippet }} - {{ end }} - - {{ template "CUSTOM_ERRORS" (buildCustomErrorDeps "upstream-default-backend" $cfg.CustomHTTPErrors $all.EnableMetrics $cfg.EnableModsecurity) }} + {{ template "CUSTOM_ERRORS" (buildCustomErrorDeps "upstream-default-backend" $cfg.CustomHTTPErrors $all.EnableMetrics) }} # OK } ## end server {{ $server.Hostname }} {{ end }} # backend for when default-backend-service is not configured or it does not have endpoints - server { - listen {{ $all.ListenPorts.Default }} default_server {{ if $all.Cfg.ReusePort }}reuseport{{ end }} backlog={{ $all.BacklogSize }}; - {{ if $IsIPV6Enabled }}listen [::]:{{ $all.ListenPorts.Default }} default_server {{ if $all.Cfg.ReusePort }}reuseport{{ end }} backlog={{ $all.BacklogSize }};{{ end }} - set $proxy_upstream_name "internal"; + server { # OK + listen {{ $all.ListenPorts.Default }} default_server {{ if $all.Cfg.ReusePort }}reuseport{{ end }} backlog={{ $all.BacklogSize }}; # OK + {{ if $IsIPV6Enabled }}listen [::]:{{ $all.ListenPorts.Default }} default_server {{ if $all.Cfg.ReusePort }}reuseport{{ end }} backlog={{ $all.BacklogSize }};{{ end }} # OK + set $proxy_upstream_name "internal"; # OK - access_log off; + access_log off; # OK - location / { - return 404; + location / { # OK + return 404; # OK } } # default server, used for NGINX healthcheck and access to nginx stats - server { - # Ensure that modsecurity will not run on an internal location as this is not accessible from outside - {{ if $all.Cfg.EnableModsecurity }} - modsecurity off; - {{ end }} + server { # OK - listen 127.0.0.1:{{ .StatusPort }}; - set $proxy_upstream_name "internal"; + listen 127.0.0.1:{{ .StatusPort }}; # OK + set $proxy_upstream_name "internal"; # OK - keepalive_timeout 0; - gzip off; + keepalive_timeout 0; # OK + gzip off; # OK - access_log off; + access_log off; # OK {{ if $cfg.EnableOpentelemetry }} - opentelemetry off; + opentelemetry off; # OK {{ end }} - location {{ $healthzURI }} { - return 200; + location {{ $healthzURI }} { # OK + return 200; # OK } - location /is-dynamic-lb-initialized { - content_by_lua_file /etc/nginx/lua/nginx/ngx_conf_is_dynamic_lb_initialized.lua; + location /is-dynamic-lb-initialized { # OK + content_by_lua_file /etc/nginx/lua/nginx/ngx_conf_is_dynamic_lb_initialized.lua; # OK } - location {{ .StatusPath }} { - stub_status on; + location {{ .StatusPath }} { # OK + stub_status on; # OK } - location /configuration { - client_max_body_size {{ luaConfigurationRequestBodySize $cfg }}; - client_body_buffer_size {{ luaConfigurationRequestBodySize $cfg }}; - proxy_buffering off; + location /configuration { # OK + client_max_body_size {{ luaConfigurationRequestBodySize $cfg }}; # OK + client_body_buffer_size {{ luaConfigurationRequestBodySize $cfg }}; # OK + proxy_buffering off; # OK - content_by_lua_file /etc/nginx/lua/nginx/ngx_conf_configuration.lua; + content_by_lua_file /etc/nginx/lua/nginx/ngx_conf_configuration.lua; # OK } - location / { - return 404; + location / { # OK + return 404; # OK } } } -stream { - lua_package_path "/etc/nginx/lua/?.lua;/etc/nginx/lua/vendor/?.lua;;"; - - lua_shared_dict tcp_udp_configuration_data 5M; - - {{ buildResolvers $cfg.Resolver $cfg.DisableIpv6DNS }} - - init_by_lua_file /etc/nginx/lua/ngx_conf_init_stream.lua; - - init_worker_by_lua_file /etc/nginx/lua/nginx/ngx_conf_init_tcp_udp.lua; - - lua_add_variable $proxy_upstream_name; - - log_format log_stream '{{ $cfg.LogFormatStream }}'; - - {{ if or $cfg.DisableAccessLog $cfg.DisableStreamAccessLog }} - access_log off; - {{ else }} - access_log {{ or $cfg.StreamAccessLogPath $cfg.AccessLogPath }} log_stream {{ $cfg.AccessLogParams }}; - {{ end }} - - - error_log {{ $cfg.ErrorLogPath }} {{ $cfg.ErrorLogLevel }}; - {{ if $cfg.EnableRealIP }} - {{ range $trusted_ip := $cfg.ProxyRealIPCIDR }} - set_real_ip_from {{ $trusted_ip }}; - {{ end }} - {{ end }} - - upstream upstream_balancer { - server 0.0.0.1:1234; # placeholder - balancer_by_lua_file /etc/nginx/lua/nginx/ngx_conf_balancer_tcp_udp.lua; - } - - server { - listen 127.0.0.1:{{ .StreamPort }}; - - access_log off; - - content_by_lua_file /etc/nginx/lua/nginx/ngx_conf_content_tcp_udp.lua; - } - - # TCP services - {{ range $tcpServer := .TCPBackends }} - server { - preread_by_lua_block { - ngx.var.proxy_upstream_name="tcp-{{ $tcpServer.Backend.Namespace }}-{{ $tcpServer.Backend.Name }}-{{ $tcpServer.Backend.Port }}"; - } - - {{ range $address := $all.Cfg.BindAddressIpv4 }} - listen {{ $address }}:{{ $tcpServer.Port }}{{ if $tcpServer.Backend.ProxyProtocol.Decode }} proxy_protocol{{ end }}; - {{ else }} - listen {{ $tcpServer.Port }}{{ if $tcpServer.Backend.ProxyProtocol.Decode }} proxy_protocol{{ end }}; - {{ end }} - {{ if $IsIPV6Enabled }} - {{ range $address := $all.Cfg.BindAddressIpv6 }} - listen {{ $address }}:{{ $tcpServer.Port }}{{ if $tcpServer.Backend.ProxyProtocol.Decode }} proxy_protocol{{ end }}; - {{ else }} - listen [::]:{{ $tcpServer.Port }}{{ if $tcpServer.Backend.ProxyProtocol.Decode }} proxy_protocol{{ end }}; - {{ end }} - {{ end }} - proxy_timeout {{ $cfg.ProxyStreamTimeout }}; - proxy_next_upstream {{ if $cfg.ProxyStreamNextUpstream }}on{{ else }}off{{ end }}; - proxy_next_upstream_timeout {{ $cfg.ProxyStreamNextUpstreamTimeout }}; - proxy_next_upstream_tries {{ $cfg.ProxyStreamNextUpstreamTries }}; - - proxy_pass upstream_balancer; - {{ if $tcpServer.Backend.ProxyProtocol.Encode }} - proxy_protocol on; - {{ end }} - } - {{ end }} - - # UDP services - {{ range $udpServer := .UDPBackends }} - server { - preread_by_lua_block { - ngx.var.proxy_upstream_name="udp-{{ $udpServer.Backend.Namespace }}-{{ $udpServer.Backend.Name }}-{{ $udpServer.Backend.Port }}"; - } - - {{ range $address := $all.Cfg.BindAddressIpv4 }} - listen {{ $address }}:{{ $udpServer.Port }} udp; - {{ else }} - listen {{ $udpServer.Port }} udp; - {{ end }} - {{ if $IsIPV6Enabled }} - {{ range $address := $all.Cfg.BindAddressIpv6 }} - listen {{ $address }}:{{ $udpServer.Port }} udp; - {{ else }} - listen [::]:{{ $udpServer.Port }} udp; - {{ end }} - {{ end }} - proxy_responses {{ $cfg.ProxyStreamResponses }}; - proxy_timeout {{ $cfg.ProxyStreamTimeout }}; - proxy_next_upstream {{ if $cfg.ProxyStreamNextUpstream }}on{{ else }}off{{ end }}; - proxy_next_upstream_timeout {{ $cfg.ProxyStreamNextUpstreamTimeout }}; - proxy_next_upstream_tries {{ $cfg.ProxyStreamNextUpstreamTries }}; - proxy_pass upstream_balancer; - } - {{ end }} - - # Stream Snippets - {{ range $snippet := .StreamSnippets }} - {{ $snippet }} - {{ end }} -} - {{/* definition of templates to avoid repetitions */}} {{ define "CUSTOM_ERRORS" }} {{ $enableMetrics := .EnableMetrics }} - {{ $modsecurityEnabled := .ModsecurityEnabled }} {{ $upstreamName := .UpstreamName }} {{ range $errCode := .ErrorCodes }} - location @custom_{{ $upstreamName }}_{{ $errCode }} { - internal; + location @custom_{{ $upstreamName }}_{{ $errCode }} { # OK + internal; # OK + proxy_intercept_errors off; # OK - # Ensure that modsecurity will not run on custom error pages or they might be blocked - {{ if $modsecurityEnabled }} - modsecurity off; - {{ end }} + proxy_set_header X-Code {{ $errCode }}; # OK + proxy_set_header X-Format $http_accept; # OK + proxy_set_header X-Original-URI $request_uri; # OK + proxy_set_header X-Namespace $namespace; # OK + proxy_set_header X-Ingress-Name $ingress_name; # OK + proxy_set_header X-Service-Name $service_name;# OK + proxy_set_header X-Service-Port $service_port; # OK + proxy_set_header X-Request-ID $req_id;# OK + proxy_set_header X-Forwarded-For $remote_addr; # OK + proxy_set_header Host $best_http_host; # OK - proxy_intercept_errors off; + set $proxy_upstream_name {{ $upstreamName | quote }}; # OK - proxy_set_header X-Code {{ $errCode }}; - proxy_set_header X-Format $http_accept; - proxy_set_header X-Original-URI $request_uri; - proxy_set_header X-Namespace $namespace; - proxy_set_header X-Ingress-Name $ingress_name; - proxy_set_header X-Service-Name $service_name; - proxy_set_header X-Service-Port $service_port; - proxy_set_header X-Request-ID $req_id; - proxy_set_header X-Forwarded-For $remote_addr; - proxy_set_header Host $best_http_host; + rewrite (.*) / break; # OK - set $proxy_upstream_name {{ $upstreamName | quote }}; - - rewrite (.*) / break; - - proxy_pass http://upstream_balancer; + proxy_pass http://upstream_balancer; # OK {{ if $enableMetrics }} - log_by_lua_file /etc/nginx/lua/nginx/ngx_conf_log.lua; + log_by_lua_file /etc/nginx/lua/nginx/ngx_conf_log.lua; # OK {{ end }} } {{ end }} @@ -807,29 +684,29 @@ stream { {{ if $cors.CorsAllowOrigin }} {{ buildCorsOriginRegex $cors.CorsAllowOrigin }} {{ end }} - if ($request_method = 'OPTIONS') { - set $cors ${cors}options; + if ($request_method = 'OPTIONS') { # OK + set $cors ${cors}options; # OK } - if ($cors = "true") { - more_set_headers 'Access-Control-Allow-Origin: $http_origin'; - {{ if $cors.CorsAllowCredentials }} more_set_headers 'Access-Control-Allow-Credentials: {{ $cors.CorsAllowCredentials }}'; {{ end }} - more_set_headers 'Access-Control-Allow-Methods: {{ $cors.CorsAllowMethods }}'; - more_set_headers 'Access-Control-Allow-Headers: {{ $cors.CorsAllowHeaders }}'; - {{ if not (empty $cors.CorsExposeHeaders) }} more_set_headers 'Access-Control-Expose-Headers: {{ $cors.CorsExposeHeaders }}'; {{ end }} - more_set_headers 'Access-Control-Max-Age: {{ $cors.CorsMaxAge }}'; + if ($cors = "true") { # OK + more_set_headers 'Access-Control-Allow-Origin: $http_origin'; # OK + {{ if $cors.CorsAllowCredentials }} more_set_headers 'Access-Control-Allow-Credentials: {{ $cors.CorsAllowCredentials }}'; {{ end }} # OK + more_set_headers 'Access-Control-Allow-Methods: {{ $cors.CorsAllowMethods }}'; # OK + more_set_headers 'Access-Control-Allow-Headers: {{ $cors.CorsAllowHeaders }}'; # OK + {{ if not (empty $cors.CorsExposeHeaders) }} more_set_headers 'Access-Control-Expose-Headers: {{ $cors.CorsExposeHeaders }}'; {{ end }} # OK + more_set_headers 'Access-Control-Max-Age: {{ $cors.CorsMaxAge }}'; # OK } - if ($cors = "trueoptions") { - more_set_headers 'Access-Control-Allow-Origin: $http_origin'; - {{ if $cors.CorsAllowCredentials }} more_set_headers 'Access-Control-Allow-Credentials: {{ $cors.CorsAllowCredentials }}'; {{ end }} - more_set_headers 'Access-Control-Allow-Methods: {{ $cors.CorsAllowMethods }}'; - more_set_headers 'Access-Control-Allow-Headers: {{ $cors.CorsAllowHeaders }}'; - {{ if not (empty $cors.CorsExposeHeaders) }} more_set_headers 'Access-Control-Expose-Headers: {{ $cors.CorsExposeHeaders }}'; {{ end }} - more_set_headers 'Access-Control-Max-Age: {{ $cors.CorsMaxAge }}'; - more_set_headers 'Content-Type: text/plain charset=UTF-8'; - more_set_headers 'Content-Length: 0'; - return 204; + if ($cors = "trueoptions") { # OK + more_set_headers 'Access-Control-Allow-Origin: $http_origin'; # OK + {{ if $cors.CorsAllowCredentials }} more_set_headers 'Access-Control-Allow-Credentials: {{ $cors.CorsAllowCredentials }}'; {{ end }} # OK + more_set_headers 'Access-Control-Allow-Methods: {{ $cors.CorsAllowMethods }}'; # OK + more_set_headers 'Access-Control-Allow-Headers: {{ $cors.CorsAllowHeaders }}'; # OK + {{ if not (empty $cors.CorsExposeHeaders) }} more_set_headers 'Access-Control-Expose-Headers: {{ $cors.CorsExposeHeaders }}'; {{ end }} # OK + more_set_headers 'Access-Control-Max-Age: {{ $cors.CorsMaxAge }}'; # OK + more_set_headers 'Content-Type: text/plain charset=UTF-8'; # OK + more_set_headers 'Content-Length: 0'; # OK + return 204; # OK } {{ end }} @@ -838,321 +715,300 @@ stream { {{ $all := .First }} {{ $server := .Second }} - {{ buildHTTPListener $all $server.Hostname }} - {{ buildHTTPSListener $all $server.Hostname }} + {{ buildHTTPListener $all $server.Hostname }} # OK + {{ buildHTTPSListener $all $server.Hostname }} # OK - set $proxy_upstream_name "-"; + set $proxy_upstream_name "-"; # OK {{ if not ( empty $server.CertificateAuth.MatchCN ) }} {{ if gt (len $server.CertificateAuth.MatchCN) 0 }} - if ( $ssl_client_s_dn !~ {{ $server.CertificateAuth.MatchCN }} ) { - return 403 "client certificate unauthorized"; + if ( $ssl_client_s_dn !~ {{ $server.CertificateAuth.MatchCN }} ) { # OK + return 403 "client certificate unauthorized"; # OK } {{ end }} {{ end }} {{ if eq $server.Hostname "_" }} - ssl_reject_handshake {{ if $all.Cfg.SSLRejectHandshake }}on{{ else }}off{{ end }}; + ssl_reject_handshake {{ if $all.Cfg.SSLRejectHandshake }}on{{ else }}off{{ end }}; # OK {{ end }} - ssl_certificate_by_lua_file /etc/nginx/lua/nginx/ngx_conf_certificate.lua; + ssl_certificate_by_lua_file /etc/nginx/lua/nginx/ngx_conf_certificate.lua; # OK {{ if not (empty $server.AuthTLSError) }} - # {{ $server.AuthTLSError }} + # {{ $server.AuthTLSError }} # NOT, WHERE THIS IF ENDS?? Couldn't reproduce this config on my ingress return 403; {{ else }} {{ if not (empty $server.CertificateAuth.CAFileName) }} # PEM sha: {{ $server.CertificateAuth.CASHA }} - ssl_client_certificate {{ $server.CertificateAuth.CAFileName }}; - ssl_verify_client {{ $server.CertificateAuth.VerifyClient }}; - ssl_verify_depth {{ $server.CertificateAuth.ValidationDepth }}; + ssl_client_certificate {{ $server.CertificateAuth.CAFileName }}; # OK + ssl_verify_client {{ $server.CertificateAuth.VerifyClient }}; # OK + ssl_verify_depth {{ $server.CertificateAuth.ValidationDepth }}; # OK {{ if not (empty $server.CertificateAuth.CRLFileName) }} # PEM sha: {{ $server.CertificateAuth.CRLSHA }} - ssl_crl {{ $server.CertificateAuth.CRLFileName }}; + ssl_crl {{ $server.CertificateAuth.CRLFileName }}; # OK {{ end }} {{ if not (empty $server.CertificateAuth.ErrorPage)}} - error_page 495 496 = {{ $server.CertificateAuth.ErrorPage }}; + error_page 495 496 = {{ $server.CertificateAuth.ErrorPage }}; # OK {{ end }} {{ end }} {{ if not (empty $server.ProxySSL.CAFileName) }} # PEM sha: {{ $server.ProxySSL.CASHA }} - proxy_ssl_trusted_certificate {{ $server.ProxySSL.CAFileName }}; - proxy_ssl_ciphers {{ $server.ProxySSL.Ciphers }}; - proxy_ssl_protocols {{ $server.ProxySSL.Protocols }}; - proxy_ssl_verify {{ $server.ProxySSL.Verify }}; - proxy_ssl_verify_depth {{ $server.ProxySSL.VerifyDepth }}; + proxy_ssl_trusted_certificate {{ $server.ProxySSL.CAFileName }}; # OK + proxy_ssl_ciphers {{ $server.ProxySSL.Ciphers }}; # OK + proxy_ssl_protocols {{ $server.ProxySSL.Protocols }}; # OK + proxy_ssl_verify {{ $server.ProxySSL.Verify }}; # OK + proxy_ssl_verify_depth {{ $server.ProxySSL.VerifyDepth }}; # OK {{ if not (empty $server.ProxySSL.ProxySSLName) }} - proxy_ssl_name {{ $server.ProxySSL.ProxySSLName }}; - proxy_ssl_server_name {{ $server.ProxySSL.ProxySSLServerName }}; + proxy_ssl_name {{ $server.ProxySSL.ProxySSLName }}; # OK + proxy_ssl_server_name {{ $server.ProxySSL.ProxySSLServerName }}; # OK {{ end }} {{ end }} {{ if not (empty $server.ProxySSL.PemFileName) }} - proxy_ssl_certificate {{ $server.ProxySSL.PemFileName }}; - proxy_ssl_certificate_key {{ $server.ProxySSL.PemFileName }}; + proxy_ssl_certificate {{ $server.ProxySSL.PemFileName }}; # OK + proxy_ssl_certificate_key {{ $server.ProxySSL.PemFileName }}; # OK {{ end }} {{ if not (empty $server.SSLCiphers) }} - ssl_ciphers {{ $server.SSLCiphers }}; + ssl_ciphers {{ $server.SSLCiphers }}; # OK {{ end }} {{ if not (empty $server.SSLPreferServerCiphers) }} - ssl_prefer_server_ciphers {{ $server.SSLPreferServerCiphers }}; + ssl_prefer_server_ciphers {{ $server.SSLPreferServerCiphers }}; # OK {{ end }} - {{ if not (empty $server.ServerSnippet) }} - # Custom code snippet configured for host {{ $server.Hostname }} - {{ $server.ServerSnippet }} + {{ range $errorLocation := (buildCustomErrorLocationsPerServer $server) }} # OK + {{ template "CUSTOM_ERRORS" (buildCustomErrorDeps $errorLocation.UpstreamName $errorLocation.Codes $all.EnableMetrics) }} # OK {{ end }} - {{ range $errorLocation := (buildCustomErrorLocationsPerServer $server) }} - {{ template "CUSTOM_ERRORS" (buildCustomErrorDeps $errorLocation.UpstreamName $errorLocation.Codes $all.EnableMetrics $all.Cfg.EnableModsecurity) }} - {{ end }} + {{ buildMirrorLocations $server.Locations }} # OK - {{ buildMirrorLocations $server.Locations }} - - {{ $enforceRegex := enforceRegexModifier $server.Locations }} + {{ $enforceRegex := enforceRegexModifier $server.Locations }} # OK {{ range $location := $server.Locations }} - {{ $path := buildLocation $location $enforceRegex }} - {{ $proxySetHeader := proxySetHeader $location }} - {{ $authPath := buildAuthLocation $location $all.Cfg.GlobalExternalAuth.URL }} - {{ $applyGlobalAuth := shouldApplyGlobalAuth $location $all.Cfg.GlobalExternalAuth.URL }} - {{ $applyAuthUpstream := shouldApplyAuthUpstream $location $all.Cfg }} + {{ $path := buildLocation $location $enforceRegex }} # OK + {{ $proxySetHeader := proxySetHeader $location }} # OK + {{ $authPath := buildAuthLocation $location $all.Cfg.GlobalExternalAuth.URL }} # OK + {{ $applyGlobalAuth := shouldApplyGlobalAuth $location $all.Cfg.GlobalExternalAuth.URL }} # OK + {{ $applyAuthUpstream := shouldApplyAuthUpstream $location $all.Cfg }} # OK - {{ $externalAuth := $location.ExternalAuth }} + {{ $externalAuth := $location.ExternalAuth }} # OK {{ if eq $applyGlobalAuth true }} - {{ $externalAuth = $all.Cfg.GlobalExternalAuth }} + {{ $externalAuth = $all.Cfg.GlobalExternalAuth }} # OK {{ end }} {{ if not (empty $location.Rewrite.AppRoot) }} - if ($uri = /) { - return 302 $scheme://$http_host{{ $location.Rewrite.AppRoot }}; + if ($uri = /) { # OK + return 302 $scheme://$http_host{{ $location.Rewrite.AppRoot }}; # OK } {{ end }} {{ if $authPath }} location = {{ $authPath }} { - internal; + internal; # OK {{ if (or $all.Cfg.EnableOpentelemetry $location.Opentelemetry.Enabled) }} - opentelemetry on; - opentelemetry_propagate; + opentelemetry on; # OK + opentelemetry_propagate; # OK {{ end }} {{ if not $all.Cfg.EnableAuthAccessLog }} - access_log off; - {{ end }} - - # Ensure that modsecurity will not run on an internal location as this is not accessible from outside - {{ if $all.Cfg.EnableModsecurity }} - modsecurity off; + access_log off; # OK {{ end }} {{ if $externalAuth.AuthCacheKey }} - set $tmp_cache_key '{{ $server.Hostname }}{{ $authPath }}{{ $externalAuth.AuthCacheKey }}'; - set $cache_key ''; + set $tmp_cache_key '{{ $server.Hostname }}{{ $authPath }}{{ $externalAuth.AuthCacheKey }}'; # OK + set $cache_key ''; # OK - rewrite_by_lua_file /etc/nginx/lua/nginx/ngx_conf_rewrite_auth.lua; + rewrite_by_lua_file /etc/nginx/lua/nginx/ngx_conf_rewrite_auth.lua; # OK - proxy_cache auth_cache; + proxy_cache auth_cache; # OK {{- range $dur := $externalAuth.AuthCacheDuration }} - proxy_cache_valid {{ $dur }}; + proxy_cache_valid {{ $dur }}; # OK {{- end }} - proxy_cache_key "$cache_key"; + proxy_cache_key "$cache_key"; # OK {{ end }} # ngx_auth_request module overrides variables in the parent request, # therefore we have to explicitly set this variable again so that when the parent request # resumes it has the correct value set for this variable so that Lua can pick backend correctly - set $proxy_upstream_name {{ buildUpstreamName $location | quote }}; + set $proxy_upstream_name {{ buildUpstreamName $location | quote }}; # OK - proxy_pass_request_body off; - proxy_set_header Content-Length ""; - proxy_set_header X-Forwarded-Proto ""; - proxy_set_header X-Request-ID $req_id; + proxy_pass_request_body off; # OK + proxy_set_header Content-Length ""; # OK + proxy_set_header X-Forwarded-Proto ""; # OK + proxy_set_header X-Request-ID $req_id; # OK {{ if $externalAuth.Method }} - proxy_method {{ $externalAuth.Method }}; - proxy_set_header X-Original-URI $request_uri; - proxy_set_header X-Scheme $pass_access_scheme; + proxy_method {{ $externalAuth.Method }}; + proxy_set_header X-Original-URI $request_uri; # OK + proxy_set_header X-Scheme $pass_access_scheme; # OK {{ end }} - proxy_set_header Host {{ $externalAuth.Host }}; - proxy_set_header X-Original-URL $scheme://$http_host$request_uri; - proxy_set_header X-Original-Method $request_method; - proxy_set_header X-Sent-From "nginx-ingress-controller"; - proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Host {{ $externalAuth.Host }}; # OK + proxy_set_header X-Original-URL $scheme://$http_host$request_uri; # OK + proxy_set_header X-Original-Method $request_method; # OK + proxy_set_header X-Sent-From "nginx-ingress-controller"; # OK + proxy_set_header X-Real-IP $remote_addr; # OK {{ if and $all.Cfg.UseForwardedHeaders $all.Cfg.ComputeFullForwardedFor }} - proxy_set_header X-Forwarded-For $full_x_forwarded_for; + proxy_set_header X-Forwarded-For $full_x_forwarded_for; # OK {{ else }} - proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Forwarded-For $remote_addr; # OK {{ end }} {{ if $externalAuth.RequestRedirect }} - proxy_set_header X-Auth-Request-Redirect {{ $externalAuth.RequestRedirect }}; + proxy_set_header X-Auth-Request-Redirect {{ $externalAuth.RequestRedirect }}; # OK {{ else }} - proxy_set_header X-Auth-Request-Redirect $request_uri; + proxy_set_header X-Auth-Request-Redirect $request_uri; # OK {{ end }} {{ if $externalAuth.AuthCacheKey }} - proxy_buffering "on"; + proxy_buffering "on"; # OK {{ else }} - proxy_buffering {{ $location.Proxy.ProxyBuffering }}; + proxy_buffering {{ $location.Proxy.ProxyBuffering }}; # OK {{ end }} - proxy_buffer_size {{ $location.Proxy.BufferSize }}; - proxy_buffers {{ $location.Proxy.BuffersNumber }} {{ $location.Proxy.BufferSize }}; - proxy_request_buffering {{ $location.Proxy.RequestBuffering }}; + proxy_buffer_size {{ $location.Proxy.BufferSize }}; # OK + proxy_buffers {{ $location.Proxy.BuffersNumber }} {{ $location.Proxy.BufferSize }}; # OK + proxy_request_buffering {{ $location.Proxy.RequestBuffering }}; # OK - proxy_ssl_server_name on; - proxy_pass_request_headers on; + proxy_ssl_server_name on; # OK + proxy_pass_request_headers on; # OK {{ if isValidByteSize $location.Proxy.BodySize true }} - client_max_body_size {{ $location.Proxy.BodySize }}; + client_max_body_size {{ $location.Proxy.BodySize }}; # OK {{ end }} {{ if isValidByteSize $location.ClientBodyBufferSize false }} - client_body_buffer_size {{ $location.ClientBodyBufferSize }}; + client_body_buffer_size {{ $location.ClientBodyBufferSize }}; # OK {{ end }} # Pass the extracted client certificate to the auth provider {{ if not (empty $server.CertificateAuth.CAFileName) }} {{ if $server.CertificateAuth.PassCertToUpstream }} - proxy_set_header ssl-client-cert $ssl_client_escaped_cert; + proxy_set_header ssl-client-cert $ssl_client_escaped_cert; # OK {{ end }} - proxy_set_header ssl-client-verify $ssl_client_verify; - proxy_set_header ssl-client-subject-dn $ssl_client_s_dn; - proxy_set_header ssl-client-issuer-dn $ssl_client_i_dn; + proxy_set_header ssl-client-verify $ssl_client_verify; # OK + proxy_set_header ssl-client-subject-dn $ssl_client_s_dn; # OK + proxy_set_header ssl-client-issuer-dn $ssl_client_i_dn; # OK {{ end }} {{- range $line := buildAuthProxySetHeaders $externalAuth.ProxySetHeaders}} - {{ $line }} + {{ $line }} # OK {{- end }} - {{ if not (empty $externalAuth.AuthSnippet) }} - {{ $externalAuth.AuthSnippet }} - {{ end }} - {{ if and (eq $applyAuthUpstream true) (eq $applyGlobalAuth false) }} {{ $authUpstreamName := buildAuthUpstreamName $location $server.Hostname }} # The target is an upstream with HTTP keepalive, that is why the # Connection header is cleared and the HTTP version is set to 1.1 as # the Nginx documentation suggests: # http://nginx.org/en/docs/http/ngx_http_upstream_module.html#keepalive - proxy_http_version 1.1; - proxy_set_header Connection ""; - set $target {{ changeHostPort $externalAuth.URL $authUpstreamName }}; + proxy_http_version 1.1; # OK + proxy_set_header Connection ""; # OK + set $target {{ changeHostPort $externalAuth.URL $authUpstreamName }}; # OK {{ else }} - proxy_http_version {{ $location.Proxy.ProxyHTTPVersion }}; - set $target {{ $externalAuth.URL }}; + proxy_http_version {{ $location.Proxy.ProxyHTTPVersion }}; # OK + set $target {{ $externalAuth.URL }}; # OK {{ end }} - proxy_pass $target; + proxy_pass $target; # OK } {{ end }} {{ if isLocationAllowed $location }} {{ if $externalAuth.SigninURL }} location {{ buildAuthSignURLLocation $location.Path $externalAuth.SigninURL }} { - internal; + internal; # OK - add_header Set-Cookie $auth_cookie; + add_header Set-Cookie $auth_cookie; # OK {{ if $location.CorsConfig.CorsEnabled }} - {{ template "CORS" $location }} + {{ template "CORS" $location }} # OK {{ end }} - # Ensure that modsecurity will not run on an internal location as this is not accessible from outside - {{ if $all.Cfg.EnableModsecurity }} - modsecurity off; - {{ end }} - - return 302 {{ buildAuthSignURL $externalAuth.SigninURL $externalAuth.SigninURLRedirectParam }}; + return 302 {{ buildAuthSignURL $externalAuth.SigninURL $externalAuth.SigninURLRedirectParam }}; # OK } {{ end }} {{ end }} location {{ $path }} { {{ $ing := (getIngressInformation $location.Ingress $server.Hostname $location.IngressPath) }} - set $namespace {{ $ing.Namespace | quote}}; - set $ingress_name {{ $ing.Rule | quote }}; - set $service_name {{ $ing.Service | quote }}; - set $service_port {{ $ing.ServicePort | quote }}; - set $location_path {{ $ing.Path | escapeLiteralDollar | quote }}; + set $namespace {{ $ing.Namespace | quote}}; # OK + set $ingress_name {{ $ing.Rule | quote }}; # OK + set $service_name {{ $ing.Service | quote }}; # OK + set $service_port {{ $ing.ServicePort | quote }}; # OK + set $location_path {{ $ing.Path | escapeLiteralDollar | quote }}; # OK - {{ buildOpentelemetryForLocation $all.Cfg.EnableOpentelemetry $all.Cfg.OpentelemetryTrustIncomingSpan $location }} + {{ buildOpentelemetryForLocation $all.Cfg.EnableOpentelemetry $all.Cfg.OpentelemetryTrustIncomingSpan $location }} # OK {{ if $location.Mirror.Source }} - mirror {{ $location.Mirror.Source }}; - mirror_request_body {{ $location.Mirror.RequestBody }}; + mirror {{ $location.Mirror.Source }}; # OK + mirror_request_body {{ $location.Mirror.RequestBody }}; # OK {{ end }} - {{ locationConfigForLua $location $all }} + {{ locationConfigForLua $location $all }} # OK - rewrite_by_lua_file /etc/nginx/lua/nginx/ngx_rewrite.lua; + rewrite_by_lua_file /etc/nginx/lua/nginx/ngx_rewrite.lua; # OK - header_filter_by_lua_file /etc/nginx/lua/nginx/ngx_conf_srv_hdr_filter.lua; + header_filter_by_lua_file /etc/nginx/lua/nginx/ngx_conf_srv_hdr_filter.lua; # OK - log_by_lua_file /etc/nginx/lua/nginx/ngx_conf_log_block.lua; + log_by_lua_file /etc/nginx/lua/nginx/ngx_conf_log_block.lua; # OK {{ if not $location.Logs.Access }} - access_log off; + access_log off; # OK {{ end }} {{ if $location.Logs.Rewrite }} - rewrite_log on; + rewrite_log on; # OK {{ end }} {{ if $location.HTTP2PushPreload }} - http2_push_preload on; + http2_push_preload on; # OK {{ end }} - port_in_redirect {{ if $location.UsePortInRedirects }}on{{ else }}off{{ end }}; + port_in_redirect {{ if $location.UsePortInRedirects }}on{{ else }}off{{ end }}; # OK - set $balancer_ewma_score -1; - set $proxy_upstream_name {{ buildUpstreamName $location | quote }}; - set $proxy_host $proxy_upstream_name; - set $pass_access_scheme $scheme; + set $balancer_ewma_score -1; # OK + set $proxy_upstream_name {{ buildUpstreamName $location | quote }}; # OK + set $proxy_host $proxy_upstream_name; # OK + set $pass_access_scheme $scheme; # OK {{ if $all.Cfg.UseProxyProtocol }} - set $pass_server_port $proxy_protocol_server_port; + set $pass_server_port $proxy_protocol_server_port; # OK {{ else }} - set $pass_server_port $server_port; + set $pass_server_port $server_port; # OK {{ end }} - set $best_http_host $http_host; - set $pass_port $pass_server_port; + set $best_http_host $http_host; # OK + set $pass_port $pass_server_port; # OK - set $proxy_alternative_upstream_name ""; + set $proxy_alternative_upstream_name ""; # OK - {{ buildModSecurityForLocation $all.Cfg $location }} + {{ if isLocationAllowed $location }} # 1 + {{ if gt (len $location.Denylist.CIDR) 0 }} # 2 + {{ range $ip := $location.Denylist.CIDR }} # 3 + deny {{ $ip }};{{ end }} # 2 # OK + {{ end }} # 1 + {{ if gt (len $location.Allowlist.CIDR) 0 }} # 2 + {{ range $ip := $location.Allowlist.CIDR }} # 3 + allow {{ $ip }};{{ end }} # 2 # OK + deny all; # OK + {{ end }} # 1 - {{ if isLocationAllowed $location }} - {{ if gt (len $location.Denylist.CIDR) 0 }} - {{ range $ip := $location.Denylist.CIDR }} - deny {{ $ip }};{{ end }} - {{ end }} - {{ if gt (len $location.Allowlist.CIDR) 0 }} - {{ range $ip := $location.Allowlist.CIDR }} - allow {{ $ip }};{{ end }} - deny all; - {{ end }} + {{ if $location.CorsConfig.CorsEnabled }} # 2 + {{ template "CORS" $location }} # OK + {{ end }} # 1 - {{ if $location.CorsConfig.CorsEnabled }} - {{ template "CORS" $location }} - {{ end }} - - {{ if not (isLocationInLocationList $location $all.Cfg.NoAuthLocations) }} - {{ if $authPath }} + {{ if not (isLocationInLocationList $location $all.Cfg.NoAuthLocations) }} # 2 + {{ if $authPath }} # 3 # this location requires authentication - {{ if and (eq $applyAuthUpstream true) (eq $applyGlobalAuth false) }} + {{ if and (eq $applyAuthUpstream true) (eq $applyGlobalAuth false) }} # 4 set $auth_cookie ''; add_header Set-Cookie $auth_cookie; - {{- range $line := buildAuthResponseHeaders $proxySetHeader $externalAuth.ResponseHeaders true }} + {{- range $line := buildAuthResponseHeaders $proxySetHeader $externalAuth.ResponseHeaders true }} # 5 {{ $line }} - {{- end }} + {{- end }} # 4 # `auth_request` module does not support HTTP keepalives in upstream block: # https://trac.nginx.org/nginx/ticket/1579 access_by_lua_block { @@ -1172,224 +1028,217 @@ stream { {{ else }} auth_request {{ $authPath }}; auth_request_set $auth_cookie $upstream_http_set_cookie; - {{ if $externalAuth.AlwaysSetCookie }} + {{ if $externalAuth.AlwaysSetCookie }} # 5 add_header Set-Cookie $auth_cookie always; {{ else }} add_header Set-Cookie $auth_cookie; - {{ end }} - {{- range $line := buildAuthResponseHeaders $proxySetHeader $externalAuth.ResponseHeaders false }} + {{ end }} # 4 + {{- range $line := buildAuthResponseHeaders $proxySetHeader $externalAuth.ResponseHeaders false }} # 5 {{ $line }} - {{- end }} - {{ end }} - {{ end }} + {{- end }} # 4 + {{ end }} # 3 + {{ end }} # 2 - {{ if $externalAuth.SigninURL }} + {{ if $externalAuth.SigninURL }} # 3 set_escape_uri $escaped_request_uri $request_uri; error_page 401 = {{ buildAuthSignURLLocation $location.Path $externalAuth.SigninURL }}; - {{ end }} + {{ end }} # 2 - {{ if $location.BasicDigestAuth.Secured }} - {{ if eq $location.BasicDigestAuth.Type "basic" }} + {{ if $location.BasicDigestAuth.Secured }} # 3 + {{ if eq $location.BasicDigestAuth.Type "basic" }} # 4 auth_basic {{ $location.BasicDigestAuth.Realm | quote }}; auth_basic_user_file {{ $location.BasicDigestAuth.File }}; {{ else }} auth_digest {{ $location.BasicDigestAuth.Realm | quote }}; auth_digest_user_file {{ $location.BasicDigestAuth.File }}; - {{ end }} + {{ end }} # 3 {{ $proxySetHeader }} Authorization ""; - {{ end }} - {{ end }} + {{ end }} # 2 + {{ end }} # 1 {{/* if the location contains a rate limit annotation, create one */}} {{ $limits := buildRateLimit $location }} - {{ range $limit := $limits }} - {{ $limit }}{{ end }} + {{ range $limit := $limits }} # 2 # OK + {{ $limit }}{{ end }} # 1 {{ if isValidByteSize $location.Proxy.BodySize true }} - client_max_body_size {{ $location.Proxy.BodySize }}; + client_max_body_size {{ $location.Proxy.BodySize }}; # OK {{ end }} {{ if isValidByteSize $location.ClientBodyBufferSize false }} - client_body_buffer_size {{ $location.ClientBodyBufferSize }}; - {{ end }} + client_body_buffer_size {{ $location.ClientBodyBufferSize }}; # OK + {{ end }} # 1 {{/* By default use vhost as Host to upstream, but allow overrides */}} {{ if not (empty $location.UpstreamVhost) }} - {{ $proxySetHeader }} Host {{ $location.UpstreamVhost | quote }}; + {{ $proxySetHeader }} Host {{ $location.UpstreamVhost | quote }}; # OK {{ else }} - {{ $proxySetHeader }} Host $best_http_host; - {{ end }} + {{ $proxySetHeader }} Host $best_http_host; # OK + {{ end }} # 1 # Pass the extracted client certificate to the backend - {{ if not (empty $server.CertificateAuth.CAFileName) }} - {{ if $server.CertificateAuth.PassCertToUpstream }} - {{ $proxySetHeader }} ssl-client-cert $ssl_client_escaped_cert; - {{ end }} - {{ $proxySetHeader }} ssl-client-verify $ssl_client_verify; - {{ $proxySetHeader }} ssl-client-subject-dn $ssl_client_s_dn; - {{ $proxySetHeader }} ssl-client-issuer-dn $ssl_client_i_dn; - {{ end }} + {{ if not (empty $server.CertificateAuth.CAFileName) }} # 2 + {{ if $server.CertificateAuth.PassCertToUpstream }} # 3 + {{ $proxySetHeader }} ssl-client-cert $ssl_client_escaped_cert; # OK + {{ end }} # 2 + {{ $proxySetHeader }} ssl-client-verify $ssl_client_verify; # OK + {{ $proxySetHeader }} ssl-client-subject-dn $ssl_client_s_dn; # OK + {{ $proxySetHeader }} ssl-client-issuer-dn $ssl_client_i_dn; # OK + {{ end }} # 1 # Allow websocket connections - {{ $proxySetHeader }} Upgrade $http_upgrade; + {{ $proxySetHeader }} Upgrade $http_upgrade; # OK {{ if $location.Connection.Enabled}} - {{ $proxySetHeader }} Connection {{ $location.Connection.Header }}; + {{ $proxySetHeader }} Connection {{ $location.Connection.Header }}; # OK {{ else }} - {{ $proxySetHeader }} Connection $connection_upgrade; + {{ $proxySetHeader }} Connection $connection_upgrade; # OK {{ end }} - {{ $proxySetHeader }} X-Request-ID $req_id; - {{ $proxySetHeader }} X-Real-IP $remote_addr; + {{ $proxySetHeader }} X-Request-ID $req_id; # OK + {{ $proxySetHeader }} X-Real-IP $remote_addr; # OK {{ if and $all.Cfg.UseForwardedHeaders $all.Cfg.ComputeFullForwardedFor }} - {{ $proxySetHeader }} X-Forwarded-For $full_x_forwarded_for; + {{ $proxySetHeader }} X-Forwarded-For $full_x_forwarded_for; # OK {{ else }} - {{ $proxySetHeader }} X-Forwarded-For $remote_addr; - {{ end }} - {{ $proxySetHeader }} X-Forwarded-Host $best_http_host; - {{ $proxySetHeader }} X-Forwarded-Port $pass_port; - {{ $proxySetHeader }} X-Forwarded-Proto $pass_access_scheme; - {{ $proxySetHeader }} X-Forwarded-Scheme $pass_access_scheme; + {{ $proxySetHeader }} X-Forwarded-For $remote_addr; # OK + {{ end }} # 1 + {{ $proxySetHeader }} X-Forwarded-Host $best_http_host; # OK + {{ $proxySetHeader }} X-Forwarded-Port $pass_port; # OK + {{ $proxySetHeader }} X-Forwarded-Proto $pass_access_scheme; # OK + {{ $proxySetHeader }} X-Forwarded-Scheme $pass_access_scheme; # OK {{ if $all.Cfg.ProxyAddOriginalURIHeader }} - {{ $proxySetHeader }} X-Original-URI $request_uri; - {{ end }} - {{ $proxySetHeader }} X-Scheme $pass_access_scheme; + {{ $proxySetHeader }} X-Original-URI $request_uri; # OK + {{ end }} # 1 + {{ $proxySetHeader }} X-Scheme $pass_access_scheme; # OK # Pass the original X-Forwarded-For - {{ $proxySetHeader }} X-Original-Forwarded-For {{ buildForwardedFor $all.Cfg.ForwardedForHeader }}; + {{ $proxySetHeader }} X-Original-Forwarded-For {{ buildForwardedFor $all.Cfg.ForwardedForHeader }}; # OK # mitigate HTTPoxy Vulnerability # https://www.nginx.com/blog/mitigating-the-httpoxy-vulnerability-with-nginx/ - {{ $proxySetHeader }} Proxy ""; + {{ $proxySetHeader }} Proxy ""; # OK # Custom headers to proxied server {{ range $k, $v := $all.ProxySetHeaders }} - {{ $proxySetHeader }} {{ $k }} {{ $v | quote }}; - {{ end }} + {{ $proxySetHeader }} {{ $k }} {{ $v | quote }}; # OK + {{ end }} # 1 - proxy_connect_timeout {{ $location.Proxy.ConnectTimeout }}s; - proxy_send_timeout {{ $location.Proxy.SendTimeout }}s; - proxy_read_timeout {{ $location.Proxy.ReadTimeout }}s; + proxy_connect_timeout {{ $location.Proxy.ConnectTimeout }}s; OK + proxy_send_timeout {{ $location.Proxy.SendTimeout }}s; # OK + proxy_read_timeout {{ $location.Proxy.ReadTimeout }}s; # OK - proxy_buffering {{ $location.Proxy.ProxyBuffering }}; - proxy_buffer_size {{ $location.Proxy.BufferSize }}; - proxy_buffers {{ $location.Proxy.BuffersNumber }} {{ $location.Proxy.BufferSize }}; + proxy_buffering {{ $location.Proxy.ProxyBuffering }}; # OK + proxy_buffer_size {{ $location.Proxy.BufferSize }}; # OK + proxy_buffers {{ $location.Proxy.BuffersNumber }} {{ $location.Proxy.BufferSize }}; # OK {{ if isValidByteSize $location.Proxy.ProxyMaxTempFileSize true }} - proxy_max_temp_file_size {{ $location.Proxy.ProxyMaxTempFileSize }}; + proxy_max_temp_file_size {{ $location.Proxy.ProxyMaxTempFileSize }}; # OK {{ end }} - proxy_request_buffering {{ $location.Proxy.RequestBuffering }}; - proxy_http_version {{ $location.Proxy.ProxyHTTPVersion }}; + proxy_request_buffering {{ $location.Proxy.RequestBuffering }}; # OK + proxy_http_version {{ $location.Proxy.ProxyHTTPVersion }}; # OK - proxy_cookie_domain {{ $location.Proxy.CookieDomain }}; - proxy_cookie_path {{ $location.Proxy.CookiePath }}; + proxy_cookie_domain {{ $location.Proxy.CookieDomain }}; # OK + proxy_cookie_path {{ $location.Proxy.CookiePath }}; # OK # In case of errors try the next upstream server before returning an error - proxy_next_upstream {{ buildNextUpstream $location.Proxy.NextUpstream $all.Cfg.RetryNonIdempotent }}; - proxy_next_upstream_timeout {{ $location.Proxy.NextUpstreamTimeout }}; - proxy_next_upstream_tries {{ $location.Proxy.NextUpstreamTries }}; + proxy_next_upstream {{ buildNextUpstream $location.Proxy.NextUpstream $all.Cfg.RetryNonIdempotent }}; # OK + proxy_next_upstream_timeout {{ $location.Proxy.NextUpstreamTimeout }}; # OK + proxy_next_upstream_tries {{ $location.Proxy.NextUpstreamTries }}; # OK {{ if or (eq $location.BackendProtocol "GRPC") (eq $location.BackendProtocol "GRPCS") }} # Grpc settings - grpc_connect_timeout {{ $location.Proxy.ConnectTimeout }}s; - grpc_send_timeout {{ $location.Proxy.SendTimeout }}s; - grpc_read_timeout {{ $location.Proxy.ReadTimeout }}s; - {{ end }} + grpc_connect_timeout {{ $location.Proxy.ConnectTimeout }}s; # OK + grpc_send_timeout {{ $location.Proxy.SendTimeout }}s; # OK + grpc_read_timeout {{ $location.Proxy.ReadTimeout }}s; # OK + {{ end }} # 1 - {{/* Add any additional configuration defined */}} - {{ $location.ConfigurationSnippet }} - - {{ if not (empty $all.Cfg.LocationSnippet) }} - # Custom code snippet configured in the configuration configmap - {{ $all.Cfg.LocationSnippet }} - {{ end }} - - {{ if $location.CustomHeaders }} + + {{ if $location.CustomHeaders }} # 2 # Custom Response Headers - {{ range $k, $v := $location.CustomHeaders.Headers }} - more_set_headers {{ printf "%s: %s" $k $v | escapeLiteralDollar | quote }}; - {{ end }} - {{ end }} + {{ range $k, $v := $location.CustomHeaders.Headers }} # 3 + more_set_headers {{ printf "%s: %s" $k $v | escapeLiteralDollar | quote }}; # OK + {{ end }} # 2 + {{ end }} # 1 {{/* if we are sending the request to a custom default backend, we add the required headers */}} {{ if (hasPrefix $location.Backend "custom-default-backend-") }} - proxy_set_header X-Code 503; - proxy_set_header X-Format $http_accept; - proxy_set_header X-Namespace $namespace; - proxy_set_header X-Ingress-Name $ingress_name; - proxy_set_header X-Service-Name $service_name; - proxy_set_header X-Service-Port $service_port; - proxy_set_header X-Request-ID $req_id; - {{ end }} + proxy_set_header X-Code 503; # OK + proxy_set_header X-Format $http_accept; # OK + proxy_set_header X-Namespace $namespace; # OK + proxy_set_header X-Ingress-Name $ingress_name; # OK + proxy_set_header X-Service-Name $service_name; # OK + proxy_set_header X-Service-Port $service_port; # OK + proxy_set_header X-Request-ID $req_id; # OK + {{ end }} # 1 {{ if $location.Satisfy }} - satisfy {{ $location.Satisfy }}; - {{ end }} + satisfy {{ $location.Satisfy }}; # OK + {{ end }} # 1 {{/* if a location-specific error override is set, add the proxy_intercept here */}} {{ if and $location.CustomHTTPErrors (not $location.DisableProxyInterceptErrors) }} # Custom error pages per ingress - proxy_intercept_errors on; - {{ end }} + proxy_intercept_errors on; # OK + {{ end }} # 1 {{ range $errCode := $location.CustomHTTPErrors }} - error_page {{ $errCode }} = @custom_{{ $location.DefaultBackendUpstreamName }}_{{ $errCode }};{{ end }} + error_page {{ $errCode }} = @custom_{{ $location.DefaultBackendUpstreamName }}_{{ $errCode }};{{ end }} # OK {{ if (eq $location.BackendProtocol "FCGI") }} - include /etc/nginx/fastcgi_params; - {{ end }} + include /etc/nginx/fastcgi_params; OK + {{ end }} # 1 {{- if $location.FastCGI.Index -}} - fastcgi_index {{ $location.FastCGI.Index | quote }}; - {{- end -}} + fastcgi_index {{ $location.FastCGI.Index | quote }}; # OK + {{- end -}} # 1 {{ range $k, $v := $location.FastCGI.Params }} - fastcgi_param {{ $k }} {{ $v | quote }}; - {{ end }} + fastcgi_param {{ $k }} {{ $v | quote }}; # OK + {{ end }} # 1 {{ if not (empty $location.Redirect.URL) }} - return {{ $location.Redirect.Code }} {{ $location.Redirect.URL }}; - {{ end }} + return {{ $location.Redirect.Code }} {{ $location.Redirect.URL }}; # OK + {{ end }} # 1 - {{ buildProxyPass $server.Hostname $all.Backends $location }} + {{ buildProxyPass $server.Hostname $all.Backends $location }} # OK {{ if (or (eq $location.Proxy.ProxyRedirectFrom "default") (eq $location.Proxy.ProxyRedirectFrom "off")) }} - proxy_redirect {{ $location.Proxy.ProxyRedirectFrom }}; - {{ else if not (eq $location.Proxy.ProxyRedirectTo "off") }} - proxy_redirect {{ $location.Proxy.ProxyRedirectFrom }} {{ $location.Proxy.ProxyRedirectTo }}; - {{ end }} + proxy_redirect {{ $location.Proxy.ProxyRedirectFrom }}; # OK + {{ else if not (eq $location.Proxy.ProxyRedirectTo "off") }} + proxy_redirect {{ $location.Proxy.ProxyRedirectFrom }} {{ $location.Proxy.ProxyRedirectTo }}; # OK + {{ end }} # 1 {{ else }} - # Location denied. Reason: {{ $location.Denied | quote }} - return 503; - {{ end }} - {{ if not (empty $location.ProxySSL.CAFileName) }} - # PEM sha: {{ $location.ProxySSL.CASHA }} - proxy_ssl_trusted_certificate {{ $location.ProxySSL.CAFileName }}; - proxy_ssl_ciphers {{ $location.ProxySSL.Ciphers }}; - proxy_ssl_protocols {{ $location.ProxySSL.Protocols }}; - proxy_ssl_verify {{ $location.ProxySSL.Verify }}; - proxy_ssl_verify_depth {{ $location.ProxySSL.VerifyDepth }}; - {{ end }} + # Location denied. Reason: {{ $location.Denied | quote }} # OK + return 503; # OK + {{ end }} # 0 + {{ if not (empty $location.ProxySSL.CAFileName) }} # 1 + # PEM sha: {{ $location.ProxySSL.CASHA }} # OK + proxy_ssl_trusted_certificate {{ $location.ProxySSL.CAFileName }}; # OK + proxy_ssl_ciphers {{ $location.ProxySSL.Ciphers }}; # OK + proxy_ssl_protocols {{ $location.ProxySSL.Protocols }}; # OK + proxy_ssl_verify {{ $location.ProxySSL.Verify }}; # OK + proxy_ssl_verify_depth {{ $location.ProxySSL.VerifyDepth }}; # OK + {{ end }} # 0 - {{ if not (empty $location.ProxySSL.ProxySSLName) }} - proxy_ssl_name {{ $location.ProxySSL.ProxySSLName }}; - {{ end }} - {{ if not (empty $location.ProxySSL.ProxySSLServerName) }} - proxy_ssl_server_name {{ $location.ProxySSL.ProxySSLServerName }}; - {{ end }} + {{ if not (empty $location.ProxySSL.ProxySSLName) }} # 1 + proxy_ssl_name {{ $location.ProxySSL.ProxySSLName }}; # OK + {{ end }} # 0 + {{ if not (empty $location.ProxySSL.ProxySSLServerName) }} # 1 + proxy_ssl_server_name {{ $location.ProxySSL.ProxySSLServerName }}; # OK + {{ end }} # 0 - {{ if not (empty $location.ProxySSL.PemFileName) }} - proxy_ssl_certificate {{ $location.ProxySSL.PemFileName }}; - proxy_ssl_certificate_key {{ $location.ProxySSL.PemFileName }}; - {{ end }} + {{ if not (empty $location.ProxySSL.PemFileName) }} # 1 + proxy_ssl_certificate {{ $location.ProxySSL.PemFileName }}; # OK + proxy_ssl_certificate_key {{ $location.ProxySSL.PemFileName }}; # OK + {{ end }} # 0 } - {{ end }} - {{ end }} + {{ end }} # End range + {{ end }} # Maybe that missing if... {{ if eq $server.Hostname "_" }} # health checks in cloud providers require the use of port {{ $all.ListenPorts.HTTP }} - location {{ $all.HealthzURI }} { + location {{ $all.HealthzURI }} { # OK {{ if $all.Cfg.EnableOpentelemetry }} - opentelemetry off; + opentelemetry off; # OK {{ end }} - access_log off; - return 200; + access_log off; # OK + return 200; # OK } # this is required to avoid error if nginx is being monitored @@ -1397,21 +1246,21 @@ stream { location /nginx_status { {{ if $all.Cfg.EnableOpentelemetry }} - opentelemetry off; + opentelemetry off; # OK {{ end }} {{ range $v := $all.NginxStatusIpv4Whitelist }} - allow {{ $v }}; + allow {{ $v }}; # OK {{ end }} {{ if $all.IsIPV6Enabled -}} {{ range $v := $all.NginxStatusIpv6Whitelist }} - allow {{ $v }}; + allow {{ $v }}; # OK {{ end }} {{ end -}} - deny all; + deny all; # OK - access_log off; - stub_status on; + access_log off; # OK + stub_status on; # OK } {{ end }} diff --git a/internal/ingress/controller/template/crossplane/utils.go b/internal/ingress/controller/template/crossplane/utils.go index 2b1b3a2df..ea0ade3c9 100644 --- a/internal/ingress/controller/template/crossplane/utils.go +++ b/internal/ingress/controller/template/crossplane/utils.go @@ -17,18 +17,72 @@ limitations under the License. package crossplane import ( + "crypto/sha1" + "encoding/base64" + "encoding/hex" "fmt" "net" + "net/url" + "regexp" "strconv" + "strings" ngx_crossplane "github.com/nginxinc/nginx-go-crossplane" + networkingv1 "k8s.io/api/networking/v1" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/klog/v2" + "k8s.io/utils/ptr" + "k8s.io/ingress-nginx/internal/ingress/annotations/parser" + "k8s.io/ingress-nginx/internal/ingress/annotations/ratelimit" "k8s.io/ingress-nginx/internal/ingress/controller/config" ing_net "k8s.io/ingress-nginx/internal/net" "k8s.io/ingress-nginx/pkg/apis/ingress" ) +const ( + slash = "/" + nonIdempotent = "non_idempotent" + defBufferSize = 65535 + writeIndentOnEmptyLines = true // backward-compatibility + httpProtocol = "HTTP" + autoHTTPProtocol = "AUTO_HTTP" + httpsProtocol = "HTTPS" + grpcProtocol = "GRPC" + grpcsProtocol = "GRPCS" + fcgiProtocol = "FCGI" +) + +var ( + nginxSizeRegex = regexp.MustCompile(`^\d+[kKmM]?$`) + nginxOffsetRegex = regexp.MustCompile(`^\d+[kKmMgG]?$`) + defaultGlobalAuthRedirectParam = "rd" +) + type seconds int +type minutes int + +func buildDirectiveWithComment(directive string, comment string, args ...any) *ngx_crossplane.Directive { + dir := buildDirective(directive, args...) + dir.Comment = ptr.To(comment) + return dir +} + +func buildStartServer(name string) *ngx_crossplane.Directive { + return buildDirective("##", "start", "server", name) +} + +func buildEndServer(name string) *ngx_crossplane.Directive { + return buildDirective("##", "end", "server", name) +} + +func buildStartAuthUpstream(name, location string) *ngx_crossplane.Directive { + return buildDirective("##", "start", "auth", "upstream", name, location) +} + +func buildEndAuthUpstream(name, location string) *ngx_crossplane.Directive { + return buildDirective("##", "end", "auth", "upstream", name, location) +} func buildDirective(directive string, args ...any) *ngx_crossplane.Directive { argsVal := make([]string, 0) @@ -36,6 +90,10 @@ func buildDirective(directive string, args ...any) *ngx_crossplane.Directive { switch v := args[k].(type) { case string: argsVal = append(argsVal, v) + case *string: + if v != nil { + argsVal = append(argsVal, *v) + } case []string: argsVal = append(argsVal, v...) case int: @@ -44,6 +102,8 @@ func buildDirective(directive string, args ...any) *ngx_crossplane.Directive { argsVal = append(argsVal, boolToStr(v)) case seconds: argsVal = append(argsVal, strconv.Itoa(int(v))+"s") + case minutes: + argsVal = append(argsVal, strconv.Itoa(int(v))+"m") } } return &ngx_crossplane.Directive{ @@ -141,3 +201,538 @@ func shouldLoadOpentelemetryModule(servers []*ingress.Server) bool { } return false } + +func buildServerName(hostname string) string { + if !strings.HasPrefix(hostname, "*") { + return hostname + } + + hostname = strings.Replace(hostname, "*.", "", 1) + parts := strings.Split(hostname, ".") + + return `~^(?[\w-]+)\.` + strings.Join(parts, "\\.") + `$` +} + +func buildListener(tc config.TemplateConfig, hostname string) ngx_crossplane.Directives { + listenDirectives := make(ngx_crossplane.Directives, 0) + + co := commonListenOptions(&tc, hostname) + + addrV4 := []string{""} + if len(tc.Cfg.BindAddressIpv4) > 0 { + addrV4 = tc.Cfg.BindAddressIpv4 + } + listenDirectives = append(listenDirectives, httpListener(addrV4, co, &tc, false)...) + listenDirectives = append(listenDirectives, httpListener(addrV4, co, &tc, true)...) + + if tc.IsIPV6Enabled { + addrV6 := []string{"[::]"} + if len(tc.Cfg.BindAddressIpv6) > 0 { + addrV6 = tc.Cfg.BindAddressIpv6 + } + listenDirectives = append(listenDirectives, httpListener(addrV6, co, &tc, false)...) + listenDirectives = append(listenDirectives, httpListener(addrV6, co, &tc, true)...) + } + + return listenDirectives +} + +// commonListenOptions defines the common directives that should be added to NGINX listeners +func commonListenOptions(template *config.TemplateConfig, hostname string) []string { + var out []string + + if template.Cfg.UseProxyProtocol { + out = append(out, "proxy_protocol") + } + + if hostname != "_" { + return out + } + + out = append(out, "default_server") + + if template.Cfg.ReusePort { + out = append(out, "reuseport") + } + out = append(out, fmt.Sprintf("backlog=%d", template.BacklogSize)) + return out +} + +func httpListener(addresses []string, co []string, tc *config.TemplateConfig, ssl bool) ngx_crossplane.Directives { + listeners := make(ngx_crossplane.Directives, 0) + port := tc.ListenPorts.HTTP + isTLSProxy := tc.IsSSLPassthroughEnabled + // If this is a SSL listener we should mutate the port properly + if ssl { + port = tc.ListenPorts.HTTPS + if isTLSProxy { + port = tc.ListenPorts.SSLProxy + } + } + for _, address := range addresses { + var listenAddress string + if address == "" { + listenAddress = fmt.Sprintf("%d", port) + } else { + listenAddress = fmt.Sprintf("%s:%d", address, port) + } + if ssl { + if isTLSProxy { + co = append(co, "proxy_protocol") + } + co = append(co, "ssl") + } + listenDirective := buildDirective("listen", listenAddress, co) + listeners = append(listeners, listenDirective) + } + + return listeners +} + +func luaConfigurationRequestBodySize(cfg config.Configuration) string { + size := cfg.LuaSharedDicts["configuration_data"] + if size < cfg.LuaSharedDicts["certificate_data"] { + size = cfg.LuaSharedDicts["certificate_data"] + } + size += 1024 + + return dictKbToStr(size) +} + +func buildLocation(location *ingress.Location, enforceRegex bool) []string { + path := location.Path + if enforceRegex { + return []string{"~*", fmt.Sprintf("^%s", path)} + } + + if location.PathType != nil && *location.PathType == networkingv1.PathTypeExact { + return []string{"=", path} + } + + return []string{path} +} + +func getProxySetHeader(location *ingress.Location) string { + if location.BackendProtocol == grpcProtocol || location.BackendProtocol == grpcsProtocol { + return "grpc_set_header" + } + + return "proxy_set_header" +} + +func buildAuthLocation(location *ingress.Location, globalExternalAuthURL string) string { + if (location.ExternalAuth.URL == "") && (!shouldApplyGlobalAuth(location, globalExternalAuthURL)) { + return "" + } + + str := base64.URLEncoding.EncodeToString([]byte(location.Path)) + // removes "=" after encoding + str = strings.ReplaceAll(str, "=", "") + + pathType := "default" + if location.PathType != nil { + pathType = string(*location.PathType) + } + + return fmt.Sprintf("/_external-auth-%v-%v", str, pathType) +} + +// shouldApplyGlobalAuth returns true only in case when ExternalAuth.URL is not set and +// GlobalExternalAuth is set and enabled +func shouldApplyGlobalAuth(location *ingress.Location, globalExternalAuthURL string) bool { + return location.ExternalAuth.URL == "" && + globalExternalAuthURL != "" && + location.EnableGlobalAuth +} + +// shouldApplyAuthUpstream returns true only in case when ExternalAuth.URL and +// ExternalAuth.KeepaliveConnections are all set +func shouldApplyAuthUpstream(location *ingress.Location, cfg config.Configuration) bool { + if location.ExternalAuth.URL == "" || location.ExternalAuth.KeepaliveConnections == 0 { + return false + } + + // Unfortunately, `auth_request` module ignores keepalive in upstream block: https://trac.nginx.org/nginx/ticket/1579 + // The workaround is to use `ngx.location.capture` Lua subrequests but it is not supported with HTTP/2 + if cfg.UseHTTP2 { + return false + } + return true +} + +func isValidByteSize(s string, isOffset bool) bool { + s = strings.TrimSpace(s) + if s == "" { + return false + } + + if isOffset { + return nginxOffsetRegex.MatchString(s) + } + + return nginxSizeRegex.MatchString(s) +} + +func buildAuthUpstreamName(input *ingress.Location, host string) string { + authPath := buildAuthLocation(input, "") + if authPath == "" || host == "" { + return "" + } + + return fmt.Sprintf("%s-%s", host, authPath[2:]) +} + +// changeHostPort will change the host:port part of the url to value +func changeHostPort(newURL, value string) string { + if newURL == "" { + return "" + } + + authURL, err := parser.StringToURL(newURL) + if err != nil { + klog.Errorf("expected a valid URL but %s was returned", newURL) + return "" + } + + authURL.Host = value + + return authURL.String() +} + +func buildAuthSignURLLocation(location, authSignURL string) string { + hasher := sha1.New() // #nosec + hasher.Write([]byte(location)) + hasher.Write([]byte(authSignURL)) + return "@" + hex.EncodeToString(hasher.Sum(nil)) +} + +func buildAuthSignURL(authSignURL, authRedirectParam string) string { + u, err := url.Parse(authSignURL) + if err != nil { + klog.Errorf("error parsing authSignURL: %v", err) + return "" + } + q := u.Query() + if authRedirectParam == "" { + authRedirectParam = defaultGlobalAuthRedirectParam + } + if len(q) == 0 { + return fmt.Sprintf("%v?%v=$pass_access_scheme://$http_host$escaped_request_uri", authSignURL, authRedirectParam) + } + + if q.Get(authRedirectParam) != "" { + return authSignURL + } + + return fmt.Sprintf("%v&%v=$pass_access_scheme://$http_host$escaped_request_uri", authSignURL, authRedirectParam) +} + +func buildCorsOriginRegex(corsOrigins []string) ngx_crossplane.Directives { + if len(corsOrigins) == 1 && corsOrigins[0] == "*" { + return ngx_crossplane.Directives{ + buildDirective("set", "$http_origin", "*"), + buildDirective("set", "$cors", "true"), + } + } + + originsArray := []string{"("} + for i, origin := range corsOrigins { + originTrimmed := strings.TrimSpace(origin) + if originTrimmed != "" { + originsArray = append(originsArray, buildOriginRegex(originTrimmed)) + } + if i != len(corsOrigins)-1 { + originsArray = append(originsArray, "|") + } + } + originsArray = append(originsArray, ")$") + + // originsArray should be converted to a single string, as it is a single directive for if. + origins := strings.Join(originsArray, "") + return ngx_crossplane.Directives{ + buildBlockDirective("if", []string{"$http_origin", "~*", origins}, ngx_crossplane.Directives{ + buildDirective("set", "$cors", "true"), + }), + } +} + +func buildOriginRegex(origin string) string { + origin = regexp.QuoteMeta(origin) + origin = strings.Replace(origin, "\\*", `[A-Za-z0-9\-]+`, 1) + return fmt.Sprintf("(%s)", origin) +} + +func buildNextUpstream(nextUpstream string, retryNonIdempotent bool) []string { + parts := strings.Split(nextUpstream, " ") + + nextUpstreamCodes := make([]string, 0, len(parts)) + for _, v := range parts { + if v != "" && v != nonIdempotent { + nextUpstreamCodes = append(nextUpstreamCodes, v) + } + + if v == nonIdempotent { + retryNonIdempotent = true + } + } + + if retryNonIdempotent { + nextUpstreamCodes = append(nextUpstreamCodes, nonIdempotent) + } + + return nextUpstreamCodes +} + +func buildProxyPass(backends []*ingress.Backend, location *ingress.Location) ngx_crossplane.Directives { + path := location.Path + proto := "http://" + proxyPass := "proxy_pass" + + switch strings.ToUpper(location.BackendProtocol) { + case autoHTTPProtocol: + proto = "$scheme://" + case httpsProtocol: + proto = "https://" + case grpcProtocol: + proto = "grpc://" + proxyPass = "grpc_pass" + case grpcsProtocol: + proto = "grpcs://" + proxyPass = "grpc_pass" + case fcgiProtocol: + proto = "" + proxyPass = "fastcgi_pass" + } + + upstreamName := "upstream_balancer" + + for _, backend := range backends { + if backend.Name == location.Backend { + if backend.SSLPassthrough { + proto = "https://" + + if location.BackendProtocol == grpcsProtocol { + proto = "grpcs://" + } + } + + break + } + } + + if location.Backend == "upstream-default-backend" { + proto = "http://" + proxyPass = "proxy_pass" + } + + // defProxyPass returns the default proxy_pass, just the name of the upstream + defProxyPass := buildDirective(proxyPass, fmt.Sprintf("%s%s", proto, upstreamName)) + + // if the path in the ingress rule is equals to the target: no special rewrite + if path == location.Rewrite.Target { + return ngx_crossplane.Directives{defProxyPass} + } + + if location.Rewrite.Target != "" { + proxySetHeader := "proxy_set_header" + dir := make(ngx_crossplane.Directives, 0) + if location.BackendProtocol == grpcProtocol || location.BackendProtocol == grpcsProtocol { + proxySetHeader = "grpc_set_header" + } + + if location.XForwardedPrefix != "" { + dir = append(dir, + buildDirective(proxySetHeader, "X-Forwarded-Prefix", location.XForwardedPrefix), + ) + } + + dir = append(dir, + buildDirective("rewrite", fmt.Sprintf("(?i)%s", path), location.Rewrite.Target, "break"), + buildDirective(proxyPass, fmt.Sprintf("%s%s", proto, upstreamName)), + ) + return dir + } + + // default proxy_pass + return ngx_crossplane.Directives{defProxyPass} +} + +func buildGeoIPDirectives(reloadTime int, files []string) ngx_crossplane.Directives { + + directives := make(ngx_crossplane.Directives, 0) + buildGeoIPBlock := func(file string, directives ngx_crossplane.Directives) *ngx_crossplane.Directive { + if reloadTime > 0 && file != "GeoIP2-Connection-Type.mmdb" { + directives = append(directives, buildDirective("auto_reload", minutes(reloadTime))) + } + fileName := fmt.Sprintf("/etc/ingress-controller/geoip/%s", file) + return buildBlockDirective("geoip2", []string{fileName}, directives) + } + + for _, file := range files { + if file == "GeoLite2-Country.mmdb" || file == "GeoIP2-Country.mmdb" { + directives = append(directives, buildGeoIPBlock(file, ngx_crossplane.Directives{ + buildDirective("$geoip2_country_code", "source=$remote_addr", "country", "iso_code"), + buildDirective("$geoip2_country_name", "source=$remote_addr", "country", "names", "en"), + buildDirective("$geoip2_country_geoname_id", "source=$remote_addr", "country", "geoname_id"), + buildDirective("$geoip2_continent_code", "source=$remote_addr", "continent", "code"), + buildDirective("$geoip2_continent_name", "source=$remote_addr", "continent", "names", "en"), + buildDirective("$geoip2_continent_geoname_id", "source=$remote_addr", "continent", "geoname_id"), + })) + } + if file == "GeoLite2-City.mmdb" || file == "GeoIP2-City.mmdb" { + directives = append(directives, buildGeoIPBlock(file, ngx_crossplane.Directives{ + buildDirective("$geoip2_city_country_code", "source=$remote_addr", "country", "iso_code"), + buildDirective("$geoip2_city_country_name", "source=$remote_addr", "country", "names", "en"), + buildDirective("$geoip2_city_country_geoname_id", "source=$remote_addr", "country", "geoname_id"), + buildDirective("$geoip2_city_continent_code", "source=$remote_addr", "continent", "code"), + buildDirective("$geoip2_city_continent_name", "source=$remote_addr", "continent", "names", "en"), + buildDirective("$geoip2_city", "source=$remote_addr", "city", "names", "en"), + buildDirective("$geoip2_city_geoname_id", "source=$remote_addr", "city", "geoname_id"), + buildDirective("$geoip2_postal_code", "source=$remote_addr", "postal", "code"), + buildDirective("$geoip2_dma_code", "source=$remote_addr", "location", "metro_code"), + buildDirective("$geoip2_latitude", "source=$remote_addr", "location", "latitude"), + buildDirective("$geoip2_longitude", "source=$remote_addr", "location", "longitude"), + buildDirective("$geoip2_time_zone", "source=$remote_addr", "location", "time_zone"), + buildDirective("$geoip2_region_code", "source=$remote_addr", "subdivisions", "0", "iso_code"), + buildDirective("$geoip2_region_name", "source=$remote_addr", "subdivisions", "0", "names", "en"), + buildDirective("$geoip2_region_geoname_id", "source=$remote_addr", "subdivisions", "0", "geoname_id"), + buildDirective("$geoip2_subregion_code", "source=$remote_addr", "subdivisions", "1", "iso_code"), + buildDirective("$geoip2_subregion_name", "source=$remote_addr", "subdivisions", "1", "names", "en"), + buildDirective("$geoip2_subregion_geoname_id", "source=$remote_addr", "subdivisions", "1", "geoname_id"), + })) + } + if file == "GeoLite2-ASN.mmdb" || file == "GeoIP2-ASN.mmdb" { + directives = append(directives, buildGeoIPBlock(file, ngx_crossplane.Directives{ + buildDirective("$geoip2_asn", "source=$remote_addr", "autonomous_system_number"), + buildDirective("$geoip2_org", "source=$remote_addr", "autonomous_system_organization"), + })) + } + if file == "GeoIP2-ISP.mmdb" { + directives = append(directives, buildGeoIPBlock(file, ngx_crossplane.Directives{ + buildDirective("$geoip2_isp", "source=$remote_addr", "isp"), + buildDirective("$geoip2_isp_org", "source=$remote_addr", "organization"), + buildDirective("$geoip2_asn", "source=$remote_addr", "autonomous_system_number"), + })) + } + if file == "GeoIP2-Anonymous-IP.mmdb" { + directives = append(directives, buildGeoIPBlock(file, ngx_crossplane.Directives{ + buildDirective("$geoip2_is_anon", "source=$remote_addr", "is_anonymous"), + buildDirective("$geoip2_is_anonymous", "source=$remote_addr", "default=0", "is_anonymous"), + buildDirective("$geoip2_is_anonymous_vpn", "source=$remote_addr", "default=0", "is_anonymous_vpn"), + buildDirective("$geoip2_is_hosting_provider", "source=$remote_addr", "default=0", "is_hosting_provider"), + buildDirective("$geoip2_is_public_proxy", "source=$remote_addr", "default=0", "is_public_proxy"), + buildDirective("$geoip2_is_tor_exit_node", "source=$remote_addr", "default=0", "is_tor_exit_node"), + })) + } + if file == "GeoIP2-Connection-Type.mmdb" { + directives = append(directives, buildGeoIPBlock(file, ngx_crossplane.Directives{ + buildDirective("$geoip2_connection_type", "connection_type"), + })) + } + } + return directives +} + +func filterRateLimits(servers []*ingress.Server) []ratelimit.Config { + ratelimits := []ratelimit.Config{} + found := sets.Set[string]{} + + for _, server := range servers { + for _, loc := range server.Locations { + if loc.RateLimit.ID != "" && !found.Has(loc.RateLimit.ID) { + found.Insert(loc.RateLimit.ID) + ratelimits = append(ratelimits, loc.RateLimit) + } + } + } + return ratelimits +} + +// buildRateLimitZones produces an array of limit_conn_zone in order to allow +// rate limiting of request. Each Ingress rule could have up to three zones, one +// for connection limit by IP address, one for limiting requests per minute, and +// one for limiting requests per second. +func buildRateLimitZones(servers []*ingress.Server) ngx_crossplane.Directives { + zones := make(map[string]bool) + directives := make(ngx_crossplane.Directives, 0) + for _, server := range servers { + for _, loc := range server.Locations { + zoneID := fmt.Sprintf("$limit_%s", loc.RateLimit.ID) + if loc.RateLimit.Connections.Limit > 0 { + zoneArg := fmt.Sprintf("zone=%s:%dm", loc.RateLimit.Connections.Name, loc.RateLimit.Connections.SharedSize) + zone := fmt.Sprintf("limit_conn_zone %s %s", zoneID, zoneArg) + if _, ok := zones[zone]; !ok { + zones[zone] = true + directives = append(directives, buildDirective("limit_conn_zone", zoneID, zoneArg)) + } + } + + if loc.RateLimit.RPM.Limit > 0 { + zoneArg := fmt.Sprintf("zone=%s:%dm", loc.RateLimit.RPM.Name, loc.RateLimit.RPM.SharedSize) + zoneRate := fmt.Sprintf("rate=%dr/m", loc.RateLimit.RPM.Limit) + zone := fmt.Sprintf("limit_req_zone %s %s %s", zoneID, zoneArg, zoneRate) + if _, ok := zones[zone]; !ok { + zones[zone] = true + directives = append(directives, buildDirective("limit_req_zone", zoneID, zoneArg, zoneRate)) + } + } + + if loc.RateLimit.RPS.Limit > 0 { + zoneArg := fmt.Sprintf("zone=%s:%dm", loc.RateLimit.RPS.Name, loc.RateLimit.RPS.SharedSize) + zoneRate := fmt.Sprintf("rate=%dr/s", loc.RateLimit.RPS.Limit) + zone := fmt.Sprintf("limit_req_zone %s %s %s", zoneID, zoneArg, zoneRate) + if _, ok := zones[zone]; !ok { + zones[zone] = true + directives = append(directives, buildDirective("limit_req_zone", zoneID, zoneArg, zoneRate)) + } + } + } + } + return directives +} + +// buildAuthResponseHeaders sets HTTP response headers when `auth-url` is used. +// Based on `auth-keepalive` value we use auth_request_set Nginx directives, or +// we use Lua and Nginx variables instead. +// +// NOTE: Unfortunately auth_request module ignores the keepalive directive (see: +// https://trac.nginx.org/nginx/ticket/1579), that is why we mimic the same +// functionality with access_by_lua_block. +// TODO: This function is duplicated with the non-crossplane and we should consolidate +func buildAuthResponseHeaders(proxySetHeader string, headers []string, lua bool) ngx_crossplane.Directives { + res := make(ngx_crossplane.Directives, 0) + + if len(headers) == 0 { + return res + } + + for i, h := range headers { + authHeader := fmt.Sprintf("$authHeader%d", i) + if lua { + res = append(res, buildDirective("set", authHeader, "")) + } else { + hvar := strings.ToLower(h) + hvar = strings.NewReplacer("-", "_").Replace(hvar) + res = append(res, buildDirective("auth_request_set", + authHeader, + fmt.Sprintf("$upstream_http_%s", hvar))) + } + res = append(res, buildDirective(proxySetHeader, h, authHeader)) + } + return res +} + +// extractHostPort will extract the host:port part from the URL specified by url +func extractHostPort(newURL string) string { + if newURL == "" { + return "" + } + + authURL, err := parser.StringToURL(newURL) + if err != nil { + klog.Errorf("expected a valid URL but %s was returned", newURL) + return "" + } + + return authURL.Host +} diff --git a/test/e2e-image/e2e.sh b/test/e2e-image/e2e.sh index f8ecd5337..f793c7681 100755 --- a/test/e2e-image/e2e.sh +++ b/test/e2e-image/e2e.sh @@ -24,7 +24,7 @@ E2E_CHECK_LEAKS=${E2E_CHECK_LEAKS:-""} reportFile="report-e2e-test-suite.xml" ginkgo_args=( - "--fail-fast" + # "--fail-fast" "--flake-attempts=2" "--junit-report=${reportFile}" "--nodes=${E2E_NODES}" diff --git a/test/e2e/admission/admission.go b/test/e2e/admission/admission.go index 873e6719d..870df21b2 100644 --- a/test/e2e/admission/admission.go +++ b/test/e2e/admission/admission.go @@ -100,6 +100,9 @@ var _ = framework.IngressNginxDescribeSerial("[Admission] admission controller", }) ginkgo.It("should return an error if there is an error validating the ingress definition", func() { + if framework.IsCrossplane() { + ginkgo.Skip("Crossplane does not support snippets") + } disableSnippet := f.AllowSnippetConfiguration() defer disableSnippet() @@ -208,6 +211,9 @@ var _ = framework.IngressNginxDescribeSerial("[Admission] admission controller", }) ginkgo.It("should return an error if the Ingress V1 definition contains invalid annotations", func() { + if framework.IsCrossplane() { + ginkgo.Skip("Crissplane does not support snippets") + } disableSnippet := f.AllowSnippetConfiguration() defer disableSnippet() @@ -222,6 +228,9 @@ var _ = framework.IngressNginxDescribeSerial("[Admission] admission controller", }) ginkgo.It("should not return an error for an invalid Ingress when it has unknown class", func() { + if framework.IsCrossplane() { + ginkgo.Skip("Crossplane does not support snippets") + } disableSnippet := f.AllowSnippetConfiguration() defer disableSnippet() out, err := createIngress(f.Namespace, invalidV1IngressWithOtherClass) diff --git a/test/e2e/annotations/affinity.go b/test/e2e/annotations/affinity.go index b64581ef6..d2adc86a5 100644 --- a/test/e2e/annotations/affinity.go +++ b/test/e2e/annotations/affinity.go @@ -58,7 +58,7 @@ var _ = framework.DescribeAnnotation("affinity session-cookie-name", func() { f.WaitForNginxServer(host, func(server string) bool { - return strings.Contains(server, fmt.Sprintf("server_name %s ;", host)) + return strings.Contains(server, fmt.Sprintf("server_name %s;", host)) }) f.HTTPTestClient(). @@ -80,7 +80,7 @@ var _ = framework.DescribeAnnotation("affinity session-cookie-name", func() { f.WaitForNginxServer(host, func(server string) bool { - return strings.Contains(server, fmt.Sprintf("server_name %s ;", host)) + return strings.Contains(server, fmt.Sprintf("server_name %s;", host)) }) f.HTTPTestClient(). @@ -115,7 +115,7 @@ var _ = framework.DescribeAnnotation("affinity session-cookie-name", func() { f.WaitForNginxServer(host, func(server string) bool { - return strings.Contains(server, fmt.Sprintf("server_name %s ;", host)) + return strings.Contains(server, fmt.Sprintf("server_name %s;", host)) }) f.HTTPTestClient(). @@ -181,7 +181,7 @@ var _ = framework.DescribeAnnotation("affinity session-cookie-name", func() { f.WaitForNginxServer(host, func(server string) bool { - return strings.Contains(server, fmt.Sprintf("server_name %s ;", host)) + return strings.Contains(server, fmt.Sprintf("server_name %s;", host)) }) f.HTTPTestClient(). @@ -212,7 +212,7 @@ var _ = framework.DescribeAnnotation("affinity session-cookie-name", func() { f.WaitForNginxServer(host, func(server string) bool { - return strings.Contains(server, fmt.Sprintf("server_name %s ;", host)) + return strings.Contains(server, fmt.Sprintf("server_name %s;", host)) }) local, err := time.LoadLocation("GMT") @@ -243,7 +243,7 @@ var _ = framework.DescribeAnnotation("affinity session-cookie-name", func() { f.WaitForNginxServer(host, func(server string) bool { - return strings.Contains(server, fmt.Sprintf("server_name %s ;", host)) + return strings.Contains(server, fmt.Sprintf("server_name %s;", host)) }) f.HTTPTestClient(). @@ -265,7 +265,7 @@ var _ = framework.DescribeAnnotation("affinity session-cookie-name", func() { f.WaitForNginxServer(host, func(server string) bool { - return strings.Contains(server, fmt.Sprintf("server_name %s ;", host)) + return strings.Contains(server, fmt.Sprintf("server_name %s;", host)) }) f.HTTPTestClient(). @@ -289,7 +289,7 @@ var _ = framework.DescribeAnnotation("affinity session-cookie-name", func() { f.WaitForNginxServer(host, func(server string) bool { - return strings.Contains(server, fmt.Sprintf("server_name %s ;", host)) + return strings.Contains(server, fmt.Sprintf("server_name %s;", host)) }) f.HTTPTestClient(). @@ -312,7 +312,7 @@ var _ = framework.DescribeAnnotation("affinity session-cookie-name", func() { f.WaitForNginxServer(host, func(server string) bool { - return strings.Contains(server, fmt.Sprintf("server_name %s ;", host)) + return strings.Contains(server, fmt.Sprintf("server_name %s;", host)) }) f.HTTPTestClient(). @@ -393,7 +393,8 @@ var _ = framework.DescribeAnnotation("affinity session-cookie-name", func() { f.WaitForNginxServer(host, func(server string) bool { // server alias sort by sort.Strings(), see: internal/ingress/annotations/alias/main.go:60 - return strings.Contains(server, fmt.Sprintf("server_name %s %s %s ;", host, alias1, alias2)) + return strings.Contains(server, fmt.Sprintf("server_name %s %s %s ;", host, alias1, alias2)) || + strings.Contains(server, fmt.Sprintf("server_name %s %s %s;", host, alias1, alias2)) }) f.HTTPTestClient(). @@ -430,7 +431,7 @@ var _ = framework.DescribeAnnotation("affinity session-cookie-name", func() { f.WaitForNginxServer(host, func(server string) bool { - return strings.Contains(server, fmt.Sprintf("server_name %s ;", host)) + return strings.Contains(server, fmt.Sprintf("server_name %s;", host)) }) f.HTTPTestClient(). @@ -453,7 +454,7 @@ var _ = framework.DescribeAnnotation("affinity session-cookie-name", func() { f.WaitForNginxServer(host, func(server string) bool { - return strings.Contains(server, fmt.Sprintf("server_name %s ;", host)) + return strings.Contains(server, fmt.Sprintf("server_name %s;", host)) }) f.HTTPTestClient(). @@ -475,7 +476,7 @@ var _ = framework.DescribeAnnotation("affinity session-cookie-name", func() { f.WaitForNginxServer(host, func(server string) bool { - return strings.Contains(server, fmt.Sprintf("server_name %s ;", host)) && + return strings.Contains(server, fmt.Sprintf("server_name %s;", host)) && strings.Contains(server, "listen 443") }) diff --git a/test/e2e/annotations/affinitymode.go b/test/e2e/annotations/affinitymode.go index e6253b6ff..a49a76721 100644 --- a/test/e2e/annotations/affinitymode.go +++ b/test/e2e/annotations/affinitymode.go @@ -56,7 +56,7 @@ var _ = framework.DescribeAnnotation("affinitymode", func() { f.WaitForNginxServer(host, func(server string) bool { - return strings.Contains(server, fmt.Sprintf("server_name %s ;", host)) + return strings.Contains(server, fmt.Sprintf("server_name %s;", host)) }) // Check configuration @@ -89,7 +89,7 @@ var _ = framework.DescribeAnnotation("affinitymode", func() { f.WaitForNginxServer(host, func(server string) bool { - return strings.Contains(server, fmt.Sprintf("server_name %s ;", host)) + return strings.Contains(server, fmt.Sprintf("server_name %s;", host)) }) // Check configuration diff --git a/test/e2e/annotations/auth.go b/test/e2e/annotations/auth.go index ddda1dce5..fd1734f35 100644 --- a/test/e2e/annotations/auth.go +++ b/test/e2e/annotations/auth.go @@ -270,6 +270,9 @@ var _ = framework.DescribeAnnotation("auth-*", func() { }) ginkgo.It(`should set snippet "proxy_set_header My-Custom-Header 42;" when external auth is configured`, func() { + if framework.IsCrossplane() { + ginkgo.Skip("crossplane does not support snippets") + } host := authHost annotations := map[string]string{ @@ -290,6 +293,9 @@ var _ = framework.DescribeAnnotation("auth-*", func() { }) ginkgo.It(`should not set snippet "proxy_set_header My-Custom-Header 42;" when external auth is not configured`, func() { + if framework.IsCrossplane() { + ginkgo.Skip("crossplane does not support snippets") + } host := authHost disableSnippet := f.AllowSnippetConfiguration() defer disableSnippet() @@ -325,7 +331,8 @@ var _ = framework.DescribeAnnotation("auth-*", func() { f.WaitForNginxServer(host, func(server string) bool { - return strings.Contains(server, `proxy_set_header 'My-Custom-Header' '42';`) + return strings.Contains(server, `proxy_set_header 'My-Custom-Header' '42';`) || + strings.Contains(server, `proxy_set_header My-Custom-Header 42;`) }) }) @@ -531,7 +538,8 @@ http { f.UpdateIngress(ing) f.WaitForNginxServer(host, func(server string) bool { - return strings.Contains(server, fmt.Sprintf("proxy_set_header '%s' $authHeader0;", rewriteHeader)) + return strings.Contains(server, fmt.Sprintf("proxy_set_header '%s' $authHeader0;", rewriteHeader)) || + strings.Contains(server, fmt.Sprintf("proxy_set_header %s $authHeader0;", rewriteHeader)) }) f.HTTPTestClient(). @@ -893,6 +901,9 @@ http { }) ginkgo.It("should add error to the config", func() { + if framework.IsCrossplane() { + ginkgo.Skip("crossplane does not allows injecting invalid configuration") + } f.WaitForNginxServer(host, func(server string) bool { return strings.Contains(server, "could not parse auth-url annotation: invalid url host") }) diff --git a/test/e2e/annotations/canary.go b/test/e2e/annotations/canary.go index ea733dbf4..443a3a486 100644 --- a/test/e2e/annotations/canary.go +++ b/test/e2e/annotations/canary.go @@ -1091,13 +1091,22 @@ var _ = framework.DescribeAnnotation("canary-*", func() { f.WaitForNginxServer("_", func(server string) bool { - upstreamName := fmt.Sprintf(`set $proxy_upstream_name "%s-%s-%s";`, f.Namespace, framework.HTTPBunService, "80") - canaryUpstreamName := fmt.Sprintf(`set $proxy_upstream_name "%s-%s-%s";`, f.Namespace, canaryService, "80") + upstreamName := fmt.Sprintf(`set $proxy_upstream_name "%s-%s-80";`, f.Namespace, framework.HTTPBunService) + upstreamNameCrossplane := fmt.Sprintf(`set $proxy_upstream_name %s-%s-80;`, f.Namespace, framework.HTTPBunService) - return strings.Contains(server, fmt.Sprintf(`set $ingress_name "%v";`, host)) && + canaryUpstreamName := fmt.Sprintf(`set $proxy_upstream_name "%s-%s-80";`, f.Namespace, canaryService) + canaryUpstreamNameCrossplane := fmt.Sprintf(`set $proxy_upstream_name %s-%s-80;`, f.Namespace, canaryService) + + return (strings.Contains(server, fmt.Sprintf(`set $ingress_name "%v";`, host)) && !strings.Contains(server, `set $proxy_upstream_name "upstream-default-backend";`) && !strings.Contains(server, canaryUpstreamName) && - strings.Contains(server, upstreamName) + strings.Contains(server, upstreamName)) || + // Crossplane assertion + (strings.Contains(server, fmt.Sprintf(`set $ingress_name %s;`, host)) && + !strings.Contains(server, `set $proxy_upstream_name "pstream-default-backend;`) && + !strings.Contains(server, canaryUpstreamNameCrossplane) && + strings.Contains(server, upstreamNameCrossplane)) + }) }) diff --git a/test/e2e/annotations/cors.go b/test/e2e/annotations/cors.go index 58f4445f7..1db7d5374 100644 --- a/test/e2e/annotations/cors.go +++ b/test/e2e/annotations/cors.go @@ -48,13 +48,21 @@ var _ = framework.DescribeAnnotation("cors-*", func() { f.WaitForNginxServer(host, func(server string) bool { - return strings.Contains(server, "more_set_headers 'Access-Control-Allow-Methods: GET, PUT, POST, DELETE, PATCH, OPTIONS';") && + return (strings.Contains(server, "more_set_headers 'Access-Control-Allow-Methods: GET, PUT, POST, DELETE, PATCH, OPTIONS';") && strings.Contains(server, "more_set_headers 'Access-Control-Allow-Origin: $http_origin';") && strings.Contains(server, "more_set_headers 'Access-Control-Allow-Headers: DNT,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,Authorization';") && strings.Contains(server, "more_set_headers 'Access-Control-Max-Age: 1728000';") && strings.Contains(server, "more_set_headers 'Access-Control-Allow-Credentials: true';") && strings.Contains(server, "set $http_origin *;") && - strings.Contains(server, "$cors 'true';") + strings.Contains(server, "$cors 'true';")) || + // Assertions for crossplane mode + (strings.Contains(server, `more_set_headers "Access-Control-Allow-Methods: GET, PUT, POST, DELETE, PATCH, OPTIONS";`) && + strings.Contains(server, `more_set_headers "Access-Control-Allow-Origin: $http_origin";`) && + strings.Contains(server, `more_set_headers "Access-Control-Allow-Headers: DNT,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,Authorization";`) && + strings.Contains(server, `more_set_headers "Access-Control-Max-Age: 1728000";`) && + strings.Contains(server, `more_set_headers "Access-Control-Allow-Credentials: true";`) && + strings.Contains(server, "set $http_origin *;") && + strings.Contains(server, "$cors true;")) }) f.HTTPTestClient(). @@ -76,7 +84,9 @@ var _ = framework.DescribeAnnotation("cors-*", func() { f.WaitForNginxServer(host, func(server string) bool { - return strings.Contains(server, "more_set_headers 'Access-Control-Allow-Methods: POST, GET';") + return strings.Contains(server, "more_set_headers 'Access-Control-Allow-Methods: POST, GET';") || + strings.Contains(server, `more_set_headers "Access-Control-Allow-Methods: POST, GET";`) + }) }) @@ -92,7 +102,8 @@ var _ = framework.DescribeAnnotation("cors-*", func() { f.WaitForNginxServer(host, func(server string) bool { - return strings.Contains(server, "more_set_headers 'Access-Control-Max-Age: 200';") + return strings.Contains(server, "more_set_headers 'Access-Control-Max-Age: 200';") || + strings.Contains(server, `more_set_headers "Access-Control-Max-Age: 200";`) }) }) @@ -151,7 +162,8 @@ var _ = framework.DescribeAnnotation("cors-*", func() { f.WaitForNginxServer(host, func(server string) bool { - return strings.Contains(server, "more_set_headers 'Access-Control-Allow-Headers: DNT, User-Agent';") + return strings.Contains(server, "more_set_headers 'Access-Control-Allow-Headers: DNT, User-Agent';") || + strings.Contains(server, `more_set_headers "Access-Control-Allow-Headers: DNT, User-Agent";`) }) }) @@ -167,7 +179,8 @@ var _ = framework.DescribeAnnotation("cors-*", func() { f.WaitForNginxServer(host, func(server string) bool { - return strings.Contains(server, "more_set_headers 'Access-Control-Expose-Headers: X-CustomResponseHeader, X-CustomSecondHeader';") + return strings.Contains(server, "more_set_headers 'Access-Control-Expose-Headers: X-CustomResponseHeader, X-CustomSecondHeader';") || + strings.Contains(server, `more_set_headers "Access-Control-Expose-Headers: X-CustomResponseHeader, X-CustomSecondHeader";`) }) }) diff --git a/test/e2e/annotations/fastcgi.go b/test/e2e/annotations/fastcgi.go index bcf1c3487..b4684faef 100644 --- a/test/e2e/annotations/fastcgi.go +++ b/test/e2e/annotations/fastcgi.go @@ -64,7 +64,8 @@ var _ = framework.DescribeAnnotation("backend-protocol - FastCGI", func() { f.WaitForNginxServer(host, func(server string) bool { - return strings.Contains(server, "fastcgi_index \"index.php\";") + return strings.Contains(server, "fastcgi_index \"index.php\";") || + strings.Contains(server, "fastcgi_index index.php;") }) }) @@ -94,8 +95,10 @@ var _ = framework.DescribeAnnotation("backend-protocol - FastCGI", func() { f.WaitForNginxServer(host, func(server string) bool { - return strings.Contains(server, "fastcgi_param SCRIPT_FILENAME \"$fastcgi_script_name\";") && - strings.Contains(server, "fastcgi_param REDIRECT_STATUS \"200\";") + return (strings.Contains(server, "fastcgi_param SCRIPT_FILENAME \"$fastcgi_script_name\";") && + strings.Contains(server, "fastcgi_param REDIRECT_STATUS \"200\";")) || + (strings.Contains(server, "fastcgi_param SCRIPT_FILENAME $fastcgi_script_name;") && + strings.Contains(server, "fastcgi_param REDIRECT_STATUS 200;")) }) }) diff --git a/test/e2e/annotations/fromtowwwredirect.go b/test/e2e/annotations/fromtowwwredirect.go index a3fb3b9b5..2f56c2315 100644 --- a/test/e2e/annotations/fromtowwwredirect.go +++ b/test/e2e/annotations/fromtowwwredirect.go @@ -62,17 +62,20 @@ var _ = framework.DescribeAnnotation("from-to-www-redirect", func() { }) ginkgo.It("should redirect from www HTTPS to HTTPS", func() { - disableSnippet := f.AllowSnippetConfiguration() - defer disableSnippet() - ginkgo.By("setting up server for redirect from www") + h := make(map[string]string) + h["ExpectedHost"] = "$http_host" + cfgMap := "add-headers-configmap" + + f.CreateConfigMap(cfgMap, h) + f.UpdateNginxConfigMapData("add-headers", fmt.Sprintf("%s/%s", f.Namespace, cfgMap)) + fromHost := fmt.Sprintf("%s.nip.io", f.GetNginxIP()) toHost := fmt.Sprintf("www.%s", fromHost) annotations := map[string]string{ - "nginx.ingress.kubernetes.io/from-to-www-redirect": "true", - "nginx.ingress.kubernetes.io/configuration-snippet": "more_set_headers \"ExpectedHost: $http_host\";", + "nginx.ingress.kubernetes.io/from-to-www-redirect": "true", } ing := framework.NewSingleIngressWithTLS(fromHost, "/", fromHost, []string{fromHost, toHost}, f.Namespace, framework.EchoService, 80, annotations) diff --git a/test/e2e/annotations/http2pushpreload.go b/test/e2e/annotations/http2pushpreload.go index b15d2df17..2c317eef8 100644 --- a/test/e2e/annotations/http2pushpreload.go +++ b/test/e2e/annotations/http2pushpreload.go @@ -25,6 +25,10 @@ import ( ) var _ = framework.DescribeAnnotation("http2-push-preload", func() { + if framework.IsCrossplane() { + // Http2 Push preload is removed from crossplane as it is deprecated + return + } f := framework.NewDefaultFramework("http2pushpreload") ginkgo.BeforeEach(func() { diff --git a/test/e2e/annotations/limitconnections.go b/test/e2e/annotations/limitconnections.go index 7d00b6df0..d44cb169c 100644 --- a/test/e2e/annotations/limitconnections.go +++ b/test/e2e/annotations/limitconnections.go @@ -41,7 +41,7 @@ var _ = framework.DescribeAnnotation("Annotation - limit-connections", func() { ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.SlowEchoService, 80, nil) f.EnsureIngress(ing) f.WaitForNginxServer(host, func(server string) bool { - return strings.Contains(server, fmt.Sprintf("server_name %s ;", host)) + return strings.Contains(server, fmt.Sprintf("server_name %s;", host)) }) // limit connections diff --git a/test/e2e/annotations/mirror.go b/test/e2e/annotations/mirror.go index 787cbfa3b..c6cd028c7 100644 --- a/test/e2e/annotations/mirror.go +++ b/test/e2e/annotations/mirror.go @@ -60,7 +60,8 @@ var _ = framework.DescribeAnnotation("mirror-*", func() { func(server string) bool { return strings.Contains(server, fmt.Sprintf("mirror /_mirror-%v;", ing.UID)) && strings.Contains(server, "mirror_request_body on;") && - strings.Contains(server, `proxy_pass "https://test.env.com/$request_uri";`) + (strings.Contains(server, `proxy_pass "https://test.env.com/$request_uri";`) || + strings.Contains(server, `proxy_pass https://test.env.com/$request_uri;`)) }) }) diff --git a/test/e2e/annotations/modsecurity/modsecurity.go b/test/e2e/annotations/modsecurity/modsecurity.go index 730fc76e7..606a50ede 100644 --- a/test/e2e/annotations/modsecurity/modsecurity.go +++ b/test/e2e/annotations/modsecurity/modsecurity.go @@ -38,6 +38,9 @@ const ( var _ = framework.DescribeAnnotation("modsecurity owasp", func() { f := framework.NewDefaultFramework("modsecuritylocation") + if framework.IsCrossplane() { + return + } ginkgo.BeforeEach(func() { f.NewEchoDeployment() diff --git a/test/e2e/annotations/rewrite.go b/test/e2e/annotations/rewrite.go index 173df29f0..416e0190e 100644 --- a/test/e2e/annotations/rewrite.go +++ b/test/e2e/annotations/rewrite.go @@ -97,8 +97,8 @@ var _ = framework.DescribeAnnotation("rewrite-target use-regex enable-rewrite-lo f.WaitForNginxServer(host, func(server string) bool { - return strings.Contains(server, `location ~* "^/" {`) && - strings.Contains(server, `location ~* "^/.well-known/acme/challenge" {`) + return (strings.Contains(server, `location ~* "^/" {`) || strings.Contains(server, `location ~* ^/ {`)) && + (strings.Contains(server, `location ~* "^/.well-known/acme/challenge" {`) || strings.Contains(server, `location ~* ^/.well-known/acme/challenge {`)) }) ginkgo.By("making a second request to the non-rewritten location") @@ -132,8 +132,8 @@ var _ = framework.DescribeAnnotation("rewrite-target use-regex enable-rewrite-lo f.WaitForNginxServer(host, func(server string) bool { - return strings.Contains(server, `location ~* "^/foo" {`) && - strings.Contains(server, `location ~* "^/foo.+" {`) + return (strings.Contains(server, `location ~* "^/foo" {`) || strings.Contains(server, `location ~* ^/foo {`)) && + (strings.Contains(server, `location ~* "^/foo.+" {`) || strings.Contains(server, `location ~* ^/foo.+ {`)) }) ginkgo.By("ensuring '/foo' matches '~* ^/foo'") @@ -174,7 +174,8 @@ var _ = framework.DescribeAnnotation("rewrite-target use-regex enable-rewrite-lo f.WaitForNginxServer(host, func(server string) bool { - return strings.Contains(server, `location ~* "^/foo/bar/bar" {`) && + return (strings.Contains(server, `location ~* "^/foo/bar/bar" {`) || + strings.Contains(server, `location ~* ^/foo/bar/bar {`)) && strings.Contains(server, `location ~* "^/foo/bar/[a-z]{3}" {`) }) @@ -202,7 +203,7 @@ var _ = framework.DescribeAnnotation("rewrite-target use-regex enable-rewrite-lo f.WaitForNginxServer(host, func(server string) bool { - return strings.Contains(server, `location ~* "^/foo/bar/(.+)" {`) + return strings.Contains(server, `location ~* "^/foo/bar/(.+)" {`) || strings.Contains(server, `location ~* ^/foo/bar/(.+) {`) }) ginkgo.By("check that '/foo/bar/bar' redirects to custom rewrite") diff --git a/test/e2e/annotations/serversnippet.go b/test/e2e/annotations/serversnippet.go index c94960a3d..0a6c09104 100644 --- a/test/e2e/annotations/serversnippet.go +++ b/test/e2e/annotations/serversnippet.go @@ -27,6 +27,9 @@ import ( var _ = framework.DescribeAnnotation("server-snippet", func() { f := framework.NewDefaultFramework("serversnippet") + if framework.IsCrossplane() { + return + } ginkgo.BeforeEach(func() { f.NewEchoDeployment() diff --git a/test/e2e/annotations/snippet.go b/test/e2e/annotations/snippet.go index 9e3160dcc..178db4202 100644 --- a/test/e2e/annotations/snippet.go +++ b/test/e2e/annotations/snippet.go @@ -30,6 +30,9 @@ var _ = framework.DescribeAnnotation("configuration-snippet", func() { "configurationsnippet", framework.WithHTTPBunEnabled(), ) + if framework.IsCrossplane() { + return + } ginkgo.It("set snippet more_set_headers in all locations", func() { host := "configurationsnippet.foo.com" diff --git a/test/e2e/annotations/streamsnippet.go b/test/e2e/annotations/streamsnippet.go index f91cdc34e..1a5f7fd5d 100644 --- a/test/e2e/annotations/streamsnippet.go +++ b/test/e2e/annotations/streamsnippet.go @@ -33,6 +33,9 @@ import ( var _ = framework.DescribeSetting("stream-snippet", func() { f := framework.NewDefaultFramework("stream-snippet") + if framework.IsCrossplane() { + return + } ginkgo.BeforeEach(func() { f.NewEchoDeployment() diff --git a/test/e2e/annotations/upstreamhashby.go b/test/e2e/annotations/upstreamhashby.go index 1b8106662..e5e3c5846 100644 --- a/test/e2e/annotations/upstreamhashby.go +++ b/test/e2e/annotations/upstreamhashby.go @@ -36,7 +36,7 @@ func startIngress(f *framework.Framework, annotations map[string]string) map[str f.EnsureIngress(ing) f.WaitForNginxServer(host, func(server string) bool { - return strings.Contains(server, fmt.Sprintf("server_name %s ;", host)) + return strings.Contains(server, fmt.Sprintf("server_name %s;", host)) }) //nolint:staticcheck // TODO: will replace it since wait.Poll is deprecated diff --git a/test/e2e/annotations/upstreamvhost.go b/test/e2e/annotations/upstreamvhost.go index e091e7c9f..fb763ba1d 100644 --- a/test/e2e/annotations/upstreamvhost.go +++ b/test/e2e/annotations/upstreamvhost.go @@ -42,7 +42,8 @@ var _ = framework.DescribeAnnotation("upstream-vhost", func() { f.WaitForNginxServer(host, func(server string) bool { - return strings.Contains(server, `proxy_set_header Host "upstreamvhost.bar.com";`) + return strings.Contains(server, `proxy_set_header Host "upstreamvhost.bar.com";`) || + strings.Contains(server, `proxy_set_header Host upstreamvhost.bar.com;`) }) }) }) diff --git a/test/e2e/annotations/xforwardedprefix.go b/test/e2e/annotations/xforwardedprefix.go index 35f2eb426..3ca449757 100644 --- a/test/e2e/annotations/xforwardedprefix.go +++ b/test/e2e/annotations/xforwardedprefix.go @@ -43,7 +43,8 @@ var _ = framework.DescribeAnnotation("x-forwarded-prefix", func() { f.WaitForNginxServer(host, func(server string) bool { return strings.Contains(server, host) && - strings.Contains(server, "proxy_set_header X-Forwarded-Prefix \"/test/value\";") + (strings.Contains(server, "proxy_set_header X-Forwarded-Prefix \"/test/value\";") || + strings.Contains(server, "proxy_set_header X-Forwarded-Prefix /test/value;")) }) f.HTTPTestClient(). diff --git a/test/e2e/defaultbackend/custom_default_backend.go b/test/e2e/defaultbackend/custom_default_backend.go index 1e30b9269..37b8c8ab2 100644 --- a/test/e2e/defaultbackend/custom_default_backend.go +++ b/test/e2e/defaultbackend/custom_default_backend.go @@ -47,7 +47,8 @@ var _ = framework.IngressNginxDescribe("[Default Backend] custom service", func( f.WaitForNginxServer("_", func(server string) bool { - return strings.Contains(server, `set $proxy_upstream_name "upstream-default-backend"`) + return strings.Contains(server, `set $proxy_upstream_name "upstream-default-backend"`) || + strings.Contains(server, `set $proxy_upstream_name upstream-default-backend`) }) f.HTTPTestClient(). diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go index 204da7df0..02cc08816 100644 --- a/test/e2e/framework/framework.go +++ b/test/e2e/framework/framework.go @@ -20,6 +20,7 @@ import ( "fmt" "net" "net/http" + "os" "strings" "time" @@ -119,6 +120,11 @@ func NewSimpleFramework(baseName string, opts ...func(*Framework)) *Framework { return f } +func IsCrossplane() bool { + isCrossplane, ok := os.LookupEnv("IS_CROSSPLANE") + return ok && isCrossplane == "true" +} + func (f *Framework) CreateEnvironment() { var err error @@ -315,7 +321,7 @@ func (f *Framework) matchNginxConditions(name string, matcher func(cfg string) b if name == "" { cmd = "cat /etc/nginx/nginx.conf" } else { - cmd = fmt.Sprintf("cat /etc/nginx/nginx.conf | awk '/## start server %v/,/## end server %v/'", name, name) + cmd = fmt.Sprintf("cat /etc/nginx/nginx.conf | awk '/## start server %s;/,/## end server %s;/'", name, name) } o, err := f.ExecCommand(f.pod, cmd) diff --git a/test/e2e/ingress/multiple_rules.go b/test/e2e/ingress/multiple_rules.go index 9247dc1d3..22ca08a96 100644 --- a/test/e2e/ingress/multiple_rules.go +++ b/test/e2e/ingress/multiple_rules.go @@ -17,11 +17,11 @@ limitations under the License. package ingress import ( + "fmt" "net/http" "strings" "github.com/onsi/ginkgo/v2" - "github.com/stretchr/testify/assert" networking "k8s.io/api/networking/v1" "k8s.io/ingress-nginx/test/e2e/framework" @@ -36,14 +36,19 @@ var _ = framework.IngressNginxDescribe("single ingress - multiple hosts", func() }) ginkgo.It("should set the correct $service_name NGINX variable", func() { - disableSnippet := f.AllowSnippetConfiguration() - defer disableSnippet() + customHeader := "Service-Name" + customHeaderValue := "$service_name" - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/configuration-snippet": `more_set_input_headers "service-name: $service_name";`, - } + h := make(map[string]string) + h[customHeader] = customHeaderValue - ing := framework.NewSingleIngress("simh", "/", "first.host", f.Namespace, "first-service", 80, annotations) + cfgMap := "custom-headers" + + f.CreateConfigMap(cfgMap, h) + + f.UpdateNginxConfigMapData("add-headers", fmt.Sprintf("%v/%v", f.Namespace, cfgMap)) + + ing := framework.NewSingleIngress("simh", "/", "first.host", f.Namespace, "first-service", 80, nil) ing.Spec.Rules = append(ing.Spec.Rules, networking.IngressRule{ Host: "second.host", @@ -79,26 +84,19 @@ var _ = framework.IngressNginxDescribe("single ingress - multiple hosts", func() return strings.Contains(server, "second.host") }) - body := f.HTTPTestClient(). + f.HTTPTestClient(). GET("/exact"). WithHeader("Host", "first.host"). Expect(). Status(http.StatusOK). - Body(). - Raw() + Headers().ValueEqual("Service-Name", []string{"first-service"}) - assert.Contains(ginkgo.GinkgoT(), body, "service-name=first-service") - assert.NotContains(ginkgo.GinkgoT(), body, "service-name=second-service") - - body = f.HTTPTestClient(). + f.HTTPTestClient(). GET("/exact"). WithHeader("Host", "second.host"). Expect(). Status(http.StatusOK). - Body(). - Raw() + Headers().ValueEqual("Service-Name", []string{"second-service"}) - assert.NotContains(ginkgo.GinkgoT(), body, "service-name=first-service") - assert.Contains(ginkgo.GinkgoT(), body, "service-name=second-service") }) }) diff --git a/test/e2e/ingress/pathtype_exact.go b/test/e2e/ingress/pathtype_exact.go index 2660e32a4..060ea9a31 100644 --- a/test/e2e/ingress/pathtype_exact.go +++ b/test/e2e/ingress/pathtype_exact.go @@ -21,7 +21,6 @@ import ( "strings" "github.com/onsi/ginkgo/v2" - "github.com/stretchr/testify/assert" networking "k8s.io/api/networking/v1" "k8s.io/ingress-nginx/test/e2e/framework" @@ -35,24 +34,31 @@ var _ = framework.IngressNginxDescribe("[Ingress] [PathType] exact", func() { }) ginkgo.It("should choose exact location for /exact", func() { - disableSnippet := f.AllowSnippetConfiguration() - defer disableSnippet() - host := "exact.path" + f.UpdateNginxConfigMapData("global-allowed-response-headers", "Pathtype,duplicated") annotations := map[string]string{ - "nginx.ingress.kubernetes.io/configuration-snippet": `more_set_input_headers "pathType: exact";`, + "nginx.ingress.kubernetes.io/custom-headers": f.Namespace + "/custom-headers-exact", } + f.CreateConfigMap("custom-headers-exact", map[string]string{ + "Pathtype": "exact", + }) + + host := "exact.path" exactPathType := networking.PathTypeExact ing := framework.NewSingleIngress("exact", "/exact", host, f.Namespace, framework.EchoService, 80, annotations) ing.Spec.Rules[0].IngressRuleValue.HTTP.Paths[0].PathType = &exactPathType f.EnsureIngress(ing) annotations = map[string]string{ - "nginx.ingress.kubernetes.io/configuration-snippet": `more_set_input_headers "pathType: prefix";`, + "nginx.ingress.kubernetes.io/custom-headers": f.Namespace + "/custom-headers-prefix", } + f.CreateConfigMap("custom-headers-prefix", map[string]string{ + "Pathtype": "prefix", + }) + ing = framework.NewSingleIngress("exact-suffix", "/exact", host, f.Namespace, framework.EchoService, 80, annotations) f.EnsureIngress(ing) @@ -63,34 +69,29 @@ var _ = framework.IngressNginxDescribe("[Ingress] [PathType] exact", func() { strings.Contains(server, "location /exact/") }) - body := f.HTTPTestClient(). + f.HTTPTestClient(). GET("/exact"). WithHeader("Host", host). Expect(). Status(http.StatusOK). - Body(). - Raw() + Headers().ValueEqual("Pathtype", []string{"exact"}) - assert.NotContains(ginkgo.GinkgoT(), body, "pathtype=prefix") - assert.Contains(ginkgo.GinkgoT(), body, "pathtype=exact") - - body = f.HTTPTestClient(). + f.HTTPTestClient(). GET("/exact/suffix"). WithHeader("Host", host). Expect(). Status(http.StatusOK). - Body(). - Raw() - - assert.Contains(ginkgo.GinkgoT(), body, "pathtype=prefix") + Headers().ValueEqual("Pathtype", []string{"prefix"}) annotations = map[string]string{ - "nginx.ingress.kubernetes.io/configuration-snippet": ` - more_set_input_headers "pathType: prefix"; - more_set_input_headers "duplicated: true"; - `, + "nginx.ingress.kubernetes.io/custom-headers": f.Namespace + "/custom-headers-duplicated", } + f.CreateConfigMap("custom-headers-duplicated", map[string]string{ + "Pathtype": "prefix", + "duplicated": "true", + }) + ing = framework.NewSingleIngress("duplicated-prefix", "/exact", host, f.Namespace, framework.EchoService, 80, annotations) f.EnsureIngress(ing) @@ -101,16 +102,12 @@ var _ = framework.IngressNginxDescribe("[Ingress] [PathType] exact", func() { strings.Contains(server, "location /exact/") }) - body = f.HTTPTestClient(). + f.HTTPTestClient(). GET("/exact/suffix"). WithHeader("Host", host). Expect(). Status(http.StatusOK). - Body(). - Raw() + Headers().ValueEqual("Pathtype", []string{"prefix"}).NotContainsKey("duplicated") - assert.Contains(ginkgo.GinkgoT(), body, "pathtype=prefix") - assert.NotContains(ginkgo.GinkgoT(), body, "pathtype=exact") - assert.NotContains(ginkgo.GinkgoT(), body, "duplicated=true") }) }) diff --git a/test/e2e/ingress/pathtype_mixed.go b/test/e2e/ingress/pathtype_mixed.go index 3212089c9..0573cc0f5 100644 --- a/test/e2e/ingress/pathtype_mixed.go +++ b/test/e2e/ingress/pathtype_mixed.go @@ -21,7 +21,6 @@ import ( "strings" "github.com/onsi/ginkgo/v2" - "github.com/stretchr/testify/assert" networking "k8s.io/api/networking/v1" "k8s.io/ingress-nginx/test/e2e/framework" @@ -37,21 +36,32 @@ var _ = framework.IngressNginxDescribe("[Ingress] [PathType] mix Exact and Prefi exactPathType := networking.PathTypeExact ginkgo.It("should choose the correct location", func() { - disableSnippet := f.AllowSnippetConfiguration() - defer disableSnippet() - host := "mixed.path" + f.UpdateNginxConfigMapData("global-allowed-response-headers", "Pathtype,Pathheader") + annotations := map[string]string{ - "nginx.ingress.kubernetes.io/configuration-snippet": `more_set_input_headers "pathType: exact";more_set_input_headers "pathheader: /";`, + "nginx.ingress.kubernetes.io/custom-headers": f.Namespace + "/custom-headers-exact", } + + f.CreateConfigMap("custom-headers-exact", map[string]string{ + "Pathtype": "exact", + "Pathheader": "/", + }) + ing := framework.NewSingleIngress("exact-root", "/", host, f.Namespace, framework.EchoService, 80, annotations) ing.Spec.Rules[0].IngressRuleValue.HTTP.Paths[0].PathType = &exactPathType f.EnsureIngress(ing) annotations = map[string]string{ - "nginx.ingress.kubernetes.io/configuration-snippet": `more_set_input_headers "pathType: prefix";more_set_input_headers "pathheader: /";`, + "nginx.ingress.kubernetes.io/custom-headers": f.Namespace + "/custom-headers-prefix", } + + f.CreateConfigMap("custom-headers-prefix", map[string]string{ + "Pathtype": "prefix", + "Pathheader": "/", + }) + ing = framework.NewSingleIngress("prefix-root", "/", host, f.Namespace, framework.EchoService, 80, annotations) f.EnsureIngress(ing) @@ -63,41 +73,42 @@ var _ = framework.IngressNginxDescribe("[Ingress] [PathType] mix Exact and Prefi }) ginkgo.By("Checking exact request to /") - body := f.HTTPTestClient(). + f.HTTPTestClient(). GET("/"). WithHeader("Host", host). Expect(). Status(http.StatusOK). - Body(). - Raw() - - assert.NotContains(ginkgo.GinkgoT(), body, "pathtype=prefix") - assert.Contains(ginkgo.GinkgoT(), body, "pathtype=exact") - assert.Contains(ginkgo.GinkgoT(), body, "pathheader=/") + Headers().ValueEqual("Pathtype", []string{"exact"}).ValueEqual("Pathheader", []string{"/"}) ginkgo.By("Checking prefix request to /bar") - body = f.HTTPTestClient(). + f.HTTPTestClient(). GET("/bar"). WithHeader("Host", host). Expect(). Status(http.StatusOK). - Body(). - Raw() - - assert.Contains(ginkgo.GinkgoT(), body, "pathtype=prefix") - assert.NotContains(ginkgo.GinkgoT(), body, "pathtype=exact") - assert.Contains(ginkgo.GinkgoT(), body, "pathheader=/") + Headers().ValueEqual("Pathtype", []string{"prefix"}).ValueEqual("Pathheader", []string{"/"}) annotations = map[string]string{ - "nginx.ingress.kubernetes.io/configuration-snippet": `more_set_input_headers "pathType: exact";more_set_input_headers "pathheader: /foo";`, + "nginx.ingress.kubernetes.io/custom-headers": f.Namespace + "/custom-headers-ex-foo", } + + f.CreateConfigMap("custom-headers-ex-foo", map[string]string{ + "Pathtype": "exact", + "Pathheader": "/foo", + }) ing = framework.NewSingleIngress("exact-foo", "/foo", host, f.Namespace, framework.EchoService, 80, annotations) ing.Spec.Rules[0].IngressRuleValue.HTTP.Paths[0].PathType = &exactPathType f.EnsureIngress(ing) annotations = map[string]string{ - "nginx.ingress.kubernetes.io/configuration-snippet": `more_set_input_headers "pathType: prefix";more_set_input_headers "pathheader: /foo";`, + "nginx.ingress.kubernetes.io/custom-headers": f.Namespace + "/custom-headers-pr-foo", } + + f.CreateConfigMap("custom-headers-pr-foo", map[string]string{ + "Pathtype": "prefix", + "Pathheader": "/foo", + }) + ing = framework.NewSingleIngress("prefix-foo", "/foo", host, f.Namespace, framework.EchoService, 80, annotations) f.EnsureIngress(ing) @@ -109,40 +120,28 @@ var _ = framework.IngressNginxDescribe("[Ingress] [PathType] mix Exact and Prefi }) ginkgo.By("Checking exact request to /foo") - body = f.HTTPTestClient(). + f.HTTPTestClient(). GET("/foo"). WithHeader("Host", host). Expect(). Status(http.StatusOK). - Body(). - Raw() - - assert.NotContains(ginkgo.GinkgoT(), body, "pathtype=prefix") - assert.Contains(ginkgo.GinkgoT(), body, "pathtype=exact") - assert.Contains(ginkgo.GinkgoT(), body, "pathheader=/foo") + Headers().ValueEqual("Pathtype", []string{"exact"}).ValueEqual("Pathheader", []string{"/foo"}) ginkgo.By("Checking prefix request to /foo/bar") - body = f.HTTPTestClient(). + f.HTTPTestClient(). GET("/foo/bar"). WithHeader("Host", host). Expect(). Status(http.StatusOK). - Body(). - Raw() - - assert.Contains(ginkgo.GinkgoT(), body, "pathtype=prefix") - assert.Contains(ginkgo.GinkgoT(), body, "pathheader=/foo") + Headers().ValueEqual("Pathtype", []string{"prefix"}).ValueEqual("Pathheader", []string{"/foo"}) ginkgo.By("Checking prefix request to /foobar") - body = f.HTTPTestClient(). + f.HTTPTestClient(). GET("/foobar"). WithHeader("Host", host). Expect(). Status(http.StatusOK). - Body(). - Raw() + Headers().ValueEqual("Pathtype", []string{"prefix"}).ValueEqual("Pathheader", []string{"/"}) - assert.Contains(ginkgo.GinkgoT(), body, "pathtype=prefix") - assert.Contains(ginkgo.GinkgoT(), body, "pathheader=/") }) }) diff --git a/test/e2e/ingress/without_host.go b/test/e2e/ingress/without_host.go index 38f89beda..24192a0b7 100644 --- a/test/e2e/ingress/without_host.go +++ b/test/e2e/ingress/without_host.go @@ -39,11 +39,17 @@ var _ = framework.IngressNginxDescribe("[Ingress] definition without host", func f.WaitForNginxServer("_", func(server string) bool { - return strings.Contains(server, fmt.Sprintf(`set $namespace "%v";`, f.Namespace)) && + return (strings.Contains(server, fmt.Sprintf(`set $namespace "%v";`, f.Namespace)) && strings.Contains(server, fmt.Sprintf(`set $ingress_name "%v";`, ing.Name)) && strings.Contains(server, fmt.Sprintf(`set $service_name "%v";`, framework.EchoService)) && strings.Contains(server, `set $service_port "80";`) && - strings.Contains(server, `set $location_path "/";`) + strings.Contains(server, `set $location_path "/";`)) || + // Crossplane assertions + (strings.Contains(server, fmt.Sprintf(`set $namespace %s;`, f.Namespace)) && + strings.Contains(server, fmt.Sprintf(`set $ingress_name %s;`, ing.Name)) && + strings.Contains(server, fmt.Sprintf(`set $service_name %s;`, framework.EchoService)) && + strings.Contains(server, `set $service_port 80;`) && + strings.Contains(server, `set $location_path /;`)) }) f.HTTPTestClient(). @@ -81,11 +87,17 @@ var _ = framework.IngressNginxDescribe("[Ingress] definition without host", func f.WaitForNginxServer("only-backend", func(server string) bool { - return strings.Contains(server, fmt.Sprintf(`set $namespace "%v";`, f.Namespace)) && + return (strings.Contains(server, fmt.Sprintf(`set $namespace "%v";`, f.Namespace)) && strings.Contains(server, fmt.Sprintf(`set $ingress_name "%v";`, ing.Name)) && strings.Contains(server, fmt.Sprintf(`set $service_name "%v";`, framework.EchoService)) && strings.Contains(server, `set $service_port "80";`) && - strings.Contains(server, `set $location_path "/";`) + strings.Contains(server, `set $location_path "/";`)) || + // Crossplane assertions + (strings.Contains(server, fmt.Sprintf(`set $namespace %s;`, f.Namespace)) && + strings.Contains(server, fmt.Sprintf(`set $ingress_name %s;`, ing.Name)) && + strings.Contains(server, fmt.Sprintf(`set $service_name %s;`, framework.EchoService)) && + strings.Contains(server, `set $service_port 80;`) && + strings.Contains(server, `set $location_path /;`)) }) f.HTTPTestClient(). diff --git a/test/e2e/lua/dynamic_configuration.go b/test/e2e/lua/dynamic_configuration.go index a5e2196ce..8f1deaeb1 100644 --- a/test/e2e/lua/dynamic_configuration.go +++ b/test/e2e/lua/dynamic_configuration.go @@ -212,7 +212,7 @@ func createIngress(f *framework.Framework, host, deploymentName string) { f.WaitForNginxServer(host, func(server string) bool { - return strings.Contains(server, fmt.Sprintf("server_name %s ;", host)) && + return strings.Contains(server, fmt.Sprintf("server_name %s;", host)) && strings.Contains(server, "proxy_pass http://upstream_balancer;") }) } diff --git a/test/e2e/metrics/metrics.go b/test/e2e/metrics/metrics.go index bec09bb37..3cdb88057 100644 --- a/test/e2e/metrics/metrics.go +++ b/test/e2e/metrics/metrics.go @@ -43,7 +43,7 @@ var _ = framework.IngressNginxDescribe("[metrics] exported prometheus metrics", f.EnsureIngress(framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, nil)) f.WaitForNginxServer(host, func(server string) bool { - return strings.Contains(server, fmt.Sprintf("server_name %s ;", host)) && + return strings.Contains(server, fmt.Sprintf("server_name %s;", host)) && strings.Contains(server, "proxy_pass http://upstream_balancer;") }) }) diff --git a/test/e2e/run-e2e-suite.sh b/test/e2e/run-e2e-suite.sh index 909368e96..621313610 100755 --- a/test/e2e/run-e2e-suite.sh +++ b/test/e2e/run-e2e-suite.sh @@ -78,6 +78,7 @@ kubectl run --rm \ --env="E2E_NODES=${E2E_NODES}" \ --env="FOCUS=${FOCUS}" \ --env="IS_CHROOT=${IS_CHROOT:-false}"\ + --env="IS_CROSSPLANE=${IS_CROSSPLANE:-false}"\ --env="SKIP_OPENTELEMETRY_TESTS=${SKIP_OPENTELEMETRY_TESTS:-false}"\ --env="E2E_CHECK_LEAKS=${E2E_CHECK_LEAKS}" \ --env="NGINX_BASE_IMAGE=${NGINX_BASE_IMAGE}" \ diff --git a/test/e2e/run-kind-e2e.sh b/test/e2e/run-kind-e2e.sh index afcfe1800..f66871952 100755 --- a/test/e2e/run-kind-e2e.sh +++ b/test/e2e/run-kind-e2e.sh @@ -39,6 +39,8 @@ fi KIND_LOG_LEVEL="1" IS_CHROOT="${IS_CHROOT:-false}" +IS_CROSSPLANE="${IS_CROSSPLANE:-false}" + export KIND_CLUSTER_NAME=${KIND_CLUSTER_NAME:-ingress-nginx-dev} DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" # Use 1.0.0-dev to make sure we use the latest configuration in the helm template diff --git a/test/e2e/security/request_smuggling.go b/test/e2e/security/request_smuggling.go index 5ede02d4b..d96ba4b4c 100644 --- a/test/e2e/security/request_smuggling.go +++ b/test/e2e/security/request_smuggling.go @@ -37,6 +37,9 @@ var _ = framework.IngressNginxDescribe("[Security] request smuggling", func() { }) ginkgo.It("should not return body content from error_page", func() { + if framework.IsCrossplane() { + ginkgo.Skip("Crossplane does not support snippets") // TODO: Re-add this test when we enable admin defined snippets + } host := "foo.bar.com" snippet := ` diff --git a/test/e2e/settings/access_log.go b/test/e2e/settings/access_log.go index 65b9dfda4..33466f29f 100644 --- a/test/e2e/settings/access_log.go +++ b/test/e2e/settings/access_log.go @@ -31,6 +31,10 @@ var _ = framework.DescribeSetting("access-log", func() { ginkgo.It("use the default configuration", func() { f.WaitForNginxConfiguration( func(cfg string) bool { + if framework.IsCrossplane() { + return strings.Contains(cfg, "access_log /var/log/nginx/access.log upstreaminfo") || + strings.Contains(cfg, "access_log syslog:server=127.0.0.1:11514 upstreaminfo") + } return (strings.Contains(cfg, "access_log /var/log/nginx/access.log upstreaminfo") && strings.Contains(cfg, "access_log /var/log/nginx/access.log log_stream")) || (strings.Contains(cfg, "access_log syslog:server=127.0.0.1:11514 upstreaminfo") && @@ -42,6 +46,9 @@ var _ = framework.DescribeSetting("access-log", func() { f.UpdateNginxConfigMapData("access-log-path", "/tmp/nginx/access.log") f.WaitForNginxConfiguration( func(cfg string) bool { + if framework.IsCrossplane() { + return strings.Contains(cfg, "access_log /tmp/nginx/access.log upstreaminfo") + } return strings.Contains(cfg, "access_log /tmp/nginx/access.log upstreaminfo") && strings.Contains(cfg, "access_log /tmp/nginx/access.log log_stream") }) @@ -53,6 +60,9 @@ var _ = framework.DescribeSetting("access-log", func() { f.UpdateNginxConfigMapData("http-access-log-path", "/tmp/nginx/http-access.log") f.WaitForNginxConfiguration( func(cfg string) bool { + if framework.IsCrossplane() { + return strings.Contains(cfg, "access_log /tmp/nginx/http-access.log upstreaminfo") + } return strings.Contains(cfg, "access_log /tmp/nginx/http-access.log upstreaminfo") && (strings.Contains(cfg, "access_log /var/log/nginx/access.log log_stream") || strings.Contains(cfg, "access_log syslog:server=127.0.0.1:11514 log_stream")) @@ -62,6 +72,9 @@ var _ = framework.DescribeSetting("access-log", func() { ginkgo.Context("stream-access-log-path", func() { ginkgo.It("use the specified configuration", func() { + if framework.IsCrossplane() { + ginkgo.Skip("Crossplane does not support stream") + } f.UpdateNginxConfigMapData("stream-access-log-path", "/tmp/nginx/stream-access.log") f.WaitForNginxConfiguration( func(cfg string) bool { @@ -74,6 +87,9 @@ var _ = framework.DescribeSetting("access-log", func() { ginkgo.Context("http-access-log-path & stream-access-log-path", func() { ginkgo.It("use the specified configuration", func() { + if framework.IsCrossplane() { + ginkgo.Skip("Crossplane does not support stream") + } f.SetNginxConfigMapData(map[string]string{ "http-access-log-path": "/tmp/nginx/http-access.log", "stream-access-log-path": "/tmp/nginx/stream-access.log", diff --git a/test/e2e/settings/badannotationvalues.go b/test/e2e/settings/badannotationvalues.go index aa9906909..bee12e66a 100644 --- a/test/e2e/settings/badannotationvalues.go +++ b/test/e2e/settings/badannotationvalues.go @@ -50,7 +50,7 @@ var _ = framework.DescribeAnnotation("Bad annotation values", func() { f.WaitForNginxServer(host, func(server string) bool { - return !strings.Contains(server, fmt.Sprintf("server_name %s ;", host)) + return !strings.Contains(server, fmt.Sprintf("server_name %s;", host)) }) f.WaitForNginxServer(host, @@ -87,7 +87,7 @@ var _ = framework.DescribeAnnotation("Bad annotation values", func() { f.WaitForNginxServer(host, func(server string) bool { - return !strings.Contains(server, fmt.Sprintf("server_name %s ;", host)) + return !strings.Contains(server, fmt.Sprintf("server_name %s;", host)) }) f.WaitForNginxServer(host, @@ -103,6 +103,9 @@ var _ = framework.DescribeAnnotation("Bad annotation values", func() { }) ginkgo.It("[BAD_ANNOTATIONS] should allow an ingress if there is a default blocklist config in place", func() { + if framework.IsCrossplane() { + ginkgo.Skip("Crossplane does not support snippets") + } disableSnippet := f.AllowSnippetConfiguration() defer disableSnippet() @@ -120,7 +123,7 @@ var _ = framework.DescribeAnnotation("Bad annotation values", func() { f.WaitForNginxServer(hostValid, func(server string) bool { - return strings.Contains(server, fmt.Sprintf("server_name %s ;", hostValid)) + return strings.Contains(server, fmt.Sprintf("server_name %s;", hostValid)) }) f.WaitForNginxServer(hostValid, @@ -153,7 +156,7 @@ var _ = framework.DescribeAnnotation("Bad annotation values", func() { f.WaitForNginxServer(host, func(server string) bool { - return !strings.Contains(server, fmt.Sprintf("server_name %s ;", host)) + return !strings.Contains(server, fmt.Sprintf("server_name %s;", host)) }) f.WaitForNginxServer(host, diff --git a/test/e2e/settings/configmap_change.go b/test/e2e/settings/configmap_change.go index 3e37b62cd..3b1878051 100644 --- a/test/e2e/settings/configmap_change.go +++ b/test/e2e/settings/configmap_change.go @@ -17,7 +17,6 @@ limitations under the License. package settings import ( - "regexp" "strings" "github.com/onsi/ginkgo/v2" @@ -43,36 +42,20 @@ var _ = framework.DescribeSetting("Configmap change", func() { f.UpdateNginxConfigMapData("whitelist-source-range", "1.1.1.1") - checksumRegex := regexp.MustCompile(`Configuration checksum:\s+(\d+)`) - checksum := "" - f.WaitForNginxConfiguration( func(cfg string) bool { - // before returning, extract the current checksum - match := checksumRegex.FindStringSubmatch(cfg) - if len(match) > 0 { - checksum = match[1] - } - return strings.Contains(cfg, "allow 1.1.1.1;") }) - assert.NotEmpty(ginkgo.GinkgoT(), checksum) ginkgo.By("changing error-log-level") f.UpdateNginxConfigMapData("error-log-level", "debug") - newChecksum := "" f.WaitForNginxConfiguration( func(cfg string) bool { - match := checksumRegex.FindStringSubmatch(cfg) - if len(match) > 0 { - newChecksum = match[1] - } - - return strings.ContainsAny(cfg, "error_log /var/log/nginx/error.log debug;") + return strings.ContainsAny(cfg, "error_log /var/log/nginx/error.log debug;") || + strings.ContainsAny(cfg, "error_log /var/log/nginx/error.log debug;") }) - assert.NotEqual(ginkgo.GinkgoT(), checksum, newChecksum) logs, err := f.NginxLogs() assert.Nil(ginkgo.GinkgoT(), err, "obtaining nginx logs") diff --git a/test/e2e/settings/custom_header.go b/test/e2e/settings/custom_header.go index 8341f7ef0..b2c8aac8d 100644 --- a/test/e2e/settings/custom_header.go +++ b/test/e2e/settings/custom_header.go @@ -79,8 +79,10 @@ var _ = framework.DescribeSetting("add-headers", func() { f.UpdateNginxConfigMapData("add-headers", fmt.Sprintf("%v/%v", f.Namespace, cfgMap)) f.WaitForNginxConfiguration(func(server string) bool { - return strings.Contains(server, fmt.Sprintf("more_set_headers \"%s: %s\";", firstCustomHeader, firstCustomHeaderValue)) && - strings.Contains(server, fmt.Sprintf("more_set_headers \"%s: %s\";", secondCustomHeader, secondCustomHeaderValue)) + return (strings.Contains(server, fmt.Sprintf(`more_set_headers "%s: %s";`, firstCustomHeader, firstCustomHeaderValue)) && + strings.Contains(server, fmt.Sprintf(`more_set_headers "%s: %s";`, secondCustomHeader, secondCustomHeaderValue))) || + (strings.Contains(server, fmt.Sprintf("more_set_headers %s: %s;", firstCustomHeader, firstCustomHeaderValue)) && + strings.Contains(server, fmt.Sprintf("more_set_headers %s: %s;", secondCustomHeader, secondCustomHeaderValue))) }) resp := f.HTTPTestClient(). diff --git a/test/e2e/settings/default_ssl_certificate.go b/test/e2e/settings/default_ssl_certificate.go index c48a1e87f..e8e219519 100644 --- a/test/e2e/settings/default_ssl_certificate.go +++ b/test/e2e/settings/default_ssl_certificate.go @@ -69,8 +69,10 @@ var _ = framework.IngressNginxDescribe("[SSL] [Flag] default-ssl-certificate", f ginkgo.By("making sure new ingress is deployed") expectedConfig := fmt.Sprintf(`set $proxy_upstream_name "%v-%v-%v";`, f.Namespace, service, port) + expectedConfigCrossPlane := fmt.Sprintf(`set $proxy_upstream_name %v-%v-%v;`, f.Namespace, service, port) + f.WaitForNginxServer("_", func(cfg string) bool { - return strings.Contains(cfg, expectedConfig) + return strings.Contains(cfg, expectedConfig) || strings.Contains(cfg, expectedConfigCrossPlane) }) ginkgo.By("making sure new ingress is responding") @@ -91,8 +93,9 @@ var _ = framework.IngressNginxDescribe("[SSL] [Flag] default-ssl-certificate", f ginkgo.By("making sure new ingress is deployed") expectedConfig := fmt.Sprintf(`set $proxy_upstream_name "%v-%v-%v";`, f.Namespace, service, port) + expectedConfigCrossPlane := fmt.Sprintf(`set $proxy_upstream_name %v-%v-%v;`, f.Namespace, service, port) f.WaitForNginxServer(host, func(cfg string) bool { - return strings.Contains(cfg, expectedConfig) + return strings.Contains(cfg, expectedConfig) || strings.Contains(cfg, expectedConfigCrossPlane) }) ginkgo.By("making sure the configured default ssl certificate is being used") diff --git a/test/e2e/settings/disable_catch_all.go b/test/e2e/settings/disable_catch_all.go index 4e7a16f4d..54fbe9d8b 100644 --- a/test/e2e/settings/disable_catch_all.go +++ b/test/e2e/settings/disable_catch_all.go @@ -62,7 +62,8 @@ var _ = framework.IngressNginxDescribe("[Flag] disable-catch-all", func() { f.WaitForNginxServer("_", func(cfg string) bool { return strings.Contains(cfg, `set $ingress_name ""`) && - strings.Contains(cfg, `set $proxy_upstream_name "upstream-default-backend"`) + (strings.Contains(cfg, `set $proxy_upstream_name "upstream-default-backend"`) || + strings.Contains(cfg, `set $proxy_upstream_name upstream-default-backend`)) }) }) @@ -74,7 +75,8 @@ var _ = framework.IngressNginxDescribe("[Flag] disable-catch-all", func() { f.WaitForNginxServer("_", func(cfg string) bool { return strings.Contains(cfg, `set $ingress_name ""`) && - strings.Contains(cfg, `set $proxy_upstream_name "upstream-default-backend"`) + (strings.Contains(cfg, `set $proxy_upstream_name "upstream-default-backend"`) || + strings.Contains(cfg, `set $proxy_upstream_name upstream-default-backend`)) }) }) diff --git a/test/e2e/settings/global_external_auth.go b/test/e2e/settings/global_external_auth.go index f589a63e9..1721c54a5 100644 --- a/test/e2e/settings/global_external_auth.go +++ b/test/e2e/settings/global_external_auth.go @@ -246,6 +246,9 @@ var _ = framework.DescribeSetting("[Security] global-auth-url", func() { }) ginkgo.It(`should set snippet when global external auth is configured`, func() { + if framework.IsCrossplane() { + ginkgo.Skip("crossplane does not support snippets") + } globalExternalAuthSnippetSetting := "global-auth-snippet" globalExternalAuthSnippet := "proxy_set_header My-Custom-Header 42;" diff --git a/test/e2e/settings/gzip.go b/test/e2e/settings/gzip.go index c7e580e07..b2972e69c 100644 --- a/test/e2e/settings/gzip.go +++ b/test/e2e/settings/gzip.go @@ -106,7 +106,7 @@ var _ = framework.DescribeSetting("gzip", func() { f.WaitForNginxConfiguration( func(cfg string) bool { return strings.Contains(cfg, "gzip on;") && - strings.Contains(cfg, `gzip_disable "msie6";`) + (strings.Contains(cfg, `gzip_disable "msie6";`) || strings.Contains(cfg, `gzip_disable msie6;`)) }, ) diff --git a/test/e2e/settings/limit_rate.go b/test/e2e/settings/limit_rate.go index 9d79dc358..16ce98277 100644 --- a/test/e2e/settings/limit_rate.go +++ b/test/e2e/settings/limit_rate.go @@ -41,7 +41,7 @@ var _ = framework.DescribeSetting("Configmap - limit-rate", func() { f.EnsureIngress(ing) f.WaitForNginxServer(host, func(server string) bool { - return strings.Contains(server, fmt.Sprintf("server_name %s ;", host)) + return strings.Contains(server, fmt.Sprintf("server_name %s;", host)) }) wlKey := "limit-rate" diff --git a/test/e2e/settings/main_snippet.go b/test/e2e/settings/main_snippet.go index ff14bab9d..50bf96cae 100644 --- a/test/e2e/settings/main_snippet.go +++ b/test/e2e/settings/main_snippet.go @@ -26,6 +26,9 @@ import ( var _ = framework.DescribeSetting("main-snippet", func() { f := framework.NewDefaultFramework("main-snippet") + if framework.IsCrossplane() { + return + } mainSnippet := "main-snippet" ginkgo.It("should add value of main-snippet setting to nginx config", func() { diff --git a/test/e2e/settings/modsecurity/modsecurity_snippet.go b/test/e2e/settings/modsecurity/modsecurity_snippet.go index 2dd92ced2..5e82ccf8e 100644 --- a/test/e2e/settings/modsecurity/modsecurity_snippet.go +++ b/test/e2e/settings/modsecurity/modsecurity_snippet.go @@ -26,6 +26,9 @@ import ( var _ = framework.DescribeSetting("[Security] modsecurity-snippet", func() { f := framework.NewDefaultFramework("modsecurity-snippet") + if framework.IsCrossplane() { + return + } ginkgo.It("should add value of modsecurity-snippet setting to nginx config", func() { expectedComment := "# modsecurity snippet" diff --git a/test/e2e/settings/no_tls_redirect_locations.go b/test/e2e/settings/no_tls_redirect_locations.go index 18fd09e26..b9ac1bb97 100644 --- a/test/e2e/settings/no_tls_redirect_locations.go +++ b/test/e2e/settings/no_tls_redirect_locations.go @@ -33,7 +33,7 @@ var _ = framework.DescribeSetting("Add no tls redirect locations", func() { f.EnsureIngress(ing) f.WaitForNginxConfiguration(func(server string) bool { - return strings.Contains(server, "set $force_no_ssl_redirect \"false\"") + return strings.Contains(server, "set $force_no_ssl_redirect \"false\"") || strings.Contains(server, "set $force_no_ssl_redirect false") }) wlKey := "no-tls-redirect-locations" @@ -42,7 +42,7 @@ var _ = framework.DescribeSetting("Add no tls redirect locations", func() { f.UpdateNginxConfigMapData(wlKey, wlValue) f.WaitForNginxConfiguration(func(server string) bool { - return strings.Contains(server, "set $force_no_ssl_redirect \"true\"") + return strings.Contains(server, "set $force_no_ssl_redirect \"true\"") || strings.Contains(server, "set $force_no_ssl_redirect true") }) }) }) diff --git a/test/e2e/settings/proxy_host.go b/test/e2e/settings/proxy_host.go index bb5dc9c01..1fcda11a1 100644 --- a/test/e2e/settings/proxy_host.go +++ b/test/e2e/settings/proxy_host.go @@ -34,14 +34,16 @@ var _ = framework.IngressNginxDescribe("Dynamic $proxy_host", func() { }) ginkgo.It("should exist a proxy_host", func() { - disableSnippet := f.AllowSnippetConfiguration() - defer disableSnippet() + + h := make(map[string]string) + h["Custom-Header"] = "$proxy_host" + cfgMap := "add-headers-configmap" + + f.CreateConfigMap(cfgMap, h) + f.UpdateNginxConfigMapData("add-headers", fmt.Sprintf("%s/%s", f.Namespace, cfgMap)) upstreamName := fmt.Sprintf("%v-%v-80", f.Namespace, framework.EchoService) - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/configuration-snippet": `more_set_headers "Custom-Header: $proxy_host"`, - } - f.EnsureIngress(framework.NewSingleIngress(test, "/", test, f.Namespace, framework.EchoService, 80, annotations)) + f.EnsureIngress(framework.NewSingleIngress(test, "/", test, f.Namespace, framework.EchoService, 80, nil)) f.WaitForNginxConfiguration( func(server string) bool { @@ -58,15 +60,20 @@ var _ = framework.IngressNginxDescribe("Dynamic $proxy_host", func() { }) ginkgo.It("should exist a proxy_host using the upstream-vhost annotation value", func() { - disableSnippet := f.AllowSnippetConfiguration() - defer disableSnippet() + + h := make(map[string]string) + h["Custom-Header"] = "$proxy_host" + cfgMap := "add-headers-configmap" + + f.CreateConfigMap(cfgMap, h) + f.UpdateNginxConfigMapData("add-headers", fmt.Sprintf("%s/%s", f.Namespace, cfgMap)) upstreamName := fmt.Sprintf("%v-%v-80", f.Namespace, framework.EchoService) upstreamVHost := "different.host" annotations := map[string]string{ - "nginx.ingress.kubernetes.io/upstream-vhost": upstreamVHost, - "nginx.ingress.kubernetes.io/configuration-snippet": `more_set_headers "Custom-Header: $proxy_host"`, + "nginx.ingress.kubernetes.io/upstream-vhost": upstreamVHost, } + f.EnsureIngress(framework.NewSingleIngress(test, "/", test, f.Namespace, framework.EchoService, 80, annotations)) f.WaitForNginxConfiguration( diff --git a/test/e2e/settings/proxy_protocol.go b/test/e2e/settings/proxy_protocol.go index cfce68bf8..4c2e2a1aa 100644 --- a/test/e2e/settings/proxy_protocol.go +++ b/test/e2e/settings/proxy_protocol.go @@ -162,6 +162,9 @@ var _ = framework.DescribeSetting("use-proxy-protocol", func() { }) ginkgo.It("should enable PROXY Protocol for TCP", func() { + if framework.IsCrossplane() { + return + } cmapData := map[string]string{} cmapData[setting] = "true" cmapData["enable-real-ip"] = "true" diff --git a/test/e2e/settings/server_snippet.go b/test/e2e/settings/server_snippet.go index 1e2084bd8..e162630db 100644 --- a/test/e2e/settings/server_snippet.go +++ b/test/e2e/settings/server_snippet.go @@ -28,6 +28,9 @@ import ( var _ = framework.DescribeSetting("configmap server-snippet", func() { f := framework.NewDefaultFramework("cm-server-snippet") + if framework.IsCrossplane() { + return + } ginkgo.BeforeEach(func() { f.NewEchoDeployment() }) diff --git a/test/e2e/settings/ssl_ciphers.go b/test/e2e/settings/ssl_ciphers.go index 241dfc92b..f80bf41a2 100644 --- a/test/e2e/settings/ssl_ciphers.go +++ b/test/e2e/settings/ssl_ciphers.go @@ -35,7 +35,7 @@ var _ = framework.DescribeSetting("ssl-ciphers", func() { f.UpdateNginxConfigMapData(wlKey, wlValue) f.WaitForNginxConfiguration(func(cfg string) bool { - return strings.Contains(cfg, fmt.Sprintf("ssl_ciphers '%s';", wlValue)) + return strings.Contains(cfg, fmt.Sprintf("ssl_ciphers '%s';", wlValue)) || strings.Contains(cfg, fmt.Sprintf("ssl_ciphers %s;", wlValue)) }) }) }) diff --git a/test/e2e/settings/ssl_passthrough.go b/test/e2e/settings/ssl_passthrough.go index b10511bde..111c0c776 100644 --- a/test/e2e/settings/ssl_passthrough.go +++ b/test/e2e/settings/ssl_passthrough.go @@ -35,6 +35,9 @@ import ( var _ = framework.IngressNginxDescribe("[Flag] enable-ssl-passthrough", func() { f := framework.NewDefaultFramework("ssl-passthrough", framework.WithHTTPBunEnabled()) + if framework.IsCrossplane() { + return + } ginkgo.BeforeEach(func() { err := f.UpdateIngressControllerDeployment(func(deployment *appsv1.Deployment) error { diff --git a/test/e2e/settings/stream_snippet.go b/test/e2e/settings/stream_snippet.go index e1552afd3..4d1757a08 100644 --- a/test/e2e/settings/stream_snippet.go +++ b/test/e2e/settings/stream_snippet.go @@ -34,6 +34,9 @@ import ( var _ = framework.DescribeSetting("configmap stream-snippet", func() { f := framework.NewDefaultFramework("cm-stream-snippet") + if framework.IsCrossplane() { + return + } ginkgo.BeforeEach(func() { f.NewEchoDeployment() diff --git a/test/e2e/settings/tls.go b/test/e2e/settings/tls.go index 1ebf358c1..219111664 100644 --- a/test/e2e/settings/tls.go +++ b/test/e2e/settings/tls.go @@ -71,7 +71,7 @@ var _ = framework.DescribeSetting("[SSL] TLS protocols, ciphers and headers", fu f.WaitForNginxConfiguration( func(cfg string) bool { - return strings.Contains(cfg, fmt.Sprintf("ssl_ciphers '%s';", testCiphers)) + return strings.Contains(cfg, fmt.Sprintf("ssl_ciphers '%s';", testCiphers)) || strings.Contains(cfg, fmt.Sprintf("ssl_ciphers %s;", testCiphers)) }) resp := f.HTTPTestClientWithTLSConfig(tlsConfig). diff --git a/test/e2e/tcpudp/tcp.go b/test/e2e/tcpudp/tcp.go index 856184d18..de45b0035 100644 --- a/test/e2e/tcpudp/tcp.go +++ b/test/e2e/tcpudp/tcp.go @@ -37,6 +37,9 @@ import ( var _ = framework.IngressNginxDescribe("[TCP] tcp-services", func() { f := framework.NewDefaultFramework("tcp") + if framework.IsCrossplane() { + return + } var ip string ginkgo.BeforeEach(func() {