From 6510535ae0164e48d60d0c3df79bffd35031b3f3 Mon Sep 17 00:00:00 2001 From: Ricardo Katz Date: Sun, 8 Sep 2024 18:48:12 -0300 Subject: [PATCH] NGINX: Remove inline Lua from template. (#11806) --- .github/workflows/ci.yaml | 16 ++ hack/verify-lualint.sh | 9 +- internal/ingress/controller/nginx.go | 30 +++ .../ingress/controller/template/template.go | 93 +++++---- internal/ingress/controller/util.go | 5 +- rootfs/etc/nginx/lua/lua_ingress.lua | 12 +- .../etc/nginx/lua/nginx/ngx_conf_balancer.lua | 2 + .../lua/nginx/ngx_conf_balancer_tcp_udp.lua | 2 + .../nginx/lua/nginx/ngx_conf_certificate.lua | 2 + .../lua/nginx/ngx_conf_configuration.lua | 2 + .../lua/nginx/ngx_conf_content_tcp_udp.lua | 2 + .../nginx/lua/nginx/ngx_conf_init_tcp_udp.lua | 2 + .../ngx_conf_is_dynamic_lb_initialized.lua | 9 + rootfs/etc/nginx/lua/nginx/ngx_conf_log.lua | 2 + .../nginx/lua/nginx/ngx_conf_log_block.lua | 11 + .../nginx/lua/nginx/ngx_conf_rewrite_auth.lua | 1 + .../lua/nginx/ngx_conf_srv_hdr_filter.lua | 2 + rootfs/etc/nginx/lua/nginx/ngx_rewrite.lua | 5 + .../etc/nginx/lua/nginx/ngx_srv_redirect.lua | 24 +++ rootfs/etc/nginx/lua/ngx_conf_init.lua | 53 +++++ rootfs/etc/nginx/lua/ngx_conf_init_stream.lua | 30 +++ rootfs/etc/nginx/lua/ngx_conf_init_worker.lua | 15 ++ rootfs/etc/nginx/lua/util.lua | 4 + rootfs/etc/nginx/template/nginx.tmpl | 189 ++---------------- test/e2e/annotations/fromtowwwredirect.go | 4 +- test/e2e/framework/framework.go | 33 +++ test/e2e/lua/dynamic_configuration.go | 7 +- .../e2e/settings/no_tls_redirect_locations.go | 4 +- test/e2e/settings/ocsp/ocsp.go | 6 +- test/e2e/settings/tls.go | 18 +- 30 files changed, 361 insertions(+), 233 deletions(-) create mode 100644 rootfs/etc/nginx/lua/nginx/ngx_conf_balancer.lua create mode 100644 rootfs/etc/nginx/lua/nginx/ngx_conf_balancer_tcp_udp.lua create mode 100644 rootfs/etc/nginx/lua/nginx/ngx_conf_certificate.lua create mode 100644 rootfs/etc/nginx/lua/nginx/ngx_conf_configuration.lua create mode 100644 rootfs/etc/nginx/lua/nginx/ngx_conf_content_tcp_udp.lua create mode 100644 rootfs/etc/nginx/lua/nginx/ngx_conf_init_tcp_udp.lua create mode 100644 rootfs/etc/nginx/lua/nginx/ngx_conf_is_dynamic_lb_initialized.lua create mode 100644 rootfs/etc/nginx/lua/nginx/ngx_conf_log.lua create mode 100644 rootfs/etc/nginx/lua/nginx/ngx_conf_log_block.lua create mode 100644 rootfs/etc/nginx/lua/nginx/ngx_conf_rewrite_auth.lua create mode 100644 rootfs/etc/nginx/lua/nginx/ngx_conf_srv_hdr_filter.lua create mode 100644 rootfs/etc/nginx/lua/nginx/ngx_rewrite.lua create mode 100644 rootfs/etc/nginx/lua/nginx/ngx_srv_redirect.lua create mode 100644 rootfs/etc/nginx/lua/ngx_conf_init.lua create mode 100644 rootfs/etc/nginx/lua/ngx_conf_init_stream.lua create mode 100644 rootfs/etc/nginx/lua/ngx_conf_init_worker.lua diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 05cf9269c..77a03fe7b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -71,6 +71,22 @@ jobs: - 'images/nginx-1.25/**' docs: - '**/*.md' + lua: + - '**/*.lua' + + lua-lint: + runs-on: ubuntu-latest + needs: changes + if: | + (needs.changes.outputs.lua == 'true') || ${{ github.event.workflow_dispatch.run_e2e == 'true' }} + steps: + - name: Checkout + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: Lint Lua + uses: lunarmodules/luacheck@v1 + with: + args: --codes --globals lua_ingress --globals configuration --globals balancer --globals monitor --globals certificate --globals tcp_udp_configuration --globals tcp_udp_balancer --no-max-comment-line-length -q rootfs/etc/nginx/lua/ test-go: runs-on: ubuntu-latest diff --git a/hack/verify-lualint.sh b/hack/verify-lualint.sh index 1f6048de0..769d7a6cd 100755 --- a/hack/verify-lualint.sh +++ b/hack/verify-lualint.sh @@ -18,6 +18,13 @@ set -o errexit set -o nounset set -o pipefail -luacheck --codes -q rootfs/etc/nginx/lua/ +luacheck --codes --globals lua_ingress \ + --globals configuration \ + --globals balancer \ + --globals monitor \ + --globals certificate \ + --globals tcp_udp_configuration \ + --globals tcp_udp_balancer \ + --no-max-comment-line-length -q rootfs/etc/nginx/lua/ find rootfs/etc/nginx/lua/ -name "*.lua" -not -path "*/test/*" -exec lj-releng -L -s {} + && echo "lj-releng validation is success!" diff --git a/internal/ingress/controller/nginx.go b/internal/ingress/controller/nginx.go index f74b3245e..7d6dcc3fa 100644 --- a/internal/ingress/controller/nginx.go +++ b/internal/ingress/controller/nginx.go @@ -690,6 +690,10 @@ func (n *NGINXController) OnUpdate(ingressCfg ingress.Configuration) error { return err } + err = n.createLuaConfig(&cfg) + if err != nil { + return err + } err = createOpentelemetryCfg(&cfg) if err != nil { return err @@ -1079,6 +1083,32 @@ func createOpentelemetryCfg(cfg *ngx_config.Configuration) error { return os.WriteFile(cfg.OpentelemetryConfig, tmplBuf.Bytes(), file.ReadWriteByUser) } +func (n *NGINXController) createLuaConfig(cfg *ngx_config.Configuration) error { + luaconfigs := &ngx_template.LuaConfig{ + EnableMetrics: n.cfg.EnableMetrics, + ListenPorts: ngx_template.LuaListenPorts{ + HTTPSPort: strconv.Itoa(n.cfg.ListenPorts.HTTPS), + StatusPort: strconv.Itoa(nginx.StatusPort), + SSLProxyPort: strconv.Itoa(n.cfg.ListenPorts.SSLProxy), + }, + UseProxyProtocol: cfg.UseProxyProtocol, + UseForwardedHeaders: cfg.UseForwardedHeaders, + IsSSLPassthroughEnabled: n.cfg.EnableSSLPassthrough, + HTTPRedirectCode: cfg.HTTPRedirectCode, + EnableOCSP: cfg.EnableOCSP, + MonitorBatchMaxSize: n.cfg.MonitorMaxBatchSize, + HSTS: cfg.HSTS, + HSTSMaxAge: cfg.HSTSMaxAge, + HSTSIncludeSubdomains: cfg.HSTSIncludeSubdomains, + HSTSPreload: cfg.HSTSPreload, + } + jsonCfg, err := json.Marshal(luaconfigs) + if err != nil { + return err + } + return os.WriteFile(luaCfgPath, jsonCfg, file.ReadWriteByUser) +} + func cleanTempNginxCfg() error { var files []string diff --git a/internal/ingress/controller/template/template.go b/internal/ingress/controller/template/template.go index 42ff7417e..d2c8a05a9 100644 --- a/internal/ingress/controller/template/template.go +++ b/internal/ingress/controller/template/template.go @@ -194,6 +194,41 @@ func cleanConf(in, out *bytes.Buffer) error { } } +/* LuaConfig defines the structure that will be written as a config for lua scripts +The json format should follow what's expected by lua: + use_forwarded_headers = %t, + use_proxy_protocol = %t, + is_ssl_passthrough_enabled = %t, + http_redirect_code = %v, + listen_ports = { ssl_proxy = "%v", https = "%v" }, + + hsts = %t, + hsts_max_age = %v, + hsts_include_subdomains = %t, + hsts_preload = %t, +*/ + +type LuaConfig struct { + EnableMetrics bool `json:"enable_metrics"` + ListenPorts LuaListenPorts `json:"listen_ports"` + UseForwardedHeaders bool `json:"use_forwarded_headers"` + UseProxyProtocol bool `json:"use_proxy_protocol"` + IsSSLPassthroughEnabled bool `json:"is_ssl_passthrough_enabled"` + HTTPRedirectCode int `json:"http_redirect_code"` + EnableOCSP bool `json:"enable_ocsp"` + MonitorBatchMaxSize int `json:"monitor_batch_max_size"` + HSTS bool `json:"hsts"` + HSTSMaxAge string `json:"hsts_max_age"` + HSTSIncludeSubdomains bool `json:"hsts_include_subdomains"` + HSTSPreload bool `json:"hsts_preload"` +} + +type LuaListenPorts struct { + HTTPSPort string `json:"https"` + StatusPort string `json:"status_port"` + SSLProxyPort string `json:"ssl_proxy"` +} + // Write populates a buffer using a template with NGINX configuration // and the servers and upstreams created by Ingress rules func (t *Template) Write(conf *config.TemplateConfig) ([]byte, error) { @@ -256,7 +291,6 @@ var funcMap = text_template.FuncMap{ "filterRateLimits": filterRateLimits, "buildRateLimitZones": buildRateLimitZones, "buildRateLimit": buildRateLimit, - "configForLua": configForLua, "locationConfigForLua": locationConfigForLua, "buildResolvers": buildResolvers, "buildUpstreamName": buildUpstreamName, @@ -383,41 +417,6 @@ func luaConfigurationRequestBodySize(c interface{}) string { return dictKbToStr(size) } -// configForLua returns some general configuration as Lua table represented as string -func configForLua(input interface{}) string { - all, ok := input.(config.TemplateConfig) - if !ok { - klog.Errorf("expected a 'config.TemplateConfig' type but %T was given", input) - return "{}" - } - - return fmt.Sprintf(`{ - use_forwarded_headers = %t, - use_proxy_protocol = %t, - is_ssl_passthrough_enabled = %t, - http_redirect_code = %v, - listen_ports = { ssl_proxy = "%v", https = "%v" }, - - hsts = %t, - hsts_max_age = %v, - hsts_include_subdomains = %t, - hsts_preload = %t, - - }`, - all.Cfg.UseForwardedHeaders, - all.Cfg.UseProxyProtocol, - all.IsSSLPassthroughEnabled, - all.Cfg.HTTPRedirectCode, - all.ListenPorts.SSLProxy, - all.ListenPorts.HTTPS, - - all.Cfg.HSTS, - all.Cfg.HSTSMaxAge, - all.Cfg.HSTSIncludeSubdomains, - all.Cfg.HSTSPreload, - ) -} - // locationConfigForLua formats some location specific configuration into Lua table represented as string func locationConfigForLua(l, a interface{}) string { location, ok := l.(*ingress.Location) @@ -432,13 +431,21 @@ func locationConfigForLua(l, a interface{}) string { return "{}" } - return fmt.Sprintf(`{ - force_ssl_redirect = %t, - ssl_redirect = %t, - force_no_ssl_redirect = %t, - preserve_trailing_slash = %t, - use_port_in_redirects = %t, - }`, + /* Lua expects the following vars + force_ssl_redirect = string_to_bool(ngx.var.force_ssl_redirect), + ssl_redirect = string_to_bool(ngx.var.ssl_redirect), + force_no_ssl_redirect = string_to_bool(ngx.var.force_no_ssl_redirect), + preserve_trailing_slash = string_to_bool(ngx.var.preserve_trailing_slash), + use_port_in_redirects = string_to_bool(ngx.var.use_port_in_redirects), + */ + + return fmt.Sprintf(` + set $force_ssl_redirect "%t"; + set $ssl_redirect "%t"; + set $force_no_ssl_redirect "%t"; + set $preserve_trailing_slash "%t"; + set $use_port_in_redirects "%t"; + `, location.Rewrite.ForceSSLRedirect, location.Rewrite.SSLRedirect, isLocationInLocationList(l, all.Cfg.NoTLSRedirectLocations), diff --git a/internal/ingress/controller/util.go b/internal/ingress/controller/util.go index 79bf931ef..975fb822a 100644 --- a/internal/ingress/controller/util.go +++ b/internal/ingress/controller/util.go @@ -98,8 +98,9 @@ func rlimitMaxNumFiles() int { } const ( - defBinary = "/usr/bin/nginx" - cfgPath = "/etc/nginx/nginx.conf" + defBinary = "/usr/bin/nginx" + cfgPath = "/etc/nginx/nginx.conf" + luaCfgPath = "/etc/nginx/lua/cfg.json" ) // NginxExecTester defines the interface to execute diff --git a/rootfs/etc/nginx/lua/lua_ingress.lua b/rootfs/etc/nginx/lua/lua_ingress.lua index 65cf5a257..a513928cf 100644 --- a/rootfs/etc/nginx/lua/lua_ingress.lua +++ b/rootfs/etc/nginx/lua/lua_ingress.lua @@ -1,4 +1,5 @@ local ngx_re_split = require("ngx.re").split +local string_to_bool = require("util").string_to_bool local certificate_configured_for_current_request = require("certificate").configured_for_current_request @@ -108,7 +109,16 @@ end -- rewrite gets called in every location context. -- This is where we do variable assignments to be used in subsequent -- phases or redirection -function _M.rewrite(location_config) +function _M.rewrite() + + local location_config = { + force_ssl_redirect = string_to_bool(ngx.var.force_ssl_redirect), + ssl_redirect = string_to_bool(ngx.var.ssl_redirect), + force_no_ssl_redirect = string_to_bool(ngx.var.force_no_ssl_redirect), + preserve_trailing_slash = string_to_bool(ngx.var.preserve_trailing_slash), + use_port_in_redirects = string_to_bool(ngx.var.use_port_in_redirects), + } + ngx.var.pass_access_scheme = ngx.var.scheme ngx.var.best_http_host = ngx.var.http_host or ngx.var.host diff --git a/rootfs/etc/nginx/lua/nginx/ngx_conf_balancer.lua b/rootfs/etc/nginx/lua/nginx/ngx_conf_balancer.lua new file mode 100644 index 000000000..977d3e964 --- /dev/null +++ b/rootfs/etc/nginx/lua/nginx/ngx_conf_balancer.lua @@ -0,0 +1,2 @@ +local balancer = require("balancer") +balancer.balance() \ No newline at end of file diff --git a/rootfs/etc/nginx/lua/nginx/ngx_conf_balancer_tcp_udp.lua b/rootfs/etc/nginx/lua/nginx/ngx_conf_balancer_tcp_udp.lua new file mode 100644 index 000000000..0442df7d8 --- /dev/null +++ b/rootfs/etc/nginx/lua/nginx/ngx_conf_balancer_tcp_udp.lua @@ -0,0 +1,2 @@ +local tcp_udp_balancer = require("tcp_udp_balancer") +tcp_udp_balancer.balance() \ No newline at end of file diff --git a/rootfs/etc/nginx/lua/nginx/ngx_conf_certificate.lua b/rootfs/etc/nginx/lua/nginx/ngx_conf_certificate.lua new file mode 100644 index 000000000..d33d2171b --- /dev/null +++ b/rootfs/etc/nginx/lua/nginx/ngx_conf_certificate.lua @@ -0,0 +1,2 @@ +local certificate = require("certificate") +certificate.call() \ No newline at end of file diff --git a/rootfs/etc/nginx/lua/nginx/ngx_conf_configuration.lua b/rootfs/etc/nginx/lua/nginx/ngx_conf_configuration.lua new file mode 100644 index 000000000..7864f40ef --- /dev/null +++ b/rootfs/etc/nginx/lua/nginx/ngx_conf_configuration.lua @@ -0,0 +1,2 @@ +local configuration = require("configuration") +configuration.call() \ No newline at end of file diff --git a/rootfs/etc/nginx/lua/nginx/ngx_conf_content_tcp_udp.lua b/rootfs/etc/nginx/lua/nginx/ngx_conf_content_tcp_udp.lua new file mode 100644 index 000000000..ed81e7ff3 --- /dev/null +++ b/rootfs/etc/nginx/lua/nginx/ngx_conf_content_tcp_udp.lua @@ -0,0 +1,2 @@ +local tcp_udp_configuration = require("tcp_udp_configuration") +tcp_udp_configuration.call() \ No newline at end of file diff --git a/rootfs/etc/nginx/lua/nginx/ngx_conf_init_tcp_udp.lua b/rootfs/etc/nginx/lua/nginx/ngx_conf_init_tcp_udp.lua new file mode 100644 index 000000000..926ab7a03 --- /dev/null +++ b/rootfs/etc/nginx/lua/nginx/ngx_conf_init_tcp_udp.lua @@ -0,0 +1,2 @@ +local tcp_udp_balancer = require("tcp_udp_balancer") +tcp_udp_balancer.init_worker() \ No newline at end of file diff --git a/rootfs/etc/nginx/lua/nginx/ngx_conf_is_dynamic_lb_initialized.lua b/rootfs/etc/nginx/lua/nginx/ngx_conf_is_dynamic_lb_initialized.lua new file mode 100644 index 000000000..ade3114b1 --- /dev/null +++ b/rootfs/etc/nginx/lua/nginx/ngx_conf_is_dynamic_lb_initialized.lua @@ -0,0 +1,9 @@ +local configuration = require("configuration") +local backend_data = configuration.get_backends_data() +if not backend_data then + ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR) + return +end + +ngx.say("OK") +ngx.exit(ngx.HTTP_OK) \ No newline at end of file diff --git a/rootfs/etc/nginx/lua/nginx/ngx_conf_log.lua b/rootfs/etc/nginx/lua/nginx/ngx_conf_log.lua new file mode 100644 index 000000000..8f3d57be6 --- /dev/null +++ b/rootfs/etc/nginx/lua/nginx/ngx_conf_log.lua @@ -0,0 +1,2 @@ +local monitor = require("monitor") +monitor.call() \ No newline at end of file diff --git a/rootfs/etc/nginx/lua/nginx/ngx_conf_log_block.lua b/rootfs/etc/nginx/lua/nginx/ngx_conf_log_block.lua new file mode 100644 index 000000000..72f6a6430 --- /dev/null +++ b/rootfs/etc/nginx/lua/nginx/ngx_conf_log_block.lua @@ -0,0 +1,11 @@ +local balancer = require("balancer") +local monitor = require("monitor") + +local luaconfig = ngx.shared.luaconfig +local enablemetrics = luaconfig:get("enablemetrics") + +balancer.log() + +if enablemetrics then + monitor.call() +end \ No newline at end of file diff --git a/rootfs/etc/nginx/lua/nginx/ngx_conf_rewrite_auth.lua b/rootfs/etc/nginx/lua/nginx/ngx_conf_rewrite_auth.lua new file mode 100644 index 000000000..0d16faba0 --- /dev/null +++ b/rootfs/etc/nginx/lua/nginx/ngx_conf_rewrite_auth.lua @@ -0,0 +1 @@ +ngx.var.cache_key = ngx.encode_base64(ngx.sha1_bin(ngx.var.tmp_cache_key)) \ No newline at end of file diff --git a/rootfs/etc/nginx/lua/nginx/ngx_conf_srv_hdr_filter.lua b/rootfs/etc/nginx/lua/nginx/ngx_conf_srv_hdr_filter.lua new file mode 100644 index 000000000..311a9b433 --- /dev/null +++ b/rootfs/etc/nginx/lua/nginx/ngx_conf_srv_hdr_filter.lua @@ -0,0 +1,2 @@ +local lua_ingress = require("lua_ingress") +lua_ingress.header() \ No newline at end of file diff --git a/rootfs/etc/nginx/lua/nginx/ngx_rewrite.lua b/rootfs/etc/nginx/lua/nginx/ngx_rewrite.lua new file mode 100644 index 000000000..66fdd6d55 --- /dev/null +++ b/rootfs/etc/nginx/lua/nginx/ngx_rewrite.lua @@ -0,0 +1,5 @@ +local lua_ingress = require("lua_ingress") +local balancer = require("balancer") + +lua_ingress.rewrite() +balancer.rewrite() \ No newline at end of file diff --git a/rootfs/etc/nginx/lua/nginx/ngx_srv_redirect.lua b/rootfs/etc/nginx/lua/nginx/ngx_srv_redirect.lua new file mode 100644 index 000000000..4b9445d00 --- /dev/null +++ b/rootfs/etc/nginx/lua/nginx/ngx_srv_redirect.lua @@ -0,0 +1,24 @@ +local request_uri = ngx.var.request_uri +local redirect_to = ngx.arg[1] + +local luaconfig = ngx.shared.luaconfig +local use_forwarded_headers = luaconfig:get("use_forwarded_headers") + +if string.sub(request_uri, -1) == "/" then + request_uri = string.sub(request_uri, 1, -2) +end + +local redirectScheme = ngx.var.scheme +local redirectPort = ngx.var.server_port + +if use_forwarded_headers then + if ngx.var.http_x_forwarded_proto then + redirectScheme = ngx.var.http_x_forwarded_proto + end + if ngx.var.http_x_forwarded_port then + redirectPort = ngx.var.http_x_forwarded_port + end +end + +return string.format("%s://%s:%s%s", redirectScheme, + redirect_to, redirectPort, request_uri) \ No newline at end of file diff --git a/rootfs/etc/nginx/lua/ngx_conf_init.lua b/rootfs/etc/nginx/lua/ngx_conf_init.lua new file mode 100644 index 000000000..9789386ac --- /dev/null +++ b/rootfs/etc/nginx/lua/ngx_conf_init.lua @@ -0,0 +1,53 @@ +local cjson = require("cjson.safe") + +collectgarbage("collect") +local f = io.open("/etc/nginx/lua/cfg.json", "r") +local content = f:read("*a") +f:close() +local configfile = cjson.decode(content) + +local luaconfig = ngx.shared.luaconfig +luaconfig:set("enablemetrics", configfile.enable_metrics) +luaconfig:set("use_forwarded_headers", configfile.use_forwarded_headers) +-- init modules +local ok, res +ok, res = pcall(require, "lua_ingress") +if not ok then + error("require failed: " .. tostring(res)) +else + lua_ingress = res + lua_ingress.set_config(configfile) +end +ok, res = pcall(require, "configuration") +if not ok then + error("require failed: " .. tostring(res)) +else + configuration = res + if not configfile.listen_ports.status_port then + error("required status port not found") + end + configuration.prohibited_localhost_port = configfile.listen_ports.status_port +end +ok, res = pcall(require, "balancer") +if not ok then + error("require failed: " .. tostring(res)) +else + balancer = res +end +if configfile.enable_metrics then + ok, res = pcall(require, "monitor") + if not ok then + error("require failed: " .. tostring(res)) + else + monitor = res + end +end +ok, res = pcall(require, "certificate") +if not ok then + error("require failed: " .. tostring(res)) +else + certificate = res + if configfile.enable_ocsp then + certificate.is_ocsp_stapling_enabled = configfile.enable_ocsp + end +end \ No newline at end of file diff --git a/rootfs/etc/nginx/lua/ngx_conf_init_stream.lua b/rootfs/etc/nginx/lua/ngx_conf_init_stream.lua new file mode 100644 index 000000000..a78062d0a --- /dev/null +++ b/rootfs/etc/nginx/lua/ngx_conf_init_stream.lua @@ -0,0 +1,30 @@ +local cjson = require("cjson.safe") +collectgarbage("collect") +local f = io.open("/etc/nginx/lua/cfg.json", "r") +local content = f:read("*a") +f:close() +local configfile = cjson.decode(content) +-- init modules +local ok, res +ok, res = pcall(require, "configuration") +if not ok then + error("require failed: " .. tostring(res)) +else + configuration = res +end +ok, res = pcall(require, "tcp_udp_configuration") +if not ok then + error("require failed: " .. tostring(res)) +else + tcp_udp_configuration = res + if not configfile.listen_ports.status_port then + error("required status port not found") + end + tcp_udp_configuration.prohibited_localhost_port = configfile.listen_ports.status_port +end +ok, res = pcall(require, "tcp_udp_balancer") +if not ok then + error("require failed: " .. tostring(res)) +else + tcp_udp_balancer = res +end diff --git a/rootfs/etc/nginx/lua/ngx_conf_init_worker.lua b/rootfs/etc/nginx/lua/ngx_conf_init_worker.lua new file mode 100644 index 000000000..cba866136 --- /dev/null +++ b/rootfs/etc/nginx/lua/ngx_conf_init_worker.lua @@ -0,0 +1,15 @@ +local cjson = require("cjson.safe") + +local f = io.open("/etc/nginx/lua/cfg.json", "r") +local content = f:read("*a") +f:close() +local configfile = cjson.decode(content) + +local lua_ingress = require("lua_ingress") +local balancer = require("balancer") +local monitor = require("monitor") +lua_ingress.init_worker() +balancer.init_worker() +if configfile.enable_metrics and configfile.monitor_batch_max_size then + monitor.init_worker(configfile.monitor_batch_max_size) +end \ No newline at end of file diff --git a/rootfs/etc/nginx/lua/util.lua b/rootfs/etc/nginx/lua/util.lua index 7389f3226..1e4cd7c01 100644 --- a/rootfs/etc/nginx/lua/util.lua +++ b/rootfs/etc/nginx/lua/util.lua @@ -146,6 +146,10 @@ function _M.is_blank(str) return str == nil or string_len(str) == 0 end +function _M.string_to_bool(str) + return str == "true" +end + -- this implementation is taken from: -- https://github.com/luafun/luafun/blob/master/fun.lua#L33 -- SHA: 04c99f9c393e54a604adde4b25b794f48104e0d0 diff --git a/rootfs/etc/nginx/template/nginx.tmpl b/rootfs/etc/nginx/template/nginx.tmpl index 4f705976f..f32860dc2 100644 --- a/rootfs/etc/nginx/template/nginx.tmpl +++ b/rootfs/etc/nginx/template/nginx.tmpl @@ -68,60 +68,11 @@ http { {{ buildLuaSharedDictionaries $cfg $servers }} - init_by_lua_block { - collectgarbage("collect") + lua_shared_dict luaconfig 5m; - -- init modules - local ok, res + init_by_lua_file /etc/nginx/lua/ngx_conf_init.lua; - ok, res = pcall(require, "lua_ingress") - if not ok then - error("require failed: " .. tostring(res)) - else - lua_ingress = res - lua_ingress.set_config({{ configForLua $all }}) - end - - ok, res = pcall(require, "configuration") - if not ok then - error("require failed: " .. tostring(res)) - else - configuration = res - configuration.prohibited_localhost_port = '{{ .StatusPort }}' - end - - ok, res = pcall(require, "balancer") - if not ok then - error("require failed: " .. tostring(res)) - else - balancer = res - end - - {{ if $all.EnableMetrics }} - ok, res = pcall(require, "monitor") - if not ok then - error("require failed: " .. tostring(res)) - else - monitor = res - end - {{ end }} - - ok, res = pcall(require, "certificate") - if not ok then - error("require failed: " .. tostring(res)) - else - certificate = res - certificate.is_ocsp_stapling_enabled = {{ $cfg.EnableOCSP }} - end - } - - init_worker_by_lua_block { - lua_ingress.init_worker() - balancer.init_worker() - {{ if $all.EnableMetrics }} - monitor.init_worker({{ $all.MonitorMaxBatchSize }}) - {{ end }} - } + init_worker_by_lua_file /etc/nginx/lua/ngx_conf_init_worker.lua; {{/* Enable the real_ip module only if we use either X-Forwarded headers or Proxy Protocol. */}} {{/* we use the value of the real IP for the geo_ip module */}} @@ -539,9 +490,7 @@ http { server 0.0.0.1; # placeholder - balancer_by_lua_block { - balancer.balance() - } + balancer_by_lua_file /etc/nginx/lua/nginx/ngx_conf_balancer.lua; {{ if (gt $cfg.UpstreamKeepaliveConnections 0) }} keepalive {{ $cfg.UpstreamKeepaliveConnections }}; @@ -606,9 +555,7 @@ http { {{ buildHTTPListener $all $redirect.From }} {{ buildHTTPSListener $all $redirect.From }} - ssl_certificate_by_lua_block { - certificate.call() - } + ssl_certificate_by_lua_file /etc/nginx/lua/nginx/ngx_conf_certificate.lua; {{ if gt (len $cfg.BlockUserAgents) 0 }} if ($block_ua) { @@ -621,30 +568,7 @@ http { } {{ end }} - set_by_lua_block $redirect_to { - local request_uri = ngx.var.request_uri - if string.sub(request_uri, -1) == "/" then - request_uri = string.sub(request_uri, 1, -2) - end - - {{ if $cfg.UseForwardedHeaders }} - local redirectScheme - if not ngx.var.http_x_forwarded_proto then - redirectScheme = ngx.var.scheme - else - redirectScheme = ngx.var.http_x_forwarded_proto - end - {{ else }} - local redirectScheme = ngx.var.scheme - {{ end }} - - {{ if ne $all.ListenPorts.HTTPS 443 }} - {{ $redirect_port := (printf ":%v" $all.ListenPorts.HTTPS) }} - return string.format("%s://%s%s%s", redirectScheme, "{{ $redirect.To }}", "{{ $redirect_port }}", request_uri) - {{ else }} - return string.format("%s://%s%s", redirectScheme, "{{ $redirect.To }}", request_uri) - {{ end }} - } + set_by_lua_file $redirect_to /etc/nginx/lua/nginx/ngx_srv_redirect.lua {{ $redirect.To }}; return {{ $all.Cfg.HTTPRedirectCode }} $redirect_to; } @@ -739,17 +663,7 @@ http { } location /is-dynamic-lb-initialized { - content_by_lua_block { - local configuration = require("configuration") - local backend_data = configuration.get_backends_data() - if not backend_data then - ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR) - return - end - - ngx.say("OK") - ngx.exit(ngx.HTTP_OK) - } + content_by_lua_file /etc/nginx/lua/nginx/ngx_conf_is_dynamic_lb_initialized.lua; } location {{ .StatusPath }} { @@ -761,15 +675,11 @@ http { client_body_buffer_size {{ luaConfigurationRequestBodySize $cfg }}; proxy_buffering off; - content_by_lua_block { - configuration.call() - } + content_by_lua_file /etc/nginx/lua/nginx/ngx_conf_configuration.lua; } location / { - content_by_lua_block { - ngx.exit(ngx.HTTP_NOT_FOUND) - } + return 404; } } } @@ -781,39 +691,9 @@ stream { {{ buildResolvers $cfg.Resolver $cfg.DisableIpv6DNS }} - init_by_lua_block { - collectgarbage("collect") + init_by_lua_file /etc/nginx/lua/ngx_conf_init_stream.lua; - -- init modules - local ok, res - - ok, res = pcall(require, "configuration") - if not ok then - error("require failed: " .. tostring(res)) - else - configuration = res - end - - ok, res = pcall(require, "tcp_udp_configuration") - if not ok then - error("require failed: " .. tostring(res)) - else - tcp_udp_configuration = res - tcp_udp_configuration.prohibited_localhost_port = '{{ .StatusPort }}' - - end - - ok, res = pcall(require, "tcp_udp_balancer") - if not ok then - error("require failed: " .. tostring(res)) - else - tcp_udp_balancer = res - end - } - - init_worker_by_lua_block { - tcp_udp_balancer.init_worker() - } + init_worker_by_lua_file /etc/nginx/lua/nginx/ngx_conf_init_tcp_udp.lua; lua_add_variable $proxy_upstream_name; @@ -835,10 +715,7 @@ stream { upstream upstream_balancer { server 0.0.0.1:1234; # placeholder - - balancer_by_lua_block { - tcp_udp_balancer.balance() - } + balancer_by_lua_file /etc/nginx/lua/nginx/ngx_conf_balancer_tcp_udp.lua; } server { @@ -846,9 +723,7 @@ stream { access_log off; - content_by_lua_block { - tcp_udp_configuration.call() - } + content_by_lua_file /etc/nginx/lua/nginx/ngx_conf_content_tcp_udp.lua; } # TCP services @@ -948,11 +823,9 @@ stream { rewrite (.*) / break; proxy_pass http://upstream_balancer; - log_by_lua_block { - {{ if $enableMetrics }} - monitor.call() - {{ end }} - } + {{ if $enableMetrics }} + log_by_lua_file /etc/nginx/lua/nginx/ngx_conf_log.lua; + {{ end }} } {{ end }} {{ end }} @@ -1012,9 +885,7 @@ stream { ssl_reject_handshake {{ if $all.Cfg.SSLRejectHandshake }}on{{ else }}off{{ end }}; {{ end }} - ssl_certificate_by_lua_block { - certificate.call() - } + ssl_certificate_by_lua_file /etc/nginx/lua/nginx/ngx_conf_certificate.lua; {{ if not (empty $server.AuthTLSError) }} # {{ $server.AuthTLSError }} @@ -1115,9 +986,7 @@ stream { set $tmp_cache_key '{{ $server.Hostname }}{{ $authPath }}{{ $externalAuth.AuthCacheKey }}'; set $cache_key ''; - rewrite_by_lua_block { - ngx.var.cache_key = ngx.encode_base64(ngx.sha1_bin(ngx.var.tmp_cache_key)) - } + rewrite_by_lua_file /etc/nginx/lua/nginx/ngx_conf_rewrite_auth.lua; proxy_cache auth_cache; @@ -1250,27 +1119,13 @@ stream { mirror_request_body {{ $location.Mirror.RequestBody }}; {{ end }} - rewrite_by_lua_block { - lua_ingress.rewrite({{ locationConfigForLua $location $all }}) - balancer.rewrite() - } + {{ locationConfigForLua $location $all }} - # be careful with `access_by_lua_block` and `satisfy any` directives as satisfy any - # will always succeed when there's `access_by_lua_block` that does not have any lua code doing `ngx.exit(ngx.DECLINED)` - # other authentication method such as basic auth or external auth useless - all requests will be allowed. - #access_by_lua_block { - #} + rewrite_by_lua_file /etc/nginx/lua/nginx/ngx_rewrite.lua; - header_filter_by_lua_block { - lua_ingress.header() - } + header_filter_by_lua_file /etc/nginx/lua/nginx/ngx_conf_srv_hdr_filter.lua; - log_by_lua_block { - balancer.log() - {{ if $all.EnableMetrics }} - monitor.call() - {{ end }} - } + log_by_lua_file /etc/nginx/lua/nginx/ngx_conf_log_block.lua; {{ if not $location.Logs.Access }} access_log off; diff --git a/test/e2e/annotations/fromtowwwredirect.go b/test/e2e/annotations/fromtowwwredirect.go index f578963ed..a3fb3b9b5 100644 --- a/test/e2e/annotations/fromtowwwredirect.go +++ b/test/e2e/annotations/fromtowwwredirect.go @@ -58,7 +58,7 @@ var _ = framework.DescribeAnnotation("from-to-www-redirect", func() { WithHeader("Host", fmt.Sprintf("%s.%s", "www", host)). Expect(). Status(http.StatusPermanentRedirect). - Header("Location").Equal("http://fromtowwwredirect.bar.com/foo") + Header("Location").Equal("http://fromtowwwredirect.bar.com:80/foo") }) ginkgo.It("should redirect from www HTTPS to HTTPS", func() { @@ -101,7 +101,7 @@ var _ = framework.DescribeAnnotation("from-to-www-redirect", func() { WithHeader("Host", toHost). Expect(). Status(http.StatusPermanentRedirect). - Header("Location").Equal(fmt.Sprintf("https://%v", fromHost)) + Header("Location").Equal(fmt.Sprintf("https://%v:443", fromHost)) ginkgo.By("sending request to domain should not redirect to www") f.HTTPTestClientWithTLSConfig(&tls.Config{ diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go index 103d8d593..204da7df0 100644 --- a/test/e2e/framework/framework.go +++ b/test/e2e/framework/framework.go @@ -16,6 +16,7 @@ package framework import ( "context" "crypto/tls" + "encoding/json" "fmt" "net" "net/http" @@ -283,6 +284,15 @@ func (f *Framework) WaitForNginxConfiguration(matcher func(cfg string) bool) { Sleep(1 * time.Second) } +// WaitForLuaConfiguration waits until the nginx configuration contains a particular configuration +// `cfg` passed to matcher is normalized by replacing all tabs and spaces with single space. +func (f *Framework) WaitForLuaConfiguration(matcher func(jsonCfg map[string]interface{}) bool) { + //nolint:staticcheck // TODO: will replace it since wait.Poll is deprecated + err := wait.Poll(Poll, DefaultTimeout, f.matchLuaConditions(matcher)) + assert.Nil(ginkgo.GinkgoT(), err, "waiting for nginx lua configuration condition/s") + Sleep(1 * time.Second) +} + // WaitForNginxCustomConfiguration waits until the nginx configuration given part (from, to) contains a particular configuration func (f *Framework) WaitForNginxCustomConfiguration(from, to string, matcher func(cfg string) bool) { //nolint:staticcheck // TODO: will replace it since wait.Poll is deprecated @@ -326,6 +336,29 @@ func (f *Framework) matchNginxConditions(name string, matcher func(cfg string) b } } +func (f *Framework) matchLuaConditions(matcher func(jsonCfg map[string]interface{}) bool) wait.ConditionFunc { + return func() (bool, error) { + cmd := "cat /etc/nginx/lua/cfg.json" + + o, err := f.ExecCommand(f.pod, cmd) + if err != nil { + return false, nil + } + + if klog.V(10).Enabled() && o != "" { + klog.InfoS("Lua", "configuration", o) + } + + luaConfig := make(map[string]interface{}) // Use unstructured so we can walk through JSON + if err := json.Unmarshal([]byte(o), &luaConfig); err != nil { + return false, err + } + + // passes the lua interface to the function + return matcher(luaConfig), nil + } +} + func (f *Framework) matchNginxCustomConditions(from, to string, matcher func(cfg string) bool) wait.ConditionFunc { return func() (bool, error) { cmd := fmt.Sprintf("cat /etc/nginx/nginx.conf| awk '/%v/,/%v/'", from, to) diff --git a/test/e2e/lua/dynamic_configuration.go b/test/e2e/lua/dynamic_configuration.go index 8ec1ef839..a5e2196ce 100644 --- a/test/e2e/lua/dynamic_configuration.go +++ b/test/e2e/lua/dynamic_configuration.go @@ -48,12 +48,7 @@ var _ = framework.IngressNginxDescribe("[Lua] dynamic configuration", func() { ginkgo.It("configures balancer Lua middleware correctly", func() { f.WaitForNginxConfiguration(func(cfg string) bool { - return strings.Contains(cfg, "balancer.init_worker()") && strings.Contains(cfg, "balancer.balance()") - }) - - host := "foo.com" - f.WaitForNginxServer(host, func(server string) bool { - return strings.Contains(server, "balancer.rewrite()") && strings.Contains(server, "balancer.log()") + return strings.Contains(cfg, "balancer_by_lua_file /etc/nginx/lua/nginx/ngx_conf_balancer.lua") }) }) diff --git a/test/e2e/settings/no_tls_redirect_locations.go b/test/e2e/settings/no_tls_redirect_locations.go index 8339eb23e..18fd09e26 100644 --- a/test/e2e/settings/no_tls_redirect_locations.go +++ b/test/e2e/settings/no_tls_redirect_locations.go @@ -33,7 +33,7 @@ var _ = framework.DescribeSetting("Add no tls redirect locations", func() { f.EnsureIngress(ing) f.WaitForNginxConfiguration(func(server string) bool { - return !strings.Contains(server, "force_no_ssl_redirect = true,") + return strings.Contains(server, "set $force_no_ssl_redirect \"false\"") }) wlKey := "no-tls-redirect-locations" @@ -42,7 +42,7 @@ var _ = framework.DescribeSetting("Add no tls redirect locations", func() { f.UpdateNginxConfigMapData(wlKey, wlValue) f.WaitForNginxConfiguration(func(server string) bool { - return strings.Contains(server, "force_no_ssl_redirect = true,") + return strings.Contains(server, "set $force_no_ssl_redirect \"true\"") }) }) }) diff --git a/test/e2e/settings/ocsp/ocsp.go b/test/e2e/settings/ocsp/ocsp.go index ef3bfb58a..5dcaccc2f 100644 --- a/test/e2e/settings/ocsp/ocsp.go +++ b/test/e2e/settings/ocsp/ocsp.go @@ -34,6 +34,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/ingress-nginx/test/e2e/framework" @@ -107,8 +108,9 @@ var _ = framework.DescribeSetting("OCSP", func() { err = framework.WaitForEndpoints(f.KubeClientSet, framework.DefaultTimeout, "ocspserve", f.Namespace, 1) assert.Nil(ginkgo.GinkgoT(), err, "waiting for endpoints to become ready") - f.WaitForNginxConfiguration(func(cfg string) bool { - return strings.Contains(cfg, "certificate.is_ocsp_stapling_enabled = true") + f.WaitForLuaConfiguration(func(jsonCfg map[string]interface{}) bool { + val, ok, err := unstructured.NestedBool(jsonCfg, "enable_ocsp") + return err == nil && ok && val }) f.WaitForNginxServer(host, diff --git a/test/e2e/settings/tls.go b/test/e2e/settings/tls.go index 51f760df8..1ebf358c1 100644 --- a/test/e2e/settings/tls.go +++ b/test/e2e/settings/tls.go @@ -25,10 +25,11 @@ import ( "github.com/onsi/ginkgo/v2" "github.com/stretchr/testify/assert" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/ingress-nginx/test/e2e/framework" ) -var _ = framework.DescribeSetting("[SSL] TLS protocols, ciphers and headers)", func() { +var _ = framework.DescribeSetting("[SSL] TLS protocols, ciphers and headers", func() { f := framework.NewDefaultFramework("settings-tls") host := "settings-tls" @@ -109,8 +110,9 @@ var _ = framework.DescribeSetting("[SSL] TLS protocols, ciphers and headers)", f ginkgo.It("setting max-age parameter", func() { f.UpdateNginxConfigMapData(hstsMaxAge, "86400") - f.WaitForNginxConfiguration(func(server string) bool { - return strings.Contains(server, `hsts_max_age = 86400,`) + f.WaitForLuaConfiguration(func(jsonCfg map[string]interface{}) bool { + val, ok, err := unstructured.NestedString(jsonCfg, "hsts_max_age") + return err == nil && ok && val == "86400" }) f.HTTPTestClientWithTLSConfig(tlsConfig). @@ -128,8 +130,9 @@ var _ = framework.DescribeSetting("[SSL] TLS protocols, ciphers and headers)", f hstsIncludeSubdomains: "false", }) - f.WaitForNginxConfiguration(func(server string) bool { - return strings.Contains(server, `hsts_include_subdomains = false,`) + f.WaitForLuaConfiguration(func(jsonCfg map[string]interface{}) bool { + val, ok, err := unstructured.NestedBool(jsonCfg, "hsts_include_subdomains") + return err == nil && ok && !val }) f.HTTPTestClientWithTLSConfig(tlsConfig). @@ -148,8 +151,9 @@ var _ = framework.DescribeSetting("[SSL] TLS protocols, ciphers and headers)", f hstsIncludeSubdomains: "false", }) - f.WaitForNginxConfiguration(func(server string) bool { - return strings.Contains(server, `hsts_preload = true,`) + f.WaitForLuaConfiguration(func(jsonCfg map[string]interface{}) bool { + val, ok, err := unstructured.NestedBool(jsonCfg, "hsts_preload") + return err == nil && ok && val }) f.HTTPTestClientWithTLSConfig(tlsConfig).