From d4b9b486e680fab8d78a078f24c96a3f378124f6 Mon Sep 17 00:00:00 2001 From: sskserk <78915702+sskserk@users.noreply.github.com> Date: Fri, 15 Apr 2022 13:09:10 +0200 Subject: [PATCH] Upstream keepalive time (#8319) * nginx 1.19.10 keepalive_time parameter * nginx v1.19.10 base image * keepalive_time documentation * base image * restore base image * e2e test * replace default value in test --- .../user-guide/nginx-configuration/configmap.md | 17 +++++++++++++---- internal/ingress/controller/config/config.go | 5 +++++ rootfs/etc/nginx/template/nginx.tmpl | 4 ++-- test/e2e/settings/keep-alive.go | 9 +++++++++ 4 files changed, 29 insertions(+), 6 deletions(-) diff --git a/docs/user-guide/nginx-configuration/configmap.md b/docs/user-guide/nginx-configuration/configmap.md index 27ef647ef..6cec6f02f 100755 --- a/docs/user-guide/nginx-configuration/configmap.md +++ b/docs/user-guide/nginx-configuration/configmap.md @@ -111,6 +111,7 @@ The following table shows a configuration option's name, type, and the default v |[variables-hash-bucket-size](#variables-hash-bucket-size)|int|128| |[variables-hash-max-size](#variables-hash-max-size)|int|2048| |[upstream-keepalive-connections](#upstream-keepalive-connections)|int|320| +|[upstream-keepalive-time](#upstream-keepalive-time)|string|"1h"| |[upstream-keepalive-timeout](#upstream-keepalive-timeout)|int|60| |[upstream-keepalive-requests](#upstream-keepalive-requests)|int|10000| |[limit-conn-zone-variable](#limit-conn-zone-variable)|string|"$binary_remote_addr"| @@ -223,13 +224,13 @@ Enables the return of the header Server from the backend instead of the generic Enables Ingress to parse and add *-snippet annotations/directives created by the user. _**default:**_ `true` -Warning: We recommend enabling this option only if you TRUST users with permission to create Ingress objects, as this +Warning: We recommend enabling this option only if you TRUST users with permission to create Ingress objects, as this may allow a user to add restricted configurations to the final nginx.conf file ## annotation-value-word-blocklist -Contains a comma-separated value of chars/words that are well known of being used to abuse Ingress configuration -and must be blocked. Related to [CVE-2021-25742](https://github.com/kubernetes/ingress-nginx/issues/7837) +Contains a comma-separated value of chars/words that are well known of being used to abuse Ingress configuration +and must be blocked. Related to [CVE-2021-25742](https://github.com/kubernetes/ingress-nginx/issues/7837) When an annotation is detected with a value that matches one of the blocked bad words, the whole Ingress won't be configured. @@ -769,6 +770,14 @@ _References:_ [https://nginx.org/en/docs/http/ngx_http_upstream_module.html#keepalive](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#keepalive) +## upstream-keepalive-time + +Sets the maximum time during which requests can be processed through one keepalive connection. + _**default:**_ "1h" + +_References:_ +[http://nginx.org/en/docs/http/ngx_http_upstream_module.html#keepalive_time](http://nginx.org/en/docs/http/ngx_http_upstream_module.html#keepalive_time) + ## upstream-keepalive-timeout Sets a timeout during which an idle keepalive connection to an upstream server will stay open. @@ -1258,7 +1267,7 @@ Configure `memcached` client for [Global Rate Limiting](https://github.com/kuber * `global-rate-limit-memcached-host`: IP/FQDN of memcached server to use. Required to enable Global Rate Limiting. * `global-rate-limit-memcached-port`: port of memcached server to use. Defaults default memcached port of `11211`. * `global-rate-limit-memcached-connect-timeout`: configure timeout for connect, send and receive operations. Unit is millisecond. Defaults to 50ms. -* `global-rate-limit-memcached-max-idle-timeout`: configure timeout for cleaning idle connections. Unit is millisecond. Defaults to 50ms. +* `global-rate-limit-memcached-max-idle-timeout`: configure timeout for cleaning idle connections. Unit is millisecond. Defaults to 50ms. * `global-rate-limit-memcached-pool-size`: configure number of max connections to keep alive. Make sure your `memcached` server can handle `global-rate-limit-memcached-pool-size * worker-processes * ` simultaneous connections. diff --git a/internal/ingress/controller/config/config.go b/internal/ingress/controller/config/config.go index 4afb3e9f5..f0dfdc191 100644 --- a/internal/ingress/controller/config/config.go +++ b/internal/ingress/controller/config/config.go @@ -468,6 +468,10 @@ type Configuration struct { // http://nginx.org/en/docs/http/ngx_http_upstream_module.html#keepalive UpstreamKeepaliveConnections int `json:"upstream-keepalive-connections,omitempty"` + // Sets the maximum time during which requests can be processed through one keepalive connection + // https://nginx.org/en/docs/http/ngx_http_upstream_module.html#keepalive_time + UpstreamKeepaliveTime string `json:"upstream-keepalive-time,omitempty"` + // Sets a timeout during which an idle keepalive connection to an upstream server will stay open. // http://nginx.org/en/docs/http/ngx_http_upstream_module.html#keepalive_timeout UpstreamKeepaliveTimeout int `json:"upstream-keepalive-timeout,omitempty"` @@ -892,6 +896,7 @@ func NewDefault() Configuration { ServiceUpstream: false, }, UpstreamKeepaliveConnections: 320, + UpstreamKeepaliveTime: "1h", UpstreamKeepaliveTimeout: 60, UpstreamKeepaliveRequests: 10000, LimitConnZoneVariable: defaultLimitConnZoneVariable, diff --git a/rootfs/etc/nginx/template/nginx.tmpl b/rootfs/etc/nginx/template/nginx.tmpl index b845988ad..1ad8458c5 100755 --- a/rootfs/etc/nginx/template/nginx.tmpl +++ b/rootfs/etc/nginx/template/nginx.tmpl @@ -156,7 +156,7 @@ http { {{ else }} modsecurity_rules_file /etc/nginx/modsecurity/modsecurity.conf; {{ end }} - + {{ if $all.Cfg.EnableOWASPCoreRules }} modsecurity_rules_file /etc/nginx/owasp-modsecurity-crs/nginx-modsecurity.conf; {{ end }} @@ -508,7 +508,7 @@ http { {{ if (gt $cfg.UpstreamKeepaliveConnections 0) }} keepalive {{ $cfg.UpstreamKeepaliveConnections }}; - + keepalive_time {{ $cfg.UpstreamKeepaliveTime }}; keepalive_timeout {{ $cfg.UpstreamKeepaliveTimeout }}s; keepalive_requests {{ $cfg.UpstreamKeepaliveRequests }}; {{ end }} diff --git a/test/e2e/settings/keep-alive.go b/test/e2e/settings/keep-alive.go index 5a2b5189e..6ef09b78c 100644 --- a/test/e2e/settings/keep-alive.go +++ b/test/e2e/settings/keep-alive.go @@ -74,6 +74,15 @@ var _ = framework.DescribeSetting("keep-alive keep-alive-requests", func() { }) }) + ginkgo.It("should set keepalive time to upstream server", func() { + f.UpdateNginxConfigMapData("upstream-keepalive-time", "75s") + + f.WaitForNginxConfiguration(func(server string) bool { + match, _ := regexp.MatchString(`upstream\supstream_balancer\s\{[\s\S]*keepalive_time\s*75s;`, server) + return match + }) + }) + ginkgo.It("should set the request count to upstream server through one keep alive connection", func() { f.UpdateNginxConfigMapData("upstream-keepalive-requests", "200")