Merge pull request #3604 from ramnes/max-worker-open-files

Add an option to automatically set worker_connections based on worker_rlimit_nofile

docs/user-guide/nginx-configuration/configmap.md

@@ -60,7 +60,8 @@ The following table shows a configuration option's name, type, and the default v
 |[log-format-upstream](#log-format-upstream)|string|`%v - [$the_real_ip] - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" $request_length $request_time [$proxy_upstream_name] $upstream_addr $upstream_response_length $upstream_response_time $upstream_status $req_id`|
 |[log-format-stream](#log-format-stream)|string|`[$time_local] $protocol $status $bytes_sent $bytes_received $session_time`|
 |[enable-multi-accept](#enable-multi-accept)|bool|"true"|
-|[max-worker-connections](#max-worker-connections)|int|16384|
+|[max-worker-connections](#max-worker-connections)|int|0|
+|[max-worker-open-files](#max-worker-open-files)|int|0|
 |[map-hash-bucket-size](#max-worker-connections)|int|64|
 |[nginx-status-ipv4-whitelist](#nginx-status-ipv4-whitelist)|[]string|"127.0.0.1"|
 |[nginx-status-ipv6-whitelist](#nginx-status-ipv6-whitelist)|[]string|"::1"|
@@ -359,7 +360,15 @@ _References:_
 
 ## max-worker-connections
 
-Sets the maximum number of simultaneous connections that can be opened by each [worker process](http://nginx.org/en/docs/ngx_core_module.html#worker_connections)
+Sets the [maximum number of simultaneous connections](http://nginx.org/en/docs/ngx_core_module.html#worker_connections) that can be opened by each worker process.
+The default of 0 uses the value of [max-worker-open-files](#max-worker-open-files).
+_**default:**_ 0
+
+## max-worker-open-files
+
+Sets the [maximum number of files](http://nginx.org/en/docs/ngx_core_module.html#worker_rlimit_nofile) that can be opened by each worker process.
+The default of 0 means "max open files (system's limit) / [worker-processes](#worker-processes) - 1024".
+_**default:**_ 0
 
 ## map-hash-bucket-size
 

internal/ingress/controller/config/config.go

@@ -235,6 +235,10 @@ type Configuration struct {
 	// http://nginx.org/en/docs/ngx_core_module.html#worker_connections
 	MaxWorkerConnections int `json:"max-worker-connections,omitempty"`
 
+	// Maximum number of files that can be opened by each worker process.
+	// http://nginx.org/en/docs/ngx_core_module.html#worker_rlimit_nofile
+	MaxWorkerOpenFiles int `json:"max-worker-open-files,omitempty"`
+
 	// Sets the bucket size for the map variables hash tables.
 	// Default value depends on the processor’s cache line size.
 	// http://nginx.org/en/docs/http/ngx_http_map_module.html#map_hash_bucket_size
@@ -605,7 +609,8 @@ func NewDefault() Configuration {
 		LogFormatStream:            logFormatStream,
 		LogFormatUpstream:          logFormatUpstream,
 		EnableMultiAccept:          true,
-		MaxWorkerConnections:       16384,
+		MaxWorkerConnections:       0,
+		MaxWorkerOpenFiles:         0,
 		MapHashBucketSize:          64,
 		NginxStatusIpv4Whitelist:   defNginxStatusIpv4Whitelist,
 		NginxStatusIpv6Whitelist:   defNginxStatusIpv6Whitelist,
@@ -697,7 +702,6 @@ func (cfg Configuration) BuildLogFormatUpstream() string {
 type TemplateConfig struct {
 	ProxySetHeaders            map[string]string
 	AddHeaders                 map[string]string
-	MaxOpenFiles               int
 	BacklogSize                int
 	Backends                   []*ingress.Backend
 	PassthroughBackends        []*ingress.SSLPassthroughBackend

internal/ingress/controller/nginx.go

@@ -522,18 +522,27 @@ func (n *NGINXController) OnUpdate(ingressCfg ingress.Configuration) error {
 		cfg.ServerNameHashMaxSize = serverNameHashMaxSize
 	}
 
-	// the limit of open files is per worker process
+	if cfg.MaxWorkerOpenFiles == 0 {
-	// and we leave some room to avoid consuming all the FDs available
+		// the limit of open files is per worker process
-	wp, err := strconv.Atoi(cfg.WorkerProcesses)
+		// and we leave some room to avoid consuming all the FDs available
-	klog.V(3).Infof("Number of worker processes: %d", wp)
+		wp, err := strconv.Atoi(cfg.WorkerProcesses)
-	if err != nil {
+		klog.V(3).Infof("Number of worker processes: %d", wp)
-		wp = 1
+		if err != nil {
+			wp = 1
+		}
+		maxOpenFiles := (sysctlFSFileMax() / wp) - 1024
+		klog.V(3).Infof("Maximum number of open file descriptors: %d", maxOpenFiles)
+		if maxOpenFiles < 1024 {
+			// this means the value of RLIMIT_NOFILE is too low.
+			maxOpenFiles = 1024
+		}
+		klog.V(3).Infof("Adjusting MaxWorkerOpenFiles variable to %d", maxOpenFiles)
+		cfg.MaxWorkerOpenFiles = maxOpenFiles
 	}
-	maxOpenFiles := (sysctlFSFileMax() / wp) - 1024
+
-	klog.V(2).Infof("Maximum number of open file descriptors: %d", maxOpenFiles)
+	if cfg.MaxWorkerConnections == 0 {
-	if maxOpenFiles < 1024 {
+		klog.V(3).Infof("Adjusting MaxWorkerConnections variable to %d", cfg.MaxWorkerOpenFiles)
-		// this means the value of RLIMIT_NOFILE is too low.
+		cfg.MaxWorkerConnections = cfg.MaxWorkerOpenFiles
-		maxOpenFiles = 1024
 	}
 
 	setHeaders := map[string]string{}
@@ -583,7 +592,6 @@ func (n *NGINXController) OnUpdate(ingressCfg ingress.Configuration) error {
 	tc := ngx_config.TemplateConfig{
 		ProxySetHeaders:            setHeaders,
 		AddHeaders:                 addHeaders,
-		MaxOpenFiles:               maxOpenFiles,
 		BacklogSize:                sysctlSomaxconn(),
 		Backends:                   ingressCfg.Backends,
 		PassthroughBackends:        ingressCfg.PassthroughBackends,

rootfs/etc/nginx/template/nginx.tmpl

@@ -29,9 +29,7 @@ worker_processes {{ $cfg.WorkerProcesses }};
 worker_cpu_affinity {{ $cfg.WorkerCPUAffinity }};
 {{ end }}
 
-{{ if ne .MaxOpenFiles 0 }}
+worker_rlimit_nofile {{ $cfg.MaxWorkerOpenFiles }};
-worker_rlimit_nofile {{ .MaxOpenFiles }};
-{{ end }}
 
 {{/* http://nginx.org/en/docs/ngx_core_module.html#worker_shutdown_timeout */}}
 {{/* avoid waiting too long during a reload */}}