From cae935d32a0388532ef11b7169aaf435ef9aeecf Mon Sep 17 00:00:00 2001 From: Andrew Davidoff Date: Thu, 9 Nov 2017 12:41:01 -0700 Subject: [PATCH] Implement loggable map for HTTP status --- docs/user-guide/nginx-configuration/configmap.md | 5 +++++ internal/ingress/controller/config/config.go | 1 + internal/ingress/controller/template/configmap.go | 8 ++++++++ .../ingress/controller/template/configmap_test.go | 2 ++ internal/ingress/defaults/main.go | 6 ++++++ rootfs/etc/nginx/template/nginx.tmpl | 13 ++++++++++++- 6 files changed, 34 insertions(+), 1 deletion(-) diff --git a/docs/user-guide/nginx-configuration/configmap.md b/docs/user-guide/nginx-configuration/configmap.md index c55b7502a..f11231a83 100644 --- a/docs/user-guide/nginx-configuration/configmap.md +++ b/docs/user-guide/nginx-configuration/configmap.md @@ -194,6 +194,7 @@ The following table shows a configuration option's name, type, and the default v |[denylist-source-range](#denylist-source-range)|[]string|[]string{}|| |[whitelist-source-range](#whitelist-source-range)|[]string|[]string{}|| |[skip-access-log-urls](#skip-access-log-urls)|[]string|[]string{}|| +|[skip-access-log-http-statuses](#skip-access-log-http-statuses)|[]string|[]string{}|| |[limit-rate](#limit-rate)|int|0|| |[limit-rate-after](#limit-rate-after)|int|0|| |[lua-shared-dicts](#lua-shared-dicts)|string|""|| @@ -1172,6 +1173,10 @@ See [ngx_http_access_module](https://nginx.org/en/docs/http/ngx_http_access_modu Sets a list of URLs that should not appear in the NGINX access log. This is useful with urls like `/health` or `health-check` that make "complex" reading the logs. _**default:**_ is empty +## skip-access-log-http-statuses + +Sets a list of HTTP statuses that should not appear in the NGINX access log. This is useful for high volume ingress where turning access logging completely off is undesirable, but not logging things like 2xx and 3xx responses is desirable. _**default:**_ is empty + ## limit-rate Limits the rate of response transmission to a client. The rate is specified in bytes per second. The zero value disables rate limiting. The limit is set per a request, and so if a client simultaneously opens two connections, the overall rate will be twice as much as the specified limit. diff --git a/internal/ingress/controller/config/config.go b/internal/ingress/controller/config/config.go index ec44b08ed..62453fa71 100644 --- a/internal/ingress/controller/config/config.go +++ b/internal/ingress/controller/config/config.go @@ -962,6 +962,7 @@ func NewDefault() Configuration { DenylistSourceRange: []string{}, WhitelistSourceRange: []string{}, SkipAccessLogURLs: []string{}, + SkipAccessLogHTTPStatuses: []string{}, LimitRate: 0, LimitRateAfter: 0, ProxyBuffering: "off", diff --git a/internal/ingress/controller/template/configmap.go b/internal/ingress/controller/template/configmap.go index c73f3b6c0..0e7e69f08 100644 --- a/internal/ingress/controller/template/configmap.go +++ b/internal/ingress/controller/template/configmap.go @@ -40,6 +40,7 @@ import ( const ( customHTTPErrors = "custom-http-errors" skipAccessLogUrls = "skip-access-log-urls" + skipAccessLogHTTPStatuses = "skip-access-log-http-statuses" whitelistSourceRange = "whitelist-source-range" denylistSourceRange = "denylist-source-range" proxyRealIPCIDR = "proxy-real-ip-cidr" @@ -101,6 +102,7 @@ func ReadConfig(src map[string]string) config.Configuration { to := config.NewDefault() errors := make([]int, 0) skipUrls := make([]string, 0) + skipHTTPStatuses := make([]string, 0) denyList := make([]string, 0) whiteList := make([]string, 0) proxyList := make([]string, 0) @@ -171,6 +173,11 @@ func ReadConfig(src map[string]string) config.Configuration { skipUrls = splitAndTrimSpace(val, ",") } + if val, ok := conf[skipAccessLogHTTPStatuses]; ok { + delete(conf, skipAccessLogHTTPStatuses) + skipHTTPStatuses = strings.Split(val, ",") + } + if val, ok := conf[denylistSourceRange]; ok { delete(conf, denylistSourceRange) denyList = append(denyList, splitAndTrimSpace(val, ",")...) @@ -402,6 +409,7 @@ func ReadConfig(src map[string]string) config.Configuration { to.CustomHTTPErrors = filterErrors(errors) to.SkipAccessLogURLs = skipUrls + to.SkipAccessLogHTTPStatuses = skipHTTPStatuses to.DenylistSourceRange = denyList to.WhitelistSourceRange = whiteList to.ProxyRealIPCIDR = proxyList diff --git a/internal/ingress/controller/template/configmap_test.go b/internal/ingress/controller/template/configmap_test.go index dad841694..3184a956a 100644 --- a/internal/ingress/controller/template/configmap_test.go +++ b/internal/ingress/controller/template/configmap_test.go @@ -58,6 +58,7 @@ func TestMergeConfigMapToStruct(t *testing.T) { "proxy-read-timeout": "1", "proxy-send-timeout": "2", "skip-access-log-urls": "/log,/demo,/test", + "skip-access-log-http-statuses": "^[23],204,302,^201", "use-proxy-protocol": "true", "disable-access-log": "true", "access-log-params": "buffer=4k gzip", @@ -85,6 +86,7 @@ func TestMergeConfigMapToStruct(t *testing.T) { def.AccessLogPath = "/var/log/test/access.log" def.ErrorLogPath = "/var/log/test/error.log" def.SkipAccessLogURLs = []string{"/log", "/demo", "/test"} + def.SkipAccessLogHTTPStatuses = []string{"^[23]", "204", "302", "^201"} def.ProxyReadTimeout = 1 def.ProxySendTimeout = 2 def.UseProxyProtocol = true diff --git a/internal/ingress/defaults/main.go b/internal/ingress/defaults/main.go index 0aab2ff47..4c89dfb14 100644 --- a/internal/ingress/defaults/main.go +++ b/internal/ingress/defaults/main.go @@ -107,6 +107,12 @@ type Backend struct { // By default this list is empty SkipAccessLogURLs []string `json:"skip-access-log-urls"` + // SkipAccessLogHTTPStatuses sets a list of HTTP statuses that should not appear in the NGINX access log + // The status strings provided are interpreted by an NGINX map as regex + // This is useful with statuses like 2xx and 3xx that make "complex" reading the logs + // By default this list is empty + SkipAccessLogHTTPStatuses []string `json:"skip-access-log-http-statuses,-"` + // Enables or disables the redirect (301) to the HTTPS port SSLRedirect bool `json:"ssl-redirect"` diff --git a/rootfs/etc/nginx/template/nginx.tmpl b/rootfs/etc/nginx/template/nginx.tmpl index a1e02aae3..70d739ae7 100644 --- a/rootfs/etc/nginx/template/nginx.tmpl +++ b/rootfs/etc/nginx/template/nginx.tmpl @@ -391,9 +391,20 @@ http { {{/* map urls that should not appear in access.log */}} {{/* http://nginx.org/en/docs/http/ngx_http_log_module.html#access_log */}} - map $request_uri $loggable { + map $request_uri $loggable_request_url { {{ range $reqUri := $cfg.SkipAccessLogURLs }} {{ $reqUri }} 0;{{ end }} + } + + {{/* map HTTP statuses that should not appear in access.log */}} + {{/* http://nginx.org/en/docs/http/ngx_http_log_module.html#access_log */}} + map $status $loggable_http_status { + {{ range $reqHTTPStatus := $cfg.SkipAccessLogHTTPStatuses }} + ~{{ $reqHTTPStatus }} 0;{{ end }} + } + + map "${loggable_request_url}${loggable_http_status}" $loggable { + ~0 0; default 1; }