From 8998f31a3ba24dadda279a985c5668a44d9bb911 Mon Sep 17 00:00:00 2001 From: Andrew Davidoff Date: Thu, 9 Nov 2017 12:41:01 -0700 Subject: [PATCH] Implement loggable map for HTTP status --- internal/ingress/controller/config/config.go | 33 ++++++++++--------- .../ingress/controller/template/configmap.go | 23 ++++++++----- .../controller/template/configmap_test.go | 30 +++++++++-------- internal/ingress/defaults/main.go | 6 ++++ rootfs/etc/nginx/template/nginx.tmpl | 13 +++++++- 5 files changed, 66 insertions(+), 39 deletions(-) diff --git a/internal/ingress/controller/config/config.go b/internal/ingress/controller/config/config.go index e35ce77e7..353bbbbeb 100644 --- a/internal/ingress/controller/config/config.go +++ b/internal/ingress/controller/config/config.go @@ -532,22 +532,23 @@ func NewDefault() Configuration { UseHTTP2: true, ProxyStreamTimeout: "600s", Backend: defaults.Backend{ - ProxyBodySize: bodySize, - ProxyConnectTimeout: 5, - ProxyReadTimeout: 60, - ProxySendTimeout: 60, - ProxyBufferSize: "4k", - ProxyCookieDomain: "off", - ProxyCookiePath: "off", - ProxyNextUpstream: "error timeout invalid_header http_502 http_503 http_504", - ProxyRequestBuffering: "on", - ProxyRedirectFrom: "off", - SSLRedirect: true, - CustomHTTPErrors: []int{}, - WhitelistSourceRange: []string{}, - SkipAccessLogURLs: []string{}, - LimitRate: 0, - LimitRateAfter: 0, + ProxyBodySize: bodySize, + ProxyConnectTimeout: 5, + ProxyReadTimeout: 60, + ProxySendTimeout: 60, + ProxyBufferSize: "4k", + ProxyCookieDomain: "off", + ProxyCookiePath: "off", + ProxyNextUpstream: "error timeout invalid_header http_502 http_503 http_504", + ProxyRequestBuffering: "on", + ProxyRedirectFrom: "off", + SSLRedirect: true, + CustomHTTPErrors: []int{}, + WhitelistSourceRange: []string{}, + SkipAccessLogURLs: []string{}, + SkipAccessLogHTTPStatuses: []string{}, + LimitRate: 0, + LimitRateAfter: 0, }, UpstreamKeepaliveConnections: 32, LimitConnZoneVariable: defaultLimitConnZoneVariable, diff --git a/internal/ingress/controller/template/configmap.go b/internal/ingress/controller/template/configmap.go index b9d7d0249..a2d9e1164 100644 --- a/internal/ingress/controller/template/configmap.go +++ b/internal/ingress/controller/template/configmap.go @@ -32,14 +32,15 @@ import ( ) const ( - customHTTPErrors = "custom-http-errors" - skipAccessLogUrls = "skip-access-log-urls" - whitelistSourceRange = "whitelist-source-range" - proxyRealIPCIDR = "proxy-real-ip-cidr" - bindAddress = "bind-address" - httpRedirectCode = "http-redirect-code" - proxyStreamResponses = "proxy-stream-responses" - hideHeaders = "hide-headers" + customHTTPErrors = "custom-http-errors" + skipAccessLogUrls = "skip-access-log-urls" + skipAccessLogHTTPStatuses = "skip-access-log-http-statuses" + whitelistSourceRange = "whitelist-source-range" + proxyRealIPCIDR = "proxy-real-ip-cidr" + bindAddress = "bind-address" + httpRedirectCode = "http-redirect-code" + proxyStreamResponses = "proxy-stream-responses" + hideHeaders = "hide-headers" ) var ( @@ -56,6 +57,7 @@ func ReadConfig(src map[string]string) config.Configuration { errors := make([]int, 0) skipUrls := make([]string, 0) + skipHTTPStatuses := make([]string, 0) whitelist := make([]string, 0) proxylist := make([]string, 0) hideHeaderslist := make([]string, 0) @@ -83,6 +85,10 @@ func ReadConfig(src map[string]string) config.Configuration { delete(conf, skipAccessLogUrls) skipUrls = strings.Split(val, ",") } + if val, ok := conf[skipAccessLogHTTPStatuses]; ok { + delete(conf, skipAccessLogHTTPStatuses) + skipHTTPStatuses = strings.Split(val, ",") + } if val, ok := conf[whitelistSourceRange]; ok { delete(conf, whitelistSourceRange) whitelist = append(whitelist, strings.Split(val, ",")...) @@ -137,6 +143,7 @@ func ReadConfig(src map[string]string) config.Configuration { to := config.NewDefault() to.CustomHTTPErrors = filterErrors(errors) to.SkipAccessLogURLs = skipUrls + to.SkipAccessLogHTTPStatuses = skipHTTPStatuses to.WhitelistSourceRange = whitelist to.ProxyRealIPCIDR = proxylist to.BindAddressIpv4 = bindAddressIpv4List diff --git a/internal/ingress/controller/template/configmap_test.go b/internal/ingress/controller/template/configmap_test.go index 1df9eae3f..fa16c1e16 100644 --- a/internal/ingress/controller/template/configmap_test.go +++ b/internal/ingress/controller/template/configmap_test.go @@ -33,20 +33,21 @@ func TestFilterErrors(t *testing.T) { func TestMergeConfigMapToStruct(t *testing.T) { conf := map[string]string{ - "custom-http-errors": "300,400,demo", - "proxy-read-timeout": "1", - "proxy-send-timeout": "2", - "skip-access-log-urls": "/log,/demo,/test", - "use-proxy-protocol": "true", - "disable-access-log": "true", - "access-log-path": "/var/log/test/access.log", - "error-log-path": "/var/log/test/error.log", - "use-gzip": "true", - "enable-dynamic-tls-records": "false", - "gzip-types": "text/html", - "proxy-real-ip-cidr": "1.1.1.1/8,2.2.2.2/24", - "bind-address": "1.1.1.1,2.2.2.2,3.3.3,2001:db8:a0b:12f0::1,3731:54:65fe:2::a7,33:33:33::33::33", - "worker-shutdown-timeout": "99s", + "custom-http-errors": "300,400,demo", + "proxy-read-timeout": "1", + "proxy-send-timeout": "2", + "skip-access-log-urls": "/log,/demo,/test", + "skip-access-log-http-statuses": "^[23],204,302,^201", + "use-proxy-protocol": "true", + "disable-access-log": "true", + "access-log-path": "/var/log/test/access.log", + "error-log-path": "/var/log/test/error.log", + "use-gzip": "true", + "enable-dynamic-tls-records": "false", + "gzip-types": "text/html", + "proxy-real-ip-cidr": "1.1.1.1/8,2.2.2.2/24", + "bind-address": "1.1.1.1,2.2.2.2,3.3.3,2001:db8:a0b:12f0::1,3731:54:65fe:2::a7,33:33:33::33::33", + "worker-shutdown-timeout": "99s", } def := config.NewDefault() def.CustomHTTPErrors = []int{300, 400} @@ -54,6 +55,7 @@ func TestMergeConfigMapToStruct(t *testing.T) { def.AccessLogPath = "/var/log/test/access.log" def.ErrorLogPath = "/var/log/test/error.log" def.SkipAccessLogURLs = []string{"/log", "/demo", "/test"} + def.SkipAccessLogHTTPStatuses = []string{"^[23]", "204", "302", "^201"} def.ProxyReadTimeout = 1 def.ProxySendTimeout = 2 def.EnableDynamicTLSRecords = false diff --git a/internal/ingress/defaults/main.go b/internal/ingress/defaults/main.go index b9ee87626..e74c0daf5 100644 --- a/internal/ingress/defaults/main.go +++ b/internal/ingress/defaults/main.go @@ -95,6 +95,12 @@ type Backend struct { // By default this list is empty SkipAccessLogURLs []string `json:"skip-access-log-urls,-"` + // SkipAccessLogHTTPStatuses sets a list of HTTP statuses that should not appear in the NGINX access log + // The status strings provided are interpreted by an NGINX map as regex + // This is useful with statuses like 2xx and 3xx that make "complex" reading the logs + // By default this list is empty + SkipAccessLogHTTPStatuses []string `json:"skip-access-log-http-statuses,-"` + // Enables or disables the redirect (301) to the HTTPS port SSLRedirect bool `json:"ssl-redirect"` diff --git a/rootfs/etc/nginx/template/nginx.tmpl b/rootfs/etc/nginx/template/nginx.tmpl index c9f1323dc..7981e1e78 100644 --- a/rootfs/etc/nginx/template/nginx.tmpl +++ b/rootfs/etc/nginx/template/nginx.tmpl @@ -157,9 +157,20 @@ http { {{/* map urls that should not appear in access.log */}} {{/* http://nginx.org/en/docs/http/ngx_http_log_module.html#access_log */}} - map $request_uri $loggable { + map $request_uri $loggable_request_url { {{ range $reqUri := $cfg.SkipAccessLogURLs }} {{ $reqUri }} 0;{{ end }} + } + + {{/* map HTTP statuses that should not appear in access.log */}} + {{/* http://nginx.org/en/docs/http/ngx_http_log_module.html#access_log */}} + map $status $loggable_http_status { + {{ range $reqHTTPStatus := $cfg.SkipAccessLogHTTPStatuses }} + ~{{ $reqHTTPStatus }} 0;{{ end }} + } + + map "${loggable_request_url}${loggable_http_status}" $loggable { + ~0 0; default 1; }