Implement reporting status classes in metrics (#8548)
This commit introduces a backwards compatible command line option --report-status-classes which will enable reporting response status classes (2xx, 3xx..) instead of status codes in exported metrics.
This commit is contained in:
Filip Petkovski
GitHub
parent
0240dd3fba
commit
4da96ea26a
7 changed files with 71 additions and 16 deletions
|
|
@ -163,6 +163,9 @@ Requires the update-status parameter.`)
|
|||
`Enables the collection of NGINX metrics`)
|
||||
metricsPerHost = flags.Bool("metrics-per-host", true,
|
||||
`Export metrics per-host`)
|
||||
reportStatusClasses = flags.Bool("report-status-classes", false,
|
||||
`Use status classes (2xx, 3xx, 4xx and 5xx) instead of status codes in metrics`)
|
||||
|
||||
timeBuckets = flags.Float64Slice("time-buckets", prometheus.DefBuckets, "Set of buckets which will be used for prometheus histogram metrics such as RequestTime, ResponseTime")
|
||||
lengthBuckets = flags.Float64Slice("length-buckets", prometheus.LinearBuckets(10, 10, 10), "Set of buckets which will be used for prometheus histogram metrics such as RequestLength, ResponseLength")
|
||||
sizeBuckets = flags.Float64Slice("size-buckets", prometheus.ExponentialBuckets(10, 10, 7), "Set of buckets which will be used for prometheus histogram metrics such as BytesSent")
|
||||
|
|
@ -313,6 +316,7 @@ https://blog.maxmind.com/2019/12/18/significant-changes-to-accessing-and-using-g
|
|||
EnableMetrics: *enableMetrics,
|
||||
MetricsPerHost: *metricsPerHost,
|
||||
MetricsBuckets: histogramBuckets,
|
||||
ReportStatusClasses: *reportStatusClasses,
|
||||
MonitorMaxBatchSize: *monitorMaxBatchSize,
|
||||
DisableServiceExternalName: *disableServiceExternalName,
|
||||
EnableSSLPassthrough: *enableSSLPassthrough,
|
||||
|
|
|
|||
|
|
@ -133,7 +133,7 @@ func main() {
|
|||
|
||||
mc := metric.NewDummyCollector()
|
||||
if conf.EnableMetrics {
|
||||
mc, err = metric.NewCollector(conf.MetricsPerHost, reg, conf.IngressClassConfiguration.Controller, *conf.MetricsBuckets)
|
||||
mc, err = metric.NewCollector(conf.MetricsPerHost, conf.ReportStatusClasses, reg, conf.IngressClassConfiguration.Controller, *conf.MetricsBuckets)
|
||||
if err != nil {
|
||||
klog.Fatalf("Error creating prometheus collector: %v", err)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -47,6 +47,7 @@ They are set in the container spec of the `ingress-nginx-controller` Deployment
|
|||
| `--publish-service` | Service fronting the Ingress controller. Takes the form "namespace/name". When used together with update-status, the controller mirrors the address of this service's endpoints to the load-balancer status of all Ingress objects it satisfies. |
|
||||
| `--publish-status-address` | Customized address (or addresses, separated by comma) to set as the load-balancer status of Ingress objects this controller satisfies. Requires the update-status parameter. |
|
||||
| `--report-node-internal-ip-address`| Set the load-balancer status of Ingress objects to internal Node addresses instead of external. Requires the update-status parameter. |
|
||||
| `--report-status-classes` | If true, report status classes in metrics (2xx, 3xx, 4xx and 5xx) instead of full status codes. (default false) |
|
||||
| `--skip_headers` | If true, avoid header prefixes in the log messages |
|
||||
| `--skip_log_headers` | If true, avoid headers when opening log files |
|
||||
| `--ssl-passthrough-proxy-port` | Port to use internally for SSL Passthrough. (default 442) |
|
||||
|
|
|
|||
|
|
@ -97,9 +97,10 @@ type Configuration struct {
|
|||
|
||||
EnableProfiling bool
|
||||
|
||||
EnableMetrics bool
|
||||
MetricsPerHost bool
|
||||
MetricsBuckets *collectors.HistogramBuckets
|
||||
EnableMetrics bool
|
||||
MetricsPerHost bool
|
||||
MetricsBuckets *collectors.HistogramBuckets
|
||||
ReportStatusClasses bool
|
||||
|
||||
FakeCertificate *ingress.SSLCert
|
||||
|
||||
|
|
|
|||
|
|
@ -85,7 +85,8 @@ type SocketCollector struct {
|
|||
|
||||
hosts sets.String
|
||||
|
||||
metricsPerHost bool
|
||||
metricsPerHost bool
|
||||
reportStatusClasses bool
|
||||
|
||||
buckets HistogramBuckets
|
||||
}
|
||||
|
|
@ -110,7 +111,7 @@ var defObjectives = map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}
|
|||
|
||||
// NewSocketCollector creates a new SocketCollector instance using
|
||||
// the ingress watch namespace and class used by the controller
|
||||
func NewSocketCollector(pod, namespace, class string, metricsPerHost bool, buckets HistogramBuckets) (*SocketCollector, error) {
|
||||
func NewSocketCollector(pod, namespace, class string, metricsPerHost, reportStatusClasses bool, buckets HistogramBuckets) (*SocketCollector, error) {
|
||||
socket := "/tmp/nginx/prometheus-nginx.socket"
|
||||
// unix sockets must be unlink()ed before being used
|
||||
_ = syscall.Unlink(socket)
|
||||
|
|
@ -139,7 +140,8 @@ func NewSocketCollector(pod, namespace, class string, metricsPerHost bool, bucke
|
|||
sc := &SocketCollector{
|
||||
listener: listener,
|
||||
|
||||
metricsPerHost: metricsPerHost,
|
||||
metricsPerHost: metricsPerHost,
|
||||
reportStatusClasses: reportStatusClasses,
|
||||
|
||||
responseTime: prometheus.NewHistogramVec(
|
||||
prometheus.HistogramOpts{
|
||||
|
|
@ -249,6 +251,10 @@ func (sc *SocketCollector) handleMessage(msg []byte) {
|
|||
continue
|
||||
}
|
||||
|
||||
if sc.reportStatusClasses && len(stats.Status) > 0 {
|
||||
stats.Status = fmt.Sprintf("%cxx", stats.Status[0])
|
||||
}
|
||||
|
||||
// Note these must match the order in requestTags at the top
|
||||
requestLabels := prometheus.Labels{
|
||||
"status": stats.Status,
|
||||
|
|
|
|||
|
|
@ -80,12 +80,13 @@ func TestCollector(t *testing.T) {
|
|||
}
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
data []string
|
||||
metrics []string
|
||||
wantBefore string
|
||||
removeIngresses []string
|
||||
wantAfter string
|
||||
name string
|
||||
data []string
|
||||
metrics []string
|
||||
useStatusClasses bool
|
||||
wantBefore string
|
||||
removeIngresses []string
|
||||
wantAfter string
|
||||
}{
|
||||
{
|
||||
name: "invalid metric object should not increase prometheus metrics",
|
||||
|
|
@ -371,13 +372,55 @@ func TestCollector(t *testing.T) {
|
|||
wantAfter: `
|
||||
`,
|
||||
},
|
||||
{
|
||||
name: "valid metric object with status classes should update prometheus metrics",
|
||||
data: []string{`[{
|
||||
"host":"testshop.com",
|
||||
"status":"200",
|
||||
"bytesSent":150.0,
|
||||
"method":"GET",
|
||||
"path":"/admin",
|
||||
"requestLength":300.0,
|
||||
"requestTime":60.0,
|
||||
"upstreamName":"test-upstream",
|
||||
"upstreamIP":"1.1.1.1:8080",
|
||||
"upstreamResponseTime":200,
|
||||
"upstreamStatus":"220",
|
||||
"namespace":"test-app-production",
|
||||
"ingress":"web-yml",
|
||||
"service":"test-app",
|
||||
"canary":""
|
||||
}]`},
|
||||
metrics: []string{"nginx_ingress_controller_response_duration_seconds"},
|
||||
useStatusClasses: true,
|
||||
wantBefore: `
|
||||
# HELP nginx_ingress_controller_response_duration_seconds The time spent on receiving the response from the upstream server
|
||||
# TYPE nginx_ingress_controller_response_duration_seconds histogram
|
||||
nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="2xx",le="0.005"} 0
|
||||
nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="2xx",le="0.01"} 0
|
||||
nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="2xx",le="0.025"} 0
|
||||
nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="2xx",le="0.05"} 0
|
||||
nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="2xx",le="0.1"} 0
|
||||
nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="2xx",le="0.25"} 0
|
||||
nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="2xx",le="0.5"} 0
|
||||
nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="2xx",le="1"} 0
|
||||
nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="2xx",le="2.5"} 0
|
||||
nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="2xx",le="5"} 0
|
||||
nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="2xx",le="10"} 0
|
||||
nginx_ingress_controller_response_duration_seconds_bucket{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="2xx",le="+Inf"} 1
|
||||
nginx_ingress_controller_response_duration_seconds_sum{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="2xx"} 200
|
||||
nginx_ingress_controller_response_duration_seconds_count{canary="",controller_class="ingress",controller_namespace="default",controller_pod="pod",host="testshop.com",ingress="web-yml",method="GET",namespace="test-app-production",path="/admin",service="test-app",status="2xx"} 1
|
||||
`,
|
||||
wantAfter: `
|
||||
`,
|
||||
},
|
||||
}
|
||||
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
registry := prometheus.NewPedanticRegistry()
|
||||
|
||||
sc, err := NewSocketCollector("pod", "default", "ingress", true, buckets)
|
||||
sc, err := NewSocketCollector("pod", "default", "ingress", true, c.useStatusClasses, buckets)
|
||||
if err != nil {
|
||||
t.Errorf("%v: unexpected error creating new SocketCollector: %v", c.name, err)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -69,7 +69,7 @@ type collector struct {
|
|||
}
|
||||
|
||||
// NewCollector creates a new metric collector the for ingress controller
|
||||
func NewCollector(metricsPerHost bool, registry *prometheus.Registry, ingressclass string, buckets collectors.HistogramBuckets) (Collector, error) {
|
||||
func NewCollector(metricsPerHost, reportStatusClasses bool, registry *prometheus.Registry, ingressclass string, buckets collectors.HistogramBuckets) (Collector, error) {
|
||||
podNamespace := os.Getenv("POD_NAMESPACE")
|
||||
if podNamespace == "" {
|
||||
podNamespace = "default"
|
||||
|
|
@ -87,7 +87,7 @@ func NewCollector(metricsPerHost bool, registry *prometheus.Registry, ingresscla
|
|||
return nil, err
|
||||
}
|
||||
|
||||
s, err := collectors.NewSocketCollector(podName, podNamespace, ingressclass, metricsPerHost, buckets)
|
||||
s, err := collectors.NewSocketCollector(podName, podNamespace, ingressclass, metricsPerHost, reportStatusClasses, buckets)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue