feature: added AdmissionController metrics (#7711)

* feature: added AdmissionController metrics

* fix: flag control on admissionCollector

* fix: admission collector disclaimer year and linting

cmd/nginx/main.go

@@ -138,7 +138,9 @@ func main() {
 			klog.Fatalf("Error creating prometheus collector:  %v", err)
 		}
 	}
-	mc.Start()
+	// Pass the ValidationWebhook status to determine if we need to start the collector
+	// for the admissionWebhook
+	mc.Start(conf.ValidationWebhook)
 
 	if conf.EnableProfiling {
 		go registerProfiler()

internal/ingress/controller/controller.go

@@ -215,6 +215,8 @@ func (n *NGINXController) syncIngress(interface{}) error {
 // CheckIngress returns an error in case the provided ingress, when added
 // to the current configuration, generates an invalid configuration
 func (n *NGINXController) CheckIngress(ing *networking.Ingress) error {
+	startCheck := time.Now().UnixNano() / 1000000
+
 	if ing == nil {
 		// no ingress to add, no state change
 		return nil
@@ -233,7 +235,7 @@ func (n *NGINXController) CheckIngress(ing *networking.Ingress) error {
 	if n.cfg.DisableCatchAll && ing.Spec.DefaultBackend != nil {
 		return fmt.Errorf("This deployment is trying to create a catch-all ingress while DisableCatchAll flag is set to true. Remove '.spec.backend' or set DisableCatchAll flag to false.")
 	}
-
+	startRender := time.Now().UnixNano() / 1000000
 	cfg := n.store.GetBackendConfiguration()
 	cfg.Resolver = n.resolver
 
@@ -267,7 +269,7 @@ func (n *NGINXController) CheckIngress(ing *networking.Ingress) error {
 		Ingress:           *ing,
 		ParsedAnnotations: annotations.NewAnnotationExtractor(n.store).Extract(ing),
 	})
-
+	startTest := time.Now().UnixNano() / 1000000
 	_, servers, pcfg := n.getConfiguration(ings)
 
 	err := checkOverlap(ing, allIngresses, servers)
@@ -275,9 +277,10 @@ func (n *NGINXController) CheckIngress(ing *networking.Ingress) error {
 		n.metricCollector.IncCheckErrorCount(ing.ObjectMeta.Namespace, ing.Name)
 		return err
 	}
-
+	testedSize := len(ings)
 	if n.cfg.DisableFullValidationTest {
 		_, _, pcfg = n.getConfiguration(ings[len(ings)-1:])
+		testedSize = 1
 	}
 
 	content, err := n.generateTemplate(cfg, *pcfg)
@@ -291,8 +294,16 @@ func (n *NGINXController) CheckIngress(ing *networking.Ingress) error {
 		n.metricCollector.IncCheckErrorCount(ing.ObjectMeta.Namespace, ing.Name)
 		return err
 	}
-
 	n.metricCollector.IncCheckCount(ing.ObjectMeta.Namespace, ing.Name)
+	endCheck := time.Now().UnixNano() / 1000000
+	n.metricCollector.SetAdmissionMetrics(
+		float64(testedSize),
+		float64(endCheck-startTest)/1000,
+		float64(len(ings)),
+		float64(startTest-startRender)/1000,
+		float64(len(content)),
+		float64(endCheck-startCheck)/1000,
+	)
 	return nil
 }
 

internal/ingress/controller/nginx.go

@@ -241,7 +241,8 @@ type NGINXController struct {
 
 	store store.Storer
 
-	metricCollector metric.Collector
+	metricCollector    metric.Collector
+	admissionCollector metric.Collector
 
 	validationWebhookServer *http.Server
 

internal/ingress/metric/collectors/admission.go

@@ -0,0 +1,157 @@
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package collectors
+
+import (
+	"fmt"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"k8s.io/klog/v2"
+)
+
+// AdmissionCollector stores prometheus metrics of the admission webhook
+type AdmissionCollector struct {
+	prometheus.Collector
+
+	testedIngressLength prometheus.Gauge
+	testedIngressTime   prometheus.Gauge
+
+	renderingIngressLength prometheus.Gauge
+	renderingIngressTime   prometheus.Gauge
+
+	admissionTime prometheus.Gauge
+
+	testedConfigurationSize prometheus.Gauge
+
+	constLabels prometheus.Labels
+	labels      prometheus.Labels
+}
+
+// NewAdmissionCollector creates a new AdmissionCollector instance for the admission collector
+func NewAdmissionCollector(pod, namespace, class string) *AdmissionCollector {
+	constLabels := prometheus.Labels{
+		"controller_namespace": namespace,
+		"controller_class":     class,
+		"controller_pod":       pod,
+	}
+
+	am := &AdmissionCollector{
+		constLabels: constLabels,
+
+		labels: prometheus.Labels{
+			"namespace": namespace,
+			"class":     class,
+		},
+
+		testedIngressLength: prometheus.NewGauge(
+			prometheus.GaugeOpts{
+				Name:        "admission_tested_ingresses",
+				Help:        "The length of ingresses processed by the admission controller",
+				Namespace:   PrometheusNamespace,
+				ConstLabels: constLabels,
+			}),
+		testedIngressTime: prometheus.NewGauge(
+			prometheus.GaugeOpts{
+				Name:        "admission_tested_duration",
+				Help:        "The processing duration of the admission controller tests (float seconds)",
+				Namespace:   PrometheusNamespace,
+				ConstLabels: constLabels,
+			}),
+		renderingIngressLength: prometheus.NewGauge(
+			prometheus.GaugeOpts{
+				Name:        "admission_render_ingresses",
+				Help:        "The length of ingresses rendered by the admission controller",
+				Namespace:   PrometheusNamespace,
+				ConstLabels: constLabels,
+			}),
+		renderingIngressTime: prometheus.NewGauge(
+			prometheus.GaugeOpts{
+				Name:        "admission_render_duration",
+				Help:        "The processing duration of ingresses rendering by the admission controller (float seconds)",
+				Namespace:   PrometheusNamespace,
+				ConstLabels: constLabels,
+			}),
+		testedConfigurationSize: prometheus.NewGauge(
+			prometheus.GaugeOpts{
+				Namespace:   PrometheusNamespace,
+				Name:        "admission_config_size",
+				Help:        "The size of the tested configuration",
+				ConstLabels: constLabels,
+			}),
+		admissionTime: prometheus.NewGauge(
+			prometheus.GaugeOpts{
+				Name:        "admission_roundtrip_duration",
+				Help:        "The complete duration of the admission controller at the time to process a new event (float seconds)",
+				Namespace:   PrometheusNamespace,
+				ConstLabels: constLabels,
+			}),
+	}
+	return am
+}
+
+// Describe implements prometheus.Collector
+func (am AdmissionCollector) Describe(ch chan<- *prometheus.Desc) {
+	am.testedIngressLength.Describe(ch)
+	am.testedIngressTime.Describe(ch)
+	am.renderingIngressLength.Describe(ch)
+	am.renderingIngressTime.Describe(ch)
+	am.testedConfigurationSize.Describe(ch)
+	am.admissionTime.Describe(ch)
+}
+
+// Collect implements the prometheus.Collector interface.
+func (am AdmissionCollector) Collect(ch chan<- prometheus.Metric) {
+	am.testedIngressLength.Collect(ch)
+	am.testedIngressTime.Collect(ch)
+	am.renderingIngressLength.Collect(ch)
+	am.renderingIngressTime.Collect(ch)
+	am.testedConfigurationSize.Collect(ch)
+	am.admissionTime.Collect(ch)
+}
+
+// ByteFormat formats humanReadable bytes
+func ByteFormat(bytes int64) string {
+	const unit = 1000
+	if bytes < unit {
+		return fmt.Sprintf("%d B", bytes)
+	}
+	div, exp := int64(unit), 0
+	for n := bytes / unit; n >= unit; n /= unit {
+		div *= unit
+		exp++
+	}
+	return fmt.Sprintf("%.1f%cB",
+		float64(bytes)/float64(div), "kMGTPE"[exp])
+}
+
+// SetAdmissionMetrics sets the values for AdmissionMetrics that can be called externally
+func (am *AdmissionCollector) SetAdmissionMetrics(testedIngressLength float64, testedIngressTime float64, renderingIngressLength float64, renderingIngressTime float64, testedConfigurationSize float64, admissionTime float64) {
+	am.testedIngressLength.Set(testedIngressLength)
+	am.testedIngressTime.Set(testedIngressTime)
+	am.renderingIngressLength.Set(renderingIngressLength)
+	am.renderingIngressTime.Set(renderingIngressTime)
+	am.testedConfigurationSize.Set(testedConfigurationSize)
+	am.admissionTime.Set(admissionTime)
+	klog.Infof("processed ingress via admission controller {testedIngressLength:%v testedIngressTime:%vs renderingIngressLength:%v renderingIngressTime:%vs admissionTime:%vs testedConfigurationSize:%v}",
+		testedIngressLength,
+		testedIngressTime,
+		renderingIngressLength,
+		renderingIngressTime,
+		ByteFormat(int64(testedConfigurationSize)),
+		admissionTime,
+	)
+}

internal/ingress/metric/collectors/admission_test.go

@@ -0,0 +1,122 @@
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package collectors
+
+import (
+	"testing"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+func TestAdmissionCounters(t *testing.T) {
+	const (
+		metadataFirst = `
+			# HELP nginx_ingress_controller_admission_config_size The size of the tested configuration
+			# TYPE nginx_ingress_controller_admission_config_size gauge
+			# HELP nginx_ingress_controller_admission_roundtrip_duration The complete duration of the admission controller at the time to process a new event (float seconds)
+			# TYPE nginx_ingress_controller_admission_roundtrip_duration gauge
+		`
+		metadataSecond = `
+			# HELP nginx_ingress_controller_admission_render_ingresses The length of ingresses rendered by the admission controller
+			# TYPE nginx_ingress_controller_admission_render_ingresses gauge
+			# HELP nginx_ingress_controller_admission_tested_duration The processing duration of the admission controller tests (float seconds)
+			# TYPE nginx_ingress_controller_admission_tested_duration gauge
+		`
+		metadataThird = `
+			# HELP nginx_ingress_controller_admission_config_size The size of the tested configuration
+			# TYPE nginx_ingress_controller_admission_config_size gauge
+			# HELP nginx_ingress_controller_admission_render_duration The processing duration of ingresses rendering by the admission controller (float seconds)
+			# TYPE nginx_ingress_controller_admission_render_duration gauge
+			# HELP nginx_ingress_controller_admission_render_ingresses The length of ingresses rendered by the admission controller
+			# TYPE nginx_ingress_controller_admission_render_ingresses gauge
+			# HELP nginx_ingress_controller_admission_roundtrip_duration The complete duration of the admission controller at the time to process a new event (float seconds)
+			# TYPE nginx_ingress_controller_admission_roundtrip_duration gauge
+			# HELP nginx_ingress_controller_admission_tested_ingresses The length of ingresses processed by the admission controller
+			# TYPE nginx_ingress_controller_admission_tested_ingresses gauge
+			# HELP nginx_ingress_controller_admission_tested_duration The processing duration of the admission controller tests (float seconds)
+			# TYPE nginx_ingress_controller_admission_tested_duration gauge
+		`
+	)
+	cases := []struct {
+		name    string
+		test    func(*AdmissionCollector)
+		metrics []string
+		want    string
+	}{
+		{
+			name: "should return 0 as values on a fresh initiated collector",
+			test: func(am *AdmissionCollector) {
+			},
+			want: metadataFirst + `
+				nginx_ingress_controller_admission_config_size{controller_class="nginx",controller_namespace="default",controller_pod="pod"} 0
+				nginx_ingress_controller_admission_roundtrip_duration{controller_class="nginx",controller_namespace="default",controller_pod="pod"} 0
+			`,
+			metrics: []string{"nginx_ingress_controller_admission_config_size", "nginx_ingress_controller_admission_roundtrip_duration"},
+		},
+		{
+			name: "set admission metrics to 1 in all fields and validate next set",
+			test: func(am *AdmissionCollector) {
+				am.SetAdmissionMetrics(1, 1, 1, 1, 1, 1)
+			},
+			want: metadataSecond + `
+				nginx_ingress_controller_admission_render_ingresses{controller_class="nginx",controller_namespace="default",controller_pod="pod"} 1
+				nginx_ingress_controller_admission_tested_duration{controller_class="nginx",controller_namespace="default",controller_pod="pod"} 1
+			`,
+			metrics: []string{"nginx_ingress_controller_admission_render_ingresses", "nginx_ingress_controller_admission_tested_duration"},
+		},
+		{
+			name: "set admission metrics to 5 in all fields and validate all sets",
+			test: func(am *AdmissionCollector) {
+				am.SetAdmissionMetrics(5, 5, 5, 5, 5, 5)
+			},
+			want: metadataThird + `
+				nginx_ingress_controller_admission_config_size{controller_class="nginx",controller_namespace="default",controller_pod="pod"} 5
+				nginx_ingress_controller_admission_render_duration{controller_class="nginx",controller_namespace="default",controller_pod="pod"} 5
+				nginx_ingress_controller_admission_render_ingresses{controller_class="nginx",controller_namespace="default",controller_pod="pod"} 5
+				nginx_ingress_controller_admission_roundtrip_duration{controller_class="nginx",controller_namespace="default",controller_pod="pod"} 5
+				nginx_ingress_controller_admission_tested_ingresses{controller_class="nginx",controller_namespace="default",controller_pod="pod"} 5
+				nginx_ingress_controller_admission_tested_duration{controller_class="nginx",controller_namespace="default",controller_pod="pod"} 5
+			`,
+			metrics: []string{
+				"nginx_ingress_controller_admission_config_size",
+				"nginx_ingress_controller_admission_render_duration",
+				"nginx_ingress_controller_admission_render_ingresses",
+				"nginx_ingress_controller_admission_roundtrip_duration",
+				"nginx_ingress_controller_admission_tested_ingresses",
+				"nginx_ingress_controller_admission_tested_duration",
+			},
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			am := NewAdmissionCollector("pod", "default", "nginx")
+			reg := prometheus.NewPedanticRegistry()
+			if err := reg.Register(am); err != nil {
+				t.Errorf("registering collector failed: %s", err)
+			}
+
+			c.test(am)
+
+			if err := GatherAndCompare(am, c.want, c.metrics, reg); err != nil {
+				t.Errorf("unexpected collecting result:\n%s", err)
+			}
+
+			reg.Unregister(am)
+		})
+	}
+}

internal/ingress/metric/dummy.go

@@ -32,6 +32,9 @@ type DummyCollector struct{}
 // ConfigSuccess ...
 func (dc DummyCollector) ConfigSuccess(uint64, bool) {}
 
+// SetAdmissionMetrics ...
+func (dc DummyCollector) SetAdmissionMetrics(float64, float64, float64, float64, float64, float64) {}
+
 // IncReloadCount ...
 func (dc DummyCollector) IncReloadCount() {}
 
@@ -48,10 +51,10 @@ func (dc DummyCollector) IncCheckErrorCount(string, string) {}
 func (dc DummyCollector) RemoveMetrics(ingresses, endpoints []string) {}
 
 // Start ...
-func (dc DummyCollector) Start() {}
+func (dc DummyCollector) Start(admissionStatus string) {}
 
 // Stop ...
-func (dc DummyCollector) Stop() {}
+func (dc DummyCollector) Stop(admissionStatus string) {}
 
 // SetSSLExpireTime ...
 func (dc DummyCollector) SetSSLExpireTime([]*ingress.Server) {}

internal/ingress/metric/main.go

@@ -36,6 +36,8 @@ type Collector interface {
 	IncReloadCount()
 	IncReloadErrorCount()
 
+	SetAdmissionMetrics(float64, float64, float64, float64, float64, float64)
+
 	OnStartedLeading(string)
 	OnStoppedLeading(string)
 
@@ -49,15 +51,16 @@ type Collector interface {
 	// SetHosts sets the hostnames that are being served by the ingress controller
 	SetHosts(sets.String)
 
-	Start()
+	Start(string)
-	Stop()
+	Stop(string)
 }
 
 type collector struct {
 	nginxStatus  collectors.NGINXStatusCollector
 	nginxProcess collectors.NGINXProcessCollector
 
-	ingressController *collectors.Controller
+	ingressController   *collectors.Controller
+	admissionController *collectors.AdmissionCollector
 
 	socket *collectors.SocketCollector
 
@@ -90,11 +93,14 @@ func NewCollector(metricsPerHost bool, registry *prometheus.Registry, ingresscla
 
 	ic := collectors.NewController(podName, podNamespace, ingressclass)
 
+	am := collectors.NewAdmissionCollector(podName, podNamespace, ingressclass)
+
 	return Collector(&collector{
 		nginxStatus:  nc,
 		nginxProcess: pc,
 
-		ingressController: ic,
+		admissionController: am,
+		ingressController:   ic,
 
 		socket: s,
 
@@ -127,9 +133,12 @@ func (c *collector) RemoveMetrics(ingresses, hosts []string) {
 	c.ingressController.RemoveMetrics(hosts, c.registry)
 }
 
-func (c *collector) Start() {
+func (c *collector) Start(admissionStatus string) {
 	c.registry.MustRegister(c.nginxStatus)
 	c.registry.MustRegister(c.nginxProcess)
+	if admissionStatus != "" {
+		c.registry.MustRegister(c.admissionController)
+	}
 	c.registry.MustRegister(c.ingressController)
 	c.registry.MustRegister(c.socket)
 
@@ -143,9 +152,12 @@ func (c *collector) Start() {
 	go c.socket.Start()
 }
 
-func (c *collector) Stop() {
+func (c *collector) Stop(admissionStatus string) {
 	c.registry.Unregister(c.nginxStatus)
 	c.registry.Unregister(c.nginxProcess)
+	if admissionStatus != "" {
+		c.registry.Unregister(c.admissionController)
+	}
 	c.registry.Unregister(c.ingressController)
 	c.registry.Unregister(c.socket)
 
@@ -167,6 +179,17 @@ func (c *collector) SetHosts(hosts sets.String) {
 	c.socket.SetHosts(hosts)
 }
 
+func (c *collector) SetAdmissionMetrics(testedIngressLength float64, testedIngressTime float64, renderingIngressLength float64, renderingIngressTime float64, testedConfigurationSize float64, admissionTime float64) {
+	c.admissionController.SetAdmissionMetrics(
+		testedIngressLength,
+		testedIngressTime,
+		renderingIngressLength,
+		renderingIngressTime,
+		testedConfigurationSize,
+		admissionTime,
+	)
+}
+
 // OnStartedLeading indicates the pod was elected as the leader
 func (c *collector) OnStartedLeading(electionID string) {
 	setLeader(true)