diff --git a/cmd/nginx/main.go b/cmd/nginx/main.go index 69d12d05e..fb87ad84a 100644 --- a/cmd/nginx/main.go +++ b/cmd/nginx/main.go @@ -138,7 +138,9 @@ func main() { klog.Fatalf("Error creating prometheus collector: %v", err) } } - mc.Start() + // Pass the ValidationWebhook status to determine if we need to start the collector + // for the admissionWebhook + mc.Start(conf.ValidationWebhook) if conf.EnableProfiling { go registerProfiler() diff --git a/internal/ingress/controller/controller.go b/internal/ingress/controller/controller.go index f935f5fff..d648bb3ae 100644 --- a/internal/ingress/controller/controller.go +++ b/internal/ingress/controller/controller.go @@ -215,6 +215,8 @@ func (n *NGINXController) syncIngress(interface{}) error { // CheckIngress returns an error in case the provided ingress, when added // to the current configuration, generates an invalid configuration func (n *NGINXController) CheckIngress(ing *networking.Ingress) error { + startCheck := time.Now().UnixNano() / 1000000 + if ing == nil { // no ingress to add, no state change return nil @@ -233,7 +235,7 @@ func (n *NGINXController) CheckIngress(ing *networking.Ingress) error { if n.cfg.DisableCatchAll && ing.Spec.DefaultBackend != nil { return fmt.Errorf("This deployment is trying to create a catch-all ingress while DisableCatchAll flag is set to true. Remove '.spec.backend' or set DisableCatchAll flag to false.") } - + startRender := time.Now().UnixNano() / 1000000 cfg := n.store.GetBackendConfiguration() cfg.Resolver = n.resolver @@ -267,7 +269,7 @@ func (n *NGINXController) CheckIngress(ing *networking.Ingress) error { Ingress: *ing, ParsedAnnotations: annotations.NewAnnotationExtractor(n.store).Extract(ing), }) - + startTest := time.Now().UnixNano() / 1000000 _, servers, pcfg := n.getConfiguration(ings) err := checkOverlap(ing, allIngresses, servers) @@ -275,9 +277,10 @@ func (n *NGINXController) CheckIngress(ing *networking.Ingress) error { n.metricCollector.IncCheckErrorCount(ing.ObjectMeta.Namespace, ing.Name) return err } - + testedSize := len(ings) if n.cfg.DisableFullValidationTest { _, _, pcfg = n.getConfiguration(ings[len(ings)-1:]) + testedSize = 1 } content, err := n.generateTemplate(cfg, *pcfg) @@ -291,8 +294,16 @@ func (n *NGINXController) CheckIngress(ing *networking.Ingress) error { n.metricCollector.IncCheckErrorCount(ing.ObjectMeta.Namespace, ing.Name) return err } - n.metricCollector.IncCheckCount(ing.ObjectMeta.Namespace, ing.Name) + endCheck := time.Now().UnixNano() / 1000000 + n.metricCollector.SetAdmissionMetrics( + float64(testedSize), + float64(endCheck-startTest)/1000, + float64(len(ings)), + float64(startTest-startRender)/1000, + float64(len(content)), + float64(endCheck-startCheck)/1000, + ) return nil } diff --git a/internal/ingress/controller/nginx.go b/internal/ingress/controller/nginx.go index ddde11bc8..b77d1d870 100644 --- a/internal/ingress/controller/nginx.go +++ b/internal/ingress/controller/nginx.go @@ -241,7 +241,8 @@ type NGINXController struct { store store.Storer - metricCollector metric.Collector + metricCollector metric.Collector + admissionCollector metric.Collector validationWebhookServer *http.Server diff --git a/internal/ingress/metric/collectors/admission.go b/internal/ingress/metric/collectors/admission.go new file mode 100644 index 000000000..cf42fbaa1 --- /dev/null +++ b/internal/ingress/metric/collectors/admission.go @@ -0,0 +1,157 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package collectors + +import ( + "fmt" + + "github.com/prometheus/client_golang/prometheus" + "k8s.io/klog/v2" +) + +// AdmissionCollector stores prometheus metrics of the admission webhook +type AdmissionCollector struct { + prometheus.Collector + + testedIngressLength prometheus.Gauge + testedIngressTime prometheus.Gauge + + renderingIngressLength prometheus.Gauge + renderingIngressTime prometheus.Gauge + + admissionTime prometheus.Gauge + + testedConfigurationSize prometheus.Gauge + + constLabels prometheus.Labels + labels prometheus.Labels +} + +// NewAdmissionCollector creates a new AdmissionCollector instance for the admission collector +func NewAdmissionCollector(pod, namespace, class string) *AdmissionCollector { + constLabels := prometheus.Labels{ + "controller_namespace": namespace, + "controller_class": class, + "controller_pod": pod, + } + + am := &AdmissionCollector{ + constLabels: constLabels, + + labels: prometheus.Labels{ + "namespace": namespace, + "class": class, + }, + + testedIngressLength: prometheus.NewGauge( + prometheus.GaugeOpts{ + Name: "admission_tested_ingresses", + Help: "The length of ingresses processed by the admission controller", + Namespace: PrometheusNamespace, + ConstLabels: constLabels, + }), + testedIngressTime: prometheus.NewGauge( + prometheus.GaugeOpts{ + Name: "admission_tested_duration", + Help: "The processing duration of the admission controller tests (float seconds)", + Namespace: PrometheusNamespace, + ConstLabels: constLabels, + }), + renderingIngressLength: prometheus.NewGauge( + prometheus.GaugeOpts{ + Name: "admission_render_ingresses", + Help: "The length of ingresses rendered by the admission controller", + Namespace: PrometheusNamespace, + ConstLabels: constLabels, + }), + renderingIngressTime: prometheus.NewGauge( + prometheus.GaugeOpts{ + Name: "admission_render_duration", + Help: "The processing duration of ingresses rendering by the admission controller (float seconds)", + Namespace: PrometheusNamespace, + ConstLabels: constLabels, + }), + testedConfigurationSize: prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: PrometheusNamespace, + Name: "admission_config_size", + Help: "The size of the tested configuration", + ConstLabels: constLabels, + }), + admissionTime: prometheus.NewGauge( + prometheus.GaugeOpts{ + Name: "admission_roundtrip_duration", + Help: "The complete duration of the admission controller at the time to process a new event (float seconds)", + Namespace: PrometheusNamespace, + ConstLabels: constLabels, + }), + } + return am +} + +// Describe implements prometheus.Collector +func (am AdmissionCollector) Describe(ch chan<- *prometheus.Desc) { + am.testedIngressLength.Describe(ch) + am.testedIngressTime.Describe(ch) + am.renderingIngressLength.Describe(ch) + am.renderingIngressTime.Describe(ch) + am.testedConfigurationSize.Describe(ch) + am.admissionTime.Describe(ch) +} + +// Collect implements the prometheus.Collector interface. +func (am AdmissionCollector) Collect(ch chan<- prometheus.Metric) { + am.testedIngressLength.Collect(ch) + am.testedIngressTime.Collect(ch) + am.renderingIngressLength.Collect(ch) + am.renderingIngressTime.Collect(ch) + am.testedConfigurationSize.Collect(ch) + am.admissionTime.Collect(ch) +} + +// ByteFormat formats humanReadable bytes +func ByteFormat(bytes int64) string { + const unit = 1000 + if bytes < unit { + return fmt.Sprintf("%d B", bytes) + } + div, exp := int64(unit), 0 + for n := bytes / unit; n >= unit; n /= unit { + div *= unit + exp++ + } + return fmt.Sprintf("%.1f%cB", + float64(bytes)/float64(div), "kMGTPE"[exp]) +} + +// SetAdmissionMetrics sets the values for AdmissionMetrics that can be called externally +func (am *AdmissionCollector) SetAdmissionMetrics(testedIngressLength float64, testedIngressTime float64, renderingIngressLength float64, renderingIngressTime float64, testedConfigurationSize float64, admissionTime float64) { + am.testedIngressLength.Set(testedIngressLength) + am.testedIngressTime.Set(testedIngressTime) + am.renderingIngressLength.Set(renderingIngressLength) + am.renderingIngressTime.Set(renderingIngressTime) + am.testedConfigurationSize.Set(testedConfigurationSize) + am.admissionTime.Set(admissionTime) + klog.Infof("processed ingress via admission controller {testedIngressLength:%v testedIngressTime:%vs renderingIngressLength:%v renderingIngressTime:%vs admissionTime:%vs testedConfigurationSize:%v}", + testedIngressLength, + testedIngressTime, + renderingIngressLength, + renderingIngressTime, + ByteFormat(int64(testedConfigurationSize)), + admissionTime, + ) +} diff --git a/internal/ingress/metric/collectors/admission_test.go b/internal/ingress/metric/collectors/admission_test.go new file mode 100644 index 000000000..68208ad3e --- /dev/null +++ b/internal/ingress/metric/collectors/admission_test.go @@ -0,0 +1,122 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package collectors + +import ( + "testing" + + "github.com/prometheus/client_golang/prometheus" +) + +func TestAdmissionCounters(t *testing.T) { + const ( + metadataFirst = ` + # HELP nginx_ingress_controller_admission_config_size The size of the tested configuration + # TYPE nginx_ingress_controller_admission_config_size gauge + # HELP nginx_ingress_controller_admission_roundtrip_duration The complete duration of the admission controller at the time to process a new event (float seconds) + # TYPE nginx_ingress_controller_admission_roundtrip_duration gauge + ` + metadataSecond = ` + # HELP nginx_ingress_controller_admission_render_ingresses The length of ingresses rendered by the admission controller + # TYPE nginx_ingress_controller_admission_render_ingresses gauge + # HELP nginx_ingress_controller_admission_tested_duration The processing duration of the admission controller tests (float seconds) + # TYPE nginx_ingress_controller_admission_tested_duration gauge + ` + metadataThird = ` + # HELP nginx_ingress_controller_admission_config_size The size of the tested configuration + # TYPE nginx_ingress_controller_admission_config_size gauge + # HELP nginx_ingress_controller_admission_render_duration The processing duration of ingresses rendering by the admission controller (float seconds) + # TYPE nginx_ingress_controller_admission_render_duration gauge + # HELP nginx_ingress_controller_admission_render_ingresses The length of ingresses rendered by the admission controller + # TYPE nginx_ingress_controller_admission_render_ingresses gauge + # HELP nginx_ingress_controller_admission_roundtrip_duration The complete duration of the admission controller at the time to process a new event (float seconds) + # TYPE nginx_ingress_controller_admission_roundtrip_duration gauge + # HELP nginx_ingress_controller_admission_tested_ingresses The length of ingresses processed by the admission controller + # TYPE nginx_ingress_controller_admission_tested_ingresses gauge + # HELP nginx_ingress_controller_admission_tested_duration The processing duration of the admission controller tests (float seconds) + # TYPE nginx_ingress_controller_admission_tested_duration gauge + ` + ) + cases := []struct { + name string + test func(*AdmissionCollector) + metrics []string + want string + }{ + { + name: "should return 0 as values on a fresh initiated collector", + test: func(am *AdmissionCollector) { + }, + want: metadataFirst + ` + nginx_ingress_controller_admission_config_size{controller_class="nginx",controller_namespace="default",controller_pod="pod"} 0 + nginx_ingress_controller_admission_roundtrip_duration{controller_class="nginx",controller_namespace="default",controller_pod="pod"} 0 + `, + metrics: []string{"nginx_ingress_controller_admission_config_size", "nginx_ingress_controller_admission_roundtrip_duration"}, + }, + { + name: "set admission metrics to 1 in all fields and validate next set", + test: func(am *AdmissionCollector) { + am.SetAdmissionMetrics(1, 1, 1, 1, 1, 1) + }, + want: metadataSecond + ` + nginx_ingress_controller_admission_render_ingresses{controller_class="nginx",controller_namespace="default",controller_pod="pod"} 1 + nginx_ingress_controller_admission_tested_duration{controller_class="nginx",controller_namespace="default",controller_pod="pod"} 1 + `, + metrics: []string{"nginx_ingress_controller_admission_render_ingresses", "nginx_ingress_controller_admission_tested_duration"}, + }, + { + name: "set admission metrics to 5 in all fields and validate all sets", + test: func(am *AdmissionCollector) { + am.SetAdmissionMetrics(5, 5, 5, 5, 5, 5) + }, + want: metadataThird + ` + nginx_ingress_controller_admission_config_size{controller_class="nginx",controller_namespace="default",controller_pod="pod"} 5 + nginx_ingress_controller_admission_render_duration{controller_class="nginx",controller_namespace="default",controller_pod="pod"} 5 + nginx_ingress_controller_admission_render_ingresses{controller_class="nginx",controller_namespace="default",controller_pod="pod"} 5 + nginx_ingress_controller_admission_roundtrip_duration{controller_class="nginx",controller_namespace="default",controller_pod="pod"} 5 + nginx_ingress_controller_admission_tested_ingresses{controller_class="nginx",controller_namespace="default",controller_pod="pod"} 5 + nginx_ingress_controller_admission_tested_duration{controller_class="nginx",controller_namespace="default",controller_pod="pod"} 5 + `, + metrics: []string{ + "nginx_ingress_controller_admission_config_size", + "nginx_ingress_controller_admission_render_duration", + "nginx_ingress_controller_admission_render_ingresses", + "nginx_ingress_controller_admission_roundtrip_duration", + "nginx_ingress_controller_admission_tested_ingresses", + "nginx_ingress_controller_admission_tested_duration", + }, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + am := NewAdmissionCollector("pod", "default", "nginx") + reg := prometheus.NewPedanticRegistry() + if err := reg.Register(am); err != nil { + t.Errorf("registering collector failed: %s", err) + } + + c.test(am) + + if err := GatherAndCompare(am, c.want, c.metrics, reg); err != nil { + t.Errorf("unexpected collecting result:\n%s", err) + } + + reg.Unregister(am) + }) + } +} diff --git a/internal/ingress/metric/dummy.go b/internal/ingress/metric/dummy.go index 59a9144e0..922a21604 100644 --- a/internal/ingress/metric/dummy.go +++ b/internal/ingress/metric/dummy.go @@ -32,6 +32,9 @@ type DummyCollector struct{} // ConfigSuccess ... func (dc DummyCollector) ConfigSuccess(uint64, bool) {} +// SetAdmissionMetrics ... +func (dc DummyCollector) SetAdmissionMetrics(float64, float64, float64, float64, float64, float64) {} + // IncReloadCount ... func (dc DummyCollector) IncReloadCount() {} @@ -48,10 +51,10 @@ func (dc DummyCollector) IncCheckErrorCount(string, string) {} func (dc DummyCollector) RemoveMetrics(ingresses, endpoints []string) {} // Start ... -func (dc DummyCollector) Start() {} +func (dc DummyCollector) Start(admissionStatus string) {} // Stop ... -func (dc DummyCollector) Stop() {} +func (dc DummyCollector) Stop(admissionStatus string) {} // SetSSLExpireTime ... func (dc DummyCollector) SetSSLExpireTime([]*ingress.Server) {} diff --git a/internal/ingress/metric/main.go b/internal/ingress/metric/main.go index 64810dd36..0cc07fe28 100644 --- a/internal/ingress/metric/main.go +++ b/internal/ingress/metric/main.go @@ -36,6 +36,8 @@ type Collector interface { IncReloadCount() IncReloadErrorCount() + SetAdmissionMetrics(float64, float64, float64, float64, float64, float64) + OnStartedLeading(string) OnStoppedLeading(string) @@ -49,15 +51,16 @@ type Collector interface { // SetHosts sets the hostnames that are being served by the ingress controller SetHosts(sets.String) - Start() - Stop() + Start(string) + Stop(string) } type collector struct { nginxStatus collectors.NGINXStatusCollector nginxProcess collectors.NGINXProcessCollector - ingressController *collectors.Controller + ingressController *collectors.Controller + admissionController *collectors.AdmissionCollector socket *collectors.SocketCollector @@ -90,11 +93,14 @@ func NewCollector(metricsPerHost bool, registry *prometheus.Registry, ingresscla ic := collectors.NewController(podName, podNamespace, ingressclass) + am := collectors.NewAdmissionCollector(podName, podNamespace, ingressclass) + return Collector(&collector{ nginxStatus: nc, nginxProcess: pc, - ingressController: ic, + admissionController: am, + ingressController: ic, socket: s, @@ -127,9 +133,12 @@ func (c *collector) RemoveMetrics(ingresses, hosts []string) { c.ingressController.RemoveMetrics(hosts, c.registry) } -func (c *collector) Start() { +func (c *collector) Start(admissionStatus string) { c.registry.MustRegister(c.nginxStatus) c.registry.MustRegister(c.nginxProcess) + if admissionStatus != "" { + c.registry.MustRegister(c.admissionController) + } c.registry.MustRegister(c.ingressController) c.registry.MustRegister(c.socket) @@ -143,9 +152,12 @@ func (c *collector) Start() { go c.socket.Start() } -func (c *collector) Stop() { +func (c *collector) Stop(admissionStatus string) { c.registry.Unregister(c.nginxStatus) c.registry.Unregister(c.nginxProcess) + if admissionStatus != "" { + c.registry.Unregister(c.admissionController) + } c.registry.Unregister(c.ingressController) c.registry.Unregister(c.socket) @@ -167,6 +179,17 @@ func (c *collector) SetHosts(hosts sets.String) { c.socket.SetHosts(hosts) } +func (c *collector) SetAdmissionMetrics(testedIngressLength float64, testedIngressTime float64, renderingIngressLength float64, renderingIngressTime float64, testedConfigurationSize float64, admissionTime float64) { + c.admissionController.SetAdmissionMetrics( + testedIngressLength, + testedIngressTime, + renderingIngressLength, + renderingIngressTime, + testedConfigurationSize, + admissionTime, + ) +} + // OnStartedLeading indicates the pod was elected as the leader func (c *collector) OnStartedLeading(electionID string) { setLeader(true)