ingress-nginx-helm/internal/ingress/metric/collectors/controller.go

/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package collectors

import (
	"fmt"
	"time"

	"github.com/prometheus/client_golang/prometheus"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/ingress-nginx/pkg/apis/ingress"
	"k8s.io/ingress-nginx/version"
	"k8s.io/klog/v2"
)

var (
	operation        = []string{"controller_namespace", "controller_class", "controller_pod"}
	ingressOperation = []string{"controller_namespace", "controller_class", "controller_pod", "namespace", "ingress"}
	sslLabelHost     = []string{"namespace", "class", "host", "secret_name", "identifier"}
	sslInfoLabels    = []string{"namespace", "class", "host", "secret_name", "identifier", "issuer_organization", "issuer_common_name", "serial_number", "public_key_algorithm"}
	orphanityLabels  = []string{"controller_namespace", "controller_class", "controller_pod", "namespace", "ingress", "type"}
)

// Controller defines base metrics about the ingress controller
type Controller struct {
	prometheus.Collector

	configHash        prometheus.Gauge
	configSuccess     prometheus.Gauge
	configSuccessTime prometheus.Gauge

	reloadOperation             *prometheus.CounterVec
	reloadOperationErrors       *prometheus.CounterVec
	checkIngressOperation       *prometheus.CounterVec
	checkIngressOperationErrors *prometheus.CounterVec
	sslExpireTime               *prometheus.GaugeVec
	sslInfo                     *prometheus.GaugeVec
	OrphanIngress               *prometheus.GaugeVec

	constLabels prometheus.Labels
	labels      prometheus.Labels

	leaderElection *prometheus.GaugeVec

	buildInfo prometheus.Collector
}

// NewController creates a new prometheus collector for the
// Ingress controller operations
func NewController(pod, namespace, class string) *Controller {
	constLabels := prometheus.Labels{
		"controller_namespace": namespace,
		"controller_class":     class,
		"controller_pod":       pod,
	}

	cm := &Controller{
		constLabels: constLabels,

		labels: prometheus.Labels{
			"namespace": namespace,
			"class":     class,
		},

		buildInfo: prometheus.NewGaugeFunc(
			prometheus.GaugeOpts{
				Namespace: PrometheusNamespace,
				Name:      "build_info",
				Help:      "A metric with a constant '1' labeled with information about the build.",
				ConstLabels: prometheus.Labels{
					"controller_namespace": namespace,
					"controller_class":     class,
					"controller_pod":       pod,
					"release":              version.RELEASE,
					"build":                version.COMMIT,
					"repository":           version.REPO,
				},
			},
			func() float64 { return 1 },
		),

		configHash: prometheus.NewGauge(
			prometheus.GaugeOpts{
				Namespace:   PrometheusNamespace,
				Name:        "config_hash",
				Help:        "Running configuration hash actually running",
				ConstLabels: constLabels,
			},
		),
		configSuccess: prometheus.NewGauge(
			prometheus.GaugeOpts{
				Namespace:   PrometheusNamespace,
				Name:        "config_last_reload_successful",
				Help:        "Whether the last configuration reload attempt was successful",
				ConstLabels: constLabels,
			}),
		configSuccessTime: prometheus.NewGauge(
			prometheus.GaugeOpts{
				Namespace:   PrometheusNamespace,
				Name:        "config_last_reload_successful_timestamp_seconds",
				Help:        "Timestamp of the last successful configuration reload.",
				ConstLabels: constLabels,
			}),
		reloadOperation: prometheus.NewCounterVec(
			prometheus.CounterOpts{
				Namespace: PrometheusNamespace,
				Name:      "success",
				Help:      `Cumulative number of Ingress controller reload operations`,
			},
			operation,
		),
		reloadOperationErrors: prometheus.NewCounterVec(
			prometheus.CounterOpts{
				Namespace: PrometheusNamespace,
				Name:      "errors",
				Help:      `Cumulative number of Ingress controller errors during reload operations`,
			},
			operation,
		),
		checkIngressOperationErrors: prometheus.NewCounterVec(
			prometheus.CounterOpts{
				Namespace: PrometheusNamespace,
				Name:      "check_errors",
				Help:      `Cumulative number of Ingress controller errors during syntax check operations`,
			},
			ingressOperation,
		),
		checkIngressOperation: prometheus.NewCounterVec(
			prometheus.CounterOpts{
				Namespace: PrometheusNamespace,
				Name:      "check_success",
				Help:      `Cumulative number of Ingress controller syntax check operations`,
			},
			ingressOperation,
		),
		sslExpireTime: prometheus.NewGaugeVec(
			prometheus.GaugeOpts{
				Namespace: PrometheusNamespace,
				Name:      "ssl_expire_time_seconds",
				Help: `Number of seconds since 1970 to the SSL Certificate expire.
			An example to check if this certificate will expire in 10 days is: "nginx_ingress_controller_ssl_expire_time_seconds < (time() + (10 * 24 * 3600))"`,
			},
			sslLabelHost,
		),
		sslInfo: prometheus.NewGaugeVec(
			prometheus.GaugeOpts{
				Namespace: PrometheusNamespace,
				Name:      "ssl_certificate_info",
				Help:      `Hold all labels associated to a certificate`,
			},
			sslInfoLabels,
		),
		leaderElection: prometheus.NewGaugeVec(
			prometheus.GaugeOpts{
				Namespace:   PrometheusNamespace,
				Name:        "leader_election_status",
				Help:        "Gauge reporting status of the leader election, 0 indicates follower, 1 indicates leader. 'name' is the string used to identify the lease",
				ConstLabels: constLabels,
			},
			[]string{"name"},
		),
		OrphanIngress: prometheus.NewGaugeVec(
			prometheus.GaugeOpts{
				Namespace: PrometheusNamespace,
				Name:      "orphan_ingress",
				Help: `Gauge reporting status of ingress orphanity, 1 indicates orphaned ingress.
			'namespace' is the string used to identify namespace of ingress, 'ingress' for ingress name and 'type' for 'no-service' or 'no-endpoint' of orphanity`,
			},
			orphanityLabels,
		),
	}

	return cm
}

// IncReloadCount increment the reload counter
func (cm *Controller) IncReloadCount() {
	cm.reloadOperation.With(cm.constLabels).Inc()
}

// IncReloadErrorCount increment the reload error counter
func (cm *Controller) IncReloadErrorCount() {
	cm.reloadOperationErrors.With(cm.constLabels).Inc()
}

// OnStartedLeading indicates the pod was elected as the leader
func (cm *Controller) OnStartedLeading(electionID string) {
	cm.leaderElection.WithLabelValues(electionID).Set(1.0)
}

// OnStoppedLeading indicates the pod stopped being the leader
func (cm *Controller) OnStoppedLeading(electionID string) {
	cm.leaderElection.WithLabelValues(electionID).Set(0)
}

// IncCheckCount increment the check counter
func (cm *Controller) IncCheckCount(namespace, name string) {
	labels := prometheus.Labels{
		"namespace": namespace,
		"ingress":   name,
	}
	cm.checkIngressOperation.MustCurryWith(cm.constLabels).With(labels).Inc()
}

// IncCheckErrorCount increment the check error counter
func (cm *Controller) IncCheckErrorCount(namespace, name string) {
	labels := prometheus.Labels{
		"namespace": namespace,
		"ingress":   name,
	}
	cm.checkIngressOperationErrors.MustCurryWith(cm.constLabels).With(labels).Inc()
}

// IncOrphanIngress sets the orphaned ingress gauge to one
func (cm *Controller) IncOrphanIngress(namespace, name, orphanityType string) {
	labels := prometheus.Labels{
		"namespace": namespace,
		"ingress":   name,
		"type":      orphanityType,
	}
	cm.OrphanIngress.MustCurryWith(cm.constLabels).With(labels).Set(1.0)
}

// DecOrphanIngress sets the orphaned ingress gauge to zero (all services has their endpoints)
func (cm *Controller) DecOrphanIngress(namespace, name, orphanityType string) {
	labels := prometheus.Labels{
		"namespace": namespace,
		"ingress":   name,
		"type":      orphanityType,
	}
	cm.OrphanIngress.MustCurryWith(cm.constLabels).With(labels).Set(0.0)
}

// ConfigSuccess set a boolean flag according to the output of the controller configuration reload
func (cm *Controller) ConfigSuccess(hash uint64, success bool) {
	if success {
		cm.configSuccessTime.Set(float64(time.Now().Unix()))
		cm.configSuccess.Set(1)

		cm.configHash.Set(float64(hash))

		return
	}

	cm.configSuccess.Set(0)
	cm.configHash.Set(0)
}

// Describe implements prometheus.Collector
func (cm *Controller) Describe(ch chan<- *prometheus.Desc) {
	cm.configHash.Describe(ch)
	cm.configSuccess.Describe(ch)
	cm.configSuccessTime.Describe(ch)
	cm.reloadOperation.Describe(ch)
	cm.reloadOperationErrors.Describe(ch)
	cm.checkIngressOperation.Describe(ch)
	cm.checkIngressOperationErrors.Describe(ch)
	cm.sslExpireTime.Describe(ch)
	cm.sslInfo.Describe(ch)
	cm.leaderElection.Describe(ch)
	cm.buildInfo.Describe(ch)
	cm.OrphanIngress.Describe(ch)
}

// Collect implements the prometheus.Collector interface.
func (cm *Controller) Collect(ch chan<- prometheus.Metric) {
	cm.configHash.Collect(ch)
	cm.configSuccess.Collect(ch)
	cm.configSuccessTime.Collect(ch)
	cm.reloadOperation.Collect(ch)
	cm.reloadOperationErrors.Collect(ch)
	cm.checkIngressOperation.Collect(ch)
	cm.checkIngressOperationErrors.Collect(ch)
	cm.sslExpireTime.Collect(ch)
	cm.sslInfo.Collect(ch)
	cm.leaderElection.Collect(ch)
	cm.buildInfo.Collect(ch)
	cm.OrphanIngress.Collect(ch)
}

// SetSSLExpireTime sets the expiration time of SSL Certificates
func (cm *Controller) SetSSLExpireTime(servers []*ingress.Server) {
	for _, s := range servers {
		if !(s.Hostname != "" && s.SSLCert != nil && s.SSLCert.ExpireTime.Unix() > 0) {
			continue
		}

		labels := make(prometheus.Labels, len(cm.labels)+1)
		for k, v := range cm.labels {
			labels[k] = v
		}
		labels["host"] = s.Hostname
		labels["secret_name"] = s.SSLCert.Name
		labels["identifier"] = s.SSLCert.Identifier()

		cm.sslExpireTime.With(labels).Set(float64(s.SSLCert.ExpireTime.Unix()))
	}
}

// SetSSLInfo creates a metric with all certificate information
func (cm *Controller) SetSSLInfo(servers []*ingress.Server) {
	for _, s := range servers {
		if s.SSLCert == nil || s.SSLCert.Certificate == nil || s.SSLCert.Certificate.SerialNumber == nil {
			continue
		}

		labels := make(prometheus.Labels, len(cm.labels)+1)
		for k, v := range cm.labels {
			labels[k] = v
		}
		labels["identifier"] = s.SSLCert.Identifier()
		labels["host"] = s.Hostname
		labels["secret_name"] = s.SSLCert.Name
		labels["namespace"] = s.SSLCert.Namespace
		labels["issuer_common_name"] = s.SSLCert.Certificate.Issuer.CommonName
		labels["issuer_organization"] = ""
		if len(s.SSLCert.Certificate.Issuer.Organization) > 0 {
			labels["issuer_organization"] = s.SSLCert.Certificate.Issuer.Organization[0]
		}
		labels["serial_number"] = s.SSLCert.Certificate.SerialNumber.String()
		labels["public_key_algorithm"] = s.SSLCert.Certificate.PublicKeyAlgorithm.String()

		cm.sslInfo.With(labels).Set(1)
	}
}

// RemoveMetrics removes metrics for certificates not available anymore by identifier
func (cm *Controller) RemoveMetrics(certificates []string, registry prometheus.Gatherer) {
	cm.removeSSLExpireMetrics(true, certificates, registry)
	cm.removeCertificatesMetrics(true, certificates, registry)
}

// RemoveAllSSLMetrics removes metrics for expiration of SSL Certificates
func (cm *Controller) RemoveAllSSLMetrics(registry prometheus.Gatherer) {
	cm.removeSSLExpireMetrics(false, []string{}, registry)
	cm.removeCertificatesMetrics(false, []string{}, registry)
}

func (cm *Controller) removeCertificatesMetrics(onlyDefinedHosts bool, certificates []string, registry prometheus.Gatherer) {
	mfs, err := registry.Gather()
	if err != nil {
		klog.Errorf("Error gathering metrics: %v", err)
		return
	}

	toRemove := sets.NewString(certificates...)

	for _, mf := range mfs {
		metricName := mf.GetName()
		if fmt.Sprintf("%v_ssl_certificate_info", PrometheusNamespace) != metricName {
			continue
		}

		for _, m := range mf.GetMetric() {
			labels := make(map[string]string, len(m.GetLabel()))
			for _, labelPair := range m.GetLabel() {
				labels[*labelPair.Name] = *labelPair.Value
			}

			// remove labels that are constant
			deleteConstants(labels)

			identifier, ok := labels["identifier"]
			if !ok {
				continue
			}
			if onlyDefinedHosts && !toRemove.Has(identifier) {
				continue
			}

			klog.V(2).Infof("Removing prometheus metric from gauge %v for identifier %v", metricName, identifier)
			removed := cm.sslInfo.Delete(labels)
			if !removed {
				klog.V(2).Infof("metric %v for identifier %v with labels not removed: %v", metricName, identifier, labels)
			}
		}
	}
}

func (cm *Controller) removeSSLExpireMetrics(onlyDefinedCerts bool, certificates []string, registry prometheus.Gatherer) {
	mfs, err := registry.Gather()
	if err != nil {
		klog.ErrorS(err, "Error gathering metrics")
		return
	}

	toRemove := sets.NewString(certificates...)

	for _, mf := range mfs {
		metricName := mf.GetName()
		if fmt.Sprintf("%v_ssl_expire_time_seconds", PrometheusNamespace) != metricName {
			continue
		}

		for _, m := range mf.GetMetric() {
			labels := make(map[string]string, len(m.GetLabel()))
			for _, labelPair := range m.GetLabel() {
				labels[*labelPair.Name] = *labelPair.Value
			}

			// remove labels that are constant
			deleteConstants(labels)

			identifier, ok := labels["identifier"]
			if !ok {
				continue
			}

			host, ok := labels["host"]
			if !ok {
				continue
			}

			if onlyDefinedCerts && !toRemove.Has(identifier) {
				continue
			}

			klog.V(2).InfoS("Removing prometheus metric", "gauge", metricName, "host", host, "identifier", identifier)
			removed := cm.sslExpireTime.Delete(labels)
			if !removed {
				klog.V(2).InfoS("metric removed", "metric", metricName, "host", host, "identifier", identifier, "labels", labels)
			}
		}
	}
}