Add new prometheus metric for orphaned ingress (#8230)

* Add new metric for orhaned ingress

* Fix const labels

* Fix after rebase

internal/ingress/controller/controller.go

@@ -54,6 +54,8 @@ const (
 	defServerName               = "_"
 	rootLocation                = "/"
 	emptyZone                   = ""
+	orphanMetricLabelNoService  = "no-service"
+	orphanMetricLabelNoEndpoint = "no-endpoint"
 )
 
 // Configuration contains all the settings required by an Ingress controller
@@ -1052,8 +1054,16 @@ func (n *NGINXController) createUpstreams(data []*ingress.Ingress, du *ingress.B
 					endp, err := n.serviceEndpoints(svcKey, port.String())
 					if err != nil {
 						klog.Warningf("Error obtaining Endpoints for Service %q: %v", svcKey, err)
+						n.metricCollector.IncOrphanIngress(ing.Namespace, ing.Name, orphanMetricLabelNoService)
 						continue
 					}
+					n.metricCollector.DecOrphanIngress(ing.Namespace, ing.Name, orphanMetricLabelNoService)
+
+					if len(endp) == 0 {
+						n.metricCollector.IncOrphanIngress(ing.Namespace, ing.Name, orphanMetricLabelNoEndpoint)
+					} else {
+						n.metricCollector.DecOrphanIngress(ing.Namespace, ing.Name, orphanMetricLabelNoEndpoint)
+					}
 					upstreams[name].Endpoints = endp
 				}
 

internal/ingress/controller/controller_test.go

@@ -2484,5 +2484,6 @@ func newDynamicNginxController(t *testing.T, setConfigMap func(string) *v1.Confi
 		store:           storer,
 		cfg:             config,
 		command:         NewNginxCommand(),
+		metricCollector: metric.DummyCollector{},
 	}
 }

internal/ingress/metric/collectors/controller.go

@@ -32,6 +32,7 @@ var (
 	ingressOperation = []string{"controller_namespace", "controller_class", "controller_pod", "namespace", "ingress"}
 	sslLabelHost     = []string{"namespace", "class", "host", "secret_name"}
 	sslInfoLabels    = []string{"namespace", "class", "host", "secret_name", "identifier", "issuer_organization", "issuer_common_name", "serial_number", "public_key_algorithm"}
+	orphanityLabels  = []string{"controller_namespace", "controller_class", "controller_pod", "namespace", "ingress", "type"}
 )
 
 // Controller defines base metrics about the ingress controller
@@ -48,6 +49,7 @@ type Controller struct {
 	checkIngressOperationErrors *prometheus.CounterVec
 	sslExpireTime               *prometheus.GaugeVec
 	sslInfo                     *prometheus.GaugeVec
+	OrphanIngress               *prometheus.GaugeVec
 
 	constLabels prometheus.Labels
 	labels      prometheus.Labels
@@ -171,6 +173,15 @@ func NewController(pod, namespace, class string) *Controller {
 			},
 			[]string{"name"},
 		),
+		OrphanIngress: prometheus.NewGaugeVec(
+			prometheus.GaugeOpts{
+				Namespace: PrometheusNamespace,
+				Name:      "orphan_ingress",
+				Help: `Gauge reporting status of ingress orphanity, 1 indicates orphaned ingress.
+			'namespace' is the string used to identify namespace of ingress, 'ingress' for ingress name and 'type' for 'no-service' or 'no-endpoint' of orphanity`,
+			},
+			orphanityLabels,
+		),
 	}
 
 	return cm
@@ -214,6 +225,26 @@ func (cm *Controller) IncCheckErrorCount(namespace, name string) {
 	cm.checkIngressOperationErrors.MustCurryWith(cm.constLabels).With(labels).Inc()
 }
 
+// IncOrphanIngress sets the the orphaned ingress gauge to one
+func (cm *Controller) IncOrphanIngress(namespace string, name string, orphanityType string) {
+	labels := prometheus.Labels{
+		"namespace": namespace,
+		"ingress":   name,
+		"type":      orphanityType,
+	}
+	cm.OrphanIngress.MustCurryWith(cm.constLabels).With(labels).Set(1.0)
+}
+
+// DecOrphanIngress sets the the orphaned ingress gauge to zero (all services has their endpoints)
+func (cm *Controller) DecOrphanIngress(namespace string, name string, orphanityType string) {
+	labels := prometheus.Labels{
+		"namespace": namespace,
+		"ingress":   name,
+		"type":      orphanityType,
+	}
+	cm.OrphanIngress.MustCurryWith(cm.constLabels).With(labels).Set(0.0)
+}
+
 // ConfigSuccess set a boolean flag according to the output of the controller configuration reload
 func (cm *Controller) ConfigSuccess(hash uint64, success bool) {
 	if success {
@@ -242,6 +273,7 @@ func (cm Controller) Describe(ch chan<- *prometheus.Desc) {
 	cm.sslInfo.Describe(ch)
 	cm.leaderElection.Describe(ch)
 	cm.buildInfo.Describe(ch)
+	cm.OrphanIngress.Describe(ch)
 }
 
 // Collect implements the prometheus.Collector interface.
@@ -257,6 +289,7 @@ func (cm Controller) Collect(ch chan<- prometheus.Metric) {
 	cm.sslInfo.Collect(ch)
 	cm.leaderElection.Collect(ch)
 	cm.buildInfo.Collect(ch)
+	cm.OrphanIngress.Collect(ch)
 }
 
 // SetSSLExpireTime sets the expiration time of SSL Certificates

internal/ingress/metric/dummy.go

@@ -41,6 +41,12 @@ func (dc DummyCollector) IncReloadCount() {}
 // IncReloadErrorCount ...
 func (dc DummyCollector) IncReloadErrorCount() {}
 
+// IncOrphanIngress ...
+func (dc DummyCollector) IncOrphanIngress(string, string, string) {}
+
+// DecOrphanIngress ...
+func (dc DummyCollector) DecOrphanIngress(string, string, string) {}
+
 // IncCheckCount ...
 func (dc DummyCollector) IncCheckCount(string, string) {}
 

internal/ingress/metric/main.go

@@ -43,6 +43,8 @@ type Collector interface {
 
 	IncCheckCount(string, string)
 	IncCheckErrorCount(string, string)
+	IncOrphanIngress(string, string, string)
+	DecOrphanIngress(string, string, string)
 
 	RemoveMetrics(ingresses, endpoints, certificates []string)
 
@@ -181,6 +183,14 @@ func (c *collector) SetSSLInfo(servers []*ingress.Server) {
 	c.ingressController.SetSSLInfo(servers)
 }
 
+func (c *collector) IncOrphanIngress(namespace string, name string, orphanityType string) {
+	c.ingressController.IncOrphanIngress(namespace, name, orphanityType)
+}
+
+func (c *collector) DecOrphanIngress(namespace string, name string, orphanityType string) {
+	c.ingressController.DecOrphanIngress(namespace, name, orphanityType)
+}
+
 func (c *collector) SetHosts(hosts sets.String) {
 	c.socket.SetHosts(hosts)
 }