feat: add --metrics-per-undefined-host to report metrics for hosts not defined in ingresses

Signed-off-by: Jon Carl <grounded042@joncarl.com>

cmd/dataplane/main.go

@@ -66,7 +66,7 @@ func main() {
 	mc := metric.NewDummyCollector()
 	if conf.EnableMetrics {
 		// TODO: Ingress class is not a part of dataplane anymore
-		mc, err = metric.NewCollector(conf.MetricsPerHost, conf.ReportStatusClasses, reg, conf.IngressClassConfiguration.Controller, *conf.MetricsBuckets, conf.MetricsBucketFactor, conf.MetricsMaxBuckets, conf.ExcludeSocketMetrics)
+		mc, err = metric.NewCollector(conf.MetricsPerHost, conf.MetricsPerUndefinedHost, conf.ReportStatusClasses, reg, conf.IngressClassConfiguration.Controller, *conf.MetricsBuckets, conf.MetricsBucketFactor, conf.MetricsMaxBuckets, conf.ExcludeSocketMetrics)
 		if err != nil {
 			klog.Fatalf("Error creating prometheus collector:  %v", err)
 		}

cmd/nginx/main.go

@@ -130,7 +130,7 @@ func main() {
 
 	mc := metric.NewDummyCollector()
 	if conf.EnableMetrics {
-		mc, err = metric.NewCollector(conf.MetricsPerHost, conf.ReportStatusClasses, reg, conf.IngressClassConfiguration.Controller, *conf.MetricsBuckets, conf.MetricsBucketFactor, conf.MetricsMaxBuckets, conf.ExcludeSocketMetrics)
+		mc, err = metric.NewCollector(conf.MetricsPerHost, conf.MetricsPerUndefinedHost, conf.ReportStatusClasses, reg, conf.IngressClassConfiguration.Controller, *conf.MetricsBuckets, conf.MetricsBucketFactor, conf.MetricsMaxBuckets, conf.ExcludeSocketMetrics)
 		if err != nil {
 			klog.Fatalf("Error creating prometheus collector:  %v", err)
 		}

docs/user-guide/cli-arguments.md

@@ -48,6 +48,7 @@ They are set in the container spec of the `ingress-nginx-controller` Deployment
 | `--maxmind-license-key`            | Maxmind license key to download GeoLite2 Databases. https://blog.maxmind.com/2019/12/significant-changes-to-accessing-and-using-geolite2-databases/ . |
 | `--maxmind-mirror`            | Maxmind mirror url (example: http://geoip.local/databases. |
 | `--metrics-per-host`               | Export metrics per-host. (default true) |
+| `--metrics-per-undefined-host`     | Export metrics per-host even if the host is not defined in an ingress. Requires --metrics-per-host to be set to true. (default false) |
 | `--monitor-max-batch-size`               | Max batch size of NGINX metrics. (default 10000)|
 | `--post-shutdown-grace-period`     | Additional delay in seconds before controller container exits. (default 10) |
 | `--profiler-port`                  | Port to use for expose the ingress controller Go profiler when it is enabled. (default 10245) |

docs/user-guide/monitoring.md

@@ -166,7 +166,9 @@ According to the above example, this URL will be http://10.192.0.3:31086
 
 #### Wildcard ingresses
 
-  - By default request metrics are labeled with the hostname. When you have a wildcard domain ingress, then there will be no metrics for that ingress (to prevent the metrics from exploding in cardinality). To get metrics in this case you need to run the ingress controller with `--metrics-per-host=false` (you will lose labeling by hostname, but still have labeling by ingress).
+  - By default request metrics are labeled with the hostname. When you have a wildcard domain ingress, then there will be no metrics for that ingress (to prevent the metrics from exploding in cardinality). To get metrics in this case you have two options:
+    - Run the ingress controller with `--metrics-per-host=false`. You will lose labeling by hostname, but still have labeling by ingress.
+    - Run the ingress controller with `--metrics-per-undefined-host=true --metrics-per-host=true`. You will get labeling by hostname even if the hostname is not explicitly defined on an ingress. Be warned that cardinality could explode due to many hostnames.
 
 ### Grafana dashboard using ingress resource
   - If you want to expose the dashboard for grafana using an ingress resource, then you can :

internal/ingress/controller/controller.go

@@ -107,6 +107,7 @@ type Configuration struct {
 
 	EnableMetrics           bool
 	MetricsPerHost          bool
+	MetricsPerUndefinedHost bool
 	MetricsBuckets          *collectors.HistogramBuckets
 	MetricsBucketFactor     float64
 	MetricsMaxBuckets       uint32

internal/ingress/metric/collectors/socket.go

@@ -82,6 +82,7 @@ type SocketCollector struct {
 	hosts sets.Set[string]
 
 	metricsPerHost          bool
+	metricsPerUndefinedHost bool
 	reportStatusClasses     bool
 }
 
@@ -99,7 +100,7 @@ var requestTags = []string{
 
 // NewSocketCollector creates a new SocketCollector instance using
 // the ingress watch namespace and class used by the controller
-func NewSocketCollector(pod, namespace, class string, metricsPerHost, reportStatusClasses bool, buckets HistogramBuckets, bucketFactor float64, maxBuckets uint32, excludeMetrics []string) (*SocketCollector, error) {
+func NewSocketCollector(pod, namespace, class string, metricsPerHost, metricsPerUndefinedHost, reportStatusClasses bool, buckets HistogramBuckets, bucketFactor float64, maxBuckets uint32, excludeMetrics []string) (*SocketCollector, error) {
 	socket := "/tmp/nginx/prometheus-nginx.socket"
 	// unix sockets must be unlink()ed before being used
 	//nolint:errcheck // Ignore unlink error
@@ -140,6 +141,7 @@ func NewSocketCollector(pod, namespace, class string, metricsPerHost, reportStat
 		listener: listener,
 
 		metricsPerHost:          metricsPerHost,
+		metricsPerUndefinedHost: metricsPerUndefinedHost,
 		reportStatusClasses:     reportStatusClasses,
 
 		connectTime: histogramMetric(
@@ -306,8 +308,8 @@ func (sc *SocketCollector) handleMessage(msg []byte) {
 
 	for i := range statsBatch {
 		stats := &statsBatch[i]
-		if sc.metricsPerHost && !sc.hosts.Has(stats.Host) {
-			klog.V(3).InfoS("Skipping metric for host not being served", "host", stats.Host)
+		if sc.metricsPerHost && !sc.hosts.Has(stats.Host) && !sc.metricsPerUndefinedHost {
+			klog.V(3).InfoS("Skipping metric for host not explicity defined in an ingress", "host", stats.Host)
 			continue
 		}
 

internal/ingress/metric/main.go

@@ -71,7 +71,7 @@ type collector struct {
 }
 
 // NewCollector creates a new metric collector the for ingress controller
-func NewCollector(metricsPerHost, reportStatusClasses bool, registry *prometheus.Registry, ingressclass string, buckets collectors.HistogramBuckets, bucketFactor float64, maxBuckets uint32, excludedSocketMetrics []string) (Collector, error) {
+func NewCollector(metricsPerHost, metricsPerUndefinedHost, reportStatusClasses bool, registry *prometheus.Registry, ingressclass string, buckets collectors.HistogramBuckets, bucketFactor float64, maxBuckets uint32, excludedSocketMetrics []string) (Collector, error) {
 	podNamespace := os.Getenv("POD_NAMESPACE")
 	if podNamespace == "" {
 		podNamespace = "default"
@@ -89,7 +89,7 @@ func NewCollector(metricsPerHost, reportStatusClasses bool, registry *prometheus
 		return nil, err
 	}
 
-	s, err := collectors.NewSocketCollector(podName, podNamespace, ingressclass, metricsPerHost, reportStatusClasses, buckets, bucketFactor, maxBuckets, excludedSocketMetrics)
+	s, err := collectors.NewSocketCollector(podName, podNamespace, ingressclass, metricsPerHost, metricsPerUndefinedHost, reportStatusClasses, buckets, bucketFactor, maxBuckets, excludedSocketMetrics)
 	if err != nil {
 		return nil, err
 	}

pkg/flags/flags.go

@@ -17,6 +17,7 @@ limitations under the License.
 package flags
 
 import (
+	"errors"
 	"flag"
 	"fmt"
 	"net"
@@ -177,6 +178,8 @@ Requires the update-status parameter.`)
 			`Enables the collection of NGINX metrics.`)
 		metricsPerHost = flags.Bool("metrics-per-host", true,
 			`Export metrics per-host.`)
+		metricsPerUndefinedHost = flags.Bool("metrics-per-undefined-host", false,
+			`Export metrics per-host even if the host is not defined in an ingress. Requires --metrics-per-host to be set to true.`)
 		reportStatusClasses = flags.Bool("report-status-classes", false,
 			`Use status classes (2xx, 3xx, 4xx and 5xx) instead of status codes in metrics.`)
 
@@ -319,6 +322,10 @@ https://blog.maxmind.com/2019/12/significant-changes-to-accessing-and-using-geol
 		}
 	}
 
+	if *metricsPerUndefinedHost && !*metricsPerHost {
+		return false, nil, errors.New("--metrics-per-undefined-host=true must be passed with --metrics-per-host=true")
+	}
+
 	if *electionTTL <= 0 {
 		*electionTTL = 30 * time.Second
 	}
@@ -340,6 +347,7 @@ https://blog.maxmind.com/2019/12/significant-changes-to-accessing-and-using-geol
 		EnableProfiling:             *profiling,
 		EnableMetrics:               *enableMetrics,
 		MetricsPerHost:              *metricsPerHost,
+		MetricsPerUndefinedHost:     *metricsPerUndefinedHost,
 		MetricsBuckets:              histogramBuckets,
 		MetricsBucketFactor:         *bucketFactor,
 		MetricsMaxBuckets:           *maxBuckets,