From f85c3866d8135d698fe6a2753b1ed17d89a9efa0 Mon Sep 17 00:00:00 2001
From: Dmitry Bashkatov <dbashkatov@gmail.com>
Date: Wed, 22 Jun 2022 22:59:43 +0300
Subject: [PATCH] add new summary metric: ingress_header_seconds (#8726)

---
 internal/ingress/metric/collectors/socket.go | 27 +++++++++++++++++++-
 rootfs/etc/nginx/lua/monitor.lua             |  1 +
 rootfs/etc/nginx/lua/test/monitor_test.lua   |  9 ++++---
 3 files changed, 33 insertions(+), 4 deletions(-)

diff --git a/internal/ingress/metric/collectors/socket.go b/internal/ingress/metric/collectors/socket.go
index 26aebbaef..3b87c0ad1 100644
--- a/internal/ingress/metric/collectors/socket.go
+++ b/internal/ingress/metric/collectors/socket.go
@@ -31,6 +31,7 @@ import (
 
 type upstream struct {
 	Latency        float64 `json:"upstreamLatency"`
+	HeaderTime     float64 `json:"upstreamHeaderTime"`
 	ResponseLength float64 `json:"upstreamResponseLength"`
 	ResponseTime   float64 `json:"upstreamResponseTime"`
 	//Status         string  `json:"upstreamStatus"`
@@ -73,7 +74,8 @@ type SocketCollector struct {
 	responseTime   *prometheus.HistogramVec
 	responseLength *prometheus.HistogramVec
 
-	upstreamLatency *prometheus.SummaryVec
+	upstreamHeaderTime *prometheus.SummaryVec
+	upstreamLatency    *prometheus.SummaryVec
 
 	bytesSent *prometheus.HistogramVec
 
@@ -206,6 +208,17 @@ func NewSocketCollector(pod, namespace, class string, metricsPerHost, reportStat
 			requestTags,
 		),
 
+		upstreamHeaderTime: prometheus.NewSummaryVec(
+			prometheus.SummaryOpts{
+				Name:        "ingress_upstream_header_seconds",
+				Help:        "The time spent on receiving first header from the upstream server",
+				Namespace:   PrometheusNamespace,
+				ConstLabels: constLabels,
+				Objectives:  defObjectives,
+			},
+			[]string{"ingress", "namespace", "service", "canary"},
+		),
+
 		upstreamLatency: prometheus.NewSummaryVec(
 			prometheus.SummaryOpts{
 				Name:        "ingress_upstream_latency_seconds",
@@ -228,6 +241,7 @@ func NewSocketCollector(pod, namespace, class string, metricsPerHost, reportStat
 
 		prometheus.BuildFQName(PrometheusNamespace, "", "bytes_sent"): sc.bytesSent,
 
+		prometheus.BuildFQName(PrometheusNamespace, "", "ingress_upstream_header_seconds"):  sc.upstreamHeaderTime,
 		prometheus.BuildFQName(PrometheusNamespace, "", "ingress_upstream_latency_seconds"): sc.upstreamLatency,
 	}
 
@@ -303,6 +317,15 @@ func (sc *SocketCollector) handleMessage(msg []byte) {
 			}
 		}
 
+		if stats.HeaderTime != -1 {
+			headerTimeMetric, err := sc.upstreamHeaderTime.GetMetricWith(latencyLabels)
+			if err != nil {
+				klog.ErrorS(err, "Error fetching header time metric")
+			} else {
+				headerTimeMetric.Observe(stats.HeaderTime)
+			}
+		}
+
 		if stats.RequestTime != -1 {
 			requestTimeMetric, err := sc.requestTime.GetMetricWith(requestLabels)
 			if err != nil {
@@ -443,6 +466,7 @@ func (sc SocketCollector) Describe(ch chan<- *prometheus.Desc) {
 	sc.requests.Describe(ch)
 
 	sc.upstreamLatency.Describe(ch)
+	sc.upstreamHeaderTime.Describe(ch)
 
 	sc.responseTime.Describe(ch)
 	sc.responseLength.Describe(ch)
@@ -458,6 +482,7 @@ func (sc SocketCollector) Collect(ch chan<- prometheus.Metric) {
 	sc.requests.Collect(ch)
 
 	sc.upstreamLatency.Collect(ch)
+	sc.upstreamHeaderTime.Collect(ch)
 
 	sc.responseTime.Collect(ch)
 	sc.responseLength.Collect(ch)
diff --git a/rootfs/etc/nginx/lua/monitor.lua b/rootfs/etc/nginx/lua/monitor.lua
index 8b0e18596..a42f26fbf 100644
--- a/rootfs/etc/nginx/lua/monitor.lua
+++ b/rootfs/etc/nginx/lua/monitor.lua
@@ -47,6 +47,7 @@ local function metrics()
     responseLength = tonumber(ngx.var.bytes_sent) or -1,
 
     upstreamLatency = tonumber(ngx.var.upstream_connect_time) or -1,
+    upstreamHeaderTime = tonumber(ngx.var.upstream_header_time) or -1,
     upstreamResponseTime = tonumber(ngx.var.upstream_response_time) or -1,
     upstreamResponseLength = tonumber(ngx.var.upstream_response_length) or -1,
     --upstreamStatus = ngx.var.upstream_status or "-",
diff --git a/rootfs/etc/nginx/lua/test/monitor_test.lua b/rootfs/etc/nginx/lua/test/monitor_test.lua
index e75018adf..99e4697f0 100644
--- a/rootfs/etc/nginx/lua/test/monitor_test.lua
+++ b/rootfs/etc/nginx/lua/test/monitor_test.lua
@@ -93,7 +93,8 @@ describe("Monitor", function()
 
         upstream_addr = "10.10.0.1",
         upstream_connect_time = "0.01",
-        upstream_response_time = "0.02",
+        upstream_header_time = "0.02",
+        upstream_response_time = "0.03",
         upstream_response_length = "456",
         upstream_status = "200",
       }
@@ -125,7 +126,8 @@ describe("Monitor", function()
           responseLength = 512,
 
           upstreamLatency = 0.01,
-          upstreamResponseTime = 0.02,
+          upstreamHeaderTime = 0.02,
+          upstreamResponseTime = 0.03,
           upstreamResponseLength = 456,
         },
         {
@@ -143,7 +145,8 @@ describe("Monitor", function()
           responseLength = 512,
 
           upstreamLatency = 0.01,
-          upstreamResponseTime = 0.02,
+          upstreamHeaderTime = 0.02,
+          upstreamResponseTime = 0.03,
           upstreamResponseLength = 456,
         },
       })