Scrap json metrics from nginx vts

controllers/nginx/pkg/cmd/controller/metrics.go

@@ -24,6 +24,7 @@ import (
 	common "github.com/ncabatoff/process-exporter"
 	"github.com/ncabatoff/process-exporter/proc"
 	"github.com/prometheus/client_golang/prometheus"
+	"reflect"
 )
 
 type exeMatcher struct {
@@ -51,24 +52,16 @@ func (n *NGINXController) setupMonitor(args []string) {
 }
 
 var (
-	numprocsDesc = prometheus.NewDesc(
+	// descriptions borrow from https://github.com/vozlt/nginx-module-vts
-		"nginx_num_procs",
-		"number of processes",
-		nil, nil)
 
 	cpuSecsDesc = prometheus.NewDesc(
 		"nginx_cpu_seconds_total",
 		"Cpu usage in seconds",
 		nil, nil)
 
-	readBytesDesc = prometheus.NewDesc(
+	numprocsDesc = prometheus.NewDesc(
-		"nginx_read_bytes_total",
+		"nginx_num_procs",
-		"number of bytes read",
+		"number of processes",
-		nil, nil)
-
-	writeBytesDesc = prometheus.NewDesc(
-		"nginx_write_bytes_total",
-		"number of bytes written",
 		nil, nil)
 
 	memResidentbytesDesc = prometheus.NewDesc(
@@ -81,45 +74,91 @@ var (
 		"number of bytes of memory in use",
 		nil, nil)
 
+	readBytesDesc = prometheus.NewDesc(
+		"nginx_read_bytes_total",
+		"number of bytes read",
+		nil, nil)
+
+	//vts metrics
+	bytesDesc = prometheus.NewDesc(
+		"nginx_bytes_total",
+		"Nginx bytes count",
+		[]string{"server_zones", "direction"}, nil)
+
+	cacheDesc = prometheus.NewDesc(
+		"nginx_cache_total",
+		"Nginx cache count",
+		[]string{"server_zones", "type"}, nil)
+
+	connectionsDesc = prometheus.NewDesc(
+		"nginx_connections_total",
+		"Nginx connections count",
+		[]string{"type"}, nil)
+
 	startTimeDesc = prometheus.NewDesc(
 		"nginx_oldest_start_time_seconds",
 		"start time in seconds since 1970/01/01",
 		nil, nil)
 
-	activeDesc = prometheus.NewDesc(
+	writeBytesDesc = prometheus.NewDesc(
-		"nginx_active_connections",
+		"nginx_write_bytes_total",
-		"total number of active connections",
+		"number of bytes written",
 		nil, nil)
 
-	acceptedDesc = prometheus.NewDesc(
+	responseDesc = prometheus.NewDesc(
-		"nginx_accepted_connections",
+		"nginx_responses_total",
-		"total number of accepted client connections",
+		"The number of responses with status codes 1xx, 2xx, 3xx, 4xx, and 5xx.",
-		nil, nil)
+		[]string{"server_zones", "status_code"}, nil)
 
-	handledDesc = prometheus.NewDesc(
+	requestDesc = prometheus.NewDesc(
-		"nginx_handled_connections",
+		"nginx_requests_total",
-		"total number of handled connections",
+		"The total number of requested client connections.",
-		nil, nil)
+		[]string{"server_zones"}, nil)
 
-	requestsDesc = prometheus.NewDesc(
+	upstreamBackupDesc = prometheus.NewDesc(
-		"nginx_total_requests",
+		"nginx_upstream_backup",
-		"total number of client requests",
+		"Current backup setting of the server.",
-		nil, nil)
+		[]string{"upstream", "server"}, nil)
 
-	readingDesc = prometheus.NewDesc(
+	upstreamBytesDesc = prometheus.NewDesc(
-		"nginx_current_reading_connections",
+		"nginx_upstream_bytes_total",
-		"current number of connections where nginx is reading the request header",
+		"The total number of bytes sent to this server.",
-		nil, nil)
+		[]string{"upstream", "server", "direction"}, nil)
 
-	writingDesc = prometheus.NewDesc(
+	upstreamDownDesc = prometheus.NewDesc(
-		"nginx_current_writing_connections",
+		"nginx_upstream_down_total",
-		"current number of connections where nginx is writing the response back to the client",
+		"Current down setting of the server.",
-		nil, nil)
+		[]string{"upstream", "server"}, nil)
 
-	waitingDesc = prometheus.NewDesc(
+	upstreamFailTimeoutDesc = prometheus.NewDesc(
-		"nginx_current_waiting_connections",
+		"nginx_upstream_fail_timeout",
-		"current number of idle client connections waiting for a request",
+		"Current fail_timeout setting of the server.",
-		nil, nil)
+		[]string{"upstream", "server"}, nil)
+
+	upstreamMaxFailsDesc = prometheus.NewDesc(
+		"nginx_upstream_maxfails",
+		"Current max_fails setting of the server.",
+		[]string{"upstream", "server"}, nil)
+
+	upstreamResponsesDesc = prometheus.NewDesc(
+		"nginx_upstream_responses_total",
+		"The number of upstream responses with status codes 1xx, 2xx, 3xx, 4xx, and 5xx.",
+		[]string{"upstream", "server", "status_code"}, nil)
+
+	upstreamRequestDesc = prometheus.NewDesc(
+		"nginx_upstream_requests_total",
+		"The total number of client connections forwarded to this server.",
+		[]string{"upstream", "server"}, nil)
+
+	upstreamResponseMsecDesc = prometheus.NewDesc(
+		"nginx_upstream_response_msecs_avg",
+		"The average of only upstream response processing times in milliseconds.",
+		[]string{"upstream", "server"}, nil)
+
+	upstreamWeightDesc = prometheus.NewDesc(
+		"nginx_upstream_weight",
+		"Current upstream weight setting of the server.",
+		[]string{"upstream", "server"}, nil)
 )
 
 type (
@@ -160,13 +199,28 @@ func newProcessCollector(
 
 // Describe implements prometheus.Collector.
 func (p *namedProcessCollector) Describe(ch chan<- *prometheus.Desc) {
+	ch <- bytesDesc
+	ch <- cacheDesc
+	ch <- connectionsDesc
 	ch <- cpuSecsDesc
-	ch <- numprocsDesc
-	ch <- readBytesDesc
-	ch <- writeBytesDesc
 	ch <- memResidentbytesDesc
 	ch <- memVirtualbytesDesc
+	ch <- numprocsDesc
+	ch <- readBytesDesc
+	ch <- requestDesc
+	ch <- responseDesc
 	ch <- startTimeDesc
+	ch <- writeBytesDesc
+	ch <- upstreamBackupDesc
+	ch <- upstreamBytesDesc
+	ch <- upstreamDownDesc
+	ch <- upstreamFailTimeoutDesc
+	ch <- upstreamMaxFailsDesc
+	ch <- upstreamRequestDesc
+	ch <- upstreamResponseMsecDesc
+	ch <- upstreamResponsesDesc
+	ch <- upstreamWeightDesc
+
 }
 
 // Collect implements prometheus.Collector.
@@ -184,27 +238,103 @@ func (p *namedProcessCollector) start() {
 	}
 }
 
+func reflectMetrics(value interface{}, desc *prometheus.Desc, ch chan<- prometheus.Metric, labels ...string) {
+
+	val := reflect.ValueOf(value).Elem()
+
+	for i := 0; i < val.NumField(); i++ {
+		tag := val.Type().Field(i).Tag
+
+		labels := append(labels, tag.Get("json"))
+		ch <- prometheus.MustNewConstMetric(desc,
+			prometheus.CounterValue, float64(val.Field(i).Interface().(float64)),
+			labels...)
+
+	}
+
+}
+
 func (p *namedProcessCollector) scrape(ch chan<- prometheus.Metric) {
-	s, err := getNginxStatus()
+	nginxMetrics, err := getNginxMetrics()
 	if err != nil {
 		glog.Warningf("unexpected error obtaining nginx status info: %v", err)
 		return
 	}
 
-	ch <- prometheus.MustNewConstMetric(activeDesc,
+	reflectMetrics(&nginxMetrics.Connections, connectionsDesc, ch)
-		prometheus.GaugeValue, float64(s.Active))
+
-	ch <- prometheus.MustNewConstMetric(acceptedDesc,
+	for name, zones := range nginxMetrics.UpstreamZones {
-		prometheus.GaugeValue, float64(s.Accepted))
+
-	ch <- prometheus.MustNewConstMetric(handledDesc,
+		for pos, value := range zones {
-		prometheus.GaugeValue, float64(s.Handled))
+
-	ch <- prometheus.MustNewConstMetric(requestsDesc,
+			reflectMetrics(&zones[pos].Responses, upstreamResponsesDesc, ch, name, value.Server)
-		prometheus.GaugeValue, float64(s.Requests))
+
-	ch <- prometheus.MustNewConstMetric(readingDesc,
+			ch <- prometheus.MustNewConstMetric(upstreamRequestDesc,
-		prometheus.GaugeValue, float64(s.Reading))
+				prometheus.CounterValue, float64(zones[pos].RequestCounter), name, value.Server)
-	ch <- prometheus.MustNewConstMetric(writingDesc,
+
-		prometheus.GaugeValue, float64(s.Writing))
+			ch <- prometheus.MustNewConstMetric(upstreamDownDesc,
-	ch <- prometheus.MustNewConstMetric(waitingDesc,
+				prometheus.CounterValue, float64(zones[pos].Down), name, value.Server)
-		prometheus.GaugeValue, float64(s.Waiting))
+
+			ch <- prometheus.MustNewConstMetric(upstreamWeightDesc,
+				prometheus.CounterValue, float64(zones[pos].Weight), name, value.Server)
+
+			ch <- prometheus.MustNewConstMetric(upstreamResponseMsecDesc,
+				prometheus.CounterValue, float64(zones[pos].ResponseMsec), name, value.Server)
+
+			ch <- prometheus.MustNewConstMetric(upstreamBackupDesc,
+				prometheus.CounterValue, float64(zones[pos].Backup), name, value.Server)
+
+			ch <- prometheus.MustNewConstMetric(upstreamFailTimeoutDesc,
+				prometheus.CounterValue, float64(zones[pos].FailTimeout), name, value.Server)
+
+			ch <- prometheus.MustNewConstMetric(upstreamMaxFailsDesc,
+				prometheus.CounterValue, float64(zones[pos].MaxFails), name, value.Server)
+
+			ch <- prometheus.MustNewConstMetric(upstreamBytesDesc,
+				prometheus.CounterValue, float64(zones[pos].InBytes), name, value.Server, "in")
+
+			ch <- prometheus.MustNewConstMetric(upstreamBytesDesc,
+				prometheus.CounterValue, float64(zones[pos].OutBytes), name, value.Server, "out")
+
+		}
+	}
+
+	for name, zone := range nginxMetrics.ServerZones {
+
+		reflectMetrics(&zone.Responses, responseDesc, ch, name)
+
+		ch <- prometheus.MustNewConstMetric(requestDesc,
+			prometheus.CounterValue, float64(zone.RequestCounter), name)
+
+		ch <- prometheus.MustNewConstMetric(bytesDesc,
+			prometheus.CounterValue, float64(zone.InBytes), name, "in")
+
+		ch <- prometheus.MustNewConstMetric(bytesDesc,
+			prometheus.CounterValue, float64(zone.OutBytes), name, "out")
+
+		//cache
+		ch <- prometheus.MustNewConstMetric(cacheDesc,
+			prometheus.CounterValue, float64(zone.Responses.CacheBypass), name, "bypass")
+
+		ch <- prometheus.MustNewConstMetric(cacheDesc,
+			prometheus.CounterValue, float64(zone.Responses.CacheExpired), name, "expired")
+
+		ch <- prometheus.MustNewConstMetric(cacheDesc,
+			prometheus.CounterValue, float64(zone.Responses.CacheHit), name, "hit")
+
+		ch <- prometheus.MustNewConstMetric(cacheDesc,
+			prometheus.CounterValue, float64(zone.Responses.CacheRevalidated), name, "revalidated")
+
+		ch <- prometheus.MustNewConstMetric(cacheDesc,
+			prometheus.CounterValue, float64(zone.Responses.CacheUpdating), name, "updating")
+
+		ch <- prometheus.MustNewConstMetric(cacheDesc,
+			prometheus.CounterValue, float64(zone.Responses.CacheStale), name, "stale")
+
+		ch <- prometheus.MustNewConstMetric(cacheDesc,
+			prometheus.CounterValue, float64(zone.Responses.CacheScarce), name, "scarce")
+
+	}
 
 	_, err = p.Update(p.fs.AllProcs())
 	if err != nil {

controllers/nginx/pkg/cmd/controller/nginx.go

@@ -46,6 +46,7 @@ const (
 	ngxHealthPort = 18080
 	ngxHealthPath = "/healthz"
 	ngxStatusPath = "/internal_nginx_status"
+	ngxVtsPath    = "/nginx_status/format/json"
 )
 
 var (

controllers/nginx/pkg/cmd/controller/status.go

@@ -17,40 +17,101 @@ limitations under the License.
 package main
 
 import (
+	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"net/http"
-	"regexp"
-	"strconv"
 )
 
-var (
+type Vts struct {
-	ac      = regexp.MustCompile(`Active connections: (\d+)`)
+	NginxVersion  string                    `json:"nginxVersion"`
-	sahr    = regexp.MustCompile(`(\d+)\s(\d+)\s(\d+)`)
+	LoadMsec      int                       `json:"loadMsec"`
-	reading = regexp.MustCompile(`Reading: (\d+)`)
+	NowMsec       int                       `json:"nowMsec"`
-	writing = regexp.MustCompile(`Writing: (\d+)`)
+	Connections   Connections               `json:"connections"`
-	waiting = regexp.MustCompile(`Waiting: (\d+)`)
+	ServerZones   map[string]ServerZones    `json:"serverZones"`
-)
+	FilterZones   map[string]FilterZone     `json:"filterZones"`
-
+	UpstreamZones map[string][]UpstreamZone `json:"upstreamZones"`
-type nginxStatus struct {
-	// Active total number of active connections
-	Active int
-	// Accepted total number of accepted client connections
-	Accepted int
-	// Handled total number of handled connections. Generally, the parameter value is the same as accepts unless some resource limits have been reached (for example, the worker_connections limit).
-	Handled int
-	// Requests total number of client requests.
-	Requests int
-	// Reading current number of connections where nginx is reading the request header.
-	Reading int
-	// Writing current number of connections where nginx is writing the response back to the client.
-	Writing int
-	// Waiting current number of idle client connections waiting for a request.
-	Waiting int
 }
 
-func getNginxStatus() (*nginxStatus, error) {
+type ServerZones struct {
-	resp, err := http.DefaultClient.Get(fmt.Sprintf("http://localhost:%v%v", ngxHealthPort, ngxStatusPath))
+	RequestCounter float64    `json:"requestCounter"`
+	InBytes        float64    `json:"inBytes"`
+	OutBytes       float64    `json:"outBytes"`
+	Responses      Response   `json:"responses"`
+	OverCounts     OverCounts `json:"overCounts"`
+}
+
+type OverCounts struct {
+	RequestCounter float64 `json:"requestCounter"`
+	InBytes        float64 `json:"inBytes"`
+	OutBytes       float64 `json:"outBytes"`
+	OneXx          float64 `json:"1xx"`
+	TwoXx          float64 `json:"2xx"`
+	TheeXx         float64 `json:"3xx"`
+	FourXx         float64 `json:"4xx"`
+	FiveXx         float64 `json:"5xx"`
+}
+
+type FilterZone struct {
+}
+
+type UpstreamZone struct {
+	Server         string        `json:"server"`
+	RequestCounter float64       `json:"requestCounter"`
+	InBytes        float64       `json:"inBytes"`
+	OutBytes       float64       `json:"outBytes"`
+	Responses      Response      `json:"responses"`
+	OverCounts     OverCounts    `json:"overcounts"`
+	ResponseMsec   float64       `json:"responseMsec"`
+	Weight         float64       `json:"weight"`
+	MaxFails       float64       `json:"maxFails"`
+	FailTimeout    float64       `json:"failTimeout"`
+	Backup         BoolToFloat64 `json:"backup"`
+	Down           BoolToFloat64 `json:"down"`
+}
+
+type Response struct {
+	OneXx            float64 `json:"1xx"`
+	TwoXx            float64 `json:"2xx"`
+	TheeXx           float64 `json:"3xx"`
+	FourXx           float64 `json:"4xx"`
+	FiveXx           float64 `json:"5xx"`
+	CacheMiss        float64 `json:"miss"`
+	CacheBypass      float64 `json:"bypass"`
+	CacheExpired     float64 `json:"expired"`
+	CacheStale       float64 `json:"stale"`
+	CacheUpdating    float64 `json:"updating"`
+	CacheRevalidated float64 `json:"revalidated"`
+	CacheHit         float64 `json:"hit"`
+	CacheScarce      float64 `json:"scarce"`
+}
+
+type Connections struct {
+	Active   float64 `json:"active"`
+	Reading  float64 `json:"reading"`
+	Writing  float64 `json:"writing"`
+	Waiting  float64 `json:"waiting"`
+	Accepted float64 `json:"accepted"`
+	Handled  float64 `json:"handled"`
+	Requests float64 `json:"requests"`
+}
+
+type BoolToFloat64 float64
+
+func (bit BoolToFloat64) UnmarshalJSON(data []byte) error {
+	asString := string(data)
+	if asString == "1" || asString == "true" {
+		bit = 1
+	} else if asString == "0" || asString == "false" {
+		bit = 0
+	} else {
+		return fmt.Errorf(fmt.Sprintf("Boolean unmarshal error: invalid input %s", asString))
+	}
+	return nil
+}
+
+func getNginxMetrics() (*Vts, error) {
+	resp, err := http.DefaultClient.Get(fmt.Sprintf("http://localhost:%v%v", ngxHealthPort, ngxVtsPath))
 	if err != nil {
 		return nil, fmt.Errorf("unexpected error scraping nginx status page: %v", err)
 	}
@@ -64,36 +125,11 @@ func getNginxStatus() (*nginxStatus, error) {
 		return nil, fmt.Errorf("unexpected error scraping nginx status page (status %v)", resp.StatusCode)
 	}
 
-	return parse(string(data)), nil
+	var vts Vts
-}
+	err = json.Unmarshal(data, &vts)
+	if err != nil {
+		return nil, fmt.Errorf("unexpected error json unmarshal (%v)", err)
+	}
 
-func parse(data string) *nginxStatus {
+	return &vts, nil
-	acr := ac.FindStringSubmatch(data)
-	sahrr := sahr.FindStringSubmatch(data)
-	readingr := reading.FindStringSubmatch(data)
-	writingr := writing.FindStringSubmatch(data)
-	waitingr := waiting.FindStringSubmatch(data)
-
-	return &nginxStatus{
-		toInt(acr, 1),
-		toInt(sahrr, 1),
-		toInt(sahrr, 2),
-		toInt(sahrr, 3),
-		toInt(readingr, 1),
-		toInt(writingr, 1),
-		toInt(waitingr, 1),
-	}
-}
-
-func toInt(data []string, pos int) int {
-	if len(data) == 0 {
-		return 0
-	}
-	if pos > len(data) {
-		return 0
-	}
-	if v, err := strconv.Atoi(data[pos]); err == nil {
-		return v
-	}
-	return 0
 }