From 9eaf7d41de2e40dd95d046f5f6091f7ef8becf59 Mon Sep 17 00:00:00 2001 From: Giancarlo Rubio Date: Wed, 22 Feb 2017 22:51:53 +0100 Subject: [PATCH] Scrap json metrics from nginx vts --- .../nginx/pkg/cmd/controller/metrics.go | 244 ++++++++++++++---- controllers/nginx/pkg/cmd/controller/nginx.go | 1 + .../nginx/pkg/cmd/controller/status.go | 152 ++++++----- 3 files changed, 282 insertions(+), 115 deletions(-) diff --git a/controllers/nginx/pkg/cmd/controller/metrics.go b/controllers/nginx/pkg/cmd/controller/metrics.go index a4e8b562d..3a5247c1f 100644 --- a/controllers/nginx/pkg/cmd/controller/metrics.go +++ b/controllers/nginx/pkg/cmd/controller/metrics.go @@ -24,6 +24,7 @@ import ( common "github.com/ncabatoff/process-exporter" "github.com/ncabatoff/process-exporter/proc" "github.com/prometheus/client_golang/prometheus" + "reflect" ) type exeMatcher struct { @@ -51,24 +52,16 @@ func (n *NGINXController) setupMonitor(args []string) { } var ( - numprocsDesc = prometheus.NewDesc( - "nginx_num_procs", - "number of processes", - nil, nil) + // descriptions borrow from https://github.com/vozlt/nginx-module-vts cpuSecsDesc = prometheus.NewDesc( "nginx_cpu_seconds_total", "Cpu usage in seconds", nil, nil) - readBytesDesc = prometheus.NewDesc( - "nginx_read_bytes_total", - "number of bytes read", - nil, nil) - - writeBytesDesc = prometheus.NewDesc( - "nginx_write_bytes_total", - "number of bytes written", + numprocsDesc = prometheus.NewDesc( + "nginx_num_procs", + "number of processes", nil, nil) memResidentbytesDesc = prometheus.NewDesc( @@ -81,45 +74,91 @@ var ( "number of bytes of memory in use", nil, nil) + readBytesDesc = prometheus.NewDesc( + "nginx_read_bytes_total", + "number of bytes read", + nil, nil) + + //vts metrics + bytesDesc = prometheus.NewDesc( + "nginx_bytes_total", + "Nginx bytes count", + []string{"server_zones", "direction"}, nil) + + cacheDesc = prometheus.NewDesc( + "nginx_cache_total", + "Nginx cache count", + []string{"server_zones", "type"}, nil) + + connectionsDesc = prometheus.NewDesc( + "nginx_connections_total", + "Nginx connections count", + []string{"type"}, nil) + startTimeDesc = prometheus.NewDesc( "nginx_oldest_start_time_seconds", "start time in seconds since 1970/01/01", nil, nil) - activeDesc = prometheus.NewDesc( - "nginx_active_connections", - "total number of active connections", + writeBytesDesc = prometheus.NewDesc( + "nginx_write_bytes_total", + "number of bytes written", nil, nil) - acceptedDesc = prometheus.NewDesc( - "nginx_accepted_connections", - "total number of accepted client connections", - nil, nil) + responseDesc = prometheus.NewDesc( + "nginx_responses_total", + "The number of responses with status codes 1xx, 2xx, 3xx, 4xx, and 5xx.", + []string{"server_zones", "status_code"}, nil) - handledDesc = prometheus.NewDesc( - "nginx_handled_connections", - "total number of handled connections", - nil, nil) + requestDesc = prometheus.NewDesc( + "nginx_requests_total", + "The total number of requested client connections.", + []string{"server_zones"}, nil) - requestsDesc = prometheus.NewDesc( - "nginx_total_requests", - "total number of client requests", - nil, nil) + upstreamBackupDesc = prometheus.NewDesc( + "nginx_upstream_backup", + "Current backup setting of the server.", + []string{"upstream", "server"}, nil) - readingDesc = prometheus.NewDesc( - "nginx_current_reading_connections", - "current number of connections where nginx is reading the request header", - nil, nil) + upstreamBytesDesc = prometheus.NewDesc( + "nginx_upstream_bytes_total", + "The total number of bytes sent to this server.", + []string{"upstream", "server", "direction"}, nil) - writingDesc = prometheus.NewDesc( - "nginx_current_writing_connections", - "current number of connections where nginx is writing the response back to the client", - nil, nil) + upstreamDownDesc = prometheus.NewDesc( + "nginx_upstream_down_total", + "Current down setting of the server.", + []string{"upstream", "server"}, nil) - waitingDesc = prometheus.NewDesc( - "nginx_current_waiting_connections", - "current number of idle client connections waiting for a request", - nil, nil) + upstreamFailTimeoutDesc = prometheus.NewDesc( + "nginx_upstream_fail_timeout", + "Current fail_timeout setting of the server.", + []string{"upstream", "server"}, nil) + + upstreamMaxFailsDesc = prometheus.NewDesc( + "nginx_upstream_maxfails", + "Current max_fails setting of the server.", + []string{"upstream", "server"}, nil) + + upstreamResponsesDesc = prometheus.NewDesc( + "nginx_upstream_responses_total", + "The number of upstream responses with status codes 1xx, 2xx, 3xx, 4xx, and 5xx.", + []string{"upstream", "server", "status_code"}, nil) + + upstreamRequestDesc = prometheus.NewDesc( + "nginx_upstream_requests_total", + "The total number of client connections forwarded to this server.", + []string{"upstream", "server"}, nil) + + upstreamResponseMsecDesc = prometheus.NewDesc( + "nginx_upstream_response_msecs_avg", + "The average of only upstream response processing times in milliseconds.", + []string{"upstream", "server"}, nil) + + upstreamWeightDesc = prometheus.NewDesc( + "nginx_upstream_weight", + "Current upstream weight setting of the server.", + []string{"upstream", "server"}, nil) ) type ( @@ -160,13 +199,28 @@ func newProcessCollector( // Describe implements prometheus.Collector. func (p *namedProcessCollector) Describe(ch chan<- *prometheus.Desc) { + ch <- bytesDesc + ch <- cacheDesc + ch <- connectionsDesc ch <- cpuSecsDesc - ch <- numprocsDesc - ch <- readBytesDesc - ch <- writeBytesDesc ch <- memResidentbytesDesc ch <- memVirtualbytesDesc + ch <- numprocsDesc + ch <- readBytesDesc + ch <- requestDesc + ch <- responseDesc ch <- startTimeDesc + ch <- writeBytesDesc + ch <- upstreamBackupDesc + ch <- upstreamBytesDesc + ch <- upstreamDownDesc + ch <- upstreamFailTimeoutDesc + ch <- upstreamMaxFailsDesc + ch <- upstreamRequestDesc + ch <- upstreamResponseMsecDesc + ch <- upstreamResponsesDesc + ch <- upstreamWeightDesc + } // Collect implements prometheus.Collector. @@ -184,27 +238,103 @@ func (p *namedProcessCollector) start() { } } +func reflectMetrics(value interface{}, desc *prometheus.Desc, ch chan<- prometheus.Metric, labels ...string) { + + val := reflect.ValueOf(value).Elem() + + for i := 0; i < val.NumField(); i++ { + tag := val.Type().Field(i).Tag + + labels := append(labels, tag.Get("json")) + ch <- prometheus.MustNewConstMetric(desc, + prometheus.CounterValue, float64(val.Field(i).Interface().(float64)), + labels...) + + } + +} + func (p *namedProcessCollector) scrape(ch chan<- prometheus.Metric) { - s, err := getNginxStatus() + nginxMetrics, err := getNginxMetrics() if err != nil { glog.Warningf("unexpected error obtaining nginx status info: %v", err) return } - ch <- prometheus.MustNewConstMetric(activeDesc, - prometheus.GaugeValue, float64(s.Active)) - ch <- prometheus.MustNewConstMetric(acceptedDesc, - prometheus.GaugeValue, float64(s.Accepted)) - ch <- prometheus.MustNewConstMetric(handledDesc, - prometheus.GaugeValue, float64(s.Handled)) - ch <- prometheus.MustNewConstMetric(requestsDesc, - prometheus.GaugeValue, float64(s.Requests)) - ch <- prometheus.MustNewConstMetric(readingDesc, - prometheus.GaugeValue, float64(s.Reading)) - ch <- prometheus.MustNewConstMetric(writingDesc, - prometheus.GaugeValue, float64(s.Writing)) - ch <- prometheus.MustNewConstMetric(waitingDesc, - prometheus.GaugeValue, float64(s.Waiting)) + reflectMetrics(&nginxMetrics.Connections, connectionsDesc, ch) + + for name, zones := range nginxMetrics.UpstreamZones { + + for pos, value := range zones { + + reflectMetrics(&zones[pos].Responses, upstreamResponsesDesc, ch, name, value.Server) + + ch <- prometheus.MustNewConstMetric(upstreamRequestDesc, + prometheus.CounterValue, float64(zones[pos].RequestCounter), name, value.Server) + + ch <- prometheus.MustNewConstMetric(upstreamDownDesc, + prometheus.CounterValue, float64(zones[pos].Down), name, value.Server) + + ch <- prometheus.MustNewConstMetric(upstreamWeightDesc, + prometheus.CounterValue, float64(zones[pos].Weight), name, value.Server) + + ch <- prometheus.MustNewConstMetric(upstreamResponseMsecDesc, + prometheus.CounterValue, float64(zones[pos].ResponseMsec), name, value.Server) + + ch <- prometheus.MustNewConstMetric(upstreamBackupDesc, + prometheus.CounterValue, float64(zones[pos].Backup), name, value.Server) + + ch <- prometheus.MustNewConstMetric(upstreamFailTimeoutDesc, + prometheus.CounterValue, float64(zones[pos].FailTimeout), name, value.Server) + + ch <- prometheus.MustNewConstMetric(upstreamMaxFailsDesc, + prometheus.CounterValue, float64(zones[pos].MaxFails), name, value.Server) + + ch <- prometheus.MustNewConstMetric(upstreamBytesDesc, + prometheus.CounterValue, float64(zones[pos].InBytes), name, value.Server, "in") + + ch <- prometheus.MustNewConstMetric(upstreamBytesDesc, + prometheus.CounterValue, float64(zones[pos].OutBytes), name, value.Server, "out") + + } + } + + for name, zone := range nginxMetrics.ServerZones { + + reflectMetrics(&zone.Responses, responseDesc, ch, name) + + ch <- prometheus.MustNewConstMetric(requestDesc, + prometheus.CounterValue, float64(zone.RequestCounter), name) + + ch <- prometheus.MustNewConstMetric(bytesDesc, + prometheus.CounterValue, float64(zone.InBytes), name, "in") + + ch <- prometheus.MustNewConstMetric(bytesDesc, + prometheus.CounterValue, float64(zone.OutBytes), name, "out") + + //cache + ch <- prometheus.MustNewConstMetric(cacheDesc, + prometheus.CounterValue, float64(zone.Responses.CacheBypass), name, "bypass") + + ch <- prometheus.MustNewConstMetric(cacheDesc, + prometheus.CounterValue, float64(zone.Responses.CacheExpired), name, "expired") + + ch <- prometheus.MustNewConstMetric(cacheDesc, + prometheus.CounterValue, float64(zone.Responses.CacheHit), name, "hit") + + ch <- prometheus.MustNewConstMetric(cacheDesc, + prometheus.CounterValue, float64(zone.Responses.CacheRevalidated), name, "revalidated") + + ch <- prometheus.MustNewConstMetric(cacheDesc, + prometheus.CounterValue, float64(zone.Responses.CacheUpdating), name, "updating") + + ch <- prometheus.MustNewConstMetric(cacheDesc, + prometheus.CounterValue, float64(zone.Responses.CacheStale), name, "stale") + + ch <- prometheus.MustNewConstMetric(cacheDesc, + prometheus.CounterValue, float64(zone.Responses.CacheScarce), name, "scarce") + + } _, err = p.Update(p.fs.AllProcs()) if err != nil { diff --git a/controllers/nginx/pkg/cmd/controller/nginx.go b/controllers/nginx/pkg/cmd/controller/nginx.go index f8e92bac0..ae0b12022 100644 --- a/controllers/nginx/pkg/cmd/controller/nginx.go +++ b/controllers/nginx/pkg/cmd/controller/nginx.go @@ -46,6 +46,7 @@ const ( ngxHealthPort = 18080 ngxHealthPath = "/healthz" ngxStatusPath = "/internal_nginx_status" + ngxVtsPath = "/nginx_status/format/json" ) var ( diff --git a/controllers/nginx/pkg/cmd/controller/status.go b/controllers/nginx/pkg/cmd/controller/status.go index bfa1c383b..60304ebde 100644 --- a/controllers/nginx/pkg/cmd/controller/status.go +++ b/controllers/nginx/pkg/cmd/controller/status.go @@ -17,40 +17,101 @@ limitations under the License. package main import ( + "encoding/json" "fmt" "io/ioutil" "net/http" - "regexp" - "strconv" ) -var ( - ac = regexp.MustCompile(`Active connections: (\d+)`) - sahr = regexp.MustCompile(`(\d+)\s(\d+)\s(\d+)`) - reading = regexp.MustCompile(`Reading: (\d+)`) - writing = regexp.MustCompile(`Writing: (\d+)`) - waiting = regexp.MustCompile(`Waiting: (\d+)`) -) - -type nginxStatus struct { - // Active total number of active connections - Active int - // Accepted total number of accepted client connections - Accepted int - // Handled total number of handled connections. Generally, the parameter value is the same as accepts unless some resource limits have been reached (for example, the worker_connections limit). - Handled int - // Requests total number of client requests. - Requests int - // Reading current number of connections where nginx is reading the request header. - Reading int - // Writing current number of connections where nginx is writing the response back to the client. - Writing int - // Waiting current number of idle client connections waiting for a request. - Waiting int +type Vts struct { + NginxVersion string `json:"nginxVersion"` + LoadMsec int `json:"loadMsec"` + NowMsec int `json:"nowMsec"` + Connections Connections `json:"connections"` + ServerZones map[string]ServerZones `json:"serverZones"` + FilterZones map[string]FilterZone `json:"filterZones"` + UpstreamZones map[string][]UpstreamZone `json:"upstreamZones"` } -func getNginxStatus() (*nginxStatus, error) { - resp, err := http.DefaultClient.Get(fmt.Sprintf("http://localhost:%v%v", ngxHealthPort, ngxStatusPath)) +type ServerZones struct { + RequestCounter float64 `json:"requestCounter"` + InBytes float64 `json:"inBytes"` + OutBytes float64 `json:"outBytes"` + Responses Response `json:"responses"` + OverCounts OverCounts `json:"overCounts"` +} + +type OverCounts struct { + RequestCounter float64 `json:"requestCounter"` + InBytes float64 `json:"inBytes"` + OutBytes float64 `json:"outBytes"` + OneXx float64 `json:"1xx"` + TwoXx float64 `json:"2xx"` + TheeXx float64 `json:"3xx"` + FourXx float64 `json:"4xx"` + FiveXx float64 `json:"5xx"` +} + +type FilterZone struct { +} + +type UpstreamZone struct { + Server string `json:"server"` + RequestCounter float64 `json:"requestCounter"` + InBytes float64 `json:"inBytes"` + OutBytes float64 `json:"outBytes"` + Responses Response `json:"responses"` + OverCounts OverCounts `json:"overcounts"` + ResponseMsec float64 `json:"responseMsec"` + Weight float64 `json:"weight"` + MaxFails float64 `json:"maxFails"` + FailTimeout float64 `json:"failTimeout"` + Backup BoolToFloat64 `json:"backup"` + Down BoolToFloat64 `json:"down"` +} + +type Response struct { + OneXx float64 `json:"1xx"` + TwoXx float64 `json:"2xx"` + TheeXx float64 `json:"3xx"` + FourXx float64 `json:"4xx"` + FiveXx float64 `json:"5xx"` + CacheMiss float64 `json:"miss"` + CacheBypass float64 `json:"bypass"` + CacheExpired float64 `json:"expired"` + CacheStale float64 `json:"stale"` + CacheUpdating float64 `json:"updating"` + CacheRevalidated float64 `json:"revalidated"` + CacheHit float64 `json:"hit"` + CacheScarce float64 `json:"scarce"` +} + +type Connections struct { + Active float64 `json:"active"` + Reading float64 `json:"reading"` + Writing float64 `json:"writing"` + Waiting float64 `json:"waiting"` + Accepted float64 `json:"accepted"` + Handled float64 `json:"handled"` + Requests float64 `json:"requests"` +} + +type BoolToFloat64 float64 + +func (bit BoolToFloat64) UnmarshalJSON(data []byte) error { + asString := string(data) + if asString == "1" || asString == "true" { + bit = 1 + } else if asString == "0" || asString == "false" { + bit = 0 + } else { + return fmt.Errorf(fmt.Sprintf("Boolean unmarshal error: invalid input %s", asString)) + } + return nil +} + +func getNginxMetrics() (*Vts, error) { + resp, err := http.DefaultClient.Get(fmt.Sprintf("http://localhost:%v%v", ngxHealthPort, ngxVtsPath)) if err != nil { return nil, fmt.Errorf("unexpected error scraping nginx status page: %v", err) } @@ -64,36 +125,11 @@ func getNginxStatus() (*nginxStatus, error) { return nil, fmt.Errorf("unexpected error scraping nginx status page (status %v)", resp.StatusCode) } - return parse(string(data)), nil -} + var vts Vts + err = json.Unmarshal(data, &vts) + if err != nil { + return nil, fmt.Errorf("unexpected error json unmarshal (%v)", err) + } -func parse(data string) *nginxStatus { - acr := ac.FindStringSubmatch(data) - sahrr := sahr.FindStringSubmatch(data) - readingr := reading.FindStringSubmatch(data) - writingr := writing.FindStringSubmatch(data) - waitingr := waiting.FindStringSubmatch(data) - - return &nginxStatus{ - toInt(acr, 1), - toInt(sahrr, 1), - toInt(sahrr, 2), - toInt(sahrr, 3), - toInt(readingr, 1), - toInt(writingr, 1), - toInt(waitingr, 1), - } -} - -func toInt(data []string, pos int) int { - if len(data) == 0 { - return 0 - } - if pos > len(data) { - return 0 - } - if v, err := strconv.Atoi(data[pos]); err == nil { - return v - } - return 0 + return &vts, nil }