DevFW-CICD/ingress-nginx-helm
16
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Scrap json metrics from nginx vts

Browse source
This commit is contained in:
Giancarlo Rubio 2017-02-22 22:51:53 +01:00
parent db96c9d574
commit 9eaf7d41de
3 changed files with 282 additions and 115 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

244
controllers/nginx/pkg/cmd/controller/metrics.go
View file

@ -24,6 +24,7 @@ import (
common "github.com/ncabatoff/process-exporter"
"github.com/ncabatoff/process-exporter/proc"
"github.com/prometheus/client_golang/prometheus"
"reflect"
)
type exeMatcher struct {
@ -51,24 +52,16 @@ func (n *NGINXController) setupMonitor(args []string) {
}
var (
numprocsDesc = prometheus.NewDesc(
"nginx_num_procs",
"number of processes",
nil, nil)
// descriptions borrow from https://github.com/vozlt/nginx-module-vts
cpuSecsDesc = prometheus.NewDesc(
"nginx_cpu_seconds_total",
"Cpu usage in seconds",
nil, nil)
readBytesDesc = prometheus.NewDesc(
"nginx_read_bytes_total",
"number of bytes read",
nil, nil)
writeBytesDesc = prometheus.NewDesc(
"nginx_write_bytes_total",
"number of bytes written",
numprocsDesc = prometheus.NewDesc(
"nginx_num_procs",
"number of processes",
nil, nil)
memResidentbytesDesc = prometheus.NewDesc(
@ -81,45 +74,91 @@ var (
"number of bytes of memory in use",
nil, nil)
readBytesDesc = prometheus.NewDesc(
"nginx_read_bytes_total",
"number of bytes read",
nil, nil)
//vts metrics
bytesDesc = prometheus.NewDesc(
"nginx_bytes_total",
"Nginx bytes count",
[]string{"server_zones", "direction"}, nil)
cacheDesc = prometheus.NewDesc(
"nginx_cache_total",
"Nginx cache count",
[]string{"server_zones", "type"}, nil)
connectionsDesc = prometheus.NewDesc(
"nginx_connections_total",
"Nginx connections count",
[]string{"type"}, nil)
startTimeDesc = prometheus.NewDesc(
"nginx_oldest_start_time_seconds",
"start time in seconds since 1970/01/01",
nil, nil)
activeDesc = prometheus.NewDesc(
"nginx_active_connections",
"total number of active connections",
writeBytesDesc = prometheus.NewDesc(
"nginx_write_bytes_total",
"number of bytes written",
nil, nil)
acceptedDesc = prometheus.NewDesc(
"nginx_accepted_connections",
"total number of accepted client connections",
nil, nil)
responseDesc = prometheus.NewDesc(
"nginx_responses_total",
"The number of responses with status codes 1xx, 2xx, 3xx, 4xx, and 5xx.",
[]string{"server_zones", "status_code"}, nil)
handledDesc = prometheus.NewDesc(
"nginx_handled_connections",
"total number of handled connections",
nil, nil)
requestDesc = prometheus.NewDesc(
"nginx_requests_total",
"The total number of requested client connections.",
[]string{"server_zones"}, nil)
requestsDesc = prometheus.NewDesc(
"nginx_total_requests",
"total number of client requests",
nil, nil)
upstreamBackupDesc = prometheus.NewDesc(
"nginx_upstream_backup",
"Current backup setting of the server.",
[]string{"upstream", "server"}, nil)
readingDesc = prometheus.NewDesc(
"nginx_current_reading_connections",
"current number of connections where nginx is reading the request header",
nil, nil)
upstreamBytesDesc = prometheus.NewDesc(
"nginx_upstream_bytes_total",
"The total number of bytes sent to this server.",
[]string{"upstream", "server", "direction"}, nil)
writingDesc = prometheus.NewDesc(
"nginx_current_writing_connections",
"current number of connections where nginx is writing the response back to the client",
nil, nil)
upstreamDownDesc = prometheus.NewDesc(
"nginx_upstream_down_total",
"Current down setting of the server.",
[]string{"upstream", "server"}, nil)
waitingDesc = prometheus.NewDesc(
"nginx_current_waiting_connections",
"current number of idle client connections waiting for a request",
nil, nil)
upstreamFailTimeoutDesc = prometheus.NewDesc(
"nginx_upstream_fail_timeout",
"Current fail_timeout setting of the server.",
[]string{"upstream", "server"}, nil)
upstreamMaxFailsDesc = prometheus.NewDesc(
"nginx_upstream_maxfails",
"Current max_fails setting of the server.",
[]string{"upstream", "server"}, nil)
upstreamResponsesDesc = prometheus.NewDesc(
"nginx_upstream_responses_total",
"The number of upstream responses with status codes 1xx, 2xx, 3xx, 4xx, and 5xx.",
[]string{"upstream", "server", "status_code"}, nil)
upstreamRequestDesc = prometheus.NewDesc(
"nginx_upstream_requests_total",
"The total number of client connections forwarded to this server.",
[]string{"upstream", "server"}, nil)
upstreamResponseMsecDesc = prometheus.NewDesc(
"nginx_upstream_response_msecs_avg",
"The average of only upstream response processing times in milliseconds.",
[]string{"upstream", "server"}, nil)
upstreamWeightDesc = prometheus.NewDesc(
"nginx_upstream_weight",
"Current upstream weight setting of the server.",
[]string{"upstream", "server"}, nil)
)
type (
@ -160,13 +199,28 @@ func newProcessCollector(
// Describe implements prometheus.Collector.
func (p *namedProcessCollector) Describe(ch chan<- *prometheus.Desc) {
ch <- bytesDesc
ch <- cacheDesc
ch <- connectionsDesc
ch <- cpuSecsDesc
ch <- numprocsDesc
ch <- readBytesDesc
ch <- writeBytesDesc
ch <- memResidentbytesDesc
ch <- memVirtualbytesDesc
ch <- numprocsDesc
ch <- readBytesDesc
ch <- requestDesc
ch <- responseDesc
ch <- startTimeDesc
ch <- writeBytesDesc
ch <- upstreamBackupDesc
ch <- upstreamBytesDesc
ch <- upstreamDownDesc
ch <- upstreamFailTimeoutDesc
ch <- upstreamMaxFailsDesc
ch <- upstreamRequestDesc
ch <- upstreamResponseMsecDesc
ch <- upstreamResponsesDesc
ch <- upstreamWeightDesc
}
// Collect implements prometheus.Collector.
@ -184,27 +238,103 @@ func (p *namedProcessCollector) start() {
}
}
func reflectMetrics(value interface{}, desc *prometheus.Desc, ch chan<- prometheus.Metric, labels ...string) {
val := reflect.ValueOf(value).Elem()
for i := 0; i < val.NumField(); i++ {
tag := val.Type().Field(i).Tag
labels := append(labels, tag.Get("json"))
ch <- prometheus.MustNewConstMetric(desc,
prometheus.CounterValue, float64(val.Field(i).Interface().(float64)),
labels...)
}
}
func (p *namedProcessCollector) scrape(ch chan<- prometheus.Metric) {
s, err := getNginxStatus()
nginxMetrics, err := getNginxMetrics()
if err != nil {
glog.Warningf("unexpected error obtaining nginx status info: %v", err)
return
}
ch <- prometheus.MustNewConstMetric(activeDesc,
prometheus.GaugeValue, float64(s.Active))
ch <- prometheus.MustNewConstMetric(acceptedDesc,
prometheus.GaugeValue, float64(s.Accepted))
ch <- prometheus.MustNewConstMetric(handledDesc,
prometheus.GaugeValue, float64(s.Handled))
ch <- prometheus.MustNewConstMetric(requestsDesc,
prometheus.GaugeValue, float64(s.Requests))
ch <- prometheus.MustNewConstMetric(readingDesc,
prometheus.GaugeValue, float64(s.Reading))
ch <- prometheus.MustNewConstMetric(writingDesc,
prometheus.GaugeValue, float64(s.Writing))
ch <- prometheus.MustNewConstMetric(waitingDesc,
prometheus.GaugeValue, float64(s.Waiting))
reflectMetrics(&nginxMetrics.Connections, connectionsDesc, ch)
for name, zones := range nginxMetrics.UpstreamZones {
for pos, value := range zones {
reflectMetrics(&zones[pos].Responses, upstreamResponsesDesc, ch, name, value.Server)
ch <- prometheus.MustNewConstMetric(upstreamRequestDesc,
prometheus.CounterValue, float64(zones[pos].RequestCounter), name, value.Server)
ch <- prometheus.MustNewConstMetric(upstreamDownDesc,
prometheus.CounterValue, float64(zones[pos].Down), name, value.Server)
ch <- prometheus.MustNewConstMetric(upstreamWeightDesc,
prometheus.CounterValue, float64(zones[pos].Weight), name, value.Server)
ch <- prometheus.MustNewConstMetric(upstreamResponseMsecDesc,
prometheus.CounterValue, float64(zones[pos].ResponseMsec), name, value.Server)
ch <- prometheus.MustNewConstMetric(upstreamBackupDesc,
prometheus.CounterValue, float64(zones[pos].Backup), name, value.Server)
ch <- prometheus.MustNewConstMetric(upstreamFailTimeoutDesc,
prometheus.CounterValue, float64(zones[pos].FailTimeout), name, value.Server)
ch <- prometheus.MustNewConstMetric(upstreamMaxFailsDesc,
prometheus.CounterValue, float64(zones[pos].MaxFails), name, value.Server)
ch <- prometheus.MustNewConstMetric(upstreamBytesDesc,
prometheus.CounterValue, float64(zones[pos].InBytes), name, value.Server, "in")
ch <- prometheus.MustNewConstMetric(upstreamBytesDesc,
prometheus.CounterValue, float64(zones[pos].OutBytes), name, value.Server, "out")
}
}
for name, zone := range nginxMetrics.ServerZones {
reflectMetrics(&zone.Responses, responseDesc, ch, name)
ch <- prometheus.MustNewConstMetric(requestDesc,
prometheus.CounterValue, float64(zone.RequestCounter), name)
ch <- prometheus.MustNewConstMetric(bytesDesc,
prometheus.CounterValue, float64(zone.InBytes), name, "in")
ch <- prometheus.MustNewConstMetric(bytesDesc,
prometheus.CounterValue, float64(zone.OutBytes), name, "out")
//cache
ch <- prometheus.MustNewConstMetric(cacheDesc,
prometheus.CounterValue, float64(zone.Responses.CacheBypass), name, "bypass")
ch <- prometheus.MustNewConstMetric(cacheDesc,
prometheus.CounterValue, float64(zone.Responses.CacheExpired), name, "expired")
ch <- prometheus.MustNewConstMetric(cacheDesc,
prometheus.CounterValue, float64(zone.Responses.CacheHit), name, "hit")
ch <- prometheus.MustNewConstMetric(cacheDesc,
prometheus.CounterValue, float64(zone.Responses.CacheRevalidated), name, "revalidated")
ch <- prometheus.MustNewConstMetric(cacheDesc,
prometheus.CounterValue, float64(zone.Responses.CacheUpdating), name, "updating")
ch <- prometheus.MustNewConstMetric(cacheDesc,
prometheus.CounterValue, float64(zone.Responses.CacheStale), name, "stale")
ch <- prometheus.MustNewConstMetric(cacheDesc,
prometheus.CounterValue, float64(zone.Responses.CacheScarce), name, "scarce")
}
_, err = p.Update(p.fs.AllProcs())
if err != nil {

1
controllers/nginx/pkg/cmd/controller/nginx.go
View file

@ -46,6 +46,7 @@ const (
ngxHealthPort = 18080
ngxHealthPath = "/healthz"
ngxStatusPath = "/internal_nginx_status"
ngxVtsPath = "/nginx_status/format/json"
)
var (

152
controllers/nginx/pkg/cmd/controller/status.go
View file

@ -17,40 +17,101 @@ limitations under the License.
package main
import (
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"regexp"
"strconv"
)
var (
ac = regexp.MustCompile(`Active connections: (\d+)`)
sahr = regexp.MustCompile(`(\d+)\s(\d+)\s(\d+)`)
reading = regexp.MustCompile(`Reading: (\d+)`)
writing = regexp.MustCompile(`Writing: (\d+)`)
waiting = regexp.MustCompile(`Waiting: (\d+)`)
)
type nginxStatus struct {
// Active total number of active connections
Active int
// Accepted total number of accepted client connections
Accepted int
// Handled total number of handled connections. Generally, the parameter value is the same as accepts unless some resource limits have been reached (for example, the worker_connections limit).
Handled int
// Requests total number of client requests.
Requests int
// Reading current number of connections where nginx is reading the request header.
Reading int
// Writing current number of connections where nginx is writing the response back to the client.
Writing int
// Waiting current number of idle client connections waiting for a request.
Waiting int
type Vts struct {
NginxVersion string `json:"nginxVersion"`
LoadMsec int `json:"loadMsec"`
NowMsec int `json:"nowMsec"`
Connections Connections `json:"connections"`
ServerZones map[string]ServerZones `json:"serverZones"`
FilterZones map[string]FilterZone `json:"filterZones"`
UpstreamZones map[string][]UpstreamZone `json:"upstreamZones"`
}
func getNginxStatus() (*nginxStatus, error) {
resp, err := http.DefaultClient.Get(fmt.Sprintf("http://localhost:%v%v", ngxHealthPort, ngxStatusPath))
type ServerZones struct {
RequestCounter float64 `json:"requestCounter"`
InBytes float64 `json:"inBytes"`
OutBytes float64 `json:"outBytes"`
Responses Response `json:"responses"`
OverCounts OverCounts `json:"overCounts"`
}
type OverCounts struct {
RequestCounter float64 `json:"requestCounter"`
InBytes float64 `json:"inBytes"`
OutBytes float64 `json:"outBytes"`
OneXx float64 `json:"1xx"`
TwoXx float64 `json:"2xx"`
TheeXx float64 `json:"3xx"`
FourXx float64 `json:"4xx"`
FiveXx float64 `json:"5xx"`
}
type FilterZone struct {
}
type UpstreamZone struct {
Server string `json:"server"`
RequestCounter float64 `json:"requestCounter"`
InBytes float64 `json:"inBytes"`
OutBytes float64 `json:"outBytes"`
Responses Response `json:"responses"`
OverCounts OverCounts `json:"overcounts"`
ResponseMsec float64 `json:"responseMsec"`
Weight float64 `json:"weight"`
MaxFails float64 `json:"maxFails"`
FailTimeout float64 `json:"failTimeout"`
Backup BoolToFloat64 `json:"backup"`
Down BoolToFloat64 `json:"down"`
}
type Response struct {
OneXx float64 `json:"1xx"`
TwoXx float64 `json:"2xx"`
TheeXx float64 `json:"3xx"`
FourXx float64 `json:"4xx"`
FiveXx float64 `json:"5xx"`
CacheMiss float64 `json:"miss"`
CacheBypass float64 `json:"bypass"`
CacheExpired float64 `json:"expired"`
CacheStale float64 `json:"stale"`
CacheUpdating float64 `json:"updating"`
CacheRevalidated float64 `json:"revalidated"`
CacheHit float64 `json:"hit"`
CacheScarce float64 `json:"scarce"`
}
type Connections struct {
Active float64 `json:"active"`
Reading float64 `json:"reading"`
Writing float64 `json:"writing"`
Waiting float64 `json:"waiting"`
Accepted float64 `json:"accepted"`
Handled float64 `json:"handled"`
Requests float64 `json:"requests"`
}
type BoolToFloat64 float64
func (bit BoolToFloat64) UnmarshalJSON(data []byte) error {
asString := string(data)
if asString == "1" || asString == "true" {
bit = 1
} else if asString == "0" || asString == "false" {
bit = 0
} else {
return fmt.Errorf(fmt.Sprintf("Boolean unmarshal error: invalid input %s", asString))
}
return nil
}
func getNginxMetrics() (*Vts, error) {
resp, err := http.DefaultClient.Get(fmt.Sprintf("http://localhost:%v%v", ngxHealthPort, ngxVtsPath))
if err != nil {
return nil, fmt.Errorf("unexpected error scraping nginx status page: %v", err)
}
@ -64,36 +125,11 @@ func getNginxStatus() (*nginxStatus, error) {
return nil, fmt.Errorf("unexpected error scraping nginx status page (status %v)", resp.StatusCode)
}
return parse(string(data)), nil
}
var vts Vts
err = json.Unmarshal(data, &vts)
if err != nil {
return nil, fmt.Errorf("unexpected error json unmarshal (%v)", err)
}
func parse(data string) *nginxStatus {
acr := ac.FindStringSubmatch(data)
sahrr := sahr.FindStringSubmatch(data)
readingr := reading.FindStringSubmatch(data)
writingr := writing.FindStringSubmatch(data)
waitingr := waiting.FindStringSubmatch(data)
return &nginxStatus{
toInt(acr, 1),
toInt(sahrr, 1),
toInt(sahrr, 2),
toInt(sahrr, 3),
toInt(readingr, 1),
toInt(writingr, 1),
toInt(waitingr, 1),
}
}
func toInt(data []string, pos int) int {
if len(data) == 0 {
return 0
}
if pos > len(data) {
return 0
}
if v, err := strconv.Atoi(data[pos]); err == nil {
return v
}
return 0
return &vts, nil
}