From 1d38e3a38425f08de2f75fcae13896a3fec4d144 Mon Sep 17 00:00:00 2001 From: Giancarlo Rubio Date: Wed, 22 Feb 2017 22:51:53 +0100 Subject: [PATCH 1/3] Scrap json metrics from nginx vts upgrade vts to the latest version --- .../nginx/pkg/cmd/controller/metrics.go | 317 +++++++++++++++--- controllers/nginx/pkg/cmd/controller/nginx.go | 10 +- .../nginx/pkg/cmd/controller/status.go | 135 +++++++- .../nginx/pkg/cmd/controller/status_test.go | 2 +- controllers/nginx/pkg/config/config.go | 1 + .../rootfs/etc/nginx/template/nginx.tmpl | 32 +- images/nginx-slim/build.sh | 2 +- 7 files changed, 424 insertions(+), 75 deletions(-) diff --git a/controllers/nginx/pkg/cmd/controller/metrics.go b/controllers/nginx/pkg/cmd/controller/metrics.go index a4e8b562d..1b6903018 100644 --- a/controllers/nginx/pkg/cmd/controller/metrics.go +++ b/controllers/nginx/pkg/cmd/controller/metrics.go @@ -24,51 +24,22 @@ import ( common "github.com/ncabatoff/process-exporter" "github.com/ncabatoff/process-exporter/proc" "github.com/prometheus/client_golang/prometheus" + "reflect" ) -type exeMatcher struct { - name string - args []string -} - -func (em exeMatcher) MatchAndName(nacl common.NameAndCmdline) (bool, string) { - if len(nacl.Cmdline) == 0 { - return false, "" - } - cmd := filepath.Base(nacl.Cmdline[0]) - return em.name == cmd, "" -} - -func (n *NGINXController) setupMonitor(args []string) { - pc, err := newProcessCollector(true, exeMatcher{"nginx", args}) - if err != nil { - glog.Fatalf("unexpected error registering nginx collector: %v", err) - } - err = prometheus.Register(pc) - if err != nil { - glog.Warningf("unexpected error registering nginx collector: %v", err) - } -} - +// TODO add current namespace +// TODO add ingress class var ( - numprocsDesc = prometheus.NewDesc( - "nginx_num_procs", - "number of processes", - nil, nil) + // descriptions borrow from https://github.com/vozlt/nginx-module-vts cpuSecsDesc = prometheus.NewDesc( "nginx_cpu_seconds_total", "Cpu usage in seconds", nil, nil) - readBytesDesc = prometheus.NewDesc( - "nginx_read_bytes_total", - "number of bytes read", - nil, nil) - - writeBytesDesc = prometheus.NewDesc( - "nginx_write_bytes_total", - "number of bytes written", + numprocsDesc = prometheus.NewDesc( + "nginx_num_procs", + "number of processes", nil, nil) memResidentbytesDesc = prometheus.NewDesc( @@ -81,11 +52,107 @@ var ( "number of bytes of memory in use", nil, nil) + readBytesDesc = prometheus.NewDesc( + "nginx_read_bytes_total", + "number of bytes read", + nil, nil) + startTimeDesc = prometheus.NewDesc( "nginx_oldest_start_time_seconds", "start time in seconds since 1970/01/01", nil, nil) + writeBytesDesc = prometheus.NewDesc( + "nginx_write_bytes_total", + "number of bytes written", + nil, nil) + + //vts metrics + vtsBytesDesc = prometheus.NewDesc( + "nginx_vts_bytes_total", + "Nginx bytes count", + []string{"server_zone", "direction"}, nil) + + vtsCacheDesc = prometheus.NewDesc( + "nginx_vts_cache_total", + "Nginx cache count", + []string{"server_zone", "type"}, nil) + + vtsConnectionsDesc = prometheus.NewDesc( + "nginx_vts_connections_total", + "Nginx connections count", + []string{"type"}, nil) + + vtsResponseDesc = prometheus.NewDesc( + "nginx_vts_responses_total", + "The number of responses with status codes 1xx, 2xx, 3xx, 4xx, and 5xx.", + []string{"server_zone", "status_code"}, nil) + + vtsRequestDesc = prometheus.NewDesc( + "nginx_vts_requests_total", + "The total number of requested client connections.", + []string{"server_zone"}, nil) + + vtsFilterZoneBytesDesc = prometheus.NewDesc( + "nginx_vts_filterzone_bytes_total", + "Nginx bytes count", + []string{"server_zone", "country", "direction"}, nil) + + vtsFilterZoneResponseDesc = prometheus.NewDesc( + "nginx_vts_filterzone_responses_total", + "The number of responses with status codes 1xx, 2xx, 3xx, 4xx, and 5xx.", + []string{"server_zone", "country", "status_code"}, nil) + + vtsFilterZoneCacheDesc = prometheus.NewDesc( + "nginx_vts_filterzone_cache_total", + "Nginx cache count", + []string{"server_zone", "country", "type"}, nil) + + vtsUpstreamBackupDesc = prometheus.NewDesc( + "nginx_vts_upstream_backup", + "Current backup setting of the server.", + []string{"upstream", "server"}, nil) + + vtsUpstreamBytesDesc = prometheus.NewDesc( + "nginx_vts_upstream_bytes_total", + "The total number of bytes sent to this server.", + []string{"upstream", "server", "direction"}, nil) + + vtsUpstreamDownDesc = prometheus.NewDesc( + "nginx_vts_upstream_down_total", + "Current down setting of the server.", + []string{"upstream", "server"}, nil) + + vtsUpstreamFailTimeoutDesc = prometheus.NewDesc( + "nginx_vts_upstream_fail_timeout", + "Current fail_timeout setting of the server.", + []string{"upstream", "server"}, nil) + + vtsUpstreamMaxFailsDesc = prometheus.NewDesc( + "nginx_vts_upstream_maxfails", + "Current max_fails setting of the server.", + []string{"upstream", "server"}, nil) + + vtsUpstreamResponsesDesc = prometheus.NewDesc( + "nginx_vts_upstream_responses_total", + "The number of upstream responses with status codes 1xx, 2xx, 3xx, 4xx, and 5xx.", + []string{"upstream", "server", "status_code"}, nil) + + vtsUpstreamRequestDesc = prometheus.NewDesc( + "nginx_vts_upstream_requests_total", + "The total number of client connections forwarded to this server.", + []string{"upstream", "server"}, nil) + + vtsUpstreamResponseMsecDesc = prometheus.NewDesc( + "nginx_vts_upstream_response_msecs_avg", + "The average of only upstream response processing times in milliseconds.", + []string{"upstream", "server"}, nil) + + vtsUpstreamWeightDesc = prometheus.NewDesc( + "nginx_vts_upstream_weight", + "Current upstream weight setting of the server.", + []string{"upstream", "server"}, nil) + activeDesc = prometheus.NewDesc( "nginx_active_connections", "total number of active connections", @@ -122,6 +189,37 @@ var ( nil, nil) ) +type exeMatcher struct { + name string + args []string +} + +func (em exeMatcher) MatchAndName(nacl common.NameAndCmdline) (bool, string) { + if len(nacl.Cmdline) == 0 { + return false, "" + } + cmd := filepath.Base(nacl.Cmdline[0]) + return em.name == cmd, "" +} + +func (n *NGINXController) setupMonitor(args []string, vtsCollector bool) { + + pc, err := newProcessCollector(true, exeMatcher{"nginx", args}, vtsCollector) + + if err != nil { + glog.Fatalf("unexpected error registering nginx collector: %v", err) + } + + err = prometheus.Register(pc) + + if err != nil { + if _, ok := err.(prometheus.AlreadyRegisteredError); !ok { + glog.Warningf("unexpected error registering nginx collector: %v", err) + } + } + +} + type ( scrapeRequest struct { results chan<- prometheus.Metric @@ -131,22 +229,25 @@ type ( namedProcessCollector struct { scrapeChan chan scrapeRequest *proc.Grouper - fs *proc.FS + fs *proc.FS + enableVtsCollector bool } ) func newProcessCollector( children bool, - n common.MatchNamer) (*namedProcessCollector, error) { + n common.MatchNamer, + enableVtsCollector bool) (*namedProcessCollector, error) { fs, err := proc.NewFS("/proc") if err != nil { return nil, err } p := &namedProcessCollector{ - scrapeChan: make(chan scrapeRequest), - Grouper: proc.NewGrouper(children, n), - fs: fs, + scrapeChan: make(chan scrapeRequest), + Grouper: proc.NewGrouper(children, n), + fs: fs, + enableVtsCollector: enableVtsCollector, } _, err = p.Update(p.fs.AllProcs()) if err != nil { @@ -160,6 +261,7 @@ func newProcessCollector( // Describe implements prometheus.Collector. func (p *namedProcessCollector) Describe(ch chan<- *prometheus.Desc) { + ch <- cpuSecsDesc ch <- numprocsDesc ch <- readBytesDesc @@ -167,6 +269,26 @@ func (p *namedProcessCollector) Describe(ch chan<- *prometheus.Desc) { ch <- memResidentbytesDesc ch <- memVirtualbytesDesc ch <- startTimeDesc + + //vts metrics + ch <- vtsBytesDesc + ch <- vtsCacheDesc + ch <- vtsConnectionsDesc + ch <- vtsRequestDesc + ch <- vtsResponseDesc + ch <- vtsUpstreamBackupDesc + ch <- vtsUpstreamBytesDesc + ch <- vtsUpstreamDownDesc + ch <- vtsUpstreamFailTimeoutDesc + ch <- vtsUpstreamMaxFailsDesc + ch <- vtsUpstreamRequestDesc + ch <- vtsUpstreamResponseMsecDesc + ch <- vtsUpstreamResponsesDesc + ch <- vtsUpstreamWeightDesc + ch <- vtsFilterZoneBytesDesc + ch <- vtsFilterZoneCacheDesc + ch <- vtsFilterZoneResponseDesc + } // Collect implements prometheus.Collector. @@ -177,15 +299,21 @@ func (p *namedProcessCollector) Collect(ch chan<- prometheus.Metric) { } func (p *namedProcessCollector) start() { + for req := range p.scrapeChan { ch := req.results - p.scrape(ch) + p.scrapeNginxStatus(ch) + p.scrapeProcs(ch) + p.scrapeVts(ch) + req.done <- struct{}{} } } -func (p *namedProcessCollector) scrape(ch chan<- prometheus.Metric) { +// scrapeNginxStatus scrap the nginx status +func (p *namedProcessCollector) scrapeNginxStatus(ch chan<- prometheus.Metric) { s, err := getNginxStatus() + if err != nil { glog.Warningf("unexpected error obtaining nginx status info: %v", err) return @@ -206,7 +334,93 @@ func (p *namedProcessCollector) scrape(ch chan<- prometheus.Metric) { ch <- prometheus.MustNewConstMetric(waitingDesc, prometheus.GaugeValue, float64(s.Waiting)) - _, err = p.Update(p.fs.AllProcs()) +} + +// scrapeVts scrape nginx vts metrics +func (p *namedProcessCollector) scrapeVts(ch chan<- prometheus.Metric) { + + nginxMetrics, err := getNginxVtsMetrics() + if err != nil { + glog.Warningf("unexpected error obtaining nginx status info: %v", err) + return + } + + reflectMetrics(&nginxMetrics.Connections, vtsConnectionsDesc, ch) + + for name, zones := range nginxMetrics.UpstreamZones { + + for pos, value := range zones { + + reflectMetrics(&zones[pos].Responses, vtsUpstreamResponsesDesc, ch, name, value.Server) + + ch <- prometheus.MustNewConstMetric(vtsUpstreamRequestDesc, + prometheus.CounterValue, float64(zones[pos].RequestCounter), name, value.Server) + + ch <- prometheus.MustNewConstMetric(vtsUpstreamDownDesc, + prometheus.CounterValue, float64(zones[pos].Down), name, value.Server) + + ch <- prometheus.MustNewConstMetric(vtsUpstreamWeightDesc, + prometheus.CounterValue, float64(zones[pos].Weight), name, value.Server) + + ch <- prometheus.MustNewConstMetric(vtsUpstreamResponseMsecDesc, + prometheus.CounterValue, float64(zones[pos].ResponseMsec), name, value.Server) + + ch <- prometheus.MustNewConstMetric(vtsUpstreamBackupDesc, + prometheus.CounterValue, float64(zones[pos].Backup), name, value.Server) + + ch <- prometheus.MustNewConstMetric(vtsUpstreamFailTimeoutDesc, + prometheus.CounterValue, float64(zones[pos].FailTimeout), name, value.Server) + + ch <- prometheus.MustNewConstMetric(vtsUpstreamMaxFailsDesc, + prometheus.CounterValue, float64(zones[pos].MaxFails), name, value.Server) + + ch <- prometheus.MustNewConstMetric(vtsUpstreamBytesDesc, + prometheus.CounterValue, float64(zones[pos].InBytes), name, value.Server, "in") + + ch <- prometheus.MustNewConstMetric(vtsUpstreamBytesDesc, + prometheus.CounterValue, float64(zones[pos].OutBytes), name, value.Server, "out") + + } + } + + for name, zone := range nginxMetrics.ServerZones { + + reflectMetrics(&zone.Responses, vtsResponseDesc, ch, name) + reflectMetrics(&zone.Cache, vtsCacheDesc, ch, name) + + ch <- prometheus.MustNewConstMetric(vtsRequestDesc, + prometheus.CounterValue, float64(zone.RequestCounter), name) + + ch <- prometheus.MustNewConstMetric(vtsBytesDesc, + prometheus.CounterValue, float64(zone.InBytes), name, "in") + + ch <- prometheus.MustNewConstMetric(vtsBytesDesc, + prometheus.CounterValue, float64(zone.OutBytes), name, "out") + + } + + for serverZone, countries := range nginxMetrics.FilterZones { + + for country, zone := range countries { + + reflectMetrics(&zone.Responses, vtsFilterZoneResponseDesc, ch, serverZone, country) + reflectMetrics(&zone.Cache, vtsFilterZoneCacheDesc, ch, serverZone, country) + + ch <- prometheus.MustNewConstMetric(vtsFilterZoneBytesDesc, + prometheus.CounterValue, float64(zone.InBytes), serverZone, country, "in") + + ch <- prometheus.MustNewConstMetric(vtsFilterZoneBytesDesc, + prometheus.CounterValue, float64(zone.OutBytes), serverZone, country, "out") + + } + + } + +} + +func (p *namedProcessCollector) scrapeProcs(ch chan<- prometheus.Metric) { + + _, err := p.Update(p.fs.AllProcs()) if err != nil { glog.Warningf("unexpected error obtaining nginx process info: %v", err) return @@ -231,3 +445,18 @@ func (p *namedProcessCollector) scrape(ch chan<- prometheus.Metric) { prometheus.CounterValue, float64(gcounts.WriteBytes)) } } + +func reflectMetrics(value interface{}, desc *prometheus.Desc, ch chan<- prometheus.Metric, labels ...string) { + + val := reflect.ValueOf(value).Elem() + + for i := 0; i < val.NumField(); i++ { + tag := val.Type().Field(i).Tag + + labels := append(labels, tag.Get("json")) + ch <- prometheus.MustNewConstMetric(desc, + prometheus.CounterValue, float64(val.Field(i).Interface().(float64)), + labels...) + } + +} diff --git a/controllers/nginx/pkg/cmd/controller/nginx.go b/controllers/nginx/pkg/cmd/controller/nginx.go index f8e92bac0..d614853eb 100644 --- a/controllers/nginx/pkg/cmd/controller/nginx.go +++ b/controllers/nginx/pkg/cmd/controller/nginx.go @@ -25,7 +25,6 @@ import ( "net/http" "os" "os/exec" - "strings" "syscall" "time" @@ -40,12 +39,14 @@ import ( "k8s.io/ingress/core/pkg/ingress" "k8s.io/ingress/core/pkg/ingress/defaults" "k8s.io/ingress/core/pkg/net/ssl" + "strings" ) const ( ngxHealthPort = 18080 ngxHealthPath = "/healthz" ngxStatusPath = "/internal_nginx_status" + ngxVtsPath = "/nginx_status/format/json" ) var ( @@ -156,8 +157,8 @@ func (n *NGINXController) start(cmd *exec.Cmd, done chan error) { done <- err return } - - n.setupMonitor(cmd.Args) + cfg := ngx_template.ReadConfig(n.configmap.Data) + n.setupMonitor(cmd.Args, cfg.EnableVtsStatus) go func() { done <- cmd.Wait() @@ -177,6 +178,7 @@ func (n NGINXController) Reload(data []byte) ([]byte, bool, error) { } o, e := exec.Command(n.binary, "-s", "reload").CombinedOutput() + return o, true, e } @@ -204,6 +206,7 @@ func (n NGINXController) isReloadRequired(data []byte) bool { } if !bytes.Equal(src, data) { + tmpfile, err := ioutil.TempFile("", "nginx-cfg-diff") if err != nil { glog.Errorf("error creating temporal file: %s", err) @@ -312,6 +315,7 @@ func (n *NGINXController) OnUpdate(ingressCfg ingress.Configuration) ([]byte, er } cfg := ngx_template.ReadConfig(n.configmap.Data) + n.setupMonitor([]string{""}, cfg.EnableVtsStatus) // NGINX cannot resize the has tables used to store server names. // For this reason we check if the defined size defined is correct diff --git a/controllers/nginx/pkg/cmd/controller/status.go b/controllers/nginx/pkg/cmd/controller/status.go index bfa1c383b..b286e3bfb 100644 --- a/controllers/nginx/pkg/cmd/controller/status.go +++ b/controllers/nginx/pkg/cmd/controller/status.go @@ -17,7 +17,9 @@ limitations under the License. package main import ( + "encoding/json" "fmt" + "github.com/golang/glog" "io/ioutil" "net/http" "regexp" @@ -49,22 +51,147 @@ type nginxStatus struct { Waiting int } +// https://github.com/vozlt/nginx-module-vts +type Vts struct { + NginxVersion string `json:"nginxVersion"` + LoadMsec int `json:"loadMsec"` + NowMsec int `json:"nowMsec"` + // Total connections and requests(same as stub_status_module in NGINX) + Connections Connections `json:"connections"` + // Traffic(in/out) and request and response counts and cache hit ratio per each server zone + ServerZones map[string]ServerZone `json:"serverZones"` + // Traffic(in/out) and request and response counts and cache hit ratio per each server zone filtered through + // the vhost_traffic_status_filter_by_set_key directive + FilterZones map[string]map[string]FilterZone `json:"filterZones"` + // Traffic(in/out) and request and response counts per server in each upstream group + UpstreamZones map[string][]UpstreamZone `json:"upstreamZones"` +} + +type ServerZone struct { + RequestCounter float64 `json:"requestCounter"` + InBytes float64 `json:"inBytes"` + OutBytes float64 `json:"outBytes"` + Responses Response `json:"responses"` + Cache Cache `json:"responses"` +} + +type FilterZone struct { + RequestCounter float64 `json:"requestCounter"` + InBytes float64 `json:"inBytes"` + OutBytes float64 `json:"outBytes"` + Cache Cache `json:"responses"` + Responses Response `json:"responses"` +} + +type UpstreamZone struct { + Responses Response `json:"responses"` + Server string `json:"server"` + RequestCounter float64 `json:"requestCounter"` + InBytes float64 `json:"inBytes"` + OutBytes float64 `json:"outBytes"` + ResponseMsec float64 `json:"responseMsec"` + Weight float64 `json:"weight"` + MaxFails float64 `json:"maxFails"` + FailTimeout float64 `json:"failTimeout"` + Backup BoolToFloat64 `json:"backup"` + Down BoolToFloat64 `json:"down"` +} + +type Cache struct { + Miss float64 `json:"miss"` + Bypass float64 `json:"bypass"` + Expired float64 `json:"expired"` + Stale float64 `json:"stale"` + Updating float64 `json:"updating"` + Revalidated float64 `json:"revalidated"` + Hit float64 `json:"hit"` + Scarce float64 `json:"scarce"` +} + +type Response struct { + OneXx float64 `json:"1xx"` + TwoXx float64 `json:"2xx"` + TheeXx float64 `json:"3xx"` + FourXx float64 `json:"4xx"` + FiveXx float64 `json:"5xx"` +} + +type Connections struct { + Active float64 `json:"active"` + Reading float64 `json:"reading"` + Writing float64 `json:"writing"` + Waiting float64 `json:"waiting"` + Accepted float64 `json:"accepted"` + Handled float64 `json:"handled"` + Requests float64 `json:"requests"` +} + +type BoolToFloat64 float64 + +func (bit BoolToFloat64) UnmarshalJSON(data []byte) error { + asString := string(data) + if asString == "1" || asString == "true" { + bit = 1 + } else if asString == "0" || asString == "false" { + bit = 0 + } else { + return fmt.Errorf(fmt.Sprintf("Boolean unmarshal error: invalid input %s", asString)) + } + return nil +} + func getNginxStatus() (*nginxStatus, error) { - resp, err := http.DefaultClient.Get(fmt.Sprintf("http://localhost:%v%v", ngxHealthPort, ngxStatusPath)) + + url := fmt.Sprintf("http://localhost:%v%v", ngxHealthPort, ngxStatusPath) + glog.V(3).Infof("start scrapping url: %v", url) + + data, err := httpBody(url) + if err != nil { return nil, fmt.Errorf("unexpected error scraping nginx status page: %v", err) } + return parse(string(data)), nil +} + +func httpBody(url string) ([]byte, error) { + resp, err := http.DefaultClient.Get(url) + if err != nil { + return nil, fmt.Errorf("unexpected error scraping nginx : %v", err) + } + data, err := ioutil.ReadAll(resp.Body) if err != nil { - return nil, fmt.Errorf("unexpected error scraping nginx status page (%v)", err) + return nil, fmt.Errorf("unexpected error scraping nginx (%v)", err) } defer resp.Body.Close() if resp.StatusCode < 200 || resp.StatusCode >= 400 { - return nil, fmt.Errorf("unexpected error scraping nginx status page (status %v)", resp.StatusCode) + return nil, fmt.Errorf("unexpected error scraping nginx (status %v)", resp.StatusCode) } - return parse(string(data)), nil + return data, nil + +} + +func getNginxVtsMetrics() (*Vts, error) { + url := fmt.Sprintf("http://localhost:%v%v", ngxHealthPort, ngxVtsPath) + glog.V(3).Infof("start scrapping url: %v", url) + + data, err := httpBody(url) + + if err != nil { + return nil, fmt.Errorf("unexpected error scraping nginx vts (%v)", err) + } + + var vts Vts + err = json.Unmarshal(data, &vts) + if err != nil { + return nil, fmt.Errorf("unexpected error json unmarshal (%v)", err) + } + + glog.V(3).Infof("scrap returned : %v", vts) + + return &vts, nil } func parse(data string) *nginxStatus { diff --git a/controllers/nginx/pkg/cmd/controller/status_test.go b/controllers/nginx/pkg/cmd/controller/status_test.go index 1dda3a01e..9d52e0691 100644 --- a/controllers/nginx/pkg/cmd/controller/status_test.go +++ b/controllers/nginx/pkg/cmd/controller/status_test.go @@ -67,4 +67,4 @@ func TestToint(t *testing.T) { t.Fatalf("expected %v but returned %v", test.exp, v) } } -} +} \ No newline at end of file diff --git a/controllers/nginx/pkg/config/config.go b/controllers/nginx/pkg/config/config.go index 0001d9502..2e59a74a9 100644 --- a/controllers/nginx/pkg/config/config.go +++ b/controllers/nginx/pkg/config/config.go @@ -289,6 +289,7 @@ func NewDefault() Configuration { UseProxyProtocol: false, UseGzip: true, WorkerProcesses: runtime.NumCPU(), + EnableVtsStatus: false, VtsStatusZoneSize: "10m", UseHTTP2: true, Backend: defaults.Backend{ diff --git a/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl b/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl index 9813bcd24..c00e53448 100644 --- a/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl +++ b/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl @@ -60,9 +60,6 @@ http { client_header_buffer_size {{ $cfg.ClientHeaderBufferSize }}; large_client_header_buffers {{ $cfg.LargeClientHeaderBuffers }}; - - http2_max_field_size {{ $cfg.HTTP2MaxFieldSize }}; - http2_max_header_size {{ $cfg.HTTP2MaxHeaderSize }}; types_hash_max_size 2048; server_names_hash_max_size {{ $cfg.ServerNameHashMaxSize }}; @@ -82,7 +79,7 @@ http { server_tokens {{ if $cfg.ShowServerTokens }}on{{ else }}off{{ end }}; - log_format upstreaminfo '{{ buildLogFormatUpstream $cfg }}'; + log_format upstreaminfo {{ buildLogFormatUpstream $cfg }}; {{/* map urls that should not appear in access.log */}} {{/* http://nginx.org/en/docs/http/ngx_http_log_module.html#access_log */}} @@ -210,10 +207,10 @@ http { {{ range $index, $server := .Servers }} server { server_name {{ $server.Hostname }}; - listen {{ if not $cfg.DisableIpv6 }}[::]:{{ end }}80{{ if $cfg.UseProxyProtocol }} proxy_protocol{{ end }}{{ if eq $server.Hostname "_"}} default_server {{ if not $cfg.DisableIpv6 }}ipv6only=off{{end}} reuseport backlog={{ $backlogSize }}{{end}}; + listen [::]:80{{ if $cfg.UseProxyProtocol }} proxy_protocol{{ end }}{{ if eq $index 0 }} ipv6only=off{{end}}{{ if eq $server.Hostname "_"}} default_server reuseport backlog={{ $backlogSize }}{{end}}; {{/* Listen on 442 because port 443 is used in the stream section */}} - {{/* This listen on port 442 cannot contains proxy_protocol directive because port 443 is in charge of decoding the protocol */}} - {{ if not (empty $server.SSLCertificate) }}listen {{ if gt (len $passthroughBackends) 0 }}442{{ else }}{{ if not $cfg.DisableIpv6 }}[::]:{{ end }}443 {{ if $cfg.UseProxyProtocol }} proxy_protocol {{ end }}{{ end }} {{ if eq $server.Hostname "_"}} default_server {{ if not $cfg.DisableIpv6 }}ipv6only=off{{end}} reuseport backlog={{ $backlogSize }}{{end}} ssl {{ if $cfg.UseHTTP2 }}http2{{ end }}; + {{/* This listen cannot contains proxy_protocol directive because port 443 is in charge of decoding the protocol */}} + {{ if not (empty $server.SSLCertificate) }}listen {{ if gt (len $passthroughBackends) 0 }}442{{ else }}[::]:443 {{ end }}{{ if eq $server.Hostname "_"}} default_server reuseport backlog={{ $backlogSize }}{{end}} ssl {{ if $cfg.UseHTTP2 }}http2{{ end }}; {{/* comment PEM sha is required to detect changes in the generated configuration and force a reload */}} # PEM sha: {{ $server.SSLPemChecksum }} ssl_certificate {{ $server.SSLCertificate }}; @@ -246,8 +243,6 @@ http { {{ end }} {{ if not (empty $location.ExternalAuth.Method) }} proxy_method {{ $location.ExternalAuth.Method }}; - proxy_set_header X-Original-URI $request_uri; - proxy_set_header X-Scheme $pass_access_scheme; {{ end }} proxy_set_header Host $host; proxy_pass_request_headers on; @@ -273,13 +268,9 @@ http { auth_request {{ $authPath }}; {{ end }} - {{ if not (empty $location.ExternalAuth.SigninURL) }} - error_page 401 = {{ $location.ExternalAuth.SigninURL }}; - {{ end }} - - {{ if (or $location.Redirect.ForceSSLRedirect (and (not (empty $server.SSLCertificate)) $location.Redirect.SSLRedirect)) }} + {{ if (and (not (empty $server.SSLCertificate)) $location.Redirect.SSLRedirect) }} # enforce ssl on server side - if ($pass_access_scheme = http) { + if ($scheme = http) { return 301 https://$host$request_uri; } {{ end }} @@ -323,8 +314,6 @@ http { proxy_set_header X-Forwarded-Host $host; proxy_set_header X-Forwarded-Port $pass_port; proxy_set_header X-Forwarded-Proto $pass_access_scheme; - proxy_set_header X-Original-URI $request_uri; - proxy_set_header X-Scheme $pass_access_scheme; # mitigate HTTPoxy Vulnerability # https://www.nginx.com/blog/mitigating-the-httpoxy-vulnerability-with-nginx/ @@ -342,7 +331,6 @@ http { proxy_redirect off; proxy_buffering off; proxy_buffer_size "{{ $location.Proxy.BufferSize }}"; - proxy_buffers 4 "{{ $location.Proxy.BufferSize }}"; proxy_http_version 1.1; @@ -376,7 +364,7 @@ http { # with an external software (like sysdig) location /nginx_status { allow 127.0.0.1; - {{ if not $cfg.DisableIpv6 }}allow ::1;{{ end }} + allow ::1; deny all; access_log off; @@ -394,7 +382,7 @@ http { # Use the port 18080 (random value just to avoid known ports) as default port for nginx. # Changing this value requires a change in: # https://github.com/kubernetes/contrib/blob/master/ingress/controllers/nginx/nginx/command.go#L104 - listen {{ if not $cfg.DisableIpv6 }}[::]:{{ end }}18080 {{ if not $cfg.DisableIpv6 }}ipv6only=off{{end}} default_server reuseport backlog={{ .BacklogSize }}; + listen [::]:18080 ipv6only=off default_server reuseport backlog={{ .BacklogSize }}; location {{ $healthzURI }} { access_log off; @@ -416,7 +404,7 @@ http { # TODO: enable extraction for vts module. location /internal_nginx_status { allow 127.0.0.1; - {{ if not $cfg.DisableIpv6 }}allow ::1;{{ end }} + allow ::1; deny all; access_log off; @@ -476,7 +464,7 @@ stream { {{ buildSSLPassthroughUpstreams $backends .PassthroughBackends }} server { - listen {{ if not $cfg.DisableIpv6 }}[::]:{{ end }}443 {{ if not $cfg.DisableIpv6 }}ipv6only=off{{ end }}{{ if $cfg.UseProxyProtocol }} proxy_protocol{{ end }}; + listen [::]:443 ipv6only=off{{ if $cfg.UseProxyProtocol }} proxy_protocol{{ end }}; proxy_pass $stream_upstream; ssl_preread on; } diff --git a/images/nginx-slim/build.sh b/images/nginx-slim/build.sh index 2236e1369..5238ccaec 100755 --- a/images/nginx-slim/build.sh +++ b/images/nginx-slim/build.sh @@ -19,7 +19,7 @@ set -e export NGINX_VERSION=1.11.10 export NDK_VERSION=0.3.0 -export VTS_VERSION=0.1.11 +export VTS_VERSION=0.1.12 export SETMISC_VERSION=0.31 export LUA_VERSION=0.10.7 export STICKY_SESSIONS_VERSION=08a395c66e42 From 7ba389c1d0760258d730c2f88f27c27f8ca43f06 Mon Sep 17 00:00:00 2001 From: Manuel de Brito Fontes Date: Fri, 10 Mar 2017 10:01:26 -0300 Subject: [PATCH 2/3] Cleanup collection of prometheus metrics --- .../nginx/pkg/cmd/controller/metrics.go | 449 +----------------- controllers/nginx/pkg/cmd/controller/nginx.go | 27 +- controllers/nginx/pkg/config/config.go | 6 - .../nginx/pkg/metric/collector/nginx.go | 130 +++++ .../nginx/pkg/metric/collector/process.go | 157 ++++++ .../nginx/pkg/metric/collector/scrape.go | 24 + .../controller => metric/collector}/status.go | 55 +-- .../collector}/status_test.go | 4 +- controllers/nginx/pkg/metric/collector/vts.go | 237 +++++++++ .../rootfs/etc/nginx/template/nginx.tmpl | 32 +- images/nginx-slim/build.sh | 2 +- 11 files changed, 643 insertions(+), 480 deletions(-) create mode 100644 controllers/nginx/pkg/metric/collector/nginx.go create mode 100644 controllers/nginx/pkg/metric/collector/process.go create mode 100644 controllers/nginx/pkg/metric/collector/scrape.go rename controllers/nginx/pkg/{cmd/controller => metric/collector}/status.go (87%) rename controllers/nginx/pkg/{cmd/controller => metric/collector}/status_test.go (98%) create mode 100644 controllers/nginx/pkg/metric/collector/vts.go diff --git a/controllers/nginx/pkg/cmd/controller/metrics.go b/controllers/nginx/pkg/cmd/controller/metrics.go index 1b6903018..a42a19230 100644 --- a/controllers/nginx/pkg/cmd/controller/metrics.go +++ b/controllers/nginx/pkg/cmd/controller/metrics.go @@ -17,446 +17,37 @@ limitations under the License. package main import ( - "path/filepath" - "github.com/golang/glog" - - common "github.com/ncabatoff/process-exporter" - "github.com/ncabatoff/process-exporter/proc" "github.com/prometheus/client_golang/prometheus" - "reflect" + + "k8s.io/ingress/controllers/nginx/pkg/metric/collector" ) -// TODO add current namespace -// TODO add ingress class -var ( - // descriptions borrow from https://github.com/vozlt/nginx-module-vts - - cpuSecsDesc = prometheus.NewDesc( - "nginx_cpu_seconds_total", - "Cpu usage in seconds", - nil, nil) - - numprocsDesc = prometheus.NewDesc( - "nginx_num_procs", - "number of processes", - nil, nil) - - memResidentbytesDesc = prometheus.NewDesc( - "nginx_resident_memory_bytes", - "number of bytes of memory in use", - nil, nil) - - memVirtualbytesDesc = prometheus.NewDesc( - "nginx_virtual_memory_bytes", - "number of bytes of memory in use", - nil, nil) - - readBytesDesc = prometheus.NewDesc( - "nginx_read_bytes_total", - "number of bytes read", - nil, nil) - - startTimeDesc = prometheus.NewDesc( - "nginx_oldest_start_time_seconds", - "start time in seconds since 1970/01/01", - nil, nil) - - writeBytesDesc = prometheus.NewDesc( - "nginx_write_bytes_total", - "number of bytes written", - nil, nil) - - //vts metrics - vtsBytesDesc = prometheus.NewDesc( - "nginx_vts_bytes_total", - "Nginx bytes count", - []string{"server_zone", "direction"}, nil) - - vtsCacheDesc = prometheus.NewDesc( - "nginx_vts_cache_total", - "Nginx cache count", - []string{"server_zone", "type"}, nil) - - vtsConnectionsDesc = prometheus.NewDesc( - "nginx_vts_connections_total", - "Nginx connections count", - []string{"type"}, nil) - - vtsResponseDesc = prometheus.NewDesc( - "nginx_vts_responses_total", - "The number of responses with status codes 1xx, 2xx, 3xx, 4xx, and 5xx.", - []string{"server_zone", "status_code"}, nil) - - vtsRequestDesc = prometheus.NewDesc( - "nginx_vts_requests_total", - "The total number of requested client connections.", - []string{"server_zone"}, nil) - - vtsFilterZoneBytesDesc = prometheus.NewDesc( - "nginx_vts_filterzone_bytes_total", - "Nginx bytes count", - []string{"server_zone", "country", "direction"}, nil) - - vtsFilterZoneResponseDesc = prometheus.NewDesc( - "nginx_vts_filterzone_responses_total", - "The number of responses with status codes 1xx, 2xx, 3xx, 4xx, and 5xx.", - []string{"server_zone", "country", "status_code"}, nil) - - vtsFilterZoneCacheDesc = prometheus.NewDesc( - "nginx_vts_filterzone_cache_total", - "Nginx cache count", - []string{"server_zone", "country", "type"}, nil) - - vtsUpstreamBackupDesc = prometheus.NewDesc( - "nginx_vts_upstream_backup", - "Current backup setting of the server.", - []string{"upstream", "server"}, nil) - - vtsUpstreamBytesDesc = prometheus.NewDesc( - "nginx_vts_upstream_bytes_total", - "The total number of bytes sent to this server.", - []string{"upstream", "server", "direction"}, nil) - - vtsUpstreamDownDesc = prometheus.NewDesc( - "nginx_vts_upstream_down_total", - "Current down setting of the server.", - []string{"upstream", "server"}, nil) - - vtsUpstreamFailTimeoutDesc = prometheus.NewDesc( - "nginx_vts_upstream_fail_timeout", - "Current fail_timeout setting of the server.", - []string{"upstream", "server"}, nil) - - vtsUpstreamMaxFailsDesc = prometheus.NewDesc( - "nginx_vts_upstream_maxfails", - "Current max_fails setting of the server.", - []string{"upstream", "server"}, nil) - - vtsUpstreamResponsesDesc = prometheus.NewDesc( - "nginx_vts_upstream_responses_total", - "The number of upstream responses with status codes 1xx, 2xx, 3xx, 4xx, and 5xx.", - []string{"upstream", "server", "status_code"}, nil) - - vtsUpstreamRequestDesc = prometheus.NewDesc( - "nginx_vts_upstream_requests_total", - "The total number of client connections forwarded to this server.", - []string{"upstream", "server"}, nil) - - vtsUpstreamResponseMsecDesc = prometheus.NewDesc( - "nginx_vts_upstream_response_msecs_avg", - "The average of only upstream response processing times in milliseconds.", - []string{"upstream", "server"}, nil) - - vtsUpstreamWeightDesc = prometheus.NewDesc( - "nginx_vts_upstream_weight", - "Current upstream weight setting of the server.", - []string{"upstream", "server"}, nil) - - activeDesc = prometheus.NewDesc( - "nginx_active_connections", - "total number of active connections", - nil, nil) - - acceptedDesc = prometheus.NewDesc( - "nginx_accepted_connections", - "total number of accepted client connections", - nil, nil) - - handledDesc = prometheus.NewDesc( - "nginx_handled_connections", - "total number of handled connections", - nil, nil) - - requestsDesc = prometheus.NewDesc( - "nginx_total_requests", - "total number of client requests", - nil, nil) - - readingDesc = prometheus.NewDesc( - "nginx_current_reading_connections", - "current number of connections where nginx is reading the request header", - nil, nil) - - writingDesc = prometheus.NewDesc( - "nginx_current_writing_connections", - "current number of connections where nginx is writing the response back to the client", - nil, nil) - - waitingDesc = prometheus.NewDesc( - "nginx_current_waiting_connections", - "current number of idle client connections waiting for a request", - nil, nil) -) - -type exeMatcher struct { - name string - args []string -} - -func (em exeMatcher) MatchAndName(nacl common.NameAndCmdline) (bool, string) { - if len(nacl.Cmdline) == 0 { - return false, "" +func (n *NGINXController) setupMonitor(sm statusModule) { + csm := n.statusModule + if csm != sm { + prometheus + n.statusModule = sm } - cmd := filepath.Base(nacl.Cmdline[0]) - return em.name == cmd, "" } -func (n *NGINXController) setupMonitor(args []string, vtsCollector bool) { - - pc, err := newProcessCollector(true, exeMatcher{"nginx", args}, vtsCollector) +type statsCollector struct { + process prometheus.Collector + basic prometheus.Collector + vts prometheus.Collector +} +func newStatsCollector() (*statsCollector, error) { + pc, err := collector.NewNamedProcess(true, collector.BinaryNameMatcher{"nginx", n.cmdArgs}) + if err != nil { + return nil, err + } + err = prometheus.Register(pc) if err != nil { glog.Fatalf("unexpected error registering nginx collector: %v", err) } - err = prometheus.Register(pc) - - if err != nil { - if _, ok := err.(prometheus.AlreadyRegisteredError); !ok { - glog.Warningf("unexpected error registering nginx collector: %v", err) - } - } - -} - -type ( - scrapeRequest struct { - results chan<- prometheus.Metric - done chan struct{} - } - - namedProcessCollector struct { - scrapeChan chan scrapeRequest - *proc.Grouper - fs *proc.FS - enableVtsCollector bool - } -) - -func newProcessCollector( - children bool, - n common.MatchNamer, - enableVtsCollector bool) (*namedProcessCollector, error) { - - fs, err := proc.NewFS("/proc") - if err != nil { - return nil, err - } - p := &namedProcessCollector{ - scrapeChan: make(chan scrapeRequest), - Grouper: proc.NewGrouper(children, n), - fs: fs, - enableVtsCollector: enableVtsCollector, - } - _, err = p.Update(p.fs.AllProcs()) - if err != nil { - return nil, err - } - - go p.start() - - return p, nil -} - -// Describe implements prometheus.Collector. -func (p *namedProcessCollector) Describe(ch chan<- *prometheus.Desc) { - - ch <- cpuSecsDesc - ch <- numprocsDesc - ch <- readBytesDesc - ch <- writeBytesDesc - ch <- memResidentbytesDesc - ch <- memVirtualbytesDesc - ch <- startTimeDesc - - //vts metrics - ch <- vtsBytesDesc - ch <- vtsCacheDesc - ch <- vtsConnectionsDesc - ch <- vtsRequestDesc - ch <- vtsResponseDesc - ch <- vtsUpstreamBackupDesc - ch <- vtsUpstreamBytesDesc - ch <- vtsUpstreamDownDesc - ch <- vtsUpstreamFailTimeoutDesc - ch <- vtsUpstreamMaxFailsDesc - ch <- vtsUpstreamRequestDesc - ch <- vtsUpstreamResponseMsecDesc - ch <- vtsUpstreamResponsesDesc - ch <- vtsUpstreamWeightDesc - ch <- vtsFilterZoneBytesDesc - ch <- vtsFilterZoneCacheDesc - ch <- vtsFilterZoneResponseDesc - -} - -// Collect implements prometheus.Collector. -func (p *namedProcessCollector) Collect(ch chan<- prometheus.Metric) { - req := scrapeRequest{results: ch, done: make(chan struct{})} - p.scrapeChan <- req - <-req.done -} - -func (p *namedProcessCollector) start() { - - for req := range p.scrapeChan { - ch := req.results - p.scrapeNginxStatus(ch) - p.scrapeProcs(ch) - p.scrapeVts(ch) - - req.done <- struct{}{} + return nil, &statsCollector{ + process: pc, } } - -// scrapeNginxStatus scrap the nginx status -func (p *namedProcessCollector) scrapeNginxStatus(ch chan<- prometheus.Metric) { - s, err := getNginxStatus() - - if err != nil { - glog.Warningf("unexpected error obtaining nginx status info: %v", err) - return - } - - ch <- prometheus.MustNewConstMetric(activeDesc, - prometheus.GaugeValue, float64(s.Active)) - ch <- prometheus.MustNewConstMetric(acceptedDesc, - prometheus.GaugeValue, float64(s.Accepted)) - ch <- prometheus.MustNewConstMetric(handledDesc, - prometheus.GaugeValue, float64(s.Handled)) - ch <- prometheus.MustNewConstMetric(requestsDesc, - prometheus.GaugeValue, float64(s.Requests)) - ch <- prometheus.MustNewConstMetric(readingDesc, - prometheus.GaugeValue, float64(s.Reading)) - ch <- prometheus.MustNewConstMetric(writingDesc, - prometheus.GaugeValue, float64(s.Writing)) - ch <- prometheus.MustNewConstMetric(waitingDesc, - prometheus.GaugeValue, float64(s.Waiting)) - -} - -// scrapeVts scrape nginx vts metrics -func (p *namedProcessCollector) scrapeVts(ch chan<- prometheus.Metric) { - - nginxMetrics, err := getNginxVtsMetrics() - if err != nil { - glog.Warningf("unexpected error obtaining nginx status info: %v", err) - return - } - - reflectMetrics(&nginxMetrics.Connections, vtsConnectionsDesc, ch) - - for name, zones := range nginxMetrics.UpstreamZones { - - for pos, value := range zones { - - reflectMetrics(&zones[pos].Responses, vtsUpstreamResponsesDesc, ch, name, value.Server) - - ch <- prometheus.MustNewConstMetric(vtsUpstreamRequestDesc, - prometheus.CounterValue, float64(zones[pos].RequestCounter), name, value.Server) - - ch <- prometheus.MustNewConstMetric(vtsUpstreamDownDesc, - prometheus.CounterValue, float64(zones[pos].Down), name, value.Server) - - ch <- prometheus.MustNewConstMetric(vtsUpstreamWeightDesc, - prometheus.CounterValue, float64(zones[pos].Weight), name, value.Server) - - ch <- prometheus.MustNewConstMetric(vtsUpstreamResponseMsecDesc, - prometheus.CounterValue, float64(zones[pos].ResponseMsec), name, value.Server) - - ch <- prometheus.MustNewConstMetric(vtsUpstreamBackupDesc, - prometheus.CounterValue, float64(zones[pos].Backup), name, value.Server) - - ch <- prometheus.MustNewConstMetric(vtsUpstreamFailTimeoutDesc, - prometheus.CounterValue, float64(zones[pos].FailTimeout), name, value.Server) - - ch <- prometheus.MustNewConstMetric(vtsUpstreamMaxFailsDesc, - prometheus.CounterValue, float64(zones[pos].MaxFails), name, value.Server) - - ch <- prometheus.MustNewConstMetric(vtsUpstreamBytesDesc, - prometheus.CounterValue, float64(zones[pos].InBytes), name, value.Server, "in") - - ch <- prometheus.MustNewConstMetric(vtsUpstreamBytesDesc, - prometheus.CounterValue, float64(zones[pos].OutBytes), name, value.Server, "out") - - } - } - - for name, zone := range nginxMetrics.ServerZones { - - reflectMetrics(&zone.Responses, vtsResponseDesc, ch, name) - reflectMetrics(&zone.Cache, vtsCacheDesc, ch, name) - - ch <- prometheus.MustNewConstMetric(vtsRequestDesc, - prometheus.CounterValue, float64(zone.RequestCounter), name) - - ch <- prometheus.MustNewConstMetric(vtsBytesDesc, - prometheus.CounterValue, float64(zone.InBytes), name, "in") - - ch <- prometheus.MustNewConstMetric(vtsBytesDesc, - prometheus.CounterValue, float64(zone.OutBytes), name, "out") - - } - - for serverZone, countries := range nginxMetrics.FilterZones { - - for country, zone := range countries { - - reflectMetrics(&zone.Responses, vtsFilterZoneResponseDesc, ch, serverZone, country) - reflectMetrics(&zone.Cache, vtsFilterZoneCacheDesc, ch, serverZone, country) - - ch <- prometheus.MustNewConstMetric(vtsFilterZoneBytesDesc, - prometheus.CounterValue, float64(zone.InBytes), serverZone, country, "in") - - ch <- prometheus.MustNewConstMetric(vtsFilterZoneBytesDesc, - prometheus.CounterValue, float64(zone.OutBytes), serverZone, country, "out") - - } - - } - -} - -func (p *namedProcessCollector) scrapeProcs(ch chan<- prometheus.Metric) { - - _, err := p.Update(p.fs.AllProcs()) - if err != nil { - glog.Warningf("unexpected error obtaining nginx process info: %v", err) - return - } - - for gname, gcounts := range p.Groups() { - glog.Infof("%v", gname) - glog.Infof("%v", gcounts) - ch <- prometheus.MustNewConstMetric(numprocsDesc, - prometheus.GaugeValue, float64(gcounts.Procs)) - ch <- prometheus.MustNewConstMetric(memResidentbytesDesc, - prometheus.GaugeValue, float64(gcounts.Memresident)) - ch <- prometheus.MustNewConstMetric(memVirtualbytesDesc, - prometheus.GaugeValue, float64(gcounts.Memvirtual)) - ch <- prometheus.MustNewConstMetric(startTimeDesc, - prometheus.GaugeValue, float64(gcounts.OldestStartTime.Unix())) - ch <- prometheus.MustNewConstMetric(cpuSecsDesc, - prometheus.CounterValue, gcounts.Cpu) - ch <- prometheus.MustNewConstMetric(readBytesDesc, - prometheus.CounterValue, float64(gcounts.ReadBytes)) - ch <- prometheus.MustNewConstMetric(writeBytesDesc, - prometheus.CounterValue, float64(gcounts.WriteBytes)) - } -} - -func reflectMetrics(value interface{}, desc *prometheus.Desc, ch chan<- prometheus.Metric, labels ...string) { - - val := reflect.ValueOf(value).Elem() - - for i := 0; i < val.NumField(); i++ { - tag := val.Type().Field(i).Tag - - labels := append(labels, tag.Get("json")) - ch <- prometheus.MustNewConstMetric(desc, - prometheus.CounterValue, float64(val.Field(i).Interface().(float64)), - labels...) - } - -} diff --git a/controllers/nginx/pkg/cmd/controller/nginx.go b/controllers/nginx/pkg/cmd/controller/nginx.go index d614853eb..7a484e7ef 100644 --- a/controllers/nginx/pkg/cmd/controller/nginx.go +++ b/controllers/nginx/pkg/cmd/controller/nginx.go @@ -33,20 +33,26 @@ import ( "k8s.io/kubernetes/pkg/api" + "strings" + "k8s.io/ingress/controllers/nginx/pkg/config" ngx_template "k8s.io/ingress/controllers/nginx/pkg/template" "k8s.io/ingress/controllers/nginx/pkg/version" "k8s.io/ingress/core/pkg/ingress" "k8s.io/ingress/core/pkg/ingress/defaults" "k8s.io/ingress/core/pkg/net/ssl" - "strings" ) +type statusModule string + const ( ngxHealthPort = 18080 ngxHealthPath = "/healthz" ngxStatusPath = "/internal_nginx_status" ngxVtsPath = "/nginx_status/format/json" + + defaultStatusModule statusModule = "default" + vtsStatusModule statusModule = "vts" ) var ( @@ -108,6 +114,10 @@ type NGINXController struct { storeLister ingress.StoreLister binary string + + cmdArgs []string + + statusModule statusModule } // Start start a new NGINX master process running in foreground. @@ -157,8 +167,17 @@ func (n *NGINXController) start(cmd *exec.Cmd, done chan error) { done <- err return } + + n.cmdArgs = cmd.Args + cfg := ngx_template.ReadConfig(n.configmap.Data) - n.setupMonitor(cmd.Args, cfg.EnableVtsStatus) + n.statusModule = defaultStatusModule + if cfg.EnableVtsStatus { + n.statusModule = vtsStatusModule + n.setupMonitor(vtsStatusModule) + } else { + n.setupMonitor(defaultStatusModule) + } go func() { done <- cmd.Wait() @@ -315,7 +334,9 @@ func (n *NGINXController) OnUpdate(ingressCfg ingress.Configuration) ([]byte, er } cfg := ngx_template.ReadConfig(n.configmap.Data) - n.setupMonitor([]string{""}, cfg.EnableVtsStatus) + + // we need to check if the status module configuration changed + n.setupMonitor() // NGINX cannot resize the has tables used to store server names. // For this reason we check if the defined size defined is correct diff --git a/controllers/nginx/pkg/config/config.go b/controllers/nginx/pkg/config/config.go index 2e59a74a9..ffd54676a 100644 --- a/controllers/nginx/pkg/config/config.go +++ b/controllers/nginx/pkg/config/config.go @@ -259,8 +259,6 @@ type Configuration struct { func NewDefault() Configuration { cfg := Configuration{ ClientHeaderBufferSize: "1k", - DisableAccessLog: false, - DisableIpv6: false, EnableDynamicTLSRecords: true, ErrorLogLevel: errorLevel, HTTP2MaxFieldSize: "4k", @@ -286,10 +284,8 @@ func NewDefault() Configuration { SSLSessionCacheSize: sslSessionCacheSize, SSLSessionTickets: true, SSLSessionTimeout: sslSessionTimeout, - UseProxyProtocol: false, UseGzip: true, WorkerProcesses: runtime.NumCPU(), - EnableVtsStatus: false, VtsStatusZoneSize: "10m", UseHTTP2: true, Backend: defaults.Backend{ @@ -301,11 +297,9 @@ func NewDefault() Configuration { ProxyCookieDomain: "off", ProxyCookiePath: "off", SSLRedirect: true, - ForceSSLRedirect: false, CustomHTTPErrors: []int{}, WhitelistSourceRange: []string{}, SkipAccessLogURLs: []string{}, - UsePortInRedirects: false, }, } diff --git a/controllers/nginx/pkg/metric/collector/nginx.go b/controllers/nginx/pkg/metric/collector/nginx.go new file mode 100644 index 000000000..7f1b869ee --- /dev/null +++ b/controllers/nginx/pkg/metric/collector/nginx.go @@ -0,0 +1,130 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package collector + +import ( + "github.com/golang/glog" + "github.com/prometheus/client_golang/prometheus" +) + +var ( + activeDesc = prometheus.NewDesc( + "nginx_active_connections", + "total number of active connections", + nil, nil) + + acceptedDesc = prometheus.NewDesc( + "nginx_accepted_connections", + "total number of accepted client connections", + nil, nil) + + handledDesc = prometheus.NewDesc( + "nginx_handled_connections", + "total number of handled connections", + nil, nil) + + requestsDesc = prometheus.NewDesc( + "nginx_total_requests", + "total number of client requests", + nil, nil) + + readingDesc = prometheus.NewDesc( + "nginx_current_reading_connections", + "current number of connections where nginx is reading the request header", + nil, nil) + + writingDesc = prometheus.NewDesc( + "nginx_current_writing_connections", + "current number of connections where nginx is writing the response back to the client", + nil, nil) + + waitingDesc = prometheus.NewDesc( + "nginx_current_waiting_connections", + "current number of idle client connections waiting for a request", + nil, nil) +) + +type ( + nginxStatusCollector struct { + scrapeChan chan scrapeRequest + } +) + +func NewNginxStatus() (prometheus.Collector, error) { + p := nginxStatusCollector{ + scrapeChan: make(chan scrapeRequest), + } + + go p.start() + + return p, nil +} + +// Describe implements prometheus.Collector. +func (p nginxStatusCollector) Describe(ch chan<- *prometheus.Desc) { + ch <- activeDesc + ch <- acceptedDesc + ch <- handledDesc + ch <- requestsDesc + ch <- readingDesc + ch <- writingDesc + ch <- waitingDesc +} + +// Collect implements prometheus.Collector. +func (p nginxStatusCollector) Collect(ch chan<- prometheus.Metric) { + req := scrapeRequest{results: ch, done: make(chan struct{})} + p.scrapeChan <- req + <-req.done +} + +func (p nginxStatusCollector) start() { + for req := range p.scrapeChan { + ch := req.results + p.scrape(ch) + req.done <- struct{}{} + } +} + +func (p nginxStatusCollector) Stop() { + close(p.scrapeChan) +} + +// nginxStatusCollector scrap the nginx status +func (p nginxStatusCollector) scrape(ch chan<- prometheus.Metric) { + s, err := getNginxStatus() + if err != nil { + glog.Warningf("unexpected error obtaining nginx status info: %v", err) + return + } + + ch <- prometheus.MustNewConstMetric(activeDesc, + prometheus.GaugeValue, float64(s.Active)) + ch <- prometheus.MustNewConstMetric(acceptedDesc, + prometheus.GaugeValue, float64(s.Accepted)) + ch <- prometheus.MustNewConstMetric(handledDesc, + prometheus.GaugeValue, float64(s.Handled)) + ch <- prometheus.MustNewConstMetric(requestsDesc, + prometheus.GaugeValue, float64(s.Requests)) + ch <- prometheus.MustNewConstMetric(readingDesc, + prometheus.GaugeValue, float64(s.Reading)) + ch <- prometheus.MustNewConstMetric(writingDesc, + prometheus.GaugeValue, float64(s.Writing)) + ch <- prometheus.MustNewConstMetric(waitingDesc, + prometheus.GaugeValue, float64(s.Waiting)) + +} diff --git a/controllers/nginx/pkg/metric/collector/process.go b/controllers/nginx/pkg/metric/collector/process.go new file mode 100644 index 000000000..3154863b3 --- /dev/null +++ b/controllers/nginx/pkg/metric/collector/process.go @@ -0,0 +1,157 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package collector + +import ( + "path/filepath" + + "github.com/golang/glog" + common "github.com/ncabatoff/process-exporter" + "github.com/ncabatoff/process-exporter/proc" + "github.com/prometheus/client_golang/prometheus" +) + +type BinaryNameMatcher struct { + name string + args []string +} + +func (em BinaryNameMatcher) MatchAndName(nacl common.NameAndCmdline) (bool, string) { + if len(nacl.Cmdline) == 0 { + return false, "" + } + cmd := filepath.Base(nacl.Cmdline[0]) + return em.name == cmd, "" +} + +var ( + numprocsDesc = prometheus.NewDesc( + "nginx_num_procs", + "number of processes", + nil, nil) + + cpuSecsDesc = prometheus.NewDesc( + "nginx_cpu_seconds_total", + "Cpu usage in seconds", + nil, nil) + + readBytesDesc = prometheus.NewDesc( + "nginx_read_bytes_total", + "number of bytes read", + nil, nil) + + writeBytesDesc = prometheus.NewDesc( + "nginx_write_bytes_total", + "number of bytes written", + nil, nil) + + memResidentbytesDesc = prometheus.NewDesc( + "nginx_resident_memory_bytes", + "number of bytes of memory in use", + nil, nil) + + memVirtualbytesDesc = prometheus.NewDesc( + "nginx_virtual_memory_bytes", + "number of bytes of memory in use", + nil, nil) + + startTimeDesc = prometheus.NewDesc( + "nginx_oldest_start_time_seconds", + "start time in seconds since 1970/01/01", + nil, nil) +) + +type namedProcess struct { + scrapeChan chan scrapeRequest + *proc.Grouper + fs *proc.FS +} + +func NewNamedProcessCollector(children bool, mn common.MatchNamer) (prometheus.Collector, error) { + fs, err := proc.NewFS("/proc") + if err != nil { + return nil, err + } + p := namedProcess{ + scrapeChan: make(chan scrapeRequest), + Grouper: proc.NewGrouper(children, mn), + fs: fs, + } + _, err = p.Update(p.fs.AllProcs()) + if err != nil { + return nil, err + } + + go p.start() + + return p, nil +} + +// Describe implements prometheus.Collector. +func (p namedProcess) Describe(ch chan<- *prometheus.Desc) { + ch <- cpuSecsDesc + ch <- numprocsDesc + ch <- readBytesDesc + ch <- writeBytesDesc + ch <- memResidentbytesDesc + ch <- memVirtualbytesDesc + ch <- startTimeDesc +} + +// Collect implements prometheus.Collector. +func (p namedProcess) Collect(ch chan<- prometheus.Metric) { + req := scrapeRequest{results: ch, done: make(chan struct{})} + p.scrapeChan <- req + <-req.done +} + +func (p namedProcess) start() { + for req := range p.scrapeChan { + ch := req.results + p.scrape(ch) + req.done <- struct{}{} + } +} + +func (p namedProcess) Stop() { + close(p.scrapeChan) +} + +func (p namedProcess) scrape(ch chan<- prometheus.Metric) { + _, err := p.Update(p.fs.AllProcs()) + if err != nil { + glog.Warningf("unexpected error obtaining nginx process info: %v", err) + return + } + + for gname, gcounts := range p.Groups() { + ch <- prometheus.MustNewConstMetric(numprocsDesc, + prometheus.GaugeValue, float64(gcounts.Procs)) + ch <- prometheus.MustNewConstMetric(memResidentbytesDesc, + prometheus.GaugeValue, float64(gcounts.Memresident)) + ch <- prometheus.MustNewConstMetric(memVirtualbytesDesc, + prometheus.GaugeValue, float64(gcounts.Memvirtual)) + ch <- prometheus.MustNewConstMetric(startTimeDesc, + prometheus.GaugeValue, float64(gcounts.OldestStartTime.Unix())) + ch <- prometheus.MustNewConstMetric(cpuSecsDesc, + prometheus.CounterValue, gcounts.Cpu) + ch <- prometheus.MustNewConstMetric(readBytesDesc, + prometheus.CounterValue, float64(gcounts.ReadBytes)) + ch <- prometheus.MustNewConstMetric(writeBytesDesc, + prometheus.CounterValue, float64(gcounts.WriteBytes)) + } +} diff --git a/controllers/nginx/pkg/metric/collector/scrape.go b/controllers/nginx/pkg/metric/collector/scrape.go new file mode 100644 index 000000000..b40ff14ad --- /dev/null +++ b/controllers/nginx/pkg/metric/collector/scrape.go @@ -0,0 +1,24 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package collector + +import "github.com/prometheus/client_golang/prometheus" + +type scrapeRequest struct { + results chan<- prometheus.Metric + done chan struct{} +} diff --git a/controllers/nginx/pkg/cmd/controller/status.go b/controllers/nginx/pkg/metric/collector/status.go similarity index 87% rename from controllers/nginx/pkg/cmd/controller/status.go rename to controllers/nginx/pkg/metric/collector/status.go index b286e3bfb..7bd0ecf52 100644 --- a/controllers/nginx/pkg/cmd/controller/status.go +++ b/controllers/nginx/pkg/metric/collector/status.go @@ -14,16 +14,17 @@ See the License for the specific language governing permissions and limitations under the License. */ -package main +package collector import ( "encoding/json" "fmt" - "github.com/golang/glog" "io/ioutil" "net/http" "regexp" "strconv" + + "github.com/golang/glog" ) var ( @@ -34,7 +35,7 @@ var ( waiting = regexp.MustCompile(`Waiting: (\d+)`) ) -type nginxStatus struct { +type basicStatus struct { // Active total number of active connections Active int // Accepted total number of accepted client connections @@ -52,39 +53,39 @@ type nginxStatus struct { } // https://github.com/vozlt/nginx-module-vts -type Vts struct { +type vts struct { NginxVersion string `json:"nginxVersion"` LoadMsec int `json:"loadMsec"` NowMsec int `json:"nowMsec"` // Total connections and requests(same as stub_status_module in NGINX) - Connections Connections `json:"connections"` + Connections connections `json:"connections"` // Traffic(in/out) and request and response counts and cache hit ratio per each server zone - ServerZones map[string]ServerZone `json:"serverZones"` + ServerZones map[string]serverZone `json:"serverZones"` // Traffic(in/out) and request and response counts and cache hit ratio per each server zone filtered through // the vhost_traffic_status_filter_by_set_key directive - FilterZones map[string]map[string]FilterZone `json:"filterZones"` + FilterZones map[string]map[string]filterZone `json:"filterZones"` // Traffic(in/out) and request and response counts per server in each upstream group - UpstreamZones map[string][]UpstreamZone `json:"upstreamZones"` + UpstreamZones map[string][]upstreamZone `json:"upstreamZones"` } -type ServerZone struct { +type serverZone struct { RequestCounter float64 `json:"requestCounter"` InBytes float64 `json:"inBytes"` OutBytes float64 `json:"outBytes"` - Responses Response `json:"responses"` - Cache Cache `json:"responses"` + Responses response `json:"responses"` + Cache cache `json:"responses"` } -type FilterZone struct { +type filterZone struct { RequestCounter float64 `json:"requestCounter"` InBytes float64 `json:"inBytes"` OutBytes float64 `json:"outBytes"` - Cache Cache `json:"responses"` - Responses Response `json:"responses"` + Cache cache `json:"responses"` + Responses response `json:"responses"` } -type UpstreamZone struct { - Responses Response `json:"responses"` +type upstreamZone struct { + Responses response `json:"responses"` Server string `json:"server"` RequestCounter float64 `json:"requestCounter"` InBytes float64 `json:"inBytes"` @@ -97,7 +98,7 @@ type UpstreamZone struct { Down BoolToFloat64 `json:"down"` } -type Cache struct { +type cache struct { Miss float64 `json:"miss"` Bypass float64 `json:"bypass"` Expired float64 `json:"expired"` @@ -108,7 +109,7 @@ type Cache struct { Scarce float64 `json:"scarce"` } -type Response struct { +type response struct { OneXx float64 `json:"1xx"` TwoXx float64 `json:"2xx"` TheeXx float64 `json:"3xx"` @@ -116,7 +117,7 @@ type Response struct { FiveXx float64 `json:"5xx"` } -type Connections struct { +type connections struct { Active float64 `json:"active"` Reading float64 `json:"reading"` Writing float64 `json:"writing"` @@ -140,8 +141,7 @@ func (bit BoolToFloat64) UnmarshalJSON(data []byte) error { return nil } -func getNginxStatus() (*nginxStatus, error) { - +func getNginxStatus() (*basicStatus, error) { url := fmt.Sprintf("http://localhost:%v%v", ngxHealthPort, ngxStatusPath) glog.V(3).Infof("start scrapping url: %v", url) @@ -170,10 +170,9 @@ func httpBody(url string) ([]byte, error) { } return data, nil - } -func getNginxVtsMetrics() (*Vts, error) { +func getNginxVtsMetrics() (*vts, error) { url := fmt.Sprintf("http://localhost:%v%v", ngxHealthPort, ngxVtsPath) glog.V(3).Infof("start scrapping url: %v", url) @@ -183,25 +182,23 @@ func getNginxVtsMetrics() (*Vts, error) { return nil, fmt.Errorf("unexpected error scraping nginx vts (%v)", err) } - var vts Vts + var vts *vts err = json.Unmarshal(data, &vts) if err != nil { return nil, fmt.Errorf("unexpected error json unmarshal (%v)", err) } - glog.V(3).Infof("scrap returned : %v", vts) - - return &vts, nil + return vts, nil } -func parse(data string) *nginxStatus { +func parse(data string) *basicStatus { acr := ac.FindStringSubmatch(data) sahrr := sahr.FindStringSubmatch(data) readingr := reading.FindStringSubmatch(data) writingr := writing.FindStringSubmatch(data) waitingr := waiting.FindStringSubmatch(data) - return &nginxStatus{ + return &basicStatus{ toInt(acr, 1), toInt(sahrr, 1), toInt(sahrr, 2), diff --git a/controllers/nginx/pkg/cmd/controller/status_test.go b/controllers/nginx/pkg/metric/collector/status_test.go similarity index 98% rename from controllers/nginx/pkg/cmd/controller/status_test.go rename to controllers/nginx/pkg/metric/collector/status_test.go index 9d52e0691..b6693d9c9 100644 --- a/controllers/nginx/pkg/cmd/controller/status_test.go +++ b/controllers/nginx/pkg/metric/collector/status_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package main +package collector import ( "reflect" @@ -67,4 +67,4 @@ func TestToint(t *testing.T) { t.Fatalf("expected %v but returned %v", test.exp, v) } } -} \ No newline at end of file +} diff --git a/controllers/nginx/pkg/metric/collector/vts.go b/controllers/nginx/pkg/metric/collector/vts.go new file mode 100644 index 000000000..a47531f65 --- /dev/null +++ b/controllers/nginx/pkg/metric/collector/vts.go @@ -0,0 +1,237 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package collector + +import ( + "reflect" + + "github.com/golang/glog" + "github.com/prometheus/client_golang/prometheus" +) + +var ( + vtsBytesDesc = prometheus.NewDesc( + "nginx_vts_bytes_total", + "Nginx bytes count", + []string{"server_zone", "direction"}, nil) + + vtsCacheDesc = prometheus.NewDesc( + "nginx_vts_cache_total", + "Nginx cache count", + []string{"server_zone", "type"}, nil) + + vtsConnectionsDesc = prometheus.NewDesc( + "nginx_vts_connections_total", + "Nginx connections count", + []string{"type"}, nil) + + vtsResponseDesc = prometheus.NewDesc( + "nginx_vts_responses_total", + "The number of responses with status codes 1xx, 2xx, 3xx, 4xx, and 5xx.", + []string{"server_zone", "status_code"}, nil) + + vtsRequestDesc = prometheus.NewDesc( + "nginx_vts_requests_total", + "The total number of requested client connections.", + []string{"server_zone"}, nil) + + vtsFilterZoneBytesDesc = prometheus.NewDesc( + "nginx_vts_filterzone_bytes_total", + "Nginx bytes count", + []string{"server_zone", "country", "direction"}, nil) + + vtsFilterZoneResponseDesc = prometheus.NewDesc( + "nginx_vts_filterzone_responses_total", + "The number of responses with status codes 1xx, 2xx, 3xx, 4xx, and 5xx.", + []string{"server_zone", "country", "status_code"}, nil) + + vtsFilterZoneCacheDesc = prometheus.NewDesc( + "nginx_vts_filterzone_cache_total", + "Nginx cache count", + []string{"server_zone", "country", "type"}, nil) + + vtsUpstreamBackupDesc = prometheus.NewDesc( + "nginx_vts_upstream_backup", + "Current backup setting of the server.", + []string{"upstream", "server"}, nil) + + vtsUpstreamBytesDesc = prometheus.NewDesc( + "nginx_vts_upstream_bytes_total", + "The total number of bytes sent to this server.", + []string{"upstream", "server", "direction"}, nil) + + vtsUpstreamDownDesc = prometheus.NewDesc( + "nginx_vts_upstream_down_total", + "Current down setting of the server.", + []string{"upstream", "server"}, nil) + + vtsUpstreamFailTimeoutDesc = prometheus.NewDesc( + "nginx_vts_upstream_fail_timeout", + "Current fail_timeout setting of the server.", + []string{"upstream", "server"}, nil) + + vtsUpstreamMaxFailsDesc = prometheus.NewDesc( + "nginx_vts_upstream_maxfails", + "Current max_fails setting of the server.", + []string{"upstream", "server"}, nil) + + vtsUpstreamResponsesDesc = prometheus.NewDesc( + "nginx_vts_upstream_responses_total", + "The number of upstream responses with status codes 1xx, 2xx, 3xx, 4xx, and 5xx.", + []string{"upstream", "server", "status_code"}, nil) + + vtsUpstreamRequestDesc = prometheus.NewDesc( + "nginx_vts_upstream_requests_total", + "The total number of client connections forwarded to this server.", + []string{"upstream", "server"}, nil) + + vtsUpstreamResponseMsecDesc = prometheus.NewDesc( + "nginx_vts_upstream_response_msecs_avg", + "The average of only upstream response processing times in milliseconds.", + []string{"upstream", "server"}, nil) + + vtsUpstreamWeightDesc = prometheus.NewDesc( + "nginx_vts_upstream_weight", + "Current upstream weight setting of the server.", + []string{"upstream", "server"}, nil) +) + +type vtsCollector struct { + scrapeChan chan scrapeRequest + } + +func NewNGINXVTSCollector() (prometheus.Collector, error) { + p := vtsCollector{ + scrapeChan: make(chan scrapeRequest), + } + + go p.start() + + return p, nil +} + +// Describe implements prometheus.Collector. +func (p *vtsCollector) Describe(ch chan<- *prometheus.Desc) { + ch <- vtsBytesDesc + ch <- vtsCacheDesc + ch <- vtsConnectionsDesc + ch <- vtsRequestDesc + ch <- vtsResponseDesc + ch <- vtsUpstreamBackupDesc + ch <- vtsUpstreamBytesDesc + ch <- vtsUpstreamDownDesc + ch <- vtsUpstreamFailTimeoutDesc + ch <- vtsUpstreamMaxFailsDesc + ch <- vtsUpstreamRequestDesc + ch <- vtsUpstreamResponseMsecDesc + ch <- vtsUpstreamResponsesDesc + ch <- vtsUpstreamWeightDesc + ch <- vtsFilterZoneBytesDesc + ch <- vtsFilterZoneCacheDesc + ch <- vtsFilterZoneResponseDesc +} + +// Collect implements prometheus.Collector. +func (p *vtsCollector) Collect(ch chan<- prometheus.Metric) { + req := scrapeRequest{results: ch, done: make(chan struct{})} + p.scrapeChan <- req + <-req.done +} + +func (p *vtsCollector) start() { + for req := range p.scrapeChan { + ch := req.results + p.scrapeVts(ch) + req.done <- struct{}{} + } +} + +func (p *vtsCollector) Stop() { + close(p.scrapeChan) +} + +// scrapeVts scrape nginx vts metrics +func (p *vtsCollector) scrapeVts(ch chan<- prometheus.Metric) { + nginxMetrics, err := getNginxVtsMetrics() + if err != nil { + glog.Warningf("unexpected error obtaining nginx status info: %v", err) + return + } + + reflectMetrics(&nginxMetrics.Connections, vtsConnectionsDesc, ch) + + for name, zones := range nginxMetrics.UpstreamZones { + for pos, value := range zones { + reflectMetrics(&zones[pos].Responses, vtsUpstreamResponsesDesc, ch, name, value.Server) + + ch <- prometheus.MustNewConstMetric(vtsUpstreamRequestDesc, + prometheus.CounterValue, float64(zones[pos].RequestCounter), name, value.Server) + ch <- prometheus.MustNewConstMetric(vtsUpstreamDownDesc, + prometheus.CounterValue, float64(zones[pos].Down), name, value.Server) + ch <- prometheus.MustNewConstMetric(vtsUpstreamWeightDesc, + prometheus.CounterValue, float64(zones[pos].Weight), name, value.Server) + ch <- prometheus.MustNewConstMetric(vtsUpstreamResponseMsecDesc, + prometheus.CounterValue, float64(zones[pos].ResponseMsec), name, value.Server) + ch <- prometheus.MustNewConstMetric(vtsUpstreamBackupDesc, + prometheus.CounterValue, float64(zones[pos].Backup), name, value.Server) + ch <- prometheus.MustNewConstMetric(vtsUpstreamFailTimeoutDesc, + prometheus.CounterValue, float64(zones[pos].FailTimeout), name, value.Server) + ch <- prometheus.MustNewConstMetric(vtsUpstreamMaxFailsDesc, + prometheus.CounterValue, float64(zones[pos].MaxFails), name, value.Server) + ch <- prometheus.MustNewConstMetric(vtsUpstreamBytesDesc, + prometheus.CounterValue, float64(zones[pos].InBytes), name, value.Server, "in") + ch <- prometheus.MustNewConstMetric(vtsUpstreamBytesDesc, + prometheus.CounterValue, float64(zones[pos].OutBytes), name, value.Server, "out") + } + } + + for name, zone := range nginxMetrics.ServerZones { + reflectMetrics(&zone.Responses, vtsResponseDesc, ch, name) + reflectMetrics(&zone.Cache, vtsCacheDesc, ch, name) + + ch <- prometheus.MustNewConstMetric(vtsRequestDesc, + prometheus.CounterValue, float64(zone.RequestCounter), name) + ch <- prometheus.MustNewConstMetric(vtsBytesDesc, + prometheus.CounterValue, float64(zone.InBytes), name, "in") + ch <- prometheus.MustNewConstMetric(vtsBytesDesc, + prometheus.CounterValue, float64(zone.OutBytes), name, "out") + } + + for serverZone, countries := range nginxMetrics.FilterZones { + for country, zone := range countries { + reflectMetrics(&zone.Responses, vtsFilterZoneResponseDesc, ch, serverZone, country) + reflectMetrics(&zone.Cache, vtsFilterZoneCacheDesc, ch, serverZone, country) + + ch <- prometheus.MustNewConstMetric(vtsFilterZoneBytesDesc, + prometheus.CounterValue, float64(zone.InBytes), serverZone, country, "in") + ch <- prometheus.MustNewConstMetric(vtsFilterZoneBytesDesc, + prometheus.CounterValue, float64(zone.OutBytes), serverZone, country, "out") + } + } +} + +func reflectMetrics(value interface{}, desc *prometheus.Desc, ch chan<- prometheus.Metric, labels ...string) { + val := reflect.ValueOf(value).Elem() + + for i := 0; i < val.NumField(); i++ { + tag := val.Type().Field(i).Tag + labels := append(labels, tag.Get("json")) + ch <- prometheus.MustNewConstMetric(desc, + prometheus.CounterValue, float64(val.Field(i).Interface().(float64)), + labels...) + } +} diff --git a/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl b/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl index c00e53448..9813bcd24 100644 --- a/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl +++ b/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl @@ -60,6 +60,9 @@ http { client_header_buffer_size {{ $cfg.ClientHeaderBufferSize }}; large_client_header_buffers {{ $cfg.LargeClientHeaderBuffers }}; + + http2_max_field_size {{ $cfg.HTTP2MaxFieldSize }}; + http2_max_header_size {{ $cfg.HTTP2MaxHeaderSize }}; types_hash_max_size 2048; server_names_hash_max_size {{ $cfg.ServerNameHashMaxSize }}; @@ -79,7 +82,7 @@ http { server_tokens {{ if $cfg.ShowServerTokens }}on{{ else }}off{{ end }}; - log_format upstreaminfo {{ buildLogFormatUpstream $cfg }}; + log_format upstreaminfo '{{ buildLogFormatUpstream $cfg }}'; {{/* map urls that should not appear in access.log */}} {{/* http://nginx.org/en/docs/http/ngx_http_log_module.html#access_log */}} @@ -207,10 +210,10 @@ http { {{ range $index, $server := .Servers }} server { server_name {{ $server.Hostname }}; - listen [::]:80{{ if $cfg.UseProxyProtocol }} proxy_protocol{{ end }}{{ if eq $index 0 }} ipv6only=off{{end}}{{ if eq $server.Hostname "_"}} default_server reuseport backlog={{ $backlogSize }}{{end}}; + listen {{ if not $cfg.DisableIpv6 }}[::]:{{ end }}80{{ if $cfg.UseProxyProtocol }} proxy_protocol{{ end }}{{ if eq $server.Hostname "_"}} default_server {{ if not $cfg.DisableIpv6 }}ipv6only=off{{end}} reuseport backlog={{ $backlogSize }}{{end}}; {{/* Listen on 442 because port 443 is used in the stream section */}} - {{/* This listen cannot contains proxy_protocol directive because port 443 is in charge of decoding the protocol */}} - {{ if not (empty $server.SSLCertificate) }}listen {{ if gt (len $passthroughBackends) 0 }}442{{ else }}[::]:443 {{ end }}{{ if eq $server.Hostname "_"}} default_server reuseport backlog={{ $backlogSize }}{{end}} ssl {{ if $cfg.UseHTTP2 }}http2{{ end }}; + {{/* This listen on port 442 cannot contains proxy_protocol directive because port 443 is in charge of decoding the protocol */}} + {{ if not (empty $server.SSLCertificate) }}listen {{ if gt (len $passthroughBackends) 0 }}442{{ else }}{{ if not $cfg.DisableIpv6 }}[::]:{{ end }}443 {{ if $cfg.UseProxyProtocol }} proxy_protocol {{ end }}{{ end }} {{ if eq $server.Hostname "_"}} default_server {{ if not $cfg.DisableIpv6 }}ipv6only=off{{end}} reuseport backlog={{ $backlogSize }}{{end}} ssl {{ if $cfg.UseHTTP2 }}http2{{ end }}; {{/* comment PEM sha is required to detect changes in the generated configuration and force a reload */}} # PEM sha: {{ $server.SSLPemChecksum }} ssl_certificate {{ $server.SSLCertificate }}; @@ -243,6 +246,8 @@ http { {{ end }} {{ if not (empty $location.ExternalAuth.Method) }} proxy_method {{ $location.ExternalAuth.Method }}; + proxy_set_header X-Original-URI $request_uri; + proxy_set_header X-Scheme $pass_access_scheme; {{ end }} proxy_set_header Host $host; proxy_pass_request_headers on; @@ -268,9 +273,13 @@ http { auth_request {{ $authPath }}; {{ end }} - {{ if (and (not (empty $server.SSLCertificate)) $location.Redirect.SSLRedirect) }} + {{ if not (empty $location.ExternalAuth.SigninURL) }} + error_page 401 = {{ $location.ExternalAuth.SigninURL }}; + {{ end }} + + {{ if (or $location.Redirect.ForceSSLRedirect (and (not (empty $server.SSLCertificate)) $location.Redirect.SSLRedirect)) }} # enforce ssl on server side - if ($scheme = http) { + if ($pass_access_scheme = http) { return 301 https://$host$request_uri; } {{ end }} @@ -314,6 +323,8 @@ http { proxy_set_header X-Forwarded-Host $host; proxy_set_header X-Forwarded-Port $pass_port; proxy_set_header X-Forwarded-Proto $pass_access_scheme; + proxy_set_header X-Original-URI $request_uri; + proxy_set_header X-Scheme $pass_access_scheme; # mitigate HTTPoxy Vulnerability # https://www.nginx.com/blog/mitigating-the-httpoxy-vulnerability-with-nginx/ @@ -331,6 +342,7 @@ http { proxy_redirect off; proxy_buffering off; proxy_buffer_size "{{ $location.Proxy.BufferSize }}"; + proxy_buffers 4 "{{ $location.Proxy.BufferSize }}"; proxy_http_version 1.1; @@ -364,7 +376,7 @@ http { # with an external software (like sysdig) location /nginx_status { allow 127.0.0.1; - allow ::1; + {{ if not $cfg.DisableIpv6 }}allow ::1;{{ end }} deny all; access_log off; @@ -382,7 +394,7 @@ http { # Use the port 18080 (random value just to avoid known ports) as default port for nginx. # Changing this value requires a change in: # https://github.com/kubernetes/contrib/blob/master/ingress/controllers/nginx/nginx/command.go#L104 - listen [::]:18080 ipv6only=off default_server reuseport backlog={{ .BacklogSize }}; + listen {{ if not $cfg.DisableIpv6 }}[::]:{{ end }}18080 {{ if not $cfg.DisableIpv6 }}ipv6only=off{{end}} default_server reuseport backlog={{ .BacklogSize }}; location {{ $healthzURI }} { access_log off; @@ -404,7 +416,7 @@ http { # TODO: enable extraction for vts module. location /internal_nginx_status { allow 127.0.0.1; - allow ::1; + {{ if not $cfg.DisableIpv6 }}allow ::1;{{ end }} deny all; access_log off; @@ -464,7 +476,7 @@ stream { {{ buildSSLPassthroughUpstreams $backends .PassthroughBackends }} server { - listen [::]:443 ipv6only=off{{ if $cfg.UseProxyProtocol }} proxy_protocol{{ end }}; + listen {{ if not $cfg.DisableIpv6 }}[::]:{{ end }}443 {{ if not $cfg.DisableIpv6 }}ipv6only=off{{ end }}{{ if $cfg.UseProxyProtocol }} proxy_protocol{{ end }}; proxy_pass $stream_upstream; ssl_preread on; } diff --git a/images/nginx-slim/build.sh b/images/nginx-slim/build.sh index 5238ccaec..2236e1369 100755 --- a/images/nginx-slim/build.sh +++ b/images/nginx-slim/build.sh @@ -19,7 +19,7 @@ set -e export NGINX_VERSION=1.11.10 export NDK_VERSION=0.3.0 -export VTS_VERSION=0.1.12 +export VTS_VERSION=0.1.11 export SETMISC_VERSION=0.31 export LUA_VERSION=0.10.7 export STICKY_SESSIONS_VERSION=08a395c66e42 From e702c558200ab7877a4b73b8b9183eed11abb37a Mon Sep 17 00:00:00 2001 From: Manuel de Brito Fontes Date: Sun, 12 Mar 2017 12:27:05 -0300 Subject: [PATCH 3/3] Fix build --- controllers/nginx/Makefile | 2 +- .../nginx/pkg/cmd/controller/metrics.go | 58 +++- controllers/nginx/pkg/cmd/controller/nginx.go | 45 +-- .../nginx/pkg/metric/collector/nginx.go | 144 +++++--- .../nginx/pkg/metric/collector/process.go | 132 +++---- .../nginx/pkg/metric/collector/scrape.go | 6 + .../nginx/pkg/metric/collector/status.go | 10 +- .../nginx/pkg/metric/collector/status_test.go | 12 +- controllers/nginx/pkg/metric/collector/vts.go | 324 ++++++++++-------- .../rootfs/etc/nginx/template/nginx.tmpl | 6 + 10 files changed, 440 insertions(+), 299 deletions(-) diff --git a/controllers/nginx/Makefile b/controllers/nginx/Makefile index 8805e2d2a..bf22ee4ab 100644 --- a/controllers/nginx/Makefile +++ b/controllers/nginx/Makefile @@ -21,7 +21,7 @@ build: clean -ldflags "-s -w -X ${PKG}/pkg/version.RELEASE=${RELEASE} -X ${PKG}/pkg/version.COMMIT=${COMMIT} -X ${PKG}/pkg/version.REPO=${REPO_INFO}" \ -o rootfs/nginx-ingress-controller ${PKG}/pkg/cmd/controller -container: build +container: $(DOCKER) build --pull -t $(PREFIX):$(RELEASE) rootfs push: container diff --git a/controllers/nginx/pkg/cmd/controller/metrics.go b/controllers/nginx/pkg/cmd/controller/metrics.go index a42a19230..b803e6756 100644 --- a/controllers/nginx/pkg/cmd/controller/metrics.go +++ b/controllers/nginx/pkg/cmd/controller/metrics.go @@ -23,31 +23,73 @@ import ( "k8s.io/ingress/controllers/nginx/pkg/metric/collector" ) +const ( + ngxStatusPath = "/internal_nginx_status" + ngxVtsPath = "/nginx_status/format/json" +) + func (n *NGINXController) setupMonitor(sm statusModule) { csm := n.statusModule if csm != sm { - prometheus + glog.Infof("changing prometheus collector from %v to %v", csm, sm) + n.stats.stop(csm) + n.stats.start(sm) n.statusModule = sm } } type statsCollector struct { process prometheus.Collector - basic prometheus.Collector - vts prometheus.Collector + basic collector.Stopable + vts collector.Stopable + + namespace string + watchClass string } -func newStatsCollector() (*statsCollector, error) { - pc, err := collector.NewNamedProcess(true, collector.BinaryNameMatcher{"nginx", n.cmdArgs}) +func (s *statsCollector) stop(sm statusModule) { + switch sm { + case defaultStatusModule: + s.basic.Stop() + prometheus.Unregister(s.basic) + break + case vtsStatusModule: + s.vts.Stop() + prometheus.Unregister(s.vts) + break + } +} + +func (s *statsCollector) start(sm statusModule) { + switch sm { + case defaultStatusModule: + s.basic = collector.NewNginxStatus(s.namespace, s.watchClass, ngxHealthPort, ngxStatusPath) + prometheus.Register(s.basic) + break + case vtsStatusModule: + s.vts = collector.NewNGINXVTSCollector(s.namespace, s.watchClass, ngxHealthPort, ngxVtsPath) + prometheus.Register(s.vts) + break + } +} + +func newStatsCollector(ns, class, binary string) *statsCollector { + glog.Infof("starting new nginx stats collector for Ingress controller running in namespace %v (class %v)", ns, class) + pc, err := collector.NewNamedProcess(true, collector.BinaryNameMatcher{ + Name: "nginx", + Binary: binary, + }) if err != nil { - return nil, err + glog.Fatalf("unexpected error registering nginx collector: %v", err) } err = prometheus.Register(pc) if err != nil { glog.Fatalf("unexpected error registering nginx collector: %v", err) } - return nil, &statsCollector{ - process: pc, + return &statsCollector{ + namespace: ns, + watchClass: class, + process: pc, } } diff --git a/controllers/nginx/pkg/cmd/controller/nginx.go b/controllers/nginx/pkg/cmd/controller/nginx.go index 7a484e7ef..1bd7ede60 100644 --- a/controllers/nginx/pkg/cmd/controller/nginx.go +++ b/controllers/nginx/pkg/cmd/controller/nginx.go @@ -48,8 +48,6 @@ type statusModule string const ( ngxHealthPort = 18080 ngxHealthPath = "/healthz" - ngxStatusPath = "/internal_nginx_status" - ngxVtsPath = "/nginx_status/format/json" defaultStatusModule statusModule = "default" vtsStatusModule statusModule = "vts" @@ -70,7 +68,7 @@ func newNGINXController() ingress.Controller { if ngx == "" { ngx = binary } - n := NGINXController{ + n := &NGINXController{ binary: ngx, configmap: &api.ConfigMap{}, } @@ -102,7 +100,7 @@ Error loading new template : %v go n.Start() - return ingress.Controller(&n) + return ingress.Controller(n) } // NGINXController ... @@ -117,11 +115,15 @@ type NGINXController struct { cmdArgs []string + watchClass string + namespace string + + stats *statsCollector statusModule statusModule } // Start start a new NGINX master process running in foreground. -func (n NGINXController) Start() { +func (n *NGINXController) Start() { glog.Info("starting NGINX process...") done := make(chan error, 1) @@ -170,15 +172,6 @@ func (n *NGINXController) start(cmd *exec.Cmd, done chan error) { n.cmdArgs = cmd.Args - cfg := ngx_template.ReadConfig(n.configmap.Data) - n.statusModule = defaultStatusModule - if cfg.EnableVtsStatus { - n.statusModule = vtsStatusModule - n.setupMonitor(vtsStatusModule) - } else { - n.setupMonitor(defaultStatusModule) - } - go func() { done <- cmd.Wait() }() @@ -264,12 +257,20 @@ func (n NGINXController) Info() *ingress.BackendInfo { } // OverrideFlags customize NGINX controller flags -func (n NGINXController) OverrideFlags(flags *pflag.FlagSet) { - ig, err := flags.GetString("ingress-class") - if err == nil && ig != "" && ig != defIngressClass { - glog.Warningf("only Ingress with class %v will be processed by this ingress controller", ig) +func (n *NGINXController) OverrideFlags(flags *pflag.FlagSet) { + ic, _ := flags.GetString("ingress-class") + wc, _ := flags.GetString("watch-namespace") + + if ic == "" { + ic = defIngressClass } - flags.Set("ingress-class", defIngressClass) + + if ic != defIngressClass { + glog.Warningf("only Ingress with class %v will be processed by this ingress controller", ic) + } + + flags.Set("ingress-class", ic) + n.stats = newStatsCollector(ic, wc, n.binary) } // DefaultIngressClass just return the default ingress class @@ -336,7 +337,11 @@ func (n *NGINXController) OnUpdate(ingressCfg ingress.Configuration) ([]byte, er cfg := ngx_template.ReadConfig(n.configmap.Data) // we need to check if the status module configuration changed - n.setupMonitor() + if cfg.EnableVtsStatus { + n.setupMonitor(vtsStatusModule) + } else { + n.setupMonitor(defaultStatusModule) + } // NGINX cannot resize the has tables used to store server names. // For this reason we check if the defined size defined is correct diff --git a/controllers/nginx/pkg/metric/collector/nginx.go b/controllers/nginx/pkg/metric/collector/nginx.go index 7f1b869ee..944eb920d 100644 --- a/controllers/nginx/pkg/metric/collector/nginx.go +++ b/controllers/nginx/pkg/metric/collector/nginx.go @@ -17,72 +17,103 @@ limitations under the License. package collector import ( + "fmt" + "github.com/golang/glog" "github.com/prometheus/client_golang/prometheus" ) -var ( - activeDesc = prometheus.NewDesc( - "nginx_active_connections", - "total number of active connections", - nil, nil) - - acceptedDesc = prometheus.NewDesc( - "nginx_accepted_connections", - "total number of accepted client connections", - nil, nil) - - handledDesc = prometheus.NewDesc( - "nginx_handled_connections", - "total number of handled connections", - nil, nil) - - requestsDesc = prometheus.NewDesc( - "nginx_total_requests", - "total number of client requests", - nil, nil) - - readingDesc = prometheus.NewDesc( - "nginx_current_reading_connections", - "current number of connections where nginx is reading the request header", - nil, nil) - - writingDesc = prometheus.NewDesc( - "nginx_current_writing_connections", - "current number of connections where nginx is writing the response back to the client", - nil, nil) - - waitingDesc = prometheus.NewDesc( - "nginx_current_waiting_connections", - "current number of idle client connections waiting for a request", - nil, nil) -) - type ( nginxStatusCollector struct { - scrapeChan chan scrapeRequest + scrapeChan chan scrapeRequest + ngxHealthPort int + ngxVtsPath string + data *nginxStatusData + } + + nginxStatusData struct { + active *prometheus.Desc + accepted *prometheus.Desc + handled *prometheus.Desc + requests *prometheus.Desc + reading *prometheus.Desc + writing *prometheus.Desc + waiting *prometheus.Desc } ) -func NewNginxStatus() (prometheus.Collector, error) { +func buildNS(namespace, class string) string { + if namespace == "" { + namespace = "all" + } + if class == "" { + class = "all" + } + + return fmt.Sprintf("%v_%v", namespace, class) +} + +// NewNginxStatus returns a new prometheus collector the default nginx status module +func NewNginxStatus(namespace, class string, ngxHealthPort int, ngxVtsPath string) Stopable { p := nginxStatusCollector{ - scrapeChan: make(chan scrapeRequest), + scrapeChan: make(chan scrapeRequest), + ngxHealthPort: ngxHealthPort, + ngxVtsPath: ngxVtsPath, + } + + ns := buildNS(namespace, class) + + p.data = &nginxStatusData{ + active: prometheus.NewDesc( + prometheus.BuildFQName(system, ns, "active_connections"), + "total number of active connections", + nil, nil), + + accepted: prometheus.NewDesc( + prometheus.BuildFQName(system, ns, "accepted_connections"), + "total number of accepted client connections", + nil, nil), + + handled: prometheus.NewDesc( + prometheus.BuildFQName(system, ns, "handled_connections"), + "total number of handled connections", + nil, nil), + + requests: prometheus.NewDesc( + prometheus.BuildFQName(system, ns, "total_requests"), + "total number of client requests", + nil, nil), + + reading: prometheus.NewDesc( + prometheus.BuildFQName(system, ns, "current_reading_connections"), + "current number of connections where nginx is reading the request header", + nil, nil), + + writing: prometheus.NewDesc( + prometheus.BuildFQName(system, ns, "current_writing_connections"), + "current number of connections where nginx is writing the response back to the client", + nil, nil), + + waiting: prometheus.NewDesc( + prometheus.BuildFQName(system, ns, "current_waiting_connections"), + "current number of idle client connections waiting for a request", + nil, nil), } go p.start() - return p, nil + return p } // Describe implements prometheus.Collector. func (p nginxStatusCollector) Describe(ch chan<- *prometheus.Desc) { - ch <- activeDesc - ch <- acceptedDesc - ch <- handledDesc - ch <- requestsDesc - ch <- readingDesc - ch <- writingDesc - ch <- waitingDesc + ch <- p.data.active + ch <- p.data.accepted + ch <- p.data.handled + ch <- p.data.requests + ch <- p.data.reading + ch <- p.data.writing + ch <- p.data.waiting } // Collect implements prometheus.Collector. @@ -106,25 +137,24 @@ func (p nginxStatusCollector) Stop() { // nginxStatusCollector scrap the nginx status func (p nginxStatusCollector) scrape(ch chan<- prometheus.Metric) { - s, err := getNginxStatus() + s, err := getNginxStatus(p.ngxHealthPort, p.ngxVtsPath) if err != nil { glog.Warningf("unexpected error obtaining nginx status info: %v", err) return } - ch <- prometheus.MustNewConstMetric(activeDesc, + ch <- prometheus.MustNewConstMetric(p.data.active, prometheus.GaugeValue, float64(s.Active)) - ch <- prometheus.MustNewConstMetric(acceptedDesc, + ch <- prometheus.MustNewConstMetric(p.data.accepted, prometheus.GaugeValue, float64(s.Accepted)) - ch <- prometheus.MustNewConstMetric(handledDesc, + ch <- prometheus.MustNewConstMetric(p.data.handled, prometheus.GaugeValue, float64(s.Handled)) - ch <- prometheus.MustNewConstMetric(requestsDesc, + ch <- prometheus.MustNewConstMetric(p.data.requests, prometheus.GaugeValue, float64(s.Requests)) - ch <- prometheus.MustNewConstMetric(readingDesc, + ch <- prometheus.MustNewConstMetric(p.data.reading, prometheus.GaugeValue, float64(s.Reading)) - ch <- prometheus.MustNewConstMetric(writingDesc, + ch <- prometheus.MustNewConstMetric(p.data.writing, prometheus.GaugeValue, float64(s.Writing)) - ch <- prometheus.MustNewConstMetric(waitingDesc, + ch <- prometheus.MustNewConstMetric(p.data.waiting, prometheus.GaugeValue, float64(s.Waiting)) - } diff --git a/controllers/nginx/pkg/metric/collector/process.go b/controllers/nginx/pkg/metric/collector/process.go index 3154863b3..8e9f3ec3f 100644 --- a/controllers/nginx/pkg/metric/collector/process.go +++ b/controllers/nginx/pkg/metric/collector/process.go @@ -25,63 +25,42 @@ import ( "github.com/prometheus/client_golang/prometheus" ) +// BinaryNameMatcher ... type BinaryNameMatcher struct { - name string - args []string + Name string + Binary string } +// MatchAndName returns false if the match failed, otherwise +// true and the resulting name. func (em BinaryNameMatcher) MatchAndName(nacl common.NameAndCmdline) (bool, string) { if len(nacl.Cmdline) == 0 { return false, "" } - cmd := filepath.Base(nacl.Cmdline[0]) - return em.name == cmd, "" + cmd := filepath.Base(em.Binary) + return em.Name == cmd, "" } -var ( - numprocsDesc = prometheus.NewDesc( - "nginx_num_procs", - "number of processes", - nil, nil) - - cpuSecsDesc = prometheus.NewDesc( - "nginx_cpu_seconds_total", - "Cpu usage in seconds", - nil, nil) - - readBytesDesc = prometheus.NewDesc( - "nginx_read_bytes_total", - "number of bytes read", - nil, nil) - - writeBytesDesc = prometheus.NewDesc( - "nginx_write_bytes_total", - "number of bytes written", - nil, nil) - - memResidentbytesDesc = prometheus.NewDesc( - "nginx_resident_memory_bytes", - "number of bytes of memory in use", - nil, nil) - - memVirtualbytesDesc = prometheus.NewDesc( - "nginx_virtual_memory_bytes", - "number of bytes of memory in use", - nil, nil) - - startTimeDesc = prometheus.NewDesc( - "nginx_oldest_start_time_seconds", - "start time in seconds since 1970/01/01", - nil, nil) -) +type namedProcessData struct { + numProcs *prometheus.Desc + cpuSecs *prometheus.Desc + readBytes *prometheus.Desc + writeBytes *prometheus.Desc + memResidentbytes *prometheus.Desc + memVirtualbytes *prometheus.Desc + startTime *prometheus.Desc +} type namedProcess struct { - scrapeChan chan scrapeRequest *proc.Grouper - fs *proc.FS + + scrapeChan chan scrapeRequest + fs *proc.FS + data namedProcessData } -func NewNamedProcessCollector(children bool, mn common.MatchNamer) (prometheus.Collector, error) { +// NewNamedProcess returns a new prometheus collector for the nginx process +func NewNamedProcess(children bool, mn common.MatchNamer) (prometheus.Collector, error) { fs, err := proc.NewFS("/proc") if err != nil { return nil, err @@ -96,6 +75,43 @@ func NewNamedProcessCollector(children bool, mn common.MatchNamer) (prometheus.C return nil, err } + p.data = namedProcessData{ + numProcs: prometheus.NewDesc( + "num_procs", + "number of processes", + nil, nil), + + cpuSecs: prometheus.NewDesc( + "cpu_seconds_total", + "Cpu usage in seconds", + nil, nil), + + readBytes: prometheus.NewDesc( + "read_bytes_total", + "number of bytes read", + nil, nil), + + writeBytes: prometheus.NewDesc( + "write_bytes_total", + "number of bytes written", + nil, nil), + + memResidentbytes: prometheus.NewDesc( + "resident_memory_bytes", + "number of bytes of memory in use", + nil, nil), + + memVirtualbytes: prometheus.NewDesc( + "virtual_memory_bytes", + "number of bytes of memory in use", + nil, nil), + + startTime: prometheus.NewDesc( + "oldest_start_time_seconds", + "start time in seconds since 1970/01/01", + nil, nil), + } + go p.start() return p, nil @@ -103,13 +119,13 @@ func NewNamedProcessCollector(children bool, mn common.MatchNamer) (prometheus.C // Describe implements prometheus.Collector. func (p namedProcess) Describe(ch chan<- *prometheus.Desc) { - ch <- cpuSecsDesc - ch <- numprocsDesc - ch <- readBytesDesc - ch <- writeBytesDesc - ch <- memResidentbytesDesc - ch <- memVirtualbytesDesc - ch <- startTimeDesc + ch <- p.data.cpuSecs + ch <- p.data.numProcs + ch <- p.data.readBytes + ch <- p.data.writeBytes + ch <- p.data.memResidentbytes + ch <- p.data.memVirtualbytes + ch <- p.data.startTime } // Collect implements prometheus.Collector. @@ -138,20 +154,20 @@ func (p namedProcess) scrape(ch chan<- prometheus.Metric) { return } - for gname, gcounts := range p.Groups() { - ch <- prometheus.MustNewConstMetric(numprocsDesc, + for _, gcounts := range p.Groups() { + ch <- prometheus.MustNewConstMetric(p.data.numProcs, prometheus.GaugeValue, float64(gcounts.Procs)) - ch <- prometheus.MustNewConstMetric(memResidentbytesDesc, + ch <- prometheus.MustNewConstMetric(p.data.memResidentbytes, prometheus.GaugeValue, float64(gcounts.Memresident)) - ch <- prometheus.MustNewConstMetric(memVirtualbytesDesc, + ch <- prometheus.MustNewConstMetric(p.data.memVirtualbytes, prometheus.GaugeValue, float64(gcounts.Memvirtual)) - ch <- prometheus.MustNewConstMetric(startTimeDesc, + ch <- prometheus.MustNewConstMetric(p.data.startTime, prometheus.GaugeValue, float64(gcounts.OldestStartTime.Unix())) - ch <- prometheus.MustNewConstMetric(cpuSecsDesc, + ch <- prometheus.MustNewConstMetric(p.data.cpuSecs, prometheus.CounterValue, gcounts.Cpu) - ch <- prometheus.MustNewConstMetric(readBytesDesc, + ch <- prometheus.MustNewConstMetric(p.data.readBytes, prometheus.CounterValue, float64(gcounts.ReadBytes)) - ch <- prometheus.MustNewConstMetric(writeBytesDesc, + ch <- prometheus.MustNewConstMetric(p.data.writeBytes, prometheus.CounterValue, float64(gcounts.WriteBytes)) } } diff --git a/controllers/nginx/pkg/metric/collector/scrape.go b/controllers/nginx/pkg/metric/collector/scrape.go index b40ff14ad..a078b2859 100644 --- a/controllers/nginx/pkg/metric/collector/scrape.go +++ b/controllers/nginx/pkg/metric/collector/scrape.go @@ -18,6 +18,12 @@ package collector import "github.com/prometheus/client_golang/prometheus" +// Stopable defines a prometheus collector that can be stopped +type Stopable interface { + prometheus.Collector + Stop() +} + type scrapeRequest struct { results chan<- prometheus.Metric done chan struct{} diff --git a/controllers/nginx/pkg/metric/collector/status.go b/controllers/nginx/pkg/metric/collector/status.go index 7bd0ecf52..1a0fcaf0e 100644 --- a/controllers/nginx/pkg/metric/collector/status.go +++ b/controllers/nginx/pkg/metric/collector/status.go @@ -73,14 +73,14 @@ type serverZone struct { InBytes float64 `json:"inBytes"` OutBytes float64 `json:"outBytes"` Responses response `json:"responses"` - Cache cache `json:"responses"` + Cache cache `json:"cache"` } type filterZone struct { RequestCounter float64 `json:"requestCounter"` InBytes float64 `json:"inBytes"` OutBytes float64 `json:"outBytes"` - Cache cache `json:"responses"` + Cache cache `json:"cache"` Responses response `json:"responses"` } @@ -127,8 +127,10 @@ type connections struct { Requests float64 `json:"requests"` } +// BoolToFloat64 ... type BoolToFloat64 float64 +// UnmarshalJSON ... func (bit BoolToFloat64) UnmarshalJSON(data []byte) error { asString := string(data) if asString == "1" || asString == "true" { @@ -141,7 +143,7 @@ func (bit BoolToFloat64) UnmarshalJSON(data []byte) error { return nil } -func getNginxStatus() (*basicStatus, error) { +func getNginxStatus(ngxHealthPort int, ngxStatusPath string) (*basicStatus, error) { url := fmt.Sprintf("http://localhost:%v%v", ngxHealthPort, ngxStatusPath) glog.V(3).Infof("start scrapping url: %v", url) @@ -172,7 +174,7 @@ func httpBody(url string) ([]byte, error) { return data, nil } -func getNginxVtsMetrics() (*vts, error) { +func getNginxVtsMetrics(ngxHealthPort int, ngxVtsPath string) (*vts, error) { url := fmt.Sprintf("http://localhost:%v%v", ngxHealthPort, ngxVtsPath) glog.V(3).Infof("start scrapping url: %v", url) diff --git a/controllers/nginx/pkg/metric/collector/status_test.go b/controllers/nginx/pkg/metric/collector/status_test.go index b6693d9c9..5d3075dae 100644 --- a/controllers/nginx/pkg/metric/collector/status_test.go +++ b/controllers/nginx/pkg/metric/collector/status_test.go @@ -17,32 +17,34 @@ limitations under the License. package collector import ( - "reflect" "testing" + + "github.com/kylelemons/godebug/pretty" ) func TestParseStatus(t *testing.T) { tests := []struct { in string - out *nginxStatus + out *basicStatus }{ {`Active connections: 43 server accepts handled requests 7368 7368 10993 Reading: 0 Writing: 5 Waiting: 38`, - &nginxStatus{43, 7368, 7368, 10993, 0, 5, 38}, + &basicStatus{43, 7368, 7368, 10993, 0, 5, 38}, }, {`Active connections: 0 server accepts handled requests 1 7 0 Reading: A Writing: B Waiting: 38`, - &nginxStatus{0, 1, 7, 0, 0, 0, 38}, + &basicStatus{0, 1, 7, 0, 0, 0, 38}, }, } for _, test := range tests { r := parse(test.in) - if !reflect.DeepEqual(r, test.out) { + if diff := pretty.Compare(r, test.out); diff != "" { + t.Logf("%v", diff) t.Fatalf("expected %v but returned %v", test.out, r) } } diff --git a/controllers/nginx/pkg/metric/collector/vts.go b/controllers/nginx/pkg/metric/collector/vts.go index a47531f65..4d80d66c4 100644 --- a/controllers/nginx/pkg/metric/collector/vts.go +++ b/controllers/nginx/pkg/metric/collector/vts.go @@ -23,136 +23,168 @@ import ( "github.com/prometheus/client_golang/prometheus" ) -var ( - vtsBytesDesc = prometheus.NewDesc( - "nginx_vts_bytes_total", - "Nginx bytes count", - []string{"server_zone", "direction"}, nil) +const system = "nginx" - vtsCacheDesc = prometheus.NewDesc( - "nginx_vts_cache_total", - "Nginx cache count", - []string{"server_zone", "type"}, nil) - - vtsConnectionsDesc = prometheus.NewDesc( - "nginx_vts_connections_total", - "Nginx connections count", - []string{"type"}, nil) - - vtsResponseDesc = prometheus.NewDesc( - "nginx_vts_responses_total", - "The number of responses with status codes 1xx, 2xx, 3xx, 4xx, and 5xx.", - []string{"server_zone", "status_code"}, nil) - - vtsRequestDesc = prometheus.NewDesc( - "nginx_vts_requests_total", - "The total number of requested client connections.", - []string{"server_zone"}, nil) - - vtsFilterZoneBytesDesc = prometheus.NewDesc( - "nginx_vts_filterzone_bytes_total", - "Nginx bytes count", - []string{"server_zone", "country", "direction"}, nil) - - vtsFilterZoneResponseDesc = prometheus.NewDesc( - "nginx_vts_filterzone_responses_total", - "The number of responses with status codes 1xx, 2xx, 3xx, 4xx, and 5xx.", - []string{"server_zone", "country", "status_code"}, nil) - - vtsFilterZoneCacheDesc = prometheus.NewDesc( - "nginx_vts_filterzone_cache_total", - "Nginx cache count", - []string{"server_zone", "country", "type"}, nil) - - vtsUpstreamBackupDesc = prometheus.NewDesc( - "nginx_vts_upstream_backup", - "Current backup setting of the server.", - []string{"upstream", "server"}, nil) - - vtsUpstreamBytesDesc = prometheus.NewDesc( - "nginx_vts_upstream_bytes_total", - "The total number of bytes sent to this server.", - []string{"upstream", "server", "direction"}, nil) - - vtsUpstreamDownDesc = prometheus.NewDesc( - "nginx_vts_upstream_down_total", - "Current down setting of the server.", - []string{"upstream", "server"}, nil) - - vtsUpstreamFailTimeoutDesc = prometheus.NewDesc( - "nginx_vts_upstream_fail_timeout", - "Current fail_timeout setting of the server.", - []string{"upstream", "server"}, nil) - - vtsUpstreamMaxFailsDesc = prometheus.NewDesc( - "nginx_vts_upstream_maxfails", - "Current max_fails setting of the server.", - []string{"upstream", "server"}, nil) - - vtsUpstreamResponsesDesc = prometheus.NewDesc( - "nginx_vts_upstream_responses_total", - "The number of upstream responses with status codes 1xx, 2xx, 3xx, 4xx, and 5xx.", - []string{"upstream", "server", "status_code"}, nil) - - vtsUpstreamRequestDesc = prometheus.NewDesc( - "nginx_vts_upstream_requests_total", - "The total number of client connections forwarded to this server.", - []string{"upstream", "server"}, nil) - - vtsUpstreamResponseMsecDesc = prometheus.NewDesc( - "nginx_vts_upstream_response_msecs_avg", - "The average of only upstream response processing times in milliseconds.", - []string{"upstream", "server"}, nil) - - vtsUpstreamWeightDesc = prometheus.NewDesc( - "nginx_vts_upstream_weight", - "Current upstream weight setting of the server.", - []string{"upstream", "server"}, nil) -) - -type vtsCollector struct { - scrapeChan chan scrapeRequest +type ( + vtsCollector struct { + scrapeChan chan scrapeRequest + ngxHealthPort int + ngxVtsPath string + data *vtsData } -func NewNGINXVTSCollector() (prometheus.Collector, error) { + vtsData struct { + bytes *prometheus.Desc + cache *prometheus.Desc + connections *prometheus.Desc + response *prometheus.Desc + request *prometheus.Desc + filterZoneBytes *prometheus.Desc + filterZoneResponse *prometheus.Desc + filterZoneCache *prometheus.Desc + upstreamBackup *prometheus.Desc + upstreamBytes *prometheus.Desc + upstreamDown *prometheus.Desc + upstreamFailTimeout *prometheus.Desc + upstreamMaxFails *prometheus.Desc + upstreamResponses *prometheus.Desc + upstreamRequest *prometheus.Desc + upstreamResponseMsec *prometheus.Desc + upstreamWeight *prometheus.Desc + } +) + +// NewNGINXVTSCollector returns a new prometheus collector for the VTS module +func NewNGINXVTSCollector(namespace, class string, ngxHealthPort int, ngxVtsPath string) Stopable { p := vtsCollector{ - scrapeChan: make(chan scrapeRequest), + scrapeChan: make(chan scrapeRequest), + ngxHealthPort: ngxHealthPort, + ngxVtsPath: ngxVtsPath, + } + + ns := buildNS(namespace, class) + + p.data = &vtsData{ + bytes: prometheus.NewDesc( + prometheus.BuildFQName(system, ns, "bytes_total"), + "Nginx bytes count", + []string{"server_zone", "direction"}, nil), + + cache: prometheus.NewDesc( + prometheus.BuildFQName(system, ns, "cache_total"), + "Nginx cache count", + []string{"server_zone", "type"}, nil), + + connections: prometheus.NewDesc( + prometheus.BuildFQName(system, ns, "connections_total"), + "Nginx connections count", + []string{"type"}, nil), + + response: prometheus.NewDesc( + prometheus.BuildFQName(system, ns, "responses_total"), + "The number of responses with status codes 1xx, 2xx, 3xx, 4xx, and 5xx.", + []string{"server_zone", "status_code"}, nil), + + request: prometheus.NewDesc( + prometheus.BuildFQName(system, ns, "requests_total"), + "The total number of requested client connections.", + []string{"server_zone"}, nil), + + filterZoneBytes: prometheus.NewDesc( + prometheus.BuildFQName(system, ns, "filterzone_bytes_total"), + "Nginx bytes count", + []string{"server_zone", "country", "direction"}, nil), + + filterZoneResponse: prometheus.NewDesc( + prometheus.BuildFQName(system, ns, "filterzone_responses_total"), + "The number of responses with status codes 1xx, 2xx, 3xx, 4xx, and 5xx.", + []string{"server_zone", "country", "status_code"}, nil), + + filterZoneCache: prometheus.NewDesc( + prometheus.BuildFQName(system, ns, "filterzone_cache_total"), + "Nginx cache count", + []string{"server_zone", "country", "type"}, nil), + + upstreamBackup: prometheus.NewDesc( + prometheus.BuildFQName(system, ns, "upstream_backup"), + "Current backup setting of the server.", + []string{"upstream", "server"}, nil), + + upstreamBytes: prometheus.NewDesc( + prometheus.BuildFQName(system, ns, "upstream_bytes_total"), + "The total number of bytes sent to this server.", + []string{"upstream", "server", "direction"}, nil), + + upstreamDown: prometheus.NewDesc( + prometheus.BuildFQName(system, ns, "vts_upstream_down_total"), + "Current down setting of the server.", + []string{"upstream", "server"}, nil), + + upstreamFailTimeout: prometheus.NewDesc( + prometheus.BuildFQName(system, ns, "upstream_fail_timeout"), + "Current fail_timeout setting of the server.", + []string{"upstream", "server"}, nil), + + upstreamMaxFails: prometheus.NewDesc( + prometheus.BuildFQName(system, ns, "upstream_maxfails"), + "Current max_fails setting of the server.", + []string{"upstream", "server"}, nil), + + upstreamResponses: prometheus.NewDesc( + prometheus.BuildFQName(system, ns, "upstream_responses_total"), + "The number of upstream responses with status codes 1xx, 2xx, 3xx, 4xx, and 5xx.", + []string{"upstream", "server", "status_code"}, nil), + + upstreamRequest: prometheus.NewDesc( + prometheus.BuildFQName(system, ns, "upstream_requests_total"), + "The total number of client connections forwarded to this server.", + []string{"upstream", "server"}, nil), + + upstreamResponseMsec: prometheus.NewDesc( + prometheus.BuildFQName(system, ns, "upstream_response_msecs_avg"), + "The average of only upstream response processing times in milliseconds.", + []string{"upstream", "server"}, nil), + + upstreamWeight: prometheus.NewDesc( + prometheus.BuildFQName(system, ns, "upstream_weight"), + "Current upstream weight setting of the server.", + []string{"upstream", "server"}, nil), } go p.start() - return p, nil + return p } // Describe implements prometheus.Collector. -func (p *vtsCollector) Describe(ch chan<- *prometheus.Desc) { - ch <- vtsBytesDesc - ch <- vtsCacheDesc - ch <- vtsConnectionsDesc - ch <- vtsRequestDesc - ch <- vtsResponseDesc - ch <- vtsUpstreamBackupDesc - ch <- vtsUpstreamBytesDesc - ch <- vtsUpstreamDownDesc - ch <- vtsUpstreamFailTimeoutDesc - ch <- vtsUpstreamMaxFailsDesc - ch <- vtsUpstreamRequestDesc - ch <- vtsUpstreamResponseMsecDesc - ch <- vtsUpstreamResponsesDesc - ch <- vtsUpstreamWeightDesc - ch <- vtsFilterZoneBytesDesc - ch <- vtsFilterZoneCacheDesc - ch <- vtsFilterZoneResponseDesc +func (p vtsCollector) Describe(ch chan<- *prometheus.Desc) { + ch <- p.data.bytes + ch <- p.data.cache + ch <- p.data.connections + ch <- p.data.request + ch <- p.data.response + ch <- p.data.upstreamBackup + ch <- p.data.upstreamBytes + ch <- p.data.upstreamDown + ch <- p.data.upstreamFailTimeout + ch <- p.data.upstreamMaxFails + ch <- p.data.upstreamRequest + ch <- p.data.upstreamResponseMsec + ch <- p.data.upstreamResponses + ch <- p.data.upstreamWeight + ch <- p.data.filterZoneBytes + ch <- p.data.filterZoneCache + ch <- p.data.filterZoneResponse } // Collect implements prometheus.Collector. -func (p *vtsCollector) Collect(ch chan<- prometheus.Metric) { +func (p vtsCollector) Collect(ch chan<- prometheus.Metric) { req := scrapeRequest{results: ch, done: make(chan struct{})} p.scrapeChan <- req <-req.done } -func (p *vtsCollector) start() { +func (p vtsCollector) start() { for req := range p.scrapeChan { ch := req.results p.scrapeVts(ch) @@ -160,65 +192,65 @@ func (p *vtsCollector) start() { } } -func (p *vtsCollector) Stop() { +func (p vtsCollector) Stop() { close(p.scrapeChan) } // scrapeVts scrape nginx vts metrics -func (p *vtsCollector) scrapeVts(ch chan<- prometheus.Metric) { - nginxMetrics, err := getNginxVtsMetrics() +func (p vtsCollector) scrapeVts(ch chan<- prometheus.Metric) { + nginxMetrics, err := getNginxVtsMetrics(p.ngxHealthPort, p.ngxVtsPath) if err != nil { glog.Warningf("unexpected error obtaining nginx status info: %v", err) return } - reflectMetrics(&nginxMetrics.Connections, vtsConnectionsDesc, ch) + reflectMetrics(&nginxMetrics.Connections, p.data.connections, ch) for name, zones := range nginxMetrics.UpstreamZones { for pos, value := range zones { - reflectMetrics(&zones[pos].Responses, vtsUpstreamResponsesDesc, ch, name, value.Server) + reflectMetrics(&zones[pos].Responses, p.data.upstreamResponses, ch, name, value.Server) - ch <- prometheus.MustNewConstMetric(vtsUpstreamRequestDesc, - prometheus.CounterValue, float64(zones[pos].RequestCounter), name, value.Server) - ch <- prometheus.MustNewConstMetric(vtsUpstreamDownDesc, + ch <- prometheus.MustNewConstMetric(p.data.upstreamRequest, + prometheus.CounterValue, zones[pos].RequestCounter, name, value.Server) + ch <- prometheus.MustNewConstMetric(p.data.upstreamDown, prometheus.CounterValue, float64(zones[pos].Down), name, value.Server) - ch <- prometheus.MustNewConstMetric(vtsUpstreamWeightDesc, - prometheus.CounterValue, float64(zones[pos].Weight), name, value.Server) - ch <- prometheus.MustNewConstMetric(vtsUpstreamResponseMsecDesc, - prometheus.CounterValue, float64(zones[pos].ResponseMsec), name, value.Server) - ch <- prometheus.MustNewConstMetric(vtsUpstreamBackupDesc, + ch <- prometheus.MustNewConstMetric(p.data.upstreamWeight, + prometheus.CounterValue, zones[pos].Weight, name, value.Server) + ch <- prometheus.MustNewConstMetric(p.data.upstreamResponseMsec, + prometheus.CounterValue, zones[pos].ResponseMsec, name, value.Server) + ch <- prometheus.MustNewConstMetric(p.data.upstreamBackup, prometheus.CounterValue, float64(zones[pos].Backup), name, value.Server) - ch <- prometheus.MustNewConstMetric(vtsUpstreamFailTimeoutDesc, - prometheus.CounterValue, float64(zones[pos].FailTimeout), name, value.Server) - ch <- prometheus.MustNewConstMetric(vtsUpstreamMaxFailsDesc, - prometheus.CounterValue, float64(zones[pos].MaxFails), name, value.Server) - ch <- prometheus.MustNewConstMetric(vtsUpstreamBytesDesc, - prometheus.CounterValue, float64(zones[pos].InBytes), name, value.Server, "in") - ch <- prometheus.MustNewConstMetric(vtsUpstreamBytesDesc, - prometheus.CounterValue, float64(zones[pos].OutBytes), name, value.Server, "out") + ch <- prometheus.MustNewConstMetric(p.data.upstreamFailTimeout, + prometheus.CounterValue, zones[pos].FailTimeout, name, value.Server) + ch <- prometheus.MustNewConstMetric(p.data.upstreamMaxFails, + prometheus.CounterValue, zones[pos].MaxFails, name, value.Server) + ch <- prometheus.MustNewConstMetric(p.data.upstreamBytes, + prometheus.CounterValue, zones[pos].InBytes, name, value.Server, "in") + ch <- prometheus.MustNewConstMetric(p.data.upstreamBytes, + prometheus.CounterValue, zones[pos].OutBytes, name, value.Server, "out") } } for name, zone := range nginxMetrics.ServerZones { - reflectMetrics(&zone.Responses, vtsResponseDesc, ch, name) - reflectMetrics(&zone.Cache, vtsCacheDesc, ch, name) + reflectMetrics(&zone.Responses, p.data.response, ch, name) + reflectMetrics(&zone.Cache, p.data.cache, ch, name) - ch <- prometheus.MustNewConstMetric(vtsRequestDesc, - prometheus.CounterValue, float64(zone.RequestCounter), name) - ch <- prometheus.MustNewConstMetric(vtsBytesDesc, - prometheus.CounterValue, float64(zone.InBytes), name, "in") - ch <- prometheus.MustNewConstMetric(vtsBytesDesc, - prometheus.CounterValue, float64(zone.OutBytes), name, "out") + ch <- prometheus.MustNewConstMetric(p.data.request, + prometheus.CounterValue, zone.RequestCounter, name) + ch <- prometheus.MustNewConstMetric(p.data.bytes, + prometheus.CounterValue, zone.InBytes, name, "in") + ch <- prometheus.MustNewConstMetric(p.data.bytes, + prometheus.CounterValue, zone.OutBytes, name, "out") } for serverZone, countries := range nginxMetrics.FilterZones { for country, zone := range countries { - reflectMetrics(&zone.Responses, vtsFilterZoneResponseDesc, ch, serverZone, country) - reflectMetrics(&zone.Cache, vtsFilterZoneCacheDesc, ch, serverZone, country) + reflectMetrics(&zone.Responses, p.data.filterZoneResponse, ch, serverZone, country) + reflectMetrics(&zone.Cache, p.data.filterZoneCache, ch, serverZone, country) - ch <- prometheus.MustNewConstMetric(vtsFilterZoneBytesDesc, + ch <- prometheus.MustNewConstMetric(p.data.filterZoneBytes, prometheus.CounterValue, float64(zone.InBytes), serverZone, country, "in") - ch <- prometheus.MustNewConstMetric(vtsFilterZoneBytesDesc, + ch <- prometheus.MustNewConstMetric(p.data.filterZoneBytes, prometheus.CounterValue, float64(zone.OutBytes), serverZone, country, "out") } } @@ -229,9 +261,9 @@ func reflectMetrics(value interface{}, desc *prometheus.Desc, ch chan<- promethe for i := 0; i < val.NumField(); i++ { tag := val.Type().Field(i).Tag - labels := append(labels, tag.Get("json")) + l := append(labels, tag.Get("json")) ch <- prometheus.MustNewConstMetric(desc, prometheus.CounterValue, float64(val.Field(i).Interface().(float64)), - labels...) + l...) } } diff --git a/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl b/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl index 9813bcd24..07a7e7921 100644 --- a/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl +++ b/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl @@ -240,6 +240,8 @@ http { {{ if not (empty $authPath) }} location = {{ $authPath }} { internal; + set $proxy_upstream_name "internal"; + {{ if not $location.ExternalAuth.SendBody }} proxy_pass_request_body off; proxy_set_header Content-Length ""; @@ -402,6 +404,8 @@ http { } location /nginx_status { + set $proxy_upstream_name "internal"; + {{ if $cfg.EnableVtsStatus }} vhost_traffic_status_display; vhost_traffic_status_display_format html; @@ -415,6 +419,8 @@ http { # using prometheus. # TODO: enable extraction for vts module. location /internal_nginx_status { + set $proxy_upstream_name "internal"; + allow 127.0.0.1; {{ if not $cfg.DisableIpv6 }}allow ::1;{{ end }} deny all;