diff --git a/controllers/nginx/Makefile b/controllers/nginx/Makefile index 8805e2d2a..bf22ee4ab 100644 --- a/controllers/nginx/Makefile +++ b/controllers/nginx/Makefile @@ -21,7 +21,7 @@ build: clean -ldflags "-s -w -X ${PKG}/pkg/version.RELEASE=${RELEASE} -X ${PKG}/pkg/version.COMMIT=${COMMIT} -X ${PKG}/pkg/version.REPO=${REPO_INFO}" \ -o rootfs/nginx-ingress-controller ${PKG}/pkg/cmd/controller -container: build +container: $(DOCKER) build --pull -t $(PREFIX):$(RELEASE) rootfs push: container diff --git a/controllers/nginx/pkg/cmd/controller/metrics.go b/controllers/nginx/pkg/cmd/controller/metrics.go index a42a19230..b803e6756 100644 --- a/controllers/nginx/pkg/cmd/controller/metrics.go +++ b/controllers/nginx/pkg/cmd/controller/metrics.go @@ -23,31 +23,73 @@ import ( "k8s.io/ingress/controllers/nginx/pkg/metric/collector" ) +const ( + ngxStatusPath = "/internal_nginx_status" + ngxVtsPath = "/nginx_status/format/json" +) + func (n *NGINXController) setupMonitor(sm statusModule) { csm := n.statusModule if csm != sm { - prometheus + glog.Infof("changing prometheus collector from %v to %v", csm, sm) + n.stats.stop(csm) + n.stats.start(sm) n.statusModule = sm } } type statsCollector struct { process prometheus.Collector - basic prometheus.Collector - vts prometheus.Collector + basic collector.Stopable + vts collector.Stopable + + namespace string + watchClass string } -func newStatsCollector() (*statsCollector, error) { - pc, err := collector.NewNamedProcess(true, collector.BinaryNameMatcher{"nginx", n.cmdArgs}) +func (s *statsCollector) stop(sm statusModule) { + switch sm { + case defaultStatusModule: + s.basic.Stop() + prometheus.Unregister(s.basic) + break + case vtsStatusModule: + s.vts.Stop() + prometheus.Unregister(s.vts) + break + } +} + +func (s *statsCollector) start(sm statusModule) { + switch sm { + case defaultStatusModule: + s.basic = collector.NewNginxStatus(s.namespace, s.watchClass, ngxHealthPort, ngxStatusPath) + prometheus.Register(s.basic) + break + case vtsStatusModule: + s.vts = collector.NewNGINXVTSCollector(s.namespace, s.watchClass, ngxHealthPort, ngxVtsPath) + prometheus.Register(s.vts) + break + } +} + +func newStatsCollector(ns, class, binary string) *statsCollector { + glog.Infof("starting new nginx stats collector for Ingress controller running in namespace %v (class %v)", ns, class) + pc, err := collector.NewNamedProcess(true, collector.BinaryNameMatcher{ + Name: "nginx", + Binary: binary, + }) if err != nil { - return nil, err + glog.Fatalf("unexpected error registering nginx collector: %v", err) } err = prometheus.Register(pc) if err != nil { glog.Fatalf("unexpected error registering nginx collector: %v", err) } - return nil, &statsCollector{ - process: pc, + return &statsCollector{ + namespace: ns, + watchClass: class, + process: pc, } } diff --git a/controllers/nginx/pkg/cmd/controller/nginx.go b/controllers/nginx/pkg/cmd/controller/nginx.go index 7a484e7ef..1bd7ede60 100644 --- a/controllers/nginx/pkg/cmd/controller/nginx.go +++ b/controllers/nginx/pkg/cmd/controller/nginx.go @@ -48,8 +48,6 @@ type statusModule string const ( ngxHealthPort = 18080 ngxHealthPath = "/healthz" - ngxStatusPath = "/internal_nginx_status" - ngxVtsPath = "/nginx_status/format/json" defaultStatusModule statusModule = "default" vtsStatusModule statusModule = "vts" @@ -70,7 +68,7 @@ func newNGINXController() ingress.Controller { if ngx == "" { ngx = binary } - n := NGINXController{ + n := &NGINXController{ binary: ngx, configmap: &api.ConfigMap{}, } @@ -102,7 +100,7 @@ Error loading new template : %v go n.Start() - return ingress.Controller(&n) + return ingress.Controller(n) } // NGINXController ... @@ -117,11 +115,15 @@ type NGINXController struct { cmdArgs []string + watchClass string + namespace string + + stats *statsCollector statusModule statusModule } // Start start a new NGINX master process running in foreground. -func (n NGINXController) Start() { +func (n *NGINXController) Start() { glog.Info("starting NGINX process...") done := make(chan error, 1) @@ -170,15 +172,6 @@ func (n *NGINXController) start(cmd *exec.Cmd, done chan error) { n.cmdArgs = cmd.Args - cfg := ngx_template.ReadConfig(n.configmap.Data) - n.statusModule = defaultStatusModule - if cfg.EnableVtsStatus { - n.statusModule = vtsStatusModule - n.setupMonitor(vtsStatusModule) - } else { - n.setupMonitor(defaultStatusModule) - } - go func() { done <- cmd.Wait() }() @@ -264,12 +257,20 @@ func (n NGINXController) Info() *ingress.BackendInfo { } // OverrideFlags customize NGINX controller flags -func (n NGINXController) OverrideFlags(flags *pflag.FlagSet) { - ig, err := flags.GetString("ingress-class") - if err == nil && ig != "" && ig != defIngressClass { - glog.Warningf("only Ingress with class %v will be processed by this ingress controller", ig) +func (n *NGINXController) OverrideFlags(flags *pflag.FlagSet) { + ic, _ := flags.GetString("ingress-class") + wc, _ := flags.GetString("watch-namespace") + + if ic == "" { + ic = defIngressClass } - flags.Set("ingress-class", defIngressClass) + + if ic != defIngressClass { + glog.Warningf("only Ingress with class %v will be processed by this ingress controller", ic) + } + + flags.Set("ingress-class", ic) + n.stats = newStatsCollector(ic, wc, n.binary) } // DefaultIngressClass just return the default ingress class @@ -336,7 +337,11 @@ func (n *NGINXController) OnUpdate(ingressCfg ingress.Configuration) ([]byte, er cfg := ngx_template.ReadConfig(n.configmap.Data) // we need to check if the status module configuration changed - n.setupMonitor() + if cfg.EnableVtsStatus { + n.setupMonitor(vtsStatusModule) + } else { + n.setupMonitor(defaultStatusModule) + } // NGINX cannot resize the has tables used to store server names. // For this reason we check if the defined size defined is correct diff --git a/controllers/nginx/pkg/metric/collector/nginx.go b/controllers/nginx/pkg/metric/collector/nginx.go index 7f1b869ee..944eb920d 100644 --- a/controllers/nginx/pkg/metric/collector/nginx.go +++ b/controllers/nginx/pkg/metric/collector/nginx.go @@ -17,72 +17,103 @@ limitations under the License. package collector import ( + "fmt" + "github.com/golang/glog" "github.com/prometheus/client_golang/prometheus" ) -var ( - activeDesc = prometheus.NewDesc( - "nginx_active_connections", - "total number of active connections", - nil, nil) - - acceptedDesc = prometheus.NewDesc( - "nginx_accepted_connections", - "total number of accepted client connections", - nil, nil) - - handledDesc = prometheus.NewDesc( - "nginx_handled_connections", - "total number of handled connections", - nil, nil) - - requestsDesc = prometheus.NewDesc( - "nginx_total_requests", - "total number of client requests", - nil, nil) - - readingDesc = prometheus.NewDesc( - "nginx_current_reading_connections", - "current number of connections where nginx is reading the request header", - nil, nil) - - writingDesc = prometheus.NewDesc( - "nginx_current_writing_connections", - "current number of connections where nginx is writing the response back to the client", - nil, nil) - - waitingDesc = prometheus.NewDesc( - "nginx_current_waiting_connections", - "current number of idle client connections waiting for a request", - nil, nil) -) - type ( nginxStatusCollector struct { - scrapeChan chan scrapeRequest + scrapeChan chan scrapeRequest + ngxHealthPort int + ngxVtsPath string + data *nginxStatusData + } + + nginxStatusData struct { + active *prometheus.Desc + accepted *prometheus.Desc + handled *prometheus.Desc + requests *prometheus.Desc + reading *prometheus.Desc + writing *prometheus.Desc + waiting *prometheus.Desc } ) -func NewNginxStatus() (prometheus.Collector, error) { +func buildNS(namespace, class string) string { + if namespace == "" { + namespace = "all" + } + if class == "" { + class = "all" + } + + return fmt.Sprintf("%v_%v", namespace, class) +} + +// NewNginxStatus returns a new prometheus collector the default nginx status module +func NewNginxStatus(namespace, class string, ngxHealthPort int, ngxVtsPath string) Stopable { p := nginxStatusCollector{ - scrapeChan: make(chan scrapeRequest), + scrapeChan: make(chan scrapeRequest), + ngxHealthPort: ngxHealthPort, + ngxVtsPath: ngxVtsPath, + } + + ns := buildNS(namespace, class) + + p.data = &nginxStatusData{ + active: prometheus.NewDesc( + prometheus.BuildFQName(system, ns, "active_connections"), + "total number of active connections", + nil, nil), + + accepted: prometheus.NewDesc( + prometheus.BuildFQName(system, ns, "accepted_connections"), + "total number of accepted client connections", + nil, nil), + + handled: prometheus.NewDesc( + prometheus.BuildFQName(system, ns, "handled_connections"), + "total number of handled connections", + nil, nil), + + requests: prometheus.NewDesc( + prometheus.BuildFQName(system, ns, "total_requests"), + "total number of client requests", + nil, nil), + + reading: prometheus.NewDesc( + prometheus.BuildFQName(system, ns, "current_reading_connections"), + "current number of connections where nginx is reading the request header", + nil, nil), + + writing: prometheus.NewDesc( + prometheus.BuildFQName(system, ns, "current_writing_connections"), + "current number of connections where nginx is writing the response back to the client", + nil, nil), + + waiting: prometheus.NewDesc( + prometheus.BuildFQName(system, ns, "current_waiting_connections"), + "current number of idle client connections waiting for a request", + nil, nil), } go p.start() - return p, nil + return p } // Describe implements prometheus.Collector. func (p nginxStatusCollector) Describe(ch chan<- *prometheus.Desc) { - ch <- activeDesc - ch <- acceptedDesc - ch <- handledDesc - ch <- requestsDesc - ch <- readingDesc - ch <- writingDesc - ch <- waitingDesc + ch <- p.data.active + ch <- p.data.accepted + ch <- p.data.handled + ch <- p.data.requests + ch <- p.data.reading + ch <- p.data.writing + ch <- p.data.waiting } // Collect implements prometheus.Collector. @@ -106,25 +137,24 @@ func (p nginxStatusCollector) Stop() { // nginxStatusCollector scrap the nginx status func (p nginxStatusCollector) scrape(ch chan<- prometheus.Metric) { - s, err := getNginxStatus() + s, err := getNginxStatus(p.ngxHealthPort, p.ngxVtsPath) if err != nil { glog.Warningf("unexpected error obtaining nginx status info: %v", err) return } - ch <- prometheus.MustNewConstMetric(activeDesc, + ch <- prometheus.MustNewConstMetric(p.data.active, prometheus.GaugeValue, float64(s.Active)) - ch <- prometheus.MustNewConstMetric(acceptedDesc, + ch <- prometheus.MustNewConstMetric(p.data.accepted, prometheus.GaugeValue, float64(s.Accepted)) - ch <- prometheus.MustNewConstMetric(handledDesc, + ch <- prometheus.MustNewConstMetric(p.data.handled, prometheus.GaugeValue, float64(s.Handled)) - ch <- prometheus.MustNewConstMetric(requestsDesc, + ch <- prometheus.MustNewConstMetric(p.data.requests, prometheus.GaugeValue, float64(s.Requests)) - ch <- prometheus.MustNewConstMetric(readingDesc, + ch <- prometheus.MustNewConstMetric(p.data.reading, prometheus.GaugeValue, float64(s.Reading)) - ch <- prometheus.MustNewConstMetric(writingDesc, + ch <- prometheus.MustNewConstMetric(p.data.writing, prometheus.GaugeValue, float64(s.Writing)) - ch <- prometheus.MustNewConstMetric(waitingDesc, + ch <- prometheus.MustNewConstMetric(p.data.waiting, prometheus.GaugeValue, float64(s.Waiting)) - } diff --git a/controllers/nginx/pkg/metric/collector/process.go b/controllers/nginx/pkg/metric/collector/process.go index 3154863b3..8e9f3ec3f 100644 --- a/controllers/nginx/pkg/metric/collector/process.go +++ b/controllers/nginx/pkg/metric/collector/process.go @@ -25,63 +25,42 @@ import ( "github.com/prometheus/client_golang/prometheus" ) +// BinaryNameMatcher ... type BinaryNameMatcher struct { - name string - args []string + Name string + Binary string } +// MatchAndName returns false if the match failed, otherwise +// true and the resulting name. func (em BinaryNameMatcher) MatchAndName(nacl common.NameAndCmdline) (bool, string) { if len(nacl.Cmdline) == 0 { return false, "" } - cmd := filepath.Base(nacl.Cmdline[0]) - return em.name == cmd, "" + cmd := filepath.Base(em.Binary) + return em.Name == cmd, "" } -var ( - numprocsDesc = prometheus.NewDesc( - "nginx_num_procs", - "number of processes", - nil, nil) - - cpuSecsDesc = prometheus.NewDesc( - "nginx_cpu_seconds_total", - "Cpu usage in seconds", - nil, nil) - - readBytesDesc = prometheus.NewDesc( - "nginx_read_bytes_total", - "number of bytes read", - nil, nil) - - writeBytesDesc = prometheus.NewDesc( - "nginx_write_bytes_total", - "number of bytes written", - nil, nil) - - memResidentbytesDesc = prometheus.NewDesc( - "nginx_resident_memory_bytes", - "number of bytes of memory in use", - nil, nil) - - memVirtualbytesDesc = prometheus.NewDesc( - "nginx_virtual_memory_bytes", - "number of bytes of memory in use", - nil, nil) - - startTimeDesc = prometheus.NewDesc( - "nginx_oldest_start_time_seconds", - "start time in seconds since 1970/01/01", - nil, nil) -) +type namedProcessData struct { + numProcs *prometheus.Desc + cpuSecs *prometheus.Desc + readBytes *prometheus.Desc + writeBytes *prometheus.Desc + memResidentbytes *prometheus.Desc + memVirtualbytes *prometheus.Desc + startTime *prometheus.Desc +} type namedProcess struct { - scrapeChan chan scrapeRequest *proc.Grouper - fs *proc.FS + + scrapeChan chan scrapeRequest + fs *proc.FS + data namedProcessData } -func NewNamedProcessCollector(children bool, mn common.MatchNamer) (prometheus.Collector, error) { +// NewNamedProcess returns a new prometheus collector for the nginx process +func NewNamedProcess(children bool, mn common.MatchNamer) (prometheus.Collector, error) { fs, err := proc.NewFS("/proc") if err != nil { return nil, err @@ -96,6 +75,43 @@ func NewNamedProcessCollector(children bool, mn common.MatchNamer) (prometheus.C return nil, err } + p.data = namedProcessData{ + numProcs: prometheus.NewDesc( + "num_procs", + "number of processes", + nil, nil), + + cpuSecs: prometheus.NewDesc( + "cpu_seconds_total", + "Cpu usage in seconds", + nil, nil), + + readBytes: prometheus.NewDesc( + "read_bytes_total", + "number of bytes read", + nil, nil), + + writeBytes: prometheus.NewDesc( + "write_bytes_total", + "number of bytes written", + nil, nil), + + memResidentbytes: prometheus.NewDesc( + "resident_memory_bytes", + "number of bytes of memory in use", + nil, nil), + + memVirtualbytes: prometheus.NewDesc( + "virtual_memory_bytes", + "number of bytes of memory in use", + nil, nil), + + startTime: prometheus.NewDesc( + "oldest_start_time_seconds", + "start time in seconds since 1970/01/01", + nil, nil), + } + go p.start() return p, nil @@ -103,13 +119,13 @@ func NewNamedProcessCollector(children bool, mn common.MatchNamer) (prometheus.C // Describe implements prometheus.Collector. func (p namedProcess) Describe(ch chan<- *prometheus.Desc) { - ch <- cpuSecsDesc - ch <- numprocsDesc - ch <- readBytesDesc - ch <- writeBytesDesc - ch <- memResidentbytesDesc - ch <- memVirtualbytesDesc - ch <- startTimeDesc + ch <- p.data.cpuSecs + ch <- p.data.numProcs + ch <- p.data.readBytes + ch <- p.data.writeBytes + ch <- p.data.memResidentbytes + ch <- p.data.memVirtualbytes + ch <- p.data.startTime } // Collect implements prometheus.Collector. @@ -138,20 +154,20 @@ func (p namedProcess) scrape(ch chan<- prometheus.Metric) { return } - for gname, gcounts := range p.Groups() { - ch <- prometheus.MustNewConstMetric(numprocsDesc, + for _, gcounts := range p.Groups() { + ch <- prometheus.MustNewConstMetric(p.data.numProcs, prometheus.GaugeValue, float64(gcounts.Procs)) - ch <- prometheus.MustNewConstMetric(memResidentbytesDesc, + ch <- prometheus.MustNewConstMetric(p.data.memResidentbytes, prometheus.GaugeValue, float64(gcounts.Memresident)) - ch <- prometheus.MustNewConstMetric(memVirtualbytesDesc, + ch <- prometheus.MustNewConstMetric(p.data.memVirtualbytes, prometheus.GaugeValue, float64(gcounts.Memvirtual)) - ch <- prometheus.MustNewConstMetric(startTimeDesc, + ch <- prometheus.MustNewConstMetric(p.data.startTime, prometheus.GaugeValue, float64(gcounts.OldestStartTime.Unix())) - ch <- prometheus.MustNewConstMetric(cpuSecsDesc, + ch <- prometheus.MustNewConstMetric(p.data.cpuSecs, prometheus.CounterValue, gcounts.Cpu) - ch <- prometheus.MustNewConstMetric(readBytesDesc, + ch <- prometheus.MustNewConstMetric(p.data.readBytes, prometheus.CounterValue, float64(gcounts.ReadBytes)) - ch <- prometheus.MustNewConstMetric(writeBytesDesc, + ch <- prometheus.MustNewConstMetric(p.data.writeBytes, prometheus.CounterValue, float64(gcounts.WriteBytes)) } } diff --git a/controllers/nginx/pkg/metric/collector/scrape.go b/controllers/nginx/pkg/metric/collector/scrape.go index b40ff14ad..a078b2859 100644 --- a/controllers/nginx/pkg/metric/collector/scrape.go +++ b/controllers/nginx/pkg/metric/collector/scrape.go @@ -18,6 +18,12 @@ package collector import "github.com/prometheus/client_golang/prometheus" +// Stopable defines a prometheus collector that can be stopped +type Stopable interface { + prometheus.Collector + Stop() +} + type scrapeRequest struct { results chan<- prometheus.Metric done chan struct{} diff --git a/controllers/nginx/pkg/metric/collector/status.go b/controllers/nginx/pkg/metric/collector/status.go index 7bd0ecf52..1a0fcaf0e 100644 --- a/controllers/nginx/pkg/metric/collector/status.go +++ b/controllers/nginx/pkg/metric/collector/status.go @@ -73,14 +73,14 @@ type serverZone struct { InBytes float64 `json:"inBytes"` OutBytes float64 `json:"outBytes"` Responses response `json:"responses"` - Cache cache `json:"responses"` + Cache cache `json:"cache"` } type filterZone struct { RequestCounter float64 `json:"requestCounter"` InBytes float64 `json:"inBytes"` OutBytes float64 `json:"outBytes"` - Cache cache `json:"responses"` + Cache cache `json:"cache"` Responses response `json:"responses"` } @@ -127,8 +127,10 @@ type connections struct { Requests float64 `json:"requests"` } +// BoolToFloat64 ... type BoolToFloat64 float64 +// UnmarshalJSON ... func (bit BoolToFloat64) UnmarshalJSON(data []byte) error { asString := string(data) if asString == "1" || asString == "true" { @@ -141,7 +143,7 @@ func (bit BoolToFloat64) UnmarshalJSON(data []byte) error { return nil } -func getNginxStatus() (*basicStatus, error) { +func getNginxStatus(ngxHealthPort int, ngxStatusPath string) (*basicStatus, error) { url := fmt.Sprintf("http://localhost:%v%v", ngxHealthPort, ngxStatusPath) glog.V(3).Infof("start scrapping url: %v", url) @@ -172,7 +174,7 @@ func httpBody(url string) ([]byte, error) { return data, nil } -func getNginxVtsMetrics() (*vts, error) { +func getNginxVtsMetrics(ngxHealthPort int, ngxVtsPath string) (*vts, error) { url := fmt.Sprintf("http://localhost:%v%v", ngxHealthPort, ngxVtsPath) glog.V(3).Infof("start scrapping url: %v", url) diff --git a/controllers/nginx/pkg/metric/collector/status_test.go b/controllers/nginx/pkg/metric/collector/status_test.go index b6693d9c9..5d3075dae 100644 --- a/controllers/nginx/pkg/metric/collector/status_test.go +++ b/controllers/nginx/pkg/metric/collector/status_test.go @@ -17,32 +17,34 @@ limitations under the License. package collector import ( - "reflect" "testing" + + "github.com/kylelemons/godebug/pretty" ) func TestParseStatus(t *testing.T) { tests := []struct { in string - out *nginxStatus + out *basicStatus }{ {`Active connections: 43 server accepts handled requests 7368 7368 10993 Reading: 0 Writing: 5 Waiting: 38`, - &nginxStatus{43, 7368, 7368, 10993, 0, 5, 38}, + &basicStatus{43, 7368, 7368, 10993, 0, 5, 38}, }, {`Active connections: 0 server accepts handled requests 1 7 0 Reading: A Writing: B Waiting: 38`, - &nginxStatus{0, 1, 7, 0, 0, 0, 38}, + &basicStatus{0, 1, 7, 0, 0, 0, 38}, }, } for _, test := range tests { r := parse(test.in) - if !reflect.DeepEqual(r, test.out) { + if diff := pretty.Compare(r, test.out); diff != "" { + t.Logf("%v", diff) t.Fatalf("expected %v but returned %v", test.out, r) } } diff --git a/controllers/nginx/pkg/metric/collector/vts.go b/controllers/nginx/pkg/metric/collector/vts.go index a47531f65..4d80d66c4 100644 --- a/controllers/nginx/pkg/metric/collector/vts.go +++ b/controllers/nginx/pkg/metric/collector/vts.go @@ -23,136 +23,168 @@ import ( "github.com/prometheus/client_golang/prometheus" ) -var ( - vtsBytesDesc = prometheus.NewDesc( - "nginx_vts_bytes_total", - "Nginx bytes count", - []string{"server_zone", "direction"}, nil) +const system = "nginx" - vtsCacheDesc = prometheus.NewDesc( - "nginx_vts_cache_total", - "Nginx cache count", - []string{"server_zone", "type"}, nil) - - vtsConnectionsDesc = prometheus.NewDesc( - "nginx_vts_connections_total", - "Nginx connections count", - []string{"type"}, nil) - - vtsResponseDesc = prometheus.NewDesc( - "nginx_vts_responses_total", - "The number of responses with status codes 1xx, 2xx, 3xx, 4xx, and 5xx.", - []string{"server_zone", "status_code"}, nil) - - vtsRequestDesc = prometheus.NewDesc( - "nginx_vts_requests_total", - "The total number of requested client connections.", - []string{"server_zone"}, nil) - - vtsFilterZoneBytesDesc = prometheus.NewDesc( - "nginx_vts_filterzone_bytes_total", - "Nginx bytes count", - []string{"server_zone", "country", "direction"}, nil) - - vtsFilterZoneResponseDesc = prometheus.NewDesc( - "nginx_vts_filterzone_responses_total", - "The number of responses with status codes 1xx, 2xx, 3xx, 4xx, and 5xx.", - []string{"server_zone", "country", "status_code"}, nil) - - vtsFilterZoneCacheDesc = prometheus.NewDesc( - "nginx_vts_filterzone_cache_total", - "Nginx cache count", - []string{"server_zone", "country", "type"}, nil) - - vtsUpstreamBackupDesc = prometheus.NewDesc( - "nginx_vts_upstream_backup", - "Current backup setting of the server.", - []string{"upstream", "server"}, nil) - - vtsUpstreamBytesDesc = prometheus.NewDesc( - "nginx_vts_upstream_bytes_total", - "The total number of bytes sent to this server.", - []string{"upstream", "server", "direction"}, nil) - - vtsUpstreamDownDesc = prometheus.NewDesc( - "nginx_vts_upstream_down_total", - "Current down setting of the server.", - []string{"upstream", "server"}, nil) - - vtsUpstreamFailTimeoutDesc = prometheus.NewDesc( - "nginx_vts_upstream_fail_timeout", - "Current fail_timeout setting of the server.", - []string{"upstream", "server"}, nil) - - vtsUpstreamMaxFailsDesc = prometheus.NewDesc( - "nginx_vts_upstream_maxfails", - "Current max_fails setting of the server.", - []string{"upstream", "server"}, nil) - - vtsUpstreamResponsesDesc = prometheus.NewDesc( - "nginx_vts_upstream_responses_total", - "The number of upstream responses with status codes 1xx, 2xx, 3xx, 4xx, and 5xx.", - []string{"upstream", "server", "status_code"}, nil) - - vtsUpstreamRequestDesc = prometheus.NewDesc( - "nginx_vts_upstream_requests_total", - "The total number of client connections forwarded to this server.", - []string{"upstream", "server"}, nil) - - vtsUpstreamResponseMsecDesc = prometheus.NewDesc( - "nginx_vts_upstream_response_msecs_avg", - "The average of only upstream response processing times in milliseconds.", - []string{"upstream", "server"}, nil) - - vtsUpstreamWeightDesc = prometheus.NewDesc( - "nginx_vts_upstream_weight", - "Current upstream weight setting of the server.", - []string{"upstream", "server"}, nil) -) - -type vtsCollector struct { - scrapeChan chan scrapeRequest +type ( + vtsCollector struct { + scrapeChan chan scrapeRequest + ngxHealthPort int + ngxVtsPath string + data *vtsData } -func NewNGINXVTSCollector() (prometheus.Collector, error) { + vtsData struct { + bytes *prometheus.Desc + cache *prometheus.Desc + connections *prometheus.Desc + response *prometheus.Desc + request *prometheus.Desc + filterZoneBytes *prometheus.Desc + filterZoneResponse *prometheus.Desc + filterZoneCache *prometheus.Desc + upstreamBackup *prometheus.Desc + upstreamBytes *prometheus.Desc + upstreamDown *prometheus.Desc + upstreamFailTimeout *prometheus.Desc + upstreamMaxFails *prometheus.Desc + upstreamResponses *prometheus.Desc + upstreamRequest *prometheus.Desc + upstreamResponseMsec *prometheus.Desc + upstreamWeight *prometheus.Desc + } +) + +// NewNGINXVTSCollector returns a new prometheus collector for the VTS module +func NewNGINXVTSCollector(namespace, class string, ngxHealthPort int, ngxVtsPath string) Stopable { p := vtsCollector{ - scrapeChan: make(chan scrapeRequest), + scrapeChan: make(chan scrapeRequest), + ngxHealthPort: ngxHealthPort, + ngxVtsPath: ngxVtsPath, + } + + ns := buildNS(namespace, class) + + p.data = &vtsData{ + bytes: prometheus.NewDesc( + prometheus.BuildFQName(system, ns, "bytes_total"), + "Nginx bytes count", + []string{"server_zone", "direction"}, nil), + + cache: prometheus.NewDesc( + prometheus.BuildFQName(system, ns, "cache_total"), + "Nginx cache count", + []string{"server_zone", "type"}, nil), + + connections: prometheus.NewDesc( + prometheus.BuildFQName(system, ns, "connections_total"), + "Nginx connections count", + []string{"type"}, nil), + + response: prometheus.NewDesc( + prometheus.BuildFQName(system, ns, "responses_total"), + "The number of responses with status codes 1xx, 2xx, 3xx, 4xx, and 5xx.", + []string{"server_zone", "status_code"}, nil), + + request: prometheus.NewDesc( + prometheus.BuildFQName(system, ns, "requests_total"), + "The total number of requested client connections.", + []string{"server_zone"}, nil), + + filterZoneBytes: prometheus.NewDesc( + prometheus.BuildFQName(system, ns, "filterzone_bytes_total"), + "Nginx bytes count", + []string{"server_zone", "country", "direction"}, nil), + + filterZoneResponse: prometheus.NewDesc( + prometheus.BuildFQName(system, ns, "filterzone_responses_total"), + "The number of responses with status codes 1xx, 2xx, 3xx, 4xx, and 5xx.", + []string{"server_zone", "country", "status_code"}, nil), + + filterZoneCache: prometheus.NewDesc( + prometheus.BuildFQName(system, ns, "filterzone_cache_total"), + "Nginx cache count", + []string{"server_zone", "country", "type"}, nil), + + upstreamBackup: prometheus.NewDesc( + prometheus.BuildFQName(system, ns, "upstream_backup"), + "Current backup setting of the server.", + []string{"upstream", "server"}, nil), + + upstreamBytes: prometheus.NewDesc( + prometheus.BuildFQName(system, ns, "upstream_bytes_total"), + "The total number of bytes sent to this server.", + []string{"upstream", "server", "direction"}, nil), + + upstreamDown: prometheus.NewDesc( + prometheus.BuildFQName(system, ns, "vts_upstream_down_total"), + "Current down setting of the server.", + []string{"upstream", "server"}, nil), + + upstreamFailTimeout: prometheus.NewDesc( + prometheus.BuildFQName(system, ns, "upstream_fail_timeout"), + "Current fail_timeout setting of the server.", + []string{"upstream", "server"}, nil), + + upstreamMaxFails: prometheus.NewDesc( + prometheus.BuildFQName(system, ns, "upstream_maxfails"), + "Current max_fails setting of the server.", + []string{"upstream", "server"}, nil), + + upstreamResponses: prometheus.NewDesc( + prometheus.BuildFQName(system, ns, "upstream_responses_total"), + "The number of upstream responses with status codes 1xx, 2xx, 3xx, 4xx, and 5xx.", + []string{"upstream", "server", "status_code"}, nil), + + upstreamRequest: prometheus.NewDesc( + prometheus.BuildFQName(system, ns, "upstream_requests_total"), + "The total number of client connections forwarded to this server.", + []string{"upstream", "server"}, nil), + + upstreamResponseMsec: prometheus.NewDesc( + prometheus.BuildFQName(system, ns, "upstream_response_msecs_avg"), + "The average of only upstream response processing times in milliseconds.", + []string{"upstream", "server"}, nil), + + upstreamWeight: prometheus.NewDesc( + prometheus.BuildFQName(system, ns, "upstream_weight"), + "Current upstream weight setting of the server.", + []string{"upstream", "server"}, nil), } go p.start() - return p, nil + return p } // Describe implements prometheus.Collector. -func (p *vtsCollector) Describe(ch chan<- *prometheus.Desc) { - ch <- vtsBytesDesc - ch <- vtsCacheDesc - ch <- vtsConnectionsDesc - ch <- vtsRequestDesc - ch <- vtsResponseDesc - ch <- vtsUpstreamBackupDesc - ch <- vtsUpstreamBytesDesc - ch <- vtsUpstreamDownDesc - ch <- vtsUpstreamFailTimeoutDesc - ch <- vtsUpstreamMaxFailsDesc - ch <- vtsUpstreamRequestDesc - ch <- vtsUpstreamResponseMsecDesc - ch <- vtsUpstreamResponsesDesc - ch <- vtsUpstreamWeightDesc - ch <- vtsFilterZoneBytesDesc - ch <- vtsFilterZoneCacheDesc - ch <- vtsFilterZoneResponseDesc +func (p vtsCollector) Describe(ch chan<- *prometheus.Desc) { + ch <- p.data.bytes + ch <- p.data.cache + ch <- p.data.connections + ch <- p.data.request + ch <- p.data.response + ch <- p.data.upstreamBackup + ch <- p.data.upstreamBytes + ch <- p.data.upstreamDown + ch <- p.data.upstreamFailTimeout + ch <- p.data.upstreamMaxFails + ch <- p.data.upstreamRequest + ch <- p.data.upstreamResponseMsec + ch <- p.data.upstreamResponses + ch <- p.data.upstreamWeight + ch <- p.data.filterZoneBytes + ch <- p.data.filterZoneCache + ch <- p.data.filterZoneResponse } // Collect implements prometheus.Collector. -func (p *vtsCollector) Collect(ch chan<- prometheus.Metric) { +func (p vtsCollector) Collect(ch chan<- prometheus.Metric) { req := scrapeRequest{results: ch, done: make(chan struct{})} p.scrapeChan <- req <-req.done } -func (p *vtsCollector) start() { +func (p vtsCollector) start() { for req := range p.scrapeChan { ch := req.results p.scrapeVts(ch) @@ -160,65 +192,65 @@ func (p *vtsCollector) start() { } } -func (p *vtsCollector) Stop() { +func (p vtsCollector) Stop() { close(p.scrapeChan) } // scrapeVts scrape nginx vts metrics -func (p *vtsCollector) scrapeVts(ch chan<- prometheus.Metric) { - nginxMetrics, err := getNginxVtsMetrics() +func (p vtsCollector) scrapeVts(ch chan<- prometheus.Metric) { + nginxMetrics, err := getNginxVtsMetrics(p.ngxHealthPort, p.ngxVtsPath) if err != nil { glog.Warningf("unexpected error obtaining nginx status info: %v", err) return } - reflectMetrics(&nginxMetrics.Connections, vtsConnectionsDesc, ch) + reflectMetrics(&nginxMetrics.Connections, p.data.connections, ch) for name, zones := range nginxMetrics.UpstreamZones { for pos, value := range zones { - reflectMetrics(&zones[pos].Responses, vtsUpstreamResponsesDesc, ch, name, value.Server) + reflectMetrics(&zones[pos].Responses, p.data.upstreamResponses, ch, name, value.Server) - ch <- prometheus.MustNewConstMetric(vtsUpstreamRequestDesc, - prometheus.CounterValue, float64(zones[pos].RequestCounter), name, value.Server) - ch <- prometheus.MustNewConstMetric(vtsUpstreamDownDesc, + ch <- prometheus.MustNewConstMetric(p.data.upstreamRequest, + prometheus.CounterValue, zones[pos].RequestCounter, name, value.Server) + ch <- prometheus.MustNewConstMetric(p.data.upstreamDown, prometheus.CounterValue, float64(zones[pos].Down), name, value.Server) - ch <- prometheus.MustNewConstMetric(vtsUpstreamWeightDesc, - prometheus.CounterValue, float64(zones[pos].Weight), name, value.Server) - ch <- prometheus.MustNewConstMetric(vtsUpstreamResponseMsecDesc, - prometheus.CounterValue, float64(zones[pos].ResponseMsec), name, value.Server) - ch <- prometheus.MustNewConstMetric(vtsUpstreamBackupDesc, + ch <- prometheus.MustNewConstMetric(p.data.upstreamWeight, + prometheus.CounterValue, zones[pos].Weight, name, value.Server) + ch <- prometheus.MustNewConstMetric(p.data.upstreamResponseMsec, + prometheus.CounterValue, zones[pos].ResponseMsec, name, value.Server) + ch <- prometheus.MustNewConstMetric(p.data.upstreamBackup, prometheus.CounterValue, float64(zones[pos].Backup), name, value.Server) - ch <- prometheus.MustNewConstMetric(vtsUpstreamFailTimeoutDesc, - prometheus.CounterValue, float64(zones[pos].FailTimeout), name, value.Server) - ch <- prometheus.MustNewConstMetric(vtsUpstreamMaxFailsDesc, - prometheus.CounterValue, float64(zones[pos].MaxFails), name, value.Server) - ch <- prometheus.MustNewConstMetric(vtsUpstreamBytesDesc, - prometheus.CounterValue, float64(zones[pos].InBytes), name, value.Server, "in") - ch <- prometheus.MustNewConstMetric(vtsUpstreamBytesDesc, - prometheus.CounterValue, float64(zones[pos].OutBytes), name, value.Server, "out") + ch <- prometheus.MustNewConstMetric(p.data.upstreamFailTimeout, + prometheus.CounterValue, zones[pos].FailTimeout, name, value.Server) + ch <- prometheus.MustNewConstMetric(p.data.upstreamMaxFails, + prometheus.CounterValue, zones[pos].MaxFails, name, value.Server) + ch <- prometheus.MustNewConstMetric(p.data.upstreamBytes, + prometheus.CounterValue, zones[pos].InBytes, name, value.Server, "in") + ch <- prometheus.MustNewConstMetric(p.data.upstreamBytes, + prometheus.CounterValue, zones[pos].OutBytes, name, value.Server, "out") } } for name, zone := range nginxMetrics.ServerZones { - reflectMetrics(&zone.Responses, vtsResponseDesc, ch, name) - reflectMetrics(&zone.Cache, vtsCacheDesc, ch, name) + reflectMetrics(&zone.Responses, p.data.response, ch, name) + reflectMetrics(&zone.Cache, p.data.cache, ch, name) - ch <- prometheus.MustNewConstMetric(vtsRequestDesc, - prometheus.CounterValue, float64(zone.RequestCounter), name) - ch <- prometheus.MustNewConstMetric(vtsBytesDesc, - prometheus.CounterValue, float64(zone.InBytes), name, "in") - ch <- prometheus.MustNewConstMetric(vtsBytesDesc, - prometheus.CounterValue, float64(zone.OutBytes), name, "out") + ch <- prometheus.MustNewConstMetric(p.data.request, + prometheus.CounterValue, zone.RequestCounter, name) + ch <- prometheus.MustNewConstMetric(p.data.bytes, + prometheus.CounterValue, zone.InBytes, name, "in") + ch <- prometheus.MustNewConstMetric(p.data.bytes, + prometheus.CounterValue, zone.OutBytes, name, "out") } for serverZone, countries := range nginxMetrics.FilterZones { for country, zone := range countries { - reflectMetrics(&zone.Responses, vtsFilterZoneResponseDesc, ch, serverZone, country) - reflectMetrics(&zone.Cache, vtsFilterZoneCacheDesc, ch, serverZone, country) + reflectMetrics(&zone.Responses, p.data.filterZoneResponse, ch, serverZone, country) + reflectMetrics(&zone.Cache, p.data.filterZoneCache, ch, serverZone, country) - ch <- prometheus.MustNewConstMetric(vtsFilterZoneBytesDesc, + ch <- prometheus.MustNewConstMetric(p.data.filterZoneBytes, prometheus.CounterValue, float64(zone.InBytes), serverZone, country, "in") - ch <- prometheus.MustNewConstMetric(vtsFilterZoneBytesDesc, + ch <- prometheus.MustNewConstMetric(p.data.filterZoneBytes, prometheus.CounterValue, float64(zone.OutBytes), serverZone, country, "out") } } @@ -229,9 +261,9 @@ func reflectMetrics(value interface{}, desc *prometheus.Desc, ch chan<- promethe for i := 0; i < val.NumField(); i++ { tag := val.Type().Field(i).Tag - labels := append(labels, tag.Get("json")) + l := append(labels, tag.Get("json")) ch <- prometheus.MustNewConstMetric(desc, prometheus.CounterValue, float64(val.Field(i).Interface().(float64)), - labels...) + l...) } } diff --git a/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl b/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl index 9813bcd24..07a7e7921 100644 --- a/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl +++ b/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl @@ -240,6 +240,8 @@ http { {{ if not (empty $authPath) }} location = {{ $authPath }} { internal; + set $proxy_upstream_name "internal"; + {{ if not $location.ExternalAuth.SendBody }} proxy_pass_request_body off; proxy_set_header Content-Length ""; @@ -402,6 +404,8 @@ http { } location /nginx_status { + set $proxy_upstream_name "internal"; + {{ if $cfg.EnableVtsStatus }} vhost_traffic_status_display; vhost_traffic_status_display_format html; @@ -415,6 +419,8 @@ http { # using prometheus. # TODO: enable extraction for vts module. location /internal_nginx_status { + set $proxy_upstream_name "internal"; + allow 127.0.0.1; {{ if not $cfg.DisableIpv6 }}allow ::1;{{ end }} deny all;