From 53a6b0fd3bd14773cd2e835e44c1692235584581 Mon Sep 17 00:00:00 2001
From: Bo0km4n <pokopenkatuya@gmail.com>
Date: Sat, 20 Jun 2020 15:58:14 +0900
Subject: [PATCH] Configurable metrics max batch size

---
 cmd/nginx/flags.go                           |  2 ++
 internal/ingress/controller/config/config.go |  1 +
 internal/ingress/controller/controller.go    |  2 ++
 internal/ingress/controller/nginx.go         |  1 +
 rootfs/etc/nginx/lua/monitor.lua             | 10 +++++++++-
 rootfs/etc/nginx/lua/test/monitor_test.lua   | 12 ++++++++++++
 rootfs/etc/nginx/template/nginx.tmpl         |  2 +-
 7 files changed, 28 insertions(+), 2 deletions(-)

diff --git a/cmd/nginx/flags.go b/cmd/nginx/flags.go
index 1466f07d7..a43b1dd6b 100644
--- a/cmd/nginx/flags.go
+++ b/cmd/nginx/flags.go
@@ -145,6 +145,7 @@ Requires the update-status parameter.`)
 			`Enables the collection of NGINX metrics`)
 		metricsPerHost = flags.Bool("metrics-per-host", true,
 			`Export metrics per-host`)
+		monitorMaxBatchSize = flags.Int("monitor-max-batch-size", 10000, "Max batch size of NGINX metrics")
 
 		httpPort  = flags.Int("http-port", 80, `Port to use for servicing HTTP traffic.`)
 		httpsPort = flags.Int("https-port", 443, `Port to use for servicing HTTPS traffic.`)
@@ -269,6 +270,7 @@ https://blog.maxmind.com/2019/12/18/significant-changes-to-accessing-and-using-g
 		EnableProfiling:        *profiling,
 		EnableMetrics:          *enableMetrics,
 		MetricsPerHost:         *metricsPerHost,
+		MonitorMaxBatchSize:    *monitorMaxBatchSize,
 		EnableSSLPassthrough:   *enableSSLPassthrough,
 		ResyncPeriod:           *resyncPeriod,
 		DefaultService:         *defaultSvc,
diff --git a/internal/ingress/controller/config/config.go b/internal/ingress/controller/config/config.go
index b39145957..65e20af7e 100644
--- a/internal/ingress/controller/config/config.go
+++ b/internal/ingress/controller/config/config.go
@@ -836,6 +836,7 @@ type TemplateConfig struct {
 	PublishService           *apiv1.Service
 	EnableMetrics            bool
 	MaxmindEditionFiles      []string
+	MonitorMaxBatchSize      int
 
 	PID        string
 	StatusPath string
diff --git a/internal/ingress/controller/controller.go b/internal/ingress/controller/controller.go
index b46b95c92..8ce417a55 100644
--- a/internal/ingress/controller/controller.go
+++ b/internal/ingress/controller/controller.go
@@ -101,6 +101,8 @@ type Configuration struct {
 
 	GlobalExternalAuth  *ngx_config.GlobalExternalAuth
 	MaxmindEditionFiles []string
+
+	MonitorMaxBatchSize int
 }
 
 // GetPublishService returns the Service used to set the load-balancer status of Ingresses.
diff --git a/internal/ingress/controller/nginx.go b/internal/ingress/controller/nginx.go
index cec908065..d697e0c51 100644
--- a/internal/ingress/controller/nginx.go
+++ b/internal/ingress/controller/nginx.go
@@ -617,6 +617,7 @@ func (n NGINXController) generateTemplate(cfg ngx_config.Configuration, ingressC
 		EnableMetrics:            n.cfg.EnableMetrics,
 		MaxmindEditionFiles:      n.cfg.MaxmindEditionFiles,
 		HealthzURI:               nginx.HealthPath,
+		MonitorMaxBatchSize:      n.cfg.MonitorMaxBatchSize,
 		PID:                      nginx.PID,
 		StatusPath:               nginx.StatusPath,
 		StatusPort:               nginx.StatusPort,
diff --git a/rootfs/etc/nginx/lua/monitor.lua b/rootfs/etc/nginx/lua/monitor.lua
index 014113b44..7592c0c5b 100644
--- a/rootfs/etc/nginx/lua/monitor.lua
+++ b/rootfs/etc/nginx/lua/monitor.lua
@@ -63,7 +63,14 @@ local function flush(premature)
   send(payload)
 end
 
-function _M.init_worker()
+local function set_metrics_max_batch_size(max_batch_size)
+  if max_batch_size > 10000 then
+    MAX_BATCH_SIZE = max_batch_size
+  end
+end
+
+function _M.init_worker(max_batch_size)
+  set_metrics_max_batch_size(max_batch_size)
   local _, err = ngx.timer.every(FLUSH_INTERVAL, flush)
   if err then
     ngx.log(ngx.ERR, string.format("error when setting up timer.every: %s", tostring(err)))
@@ -83,6 +90,7 @@ end
 if _TEST then
   _M.flush = flush
   _M.get_metrics_batch = function() return metrics_batch end
+  _M.set_metrics_max_batch_size = set_metrics_max_batch_size
 end
 
 return _M
diff --git a/rootfs/etc/nginx/lua/test/monitor_test.lua b/rootfs/etc/nginx/lua/test/monitor_test.lua
index 8796d0a62..19a576c33 100644
--- a/rootfs/etc/nginx/lua/test/monitor_test.lua
+++ b/rootfs/etc/nginx/lua/test/monitor_test.lua
@@ -30,6 +30,18 @@ describe("Monitor", function()
     package.loaded["monitor"] = nil
   end)
 
+  it("extended batch size", function()
+    local monitor = require("monitor")
+    mock_ngx({ var = {} })
+    monitor.set_metrics_max_batch_size(20000)
+
+    for i = 1,20000,1 do
+      monitor.call()
+    end
+
+    assert.equal(20000, #monitor.get_metrics_batch())
+  end)
+
   it("batches metrics", function()
     local monitor = require("monitor")
     mock_ngx({ var = {} })
diff --git a/rootfs/etc/nginx/template/nginx.tmpl b/rootfs/etc/nginx/template/nginx.tmpl
index 9cf9cd22d..86a50c683 100755
--- a/rootfs/etc/nginx/template/nginx.tmpl
+++ b/rootfs/etc/nginx/template/nginx.tmpl
@@ -111,7 +111,7 @@ http {
         lua_ingress.init_worker()
         balancer.init_worker()
         {{ if $all.EnableMetrics }}
-        monitor.init_worker()
+        monitor.init_worker({{ $all.MonitorMaxBatchSize }})
         {{ end }}
 
         plugins.run()