DevFW-CICD/ingress-nginx-helm
12
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Live Nginx (re)configuration without reloading (#2174)

Browse source
This commit is contained in:
Elvin Efendi 2018-03-18 09:13:41 -04:00 committed by Manuel Alejandro de Brito Fontes
parent 41cefeb178
commit c90a4e811e
13 changed files with 759 additions and 114 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
Makefile
View file

@ -50,7 +50,7 @@ IMAGE = $(REGISTRY)/$(IMGNAME)
MULTI_ARCH_IMG = $(IMAGE)-$(ARCH)
# Set default base image dynamically for each arch
BASEIMAGE?=quay.io/kubernetes-ingress-controller/nginx-$(ARCH):0.34
BASEIMAGE?=quay.io/kubernetes-ingress-controller/nginx-$(ARCH):0.37
ifeq ($(ARCH),arm)
QEMUARCH=arm

48
cmd/nginx/flags.go
View file

@ -131,6 +131,9 @@ func parseFlags() (bool, *controller.Configuration, error) {
publishStatusAddress = flags.String("publish-status-address", "",
`User customized address to be set in the status of ingress resources. The controller will set the
endpoint records on the ingress using this address.`)
dynamicConfigurationEnabled = flags.Bool("enable-dynamic-configuration", false,
`When enabled controller will try to avoid Nginx reloads as much as possible by using Lua. Disabled by default.`)
)
flag.Set("logtostderr", "true")
@ -192,28 +195,29 @@ func parseFlags() (bool, *controller.Configuration, error) {
}
config := &controller.Configuration{
APIServerHost: *apiserverHost,
KubeConfigFile: *kubeConfigFile,
UpdateStatus: *updateStatus,
ElectionID: *electionID,
EnableProfiling: *profiling,
EnableSSLPassthrough: *enableSSLPassthrough,
EnableSSLChainCompletion: *enableSSLChainCompletion,
ResyncPeriod: *resyncPeriod,
DefaultService: *defaultSvc,
Namespace: *watchNamespace,
ConfigMapName: *configMap,
TCPConfigMapName: *tcpConfigMapName,
UDPConfigMapName: *udpConfigMapName,
DefaultSSLCertificate: *defSSLCertificate,
DefaultHealthzURL: *defHealthzURL,
PublishService: *publishSvc,
PublishStatusAddress: *publishStatusAddress,
ForceNamespaceIsolation: *forceIsolation,
UpdateStatusOnShutdown: *updateStatusOnShutdown,
SortBackends: *sortBackends,
UseNodeInternalIP: *useNodeInternalIP,
SyncRateLimit: *syncRateLimit,
APIServerHost: *apiserverHost,
KubeConfigFile: *kubeConfigFile,
UpdateStatus: *updateStatus,
ElectionID: *electionID,
EnableProfiling: *profiling,
EnableSSLPassthrough: *enableSSLPassthrough,
EnableSSLChainCompletion: *enableSSLChainCompletion,
ResyncPeriod: *resyncPeriod,
DefaultService: *defaultSvc,
Namespace: *watchNamespace,
ConfigMapName: *configMap,
TCPConfigMapName: *tcpConfigMapName,
UDPConfigMapName: *udpConfigMapName,
DefaultSSLCertificate: *defSSLCertificate,
DefaultHealthzURL: *defHealthzURL,
PublishService: *publishSvc,
PublishStatusAddress: *publishStatusAddress,
ForceNamespaceIsolation: *forceIsolation,
UpdateStatusOnShutdown: *updateStatusOnShutdown,
SortBackends: *sortBackends,
UseNodeInternalIP: *useNodeInternalIP,
SyncRateLimit: *syncRateLimit,
DynamicConfigurationEnabled: *dynamicConfigurationEnabled,
ListenPorts: &ngx_config.ListenPorts{
Default: *defServerPort,
Health: *healthzPort,

79
internal/file/bindata.go
View file

File diff suppressed because one or more lines are too long

35
internal/ingress/controller/config/config.go
View file

@ -604,23 +604,24 @@ func (cfg Configuration) BuildLogFormatUpstream() string {
// TemplateConfig contains the nginx configuration to render the file nginx.conf
type TemplateConfig struct {
ProxySetHeaders map[string]string
AddHeaders map[string]string
MaxOpenFiles int
BacklogSize int
Backends []*ingress.Backend
PassthroughBackends []*ingress.SSLPassthroughBackend
Servers []*ingress.Server
TCPBackends []ingress.L4Service
UDPBackends []ingress.L4Service
HealthzURI string
CustomErrors bool
Cfg Configuration
IsIPV6Enabled bool
IsSSLPassthroughEnabled bool
RedirectServers map[string]string
ListenPorts *ListenPorts
PublishService *apiv1.Service
ProxySetHeaders map[string]string
AddHeaders map[string]string
MaxOpenFiles int
BacklogSize int
Backends []*ingress.Backend
PassthroughBackends []*ingress.SSLPassthroughBackend
Servers []*ingress.Server
TCPBackends []ingress.L4Service
UDPBackends []ingress.L4Service
HealthzURI string
CustomErrors bool
Cfg Configuration
IsIPV6Enabled bool
IsSSLPassthroughEnabled bool
RedirectServers map[string]string
ListenPorts *ListenPorts
PublishService *apiv1.Service
DynamicConfigurationEnabled bool
}
// ListenPorts describe the ports required to run the

24
internal/ingress/controller/controller.go
View file

@ -95,6 +95,8 @@ type Configuration struct {
FakeCertificateSHA string
SyncRateLimit float32
DynamicConfigurationEnabled bool
}
// GetPublishService returns the configured service used to set ingress status
@ -167,6 +169,15 @@ func (n *NGINXController) syncIngress(item interface{}) error {
if !n.isForceReload() && n.runningConfig.Equal(&pcfg) {
glog.V(3).Infof("skipping backend reload (no changes detected)")
return nil
} else if !n.isForceReload() && n.cfg.DynamicConfigurationEnabled && n.IsDynamicallyConfigurable(&pcfg) {
err := n.ConfigureDynamically(&pcfg)
if err == nil {
glog.Infof("dynamic reconfiguration succeeded, skipping reload")
n.runningConfig = &pcfg
return nil
}
glog.Warningf("falling back to reload, could not dynamically reconfigure: %v", err)
}
glog.Infof("backend reload required")
@ -182,6 +193,19 @@ func (n *NGINXController) syncIngress(item interface{}) error {
incReloadCount()
setSSLExpireTime(servers)
if n.isForceReload() && n.cfg.DynamicConfigurationEnabled {
go func() {
// it takes time for Nginx to start listening on the port
time.Sleep(1 * time.Second)
err := n.ConfigureDynamically(&pcfg)
if err == nil {
glog.Infof("dynamic reconfiguration succeeded")
} else {
glog.Warningf("could not dynamically reconfigure: %v", err)
}
}()
}
n.runningConfig = &pcfg
n.SetForceReload(false)

77
internal/ingress/controller/nginx.go
View file

@ -18,10 +18,12 @@ package controller
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"net"
"net/http"
"os"
"os/exec"
"strconv"
@ -606,23 +608,24 @@ func (n *NGINXController) OnUpdate(ingressCfg ingress.Configuration) error {
cfg.SSLDHParam = sslDHParam
tc := ngx_config.TemplateConfig{
ProxySetHeaders: setHeaders,
AddHeaders: addHeaders,
MaxOpenFiles: maxOpenFiles,
BacklogSize: sysctlSomaxconn(),
Backends: ingressCfg.Backends,
PassthroughBackends: ingressCfg.PassthroughBackends,
Servers: ingressCfg.Servers,
TCPBackends: ingressCfg.TCPEndpoints,
UDPBackends: ingressCfg.UDPEndpoints,
HealthzURI: ngxHealthPath,
CustomErrors: len(cfg.CustomHTTPErrors) > 0,
Cfg: cfg,
IsIPV6Enabled: n.isIPV6Enabled && !cfg.DisableIpv6,
RedirectServers: redirectServers,
IsSSLPassthroughEnabled: n.cfg.EnableSSLPassthrough,
ListenPorts: n.cfg.ListenPorts,
PublishService: n.GetPublishService(),
ProxySetHeaders: setHeaders,
AddHeaders: addHeaders,
MaxOpenFiles: maxOpenFiles,
BacklogSize: sysctlSomaxconn(),
Backends: ingressCfg.Backends,
PassthroughBackends: ingressCfg.PassthroughBackends,
Servers: ingressCfg.Servers,
TCPBackends: ingressCfg.TCPEndpoints,
UDPBackends: ingressCfg.UDPEndpoints,
HealthzURI: ngxHealthPath,
CustomErrors: len(cfg.CustomHTTPErrors) > 0,
Cfg: cfg,
IsIPV6Enabled: n.isIPV6Enabled && !cfg.DisableIpv6,
RedirectServers: redirectServers,
IsSSLPassthroughEnabled: n.cfg.EnableSSLPassthrough,
ListenPorts: n.cfg.ListenPorts,
PublishService: n.GetPublishService(),
DynamicConfigurationEnabled: n.cfg.DynamicConfigurationEnabled,
}
content, err := n.t.Write(tc)
@ -745,3 +748,43 @@ func (n *NGINXController) setupSSLProxy() {
}
}()
}
// IsDynamicallyConfigurable decides if the new configuration can be dynamically configured without reloading
func (n *NGINXController) IsDynamicallyConfigurable(pcfg *ingress.Configuration) bool {
var copyOfRunningConfig ingress.Configuration = *n.runningConfig
var copyOfPcfg ingress.Configuration = *pcfg
copyOfRunningConfig.Backends = []*ingress.Backend{}
copyOfPcfg.Backends = []*ingress.Backend{}
return copyOfRunningConfig.Equal(&copyOfPcfg)
}
// ConfigureDynamically JSON encodes new Backends and POSTs it to an internal HTTP endpoint
// that is handled by Lua
func (n *NGINXController) ConfigureDynamically(pcfg *ingress.Configuration) error {
buf, err := json.Marshal(pcfg.Backends)
if err != nil {
return err
}
glog.V(2).Infof("posting backends configuration: %s", buf)
url := fmt.Sprintf("http://localhost:%d/configuration/backends", n.cfg.ListenPorts.Status)
resp, err := http.Post(url, "application/json", bytes.NewReader(buf))
if err != nil {
return err
}
defer func() {
if err := resp.Body.Close(); err != nil {
glog.Warningf("error while closing response body: \n%v", err)
}
}()
if resp.StatusCode != http.StatusCreated {
return fmt.Errorf("Unexpected error code: %d", resp.StatusCode)
}
return nil
}

73
internal/ingress/controller/nginx_test.go
View file

@ -16,7 +16,78 @@ limitations under the License.
package controller
import "testing"
import (
"testing"
"k8s.io/ingress-nginx/internal/ingress"
)
func TestIsDynamicallyConfigurable(t *testing.T) {
backends := []*ingress.Backend{{
Name: "fakenamespace-myapp-80",
Endpoints: []ingress.Endpoint{
{
Address: "10.0.0.1",
Port: "8080",
},
{
Address: "10.0.0.2",
Port: "8080",
},
},
}}
servers := []*ingress.Server{{
Hostname: "myapp.fake",
Locations: []*ingress.Location{
{
Path: "/",
Backend: "fakenamespace-myapp-80",
},
},
}}
commonConfig := &ingress.Configuration{
Backends: backends,
Servers: servers,
}
n := &NGINXController{
runningConfig: &ingress.Configuration{
Backends: backends,
Servers: servers,
},
}
newConfig := commonConfig
if !n.IsDynamicallyConfigurable(newConfig) {
t.Errorf("When new config is same as the running config it should be deemed as dynamically configurable")
}
newConfig = &ingress.Configuration{
Backends: []*ingress.Backend{{Name: "another-backend-8081"}},
Servers: []*ingress.Server{{Hostname: "myapp1.fake"}},
}
if n.IsDynamicallyConfigurable(newConfig) {
t.Errorf("Expected to not be dynamically configurable when there's more than just backends change")
}
newConfig = &ingress.Configuration{
Backends: []*ingress.Backend{{Name: "a-backend-8080"}},
Servers: servers,
}
if !n.IsDynamicallyConfigurable(newConfig) {
t.Errorf("Expected to be dynamically configurable when only backends change")
}
if !n.runningConfig.Equal(commonConfig) {
t.Errorf("Expected running config to not change")
}
if !newConfig.Equal(&ingress.Configuration{Backends: []*ingress.Backend{{Name: "a-backend-8080"}}, Servers: servers}) {
t.Errorf("Expected new config to not change")
}
}
func TestNginxHashBucketSize(t *testing.T) {
tests := []struct {

12
internal/ingress/controller/template/template.go
View file

@ -307,7 +307,7 @@ func buildLoadBalancingConfig(b interface{}, fallbackLoadBalancing string) strin
// (specified through the nginx.ingress.kubernetes.io/rewrite-to annotation)
// If the annotation nginx.ingress.kubernetes.io/add-base-url:"true" is specified it will
// add a base tag in the head of the response from the service
func buildProxyPass(host string, b interface{}, loc interface{}) string {
func buildProxyPass(host string, b interface{}, loc interface{}, dynamicConfigurationEnabled bool) string {
backends, ok := b.([]*ingress.Backend)
if !ok {
glog.Errorf("expected an '[]*ingress.Backend' type but %T was returned", b)
@ -323,14 +323,19 @@ func buildProxyPass(host string, b interface{}, loc interface{}) string {
path := location.Path
proto := "http"
upstreamName := location.Backend
upstreamName := "upstream_balancer"
if !dynamicConfigurationEnabled {
upstreamName = location.Backend
}
for _, backend := range backends {
if backend.Name == location.Backend {
if backend.Secure || backend.SSLPassthrough {
proto = "https"
}
if isSticky(host, location, backend.SessionAffinity.CookieSessionAffinity.Locations) {
if !dynamicConfigurationEnabled && isSticky(host, location, backend.SessionAffinity.CookieSessionAffinity.Locations) {
upstreamName = fmt.Sprintf("sticky-%v", upstreamName)
}
@ -340,6 +345,7 @@ func buildProxyPass(host string, b interface{}, loc interface{}) string {
// defProxyPass returns the default proxy_pass, just the name of the upstream
defProxyPass := fmt.Sprintf("proxy_pass %s://%s;", proto, upstreamName)
// if the path in the ingress rule is equals to the target: no special rewrite
if path == location.Rewrite.Target {
return defProxyPass

279
internal/ingress/controller/template/template_test.go
View file

@ -37,72 +37,262 @@ import (
)
var (
// TODO: add tests for secure endpoints
// TODO: add tests for SSLPassthrough
tmplFuncTestcases = map[string]struct {
Path string
Target string
Location string
ProxyPass string
AddBaseURL bool
BaseURLScheme string
Sticky bool
XForwardedPrefix bool
Path string
Target string
Location string
ProxyPass string
AddBaseURL bool
BaseURLScheme string
Sticky bool
XForwardedPrefix bool
DynamicConfigurationEnabled bool
SecureBackend bool
}{
"invalid redirect / to /": {"/", "/", "/", "proxy_pass http://upstream-name;", false, "", false, false},
"redirect / to /jenkins": {"/", "/jenkins", "~* /",
"when secure backend enabled": {
"/",
"/",
"/",
"proxy_pass https://upstream-name;",
false,
"",
false,
false,
false,
true},
"when secure backend and stickeness enabled": {
"/",
"/",
"/",
"proxy_pass https://sticky-upstream-name;",
false,
"",
true,
false,
false,
true},
"when secure backend and dynamic config enabled": {
"/",
"/",
"/",
"proxy_pass https://upstream_balancer;",
false,
"",
false,
false,
true,
true},
"when secure backend, stickeness and dynamic config enabled": {
"/",
"/",
"/",
"proxy_pass https://upstream_balancer;",
false,
"",
true,
false,
true,
true},
"invalid redirect / to / with dynamic config enabled": {
"/",
"/",
"/",
"proxy_pass http://upstream_balancer;",
false,
"",
false,
false,
true,
false},
"invalid redirect / to /": {
"/",
"/",
"/",
"proxy_pass http://upstream-name;",
false,
"",
false,
false,
false,
false},
"redirect / to /jenkins": {
"/",
"/jenkins",
"~* /",
`
rewrite /(.*) /jenkins/$1 break;
proxy_pass http://upstream-name;
`, false, "", false, false},
"redirect /something to /": {"/something", "/", `~* ^/something\/?(?<baseuri>.*)`, `
`,
false,
"",
false,
false,
false,
false},
"redirect /something to /": {
"/something",
"/",
`~* ^/something\/?(?<baseuri>.*)`,
`
rewrite /something/(.*) /$1 break;
rewrite /something / break;
proxy_pass http://upstream-name;
`, false, "", false, false},
"redirect /end-with-slash/ to /not-root": {"/end-with-slash/", "/not-root", "~* ^/end-with-slash/(?<baseuri>.*)", `
`,
false,
"",
false,
false,
false,
false},
"redirect /end-with-slash/ to /not-root": {
"/end-with-slash/",
"/not-root",
"~* ^/end-with-slash/(?<baseuri>.*)",
`
rewrite /end-with-slash/(.*) /not-root/$1 break;
proxy_pass http://upstream-name;
`, false, "", false, false},
"redirect /something-complex to /not-root": {"/something-complex", "/not-root", `~* ^/something-complex\/?(?<baseuri>.*)`, `
`,
false,
"",
false,
false,
false,
false},
"redirect /something-complex to /not-root": {
"/something-complex",
"/not-root",
`~* ^/something-complex\/?(?<baseuri>.*)`,
`
rewrite /something-complex/(.*) /not-root/$1 break;
proxy_pass http://upstream-name;
`, false, "", false, false},
"redirect / to /jenkins and rewrite": {"/", "/jenkins", "~* /", `
`,
false,
"",
false,
false,
false,
false},
"redirect / to /jenkins and rewrite": {
"/",
"/jenkins",
"~* /",
`
rewrite /(.*) /jenkins/$1 break;
proxy_pass http://upstream-name;
subs_filter '(<(?:H|h)(?:E|e)(?:A|a)(?:D|d)(?:[^">]|"[^"]*")*>)' '$1<base href="$scheme://$http_host/$baseuri">' ro;
`, true, "", false, false},
"redirect /something to / and rewrite": {"/something", "/", `~* ^/something\/?(?<baseuri>.*)`, `
`,
true,
"",
false,
false,
false,
false},
"redirect /something to / and rewrite": {
"/something",
"/",
`~* ^/something\/?(?<baseuri>.*)`,
`
rewrite /something/(.*) /$1 break;
rewrite /something / break;
proxy_pass http://upstream-name;
subs_filter '(<(?:H|h)(?:E|e)(?:A|a)(?:D|d)(?:[^">]|"[^"]*")*>)' '$1<base href="$scheme://$http_host/something/$baseuri">' ro;
`, true, "", false, false},
"redirect /end-with-slash/ to /not-root and rewrite": {"/end-with-slash/", "/not-root", `~* ^/end-with-slash/(?<baseuri>.*)`, `
`,
true,
"",
false,
false,
false,
false},
"redirect /end-with-slash/ to /not-root and rewrite": {
"/end-with-slash/",
"/not-root",
`~* ^/end-with-slash/(?<baseuri>.*)`,
`
rewrite /end-with-slash/(.*) /not-root/$1 break;
proxy_pass http://upstream-name;
subs_filter '(<(?:H|h)(?:E|e)(?:A|a)(?:D|d)(?:[^">]|"[^"]*")*>)' '$1<base href="$scheme://$http_host/end-with-slash/$baseuri">' ro;
`, true, "", false, false},
"redirect /something-complex to /not-root and rewrite": {"/something-complex", "/not-root", `~* ^/something-complex\/?(?<baseuri>.*)`, `
`,
true,
"",
false,
false,
false,
false},
"redirect /something-complex to /not-root and rewrite": {
"/something-complex",
"/not-root",
`~* ^/something-complex\/?(?<baseuri>.*)`,
`
rewrite /something-complex/(.*) /not-root/$1 break;
proxy_pass http://upstream-name;
subs_filter '(<(?:H|h)(?:E|e)(?:A|a)(?:D|d)(?:[^">]|"[^"]*")*>)' '$1<base href="$scheme://$http_host/something-complex/$baseuri">' ro;
`, true, "", false, false},
"redirect /something to / and rewrite with specific scheme": {"/something", "/", `~* ^/something\/?(?<baseuri>.*)`, `
`,
true,
"",
false,
false,
false,
false},
"redirect /something to / and rewrite with specific scheme": {
"/something",
"/",
`~* ^/something\/?(?<baseuri>.*)`,
`
rewrite /something/(.*) /$1 break;
rewrite /something / break;
proxy_pass http://upstream-name;
subs_filter '(<(?:H|h)(?:E|e)(?:A|a)(?:D|d)(?:[^">]|"[^"]*")*>)' '$1<base href="http://$http_host/something/$baseuri">' ro;
`, true, "http", false, false},
"redirect / to /something with sticky enabled": {"/", "/something", `~* /`, `
`,
true,
"http",
false,
false,
false,
false},
"redirect / to /something with sticky enabled": {
"/",
"/something",
`~* /`,
`
rewrite /(.*) /something/$1 break;
proxy_pass http://sticky-upstream-name;
`, false, "http", true, false},
"add the X-Forwarded-Prefix header": {"/there", "/something", `~* ^/there\/?(?<baseuri>.*)`, `
`,
false,
"http",
true,
false,
false,
false},
"redirect / to /something with sticky and dynamic config enabled": {
"/",
"/something",
`~* /`,
`
rewrite /(.*) /something/$1 break;
proxy_pass http://upstream_balancer;
`,
false,
"http",
true,
false,
true,
false},
"add the X-Forwarded-Prefix header": {
"/there",
"/something",
`~* ^/there\/?(?<baseuri>.*)`,
`
rewrite /there/(.*) /something/$1 break;
proxy_set_header X-Forwarded-Prefix "/there/";
proxy_pass http://sticky-upstream-name;
`, false, "http", true, true},
`,
false,
"http",
true,
true,
false,
false},
}
)
@ -151,24 +341,25 @@ func TestBuildProxyPass(t *testing.T) {
XForwardedPrefix: tc.XForwardedPrefix,
}
backends := []*ingress.Backend{}
backend := &ingress.Backend{
Name: defaultBackend,
Secure: tc.SecureBackend,
}
if tc.Sticky {
backends = []*ingress.Backend{
{
Name: defaultBackend,
SessionAffinity: ingress.SessionAffinityConfig{
AffinityType: "cookie",
CookieSessionAffinity: ingress.CookieSessionAffinity{
Locations: map[string][]string{
defaultHost: {tc.Path},
},
},
backend.SessionAffinity = ingress.SessionAffinityConfig{
AffinityType: "cookie",
CookieSessionAffinity: ingress.CookieSessionAffinity{
Locations: map[string][]string{
defaultHost: {tc.Path},
},
},
}
}
pp := buildProxyPass(defaultHost, backends, loc)
backends := []*ingress.Backend{backend}
pp := buildProxyPass(defaultHost, backends, loc, tc.DynamicConfigurationEnabled)
if !strings.EqualFold(tc.ProxyPass, pp) {
t.Errorf("%s: expected \n'%v'\nbut returned \n'%v'", k, tc.ProxyPass, pp)
}

107
rootfs/etc/nginx/lua/balancer.lua Normal file
View file

@ -0,0 +1,107 @@
local ngx_balancer = require("ngx.balancer")
local json = require("cjson")
local configuration = require("configuration")
local util = require("util")
local lrucache = require("resty.lrucache")
local resty_lock = require("resty.lock")
-- measured in seconds
-- for an Nginx worker to pick up the new list of upstream peers
-- it will take <the delay until controller POSTed the backend object to the Nginx endpoint> + BACKENDS_SYNC_INTERVAL
local BACKENDS_SYNC_INTERVAL = 1
ROUND_ROBIN_LOCK_KEY = "round_robin"
local round_robin_state = ngx.shared.round_robin_state
local _M = {}
local round_robin_lock = resty_lock:new("locks", {timeout = 0, exptime = 0.1})
local backends, err = lrucache.new(1024)
if not backends then
return error("failed to create the cache for backends: " .. (err or "unknown"))
end
local function balance()
local backend_name = ngx.var.proxy_upstream_name
local backend = backends:get(backend_name)
-- lb_alg field does not exist for ingress.Backend struct for now, so lb_alg
-- will always be round_robin
local lb_alg = backend.lb_alg or "round_robin"
if lb_alg == "ip_hash" then
-- TODO(elvinefendi) implement me
return backend.endpoints[0].address, backend.endpoints[0].port
end
-- Round-Robin
round_robin_lock:lock(backend_name .. ROUND_ROBIN_LOCK_KEY)
local index = round_robin_state:get(backend_name)
local index, endpoint = next(backend.endpoints, index)
if not index then
index = 1
endpoint = backend.endpoints[index]
end
round_robin_state:set(backend_name, index)
round_robin_lock:unlock(backend_name .. ROUND_ROBIN_LOCK_KEY)
return endpoint.address, endpoint.port
end
local function sync_backend(backend)
backends:set(backend.name, backend)
-- also reset the respective balancer state since backend has changed
round_robin_state:delete(backend.name)
ngx.log(ngx.INFO, "syncronization completed for: " .. backend.name)
end
local function sync_backends()
local backends_data = configuration.get_backends_data()
if not backends_data then
return
end
local ok, new_backends = pcall(json.decode, backends_data)
if not ok then
ngx.log(ngx.ERR, "could not parse backends data: " .. tostring(new_backends))
return
end
for _, new_backend in pairs(new_backends) do
local backend = backends:get(new_backend.name)
local backend_changed = true
if backend then
backend_changed = not util.deep_compare(backend, new_backend)
end
if backend_changed then
sync_backend(new_backend)
end
end
end
function _M.init_worker()
_, err = ngx.timer.every(BACKENDS_SYNC_INTERVAL, sync_backends)
if err then
ngx.log(ngx.ERR, "error when setting up timer.every for sync_backends: " .. tostring(err))
end
end
function _M.call()
ngx_balancer.set_more_tries(1)
local host, port = balance()
local ok, err = ngx_balancer.set_current_peer(host, port)
if ok then
ngx.log(ngx.INFO, "current peer is set to " .. host .. ":" .. port)
else
ngx.log(ngx.ERR, "error while setting current upstream peer to: " .. tostring(err))
end
end
return _M

41
rootfs/etc/nginx/lua/configuration.lua Normal file
View file

@ -0,0 +1,41 @@
-- this is the Lua representation of Configuration struct in internal/ingress/types.go
local configuration_data = ngx.shared.configuration_data
local _M = {}
function _M.get_backends_data()
return configuration_data:get("backends")
end
function _M.call()
if ngx.var.request_method ~= "POST" and ngx.var.request_method ~= "GET" then
ngx.status = ngx.HTTP_BAD_REQUEST
ngx.print("Only POST and GET requests are allowed!")
return
end
if ngx.var.request_uri ~= "/configuration/backends" then
ngx.status = ngx.HTTP_NOT_FOUND
ngx.print("Not found!")
return
end
if ngx.var.request_method == "GET" then
ngx.status = ngx.HTTP_OK
ngx.print(_M.get_backends_data())
return
end
ngx.req.read_body()
local success, err = configuration_data:set("backends", ngx.req.get_body_data())
if not success then
ngx.log(ngx.ERR, "error while saving configuration: " .. tostring(err))
ngx.status = ngx.HTTP_BAD_REQUEST
return
end
ngx.status = ngx.HTTP_CREATED
end
return _M

27
rootfs/etc/nginx/lua/util.lua Normal file
View file

@ -0,0 +1,27 @@
local _M = {}
-- this implementation is taken from
-- https://web.archive.org/web/20131225070434/http://snippets.luacode.org/snippets/Deep_Comparison_of_Two_Values_3
-- and modified for use in this project
local function deep_compare(t1, t2, ignore_mt)
local ty1 = type(t1)
local ty2 = type(t2)
if ty1 ~= ty2 then return false end
-- non-table types can be directly compared
if ty1 ~= 'table' and ty2 ~= 'table' then return t1 == t2 end
-- as well as tables which have the metamethod __eq
local mt = getmetatable(t1)
if not ignore_mt and mt and mt.__eq then return t1 == t2 end
for k1,v1 in pairs(t1) do
local v2 = t2[k1]
if v2 == nil or not deep_compare(v1,v2) then return false end
end
for k2,v2 in pairs(t2) do
local v1 = t1[k2]
if v1 == nil or not deep_compare(v1,v2) then return false end
end
return true
end
_M.deep_compare = deep_compare
return _M

69
rootfs/etc/nginx/template/nginx.tmpl
View file

@ -36,6 +36,39 @@ events {
}
http {
lua_package_cpath "/usr/local/lib/lua/?.so;/usr/lib/x86_64-linux-gnu/lua/5.1/?.so;;";
lua_package_path "/etc/nginx/lua/?.lua;/etc/nginx/lua/vendor/?.lua;/usr/local/lib/lua/?.lua;;";
lua_shared_dict configuration_data 5M;
lua_shared_dict round_robin_state 1M;
lua_shared_dict locks 512k;
init_by_lua_block {
require("resty.core")
collectgarbage("collect")
-- init modules
local ok, res
ok, res = pcall(require, "configuration")
if not ok then
error("require failed: " .. tostring(res))
else
configuration = res
end
ok, res = pcall(require, "balancer")
if not ok then
error("require failed: " .. tostring(res))
else
balancer = res
end
}
init_worker_by_lua_block {
balancer.init_worker()
}
{{/* we use the value of the header X-Forwarded-For to be able to use the geo_ip module */}}
{{ if $cfg.UseProxyProtocol }}
real_ip_header proxy_protocol;
@ -308,6 +341,7 @@ http {
{{ $cfg.HTTPSnippet }}
{{ end }}
{{ if not $all.DynamicConfigurationEnabled }}
{{ range $name, $upstream := $backends }}
{{ if eq $upstream.SessionAffinity.AffinityType "cookie" }}
upstream sticky-{{ $upstream.Name }} {
@ -319,9 +353,7 @@ http {
{{ range $server := $upstream.Endpoints }}server {{ $server.Address | formatIP }}:{{ $server.Port }} max_fails={{ $server.MaxFails }} fail_timeout={{ $server.FailTimeout }};
{{ end }}
}
{{ end }}
upstream {{ $upstream.Name }} {
@ -334,8 +366,18 @@ http {
{{ range $server := $upstream.Endpoints }}server {{ $server.Address | formatIP }}:{{ $server.Port }} max_fails={{ $server.MaxFails }} fail_timeout={{ $server.FailTimeout }};
{{ end }}
}
{{ end }}
{{ end }}
upstream upstream_balancer {
server 0.0.0.1; # placeholder
balancer_by_lua_block {
balancer.call()
}
keepalive 1000;
}
{{/* build the maps that will be use to validate the Whitelist */}}
{{ range $index, $server := $servers }}
@ -452,12 +494,24 @@ http {
{{ end }}
}
location /configuration {
allow 127.0.0.1;
deny all;
content_by_lua_block {
configuration.call()
}
}
location / {
{{ if .CustomErrors }}
proxy_set_header X-Code 404;
{{ end }}
set $proxy_upstream_name "upstream-default-backend";
{{ if $all.DynamicConfigurationEnabled }}
proxy_pass http://upstream_balancer;
{{ else }}
proxy_pass http://upstream-default-backend;
{{ end }}
}
{{ template "CUSTOM_ERRORS" $all }}
@ -550,7 +604,12 @@ stream {
proxy_set_header X-Service-Name $service_name;
rewrite (.*) / break;
proxy_pass http://upstream-default-backend;
{{ if .DynamicConfigurationEnabled }}
proxy_pass http://upstream_balancer;
{{ else }}
proxy_pass http://upstream-default-backend;
{{ end }}
}
{{ end }}
{{ end }}
@ -887,7 +946,7 @@ stream {
{{ end }}
{{ if not (empty $location.Backend) }}
{{ buildProxyPass $server.Hostname $all.Backends $location }}
{{ buildProxyPass $server.Hostname $all.Backends $location $all.DynamicConfigurationEnabled }}
{{ if (or (eq $location.Proxy.ProxyRedirectFrom "default") (eq $location.Proxy.ProxyRedirectFrom "off")) }}
proxy_redirect {{ $location.Proxy.ProxyRedirectFrom }};
{{ else }}