Live Nginx (re)configuration without reloading (#2174)

Makefile

@@ -50,7 +50,7 @@ IMAGE = $(REGISTRY)/$(IMGNAME)
 MULTI_ARCH_IMG = $(IMAGE)-$(ARCH)
 
 # Set default base image dynamically for each arch
-BASEIMAGE?=quay.io/kubernetes-ingress-controller/nginx-$(ARCH):0.34
+BASEIMAGE?=quay.io/kubernetes-ingress-controller/nginx-$(ARCH):0.37
 
 ifeq ($(ARCH),arm)
 	QEMUARCH=arm

cmd/nginx/flags.go

@@ -131,6 +131,9 @@ func parseFlags() (bool, *controller.Configuration, error) {
 		publishStatusAddress = flags.String("publish-status-address", "",
 			`User customized address to be set in the status of ingress resources. The controller will set the
 		endpoint records on the ingress using this address.`)
+
+		dynamicConfigurationEnabled = flags.Bool("enable-dynamic-configuration", false,
+			`When enabled controller will try to avoid Nginx reloads as much as possible by using Lua. Disabled by default.`)
 	)
 
 	flag.Set("logtostderr", "true")
@@ -192,28 +195,29 @@ func parseFlags() (bool, *controller.Configuration, error) {
 	}
 
 	config := &controller.Configuration{
-		APIServerHost:            *apiserverHost,
-		KubeConfigFile:           *kubeConfigFile,
-		UpdateStatus:             *updateStatus,
-		ElectionID:               *electionID,
-		EnableProfiling:          *profiling,
-		EnableSSLPassthrough:     *enableSSLPassthrough,
-		EnableSSLChainCompletion: *enableSSLChainCompletion,
-		ResyncPeriod:             *resyncPeriod,
-		DefaultService:           *defaultSvc,
-		Namespace:                *watchNamespace,
-		ConfigMapName:            *configMap,
-		TCPConfigMapName:         *tcpConfigMapName,
-		UDPConfigMapName:         *udpConfigMapName,
-		DefaultSSLCertificate:    *defSSLCertificate,
-		DefaultHealthzURL:        *defHealthzURL,
-		PublishService:           *publishSvc,
-		PublishStatusAddress:     *publishStatusAddress,
-		ForceNamespaceIsolation:  *forceIsolation,
-		UpdateStatusOnShutdown:   *updateStatusOnShutdown,
-		SortBackends:             *sortBackends,
-		UseNodeInternalIP:        *useNodeInternalIP,
-		SyncRateLimit:            *syncRateLimit,
+		APIServerHost:               *apiserverHost,
+		KubeConfigFile:              *kubeConfigFile,
+		UpdateStatus:                *updateStatus,
+		ElectionID:                  *electionID,
+		EnableProfiling:             *profiling,
+		EnableSSLPassthrough:        *enableSSLPassthrough,
+		EnableSSLChainCompletion:    *enableSSLChainCompletion,
+		ResyncPeriod:                *resyncPeriod,
+		DefaultService:              *defaultSvc,
+		Namespace:                   *watchNamespace,
+		ConfigMapName:               *configMap,
+		TCPConfigMapName:            *tcpConfigMapName,
+		UDPConfigMapName:            *udpConfigMapName,
+		DefaultSSLCertificate:       *defSSLCertificate,
+		DefaultHealthzURL:           *defHealthzURL,
+		PublishService:              *publishSvc,
+		PublishStatusAddress:        *publishStatusAddress,
+		ForceNamespaceIsolation:     *forceIsolation,
+		UpdateStatusOnShutdown:      *updateStatusOnShutdown,
+		SortBackends:                *sortBackends,
+		UseNodeInternalIP:           *useNodeInternalIP,
+		SyncRateLimit:               *syncRateLimit,
+		DynamicConfigurationEnabled: *dynamicConfigurationEnabled,
 		ListenPorts: &ngx_config.ListenPorts{
 			Default:  *defServerPort,
 			Health:   *healthzPort,

internal/ingress/controller/config/config.go

@@ -604,23 +604,24 @@ func (cfg Configuration) BuildLogFormatUpstream() string {
 
 // TemplateConfig contains the nginx configuration to render the file nginx.conf
 type TemplateConfig struct {
-	ProxySetHeaders         map[string]string
-	AddHeaders              map[string]string
-	MaxOpenFiles            int
-	BacklogSize             int
-	Backends                []*ingress.Backend
-	PassthroughBackends     []*ingress.SSLPassthroughBackend
-	Servers                 []*ingress.Server
-	TCPBackends             []ingress.L4Service
-	UDPBackends             []ingress.L4Service
-	HealthzURI              string
-	CustomErrors            bool
-	Cfg                     Configuration
-	IsIPV6Enabled           bool
-	IsSSLPassthroughEnabled bool
-	RedirectServers         map[string]string
-	ListenPorts             *ListenPorts
-	PublishService          *apiv1.Service
+	ProxySetHeaders             map[string]string
+	AddHeaders                  map[string]string
+	MaxOpenFiles                int
+	BacklogSize                 int
+	Backends                    []*ingress.Backend
+	PassthroughBackends         []*ingress.SSLPassthroughBackend
+	Servers                     []*ingress.Server
+	TCPBackends                 []ingress.L4Service
+	UDPBackends                 []ingress.L4Service
+	HealthzURI                  string
+	CustomErrors                bool
+	Cfg                         Configuration
+	IsIPV6Enabled               bool
+	IsSSLPassthroughEnabled     bool
+	RedirectServers             map[string]string
+	ListenPorts                 *ListenPorts
+	PublishService              *apiv1.Service
+	DynamicConfigurationEnabled bool
 }
 
 // ListenPorts describe the ports required to run the

internal/ingress/controller/controller.go

@@ -95,6 +95,8 @@ type Configuration struct {
 	FakeCertificateSHA  string
 
 	SyncRateLimit float32
+
+	DynamicConfigurationEnabled bool
 }
 
 // GetPublishService returns the configured service used to set ingress status
@@ -167,6 +169,15 @@ func (n *NGINXController) syncIngress(item interface{}) error {
 	if !n.isForceReload() && n.runningConfig.Equal(&pcfg) {
 		glog.V(3).Infof("skipping backend reload (no changes detected)")
 		return nil
+	} else if !n.isForceReload() && n.cfg.DynamicConfigurationEnabled && n.IsDynamicallyConfigurable(&pcfg) {
+		err := n.ConfigureDynamically(&pcfg)
+		if err == nil {
+			glog.Infof("dynamic reconfiguration succeeded, skipping reload")
+			n.runningConfig = &pcfg
+			return nil
+		}
+
+		glog.Warningf("falling back to reload, could not dynamically reconfigure: %v", err)
 	}
 
 	glog.Infof("backend reload required")
@@ -182,6 +193,19 @@ func (n *NGINXController) syncIngress(item interface{}) error {
 	incReloadCount()
 	setSSLExpireTime(servers)
 
+	if n.isForceReload() && n.cfg.DynamicConfigurationEnabled {
+		go func() {
+			// it takes time for Nginx to start listening on the port
+			time.Sleep(1 * time.Second)
+			err := n.ConfigureDynamically(&pcfg)
+			if err == nil {
+				glog.Infof("dynamic reconfiguration succeeded")
+			} else {
+				glog.Warningf("could not dynamically reconfigure: %v", err)
+			}
+		}()
+	}
+
 	n.runningConfig = &pcfg
 	n.SetForceReload(false)
 

internal/ingress/controller/nginx.go

@@ -18,10 +18,12 @@ package controller
 
 import (
 	"bytes"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io/ioutil"
 	"net"
+	"net/http"
 	"os"
 	"os/exec"
 	"strconv"
@@ -606,23 +608,24 @@ func (n *NGINXController) OnUpdate(ingressCfg ingress.Configuration) error {
 	cfg.SSLDHParam = sslDHParam
 
 	tc := ngx_config.TemplateConfig{
-		ProxySetHeaders:         setHeaders,
-		AddHeaders:              addHeaders,
-		MaxOpenFiles:            maxOpenFiles,
-		BacklogSize:             sysctlSomaxconn(),
-		Backends:                ingressCfg.Backends,
-		PassthroughBackends:     ingressCfg.PassthroughBackends,
-		Servers:                 ingressCfg.Servers,
-		TCPBackends:             ingressCfg.TCPEndpoints,
-		UDPBackends:             ingressCfg.UDPEndpoints,
-		HealthzURI:              ngxHealthPath,
-		CustomErrors:            len(cfg.CustomHTTPErrors) > 0,
-		Cfg:                     cfg,
-		IsIPV6Enabled:           n.isIPV6Enabled && !cfg.DisableIpv6,
-		RedirectServers:         redirectServers,
-		IsSSLPassthroughEnabled: n.cfg.EnableSSLPassthrough,
-		ListenPorts:             n.cfg.ListenPorts,
-		PublishService:          n.GetPublishService(),
+		ProxySetHeaders:             setHeaders,
+		AddHeaders:                  addHeaders,
+		MaxOpenFiles:                maxOpenFiles,
+		BacklogSize:                 sysctlSomaxconn(),
+		Backends:                    ingressCfg.Backends,
+		PassthroughBackends:         ingressCfg.PassthroughBackends,
+		Servers:                     ingressCfg.Servers,
+		TCPBackends:                 ingressCfg.TCPEndpoints,
+		UDPBackends:                 ingressCfg.UDPEndpoints,
+		HealthzURI:                  ngxHealthPath,
+		CustomErrors:                len(cfg.CustomHTTPErrors) > 0,
+		Cfg:                         cfg,
+		IsIPV6Enabled:               n.isIPV6Enabled && !cfg.DisableIpv6,
+		RedirectServers:             redirectServers,
+		IsSSLPassthroughEnabled:     n.cfg.EnableSSLPassthrough,
+		ListenPorts:                 n.cfg.ListenPorts,
+		PublishService:              n.GetPublishService(),
+		DynamicConfigurationEnabled: n.cfg.DynamicConfigurationEnabled,
 	}
 
 	content, err := n.t.Write(tc)
@@ -745,3 +748,43 @@ func (n *NGINXController) setupSSLProxy() {
 		}
 	}()
 }
+
+// IsDynamicallyConfigurable decides if the new configuration can be dynamically configured without reloading
+func (n *NGINXController) IsDynamicallyConfigurable(pcfg *ingress.Configuration) bool {
+	var copyOfRunningConfig ingress.Configuration = *n.runningConfig
+	var copyOfPcfg ingress.Configuration = *pcfg
+
+	copyOfRunningConfig.Backends = []*ingress.Backend{}
+	copyOfPcfg.Backends = []*ingress.Backend{}
+
+	return copyOfRunningConfig.Equal(&copyOfPcfg)
+}
+
+// ConfigureDynamically JSON encodes new Backends and POSTs it to an internal HTTP endpoint
+// that is handled by Lua
+func (n *NGINXController) ConfigureDynamically(pcfg *ingress.Configuration) error {
+	buf, err := json.Marshal(pcfg.Backends)
+	if err != nil {
+		return err
+	}
+
+	glog.V(2).Infof("posting backends configuration: %s", buf)
+
+	url := fmt.Sprintf("http://localhost:%d/configuration/backends", n.cfg.ListenPorts.Status)
+	resp, err := http.Post(url, "application/json", bytes.NewReader(buf))
+	if err != nil {
+		return err
+	}
+
+	defer func() {
+		if err := resp.Body.Close(); err != nil {
+			glog.Warningf("error while closing response body: \n%v", err)
+		}
+	}()
+
+	if resp.StatusCode != http.StatusCreated {
+		return fmt.Errorf("Unexpected error code: %d", resp.StatusCode)
+	}
+
+	return nil
+}

internal/ingress/controller/nginx_test.go

@@ -16,7 +16,78 @@ limitations under the License.
 
 package controller
 
-import "testing"
+import (
+	"testing"
+
+	"k8s.io/ingress-nginx/internal/ingress"
+)
+
+func TestIsDynamicallyConfigurable(t *testing.T) {
+	backends := []*ingress.Backend{{
+		Name: "fakenamespace-myapp-80",
+		Endpoints: []ingress.Endpoint{
+			{
+				Address: "10.0.0.1",
+				Port:    "8080",
+			},
+			{
+				Address: "10.0.0.2",
+				Port:    "8080",
+			},
+		},
+	}}
+
+	servers := []*ingress.Server{{
+		Hostname: "myapp.fake",
+		Locations: []*ingress.Location{
+			{
+				Path:    "/",
+				Backend: "fakenamespace-myapp-80",
+			},
+		},
+	}}
+
+	commonConfig := &ingress.Configuration{
+		Backends: backends,
+		Servers:  servers,
+	}
+
+	n := &NGINXController{
+		runningConfig: &ingress.Configuration{
+			Backends: backends,
+			Servers:  servers,
+		},
+	}
+
+	newConfig := commonConfig
+	if !n.IsDynamicallyConfigurable(newConfig) {
+		t.Errorf("When new config is same as the running config it should be deemed as dynamically configurable")
+	}
+
+	newConfig = &ingress.Configuration{
+		Backends: []*ingress.Backend{{Name: "another-backend-8081"}},
+		Servers:  []*ingress.Server{{Hostname: "myapp1.fake"}},
+	}
+	if n.IsDynamicallyConfigurable(newConfig) {
+		t.Errorf("Expected to not be dynamically configurable when there's more than just backends change")
+	}
+
+	newConfig = &ingress.Configuration{
+		Backends: []*ingress.Backend{{Name: "a-backend-8080"}},
+		Servers:  servers,
+	}
+	if !n.IsDynamicallyConfigurable(newConfig) {
+		t.Errorf("Expected to be dynamically configurable when only backends change")
+	}
+
+	if !n.runningConfig.Equal(commonConfig) {
+		t.Errorf("Expected running config to not change")
+	}
+
+	if !newConfig.Equal(&ingress.Configuration{Backends: []*ingress.Backend{{Name: "a-backend-8080"}}, Servers: servers}) {
+		t.Errorf("Expected new config to not change")
+	}
+}
 
 func TestNginxHashBucketSize(t *testing.T) {
 	tests := []struct {

internal/ingress/controller/template/template.go

@@ -307,7 +307,7 @@ func buildLoadBalancingConfig(b interface{}, fallbackLoadBalancing string) strin
 // (specified through the nginx.ingress.kubernetes.io/rewrite-to annotation)
 // If the annotation nginx.ingress.kubernetes.io/add-base-url:"true" is specified it will
 // add a base tag in the head of the response from the service
-func buildProxyPass(host string, b interface{}, loc interface{}) string {
+func buildProxyPass(host string, b interface{}, loc interface{}, dynamicConfigurationEnabled bool) string {
 	backends, ok := b.([]*ingress.Backend)
 	if !ok {
 		glog.Errorf("expected an '[]*ingress.Backend' type but %T was returned", b)
@@ -323,14 +323,19 @@ func buildProxyPass(host string, b interface{}, loc interface{}) string {
 	path := location.Path
 	proto := "http"
 
-	upstreamName := location.Backend
+	upstreamName := "upstream_balancer"
+
+	if !dynamicConfigurationEnabled {
+		upstreamName = location.Backend
+	}
+
 	for _, backend := range backends {
 		if backend.Name == location.Backend {
 			if backend.Secure || backend.SSLPassthrough {
 				proto = "https"
 			}
 
-			if isSticky(host, location, backend.SessionAffinity.CookieSessionAffinity.Locations) {
+			if !dynamicConfigurationEnabled && isSticky(host, location, backend.SessionAffinity.CookieSessionAffinity.Locations) {
 				upstreamName = fmt.Sprintf("sticky-%v", upstreamName)
 			}
 
@@ -340,6 +345,7 @@ func buildProxyPass(host string, b interface{}, loc interface{}) string {
 
 	// defProxyPass returns the default proxy_pass, just the name of the upstream
 	defProxyPass := fmt.Sprintf("proxy_pass %s://%s;", proto, upstreamName)
+
 	// if the path in the ingress rule is equals to the target: no special rewrite
 	if path == location.Rewrite.Target {
 		return defProxyPass

internal/ingress/controller/template/template_test.go

@@ -37,72 +37,262 @@ import (
 )
 
 var (
-	// TODO: add tests for secure endpoints
+	// TODO: add tests for SSLPassthrough
 	tmplFuncTestcases = map[string]struct {
-		Path             string
-		Target           string
-		Location         string
-		ProxyPass        string
-		AddBaseURL       bool
-		BaseURLScheme    string
-		Sticky           bool
-		XForwardedPrefix bool
+		Path                        string
+		Target                      string
+		Location                    string
+		ProxyPass                   string
+		AddBaseURL                  bool
+		BaseURLScheme               string
+		Sticky                      bool
+		XForwardedPrefix            bool
+		DynamicConfigurationEnabled bool
+		SecureBackend               bool
 	}{
-		"invalid redirect / to /": {"/", "/", "/", "proxy_pass http://upstream-name;", false, "", false, false},
-		"redirect / to /jenkins": {"/", "/jenkins", "~* /",
+		"when secure backend enabled": {
+			"/",
+			"/",
+			"/",
+			"proxy_pass https://upstream-name;",
+			false,
+			"",
+			false,
+			false,
+			false,
+			true},
+		"when secure backend and stickeness enabled": {
+			"/",
+			"/",
+			"/",
+			"proxy_pass https://sticky-upstream-name;",
+			false,
+			"",
+			true,
+			false,
+			false,
+			true},
+		"when secure backend and dynamic config enabled": {
+			"/",
+			"/",
+			"/",
+			"proxy_pass https://upstream_balancer;",
+			false,
+			"",
+			false,
+			false,
+			true,
+			true},
+		"when secure backend, stickeness and dynamic config enabled": {
+			"/",
+			"/",
+			"/",
+			"proxy_pass https://upstream_balancer;",
+			false,
+			"",
+			true,
+			false,
+			true,
+			true},
+		"invalid redirect / to / with dynamic config enabled": {
+			"/",
+			"/",
+			"/",
+			"proxy_pass http://upstream_balancer;",
+			false,
+			"",
+			false,
+			false,
+			true,
+			false},
+		"invalid redirect / to /": {
+			"/",
+			"/",
+			"/",
+			"proxy_pass http://upstream-name;",
+			false,
+			"",
+			false,
+			false,
+			false,
+			false},
+		"redirect / to /jenkins": {
+			"/",
+			"/jenkins",
+			"~* /",
 			`
 	    rewrite /(.*) /jenkins/$1 break;
 	    proxy_pass http://upstream-name;
-	    `, false, "", false, false},
-		"redirect /something to /": {"/something", "/", `~* ^/something\/?(?<baseuri>.*)`, `
+	    `,
+			false,
+			"",
+			false,
+			false,
+			false,
+			false},
+		"redirect /something to /": {
+			"/something",
+			"/",
+			`~* ^/something\/?(?<baseuri>.*)`,
+			`
 	    rewrite /something/(.*) /$1 break;
 	    rewrite /something / break;
 	    proxy_pass http://upstream-name;
-	    `, false, "", false, false},
-		"redirect /end-with-slash/ to /not-root": {"/end-with-slash/", "/not-root", "~* ^/end-with-slash/(?<baseuri>.*)", `
+	    `,
+			false,
+			"",
+			false,
+			false,
+			false,
+			false},
+		"redirect /end-with-slash/ to /not-root": {
+			"/end-with-slash/",
+			"/not-root",
+			"~* ^/end-with-slash/(?<baseuri>.*)",
+			`
 	    rewrite /end-with-slash/(.*) /not-root/$1 break;
 	    proxy_pass http://upstream-name;
-	    `, false, "", false, false},
-		"redirect /something-complex to /not-root": {"/something-complex", "/not-root", `~* ^/something-complex\/?(?<baseuri>.*)`, `
+	    `,
+			false,
+			"",
+			false,
+			false,
+			false,
+			false},
+		"redirect /something-complex to /not-root": {
+			"/something-complex",
+			"/not-root",
+			`~* ^/something-complex\/?(?<baseuri>.*)`,
+			`
 	    rewrite /something-complex/(.*) /not-root/$1 break;
 	    proxy_pass http://upstream-name;
-	    `, false, "", false, false},
-		"redirect / to /jenkins and rewrite": {"/", "/jenkins", "~* /", `
+	    `,
+			false,
+			"",
+			false,
+			false,
+			false,
+			false},
+		"redirect / to /jenkins and rewrite": {
+			"/",
+			"/jenkins",
+			"~* /",
+			`
 	    rewrite /(.*) /jenkins/$1 break;
 	    proxy_pass http://upstream-name;
 	    subs_filter '(<(?:H|h)(?:E|e)(?:A|a)(?:D|d)(?:[^">]|"[^"]*")*>)' '$1<base href="$scheme://$http_host/$baseuri">' ro;
-	    `, true, "", false, false},
-		"redirect /something to / and rewrite": {"/something", "/", `~* ^/something\/?(?<baseuri>.*)`, `
+	    `,
+			true,
+			"",
+			false,
+			false,
+			false,
+			false},
+		"redirect /something to / and rewrite": {
+			"/something",
+			"/",
+			`~* ^/something\/?(?<baseuri>.*)`,
+			`
 	    rewrite /something/(.*) /$1 break;
 	    rewrite /something / break;
 	    proxy_pass http://upstream-name;
 	    subs_filter '(<(?:H|h)(?:E|e)(?:A|a)(?:D|d)(?:[^">]|"[^"]*")*>)' '$1<base href="$scheme://$http_host/something/$baseuri">' ro;
-	    `, true, "", false, false},
-		"redirect /end-with-slash/ to /not-root and rewrite": {"/end-with-slash/", "/not-root", `~* ^/end-with-slash/(?<baseuri>.*)`, `
+	    `,
+			true,
+			"",
+			false,
+			false,
+			false,
+			false},
+		"redirect /end-with-slash/ to /not-root and rewrite": {
+			"/end-with-slash/",
+			"/not-root",
+			`~* ^/end-with-slash/(?<baseuri>.*)`,
+			`
 	    rewrite /end-with-slash/(.*) /not-root/$1 break;
 	    proxy_pass http://upstream-name;
 	    subs_filter '(<(?:H|h)(?:E|e)(?:A|a)(?:D|d)(?:[^">]|"[^"]*")*>)' '$1<base href="$scheme://$http_host/end-with-slash/$baseuri">' ro;
-	    `, true, "", false, false},
-		"redirect /something-complex to /not-root and rewrite": {"/something-complex", "/not-root", `~* ^/something-complex\/?(?<baseuri>.*)`, `
+	    `,
+			true,
+			"",
+			false,
+			false,
+			false,
+			false},
+		"redirect /something-complex to /not-root and rewrite": {
+			"/something-complex",
+			"/not-root",
+			`~* ^/something-complex\/?(?<baseuri>.*)`,
+			`
 	    rewrite /something-complex/(.*) /not-root/$1 break;
 	    proxy_pass http://upstream-name;
 	    subs_filter '(<(?:H|h)(?:E|e)(?:A|a)(?:D|d)(?:[^">]|"[^"]*")*>)' '$1<base href="$scheme://$http_host/something-complex/$baseuri">' ro;
-	    `, true, "", false, false},
-		"redirect /something to / and rewrite with specific scheme": {"/something", "/", `~* ^/something\/?(?<baseuri>.*)`, `
+	    `,
+			true,
+			"",
+			false,
+			false,
+			false,
+			false},
+		"redirect /something to / and rewrite with specific scheme": {
+			"/something",
+			"/",
+			`~* ^/something\/?(?<baseuri>.*)`,
+			`
 	    rewrite /something/(.*) /$1 break;
 	    rewrite /something / break;
 	    proxy_pass http://upstream-name;
 	    subs_filter '(<(?:H|h)(?:E|e)(?:A|a)(?:D|d)(?:[^">]|"[^"]*")*>)' '$1<base href="http://$http_host/something/$baseuri">' ro;
-	    `, true, "http", false, false},
-		"redirect / to /something with sticky enabled": {"/", "/something", `~* /`, `
+	    `,
+			true,
+			"http",
+			false,
+			false,
+			false,
+			false},
+		"redirect / to /something with sticky enabled": {
+			"/",
+			"/something",
+			`~* /`,
+			`
 	    rewrite /(.*) /something/$1 break;
 	    proxy_pass http://sticky-upstream-name;
-	    `, false, "http", true, false},
-		"add the X-Forwarded-Prefix header": {"/there", "/something", `~* ^/there\/?(?<baseuri>.*)`, `
+	    `,
+			false,
+			"http",
+			true,
+			false,
+			false,
+			false},
+		"redirect / to /something with sticky and dynamic config enabled": {
+			"/",
+			"/something",
+			`~* /`,
+			`
+	    rewrite /(.*) /something/$1 break;
+	    proxy_pass http://upstream_balancer;
+	    `,
+			false,
+			"http",
+			true,
+			false,
+			true,
+			false},
+		"add the X-Forwarded-Prefix header": {
+			"/there",
+			"/something",
+			`~* ^/there\/?(?<baseuri>.*)`,
+			`
 	    rewrite /there/(.*) /something/$1 break;
 	    proxy_set_header X-Forwarded-Prefix "/there/";
 	    proxy_pass http://sticky-upstream-name;
-	    `, false, "http", true, true},
+	    `,
+			false,
+			"http",
+			true,
+			true,
+			false,
+			false},
 	}
 )
 
@@ -151,24 +341,25 @@ func TestBuildProxyPass(t *testing.T) {
 			XForwardedPrefix: tc.XForwardedPrefix,
 		}
 
-		backends := []*ingress.Backend{}
+		backend := &ingress.Backend{
+			Name:   defaultBackend,
+			Secure: tc.SecureBackend,
+		}
+
 		if tc.Sticky {
-			backends = []*ingress.Backend{
-				{
-					Name: defaultBackend,
-					SessionAffinity: ingress.SessionAffinityConfig{
-						AffinityType: "cookie",
-						CookieSessionAffinity: ingress.CookieSessionAffinity{
-							Locations: map[string][]string{
-								defaultHost: {tc.Path},
-							},
-						},
+			backend.SessionAffinity = ingress.SessionAffinityConfig{
+				AffinityType: "cookie",
+				CookieSessionAffinity: ingress.CookieSessionAffinity{
+					Locations: map[string][]string{
+						defaultHost: {tc.Path},
 					},
 				},
 			}
 		}
 
-		pp := buildProxyPass(defaultHost, backends, loc)
+		backends := []*ingress.Backend{backend}
+
+		pp := buildProxyPass(defaultHost, backends, loc, tc.DynamicConfigurationEnabled)
 		if !strings.EqualFold(tc.ProxyPass, pp) {
 			t.Errorf("%s: expected \n'%v'\nbut returned \n'%v'", k, tc.ProxyPass, pp)
 		}

rootfs/etc/nginx/lua/balancer.lua

@@ -0,0 +1,107 @@
+local ngx_balancer = require("ngx.balancer")
+local json = require("cjson")
+local configuration = require("configuration")
+local util = require("util")
+local lrucache = require("resty.lrucache")
+local resty_lock = require("resty.lock")
+
+-- measured in seconds
+-- for an Nginx worker to pick up the new list of upstream peers 
+-- it will take <the delay until controller POSTed the backend object to the Nginx endpoint> + BACKENDS_SYNC_INTERVAL
+local BACKENDS_SYNC_INTERVAL = 1
+
+ROUND_ROBIN_LOCK_KEY = "round_robin"
+
+local round_robin_state = ngx.shared.round_robin_state
+
+local _M = {}
+
+local round_robin_lock = resty_lock:new("locks", {timeout = 0, exptime = 0.1})
+
+local backends, err = lrucache.new(1024)
+if not backends then
+  return error("failed to create the cache for backends: " .. (err or "unknown"))
+end
+
+local function balance()
+  local backend_name = ngx.var.proxy_upstream_name
+  local backend = backends:get(backend_name)
+  -- lb_alg field does not exist for ingress.Backend struct for now, so lb_alg
+  -- will always be round_robin
+  local lb_alg = backend.lb_alg or "round_robin"
+
+  if lb_alg == "ip_hash" then
+    -- TODO(elvinefendi) implement me
+    return backend.endpoints[0].address, backend.endpoints[0].port
+  end
+
+  -- Round-Robin
+  round_robin_lock:lock(backend_name .. ROUND_ROBIN_LOCK_KEY)
+  local index = round_robin_state:get(backend_name)
+  local index, endpoint = next(backend.endpoints, index)
+  if not index then
+    index = 1
+    endpoint = backend.endpoints[index]
+  end
+  round_robin_state:set(backend_name, index)
+  round_robin_lock:unlock(backend_name .. ROUND_ROBIN_LOCK_KEY)
+
+  return endpoint.address, endpoint.port
+end
+
+local function sync_backend(backend)
+  backends:set(backend.name, backend)
+
+  -- also reset the respective balancer state since backend has changed
+  round_robin_state:delete(backend.name)
+
+  ngx.log(ngx.INFO, "syncronization completed for: " .. backend.name)
+end
+
+local function sync_backends()
+  local backends_data = configuration.get_backends_data()
+  if not backends_data then
+    return
+  end
+
+  local ok, new_backends = pcall(json.decode, backends_data)
+  if not ok then
+    ngx.log(ngx.ERR,  "could not parse backends data: " .. tostring(new_backends))
+    return
+  end
+
+  for _, new_backend in pairs(new_backends) do
+    local backend = backends:get(new_backend.name)
+    local backend_changed = true
+
+    if backend then
+      backend_changed = not util.deep_compare(backend, new_backend)
+    end
+
+    if backend_changed then
+      sync_backend(new_backend)
+    end
+  end
+end
+
+function _M.init_worker()
+  _, err = ngx.timer.every(BACKENDS_SYNC_INTERVAL, sync_backends)
+  if err then
+    ngx.log(ngx.ERR, "error when setting up timer.every for sync_backends: " .. tostring(err))
+  end
+end
+
+function _M.call()
+  ngx_balancer.set_more_tries(1)
+
+  local host, port = balance()
+
+  local ok, err = ngx_balancer.set_current_peer(host, port)
+  if ok then
+    ngx.log(ngx.INFO, "current peer is set to " .. host .. ":" .. port)
+  else
+    ngx.log(ngx.ERR, "error while setting current upstream peer to: " .. tostring(err))
+  end
+end
+
+return _M

rootfs/etc/nginx/lua/configuration.lua

@@ -0,0 +1,41 @@
+-- this is the Lua representation of Configuration struct in internal/ingress/types.go
+local configuration_data = ngx.shared.configuration_data
+
+local _M = {}
+
+function _M.get_backends_data()
+  return configuration_data:get("backends")
+end
+
+function _M.call()
+  if ngx.var.request_method ~= "POST" and ngx.var.request_method ~= "GET" then
+    ngx.status = ngx.HTTP_BAD_REQUEST
+    ngx.print("Only POST and GET requests are allowed!")
+    return
+  end
+
+  if ngx.var.request_uri ~= "/configuration/backends" then
+    ngx.status = ngx.HTTP_NOT_FOUND
+    ngx.print("Not found!")
+    return
+  end
+
+  if ngx.var.request_method == "GET" then
+    ngx.status = ngx.HTTP_OK
+    ngx.print(_M.get_backends_data())
+    return
+  end
+
+  ngx.req.read_body()
+
+  local success, err = configuration_data:set("backends", ngx.req.get_body_data())
+  if not success then
+    ngx.log(ngx.ERR, "error while saving configuration: " .. tostring(err))
+    ngx.status = ngx.HTTP_BAD_REQUEST
+    return
+  end
+
+  ngx.status = ngx.HTTP_CREATED
+end
+
+return _M

rootfs/etc/nginx/lua/util.lua

@@ -0,0 +1,27 @@
+local _M = {}
+
+-- this implementation is taken from
+-- https://web.archive.org/web/20131225070434/http://snippets.luacode.org/snippets/Deep_Comparison_of_Two_Values_3
+-- and modified for use in this project
+local function deep_compare(t1, t2, ignore_mt)
+  local ty1 = type(t1)
+  local ty2 = type(t2)
+  if ty1 ~= ty2 then return false end
+  -- non-table types can be directly compared
+  if ty1 ~= 'table' and ty2 ~= 'table' then return t1 == t2 end
+  -- as well as tables which have the metamethod __eq
+  local mt = getmetatable(t1)
+  if not ignore_mt and mt and mt.__eq then return t1 == t2 end
+  for k1,v1 in pairs(t1) do
+    local v2 = t2[k1]
+    if v2 == nil or not deep_compare(v1,v2) then return false end
+  end
+  for k2,v2 in pairs(t2) do
+    local v1 = t1[k2]
+    if v1 == nil or not deep_compare(v1,v2) then return false end
+  end
+  return true
+end
+_M.deep_compare = deep_compare
+
+return _M

rootfs/etc/nginx/template/nginx.tmpl

@@ -36,6 +36,39 @@ events {
 }
 
 http {
+    lua_package_cpath "/usr/local/lib/lua/?.so;/usr/lib/x86_64-linux-gnu/lua/5.1/?.so;;";
+    lua_package_path "/etc/nginx/lua/?.lua;/etc/nginx/lua/vendor/?.lua;/usr/local/lib/lua/?.lua;;";
+
+    lua_shared_dict configuration_data 5M;
+    lua_shared_dict round_robin_state 1M;
+    lua_shared_dict locks 512k;
+
+    init_by_lua_block {
+        require("resty.core")
+        collectgarbage("collect")
+
+        -- init modules
+        local ok, res
+
+        ok, res = pcall(require, "configuration")
+        if not ok then
+          error("require failed: " .. tostring(res))
+        else
+          configuration = res
+        end
+
+        ok, res = pcall(require, "balancer")
+        if not ok then
+          error("require failed: " .. tostring(res))
+        else
+          balancer = res
+        end
+    }
+
+    init_worker_by_lua_block {
+        balancer.init_worker()
+    }
+
     {{/* we use the value of the header X-Forwarded-For to be able to use the geo_ip module */}}
     {{ if $cfg.UseProxyProtocol }}
     real_ip_header      proxy_protocol;
@@ -308,6 +341,7 @@ http {
     {{ $cfg.HTTPSnippet }}
     {{ end }}
 
+    {{ if not $all.DynamicConfigurationEnabled }}
     {{ range $name, $upstream := $backends }}
     {{ if eq $upstream.SessionAffinity.AffinityType "cookie" }}
     upstream sticky-{{ $upstream.Name }} {
@@ -319,9 +353,7 @@ http {
 
         {{ range $server := $upstream.Endpoints }}server {{ $server.Address | formatIP }}:{{ $server.Port }} max_fails={{ $server.MaxFails }} fail_timeout={{ $server.FailTimeout }};
         {{ end }}
-
     }
-
     {{ end }}
 
     upstream {{ $upstream.Name }} {
@@ -334,8 +366,18 @@ http {
         {{ range $server := $upstream.Endpoints }}server {{ $server.Address | formatIP }}:{{ $server.Port }} max_fails={{ $server.MaxFails }} fail_timeout={{ $server.FailTimeout }};
         {{ end }}
     }
-
     {{ end }}
+    {{ end }}
+
+    upstream upstream_balancer {
+        server 0.0.0.1; # placeholder
+
+        balancer_by_lua_block {
+          balancer.call()
+        }
+
+        keepalive 1000;
+    }
 
     {{/* build the maps that will be use to validate the Whitelist */}}
     {{ range $index, $server := $servers }}
@@ -452,12 +494,24 @@ http {
             {{ end }}
         }
 
+        location /configuration {
+            allow 127.0.0.1;
+            deny all;
+            content_by_lua_block {
+              configuration.call()
+            }
+        }
+
         location / {
             {{ if .CustomErrors }}
             proxy_set_header    X-Code 404;
             {{ end }}
             set $proxy_upstream_name "upstream-default-backend";
+            {{ if $all.DynamicConfigurationEnabled }}
+            proxy_pass          http://upstream_balancer;
+            {{ else }}
             proxy_pass          http://upstream-default-backend;
+            {{ end }}
         }
 
         {{ template "CUSTOM_ERRORS" $all }}
@@ -550,7 +604,12 @@ stream {
             proxy_set_header       X-Service-Name     $service_name;
 
             rewrite                (.*) / break;
-            proxy_pass             http://upstream-default-backend;
+
+            {{ if .DynamicConfigurationEnabled }}
+            proxy_pass            http://upstream_balancer;
+            {{ else }}
+            proxy_pass            http://upstream-default-backend;
+            {{ end }}
         }
         {{ end }}
 {{ end }}
@@ -887,7 +946,7 @@ stream {
             {{ end }}
 
             {{ if not (empty $location.Backend) }}
-            {{ buildProxyPass $server.Hostname $all.Backends $location }}
+            {{ buildProxyPass $server.Hostname $all.Backends $location $all.DynamicConfigurationEnabled }}
             {{ if (or (eq $location.Proxy.ProxyRedirectFrom "default") (eq $location.Proxy.ProxyRedirectFrom "off")) }}
             proxy_redirect                          {{ $location.Proxy.ProxyRedirectFrom }};
             {{ else }}