From 051b5bb35c2a33d94d71fe9d284becdedf570c2b Mon Sep 17 00:00:00 2001 From: Corey O'Brien Date: Thu, 11 Oct 2018 23:47:50 -0400 Subject: [PATCH 1/2] Retry initial backend configuration --- internal/ingress/controller/controller.go | 25 +++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/internal/ingress/controller/controller.go b/internal/ingress/controller/controller.go index b2109bd6b..052d9be73 100644 --- a/internal/ingress/controller/controller.go +++ b/internal/ingress/controller/controller.go @@ -185,17 +185,26 @@ func (n *NGINXController) syncIngress(interface{}) error { isFirstSync := n.runningConfig.Equal(&ingress.Configuration{}) go func(isFirstSync bool) { + maxAttempts := 1 if isFirstSync { - glog.Infof("Initial synchronization of the NGINX configuration.") - - // it takes time for NGINX to start listening on the configured ports + // For the initial sync it always takes some time for NGINX to + // start listening on the configured port (default 18080) + // For large configurations it might take a while so we loop + // and back off + maxAttempts = 15 time.Sleep(1 * time.Second) } - err := configureDynamically(pcfg, n.cfg.ListenPorts.Status, n.cfg.DynamicCertificatesEnabled) - if err == nil { - glog.Infof("Dynamic reconfiguration succeeded.") - } else { - glog.Warningf("Dynamic reconfiguration failed: %v", err) + + for i := 0; i < maxAttempts; i++ { + err := configureDynamically(pcfg, n.cfg.ListenPorts.Status, n.cfg.DynamicCertificatesEnabled) + if err == nil { + glog.Infof("Dynamic reconfiguration succeeded.") + break + } else { + glog.Warningf("Dynamic reconfiguration failed: %v", err) + // Sleep between retries backing off up to 120s total + time.Sleep(time.Duration(i+1) * time.Second) + } } }(isFirstSync) From 32d74c4ac0cf37fd411c8497514cf3f8c25727ee Mon Sep 17 00:00:00 2001 From: Corey O'Brien Date: Fri, 12 Oct 2018 09:16:33 -0400 Subject: [PATCH 2/2] Switch to using wait.ExponentialBackoff --- internal/ingress/controller/controller.go | 24 ++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/internal/ingress/controller/controller.go b/internal/ingress/controller/controller.go index 052d9be73..2f2b63ec2 100644 --- a/internal/ingress/controller/controller.go +++ b/internal/ingress/controller/controller.go @@ -30,6 +30,7 @@ import ( extensions "k8s.io/api/extensions/v1beta1" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" "k8s.io/ingress-nginx/internal/ingress" @@ -185,27 +186,32 @@ func (n *NGINXController) syncIngress(interface{}) error { isFirstSync := n.runningConfig.Equal(&ingress.Configuration{}) go func(isFirstSync bool) { - maxAttempts := 1 + steps := 1 if isFirstSync { // For the initial sync it always takes some time for NGINX to // start listening on the configured port (default 18080) // For large configurations it might take a while so we loop // and back off - maxAttempts = 15 + steps = 10 time.Sleep(1 * time.Second) } - for i := 0; i < maxAttempts; i++ { + retry := wait.Backoff{ + Steps: steps, + Duration: 1 * time.Second, + Factor: 1.5, + Jitter: 0.1, + } + + wait.ExponentialBackoff(retry, func() (bool, error) { err := configureDynamically(pcfg, n.cfg.ListenPorts.Status, n.cfg.DynamicCertificatesEnabled) if err == nil { glog.Infof("Dynamic reconfiguration succeeded.") - break - } else { - glog.Warningf("Dynamic reconfiguration failed: %v", err) - // Sleep between retries backing off up to 120s total - time.Sleep(time.Duration(i+1) * time.Second) + return true, nil } - } + glog.Warningf("Dynamic reconfiguration failed: %v", err) + return false, nil + }) }(isFirstSync) ri := getRemovedIngresses(n.runningConfig, pcfg)