Fix for buggy ingress sync with retries (#8325)

cmd/nginx/flags.go

@@ -203,6 +203,8 @@ Takes the form "<host>:port". If not provided, no admission controller is starte
 		postShutdownGracePeriod = flags.Int("post-shutdown-grace-period", 10, "Seconds to wait after the nginx process has stopped before controller exits.")
 
 		deepInspector = flags.Bool("deep-inspect", true, "Enables ingress object security deep inspector")
+
+		dynamicConfigurationRetries = flags.Int("dynamic-configuration-retries", 15, "Number of times to retry failed dynamic configuration before failing to sync an ingress.")
 	)
 
 	flags.StringVar(&nginx.MaxmindMirror, "maxmind-mirror", "", `Maxmind mirror url (example: http://geoip.local/databases`)
@@ -332,6 +334,7 @@ https://blog.maxmind.com/2019/12/18/significant-changes-to-accessing-and-using-g
 		UseNodeInternalIP:           *useNodeInternalIP,
 		SyncRateLimit:               *syncRateLimit,
 		HealthCheckHost:             *healthzHost,
+		DynamicConfigurationRetries: *dynamicConfigurationRetries,
 		ListenPorts: &ngx_config.ListenPorts{
 			Default:  *defServerPort,
 			Health:   *healthzPort,

internal/ingress/controller/controller.go

@@ -125,6 +125,8 @@ type Configuration struct {
 	InternalLoggerAddress string
 	IsChroot              bool
 	DeepInspector         bool
+
+	DynamicConfigurationRetries int
 }
 
 // GetPublishService returns the Service used to set the load-balancer status of Ingresses.
@@ -194,19 +196,24 @@ func (n *NGINXController) syncIngress(interface{}) error {
 	}
 
 	retry := wait.Backoff{
-		Steps:    15,
-		Duration: 1 * time.Second,
-		Factor:   0.8,
+		Steps:    1 + n.cfg.DynamicConfigurationRetries,
+		Duration: time.Second,
+		Factor:   1.3,
 		Jitter:   0.1,
 	}
 
+	retriesRemaining := retry.Steps
 	err := wait.ExponentialBackoff(retry, func() (bool, error) {
 		err := n.configureDynamically(pcfg)
 		if err == nil {
 			klog.V(2).Infof("Dynamic reconfiguration succeeded.")
 			return true, nil
 		}
-
+		retriesRemaining--
+		if retriesRemaining > 0 {
+			klog.Warningf("Dynamic reconfiguration failed (retrying; %d retries left): %v", retriesRemaining, err)
+			return false, nil
+		}
 		klog.Warningf("Dynamic reconfiguration failed: %v", err)
 		return false, err
 	})