Fix for buggy ingress sync with retries (#8325)

cmd/nginx/flags.go

@@ -65,7 +65,7 @@ The parameter --controller-class has precedence over this.`)
 
 		ingressClassController = flags.String("controller-class", ingressclass.DefaultControllerName,
 			`Ingress Class Controller value this Ingress satisfies.
-The class of an Ingress object is set using the field IngressClassName in Kubernetes clusters version v1.19.0 or higher. The .spec.controller value of the IngressClass 
+The class of an Ingress object is set using the field IngressClassName in Kubernetes clusters version v1.19.0 or higher. The .spec.controller value of the IngressClass
 referenced in an Ingress Object should be the same value specified here to make this object be watched.`)
 
 		watchWithoutClass = flags.Bool("watch-ingress-without-class", false,
@@ -203,6 +203,8 @@ Takes the form "<host>:port". If not provided, no admission controller is starte
 		postShutdownGracePeriod = flags.Int("post-shutdown-grace-period", 10, "Seconds to wait after the nginx process has stopped before controller exits.")
 
 		deepInspector = flags.Bool("deep-inspect", true, "Enables ingress object security deep inspector")
+
+		dynamicConfigurationRetries = flags.Int("dynamic-configuration-retries", 15, "Number of times to retry failed dynamic configuration before failing to sync an ingress.")
 	)
 
 	flags.StringVar(&nginx.MaxmindMirror, "maxmind-mirror", "", `Maxmind mirror url (example: http://geoip.local/databases`)
@@ -303,35 +305,36 @@ https://blog.maxmind.com/2019/12/18/significant-changes-to-accessing-and-using-g
 	ngx_config.EnableSSLChainCompletion = *enableSSLChainCompletion
 
 	config := &controller.Configuration{
-		APIServerHost:              *apiserverHost,
+		APIServerHost:               *apiserverHost,
-		KubeConfigFile:             *kubeConfigFile,
+		KubeConfigFile:              *kubeConfigFile,
-		UpdateStatus:               *updateStatus,
+		UpdateStatus:                *updateStatus,
-		ElectionID:                 *electionID,
+		ElectionID:                  *electionID,
-		EnableProfiling:            *profiling,
+		EnableProfiling:             *profiling,
-		EnableMetrics:              *enableMetrics,
+		EnableMetrics:               *enableMetrics,
-		MetricsPerHost:             *metricsPerHost,
+		MetricsPerHost:              *metricsPerHost,
-		MetricsBuckets:             histogramBuckets,
+		MetricsBuckets:              histogramBuckets,
-		MonitorMaxBatchSize:        *monitorMaxBatchSize,
+		MonitorMaxBatchSize:         *monitorMaxBatchSize,
-		DisableServiceExternalName: *disableServiceExternalName,
+		DisableServiceExternalName:  *disableServiceExternalName,
-		EnableSSLPassthrough:       *enableSSLPassthrough,
+		EnableSSLPassthrough:        *enableSSLPassthrough,
-		ResyncPeriod:               *resyncPeriod,
+		ResyncPeriod:                *resyncPeriod,
-		DefaultService:             *defaultSvc,
+		DefaultService:              *defaultSvc,
-		Namespace:                  *watchNamespace,
+		Namespace:                   *watchNamespace,
-		WatchNamespaceSelector:     namespaceSelector,
+		WatchNamespaceSelector:      namespaceSelector,
-		ConfigMapName:              *configMap,
+		ConfigMapName:               *configMap,
-		TCPConfigMapName:           *tcpConfigMapName,
+		TCPConfigMapName:            *tcpConfigMapName,
-		UDPConfigMapName:           *udpConfigMapName,
+		UDPConfigMapName:            *udpConfigMapName,
-		DisableFullValidationTest:  *disableFullValidationTest,
+		DisableFullValidationTest:   *disableFullValidationTest,
-		DefaultSSLCertificate:      *defSSLCertificate,
+		DefaultSSLCertificate:       *defSSLCertificate,
-		DeepInspector:              *deepInspector,
+		DeepInspector:               *deepInspector,
-		PublishService:             *publishSvc,
+		PublishService:              *publishSvc,
-		PublishStatusAddress:       *publishStatusAddress,
+		PublishStatusAddress:        *publishStatusAddress,
-		UpdateStatusOnShutdown:     *updateStatusOnShutdown,
+		UpdateStatusOnShutdown:      *updateStatusOnShutdown,
-		ShutdownGracePeriod:        *shutdownGracePeriod,
+		ShutdownGracePeriod:         *shutdownGracePeriod,
-		PostShutdownGracePeriod:    *postShutdownGracePeriod,
+		PostShutdownGracePeriod:     *postShutdownGracePeriod,
-		UseNodeInternalIP:          *useNodeInternalIP,
+		UseNodeInternalIP:           *useNodeInternalIP,
-		SyncRateLimit:              *syncRateLimit,
+		SyncRateLimit:               *syncRateLimit,
-		HealthCheckHost:            *healthzHost,
+		HealthCheckHost:             *healthzHost,
+		DynamicConfigurationRetries: *dynamicConfigurationRetries,
 		ListenPorts: &ngx_config.ListenPorts{
 			Default:  *defServerPort,
 			Health:   *healthzPort,

internal/ingress/controller/controller.go

@@ -125,6 +125,8 @@ type Configuration struct {
 	InternalLoggerAddress string
 	IsChroot              bool
 	DeepInspector         bool
+
+	DynamicConfigurationRetries int
 }
 
 // GetPublishService returns the Service used to set the load-balancer status of Ingresses.
@@ -194,19 +196,24 @@ func (n *NGINXController) syncIngress(interface{}) error {
 	}
 
 	retry := wait.Backoff{
-		Steps:    15,
+		Steps:    1 + n.cfg.DynamicConfigurationRetries,
-		Duration: 1 * time.Second,
+		Duration: time.Second,
-		Factor:   0.8,
+		Factor:   1.3,
 		Jitter:   0.1,
 	}
 
+	retriesRemaining := retry.Steps
 	err := wait.ExponentialBackoff(retry, func() (bool, error) {
 		err := n.configureDynamically(pcfg)
 		if err == nil {
 			klog.V(2).Infof("Dynamic reconfiguration succeeded.")
 			return true, nil
 		}
-
+		retriesRemaining--
+		if retriesRemaining > 0 {
+			klog.Warningf("Dynamic reconfiguration failed (retrying; %d retries left): %v", retriesRemaining, err)
+			return false, nil
+		}
 		klog.Warningf("Dynamic reconfiguration failed: %v", err)
 		return false, err
 	})