DevFW-CICD/ingress-nginx-helm
12
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix for buggy ingress sync with retries (#8325)

Browse source
This commit is contained in:
David Shay 2022-04-11 14:42:06 -04:00 committed by GitHub
parent 89ed571d2a
commit 47a266df45
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 44 additions and 34 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

61
cmd/nginx/flags.go
View file

@ -203,6 +203,8 @@ Takes the form "<host>:port". If not provided, no admission controller is starte
postShutdownGracePeriod = flags.Int("post-shutdown-grace-period", 10, "Seconds to wait after the nginx process has stopped before controller exits.") postShutdownGracePeriod = flags.Int("post-shutdown-grace-period", 10, "Seconds to wait after the nginx process has stopped before controller exits.")
deepInspector = flags.Bool("deep-inspect", true, "Enables ingress object security deep inspector") deepInspector = flags.Bool("deep-inspect", true, "Enables ingress object security deep inspector")
dynamicConfigurationRetries = flags.Int("dynamic-configuration-retries", 15, "Number of times to retry failed dynamic configuration before failing to sync an ingress.")
) )
flags.StringVar(&nginx.MaxmindMirror, "maxmind-mirror", "", `Maxmind mirror url (example: http://geoip.local/databases`) flags.StringVar(&nginx.MaxmindMirror, "maxmind-mirror", "", `Maxmind mirror url (example: http://geoip.local/databases`)
@ -303,35 +305,36 @@ https://blog.maxmind.com/2019/12/18/significant-changes-to-accessing-and-using-g
ngx_config.EnableSSLChainCompletion = *enableSSLChainCompletion ngx_config.EnableSSLChainCompletion = *enableSSLChainCompletion
config := &controller.Configuration{ config := &controller.Configuration{
APIServerHost: *apiserverHost, APIServerHost: *apiserverHost,
KubeConfigFile: *kubeConfigFile, KubeConfigFile: *kubeConfigFile,
UpdateStatus: *updateStatus, UpdateStatus: *updateStatus,
ElectionID: *electionID, ElectionID: *electionID,
EnableProfiling: *profiling, EnableProfiling: *profiling,
EnableMetrics: *enableMetrics, EnableMetrics: *enableMetrics,
MetricsPerHost: *metricsPerHost, MetricsPerHost: *metricsPerHost,
MetricsBuckets: histogramBuckets, MetricsBuckets: histogramBuckets,
MonitorMaxBatchSize: *monitorMaxBatchSize, MonitorMaxBatchSize: *monitorMaxBatchSize,
DisableServiceExternalName: *disableServiceExternalName, DisableServiceExternalName: *disableServiceExternalName,
EnableSSLPassthrough: *enableSSLPassthrough, EnableSSLPassthrough: *enableSSLPassthrough,
ResyncPeriod: *resyncPeriod, ResyncPeriod: *resyncPeriod,
DefaultService: *defaultSvc, DefaultService: *defaultSvc,
Namespace: *watchNamespace, Namespace: *watchNamespace,
WatchNamespaceSelector: namespaceSelector, WatchNamespaceSelector: namespaceSelector,
ConfigMapName: *configMap, ConfigMapName: *configMap,
TCPConfigMapName: *tcpConfigMapName, TCPConfigMapName: *tcpConfigMapName,
UDPConfigMapName: *udpConfigMapName, UDPConfigMapName: *udpConfigMapName,
DisableFullValidationTest: *disableFullValidationTest, DisableFullValidationTest: *disableFullValidationTest,
DefaultSSLCertificate: *defSSLCertificate, DefaultSSLCertificate: *defSSLCertificate,
DeepInspector: *deepInspector, DeepInspector: *deepInspector,
PublishService: *publishSvc, PublishService: *publishSvc,
PublishStatusAddress: *publishStatusAddress, PublishStatusAddress: *publishStatusAddress,
UpdateStatusOnShutdown: *updateStatusOnShutdown, UpdateStatusOnShutdown: *updateStatusOnShutdown,
ShutdownGracePeriod: *shutdownGracePeriod, ShutdownGracePeriod: *shutdownGracePeriod,
PostShutdownGracePeriod: *postShutdownGracePeriod, PostShutdownGracePeriod: *postShutdownGracePeriod,
UseNodeInternalIP: *useNodeInternalIP, UseNodeInternalIP: *useNodeInternalIP,
SyncRateLimit: *syncRateLimit, SyncRateLimit: *syncRateLimit,
HealthCheckHost: *healthzHost, HealthCheckHost: *healthzHost,
DynamicConfigurationRetries: *dynamicConfigurationRetries,
ListenPorts: &ngx_config.ListenPorts{ ListenPorts: &ngx_config.ListenPorts{
Default: *defServerPort, Default: *defServerPort,
Health: *healthzPort, Health: *healthzPort,

15
internal/ingress/controller/controller.go
View file

@ -125,6 +125,8 @@ type Configuration struct {
InternalLoggerAddress string InternalLoggerAddress string
IsChroot bool IsChroot bool
DeepInspector bool DeepInspector bool
DynamicConfigurationRetries int
} }
// GetPublishService returns the Service used to set the load-balancer status of Ingresses. // GetPublishService returns the Service used to set the load-balancer status of Ingresses.
@ -194,19 +196,24 @@ func (n *NGINXController) syncIngress(interface{}) error {
} }
retry := wait.Backoff{ retry := wait.Backoff{
Steps: 15, Steps: 1 + n.cfg.DynamicConfigurationRetries,
Duration: 1 * time.Second, Duration: time.Second,
Factor: 0.8, Factor: 1.3,
Jitter: 0.1, Jitter: 0.1,
} }
retriesRemaining := retry.Steps
err := wait.ExponentialBackoff(retry, func() (bool, error) { err := wait.ExponentialBackoff(retry, func() (bool, error) {
err := n.configureDynamically(pcfg) err := n.configureDynamically(pcfg)
if err == nil { if err == nil {
klog.V(2).Infof("Dynamic reconfiguration succeeded.") klog.V(2).Infof("Dynamic reconfiguration succeeded.")
return true, nil return true, nil
} }
retriesRemaining--
if retriesRemaining > 0 {
klog.Warningf("Dynamic reconfiguration failed (retrying; %d retries left): %v", retriesRemaining, err)
return false, nil
}
klog.Warningf("Dynamic reconfiguration failed: %v", err) klog.Warningf("Dynamic reconfiguration failed: %v", err)
return false, err return false, err
}) })