Retry to download maxmind DB if it fails (#7242)

* Retry to download maxmind DB if it fails.

Signed-off-by: Sergey Shakuto <sshakuto@infoblox.com>

* Add retries count arg, move retry logic into DownloadGeoLite2DB function

Signed-off-by: Sergey Shakuto <sshakuto@infoblox.com>

* Reorder parameters in DownloadGeoLite2DB

Signed-off-by: Sergey Shakuto <sshakuto@infoblox.com>

* Remove hardcoded value

Signed-off-by: Sergey Shakuto <sshakuto@infoblox.com>

cmd/nginx/flags.go

@@ -23,10 +23,7 @@ import (
 	"time"
 
 	"github.com/spf13/pflag"
-
 	apiv1 "k8s.io/api/core/v1"
-	"k8s.io/klog/v2"
-
 	"k8s.io/ingress-nginx/internal/ingress/annotations/class"
 	"k8s.io/ingress-nginx/internal/ingress/annotations/parser"
 	"k8s.io/ingress-nginx/internal/ingress/controller"
@@ -34,6 +31,7 @@ import (
 	"k8s.io/ingress-nginx/internal/ingress/status"
 	ing_net "k8s.io/ingress-nginx/internal/net"
 	"k8s.io/ingress-nginx/internal/nginx"
+	klog "k8s.io/klog/v2"
 )
 
 func parseFlags() (bool, *controller.Configuration, error) {
@@ -182,6 +180,8 @@ Takes the form "<host>:port". If not provided, no admission controller is starte
 	flags.StringVar(&nginx.MaxmindLicenseKey, "maxmind-license-key", "", `Maxmind license key to download GeoLite2 Databases.
 https://blog.maxmind.com/2019/12/18/significant-changes-to-accessing-and-using-geolite2-databases`)
 	flags.StringVar(&nginx.MaxmindEditionIDs, "maxmind-edition-ids", "GeoLite2-City,GeoLite2-ASN", `Maxmind edition ids to download GeoLite2 Databases.`)
+	flags.IntVar(&nginx.MaxmindRetriesCount, "maxmind-retries-count", 1, "Number of attempts to download the GeoIP DB.")
+	flags.DurationVar(&nginx.MaxmindRetriesTimeout, "maxmind-retries-timeout", time.Second*0, "Maxmind downloading delay between 1st and 2nd attempt, 0s - do not retry to download if something went wrong.")
 
 	flag.Set("logtostderr", "true")
 
@@ -307,16 +307,17 @@ https://blog.maxmind.com/2019/12/18/significant-changes-to-accessing-and-using-g
 		config.RootCAFile = *rootCAFile
 	}
 
+	var err error
 	if (nginx.MaxmindLicenseKey != "" || nginx.MaxmindMirror != "") && nginx.MaxmindEditionIDs != "" {
-		if err := nginx.ValidateGeoLite2DBEditions(); err != nil {
+		if err = nginx.ValidateGeoLite2DBEditions(); err != nil {
 			return false, nil, err
 		}
 		klog.InfoS("downloading maxmind GeoIP2 databases")
-		if err := nginx.DownloadGeoLite2DB(); err != nil {
+		if err = nginx.DownloadGeoLite2DB(nginx.MaxmindRetriesCount, nginx.MaxmindRetriesTimeout); err != nil {
 			klog.ErrorS(err, "unexpected error downloading GeoIP2 database")
 		}
 		config.MaxmindEditionFiles = nginx.MaxmindEditionFiles
 	}
 
-	return false, config, nil
+	return false, config, err
 }

cmd/nginx/flags_test.go

@@ -105,3 +105,16 @@ func TestMaxmindMirror(t *testing.T) {
 		t.Fatalf("Expected an error parsing flags but none returned")
 	}
 }
+
+func TestMaxmindRetryDownload(t *testing.T) {
+	resetForTesting(func() { t.Fatal("Parsing failed") })
+
+	oldArgs := os.Args
+	defer func() { os.Args = oldArgs }()
+	os.Args = []string{"cmd", "--publish-service", "namespace/test", "--http-port", "0", "--https-port", "0", "--maxmind-mirror", "http://127.0.0.1", "--maxmind-license-key", "0000000", "--maxmind-edition-ids", "GeoLite2-City", "--maxmind-retries-timeout", "1s", "--maxmind-retries-count", "3"}
+
+	_, _, err := parseFlags()
+	if err == nil {
+		t.Fatalf("Expected an error parsing flags but none returned")
+	}
+}

docs/user-guide/cli-arguments.md

@@ -33,6 +33,8 @@ They are set in the container spec of the `nginx-ingress-controller` Deployment
 | `--log_file_max_size`              | Defines the maximum size a log file can grow to. Unit is megabytes. If the value is 0, the maximum file size is unlimited. (default 1800) |
 | `--logtostderr`                    | log to standard error instead of files (default true) |
 | `--maxmind-edition-ids`            | Maxmind edition ids to download GeoLite2 Databases. (default "GeoLite2-City,GeoLite2-ASN") |
+| `--maxmind-retries-timeout`        | Maxmind downloading delay between 1st and 2nd attempt, 0s - do not retry to download if something went wrong. (default 0s) |
+| `--maxmind-retries-count`          | Number of attempts to download the GeoIP DB. (default 1) |
 | `--maxmind-license-key`            | Maxmind license key to download GeoLite2 Databases. https://blog.maxmind.com/2019/12/18/significant-changes-to-accessing-and-using-geolite2-databases |
 | `--metrics-per-host`               | Export metrics per-host (default true) |
 | `--profiler-port`                  | Port to use for expose the ingress controller Go profiler when it is enabled. (default 10245) |

go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/imdario/mergo v0.3.12
 	github.com/json-iterator/go v1.1.11
 	github.com/kylelemons/godebug v1.1.0
+	github.com/mattn/go-isatty v0.0.12 // indirect
 	github.com/mitchellh/go-ps v1.0.0
 	github.com/mitchellh/hashstructure v1.1.0
 	github.com/mitchellh/mapstructure v1.4.1

go.sum

@@ -468,8 +468,9 @@ github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx
 github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
-github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE=
 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
+github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI=
@@ -929,6 +930,7 @@ golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

internal/nginx/maxmind.go

@@ -21,10 +21,17 @@ import (
 	"compress/gzip"
 	"fmt"
 	"io"
+	"net"
 	"net/http"
+	"net/url"
 	"os"
 	"path"
 	"strings"
+	"syscall"
+	"time"
+
+	"k8s.io/apimachinery/pkg/util/wait"
+	klog "k8s.io/klog/v2"
 )
 
 // MaxmindLicenseKey maxmind license key to download databases
@@ -39,6 +46,15 @@ var MaxmindEditionFiles []string
 // MaxmindMirror maxmind database mirror url (http://geoip.local)
 var MaxmindMirror = ""
 
+// MaxmindRetriesCount number of attempts to download the GeoIP DB
+var MaxmindRetriesCount = 1
+
+// MaxmindRetriesTimeout maxmind download retries timeout in seconds, 0 - do not retry to download if something went wrong
+var MaxmindRetriesTimeout = time.Second * 0
+
+// minimumRetriesCount minimum value of the MaxmindRetriesCount parameter. If MaxmindRetriesCount less than minimumRetriesCount, it will be set to minimumRetriesCount
+const minimumRetriesCount = 1
+
 const (
 	geoIPPath   = "/etc/nginx/geoip"
 	dbExtension = ".mmdb"
@@ -60,15 +76,53 @@ func GeoLite2DBExists() bool {
 
 // DownloadGeoLite2DB downloads the required databases by the
 // GeoIP2 NGINX module using a license key from MaxMind.
-func DownloadGeoLite2DB() error {
+func DownloadGeoLite2DB(attempts int, period time.Duration) error {
+	if attempts < minimumRetriesCount {
+		attempts = minimumRetriesCount
+	}
+
+	defaultRetry := wait.Backoff{
+		Steps:    attempts,
+		Duration: period,
+		Factor:   1.5,
+		Jitter:   0.1,
+	}
+	if period == time.Duration(0) {
+		defaultRetry.Steps = minimumRetriesCount
+	}
+
+	var lastErr error
+	retries := 0
+
+	_ = wait.ExponentialBackoff(defaultRetry, func() (bool, error) {
+		var dlError error
 		for _, dbName := range strings.Split(MaxmindEditionIDs, ",") {
-		err := downloadDatabase(dbName)
+			dlError = downloadDatabase(dbName)
-		if err != nil {
+			if dlError != nil {
-			return err
+				break
 			}
 			MaxmindEditionFiles = append(MaxmindEditionFiles, dbName+dbExtension)
 		}
-	return nil
+
+		lastErr = dlError
+		if dlError == nil {
+			return true, nil
+		}
+
+		if e, ok := dlError.(*url.Error); ok {
+			if e, ok := e.Err.(*net.OpError); ok {
+				if e, ok := e.Err.(*os.SyscallError); ok {
+					if e.Err == syscall.ECONNREFUSED {
+						retries++
+						klog.InfoS("download failed on attempt " + fmt.Sprint(retries))
+						return false, nil
+					}
+				}
+			}
+		}
+		return true, nil
+	})
+	return lastErr
 }
 
 func createURL(mirror, licenseKey, dbName string) string {