From 45995525e7b3cd7b06194770464dace1672b0e06 Mon Sep 17 00:00:00 2001 From: Sergey Shakuto Date: Fri, 9 Jul 2021 01:08:53 +0300 Subject: [PATCH] Retry to download maxmind DB if it fails (#7242) * Retry to download maxmind DB if it fails. Signed-off-by: Sergey Shakuto * Add retries count arg, move retry logic into DownloadGeoLite2DB function Signed-off-by: Sergey Shakuto * Reorder parameters in DownloadGeoLite2DB Signed-off-by: Sergey Shakuto * Remove hardcoded value Signed-off-by: Sergey Shakuto --- cmd/nginx/flags.go | 13 +++--- cmd/nginx/flags_test.go | 13 ++++++ docs/user-guide/cli-arguments.md | 2 + go.mod | 1 + go.sum | 4 +- internal/nginx/maxmind.go | 70 ++++++++++++++++++++++++++++---- 6 files changed, 88 insertions(+), 15 deletions(-) diff --git a/cmd/nginx/flags.go b/cmd/nginx/flags.go index 62c4ced23..1abe7d9e3 100644 --- a/cmd/nginx/flags.go +++ b/cmd/nginx/flags.go @@ -23,10 +23,7 @@ import ( "time" "github.com/spf13/pflag" - apiv1 "k8s.io/api/core/v1" - "k8s.io/klog/v2" - "k8s.io/ingress-nginx/internal/ingress/annotations/class" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" "k8s.io/ingress-nginx/internal/ingress/controller" @@ -34,6 +31,7 @@ import ( "k8s.io/ingress-nginx/internal/ingress/status" ing_net "k8s.io/ingress-nginx/internal/net" "k8s.io/ingress-nginx/internal/nginx" + klog "k8s.io/klog/v2" ) func parseFlags() (bool, *controller.Configuration, error) { @@ -182,6 +180,8 @@ Takes the form ":port". If not provided, no admission controller is starte flags.StringVar(&nginx.MaxmindLicenseKey, "maxmind-license-key", "", `Maxmind license key to download GeoLite2 Databases. https://blog.maxmind.com/2019/12/18/significant-changes-to-accessing-and-using-geolite2-databases`) flags.StringVar(&nginx.MaxmindEditionIDs, "maxmind-edition-ids", "GeoLite2-City,GeoLite2-ASN", `Maxmind edition ids to download GeoLite2 Databases.`) + flags.IntVar(&nginx.MaxmindRetriesCount, "maxmind-retries-count", 1, "Number of attempts to download the GeoIP DB.") + flags.DurationVar(&nginx.MaxmindRetriesTimeout, "maxmind-retries-timeout", time.Second*0, "Maxmind downloading delay between 1st and 2nd attempt, 0s - do not retry to download if something went wrong.") flag.Set("logtostderr", "true") @@ -307,16 +307,17 @@ https://blog.maxmind.com/2019/12/18/significant-changes-to-accessing-and-using-g config.RootCAFile = *rootCAFile } + var err error if (nginx.MaxmindLicenseKey != "" || nginx.MaxmindMirror != "") && nginx.MaxmindEditionIDs != "" { - if err := nginx.ValidateGeoLite2DBEditions(); err != nil { + if err = nginx.ValidateGeoLite2DBEditions(); err != nil { return false, nil, err } klog.InfoS("downloading maxmind GeoIP2 databases") - if err := nginx.DownloadGeoLite2DB(); err != nil { + if err = nginx.DownloadGeoLite2DB(nginx.MaxmindRetriesCount, nginx.MaxmindRetriesTimeout); err != nil { klog.ErrorS(err, "unexpected error downloading GeoIP2 database") } config.MaxmindEditionFiles = nginx.MaxmindEditionFiles } - return false, config, nil + return false, config, err } diff --git a/cmd/nginx/flags_test.go b/cmd/nginx/flags_test.go index 4286d4a20..b25fa7557 100644 --- a/cmd/nginx/flags_test.go +++ b/cmd/nginx/flags_test.go @@ -105,3 +105,16 @@ func TestMaxmindMirror(t *testing.T) { t.Fatalf("Expected an error parsing flags but none returned") } } + +func TestMaxmindRetryDownload(t *testing.T) { + resetForTesting(func() { t.Fatal("Parsing failed") }) + + oldArgs := os.Args + defer func() { os.Args = oldArgs }() + os.Args = []string{"cmd", "--publish-service", "namespace/test", "--http-port", "0", "--https-port", "0", "--maxmind-mirror", "http://127.0.0.1", "--maxmind-license-key", "0000000", "--maxmind-edition-ids", "GeoLite2-City", "--maxmind-retries-timeout", "1s", "--maxmind-retries-count", "3"} + + _, _, err := parseFlags() + if err == nil { + t.Fatalf("Expected an error parsing flags but none returned") + } +} diff --git a/docs/user-guide/cli-arguments.md b/docs/user-guide/cli-arguments.md index 799ce7324..d51c75994 100644 --- a/docs/user-guide/cli-arguments.md +++ b/docs/user-guide/cli-arguments.md @@ -33,6 +33,8 @@ They are set in the container spec of the `nginx-ingress-controller` Deployment | `--log_file_max_size` | Defines the maximum size a log file can grow to. Unit is megabytes. If the value is 0, the maximum file size is unlimited. (default 1800) | | `--logtostderr` | log to standard error instead of files (default true) | | `--maxmind-edition-ids` | Maxmind edition ids to download GeoLite2 Databases. (default "GeoLite2-City,GeoLite2-ASN") | +| `--maxmind-retries-timeout` | Maxmind downloading delay between 1st and 2nd attempt, 0s - do not retry to download if something went wrong. (default 0s) | +| `--maxmind-retries-count` | Number of attempts to download the GeoIP DB. (default 1) | | `--maxmind-license-key` | Maxmind license key to download GeoLite2 Databases. https://blog.maxmind.com/2019/12/18/significant-changes-to-accessing-and-using-geolite2-databases | | `--metrics-per-host` | Export metrics per-host (default true) | | `--profiler-port` | Port to use for expose the ingress controller Go profiler when it is enabled. (default 10245) | diff --git a/go.mod b/go.mod index 02de2cecf..c0e2b350b 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/imdario/mergo v0.3.12 github.com/json-iterator/go v1.1.11 github.com/kylelemons/godebug v1.1.0 + github.com/mattn/go-isatty v0.0.12 // indirect github.com/mitchellh/go-ps v1.0.0 github.com/mitchellh/hashstructure v1.1.0 github.com/mitchellh/mapstructure v1.4.1 diff --git a/go.sum b/go.sum index 795d1bf16..2a19bfd54 100644 --- a/go.sum +++ b/go.sum @@ -468,8 +468,9 @@ github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= @@ -929,6 +930,7 @@ golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/internal/nginx/maxmind.go b/internal/nginx/maxmind.go index 0f3c37016..c764be161 100644 --- a/internal/nginx/maxmind.go +++ b/internal/nginx/maxmind.go @@ -21,10 +21,17 @@ import ( "compress/gzip" "fmt" "io" + "net" "net/http" + "net/url" "os" "path" "strings" + "syscall" + "time" + + "k8s.io/apimachinery/pkg/util/wait" + klog "k8s.io/klog/v2" ) // MaxmindLicenseKey maxmind license key to download databases @@ -39,6 +46,15 @@ var MaxmindEditionFiles []string // MaxmindMirror maxmind database mirror url (http://geoip.local) var MaxmindMirror = "" +// MaxmindRetriesCount number of attempts to download the GeoIP DB +var MaxmindRetriesCount = 1 + +// MaxmindRetriesTimeout maxmind download retries timeout in seconds, 0 - do not retry to download if something went wrong +var MaxmindRetriesTimeout = time.Second * 0 + +// minimumRetriesCount minimum value of the MaxmindRetriesCount parameter. If MaxmindRetriesCount less than minimumRetriesCount, it will be set to minimumRetriesCount +const minimumRetriesCount = 1 + const ( geoIPPath = "/etc/nginx/geoip" dbExtension = ".mmdb" @@ -60,15 +76,53 @@ func GeoLite2DBExists() bool { // DownloadGeoLite2DB downloads the required databases by the // GeoIP2 NGINX module using a license key from MaxMind. -func DownloadGeoLite2DB() error { - for _, dbName := range strings.Split(MaxmindEditionIDs, ",") { - err := downloadDatabase(dbName) - if err != nil { - return err - } - MaxmindEditionFiles = append(MaxmindEditionFiles, dbName+dbExtension) +func DownloadGeoLite2DB(attempts int, period time.Duration) error { + if attempts < minimumRetriesCount { + attempts = minimumRetriesCount } - return nil + + defaultRetry := wait.Backoff{ + Steps: attempts, + Duration: period, + Factor: 1.5, + Jitter: 0.1, + } + if period == time.Duration(0) { + defaultRetry.Steps = minimumRetriesCount + } + + var lastErr error + retries := 0 + + _ = wait.ExponentialBackoff(defaultRetry, func() (bool, error) { + var dlError error + for _, dbName := range strings.Split(MaxmindEditionIDs, ",") { + dlError = downloadDatabase(dbName) + if dlError != nil { + break + } + MaxmindEditionFiles = append(MaxmindEditionFiles, dbName+dbExtension) + } + + lastErr = dlError + if dlError == nil { + return true, nil + } + + if e, ok := dlError.(*url.Error); ok { + if e, ok := e.Err.(*net.OpError); ok { + if e, ok := e.Err.(*os.SyscallError); ok { + if e.Err == syscall.ECONNREFUSED { + retries++ + klog.InfoS("download failed on attempt " + fmt.Sprint(retries)) + return false, nil + } + } + } + } + return true, nil + }) + return lastErr } func createURL(mirror, licenseKey, dbName string) string {