keep backwards compatibility

This commit is contained in:
Moleus 2024-02-14 17:29:09 +03:00
parent 83eff5d1c8
commit 5101d6e12f
4 changed files with 71 additions and 33 deletions

View file

@ -137,7 +137,7 @@ func DownloadGeoLite2DB(attempts int, period time.Duration) error {
lastErr = wait.ExponentialBackoff(defaultRetry, func() (bool, error) { lastErr = wait.ExponentialBackoff(defaultRetry, func() (bool, error) {
var dlError error var dlError error
for _, editionID := range strings.Split(MaxmindEditionIDs, ",") { for _, editionID := range strings.Split(MaxmindEditionIDs, ",") {
dlError = fetchDatabase(editionID) dlError = downloadDatabase(editionID)
if dlError != nil { if dlError != nil {
break break
} }
@ -164,19 +164,10 @@ func DownloadGeoLite2DB(attempts int, period time.Duration) error {
return lastErr return lastErr
} }
func createURL(mirror, editionID, hash string) string { func downloadDatabase(editionID string) error {
if len(mirror) > 0 {
return fmt.Sprintf("%s/%s.tar.gz", mirror, editionID)
}
return fmt.Sprintf(maxmindURLFormat, MaxmindURL, editionID, hash)
}
func fetchDatabase(editionID string) error {
md5hash := getMD5Hash(editionID)
mmdbFile := editionID + dbExtension mmdbFile := editionID + dbExtension
for { result, err := fetchDatabase(editionID)
result, err := requestDatabase(editionID, md5hash)
if err != nil { if err != nil {
return err return err
} }
@ -185,11 +176,13 @@ func fetchDatabase(editionID string) error {
result.Reader.Close() result.Reader.Close()
} }
}() }()
if strings.EqualFold(result.OldHash, result.NewHash) {
if len(MaxmindMirror) == 0 && strings.EqualFold(result.OldHash, result.NewHash) {
return nil return nil
} }
tarReader := tar.NewReader(result.Reader) tarReader := tar.NewReader(result.Reader)
for {
header, err := tarReader.Next() header, err := tarReader.Next()
if err == io.EOF { if err == io.EOF {
break break
@ -219,12 +212,49 @@ func fetchDatabase(editionID string) error {
} }
} }
return fmt.Errorf("the URL %v does not contains the database %v", createURL(MaxmindMirror, editionID, md5hash), mmdbFile) return fmt.Errorf("no %s file in the db archive", mmdbFile)
} }
func requestDatabase(editionID, md5hash string) (*FetchResult, error) { func fetchDatabase(editionID string) (*FetchResult, error) {
newURL := createURL(MaxmindMirror, editionID, md5hash) if len(MaxmindMirror) > 0 {
req, err := http.NewRequest(http.MethodGet, newURL, http.NoBody) return fetchDatabaseFromMirror(editionID)
}
return fetchDatabaseIfUpdated(editionID, calculateMD5Hash(editionID))
}
// backwards compatibility support - fetch directly from mirror without checking md5
// without md5 check and no auth
func fetchDatabaseFromMirror(editionID string) (result *FetchResult, err error) {
mirrorUrl := fmt.Sprintf("%s/%s.tar.gz", MaxmindMirror, editionID)
resp, err := http.Get(mirrorUrl)
if err != nil {
return nil, err
}
defer func() {
if err != nil {
resp.Body.Close()
}
}()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("unexpected status code: %v", resp.Status)
}
archive, err := gzip.NewReader(resp.Body)
if err != nil {
return nil, err
}
return &FetchResult{
Reader: archive,
EditionID: editionID,
}, nil
}
func fetchDatabaseIfUpdated(editionID, md5hash string) (result *FetchResult, err error) {
updateDbUrl := fmt.Sprintf(maxmindURLFormat, MaxmindURL, editionID, md5hash)
req, err := http.NewRequest(http.MethodGet, updateDbUrl, http.NoBody)
if err != nil { if err != nil {
return nil, fmt.Errorf("create request: %w", err) return nil, fmt.Errorf("create request: %w", err)
} }
@ -237,7 +267,7 @@ func requestDatabase(editionID, md5hash string) (*FetchResult, error) {
} }
defer func() { defer func() {
if resp.Body != nil { if err != nil {
resp.Body.Close() resp.Body.Close()
} }
}() }()
@ -280,10 +310,9 @@ func requestDatabase(editionID, md5hash string) (*FetchResult, error) {
}, nil }, nil
} }
func getMD5Hash(editionID string) string { func calculateMD5Hash(editionID string) string {
file, err := os.Open(path.Join(geoIPPath, editionID+dbExtension)) file, err := os.Open(path.Join(geoIPPath, editionID+dbExtension))
if err != nil { if err != nil {
klog.ErrorS(err, "error opening file", "file", editionID+dbExtension)
return "" return ""
} }

View file

@ -141,7 +141,7 @@ func TestRequestDatabase(t *testing.T) {
MaxmindURL = server.URL MaxmindURL = server.URL
result, err := requestDatabase(test.requestEdition, test.requestHash) result, err := fetchDatabaseIfUpdated(test.requestEdition, test.requestHash)
test.checkErr(t, err) test.checkErr(t, err)
if err == nil { if err == nil {
require.Equal(t, result.EditionID, test.result.EditionID) require.Equal(t, result.EditionID, test.result.EditionID)

View file

@ -233,7 +233,7 @@ https://blog.maxmind.com/2019/12/18/significant-changes-to-accessing-and-using-g
flags.DurationVar(&nginx.MaxmindRetriesTimeout, "maxmind-retries-timeout", time.Second*0, "Maxmind downloading delay between 1st and 2nd attempt, 0s - do not retry to download if something went wrong.") flags.DurationVar(&nginx.MaxmindRetriesTimeout, "maxmind-retries-timeout", time.Second*0, "Maxmind downloading delay between 1st and 2nd attempt, 0s - do not retry to download if something went wrong.")
flags.BoolVar(&nginx.MaxmindEnableSync, "maxmind-enable-sync", false, "Enable periodic sync of maxmind databases.") flags.BoolVar(&nginx.MaxmindEnableSync, "maxmind-enable-sync", false, "Enable periodic sync of maxmind databases.")
flags.DurationVar(&nginx.MaxmindSyncPeriod, "maxmind-sync-period", time.Hour*24, "Maxmind databases sync period.") flags.DurationVar(&nginx.MaxmindSyncPeriod, "maxmind-sync-period", time.Hour*24, "Maxmind databases sync period.")
flags.StringVar(&nginx.MaxmindURL, "maxmind-url", "https://updates.maxmind.com", "Maxmind URL to download GeoLite2 Databases.") flags.StringVar(&nginx.MaxmindURL, "maxmind-url", "https://updates.maxmind.com", "Maxmind url to download GeoLite2 Databases.")
flags.AddGoFlagSet(flag.CommandLine) flags.AddGoFlagSet(flag.CommandLine)
if err := flags.Parse(os.Args); err != nil { if err := flags.Parse(os.Args); err != nil {

View file

@ -111,5 +111,14 @@ func TestMaxmindRetryDownload(t *testing.T) {
} }
func TestMaxmindSyncPeriod(t *testing.T) { func TestMaxmindSyncPeriod(t *testing.T) {
ResetForTesting() ResetForTesting(func() { t.Fatal("Parsing failed") })
oldArgs := os.Args
defer func() { os.Args = oldArgs }()
os.Args = []string{"cmd", "--publish-service", "namespace/test", "--http-port", "0", "--https-port", "0", "--maxmind-enable-sync", "--maxmind-sync-period", "30s"}
_, _, err := ParseFlags()
if err == nil {
t.Fatalf("Expected an error parsing flags but none returned")
}
} }