keep backwards compatibility
This commit is contained in:
parent
83eff5d1c8
commit
5101d6e12f
4 changed files with 71 additions and 33 deletions
|
@ -137,7 +137,7 @@ func DownloadGeoLite2DB(attempts int, period time.Duration) error {
|
||||||
lastErr = wait.ExponentialBackoff(defaultRetry, func() (bool, error) {
|
lastErr = wait.ExponentialBackoff(defaultRetry, func() (bool, error) {
|
||||||
var dlError error
|
var dlError error
|
||||||
for _, editionID := range strings.Split(MaxmindEditionIDs, ",") {
|
for _, editionID := range strings.Split(MaxmindEditionIDs, ",") {
|
||||||
dlError = fetchDatabase(editionID)
|
dlError = downloadDatabase(editionID)
|
||||||
if dlError != nil {
|
if dlError != nil {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
|
@ -164,32 +164,25 @@ func DownloadGeoLite2DB(attempts int, period time.Duration) error {
|
||||||
return lastErr
|
return lastErr
|
||||||
}
|
}
|
||||||
|
|
||||||
func createURL(mirror, editionID, hash string) string {
|
func downloadDatabase(editionID string) error {
|
||||||
if len(mirror) > 0 {
|
|
||||||
return fmt.Sprintf("%s/%s.tar.gz", mirror, editionID)
|
|
||||||
}
|
|
||||||
return fmt.Sprintf(maxmindURLFormat, MaxmindURL, editionID, hash)
|
|
||||||
}
|
|
||||||
|
|
||||||
func fetchDatabase(editionID string) error {
|
|
||||||
md5hash := getMD5Hash(editionID)
|
|
||||||
mmdbFile := editionID + dbExtension
|
mmdbFile := editionID + dbExtension
|
||||||
|
|
||||||
for {
|
result, err := fetchDatabase(editionID)
|
||||||
result, err := requestDatabase(editionID, md5hash)
|
if err != nil {
|
||||||
if err != nil {
|
return err
|
||||||
return err
|
}
|
||||||
}
|
defer func() {
|
||||||
defer func() {
|
if result.Reader != nil {
|
||||||
if result.Reader != nil {
|
result.Reader.Close()
|
||||||
result.Reader.Close()
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
if strings.EqualFold(result.OldHash, result.NewHash) {
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
tarReader := tar.NewReader(result.Reader)
|
if len(MaxmindMirror) == 0 && strings.EqualFold(result.OldHash, result.NewHash) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
tarReader := tar.NewReader(result.Reader)
|
||||||
|
for {
|
||||||
header, err := tarReader.Next()
|
header, err := tarReader.Next()
|
||||||
if err == io.EOF {
|
if err == io.EOF {
|
||||||
break
|
break
|
||||||
|
@ -219,12 +212,49 @@ func fetchDatabase(editionID string) error {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return fmt.Errorf("the URL %v does not contains the database %v", createURL(MaxmindMirror, editionID, md5hash), mmdbFile)
|
return fmt.Errorf("no %s file in the db archive", mmdbFile)
|
||||||
}
|
}
|
||||||
|
|
||||||
func requestDatabase(editionID, md5hash string) (*FetchResult, error) {
|
func fetchDatabase(editionID string) (*FetchResult, error) {
|
||||||
newURL := createURL(MaxmindMirror, editionID, md5hash)
|
if len(MaxmindMirror) > 0 {
|
||||||
req, err := http.NewRequest(http.MethodGet, newURL, http.NoBody)
|
return fetchDatabaseFromMirror(editionID)
|
||||||
|
}
|
||||||
|
return fetchDatabaseIfUpdated(editionID, calculateMD5Hash(editionID))
|
||||||
|
}
|
||||||
|
|
||||||
|
// backwards compatibility support - fetch directly from mirror without checking md5
|
||||||
|
// without md5 check and no auth
|
||||||
|
func fetchDatabaseFromMirror(editionID string) (result *FetchResult, err error) {
|
||||||
|
mirrorUrl := fmt.Sprintf("%s/%s.tar.gz", MaxmindMirror, editionID)
|
||||||
|
resp, err := http.Get(mirrorUrl)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
defer func() {
|
||||||
|
if err != nil {
|
||||||
|
resp.Body.Close()
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
return nil, fmt.Errorf("unexpected status code: %v", resp.Status)
|
||||||
|
}
|
||||||
|
|
||||||
|
archive, err := gzip.NewReader(resp.Body)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return &FetchResult{
|
||||||
|
Reader: archive,
|
||||||
|
EditionID: editionID,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func fetchDatabaseIfUpdated(editionID, md5hash string) (result *FetchResult, err error) {
|
||||||
|
updateDbUrl := fmt.Sprintf(maxmindURLFormat, MaxmindURL, editionID, md5hash)
|
||||||
|
req, err := http.NewRequest(http.MethodGet, updateDbUrl, http.NoBody)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("create request: %w", err)
|
return nil, fmt.Errorf("create request: %w", err)
|
||||||
}
|
}
|
||||||
|
@ -237,7 +267,7 @@ func requestDatabase(editionID, md5hash string) (*FetchResult, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
defer func() {
|
defer func() {
|
||||||
if resp.Body != nil {
|
if err != nil {
|
||||||
resp.Body.Close()
|
resp.Body.Close()
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
@ -280,10 +310,9 @@ func requestDatabase(editionID, md5hash string) (*FetchResult, error) {
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func getMD5Hash(editionID string) string {
|
func calculateMD5Hash(editionID string) string {
|
||||||
file, err := os.Open(path.Join(geoIPPath, editionID+dbExtension))
|
file, err := os.Open(path.Join(geoIPPath, editionID+dbExtension))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
klog.ErrorS(err, "error opening file", "file", editionID+dbExtension)
|
|
||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -141,7 +141,7 @@ func TestRequestDatabase(t *testing.T) {
|
||||||
|
|
||||||
MaxmindURL = server.URL
|
MaxmindURL = server.URL
|
||||||
|
|
||||||
result, err := requestDatabase(test.requestEdition, test.requestHash)
|
result, err := fetchDatabaseIfUpdated(test.requestEdition, test.requestHash)
|
||||||
test.checkErr(t, err)
|
test.checkErr(t, err)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
require.Equal(t, result.EditionID, test.result.EditionID)
|
require.Equal(t, result.EditionID, test.result.EditionID)
|
||||||
|
|
|
@ -233,7 +233,7 @@ https://blog.maxmind.com/2019/12/18/significant-changes-to-accessing-and-using-g
|
||||||
flags.DurationVar(&nginx.MaxmindRetriesTimeout, "maxmind-retries-timeout", time.Second*0, "Maxmind downloading delay between 1st and 2nd attempt, 0s - do not retry to download if something went wrong.")
|
flags.DurationVar(&nginx.MaxmindRetriesTimeout, "maxmind-retries-timeout", time.Second*0, "Maxmind downloading delay between 1st and 2nd attempt, 0s - do not retry to download if something went wrong.")
|
||||||
flags.BoolVar(&nginx.MaxmindEnableSync, "maxmind-enable-sync", false, "Enable periodic sync of maxmind databases.")
|
flags.BoolVar(&nginx.MaxmindEnableSync, "maxmind-enable-sync", false, "Enable periodic sync of maxmind databases.")
|
||||||
flags.DurationVar(&nginx.MaxmindSyncPeriod, "maxmind-sync-period", time.Hour*24, "Maxmind databases sync period.")
|
flags.DurationVar(&nginx.MaxmindSyncPeriod, "maxmind-sync-period", time.Hour*24, "Maxmind databases sync period.")
|
||||||
flags.StringVar(&nginx.MaxmindURL, "maxmind-url", "https://updates.maxmind.com", "Maxmind URL to download GeoLite2 Databases.")
|
flags.StringVar(&nginx.MaxmindURL, "maxmind-url", "https://updates.maxmind.com", "Maxmind url to download GeoLite2 Databases.")
|
||||||
|
|
||||||
flags.AddGoFlagSet(flag.CommandLine)
|
flags.AddGoFlagSet(flag.CommandLine)
|
||||||
if err := flags.Parse(os.Args); err != nil {
|
if err := flags.Parse(os.Args); err != nil {
|
||||||
|
|
|
@ -111,5 +111,14 @@ func TestMaxmindRetryDownload(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMaxmindSyncPeriod(t *testing.T) {
|
func TestMaxmindSyncPeriod(t *testing.T) {
|
||||||
ResetForTesting()
|
ResetForTesting(func() { t.Fatal("Parsing failed") })
|
||||||
|
|
||||||
|
oldArgs := os.Args
|
||||||
|
defer func() { os.Args = oldArgs }()
|
||||||
|
os.Args = []string{"cmd", "--publish-service", "namespace/test", "--http-port", "0", "--https-port", "0", "--maxmind-enable-sync", "--maxmind-sync-period", "30s"}
|
||||||
|
|
||||||
|
_, _, err := ParseFlags()
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("Expected an error parsing flags but none returned")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue