//go:build mage package main import ( "context" "errors" "fmt" "github.com/google/go-github/v48/github" "github.com/magefile/mage/mg" "github.com/magefile/mage/sh" "golang.org/x/oauth2" "gopkg.in/yaml.v3" "io" "net" "net/http" "os" "text/template" "regexp" "strings" "time" ) type Release mg.Namespace var INGRESS_ORG = "strongjz" // the owner so we can test from forks var INGRESS_REPO = "ingress-nginx" // the repo to pull from var RELEASE_BRANCH = "main" //we only release from main var GITHUB_TOKEN string // the Google/gogithub lib needs an PAT to access the GitHub API var K8S_IO_ORG = "strongjz" //the owner or organization for the k8s.io repo var K8S_IO_REPO = "k8s.io" //the repo that holds the images yaml for production promotion var INGRESS_REGISTRY = "registry.k8s.io" //Container registry for storage Ingress-nginx images // ingress-nginx releases start with a TAG then a cloudbuild, then a promotion through a PR, this the location of that PR // var IMAGES_YAML = "https://raw.githubusercontent.com/kubernetes/k8s.io/main/k8s.gcr.io/images/k8s-staging-ingress-nginx/images.yaml" var IMAGES_YAML = "https://raw.githubusercontent.com/strongjz/k8s.io/testing/k8s.gcr.io/images/k8s-staging-ingress-nginx/images.yaml" var ctx = context.Background() // Context used for GitHub Client const CHANGELOG = "Changelog.md" //Name of the changelog func init() { GITHUB_TOKEN = os.Getenv("GITHUB_TOKEN") } // PromoteImage Creates PR into the k8s.io repo for promotion of ingress from staging to production func (Release) PromoteImage(version, sha string) { } // Release Create a new release of ingress nginx controller func (Release) NewRelease(version string) { //newRelease := Release{} //update ingress-nginx version //This is the step that kicks all the release process //it is already done, so it kicks off the gcloud build of the controller images //mg.Deps(mg.F(Tag.BumpNginx, version)) tag, err := getIngressNGINXVersion() CheckIfError(err, "RELEASE Retrieving the current Ingress Nginx Version") Info("RELEASE Checking Current Version %s to New Version %s", tag, version) //if the version were upgrading does not match the TAG file, lets update the TAG file if tag != version { Warning("RELEASE Ingress Nginx TAG %s and new version %s do not match", tag, version) mg.Deps(mg.F(Tag.BumpNginx, fmt.Sprintf("v%s", version))) } //update git controller tag controller-v$version mg.Deps(mg.F(Tag.NewControllerTag, version)) //make release notes releaseNotes, err := makeReleaseNotes(version) CheckIfError(err, "RELEASE Creating Release Notes for version %s", version) Info("RELEASE Release Notes %s completed", releaseNotes.Version) //update chart values.yaml new controller tag and image digest releaseNotes.PreviousHelmChartVersion = currentChartVersion() //controller tag updateChartValue("controller.image.tag", fmt.Sprintf("v%s", releaseNotes.Version)) //controller digest if releaseNotes.ControllerImages[0].Name == "controller" { updateChartValue("controller.image.digest", releaseNotes.ControllerImages[0].Digest) } //controller chroot digest if releaseNotes.ControllerImages[1].Name == "controller-chroot" { updateChartValue("controller.image.digestChroot", releaseNotes.ControllerImages[1].Digest) } //update helm chart app version mg.Deps(mg.F(Helm.UpdateVersion, version)) releaseNotes.NewHelmChartVersion = currentChartVersion() //update helm chart release notes updateChartReleaseNotes(releaseNotes.HelmUpdates) //Run helm docs update CheckIfError(runHelmDocs(), "Error Updating Helm Docs ") releaseNotes.helmTemplate() //update static manifest CheckIfError(updateStaticManifest(), "Error Updating Static manifests") ////update e2e docs TODO updateE2EDocs() //update documentation with ingress-nginx version CheckIfError(updateIndexMD(releaseNotes.PreviousControllerVersion, releaseNotes.NewControllerVersion), "Error Updating %s", INDEX_DOCS) //git commit TODO //make Pull Request TODO //make release TODO //mg.Deps(mg.F(Release.CreateRelease, version)) } const INDEX_DOCS = "docs/deploy/index.md" func updateIndexMD(old, new string) error { Info("Updating Deploy docs with new version") data, err := os.ReadFile(INDEX_DOCS) CheckIfError(err, "Could not read INDEX_DOCS file %s", INDEX_DOCS) datString := string(data) datString = strings.Replace(datString, old, new, -1) err = os.WriteFile(INDEX_DOCS, []byte(datString), 644) if err != nil { ErrorF("Could not write new %s %s", INDEX_DOCS, err) return err } return nil } func updateE2EDocs() { //hack/generate-deploy-scripts.sh updates, err := sh.Output("./hack/generate-e2e-suite-doc.sh") CheckIfError(err, "Could not run update hack script") err = os.WriteFile("docs/e2e-tests.md", []byte(updates), 644) CheckIfError(err, "Could not write new e2e test file ") } func installKustomize() error { Info("Install Kustomize") var g0 = sh.RunCmd("go") // somewhere in your main code err := g0("install", "sigs.k8s.io/kustomize/kustomize/v4@v4.5.4") if err != nil { return err } return nil } func updateStaticManifest() error { CheckIfError(installKustomize(), "error installing kustomize") //hack/generate-deploy-scripts.sh err := sh.RunV("./hack/generate-deploy-scripts.sh") if err != nil { return err } return nil } //// CreateRelease Creates a new GitHub Release //func (Release) CreateRelease(name string) { // releaser, err := gh_release.NewReleaser(INGRESS_ORG, INGRESS_REPO, GITHUB_TOKEN) // CheckIfError(err, "GitHub Release Client error") // newRelease, err := releaser.Create(fmt.Sprintf("controller-%s", name)) // CheckIfError(err, "Create release error") // Info("New Release: Tag %v, ID: %v", newRelease.TagName, newRelease.ID) //} type ControllerImage struct { Tag string Digest string Registry string Name string } type IngressRelease struct { ControllerVersion string ControllerImage ControllerImage ReleaseNote ReleaseNote Release *github.RepositoryRelease } type ReleaseNote struct { Version string NewControllerVersion string PreviousControllerVersion string ControllerImages []ControllerImage DepUpdates []string Updates []string HelmUpdates []string NewHelmChartVersion string PreviousHelmChartVersion string } // Returns a GitHub client ready for use func githubClient() *github.Client { ts := oauth2.StaticTokenSource( &oauth2.Token{AccessToken: GITHUB_TOKEN}, ) oauthClient := oauth2.NewClient(ctx, ts) return github.NewClient(oauthClient) } // LatestCommitLogs Retrieves the commit log between the latest two controller versions. func (Release) LatestCommitLogs() { commitLog := commitsBetweenTags() for i, s := range commitLog { Info("#%v Version %v", i, s) } } func commitsBetweenTags() []string { tags := getAllControllerTags() Info("Getting Commits between %v and %v", tags[0], tags[1]) commitLog, err := git("log", "--full-history", "--pretty", "--oneline", fmt.Sprintf("%v..%v", tags[1], tags[0])) if commitLog == "" { Warning("All Controller Tags is empty") } CheckIfError(err, "Retrieving Commit log") return strings.Split(commitLog, "\n") } // Generate Release Notes func (Release) ReleaseNotes(newVersion string) error { notes, err := makeReleaseNotes(newVersion) CheckIfError(err, "Creating Release Notes for version %s", newVersion) Info("Release Notes %s completed", notes.Version) return nil } func makeReleaseNotes(newVersion string) (*ReleaseNote, error) { var newReleaseNotes = ReleaseNote{} newReleaseNotes.Version = newVersion allControllerTags := getAllControllerTags() //new version newReleaseNotes.NewControllerVersion = allControllerTags[0] newControllerVersion := fmt.Sprintf("controller-v%s", newVersion) //the newControllerVersion should match the latest tag if newControllerVersion != allControllerTags[0] { return nil, errors.New(fmt.Sprintf("Generating release new version %s didnt match the current latest tag %s", newControllerVersion, allControllerTags[0])) } //previous version newReleaseNotes.PreviousControllerVersion = allControllerTags[1] Info("New Version: %s Old Version: %s", newReleaseNotes.NewControllerVersion, newReleaseNotes.PreviousControllerVersion) commits := commitsBetweenTags() //dependency_updates //all_updates var allUpdates []string var depUpdates []string var helmUpdates []string prRegex := regexp.MustCompile("\\(#\\d+\\)") depBot := regexp.MustCompile("^(\\w){1,10} Bump ") helmRegex := regexp.MustCompile("helm|chart") for i, s := range commits { //matches on PR if prRegex.Match([]byte(s)) { //matches a dependant bot update if depBot.Match([]byte(s)) { // Debug("#%v DEPENDABOT %v", i, s) u := strings.SplitN(s, " ", 2) depUpdates = append(depUpdates, u[1]) } else { // add it to the all updates slice Debug("#%v ALL UPDATES %v", i, s) u := strings.SplitN(s, " ", 2) allUpdates = append(allUpdates, u[1]) //helm chart updates if helmRegex.Match([]byte(s)) { u := strings.SplitN(s, " ", 2) helmUpdates = append(helmUpdates, u[1]) } } } } helmUpdates = append(helmUpdates, fmt.Sprintf("Update Ingress-Nginx version %s", newReleaseNotes.NewControllerVersion)) newReleaseNotes.Updates = allUpdates newReleaseNotes.DepUpdates = depUpdates newReleaseNotes.HelmUpdates = helmUpdates //controller_image_digests imagesYaml, err := downloadFile(IMAGES_YAML) if err != nil { ErrorF("Could not download file %s : %s", IMAGES_YAML, err) return nil, err } Debug("%s", imagesYaml) data := ImageYamls{} err = yaml.Unmarshal([]byte(imagesYaml), &data) if err != nil { ErrorF("Could not unmarshal images yaml %s", err) return nil, err } //controller controllerDigest := findImageDigest(data, "controller", newVersion) if len(controllerDigest) == 0 { ErrorF("Controller Digest could not be found") return nil, errors.New("Controller digest could not be found") } controllerChrootDigest := findImageDigest(data, "controller-chroot", newVersion) if len(controllerChrootDigest) == 0 { ErrorF("Controller Chroot Digest could not be found") return nil, errors.New("Controller Chroot digest could not be found") } Debug("Latest Controller Digest %v", controllerDigest) Debug("Latest Controller Chroot Digest %v", controllerChrootDigest) c1 := ControllerImage{ Digest: controllerDigest, Registry: INGRESS_REGISTRY, Name: "controller", Tag: newReleaseNotes.NewControllerVersion, } c2 := ControllerImage{ Digest: controllerChrootDigest, Registry: INGRESS_REGISTRY, Name: "controller-chroot", Tag: newReleaseNotes.NewControllerVersion, } newReleaseNotes.ControllerImages = append(newReleaseNotes.ControllerImages, c1) newReleaseNotes.ControllerImages = append(newReleaseNotes.ControllerImages, c2) Debug("New Release Controller Images %s %s", newReleaseNotes.ControllerImages[0].Digest, newReleaseNotes.ControllerImages[1].Digest) if DEBUG { newReleaseNotes.printRelease() } //write it all out to the changelog file newReleaseNotes.template() return &newReleaseNotes, nil } func (i ControllerImage) print() string { return fmt.Sprintf("%s/%s:%s@%s", i.Registry, i.Name, i.Tag, i.Digest) } func (r ReleaseNote) template() { // Files are provided as a slice of strings. changelogTemplate, err := os.ReadFile("Changelog.md.gotmpl") if err != nil { ErrorF("Could not read changelog template file %s", err) } Debug("ChangeLog Templates %s", string(changelogTemplate)) t := template.Must(template.New("changelog").Parse(string(changelogTemplate))) // create a new file file, err := os.Create(fmt.Sprintf("changelog/Changelog-%s.md", r.Version)) if err != nil { ErrorF("Could not create changelog file %s", err) } defer file.Close() err = t.Execute(file, r) if err != nil { ErrorF("executing template:", err) } } func (r ReleaseNote) helmTemplate() { // Files are provided as a slice of strings. changelogTemplate, err := os.ReadFile("charts/ingress-nginx/changelog.md.gotmpl") if err != nil { ErrorF("Could not read changelog template file %s", err) } Debug("ChangeLog Templates %s", string(changelogTemplate)) t := template.Must(template.New("changelog").Parse(string(changelogTemplate))) // create a new file file, err := os.Create(fmt.Sprintf("charts/ingress-nginx/changelog/Changelog-%s.md", r.Version)) if err != nil { ErrorF("Could not create changelog file %s", err) } defer file.Close() err = t.Execute(file, r) if err != nil { ErrorF("executing template:", err) } } func (r ReleaseNote) printRelease() { Info("Release Version: %v", r.NewControllerVersion) Info("Previous Version: %v", r.PreviousControllerVersion) Info("Controller Image: %v", r.ControllerImages[0].print()) Info("Controller Chroot Image: %v", r.ControllerImages[1].print()) for i := range r.Updates { Info("Update #%v - %v", i, r.Updates[i]) } for j := range r.DepUpdates { Info("Dependabot Update #%v - %v", j, r.DepUpdates[j]) } } type ImageYamls []ImageElement type ImageElement struct { Name string `json:"name"` Dmap map[string][]string `json:"dmap"` } func findImageDigest(yaml ImageYamls, image, version string) string { version = fmt.Sprintf("v%s", version) Info("Searching Digest for %s:%s", image, version) for i := range yaml { if yaml[i].Name == image { for k, v := range yaml[i].Dmap { if v[0] == version { return k } } return "" } } return "" } func downloadFile(url string) (string, error) { client := &http.Client{ Transport: &http.Transport{ DialContext: (&net.Dialer{ Timeout: 5 * time.Second, KeepAlive: 5 * time.Second, }).DialContext, TLSHandshakeTimeout: 5 * time.Second, ResponseHeaderTimeout: 5 * time.Second, ExpectContinueTimeout: 1 * time.Second, MaxIdleConnsPerHost: -1, }, } resp, err := client.Get(url) if err != nil { return "", nil } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return "", errors.New(fmt.Sprintf("Could not retrieve file, response from server %s for file %s", resp.StatusCode, url)) } bodyBytes, err := io.ReadAll(resp.Body) if err != nil { return "", nil } return string(bodyBytes), nil } // Latest returns latest Github Release func (Release) Latest() error { r, _, err := latestRelease() if err != nil { ErrorF("Latest Release error %s", err) return err } Info("Latest Release %v", r.String()) return nil } func (Release) ReleaseByTag(tag string) error { r, _, err := releaseByTag(tag) if err != nil { ErrorF("Release retrieve tag error %s", tag, err) return err } Info("Latest Release %v", r.String()) return nil } func releaseByTag(tag string) (*github.RepositoryRelease, *github.Response, error) { ghClient := githubClient() return ghClient.Repositories.GetReleaseByTag(ctx, INGRESS_ORG, INGRESS_REPO, tag) } func latestRelease() (*github.RepositoryRelease, *github.Response, error) { ghClient := githubClient() return ghClient.Repositories.GetLatestRelease(ctx, INGRESS_ORG, INGRESS_REPO) } // Copy Test function to copy a release func (Release) Copy() error { ghClient := githubClient() kRelease, _, err := ghClient.Repositories.GetLatestRelease(ctx, "kubernetes", "ingress-nginx") if err != nil { ErrorF("Get Release from kubernetes %s", err) return err } sRelease := &github.RepositoryRelease{ TagName: kRelease.TagName, Name: kRelease.Name, Body: kRelease.Body, Draft: kRelease.Draft, Prerelease: kRelease.GenerateReleaseNotes, DiscussionCategoryName: kRelease.DiscussionCategoryName, GenerateReleaseNotes: kRelease.GenerateReleaseNotes, } sRelease, _, err = ghClient.Repositories.CreateRelease(ctx, "strongjz", "ingress-nginx", sRelease) if err != nil { ErrorF("Creating Strongjz release %s", err) return err } Info("Copied over Kubernetes Release %v to Strongjz %v", &kRelease.Name, &sRelease.Name) return nil }