//go:build mage /* Copyright 2023 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package main import ( "context" "errors" "fmt" "github.com/google/go-github/v48/github" "github.com/magefile/mage/mg" "github.com/magefile/mage/sh" "golang.org/x/oauth2" "gopkg.in/yaml.v3" "io" "net" "net/http" "os" "text/template" "regexp" "strings" "time" ) type Release mg.Namespace var INGRESS_ORG = "kubernetes" // the owner so we can test from forks var INGRESS_REPO = "ingress-nginx" // the repo to pull from var RELEASE_BRANCH = "main" //we only release from main var GITHUB_TOKEN string // the Google/gogithub lib needs an PAT to access the GitHub API var K8S_IO_ORG = "kubernetes" //the owner or organization for the k8s.io repo var K8S_IO_REPO = "k8s.io" //the repo that holds the images yaml for production promotion var INGRESS_REGISTRY = "registry.k8s.io" //Container registry for storage Ingress-nginx images var KUSTOMIZE_INSTALL_VERSION = "sigs.k8s.io/kustomize/kustomize/v4@v4.5.4" //static deploys needs kustomize to generate the template // ingress-nginx releases start with a TAG then a cloudbuild, then a promotion through a PR, this the location of that PR var IMAGES_YAML = "https://raw.githubusercontent.com/kubernetes/k8s.io/main/registry.k8s.io/images/k8s-staging-ingress-nginx/images.yaml" var ctx = context.Background() // Context used for GitHub Client const INDEX_DOCS = "docs/deploy/index.md" //index.md has a version of the controller and needs to updated const CHANGELOG = "Changelog.md" //Name of the changelog // ControllerImage - struct with info about controllers type ControllerImage struct { Tag string Digest string Registry string Name string } // IngressRelease All the information about an ingress-nginx release that gets updated type IngressRelease struct { ControllerVersion string ControllerImage ControllerImage ReleaseNote ReleaseNote Release *github.RepositoryRelease } // ReleaseNote - All the pieces of information/documents that get updated during a release type ReleaseNote struct { Version string NewControllerVersion string PreviousControllerVersion string ControllerImages []ControllerImage DepUpdates []string Updates []string HelmUpdates []string NewHelmChartVersion string PreviousHelmChartVersion string } // IMAGES_YAML returns this data structure type ImageYamls []ImageElement // ImageElement - a specific image and it's data structure the dmap is a list of shas and container versions type ImageElement struct { Name string `json:"name"` Dmap map[string][]string `json:"dmap"` } // init will set the GitHub token from the committers/releasers env var func init() { GITHUB_TOKEN = os.Getenv("GITHUB_TOKEN") } // PromoteImage Creates PR into the k8s.io repo for promotion of ingress from staging to production func (Release) PromoteImage(version, sha string) { } // Release Create a new release of ingress nginx controller func (Release) NewRelease(version string) { //newRelease := Release{} //update ingress-nginx version //This is the step that kicks all the release process //it is already done, so it kicks off the gcloud build of the controller images //mg.Deps(mg.F(Tag.BumpNginx, version)) tag, err := getIngressNGINXVersion() CheckIfError(err, "RELEASE Retrieving the current Ingress Nginx Version") Info("RELEASE Checking Current Version %s to New Version %s", tag, version) //if the version were upgrading does not match the TAG file, lets update the TAG file if tag[1:] != version { Warning("RELEASE Ingress Nginx TAG %s and new version %s do not match", tag, version) mg.Deps(mg.F(Tag.BumpNginx, fmt.Sprintf("v%s", version))) } //update git controller tag controller-v$version mg.Deps(mg.F(Tag.NewControllerTag, version)) //make release notes releaseNotes, err := makeReleaseNotes(version) CheckIfError(err, "RELEASE Creating Release Notes for version %s", version) Info("RELEASE Release Notes %s completed", releaseNotes.Version) //update chart values.yaml new controller tag and image digest releaseNotes.PreviousHelmChartVersion = currentChartVersion() //controller tag updateChartValue("controller.image.tag", fmt.Sprintf("v%s", releaseNotes.Version)) //controller digest if releaseNotes.ControllerImages[0].Name == "controller" { updateChartValue("controller.image.digest", releaseNotes.ControllerImages[0].Digest) } //controller chroot digest if releaseNotes.ControllerImages[1].Name == "controller-chroot" { updateChartValue("controller.image.digestChroot", releaseNotes.ControllerImages[1].Digest) } //update helm chart app version mg.Deps(mg.F(Helm.UpdateVersion, version)) releaseNotes.NewHelmChartVersion = currentChartVersion() //update helm chart release notes updateChartReleaseNotes(releaseNotes.HelmUpdates) //Run helm docs update CheckIfError(runHelmDocs(), "Error Updating Helm Docs ") releaseNotes.helmTemplate() //update static manifest CheckIfError(updateStaticManifest(), "Error Updating Static manifests") ////update e2e docs updateE2EDocs() //update documentation with ingress-nginx version CheckIfError(updateIndexMD(releaseNotes.PreviousControllerVersion, releaseNotes.NewControllerVersion), "Error Updating %s", INDEX_DOCS) //keeping these manual for now //git commit TODO //make Pull Request TODO //make release TODO //mg.Deps(mg.F(Release.CreateRelease, version)) } // the index.md doc needs the controller version updated func updateIndexMD(old, new string) error { Info("Updating Deploy docs with new version") data, err := os.ReadFile(INDEX_DOCS) CheckIfError(err, "Could not read INDEX_DOCS file %s", INDEX_DOCS) datString := string(data) datString = strings.Replace(datString, old, new, -1) err = os.WriteFile(INDEX_DOCS, []byte(datString), 644) if err != nil { ErrorF("Could not write new %s %s", INDEX_DOCS, err) return err } return nil } // runs the hack/generate-deploy-scripts.sh func updateE2EDocs() { updates, err := sh.Output("./hack/generate-e2e-suite-doc.sh") CheckIfError(err, "Could not run update hack script") err = os.WriteFile("docs/e2e-tests.md", []byte(updates), 644) CheckIfError(err, "Could not write new e2e test file ") } // The static deploy scripts use kustomize to generate them, this function ensures kustomize is installed func installKustomize() error { Info("Install Kustomize") var g0 = sh.RunCmd("go") // somewhere in your main code err := g0("install", KUSTOMIZE_INSTALL_VERSION) if err != nil { return err } return nil } func updateStaticManifest() error { CheckIfError(installKustomize(), "error installing kustomize") //hack/generate-deploy-scripts.sh err := sh.RunV("./hack/generate-deploy-scripts.sh") if err != nil { return err } return nil } //// CreateRelease Creates a new GitHub Release //func (Release) CreateRelease(name string) { // releaser, err := gh_release.NewReleaser(INGRESS_ORG, INGRESS_REPO, GITHUB_TOKEN) // CheckIfError(err, "GitHub Release Client error") // newRelease, err := releaser.Create(fmt.Sprintf("controller-%s", name)) // CheckIfError(err, "Create release error") // Info("New Release: Tag %v, ID: %v", newRelease.TagName, newRelease.ID) //} // Returns a GitHub client ready for use func githubClient() *github.Client { ts := oauth2.StaticTokenSource( &oauth2.Token{AccessToken: GITHUB_TOKEN}, ) oauthClient := oauth2.NewClient(ctx, ts) return github.NewClient(oauthClient) } // LatestCommitLogs Retrieves the commit log between the latest two controller versions. func (Release) LatestCommitLogs() { commitLog := commitsBetweenTags() for i, s := range commitLog { Info("#%v Version %v", i, s) } } func commitsBetweenTags() []string { tags := getAllControllerTags() Info("Getting Commits between %v and %v", tags[0], tags[1]) commitLog, err := git("log", "--full-history", "--pretty", "--oneline", fmt.Sprintf("%v..%v", tags[1], tags[0])) if commitLog == "" { Warning("All Controller Tags is empty") } CheckIfError(err, "Retrieving Commit log") return strings.Split(commitLog, "\n") } // Generate Release Notes func (Release) ReleaseNotes(newVersion string) error { notes, err := makeReleaseNotes(newVersion) CheckIfError(err, "Creating Release Notes for version %s", newVersion) Info("Release Notes %s completed", notes.Version) return nil } func makeReleaseNotes(newVersion string) (*ReleaseNote, error) { var newReleaseNotes = ReleaseNote{} newReleaseNotes.Version = newVersion allControllerTags := getAllControllerTags() //new version newReleaseNotes.NewControllerVersion = allControllerTags[0] newControllerVersion := fmt.Sprintf("controller-v%s", newVersion) //the newControllerVersion should match the latest tag if newControllerVersion != allControllerTags[0] { return nil, errors.New(fmt.Sprintf("Generating release new version %s didnt match the current latest tag %s", newControllerVersion, allControllerTags[0])) } //previous version newReleaseNotes.PreviousControllerVersion = allControllerTags[1] Info("New Version: %s Old Version: %s", newReleaseNotes.NewControllerVersion, newReleaseNotes.PreviousControllerVersion) commits := commitsBetweenTags() //dependency_updates //all_updates var allUpdates []string var depUpdates []string var helmUpdates []string prRegex := regexp.MustCompile("\\(#\\d+\\)") depBot := regexp.MustCompile("^(\\w){1,10} Bump ") helmRegex := regexp.MustCompile("helm|chart") for i, s := range commits { //matches on PR if prRegex.Match([]byte(s)) { //matches a dependant bot update if depBot.Match([]byte(s)) { // Debug("#%v DEPENDABOT %v", i, s) u := strings.SplitN(s, " ", 2) depUpdates = append(depUpdates, u[1]) } else { // add it to the all updates slice Debug("#%v ALL UPDATES %v", i, s) u := strings.SplitN(s, " ", 2) allUpdates = append(allUpdates, u[1]) //helm chart updates if helmRegex.Match([]byte(s)) { u := strings.SplitN(s, " ", 2) helmUpdates = append(helmUpdates, u[1]) } } } } helmUpdates = append(helmUpdates, fmt.Sprintf("Update Ingress-Nginx version %s", newReleaseNotes.NewControllerVersion)) newReleaseNotes.Updates = allUpdates newReleaseNotes.DepUpdates = depUpdates newReleaseNotes.HelmUpdates = helmUpdates //controller_image_digests imagesYaml, err := downloadFile(IMAGES_YAML) if err != nil { ErrorF("Could not download file %s : %s", IMAGES_YAML, err) return nil, err } Debug("%s", imagesYaml) data := ImageYamls{} err = yaml.Unmarshal([]byte(imagesYaml), &data) if err != nil { ErrorF("Could not unmarshal images yaml %s", err) return nil, err } //controller controllerDigest := findImageDigest(data, "controller", newVersion) if len(controllerDigest) == 0 { ErrorF("Controller Digest could not be found") return nil, errors.New("Controller digest could not be found") } controllerChrootDigest := findImageDigest(data, "controller-chroot", newVersion) if len(controllerChrootDigest) == 0 { ErrorF("Controller Chroot Digest could not be found") return nil, errors.New("Controller Chroot digest could not be found") } Debug("Latest Controller Digest %v", controllerDigest) Debug("Latest Controller Chroot Digest %v", controllerChrootDigest) c1 := ControllerImage{ Digest: controllerDigest, Registry: INGRESS_REGISTRY, Name: "ingress-nginx/controller", Tag: fmt.Sprintf("v%s", newReleaseNotes.Version), } c2 := ControllerImage{ Digest: controllerChrootDigest, Registry: INGRESS_REGISTRY, Name: "ingress-nginx/controller-chroot", Tag: fmt.Sprintf("v%s", newReleaseNotes.Version), } newReleaseNotes.ControllerImages = append(newReleaseNotes.ControllerImages, c1) newReleaseNotes.ControllerImages = append(newReleaseNotes.ControllerImages, c2) Debug("New Release Controller Images %s %s", newReleaseNotes.ControllerImages[0].Digest, newReleaseNotes.ControllerImages[1].Digest) if DEBUG { newReleaseNotes.printRelease() } //write it all out to the changelog file newReleaseNotes.template() return &newReleaseNotes, nil } func (i ControllerImage) print() string { return fmt.Sprintf("%s/%s:%s@%s", i.Registry, i.Name, i.Tag, i.Digest) } func (r ReleaseNote) template() { // Files are provided as a slice of strings. changelogTemplate, err := os.ReadFile("Changelog.md.gotmpl") if err != nil { ErrorF("Could not read changelog template file %s", err) } Debug("ChangeLog Templates %s", string(changelogTemplate)) t := template.Must(template.New("changelog").Parse(string(changelogTemplate))) // create a new file file, err := os.Create(fmt.Sprintf("changelog/Changelog-%s.md", r.Version)) if err != nil { ErrorF("Could not create changelog file %s", err) } defer file.Close() err = t.Execute(file, r) if err != nil { ErrorF("executing template:", err) } } func (r ReleaseNote) helmTemplate() { // Files are provided as a slice of strings. changelogTemplate, err := os.ReadFile("charts/ingress-nginx/changelog.md.gotmpl") if err != nil { ErrorF("Could not read changelog template file %s", err) } Debug("ChangeLog Templates %s", string(changelogTemplate)) t := template.Must(template.New("changelog").Parse(string(changelogTemplate))) // create a new file file, err := os.Create(fmt.Sprintf("charts/ingress-nginx/changelog/Changelog-%s.md", r.NewHelmChartVersion)) if err != nil { ErrorF("Could not create changelog file %s", err) } defer file.Close() err = t.Execute(file, r) if err != nil { ErrorF("executing template:", err) } } func (r ReleaseNote) printRelease() { Info("Release Version: %v", r.NewControllerVersion) Info("Previous Version: %v", r.PreviousControllerVersion) Info("Controller Image: %v", r.ControllerImages[0].print()) Info("Controller Chroot Image: %v", r.ControllerImages[1].print()) for i := range r.Updates { Info("Update #%v - %v", i, r.Updates[i]) } for j := range r.DepUpdates { Info("Dependabot Update #%v - %v", j, r.DepUpdates[j]) } } func findImageDigest(yaml ImageYamls, image, version string) string { version = fmt.Sprintf("v%s", version) Info("Searching Digest for %s:%s", image, version) for i := range yaml { if yaml[i].Name == image { for k, v := range yaml[i].Dmap { if v[0] == version { return k } } return "" } } return "" } func downloadFile(url string) (string, error) { client := &http.Client{ Transport: &http.Transport{ DialContext: (&net.Dialer{ Timeout: 5 * time.Second, KeepAlive: 5 * time.Second, }).DialContext, TLSHandshakeTimeout: 5 * time.Second, ResponseHeaderTimeout: 5 * time.Second, ExpectContinueTimeout: 1 * time.Second, MaxIdleConnsPerHost: -1, }, } resp, err := client.Get(url) if err != nil { return "", nil } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return "", errors.New(fmt.Sprintf("Could not retrieve file, response from server %s for file %s", resp.StatusCode, url)) } bodyBytes, err := io.ReadAll(resp.Body) if err != nil { return "", nil } return string(bodyBytes), nil } // Latest returns latest Github Release func (Release) Latest() error { r, _, err := latestRelease() if err != nil { ErrorF("Latest Release error %s", err) return err } Info("Latest Release %v", r.String()) return nil } func (Release) ReleaseByTag(tag string) error { r, _, err := releaseByTag(tag) if err != nil { ErrorF("Release retrieve tag error %s", tag, err) return err } Info("Latest Release %v", r.String()) return nil } func releaseByTag(tag string) (*github.RepositoryRelease, *github.Response, error) { ghClient := githubClient() return ghClient.Repositories.GetReleaseByTag(ctx, INGRESS_ORG, INGRESS_REPO, tag) } func latestRelease() (*github.RepositoryRelease, *github.Response, error) { ghClient := githubClient() return ghClient.Repositories.GetLatestRelease(ctx, INGRESS_ORG, INGRESS_REPO) } // Copy Test function to copy a release func (Release) Copy() error { ghClient := githubClient() kRelease, _, err := ghClient.Repositories.GetLatestRelease(ctx, "kubernetes", "ingress-nginx") if err != nil { ErrorF("Get Release from kubernetes %s", err) return err } sRelease := &github.RepositoryRelease{ TagName: kRelease.TagName, Name: kRelease.Name, Body: kRelease.Body, Draft: kRelease.Draft, Prerelease: kRelease.GenerateReleaseNotes, DiscussionCategoryName: kRelease.DiscussionCategoryName, GenerateReleaseNotes: kRelease.GenerateReleaseNotes, } sRelease, _, err = ghClient.Repositories.CreateRelease(ctx, "strongjz", "ingress-nginx", sRelease) if err != nil { ErrorF("Creating Strongjz release %s", err) return err } Info("Copied over Kubernetes Release %v to Strongjz %v", &kRelease.Name, &sRelease.Name) return nil }