diff --git a/ingressctl/cmd/docker.go b/ingressctl/cmd/docker.go index 3eb779070..280662fb2 100644 --- a/ingressctl/cmd/docker.go +++ b/ingressctl/cmd/docker.go @@ -50,7 +50,7 @@ func init() { dockerBuildCmd.Flags().StringVar(&dco.BuildArgs.TargetArch, "targetarch", "", "target arch to build") dockerBuildCmd.Flags().StringVar(&dco.BuildArgs.CommitSHA, "commitsha", "", "build arg commit sha to add to build") dockerBuildCmd.Flags().StringVar(&dco.BuildArgs.BuildId, "build-id", "", "build id to add to container metadata") - dockerBuildCmd.Flags().StringVar(&dco.DockerFile, "dockerfile", "", "dockerfile of image to build") + dockerBuildCmd.Flags().StringVar(&dco.Dockerfile, "dockerfile", "Dockerfile", "dockerfile of image to build") dockerBuildCmd.Flags().StringVar(&dco.Image.Name, "name", "", "container image name registry/name:tag@digest") dockerBuildCmd.Flags().StringVar(&dco.Image.Registry, "registry", Registry, "Registry to tag image and push container registry/name:tag@digest") dockerBuildCmd.Flags().StringVar(&dco.Image.Tag, "tag", "", "container tag registry/name:tag@digest") @@ -81,7 +81,7 @@ type dockerBuildOpts struct { PlatformFlag string Platform []string BuildArgs BuildArgs - DockerFile string + Dockerfile string Image Image Path string } @@ -136,6 +136,7 @@ func dockerBuild() error { "--build-arg", "TARGETARCH="+dco.BuildArgs.TargetArch, "--build-arg", "COMMIT_SHA="+dco.BuildArgs.CommitSHA, "--build-arg", "BUILD_ID="+dco.BuildArgs.BuildId, + "-f", dco.Path+dco.Dockerfile, dco.Path).Run() return nil diff --git a/ingressctl/cmd/release.go b/ingressctl/cmd/release.go new file mode 100644 index 000000000..f6f02094c --- /dev/null +++ b/ingressctl/cmd/release.go @@ -0,0 +1,573 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "errors" + "fmt" + "github.com/google/go-github/v48/github" + "golang.org/x/oauth2" + "gopkg.in/yaml.v3" + "io" + "net" + "net/http" + "os" + "text/template" + + "regexp" + "strings" + "time" +) + + +var INGRESS_ORG = "kubernetes" // the owner so we can test from forks +var INGRESS_REPO = "ingress-nginx" // the repo to pull from +var RELEASE_BRANCH = "main" //we only release from main +var GITHUB_TOKEN string // the Google/gogithub lib needs an PAT to access the GitHub API +var K8S_IO_ORG = "kubernetes" //the owner or organization for the k8s.io repo +var K8S_IO_REPO = "k8s.io" //the repo that holds the images yaml for production promotion +var INGRESS_REGISTRY = "registry.k8s.io" //Container registry for storage Ingress-nginx images +var KUSTOMIZE_INSTALL_VERSION = "sigs.k8s.io/kustomize/kustomize/v4@v4.5.4" //static deploys needs kustomize to generate the template + +// ingress-nginx releases start with a TAG then a cloudbuild, then a promotion through a PR, this the location of that PR +var IMAGES_YAML = "https://raw.githubusercontent.com/kubernetes/k8s.io/main/registry.k8s.io/images/k8s-staging-ingress-nginx/images.yaml" +var ctx = context.Background() // Context used for GitHub Client + +const INDEX_DOCS = "docs/deploy/index.md" //index.md has a version of the controller and needs to updated +const CHANGELOG = "Changelog.md" //Name of the changelog + + +var releaseCmd = &cobra.Command{ + Use: "release", + Short: "Start a release" + Long: "Start a new release for ingress-nginx" + Run: func(cmd *cobra.Command, args []string){ + + } +} + + + +var helmReleaseCmd = &cobra.Command{ + Use: "helm", + Short: "Start a new helm chart release", + Long: "Start a new helm chart release", + Run: func(cmd *cobra.Command, args []strings) { + + } +} + +func init(){ + rootCmd.AddCommand(releaseCmd) + releaseCmd.AddCommand(helmReleaseCmd) + releaseCmd.AddCommand(controllerReleaseCmd) + +} +// ControllerImage - struct with info about controllers +type ControllerImage struct { + Tag string + Digest string + Registry string + Name string +} + +// IngressRelease All the information about an ingress-nginx release that gets updated +type IngressRelease struct { + ControllerVersion string + ControllerImage ControllerImage + ReleaseNote ReleaseNote + Release *github.RepositoryRelease +} + +// ReleaseNote - All the pieces of information/documents that get updated during a release +type ReleaseNote struct { + Version string + NewControllerVersion string + PreviousControllerVersion string + ControllerImages []ControllerImage + DepUpdates []string + Updates []string + HelmUpdates []string + NewHelmChartVersion string + PreviousHelmChartVersion string +} + +// IMAGES_YAML returns this data structure +type ImageYamls []ImageElement + +// ImageElement - a specific image and it's data structure the dmap is a list of shas and container versions +type ImageElement struct { + Name string `json:"name"` + Dmap map[string][]string `json:"dmap"` +} + +// init will set the GitHub token from the committers/releasers env var +func init() { + GITHUB_TOKEN = os.Getenv("GITHUB_TOKEN") +} + +// PromoteImage Creates PR into the k8s.io repo for promotion of ingress from staging to production +func (Release) PromoteImage(version, sha string) { + +} + +// Release Create a new release of ingress nginx controller +func (Release) NewRelease(version string) { + //newRelease := Release{} + + //update ingress-nginx version + //This is the step that kicks all the release process + //it is already done, so it kicks off the gcloud build of the controller images + //mg.Deps(mg.F(Tag.BumpNginx, version)) + + tag, err := getIngressNGINXVersion() + CheckIfError(err, "RELEASE Retrieving the current Ingress Nginx Version") + + Info("RELEASE Checking Current Version %s to New Version %s", tag, version) + //if the version were upgrading does not match the TAG file, lets update the TAG file + if tag[1:] != version { + Warning("RELEASE Ingress Nginx TAG %s and new version %s do not match", tag, version) + mg.Deps(mg.F(Tag.BumpNginx, fmt.Sprintf("v%s", version))) + } + + //update git controller tag controller-v$version + mg.Deps(mg.F(Tag.NewControllerTag, version)) + + //make release notes + releaseNotes, err := makeReleaseNotes(version) + CheckIfError(err, "RELEASE Creating Release Notes for version %s", version) + Info("RELEASE Release Notes %s completed", releaseNotes.Version) + + //update chart values.yaml new controller tag and image digest + releaseNotes.PreviousHelmChartVersion = currentChartVersion() + + //controller tag + updateChartValue("controller.image.tag", fmt.Sprintf("v%s", releaseNotes.Version)) + Debug("releaseNotes.ControllerImages[0].Name %s", releaseNotes.ControllerImages[0].Name) + Debug("releaseNotes.ControllerImages[1].Name %s", releaseNotes.ControllerImages[1].Name) + //controller digest + if releaseNotes.ControllerImages[0].Name == "ingress-nginx/controller" { + Debug("Updating Chart Value %s with %s", "controller.image.digest", releaseNotes.ControllerImages[0].Digest) + updateChartValue("controller.image.digest", releaseNotes.ControllerImages[0].Digest) + } + //controller chroot digest + if releaseNotes.ControllerImages[1].Name == "ingress-nginx/controller-chroot" { + Debug("Updating Chart Value %s with %s", "controller.image.digestChroot", releaseNotes.ControllerImages[1].Digest) + updateChartValue("controller.image.digestChroot", releaseNotes.ControllerImages[1].Digest) + } + + //update helm chart app version + mg.Deps(mg.F(Helm.UpdateVersion, version)) + + releaseNotes.NewHelmChartVersion = currentChartVersion() + + //update helm chart release notes + updateChartReleaseNotes(releaseNotes.HelmUpdates) + + //Run helm docs update + CheckIfError(runHelmDocs(), "Error Updating Helm Docs ") + + releaseNotes.helmTemplate() + + //update static manifest + CheckIfError(updateStaticManifest(), "Error Updating Static manifests") + + ////update e2e docs + updateE2EDocs() + + //update documentation with ingress-nginx version + CheckIfError(updateIndexMD(releaseNotes.PreviousControllerVersion, releaseNotes.NewControllerVersion), "Error Updating %s", INDEX_DOCS) + + //keeping these manual for now + //git commit TODO + //make Pull Request TODO + //make release TODO + //mg.Deps(mg.F(Release.CreateRelease, version)) +} + +// the index.md doc needs the controller version updated +func updateIndexMD(old, new string) error { + Info("Updating Deploy docs with new version") + data, err := os.ReadFile(INDEX_DOCS) + CheckIfError(err, "Could not read INDEX_DOCS file %s", INDEX_DOCS) + datString := string(data) + datString = strings.Replace(datString, old, new, -1) + err = os.WriteFile(INDEX_DOCS, []byte(datString), 644) + if err != nil { + ErrorF("Could not write new %s %s", INDEX_DOCS, err) + return err + } + return nil +} + +// runs the hack/generate-deploy-scripts.sh +func updateE2EDocs() { + updates, err := sh.Output("./hack/generate-e2e-suite-doc.sh") + CheckIfError(err, "Could not run update hack script") + err = os.WriteFile("docs/e2e-tests.md", []byte(updates), 644) + CheckIfError(err, "Could not write new e2e test file ") +} + +// The static deploy scripts use kustomize to generate them, this function ensures kustomize is installed +func installKustomize() error { + Info("Install Kustomize") + var g0 = sh.RunCmd("go") + // somewhere in your main code + err := g0("install", KUSTOMIZE_INSTALL_VERSION) + if err != nil { + return err + } + return nil +} + +func updateStaticManifest() error { + CheckIfError(installKustomize(), "error installing kustomize") + //hack/generate-deploy-scripts.sh + err := sh.RunV("./hack/generate-deploy-scripts.sh") + if err != nil { + return err + } + return nil +} + +//// CreateRelease Creates a new GitHub Release +//func (Release) CreateRelease(name string) { +// releaser, err := gh_release.NewReleaser(INGRESS_ORG, INGRESS_REPO, GITHUB_TOKEN) +// CheckIfError(err, "GitHub Release Client error") +// newRelease, err := releaser.Create(fmt.Sprintf("controller-%s", name)) +// CheckIfError(err, "Create release error") +// Info("New Release: Tag %v, ID: %v", newRelease.TagName, newRelease.ID) +//} + +// Returns a GitHub client ready for use +func githubClient() *github.Client { + ts := oauth2.StaticTokenSource( + &oauth2.Token{AccessToken: GITHUB_TOKEN}, + ) + oauthClient := oauth2.NewClient(ctx, ts) + return github.NewClient(oauthClient) +} + +// LatestCommitLogs Retrieves the commit log between the latest two controller versions. +func (Release) LatestCommitLogs() { + commitLog := commitsBetweenTags() + for i, s := range commitLog { + Info("#%v Version %v", i, s) + } +} + +func commitsBetweenTags() []string { + tags := getAllControllerTags() + Info("Getting Commits between %v and %v", tags[0], tags[1]) + commitLog, err := git("log", "--full-history", "--pretty", "--oneline", fmt.Sprintf("%v..%v", tags[1], tags[0])) + + if commitLog == "" { + Warning("All Controller Tags is empty") + } + CheckIfError(err, "Retrieving Commit log") + return strings.Split(commitLog, "\n") +} + +// Generate Release Notes +func (Release) ReleaseNotes(newVersion string) error { + notes, err := makeReleaseNotes(newVersion) + CheckIfError(err, "Creating Release Notes for version %s", newVersion) + Info("Release Notes %s completed", notes.Version) + return nil +} + +func makeReleaseNotes(newVersion string) (*ReleaseNote, error) { + var newReleaseNotes = ReleaseNote{} + + newReleaseNotes.Version = newVersion + allControllerTags := getAllControllerTags() + + //new version + newReleaseNotes.NewControllerVersion = allControllerTags[0] + newControllerVersion := fmt.Sprintf("controller-v%s", newVersion) + + //the newControllerVersion should match the latest tag + if newControllerVersion != allControllerTags[0] { + return nil, errors.New(fmt.Sprintf("Generating release new version %s didnt match the current latest tag %s", newControllerVersion, allControllerTags[0])) + } + //previous version + newReleaseNotes.PreviousControllerVersion = allControllerTags[1] + + Info("New Version: %s Old Version: %s", newReleaseNotes.NewControllerVersion, newReleaseNotes.PreviousControllerVersion) + + commits := commitsBetweenTags() + + //dependency_updates + //all_updates + var allUpdates []string + var depUpdates []string + var helmUpdates []string + prRegex := regexp.MustCompile("\\(#\\d+\\)") + depBot := regexp.MustCompile("^(\\w){1,10} Bump ") + helmRegex := regexp.MustCompile("helm|chart") + for i, s := range commits { + //matches on PR + if prRegex.Match([]byte(s)) { + //matches a dependant bot update + if depBot.Match([]byte(s)) { // + Debug("#%v DEPENDABOT %v", i, s) + u := strings.SplitN(s, " ", 2) + depUpdates = append(depUpdates, u[1]) + } else { // add it to the all updates slice + Debug("#%v ALL UPDATES %v", i, s) + u := strings.SplitN(s, " ", 2) + allUpdates = append(allUpdates, u[1]) + + //helm chart updates + if helmRegex.Match([]byte(s)) { + u := strings.SplitN(s, " ", 2) + helmUpdates = append(helmUpdates, u[1]) + } + } + + } + } + helmUpdates = append(helmUpdates, fmt.Sprintf("Update Ingress-Nginx version %s", newReleaseNotes.NewControllerVersion)) + + newReleaseNotes.Updates = allUpdates + newReleaseNotes.DepUpdates = depUpdates + newReleaseNotes.HelmUpdates = helmUpdates + + //controller_image_digests + imagesYaml, err := downloadFile(IMAGES_YAML) + if err != nil { + ErrorF("Could not download file %s : %s", IMAGES_YAML, err) + return nil, err + } + Debug("%s", imagesYaml) + + data := ImageYamls{} + + err = yaml.Unmarshal([]byte(imagesYaml), &data) + if err != nil { + ErrorF("Could not unmarshal images yaml %s", err) + return nil, err + } + + //controller + controllerDigest := findImageDigest(data, "controller", newVersion) + if len(controllerDigest) == 0 { + ErrorF("Controller Digest could not be found") + return nil, errors.New("Controller digest could not be found") + } + + controllerChrootDigest := findImageDigest(data, "controller-chroot", newVersion) + if len(controllerChrootDigest) == 0 { + ErrorF("Controller Chroot Digest could not be found") + return nil, errors.New("Controller Chroot digest could not be found") + } + + Debug("Latest Controller Digest %v", controllerDigest) + Debug("Latest Controller Chroot Digest %v", controllerChrootDigest) + c1 := ControllerImage{ + Digest: controllerDigest, + Registry: INGRESS_REGISTRY, + Name: "ingress-nginx/controller", + Tag: fmt.Sprintf("v%s", newReleaseNotes.Version), + } + + c2 := ControllerImage{ + Digest: controllerChrootDigest, + Registry: INGRESS_REGISTRY, + Name: "ingress-nginx/controller-chroot", + Tag: fmt.Sprintf("v%s", newReleaseNotes.Version), + } + + newReleaseNotes.ControllerImages = append(newReleaseNotes.ControllerImages, c1) + newReleaseNotes.ControllerImages = append(newReleaseNotes.ControllerImages, c2) + Debug("New Release Controller Images %s %s", newReleaseNotes.ControllerImages[0].Digest, newReleaseNotes.ControllerImages[1].Digest) + + if DEBUG { + newReleaseNotes.printRelease() + } + + //write it all out to the changelog file + newReleaseNotes.template() + + return &newReleaseNotes, nil +} + +func (i ControllerImage) print() string { + return fmt.Sprintf("%s/%s:%s@%s", i.Registry, i.Name, i.Tag, i.Digest) +} + +func (r ReleaseNote) template() { + // Files are provided as a slice of strings. + changelogTemplate, err := os.ReadFile("Changelog.md.gotmpl") + if err != nil { + ErrorF("Could not read changelog template file %s", err) + } + Debug("ChangeLog Templates %s", string(changelogTemplate)) + t := template.Must(template.New("changelog").Parse(string(changelogTemplate))) + // create a new file + file, err := os.Create(fmt.Sprintf("changelog/Changelog-%s.md", r.Version)) + if err != nil { + ErrorF("Could not create changelog file %s", err) + } + defer file.Close() + + err = t.Execute(file, r) + if err != nil { + ErrorF("executing template:", err) + } +} + +func (r ReleaseNote) helmTemplate() { + // Files are provided as a slice of strings. + changelogTemplate, err := os.ReadFile("charts/ingress-nginx/changelog.md.gotmpl") + if err != nil { + ErrorF("Could not read changelog template file %s", err) + } + Debug("ChangeLog Templates %s", string(changelogTemplate)) + t := template.Must(template.New("changelog").Parse(string(changelogTemplate))) + // create a new file + file, err := os.Create(fmt.Sprintf("charts/ingress-nginx/changelog/Changelog-%s.md", r.NewHelmChartVersion)) + if err != nil { + ErrorF("Could not create changelog file %s", err) + } + defer file.Close() + + err = t.Execute(file, r) + if err != nil { + ErrorF("executing template:", err) + } +} + +func (r ReleaseNote) printRelease() { + Info("Release Version: %v", r.NewControllerVersion) + Info("Previous Version: %v", r.PreviousControllerVersion) + Info("Controller Image: %v", r.ControllerImages[0].print()) + Info("Controller Chroot Image: %v", r.ControllerImages[1].print()) + for i := range r.Updates { + Info("Update #%v - %v", i, r.Updates[i]) + } + for j := range r.DepUpdates { + Info("Dependabot Update #%v - %v", j, r.DepUpdates[j]) + } +} + +func findImageDigest(yaml ImageYamls, image, version string) string { + version = fmt.Sprintf("v%s", version) + Info("Searching Digest for %s:%s", image, version) + for i := range yaml { + if yaml[i].Name == image { + for k, v := range yaml[i].Dmap { + if v[0] == version { + return k + } + } + return "" + } + } + return "" +} + +func downloadFile(url string) (string, error) { + client := &http.Client{ + Transport: &http.Transport{ + DialContext: (&net.Dialer{ + Timeout: 5 * time.Second, + KeepAlive: 5 * time.Second, + }).DialContext, + TLSHandshakeTimeout: 5 * time.Second, + ResponseHeaderTimeout: 5 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + MaxIdleConnsPerHost: -1, + }, + } + resp, err := client.Get(url) + if err != nil { + return "", nil + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return "", errors.New(fmt.Sprintf("Could not retrieve file, response from server %s for file %s", resp.StatusCode, url)) + } + bodyBytes, err := io.ReadAll(resp.Body) + if err != nil { + return "", nil + } + return string(bodyBytes), nil +} + +// Latest returns latest Github Release +func (Release) Latest() error { + r, _, err := latestRelease() + if err != nil { + ErrorF("Latest Release error %s", err) + return err + } + Info("Latest Release %v", r.String()) + return nil +} + +func (Release) ReleaseByTag(tag string) error { + r, _, err := releaseByTag(tag) + if err != nil { + ErrorF("Release retrieve tag error %s", tag, err) + return err + } + + Info("Latest Release %v", r.String()) + + return nil +} + +func releaseByTag(tag string) (*github.RepositoryRelease, *github.Response, error) { + ghClient := githubClient() + return ghClient.Repositories.GetReleaseByTag(ctx, INGRESS_ORG, INGRESS_REPO, tag) +} + +func latestRelease() (*github.RepositoryRelease, *github.Response, error) { + ghClient := githubClient() + return ghClient.Repositories.GetLatestRelease(ctx, INGRESS_ORG, INGRESS_REPO) +} + +// Copy Test function to copy a release +func (Release) Copy() error { + ghClient := githubClient() + kRelease, _, err := ghClient.Repositories.GetLatestRelease(ctx, "kubernetes", "ingress-nginx") + if err != nil { + ErrorF("Get Release from kubernetes %s", err) + return err + } + + sRelease := &github.RepositoryRelease{ + TagName: kRelease.TagName, + Name: kRelease.Name, + Body: kRelease.Body, + Draft: kRelease.Draft, + Prerelease: kRelease.GenerateReleaseNotes, + DiscussionCategoryName: kRelease.DiscussionCategoryName, + GenerateReleaseNotes: kRelease.GenerateReleaseNotes, + } + + sRelease, _, err = ghClient.Repositories.CreateRelease(ctx, "strongjz", "ingress-nginx", sRelease) + if err != nil { + ErrorF("Creating Strongjz release %s", err) + return err + } + Info("Copied over Kubernetes Release %v to Strongjz %v", &kRelease.Name, &sRelease.Name) + return nil +} diff --git a/magefiles/docker.go b/magefiles/docker.go deleted file mode 100644 index d7c7f588e..000000000 --- a/magefiles/docker.go +++ /dev/null @@ -1,19 +0,0 @@ -//go:build mage - -/* -Copyright 2023 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main diff --git a/magefiles/go.go b/magefiles/go.go deleted file mode 100644 index 2385b089e..000000000 --- a/magefiles/go.go +++ /dev/null @@ -1,31 +0,0 @@ -//go:build mage - -/* -Copyright 2023 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -//import ( -// "github.com/magefile/mage/mg" -// "github.com/mysteriumnetwork/go-ci/commands" -//) -// -//type Go mg.Namespace -// -//// Checks for issues with go imports -//func (Go) CheckGoImports() error { -// return commands.GoImports("./...") -//}