From c34e26ed1c1948ec48d8e3270053fe493ad11868 Mon Sep 17 00:00:00 2001
From: k8s-infra-cherrypick-robot
 <90416843+k8s-infra-cherrypick-robot@users.noreply.github.com>
Date: Sun, 23 Mar 2025 09:26:32 -0700
Subject: [PATCH] Images: Rework. (1/3) (#13014)

Co-authored-by: Marco Ebert <marco_ebert@icloud.com>
---
 Makefile                             | 10 +--
 cloudbuild.yaml                      | 18 +++---
 hack/init-buildx.sh                  | 56 ----------------
 images/Makefile                      | 92 ++++++++++----------------
 images/nginx/Makefile                | 67 ++++++++-----------
 images/nginx/cloudbuild.yaml         | 12 ++--
 images/nginx/rootfs/Dockerfile       |  2 +-
 images/nginx/rootfs/build.sh         |  4 --
 images/test-runner/Makefile          | 96 ++++++++--------------------
 images/test-runner/cloudbuild.yaml   | 12 ++--
 images/test-runner/rootfs/Dockerfile | 10 +--
 11 files changed, 119 insertions(+), 260 deletions(-)
 delete mode 100755 hack/init-buildx.sh

diff --git a/Makefile b/Makefile
index 0b8f1f5c2..dc7f0a204 100644
--- a/Makefile
+++ b/Makefile
@@ -232,19 +232,21 @@ misspell:  ## Check for spelling errors.
 run-ingress-controller: ## Run the ingress controller locally using a kubectl proxy connection.
 	@build/run-ingress-controller.sh
 
-.PHONY: ensure-buildx
-ensure-buildx:
-	./hack/init-buildx.sh
+.PHONY: builder
+builder:
+	docker buildx create --name $(BUILDER) --bootstrap --use || :
+	docker buildx inspect $(BUILDER)
 
 .PHONY: show-version
 show-version:
 	echo -n $(TAG)
 
+BUILDER ?= ingress-nginx
 PLATFORMS ?= amd64 arm arm64
 BUILDX_PLATFORMS ?= linux/amd64,linux/arm,linux/arm64
 
 .PHONY: release # Build a multi-arch docker image
-release: ensure-buildx clean
+release: builder clean
 	echo "Building binaries..."
 	$(foreach PLATFORM,$(PLATFORMS), echo -n "$(PLATFORM)..."; ARCH=$(PLATFORM) make build;)
 
diff --git a/cloudbuild.yaml b/cloudbuild.yaml
index 0bb2b60a4..1d0228358 100644
--- a/cloudbuild.yaml
+++ b/cloudbuild.yaml
@@ -2,13 +2,11 @@ options:
   # Ignore Prow provided substitutions.
   substitution_option: ALLOW_LOOSE
 steps:
-  - name: gcr.io/k8s-staging-test-infra/gcb-docker-gcloud:v20250116-2a05ea7e3d
-    env:
-      - REGISTRY=us-central1-docker.pkg.dev/k8s-staging-images/ingress-nginx
-      - REPO_INFO=https://github.com/kubernetes/ingress-nginx
-      - COMMIT_SHA=${_PULL_BASE_SHA}
-      - BUILD_ID=${BUILD_ID}
-    entrypoint: bash
-    args:
-      - -c
-      - gcloud auth configure-docker && make release
+- name: gcr.io/k8s-staging-test-infra/gcb-docker-gcloud:v20250116-2a05ea7e3d
+  env:
+  - REPO_INFO=https://github.com/kubernetes/ingress-nginx
+  - COMMIT_SHA=${_PULL_BASE_SHA}
+  - BUILD_ID=${BUILD_ID}
+  entrypoint: make
+  args:
+  - release
diff --git a/hack/init-buildx.sh b/hack/init-buildx.sh
deleted file mode 100755
index bac68e1ae..000000000
--- a/hack/init-buildx.sh
+++ /dev/null
@@ -1,56 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright 2020 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-if [ -n "$DEBUG" ]; then
-	set -x
-fi
-
-set -o errexit
-set -o nounset
-set -o pipefail
-
-export DOCKER_CLI_EXPERIMENTAL=enabled
-
-if ! docker buildx 2>&1 >/dev/null; then
-  echo "buildx not available. Docker 19.03 or higher is required with experimental features enabled"
-  exit 1
-fi
-
-# Ensure qemu is in binfmt_misc
-# Docker desktop already has these in versions recent enough to have buildx
-# We only need to do this setup on linux hosts
-if [ "$(uname)" == 'Linux' ]; then
-  # NOTE: this is pinned to a digest for a reason!
-  # Note2 (@rikatz) - Removing the pin, as apparently it's breaking new alpine builds
-  # docker run --rm --privileged multiarch/qemu-user-static@sha256:28ebe2e48220ae8fd5d04bb2c847293b24d7fbfad84f0b970246e0a4efd48ad6 --reset -p yes
-  docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
-fi
-
-# We can skip setup if the current builder already has multi-arch
-# AND if it isn't the docker driver, which doesn't work
-current_builder="$(docker buildx inspect)"
-# linux/amd64, linux/arm, linux/arm64
-if ! grep -q "^Driver: docker$"  <<<"${current_builder}" && \
-     grep -q "linux/amd64" <<<"${current_builder}" && \
-     grep -q "linux/arm"   <<<"${current_builder}" && \
-     grep -q "linux/arm64" <<<"${current_builder}"; then
-  exit 0
-fi
-
-
-# Ensure we use a builder that can leverage it (the default on linux will not)
-docker buildx rm ingress-nginx || true
-docker buildx create --use --name=ingress-nginx
diff --git a/images/Makefile b/images/Makefile
index 31560168d..89d98f7a2 100644
--- a/images/Makefile
+++ b/images/Makefile
@@ -1,4 +1,4 @@
-# Copyright 2024 The Kubernetes Authors.
+# Copyright 2025 The Kubernetes Authors. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,75 +12,51 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-.DEFAULT_GOAL:=build
-
-# set default shell
-SHELL=/bin/bash -o pipefail -o errexit
-
-DIR:=$(strip $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))))
-INIT_BUILDX=$(DIR)/../hack/init-buildx.sh
-
-
-BASE_IMAGE = $(shell cat $(DIR)/../NGINX_BASE)
-
-# The env below is called GO_VERSION and not GOLANG_VERSION because 
-# the gcb image we use to build already defines GOLANG_VERSION and is a 
-# really old version
-GO_VERSION = $(shell cat $(DIR)/../GOLANG_VERSION)
-
-REGISTRY ?= local
 NAME ?=
 
-IMAGE = $(REGISTRY)/$(NAME)
+BUILDER ?= ingress-nginx
+PLATFORMS ?= linux/amd64,linux/arm,linux/arm64
+REGISTRY ?= us-central1-docker.pkg.dev/k8s-staging-images/ingress-nginx
+IMAGE ?= $(REGISTRY)/$(NAME)
 TAG ?= $(shell cat $(NAME)/TAG)
 
+DIR = $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
+BASE_IMAGE ?= $(shell cat $(DIR)/../NGINX_BASE)
+GOLANG_VERSION ?= $(shell cat $(DIR)/../GOLANG_VERSION)
 EXTRAARGS ?= $(shell cat $(NAME)/EXTRAARGS)
 
+.PHONY: builder
+builder:
+	docker buildx create --name $(BUILDER) --bootstrap || :
+	docker buildx inspect $(BUILDER)
 
-# required to enable buildx
-export DOCKER_CLI_EXPERIMENTAL=enabled
-
-# build with buildx
-PLATFORMS?=linux/amd64,linux/arm,linux/arm64
-OUTPUT=
-PROGRESS=plain
-
-
-precheck:
-ifndef NAME
-	$(error NAME variable is required)
-endif
-
-build: precheck ensure-buildx
+.PHONY: build
+build: builder
 	docker buildx build \
-		--label=org.opencontainers.image.source=https://github.com/kubernetes/ingress-nginx \
-		--label=org.opencontainers.image.licenses=Apache-2.0 \
-		--label=org.opencontainers.image.description="Ingress NGINX $(NAME) image" \
+		--builder $(BUILDER) \
+		--platform $(PLATFORMS) \
+		--label org.opencontainers.image.description="Ingress NGINX $(NAME)" \
+		--label org.opencontainers.image.source="https://github.com/kubernetes/ingress-nginx" \
+		--label org.opencontainers.image.licenses="Apache-2.0" \
 		--build-arg BASE_IMAGE=$(BASE_IMAGE) \
-		--build-arg GOLANG_VERSION=$(GO_VERSION) \
-		--platform=${PLATFORMS} $(OUTPUT) \
-		--progress=$(PROGRESS) \
-		--pull $(EXTRAARGS) \
-		-t $(IMAGE):$(TAG) $(NAME)/rootfs
+		--build-arg GOLANG_VERSION=$(GOLANG_VERSION) \
+		$(EXTRAARGS) \
+		$(NAME)/rootfs \
+		--tag $(IMAGE):$(TAG) \
+		$(OUTPUT)
 
-# push the cross built image
-push: OUTPUT=--push
+.PHONY: push
+push: OUTPUT = --push
 push: build
 
-test: precheck
+.PHONY: test
+test:
 	cd $(NAME)/rootfs && go test ./...
 
-test-e2e: precheck
-	cd $(NAME) && ./hack/e2e.sh
+.PHONY: test-e2e
+test-e2e:
+	cd $(NAME) && hack/e2e.sh
 
-# enable buildx
-ensure-buildx:
-# this is required for cloudbuild
-ifeq ("$(wildcard $(INIT_BUILDX))","")
-	@curl -sSL https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/hack/init-buildx.sh | bash
-else
-	@exec $(INIT_BUILDX)
-endif
-	@echo "done"
-
-.PHONY: build push ensure-buildx test test-e2e precheck
+.PHONY: clean
+clean:
+	docker buildx rm $(BUILDER) || :
diff --git a/images/nginx/Makefile b/images/nginx/Makefile
index 3cc14e5b3..803f8ae80 100644
--- a/images/nginx/Makefile
+++ b/images/nginx/Makefile
@@ -1,4 +1,4 @@
-# Copyright 2024 The Kubernetes Authors. All rights reserved.
+# Copyright 2025 The Kubernetes Authors. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,48 +12,37 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-.DEFAULT_GOAL:=build
-
-# set default shell
-SHELL=/bin/bash -o pipefail -o errexit
-
-DIR:=$(strip $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))))
-INIT_BUILDX=$(DIR)/../../hack/init-buildx.sh
-
-# 0.0.0 shouldn't clobber any released builds
-SHORT_SHA ?=$(shell git rev-parse --short HEAD)
-TAG ?=$(shell cat TAG)
-
+BUILDER ?= ingress-nginx
+PLATFORMS ?= linux/amd64,linux/arm,linux/arm64
 REGISTRY ?= us-central1-docker.pkg.dev/k8s-staging-images/ingress-nginx
+IMAGE ?= $(REGISTRY)/nginx
+TAG ?= $(shell cat TAG)
 
-IMAGE = $(REGISTRY)/nginx
+.PHONY: builder
+builder:
+	docker buildx create --name $(BUILDER) --bootstrap || :
+	docker buildx inspect $(BUILDER)
 
-# required to enable buildx
-export DOCKER_CLI_EXPERIMENTAL=enabled
-
-# build with buildx
-PLATFORMS?=linux/amd64,linux/arm,linux/arm64
-OUTPUT=
-PROGRESS=plain
-build: ensure-buildx
+.PHONY: build
+build: builder
 	docker buildx build \
-		--platform=${PLATFORMS} $(OUTPUT) \
-		--progress=$(PROGRESS) \
-		--pull \
-		--tag $(IMAGE):$(TAG) rootfs
+		--builder $(BUILDER) \
+		--platform $(PLATFORMS) \
+		rootfs \
+		--tag $(IMAGE):$(TAG)
 
-# push the cross built image
-push: OUTPUT=--push
+# Pushing in the `build` target does not work as authentication times out after one hour.
+#
+# Therefore we need to build and push in separate commands.
+.PHONY: push
 push: build
+	docker buildx build \
+		--builder $(BUILDER) \
+		--platform $(PLATFORMS) \
+		rootfs \
+		--tag $(IMAGE):$(TAG) \
+		--push
 
-# enable buildx
-ensure-buildx:
-# this is required for cloudbuild
-ifeq ("$(wildcard $(INIT_BUILDX))","")
-	@curl -sSL https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/hack/init-buildx.sh | bash
-else
-	@exec $(INIT_BUILDX)
-endif
-	@echo "done"
-
-.PHONY: build push ensure-buildx
+.PHONY: clean
+clean:
+	docker buildx rm $(BUILDER) || :
diff --git a/images/nginx/cloudbuild.yaml b/images/nginx/cloudbuild.yaml
index 2563692d7..d28af1aa5 100644
--- a/images/nginx/cloudbuild.yaml
+++ b/images/nginx/cloudbuild.yaml
@@ -4,11 +4,9 @@ options:
   # Ignore Prow provided substitutions.
   substitution_option: ALLOW_LOOSE
 steps:
-  - name: gcr.io/k8s-staging-test-infra/gcb-docker-gcloud:v20250116-2a05ea7e3d
-    env:
-      - REGISTRY=us-central1-docker.pkg.dev/k8s-staging-images/ingress-nginx
-    entrypoint: bash
-    args:
-      - -c
-      - gcloud auth configure-docker && cd images/nginx && make push
+- name: gcr.io/cloud-builders/docker
+  dir: images/nginx
+  entrypoint: make
+  args:
+  - push
 timeout: 7200s
diff --git a/images/nginx/rootfs/Dockerfile b/images/nginx/rootfs/Dockerfile
index 834a9bcf3..8f6bab137 100644
--- a/images/nginx/rootfs/Dockerfile
+++ b/images/nginx/rootfs/Dockerfile
@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-FROM alpine:3.21 as builder
+FROM alpine:3.21 AS builder
 
 COPY . /
 
diff --git a/images/nginx/rootfs/build.sh b/images/nginx/rootfs/build.sh
index 63d65210e..e352be10c 100755
--- a/images/nginx/rootfs/build.sh
+++ b/images/nginx/rootfs/build.sh
@@ -183,10 +183,6 @@ apk add \
 
 # apk add -X http://dl-cdn.alpinelinux.org/alpine/edge/testing opentelemetry-cpp-dev
 
-# There is some bug with some platforms and git, so force HTTP/1.1
-git config --global http.version HTTP/1.1
-git config --global http.postBuffer 157286400
-
 mkdir -p /etc/nginx
 
 mkdir --verbose -p "$BUILD_PATH"
diff --git a/images/test-runner/Makefile b/images/test-runner/Makefile
index f3987204e..78be8179d 100644
--- a/images/test-runner/Makefile
+++ b/images/test-runner/Makefile
@@ -1,4 +1,4 @@
-# Copyright 2018 The Kubernetes Authors. All rights reserved.
+# Copyright 2025 The Kubernetes Authors. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,64 +12,28 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# set default shell
-SHELL=/bin/bash -o pipefail -o errexit
+BUILDER ?= ingress-nginx
+PLATFORMS ?= linux/amd64,linux/arm64
+REGISTRY ?= us-central1-docker.pkg.dev/k8s-staging-images/ingress-nginx
+IMAGE ?= $(REGISTRY)/e2e-test-runner
+TAG ?= $(shell cat TAG)
 
-DIR:=$(strip $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))))
-INIT_BUILDX=$(DIR)/../../hack/init-buildx.sh
+DIR = $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
+BASE_IMAGE ?= $(shell cat $(DIR)/../../NGINX_BASE)
+GOLANG_VERSION ?= $(shell cat $(DIR)/../../GOLANG_VERSION)
 
-SHORT_SHA ?=$(shell git rev-parse --short HEAD)
-TAG ?=v$(shell date +%Y%m%d)-$(SHORT_SHA)
+.PHONY: builder
+builder:
+	docker buildx create --name $(BUILDER) --bootstrap || :
+	docker buildx inspect $(BUILDER)
 
-REGISTRY ?= local
-
-IMAGE = $(REGISTRY)/e2e-test-runner
-
-NGINX_BASE_IMAGE ?= $(shell cat $(DIR)/../../NGINX_BASE)
-
-# The env below is called GO_VERSION and not GOLANG_VERSION because 
-# the gcb image we use to build already defines GOLANG_VERSION and is a 
-# really old version
-GO_VERSION ?= $(shell cat $(DIR)/../../GOLANG_VERSION)
-
-# required to enable buildx
-export DOCKER_CLI_EXPERIMENTAL=enabled
-
-# build with buildx
-PLATFORMS?=linux/amd64,linux/arm64
-OUTPUT?=
-PROGRESS=plain
-
-image:
-	echo "Building docker image ($(ARCH))..."
-	docker build \
-		${PLATFORM_FLAG} ${PLATFORM} \
-		--no-cache \
-		--pull \
-		--push \
-		--build-arg BASE_IMAGE=${NGINX_BASE_IMAGE} \
-		--build-arg GOLANG_VERSION=${GO_VERSION} \
-		--build-arg ETCD_VERSION=3.5.13-0 \
-		--build-arg K8S_RELEASE=v1.32.2 \
-		--build-arg RESTY_CLI_VERSION=0.27 \
-		--build-arg RESTY_CLI_SHA=e5f4f3128af49ba5c4d039d0554e5ae91bbe05866f60eccfa96d3653274bff90 \
-		--build-arg LUAROCKS_VERSION=3.8.0 \
-		--build-arg LUAROCKS_SHA=ab6612ca9ab87c6984871d2712d05525775e8b50172701a0a1cabddf76de2be7 \
-		--build-arg CHART_TESTING_VERSION=3.8.0 \
-		--build-arg YAML_LINT_VERSION=1.33.0 \
-		--build-arg YAMALE_VERSION=4.0.4 \
-		--build-arg HELM_VERSION=3.14.4 \
-		--build-arg GINKGO_VERSION=2.23.0 \
-		--build-arg GOLINT_VERSION=latest \
-		-t ${IMAGE}:${TAG} rootfs
-
-build: ensure-buildx
+.PHONY: build
+build: builder
 	docker buildx build \
-		--platform=${PLATFORMS} ${OUTPUT} \
-		--progress=${PROGRESS} \
-		--pull \
-		--build-arg BASE_IMAGE=${NGINX_BASE_IMAGE} \
-		--build-arg GOLANG_VERSION=${GO_VERSION} \
+		--builder $(BUILDER) \
+		--platform $(PLATFORMS) \
+		--build-arg BASE_IMAGE=$(BASE_IMAGE) \
+		--build-arg GOLANG_VERSION=$(GOLANG_VERSION) \
 		--build-arg ETCD_VERSION=3.5.13-0 \
 		--build-arg K8S_RELEASE=v1.32.2 \
 		--build-arg RESTY_CLI_VERSION=0.27 \
@@ -82,20 +46,14 @@ build: ensure-buildx
 		--build-arg HELM_VERSION=3.14.4 \
 		--build-arg GINKGO_VERSION=2.23.0 \
 		--build-arg GOLINT_VERSION=latest \
-		-t ${IMAGE}:${TAG} rootfs
+		rootfs \
+		--tag $(IMAGE):$(TAG) \
+		$(OUTPUT)
 
-# push the cross built image
-push: OUTPUT=--push
+.PHONY: push
+push: OUTPUT = --push
 push: build
 
-# enable buildx
-ensure-buildx:
-# this is required for cloudbuild
-ifeq ("$(wildcard $(INIT_BUILDX))","")
-	@curl -sSL https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/hack/init-buildx.sh | bash
-else
-	@exec $(INIT_BUILDX)
-endif
-	@echo "done"
-
-.PHONY: build push ensure-buildx
+.PHONY: clean
+clean:
+	docker buildx rm $(BUILDER) || :
diff --git a/images/test-runner/cloudbuild.yaml b/images/test-runner/cloudbuild.yaml
index 93dce3ec9..2f49ddafc 100644
--- a/images/test-runner/cloudbuild.yaml
+++ b/images/test-runner/cloudbuild.yaml
@@ -2,10 +2,8 @@ options:
   # Ignore Prow provided substitutions.
   substitution_option: ALLOW_LOOSE
 steps:
-  - name: gcr.io/k8s-staging-test-infra/gcb-docker-gcloud:v20250116-2a05ea7e3d
-    env:
-      - REGISTRY=us-central1-docker.pkg.dev/k8s-staging-images/ingress-nginx
-    entrypoint: bash
-    args:
-      - -c
-      - gcloud auth configure-docker && cd images/test-runner && make push
+- name: gcr.io/cloud-builders/docker
+  dir: images/test-runner
+  entrypoint: make
+  args:
+  - push
diff --git a/images/test-runner/rootfs/Dockerfile b/images/test-runner/rootfs/Dockerfile
index d871461bf..69fae92d7 100644
--- a/images/test-runner/rootfs/Dockerfile
+++ b/images/test-runner/rootfs/Dockerfile
@@ -15,8 +15,8 @@ ARG BASE_IMAGE
 ARG GOLANG_VERSION
 ARG ETCD_VERSION
 
-FROM golang:${GOLANG_VERSION}-alpine3.21 as GO
-FROM registry.k8s.io/etcd:${ETCD_VERSION} as etcd
+FROM golang:${GOLANG_VERSION}-alpine3.21 AS go
+FROM registry.k8s.io/etcd:${ETCD_VERSION} AS etcd
 
 FROM ${BASE_IMAGE}
 
@@ -41,9 +41,9 @@ RUN set -eux; \
 		echo 'hosts: files dns' > /etc/nsswitch.conf; \
 	fi
 
-COPY --from=GO   /usr/local/go /usr/local/go
-ENV GOPATH /go
-ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH
+COPY --from=go   /usr/local/go /usr/local/go
+ENV GOPATH=/go
+ENV PATH=$GOPATH/bin:/usr/local/go/bin:$PATH
 RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH"
 
 COPY --from=etcd /usr/local/bin/etcd /usr/local/bin/etcd