Refactor build of docker images

Makefile

@@ -42,7 +42,7 @@ endif
 # Allow limiting the scope of the e2e tests. By default run everything
 FOCUS ?= .*
 # number of parallel test
-E2E_NODES ?= 14
+E2E_NODES ?= 15
 # slow test only if takes > 50s
 SLOW_E2E_THRESHOLD ?= 50
 # run e2e test suite with tests that check for memory leaks? (default is false)
@@ -61,75 +61,31 @@ endif
 
 REGISTRY ?= quay.io/kubernetes-ingress-controller
 
-BASE_IMAGE ?= quay.io/kubernetes-ingress-controller/nginx:e3c49c52f4b74fe47ad65d6f3266a02e8b6b622f
+BASE_IMAGE ?= $(REGISTRY)/nginx:e3c49c52f4b74fe47ad65d6f3266a02e8b6b622f
 
 GOARCH=$(ARCH)
-GOBUILD_FLAGS := -v
 
 # use vendor directory instead of go modules https://github.com/golang/go/wiki/Modules
 GO111MODULE=off
 
-TEMP_DIR := $(shell mktemp -d)
-DOCKERFILE := $(TEMP_DIR)/rootfs/Dockerfile
-
 help:  ## Display this help
 	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_-]+:.*?##/ { printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
 
-# internal task
-.PHONY: sub-image-%
-sub-image-%:
-	$(MAKE) ARCH=$* build image
-
-# internal task
-.PHONY: sub-push-%
-sub-push-%: ## Publish image for a particular arch.
-	$(MAKE) ARCH=$* push
-
 .PHONY: image
-image: clean-image .image-$(ARCH) ## Build image for a particular arch.
+image: clean-image ## Build image for a particular arch.
-
-# internal task to build image for a particular arch.
-.PHONY: .image-$(ARCH)
-.image-$(ARCH): init-docker-buildx
-	mkdir -p $(TEMP_DIR)/rootfs
-	cp bin/$(ARCH)/nginx-ingress-controller $(TEMP_DIR)/rootfs/nginx-ingress-controller
-	cp bin/$(ARCH)/dbg $(TEMP_DIR)/rootfs/dbg
-	cp bin/$(ARCH)/wait-shutdown $(TEMP_DIR)/rootfs/wait-shutdown
-
-	cp -RP rootfs/* $(TEMP_DIR)/rootfs
-
 	echo "Building docker image ($(ARCH))..."
-	# buildx assumes images are multi-arch
+	@cp -R bin/ rootfs/
-	docker buildx build \
+	@docker build \
-		--pull \
-		--load \
 		--no-cache \
-		--progress plain \
-		--platform linux/$(ARCH) \
 		--build-arg BASE_IMAGE="$(BASE_IMAGE)" \
 		--build-arg VERSION="$(TAG)" \
-		-t $(REGISTRY)/nginx-ingress-controller-${ARCH}:$(TAG) $(TEMP_DIR)/rootfs
+		--build-arg TARGETARCH="$(ARCH)" \
+		-t $(REGISTRY)/nginx-ingress-controller:$(TAG) rootfs
 
 .PHONY: clean-image
 clean-image: ## Removes local image
-	echo "removing old image $(BASE_IMAGE)-$(ARCH):$(TAG)"
+	echo "removing old image $(BASE_IMAGE):$(TAG)"
-	@docker rmi -f $(BASE_IMAGE)-$(ARCH):$(TAG) || true
+	@docker rmi -f $(BASE_IMAGE):$(TAG) || true
-
-.PHONY: push
-push: .push-$(ARCH) ## Publish image for a particular arch.
-
-# internal task
-.PHONY: .push-$(ARCH)
-.push-$(ARCH):
-	docker push $(REGISTRY)/nginx-ingress-controller-${ARCH}:$(TAG)
-
-.PHONY: push-manifest
-push-manifest:
-	docker manifest create $(REGISTRY)/nginx-ingress-controller:$(TAG) \
-		$(REGISTRY)/nginx-ingress-controller-amd64:$(TAG) \
-		$(REGISTRY)/nginx-ingress-controller-arm:$(TAG) \
-		$(REGISTRY)/nginx-ingress-controller-arm64:$(TAG)
-	docker manifest push --purge $(REGISTRY)/nginx-ingress-controller:$(TAG)
 
 .PHONY: build
 build: check-go-version ## Build ingress controller, debug tool and pre-stop hook.
@@ -300,9 +256,25 @@ endif
 show-version:
 	echo -n $(TAG)
 
-.PHONY: staging-gcr
+PLATFORMS ?= amd64 arm arm64 s390x
-staging-gcr:
+
-	echo "Building NGINX image..."
+EMPTY :=
-	ARCH=amd64 make build image push
+SPACE := $(EMPTY) $(EMPTY)
-	ARCH=arm   make build image push
+COMMA := ,
-	ARCH=arm64 make build image push
+
+.PHONY: release # Build a multi-arch docker image
+release: init-docker-buildx clean
+	echo "Building binaries..."
+	$(foreach PLATFORM,$(PLATFORMS), ARCH=$(PLATFORM) make build;)
+
+	@cp -R bin/ rootfs/
+
+	echo "Building and pushing ingress-nginx image..."
+	@docker buildx build \
+		--no-cache \
+		--push \
+		--progress plain \
+		--platform $(subst $(SPACE),$(COMMA),$(PLATFORMS)) \
+		--build-arg BASE_IMAGE="$(BASE_IMAGE)" \
+		--build-arg VERSION="$(TAG)" \
+		-t $(REGISTRY)/nginx-ingress-controller:$(TAG) rootfs

build/build-ingress-controller.sh

@@ -38,10 +38,7 @@ if [ ! -f "${ENV_FILE}" ]; then
 fi
 
 # build local terraform image to build nginx
-docker buildx build \
+docker build \
-  --load \
-  --progress plain \
-  --platform linux/amd64 \
   --tag build-ingress-controller-terraform $DIR/images/ingress-controller
 
 # build nginx and publish docker images to quay.io.

build/build-nginx-image.sh

@@ -37,12 +37,7 @@ if [ ! -f "${ENV_FILE}" ]; then
   exit 1
 fi
 
-# build local terraform image to build nginx
+docker build \
-export DOCKER_CLI_EXPERIMENTAL=enabled
-docker buildx build \
-  --load \
-  --progress plain \
-  --platform linux/amd64 \
   --tag build-nginx-terraform $DIR/images/nginx
 
 # build nginx and publish docker images to quay.io.

build/build.sh

@@ -47,7 +47,6 @@ export CGO_ENABLED=0
 export GOARCH=${ARCH}
 
 go build \
-  "${GOBUILD_FLAGS}" \
   -ldflags "-s -w \
     -X ${PKG}/version.RELEASE=${TAG} \
     -X ${PKG}/version.COMMIT=${GIT_COMMIT} \
@@ -55,16 +54,13 @@ go build \
   -o "bin/${ARCH}/nginx-ingress-controller" "${PKG}/cmd/nginx"
 
 go build \
-  "${GOBUILD_FLAGS}" \
   -ldflags "-s -w \
     -X ${PKG}/version.RELEASE=${TAG} \
     -X ${PKG}/version.COMMIT=${GIT_COMMIT} \
     -X ${PKG}/version.REPO=${REPO_INFO}" \
   -o "bin/${ARCH}/dbg" "${PKG}/cmd/dbg"
 
-
 go build \
-  "${GOBUILD_FLAGS}" \
   -ldflags "-s -w \
     -X ${PKG}/version.RELEASE=${TAG} \
     -X ${PKG}/version.COMMIT=${GIT_COMMIT} \

build/run-in-docker.sh

@@ -34,7 +34,7 @@ function cleanup {
 }
 trap cleanup EXIT
 
-E2E_IMAGE=quay.io/kubernetes-ingress-controller/e2e:v05312020-d250b97b4
+E2E_IMAGE=${E2E_IMAGE:-quay.io/kubernetes-ingress-controller/e2e:v05312020-d250b97b4}
 
 DOCKER_OPTS=${DOCKER_OPTS:-}
 

cloudbuild.yaml

@@ -12,10 +12,8 @@ steps:
       - TAG=$_GIT_TAG
       - BASE_REF=$_PULL_BASE_REF
       - REGISTRY=gcr.io/k8s-staging-ingress-nginx
-      - DIND_TASKS=0
     args:
-      - staging-gcr
+      - release
-      - push-manifest
 
 substitutions:
   # _GIT_TAG will be filled with a git-based tag for the image, of the form vYYYYMMDD-hash, and

images/cfssl/Makefile

@@ -17,27 +17,12 @@
 # Use the 0.0 tag for testing, it shouldn't clobber any release builds
 TAG ?= 0.0
 
-HOSTARCH := $(shell uname -m | sed -e s/x86_64/amd64/ \
-	-e s/s390x/s390x/ \
-	-e s/armv7l/arm/ \
-	-e s/aarch64.*/arm64/)
-
-ifndef ARCH
-ARCH := $(HOSTARCH)
-endif
-ifeq ($(ARCH),)
-    $(error mandatory variable ARCH is empty)
-endif
-
 REGISTRY ?= ingress-controller
 
 IMAGE = $(REGISTRY)/cfssl
 
 image:
-	docker buildx build \
+	docker build \
-		--load \
-		--progress plain \
-		--platform $(ARCH) \
 		-t $(IMAGE):$(TAG) rootfs
 
 clean:

images/e2e-prow/Makefile

@@ -1,3 +1,17 @@
+# Copyright 2018 The Kubernetes Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 TAG ?=v$(shell date +%m%d%Y)-$(shell git rev-parse --short HEAD)
 REGISTRY ?= quay.io/kubernetes-ingress-controller
 
@@ -5,13 +19,8 @@ IMAGE = $(REGISTRY)/e2e-prow
 
 .PHONY: image
 image:
-	docker buildx build \
+	docker build \
-		--pull \
-		--push \
-		--progress plain \
-		--platform amd64 \
 		--build-arg K8S_RELEASE=v1.17.0 \
-		--build-arg ETCD_VERSION=v3.3.18 \
 		--build-arg KIND_VERSION=v0.8.0 \
 		--build-arg GO_VERSION=1.14.2 \
-		-t $(IMAGE):$(TAG) .
+		-t $(IMAGE):$(TAG) rootfs

images/e2e-prow/rootfs/Dockerfile

@@ -16,6 +16,9 @@
 # unit and integration tests
 
 FROM gcr.io/k8s-testimages/bootstrap
+FROM k8s.gcr.io/etcd:3.4.3-0 as etcd
+
+COPY --from=etcd /usr/local/bin/etcd /usr/local/bin/etcd
 
 # hint to kubetest that it is in CI
 ENV KUBETEST_IN_DOCKER="true"
@@ -43,7 +46,6 @@ RUN apt-get update \
     && rm -rf /var/lib/apt/lists/*
 
 ARG K8S_RELEASE
-ARG ETCD_VERSION
 ARG KIND_VERSION
 ARG GO_VERSION
 
@@ -53,12 +55,6 @@ RUN curl -sSL https://storage.googleapis.com/kubernetes-release/release/${K8S_RE
 RUN curl -sSL https://storage.googleapis.com/kubernetes-release/release/${K8S_RELEASE}/bin/linux/amd64/kube-apiserver -o /usr/local/bin/kube-apiserver \
     && chmod +x /usr/local/bin/kube-apiserver
 
-RUN curl -sSL https://storage.googleapis.com/etcd/${ETCD_VERSION}/etcd-${ETCD_VERSION}-linux-amd64.tar.gz -o /tmp/etcd-${ETCD_VERSION}-linux-amd64.tar.gz \
-    && mkdir -p /tmp/etcd-download \
-    && tar xzvf /tmp/etcd-${ETCD_VERSION}-linux-amd64.tar.gz -C /tmp/etcd-download --strip-components=1 \
-    && cp /tmp/etcd-download/etcd /usr/local/bin \
-    && rm -rf /tmp/etcd-download
-
 RUN curl -sSL https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-linux-amd64 -o /usr/local/bin/kind \
     && chmod +x /usr/local/bin/kind
 

images/e2e/Makefile

@@ -17,25 +17,18 @@ REGISTRY ?= quay.io/kubernetes-ingress-controller
 
 IMAGE = $(REGISTRY)/e2e
 
-PLATFORM ?= amd64
-
 .PHONY: image
 image:
-	docker buildx build \
+	docker build \
-		--pull \
-		--push \
-		--progress plain \
-		--platform $(PLATFORM) \
 		--build-arg K8S_RELEASE=v1.15.7 \
-		--build-arg ETCD_VERSION=v3.3.18 \
 		--build-arg RESTY_CLI_VERSION=0.25rc2 \
 		--build-arg RESTY_CLI_SHA=a38d850441384fa037a5922ca012dcce8708d0e4abe34ad2fe4164a01b28bdfb \
 		--build-arg CHART_TESTING_VERSION=3.0.0-beta.1 \
 		--build-arg YAML_LINT_VERSION=1.13.0 \
 		--build-arg YAMALE_VERSION=1.8.0 \
-		--build-arg LUA_CHECK_VERSION=0.23.0 \
-		--build-arg LUA_CHECK_SHA=b4edf3a7702519502696d4ac7372ed1bd6a82ded63bf81f2b1d7e9b37711be2b \
-		--build-arg BUSTED_VERSION=v2.0.rc13-0 \
-		--build-arg BUSTED_SHA=74ef88ae04545d78b4922e459c1cd459fb5a0356b73a061549d2e4601b5de254 \
 		--build-arg HELM_VERSION=v3.2.0 \
-		-t $(IMAGE):$(TAG) .
+		-t $(IMAGE):$(TAG) rootfs
+
+.PHONY: show-image
+show-image:
+	echo -n $(IMAGE):$(TAG)

images/e2e/rootfs/Dockerfile

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 FROM golang:1.14.3-alpine3.11 as GO
+FROM k8s.gcr.io/etcd:3.4.3-0 as etcd
 
 FROM quay.io/kubernetes-ingress-controller/nginx:e3c49c52f4b74fe47ad65d6f3266a02e8b6b622f
 
@@ -25,7 +26,8 @@ ARG CHART_TESTING_VERSION
 
 RUN [ ! -e /etc/nsswitch.conf ] && echo 'hosts: files dns' > /etc/nsswitch.conf
 
-COPY --from=GO /usr/local/go /usr/local/go
+COPY --from=GO   /usr/local/go /usr/local/go
+COPY --from=etcd /usr/local/bin/etcd /usr/local/bin/etcd
 
 RUN apk add --no-cache \
   bash \
@@ -56,16 +58,13 @@ RUN wget -O /tmp/resty_cli.tgz https://github.com/openresty/resty-cli/archive/v$
   && resty -V \
   && rm -rf /tmp/*
 
-ARG LUA_CHECK_VERSION
-ARG LUA_CHECK_SHA
-
 RUN wget -O /tmp/luarocks.tgz https://github.com/luarocks/luarocks/archive/v3.3.1.tar.gz \
   && tar -C /tmp -xzf /tmp/luarocks.tgz \
   && cd /tmp/luarocks* \
   && ./configure \
   && make install
 
-RUN luarocks install busted \
+RUN  luarocks install busted \
   && luarocks install luacheck
 
 ARG BUSTED_VERSION
@@ -81,13 +80,6 @@ RUN wget -O /usr/local/bin/kube-apiserver \
   https://storage.googleapis.com/kubernetes-release/release/${K8S_RELEASE}/bin/linux/${TARGETARCH}/kube-apiserver \
   && chmod +x /usr/local/bin/kube-apiserver
 
-RUN wget -O /tmp/etcd-${ETCD_VERSION}-linux-${TARGETARCH}.tar.gz \
-  https://storage.googleapis.com/etcd/${ETCD_VERSION}/etcd-${ETCD_VERSION}-linux-${TARGETARCH}.tar.gz \
-  && mkdir -p /tmp/etcd-download \
-  && tar xzvf /tmp/etcd-${ETCD_VERSION}-linux-${TARGETARCH}.tar.gz -C /tmp/etcd-download --strip-components=1 \
-  && cp /tmp/etcd-download/etcd /usr/local/bin \
-  && rm -rf /tmp/*
-
 RUN wget -O /tmp/ct-${CHART_TESTING_VERSION}-linux-amd64.tar.gz \
   https://github.com/helm/chart-testing/releases/download/v${CHART_TESTING_VERSION}/chart-testing_${CHART_TESTING_VERSION}_linux_amd64.tar.gz \
   && mkdir -p /tmp/ct-download \

images/echo/Makefile

@@ -17,28 +17,13 @@
 # Use the 0.0 tag for testing, it shouldn't clobber any release builds
 TAG ?= 0.0
 
-HOSTARCH := $(shell uname -m | sed -e s/x86_64/amd64/ \
-	-e s/s390x/s390x/ \
-	-e s/armv7l/arm/ \
-	-e s/aarch64.*/arm64/)
-
-ifndef ARCH
-ARCH := $(HOSTARCH)
-endif
-ifeq ($(ARCH),)
-    $(error mandatory variable ARCH is empty)
-endif
-
 REGISTRY ?= ingress-controller
 
 IMAGE = $(REGISTRY)/echo
 
 image:
-	docker buildx build \
+	docker build \
-		--load \
+		-t $(IMAGE):$(TAG) rootfs
-		--progress plain \
-		--platform $(ARCH) \
-		-t $(IMAGE):$(TAG) .
 
 clean:
 	docker rmi -f $(IMAGE):$(TAG) || true

images/fastcgi-helloserver/Makefile

@@ -37,10 +37,7 @@ PKG=k8s.io/ingress-nginx/images/fastcgi-helloserver
 
 .PHONY: image
 image: build
-	docker buildx build \
+	docker build \
-		--load \
-		--progress plain \
-		--platform $(ARCH) \
 		-t $(IMAGE):$(TAG) rootfs
 
 .PHONY: build

images/httpbin/Makefile

@@ -17,26 +17,11 @@
 # Use the 0.0 tag for testing, it shouldn't clobber any release builds
 TAG ?= 0.0
 
-HOSTARCH := $(shell uname -m | sed -e s/x86_64/amd64/ \
-	-e s/s390x/s390x/ \
-	-e s/armv7l/arm/ \
-	-e s/aarch64.*/arm64/)
-
-ifndef ARCH
-ARCH := $(HOSTARCH)
-endif
-ifeq ($(ARCH),)
-    $(error mandatory variable ARCH is empty)
-endif
-
 REGISTRY ?= ingress-controller
 
 IMAGE = $(REGISTRY)/httpbin
 
 .PHONY: image
 image:
-	docker buildx build \
+	docker build \
-		--load \
-		--progress plain \
-		--platform $(ARCH) \
 		-t $(IMAGE):$(TAG) rootfs

rootfs/.dockerignore

@@ -0,0 +1,2 @@
+Dockerfile
+.dockerignore

rootfs/.gitignore

@@ -0,0 +1,2 @@
+bin/*
+

rootfs/Dockerfile

@@ -16,6 +16,7 @@ ARG BASE_IMAGE
 
 FROM ${BASE_IMAGE}
 
+ARG TARGETARCH
 ARG VERSION
 
 LABEL org.opencontainers.image.title="NGINX Ingress Controller for Kubernetes"
@@ -29,12 +30,16 @@ WORKDIR  /etc/nginx
 
 RUN apk update \
   && apk upgrade \
-  && apk add -U --no-cache \
+  && apk add --no-cache \
     diffutils \
-    libcap \
   && rm -rf /var/cache/apk/*
 
-COPY --chown=www-data:www-data . /
+COPY --chown=www-data:www-data etc /etc
+COPY --chown=www-data:www-data ingress-controller /ingress-controller
+
+COPY --chown=www-data:www-data bin/${TARGETARCH}/dbg /
+COPY --chown=www-data:www-data bin/${TARGETARCH}/nginx-ingress-controller /
+COPY --chown=www-data:www-data bin/${TARGETARCH}/wait-shutdown /
 
 # Fix permission during the build to avoid issues at runtime
 # with volumes (custom templates)
@@ -51,11 +56,12 @@ RUN bash -xeu -c ' \
     chown -R www-data.www-data ${dir}; \
   done'
 
-RUN  setcap    cap_net_bind_service=+ep /nginx-ingress-controller \
+RUN apk add --no-cache libcap \
-  && setcap -v cap_net_bind_service=+ep /nginx-ingress-controller
+  && setcap    cap_net_bind_service=+ep /nginx-ingress-controller \
-
+  && setcap -v cap_net_bind_service=+ep /nginx-ingress-controller \
-RUN  setcap    cap_net_bind_service=+ep /usr/local/nginx/sbin/nginx \
+  && setcap    cap_net_bind_service=+ep /usr/local/nginx/sbin/nginx \
-  && setcap -v cap_net_bind_service=+ep /usr/local/nginx/sbin/nginx
+  && setcap -v cap_net_bind_service=+ep /usr/local/nginx/sbin/nginx \
+  && apk del libcap
 
 USER www-data
 

test/e2e-image/Makefile

@@ -27,9 +27,7 @@ endif
 	# TODO: avoid manual copy
 	cp -R $(DIR)/../../test/e2e/settings/ocsp/* .
 
-	docker buildx build \
+	docker build \
-		--load \
-		--progress plain \
 		--tag nginx-ingress-controller:e2e .
 
 .PHONY: clean

test/e2e/run.sh

@@ -96,11 +96,9 @@ then
 else
   echo "Image builds were not ok! Log:"
   cat /tmp/log
-  exit
+  exit 1
 fi
 
-docker tag ${REGISTRY}/nginx-ingress-controller-${ARCH}:${TAG} ${REGISTRY}/nginx-ingress-controller:${TAG}
-
 # Preload images used in e2e tests
 docker pull moul/grpcbin
 docker pull quay.io/kubernetes-ingress-controller/nginx:e3c49c52f4b74fe47ad65d6f3266a02e8b6b622f