Cleanup of e2e docker images (#4962)

Makefile

@@ -251,7 +251,7 @@ dep-ensure: check-go-version ## Update and vendo go dependencies.
 
 .PHONY: dev-env
 dev-env: check-go-version ## Starts a local Kubernetes cluster using minikube, building and deploying the ingress controller.
-	@DIND_DOCKER=0 USE_DOCKER=false build/dev-env.sh
+	@DIND_TASKS=0 USE_DOCKER=false build/dev-env.sh
 
 .PHONY: live-docs
 live-docs: ## Build and launch a local copy of the documentation website in http://localhost:3000
@@ -273,7 +273,7 @@ misspell: check-go-version ## Check for spelling errors.
 
 .PHONY: kind-e2e-test
 kind-e2e-test: check-go-version ## Run e2e tests using kind.
-	@test/e2e/run.sh
+	@DIND_TASKS=0 test/e2e/run.sh
 
 .PHONY: run-ingress-controller
 run-ingress-controller: ## Run the ingress controller locally using a kubectl proxy connection.

build/build-ingress-controller.sh

@@ -38,7 +38,10 @@ if [ ! -f "${ENV_FILE}" ]; then
 fi
 
 # build local terraform image to build nginx
-docker build -t build-ingress-controller-terraform $DIR/images/ingress-controller
+docker buildx build \
+  --load \
+  --platform linux/amd64 \
+  --tag build-ingress-controller-terraform $DIR/images/ingress-controller
 
 # build nginx and publish docker images to quay.io.
 # this can take up to two hours.

build/build-nginx-image.sh

@@ -40,6 +40,7 @@ fi
 # build local terraform image to build nginx
 export DOCKER_CLI_EXPERIMENTAL=enabled
 docker buildx build \
+  --load \
   --no-cache \
   --platform linux/amd64 \
   --tag build-nginx-terraform $DIR/images/nginx

images/e2e-prow/Makefile

@@ -7,11 +7,12 @@ IMAGE = $(REGISTRY)/e2e-prow
 all: docker-build docker-push
 
 docker-build:
-	$(DOCKER) build \
+	$(DOCKER) buildx build \
 		--pull \
+		--load \
 		--build-arg K8S_RELEASE=v1.17.0 \
 		--build-arg ETCD_VERSION=v3.3.15 \
-		--build-arg KIND_VERSION=v0.6.1 \
+		--build-arg KIND_VERSION=v0.7.0 \
 		-t $(IMAGE):$(TAG) .
 
 docker-push:

images/e2e/Makefile

@@ -20,7 +20,7 @@ IMAGE = $(REGISTRY)/e2e
 all: docker-build docker-push
 
 docker-build:
-	docker build \
+	docker buildx build \
 		--pull \
 		--load \
 		--build-arg K8S_RELEASE=v1.15.7 \

images/fastcgi-helloserver/Makefile

@@ -1,101 +1,40 @@
-all: all-container
+# Copyright 2020 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
-BUILDTAGS=
+# Docker image for e2e testing.
 
 # Use the 0.0 tag for testing, it shouldn't clobber any release builds
-TAG?=0.1
+TAG ?= 0.0
-REGISTRY?=kubernetes-ingress-controller
-GOOS?=linux
-DOCKER?=docker
-SED_I?=sed -i
-GOHOSTOS ?= $(shell go env GOHOSTOS)
 
-PKG=k8s.io/ingress-nginx/images/fastcgi-helloserver
+REGISTRY ?= ingress-controller
-
+DOCKER ?= docker
-ifeq ($(GOHOSTOS),darwin)
-  SED_I=sed -i ''
-endif
-
-REPO_INFO=$(shell git config --get remote.origin.url)
-
-ARCH ?= $(shell go env GOARCH)
-GOARCH = ${ARCH}
-
-BASEIMAGE?=alpine:3.10
-
-ALL_ARCH = amd64 arm arm64
-
-QEMUVERSION=v4.1.0-1
 
 IMGNAME = fastcgi-helloserver
 IMAGE = $(REGISTRY)/$(IMGNAME)
-MULTI_ARCH_IMG = $(IMAGE)-$(ARCH)
 
-ifeq ($(ARCH),arm)
+PKG=k8s.io/ingress-nginx/images/fastcgi-helloserver
-    QEMUARCH=arm
-	GOARCH=arm
-endif
-ifeq ($(ARCH),arm64)
-	QEMUARCH=aarch64
-endif
 
-TEMP_DIR := $(shell mktemp -d)
+container: clean build
-
+	$(DOCKER) buildx build \
-DOCKERFILE := $(TEMP_DIR)/rootfs/Dockerfile
+		--load \
-
+		--platform linux/amd64 \
-sub-container-%:
+		-t $(IMAGE):$(TAG) rootfs
-	$(MAKE) ARCH=$* build container
-
-sub-push-%:
-	$(MAKE) ARCH=$* push
-
-all-container: $(addprefix sub-container-,$(ALL_ARCH))
-
-all-push: $(addprefix sub-push-,$(ALL_ARCH))
-
-container: .container-$(ARCH)
-.container-$(ARCH):
-	cp -r ./* $(TEMP_DIR)
-	$(SED_I) 's|BASEIMAGE|$(BASEIMAGE)|g' $(DOCKERFILE)
-	$(SED_I) "s|QEMUARCH|$(QEMUARCH)|g" $(DOCKERFILE)
-
-ifeq ($(ARCH),amd64)
-	# When building "normally" for amd64, remove the whole line, it has no part in the amd64 image
-	$(SED_I) "/CROSS_BUILD_/d" $(DOCKERFILE)
-else
-	# When cross-building, only the placeholder "CROSS_BUILD_" should be removed
-	# Register /usr/bin/qemu-ARCH-static as the handler for ARM binaries in the kernel
-	# $(DOCKER) run --rm --privileged multiarch/qemu-user-static:register --reset
-	curl -sSL https://github.com/multiarch/qemu-user-static/releases/download/$(QEMUVERSION)/x86_64_qemu-$(QEMUARCH)-static.tar.gz | tar -xz -C $(TEMP_DIR)/rootfs
-	$(SED_I) "s/CROSS_BUILD_//g" $(DOCKERFILE)
-endif
-
-	$(DOCKER) build -t $(MULTI_ARCH_IMG):$(TAG) $(TEMP_DIR)/rootfs
-
-ifeq ($(ARCH), amd64)
-	# This is for to maintain the backward compatibility
-	$(DOCKER) tag $(MULTI_ARCH_IMG):$(TAG) $(IMAGE):$(TAG)
-endif
-
-push: .push-$(ARCH)
-.push-$(ARCH):
-	$(DOCKER) push $(MULTI_ARCH_IMG):$(TAG)
-ifeq ($(ARCH), amd64)
-	$(DOCKER) push $(IMAGE):$(TAG)
-endif
-
-clean:
-	$(DOCKER) rmi -f $(MULTI_ARCH_IMG):$(TAG) || true
 
 build: clean
-	CGO_ENABLED=0 GOOS=${GOOS} GOARCH=${GOARCH} go build -a -installsuffix cgo \
+	CGO_ENABLED=0 go build -a -installsuffix cgo \
 		-ldflags "-s -w" \
-		-o ${TEMP_DIR}/rootfs/fastcgi-helloserver ${PKG}/...
+		-o rootfs/fastcgi-helloserver ${PKG}/...
 
-release: all-container all-push
+clean:
-	echo "done"
+	$(DOCKER) rmi -f $(IMAGE):$(TAG) || true
-
-.PHONY: register-qemu
-register-qemu:
-	# Register /usr/bin/qemu-ARCH-static as the handler for binaries in multiple platforms
-	$(DOCKER) run --rm --privileged multiarch/qemu-user-static:register --reset

images/fastcgi-helloserver/rootfs/.gitignore

@@ -0,0 +1 @@
+fastcgi-helloserver

images/fastcgi-helloserver/rootfs/Dockerfile

@@ -12,9 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM BASEIMAGE
+FROM k8s.gcr.io/debian-base:v2.0.0
-
-CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/
 
 COPY . /
 

images/httpbin/Makefile

@@ -1,94 +1,33 @@
-all: all-container
+# Copyright 2020 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
-BUILDTAGS=
+# Docker image for e2e testing.
 
 # Use the 0.0 tag for testing, it shouldn't clobber any release builds
-TAG?=0.1
+TAG ?= 0.0
-REGISTRY?=kubernetes-ingress-controller
-GOOS?=linux
-DOCKER?=docker
-SED_I?=sed -i
-GOHOSTOS ?= $(shell go env GOHOSTOS)
 
-ifeq ($(GOHOSTOS),darwin)
+REGISTRY ?= ingress-controller
-  SED_I=sed -i ''
+DOCKER ?= docker
-endif
-
-REPO_INFO=$(shell git config --get remote.origin.url)
-
-ARCH ?= $(shell go env GOARCH)
-GOARCH = ${ARCH}
-
-# Set default base image dynamically for each arch
-BASEIMAGE=quay.io/kubernetes-ingress-controller/debian-base-$(ARCH):0.1
-
-ALL_ARCH = amd64 arm arm64
-
-QEMUVERSION=v4.1.0-1
 
 IMGNAME = httpbin
 IMAGE = $(REGISTRY)/$(IMGNAME)
-MULTI_ARCH_IMG = $(IMAGE)-$(ARCH)
 
-ifeq ($(ARCH),arm)
+container:
-    QEMUARCH=arm
+	$(DOCKER) buildx build \
-endif
+		--load \
-ifeq ($(ARCH),arm64)
+		--platform linux/amd64 \
-	QEMUARCH=aarch64
+		-t $(IMAGE):$(TAG) rootfs
-endif
-
-TEMP_DIR := $(shell mktemp -d)
-
-DOCKERFILE := $(TEMP_DIR)/rootfs/Dockerfile
-
-sub-container-%:
-	$(MAKE) ARCH=$* container
-
-sub-push-%:
-	$(MAKE) ARCH=$* push
-
-all-container: $(addprefix sub-container-,$(ALL_ARCH))
-
-all-push: $(addprefix sub-push-,$(ALL_ARCH))
-
-container: .container-$(ARCH)
-.container-$(ARCH):
-	cp -r ./* $(TEMP_DIR)
-	$(SED_I) 's|BASEIMAGE|$(BASEIMAGE)|g' $(DOCKERFILE)
-	$(SED_I) "s|QEMUARCH|$(QEMUARCH)|g" $(DOCKERFILE)
-
-ifeq ($(ARCH),amd64)
-	# When building "normally" for amd64, remove the whole line, it has no part in the amd64 image
-	$(SED_I) "/CROSS_BUILD_/d" $(DOCKERFILE)
-else
-	# When cross-building, only the placeholder "CROSS_BUILD_" should be removed
-	# Register /usr/bin/qemu-ARCH-static as the handler for ARM binaries in the kernel
-	# $(DOCKER) run --rm --privileged multiarch/qemu-user-static:register --reset
-	curl -sSL https://github.com/multiarch/qemu-user-static/releases/download/$(QEMUVERSION)/x86_64_qemu-$(QEMUARCH)-static.tar.gz | tar -xz -C $(TEMP_DIR)/rootfs
-	$(SED_I) "s/CROSS_BUILD_//g" $(DOCKERFILE)
-endif
-
-	$(DOCKER) build -t $(MULTI_ARCH_IMG):$(TAG) $(TEMP_DIR)/rootfs
-
-ifeq ($(ARCH), amd64)
-	# This is for to maintain the backward compatibility
-	$(DOCKER) tag $(MULTI_ARCH_IMG):$(TAG) $(IMAGE):$(TAG)
-endif
-
-push: .push-$(ARCH)
-.push-$(ARCH):
-	$(DOCKER) push $(MULTI_ARCH_IMG):$(TAG)
-ifeq ($(ARCH), amd64)
-	$(DOCKER) push $(IMAGE):$(TAG)
-endif
 
 clean:
-	$(DOCKER) rmi -f $(MULTI_ARCH_IMG):$(TAG) || true
+	$(DOCKER) rmi -f $(IMAGE):$(TAG) || true
-
-release: all-container all-push
-	echo "done"
-
-.PHONY: register-qemu
-register-qemu:
-	# Register /usr/bin/qemu-ARCH-static as the handler for binaries in multiple platforms
-	$(DOCKER) run --rm --privileged multiarch/qemu-user-static:register --reset

images/httpbin/rootfs/Dockerfile

@@ -12,9 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM BASEIMAGE
+FROM k8s.gcr.io/debian-base:v2.0.0
-
-CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/
 
 ENV LC_ALL=C.UTF-8
 ENV LANG=C.UTF-8

images/mkdocs/Dockerfile

@@ -12,9 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM alpine:3.10
+FROM alpine:3.11
 
-RUN apk update && apk add --no-cache \
+RUN apk add --no-cache \
   bash \
   git \
   git-fast-import \

test/e2e/defaultbackend/default_backend.go

@@ -96,12 +96,13 @@ var _ = framework.IngressNginxDescribe("Default backend", func() {
 			Expect(resp.StatusCode).Should(Equal(test.Status))
 		}
 	})
+
 	It("enables access logging for default backend", func() {
 		f.UpdateNginxConfigMapData("enable-access-log-for-default-backend", "true")
-		host := "foo"
+
 		resp, _, errs := gorequest.New().
 			Get(f.GetURL(framework.HTTP)+"/somethingOne").
-			Set("Host", host).
+			Set("Host", "foo").
 			End()
 
 		Expect(len(errs)).Should(Equal(0))
@@ -114,10 +115,10 @@ var _ = framework.IngressNginxDescribe("Default backend", func() {
 
 	It("disables access logging for default backend", func() {
 		f.UpdateNginxConfigMapData("enable-access-log-for-default-backend", "false")
-		host := "bar"
+
 		resp, _, errs := gorequest.New().
 			Get(f.GetURL(framework.HTTP)+"/somethingTwo").
-			Set("Host", host).
+			Set("Host", "bar").
 			End()
 
 		Expect(len(errs)).Should(Equal(0))

test/e2e/defaultbackend/ssl.go

@@ -59,6 +59,7 @@ var _ = framework.IngressNginxDescribe("Default backend - SSL", func() {
 				InsecureSkipVerify: true,
 			}).
 			Set("Host", "foo.bar.com").End()
+
 		Expect(errs).Should(BeEmpty())
 		Expect(len(resp.TLS.PeerCertificates)).Should(BeNumerically("==", 1))
 		for _, pc := range resp.TLS.PeerCertificates {

test/e2e/framework/framework.go

@@ -332,6 +332,7 @@ func (f *Framework) UpdateNginxConfigMapData(key string, value string) {
 	config[key] = value
 
 	f.SetNginxConfigMapData(config)
+	time.Sleep(1 * time.Second)
 }
 
 // DeleteNGINXPod deletes the currently running pod. It waits for the replacement pod to be up.

test/e2e/gracefulshutdown/shutdown.go

@@ -50,6 +50,8 @@ var _ = framework.IngressNginxDescribe("Shutdown ingress controller", func() {
 				return Expect(server).Should(ContainSubstring("server_name shutdown"))
 			})
 
+		time.Sleep(1 * time.Second)
+
 		resp, _, _ := gorequest.New().
 			Get(f.GetURL(framework.HTTP)+"/sleep/1").
 			Set("Host", host).
@@ -89,6 +91,8 @@ var _ = framework.IngressNginxDescribe("Shutdown ingress controller", func() {
 				return Expect(server).Should(ContainSubstring("server_name shutdown"))
 			})
 
+		time.Sleep(1 * time.Second)
+
 		result := make(chan *asyncResult)
 
 		startTime := time.Now()
@@ -148,6 +152,8 @@ var _ = framework.IngressNginxDescribe("Shutdown ingress controller", func() {
 				return Expect(server).Should(ContainSubstring("server_name shutdown"))
 			})
 
+		time.Sleep(1 * time.Second)
+
 		result := make(chan *asyncResult)
 
 		startTime := time.Now()