DevFW-CICD/ingress-nginx-helm
14
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Refactor build of docker images

Browse source
This commit is contained in:
Manuel Alejandro de Brito Fontes 2020-06-01 21:22:32 -04:00
parent 8e2eebb197
commit ea8e711d2c
21 changed files with 89 additions and 183 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

90
Makefile
View file

@ -42,7 +42,7 @@ endif
# Allow limiting the scope of the e2e tests. By default run everything
FOCUS ?= .*
# number of parallel test
E2E_NODES ?= 14
E2E_NODES ?= 15
# slow test only if takes > 50s
SLOW_E2E_THRESHOLD ?= 50
# run e2e test suite with tests that check for memory leaks? (default is false)
@ -61,75 +61,31 @@ endif
REGISTRY ?= quay.io/kubernetes-ingress-controller
BASE_IMAGE ?= quay.io/kubernetes-ingress-controller/nginx:e3c49c52f4b74fe47ad65d6f3266a02e8b6b622f
BASE_IMAGE ?= $(REGISTRY)/nginx:e3c49c52f4b74fe47ad65d6f3266a02e8b6b622f
GOARCH=$(ARCH)
GOBUILD_FLAGS := -v
# use vendor directory instead of go modules https://github.com/golang/go/wiki/Modules
GO111MODULE=off
TEMP_DIR := $(shell mktemp -d)
DOCKERFILE := $(TEMP_DIR)/rootfs/Dockerfile
help: ## Display this help
@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m<target>\033[0m\n"} /^[a-zA-Z_-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
# internal task
.PHONY: sub-image-%
sub-image-%:
$(MAKE) ARCH=$* build image
# internal task
.PHONY: sub-push-%
sub-push-%: ## Publish image for a particular arch.
$(MAKE) ARCH=$* push
.PHONY: image
image: clean-image .image-$(ARCH) ## Build image for a particular arch.
# internal task to build image for a particular arch.
.PHONY: .image-$(ARCH)
.image-$(ARCH): init-docker-buildx
mkdir -p $(TEMP_DIR)/rootfs
cp bin/$(ARCH)/nginx-ingress-controller $(TEMP_DIR)/rootfs/nginx-ingress-controller
cp bin/$(ARCH)/dbg $(TEMP_DIR)/rootfs/dbg
cp bin/$(ARCH)/wait-shutdown $(TEMP_DIR)/rootfs/wait-shutdown
cp -RP rootfs/* $(TEMP_DIR)/rootfs
image: clean-image ## Build image for a particular arch.
echo "Building docker image ($(ARCH))..."
# buildx assumes images are multi-arch
docker buildx build \
--pull \
--load \
@cp -R bin/ rootfs/
@docker build \
--no-cache \
--progress plain \
--platform linux/$(ARCH) \
--build-arg BASE_IMAGE="$(BASE_IMAGE)" \
--build-arg VERSION="$(TAG)" \
-t $(REGISTRY)/nginx-ingress-controller-${ARCH}:$(TAG) $(TEMP_DIR)/rootfs
--build-arg TARGETARCH="$(ARCH)" \
-t $(REGISTRY)/nginx-ingress-controller:$(TAG) rootfs
.PHONY: clean-image
clean-image: ## Removes local image
echo "removing old image $(BASE_IMAGE)-$(ARCH):$(TAG)"
@docker rmi -f $(BASE_IMAGE)-$(ARCH):$(TAG) || true
.PHONY: push
push: .push-$(ARCH) ## Publish image for a particular arch.
# internal task
.PHONY: .push-$(ARCH)
.push-$(ARCH):
docker push $(REGISTRY)/nginx-ingress-controller-${ARCH}:$(TAG)
.PHONY: push-manifest
push-manifest:
docker manifest create $(REGISTRY)/nginx-ingress-controller:$(TAG) \
$(REGISTRY)/nginx-ingress-controller-amd64:$(TAG) \
$(REGISTRY)/nginx-ingress-controller-arm:$(TAG) \
$(REGISTRY)/nginx-ingress-controller-arm64:$(TAG)
docker manifest push --purge $(REGISTRY)/nginx-ingress-controller:$(TAG)
echo "removing old image $(BASE_IMAGE):$(TAG)"
@docker rmi -f $(BASE_IMAGE):$(TAG) || true
.PHONY: build
build: check-go-version ## Build ingress controller, debug tool and pre-stop hook.
@ -300,9 +256,25 @@ endif
show-version:
echo -n $(TAG)
.PHONY: staging-gcr
staging-gcr:
echo "Building NGINX image..."
ARCH=amd64 make build image push
ARCH=arm make build image push
ARCH=arm64 make build image push
PLATFORMS ?= amd64 arm arm64 s390x
EMPTY :=
SPACE := $(EMPTY) $(EMPTY)
COMMA := ,
.PHONY: release # Build a multi-arch docker image
release: init-docker-buildx clean
echo "Building binaries..."
$(foreach PLATFORM,$(PLATFORMS), ARCH=$(PLATFORM) make build;)
@cp -R bin/ rootfs/
echo "Building and pushing ingress-nginx image..."
@docker buildx build \
--no-cache \
--push \
--progress plain \
--platform $(subst $(SPACE),$(COMMA),$(PLATFORMS)) \
--build-arg BASE_IMAGE="$(BASE_IMAGE)" \
--build-arg VERSION="$(TAG)" \
-t $(REGISTRY)/nginx-ingress-controller:$(TAG) rootfs

5
build/build-ingress-controller.sh
View file

@ -38,10 +38,7 @@ if [ ! -f "${ENV_FILE}" ]; then
fi
# build local terraform image to build nginx
docker buildx build \
--load \
--progress plain \
--platform linux/amd64 \
docker build \
--tag build-ingress-controller-terraform $DIR/images/ingress-controller
# build nginx and publish docker images to quay.io.

7
build/build-nginx-image.sh
View file

@ -37,12 +37,7 @@ if [ ! -f "${ENV_FILE}" ]; then
exit 1
fi
# build local terraform image to build nginx
export DOCKER_CLI_EXPERIMENTAL=enabled
docker buildx build \
--load \
--progress plain \
--platform linux/amd64 \
docker build \
--tag build-nginx-terraform $DIR/images/nginx
# build nginx and publish docker images to quay.io.

4
build/build.sh
View file

@ -47,7 +47,6 @@ export CGO_ENABLED=0
export GOARCH=${ARCH}
go build \
"${GOBUILD_FLAGS}" \
-ldflags "-s -w \
-X ${PKG}/version.RELEASE=${TAG} \
-X ${PKG}/version.COMMIT=${GIT_COMMIT} \
@ -55,16 +54,13 @@ go build \
-o "bin/${ARCH}/nginx-ingress-controller" "${PKG}/cmd/nginx"
go build \
"${GOBUILD_FLAGS}" \
-ldflags "-s -w \
-X ${PKG}/version.RELEASE=${TAG} \
-X ${PKG}/version.COMMIT=${GIT_COMMIT} \
-X ${PKG}/version.REPO=${REPO_INFO}" \
-o "bin/${ARCH}/dbg" "${PKG}/cmd/dbg"
go build \
"${GOBUILD_FLAGS}" \
-ldflags "-s -w \
-X ${PKG}/version.RELEASE=${TAG} \
-X ${PKG}/version.COMMIT=${GIT_COMMIT} \

2
build/run-in-docker.sh
View file

@ -34,7 +34,7 @@ function cleanup {
}
trap cleanup EXIT
E2E_IMAGE=quay.io/kubernetes-ingress-controller/e2e:v05312020-d250b97b4
E2E_IMAGE=${E2E_IMAGE:-quay.io/kubernetes-ingress-controller/e2e:v05312020-d250b97b4}
DOCKER_OPTS=${DOCKER_OPTS:-}

4
cloudbuild.yaml
View file

@ -12,10 +12,8 @@ steps:
- TAG=$_GIT_TAG
- BASE_REF=$_PULL_BASE_REF
- REGISTRY=gcr.io/k8s-staging-ingress-nginx
- DIND_TASKS=0
args:
- staging-gcr
- push-manifest
- release
substitutions:
# _GIT_TAG will be filled with a git-based tag for the image, of the form vYYYYMMDD-hash, and

17
images/cfssl/Makefile
View file

@ -17,27 +17,12 @@
# Use the 0.0 tag for testing, it shouldn't clobber any release builds
TAG ?= 0.0
HOSTARCH := $(shell uname -m | sed -e s/x86_64/amd64/ \
-e s/s390x/s390x/ \
-e s/armv7l/arm/ \
-e s/aarch64.*/arm64/)
ifndef ARCH
ARCH := $(HOSTARCH)
endif
ifeq ($(ARCH),)
$(error mandatory variable ARCH is empty)
endif
REGISTRY ?= ingress-controller
IMAGE = $(REGISTRY)/cfssl
image:
docker buildx build \
--load \
--progress plain \
--platform $(ARCH) \
docker build \
-t $(IMAGE):$(TAG) rootfs
clean:

23
images/e2e-prow/Makefile
View file

@ -1,3 +1,17 @@
# Copyright 2018 The Kubernetes Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
TAG ?=v$(shell date +%m%d%Y)-$(shell git rev-parse --short HEAD)
REGISTRY ?= quay.io/kubernetes-ingress-controller
@ -5,13 +19,8 @@ IMAGE = $(REGISTRY)/e2e-prow
.PHONY: image
image:
docker buildx build \
--pull \
--push \
--progress plain \
--platform amd64 \
docker build \
--build-arg K8S_RELEASE=v1.17.0 \
--build-arg ETCD_VERSION=v3.3.18 \
--build-arg KIND_VERSION=v0.8.0 \
--build-arg GO_VERSION=1.14.2 \
-t $(IMAGE):$(TAG) .
-t $(IMAGE):$(TAG) rootfs

10
images/e2e-prow/Dockerfile → images/e2e-prow/rootfs/Dockerfile
View file

@ -16,6 +16,9 @@
# unit and integration tests
FROM gcr.io/k8s-testimages/bootstrap
FROM k8s.gcr.io/etcd:3.4.3-0 as etcd
COPY --from=etcd /usr/local/bin/etcd /usr/local/bin/etcd
# hint to kubetest that it is in CI
ENV KUBETEST_IN_DOCKER="true"
@ -43,7 +46,6 @@ RUN apt-get update \
&& rm -rf /var/lib/apt/lists/*
ARG K8S_RELEASE
ARG ETCD_VERSION
ARG KIND_VERSION
ARG GO_VERSION
@ -53,12 +55,6 @@ RUN curl -sSL https://storage.googleapis.com/kubernetes-release/release/${K8S_RE
RUN curl -sSL https://storage.googleapis.com/kubernetes-release/release/${K8S_RELEASE}/bin/linux/amd64/kube-apiserver -o /usr/local/bin/kube-apiserver \
&& chmod +x /usr/local/bin/kube-apiserver
RUN curl -sSL https://storage.googleapis.com/etcd/${ETCD_VERSION}/etcd-${ETCD_VERSION}-linux-amd64.tar.gz -o /tmp/etcd-${ETCD_VERSION}-linux-amd64.tar.gz \
&& mkdir -p /tmp/etcd-download \
&& tar xzvf /tmp/etcd-${ETCD_VERSION}-linux-amd64.tar.gz -C /tmp/etcd-download --strip-components=1 \
&& cp /tmp/etcd-download/etcd /usr/local/bin \
&& rm -rf /tmp/etcd-download
RUN curl -sSL https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-linux-amd64 -o /usr/local/bin/kind \
&& chmod +x /usr/local/bin/kind

19
images/e2e/Makefile
View file

@ -17,25 +17,18 @@ REGISTRY ?= quay.io/kubernetes-ingress-controller
IMAGE = $(REGISTRY)/e2e
PLATFORM ?= amd64
.PHONY: image
image:
docker buildx build \
--pull \
--push \
--progress plain \
--platform $(PLATFORM) \
docker build \
--build-arg K8S_RELEASE=v1.15.7 \
--build-arg ETCD_VERSION=v3.3.18 \
--build-arg RESTY_CLI_VERSION=0.25rc2 \
--build-arg RESTY_CLI_SHA=a38d850441384fa037a5922ca012dcce8708d0e4abe34ad2fe4164a01b28bdfb \
--build-arg CHART_TESTING_VERSION=3.0.0-beta.1 \
--build-arg YAML_LINT_VERSION=1.13.0 \
--build-arg YAMALE_VERSION=1.8.0 \
--build-arg LUA_CHECK_VERSION=0.23.0 \
--build-arg LUA_CHECK_SHA=b4edf3a7702519502696d4ac7372ed1bd6a82ded63bf81f2b1d7e9b37711be2b \
--build-arg BUSTED_VERSION=v2.0.rc13-0 \
--build-arg BUSTED_SHA=74ef88ae04545d78b4922e459c1cd459fb5a0356b73a061549d2e4601b5de254 \
--build-arg HELM_VERSION=v3.2.0 \
-t $(IMAGE):$(TAG) .
-t $(IMAGE):$(TAG) rootfs
.PHONY: show-image
show-image:
echo -n $(IMAGE):$(TAG)

16
images/e2e/Dockerfile → images/e2e/rootfs/Dockerfile
View file

@ -13,6 +13,7 @@
# limitations under the License.
FROM golang:1.14.3-alpine3.11 as GO
FROM k8s.gcr.io/etcd:3.4.3-0 as etcd
FROM quay.io/kubernetes-ingress-controller/nginx:e3c49c52f4b74fe47ad65d6f3266a02e8b6b622f
@ -25,7 +26,8 @@ ARG CHART_TESTING_VERSION
RUN [ ! -e /etc/nsswitch.conf ] && echo 'hosts: files dns' > /etc/nsswitch.conf
COPY --from=GO /usr/local/go /usr/local/go
COPY --from=GO /usr/local/go /usr/local/go
COPY --from=etcd /usr/local/bin/etcd /usr/local/bin/etcd
RUN apk add --no-cache \
bash \
@ -56,16 +58,13 @@ RUN wget -O /tmp/resty_cli.tgz https://github.com/openresty/resty-cli/archive/v$
&& resty -V \
&& rm -rf /tmp/*
ARG LUA_CHECK_VERSION
ARG LUA_CHECK_SHA
RUN wget -O /tmp/luarocks.tgz https://github.com/luarocks/luarocks/archive/v3.3.1.tar.gz \
&& tar -C /tmp -xzf /tmp/luarocks.tgz \
&& cd /tmp/luarocks* \
&& ./configure \
&& make install
RUN luarocks install busted \
RUN luarocks install busted \
&& luarocks install luacheck
ARG BUSTED_VERSION
@ -81,13 +80,6 @@ RUN wget -O /usr/local/bin/kube-apiserver \
https://storage.googleapis.com/kubernetes-release/release/${K8S_RELEASE}/bin/linux/${TARGETARCH}/kube-apiserver \
&& chmod +x /usr/local/bin/kube-apiserver
RUN wget -O /tmp/etcd-${ETCD_VERSION}-linux-${TARGETARCH}.tar.gz \
https://storage.googleapis.com/etcd/${ETCD_VERSION}/etcd-${ETCD_VERSION}-linux-${TARGETARCH}.tar.gz \
&& mkdir -p /tmp/etcd-download \
&& tar xzvf /tmp/etcd-${ETCD_VERSION}-linux-${TARGETARCH}.tar.gz -C /tmp/etcd-download --strip-components=1 \
&& cp /tmp/etcd-download/etcd /usr/local/bin \
&& rm -rf /tmp/*
RUN wget -O /tmp/ct-${CHART_TESTING_VERSION}-linux-amd64.tar.gz \
https://github.com/helm/chart-testing/releases/download/v${CHART_TESTING_VERSION}/chart-testing_${CHART_TESTING_VERSION}_linux_amd64.tar.gz \
&& mkdir -p /tmp/ct-download \

19
images/echo/Makefile
View file

@ -17,28 +17,13 @@
# Use the 0.0 tag for testing, it shouldn't clobber any release builds
TAG ?= 0.0
HOSTARCH := $(shell uname -m | sed -e s/x86_64/amd64/ \
-e s/s390x/s390x/ \
-e s/armv7l/arm/ \
-e s/aarch64.*/arm64/)
ifndef ARCH
ARCH := $(HOSTARCH)
endif
ifeq ($(ARCH),)
$(error mandatory variable ARCH is empty)
endif
REGISTRY ?= ingress-controller
IMAGE = $(REGISTRY)/echo
image:
docker buildx build \
--load \
--progress plain \
--platform $(ARCH) \
-t $(IMAGE):$(TAG) .
docker build \
-t $(IMAGE):$(TAG) rootfs
clean:
docker rmi -f $(IMAGE):$(TAG) || true

0
images/echo/Dockerfile → images/echo/rootfs/Dockerfile
View file

0
images/echo/nginx.conf → images/echo/rootfs/nginx.conf
View file

5
images/fastcgi-helloserver/Makefile
View file

@ -37,10 +37,7 @@ PKG=k8s.io/ingress-nginx/images/fastcgi-helloserver
.PHONY: image
image: build
docker buildx build \
--load \
--progress plain \
--platform $(ARCH) \
docker build \
-t $(IMAGE):$(TAG) rootfs
.PHONY: build

17
images/httpbin/Makefile
View file

@ -17,26 +17,11 @@
# Use the 0.0 tag for testing, it shouldn't clobber any release builds
TAG ?= 0.0
HOSTARCH := $(shell uname -m | sed -e s/x86_64/amd64/ \
-e s/s390x/s390x/ \
-e s/armv7l/arm/ \
-e s/aarch64.*/arm64/)
ifndef ARCH
ARCH := $(HOSTARCH)
endif
ifeq ($(ARCH),)
$(error mandatory variable ARCH is empty)
endif
REGISTRY ?= ingress-controller
IMAGE = $(REGISTRY)/httpbin
.PHONY: image
image:
docker buildx build \
--load \
--progress plain \
--platform $(ARCH) \
docker build \
-t $(IMAGE):$(TAG) rootfs

2
rootfs/.dockerignore Normal file
View file

@ -0,0 +1,2 @@
Dockerfile
.dockerignore

2
rootfs/.gitignore vendored Normal file
View file

@ -0,0 +1,2 @@
bin/*

22
rootfs/Dockerfile
View file

@ -16,6 +16,7 @@ ARG BASE_IMAGE
FROM ${BASE_IMAGE}
ARG TARGETARCH
ARG VERSION
LABEL org.opencontainers.image.title="NGINX Ingress Controller for Kubernetes"
@ -29,12 +30,16 @@ WORKDIR /etc/nginx
RUN apk update \
&& apk upgrade \
&& apk add -U --no-cache \
&& apk add --no-cache \
diffutils \
libcap \
&& rm -rf /var/cache/apk/*
COPY --chown=www-data:www-data . /
COPY --chown=www-data:www-data etc /etc
COPY --chown=www-data:www-data ingress-controller /ingress-controller
COPY --chown=www-data:www-data bin/${TARGETARCH}/dbg /
COPY --chown=www-data:www-data bin/${TARGETARCH}/nginx-ingress-controller /
COPY --chown=www-data:www-data bin/${TARGETARCH}/wait-shutdown /
# Fix permission during the build to avoid issues at runtime
# with volumes (custom templates)
@ -51,11 +56,12 @@ RUN bash -xeu -c ' \
chown -R www-data.www-data ${dir}; \
done'
RUN setcap cap_net_bind_service=+ep /nginx-ingress-controller \
&& setcap -v cap_net_bind_service=+ep /nginx-ingress-controller
RUN setcap cap_net_bind_service=+ep /usr/local/nginx/sbin/nginx \
&& setcap -v cap_net_bind_service=+ep /usr/local/nginx/sbin/nginx
RUN apk add --no-cache libcap \
&& setcap cap_net_bind_service=+ep /nginx-ingress-controller \
&& setcap -v cap_net_bind_service=+ep /nginx-ingress-controller \
&& setcap cap_net_bind_service=+ep /usr/local/nginx/sbin/nginx \
&& setcap -v cap_net_bind_service=+ep /usr/local/nginx/sbin/nginx \
&& apk del libcap
USER www-data

4
test/e2e-image/Makefile
View file

@ -27,9 +27,7 @@ endif
# TODO: avoid manual copy
cp -R $(DIR)/../../test/e2e/settings/ocsp/* .
docker buildx build \
--load \
--progress plain \
docker build \
--tag nginx-ingress-controller:e2e .
.PHONY: clean

4
test/e2e/run.sh
View file

@ -96,11 +96,9 @@ then
else
echo "Image builds were not ok! Log:"
cat /tmp/log
exit
exit 1
fi
docker tag ${REGISTRY}/nginx-ingress-controller-${ARCH}:${TAG} ${REGISTRY}/nginx-ingress-controller:${TAG}
# Preload images used in e2e tests
docker pull moul/grpcbin
docker pull quay.io/kubernetes-ingress-controller/nginx:e3c49c52f4b74fe47ad65d6f3266a02e8b6b622f