Fix terraform build of nginx images (#4547)

build/build-nginx-image.sh

@@ -22,25 +22,20 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
-declare -a mandatory
-mandatory=(
-  AWS_ACCESS_KEY
-  AWS_SECRET_KEY
-)
+DIR=$(cd $(dirname "${BASH_SOURCE}") && pwd -P)
 
-missing=false
-for var in "${mandatory[@]}"; do
-  if [[ -z "${!var:-}" ]]; then
-    echo "Environment variable $var must be set"
-    missing=true
-  fi
-done
+AWS_FILE="${DIR}/images/nginx/aws.tfvars"
+ENV_FILE="${DIR}/images/nginx/env.tfvars"
 
-if [ "$missing" = true ]; then
+if [ ! -f "${AWS_FILE}" ]; then
+  echo "File $AWS_FILE does not exist. Please create this file with keys access_key an secret_key"
   exit 1
 fi
 
-DIR=$(cd $(dirname "${BASH_SOURCE}") && pwd -P)
+if [ ! -f "${ENV_FILE}" ]; then
+  echo "File $ENV_FILE does not exist. Please create this file with keys docker_username and docker_password"
+  exit 1
+fi
 
 # build local terraform image to build nginx
 docker build -t build-nginx-terraform $DIR/images/nginx
@@ -50,9 +45,6 @@ docker build -t build-nginx-terraform $DIR/images/nginx
 docker run --rm -it \
   --volume $DIR/images/nginx:/tf \
   -w /tf \
-  --env AWS_ACCESS_KEY=${AWS_ACCESS_KEY} \
-  --env AWS_SECRET_KEY=${AWS_SECRET_KEY} \
-  --env AWS_SECRET_KEY=${AWS_SECRET_KEY} \
-  --env QUAY_USERNAME=${QUAY_USERNAME} \
-  --env QUAY_PASSWORD="${QUAY_PASSWORD}" \
+  -v ${AWS_FILE}:/root/aws.tfvars:ro \
+  -v ${ENV_FILE}:/root/env.tfvars:ro \
   build-nginx-terraform

build/images/nginx/.dockerignore

@@ -0,0 +1 @@
+*.tfvars

build/images/nginx/.gitignore

@@ -1,6 +1,7 @@
-.terraform
 .terraform*
 terraform*
 *.tfstate
 *.tfstate.backup
 id_rsa*
+aws.tfvars
+env.tfvars

build/images/nginx/build-nginx.sh

@@ -18,24 +18,39 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
+function source_tfvars() {
+  eval "$(
+    awk 'BEGIN {FS=OFS="="}
+    !/^(#| *$)/ && /^.+=.+$/ {
+      gsub(/^[ \t]+|[ \t]+$/, "", $1);
+      gsub(/\./, "_", $1);
+      gsub(/^[ \t]+|[ \t]+$/, "", $2);
+      if ($1 && $2) print $0
+    }' "$@"
+  )"
+}
+
+source_tfvars /tmp/env
+
 export DEBIAN_FRONTEND=noninteractive
 export AR_FLAGS=cr
 
-apt update
+apt -q=3 update
 
-apt dist-upgrade --yes
+apt -q=3 dist-upgrade --yes
 
 add-apt-repository universe   --yes
 add-apt-repository multiverse --yes
 
-apt update
+apt -q=3 update
 
-apt install \
+apt -q=3 install \
   apt-transport-https \
   ca-certificates \
   curl \
   make \
   htop \
+  parallel \
   software-properties-common --yes
 
 curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
@@ -45,15 +60,16 @@ add-apt-repository \
   $(lsb_release -cs) \
   stable" --yes
 
-apt update
+apt -q=3 update
 
-apt install docker-ce --yes
+apt -q=3 install docker-ce --yes
+
+echo ${docker_password} | docker login -u ${docker_username} --password-stdin quay.io
 
 curl -sL -o /usr/local/bin/gimme https://raw.githubusercontent.com/travis-ci/gimme/master/gimme
 chmod +x /usr/local/bin/gimme
 
 eval "$(gimme 1.13)"
-gimme 1.13
 
 git clone https://github.com/kubernetes/ingress-nginx
 
@@ -61,34 +77,21 @@ cd ingress-nginx/images/nginx
 
 make register-qemu
 
-PARALLELISM=${PARALLELISM:-3}
-
 export TAG=$(git rev-parse HEAD)
 
-# Borrowed from https://github.com/kubernetes-sigs/kind/blob/master/hack/release/build/cross.sh#L27
-echo "Building in parallel for:"
-# What we do here:
-# - use xargs to build in parallel (-P) while collecting a combined exit code
-# - use cat to supply the individual args to xargs (one line each)
-# - use env -S to split the line into environment variables and execute
-# - ... the build
-# shellcheck disable=SC2016
-if xargs -0 -n1 -P "${PARALLELISM}" bash -c 'eval $0; TAG=${TAG} make sub-container-${ARCH} > build-${ARCH}.log'; then
-  echo "Docker build finished without issues" 1>&2
-else
-  echo "Docker build failed!" 1>&2
-  cat build-amd64.log
-  cat build-arm.log
-  cat build-arm64.log
-  exit 1
-fi < <(cat <<EOF | tr '\n' '\0'
-ARCH=amd64
-ARCH=arm
-ARCH=arm64
-EOF
-)
+echo "Building NGINX image in parallel:"
+echo "
+make sub-container-amd64
+make sub-container-arm
+make sub-container-arm64
+" | parallel {}
 
+echo "Docker images:"
 docker images
 
-echo $QUAY_PASSWORD | sudo docker login -u $QUAY_USERNAME --password-stdin quay.io
-make all-push
+echo "Publishing docker images..."
+echo "
+make sub-push-amd64
+make sub-push-arm
+make sub-push-arm64
+" | parallel {}

build/images/nginx/entrypoint.sh

@@ -18,18 +18,37 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
+catch() {
+  if [ "$1" == "0" ]; then
+    exit 0
+  fi
+
+  echo "Error $1 occurred on $2"
+
+  echo "Removing temporal resources..."
+  terraform destroy -auto-approve \
+    -var-file /root/aws.tfvars \
+    -var-file /root/env.tfvars \
+    -var valid_until="${EC2_VALID_UNTIL}"
+}
+trap 'catch $? $LINENO' ERR
+
 terraform init
 
+# destroy spot instance after two hours
+EC2_VALID_UNTIL=$(date -d "+2 hours" +%Y-%m-%dT%H:%M:%SZ)
+
 terraform plan \
-  -var access_key="${AWS_ACCESS_KEY}" \
-  -var secret_key="${AWS_SECRET_KEY}" \
-  .
+  -var-file /root/aws.tfvars \
+  -var-file /root/env.tfvars \
+  -var valid_until="${EC2_VALID_UNTIL}"
 
 terraform apply -auto-approve \
-  -var access_key="${AWS_ACCESS_KEY}" \
-  -var secret_key="${AWS_SECRET_KEY}" \
-  .
+  -var-file /root/aws.tfvars \
+  -var-file /root/env.tfvars \
+  -var valid_until="${EC2_VALID_UNTIL}"
 
 terraform destroy -auto-approve \
-  -var access_key="${AWS_ACCESS_KEY}" \
-  -var secret_key="${AWS_SECRET_KEY}" \
+  -var-file /root/aws.tfvars \
+  -var-file /root/env.tfvars \
+  -var valid_until="${EC2_VALID_UNTIL}"

build/images/nginx/main.tf

@@ -133,9 +133,12 @@ resource "aws_spot_instance_request" "build_worker" {
   subnet_id              = aws_subnet.subnet_public.id
   vpc_security_group_ids = [aws_security_group.allow_ssh.id]
 
+  valid_until = var.valid_until
+
   key_name = aws_key_pair.ssh_key.key_name
 
   spot_price = "2"
+  spot_type = "one-time"
 
   ebs_optimized = true
 
@@ -146,6 +149,7 @@ resource "aws_spot_instance_request" "build_worker" {
   }
 
   wait_for_fulfillment = true
+  instance_initiated_shutdown_behavior = "terminate"
 
   associate_public_ip_address = true
 
@@ -165,10 +169,14 @@ resource "aws_spot_instance_request" "build_worker" {
     destination = "/tmp/build-nginx.sh"
   }
 
+  provisioner "file" {
+    source      = "/root/env.tfvars"
+    destination = "/tmp/env"
+  }
+
   provisioner "remote-exec" {
     inline = [
       "echo Building nginx images...",
-      "ls /tmp",
       "chmod +x /tmp/build-nginx.sh",
       "sudo /tmp/build-nginx.sh",
     ]

build/images/nginx/variables.tf

@@ -4,6 +4,15 @@ variable "access_key" {
 variable "secret_key" {
 }
 
+variable "valid_until" {
+}
+
+variable "docker_username" {
+}
+
+variable "docker_password" {
+}
+
 variable "region" {
   default = "us-west-2"
 }
@@ -20,7 +29,7 @@ variable "cidr_subnet" {
 
 variable "availability_zone" {
   description = "availability zone to create subnet"
-  default     = "us-west-2a"
+  default     = "us-west-2b"
 }
 
 variable "ssh_key_path" {
@@ -35,7 +44,7 @@ variable "ssh_public_key_path" {
 
 variable "instance_type" {
   description = "EC2 instance"
-  default     = "m5.16xlarge"
+  default     = "c5.18xlarge"
 }
 
 variable "project_tag" {