Add terraform scripts to build nginx image (#4484)

build/build-nginx-image.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+
+# Copyright 2019 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ -n "$DEBUG" ]; then
+	set -x
+fi
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+declare -a mandatory
+mandatory=(
+  AWS_ACCESS_KEY
+  AWS_SECRET_KEY
+)
+
+missing=false
+for var in "${mandatory[@]}"; do
+  if [[ -z "${!var:-}" ]]; then
+    echo "Environment variable $var must be set"
+    missing=true
+  fi
+done
+
+if [ "$missing" = true ]; then
+  exit 1
+fi
+
+DIR=$(cd $(dirname "${BASH_SOURCE}") && pwd -P)
+
+# build local terraform image to build nginx
+docker build -t build-nginx-terraform $DIR/images/nginx
+
+# build nginx and publish docker images to quay.io.
+# this can take up to two hours.
+docker run --rm -it \
+  --volume $DIR/images/nginx:/tf \
+  -w /tf \
+  --env AWS_ACCESS_KEY=${AWS_ACCESS_KEY} \
+  --env AWS_SECRET_KEY=${AWS_SECRET_KEY} \
+  --env AWS_SECRET_KEY=${AWS_SECRET_KEY} \
+  --env QUAY_USERNAME=${QUAY_USERNAME} \
+  --env QUAY_PASSWORD="${QUAY_PASSWORD}" \
+  build-nginx-terraform

build/images/nginx/.gitignore

@@ -0,0 +1,6 @@
+.terraform
+.terraform*
+terraform*
+*.tfstate
+*.tfstate.backup
+id_rsa*

build/images/nginx/Dockerfile

@@ -0,0 +1,19 @@
+FROM quay.io/kubernetes-ingress-controller/debian-base-amd64:0.1
+
+ENV TERRAFORM_VERSION 0.12.6
+
+RUN clean-install \
+    bash \
+    curl \
+    ca-certificates \
+    unzip \
+    git \
+    openssh-client
+
+RUN curl -sSL -o /terraform.zip "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip" \
+    && unzip /terraform.zip -d /usr/bin \
+    && rm -rf /terraform.zip
+
+COPY entrypoint.sh /
+
+CMD [ "/entrypoint.sh" ]

build/images/nginx/build-nginx.sh

@@ -0,0 +1,94 @@
+#!/bin/bash
+
+# Copyright 2019 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+export DEBIAN_FRONTEND=noninteractive
+export AR_FLAGS=cr
+
+apt update
+
+apt dist-upgrade --yes
+
+add-apt-repository universe   --yes
+add-apt-repository multiverse --yes
+
+apt update
+
+apt install \
+  apt-transport-https \
+  ca-certificates \
+  curl \
+  make \
+  htop \
+  software-properties-common --yes
+
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+
+add-apt-repository \
+  "deb [arch=amd64] https://download.docker.com/linux/ubuntu \
+  $(lsb_release -cs) \
+  stable" --yes
+
+apt update
+
+apt install docker-ce --yes
+
+curl -sL -o /usr/local/bin/gimme https://raw.githubusercontent.com/travis-ci/gimme/master/gimme
+chmod +x /usr/local/bin/gimme
+
+eval "$(gimme 1.13)"
+gimme 1.13
+
+git clone https://github.com/kubernetes/ingress-nginx
+
+cd ingress-nginx/images/nginx
+
+make register-qemu
+
+PARALLELISM=${PARALLELISM:-3}
+
+export TAG=$(git rev-parse HEAD)
+
+# Borrowed from https://github.com/kubernetes-sigs/kind/blob/master/hack/release/build/cross.sh#L27
+echo "Building in parallel for:"
+# What we do here:
+# - use xargs to build in parallel (-P) while collecting a combined exit code
+# - use cat to supply the individual args to xargs (one line each)
+# - use env -S to split the line into environment variables and execute
+# - ... the build
+# shellcheck disable=SC2016
+if xargs -0 -n1 -P "${PARALLELISM}" bash -c 'eval $0; TAG=${TAG} make sub-container-${ARCH} > build-${ARCH}.log'; then
+  echo "Docker build finished without issues" 1>&2
+else
+  echo "Docker build failed!" 1>&2
+  cat build-amd64.log
+  cat build-arm.log
+  cat build-arm64.log
+  exit 1
+fi < <(cat <<EOF | tr '\n' '\0'
+ARCH=amd64
+ARCH=arm
+ARCH=arm64
+EOF
+)
+
+docker images
+
+echo $QUAY_PASSWORD | sudo docker login -u $QUAY_USERNAME --password-stdin quay.io
+make all-push

build/images/nginx/entrypoint.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+# Copyright 2019 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+terraform init
+
+terraform plan \
+  -var access_key="${AWS_ACCESS_KEY}" \
+  -var secret_key="${AWS_SECRET_KEY}" \
+  .
+
+terraform apply -auto-approve \
+  -var access_key="${AWS_ACCESS_KEY}" \
+  -var secret_key="${AWS_SECRET_KEY}" \
+  .
+
+terraform destroy -auto-approve \
+  -var access_key="${AWS_ACCESS_KEY}" \
+  -var secret_key="${AWS_SECRET_KEY}" \

build/images/nginx/main.tf

@@ -0,0 +1,176 @@
+terraform {
+  backend "local" {
+    path = "terraform.tfstate"
+  }
+}
+
+provider "aws" {
+  access_key = var.access_key
+  secret_key = var.secret_key
+  region     = var.region
+}
+
+resource "aws_vpc" "vpc" {
+  cidr_block           = var.cidr_vpc
+  enable_dns_support   = true
+  enable_dns_hostnames = true
+  tags = {
+    "Project" = var.project_tag
+  }
+}
+
+resource "aws_internet_gateway" "igw" {
+  vpc_id = aws_vpc.vpc.id
+  tags = {
+    "Project" = var.project_tag
+  }
+}
+
+resource "aws_subnet" "subnet_public" {
+  vpc_id                  = aws_vpc.vpc.id
+  cidr_block              = var.cidr_subnet
+  map_public_ip_on_launch = "true"
+  availability_zone       = var.availability_zone
+  tags = {
+    "Project" = var.project_tag
+  }
+}
+
+resource "aws_route_table" "rtb_public" {
+  vpc_id = aws_vpc.vpc.id
+
+  route {
+    cidr_block = "0.0.0.0/0"
+    gateway_id = aws_internet_gateway.igw.id
+  }
+
+  tags = {
+    "Project" = var.project_tag
+  }
+}
+
+resource "aws_route_table_association" "rta_subnet_public" {
+  subnet_id      = aws_subnet.subnet_public.id
+  route_table_id = aws_route_table.rtb_public.id
+}
+
+resource "aws_security_group" "allow_ssh" {
+  name   = "ssh"
+  vpc_id = aws_vpc.vpc.id
+
+  ingress {
+    from_port   = 22
+    to_port     = 22
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  egress {
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  tags = {
+    "Project" = var.project_tag
+  }
+}
+
+resource "tls_private_key" "bootstrap_private_key" {
+    algorithm = "RSA"
+    rsa_bits  = "4096"
+}
+
+resource "aws_key_pair" "ssh_key" {
+  key_name   = "ssh-key_${var.project_tag}"
+  public_key = chomp(tls_private_key.bootstrap_private_key.public_key_openssh)
+}
+
+resource "local_file" "public_key_openssh" {
+  count      = 1
+  depends_on = [tls_private_key.bootstrap_private_key]
+  content    = tls_private_key.bootstrap_private_key.public_key_pem
+  filename   = "id_rsa.pub"
+}
+
+resource "local_file" "private_key_openssh" {
+  count      = 1
+  depends_on = [tls_private_key.bootstrap_private_key]
+  content    = tls_private_key.bootstrap_private_key.private_key_pem
+  filename   = "id_rsa"
+}
+
+data "aws_ami" "latest-ubuntu" {
+  most_recent = true
+
+  owners = ["099720109477"]
+
+  filter {
+    name   = "name"
+    values = ["ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-*"]
+  }
+
+  filter {
+    name   = "virtualization-type"
+    values = ["hvm"]
+  }
+
+  filter {
+    name = "root-device-type"
+    values = ["ebs"]
+  }
+
+  filter {
+    name = "block-device-mapping.volume-type"
+    values = ["gp2"]
+  }
+}
+
+resource "aws_spot_instance_request" "build_worker" {
+  ami                    = data.aws_ami.latest-ubuntu.id
+  instance_type          = var.instance_type
+  subnet_id              = aws_subnet.subnet_public.id
+  vpc_security_group_ids = [aws_security_group.allow_ssh.id]
+
+  key_name = aws_key_pair.ssh_key.key_name
+
+  spot_price = "2"
+
+  ebs_optimized = true
+
+  root_block_device {
+    volume_size           = 32
+    volume_type           = "gp2"
+    delete_on_termination = true
+  }
+
+  wait_for_fulfillment = true
+
+  associate_public_ip_address = true
+
+  tags = {
+    "Project" = var.project_tag
+  }
+
+  connection {
+    host        = coalesce(self.public_ip, self.private_ip)
+    type        = "ssh"
+    user        = "ubuntu"
+    private_key = tls_private_key.bootstrap_private_key.private_key_pem
+  }
+
+  provisioner "file" {
+    source      = "build-nginx.sh"
+    destination = "/tmp/build-nginx.sh"
+  }
+
+  provisioner "remote-exec" {
+    inline = [
+      "echo Building nginx images...",
+      "ls /tmp",
+      "chmod +x /tmp/build-nginx.sh",
+      "sudo /tmp/build-nginx.sh",
+    ]
+  }
+}

build/images/nginx/variables.tf

@@ -0,0 +1,43 @@
+variable "access_key" {
+}
+
+variable "secret_key" {
+}
+
+variable "region" {
+  default = "us-west-2"
+}
+
+variable "cidr_vpc" {
+  description = "CIDR block for the VPC"
+  default     = "10.4.0.0/16"
+}
+
+variable "cidr_subnet" {
+  description = "CIDR block for the subnet"
+  default     = "10.4.0.0/24"
+}
+
+variable "availability_zone" {
+  description = "availability zone to create subnet"
+  default     = "us-west-2a"
+}
+
+variable "ssh_key_path" {
+  description = "Path to the SSH key"
+  default     = "~/.ssh/id_rsa"
+}
+
+variable "ssh_public_key_path" {
+  description = "Path to the public SSH key"
+  default     = "~/.ssh/id_rsa.pub"
+}
+
+variable "instance_type" {
+  description = "EC2 instance"
+  default     = "m5.16xlarge"
+}
+
+variable "project_tag" {
+  default = "kubernetes/ingress-nginx"
+}

build/images/nginx/versions.tf

@@ -0,0 +1,11 @@
+
+terraform {
+  required_version = ">= 0.12"
+
+  required_providers {
+    aws   = "~> 2.0"
+    tls   = "~> 2.0"
+    local = "~> 1.3"
+    null  = "~> 2.1"
+  }
+}

images/nginx/rootfs/build.sh

@@ -475,7 +475,12 @@ cd /usr/local/openresty
 export LUA_LIB_DIR=/usr/local/openresty/lualib
 export LUA_INCLUDE_DIR=/tmp/build/openresty-$OPENRESTY_VERSION/build/luajit-root/usr/local/openresty/luajit/include/luajit-2.1
 
-luarocks install lrexlib-pcre 2.7.2-1 PCRE_LIBDIR=${PCRE_DIR}
+ln -s $LUA_INCLUDE_DIR /usr/include/lua5.1
+
+if [[ ${ARCH} != "armv7l" ]]; then
+  luarocks install lrexlib-pcre 2.7.2-1 PCRE_LIBDIR=${PCRE_DIR}
+fi
+
 luarocks install lua-resty-iputils 0.3.0-1
 luarocks install lua-resty-cookie 0.1.0-1
 
@@ -484,8 +489,6 @@ cd "$BUILD_PATH/lua-resty-balancer-$LUA_RESTY_BALANCER_VERSION"
 make
 make install
 
-ln -s $LUA_INCLUDE_DIR /usr/include/lua5.1
-
 if [[ ${ARCH} != "armv7l" ]]; then
   /install_lua_resty_waf.sh
 fi