Merge pull request #3852 from aledbf/arm

Enable arm again

Makefile

@@ -46,7 +46,7 @@ DUMB_ARCH = ${ARCH}
 
 GOBUILD_FLAGS := -v
 
-ALL_ARCH = amd64 arm64
+ALL_ARCH = amd64 arm arm64
 
 QEMUVERSION = v4.0.0
 
@@ -54,6 +54,11 @@ BUSTED_ARGS =-v --pattern=_test
 
 GOOS = linux
 
+IMGNAME = nginx-ingress-controller
+IMAGE = $(REGISTRY)/$(IMGNAME)
+
+MULTI_ARCH_IMG = ${IMAGE}-${ARCH}
+
 export ARCH
 export DUMB_ARCH
 export TAG
@@ -64,14 +69,14 @@ export GIT_COMMIT
 export GOBUILD_FLAGS
 export REPO_INFO
 export BUSTED_ARGS
-
+export IMAGE
-IMGNAME = nginx-ingress-controller
-IMAGE = $(REGISTRY)/$(IMGNAME)
-MULTI_ARCH_IMG = $(IMAGE)-$(ARCH)
 
 # Set default base image dynamically for each arch
 BASEIMAGE?=quay.io/kubernetes-ingress-controller/nginx-$(ARCH):0.90
 
+ifeq ($(ARCH),arm)
+	QEMUARCH=arm
+endif
 ifeq ($(ARCH),arm64)
 	QEMUARCH=aarch64
 endif
@@ -131,7 +136,7 @@ clean-container:
 .PHONY: register-qemu
 register-qemu:
 	# Register /usr/bin/qemu-ARCH-static as the handler for binaries in multiple platforms
-	@$(DOCKER) run --rm --privileged multiarch/qemu-user-static:register --reset
+	@$(DOCKER) run --rm --privileged multiarch/qemu-user-static:register --reset >&2
 
 .PHONY: push
 push: .push-$(ARCH)
@@ -249,6 +254,10 @@ misspell:
 		-error \
 		cmd/* internal/* deploy/* docs/* design/* test/* README.md
 
-.PHONE: kind-e2e-test
+.PHONY: kind-e2e-test
 kind-e2e-test:
 	test/e2e/run.sh
+
+.PHONY: run-ingress-controller
+run-ingress-controller:
+	@build/run-ingress-controller.sh

build/run-ingress-controller.sh

@@ -0,0 +1,103 @@
+#!/bin/bash
+
+# Copyright 2018 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if ! [ -z "$DEBUG" ]; then
+	set -x
+fi
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+RED='\e[35m'
+NC='\e[0m'
+BGREEN='\e[32m'
+
+declare -a mandatory
+mandatory=(
+  IMAGE
+  ARCH
+)
+
+missing=false
+for var in "${mandatory[@]}"; do
+  if [[ -z "${!var:-}" ]]; then
+    echo "${RED}Environment variable $var must be set${NC}"
+    missing=true
+  fi
+done
+
+if [ "$missing" = true ]; then
+  exit 1
+fi
+
+function cleanup {
+    echo -e "${BGREEN}Stoping kubectl proxy${NC}"
+    rm -rf "${SSL_VOLUME}"
+    kill $proxy_pid
+}
+trap cleanup EXIT
+
+KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
+
+# the ingress controller needs this two variables. To avoid the 
+# creation of any object in the cluster we use invalid names.
+POD_NAMESPACE="invalid-namespace"
+POD_NAME="invalid-namespace"
+
+export TAG=local
+
+if [[ "${ARCH}" != "amd64" ]]; then
+  echo -e "${BGREEN}Register ${RED}/usr/bin/qemu-ARCH-static${BGREEN} as the handler for binaries in multiple platforms${NC}"
+  make -C "${KUBE_ROOT}" register-qemu
+fi
+
+echo -e "${BGREEN}Building ingress controller image${NC}"
+make -C "${KUBE_ROOT}" build "sub-container-${ARCH}"
+
+CONTEXT=$(kubectl config current-context)
+
+echo -e "Running against kubectl cluster ${BGREEN}${CONTEXT}${NC}"
+
+kubectl proxy --accept-hosts=.* --address=0.0.0.0 &
+proxy_pid=$!
+sleep 1
+
+echo -e "\n${BGREEN}kubectl proxy PID: ${BGREEN}$proxy_pid${NC}"
+
+until $(curl --output /dev/null -fsSL http://localhost:8001/); do
+  echo -e "${RED}waiting for kubectl proxy${NC}"
+  sleep 5
+done
+
+# temporal directory for the fake SSL certificate
+SSL_VOLUME=$(mktemp -d)
+
+# if we run as user we cannot bind to port 80 and 443
+docker run \
+  --rm \
+  --name local-ingress-controller \
+  --net=host \
+  --user="root:root" \
+  -e POD_NAMESPACE="${POD_NAMESPACE}" \
+  -e POD_NAME="${POD_NAME}" \
+  -v "${SSL_VOLUME}:/etc/ingress-controller/ssl/" \
+  -v "${HOME}/.kube:${HOME}/.kube:ro" \
+  "${IMAGE}-${ARCH}:local" /nginx-ingress-controller \
+  --update-status=false \
+  --v=2 \
+  --apiserver-host=http://0.0.0.0:8001 \
+  --kubeconfig="${HOME}/.kube/config"

images/nginx/Makefile

@@ -13,12 +13,12 @@
 # limitations under the License.
 
 # 0.0.0 shouldn't clobber any released builds
-TAG ?= 0.89
+TAG ?= 0.90
 REGISTRY ?= quay.io/kubernetes-ingress-controller
 ARCH ?= $(shell go env GOARCH)
 DOCKER ?= docker
 
-ALL_ARCH = amd64 arm64
+ALL_ARCH = amd64 arm arm64
 SED_I?=sed -i
 GOHOSTOS ?= $(shell go env GOHOSTOS)
 
@@ -26,7 +26,7 @@ ifeq ($(GOHOSTOS),darwin)
   SED_I=sed -i ''
 endif
 
-QEMUVERSION=v3.0.0
+QEMUVERSION=v4.0.0-2
 
 IMGNAME = nginx
 IMAGE = $(REGISTRY)/$(IMGNAME)
@@ -35,6 +35,9 @@ MULTI_ARCH_IMG = $(IMAGE)-$(ARCH)
 # Set default base image dynamically for each arch
 BASEIMAGE?=quay.io/kubernetes-ingress-controller/debian-base-$(ARCH):0.1
 
+ifeq ($(ARCH),arm)
+	QEMUARCH=arm
+endif
 ifeq ($(ARCH),arm64)
 	QEMUARCH=aarch64
 endif

images/nginx/rootfs/build.sh

@@ -19,6 +19,8 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
+export DEBIAN_FRONTEND=noninteractive
+
 export OPENRESTY_VERSION=1.15.8.1
 export NGINX_DIGEST_AUTH=cd8641886c873cf543255aeda20d23e4cd603d05
 export NGINX_SUBSTITUTIONS=bc58cb11844bc42735bbaef7085ea86ace46d05b
@@ -56,7 +58,6 @@ get_src()
   rm -rf "$f"
 }
 
-
 apt-get update && apt-get dist-upgrade -y
 
 # install required packages to build
@@ -92,8 +93,16 @@ clean-install \
   bc \
   unzip \
   nano \
+  ssdeep \
   || exit 1
 
+# https://www.mail-archive.com/debian-bugs-dist@lists.debian.org/msg1667178.html
+if [[ ${ARCH} == "armv7l" ]]; then
+  echo "Fixing ca-certificates"
+  touch /etc/ssl/certs/ca-certificates.crt
+  c_rehash
+fi
+
 mkdir -p /etc/nginx
 
 # Get the GeoIP data
@@ -173,7 +182,10 @@ export HUNTER_JOBS_NUMBER=${CORES}
 export HUNTER_KEEP_PACKAGE_SOURCES=false
 export HUNTER_USE_CACHE_SERVERS=true
 
-# Installing luarocks packages
+if [[ ${ARCH} == "armv7l" ]]; then
+  export PCRE_DIR=/usr/lib/arm-linux-gnueabihf
+fi
+
 if [[ ${ARCH} == "x86_64" ]]; then
   export PCRE_DIR=/usr/lib/x86_64-linux-gnu
 fi
@@ -216,11 +228,12 @@ cmake   -DCMAKE_BUILD_TYPE=Release \
 make
 make install
 
-# build jaeger lib
+if [[ ${ARCH} != "armv7l" ]]; then
-cd "$BUILD_PATH/jaeger-client-cpp-$JAEGER_VERSION"
+  # build jaeger lib
-sed -i 's/-Werror/-Wno-psabi/' CMakeLists.txt
+  cd "$BUILD_PATH/jaeger-client-cpp-$JAEGER_VERSION"
+  sed -i 's/-Werror/-Wno-psabi/' CMakeLists.txt
 
-cat <<EOF > export.map
+  cat <<EOF > export.map
 {
     global:
         OpenTracingMakeTracerFactory;
@@ -228,24 +241,25 @@ cat <<EOF > export.map
 };
 EOF
 
-mkdir .build
+  mkdir .build
-cd .build
+  cd .build
 
-cmake -DCMAKE_BUILD_TYPE=Release \
+  cmake -DCMAKE_BUILD_TYPE=Release \
-      -DBUILD_TESTING=OFF \
+        -DBUILD_TESTING=OFF \
-      -DJAEGERTRACING_BUILD_EXAMPLES=OFF \
+        -DJAEGERTRACING_BUILD_EXAMPLES=OFF \
-      -DJAEGERTRACING_BUILD_CROSSDOCK=OFF \
+        -DJAEGERTRACING_BUILD_CROSSDOCK=OFF \
-      -DJAEGERTRACING_COVERAGE=OFF \
+        -DJAEGERTRACING_COVERAGE=OFF \
-      -DJAEGERTRACING_PLUGIN=ON \
+        -DJAEGERTRACING_PLUGIN=ON \
-      -DHUNTER_CONFIGURATION_TYPES=Release \
+        -DHUNTER_CONFIGURATION_TYPES=Release \
-      -DJAEGERTRACING_WITH_YAML_CPP=ON ..
+        -DJAEGERTRACING_WITH_YAML_CPP=ON ..
 
-make
+  make
-make install
+  make install
 
-export HUNTER_INSTALL_DIR=$(cat _3rdParty/Hunter/install-root-dir) \
+  export HUNTER_INSTALL_DIR=$(cat _3rdParty/Hunter/install-root-dir) \
 
-mv libjaegertracing_plugin.so /usr/local/lib/libjaegertracing_plugin.so
+  mv libjaegertracing_plugin.so /usr/local/lib/libjaegertracing_plugin.so
+fi
 
 # build zipkin lib
 cd "$BUILD_PATH/zipkin-cpp-opentracing-$ZIPKIN_CPP_VERSION"
@@ -385,10 +399,6 @@ WITH_FLAGS="--with-debug \
   --with-sha1-asm \
   -j${CORES} "
 
-if [[ ${ARCH} != "aarch64" ]]; then
-  WITH_FLAGS+=" --with-file-aio"
-fi
-
 # "Combining -flto with -g is currently experimental and expected to produce unexpected results."
 # https://gcc.gnu.org/onlinedocs/gcc/Optimize-Options.html
 CC_OPT="-g -Og -fPIE -fstack-protector-strong \
@@ -400,10 +410,18 @@ CC_OPT="-g -Og -fPIE -fstack-protector-strong \
   --param=ssp-buffer-size=4 \
   -DTCP_FASTOPEN=23 \
   -fPIC \
-  -I$HUNTER_INSTALL_DIR/include \
   -Wno-cast-function-type"
 
-LD_OPT="-fPIE -fPIC -pie -Wl,-z,relro -Wl,-z,now -L$HUNTER_INSTALL_DIR/lib"
+LD_OPT="-fPIE -fPIC -pie -Wl,-z,relro -Wl,-z,now"
+
+if [[ ${ARCH} != "armv7l" ]]; then
+  CC_OPT+=" -I$HUNTER_INSTALL_DIR/include"
+  LD_OPT+=" -L$HUNTER_INSTALL_DIR/lib"
+fi
+
+if [[ ${ARCH} != "aarch64" ]]; then
+  WITH_FLAGS+=" --with-file-aio"
+fi
 
 if [[ ${ARCH} == "x86_64" ]]; then
   CC_OPT+=' -m64 -mtune=native'
@@ -450,7 +468,7 @@ cd /usr/local/openresty
 
 # build and install lua-resty-waf with dependencies
 export LUA_LIB_DIR=/usr/local/openresty/lualib
-export LUA_INCLUDE_DIR=/tmp/build/openresty-1.15.8.1/build/luajit-root/usr/local/openresty/luajit/include/luajit-2.1
+export LUA_INCLUDE_DIR=/tmp/build/openresty-$OPENRESTY_VERSION/build/luajit-root/usr/local/openresty/luajit/include/luajit-2.1
 
 luarocks install lrexlib-pcre 2.7.2-1 PCRE_LIBDIR=${PCRE_DIR}
 luarocks install lua-resty-iputils 0.3.0-1
@@ -463,7 +481,9 @@ make install
 
 ln -s $LUA_INCLUDE_DIR /usr/include/lua5.1
 
-/install_lua_resty_waf.sh
+if [[ ${ARCH} != "armv7l" ]]; then
+  /install_lua_resty_waf.sh
+fi
 
 # build Lua bridge tracer
 cd "$BUILD_PATH/lua-bridge-tracer-$LUA_BRIDGE_TRACER_VERSION"

internal/ingress/controller/template/template.go

@@ -28,6 +28,7 @@ import (
 	"os/exec"
 	"reflect"
 	"regexp"
+	"runtime"
 	"sort"
 	"strings"
 	text_template "text/template"
@@ -91,6 +92,11 @@ func (t *Template) Write(conf config.TemplateConfig) ([]byte, error) {
 	outCmdBuf := t.bp.Get()
 	defer t.bp.Put(outCmdBuf)
 
+	// TODO: remove once we found a fix for coredump running luarocks install lrexlib
+	if runtime.GOARCH == "arm" {
+		conf.Cfg.DisableLuaRestyWAF = true
+	}
+
 	if klog.V(3) {
 		b, err := json.Marshal(conf)
 		if err != nil {
@@ -914,7 +920,11 @@ func buildOpentracing(input interface{}) string {
 	if cfg.ZipkinCollectorHost != "" {
 		buf.WriteString("opentracing_load_tracer /usr/local/lib/libzipkin_opentracing.so /etc/nginx/opentracing.json;")
 	} else if cfg.JaegerCollectorHost != "" {
-		buf.WriteString("opentracing_load_tracer /usr/local/lib/libjaegertracing_plugin.so /etc/nginx/opentracing.json;")
+		if runtime.GOARCH == "arm" {
+			buf.WriteString("# Jaeger tracer is not available for ARM https://github.com/jaegertracing/jaeger-client-cpp/issues/151")
+		} else {
+			buf.WriteString("opentracing_load_tracer /usr/local/lib/libjaegertracing_plugin.so /etc/nginx/opentracing.json;")
+		}
 	} else if cfg.DatadogCollectorHost != "" {
 		buf.WriteString("opentracing_load_tracer /usr/local/lib/libdd_opentracing.so /etc/nginx/opentracing.json;")
 	}

rootfs/etc/nginx/template/nginx.tmpl

@@ -57,8 +57,10 @@ http {
         require("resty.core")
         collectgarbage("collect")
 
+        {{ if not $all.Cfg.DisableLuaRestyWAF }}
         local lua_resty_waf = require("resty.waf")
         lua_resty_waf.init()
+        {{ end }}
 
         -- init modules
         local ok, res