diff --git a/Makefile b/Makefile index b1a422a07..45b1577d2 100644 --- a/Makefile +++ b/Makefile @@ -46,7 +46,7 @@ DUMB_ARCH = ${ARCH} GOBUILD_FLAGS := -v -ALL_ARCH = amd64 arm64 +ALL_ARCH = amd64 arm arm64 QEMUVERSION = v4.0.0 @@ -54,6 +54,11 @@ BUSTED_ARGS =-v --pattern=_test GOOS = linux +IMGNAME = nginx-ingress-controller +IMAGE = $(REGISTRY)/$(IMGNAME) + +MULTI_ARCH_IMG = ${IMAGE}-${ARCH} + export ARCH export DUMB_ARCH export TAG @@ -64,14 +69,14 @@ export GIT_COMMIT export GOBUILD_FLAGS export REPO_INFO export BUSTED_ARGS - -IMGNAME = nginx-ingress-controller -IMAGE = $(REGISTRY)/$(IMGNAME) -MULTI_ARCH_IMG = $(IMAGE)-$(ARCH) +export IMAGE # Set default base image dynamically for each arch BASEIMAGE?=quay.io/kubernetes-ingress-controller/nginx-$(ARCH):0.90 +ifeq ($(ARCH),arm) + QEMUARCH=arm +endif ifeq ($(ARCH),arm64) QEMUARCH=aarch64 endif @@ -131,7 +136,7 @@ clean-container: .PHONY: register-qemu register-qemu: # Register /usr/bin/qemu-ARCH-static as the handler for binaries in multiple platforms - @$(DOCKER) run --rm --privileged multiarch/qemu-user-static:register --reset + @$(DOCKER) run --rm --privileged multiarch/qemu-user-static:register --reset >&2 .PHONY: push push: .push-$(ARCH) @@ -249,6 +254,10 @@ misspell: -error \ cmd/* internal/* deploy/* docs/* design/* test/* README.md -.PHONE: kind-e2e-test +.PHONY: kind-e2e-test kind-e2e-test: test/e2e/run.sh + +.PHONY: run-ingress-controller +run-ingress-controller: + @build/run-ingress-controller.sh diff --git a/build/run-ingress-controller.sh b/build/run-ingress-controller.sh new file mode 100755 index 000000000..6e7aba01b --- /dev/null +++ b/build/run-ingress-controller.sh @@ -0,0 +1,103 @@ +#!/bin/bash + +# Copyright 2018 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if ! [ -z "$DEBUG" ]; then + set -x +fi + +set -o errexit +set -o nounset +set -o pipefail + +RED='\e[35m' +NC='\e[0m' +BGREEN='\e[32m' + +declare -a mandatory +mandatory=( + IMAGE + ARCH +) + +missing=false +for var in "${mandatory[@]}"; do + if [[ -z "${!var:-}" ]]; then + echo "${RED}Environment variable $var must be set${NC}" + missing=true + fi +done + +if [ "$missing" = true ]; then + exit 1 +fi + +function cleanup { + echo -e "${BGREEN}Stoping kubectl proxy${NC}" + rm -rf "${SSL_VOLUME}" + kill $proxy_pid +} +trap cleanup EXIT + +KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. + +# the ingress controller needs this two variables. To avoid the +# creation of any object in the cluster we use invalid names. +POD_NAMESPACE="invalid-namespace" +POD_NAME="invalid-namespace" + +export TAG=local + +if [[ "${ARCH}" != "amd64" ]]; then + echo -e "${BGREEN}Register ${RED}/usr/bin/qemu-ARCH-static${BGREEN} as the handler for binaries in multiple platforms${NC}" + make -C "${KUBE_ROOT}" register-qemu +fi + +echo -e "${BGREEN}Building ingress controller image${NC}" +make -C "${KUBE_ROOT}" build "sub-container-${ARCH}" + +CONTEXT=$(kubectl config current-context) + +echo -e "Running against kubectl cluster ${BGREEN}${CONTEXT}${NC}" + +kubectl proxy --accept-hosts=.* --address=0.0.0.0 & +proxy_pid=$! +sleep 1 + +echo -e "\n${BGREEN}kubectl proxy PID: ${BGREEN}$proxy_pid${NC}" + +until $(curl --output /dev/null -fsSL http://localhost:8001/); do + echo -e "${RED}waiting for kubectl proxy${NC}" + sleep 5 +done + +# temporal directory for the fake SSL certificate +SSL_VOLUME=$(mktemp -d) + +# if we run as user we cannot bind to port 80 and 443 +docker run \ + --rm \ + --name local-ingress-controller \ + --net=host \ + --user="root:root" \ + -e POD_NAMESPACE="${POD_NAMESPACE}" \ + -e POD_NAME="${POD_NAME}" \ + -v "${SSL_VOLUME}:/etc/ingress-controller/ssl/" \ + -v "${HOME}/.kube:${HOME}/.kube:ro" \ + "${IMAGE}-${ARCH}:local" /nginx-ingress-controller \ + --update-status=false \ + --v=2 \ + --apiserver-host=http://0.0.0.0:8001 \ + --kubeconfig="${HOME}/.kube/config" diff --git a/images/nginx/Makefile b/images/nginx/Makefile index 1e3aed385..500f0d782 100644 --- a/images/nginx/Makefile +++ b/images/nginx/Makefile @@ -13,12 +13,12 @@ # limitations under the License. # 0.0.0 shouldn't clobber any released builds -TAG ?= 0.89 +TAG ?= 0.90 REGISTRY ?= quay.io/kubernetes-ingress-controller ARCH ?= $(shell go env GOARCH) DOCKER ?= docker -ALL_ARCH = amd64 arm64 +ALL_ARCH = amd64 arm arm64 SED_I?=sed -i GOHOSTOS ?= $(shell go env GOHOSTOS) @@ -26,7 +26,7 @@ ifeq ($(GOHOSTOS),darwin) SED_I=sed -i '' endif -QEMUVERSION=v3.0.0 +QEMUVERSION=v4.0.0-2 IMGNAME = nginx IMAGE = $(REGISTRY)/$(IMGNAME) @@ -35,6 +35,9 @@ MULTI_ARCH_IMG = $(IMAGE)-$(ARCH) # Set default base image dynamically for each arch BASEIMAGE?=quay.io/kubernetes-ingress-controller/debian-base-$(ARCH):0.1 +ifeq ($(ARCH),arm) + QEMUARCH=arm +endif ifeq ($(ARCH),arm64) QEMUARCH=aarch64 endif diff --git a/images/nginx/rootfs/build.sh b/images/nginx/rootfs/build.sh index d62632789..cbda0c18f 100755 --- a/images/nginx/rootfs/build.sh +++ b/images/nginx/rootfs/build.sh @@ -19,6 +19,8 @@ set -o errexit set -o nounset set -o pipefail +export DEBIAN_FRONTEND=noninteractive + export OPENRESTY_VERSION=1.15.8.1 export NGINX_DIGEST_AUTH=cd8641886c873cf543255aeda20d23e4cd603d05 export NGINX_SUBSTITUTIONS=bc58cb11844bc42735bbaef7085ea86ace46d05b @@ -56,7 +58,6 @@ get_src() rm -rf "$f" } - apt-get update && apt-get dist-upgrade -y # install required packages to build @@ -92,8 +93,16 @@ clean-install \ bc \ unzip \ nano \ + ssdeep \ || exit 1 +# https://www.mail-archive.com/debian-bugs-dist@lists.debian.org/msg1667178.html +if [[ ${ARCH} == "armv7l" ]]; then + echo "Fixing ca-certificates" + touch /etc/ssl/certs/ca-certificates.crt + c_rehash +fi + mkdir -p /etc/nginx # Get the GeoIP data @@ -173,7 +182,10 @@ export HUNTER_JOBS_NUMBER=${CORES} export HUNTER_KEEP_PACKAGE_SOURCES=false export HUNTER_USE_CACHE_SERVERS=true -# Installing luarocks packages +if [[ ${ARCH} == "armv7l" ]]; then + export PCRE_DIR=/usr/lib/arm-linux-gnueabihf +fi + if [[ ${ARCH} == "x86_64" ]]; then export PCRE_DIR=/usr/lib/x86_64-linux-gnu fi @@ -216,11 +228,12 @@ cmake -DCMAKE_BUILD_TYPE=Release \ make make install -# build jaeger lib -cd "$BUILD_PATH/jaeger-client-cpp-$JAEGER_VERSION" -sed -i 's/-Werror/-Wno-psabi/' CMakeLists.txt +if [[ ${ARCH} != "armv7l" ]]; then + # build jaeger lib + cd "$BUILD_PATH/jaeger-client-cpp-$JAEGER_VERSION" + sed -i 's/-Werror/-Wno-psabi/' CMakeLists.txt -cat < export.map + cat < export.map { global: OpenTracingMakeTracerFactory; @@ -228,24 +241,25 @@ cat < export.map }; EOF -mkdir .build -cd .build + mkdir .build + cd .build -cmake -DCMAKE_BUILD_TYPE=Release \ - -DBUILD_TESTING=OFF \ - -DJAEGERTRACING_BUILD_EXAMPLES=OFF \ - -DJAEGERTRACING_BUILD_CROSSDOCK=OFF \ - -DJAEGERTRACING_COVERAGE=OFF \ - -DJAEGERTRACING_PLUGIN=ON \ - -DHUNTER_CONFIGURATION_TYPES=Release \ - -DJAEGERTRACING_WITH_YAML_CPP=ON .. + cmake -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_TESTING=OFF \ + -DJAEGERTRACING_BUILD_EXAMPLES=OFF \ + -DJAEGERTRACING_BUILD_CROSSDOCK=OFF \ + -DJAEGERTRACING_COVERAGE=OFF \ + -DJAEGERTRACING_PLUGIN=ON \ + -DHUNTER_CONFIGURATION_TYPES=Release \ + -DJAEGERTRACING_WITH_YAML_CPP=ON .. -make -make install + make + make install -export HUNTER_INSTALL_DIR=$(cat _3rdParty/Hunter/install-root-dir) \ + export HUNTER_INSTALL_DIR=$(cat _3rdParty/Hunter/install-root-dir) \ -mv libjaegertracing_plugin.so /usr/local/lib/libjaegertracing_plugin.so + mv libjaegertracing_plugin.so /usr/local/lib/libjaegertracing_plugin.so +fi # build zipkin lib cd "$BUILD_PATH/zipkin-cpp-opentracing-$ZIPKIN_CPP_VERSION" @@ -385,10 +399,6 @@ WITH_FLAGS="--with-debug \ --with-sha1-asm \ -j${CORES} " -if [[ ${ARCH} != "aarch64" ]]; then - WITH_FLAGS+=" --with-file-aio" -fi - # "Combining -flto with -g is currently experimental and expected to produce unexpected results." # https://gcc.gnu.org/onlinedocs/gcc/Optimize-Options.html CC_OPT="-g -Og -fPIE -fstack-protector-strong \ @@ -400,10 +410,18 @@ CC_OPT="-g -Og -fPIE -fstack-protector-strong \ --param=ssp-buffer-size=4 \ -DTCP_FASTOPEN=23 \ -fPIC \ - -I$HUNTER_INSTALL_DIR/include \ -Wno-cast-function-type" -LD_OPT="-fPIE -fPIC -pie -Wl,-z,relro -Wl,-z,now -L$HUNTER_INSTALL_DIR/lib" +LD_OPT="-fPIE -fPIC -pie -Wl,-z,relro -Wl,-z,now" + +if [[ ${ARCH} != "armv7l" ]]; then + CC_OPT+=" -I$HUNTER_INSTALL_DIR/include" + LD_OPT+=" -L$HUNTER_INSTALL_DIR/lib" +fi + +if [[ ${ARCH} != "aarch64" ]]; then + WITH_FLAGS+=" --with-file-aio" +fi if [[ ${ARCH} == "x86_64" ]]; then CC_OPT+=' -m64 -mtune=native' @@ -450,7 +468,7 @@ cd /usr/local/openresty # build and install lua-resty-waf with dependencies export LUA_LIB_DIR=/usr/local/openresty/lualib -export LUA_INCLUDE_DIR=/tmp/build/openresty-1.15.8.1/build/luajit-root/usr/local/openresty/luajit/include/luajit-2.1 +export LUA_INCLUDE_DIR=/tmp/build/openresty-$OPENRESTY_VERSION/build/luajit-root/usr/local/openresty/luajit/include/luajit-2.1 luarocks install lrexlib-pcre 2.7.2-1 PCRE_LIBDIR=${PCRE_DIR} luarocks install lua-resty-iputils 0.3.0-1 @@ -463,7 +481,9 @@ make install ln -s $LUA_INCLUDE_DIR /usr/include/lua5.1 -/install_lua_resty_waf.sh +if [[ ${ARCH} != "armv7l" ]]; then + /install_lua_resty_waf.sh +fi # build Lua bridge tracer cd "$BUILD_PATH/lua-bridge-tracer-$LUA_BRIDGE_TRACER_VERSION" diff --git a/internal/ingress/controller/template/template.go b/internal/ingress/controller/template/template.go index 4a1d4b1f3..b5039f53a 100644 --- a/internal/ingress/controller/template/template.go +++ b/internal/ingress/controller/template/template.go @@ -28,6 +28,7 @@ import ( "os/exec" "reflect" "regexp" + "runtime" "sort" "strings" text_template "text/template" @@ -91,6 +92,11 @@ func (t *Template) Write(conf config.TemplateConfig) ([]byte, error) { outCmdBuf := t.bp.Get() defer t.bp.Put(outCmdBuf) + // TODO: remove once we found a fix for coredump running luarocks install lrexlib + if runtime.GOARCH == "arm" { + conf.Cfg.DisableLuaRestyWAF = true + } + if klog.V(3) { b, err := json.Marshal(conf) if err != nil { @@ -914,7 +920,11 @@ func buildOpentracing(input interface{}) string { if cfg.ZipkinCollectorHost != "" { buf.WriteString("opentracing_load_tracer /usr/local/lib/libzipkin_opentracing.so /etc/nginx/opentracing.json;") } else if cfg.JaegerCollectorHost != "" { - buf.WriteString("opentracing_load_tracer /usr/local/lib/libjaegertracing_plugin.so /etc/nginx/opentracing.json;") + if runtime.GOARCH == "arm" { + buf.WriteString("# Jaeger tracer is not available for ARM https://github.com/jaegertracing/jaeger-client-cpp/issues/151") + } else { + buf.WriteString("opentracing_load_tracer /usr/local/lib/libjaegertracing_plugin.so /etc/nginx/opentracing.json;") + } } else if cfg.DatadogCollectorHost != "" { buf.WriteString("opentracing_load_tracer /usr/local/lib/libdd_opentracing.so /etc/nginx/opentracing.json;") } diff --git a/rootfs/etc/nginx/template/nginx.tmpl b/rootfs/etc/nginx/template/nginx.tmpl index a90d0b2f4..6d66bb329 100755 --- a/rootfs/etc/nginx/template/nginx.tmpl +++ b/rootfs/etc/nginx/template/nginx.tmpl @@ -57,8 +57,10 @@ http { require("resty.core") collectgarbage("collect") + {{ if not $all.Cfg.DisableLuaRestyWAF }} local lua_resty_waf = require("resty.waf") lua_resty_waf.init() + {{ end }} -- init modules local ok, res