First sidecar module: OpenTelemetry (#8013)

* remove opentelemetry from main nginx image

* add opentelemetry sidecar image

* handle extra modules in helm chart

* fix running helm chart

* mount the modules volume in the init container

* merge the mounted folder

* fix the otel image

* fix licence year

* fix cloudbuild image

* use the same nginx version as in the main image

* only retrieve /etc/nginx/modules for now

charts/ingress-nginx/ci/daemonset-extra-modules.yaml

@@ -0,0 +1,10 @@
+controller:
+  kind: DaemonSet
+  image:
+    repository: ingress-controller/controller
+    tag: 1.0.0-dev
+  service:
+    type: ClusterIP
+  extraModules:
+    - name: opentelemetry
+      image: busybox

charts/ingress-nginx/ci/deployment-extra-modules.yaml

@@ -0,0 +1,10 @@
+controller:
+  image:
+    repository: ingress-controller/controller
+    tag: 1.0.0-dev
+    digest: null
+  service:
+    type: ClusterIP
+  extraModules:
+    - name: opentelemetry
+      image: busybox

charts/ingress-nginx/templates/controller-daemonset.yaml

@@ -143,8 +143,12 @@ spec:
               hostPort: {{ $key }}
               {{- end }}
           {{- end }}
-        {{- if (or .Values.controller.customTemplate.configMapName .Values.controller.extraVolumeMounts .Values.controller.admissionWebhooks.enabled) }}
+        {{- if (or .Values.controller.customTemplate.configMapName .Values.controller.extraVolumeMounts .Values.controller.admissionWebhooks.enabled .Values.controller.extraModules) }}
           volumeMounts:
+          {{- if .Values.controller.extraModules }}
+            - name: modules
+              mountPath: /modules_mount
+          {{- end }}
           {{- if .Values.controller.customTemplate.configMapName }}
             - mountPath: /etc/nginx/template
               name: nginx-template-volume
@@ -165,8 +169,20 @@ spec:
       {{- if .Values.controller.extraContainers }}
         {{ toYaml .Values.controller.extraContainers | nindent 8 }}
       {{- end }}
-    {{- if .Values.controller.extraInitContainers }}
+
-      initContainers: {{ toYaml .Values.controller.extraInitContainers | nindent 8 }}
+
+    {{- if (or .Values.controller.extraInitContainers .Values.controller.extraModules) }}
+      initContainers:
+      {{- if .Values.controller.extraInitContainers }}
+        {{ toYaml .Values.controller.extraInitContainers | nindent 8 }}
+      {{- end }}
+      {{- if .Values.controller.extraModules }}
+        {{- range .Values.controller.extraModules }}
+          - name: {{ .Name }}
+            image: {{ .Image }}
+            command: ['sh', '-c', '/usr/local/bin/init_module.sh']
+        {{- end }}
+      {{- end }}
     {{- end }}
     {{- if .Values.controller.hostNetwork }}
       hostNetwork: {{ .Values.controller.hostNetwork }}
@@ -185,8 +201,12 @@ spec:
     {{- end }}
       serviceAccountName: {{ template "ingress-nginx.serviceAccountName" . }}
       terminationGracePeriodSeconds: {{ .Values.controller.terminationGracePeriodSeconds }}
-    {{- if (or .Values.controller.customTemplate.configMapName .Values.controller.extraVolumeMounts .Values.controller.admissionWebhooks.enabled .Values.controller.extraVolumes) }}
+    {{- if (or .Values.controller.customTemplate.configMapName .Values.controller.extraVolumeMounts .Values.controller.admissionWebhooks.enabled .Values.controller.extraVolumes .Values.controller.extraModules) }}
       volumes:
+      {{- if .Values.controller.extraModules }}
+        - name: modules
+          emptyDir: {}
+      {{- end }}
       {{- if .Values.controller.customTemplate.configMapName }}
         - name: nginx-template-volume
           configMap:

charts/ingress-nginx/templates/controller-deployment.yaml

@@ -140,8 +140,12 @@ spec:
               hostPort: {{ $key }}
               {{- end }}
           {{- end }}
-        {{- if (or .Values.controller.customTemplate.configMapName .Values.controller.extraVolumeMounts .Values.controller.admissionWebhooks.enabled) }}
+        {{- if (or .Values.controller.customTemplate.configMapName .Values.controller.extraVolumeMounts .Values.controller.admissionWebhooks.enabled .Values.controller.extraModules) }}
           volumeMounts:
+          {{- if .Values.controller.extraModules }}
+            - name: modules
+              mountPath: /modules_mount
+          {{- end }}
           {{- if .Values.controller.customTemplate.configMapName }}
             - mountPath: /etc/nginx/template
               name: nginx-template-volume
@@ -162,8 +166,21 @@ spec:
       {{- if .Values.controller.extraContainers }}
         {{ toYaml .Values.controller.extraContainers | nindent 8 }}
       {{- end }}
-    {{- if .Values.controller.extraInitContainers }}
+    {{- if (or .Values.controller.extraInitContainers .Values.controller.extraModules) }}
-      initContainers: {{ toYaml .Values.controller.extraInitContainers | nindent 8 }}
+      initContainers:
+      {{- if .Values.controller.extraInitContainers }}
+        {{ toYaml .Values.controller.extraInitContainers | nindent 8 }}
+      {{- end }}
+      {{- if .Values.controller.extraModules }}
+        {{- range .Values.controller.extraModules }}
+          - name: {{ .name }}
+            image: {{ .image }}
+            command: ['sh', '-c', '/usr/local/bin/init_module.sh']
+            volumeMounts:
+              - name: modules
+                mountPath: /modules_mount
+        {{- end }}
+      {{- end }}
     {{- end }}
     {{- if .Values.controller.hostNetwork }}
       hostNetwork: {{ .Values.controller.hostNetwork }}
@@ -182,8 +199,12 @@ spec:
     {{- end }}
       serviceAccountName: {{ template "ingress-nginx.serviceAccountName" . }}
       terminationGracePeriodSeconds: {{ .Values.controller.terminationGracePeriodSeconds }}
-    {{- if (or .Values.controller.customTemplate.configMapName .Values.controller.extraVolumeMounts .Values.controller.admissionWebhooks.enabled .Values.controller.extraVolumes) }}
+    {{- if (or .Values.controller.customTemplate.configMapName .Values.controller.extraVolumeMounts .Values.controller.admissionWebhooks.enabled .Values.controller.extraVolumes .Values.controller.extraModules) }}
       volumes:
+      {{- if .Values.controller.extraModules }}
+        - name: modules
+          emptyDir: {}
+      {{- end }}
       {{- if .Values.controller.customTemplate.configMapName }}
         - name: nginx-template-volume
           configMap:

charts/ingress-nginx/values.yaml

@@ -559,6 +559,15 @@ controller:
   #   image: busybox
   #   command: ['sh', '-c', 'until nslookup myservice; do echo waiting for myservice; sleep 2; done;']
 
+  extraModules: []
+  ## Modules, which are mounted into the core nginx image
+  # - name: opentelemetry
+  #   image: busybox
+  #
+  # The image must contain a `/usr/local/bin/init_module.sh` executable, which
+  # will be executed as initContainers, to move its config files within the
+  # mounted volume.
+
   admissionWebhooks:
     annotations: {}
     # ignore-check.kube-linter.io/no-read-only-rootfs: "This deployment needs write access to root filesystem".

images/nginx/rootfs/Dockerfile

@@ -33,6 +33,7 @@ ENV LUA_CPATH="/usr/local/lib/lua/?/?.so;/usr/local/lib/lua/?.so;;"
 COPY --from=builder /usr/local /usr/local
 COPY --from=builder /opt /opt
 COPY --from=builder /etc/nginx /etc/nginx
+COPY --from=builder entrypoint.sh /usr/local/entrypoint.sh
 
 RUN apk update \
   && apk upgrade \
@@ -72,4 +73,5 @@ RUN apk update \
 
 EXPOSE 80 443
 
+ENTRYPOINT ["/usr/local/entrypoint.sh"]
 CMD ["nginx", "-g", "daemon off;"]

images/nginx/rootfs/build.sh

@@ -35,12 +35,6 @@ export NGINX_DIGEST_AUTH=1.0.0
 # Check for recent changes: https://github.com/yaoweibin/ngx_http_substitutions_filter_module/compare/v0.6.4...master
 export NGINX_SUBSTITUTIONS=b8a71eacc7f986ba091282ab8b1bbbc6ae1807e0
 
-# Check for recent changes: https://github.com/open-telemetry/opentelemetry-cpp/compare/v1.0.0...main
-export OPENTELEMETRY_CPP_VERSION=1.0.0
-
-# Check for recent changes: https://github.com/open-telemetry/opentelemetry-cpp-contrib/compare/c655b8...main
-export OPENTELEMETRY_CONTRIB_COMMIT=c655b849f017a5363085a4b4e6fcae8a4b7621ad
-
 # Check for recent changes: https://github.com/opentracing-contrib/nginx-opentracing/compare/v0.19.0...master
 export NGINX_OPENTRACING_VERSION=0.19.0
 
@@ -144,11 +138,6 @@ if [[ ${ARCH} == "s390x" ]]; then
   export LUA_STREAM_NGX_VERSION=0.0.7
 fi
 
-export USE_OPENTELEMETRY=true
-if [[ ${ARCH} == "s390x" ]] || [[ ${ARCH} == "armv7l" ]]; then
-  export USE_OPENTELEMETRY=false
-fi
-
 get_src()
 {
   hash="$1"
@@ -226,9 +215,6 @@ get_src f09851e6309560a8ff3e901548405066c83f1f6ff88aa7171e0763bd9514762b \
 get_src a98b48947359166326d58700ccdc27256d2648218072da138ab6b47de47fbd8f \
         "https://github.com/yaoweibin/ngx_http_substitutions_filter_module/archive/$NGINX_SUBSTITUTIONS.tar.gz"
 
-get_src 37b2a2abf75e865449ff1425cee96dbd74659ac0c612c84ee5f381244360cab2 \
-        "https://github.com/open-telemetry/opentelemetry-cpp-contrib/archive/$OPENTELEMETRY_CONTRIB_COMMIT.tar.gz"
-
 get_src 6f97776ebdf019b105a755c7736b70bdbd7e575c7f0d39db5fe127873c7abf17 \
         "https://github.com/opentracing-contrib/nginx-opentracing/archive/v$NGINX_OPENTRACING_VERSION.tar.gz"
 
@@ -482,32 +468,6 @@ cmake -DCMAKE_BUILD_TYPE=Release \
 make
 make install
 
-if [ $USE_OPENTELEMETRY = true ]; then
-  # build opentelemetry lib
-  apk add protobuf-dev \
-    grpc \
-    grpc-dev \
-    gtest-dev \
-    c-ares-dev
-
-  cd $BUILD_PATH
-  git clone --recursive https://github.com/open-telemetry/opentelemetry-cpp opentelemetry-cpp-$OPENTELEMETRY_CPP_VERSION
-  cd "opentelemetry-cpp-$OPENTELEMETRY_CPP_VERSION"
-  git checkout v$OPENTELEMETRY_CPP_VERSION
-  mkdir .build
-  cd .build
-
-  cmake -DCMAKE_BUILD_TYPE=Release \
-        -DBUILD_TESTING=OFF \
-        -DWITH_EXAMPLES=OFF \
-        -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
-        -DWITH_OTLP=ON \
-        -DWITH_OTLP_HTTP=OFF \
-        ..
-  make
-  make install
-fi
-
 # Get Brotli source and deps
 cd "$BUILD_PATH"
 git clone --depth=1 https://github.com/google/ngx_brotli.git
@@ -675,11 +635,6 @@ WITH_MODULES=" \
   --add-dynamic-module=$BUILD_PATH/ngx_http_geoip2_module-${GEOIP2_VERSION} \
   --add-dynamic-module=$BUILD_PATH/ngx_brotli"
 
-if [ $USE_OPENTELEMETRY = true ]; then
-  WITH_MODULES+=" \
-    --add-dynamic-module=$BUILD_PATH/opentelemetry-cpp-contrib-$OPENTELEMETRY_CONTRIB_COMMIT/instrumentation/nginx"
-fi
-
 ./configure \
   --prefix=/usr/local/nginx \
   --conf-path=/etc/nginx/nginx.conf \

images/nginx/rootfs/entrypoint.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+# Copyright 2022 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+if [ -d "/modules_mount/etc/nginx/modules" ]; then
+	for dir in /modules_mount/etc/nginx/modules/*; do
+		cp "$dir"/* "/etc/nginx/modules/$(basename "$dir")"
+	done
+fi
+
+exec "$@"

images/opentelemetry/Makefile

@@ -0,0 +1,57 @@
+# Copyright 2021 The Kubernetes Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+.DEFAULT_GOAL:=build
+
+# set default shell
+SHELL=/bin/bash -o pipefail -o errexit
+
+DIR:=$(strip $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))))
+INIT_BUILDX=$(DIR)/../../hack/init-buildx.sh
+
+# 0.0.0 shouldn't clobber any released builds
+TAG ?= 0.0
+REGISTRY ?= gcr.io/k8s-staging-ingress-nginx
+
+IMAGE = $(REGISTRY)/opentelemetry
+
+# required to enable buildx
+export DOCKER_CLI_EXPERIMENTAL=enabled
+
+# build with buildx
+PLATFORMS?=linux/amd64,linux/arm
+OUTPUT=
+PROGRESS=plain
+build: ensure-buildx
+	docker buildx build \
+		--platform=${PLATFORMS} $(OUTPUT) \
+		--progress=$(PROGRESS) \
+		--pull \
+		--tag $(IMAGE):$(TAG) rootfs
+
+# push the cross built image
+push: OUTPUT=--push
+push: build
+
+# enable buildx
+ensure-buildx:
+# this is required for cloudbuild
+ifeq ("$(wildcard $(INIT_BUILDX))","")
+	@curl -sSL https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/hack/init-buildx.sh | bash
+else
+	@exec $(INIT_BUILDX)
+endif
+	@echo "done"
+
+.PHONY: build push ensure-buildx

images/opentelemetry/README.md

@@ -0,0 +1,5 @@
+# OpenTelemetry library builder
+
+**How to use this image:**
+This image only contains the necessary files in /usr/local and /etc/nginx/opentelemetry to 
+be copied to Ingress Controller deployment when OpenTelemetry is enabled

images/opentelemetry/cloudbuild.yaml

@@ -0,0 +1,24 @@
+timeout: 10800s
+options:
+  substitution_option: ALLOW_LOOSE
+  # job builds a multi-arch docker image for amd64,arm,arm64 and s390x.
+  machineType: N1_HIGHCPU_32
+steps:
+  - name: gcr.io/k8s-staging-test-infra/gcb-docker-gcloud:v20211118-2f2d816b90
+    entrypoint: bash
+    env:
+      - DOCKER_CLI_EXPERIMENTAL=enabled
+      - TAG=$_GIT_TAG
+      - BASE_REF=$_PULL_BASE_REF
+      - REGISTRY=gcr.io/k8s-staging-ingress-nginx
+      # default cloudbuild has HOME=/builder/home and docker buildx is in /root/.docker/cli-plugins/docker-buildx
+      # set the home to /root explicitly to if using docker buildx
+      - HOME=/root
+    args:
+    - -c
+    - |
+      gcloud auth configure-docker \
+      && make push
+substitutions:
+  _GIT_TAG: "12345"
+  _PULL_BASE_REF: "master"

images/opentelemetry/rootfs/Dockerfile

@@ -0,0 +1,28 @@
+# Copyright 2021 The Kubernetes Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+FROM alpine:3.14.2 as builder
+
+COPY . /
+
+RUN apk update \
+	&& apk upgrade \
+	&& apk add -U bash \
+	&& /build.sh
+
+FROM busybox:latest 
+
+COPY --from=builder init_module.sh /usr/local/bin/init_module.sh
+COPY --from=builder /etc/nginx/modules /etc/nginx/modules

images/opentelemetry/rootfs/build.sh

@@ -0,0 +1,111 @@
+#!/bin/bash
+
+# Copyright 2021 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+export NGINX_VERSION=1.19.9
+
+# Check for recent changes: https://github.com/open-telemetry/opentelemetry-cpp/compare/v1.0.0...main
+export OPENTELEMETRY_CPP_VERSION=1.0.0
+
+# Check for recent changes: https://github.com/open-telemetry/opentelemetry-cpp-contrib/compare/f4850...main
+export OPENTELEMETRY_CONTRIB_COMMIT=f48500884b1b32efc456790bbcdc2e6cf7a8e630
+
+export BUILD_PATH=/tmp/build
+
+rm -rf \
+   /var/cache/debconf/* \
+   /var/lib/apt/lists/* \
+   /var/log/* \
+   /tmp/* \
+   /var/tmp/*
+
+
+mkdir -p /etc/nginx
+mkdir --verbose -p "$BUILD_PATH"
+cd "$BUILD_PATH"
+
+apk add \
+  curl \
+  git \
+  build-base
+
+get_src()
+{
+  hash="$1"
+  url="$2"
+  f=$(basename "$url")
+
+  echo "Downloading $url"
+
+  curl -sSL "$url" -o "$f"
+  echo "$hash  $f" | sha256sum -c - || exit 10
+  tar xzf "$f"
+  rm -rf "$f"
+}
+
+
+get_src e462e11533d5c30baa05df7652160ff5979591d291736cfa5edb9fd2edb48c49 \
+        "https://nginx.org/download/nginx-$NGINX_VERSION.tar.gz"
+
+get_src 45c52498788e47131b20a4786dbb08f4390b8cb419bd3d61c88b503cafff3324 \
+        "https://github.com/open-telemetry/opentelemetry-cpp-contrib/archive/$OPENTELEMETRY_CONTRIB_COMMIT.tar.gz"
+
+# improve compilation times
+CORES=$(($(grep -c ^processor /proc/cpuinfo) - 1))
+
+export MAKEFLAGS=-j${CORES}
+
+apk add \
+  protobuf-dev \
+  grpc \
+  grpc-dev \
+  gtest-dev \
+  c-ares-dev \
+  pcre-dev
+
+cd $BUILD_PATH
+git clone --recursive https://github.com/open-telemetry/opentelemetry-cpp opentelemetry-cpp-$OPENTELEMETRY_CPP_VERSION
+cd "opentelemetry-cpp-$OPENTELEMETRY_CPP_VERSION"
+git checkout v$OPENTELEMETRY_CPP_VERSION
+mkdir .build
+cd .build
+
+cmake -DCMAKE_BUILD_TYPE=Release \
+      -DBUILD_TESTING=OFF \
+      -DWITH_EXAMPLES=OFF \
+      -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
+      -DWITH_OTLP=ON \
+      -DWITH_OTLP_HTTP=OFF \
+      ..
+make
+make install
+
+# build nginx
+cd "$BUILD_PATH/nginx-$NGINX_VERSION"
+./configure \
+  --prefix=/usr/local/nginx \
+  --with-compat \
+  --add-dynamic-module=$BUILD_PATH/opentelemetry-cpp-contrib-$OPENTELEMETRY_CONTRIB_COMMIT/instrumentation/nginx
+
+make modules
+mkdir -p /etc/nginx/modules
+cp objs/otel_ngx_module.so /etc/nginx/modules/otel_ngx_module.so
+
+# remove .a files
+find /usr/local -name "*.a" -print | xargs /bin/rm

images/opentelemetry/rootfs/init_module.sh

@@ -0,0 +1,21 @@
+#!/bin/sh
+
+# Copyright 2021 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+cp -R /etc/nginx/modules /modules_mount/etc/nginx/modules