First sidecar module: OpenTelemetry (#8013)

* remove opentelemetry from main nginx image

* add opentelemetry sidecar image

* handle extra modules in helm chart

* fix running helm chart

* mount the modules volume in the init container

* merge the mounted folder

* fix the otel image

* fix licence year

* fix cloudbuild image

* use the same nginx version as in the main image

* only retrieve /etc/nginx/modules for now
This commit is contained in:
Damien Mathieu 2022-01-16 22:33:28 +01:00 committed by GitHub
parent c0098f305c
commit 15b0aba03b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 353 additions and 53 deletions

View file

@ -0,0 +1,10 @@
controller:
kind: DaemonSet
image:
repository: ingress-controller/controller
tag: 1.0.0-dev
service:
type: ClusterIP
extraModules:
- name: opentelemetry
image: busybox

View file

@ -0,0 +1,10 @@
controller:
image:
repository: ingress-controller/controller
tag: 1.0.0-dev
digest: null
service:
type: ClusterIP
extraModules:
- name: opentelemetry
image: busybox

View file

@ -143,8 +143,12 @@ spec:
hostPort: {{ $key }}
{{- end }}
{{- end }}
{{- if (or .Values.controller.customTemplate.configMapName .Values.controller.extraVolumeMounts .Values.controller.admissionWebhooks.enabled) }}
{{- if (or .Values.controller.customTemplate.configMapName .Values.controller.extraVolumeMounts .Values.controller.admissionWebhooks.enabled .Values.controller.extraModules) }}
volumeMounts:
{{- if .Values.controller.extraModules }}
- name: modules
mountPath: /modules_mount
{{- end }}
{{- if .Values.controller.customTemplate.configMapName }}
- mountPath: /etc/nginx/template
name: nginx-template-volume
@ -165,8 +169,20 @@ spec:
{{- if .Values.controller.extraContainers }}
{{ toYaml .Values.controller.extraContainers | nindent 8 }}
{{- end }}
{{- if .Values.controller.extraInitContainers }}
initContainers: {{ toYaml .Values.controller.extraInitContainers | nindent 8 }}
{{- if (or .Values.controller.extraInitContainers .Values.controller.extraModules) }}
initContainers:
{{- if .Values.controller.extraInitContainers }}
{{ toYaml .Values.controller.extraInitContainers | nindent 8 }}
{{- end }}
{{- if .Values.controller.extraModules }}
{{- range .Values.controller.extraModules }}
- name: {{ .Name }}
image: {{ .Image }}
command: ['sh', '-c', '/usr/local/bin/init_module.sh']
{{- end }}
{{- end }}
{{- end }}
{{- if .Values.controller.hostNetwork }}
hostNetwork: {{ .Values.controller.hostNetwork }}
@ -185,8 +201,12 @@ spec:
{{- end }}
serviceAccountName: {{ template "ingress-nginx.serviceAccountName" . }}
terminationGracePeriodSeconds: {{ .Values.controller.terminationGracePeriodSeconds }}
{{- if (or .Values.controller.customTemplate.configMapName .Values.controller.extraVolumeMounts .Values.controller.admissionWebhooks.enabled .Values.controller.extraVolumes) }}
{{- if (or .Values.controller.customTemplate.configMapName .Values.controller.extraVolumeMounts .Values.controller.admissionWebhooks.enabled .Values.controller.extraVolumes .Values.controller.extraModules) }}
volumes:
{{- if .Values.controller.extraModules }}
- name: modules
emptyDir: {}
{{- end }}
{{- if .Values.controller.customTemplate.configMapName }}
- name: nginx-template-volume
configMap:

View file

@ -140,8 +140,12 @@ spec:
hostPort: {{ $key }}
{{- end }}
{{- end }}
{{- if (or .Values.controller.customTemplate.configMapName .Values.controller.extraVolumeMounts .Values.controller.admissionWebhooks.enabled) }}
{{- if (or .Values.controller.customTemplate.configMapName .Values.controller.extraVolumeMounts .Values.controller.admissionWebhooks.enabled .Values.controller.extraModules) }}
volumeMounts:
{{- if .Values.controller.extraModules }}
- name: modules
mountPath: /modules_mount
{{- end }}
{{- if .Values.controller.customTemplate.configMapName }}
- mountPath: /etc/nginx/template
name: nginx-template-volume
@ -162,8 +166,21 @@ spec:
{{- if .Values.controller.extraContainers }}
{{ toYaml .Values.controller.extraContainers | nindent 8 }}
{{- end }}
{{- if .Values.controller.extraInitContainers }}
initContainers: {{ toYaml .Values.controller.extraInitContainers | nindent 8 }}
{{- if (or .Values.controller.extraInitContainers .Values.controller.extraModules) }}
initContainers:
{{- if .Values.controller.extraInitContainers }}
{{ toYaml .Values.controller.extraInitContainers | nindent 8 }}
{{- end }}
{{- if .Values.controller.extraModules }}
{{- range .Values.controller.extraModules }}
- name: {{ .name }}
image: {{ .image }}
command: ['sh', '-c', '/usr/local/bin/init_module.sh']
volumeMounts:
- name: modules
mountPath: /modules_mount
{{- end }}
{{- end }}
{{- end }}
{{- if .Values.controller.hostNetwork }}
hostNetwork: {{ .Values.controller.hostNetwork }}
@ -182,8 +199,12 @@ spec:
{{- end }}
serviceAccountName: {{ template "ingress-nginx.serviceAccountName" . }}
terminationGracePeriodSeconds: {{ .Values.controller.terminationGracePeriodSeconds }}
{{- if (or .Values.controller.customTemplate.configMapName .Values.controller.extraVolumeMounts .Values.controller.admissionWebhooks.enabled .Values.controller.extraVolumes) }}
{{- if (or .Values.controller.customTemplate.configMapName .Values.controller.extraVolumeMounts .Values.controller.admissionWebhooks.enabled .Values.controller.extraVolumes .Values.controller.extraModules) }}
volumes:
{{- if .Values.controller.extraModules }}
- name: modules
emptyDir: {}
{{- end }}
{{- if .Values.controller.customTemplate.configMapName }}
- name: nginx-template-volume
configMap:

View file

@ -559,6 +559,15 @@ controller:
# image: busybox
# command: ['sh', '-c', 'until nslookup myservice; do echo waiting for myservice; sleep 2; done;']
extraModules: []
## Modules, which are mounted into the core nginx image
# - name: opentelemetry
# image: busybox
#
# The image must contain a `/usr/local/bin/init_module.sh` executable, which
# will be executed as initContainers, to move its config files within the
# mounted volume.
admissionWebhooks:
annotations: {}
# ignore-check.kube-linter.io/no-read-only-rootfs: "This deployment needs write access to root filesystem".

View file

@ -33,6 +33,7 @@ ENV LUA_CPATH="/usr/local/lib/lua/?/?.so;/usr/local/lib/lua/?.so;;"
COPY --from=builder /usr/local /usr/local
COPY --from=builder /opt /opt
COPY --from=builder /etc/nginx /etc/nginx
COPY --from=builder entrypoint.sh /usr/local/entrypoint.sh
RUN apk update \
&& apk upgrade \
@ -72,4 +73,5 @@ RUN apk update \
EXPOSE 80 443
ENTRYPOINT ["/usr/local/entrypoint.sh"]
CMD ["nginx", "-g", "daemon off;"]

View file

@ -35,12 +35,6 @@ export NGINX_DIGEST_AUTH=1.0.0
# Check for recent changes: https://github.com/yaoweibin/ngx_http_substitutions_filter_module/compare/v0.6.4...master
export NGINX_SUBSTITUTIONS=b8a71eacc7f986ba091282ab8b1bbbc6ae1807e0
# Check for recent changes: https://github.com/open-telemetry/opentelemetry-cpp/compare/v1.0.0...main
export OPENTELEMETRY_CPP_VERSION=1.0.0
# Check for recent changes: https://github.com/open-telemetry/opentelemetry-cpp-contrib/compare/c655b8...main
export OPENTELEMETRY_CONTRIB_COMMIT=c655b849f017a5363085a4b4e6fcae8a4b7621ad
# Check for recent changes: https://github.com/opentracing-contrib/nginx-opentracing/compare/v0.19.0...master
export NGINX_OPENTRACING_VERSION=0.19.0
@ -144,11 +138,6 @@ if [[ ${ARCH} == "s390x" ]]; then
export LUA_STREAM_NGX_VERSION=0.0.7
fi
export USE_OPENTELEMETRY=true
if [[ ${ARCH} == "s390x" ]] || [[ ${ARCH} == "armv7l" ]]; then
export USE_OPENTELEMETRY=false
fi
get_src()
{
hash="$1"
@ -226,9 +215,6 @@ get_src f09851e6309560a8ff3e901548405066c83f1f6ff88aa7171e0763bd9514762b \
get_src a98b48947359166326d58700ccdc27256d2648218072da138ab6b47de47fbd8f \
"https://github.com/yaoweibin/ngx_http_substitutions_filter_module/archive/$NGINX_SUBSTITUTIONS.tar.gz"
get_src 37b2a2abf75e865449ff1425cee96dbd74659ac0c612c84ee5f381244360cab2 \
"https://github.com/open-telemetry/opentelemetry-cpp-contrib/archive/$OPENTELEMETRY_CONTRIB_COMMIT.tar.gz"
get_src 6f97776ebdf019b105a755c7736b70bdbd7e575c7f0d39db5fe127873c7abf17 \
"https://github.com/opentracing-contrib/nginx-opentracing/archive/v$NGINX_OPENTRACING_VERSION.tar.gz"
@ -482,32 +468,6 @@ cmake -DCMAKE_BUILD_TYPE=Release \
make
make install
if [ $USE_OPENTELEMETRY = true ]; then
# build opentelemetry lib
apk add protobuf-dev \
grpc \
grpc-dev \
gtest-dev \
c-ares-dev
cd $BUILD_PATH
git clone --recursive https://github.com/open-telemetry/opentelemetry-cpp opentelemetry-cpp-$OPENTELEMETRY_CPP_VERSION
cd "opentelemetry-cpp-$OPENTELEMETRY_CPP_VERSION"
git checkout v$OPENTELEMETRY_CPP_VERSION
mkdir .build
cd .build
cmake -DCMAKE_BUILD_TYPE=Release \
-DBUILD_TESTING=OFF \
-DWITH_EXAMPLES=OFF \
-DCMAKE_POSITION_INDEPENDENT_CODE=ON \
-DWITH_OTLP=ON \
-DWITH_OTLP_HTTP=OFF \
..
make
make install
fi
# Get Brotli source and deps
cd "$BUILD_PATH"
git clone --depth=1 https://github.com/google/ngx_brotli.git
@ -675,11 +635,6 @@ WITH_MODULES=" \
--add-dynamic-module=$BUILD_PATH/ngx_http_geoip2_module-${GEOIP2_VERSION} \
--add-dynamic-module=$BUILD_PATH/ngx_brotli"
if [ $USE_OPENTELEMETRY = true ]; then
WITH_MODULES+=" \
--add-dynamic-module=$BUILD_PATH/opentelemetry-cpp-contrib-$OPENTELEMETRY_CONTRIB_COMMIT/instrumentation/nginx"
fi
./configure \
--prefix=/usr/local/nginx \
--conf-path=/etc/nginx/nginx.conf \

View file

@ -0,0 +1,27 @@
#!/bin/bash
# Copyright 2022 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -o errexit
set -o nounset
set -o pipefail
if [ -d "/modules_mount/etc/nginx/modules" ]; then
for dir in /modules_mount/etc/nginx/modules/*; do
cp "$dir"/* "/etc/nginx/modules/$(basename "$dir")"
done
fi
exec "$@"

View file

@ -0,0 +1,57 @@
# Copyright 2021 The Kubernetes Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
.DEFAULT_GOAL:=build
# set default shell
SHELL=/bin/bash -o pipefail -o errexit
DIR:=$(strip $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))))
INIT_BUILDX=$(DIR)/../../hack/init-buildx.sh
# 0.0.0 shouldn't clobber any released builds
TAG ?= 0.0
REGISTRY ?= gcr.io/k8s-staging-ingress-nginx
IMAGE = $(REGISTRY)/opentelemetry
# required to enable buildx
export DOCKER_CLI_EXPERIMENTAL=enabled
# build with buildx
PLATFORMS?=linux/amd64,linux/arm
OUTPUT=
PROGRESS=plain
build: ensure-buildx
docker buildx build \
--platform=${PLATFORMS} $(OUTPUT) \
--progress=$(PROGRESS) \
--pull \
--tag $(IMAGE):$(TAG) rootfs
# push the cross built image
push: OUTPUT=--push
push: build
# enable buildx
ensure-buildx:
# this is required for cloudbuild
ifeq ("$(wildcard $(INIT_BUILDX))","")
@curl -sSL https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/hack/init-buildx.sh | bash
else
@exec $(INIT_BUILDX)
endif
@echo "done"
.PHONY: build push ensure-buildx

View file

@ -0,0 +1,5 @@
# OpenTelemetry library builder
**How to use this image:**
This image only contains the necessary files in /usr/local and /etc/nginx/opentelemetry to
be copied to Ingress Controller deployment when OpenTelemetry is enabled

View file

@ -0,0 +1,24 @@
timeout: 10800s
options:
substitution_option: ALLOW_LOOSE
# job builds a multi-arch docker image for amd64,arm,arm64 and s390x.
machineType: N1_HIGHCPU_32
steps:
- name: gcr.io/k8s-staging-test-infra/gcb-docker-gcloud:v20211118-2f2d816b90
entrypoint: bash
env:
- DOCKER_CLI_EXPERIMENTAL=enabled
- TAG=$_GIT_TAG
- BASE_REF=$_PULL_BASE_REF
- REGISTRY=gcr.io/k8s-staging-ingress-nginx
# default cloudbuild has HOME=/builder/home and docker buildx is in /root/.docker/cli-plugins/docker-buildx
# set the home to /root explicitly to if using docker buildx
- HOME=/root
args:
- -c
- |
gcloud auth configure-docker \
&& make push
substitutions:
_GIT_TAG: "12345"
_PULL_BASE_REF: "master"

View file

@ -0,0 +1,28 @@
# Copyright 2021 The Kubernetes Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
FROM alpine:3.14.2 as builder
COPY . /
RUN apk update \
&& apk upgrade \
&& apk add -U bash \
&& /build.sh
FROM busybox:latest
COPY --from=builder init_module.sh /usr/local/bin/init_module.sh
COPY --from=builder /etc/nginx/modules /etc/nginx/modules

View file

@ -0,0 +1,111 @@
#!/bin/bash
# Copyright 2021 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -o errexit
set -o nounset
set -o pipefail
export NGINX_VERSION=1.19.9
# Check for recent changes: https://github.com/open-telemetry/opentelemetry-cpp/compare/v1.0.0...main
export OPENTELEMETRY_CPP_VERSION=1.0.0
# Check for recent changes: https://github.com/open-telemetry/opentelemetry-cpp-contrib/compare/f4850...main
export OPENTELEMETRY_CONTRIB_COMMIT=f48500884b1b32efc456790bbcdc2e6cf7a8e630
export BUILD_PATH=/tmp/build
rm -rf \
/var/cache/debconf/* \
/var/lib/apt/lists/* \
/var/log/* \
/tmp/* \
/var/tmp/*
mkdir -p /etc/nginx
mkdir --verbose -p "$BUILD_PATH"
cd "$BUILD_PATH"
apk add \
curl \
git \
build-base
get_src()
{
hash="$1"
url="$2"
f=$(basename "$url")
echo "Downloading $url"
curl -sSL "$url" -o "$f"
echo "$hash $f" | sha256sum -c - || exit 10
tar xzf "$f"
rm -rf "$f"
}
get_src e462e11533d5c30baa05df7652160ff5979591d291736cfa5edb9fd2edb48c49 \
"https://nginx.org/download/nginx-$NGINX_VERSION.tar.gz"
get_src 45c52498788e47131b20a4786dbb08f4390b8cb419bd3d61c88b503cafff3324 \
"https://github.com/open-telemetry/opentelemetry-cpp-contrib/archive/$OPENTELEMETRY_CONTRIB_COMMIT.tar.gz"
# improve compilation times
CORES=$(($(grep -c ^processor /proc/cpuinfo) - 1))
export MAKEFLAGS=-j${CORES}
apk add \
protobuf-dev \
grpc \
grpc-dev \
gtest-dev \
c-ares-dev \
pcre-dev
cd $BUILD_PATH
git clone --recursive https://github.com/open-telemetry/opentelemetry-cpp opentelemetry-cpp-$OPENTELEMETRY_CPP_VERSION
cd "opentelemetry-cpp-$OPENTELEMETRY_CPP_VERSION"
git checkout v$OPENTELEMETRY_CPP_VERSION
mkdir .build
cd .build
cmake -DCMAKE_BUILD_TYPE=Release \
-DBUILD_TESTING=OFF \
-DWITH_EXAMPLES=OFF \
-DCMAKE_POSITION_INDEPENDENT_CODE=ON \
-DWITH_OTLP=ON \
-DWITH_OTLP_HTTP=OFF \
..
make
make install
# build nginx
cd "$BUILD_PATH/nginx-$NGINX_VERSION"
./configure \
--prefix=/usr/local/nginx \
--with-compat \
--add-dynamic-module=$BUILD_PATH/opentelemetry-cpp-contrib-$OPENTELEMETRY_CONTRIB_COMMIT/instrumentation/nginx
make modules
mkdir -p /etc/nginx/modules
cp objs/otel_ngx_module.so /etc/nginx/modules/otel_ngx_module.so
# remove .a files
find /usr/local -name "*.a" -print | xargs /bin/rm

View file

@ -0,0 +1,21 @@
#!/bin/sh
# Copyright 2021 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -o errexit
set -o nounset
set -o pipefail
cp -R /etc/nginx/modules /modules_mount/etc/nginx/modules