DevFW-CICD/ingress-nginx-helm
16
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add OTEL build test and for NGINX v1.25 (#10889)

Browse source 
* Add OTEL build test

* Simplify otel compilation

* Remove http2 deprecated arg

* Move image build to CI

* Turn image from scratch to optimize usage

* rollback image from scratch

* Final reviews on nginx v1.25 image

* Remove s390x from final image
This commit is contained in:
Ricardo Katz 2024-01-27 12:33:50 -03:00 committed by GitHub
parent 3d1cbb803b
commit 915070453d
20 changed files with 247 additions and 409 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

35
.github/workflows/ci.yaml vendored
View file

@ -9,6 +9,7 @@ on:
- 'deploy/**'
- '**.md'
- 'images/**' # Images changes should be tested on their own workflow
- '!images/nginx-1.25/**'
push:
branches:
@ -41,6 +42,7 @@ jobs:
outputs:
go: ${{ steps.filter.outputs.go }}
charts: ${{ steps.filter.outputs.charts }}
baseimage: ${{ steps.filter.outputs.baseimage }}
steps:
@ -64,6 +66,8 @@ jobs:
- 'charts/ingress-nginx/Chart.yaml'
- 'charts/ingress-nginx/**/*'
- 'NGINX_BASE'
baseimage:
- 'images/nginx-1.25/**'
test-go:
runs-on: ubuntu-latest
@ -89,8 +93,10 @@ jobs:
runs-on: ubuntu-latest
needs: changes
if: |
(needs.changes.outputs.go == 'true') || (needs.changes.outputs.charts == 'true')
(needs.changes.outputs.go == 'true') || (needs.changes.outputs.charts == 'true') || (needs.changes.outputs.baseimage == 'true')
env:
PLATFORMS: linux/amd64
steps:
- name: Checkout
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
@ -120,6 +126,13 @@ jobs:
chmod +x ./kubectl
sudo mv ./kubectl /usr/local/bin/kubectl
- name: Build NGINX Base image
if: |
needs.changes.outputs.baseimage == 'true'
run: |
export TAG=$(cat images/nginx-1.25/TAG)
cd images/nginx-1.25/rootfs && docker buildx build --platform=${{ env.PLATFORMS }} --load -t gcr.io/k8s-staging-ingress-nginx/nginx-1.25:${TAG} .
- name: Build images
env:
TAG: 1.0.0-dev
@ -127,7 +140,8 @@ jobs:
REGISTRY: ingress-controller
run: |
echo "building images..."
make clean-image build image image-chroot
export TAGNGINX=$(cat images/nginx-1.25/TAG)
make BASE_IMAGE=gcr.io/k8s-staging-ingress-nginx/nginx-1.25:${TAGNGINX} clean-image build image image-chroot
make -C test/e2e-image image
echo "creating images cache..."
@ -150,11 +164,11 @@ jobs:
- changes
- build
if: |
(needs.changes.outputs.charts == 'true')
(needs.changes.outputs.charts == 'true') || (needs.changes.outputs.baseimage == 'true')
strategy:
matrix:
k8s: [v1.25.11, v1.26.6, v1.27.3, v1.28.0, v1.29.0]
k8s: [v1.26.6, v1.27.3, v1.28.0, v1.29.0]
steps:
- name: Checkout
@ -222,6 +236,7 @@ jobs:
KIND_CLUSTER_NAME: kind
SKIP_CLUSTER_CREATION: true
SKIP_IMAGE_CREATION: true
SKIP_INGRESS_IMAGE_CREATION: true
run: |
kind get kubeconfig > $HOME/.kube/kind-config-kind
make kind-e2e-chart-tests
@ -232,10 +247,10 @@ jobs:
- changes
- build
if: |
(needs.changes.outputs.go == 'true')
(needs.changes.outputs.go == 'true') || (needs.changes.outputs.baseimage == 'true')
strategy:
matrix:
k8s: [v1.25.11, v1.26.6, v1.27.3, v1.28.0, v1.29.0]
k8s: [v1.26.6, v1.27.3, v1.28.0, v1.29.0]
uses: ./.github/workflows/zz-tmpl-k8s-e2e.yaml
with:
k8s-version: ${{ matrix.k8s }}
@ -246,10 +261,10 @@ jobs:
- changes
- build
if: |
(needs.changes.outputs.go == 'true')
(needs.changes.outputs.go == 'true') || (needs.changes.outputs.baseimage == 'true')
strategy:
matrix:
k8s: [v1.25.11, v1.26.6, v1.27.3, v1.28.0, v1.29.0]
k8s: [v1.26.6, v1.27.3, v1.28.0, v1.29.0]
uses: ./.github/workflows/zz-tmpl-k8s-e2e.yaml
with:
k8s-version: ${{ matrix.k8s }}
@ -261,10 +276,10 @@ jobs:
- changes
- build
if: |
(needs.changes.outputs.go == 'true')
(needs.changes.outputs.go == 'true') || (needs.changes.outputs.baseimage == 'true')
strategy:
matrix:
k8s: [v1.25.11, v1.26.6, v1.27.3, v1.28.0, v1.29.0]
k8s: [v1.26.6, v1.27.3, v1.28.0, v1.29.0]
uses: ./.github/workflows/zz-tmpl-k8s-e2e.yaml
with:
k8s-version: ${{ matrix.k8s }}

56
.github/workflows/images.yaml vendored
View file

@ -36,6 +36,8 @@ jobs:
kube-webhook-certgen: ${{ steps.filter.outputs.kube-webhook-certgen }}
ext-auth-example-authsvc: ${{ steps.filter.outputs.ext-auth-example-authsvc }}
nginx: ${{ steps.filter.outputs.nginx }}
nginx125: ${{ steps.filter.outputs.nginx125 }}
opentelemetry: ${{ steps.filter.outputs.opentelemetry }}
steps:
- name: Checkout
@ -63,6 +65,10 @@ jobs:
- 'images/ext-auth-example-authsvc/**'
nginx:
- 'images/nginx/**'
opentelemetry:
- 'images/opentelemetry/**'
nginx125:
- 'images/nginx-1.25/TAG'
#### TODO: Make the below jobs 'less dumb' and use the job name as parameter (the github.job context does not work here)
cfssl:
@ -179,3 +185,53 @@ jobs:
uses: github/codeql-action/upload-sarif@v3.23.1
with:
sarif_file: 'trivy-results.sarif'
opentelemetry:
runs-on: ubuntu-latest
env:
PLATFORMS: linux/amd64,linux/arm,linux/arm64
needs: changes
if: |
(needs.changes.outputs.opentelemetry == 'true')
strategy:
matrix:
nginx: ['1.25.3', '1.21.6']
steps:
- name: Checkout
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
- name: image build
run: |
cd images/opentelemetry && make NGINX_VERSION=${{ matrix.nginx }} build
nginx125:
permissions:
contents: write
packages: write
runs-on: ubuntu-latest
needs: changes
if: |
(github.event_name == 'push' && github.ref == 'refs/heads/main' && needs.changes.outputs.nginx125 == 'true')
env:
PLATFORMS: linux/amd64,linux/arm,linux/arm64
steps:
- name: Checkout
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
- name: Set up QEMU
uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0
with:
version: latest
platforms: ${{ env.PLATFORMS }}
- name: Login to GitHub Container Registry
uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: build-image
run: |
export TAG=$(cat images/nginx-1.25/TAG)
cd images/nginx-1.25/rootfs && docker buildx build --platform=${{ env.PLATFORMS }} --push --load -t ingressnginx/nginx-1.25:${TAG} .

167
.github/workflows/nginx125.yaml vendored
View file

@ -1,167 +0,0 @@
name: NGINX v1.25 Image
on:
pull_request:
branches:
- "*"
paths:
- 'images/nginx-1.25/**'
push:
branches:
- main
paths:
- 'images/nginx-1.25/**'
permissions:
contents: read
jobs:
changes:
permissions:
contents: read # for dorny/paths-filter to fetch a list of changed files
pull-requests: read # for dorny/paths-filter to read pull requests
runs-on: ubuntu-latest
outputs:
nginx: ${{ steps.filter.outputs.nginx }}
tag: ${{ steps.filter.outputs.tag }}
steps:
- name: Checkout
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
- uses: dorny/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1
id: filter
with:
token: ${{ secrets.GITHUB_TOKEN }}
filters: |
nginx:
- 'images/nginx-1.25/**'
tag:
- 'images/nginx-1.25/TAG'
build:
permissions:
contents: read # for dorny/paths-filter to fetch a list of changed files
pull-requests: read # for dorny/paths-filter to read pull requests
runs-on: ubuntu-latest
needs: changes
if: |
(github.event_name != 'push' && github.ref != 'refs/heads/main' && needs.changes.outputs.nginx == 'true')
env:
PLATFORMS: linux/amd64
steps:
- name: Checkout
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
- name: Set up Go
id: go
uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
with:
go-version: '1.21.5'
check-latest: true
- name: Set up QEMU
uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0
with:
version: latest
platforms: ${{ env.PLATFORMS }}
- name: Prepare Host
run: |
curl -LO https://dl.k8s.io/release/v1.27.3/bin/linux/amd64/kubectl
chmod +x ./kubectl
sudo mv ./kubectl /usr/local/bin/kubectl
- name: build-image
run: |
cd images/nginx-1.25/rootfs && docker buildx build --platform=${{ env.PLATFORMS }} --load -t nginx-1.25:1.0.0-dev .
- name: load-image
run: |
make clean-image build
make -C test/e2e-image image
docker build \
--platform linux \
--no-cache \
--build-arg BASE_IMAGE="nginx-1.25:1.0.0-dev" \
--build-arg VERSION="0.0.1-${{ github.sha }}" \
--build-arg TARGETARCH="amd64" \
--build-arg COMMIT_SHA="git-${{ github.sha }}" \
--build-arg BUILD_ID=""UNSET"" \
-t ingress-controller/controller:1.0.0-dev rootfs
docker save \
nginx-ingress-controller:e2e \
ingress-controller/controller:1.0.0-dev \
nginx-1.25:1.0.0-dev \
| gzip > docker.tar.gz
- name: cache
uses: actions/upload-artifact@694cdabd8bdb0f10b2cea11669e1bf5453eed0a6 # v4.2.0
with:
name: docker.tar.gz
path: docker.tar.gz
retention-days: 2
e2e-test:
name: Kubernetes
runs-on: ubuntu-latest
needs:
- build
strategy:
matrix:
k8s: [v1.27.3, v1.28.0, v1.29.0]
steps:
- name: Checkout
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
- name: cache
uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1
with:
name: docker.tar.gz
- name: Create Kubernetes ${{ matrix.k8s }} cluster
id: kind
run: |
kind create cluster --image=kindest/node:${{ matrix.k8s }} --config test/e2e/kind.yaml
- name: Load images from cache
run: |
echo "loading docker images..."
gzip -dc docker.tar.gz | docker load
- name: Run e2e tests
env:
KIND_CLUSTER_NAME: kind
SKIP_CLUSTER_CREATION: true
SKIP_IMAGE_CREATION: true
SKIP_OPENTELEMETRY_TESTS: true
run: |
kind get kubeconfig > $HOME/.kube/kind-config-kind
make NGINX_BASE_IMAGE="nginx-1.25:1.0.0-dev" kind-e2e-test
push:
permissions:
contents: write
packages: write
runs-on: ubuntu-latest
needs: changes
if: |
(github.event_name == 'push' && github.ref == 'refs/heads/main' && needs.changes.outputs.tag == 'true')
env:
PLATFORMS: linux/amd64,linux/arm,linux/arm64,linux/s390x
steps:
- name: Checkout
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
- name: Set up QEMU
uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0
with:
version: latest
platforms: ${{ env.PLATFORMS }}
- name: Login to GitHub Container Registry
uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: build-image
run: |
export TAG=$(cat images/nginx-1.25/TAG)
cd images/nginx-1.25/rootfs && docker buildx build --platform=${{ env.PLATFORMS }} --push --load -t ingressnginx/nginx-1.25:${TAG} .

3
.github/workflows/zz-tmpl-k8s-e2e.yaml vendored
View file

@ -41,7 +41,8 @@ jobs:
env:
KIND_CLUSTER_NAME: kind
SKIP_CLUSTER_CREATION: true
SKIP_IMAGE_CREATION: true
SKIP_INGRESS_IMAGE_CREATION: true
SKIP_E2E_IMAGE_CREATION: true
ENABLE_VALIDATIONS: ${{ inputs.variation == 'VALIDATIONS' }}
IS_CHROOT: ${{ inputs.variation == 'CHROOT' }}
run: |

4
Makefile
View file

@ -231,8 +231,8 @@ ensure-buildx:
show-version:
echo -n $(TAG)
PLATFORMS ?= amd64 arm arm64 s390x
BUILDX_PLATFORMS ?= linux/amd64,linux/arm,linux/arm64,linux/s390x
PLATFORMS ?= amd64 arm arm64
BUILDX_PLATFORMS ?= linux/amd64,linux/arm,linux/arm64
.PHONY: release # Build a multi-arch docker image
release: ensure-buildx clean

6
images/nginx-1.25/Makefile
View file

@ -22,17 +22,17 @@ INIT_BUILDX=$(DIR)/../../hack/init-buildx.sh
# 0.0.0 shouldn't clobber any released builds
SHORT_SHA ?=$(shell git rev-parse --short HEAD)
TAG ?=v$(shell date +%Y%m%d)-$(SHORT_SHA)
TAG ?=$(shell cat TAG)
REGISTRY ?= gcr.io/k8s-staging-ingress-nginx
IMAGE = $(REGISTRY)/nginx
IMAGE = $(REGISTRY)/nginx-1.25
# required to enable buildx
export DOCKER_CLI_EXPERIMENTAL=enabled
# build with buildx
PLATFORMS?=linux/amd64,linux/arm,linux/arm64,linux/s390x
PLATFORMS?=linux/amd64,linux/arm,linux/arm64
OUTPUT=
PROGRESS=plain
build: ensure-buildx

2
images/nginx-1.25/cloudbuild.yaml
View file

@ -1,7 +1,7 @@
timeout: 10800s
options:
substitution_option: ALLOW_LOOSE
# job builds a multi-arch docker image for amd64,arm,arm64 and s390x.
# job builds a multi-arch docker image for amd64,arm,arm64
machineType: E2_HIGHCPU_32
steps:
- name: 'gcr.io/k8s-staging-test-infra/gcb-docker-gcloud:v20211118-2f2d816b90'

3
images/nginx-1.25/rootfs/Dockerfile
View file

@ -48,6 +48,9 @@ RUN apk update \
yaml-cpp \
dumb-init \
tzdata \
grpc-cpp \
libprotobuf \
&& apk add -X http://dl-cdn.alpinelinux.org/alpine/edge/testing opentelemetry-cpp-exporter-otlp-common opentelemetry-cpp-exporter-otlp-grpc \
&& ln -s /usr/local/nginx/sbin/nginx /sbin/nginx \
&& adduser -S -D -H -u 101 -h /usr/local/nginx \
-s /sbin/nologin -G www-data -g www-data www-data \

45
images/nginx-1.25/rootfs/build.sh
View file

@ -129,6 +129,7 @@ get_src()
}
# install required packages to build
# Dependencies from "ninja" and below are OTEL dependencies
apk add \
bash \
gcc \
@ -165,7 +166,22 @@ apk add \
unzip \
dos2unix \
yaml-cpp \
coreutils
coreutils \
ninja \
gtest-dev \
git \
build-base \
pkgconfig \
c-ares-dev \
re2-dev \
grpc-dev \
protobuf-dev
apk add -X http://dl-cdn.alpinelinux.org/alpine/edge/testing opentelemetry-cpp-dev
# There is some bug with some platforms and git, so force HTTP/1.1
git config --global http.version HTTP/1.1
git config --global http.postBuffer 157286400
mkdir -p /etc/nginx
@ -472,6 +488,33 @@ make
make modules
make install
export OPENTELEMETRY_CONTRIB_COMMIT=aaa51e2297bcb34297f3c7aa44fa790497d2f7f3
cd "$BUILD_PATH"
git clone https://github.com/open-telemetry/opentelemetry-cpp-contrib.git opentelemetry-cpp-contrib-${OPENTELEMETRY_CONTRIB_COMMIT}
cd ${BUILD_PATH}/opentelemetry-cpp-contrib-${OPENTELEMETRY_CONTRIB_COMMIT}
git reset --hard ${OPENTELEMETRY_CONTRIB_COMMIT}
export OTEL_TEMP_INSTALL=/tmp/otel
mkdir -p ${OTEL_TEMP_INSTALL}
cd ${BUILD_PATH}/opentelemetry-cpp-contrib-${OPENTELEMETRY_CONTRIB_COMMIT}/instrumentation/nginx
mkdir -p build
cd build
cmake -DCMAKE_BUILD_TYPE=Release \
-G Ninja \
-DCMAKE_CXX_STANDARD=17 \
-DCMAKE_INSTALL_PREFIX=${OTEL_TEMP_INSTALL} \
-DBUILD_SHARED_LIBS=ON \
-DNGINX_VERSION=${NGINX_VERSION} \
..
cmake --build . -j ${CORES} --target install
mkdir -p /etc/nginx/modules
cp ${OTEL_TEMP_INSTALL}/otel_ngx_module.so /etc/nginx/modules/otel_ngx_module.so
cd "$BUILD_PATH/lua-resty-core"
make install

69
images/opentelemetry/Makefile Normal file
View file

@ -0,0 +1,69 @@
# Copyright 2024 The Kubernetes Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
.DEFAULT_GOAL:=build
# set default shell
SHELL=/bin/bash -o pipefail -o errexit
DIR:=$(strip $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))))
INIT_BUILDX=$(DIR)/../../hack/init-buildx.sh
# 0.0.0 shouldn't clobber any released builds
SHORT_SHA ?=$(shell git rev-parse --short HEAD)
TAG ?=v$(shell date +%Y%m%d)-$(SHORT_SHA)
REGISTRY ?= gcr.io/k8s-staging-ingress-nginx
IMAGE = $(REGISTRY)/opentelemetry
# required to enable buildx
export DOCKER_CLI_EXPERIMENTAL=enabled
# build with buildx
PLATFORMS?=linux/amd64,linux/arm,linux/arm64
OUTPUT=
PROGRESS=plain
precheck:
ifndef NGINX_VERSION
$(error NGINX_VERSION variable is required)
endif
build: precheck ensure-buildx
docker buildx build \
--label=org.opencontainers.image.source=https://github.com/kubernetes/ingress-nginx \
--label=org.opencontainers.image.licenses=Apache-2.0 \
--label=org.opencontainers.image.description="Ingress NGINX Opentelemetry image" \
--platform=${PLATFORMS} $(OUTPUT) \
--progress=$(PROGRESS) \
--build-arg=NGINX_VERSION=$(NGINX_VERSION) \
--pull \
--tag $(IMAGE)-$(NGINX_VERSION):$(TAG) rootfs
# push the cross built image
push: OUTPUT=--push
push: build
# enable buildx
ensure-buildx:
# this is required for cloudbuild
ifeq ("$(wildcard $(INIT_BUILDX))","")
@curl -sSL https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/hack/init-buildx.sh | bash
else
@exec $(INIT_BUILDX)
endif
@echo "done"
.PHONY: build precheck push ensure-buildx

3
images/opentelemetry/cloudbuild.yaml
View file

@ -16,4 +16,5 @@ steps:
- -c
- |
gcloud auth configure-docker \
&& cd images/ && make NAME=opentelemetry push
&& cd images/opentelemetry && make NGINX_VERSION=1.25.3 push \
&& make NGINX_VERSION=1.21.6 push

95
images/opentelemetry/rootfs/CMakeLists.txt
View file

@ -64,98 +64,9 @@ find_package(OpenSSL REQUIRED)
message("OpenSSL include dir: ${OPENSSL_INCLUDE_DIR}")
message("OpenSSL libraries: ${OPENSSL_LIBRARIES}")
set(GRPC_GIT_TAG
"v1.45.2"
CACHE STRING "gRPC version")
include(ExternalProject)
set(ABSEIL_GIT_TAG "20230802.1" CACHE STRING "Abseil version")
ExternalProject_Add(
Abseil
GIT_REPOSITORY https://github.com/abseil/abseil-cpp.git
GIT_TAG ${ABSEIL_GIT_TAG}
GIT_SHALLOW 1
UPDATE_COMMAND ""
CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${STAGED_INSTALL_PREFIX}
-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
-DCMAKE_CXX_STANDARD=${CMAKE_CXX_STANDARD}
-DCMAKE_CXX_FLAGS:STRING=${CMAKE_CXX_FLAGS}
USES_TERMINAL_BUILD TRUE
DOWNLOAD_NO_PROGRESS TRUE
LOG_CONFIGURE TRUE
LOG_BUILD TRUE
LOG_INSTALL TRUE)
# RE2 settings
set(RE2_GIT_TAG "2023-11-01" CACHE STRING "RE2 version")
ExternalProject_Add(
RE2
DEPENDS Abseil
GIT_REPOSITORY https://github.com/google/re2.git
GIT_TAG ${RE2_GIT_TAG}
GIT_SHALLOW 1
UPDATE_COMMAND ""
CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${STAGED_INSTALL_PREFIX}
-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
-DCMAKE_CXX_STANDARD=${CMAKE_CXX_STANDARD}
-DCMAKE_CXX_FLAGS:STRING=${CMAKE_CXX_FLAGS}
USES_TERMINAL_BUILD TRUE
DOWNLOAD_NO_PROGRESS TRUE
LOG_CONFIGURE TRUE
LOG_BUILD TRUE
LOG_INSTALL TRUE)
ExternalProject_Add(
gRPC
DEPENDS Abseil RE2
GIT_REPOSITORY https://github.com/grpc/grpc.git
GIT_TAG ${GRPC_GIT_TAG}
GIT_SHALLOW 1
UPDATE_COMMAND ""
CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${STAGED_INSTALL_PREFIX}
-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
-DgRPC_SSL_PROVIDER=package
-DgRPC_RE2_PROVIDER=package
-DgRPC_ABSEIL_PROVIDER=package
-DOPENSSL_ROOT_DIR=OpenSSL
-DgRPC_BUILD_TESTS=OFF
-DBUILD_SHARED_LIBS=OFF
-DgRPC_INSTALL=ON
-DgRPC_BUILD_GRPC_NODE_PLUGIN=OFF
-DgRPC_BUILD_GRPC_OBJECTIVE_C_PLUGIN=OFF
-DgRPC_BUILD_GRPC_PHP_PLUGIN=OFF
-DgRPC_BUILD_GRPC_PYTHON_PLUGIN=OFF
-DgRPC_BUILD_GRPC_RUBY_PLUGIN=OFF
-DgRPC_BUILD_GRPC_NODE_PLUGIN=OFF
-DgRPC_BUILD_GRPC_OBJECTIVE_C_PLUGIN=OFF
-DgRPC_BUILD_GRPC_PHP_PLUGIN=OFF
-DgRPC_BUILD_GRPC_PYTHON_PLUGIN=OFF
-DgRPC_BUILD_GRPC_RUBY_PLUGIN=OFF
-DgRPC_USE_PROTO_LITE=OFF
-DgRPC_BUILD_CSHARP_EXT=OFF
-DgRPC_BUILD_GRPC_CSHARP_PLUGIN=OFF
-DgRPC_BUILD_GRPC_JAVA_PLUGIN=OFF
-DgRPC_BUILD_GRPC_GRPC_JAVA_PLUGIN=OFF
-DgRPC_BUILD_GRPC_WEB_PLUGIN=OFF
-DgRPC_BUILD_GRPC_C_PLUGIN=OFF
-DgRPC_BUILD_GRPC_NODE_PLUGIN=OFF
-DgRPC_BUILD_GRPC_OBJECTIVE_C_PLUGIN=OFF
-DgRPC_BUILD_GRPC_PHP_PLUGIN=OFF
-DgRPC_BUILD_GRPC_PYTHON_PLUGIN=OFF
-DgRPC_BUILD_GRPC_RUBY_PLUGIN=OFF
-DCMAKE_CXX_STANDARD=${CMAKE_CXX_STANDARD}
CMAKE_CACHE_ARGS -DCMAKE_CXX_FLAGS:STRING=${CMAKE_CXX_FLAGS}
TEST_AFTER_INSTALL FALSE
USES_TERMINAL_BUILD TRUE
DOWNLOAD_NO_PROGRESS TRUE
LOG_CONFIGURE TRUE
LOG_BUILD TRUE
LOG_INSTALL TRUE)
find_package(Protobuf REQUIRED)
find_package(gRPC REQUIRED)
find_package(OpentelemetryCPP REQUIRED)
install(
DIRECTORY ${STAGED_INSTALL_PREFIX}/

17
images/opentelemetry/rootfs/Dockerfile
View file

@ -26,23 +26,12 @@ RUN apk update \
ENV NINJA_STATUS "[%p/%f/%t] "
# install gRPC
FROM base as grpc
RUN bash /opt/third_party/build.sh -g v1.60.0
# install OpenTelemetry-cpp
FROM base as otel-cpp
COPY --from=grpc /opt/third_party/install/ /usr
RUN bash /opt/third_party/build.sh -o v1.11.0
# install otel_ngx_module.so
FROM base as nginx
COPY --from=grpc /opt/third_party/install/ /usr
COPY --from=otel-cpp /opt/third_party/install/ /usr
RUN bash /opt/third_party/build.sh -n
ARG NGINX_VERSION=1.25.3
RUN bash /opt/third_party/build.sh -n ${NGINX_VERSION}
FROM cgr.dev/chainguard/go:latest as build-init
FROM golang:1.21.6-bullseye as build-init
WORKDIR /go/src/app
COPY . .

100
images/opentelemetry/rootfs/build.sh
View file

@ -17,12 +17,12 @@
set -o errexit
set -o nounset
set -o pipefail
export GRPC_GIT_TAG=${GRPC_GIT_TAG:="v1.43.2"}
set -x
# Check for recent changes: https://github.com/open-telemetry/opentelemetry-cpp/compare/v1.2.0...main
export OPENTELEMETRY_CPP_VERSION=${OPENTELEMETRY_CPP_VERSION:="1.2.0"}
export ABSL_CPP_VERSION=${ABSL_CPP_VERSION:="20230802.0"}
export INSTAL_DIR=/opt/third_party/install
export OPENTELEMETRY_CPP_VERSION=${OPENTELEMETRY_CPP_VERSION:="v1.11.0"}
export INSTALL_DIR=/opt/third_party/install
export NGINX_VERSION=${NGINX_VERSION:="1.25.3"}
# improve compilation times
CORES=$(($(grep -c ^processor /proc/cpuinfo) - 1))
@ -41,10 +41,9 @@ Help()
# Display Help
echo "Add description of the script functions here."
echo
echo "Syntax: scriptTemplate [-h|g|o|n|p|]"
echo "Syntax: scriptTemplate [-h|o|n|p|]"
echo "options:"
echo "h Print Help."
echo "g gRPC git tag"
echo "o OpenTelemetry git tag"
echo "n install nginx"
echo "p prepare"
@ -53,6 +52,7 @@ Help()
prepare()
{
echo "http://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories
apk add \
linux-headers \
cmake \
@ -65,48 +65,26 @@ prepare()
pcre-dev \
curl \
git \
build-base
}
build-base \
coreutils \
build-base \
openssl-dev \
pkgconfig \
c-ares-dev \
re2-dev \
grpc-dev \
protobuf-dev \
opentelemetry-cpp-dev
install_grpc()
{
mkdir -p $BUILD_PATH/grpc
cd ${BUILD_PATH}/grpc
cmake -DCMAKE_INSTALL_PREFIX=${INSTAL_DIR} \
-G Ninja \
-DGRPC_GIT_TAG=${GRPC_GIT_TAG} /opt/third_party
cmake --build . -j ${CORES} --target all install --verbose
}
install_absl()
{
cd ${BUILD_PATH}
export LD_LIBRARY_PATH="${LD_LIBRARY_PATH:+LD_LIBRARY_PATH:}${INSTAL_DIR}/lib:/usr/local"
export PATH="${PATH}:${INSTAL_DIR}/bin"
git clone --recurse-submodules -j ${CORES} --depth=1 -b \
${ABSL_CPP_VERSION} https://github.com/abseil/abseil-cpp.git abseil-cpp-${ABSL_CPP_VERSION}
cd "abseil-cpp-${ABSL_CPP_VERSION}"
mkdir -p .build
cd .build
cmake -DCMAKE_BUILD_TYPE=Release \
-G Ninja \
-DCMAKE_CXX_STANDARD=17 \
-DCMAKE_POSITION_INDEPENDENT_CODE=TRUE \
-DBUILD_TESTING=OFF \
-DCMAKE_INSTALL_PREFIX=${INSTAL_DIR} \
-DABSL_PROPAGATE_CXX_STD=ON \
-DBUILD_SHARED_LIBS=OFF \
..
cmake --build . -j ${CORES} --target install
git config --global http.version HTTP/1.1
git config --global http.postBuffer 157286400
}
install_otel()
{
cd ${BUILD_PATH}
export LD_LIBRARY_PATH="${LD_LIBRARY_PATH:+LD_LIBRARY_PATH:}${INSTAL_DIR}/lib:/usr/local"
export PATH="${PATH}:${INSTAL_DIR}/bin"
export LD_LIBRARY_PATH="${LD_LIBRARY_PATH:+LD_LIBRARY_PATH:}${INSTALL_DIR}/lib:/usr/local"
export PATH="${PATH}:${INSTALL_DIR}/bin"
git clone --recurse-submodules -j ${CORES} --depth=1 -b \
${OPENTELEMETRY_CPP_VERSION} https://github.com/open-telemetry/opentelemetry-cpp.git opentelemetry-cpp-${OPENTELEMETRY_CPP_VERSION}
cd "opentelemetry-cpp-${OPENTELEMETRY_CPP_VERSION}"
@ -118,7 +96,7 @@ install_otel()
-DCMAKE_CXX_STANDARD=17 \
-DCMAKE_POSITION_INDEPENDENT_CODE=TRUE \
-DWITH_ZIPKIN=OFF \
-DCMAKE_INSTALL_PREFIX=${INSTAL_DIR} \
-DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \
-DBUILD_TESTING=OFF \
-DWITH_BENCHMARK=OFF \
-DWITH_FUNC_TESTS=OFF \
@ -132,23 +110,8 @@ install_otel()
cmake --build . -j ${CORES} --target install
}
get_src()
{
hash="$1"
url="$2"
f=$(basename "$url")
echo "Downloading $url"
curl -sSL --fail-with-body "$url" -o "$f"
echo "$hash $f" | sha256sum -c - || exit 10
tar xzf "$f"
rm -rf "$f"
}
install_nginx()
{
export NGINX_VERSION=1.25.3
# Check for recent changes: https://github.com/open-telemetry/opentelemetry-cpp-contrib/compare/2656a4...main
export OPENTELEMETRY_CONTRIB_COMMIT=aaa51e2297bcb34297f3c7aa44fa790497d2f7f3
@ -170,39 +133,28 @@ install_nginx()
cmake -DCMAKE_BUILD_TYPE=Release \
-G Ninja \
-DCMAKE_CXX_STANDARD=17 \
-DCMAKE_INSTALL_PREFIX=${INSTAL_DIR} \
-DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \
-DBUILD_SHARED_LIBS=ON \
-DNGINX_VERSION=${NGINX_VERSION} \
..
cmake --build . -j ${CORES} --target install
mkdir -p /etc/nginx/modules
cp ${INSTAL_DIR}/otel_ngx_module.so /etc/nginx/modules/otel_ngx_module.so
cp ${INSTALL_DIR}/otel_ngx_module.so /etc/nginx/modules/otel_ngx_module.so
}
while getopts ":pha:g:o:n" option; do
while getopts ":phn:" option; do
case $option in
h) # display Help
Help
exit;;
g) # install gRPC with git tag
GRPC_GIT_TAG=${OPTARG}
install_grpc
exit;;
o) # install OpenTelemetry tag
OPENTELEMETRY_CPP_VERSION=${OPTARG}
install_otel
exit;;
p) # prepare
prepare
exit;;
n) # install nginx
NGINX_VERSION=${OPTARG}
install_nginx
exit;;
a) # install abseil
ABSL_CPP_VERSION=${OPTARG}
install_absl
exit;;
\?)
Help
exit;;

7
internal/ingress/controller/template/template.go
View file

@ -1501,13 +1501,8 @@ func httpsListener(addresses []string, co string, tc *config.TemplateConfig) []s
}
}
lo = append(lo, co, "ssl")
lo = append(lo, co, "ssl;")
if tc.Cfg.UseHTTP2 {
lo = append(lo, "http2")
}
lo = append(lo, ";")
out = append(out, strings.Join(lo, " "))
}

1
rootfs/Dockerfile
View file

@ -84,5 +84,4 @@ RUN ln -sf /dev/stdout /var/log/nginx/access.log \
&& ln -sf /dev/stderr /var/log/nginx/error.log
ENTRYPOINT ["/usr/bin/dumb-init", "--"]
CMD ["/nginx-ingress-controller"]

1
rootfs/Dockerfile-chroot
View file

@ -119,7 +119,6 @@ RUN mkdir -p /chroot/modules_mount \
USER www-data
EXPOSE 80 443
ENTRYPOINT ["/usr/bin/dumb-init", "--"]
CMD ["/nginx-ingress-controller"]

6
rootfs/etc/nginx/template/nginx.tmpl
View file

@ -30,7 +30,7 @@ load_module /etc/nginx/modules/ngx_http_modsecurity_module.so;
{{ end }}
{{ if (shouldLoadOpentelemetryModule $cfg $servers) }}
load_module /modules_mount/etc/nginx/modules/otel/otel_ngx_module.so;
load_module /etc/nginx/modules/otel_ngx_module.so;
{{ end }}
daemon off;
@ -658,6 +658,10 @@ http {
server {
server_name {{ buildServerName $server.Hostname }} {{range $server.Aliases }}{{ . }} {{ end }};
{{ if $cfg.UseHTTP2 }}
http2 on;
{{ end }}
{{ if gt (len $cfg.BlockUserAgents) 0 }}
if ($block_ua) {
return 403;

22
test/e2e/settings/opentelemetry.go
View file

@ -17,7 +17,6 @@ limitations under the License.
package settings
import (
"os"
"strings"
"github.com/onsi/ginkgo/v2"
@ -40,12 +39,6 @@ const (
var _ = framework.IngressNginxDescribe("Configure Opentelemetry", func() {
f := framework.NewDefaultFramework("enable-opentelemetry")
shouldSkip := false
skip, ok := os.LookupEnv("SKIP_OPENTELEMETRY_TESTS")
if ok && skip == enable {
shouldSkip = true
}
ginkgo.BeforeEach(func() {
f.NewEchoDeployment()
})
@ -54,9 +47,6 @@ var _ = framework.IngressNginxDescribe("Configure Opentelemetry", func() {
})
ginkgo.It("should not exists opentelemetry directive", func() {
if shouldSkip {
ginkgo.Skip("skipped")
}
config := map[string]string{}
config[enableOpentelemetry] = disable
f.SetNginxConfigMapData(config)
@ -70,9 +60,6 @@ var _ = framework.IngressNginxDescribe("Configure Opentelemetry", func() {
})
ginkgo.It("should exists opentelemetry directive when is enabled", func() {
if shouldSkip {
ginkgo.Skip("skipped")
}
config := map[string]string{}
config[enableOpentelemetry] = enable
config[opentelemetryConfig] = opentelemetryConfigPath
@ -87,9 +74,6 @@ var _ = framework.IngressNginxDescribe("Configure Opentelemetry", func() {
})
ginkgo.It("should include opentelemetry_trust_incoming_spans on directive when enabled", func() {
if shouldSkip {
ginkgo.Skip("skipped")
}
config := map[string]string{}
config[enableOpentelemetry] = enable
config[opentelemetryConfig] = opentelemetryConfigPath
@ -105,9 +89,6 @@ var _ = framework.IngressNginxDescribe("Configure Opentelemetry", func() {
})
ginkgo.It("should not exists opentelemetry_operation_name directive when is empty", func() {
if shouldSkip {
ginkgo.Skip("skipped")
}
config := map[string]string{}
config[enableOpentelemetry] = enable
config[opentelemetryConfig] = opentelemetryConfigPath
@ -123,9 +104,6 @@ var _ = framework.IngressNginxDescribe("Configure Opentelemetry", func() {
})
ginkgo.It("should exists opentelemetry_operation_name directive when is configured", func() {
if shouldSkip {
ginkgo.Skip("skipped")
}
config := map[string]string{}
config[enableOpentelemetry] = enable
config[opentelemetryConfig] = opentelemetryConfigPath

10
test/e2e/wait-for-nginx.sh
View file

@ -48,16 +48,6 @@ metadata:
EOF
OTEL_MODULE=$(cat <<EOF
opentelemetry:
enabled: true
EOF
)
if [[ "$NAMESPACE_OVERLAY" != "enable-opentelemetry" ]]; then
OTEL_MODULE=""
fi
# Use the namespace overlay if it was requested
if [[ ! -z "$NAMESPACE_OVERLAY" && -d "$DIR/namespace-overlays/$NAMESPACE_OVERLAY" ]]; then
echo "Namespace overlay $NAMESPACE_OVERLAY is being used for namespace $NAMESPACE"