Add OTEL build test and for NGINX v1.25 (#10889)

* Add OTEL build test

* Simplify otel compilation

* Remove http2 deprecated arg

* Move image build to CI

* Turn image from scratch to optimize usage

* rollback image from scratch

* Final reviews on nginx v1.25 image

* Remove s390x from final image

.github/workflows/ci.yaml

@@ -9,6 +9,7 @@ on:
       - 'deploy/**'
       - '**.md'
       - 'images/**' # Images changes should be tested on their own workflow
+      - '!images/nginx-1.25/**'
 
   push:
     branches:
@@ -41,6 +42,7 @@ jobs:
     outputs:
       go: ${{ steps.filter.outputs.go }}
       charts: ${{ steps.filter.outputs.charts }}
+      baseimage: ${{ steps.filter.outputs.baseimage }}
 
     steps:
 
@@ -64,6 +66,8 @@ jobs:
               - 'charts/ingress-nginx/Chart.yaml'
               - 'charts/ingress-nginx/**/*'
               - 'NGINX_BASE'
+            baseimage:
+              - 'images/nginx-1.25/**'
 
   test-go:
     runs-on: ubuntu-latest
@@ -89,8 +93,10 @@ jobs:
     runs-on: ubuntu-latest
     needs: changes
     if: |
-      (needs.changes.outputs.go == 'true') || (needs.changes.outputs.charts == 'true')
+      (needs.changes.outputs.go == 'true') || (needs.changes.outputs.charts == 'true') || (needs.changes.outputs.baseimage == 'true')
-
+     
+    env:
+        PLATFORMS: linux/amd64
     steps:
       - name: Checkout
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
@@ -119,7 +125,14 @@ jobs:
           curl -LO https://dl.k8s.io/release/v1.27.3/bin/linux/amd64/kubectl
           chmod +x ./kubectl
           sudo mv ./kubectl /usr/local/bin/kubectl
-
+      
+      - name: Build NGINX Base image
+        if: |
+          needs.changes.outputs.baseimage == 'true'
+        run: |
+            export TAG=$(cat images/nginx-1.25/TAG)
+            cd images/nginx-1.25/rootfs && docker buildx build --platform=${{ env.PLATFORMS }} --load -t gcr.io/k8s-staging-ingress-nginx/nginx-1.25:${TAG} .
+    
       - name: Build images
         env:
           TAG: 1.0.0-dev
@@ -127,7 +140,8 @@ jobs:
           REGISTRY: ingress-controller
         run: |
           echo "building images..."
-          make clean-image build image image-chroot
+          export TAGNGINX=$(cat images/nginx-1.25/TAG)
+          make BASE_IMAGE=gcr.io/k8s-staging-ingress-nginx/nginx-1.25:${TAGNGINX} clean-image build image image-chroot
           make -C test/e2e-image image
 
           echo "creating images cache..."
@@ -150,11 +164,11 @@ jobs:
       - changes
       - build
     if: |
-      (needs.changes.outputs.charts == 'true')
+      (needs.changes.outputs.charts == 'true') || (needs.changes.outputs.baseimage == 'true')
 
     strategy:
       matrix:
-        k8s: [v1.25.11, v1.26.6, v1.27.3, v1.28.0, v1.29.0]
+        k8s: [v1.26.6, v1.27.3, v1.28.0, v1.29.0]
 
     steps:
       - name: Checkout
@@ -222,6 +236,7 @@ jobs:
           KIND_CLUSTER_NAME: kind
           SKIP_CLUSTER_CREATION: true
           SKIP_IMAGE_CREATION: true
+          SKIP_INGRESS_IMAGE_CREATION: true
         run: |
           kind get kubeconfig > $HOME/.kube/kind-config-kind
           make kind-e2e-chart-tests
@@ -232,10 +247,10 @@ jobs:
       - changes
       - build
     if: |
-      (needs.changes.outputs.go == 'true')
+      (needs.changes.outputs.go == 'true') || (needs.changes.outputs.baseimage == 'true')
     strategy:
       matrix:
-        k8s: [v1.25.11, v1.26.6, v1.27.3, v1.28.0, v1.29.0]
+        k8s: [v1.26.6, v1.27.3, v1.28.0, v1.29.0]
     uses: ./.github/workflows/zz-tmpl-k8s-e2e.yaml
     with:
       k8s-version: ${{ matrix.k8s }}
@@ -246,10 +261,10 @@ jobs:
       - changes
       - build
     if: |
-      (needs.changes.outputs.go == 'true')
+      (needs.changes.outputs.go == 'true') || (needs.changes.outputs.baseimage == 'true')
     strategy:
       matrix:
-        k8s: [v1.25.11, v1.26.6, v1.27.3, v1.28.0, v1.29.0]
+        k8s: [v1.26.6, v1.27.3, v1.28.0, v1.29.0]
     uses: ./.github/workflows/zz-tmpl-k8s-e2e.yaml
     with:
       k8s-version: ${{ matrix.k8s }}
@@ -261,10 +276,10 @@ jobs:
       - changes
       - build
     if: |
-      (needs.changes.outputs.go == 'true')
+      (needs.changes.outputs.go == 'true') || (needs.changes.outputs.baseimage == 'true')
     strategy:
       matrix:
-        k8s: [v1.25.11, v1.26.6, v1.27.3, v1.28.0, v1.29.0]
+        k8s: [v1.26.6, v1.27.3, v1.28.0, v1.29.0]
     uses: ./.github/workflows/zz-tmpl-k8s-e2e.yaml
     with:
       k8s-version: ${{ matrix.k8s }}

.github/workflows/images.yaml

@@ -36,6 +36,8 @@ jobs:
       kube-webhook-certgen: ${{ steps.filter.outputs.kube-webhook-certgen }}
       ext-auth-example-authsvc: ${{ steps.filter.outputs.ext-auth-example-authsvc }}
       nginx: ${{ steps.filter.outputs.nginx }}
+      nginx125: ${{ steps.filter.outputs.nginx125 }}
+      opentelemetry: ${{ steps.filter.outputs.opentelemetry }}
 
     steps:
       - name: Checkout
@@ -63,6 +65,10 @@ jobs:
               - 'images/ext-auth-example-authsvc/**'
             nginx:
               - 'images/nginx/**'
+            opentelemetry:
+              - 'images/opentelemetry/**'
+            nginx125:
+              - 'images/nginx-1.25/TAG'
  
   #### TODO: Make the below jobs 'less dumb' and use the job name as parameter (the github.job context does not work here)
   cfssl:
@@ -179,3 +185,53 @@ jobs:
       uses: github/codeql-action/upload-sarif@v3.23.1
       with:
         sarif_file: 'trivy-results.sarif'
+
+  opentelemetry:
+    runs-on: ubuntu-latest
+    env:
+      PLATFORMS: linux/amd64,linux/arm,linux/arm64
+    needs: changes
+    if: |
+      (needs.changes.outputs.opentelemetry == 'true')
+    strategy:
+      matrix:
+        nginx: ['1.25.3', '1.21.6']
+    steps:
+    - name: Checkout
+      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+    - name: image build
+      run: |
+        cd images/opentelemetry && make NGINX_VERSION=${{ matrix.nginx }} build
+
+  nginx125:
+    permissions:
+      contents: write
+      packages: write
+    runs-on: ubuntu-latest
+    needs: changes
+    if: |
+      (github.event_name == 'push' && github.ref == 'refs/heads/main' && needs.changes.outputs.nginx125 == 'true')
+    env:
+      PLATFORMS: linux/amd64,linux/arm,linux/arm64
+    steps:
+      - name: Checkout
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0
+        with:
+          version: latest
+          platforms: ${{ env.PLATFORMS }}
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: build-image
+        run: |
+          export TAG=$(cat images/nginx-1.25/TAG)
+          cd images/nginx-1.25/rootfs && docker buildx build --platform=${{ env.PLATFORMS }} --push --load -t ingressnginx/nginx-1.25:${TAG} .
+    
+    

.github/workflows/nginx125.yaml

@@ -1,167 +0,0 @@
-name: NGINX v1.25 Image
-
-on:
-  pull_request:
-    branches:
-    - "*"
-    paths:
-    - 'images/nginx-1.25/**'
-  push:
-    branches:
-    - main
-    paths:
-    - 'images/nginx-1.25/**'
-
-permissions:
-    contents: read
-
-jobs:
-  changes:
-    permissions:
-      contents: read  # for dorny/paths-filter to fetch a list of changed files
-      pull-requests: read  # for dorny/paths-filter to read pull requests
-    runs-on: ubuntu-latest
-    outputs:
-      nginx: ${{ steps.filter.outputs.nginx }}
-      tag: ${{ steps.filter.outputs.tag }}
-    steps:
-    - name: Checkout
-      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
-    - uses: dorny/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1
-      id: filter
-      with:
-        token: ${{ secrets.GITHUB_TOKEN }}
-        filters: |
-          nginx:
-          - 'images/nginx-1.25/**'
-          tag:
-          - 'images/nginx-1.25/TAG'
-
-  build:
-    permissions:
-      contents: read  # for dorny/paths-filter to fetch a list of changed files
-      pull-requests: read  # for dorny/paths-filter to read pull requests
-    runs-on: ubuntu-latest
-    needs: changes
-    if: |
-      (github.event_name != 'push' && github.ref != 'refs/heads/main' && needs.changes.outputs.nginx == 'true')
-    env:
-      PLATFORMS: linux/amd64
-    steps:
-    - name: Checkout
-      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
-    - name: Set up Go
-      id: go
-      uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
-      with:
-        go-version: '1.21.5'
-        check-latest: true
-    - name: Set up QEMU
-      uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0
-    - name: Set up Docker Buildx
-      id: buildx
-      uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0
-      with:
-        version: latest
-        platforms: ${{ env.PLATFORMS }}
-    - name: Prepare Host
-      run: |
-        curl -LO https://dl.k8s.io/release/v1.27.3/bin/linux/amd64/kubectl
-        chmod +x ./kubectl
-        sudo mv ./kubectl /usr/local/bin/kubectl
-    - name: build-image
-      run: |
-        cd images/nginx-1.25/rootfs && docker buildx build --platform=${{ env.PLATFORMS }} --load -t nginx-1.25:1.0.0-dev .
-    - name: load-image
-      run: |
-        make clean-image build
-        make -C test/e2e-image image
-        docker build \
-          --platform linux \
-          --no-cache \
-          --build-arg BASE_IMAGE="nginx-1.25:1.0.0-dev" \
-          --build-arg VERSION="0.0.1-${{ github.sha }}" \
-          --build-arg TARGETARCH="amd64" \
-          --build-arg COMMIT_SHA="git-${{ github.sha }}" \
-          --build-arg BUILD_ID=""UNSET"" \
-          -t ingress-controller/controller:1.0.0-dev rootfs
-        docker save \
-          nginx-ingress-controller:e2e \
-          ingress-controller/controller:1.0.0-dev \
-          nginx-1.25:1.0.0-dev \
-          | gzip > docker.tar.gz
-    - name: cache
-      uses: actions/upload-artifact@694cdabd8bdb0f10b2cea11669e1bf5453eed0a6 # v4.2.0
-      with:
-        name: docker.tar.gz
-        path: docker.tar.gz
-        retention-days: 2
-
-  e2e-test:
-    name: Kubernetes
-    runs-on: ubuntu-latest
-    needs:
-      - build
-    strategy:
-      matrix:
-        k8s: [v1.27.3, v1.28.0, v1.29.0]
-    steps:
-    - name: Checkout
-      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
-  
-    - name: cache
-      uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1
-      with:
-        name: docker.tar.gz
-  
-    - name: Create Kubernetes ${{ matrix.k8s }} cluster
-      id: kind
-      run: |
-        kind create cluster --image=kindest/node:${{ matrix.k8s }} --config test/e2e/kind.yaml
-  
-    - name: Load images from cache
-      run: |
-        echo "loading docker images..."
-        gzip -dc docker.tar.gz | docker load
-  
-    - name: Run e2e tests
-      env:
-        KIND_CLUSTER_NAME: kind
-        SKIP_CLUSTER_CREATION: true
-        SKIP_IMAGE_CREATION: true
-        SKIP_OPENTELEMETRY_TESTS: true
-      run: |
-        kind get kubeconfig > $HOME/.kube/kind-config-kind
-        make NGINX_BASE_IMAGE="nginx-1.25:1.0.0-dev" kind-e2e-test
-        
-  push:
-    permissions:
-      contents: write
-      packages: write
-    runs-on: ubuntu-latest
-    needs: changes
-    if: |
-      (github.event_name == 'push' && github.ref == 'refs/heads/main' && needs.changes.outputs.tag == 'true')
-    env:
-      PLATFORMS: linux/amd64,linux/arm,linux/arm64,linux/s390x
-    steps:
-    - name: Checkout
-      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
-    - name: Set up QEMU
-      uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0
-    - name: Set up Docker Buildx
-      id: buildx
-      uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0
-      with:
-        version: latest
-        platforms: ${{ env.PLATFORMS }}
-    - name: Login to GitHub Container Registry
-      uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
-      with:
-        username: ${{ secrets.DOCKERHUB_USERNAME }}
-        password: ${{ secrets.DOCKERHUB_TOKEN }}
-    - name: build-image
-      run: |
-        export TAG=$(cat images/nginx-1.25/TAG)
-        cd images/nginx-1.25/rootfs && docker buildx build --platform=${{ env.PLATFORMS }} --push --load -t ingressnginx/nginx-1.25:${TAG} .
-

.github/workflows/zz-tmpl-k8s-e2e.yaml

@@ -41,7 +41,8 @@ jobs:
         env:
           KIND_CLUSTER_NAME: kind
           SKIP_CLUSTER_CREATION: true
-          SKIP_IMAGE_CREATION: true
+          SKIP_INGRESS_IMAGE_CREATION: true
+          SKIP_E2E_IMAGE_CREATION: true
           ENABLE_VALIDATIONS: ${{ inputs.variation == 'VALIDATIONS' }}
           IS_CHROOT: ${{ inputs.variation == 'CHROOT' }}
         run: |

Makefile

@@ -231,8 +231,8 @@ ensure-buildx:
 show-version:
 	echo -n $(TAG)
 
-PLATFORMS ?= amd64 arm arm64 s390x
+PLATFORMS ?= amd64 arm arm64
-BUILDX_PLATFORMS ?= linux/amd64,linux/arm,linux/arm64,linux/s390x
+BUILDX_PLATFORMS ?= linux/amd64,linux/arm,linux/arm64
 
 .PHONY: release # Build a multi-arch docker image
 release: ensure-buildx clean

images/nginx-1.25/Makefile

@@ -22,17 +22,17 @@ INIT_BUILDX=$(DIR)/../../hack/init-buildx.sh
 
 # 0.0.0 shouldn't clobber any released builds
 SHORT_SHA ?=$(shell git rev-parse --short HEAD)
-TAG ?=v$(shell date +%Y%m%d)-$(SHORT_SHA)
+TAG ?=$(shell cat TAG)
 
 REGISTRY ?= gcr.io/k8s-staging-ingress-nginx
 
-IMAGE = $(REGISTRY)/nginx
+IMAGE = $(REGISTRY)/nginx-1.25
 
 # required to enable buildx
 export DOCKER_CLI_EXPERIMENTAL=enabled
 
 # build with buildx
-PLATFORMS?=linux/amd64,linux/arm,linux/arm64,linux/s390x
+PLATFORMS?=linux/amd64,linux/arm,linux/arm64
 OUTPUT=
 PROGRESS=plain
 build: ensure-buildx

images/nginx-1.25/cloudbuild.yaml

@@ -1,7 +1,7 @@
 timeout: 10800s
 options:
   substitution_option: ALLOW_LOOSE
-  # job builds a multi-arch docker image for amd64,arm,arm64 and s390x.
+  # job builds a multi-arch docker image for amd64,arm,arm64
   machineType: E2_HIGHCPU_32
 steps:
   - name: 'gcr.io/k8s-staging-test-infra/gcb-docker-gcloud:v20211118-2f2d816b90'

images/nginx-1.25/rootfs/Dockerfile

@@ -48,6 +48,9 @@ RUN apk update \
   yaml-cpp \
   dumb-init \
   tzdata \
+  grpc-cpp \
+  libprotobuf \
+  && apk add -X http://dl-cdn.alpinelinux.org/alpine/edge/testing opentelemetry-cpp-exporter-otlp-common opentelemetry-cpp-exporter-otlp-grpc \
   && ln -s /usr/local/nginx/sbin/nginx /sbin/nginx \
   && adduser -S -D -H -u 101 -h /usr/local/nginx \
   -s /sbin/nologin -G www-data -g www-data www-data \

images/nginx-1.25/rootfs/build.sh

@@ -129,6 +129,7 @@ get_src()
 }
 
 # install required packages to build
+# Dependencies from "ninja" and below are OTEL dependencies
 apk add \
   bash \
   gcc \
@@ -165,7 +166,22 @@ apk add \
   unzip \
   dos2unix \
   yaml-cpp \
-  coreutils
+  coreutils \
+  ninja \
+  gtest-dev \
+  git \
+  build-base \
+  pkgconfig \
+  c-ares-dev \
+  re2-dev \
+  grpc-dev \
+  protobuf-dev 
+
+apk add -X http://dl-cdn.alpinelinux.org/alpine/edge/testing opentelemetry-cpp-dev
+
+# There is some bug with some platforms and git, so force HTTP/1.1
+git config --global http.version HTTP/1.1
+git config --global http.postBuffer 157286400
 
 mkdir -p /etc/nginx
 
@@ -472,6 +488,33 @@ make
 make modules
 make install
 
+export OPENTELEMETRY_CONTRIB_COMMIT=aaa51e2297bcb34297f3c7aa44fa790497d2f7f3
+cd "$BUILD_PATH"
+
+git clone https://github.com/open-telemetry/opentelemetry-cpp-contrib.git opentelemetry-cpp-contrib-${OPENTELEMETRY_CONTRIB_COMMIT}
+
+cd ${BUILD_PATH}/opentelemetry-cpp-contrib-${OPENTELEMETRY_CONTRIB_COMMIT}
+git reset --hard ${OPENTELEMETRY_CONTRIB_COMMIT}
+
+export OTEL_TEMP_INSTALL=/tmp/otel
+mkdir -p ${OTEL_TEMP_INSTALL}
+
+cd ${BUILD_PATH}/opentelemetry-cpp-contrib-${OPENTELEMETRY_CONTRIB_COMMIT}/instrumentation/nginx
+mkdir -p build
+cd build
+cmake -DCMAKE_BUILD_TYPE=Release \
+        -G Ninja \
+        -DCMAKE_CXX_STANDARD=17 \
+        -DCMAKE_INSTALL_PREFIX=${OTEL_TEMP_INSTALL} \
+        -DBUILD_SHARED_LIBS=ON \
+        -DNGINX_VERSION=${NGINX_VERSION} \
+        ..
+cmake --build . -j ${CORES} --target install
+
+mkdir -p /etc/nginx/modules
+cp ${OTEL_TEMP_INSTALL}/otel_ngx_module.so /etc/nginx/modules/otel_ngx_module.so
+
+
 cd "$BUILD_PATH/lua-resty-core"
 make install
 

images/opentelemetry/Makefile

@@ -0,0 +1,69 @@
+# Copyright 2024 The Kubernetes Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+.DEFAULT_GOAL:=build
+
+# set default shell
+SHELL=/bin/bash -o pipefail -o errexit
+
+DIR:=$(strip $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))))
+INIT_BUILDX=$(DIR)/../../hack/init-buildx.sh
+
+# 0.0.0 shouldn't clobber any released builds
+SHORT_SHA ?=$(shell git rev-parse --short HEAD)
+TAG ?=v$(shell date +%Y%m%d)-$(SHORT_SHA)
+
+REGISTRY ?= gcr.io/k8s-staging-ingress-nginx
+
+IMAGE = $(REGISTRY)/opentelemetry
+
+# required to enable buildx
+export DOCKER_CLI_EXPERIMENTAL=enabled
+
+# build with buildx
+PLATFORMS?=linux/amd64,linux/arm,linux/arm64
+OUTPUT=
+PROGRESS=plain
+
+precheck:
+ifndef NGINX_VERSION
+	$(error NGINX_VERSION variable is required)
+endif
+
+build: precheck ensure-buildx
+	docker buildx build \
+		--label=org.opencontainers.image.source=https://github.com/kubernetes/ingress-nginx \
+		--label=org.opencontainers.image.licenses=Apache-2.0 \
+		--label=org.opencontainers.image.description="Ingress NGINX Opentelemetry image" \
+		--platform=${PLATFORMS} $(OUTPUT) \
+		--progress=$(PROGRESS) \
+		--build-arg=NGINX_VERSION=$(NGINX_VERSION) \
+		--pull \
+		--tag $(IMAGE)-$(NGINX_VERSION):$(TAG) rootfs
+
+# push the cross built image
+push: OUTPUT=--push
+push: build
+
+# enable buildx
+ensure-buildx:
+# this is required for cloudbuild
+ifeq ("$(wildcard $(INIT_BUILDX))","")
+	@curl -sSL https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/hack/init-buildx.sh | bash
+else
+	@exec $(INIT_BUILDX)
+endif
+	@echo "done"
+
+.PHONY: build precheck push ensure-buildx

images/opentelemetry/cloudbuild.yaml

@@ -16,4 +16,5 @@ steps:
     - -c
     - |
       gcloud auth configure-docker \
-      && cd images/ && make NAME=opentelemetry push
+      && cd images/opentelemetry && make NGINX_VERSION=1.25.3 push \
+      && make NGINX_VERSION=1.21.6 push

images/opentelemetry/rootfs/CMakeLists.txt

@@ -64,98 +64,9 @@ find_package(OpenSSL REQUIRED)
 message("OpenSSL include dir: ${OPENSSL_INCLUDE_DIR}")
 message("OpenSSL libraries: ${OPENSSL_LIBRARIES}")
 
-set(GRPC_GIT_TAG
+find_package(Protobuf REQUIRED)
-    "v1.45.2"
+find_package(gRPC REQUIRED)
-    CACHE STRING "gRPC version")
+find_package(OpentelemetryCPP REQUIRED)
-
-include(ExternalProject)
-set(ABSEIL_GIT_TAG "20230802.1" CACHE STRING "Abseil version")
-
-ExternalProject_Add(
-  Abseil
-  GIT_REPOSITORY https://github.com/abseil/abseil-cpp.git
-  GIT_TAG ${ABSEIL_GIT_TAG}
-  GIT_SHALLOW 1
-  UPDATE_COMMAND ""
-  CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${STAGED_INSTALL_PREFIX}
-             -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
-             -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
-             -DCMAKE_CXX_STANDARD=${CMAKE_CXX_STANDARD}
-             -DCMAKE_CXX_FLAGS:STRING=${CMAKE_CXX_FLAGS}
-  USES_TERMINAL_BUILD TRUE
-  DOWNLOAD_NO_PROGRESS TRUE
-  LOG_CONFIGURE TRUE
-  LOG_BUILD TRUE
-  LOG_INSTALL TRUE)
-
-# RE2 settings
-set(RE2_GIT_TAG "2023-11-01" CACHE STRING "RE2 version")
-
-ExternalProject_Add(
-  RE2
-  DEPENDS Abseil
-  GIT_REPOSITORY https://github.com/google/re2.git
-  GIT_TAG ${RE2_GIT_TAG}
-  GIT_SHALLOW 1
-  UPDATE_COMMAND ""
-  CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${STAGED_INSTALL_PREFIX}
-             -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
-             -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
-             -DCMAKE_CXX_STANDARD=${CMAKE_CXX_STANDARD}
-             -DCMAKE_CXX_FLAGS:STRING=${CMAKE_CXX_FLAGS}
-  USES_TERMINAL_BUILD TRUE
-  DOWNLOAD_NO_PROGRESS TRUE
-  LOG_CONFIGURE TRUE
-  LOG_BUILD TRUE
-  LOG_INSTALL TRUE)
-
-ExternalProject_Add(
-  gRPC
-  DEPENDS Abseil RE2
-  GIT_REPOSITORY https://github.com/grpc/grpc.git
-  GIT_TAG ${GRPC_GIT_TAG}
-  GIT_SHALLOW 1
-  UPDATE_COMMAND ""
-  CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${STAGED_INSTALL_PREFIX}
-             -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
-             -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
-             -DgRPC_SSL_PROVIDER=package
-             -DgRPC_RE2_PROVIDER=package
-             -DgRPC_ABSEIL_PROVIDER=package
-             -DOPENSSL_ROOT_DIR=OpenSSL
-             -DgRPC_BUILD_TESTS=OFF
-             -DBUILD_SHARED_LIBS=OFF
-             -DgRPC_INSTALL=ON
-             -DgRPC_BUILD_GRPC_NODE_PLUGIN=OFF
-             -DgRPC_BUILD_GRPC_OBJECTIVE_C_PLUGIN=OFF
-             -DgRPC_BUILD_GRPC_PHP_PLUGIN=OFF
-             -DgRPC_BUILD_GRPC_PYTHON_PLUGIN=OFF
-             -DgRPC_BUILD_GRPC_RUBY_PLUGIN=OFF
-             -DgRPC_BUILD_GRPC_NODE_PLUGIN=OFF
-             -DgRPC_BUILD_GRPC_OBJECTIVE_C_PLUGIN=OFF
-             -DgRPC_BUILD_GRPC_PHP_PLUGIN=OFF
-             -DgRPC_BUILD_GRPC_PYTHON_PLUGIN=OFF
-             -DgRPC_BUILD_GRPC_RUBY_PLUGIN=OFF
-             -DgRPC_USE_PROTO_LITE=OFF
-             -DgRPC_BUILD_CSHARP_EXT=OFF
-             -DgRPC_BUILD_GRPC_CSHARP_PLUGIN=OFF
-             -DgRPC_BUILD_GRPC_JAVA_PLUGIN=OFF
-             -DgRPC_BUILD_GRPC_GRPC_JAVA_PLUGIN=OFF
-             -DgRPC_BUILD_GRPC_WEB_PLUGIN=OFF
-             -DgRPC_BUILD_GRPC_C_PLUGIN=OFF
-             -DgRPC_BUILD_GRPC_NODE_PLUGIN=OFF
-             -DgRPC_BUILD_GRPC_OBJECTIVE_C_PLUGIN=OFF
-             -DgRPC_BUILD_GRPC_PHP_PLUGIN=OFF
-             -DgRPC_BUILD_GRPC_PYTHON_PLUGIN=OFF
-             -DgRPC_BUILD_GRPC_RUBY_PLUGIN=OFF
-             -DCMAKE_CXX_STANDARD=${CMAKE_CXX_STANDARD}
-             CMAKE_CACHE_ARGS -DCMAKE_CXX_FLAGS:STRING=${CMAKE_CXX_FLAGS}
-  TEST_AFTER_INSTALL FALSE
-  USES_TERMINAL_BUILD TRUE
-  DOWNLOAD_NO_PROGRESS TRUE
-  LOG_CONFIGURE TRUE
-  LOG_BUILD TRUE
-  LOG_INSTALL TRUE)
 
 install(
   DIRECTORY ${STAGED_INSTALL_PREFIX}/

images/opentelemetry/rootfs/Dockerfile

@@ -26,23 +26,12 @@ RUN apk update \
 
 ENV NINJA_STATUS "[%p/%f/%t] "
 
-# install gRPC
-FROM base as grpc
-RUN bash /opt/third_party/build.sh -g v1.60.0
-
-
-# install OpenTelemetry-cpp
-FROM base as otel-cpp
-COPY --from=grpc /opt/third_party/install/ /usr
-RUN bash /opt/third_party/build.sh -o v1.11.0
-
 # install otel_ngx_module.so
 FROM base as nginx
-COPY --from=grpc /opt/third_party/install/ /usr
+ARG NGINX_VERSION=1.25.3
-COPY --from=otel-cpp /opt/third_party/install/ /usr
+RUN bash /opt/third_party/build.sh -n ${NGINX_VERSION}
-RUN bash /opt/third_party/build.sh -n
 
-FROM cgr.dev/chainguard/go:latest as build-init
+FROM golang:1.21.6-bullseye as build-init
 
 WORKDIR /go/src/app
 COPY . .

images/opentelemetry/rootfs/build.sh

@@ -17,12 +17,12 @@
 set -o errexit
 set -o nounset
 set -o pipefail
-
+set -x
-export GRPC_GIT_TAG=${GRPC_GIT_TAG:="v1.43.2"}
 # Check for recent changes: https://github.com/open-telemetry/opentelemetry-cpp/compare/v1.2.0...main
-export OPENTELEMETRY_CPP_VERSION=${OPENTELEMETRY_CPP_VERSION:="1.2.0"}
+export OPENTELEMETRY_CPP_VERSION=${OPENTELEMETRY_CPP_VERSION:="v1.11.0"}
-export ABSL_CPP_VERSION=${ABSL_CPP_VERSION:="20230802.0"}
+export INSTALL_DIR=/opt/third_party/install
-export INSTAL_DIR=/opt/third_party/install
+
+export NGINX_VERSION=${NGINX_VERSION:="1.25.3"}
 # improve compilation times
 CORES=$(($(grep -c ^processor /proc/cpuinfo) - 1))
 
@@ -41,10 +41,9 @@ Help()
    # Display Help
    echo "Add description of the script functions here."
    echo
-   echo "Syntax: scriptTemplate [-h|g|o|n|p|]"
+   echo "Syntax: scriptTemplate [-h|o|n|p|]"
    echo "options:"
    echo "h     Print Help."
-   echo "g     gRPC git tag"
    echo "o     OpenTelemetry git tag"
    echo "n     install nginx"
    echo "p     prepare"
@@ -53,6 +52,7 @@ Help()
 
 prepare()
 {
+  echo "http://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories
   apk add \
     linux-headers \
     cmake \
@@ -65,48 +65,26 @@ prepare()
     pcre-dev \
     curl \
     git \
-    build-base
+    build-base \
-}
+    coreutils \
+    build-base \
+    openssl-dev \
+    pkgconfig \
+    c-ares-dev \
+    re2-dev \
+    grpc-dev \
+    protobuf-dev \
+    opentelemetry-cpp-dev
 
-install_grpc()
+  git config --global http.version HTTP/1.1
-{
+  git config --global http.postBuffer 157286400
-  mkdir -p $BUILD_PATH/grpc
-  cd ${BUILD_PATH}/grpc
-  cmake -DCMAKE_INSTALL_PREFIX=${INSTAL_DIR} \
-    -G Ninja \
-    -DGRPC_GIT_TAG=${GRPC_GIT_TAG} /opt/third_party
-
-  cmake --build . -j ${CORES} --target all install --verbose
-}
-
-install_absl()
-{
-  cd ${BUILD_PATH}
-  export LD_LIBRARY_PATH="${LD_LIBRARY_PATH:+LD_LIBRARY_PATH:}${INSTAL_DIR}/lib:/usr/local"
-  export PATH="${PATH}:${INSTAL_DIR}/bin"
-  git clone --recurse-submodules -j ${CORES} --depth=1 -b \
-    ${ABSL_CPP_VERSION} https://github.com/abseil/abseil-cpp.git abseil-cpp-${ABSL_CPP_VERSION}
-  cd "abseil-cpp-${ABSL_CPP_VERSION}"
-  mkdir -p .build
-  cd .build
-
-  cmake -DCMAKE_BUILD_TYPE=Release \
-        -G Ninja \
-        -DCMAKE_CXX_STANDARD=17 \
-        -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE \
-        -DBUILD_TESTING=OFF \
-        -DCMAKE_INSTALL_PREFIX=${INSTAL_DIR} \
-        -DABSL_PROPAGATE_CXX_STD=ON \
-        -DBUILD_SHARED_LIBS=OFF \
-        ..
-  cmake --build . -j ${CORES} --target install
 }
 
 install_otel()
 {
   cd ${BUILD_PATH}
-  export LD_LIBRARY_PATH="${LD_LIBRARY_PATH:+LD_LIBRARY_PATH:}${INSTAL_DIR}/lib:/usr/local"
+  export LD_LIBRARY_PATH="${LD_LIBRARY_PATH:+LD_LIBRARY_PATH:}${INSTALL_DIR}/lib:/usr/local"
-  export PATH="${PATH}:${INSTAL_DIR}/bin"
+  export PATH="${PATH}:${INSTALL_DIR}/bin"
   git clone --recurse-submodules -j ${CORES} --depth=1 -b \
     ${OPENTELEMETRY_CPP_VERSION} https://github.com/open-telemetry/opentelemetry-cpp.git opentelemetry-cpp-${OPENTELEMETRY_CPP_VERSION}
   cd "opentelemetry-cpp-${OPENTELEMETRY_CPP_VERSION}"
@@ -118,7 +96,7 @@ install_otel()
         -DCMAKE_CXX_STANDARD=17 \
         -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE  \
         -DWITH_ZIPKIN=OFF \
-        -DCMAKE_INSTALL_PREFIX=${INSTAL_DIR} \
+        -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \
         -DBUILD_TESTING=OFF \
         -DWITH_BENCHMARK=OFF \
         -DWITH_FUNC_TESTS=OFF \
@@ -132,23 +110,8 @@ install_otel()
   cmake --build . -j ${CORES} --target install
 }
 
-get_src()
-{
-  hash="$1"
-  url="$2"
-  f=$(basename "$url")
-
-  echo "Downloading $url"
-
-  curl -sSL --fail-with-body "$url" -o "$f"
-  echo "$hash  $f" | sha256sum -c - || exit 10
-  tar xzf "$f"
-  rm -rf "$f"
-}
-
 install_nginx()
 {
-  export NGINX_VERSION=1.25.3
 
   # Check for recent changes: https://github.com/open-telemetry/opentelemetry-cpp-contrib/compare/2656a4...main
   export OPENTELEMETRY_CONTRIB_COMMIT=aaa51e2297bcb34297f3c7aa44fa790497d2f7f3
@@ -170,39 +133,28 @@ install_nginx()
   cmake -DCMAKE_BUILD_TYPE=Release \
         -G Ninja \
         -DCMAKE_CXX_STANDARD=17 \
-        -DCMAKE_INSTALL_PREFIX=${INSTAL_DIR} \
+        -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \
         -DBUILD_SHARED_LIBS=ON \
         -DNGINX_VERSION=${NGINX_VERSION} \
         ..
   cmake --build . -j ${CORES} --target install
 
   mkdir -p /etc/nginx/modules
-  cp ${INSTAL_DIR}/otel_ngx_module.so /etc/nginx/modules/otel_ngx_module.so
+  cp ${INSTALL_DIR}/otel_ngx_module.so /etc/nginx/modules/otel_ngx_module.so
 }
 
-while getopts ":pha:g:o:n" option; do
+while getopts ":phn:" option; do
    case $option in
     h) # display Help
          Help
          exit;;
-    g) # install gRPC with git tag
-        GRPC_GIT_TAG=${OPTARG}
-        install_grpc
-        exit;;
-    o) # install OpenTelemetry tag
-        OPENTELEMETRY_CPP_VERSION=${OPTARG}
-        install_otel
-        exit;;
     p) # prepare
         prepare
         exit;;
     n) # install nginx
+        NGINX_VERSION=${OPTARG}
         install_nginx
         exit;;
-    a) # install abseil
-        ABSL_CPP_VERSION=${OPTARG}
-        install_absl
-        exit;;
     \?)
         Help
         exit;;

internal/ingress/controller/template/template.go

@@ -1501,13 +1501,8 @@ func httpsListener(addresses []string, co string, tc *config.TemplateConfig) []s
 			}
 		}
 
-		lo = append(lo, co, "ssl")
+		lo = append(lo, co, "ssl;")
 
-		if tc.Cfg.UseHTTP2 {
-			lo = append(lo, "http2")
-		}
-
-		lo = append(lo, ";")
 		out = append(out, strings.Join(lo, " "))
 	}
 

rootfs/Dockerfile

@@ -84,5 +84,4 @@ RUN  ln -sf /dev/stdout /var/log/nginx/access.log \
   && ln -sf /dev/stderr /var/log/nginx/error.log
 
 ENTRYPOINT ["/usr/bin/dumb-init", "--"]
-
 CMD ["/nginx-ingress-controller"]

rootfs/Dockerfile-chroot

@@ -119,7 +119,6 @@ RUN mkdir -p /chroot/modules_mount \
 USER www-data
 
 EXPOSE 80 443
-
 ENTRYPOINT ["/usr/bin/dumb-init", "--"]
 
 CMD ["/nginx-ingress-controller"]

rootfs/etc/nginx/template/nginx.tmpl

@@ -30,7 +30,7 @@ load_module /etc/nginx/modules/ngx_http_modsecurity_module.so;
 {{ end }}
 
 {{ if (shouldLoadOpentelemetryModule $cfg $servers) }}
-load_module /modules_mount/etc/nginx/modules/otel/otel_ngx_module.so;
+load_module /etc/nginx/modules/otel_ngx_module.so;
 {{ end }}
 
 daemon off;
@@ -658,6 +658,10 @@ http {
     server {
         server_name {{ buildServerName $server.Hostname }} {{range $server.Aliases }}{{ . }} {{ end }};
 
+        {{ if $cfg.UseHTTP2 }}
+            http2 on;
+        {{ end }}
+
         {{ if gt (len $cfg.BlockUserAgents) 0 }}
         if ($block_ua) {
            return 403;

test/e2e/settings/opentelemetry.go

@@ -17,7 +17,6 @@ limitations under the License.
 package settings
 
 import (
-	"os"
 	"strings"
 
 	"github.com/onsi/ginkgo/v2"
@@ -40,12 +39,6 @@ const (
 var _ = framework.IngressNginxDescribe("Configure Opentelemetry", func() {
 	f := framework.NewDefaultFramework("enable-opentelemetry")
 
-	shouldSkip := false
-	skip, ok := os.LookupEnv("SKIP_OPENTELEMETRY_TESTS")
-	if ok && skip == enable {
-		shouldSkip = true
-	}
-
 	ginkgo.BeforeEach(func() {
 		f.NewEchoDeployment()
 	})
@@ -54,9 +47,6 @@ var _ = framework.IngressNginxDescribe("Configure Opentelemetry", func() {
 	})
 
 	ginkgo.It("should not exists opentelemetry directive", func() {
-		if shouldSkip {
-			ginkgo.Skip("skipped")
-		}
 		config := map[string]string{}
 		config[enableOpentelemetry] = disable
 		f.SetNginxConfigMapData(config)
@@ -70,9 +60,6 @@ var _ = framework.IngressNginxDescribe("Configure Opentelemetry", func() {
 	})
 
 	ginkgo.It("should exists opentelemetry directive when is enabled", func() {
-		if shouldSkip {
-			ginkgo.Skip("skipped")
-		}
 		config := map[string]string{}
 		config[enableOpentelemetry] = enable
 		config[opentelemetryConfig] = opentelemetryConfigPath
@@ -87,9 +74,6 @@ var _ = framework.IngressNginxDescribe("Configure Opentelemetry", func() {
 	})
 
 	ginkgo.It("should include opentelemetry_trust_incoming_spans on directive when enabled", func() {
-		if shouldSkip {
-			ginkgo.Skip("skipped")
-		}
 		config := map[string]string{}
 		config[enableOpentelemetry] = enable
 		config[opentelemetryConfig] = opentelemetryConfigPath
@@ -105,9 +89,6 @@ var _ = framework.IngressNginxDescribe("Configure Opentelemetry", func() {
 	})
 
 	ginkgo.It("should not exists opentelemetry_operation_name directive when is empty", func() {
-		if shouldSkip {
-			ginkgo.Skip("skipped")
-		}
 		config := map[string]string{}
 		config[enableOpentelemetry] = enable
 		config[opentelemetryConfig] = opentelemetryConfigPath
@@ -123,9 +104,6 @@ var _ = framework.IngressNginxDescribe("Configure Opentelemetry", func() {
 	})
 
 	ginkgo.It("should exists opentelemetry_operation_name directive when is configured", func() {
-		if shouldSkip {
-			ginkgo.Skip("skipped")
-		}
 		config := map[string]string{}
 		config[enableOpentelemetry] = enable
 		config[opentelemetryConfig] = opentelemetryConfigPath

test/e2e/wait-for-nginx.sh

@@ -48,16 +48,6 @@ metadata:
 
 EOF
 
-OTEL_MODULE=$(cat <<EOF
-  opentelemetry:
-    enabled: true
-EOF
-)
-
-if [[ "$NAMESPACE_OVERLAY" != "enable-opentelemetry" ]]; then
-  OTEL_MODULE=""
-fi
-
 # Use the namespace overlay if it was requested
 if [[ ! -z "$NAMESPACE_OVERLAY" && -d "$DIR/namespace-overlays/$NAMESPACE_OVERLAY" ]]; then
     echo "Namespace overlay $NAMESPACE_OVERLAY is being used for namespace $NAMESPACE"