diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index eb8815fb8..2c6bef636 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -9,6 +9,7 @@ on: - 'deploy/**' - '**.md' - 'images/**' # Images changes should be tested on their own workflow + - '!images/nginx-1.25/**' push: branches: @@ -41,6 +42,7 @@ jobs: outputs: go: ${{ steps.filter.outputs.go }} charts: ${{ steps.filter.outputs.charts }} + baseimage: ${{ steps.filter.outputs.baseimage }} steps: @@ -64,6 +66,8 @@ jobs: - 'charts/ingress-nginx/Chart.yaml' - 'charts/ingress-nginx/**/*' - 'NGINX_BASE' + baseimage: + - 'images/nginx-1.25/**' test-go: runs-on: ubuntu-latest @@ -89,8 +93,10 @@ jobs: runs-on: ubuntu-latest needs: changes if: | - (needs.changes.outputs.go == 'true') || (needs.changes.outputs.charts == 'true') - + (needs.changes.outputs.go == 'true') || (needs.changes.outputs.charts == 'true') || (needs.changes.outputs.baseimage == 'true') + + env: + PLATFORMS: linux/amd64 steps: - name: Checkout uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 @@ -119,7 +125,14 @@ jobs: curl -LO https://dl.k8s.io/release/v1.27.3/bin/linux/amd64/kubectl chmod +x ./kubectl sudo mv ./kubectl /usr/local/bin/kubectl - + + - name: Build NGINX Base image + if: | + needs.changes.outputs.baseimage == 'true' + run: | + export TAG=$(cat images/nginx-1.25/TAG) + cd images/nginx-1.25/rootfs && docker buildx build --platform=${{ env.PLATFORMS }} --load -t gcr.io/k8s-staging-ingress-nginx/nginx-1.25:${TAG} . + - name: Build images env: TAG: 1.0.0-dev @@ -127,7 +140,8 @@ jobs: REGISTRY: ingress-controller run: | echo "building images..." - make clean-image build image image-chroot + export TAGNGINX=$(cat images/nginx-1.25/TAG) + make BASE_IMAGE=gcr.io/k8s-staging-ingress-nginx/nginx-1.25:${TAGNGINX} clean-image build image image-chroot make -C test/e2e-image image echo "creating images cache..." @@ -150,11 +164,11 @@ jobs: - changes - build if: | - (needs.changes.outputs.charts == 'true') + (needs.changes.outputs.charts == 'true') || (needs.changes.outputs.baseimage == 'true') strategy: matrix: - k8s: [v1.25.11, v1.26.6, v1.27.3, v1.28.0, v1.29.0] + k8s: [v1.26.6, v1.27.3, v1.28.0, v1.29.0] steps: - name: Checkout @@ -222,6 +236,7 @@ jobs: KIND_CLUSTER_NAME: kind SKIP_CLUSTER_CREATION: true SKIP_IMAGE_CREATION: true + SKIP_INGRESS_IMAGE_CREATION: true run: | kind get kubeconfig > $HOME/.kube/kind-config-kind make kind-e2e-chart-tests @@ -232,10 +247,10 @@ jobs: - changes - build if: | - (needs.changes.outputs.go == 'true') + (needs.changes.outputs.go == 'true') || (needs.changes.outputs.baseimage == 'true') strategy: matrix: - k8s: [v1.25.11, v1.26.6, v1.27.3, v1.28.0, v1.29.0] + k8s: [v1.26.6, v1.27.3, v1.28.0, v1.29.0] uses: ./.github/workflows/zz-tmpl-k8s-e2e.yaml with: k8s-version: ${{ matrix.k8s }} @@ -246,10 +261,10 @@ jobs: - changes - build if: | - (needs.changes.outputs.go == 'true') + (needs.changes.outputs.go == 'true') || (needs.changes.outputs.baseimage == 'true') strategy: matrix: - k8s: [v1.25.11, v1.26.6, v1.27.3, v1.28.0, v1.29.0] + k8s: [v1.26.6, v1.27.3, v1.28.0, v1.29.0] uses: ./.github/workflows/zz-tmpl-k8s-e2e.yaml with: k8s-version: ${{ matrix.k8s }} @@ -261,10 +276,10 @@ jobs: - changes - build if: | - (needs.changes.outputs.go == 'true') + (needs.changes.outputs.go == 'true') || (needs.changes.outputs.baseimage == 'true') strategy: matrix: - k8s: [v1.25.11, v1.26.6, v1.27.3, v1.28.0, v1.29.0] + k8s: [v1.26.6, v1.27.3, v1.28.0, v1.29.0] uses: ./.github/workflows/zz-tmpl-k8s-e2e.yaml with: k8s-version: ${{ matrix.k8s }} diff --git a/.github/workflows/images.yaml b/.github/workflows/images.yaml index fcb02f635..7eb07cfc3 100644 --- a/.github/workflows/images.yaml +++ b/.github/workflows/images.yaml @@ -36,6 +36,8 @@ jobs: kube-webhook-certgen: ${{ steps.filter.outputs.kube-webhook-certgen }} ext-auth-example-authsvc: ${{ steps.filter.outputs.ext-auth-example-authsvc }} nginx: ${{ steps.filter.outputs.nginx }} + nginx125: ${{ steps.filter.outputs.nginx125 }} + opentelemetry: ${{ steps.filter.outputs.opentelemetry }} steps: - name: Checkout @@ -63,6 +65,10 @@ jobs: - 'images/ext-auth-example-authsvc/**' nginx: - 'images/nginx/**' + opentelemetry: + - 'images/opentelemetry/**' + nginx125: + - 'images/nginx-1.25/TAG' #### TODO: Make the below jobs 'less dumb' and use the job name as parameter (the github.job context does not work here) cfssl: @@ -179,3 +185,53 @@ jobs: uses: github/codeql-action/upload-sarif@v3.23.1 with: sarif_file: 'trivy-results.sarif' + + opentelemetry: + runs-on: ubuntu-latest + env: + PLATFORMS: linux/amd64,linux/arm,linux/arm64 + needs: changes + if: | + (needs.changes.outputs.opentelemetry == 'true') + strategy: + matrix: + nginx: ['1.25.3', '1.21.6'] + steps: + - name: Checkout + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - name: image build + run: | + cd images/opentelemetry && make NGINX_VERSION=${{ matrix.nginx }} build + + nginx125: + permissions: + contents: write + packages: write + runs-on: ubuntu-latest + needs: changes + if: | + (github.event_name == 'push' && github.ref == 'refs/heads/main' && needs.changes.outputs.nginx125 == 'true') + env: + PLATFORMS: linux/amd64,linux/arm,linux/arm64 + steps: + - name: Checkout + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - name: Set up QEMU + uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0 + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0 + with: + version: latest + platforms: ${{ env.PLATFORMS }} + - name: Login to GitHub Container Registry + uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: build-image + run: | + export TAG=$(cat images/nginx-1.25/TAG) + cd images/nginx-1.25/rootfs && docker buildx build --platform=${{ env.PLATFORMS }} --push --load -t ingressnginx/nginx-1.25:${TAG} . + + \ No newline at end of file diff --git a/.github/workflows/nginx125.yaml b/.github/workflows/nginx125.yaml deleted file mode 100644 index 4dfed9567..000000000 --- a/.github/workflows/nginx125.yaml +++ /dev/null @@ -1,167 +0,0 @@ -name: NGINX v1.25 Image - -on: - pull_request: - branches: - - "*" - paths: - - 'images/nginx-1.25/**' - push: - branches: - - main - paths: - - 'images/nginx-1.25/**' - -permissions: - contents: read - -jobs: - changes: - permissions: - contents: read # for dorny/paths-filter to fetch a list of changed files - pull-requests: read # for dorny/paths-filter to read pull requests - runs-on: ubuntu-latest - outputs: - nginx: ${{ steps.filter.outputs.nginx }} - tag: ${{ steps.filter.outputs.tag }} - steps: - - name: Checkout - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - - uses: dorny/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1 - id: filter - with: - token: ${{ secrets.GITHUB_TOKEN }} - filters: | - nginx: - - 'images/nginx-1.25/**' - tag: - - 'images/nginx-1.25/TAG' - - build: - permissions: - contents: read # for dorny/paths-filter to fetch a list of changed files - pull-requests: read # for dorny/paths-filter to read pull requests - runs-on: ubuntu-latest - needs: changes - if: | - (github.event_name != 'push' && github.ref != 'refs/heads/main' && needs.changes.outputs.nginx == 'true') - env: - PLATFORMS: linux/amd64 - steps: - - name: Checkout - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - - name: Set up Go - id: go - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 - with: - go-version: '1.21.5' - check-latest: true - - name: Set up QEMU - uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0 - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0 - with: - version: latest - platforms: ${{ env.PLATFORMS }} - - name: Prepare Host - run: | - curl -LO https://dl.k8s.io/release/v1.27.3/bin/linux/amd64/kubectl - chmod +x ./kubectl - sudo mv ./kubectl /usr/local/bin/kubectl - - name: build-image - run: | - cd images/nginx-1.25/rootfs && docker buildx build --platform=${{ env.PLATFORMS }} --load -t nginx-1.25:1.0.0-dev . - - name: load-image - run: | - make clean-image build - make -C test/e2e-image image - docker build \ - --platform linux \ - --no-cache \ - --build-arg BASE_IMAGE="nginx-1.25:1.0.0-dev" \ - --build-arg VERSION="0.0.1-${{ github.sha }}" \ - --build-arg TARGETARCH="amd64" \ - --build-arg COMMIT_SHA="git-${{ github.sha }}" \ - --build-arg BUILD_ID=""UNSET"" \ - -t ingress-controller/controller:1.0.0-dev rootfs - docker save \ - nginx-ingress-controller:e2e \ - ingress-controller/controller:1.0.0-dev \ - nginx-1.25:1.0.0-dev \ - | gzip > docker.tar.gz - - name: cache - uses: actions/upload-artifact@694cdabd8bdb0f10b2cea11669e1bf5453eed0a6 # v4.2.0 - with: - name: docker.tar.gz - path: docker.tar.gz - retention-days: 2 - - e2e-test: - name: Kubernetes - runs-on: ubuntu-latest - needs: - - build - strategy: - matrix: - k8s: [v1.27.3, v1.28.0, v1.29.0] - steps: - - name: Checkout - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - - - name: cache - uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1 - with: - name: docker.tar.gz - - - name: Create Kubernetes ${{ matrix.k8s }} cluster - id: kind - run: | - kind create cluster --image=kindest/node:${{ matrix.k8s }} --config test/e2e/kind.yaml - - - name: Load images from cache - run: | - echo "loading docker images..." - gzip -dc docker.tar.gz | docker load - - - name: Run e2e tests - env: - KIND_CLUSTER_NAME: kind - SKIP_CLUSTER_CREATION: true - SKIP_IMAGE_CREATION: true - SKIP_OPENTELEMETRY_TESTS: true - run: | - kind get kubeconfig > $HOME/.kube/kind-config-kind - make NGINX_BASE_IMAGE="nginx-1.25:1.0.0-dev" kind-e2e-test - - push: - permissions: - contents: write - packages: write - runs-on: ubuntu-latest - needs: changes - if: | - (github.event_name == 'push' && github.ref == 'refs/heads/main' && needs.changes.outputs.tag == 'true') - env: - PLATFORMS: linux/amd64,linux/arm,linux/arm64,linux/s390x - steps: - - name: Checkout - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - - name: Set up QEMU - uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0 - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0 - with: - version: latest - platforms: ${{ env.PLATFORMS }} - - name: Login to GitHub Container Registry - uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: build-image - run: | - export TAG=$(cat images/nginx-1.25/TAG) - cd images/nginx-1.25/rootfs && docker buildx build --platform=${{ env.PLATFORMS }} --push --load -t ingressnginx/nginx-1.25:${TAG} . - diff --git a/.github/workflows/zz-tmpl-k8s-e2e.yaml b/.github/workflows/zz-tmpl-k8s-e2e.yaml index d2941eb15..7709e9637 100644 --- a/.github/workflows/zz-tmpl-k8s-e2e.yaml +++ b/.github/workflows/zz-tmpl-k8s-e2e.yaml @@ -41,7 +41,8 @@ jobs: env: KIND_CLUSTER_NAME: kind SKIP_CLUSTER_CREATION: true - SKIP_IMAGE_CREATION: true + SKIP_INGRESS_IMAGE_CREATION: true + SKIP_E2E_IMAGE_CREATION: true ENABLE_VALIDATIONS: ${{ inputs.variation == 'VALIDATIONS' }} IS_CHROOT: ${{ inputs.variation == 'CHROOT' }} run: | diff --git a/Makefile b/Makefile index 964889532..5a2d81dd4 100644 --- a/Makefile +++ b/Makefile @@ -231,8 +231,8 @@ ensure-buildx: show-version: echo -n $(TAG) -PLATFORMS ?= amd64 arm arm64 s390x -BUILDX_PLATFORMS ?= linux/amd64,linux/arm,linux/arm64,linux/s390x +PLATFORMS ?= amd64 arm arm64 +BUILDX_PLATFORMS ?= linux/amd64,linux/arm,linux/arm64 .PHONY: release # Build a multi-arch docker image release: ensure-buildx clean diff --git a/images/nginx-1.25/Makefile b/images/nginx-1.25/Makefile index 527dfa8fb..e03197109 100644 --- a/images/nginx-1.25/Makefile +++ b/images/nginx-1.25/Makefile @@ -22,17 +22,17 @@ INIT_BUILDX=$(DIR)/../../hack/init-buildx.sh # 0.0.0 shouldn't clobber any released builds SHORT_SHA ?=$(shell git rev-parse --short HEAD) -TAG ?=v$(shell date +%Y%m%d)-$(SHORT_SHA) +TAG ?=$(shell cat TAG) REGISTRY ?= gcr.io/k8s-staging-ingress-nginx -IMAGE = $(REGISTRY)/nginx +IMAGE = $(REGISTRY)/nginx-1.25 # required to enable buildx export DOCKER_CLI_EXPERIMENTAL=enabled # build with buildx -PLATFORMS?=linux/amd64,linux/arm,linux/arm64,linux/s390x +PLATFORMS?=linux/amd64,linux/arm,linux/arm64 OUTPUT= PROGRESS=plain build: ensure-buildx diff --git a/images/nginx-1.25/cloudbuild.yaml b/images/nginx-1.25/cloudbuild.yaml index e3902c785..c94539165 100644 --- a/images/nginx-1.25/cloudbuild.yaml +++ b/images/nginx-1.25/cloudbuild.yaml @@ -1,7 +1,7 @@ timeout: 10800s options: substitution_option: ALLOW_LOOSE - # job builds a multi-arch docker image for amd64,arm,arm64 and s390x. + # job builds a multi-arch docker image for amd64,arm,arm64 machineType: E2_HIGHCPU_32 steps: - name: 'gcr.io/k8s-staging-test-infra/gcb-docker-gcloud:v20211118-2f2d816b90' diff --git a/images/nginx-1.25/rootfs/Dockerfile b/images/nginx-1.25/rootfs/Dockerfile index 8996547e6..ad0c9d654 100644 --- a/images/nginx-1.25/rootfs/Dockerfile +++ b/images/nginx-1.25/rootfs/Dockerfile @@ -48,6 +48,9 @@ RUN apk update \ yaml-cpp \ dumb-init \ tzdata \ + grpc-cpp \ + libprotobuf \ + && apk add -X http://dl-cdn.alpinelinux.org/alpine/edge/testing opentelemetry-cpp-exporter-otlp-common opentelemetry-cpp-exporter-otlp-grpc \ && ln -s /usr/local/nginx/sbin/nginx /sbin/nginx \ && adduser -S -D -H -u 101 -h /usr/local/nginx \ -s /sbin/nologin -G www-data -g www-data www-data \ diff --git a/images/nginx-1.25/rootfs/build.sh b/images/nginx-1.25/rootfs/build.sh index d88527e88..0fded23d6 100755 --- a/images/nginx-1.25/rootfs/build.sh +++ b/images/nginx-1.25/rootfs/build.sh @@ -129,6 +129,7 @@ get_src() } # install required packages to build +# Dependencies from "ninja" and below are OTEL dependencies apk add \ bash \ gcc \ @@ -165,7 +166,22 @@ apk add \ unzip \ dos2unix \ yaml-cpp \ - coreutils + coreutils \ + ninja \ + gtest-dev \ + git \ + build-base \ + pkgconfig \ + c-ares-dev \ + re2-dev \ + grpc-dev \ + protobuf-dev + +apk add -X http://dl-cdn.alpinelinux.org/alpine/edge/testing opentelemetry-cpp-dev + +# There is some bug with some platforms and git, so force HTTP/1.1 +git config --global http.version HTTP/1.1 +git config --global http.postBuffer 157286400 mkdir -p /etc/nginx @@ -472,6 +488,33 @@ make make modules make install +export OPENTELEMETRY_CONTRIB_COMMIT=aaa51e2297bcb34297f3c7aa44fa790497d2f7f3 +cd "$BUILD_PATH" + +git clone https://github.com/open-telemetry/opentelemetry-cpp-contrib.git opentelemetry-cpp-contrib-${OPENTELEMETRY_CONTRIB_COMMIT} + +cd ${BUILD_PATH}/opentelemetry-cpp-contrib-${OPENTELEMETRY_CONTRIB_COMMIT} +git reset --hard ${OPENTELEMETRY_CONTRIB_COMMIT} + +export OTEL_TEMP_INSTALL=/tmp/otel +mkdir -p ${OTEL_TEMP_INSTALL} + +cd ${BUILD_PATH}/opentelemetry-cpp-contrib-${OPENTELEMETRY_CONTRIB_COMMIT}/instrumentation/nginx +mkdir -p build +cd build +cmake -DCMAKE_BUILD_TYPE=Release \ + -G Ninja \ + -DCMAKE_CXX_STANDARD=17 \ + -DCMAKE_INSTALL_PREFIX=${OTEL_TEMP_INSTALL} \ + -DBUILD_SHARED_LIBS=ON \ + -DNGINX_VERSION=${NGINX_VERSION} \ + .. +cmake --build . -j ${CORES} --target install + +mkdir -p /etc/nginx/modules +cp ${OTEL_TEMP_INSTALL}/otel_ngx_module.so /etc/nginx/modules/otel_ngx_module.so + + cd "$BUILD_PATH/lua-resty-core" make install diff --git a/images/opentelemetry/Makefile b/images/opentelemetry/Makefile new file mode 100644 index 000000000..eae435bef --- /dev/null +++ b/images/opentelemetry/Makefile @@ -0,0 +1,69 @@ +# Copyright 2024 The Kubernetes Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +.DEFAULT_GOAL:=build + +# set default shell +SHELL=/bin/bash -o pipefail -o errexit + +DIR:=$(strip $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))) +INIT_BUILDX=$(DIR)/../../hack/init-buildx.sh + +# 0.0.0 shouldn't clobber any released builds +SHORT_SHA ?=$(shell git rev-parse --short HEAD) +TAG ?=v$(shell date +%Y%m%d)-$(SHORT_SHA) + +REGISTRY ?= gcr.io/k8s-staging-ingress-nginx + +IMAGE = $(REGISTRY)/opentelemetry + +# required to enable buildx +export DOCKER_CLI_EXPERIMENTAL=enabled + +# build with buildx +PLATFORMS?=linux/amd64,linux/arm,linux/arm64 +OUTPUT= +PROGRESS=plain + +precheck: +ifndef NGINX_VERSION + $(error NGINX_VERSION variable is required) +endif + +build: precheck ensure-buildx + docker buildx build \ + --label=org.opencontainers.image.source=https://github.com/kubernetes/ingress-nginx \ + --label=org.opencontainers.image.licenses=Apache-2.0 \ + --label=org.opencontainers.image.description="Ingress NGINX Opentelemetry image" \ + --platform=${PLATFORMS} $(OUTPUT) \ + --progress=$(PROGRESS) \ + --build-arg=NGINX_VERSION=$(NGINX_VERSION) \ + --pull \ + --tag $(IMAGE)-$(NGINX_VERSION):$(TAG) rootfs + +# push the cross built image +push: OUTPUT=--push +push: build + +# enable buildx +ensure-buildx: +# this is required for cloudbuild +ifeq ("$(wildcard $(INIT_BUILDX))","") + @curl -sSL https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/hack/init-buildx.sh | bash +else + @exec $(INIT_BUILDX) +endif + @echo "done" + +.PHONY: build precheck push ensure-buildx diff --git a/images/opentelemetry/cloudbuild.yaml b/images/opentelemetry/cloudbuild.yaml index 6705dac87..bc48a93f4 100644 --- a/images/opentelemetry/cloudbuild.yaml +++ b/images/opentelemetry/cloudbuild.yaml @@ -16,4 +16,5 @@ steps: - -c - | gcloud auth configure-docker \ - && cd images/ && make NAME=opentelemetry push + && cd images/opentelemetry && make NGINX_VERSION=1.25.3 push \ + && make NGINX_VERSION=1.21.6 push diff --git a/images/opentelemetry/rootfs/CMakeLists.txt b/images/opentelemetry/rootfs/CMakeLists.txt index a1ff85b49..1c68d6fc6 100644 --- a/images/opentelemetry/rootfs/CMakeLists.txt +++ b/images/opentelemetry/rootfs/CMakeLists.txt @@ -64,98 +64,9 @@ find_package(OpenSSL REQUIRED) message("OpenSSL include dir: ${OPENSSL_INCLUDE_DIR}") message("OpenSSL libraries: ${OPENSSL_LIBRARIES}") -set(GRPC_GIT_TAG - "v1.45.2" - CACHE STRING "gRPC version") - -include(ExternalProject) -set(ABSEIL_GIT_TAG "20230802.1" CACHE STRING "Abseil version") - -ExternalProject_Add( - Abseil - GIT_REPOSITORY https://github.com/abseil/abseil-cpp.git - GIT_TAG ${ABSEIL_GIT_TAG} - GIT_SHALLOW 1 - UPDATE_COMMAND "" - CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${STAGED_INSTALL_PREFIX} - -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} - -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} - -DCMAKE_CXX_STANDARD=${CMAKE_CXX_STANDARD} - -DCMAKE_CXX_FLAGS:STRING=${CMAKE_CXX_FLAGS} - USES_TERMINAL_BUILD TRUE - DOWNLOAD_NO_PROGRESS TRUE - LOG_CONFIGURE TRUE - LOG_BUILD TRUE - LOG_INSTALL TRUE) - -# RE2 settings -set(RE2_GIT_TAG "2023-11-01" CACHE STRING "RE2 version") - -ExternalProject_Add( - RE2 - DEPENDS Abseil - GIT_REPOSITORY https://github.com/google/re2.git - GIT_TAG ${RE2_GIT_TAG} - GIT_SHALLOW 1 - UPDATE_COMMAND "" - CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${STAGED_INSTALL_PREFIX} - -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} - -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} - -DCMAKE_CXX_STANDARD=${CMAKE_CXX_STANDARD} - -DCMAKE_CXX_FLAGS:STRING=${CMAKE_CXX_FLAGS} - USES_TERMINAL_BUILD TRUE - DOWNLOAD_NO_PROGRESS TRUE - LOG_CONFIGURE TRUE - LOG_BUILD TRUE - LOG_INSTALL TRUE) - -ExternalProject_Add( - gRPC - DEPENDS Abseil RE2 - GIT_REPOSITORY https://github.com/grpc/grpc.git - GIT_TAG ${GRPC_GIT_TAG} - GIT_SHALLOW 1 - UPDATE_COMMAND "" - CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${STAGED_INSTALL_PREFIX} - -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} - -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} - -DgRPC_SSL_PROVIDER=package - -DgRPC_RE2_PROVIDER=package - -DgRPC_ABSEIL_PROVIDER=package - -DOPENSSL_ROOT_DIR=OpenSSL - -DgRPC_BUILD_TESTS=OFF - -DBUILD_SHARED_LIBS=OFF - -DgRPC_INSTALL=ON - -DgRPC_BUILD_GRPC_NODE_PLUGIN=OFF - -DgRPC_BUILD_GRPC_OBJECTIVE_C_PLUGIN=OFF - -DgRPC_BUILD_GRPC_PHP_PLUGIN=OFF - -DgRPC_BUILD_GRPC_PYTHON_PLUGIN=OFF - -DgRPC_BUILD_GRPC_RUBY_PLUGIN=OFF - -DgRPC_BUILD_GRPC_NODE_PLUGIN=OFF - -DgRPC_BUILD_GRPC_OBJECTIVE_C_PLUGIN=OFF - -DgRPC_BUILD_GRPC_PHP_PLUGIN=OFF - -DgRPC_BUILD_GRPC_PYTHON_PLUGIN=OFF - -DgRPC_BUILD_GRPC_RUBY_PLUGIN=OFF - -DgRPC_USE_PROTO_LITE=OFF - -DgRPC_BUILD_CSHARP_EXT=OFF - -DgRPC_BUILD_GRPC_CSHARP_PLUGIN=OFF - -DgRPC_BUILD_GRPC_JAVA_PLUGIN=OFF - -DgRPC_BUILD_GRPC_GRPC_JAVA_PLUGIN=OFF - -DgRPC_BUILD_GRPC_WEB_PLUGIN=OFF - -DgRPC_BUILD_GRPC_C_PLUGIN=OFF - -DgRPC_BUILD_GRPC_NODE_PLUGIN=OFF - -DgRPC_BUILD_GRPC_OBJECTIVE_C_PLUGIN=OFF - -DgRPC_BUILD_GRPC_PHP_PLUGIN=OFF - -DgRPC_BUILD_GRPC_PYTHON_PLUGIN=OFF - -DgRPC_BUILD_GRPC_RUBY_PLUGIN=OFF - -DCMAKE_CXX_STANDARD=${CMAKE_CXX_STANDARD} - CMAKE_CACHE_ARGS -DCMAKE_CXX_FLAGS:STRING=${CMAKE_CXX_FLAGS} - TEST_AFTER_INSTALL FALSE - USES_TERMINAL_BUILD TRUE - DOWNLOAD_NO_PROGRESS TRUE - LOG_CONFIGURE TRUE - LOG_BUILD TRUE - LOG_INSTALL TRUE) +find_package(Protobuf REQUIRED) +find_package(gRPC REQUIRED) +find_package(OpentelemetryCPP REQUIRED) install( DIRECTORY ${STAGED_INSTALL_PREFIX}/ diff --git a/images/opentelemetry/rootfs/Dockerfile b/images/opentelemetry/rootfs/Dockerfile index e568237c2..b6a0a595b 100644 --- a/images/opentelemetry/rootfs/Dockerfile +++ b/images/opentelemetry/rootfs/Dockerfile @@ -26,23 +26,12 @@ RUN apk update \ ENV NINJA_STATUS "[%p/%f/%t] " -# install gRPC -FROM base as grpc -RUN bash /opt/third_party/build.sh -g v1.60.0 - - -# install OpenTelemetry-cpp -FROM base as otel-cpp -COPY --from=grpc /opt/third_party/install/ /usr -RUN bash /opt/third_party/build.sh -o v1.11.0 - # install otel_ngx_module.so FROM base as nginx -COPY --from=grpc /opt/third_party/install/ /usr -COPY --from=otel-cpp /opt/third_party/install/ /usr -RUN bash /opt/third_party/build.sh -n +ARG NGINX_VERSION=1.25.3 +RUN bash /opt/third_party/build.sh -n ${NGINX_VERSION} -FROM cgr.dev/chainguard/go:latest as build-init +FROM golang:1.21.6-bullseye as build-init WORKDIR /go/src/app COPY . . diff --git a/images/opentelemetry/rootfs/build.sh b/images/opentelemetry/rootfs/build.sh index 8b491589f..d46ab9fe5 100755 --- a/images/opentelemetry/rootfs/build.sh +++ b/images/opentelemetry/rootfs/build.sh @@ -17,12 +17,12 @@ set -o errexit set -o nounset set -o pipefail - -export GRPC_GIT_TAG=${GRPC_GIT_TAG:="v1.43.2"} +set -x # Check for recent changes: https://github.com/open-telemetry/opentelemetry-cpp/compare/v1.2.0...main -export OPENTELEMETRY_CPP_VERSION=${OPENTELEMETRY_CPP_VERSION:="1.2.0"} -export ABSL_CPP_VERSION=${ABSL_CPP_VERSION:="20230802.0"} -export INSTAL_DIR=/opt/third_party/install +export OPENTELEMETRY_CPP_VERSION=${OPENTELEMETRY_CPP_VERSION:="v1.11.0"} +export INSTALL_DIR=/opt/third_party/install + +export NGINX_VERSION=${NGINX_VERSION:="1.25.3"} # improve compilation times CORES=$(($(grep -c ^processor /proc/cpuinfo) - 1)) @@ -41,10 +41,9 @@ Help() # Display Help echo "Add description of the script functions here." echo - echo "Syntax: scriptTemplate [-h|g|o|n|p|]" + echo "Syntax: scriptTemplate [-h|o|n|p|]" echo "options:" echo "h Print Help." - echo "g gRPC git tag" echo "o OpenTelemetry git tag" echo "n install nginx" echo "p prepare" @@ -53,6 +52,7 @@ Help() prepare() { + echo "http://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories apk add \ linux-headers \ cmake \ @@ -65,48 +65,26 @@ prepare() pcre-dev \ curl \ git \ - build-base -} + build-base \ + coreutils \ + build-base \ + openssl-dev \ + pkgconfig \ + c-ares-dev \ + re2-dev \ + grpc-dev \ + protobuf-dev \ + opentelemetry-cpp-dev -install_grpc() -{ - mkdir -p $BUILD_PATH/grpc - cd ${BUILD_PATH}/grpc - cmake -DCMAKE_INSTALL_PREFIX=${INSTAL_DIR} \ - -G Ninja \ - -DGRPC_GIT_TAG=${GRPC_GIT_TAG} /opt/third_party - - cmake --build . -j ${CORES} --target all install --verbose -} - -install_absl() -{ - cd ${BUILD_PATH} - export LD_LIBRARY_PATH="${LD_LIBRARY_PATH:+LD_LIBRARY_PATH:}${INSTAL_DIR}/lib:/usr/local" - export PATH="${PATH}:${INSTAL_DIR}/bin" - git clone --recurse-submodules -j ${CORES} --depth=1 -b \ - ${ABSL_CPP_VERSION} https://github.com/abseil/abseil-cpp.git abseil-cpp-${ABSL_CPP_VERSION} - cd "abseil-cpp-${ABSL_CPP_VERSION}" - mkdir -p .build - cd .build - - cmake -DCMAKE_BUILD_TYPE=Release \ - -G Ninja \ - -DCMAKE_CXX_STANDARD=17 \ - -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE \ - -DBUILD_TESTING=OFF \ - -DCMAKE_INSTALL_PREFIX=${INSTAL_DIR} \ - -DABSL_PROPAGATE_CXX_STD=ON \ - -DBUILD_SHARED_LIBS=OFF \ - .. - cmake --build . -j ${CORES} --target install + git config --global http.version HTTP/1.1 + git config --global http.postBuffer 157286400 } install_otel() { cd ${BUILD_PATH} - export LD_LIBRARY_PATH="${LD_LIBRARY_PATH:+LD_LIBRARY_PATH:}${INSTAL_DIR}/lib:/usr/local" - export PATH="${PATH}:${INSTAL_DIR}/bin" + export LD_LIBRARY_PATH="${LD_LIBRARY_PATH:+LD_LIBRARY_PATH:}${INSTALL_DIR}/lib:/usr/local" + export PATH="${PATH}:${INSTALL_DIR}/bin" git clone --recurse-submodules -j ${CORES} --depth=1 -b \ ${OPENTELEMETRY_CPP_VERSION} https://github.com/open-telemetry/opentelemetry-cpp.git opentelemetry-cpp-${OPENTELEMETRY_CPP_VERSION} cd "opentelemetry-cpp-${OPENTELEMETRY_CPP_VERSION}" @@ -118,7 +96,7 @@ install_otel() -DCMAKE_CXX_STANDARD=17 \ -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE \ -DWITH_ZIPKIN=OFF \ - -DCMAKE_INSTALL_PREFIX=${INSTAL_DIR} \ + -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ -DBUILD_TESTING=OFF \ -DWITH_BENCHMARK=OFF \ -DWITH_FUNC_TESTS=OFF \ @@ -132,23 +110,8 @@ install_otel() cmake --build . -j ${CORES} --target install } -get_src() -{ - hash="$1" - url="$2" - f=$(basename "$url") - - echo "Downloading $url" - - curl -sSL --fail-with-body "$url" -o "$f" - echo "$hash $f" | sha256sum -c - || exit 10 - tar xzf "$f" - rm -rf "$f" -} - install_nginx() { - export NGINX_VERSION=1.25.3 # Check for recent changes: https://github.com/open-telemetry/opentelemetry-cpp-contrib/compare/2656a4...main export OPENTELEMETRY_CONTRIB_COMMIT=aaa51e2297bcb34297f3c7aa44fa790497d2f7f3 @@ -170,39 +133,28 @@ install_nginx() cmake -DCMAKE_BUILD_TYPE=Release \ -G Ninja \ -DCMAKE_CXX_STANDARD=17 \ - -DCMAKE_INSTALL_PREFIX=${INSTAL_DIR} \ + -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ -DBUILD_SHARED_LIBS=ON \ -DNGINX_VERSION=${NGINX_VERSION} \ .. cmake --build . -j ${CORES} --target install mkdir -p /etc/nginx/modules - cp ${INSTAL_DIR}/otel_ngx_module.so /etc/nginx/modules/otel_ngx_module.so + cp ${INSTALL_DIR}/otel_ngx_module.so /etc/nginx/modules/otel_ngx_module.so } -while getopts ":pha:g:o:n" option; do +while getopts ":phn:" option; do case $option in h) # display Help Help exit;; - g) # install gRPC with git tag - GRPC_GIT_TAG=${OPTARG} - install_grpc - exit;; - o) # install OpenTelemetry tag - OPENTELEMETRY_CPP_VERSION=${OPTARG} - install_otel - exit;; p) # prepare prepare exit;; n) # install nginx + NGINX_VERSION=${OPTARG} install_nginx exit;; - a) # install abseil - ABSL_CPP_VERSION=${OPTARG} - install_absl - exit;; \?) Help exit;; diff --git a/internal/ingress/controller/template/template.go b/internal/ingress/controller/template/template.go index 7410ce6e0..f08eee498 100644 --- a/internal/ingress/controller/template/template.go +++ b/internal/ingress/controller/template/template.go @@ -1501,13 +1501,8 @@ func httpsListener(addresses []string, co string, tc *config.TemplateConfig) []s } } - lo = append(lo, co, "ssl") + lo = append(lo, co, "ssl;") - if tc.Cfg.UseHTTP2 { - lo = append(lo, "http2") - } - - lo = append(lo, ";") out = append(out, strings.Join(lo, " ")) } diff --git a/rootfs/Dockerfile b/rootfs/Dockerfile index 86517c6c1..a04cfe3de 100644 --- a/rootfs/Dockerfile +++ b/rootfs/Dockerfile @@ -84,5 +84,4 @@ RUN ln -sf /dev/stdout /var/log/nginx/access.log \ && ln -sf /dev/stderr /var/log/nginx/error.log ENTRYPOINT ["/usr/bin/dumb-init", "--"] - CMD ["/nginx-ingress-controller"] diff --git a/rootfs/Dockerfile-chroot b/rootfs/Dockerfile-chroot index a210aa7bf..b52cbddf9 100644 --- a/rootfs/Dockerfile-chroot +++ b/rootfs/Dockerfile-chroot @@ -119,7 +119,6 @@ RUN mkdir -p /chroot/modules_mount \ USER www-data EXPOSE 80 443 - ENTRYPOINT ["/usr/bin/dumb-init", "--"] CMD ["/nginx-ingress-controller"] diff --git a/rootfs/etc/nginx/template/nginx.tmpl b/rootfs/etc/nginx/template/nginx.tmpl index 94dc12412..d58be2880 100644 --- a/rootfs/etc/nginx/template/nginx.tmpl +++ b/rootfs/etc/nginx/template/nginx.tmpl @@ -30,7 +30,7 @@ load_module /etc/nginx/modules/ngx_http_modsecurity_module.so; {{ end }} {{ if (shouldLoadOpentelemetryModule $cfg $servers) }} -load_module /modules_mount/etc/nginx/modules/otel/otel_ngx_module.so; +load_module /etc/nginx/modules/otel_ngx_module.so; {{ end }} daemon off; @@ -658,6 +658,10 @@ http { server { server_name {{ buildServerName $server.Hostname }} {{range $server.Aliases }}{{ . }} {{ end }}; + {{ if $cfg.UseHTTP2 }} + http2 on; + {{ end }} + {{ if gt (len $cfg.BlockUserAgents) 0 }} if ($block_ua) { return 403; diff --git a/test/e2e/settings/opentelemetry.go b/test/e2e/settings/opentelemetry.go index af946f277..b5fc6ff4e 100644 --- a/test/e2e/settings/opentelemetry.go +++ b/test/e2e/settings/opentelemetry.go @@ -17,7 +17,6 @@ limitations under the License. package settings import ( - "os" "strings" "github.com/onsi/ginkgo/v2" @@ -40,12 +39,6 @@ const ( var _ = framework.IngressNginxDescribe("Configure Opentelemetry", func() { f := framework.NewDefaultFramework("enable-opentelemetry") - shouldSkip := false - skip, ok := os.LookupEnv("SKIP_OPENTELEMETRY_TESTS") - if ok && skip == enable { - shouldSkip = true - } - ginkgo.BeforeEach(func() { f.NewEchoDeployment() }) @@ -54,9 +47,6 @@ var _ = framework.IngressNginxDescribe("Configure Opentelemetry", func() { }) ginkgo.It("should not exists opentelemetry directive", func() { - if shouldSkip { - ginkgo.Skip("skipped") - } config := map[string]string{} config[enableOpentelemetry] = disable f.SetNginxConfigMapData(config) @@ -70,9 +60,6 @@ var _ = framework.IngressNginxDescribe("Configure Opentelemetry", func() { }) ginkgo.It("should exists opentelemetry directive when is enabled", func() { - if shouldSkip { - ginkgo.Skip("skipped") - } config := map[string]string{} config[enableOpentelemetry] = enable config[opentelemetryConfig] = opentelemetryConfigPath @@ -87,9 +74,6 @@ var _ = framework.IngressNginxDescribe("Configure Opentelemetry", func() { }) ginkgo.It("should include opentelemetry_trust_incoming_spans on directive when enabled", func() { - if shouldSkip { - ginkgo.Skip("skipped") - } config := map[string]string{} config[enableOpentelemetry] = enable config[opentelemetryConfig] = opentelemetryConfigPath @@ -105,9 +89,6 @@ var _ = framework.IngressNginxDescribe("Configure Opentelemetry", func() { }) ginkgo.It("should not exists opentelemetry_operation_name directive when is empty", func() { - if shouldSkip { - ginkgo.Skip("skipped") - } config := map[string]string{} config[enableOpentelemetry] = enable config[opentelemetryConfig] = opentelemetryConfigPath @@ -123,9 +104,6 @@ var _ = framework.IngressNginxDescribe("Configure Opentelemetry", func() { }) ginkgo.It("should exists opentelemetry_operation_name directive when is configured", func() { - if shouldSkip { - ginkgo.Skip("skipped") - } config := map[string]string{} config[enableOpentelemetry] = enable config[opentelemetryConfig] = opentelemetryConfigPath diff --git a/test/e2e/wait-for-nginx.sh b/test/e2e/wait-for-nginx.sh index 0726bde10..d2529e348 100755 --- a/test/e2e/wait-for-nginx.sh +++ b/test/e2e/wait-for-nginx.sh @@ -48,16 +48,6 @@ metadata: EOF -OTEL_MODULE=$(cat <