Simplify otel compilation

This commit is contained in:
Ricardo Katz 2024-01-21 22:08:41 -03:00
parent dbfea38a17
commit de1f52afee
11 changed files with 101 additions and 210 deletions

View file

@ -186,7 +186,7 @@ jobs:
opentelemetry:
runs-on: ubuntu-latest
env:
PLATFORMS: linux/amd64,linux/arm,linux/arm64,linux/s390x
PLATFORMS: linux/amd64,linux/arm,linux/arm64
needs: changes
if: |
(needs.changes.outputs.opentelemetry == 'true')

View file

@ -26,7 +26,7 @@ TAG ?=v$(shell date +%Y%m%d)-$(SHORT_SHA)
REGISTRY ?= gcr.io/k8s-staging-ingress-nginx
IMAGE = $(REGISTRY)/nginx
IMAGE = $(REGISTRY)/nginx-1.25
# required to enable buildx
export DOCKER_CLI_EXPERIMENTAL=enabled

View file

@ -32,7 +32,8 @@ COPY --from=builder /usr/local /usr/local
COPY --from=builder /opt /opt
COPY --from=builder /etc/nginx /etc/nginx
RUN apk update \
RUN echo "http://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories \
&& apk update \
&& apk upgrade \
&& apk add -U --no-cache \
bash \
@ -48,6 +49,10 @@ RUN apk update \
yaml-cpp \
dumb-init \
tzdata \
grpc-cpp \
libprotobuf \
opentelemetry-cpp-exporter-otlp-common \
opentelemetry-cpp-exporter-otlp-grpc \
&& ln -s /usr/local/nginx/sbin/nginx /sbin/nginx \
&& adduser -S -D -H -u 101 -h /usr/local/nginx \
-s /sbin/nologin -G www-data -g www-data www-data \

View file

@ -128,7 +128,11 @@ get_src()
rm -rf "$f"
}
# Enable testing/edge repo for OTEL
echo "http://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories
# install required packages to build
# Dependencies from "ninja" and below are OTEL dependencies
apk add \
bash \
gcc \
@ -165,7 +169,22 @@ apk add \
unzip \
dos2unix \
yaml-cpp \
coreutils
coreutils \
ninja \
gtest-dev \
git \
build-base \
pkgconfig \
c-ares-dev \
re2-dev \
grpc-dev \
protobuf-dev \
opentelemetry-cpp-dev
# There is some bug with some platforms and git, so force HTTP/1.1
git config --global http.version HTTP/1.1
git config --global http.postBuffer 157286400
mkdir -p /etc/nginx
@ -472,6 +491,35 @@ make
make modules
make install
# TODO: Ricardo Move to the right place
export OPENTELEMETRY_CONTRIB_COMMIT=aaa51e2297bcb34297f3c7aa44fa790497d2f7f3
cd "$BUILD_PATH"
git clone https://github.com/open-telemetry/opentelemetry-cpp-contrib.git opentelemetry-cpp-contrib-${OPENTELEMETRY_CONTRIB_COMMIT}
cd ${BUILD_PATH}/opentelemetry-cpp-contrib-${OPENTELEMETRY_CONTRIB_COMMIT}
git reset --hard ${OPENTELEMETRY_CONTRIB_COMMIT}
export OTEL_TEMP_INSTALL=/tmp/otel
mkdir -p ${OTEL_TEMP_INSTALL}
cd ${BUILD_PATH}/opentelemetry-cpp-contrib-${OPENTELEMETRY_CONTRIB_COMMIT}/instrumentation/nginx
mkdir -p build
cd build
cmake -DCMAKE_BUILD_TYPE=Release \
-G Ninja \
-DCMAKE_CXX_STANDARD=17 \
-DCMAKE_INSTALL_PREFIX=${OTEL_TEMP_INSTALL} \
-DBUILD_SHARED_LIBS=ON \
-DNGINX_VERSION=${NGINX_VERSION} \
..
cmake --build . -j ${CORES} --target install
mkdir -p /etc/nginx/modules
cp ${OTEL_TEMP_INSTALL}/otel_ngx_module.so /etc/nginx/modules/otel_ngx_module.so
cd "$BUILD_PATH/lua-resty-core"
make install

View file

@ -32,7 +32,7 @@ IMAGE = $(REGISTRY)/opentelemetry
export DOCKER_CLI_EXPERIMENTAL=enabled
# build with buildx
PLATFORMS?=linux/amd64,linux/arm,linux/arm64,linux/s390x
PLATFORMS?=linux/amd64,linux/arm,linux/arm64
OUTPUT=
PROGRESS=plain
@ -48,7 +48,7 @@ build: precheck ensure-buildx
--label=org.opencontainers.image.description="Ingress NGINX Opentelemetry image" \
--platform=${PLATFORMS} $(OUTPUT) \
--progress=$(PROGRESS) \
--build-arg=$(NGINX_VERSION) \
--build-arg=NGINX_VERSION=$(NGINX_VERSION) \
--pull \
--tag $(IMAGE)-$(NGINX_VERSION):$(TAG) rootfs

View file

@ -64,98 +64,9 @@ find_package(OpenSSL REQUIRED)
message("OpenSSL include dir: ${OPENSSL_INCLUDE_DIR}")
message("OpenSSL libraries: ${OPENSSL_LIBRARIES}")
set(GRPC_GIT_TAG
"v1.45.2"
CACHE STRING "gRPC version")
include(ExternalProject)
set(ABSEIL_GIT_TAG "20230802.1" CACHE STRING "Abseil version")
ExternalProject_Add(
Abseil
GIT_REPOSITORY https://github.com/abseil/abseil-cpp.git
GIT_TAG ${ABSEIL_GIT_TAG}
GIT_SHALLOW 1
UPDATE_COMMAND ""
CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${STAGED_INSTALL_PREFIX}
-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
-DCMAKE_CXX_STANDARD=${CMAKE_CXX_STANDARD}
-DCMAKE_CXX_FLAGS:STRING=${CMAKE_CXX_FLAGS}
USES_TERMINAL_BUILD TRUE
DOWNLOAD_NO_PROGRESS TRUE
LOG_CONFIGURE TRUE
LOG_BUILD TRUE
LOG_INSTALL TRUE)
# RE2 settings
set(RE2_GIT_TAG "2023-11-01" CACHE STRING "RE2 version")
ExternalProject_Add(
RE2
DEPENDS Abseil
GIT_REPOSITORY https://github.com/google/re2.git
GIT_TAG ${RE2_GIT_TAG}
GIT_SHALLOW 1
UPDATE_COMMAND ""
CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${STAGED_INSTALL_PREFIX}
-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
-DCMAKE_CXX_STANDARD=${CMAKE_CXX_STANDARD}
-DCMAKE_CXX_FLAGS:STRING=${CMAKE_CXX_FLAGS}
USES_TERMINAL_BUILD TRUE
DOWNLOAD_NO_PROGRESS TRUE
LOG_CONFIGURE TRUE
LOG_BUILD TRUE
LOG_INSTALL TRUE)
ExternalProject_Add(
gRPC
DEPENDS Abseil RE2
GIT_REPOSITORY https://github.com/grpc/grpc.git
GIT_TAG ${GRPC_GIT_TAG}
GIT_SHALLOW 1
UPDATE_COMMAND ""
CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${STAGED_INSTALL_PREFIX}
-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
-DgRPC_SSL_PROVIDER=package
-DgRPC_RE2_PROVIDER=package
-DgRPC_ABSEIL_PROVIDER=package
-DOPENSSL_ROOT_DIR=OpenSSL
-DgRPC_BUILD_TESTS=OFF
-DBUILD_SHARED_LIBS=OFF
-DgRPC_INSTALL=ON
-DgRPC_BUILD_GRPC_NODE_PLUGIN=OFF
-DgRPC_BUILD_GRPC_OBJECTIVE_C_PLUGIN=OFF
-DgRPC_BUILD_GRPC_PHP_PLUGIN=OFF
-DgRPC_BUILD_GRPC_PYTHON_PLUGIN=OFF
-DgRPC_BUILD_GRPC_RUBY_PLUGIN=OFF
-DgRPC_BUILD_GRPC_NODE_PLUGIN=OFF
-DgRPC_BUILD_GRPC_OBJECTIVE_C_PLUGIN=OFF
-DgRPC_BUILD_GRPC_PHP_PLUGIN=OFF
-DgRPC_BUILD_GRPC_PYTHON_PLUGIN=OFF
-DgRPC_BUILD_GRPC_RUBY_PLUGIN=OFF
-DgRPC_USE_PROTO_LITE=OFF
-DgRPC_BUILD_CSHARP_EXT=OFF
-DgRPC_BUILD_GRPC_CSHARP_PLUGIN=OFF
-DgRPC_BUILD_GRPC_JAVA_PLUGIN=OFF
-DgRPC_BUILD_GRPC_GRPC_JAVA_PLUGIN=OFF
-DgRPC_BUILD_GRPC_WEB_PLUGIN=OFF
-DgRPC_BUILD_GRPC_C_PLUGIN=OFF
-DgRPC_BUILD_GRPC_NODE_PLUGIN=OFF
-DgRPC_BUILD_GRPC_OBJECTIVE_C_PLUGIN=OFF
-DgRPC_BUILD_GRPC_PHP_PLUGIN=OFF
-DgRPC_BUILD_GRPC_PYTHON_PLUGIN=OFF
-DgRPC_BUILD_GRPC_RUBY_PLUGIN=OFF
-DCMAKE_CXX_STANDARD=${CMAKE_CXX_STANDARD}
CMAKE_CACHE_ARGS -DCMAKE_CXX_FLAGS:STRING=${CMAKE_CXX_FLAGS}
TEST_AFTER_INSTALL FALSE
USES_TERMINAL_BUILD TRUE
DOWNLOAD_NO_PROGRESS TRUE
LOG_CONFIGURE TRUE
LOG_BUILD TRUE
LOG_INSTALL TRUE)
find_package(Protobuf REQUIRED)
find_package(gRPC REQUIRED)
find_package(OpentelemetryCPP REQUIRED)
install(
DIRECTORY ${STAGED_INSTALL_PREFIX}/

View file

@ -26,21 +26,9 @@ RUN apk update \
ENV NINJA_STATUS "[%p/%f/%t] "
# install gRPC
FROM base as grpc
RUN bash /opt/third_party/build.sh -g v1.60.0
# install OpenTelemetry-cpp
FROM base as otel-cpp
COPY --from=grpc /opt/third_party/install/ /usr
RUN bash /opt/third_party/build.sh -o v1.11.0
# install otel_ngx_module.so
FROM base as nginx
ARG NGINX_VERSION=1.25.3
COPY --from=grpc /opt/third_party/install/ /usr
COPY --from=otel-cpp /opt/third_party/install/ /usr
RUN bash /opt/third_party/build.sh -n ${NGINX_VERSION}
FROM golang:1.21.6-bullseye as build-init

View file

@ -17,14 +17,12 @@
set -o errexit
set -o nounset
set -o pipefail
export GRPC_GIT_TAG=${GRPC_GIT_TAG:="v1.43.2"}
set -x
# Check for recent changes: https://github.com/open-telemetry/opentelemetry-cpp/compare/v1.2.0...main
export OPENTELEMETRY_CPP_VERSION=${OPENTELEMETRY_CPP_VERSION:="1.2.0"}
export ABSL_CPP_VERSION=${ABSL_CPP_VERSION:="20230802.0"}
export INSTAL_DIR=/opt/third_party/install
export OPENTELEMETRY_CPP_VERSION=${OPENTELEMETRY_CPP_VERSION:="v1.11.0"}
export INSTALL_DIR=/opt/third_party/install
export NGINX_VERSION=${NGINX_VERSION:-"1.25.3"}
export NGINX_VERSION=${NGINX_VERSION:="1.25.3"}
# improve compilation times
CORES=$(($(grep -c ^processor /proc/cpuinfo) - 1))
@ -43,10 +41,9 @@ Help()
# Display Help
echo "Add description of the script functions here."
echo
echo "Syntax: scriptTemplate [-h|g|o|n|p|]"
echo "Syntax: scriptTemplate [-h|o|n|p|]"
echo "options:"
echo "h Print Help."
echo "g gRPC git tag"
echo "o OpenTelemetry git tag"
echo "n install nginx"
echo "p prepare"
@ -55,6 +52,7 @@ Help()
prepare()
{
echo "http://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories
apk add \
linux-headers \
cmake \
@ -67,48 +65,26 @@ prepare()
pcre-dev \
curl \
git \
build-base
}
build-base \
coreutils \
build-base \
openssl-dev \
pkgconfig \
c-ares-dev \
re2-dev \
grpc-dev \
protobuf-dev \
opentelemetry-cpp-dev
install_grpc()
{
mkdir -p $BUILD_PATH/grpc
cd ${BUILD_PATH}/grpc
cmake -DCMAKE_INSTALL_PREFIX=${INSTAL_DIR} \
-G Ninja \
-DGRPC_GIT_TAG=${GRPC_GIT_TAG} /opt/third_party
cmake --build . -j ${CORES} --target all install --verbose
}
install_absl()
{
cd ${BUILD_PATH}
export LD_LIBRARY_PATH="${LD_LIBRARY_PATH:+LD_LIBRARY_PATH:}${INSTAL_DIR}/lib:/usr/local"
export PATH="${PATH}:${INSTAL_DIR}/bin"
git clone --recurse-submodules -j ${CORES} --depth=1 -b \
${ABSL_CPP_VERSION} https://github.com/abseil/abseil-cpp.git abseil-cpp-${ABSL_CPP_VERSION}
cd "abseil-cpp-${ABSL_CPP_VERSION}"
mkdir -p .build
cd .build
cmake -DCMAKE_BUILD_TYPE=Release \
-G Ninja \
-DCMAKE_CXX_STANDARD=17 \
-DCMAKE_POSITION_INDEPENDENT_CODE=TRUE \
-DBUILD_TESTING=OFF \
-DCMAKE_INSTALL_PREFIX=${INSTAL_DIR} \
-DABSL_PROPAGATE_CXX_STD=ON \
-DBUILD_SHARED_LIBS=OFF \
..
cmake --build . -j ${CORES} --target install
git config --global http.version HTTP/1.1
git config --global http.postBuffer 157286400
}
install_otel()
{
cd ${BUILD_PATH}
export LD_LIBRARY_PATH="${LD_LIBRARY_PATH:+LD_LIBRARY_PATH:}${INSTAL_DIR}/lib:/usr/local"
export PATH="${PATH}:${INSTAL_DIR}/bin"
export LD_LIBRARY_PATH="${LD_LIBRARY_PATH:+LD_LIBRARY_PATH:}${INSTALL_DIR}/lib:/usr/local"
export PATH="${PATH}:${INSTALL_DIR}/bin"
git clone --recurse-submodules -j ${CORES} --depth=1 -b \
${OPENTELEMETRY_CPP_VERSION} https://github.com/open-telemetry/opentelemetry-cpp.git opentelemetry-cpp-${OPENTELEMETRY_CPP_VERSION}
cd "opentelemetry-cpp-${OPENTELEMETRY_CPP_VERSION}"
@ -120,7 +96,7 @@ install_otel()
-DCMAKE_CXX_STANDARD=17 \
-DCMAKE_POSITION_INDEPENDENT_CODE=TRUE \
-DWITH_ZIPKIN=OFF \
-DCMAKE_INSTALL_PREFIX=${INSTAL_DIR} \
-DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \
-DBUILD_TESTING=OFF \
-DWITH_BENCHMARK=OFF \
-DWITH_FUNC_TESTS=OFF \
@ -134,20 +110,6 @@ install_otel()
cmake --build . -j ${CORES} --target install
}
get_src()
{
hash="$1"
url="$2"
f=$(basename "$url")
echo "Downloading $url"
curl -sSL --fail-with-body "$url" -o "$f"
echo "$hash $f" | sha256sum -c - || exit 10
tar xzf "$f"
rm -rf "$f"
}
install_nginx()
{
@ -171,29 +133,21 @@ install_nginx()
cmake -DCMAKE_BUILD_TYPE=Release \
-G Ninja \
-DCMAKE_CXX_STANDARD=17 \
-DCMAKE_INSTALL_PREFIX=${INSTAL_DIR} \
-DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \
-DBUILD_SHARED_LIBS=ON \
-DNGINX_VERSION=${NGINX_VERSION} \
..
cmake --build . -j ${CORES} --target install
mkdir -p /etc/nginx/modules
cp ${INSTAL_DIR}/otel_ngx_module.so /etc/nginx/modules/otel_ngx_module.so
cp ${INSTALL_DIR}/otel_ngx_module.so /etc/nginx/modules/otel_ngx_module.so
}
while getopts ":pha:g:o:n" option; do
while getopts ":phn:" option; do
case $option in
h) # display Help
Help
exit;;
g) # install gRPC with git tag
GRPC_GIT_TAG=${OPTARG}
install_grpc
exit;;
o) # install OpenTelemetry tag
OPENTELEMETRY_CPP_VERSION=${OPTARG}
install_otel
exit;;
p) # prepare
prepare
exit;;
@ -201,10 +155,6 @@ while getopts ":pha:g:o:n" option; do
NGINX_VERSION=${OPTARG}
install_nginx
exit;;
a) # install abseil
ABSL_CPP_VERSION=${OPTARG}
install_absl
exit;;
\?)
Help
exit;;

View file

@ -30,7 +30,7 @@ load_module /etc/nginx/modules/ngx_http_modsecurity_module.so;
{{ end }}
{{ if (shouldLoadOpentelemetryModule $cfg $servers) }}
load_module /modules_mount/etc/nginx/modules/otel/otel_ngx_module.so;
load_module /etc/nginx/modules/otel_ngx_module.so;
{{ end }}
daemon off;

View file

@ -17,7 +17,6 @@ limitations under the License.
package settings
import (
"os"
"strings"
"github.com/onsi/ginkgo/v2"
@ -40,11 +39,11 @@ const (
var _ = framework.IngressNginxDescribe("Configure Opentelemetry", func() {
f := framework.NewDefaultFramework("enable-opentelemetry")
shouldSkip := false
/*shouldSkip := false
skip, ok := os.LookupEnv("SKIP_OPENTELEMETRY_TESTS")
if ok && skip == enable {
shouldSkip = true
}
}*/
ginkgo.BeforeEach(func() {
f.NewEchoDeployment()
@ -54,9 +53,9 @@ var _ = framework.IngressNginxDescribe("Configure Opentelemetry", func() {
})
ginkgo.It("should not exists opentelemetry directive", func() {
if shouldSkip {
/*if shouldSkip {
ginkgo.Skip("skipped")
}
}*/
config := map[string]string{}
config[enableOpentelemetry] = disable
f.SetNginxConfigMapData(config)
@ -70,9 +69,9 @@ var _ = framework.IngressNginxDescribe("Configure Opentelemetry", func() {
})
ginkgo.It("should exists opentelemetry directive when is enabled", func() {
if shouldSkip {
/*if shouldSkip {
ginkgo.Skip("skipped")
}
}*/
config := map[string]string{}
config[enableOpentelemetry] = enable
config[opentelemetryConfig] = opentelemetryConfigPath
@ -87,9 +86,9 @@ var _ = framework.IngressNginxDescribe("Configure Opentelemetry", func() {
})
ginkgo.It("should include opentelemetry_trust_incoming_spans on directive when enabled", func() {
if shouldSkip {
/*if shouldSkip {
ginkgo.Skip("skipped")
}
}*/
config := map[string]string{}
config[enableOpentelemetry] = enable
config[opentelemetryConfig] = opentelemetryConfigPath
@ -105,9 +104,9 @@ var _ = framework.IngressNginxDescribe("Configure Opentelemetry", func() {
})
ginkgo.It("should not exists opentelemetry_operation_name directive when is empty", func() {
if shouldSkip {
/*if shouldSkip {
ginkgo.Skip("skipped")
}
}*/
config := map[string]string{}
config[enableOpentelemetry] = enable
config[opentelemetryConfig] = opentelemetryConfigPath
@ -123,9 +122,9 @@ var _ = framework.IngressNginxDescribe("Configure Opentelemetry", func() {
})
ginkgo.It("should exists opentelemetry_operation_name directive when is configured", func() {
if shouldSkip {
/*if shouldSkip {
ginkgo.Skip("skipped")
}
}*/
config := map[string]string{}
config[enableOpentelemetry] = enable
config[opentelemetryConfig] = opentelemetryConfigPath

View file

@ -48,16 +48,6 @@ metadata:
EOF
OTEL_MODULE=$(cat <<EOF
opentelemetry:
enabled: true
EOF
)
if [[ "$NAMESPACE_OVERLAY" != "enable-opentelemetry" ]]; then
OTEL_MODULE=""
fi
# Use the namespace overlay if it was requested
if [[ ! -z "$NAMESPACE_OVERLAY" && -d "$DIR/namespace-overlays/$NAMESPACE_OVERLAY" ]]; then
echo "Namespace overlay $NAMESPACE_OVERLAY is being used for namespace $NAMESPACE"