diff --git a/docs/images/otel-grafana-demo.png b/docs/images/otel-grafana-demo.png new file mode 100644 index 000000000..61b50c861 Binary files /dev/null and b/docs/images/otel-grafana-demo.png differ diff --git a/docs/images/otel-jaeger-demo.png b/docs/images/otel-jaeger-demo.png new file mode 100644 index 000000000..006b23c34 Binary files /dev/null and b/docs/images/otel-jaeger-demo.png differ diff --git a/docs/images/otel-zipkin-demo.png b/docs/images/otel-zipkin-demo.png new file mode 100644 index 000000000..a3af04e46 Binary files /dev/null and b/docs/images/otel-zipkin-demo.png differ diff --git a/docs/kubectl-plugin.md b/docs/kubectl-plugin.md index 01be19f59..2b6381f6f 100644 --- a/docs/kubectl-plugin.md +++ b/docs/kubectl-plugin.md @@ -208,6 +208,7 @@ modsecurity modules nginx.conf opentracing.json +opentelemetry.toml owasp-modsecurity-crs template ``` diff --git a/docs/user-guide/nginx-configuration/annotations.md b/docs/user-guide/nginx-configuration/annotations.md index 9c79d9d9c..dac39f752 100755 --- a/docs/user-guide/nginx-configuration/annotations.md +++ b/docs/user-guide/nginx-configuration/annotations.md @@ -121,6 +121,8 @@ You can add these Kubernetes annotations to specific Ingress objects to customiz |[nginx.ingress.kubernetes.io/enable-access-log](#enable-access-log)|"true" or "false"| |[nginx.ingress.kubernetes.io/enable-opentracing](#enable-opentracing)|"true" or "false"| |[nginx.ingress.kubernetes.io/opentracing-trust-incoming-span](#opentracing-trust-incoming-span)|"true" or "false"| +|[nginx.ingress.kubernetes.io/enable-opentelemetry](#enable-opentelemetry)|"true" or "false"| +|[nginx.ingress.kubernetes.io/opentelemetry-trust-incoming-span](#opentelemetry-trust-incoming-spans)|"true" or "false"| |[nginx.ingress.kubernetes.io/enable-influxdb](#influxdb)|"true" or "false"| |[nginx.ingress.kubernetes.io/influxdb-measurement](#influxdb)|string| |[nginx.ingress.kubernetes.io/influxdb-port](#influxdb)|string| @@ -821,6 +823,24 @@ sometimes need to be overridden to enable it or disable it for a specific ingres nginx.ingress.kubernetes.io/opentracing-trust-incoming-span: "true" ``` +### Enable Opentelemetry + +Opentelemetry can be enabled or disabled globally through the ConfigMap but this will sometimes need to be overridden +to enable it or disable it for a specific ingress (e.g. to turn off telemetry of external health check endpoints) + +```yaml +nginx.ingress.kubernetes.io/enable-opentelemetry: "true" +``` + +### Opentelemetry Trust Incoming Span + +The option to trust incoming trace spans can be enabled or disabled globally through the ConfigMap but this will +sometimes need to be overridden to enable it or disable it for a specific ingress (e.g. only enable on a private endpoint) + +```yaml +nginx.ingress.kubernetes.io/opentelemetry-trust-incoming-spans: "true" +``` + ### X-Forwarded-Prefix Header To add the non-standard `X-Forwarded-Prefix` header to the upstream request with a string value, the following annotation can be used: diff --git a/docs/user-guide/nginx-configuration/configmap.md b/docs/user-guide/nginx-configuration/configmap.md old mode 100755 new mode 100644 index fc5bd07c2..ccb115a0f --- a/docs/user-guide/nginx-configuration/configmap.md +++ b/docs/user-guide/nginx-configuration/configmap.md @@ -157,6 +157,19 @@ The following table shows a configuration option's name, type, and the default v |[datadog-operation-name-override](#datadog-operation-name-override)|string|"nginx.handle"| |[datadog-priority-sampling](#datadog-priority-sampling)|bool|"true"| |[datadog-sample-rate](#datadog-sample-rate)|float|1.0| +|[enable-opentelemetry](#enable-opentelemetry)|bool|"false"| +|[opentelemetry-trust-incoming-span](#opentelemetry-trust-incoming-span)|bool|"true"| +|[opentelemetry-operation-name](#opentelemetry-operation-name)|string|""| +|[opentelemetry-config](#/etc/nginx/opentelemetry.toml)|string|"/etc/nginx/opentelemetry.toml"| +|[otlp-collector-host](#otlp-collector-host)|string|""| +|[otlp-collector-port](#otlp-collector-port)|int|4317| +|[otel-max-queuesize](#otel-max-queuesize)|int|| +|[otel-schedule-delay-millis](#otel-schedule-delay-millis)|int|| +|[otel-max-export-batch-size](#otel-max-export-batch-size)|int|| +|[otel-service-name](#otel-service-name)|string|"nginx"| +|[otel-sampler](#otel-sampler)|string|"AlwaysOff"| +|[otel-sampler-parent-based](#otel-sampler-parent-based)|bool|"false"| +|[otel-sampler-ratio](#otel-sampler-ratio)|float|0.01| |[main-snippet](#main-snippet)|string|""| |[http-snippet](#http-snippet)|string|""| |[server-snippet](#server-snippet)|string|""| @@ -1009,6 +1022,46 @@ If true disables client-side sampling (thus ignoring `sample_rate`) and enables Specifies sample rate for any traces created. This is effective only when `datadog-priority-sampling` is `false` _**default:**_ 1.0 +## enable-opentelemetry + +Enables the nginx OpenTelemetry extension. _**default:**_ is disabled + +_References:_ +[https://github.com/open-telemetry/opentelemetry-cpp-contrib](https://github.com/open-telemetry/opentelemetry-cpp-contrib/tree/main/instrumentation/nginx) + +## opentelemetry-operation-name + +Specifies a custom name for the server span. _**default:**_ is empty + +For example, set to "HTTP $request_method $uri". + +## otlp-collector-host + +Specifies the host to use when uploading traces. It must be a valid URL. + +## otlp-collector-port + +Specifies the port to use when uploading traces. _**default:**_ 4317 + +## otel-service-name + +Specifies the service name to use for any traces created. _**default:**_ nginx + +## opentelemetry-trust-incoming-span: "true" +Enables or disables using spans from incoming requests as parent for created ones. _**default:**_ true + +## otel-sampler-parent-based + +Uses sampler implementation which by default will take a sample if parent Activity is sampled. _**default:**_ false + +## otel-sampler-ratio + +Specifies sample rate for any traces created. _**default:**_ 0.01 + +## otel-sampler + +Specifies the sampler to be used when sampling traces. The available samplers are: AlwaysOff, AlwaysOn, TraceIdRatioBased, remote. _**default:**_ AlwaysOff + ## main-snippet Adds custom configuration to the main section of the nginx configuration. diff --git a/docs/user-guide/third-party-addons/opentelemetry.md b/docs/user-guide/third-party-addons/opentelemetry.md new file mode 100644 index 000000000..d966dff1f --- /dev/null +++ b/docs/user-guide/third-party-addons/opentelemetry.md @@ -0,0 +1,260 @@ +# OpenTelemetry + +Enables requests served by NGINX for distributed telemetry via The OpenTelemetry Project. + +Using the third party module [opentelemetry-cpp-contrib/nginx](https://github.com/open-telemetry/opentelemetry-cpp-contrib/tree/main/instrumentation/nginx) the NGINX ingress controller can configure NGINX to enable [OpenTelemetry](http://opentelemetry.io) instrumentation. +By default this feature is disabled. + +## Usage + +To enable the instrumentation we must enable OpenTelemetry in the configuration ConfigMap: +```yaml +data: + enable-opentelemetry: "true" +``` + +To enable or disable instrumentation for a single Ingress, use +the `enable-opentelemetry` annotation: +```yaml +kind: Ingress +metadata: + annotations: + nginx.ingress.kubernetes.io/enable-opentelemetry: "true" +``` + +We must also set the host to use when uploading traces: + +```yaml +otlp-collector-host: "otel-coll-collector.otel.svc" +``` +NOTE: While the option is called `otlp-collector-host`, you will need to point this to any backend that recieves otlp-grpc. + +Next you will need to deploy a distributed telemetry system which uses OpenTelemetry. +[opentelemetry-collector](https://github.com/open-telemetry/opentelemetry-collector), [Jaeger](https://www.jaegertracing.io/) +[Tempo](https://github.com/grafana/tempo), and [zipkin](https://zipkin.io/) +have been tested. + +Other optional configuration options: +```yaml +# specifies the name to use for the server span +opentelemetry-operation-name + +# sets whether or not to trust incoming telemetry spans +opentelemetry-trust-incoming-span + +# specifies the port to use when uploading traces, Default: 4317 +otlp-collector-port + +# specifies the service name to use for any traces created, Default: nginx +otel-service-name + +# The maximum queue size. After the size is reached data are dropped. +otel-max-queuesize + +# The delay interval in milliseconds between two consecutive exports. +otel-schedule-delay-millis + +# How long the export can run before it is cancelled. +otel-schedule-delay-millis + +# The maximum batch size of every export. It must be smaller or equal to maxQueueSize. +otel-max-export-batch-size + +# specifies sample rate for any traces created, Default: 0.01 +otel-sampler-ratio + +# specifies the sampler to be used when sampling traces. +# The available samplers are: AlwaysOn, AlwaysOff, TraceIdRatioBased, Default: AlwaysOff +otel-sampler + +# Uses sampler implementation which by default will take a sample if parent Activity is sampled, Default: false +otel-sampler-parent-based +``` + +Note that you can also set whether to trust incoming spans (global default is true) per-location using annotations like the following: +```yaml +kind: Ingress +metadata: + annotations: + nginx.ingress.kubernetes.io/opentelemetry-trust-incoming-span: "true" +``` + +## Examples + +The following examples show how to deploy and test different distributed telemetry systems. These example can be performed using Docker Desktop. + +In the [esigo/nginx-example](https://github.com/esigo/nginx-example) +GitHub repository is an example of a simple hello service: + +```mermaid +graph TB + subgraph Browser + start["http://esigo.dev/hello/nginx"] + end + + subgraph app + sa[service-a] + sb[service-b] + sa --> |name: nginx| sb + sb --> |hello nginx!| sa + end + + subgraph otel + otc["Otel Collector"] + end + + subgraph observability + tempo["Tempo"] + grafana["Grafana"] + backend["Jaeger"] + zipkin["Zipkin"] + end + + subgraph ingress-nginx + ngx[nginx] + end + + subgraph ngx[nginx] + ng[nginx] + om[OpenTelemetry module] + end + + subgraph Node + app + otel + observability + ingress-nginx + om --> |otlp-gRPC| otc --> |jaeger| backend + otc --> |zipkin| zipkin + otc --> |otlp-gRPC| tempo --> grafana + sa --> |otlp-gRPC| otc + sb --> |otlp-gRPC| otc + start --> ng --> sa + end +``` + +To install the example and collectors run: + +1. Enable Ingress addon with: + + ```yaml + opentelemetry: + enabled: true + image: registry.k8s.io/ingress-nginx/opentelemetry:v20230107-helm-chart-4.4.2-2-g96b3d2165@sha256:331b9bebd6acfcd2d3048abbdd86555f5be76b7e3d0b5af4300b04235c6056c9 + containerSecurityContext: + allowPrivilegeEscalation: false + ``` + +2. Enable OpenTelemetry and set the otlp-collector-host: + + ```yaml + $ echo ' + apiVersion: v1 + kind: ConfigMap + data: + enable-opentelemetry: "true" + opentelemetry-config: "/etc/nginx/opentelemetry.toml" + opentelemetry-operation-name: "HTTP $request_method $service_name $uri" + opentelemetry-trust-incoming-span: "true" + otlp-collector-host: "otel-coll-collector.otel.svc" + otlp-collector-port: "4317" + otel-max-queuesize: "2048" + otel-schedule-delay-millis: "5000" + otel-max-export-batch-size: "512" + otel-service-name: "nginx-proxy" # Opentelemetry resource name + otel-sampler: "AlwaysOn" # Also: AlwaysOff, TraceIdRatioBased + otel-sampler-ratio: "1.0" + otel-sampler-parent-based: "false" + metadata: + name: ingress-nginx-controller + namespace: ingress-nginx + ' | kubectl replace -f - + ``` + +4. Deploy otel-collector, grafana and Jaeger backend: + + ```bash + # add helm charts needed for grafana and OpenTelemetry collector + helm repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts + helm repo add grafana https://grafana.github.io/helm-charts + helm repo update + # deply cert-manager needed for OpenTelemetry collector operator + kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.9.1/cert-manager.yaml + # create observability namespace + kubectl apply -f https://raw.githubusercontent.com/esigo/nginx-example/main/observability/namespace.yaml + # install OpenTelemetry collector operator + helm upgrade --install otel-collector-operator -n otel --create-namespace open-telemetry/opentelemetry-operator + # deploy OpenTelemetry collector + kubectl apply -f https://raw.githubusercontent.com/esigo/nginx-example/main/observability/collector.yaml + # deploy Jaeger all-in-one + kubectl apply -f https://github.com/jaegertracing/jaeger-operator/releases/download/v1.37.0/jaeger-operator.yaml -n observability + kubectl apply -f https://raw.githubusercontent.com/esigo/nginx-example/main/observability/jaeger.yaml -n observability + # deploy zipkin + kubectl apply -f https://raw.githubusercontent.com/esigo/nginx-example/main/observability/zipkin.yaml -n observability + # deploy tempo and grafana + helm upgrade --install tempo grafana/tempo --create-namespace -n observability + helm upgrade -f https://raw.githubusercontent.com/esigo/nginx-example/main/observability/grafana/grafana-values.yaml --install grafana grafana/grafana --create-namespace -n observability + ``` + +3. Build and deploy demo app: + + ```bash + # build images + make images + + # deploy demo app: + make deploy-app + ``` + +5. Make a few requests to the Service: + + ```bash + kubectl port-forward --namespace=ingress-nginx service/ingress-nginx-controller 8090:80 + curl http://esigo.dev:8090/hello/nginx + + + StatusCode : 200 + StatusDescription : OK + Content : {"v":"hello nginx!"} + + RawContent : HTTP/1.1 200 OK + Connection: keep-alive + Content-Length: 21 + Content-Type: text/plain; charset=utf-8 + Date: Mon, 10 Oct 2022 17:43:33 GMT + + {"v":"hello nginx!"} + + Forms : {} + Headers : {[Connection, keep-alive], [Content-Length, 21], [Content-Type, text/plain; charset=utf-8], [Date, + Mon, 10 Oct 2022 17:43:33 GMT]} + Images : {} + InputFields : {} + Links : {} + ParsedHtml : System.__ComObject + RawContentLength : 21 + ``` + +6. View the Grafana UI: + + ```bash + kubectl port-forward --namespace=observability service/grafana 3000:80 + ``` + In the Grafana interface we can see the details: + ![grafana screenshot](../../images/otel-grafana-demo.png "grafana screenshot") + +7. View the Jaeger UI: + + ```bash + kubectl port-forward --namespace=observability service/jaeger-all-in-one-query 16686:16686 + ``` + In the Jaeger interface we can see the details: + ![Jaeger screenshot](../../images/otel-jaeger-demo.png "Jaeger screenshot") + +8. View the Zipkin UI: + + ```bash + kubectl port-forward --namespace=observability service/zipkin 9411:9411 + ``` + In the Zipkin interface we can see the details: + ![zipkin screenshot](../../images/otel-zipkin-demo.png "zipkin screenshot") diff --git a/internal/ingress/annotations/annotations.go b/internal/ingress/annotations/annotations.go index 14415a4f7..6a9e7367c 100644 --- a/internal/ingress/annotations/annotations.go +++ b/internal/ingress/annotations/annotations.go @@ -20,6 +20,7 @@ import ( "github.com/imdario/mergo" "k8s.io/ingress-nginx/internal/ingress/annotations/canary" "k8s.io/ingress-nginx/internal/ingress/annotations/modsecurity" + "k8s.io/ingress-nginx/internal/ingress/annotations/opentelemetry" "k8s.io/ingress-nginx/internal/ingress/annotations/proxyssl" "k8s.io/ingress-nginx/internal/ingress/annotations/sslcipher" "k8s.io/ingress-nginx/internal/ingress/annotations/streamsnippet" @@ -94,6 +95,7 @@ type Ingress struct { EnableGlobalAuth bool HTTP2PushPreload bool Opentracing opentracing.Config + Opentelemetry opentelemetry.Config Proxy proxy.Config ProxySSL proxyssl.Config RateLimit ratelimit.Config @@ -145,6 +147,7 @@ func NewAnnotationExtractor(cfg resolver.Resolver) Extractor { "EnableGlobalAuth": authreqglobal.NewParser(cfg), "HTTP2PushPreload": http2pushpreload.NewParser(cfg), "Opentracing": opentracing.NewParser(cfg), + "Opentelemetry": opentelemetry.NewParser(cfg), "Proxy": proxy.NewParser(cfg), "ProxySSL": proxyssl.NewParser(cfg), "RateLimit": ratelimit.NewParser(cfg), diff --git a/internal/ingress/annotations/opentelemetry/main.go b/internal/ingress/annotations/opentelemetry/main.go new file mode 100644 index 000000000..7dd292322 --- /dev/null +++ b/internal/ingress/annotations/opentelemetry/main.go @@ -0,0 +1,101 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package opentelemetry + +import ( + networking "k8s.io/api/networking/v1" + + "k8s.io/ingress-nginx/internal/ingress/annotations/parser" + "k8s.io/ingress-nginx/internal/ingress/resolver" +) + +type opentelemetry struct { + r resolver.Resolver +} + +// Config contains the configuration to be used in the Ingress +type Config struct { + Enabled bool `json:"enabled"` + Set bool `json:"set"` + TrustEnabled bool `json:"trust-enabled"` + TrustSet bool `json:"trust-set"` + OperationName string `json:"operation-name"` +} + +// Equal tests for equality between two Config types +func (bd1 *Config) Equal(bd2 *Config) bool { + + if bd1.Set != bd2.Set { + return false + } + + if bd1.Enabled != bd2.Enabled { + return false + } + + if bd1.TrustSet != bd2.TrustSet { + return false + } + + if bd1.TrustEnabled != bd2.TrustEnabled { + return false + } + + if bd1.OperationName != bd2.OperationName { + return false + } + + return true +} + +// NewParser creates a new serviceUpstream annotation parser +func NewParser(r resolver.Resolver) parser.IngressAnnotation { + return opentelemetry{r} +} + +// Parse parses the annotations to look for opentelemetry configurations +func (c opentelemetry) Parse(ing *networking.Ingress) (interface{}, error) { + cfg := Config{} + enabled, err := parser.GetBoolAnnotation("enable-opentelemetry", ing) + if err != nil { + return &cfg, nil + } + cfg.Set = true + cfg.Enabled = enabled + if !enabled { + return &cfg, nil + } + + trustEnabled, err := parser.GetBoolAnnotation("opentelemetry-trust-incoming-span", ing) + if err != nil { + operationName, err := parser.GetStringAnnotation("opentelemetry-operation-name", ing) + if err != nil { + return &cfg, nil + } + cfg.OperationName = operationName + return &cfg, nil + } + + cfg.TrustSet = true + cfg.TrustEnabled = trustEnabled + operationName, err := parser.GetStringAnnotation("opentelemetry-operation-name", ing) + if err != nil { + return &cfg, nil + } + cfg.OperationName = operationName + return &cfg, nil +} diff --git a/internal/ingress/annotations/opentelemetry/main_test.go b/internal/ingress/annotations/opentelemetry/main_test.go new file mode 100644 index 000000000..619108fa6 --- /dev/null +++ b/internal/ingress/annotations/opentelemetry/main_test.go @@ -0,0 +1,170 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package opentelemetry + +import ( + "testing" + + api "k8s.io/api/core/v1" + networking "k8s.io/api/networking/v1" + meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/ingress-nginx/internal/ingress/annotations/parser" + "k8s.io/ingress-nginx/internal/ingress/resolver" +) + +func buildIngress() *networking.Ingress { + defaultBackend := networking.IngressBackend{ + Service: &networking.IngressServiceBackend{ + Name: "default-backend", + Port: networking.ServiceBackendPort{ + Number: 80, + }, + }, + } + + return &networking.Ingress{ + ObjectMeta: meta_v1.ObjectMeta{ + Name: "foo", + Namespace: api.NamespaceDefault, + }, + Spec: networking.IngressSpec{ + DefaultBackend: &networking.IngressBackend{ + Service: &networking.IngressServiceBackend{ + Name: "default-backend", + Port: networking.ServiceBackendPort{ + Number: 80, + }, + }, + }, + Rules: []networking.IngressRule{ + { + Host: "foo.bar.com", + IngressRuleValue: networking.IngressRuleValue{ + HTTP: &networking.HTTPIngressRuleValue{ + Paths: []networking.HTTPIngressPath{ + { + Path: "/foo", + Backend: defaultBackend, + }, + }, + }, + }, + }, + }, + }, + } +} + +func TestIngressAnnotationOpentelemetrySetTrue(t *testing.T) { + ing := buildIngress() + + data := map[string]string{} + data[parser.GetAnnotationWithPrefix("enable-opentelemetry")] = "true" + ing.SetAnnotations(data) + + val, _ := NewParser(&resolver.Mock{}).Parse(ing) + openTelemetry, ok := val.(*Config) + if !ok { + t.Errorf("expected a Config type") + } + + if !openTelemetry.Enabled { + t.Errorf("expected annotation value to be true, got false") + } + + if !openTelemetry.Set { + t.Errorf("expected annotation value to be true, got false") + } + + if openTelemetry.TrustSet { + t.Errorf("expected annotation value to be false, got true") + } +} + +func TestIngressAnnotationOpentelemetrySetFalse(t *testing.T) { + ing := buildIngress() + + // Test with explicitly set to false + data := map[string]string{} + data[parser.GetAnnotationWithPrefix("enable-opentelemetry")] = "false" + ing.SetAnnotations(data) + + val, _ := NewParser(&resolver.Mock{}).Parse(ing) + openTelemetry, ok := val.(*Config) + if !ok { + t.Errorf("expected a Config type") + } + + if openTelemetry.Enabled { + t.Errorf("expected annotation value to be false, got true") + } + + if !openTelemetry.Set { + t.Errorf("expected annotation value to be true, got false") + } +} + +func TestIngressAnnotationOpentelemetryTrustSetTrue(t *testing.T) { + ing := buildIngress() + + data := map[string]string{} + opName := "foo-op" + data[parser.GetAnnotationWithPrefix("enable-opentelemetry")] = "true" + data[parser.GetAnnotationWithPrefix("opentelemetry-trust-incoming-span")] = "true" + data[parser.GetAnnotationWithPrefix("opentelemetry-operation-name")] = opName + ing.SetAnnotations(data) + + val, _ := NewParser(&resolver.Mock{}).Parse(ing) + openTelemetry, ok := val.(*Config) + if !ok { + t.Errorf("expected a Config type") + } + + if !openTelemetry.Enabled { + t.Errorf("expected annotation value to be true, got false") + } + + if !openTelemetry.Set { + t.Errorf("expected annotation value to be true, got false") + } + + if !openTelemetry.TrustEnabled { + t.Errorf("expected annotation value to be true, got false") + } + + if !openTelemetry.TrustSet { + t.Errorf("expected annotation value to be true, got false") + } + + if openTelemetry.OperationName != opName { + t.Errorf("expected annotation value to be %v, got %v", opName, openTelemetry.OperationName) + } +} + +func TestIngressAnnotationOpentelemetryUnset(t *testing.T) { + ing := buildIngress() + + // Test with no annotation specified + data := map[string]string{} + ing.SetAnnotations(data) + + val, _ := NewParser(&resolver.Mock{}).Parse(ing) + _, ok := val.(*Config) + if !ok { + t.Errorf("expected a Config type") + } +} diff --git a/internal/ingress/controller/config/config.go b/internal/ingress/controller/config/config.go index 5786b6776..da37ea3c9 100644 --- a/internal/ingress/controller/config/config.go +++ b/internal/ingress/controller/config/config.go @@ -568,6 +568,54 @@ type Configuration struct { // Default: true OpentracingTrustIncomingSpan bool `json:"opentracing-trust-incoming-span"` + // EnableOpentelemetry enables the nginx Opentelemetry extension + // By default this is disabled + EnableOpentelemetry bool `json:"enable-opentelemetry"` + + // OpentelemetryConfig sets the opentelemetry config file + // Default: /etc/nginx/opentelemetry.toml + OpentelemetryConfig string `json:"opentelemetry-config"` + + // OpentelemetryOperationName specifies a custom name for the server span + OpentelemetryOperationName string `json:"opentelemetry-operation-name"` + + // OpentelemetryTrustIncomingSpan sets whether or not to trust incoming trace spans + // If false, incoming span headers will be rejected + // Default: true + OpentelemetryTrustIncomingSpan bool `json:"opentelemetry-trust-incoming-span"` + + // OtlpCollectorHost specifies the host to use when uploading traces + OtlpCollectorHost string `json:"otlp-collector-host"` + + // OtlpCollectorPort specifies the port to use when uploading traces + // Default: 4317 + OtlpCollectorPort string `json:"otlp-collector-port"` + + // OtelServiceName specifies the service name to use for any traces created + // Default: nginx + OtelServiceName string `json:"otel-service-name"` + + // OtelSampler specifies the sampler to use for any traces created + // Default: AlwaysOff + OtelSampler string `json:"otel-sampler"` + + // OtelSamplerRatio specifies the sampler ratio to use for any traces created + // Default: 0.01 + OtelSamplerRatio float32 `json:"otel-sampler-ratio"` + + //OtelSamplerParentBased specifies the parent based sampler to be use for any traces created + // Default: false + OtelSamplerParentBased bool `json:"otel-sampler-parent-based"` + + // MaxQueueSize specifies the max queue size for uploading traces + OtelMaxQueueSize int32 `json:"otel-max-queuesize"` + + // ScheduleDelayMillis specifies the max delay between uploading traces + OtelScheduleDelayMillis int32 `json:"otel-schedule-delay-millis"` + + // MaxExportBatchSize specifies the max export batch size to used when uploading traces + OtelMaxExportBatchSize int32 `json:"otel-max-export-batch-size"` + // ZipkinCollectorHost specifies the host to use when uploading traces ZipkinCollectorHost string `json:"zipkin-collector-host"` @@ -917,6 +965,13 @@ func NewDefault() Configuration { BindAddressIpv4: defBindAddress, BindAddressIpv6: defBindAddress, OpentracingTrustIncomingSpan: true, + OpentelemetryTrustIncomingSpan: true, + OpentelemetryConfig: "/etc/nginx/opentelemetry.toml", + OtlpCollectorPort: "4317", + OtelServiceName: "nginx", + OtelSampler: "AlwaysOff", + OtelSamplerRatio: 0.01, + OtelSamplerParentBased: false, ZipkinCollectorPort: 9411, ZipkinServiceName: "nginx", ZipkinSampleRate: 1.0, diff --git a/internal/ingress/controller/controller.go b/internal/ingress/controller/controller.go index 20f8c8687..5d94605b5 100644 --- a/internal/ingress/controller/controller.go +++ b/internal/ingress/controller/controller.go @@ -1444,6 +1444,7 @@ func locationApplyAnnotations(loc *ingress.Location, anns *annotations.Ingress) loc.EnableGlobalAuth = anns.EnableGlobalAuth loc.HTTP2PushPreload = anns.HTTP2PushPreload loc.Opentracing = anns.Opentracing + loc.Opentelemetry = anns.Opentelemetry loc.Proxy = anns.Proxy loc.ProxySSL = anns.ProxySSL loc.RateLimit = anns.RateLimit diff --git a/internal/ingress/controller/nginx.go b/internal/ingress/controller/nginx.go index 5575009ea..80693db5c 100644 --- a/internal/ingress/controller/nginx.go +++ b/internal/ingress/controller/nginx.go @@ -673,6 +673,11 @@ func (n *NGINXController) OnUpdate(ingressCfg ingress.Configuration) error { return err } + err = createOpentelemetryCfg(cfg) + if err != nil { + return err + } + err = n.testTemplate(content) if err != nil { return err @@ -1056,6 +1061,29 @@ const datadogTmpl = `{ "dd.priority.sampling": {{ .DatadogPrioritySampling }} }` +const otelTmpl = ` +exporter = "otlp" +processor = "batch" + +[exporters.otlp] +# Alternatively the OTEL_EXPORTER_OTLP_ENDPOINT environment variable can also be used. +host = "{{ .OtlpCollectorHost }}" +port = {{ .OtlpCollectorPort }} + +[processors.batch] +max_queue_size = {{ .OtelMaxQueueSize }} +schedule_delay_millis = {{ .OtelScheduleDelayMillis }} +max_export_batch_size = {{ .OtelMaxExportBatchSize }} + +[service] +name = "{{ .OtelServiceName }}" # Opentelemetry resource name + +[sampler] +name = "{{ .OtelSampler }}" # Also: AlwaysOff, TraceIdRatioBased +ratio = {{ .OtelSamplerRatio }} +parent_based = {{ .OtelSamplerParentBased }} +` + func createOpentracingCfg(cfg ngx_config.Configuration) error { var tmpl *template.Template var err error @@ -1091,6 +1119,21 @@ func createOpentracingCfg(cfg ngx_config.Configuration) error { return os.WriteFile("/etc/nginx/opentracing.json", []byte(expanded), file.ReadWriteByUser) } +func createOpentelemetryCfg(cfg ngx_config.Configuration) error { + + tmpl, err := template.New("otel").Parse(otelTmpl) + if err != nil { + return err + } + tmplBuf := bytes.NewBuffer(make([]byte, 0)) + err = tmpl.Execute(tmplBuf, cfg) + if err != nil { + return err + } + + return os.WriteFile(cfg.OpentelemetryConfig, tmplBuf.Bytes(), file.ReadWriteByUser) +} + func cleanTempNginxCfg() error { var files []string diff --git a/internal/ingress/controller/template/template.go b/internal/ingress/controller/template/template.go index f3283a782..9dc9fb3b8 100644 --- a/internal/ingress/controller/template/template.go +++ b/internal/ingress/controller/template/template.go @@ -265,6 +265,7 @@ var ( "buildAuthSignURL": buildAuthSignURL, "buildAuthSignURLLocation": buildAuthSignURLLocation, "buildOpentracing": buildOpentracing, + "buildOpentelemetry": buildOpentelemetry, "proxySetHeader": proxySetHeader, "buildInfluxDB": buildInfluxDB, "enforceRegexModifier": enforceRegexModifier, @@ -274,7 +275,9 @@ var ( "buildHTTPListener": buildHTTPListener, "buildHTTPSListener": buildHTTPSListener, "buildOpentracingForLocation": buildOpentracingForLocation, + "buildOpentelemetryForLocation": buildOpentelemetryForLocation, "shouldLoadOpentracingModule": shouldLoadOpentracingModule, + "shouldLoadOpentelemetryModule": shouldLoadOpentelemetryModule, "buildModSecurityForLocation": buildModSecurityForLocation, "buildMirrorLocations": buildMirrorLocations, "shouldLoadAuthDigestModule": shouldLoadAuthDigestModule, @@ -1239,6 +1242,33 @@ func buildOpentracing(c interface{}, s interface{}) string { return buf.String() } +func buildOpentelemetry(c interface{}, s interface{}) string { + cfg, ok := c.(config.Configuration) + if !ok { + klog.Errorf("expected a 'config.Configuration' type but %T was returned", c) + return "" + } + + servers, ok := s.([]*ingress.Server) + if !ok { + klog.Errorf("expected an '[]*ingress.Server' type but %T was returned", s) + return "" + } + + if !shouldLoadOpentelemetryModule(cfg, servers) { + return "" + } + + buf := bytes.NewBufferString("") + + buf.WriteString("\r\n") + + if cfg.OpentelemetryOperationName != "" { + buf.WriteString(fmt.Sprintf("opentelemetry_operation_name \"%s\";\n", cfg.OpentelemetryOperationName)) + } + return buf.String() +} + // buildInfluxDB produces the single line configuration // needed by the InfluxDB module to send request's metrics // for the current resource @@ -1360,6 +1390,13 @@ func opentracingPropagateContext(location *ingress.Location) string { return "opentracing_propagate_context;" } +func opentelemetryPropagateContext(location *ingress.Location) string { + if location == nil { + return "" + } + return "opentelemetry_propagate;" +} + // shouldLoadModSecurityModule determines whether or not the ModSecurity module needs to be loaded. // First, it checks if `enable-modsecurity` is set in the ConfigMap. If it is not, it iterates over all locations to // check if ModSecurity is enabled by the annotation `nginx.ingress.kubernetes.io/enable-modsecurity`. @@ -1575,6 +1612,36 @@ func buildOpentracingForLocation(isOTEnabled bool, isOTTrustSet bool, location * return opc } +func buildOpentelemetryForLocation(isOTEnabled bool, isOTTrustSet bool, location *ingress.Location) string { + isOTEnabledInLoc := location.Opentelemetry.Enabled + isOTSetInLoc := location.Opentelemetry.Set + + if isOTEnabled { + if isOTSetInLoc && !isOTEnabledInLoc { + return "opentelemetry off;" + } + } else if !isOTSetInLoc || !isOTEnabledInLoc { + return "" + } + + opc := opentelemetryPropagateContext(location) + if opc != "" { + opc = fmt.Sprintf("opentelemetry on;\n%v", opc) + } + + if location.Opentelemetry.OperationName != "" { + opc = opc + "\nopentelemetry_operation_name " + location.Opentelemetry.OperationName + ";" + } + + if (!isOTTrustSet && !location.Opentelemetry.TrustSet) || + (location.Opentelemetry.TrustSet && !location.Opentelemetry.TrustEnabled) { + opc = opc + "\nopentelemetry_trust_incoming_spans off;" + } else { + opc = opc + "\nopentelemetry_trust_incoming_spans on;" + } + return opc +} + // shouldLoadOpentracingModule determines whether or not the Opentracing module needs to be loaded. // First, it checks if `enable-opentracing` is set in the ConfigMap. If it is not, it iterates over all locations to // check if Opentracing is enabled by the annotation `nginx.ingress.kubernetes.io/enable-opentracing`. @@ -1606,6 +1673,35 @@ func shouldLoadOpentracingModule(c interface{}, s interface{}) bool { return false } +// shouldLoadOpentelemetryModule determines whether or not the Opentelemetry module needs to be loaded. +// It checks if `enable-opentelemetry` is set in the ConfigMap. +func shouldLoadOpentelemetryModule(c interface{}, s interface{}) bool { + cfg, ok := c.(config.Configuration) + if !ok { + klog.Errorf("expected a 'config.Configuration' type but %T was returned", c) + return false + } + + servers, ok := s.([]*ingress.Server) + if !ok { + klog.Errorf("expected an '[]*ingress.Server' type but %T was returned", s) + return false + } + + if cfg.EnableOpentelemetry { + return true + } + + for _, server := range servers { + for _, location := range server.Locations { + if location.Opentelemetry.Enabled { + return true + } + } + } + return false +} + func buildModSecurityForLocation(cfg config.Configuration, location *ingress.Location) string { isMSEnabledInLoc := location.ModSecurity.Enable isMSEnableSetInLoc := location.ModSecurity.EnableSet diff --git a/internal/ingress/controller/template/template_test.go b/internal/ingress/controller/template/template_test.go index cb1ebd1b7..343da506b 100644 --- a/internal/ingress/controller/template/template_test.go +++ b/internal/ingress/controller/template/template_test.go @@ -37,6 +37,7 @@ import ( "k8s.io/ingress-nginx/internal/ingress/annotations/authreq" "k8s.io/ingress-nginx/internal/ingress/annotations/influxdb" "k8s.io/ingress-nginx/internal/ingress/annotations/modsecurity" + "k8s.io/ingress-nginx/internal/ingress/annotations/opentelemetry" "k8s.io/ingress-nginx/internal/ingress/annotations/opentracing" "k8s.io/ingress-nginx/internal/ingress/annotations/ratelimit" "k8s.io/ingress-nginx/internal/ingress/annotations/rewrite" @@ -1150,6 +1151,26 @@ func TestOpentracingPropagateContext(t *testing.T) { } } +func TestOpentelemetryPropagateContext(t *testing.T) { + tests := map[*ingress.Location]string{ + {BackendProtocol: "HTTP"}: "opentelemetry_propagate;", + {BackendProtocol: "HTTPS"}: "opentelemetry_propagate;", + {BackendProtocol: "AUTO_HTTP"}: "opentelemetry_propagate;", + {BackendProtocol: "GRPC"}: "opentelemetry_propagate;", + {BackendProtocol: "GRPCS"}: "opentelemetry_propagate;", + {BackendProtocol: "AJP"}: "opentelemetry_propagate;", + {BackendProtocol: "FCGI"}: "opentelemetry_propagate;", + nil: "", + } + + for loc, expectedDirective := range tests { + actualDirective := opentelemetryPropagateContext(loc) + if actualDirective != expectedDirective { + t.Errorf("Expected %v but returned %v", expectedDirective, actualDirective) + } + } +} + func TestGetIngressInformation(t *testing.T) { testcases := map[string]struct { @@ -1723,6 +1744,37 @@ func TestBuildOpenTracing(t *testing.T) { } +func TestBuildOpenTelemetry(t *testing.T) { + invalidType := &ingress.Ingress{} + expected := "" + actual := buildOpentelemetry(invalidType, []*ingress.Server{}) + + if expected != actual { + t.Errorf("Expected '%v' but returned '%v'", expected, actual) + } + + cfgNoHost := config.Configuration{ + EnableOpentelemetry: true, + } + expected = "\r\n" + actual = buildOpentelemetry(cfgNoHost, []*ingress.Server{}) + + if expected != actual { + t.Errorf("Expected '%v' but returned '%v'", expected, actual) + } + + cfgOpenTelemetry := config.Configuration{ + EnableOpentelemetry: true, + OpentelemetryOperationName: "my-operation-name", + } + expected = "\r\n" + expected += "opentelemetry_operation_name \"my-operation-name\";\n" + actual = buildOpentelemetry(cfgOpenTelemetry, []*ingress.Server{}) + if expected != actual { + t.Errorf("Expected '%v' but returned '%v'", expected, actual) + } +} + func TestEnforceRegexModifier(t *testing.T) { invalidType := &ingress.Ingress{} expected := false @@ -1903,6 +1955,107 @@ func TestShouldLoadOpentracingModule(t *testing.T) { } } +func TestOpentelemetryForLocation(t *testing.T) { + trueVal := true + falseVal := false + + loadOT := `opentelemetry on; +opentelemetry_propagate; +opentelemetry_trust_incoming_spans on;` + loadOTUntrustedSpan := `opentelemetry on; +opentelemetry_propagate; +opentelemetry_trust_incoming_spans off;` + testCases := []struct { + description string + globalOT bool + isSetInLoc bool + isOTInLoc *bool + globalTrust bool + isTrustSetInLoc bool + isTrustInLoc *bool + expected string + }{ + {"globally enabled, without annotation", true, false, nil, true, false, nil, loadOT}, + {"globally enabled and enabled in location", true, true, &trueVal, true, false, nil, loadOT}, + {"globally disabled and not enabled in location", false, false, nil, true, false, nil, ""}, + {"globally disabled but enabled in location", false, true, &trueVal, true, false, nil, loadOT}, + {"globally trusted, not trusted in location", true, false, nil, true, true, &falseVal, loadOTUntrustedSpan}, + {"not globally trusted, trust set in location", true, false, nil, false, true, &trueVal, loadOT}, + {"not globally trusted, trust not set in location", true, false, nil, false, false, nil, loadOTUntrustedSpan}, + } + + for _, testCase := range testCases { + il := &ingress.Location{ + Opentelemetry: opentelemetry.Config{Set: testCase.isSetInLoc, TrustSet: testCase.isTrustSetInLoc}, + } + if il.Opentelemetry.Set { + il.Opentelemetry.Enabled = *testCase.isOTInLoc + } + if il.Opentelemetry.TrustSet { + il.Opentelemetry.TrustEnabled = *testCase.isTrustInLoc + } + + actual := buildOpentelemetryForLocation(testCase.globalOT, testCase.globalTrust, il) + + if testCase.expected != actual { + t.Errorf("%v: expected '%v' but returned '%v'", testCase.description, testCase.expected, actual) + } + } +} + +func TestShouldLoadOpentelemetryModule(t *testing.T) { + // ### Invalid argument type tests ### + // The first tests should return false. + expected := false + + invalidType := &ingress.Ingress{} + actual := shouldLoadOpentelemetryModule(config.Configuration{}, invalidType) + if expected != actual { + t.Errorf("Expected '%v' but returned '%v'", expected, actual) + } + + actual = shouldLoadOpentelemetryModule(invalidType, []*ingress.Server{}) + if expected != actual { + t.Errorf("Expected '%v' but returned '%v'", expected, actual) + } + + // ### Functional tests ### + actual = shouldLoadOpentelemetryModule(config.Configuration{}, []*ingress.Server{}) + if expected != actual { + t.Errorf("Expected '%v' but returned '%v'", expected, actual) + } + + // All further tests should return true. + expected = true + + configuration := config.Configuration{EnableOpentelemetry: true} + actual = shouldLoadOpentelemetryModule(configuration, []*ingress.Server{}) + if expected != actual { + t.Errorf("Expected '%v' but returned '%v'", expected, actual) + } + + servers := []*ingress.Server{ + { + Locations: []*ingress.Location{ + { + Opentelemetry: opentelemetry.Config{ + Enabled: true, + }, + }, + }, + }, + } + actual = shouldLoadOpentelemetryModule(config.Configuration{}, servers) + if expected != actual { + t.Errorf("Expected '%v' but returned '%v'", expected, actual) + } + + actual = shouldLoadOpentelemetryModule(configuration, servers) + if expected != actual { + t.Errorf("Expected '%v' but returned '%v'", expected, actual) + } +} + func TestModSecurityForLocation(t *testing.T) { loadModule := `modsecurity on; ` diff --git a/mkdocs.yml b/mkdocs.yml index c76a640f6..62c0ccf48 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -101,6 +101,7 @@ nav: - Third party addons: - ModSecurity Web Application Firewall: "user-guide/third-party-addons/modsecurity.md" - OpenTracing: "user-guide/third-party-addons/opentracing.md" + - OpenTelemetry: "user-guide/third-party-addons/opentelemetry.md" - Examples: - Introduction: "examples/index.md" - Prerequisites: "examples/PREREQUISITES.md" diff --git a/pkg/apis/ingress/types.go b/pkg/apis/ingress/types.go index 9395683ec..08a2f7575 100644 --- a/pkg/apis/ingress/types.go +++ b/pkg/apis/ingress/types.go @@ -35,6 +35,7 @@ import ( "k8s.io/ingress-nginx/internal/ingress/annotations/log" "k8s.io/ingress-nginx/internal/ingress/annotations/mirror" "k8s.io/ingress-nginx/internal/ingress/annotations/modsecurity" + "k8s.io/ingress-nginx/internal/ingress/annotations/opentelemetry" "k8s.io/ingress-nginx/internal/ingress/annotations/opentracing" "k8s.io/ingress-nginx/internal/ingress/annotations/proxy" "k8s.io/ingress-nginx/internal/ingress/annotations/proxyssl" @@ -360,6 +361,9 @@ type Location struct { // Opentracing allows the global opentracing setting to be overridden for a location // +optional Opentracing opentracing.Config `json:"opentracing"` + // Opentelemetry allows the global opentelemetry setting to be overridden for a location + // +optional + Opentelemetry opentelemetry.Config `json:"opentelemetry"` } // SSLPassthroughBackend describes a SSL upstream server configured diff --git a/pkg/apis/ingress/types_equals.go b/pkg/apis/ingress/types_equals.go index 2485f8a7b..8a54f3758 100644 --- a/pkg/apis/ingress/types_equals.go +++ b/pkg/apis/ingress/types_equals.go @@ -464,6 +464,10 @@ func (l1 *Location) Equal(l2 *Location) bool { return false } + if !l1.Opentelemetry.Equal(&l2.Opentelemetry) { + return false + } + if !l1.Mirror.Equal(&l2.Mirror) { return false } diff --git a/rootfs/etc/nginx/template/nginx.tmpl b/rootfs/etc/nginx/template/nginx.tmpl old mode 100755 new mode 100644 index 958397dd5..014c8cc2a --- a/rootfs/etc/nginx/template/nginx.tmpl +++ b/rootfs/etc/nginx/template/nginx.tmpl @@ -37,6 +37,10 @@ load_module /etc/nginx/modules/ngx_http_modsecurity_module.so; load_module /etc/nginx/modules/ngx_http_opentracing_module.so; {{ end }} +{{ if (shouldLoadOpentelemetryModule $cfg $servers) }} +load_module /modules_mount/etc/nginx/modules/otel/otel_ngx_module.so; +{{ end }} + daemon off; worker_processes {{ $cfg.WorkerProcesses }}; @@ -64,6 +68,10 @@ events { } http { + {{ if (shouldLoadOpentelemetryModule $cfg $servers) }} + opentelemetry_config {{ $cfg.OpentelemetryConfig }}; + {{ end }} + lua_package_path "/etc/nginx/lua/?.lua;;"; {{ buildLuaSharedDictionaries $cfg $servers }} @@ -322,6 +330,7 @@ http { limit_conn_status {{ $cfg.LimitConnStatusCode }}; {{ buildOpentracing $cfg $servers }} + {{ buildOpentelemetry $cfg $servers }} include /etc/nginx/mime.types; default_type {{ $cfg.DefaultType }}; @@ -695,6 +704,9 @@ http { opentracing off; {{ end }} + {{ if $cfg.EnableOpentelemetry }} + opentelemetry off; + {{ end }} location {{ $healthzURI }} { return 200; } @@ -1063,6 +1075,11 @@ stream { opentracing_propagate_context; {{ end }} + {{ if (or $all.Cfg.EnableOpentelemetry $location.Opentelemetry.Enabled) }} + opentelemetry on; + opentelemetry_propagate; + {{ end }} + access_log off; # Ensure that modsecurity will not run on an internal location as this is not accessible from outside @@ -1204,6 +1221,7 @@ stream { set $global_rate_limit_exceeding n; {{ buildOpentracingForLocation $all.Cfg.EnableOpentracing $all.Cfg.OpentracingTrustIncomingSpan $location }} + {{ buildOpentelemetryForLocation $all.Cfg.EnableOpentelemetry $all.Cfg.OpentelemetryTrustIncomingSpan $location }} {{ if $location.Mirror.Source }} mirror {{ $location.Mirror.Source }}; @@ -1520,6 +1538,10 @@ stream { opentracing off; {{ end }} + {{ if $all.Cfg.EnableOpentelemetry }} + opentelemetry off; + {{ end }} + access_log off; return 200; } @@ -1531,6 +1553,10 @@ stream { opentracing off; {{ end }} + {{ if $all.Cfg.EnableOpentelemetry }} + opentelemetry off; + {{ end }} + {{ range $v := $all.NginxStatusIpv4Whitelist }} allow {{ $v }}; {{ end }} diff --git a/test/e2e/framework/exec.go b/test/e2e/framework/exec.go index 0284e768c..d91d36551 100644 --- a/test/e2e/framework/exec.go +++ b/test/e2e/framework/exec.go @@ -21,6 +21,7 @@ import ( "encoding/json" "fmt" "io" + "os" "os/exec" "regexp" "strconv" @@ -111,7 +112,11 @@ func (f *Framework) NamespaceContent() (string, error) { // newIngressController deploys a new NGINX Ingress controller in a namespace func (f *Framework) newIngressController(namespace string, namespaceOverlay string) error { // Creates an nginx deployment - cmd := exec.Command("./wait-for-nginx.sh", namespace, namespaceOverlay) + isChroot, ok := os.LookupEnv("IS_CHROOT") + if !ok { + isChroot = "false" + } + cmd := exec.Command("./wait-for-nginx.sh", namespace, namespaceOverlay, isChroot) out, err := cmd.CombinedOutput() if err != nil { return fmt.Errorf("unexpected error waiting for ingress controller deployment: %v.\nLogs:\n%v", err, string(out)) diff --git a/test/e2e/run-e2e-suite.sh b/test/e2e/run-e2e-suite.sh index b1de8bf9a..a3bf589cd 100755 --- a/test/e2e/run-e2e-suite.sh +++ b/test/e2e/run-e2e-suite.sh @@ -76,6 +76,7 @@ kubectl run --rm \ --restart=Never \ --env="E2E_NODES=${E2E_NODES}" \ --env="FOCUS=${FOCUS}" \ + --env="IS_CHROOT=${IS_CHROOT:-false}"\ --env="E2E_CHECK_LEAKS=${E2E_CHECK_LEAKS}" \ --env="NGINX_BASE_IMAGE=${NGINX_BASE_IMAGE}" \ --overrides='{ "apiVersion": "v1", "spec":{"serviceAccountName": "ingress-nginx-e2e"}}' \ diff --git a/test/e2e/settings/opentelemetry.go b/test/e2e/settings/opentelemetry.go new file mode 100644 index 000000000..92d202cb3 --- /dev/null +++ b/test/e2e/settings/opentelemetry.go @@ -0,0 +1,119 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package settings + +import ( + "strings" + + "github.com/onsi/ginkgo/v2" + + "k8s.io/ingress-nginx/test/e2e/framework" +) + +const ( + enableOpentelemetry = "enable-opentelemetry" + opentelemetryTrustIncomingSpan = "opentelemetry-trust-incoming-span" + + opentelemetryOperationName = "opentelemetry-operation-name" + opentelemetryLocationOperationName = "opentelemetry-location-operation-name" + opentelemetryConfig = "opentelemetry-config" + opentelemetryConfigPath = "/etc/nginx/opentelemetry.toml" +) + +var _ = framework.IngressNginxDescribe("Configure Opentelemetry", func() { + f := framework.NewDefaultFramework("enable-opentelemetry") + + ginkgo.BeforeEach(func() { + f.NewEchoDeployment() + }) + + ginkgo.AfterEach(func() { + }) + + ginkgo.It("should not exists opentelemetry directive", func() { + config := map[string]string{} + config[enableOpentelemetry] = "false" + f.SetNginxConfigMapData(config) + + f.EnsureIngress(framework.NewSingleIngress(enableOpentelemetry, "/", enableOpentelemetry, f.Namespace, "http-svc", 80, nil)) + + f.WaitForNginxConfiguration( + func(cfg string) bool { + return !strings.Contains(cfg, "opentelemetry on") + }) + }) + + ginkgo.It("should exists opentelemetry directive when is enabled", func() { + config := map[string]string{} + config[enableOpentelemetry] = "true" + config[opentelemetryConfig] = opentelemetryConfigPath + f.SetNginxConfigMapData(config) + + f.EnsureIngress(framework.NewSingleIngress(enableOpentelemetry, "/", enableOpentelemetry, f.Namespace, "http-svc", 80, nil)) + + f.WaitForNginxConfiguration( + func(cfg string) bool { + return strings.Contains(cfg, "opentelemetry on") + }) + }) + + ginkgo.It("should include opentelemetry_trust_incoming_spans on directive when enabled", func() { + config := map[string]string{} + config[enableOpentelemetry] = "true" + config[opentelemetryConfig] = opentelemetryConfigPath + config[opentelemetryTrustIncomingSpan] = "true" + f.SetNginxConfigMapData(config) + + f.EnsureIngress(framework.NewSingleIngress(enableOpentelemetry, "/", enableOpentelemetry, f.Namespace, "http-svc", 80, nil)) + + f.WaitForNginxConfiguration( + func(cfg string) bool { + return strings.Contains(cfg, "opentelemetry_trust_incoming_spans on") + }) + }) + + ginkgo.It("should not exists opentelemetry_operation_name directive when is empty", func() { + config := map[string]string{} + config[enableOpentelemetry] = "true" + config[opentelemetryConfig] = opentelemetryConfigPath + config[opentelemetryOperationName] = "" + f.SetNginxConfigMapData(config) + + f.EnsureIngress(framework.NewSingleIngress(enableOpentelemetry, "/", enableOpentelemetry, f.Namespace, "http-svc", 80, nil)) + + f.WaitForNginxConfiguration( + func(cfg string) bool { + return !strings.Contains(cfg, "opentelemetry_operation_name") + }) + }) + + ginkgo.It("should exists opentelemetry_operation_name directive when is configured", func() { + config := map[string]string{} + config[enableOpentelemetry] = "true" + config[opentelemetryConfig] = opentelemetryConfigPath + config[opentelemetryOperationName] = "HTTP $request_method $uri" + f.SetNginxConfigMapData(config) + + f.EnsureIngress(framework.NewSingleIngress(enableOpentelemetry, "/", enableOpentelemetry, f.Namespace, "http-svc", 80, nil)) + + f.WaitForNginxConfiguration( + func(cfg string) bool { + return strings.Contains(cfg, `opentelemetry_operation_name "HTTP $request_method $uri"`) + }) + }) + +}) diff --git a/test/e2e/wait-for-nginx.sh b/test/e2e/wait-for-nginx.sh index 190f71e6d..153d348c2 100755 --- a/test/e2e/wait-for-nginx.sh +++ b/test/e2e/wait-for-nginx.sh @@ -23,6 +23,7 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" export NAMESPACE=$1 export NAMESPACE_OVERLAY=$2 +export IS_CHROOT=$3 echo "deploying NGINX Ingress controller in namespace $NAMESPACE" @@ -46,6 +47,16 @@ metadata: EOF +OTEL_MODULE=$(cat <