feat: OpenTelemetry module integration (#9062)

* OpenTelemetry module integration * e2e test * e2e test fix * default OpentelemetryConfig * e2e values * mount otel module for otel test only * propagate IS_CHROOT * propagate IS_CHROOT e2e test * code doc * comments * golint * opentelemetry doc * zipkin * zipkin * typo * update e2e test OpenTelemetry value * use opentelemetry value * revert merge conflict * fix * format * review comments * clean
2023-03-22 19:58:22 +01:00 · 2023-03-22 19:58:22 +01:00 · c8cb9167d3
commit c8cb9167d3
parent c075793ae5
23 changed files with 1131 additions and 2 deletions
--- a/docs/images/otel-grafana-demo.png
+++ b/docs/images/otel-grafana-demo.png
--- a/docs/images/otel-jaeger-demo.png
+++ b/docs/images/otel-jaeger-demo.png
--- a/docs/images/otel-zipkin-demo.png
+++ b/docs/images/otel-zipkin-demo.png
--- a/docs/kubectl-plugin.md
+++ b/docs/kubectl-plugin.md
@ -208,6 +208,7 @@ modsecurity
 modules
 nginx.conf
 opentracing.json
+opentelemetry.toml
 owasp-modsecurity-crs
 template
 ```
--- a/docs/user-guide/nginx-configuration/annotations.md
+++ b/docs/user-guide/nginx-configuration/annotations.md
@ -121,6 +121,8 @@ You can add these Kubernetes annotations to specific Ingress objects to customiz
 |[nginx.ingress.kubernetes.io/enable-access-log](#enable-access-log)|"true" or "false"|
 |[nginx.ingress.kubernetes.io/enable-opentracing](#enable-opentracing)|"true" or "false"|
 |[nginx.ingress.kubernetes.io/opentracing-trust-incoming-span](#opentracing-trust-incoming-span)|"true" or "false"|
+|[nginx.ingress.kubernetes.io/enable-opentelemetry](#enable-opentelemetry)|"true" or "false"|
+|[nginx.ingress.kubernetes.io/opentelemetry-trust-incoming-span](#opentelemetry-trust-incoming-spans)|"true" or "false"|
 |[nginx.ingress.kubernetes.io/enable-influxdb](#influxdb)|"true" or "false"|
 |[nginx.ingress.kubernetes.io/influxdb-measurement](#influxdb)|string|
 |[nginx.ingress.kubernetes.io/influxdb-port](#influxdb)|string|
@ -821,6 +823,24 @@ sometimes need to be overridden to enable it or disable it for a specific ingres
 nginx.ingress.kubernetes.io/opentracing-trust-incoming-span: "true"
 ```

+### Enable Opentelemetry
+
+Opentelemetry can be enabled or disabled globally through the ConfigMap but this will sometimes need to be overridden
+to enable it or disable it for a specific ingress (e.g. to turn off telemetry of external health check endpoints)
+
+```yaml
+nginx.ingress.kubernetes.io/enable-opentelemetry: "true"
+```
+
+### Opentelemetry Trust Incoming Span
+
+The option to trust incoming trace spans can be enabled or disabled globally through the ConfigMap but this will
+sometimes need to be overridden to enable it or disable it for a specific ingress (e.g. only enable on a private endpoint)
+
+```yaml
+nginx.ingress.kubernetes.io/opentelemetry-trust-incoming-spans: "true"
+```
+
 ### X-Forwarded-Prefix Header
 To add the non-standard `X-Forwarded-Prefix` header to the upstream request with a string value, the following annotation can be used:

--- a/docs/user-guide/nginx-configuration/configmap.md
+++ b/docs/user-guide/nginx-configuration/configmap.md
@ -157,6 +157,19 @@ The following table shows a configuration option's name, type, and the default v
 |[datadog-operation-name-override](#datadog-operation-name-override)|string|"nginx.handle"|
 |[datadog-priority-sampling](#datadog-priority-sampling)|bool|"true"|
 |[datadog-sample-rate](#datadog-sample-rate)|float|1.0|
+|[enable-opentelemetry](#enable-opentelemetry)|bool|"false"|
+|[opentelemetry-trust-incoming-span](#opentelemetry-trust-incoming-span)|bool|"true"|
+|[opentelemetry-operation-name](#opentelemetry-operation-name)|string|""|
+|[opentelemetry-config](#/etc/nginx/opentelemetry.toml)|string|"/etc/nginx/opentelemetry.toml"|
+|[otlp-collector-host](#otlp-collector-host)|string|""|
+|[otlp-collector-port](#otlp-collector-port)|int|4317|
+|[otel-max-queuesize](#otel-max-queuesize)|int||
+|[otel-schedule-delay-millis](#otel-schedule-delay-millis)|int||
+|[otel-max-export-batch-size](#otel-max-export-batch-size)|int||
+|[otel-service-name](#otel-service-name)|string|"nginx"|
+|[otel-sampler](#otel-sampler)|string|"AlwaysOff"|
+|[otel-sampler-parent-based](#otel-sampler-parent-based)|bool|"false"|
+|[otel-sampler-ratio](#otel-sampler-ratio)|float|0.01|
 |[main-snippet](#main-snippet)|string|""|
 |[http-snippet](#http-snippet)|string|""|
 |[server-snippet](#server-snippet)|string|""|
@ -1009,6 +1022,46 @@ If true disables client-side sampling (thus ignoring `sample_rate`) and enables
 Specifies sample rate for any traces created.
 This is effective only when `datadog-priority-sampling` is `false` _**default:**_ 1.0

+## enable-opentelemetry
+
+Enables the nginx OpenTelemetry extension. _**default:**_ is disabled
+
+_References:_
+[https://github.com/open-telemetry/opentelemetry-cpp-contrib](https://github.com/open-telemetry/opentelemetry-cpp-contrib/tree/main/instrumentation/nginx)
+
+## opentelemetry-operation-name
+
+Specifies a custom name for the server span. _**default:**_ is empty
+
+For example, set to "HTTP $request_method $uri".
+
+## otlp-collector-host
+
+Specifies the host to use when uploading traces. It must be a valid URL.
+
+## otlp-collector-port
+
+Specifies the port to use when uploading traces. _**default:**_ 4317
+
+## otel-service-name
+
+Specifies the service name to use for any traces created. _**default:**_ nginx
+
+##  opentelemetry-trust-incoming-span: "true"
+Enables or disables using spans from incoming requests as parent for created ones. _**default:**_ true
+
+##  otel-sampler-parent-based
+
+Uses sampler implementation which by default will take a sample if parent Activity is sampled. _**default:**_ false
+
+## otel-sampler-ratio
+
+Specifies sample rate for any traces created. _**default:**_ 0.01
+
+## otel-sampler
+
+Specifies the sampler to be used when sampling traces. The available samplers are: AlwaysOff, AlwaysOn, TraceIdRatioBased, remote. _**default:**_ AlwaysOff
+
 ## main-snippet

 Adds custom configuration to the main section of the nginx configuration.
--- a/docs/user-guide/third-party-addons/opentelemetry.md
+++ b/docs/user-guide/third-party-addons/opentelemetry.md
@ -0,0 +1,260 @@
+# OpenTelemetry
+
+Enables requests served by NGINX for distributed telemetry via The OpenTelemetry Project.
+
+Using the third party module [opentelemetry-cpp-contrib/nginx](https://github.com/open-telemetry/opentelemetry-cpp-contrib/tree/main/instrumentation/nginx) the NGINX ingress controller can configure NGINX to enable [OpenTelemetry](http://opentelemetry.io) instrumentation.
+By default this feature is disabled.
+
+## Usage
+
+To enable the instrumentation we must enable OpenTelemetry in the configuration ConfigMap:
+```yaml
+data:
+  enable-opentelemetry: "true"
+```
+
+To enable or disable instrumentation for a single Ingress, use
+the `enable-opentelemetry` annotation:
+```yaml
+kind: Ingress
+metadata:
+  annotations:
+    nginx.ingress.kubernetes.io/enable-opentelemetry: "true"
+```
+
+We must also set the host to use when uploading traces:
+
+```yaml
+otlp-collector-host: "otel-coll-collector.otel.svc"
+```
+NOTE: While the option is called `otlp-collector-host`, you will need to point this to any backend that recieves otlp-grpc.
+
+Next you will need to deploy a distributed telemetry system which uses OpenTelemetry.
+[opentelemetry-collector](https://github.com/open-telemetry/opentelemetry-collector), [Jaeger](https://www.jaegertracing.io/)
+[Tempo](https://github.com/grafana/tempo), and [zipkin](https://zipkin.io/)
+have been tested.
+
+Other optional configuration options:
+```yaml
+# specifies the name to use for the server span
+opentelemetry-operation-name
+
+# sets whether or not to trust incoming telemetry spans
+opentelemetry-trust-incoming-span
+
+# specifies the port to use when uploading traces, Default: 4317
+otlp-collector-port
+
+# specifies the service name to use for any traces created, Default: nginx
+otel-service-name
+
+# The maximum queue size. After the size is reached data are dropped.
+otel-max-queuesize
+
+# The delay interval in milliseconds between two consecutive exports.
+otel-schedule-delay-millis
+        
+# How long the export can run before it is cancelled.
+otel-schedule-delay-millis
+
+# The maximum batch size of every export. It must be smaller or equal to maxQueueSize.
+otel-max-export-batch-size
+
+# specifies sample rate for any traces created, Default: 0.01
+otel-sampler-ratio
+
+# specifies the sampler to be used when sampling traces.
+# The available samplers are: AlwaysOn,  AlwaysOff, TraceIdRatioBased, Default: AlwaysOff
+otel-sampler
+
+# Uses sampler implementation which by default will take a sample if parent Activity is sampled, Default: false
+otel-sampler-parent-based
+```
+
+Note that you can also set whether to trust incoming spans (global default is true) per-location using annotations like the following:
+```yaml
+kind: Ingress
+metadata:
+  annotations:
+    nginx.ingress.kubernetes.io/opentelemetry-trust-incoming-span: "true"
+```
+
+## Examples
+
+The following examples show how to deploy and test different distributed telemetry systems. These example can be performed using Docker Desktop.
+
+In the [esigo/nginx-example](https://github.com/esigo/nginx-example)
+GitHub repository is an example of a simple hello service:
+
+```mermaid
+graph TB
+    subgraph Browser
+    start["http://esigo.dev/hello/nginx"]
+    end
+
+    subgraph app
+        sa[service-a]
+        sb[service-b]
+        sa --> |name: nginx| sb
+        sb --> |hello nginx!| sa
+    end
+
+    subgraph otel
+        otc["Otel Collector"] 
+    end
+
+    subgraph observability
+        tempo["Tempo"]
+        grafana["Grafana"]
+        backend["Jaeger"]
+        zipkin["Zipkin"]
+    end
+
+    subgraph ingress-nginx
+        ngx[nginx]
+    end
+
+    subgraph ngx[nginx]
+        ng[nginx]
+        om[OpenTelemetry module]
+    end
+
+    subgraph Node
+        app
+        otel
+        observability
+        ingress-nginx
+        om --> |otlp-gRPC| otc --> |jaeger| backend
+        otc --> |zipkin| zipkin
+        otc --> |otlp-gRPC| tempo --> grafana
+        sa --> |otlp-gRPC| otc
+        sb --> |otlp-gRPC| otc
+        start --> ng --> sa
+    end
+```
+
+To install the example and collectors run:
+
+1. Enable Ingress addon with:
+
+    ```yaml
+      opentelemetry:
+        enabled: true
+        image: registry.k8s.io/ingress-nginx/opentelemetry:v20230107-helm-chart-4.4.2-2-g96b3d2165@sha256:331b9bebd6acfcd2d3048abbdd86555f5be76b7e3d0b5af4300b04235c6056c9
+        containerSecurityContext:
+        allowPrivilegeEscalation: false
+    ```
+
+2. Enable OpenTelemetry and set the otlp-collector-host:
+
+    ```yaml
+    $ echo '
+      apiVersion: v1
+      kind: ConfigMap
+      data:
+        enable-opentelemetry: "true"
+        opentelemetry-config: "/etc/nginx/opentelemetry.toml"
+        opentelemetry-operation-name: "HTTP $request_method $service_name $uri"
+        opentelemetry-trust-incoming-span: "true"
+        otlp-collector-host: "otel-coll-collector.otel.svc"
+        otlp-collector-port: "4317"
+        otel-max-queuesize: "2048"
+        otel-schedule-delay-millis: "5000"
+        otel-max-export-batch-size: "512"
+        otel-service-name: "nginx-proxy" # Opentelemetry resource name
+        otel-sampler: "AlwaysOn" # Also: AlwaysOff, TraceIdRatioBased
+        otel-sampler-ratio: "1.0"
+        otel-sampler-parent-based: "false"
+      metadata:
+        name: ingress-nginx-controller
+        namespace: ingress-nginx
+      ' | kubectl replace -f -
+    ```
+
+4. Deploy otel-collector, grafana and Jaeger backend:
+
+    ```bash
+    # add helm charts needed for grafana and OpenTelemetry collector
+    helm repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts
+    helm repo add grafana https://grafana.github.io/helm-charts
+    helm repo update
+    # deply cert-manager needed for OpenTelemetry collector operator
+    kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.9.1/cert-manager.yaml
+    # create observability namespace
+    kubectl apply -f https://raw.githubusercontent.com/esigo/nginx-example/main/observability/namespace.yaml
+    # install OpenTelemetry collector operator
+    helm upgrade --install otel-collector-operator -n otel --create-namespace open-telemetry/opentelemetry-operator
+    # deploy OpenTelemetry collector
+    kubectl apply -f https://raw.githubusercontent.com/esigo/nginx-example/main/observability/collector.yaml
+    # deploy Jaeger all-in-one
+    kubectl apply -f https://github.com/jaegertracing/jaeger-operator/releases/download/v1.37.0/jaeger-operator.yaml -n observability
+    kubectl apply -f https://raw.githubusercontent.com/esigo/nginx-example/main/observability/jaeger.yaml -n observability
+    # deploy zipkin
+    kubectl apply -f https://raw.githubusercontent.com/esigo/nginx-example/main/observability/zipkin.yaml -n observability
+    # deploy tempo and grafana
+	helm upgrade --install tempo grafana/tempo --create-namespace -n observability
+	helm upgrade -f https://raw.githubusercontent.com/esigo/nginx-example/main/observability/grafana/grafana-values.yaml --install grafana grafana/grafana --create-namespace -n observability
+    ```
+
+3. Build and deploy demo app:
+
+    ```bash
+    # build images
+    make images
+
+    # deploy demo app:
+    make deploy-app
+    ```
+
+5. Make a few requests to the Service:
+
+    ```bash
+    kubectl port-forward --namespace=ingress-nginx service/ingress-nginx-controller 8090:80
+    curl http://esigo.dev:8090/hello/nginx
+
+
+    StatusCode        : 200
+    StatusDescription : OK
+    Content           : {"v":"hello nginx!"}
+
+    RawContent        : HTTP/1.1 200 OK
+                        Connection: keep-alive
+                        Content-Length: 21
+                        Content-Type: text/plain; charset=utf-8
+                        Date: Mon, 10 Oct 2022 17:43:33 GMT
+
+                        {"v":"hello nginx!"}
+
+    Forms             : {}
+    Headers           : {[Connection, keep-alive], [Content-Length, 21], [Content-Type, text/plain; charset=utf-8], [Date,
+                        Mon, 10 Oct 2022 17:43:33 GMT]}
+    Images            : {}
+    InputFields       : {}
+    Links             : {}
+    ParsedHtml        : System.__ComObject
+    RawContentLength  : 21
+    ```
+
+6. View the Grafana UI:
+
+    ```bash
+    kubectl port-forward --namespace=observability service/grafana 3000:80
+    ```
+    In the Grafana interface we can see the details:
+    ![grafana screenshot](../../images/otel-grafana-demo.png "grafana screenshot")
+
+7. View the Jaeger UI:
+
+    ```bash
+    kubectl port-forward --namespace=observability service/jaeger-all-in-one-query 16686:16686
+    ```
+    In the Jaeger interface we can see the details:
+    ![Jaeger screenshot](../../images/otel-jaeger-demo.png "Jaeger screenshot")
+
+8. View the Zipkin UI:
+
+    ```bash
+    kubectl port-forward --namespace=observability service/zipkin 9411:9411
+    ```
+    In the Zipkin interface we can see the details:
+    ![zipkin screenshot](../../images/otel-zipkin-demo.png "zipkin screenshot")
--- a/internal/ingress/annotations/annotations.go
+++ b/internal/ingress/annotations/annotations.go
@ -20,6 +20,7 @@ import (
 	"github.com/imdario/mergo"
 	"k8s.io/ingress-nginx/internal/ingress/annotations/canary"
 	"k8s.io/ingress-nginx/internal/ingress/annotations/modsecurity"
+	"k8s.io/ingress-nginx/internal/ingress/annotations/opentelemetry"
 	"k8s.io/ingress-nginx/internal/ingress/annotations/proxyssl"
 	"k8s.io/ingress-nginx/internal/ingress/annotations/sslcipher"
 	"k8s.io/ingress-nginx/internal/ingress/annotations/streamsnippet"
@ -94,6 +95,7 @@ type Ingress struct {
 	EnableGlobalAuth   bool
 	HTTP2PushPreload   bool
 	Opentracing        opentracing.Config
+	Opentelemetry      opentelemetry.Config
 	Proxy              proxy.Config
 	ProxySSL           proxyssl.Config
 	RateLimit          ratelimit.Config
@ -145,6 +147,7 @@ func NewAnnotationExtractor(cfg resolver.Resolver) Extractor {
 			"EnableGlobalAuth":     authreqglobal.NewParser(cfg),
 			"HTTP2PushPreload":     http2pushpreload.NewParser(cfg),
 			"Opentracing":          opentracing.NewParser(cfg),
+			"Opentelemetry":        opentelemetry.NewParser(cfg),
 			"Proxy":                proxy.NewParser(cfg),
 			"ProxySSL":             proxyssl.NewParser(cfg),
 			"RateLimit":            ratelimit.NewParser(cfg),
--- a/internal/ingress/annotations/opentelemetry/main.go
+++ b/internal/ingress/annotations/opentelemetry/main.go
@ -0,0 +1,101 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package opentelemetry
+
+import (
+	networking "k8s.io/api/networking/v1"
+
+	"k8s.io/ingress-nginx/internal/ingress/annotations/parser"
+	"k8s.io/ingress-nginx/internal/ingress/resolver"
+)
+
+type opentelemetry struct {
+	r resolver.Resolver
+}
+
+// Config contains the configuration to be used in the Ingress
+type Config struct {
+	Enabled       bool   `json:"enabled"`
+	Set           bool   `json:"set"`
+	TrustEnabled  bool   `json:"trust-enabled"`
+	TrustSet      bool   `json:"trust-set"`
+	OperationName string `json:"operation-name"`
+}
+
+// Equal tests for equality between two Config types
+func (bd1 *Config) Equal(bd2 *Config) bool {
+
+	if bd1.Set != bd2.Set {
+		return false
+	}
+
+	if bd1.Enabled != bd2.Enabled {
+		return false
+	}
+
+	if bd1.TrustSet != bd2.TrustSet {
+		return false
+	}
+
+	if bd1.TrustEnabled != bd2.TrustEnabled {
+		return false
+	}
+
+	if bd1.OperationName != bd2.OperationName {
+		return false
+	}
+
+	return true
+}
+
+// NewParser creates a new serviceUpstream annotation parser
+func NewParser(r resolver.Resolver) parser.IngressAnnotation {
+	return opentelemetry{r}
+}
+
+// Parse parses the annotations to look for opentelemetry configurations
+func (c opentelemetry) Parse(ing *networking.Ingress) (interface{}, error) {
+	cfg := Config{}
+	enabled, err := parser.GetBoolAnnotation("enable-opentelemetry", ing)
+	if err != nil {
+		return &cfg, nil
+	}
+	cfg.Set = true
+	cfg.Enabled = enabled
+	if !enabled {
+		return &cfg, nil
+	}
+
+	trustEnabled, err := parser.GetBoolAnnotation("opentelemetry-trust-incoming-span", ing)
+	if err != nil {
+		operationName, err := parser.GetStringAnnotation("opentelemetry-operation-name", ing)
+		if err != nil {
+			return &cfg, nil
+		}
+		cfg.OperationName = operationName
+		return &cfg, nil
+	}
+
+	cfg.TrustSet = true
+	cfg.TrustEnabled = trustEnabled
+	operationName, err := parser.GetStringAnnotation("opentelemetry-operation-name", ing)
+	if err != nil {
+		return &cfg, nil
+	}
+	cfg.OperationName = operationName
+	return &cfg, nil
+}
--- a/internal/ingress/annotations/opentelemetry/main_test.go
+++ b/internal/ingress/annotations/opentelemetry/main_test.go
@ -0,0 +1,170 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package opentelemetry
+
+import (
+	"testing"
+
+	api "k8s.io/api/core/v1"
+	networking "k8s.io/api/networking/v1"
+	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/ingress-nginx/internal/ingress/annotations/parser"
+	"k8s.io/ingress-nginx/internal/ingress/resolver"
+)
+
+func buildIngress() *networking.Ingress {
+	defaultBackend := networking.IngressBackend{
+		Service: &networking.IngressServiceBackend{
+			Name: "default-backend",
+			Port: networking.ServiceBackendPort{
+				Number: 80,
+			},
+		},
+	}
+
+	return &networking.Ingress{
+		ObjectMeta: meta_v1.ObjectMeta{
+			Name:      "foo",
+			Namespace: api.NamespaceDefault,
+		},
+		Spec: networking.IngressSpec{
+			DefaultBackend: &networking.IngressBackend{
+				Service: &networking.IngressServiceBackend{
+					Name: "default-backend",
+					Port: networking.ServiceBackendPort{
+						Number: 80,
+					},
+				},
+			},
+			Rules: []networking.IngressRule{
+				{
+					Host: "foo.bar.com",
+					IngressRuleValue: networking.IngressRuleValue{
+						HTTP: &networking.HTTPIngressRuleValue{
+							Paths: []networking.HTTPIngressPath{
+								{
+									Path:    "/foo",
+									Backend: defaultBackend,
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+}
+
+func TestIngressAnnotationOpentelemetrySetTrue(t *testing.T) {
+	ing := buildIngress()
+
+	data := map[string]string{}
+	data[parser.GetAnnotationWithPrefix("enable-opentelemetry")] = "true"
+	ing.SetAnnotations(data)
+
+	val, _ := NewParser(&resolver.Mock{}).Parse(ing)
+	openTelemetry, ok := val.(*Config)
+	if !ok {
+		t.Errorf("expected a Config type")
+	}
+
+	if !openTelemetry.Enabled {
+		t.Errorf("expected annotation value to be true, got false")
+	}
+
+	if !openTelemetry.Set {
+		t.Errorf("expected annotation value to be true, got false")
+	}
+
+	if openTelemetry.TrustSet {
+		t.Errorf("expected annotation value to be false, got true")
+	}
+}
+
+func TestIngressAnnotationOpentelemetrySetFalse(t *testing.T) {
+	ing := buildIngress()
+
+	// Test with explicitly set to false
+	data := map[string]string{}
+	data[parser.GetAnnotationWithPrefix("enable-opentelemetry")] = "false"
+	ing.SetAnnotations(data)
+
+	val, _ := NewParser(&resolver.Mock{}).Parse(ing)
+	openTelemetry, ok := val.(*Config)
+	if !ok {
+		t.Errorf("expected a Config type")
+	}
+
+	if openTelemetry.Enabled {
+		t.Errorf("expected annotation value to be false, got true")
+	}
+
+	if !openTelemetry.Set {
+		t.Errorf("expected annotation value to be true, got false")
+	}
+}
+
+func TestIngressAnnotationOpentelemetryTrustSetTrue(t *testing.T) {
+	ing := buildIngress()
+
+	data := map[string]string{}
+	opName := "foo-op"
+	data[parser.GetAnnotationWithPrefix("enable-opentelemetry")] = "true"
+	data[parser.GetAnnotationWithPrefix("opentelemetry-trust-incoming-span")] = "true"
+	data[parser.GetAnnotationWithPrefix("opentelemetry-operation-name")] = opName
+	ing.SetAnnotations(data)
+
+	val, _ := NewParser(&resolver.Mock{}).Parse(ing)
+	openTelemetry, ok := val.(*Config)
+	if !ok {
+		t.Errorf("expected a Config type")
+	}
+
+	if !openTelemetry.Enabled {
+		t.Errorf("expected annotation value to be true, got false")
+	}
+
+	if !openTelemetry.Set {
+		t.Errorf("expected annotation value to be true, got false")
+	}
+
+	if !openTelemetry.TrustEnabled {
+		t.Errorf("expected annotation value to be true, got false")
+	}
+
+	if !openTelemetry.TrustSet {
+		t.Errorf("expected annotation value to be true, got false")
+	}
+
+	if openTelemetry.OperationName != opName {
+		t.Errorf("expected annotation value to be %v, got %v", opName, openTelemetry.OperationName)
+	}
+}
+
+func TestIngressAnnotationOpentelemetryUnset(t *testing.T) {
+	ing := buildIngress()
+
+	// Test with no annotation specified
+	data := map[string]string{}
+	ing.SetAnnotations(data)
+
+	val, _ := NewParser(&resolver.Mock{}).Parse(ing)
+	_, ok := val.(*Config)
+	if !ok {
+		t.Errorf("expected a Config type")
+	}
+}
--- a/internal/ingress/controller/config/config.go
+++ b/internal/ingress/controller/config/config.go
@ -568,6 +568,54 @@ type Configuration struct {
 	// Default: true
 	OpentracingTrustIncomingSpan bool `json:"opentracing-trust-incoming-span"`

+	// EnableOpentelemetry enables the nginx Opentelemetry extension
+	// By default this is disabled
+	EnableOpentelemetry bool `json:"enable-opentelemetry"`
+
+	// OpentelemetryConfig sets the opentelemetry config file
+	// Default: /etc/nginx/opentelemetry.toml
+	OpentelemetryConfig string `json:"opentelemetry-config"`
+
+	// OpentelemetryOperationName specifies a custom name for the server span
+	OpentelemetryOperationName string `json:"opentelemetry-operation-name"`
+
+	// OpentelemetryTrustIncomingSpan sets whether or not to trust incoming trace spans
+	// If false, incoming span headers will be rejected
+	// Default: true
+	OpentelemetryTrustIncomingSpan bool `json:"opentelemetry-trust-incoming-span"`
+
+	// OtlpCollectorHost specifies the host to use when uploading traces
+	OtlpCollectorHost string `json:"otlp-collector-host"`
+
+	// OtlpCollectorPort specifies the port to use when uploading traces
+	// Default: 4317
+	OtlpCollectorPort string `json:"otlp-collector-port"`
+
+	// OtelServiceName specifies the service name to use for any traces created
+	// Default: nginx
+	OtelServiceName string `json:"otel-service-name"`
+
+	// OtelSampler specifies the sampler to use for any traces created
+	// Default: AlwaysOff
+	OtelSampler string `json:"otel-sampler"`
+
+	// OtelSamplerRatio specifies the sampler ratio to use for any traces created
+	// Default: 0.01
+	OtelSamplerRatio float32 `json:"otel-sampler-ratio"`
+
+	//OtelSamplerParentBased specifies the parent based sampler to be use for any traces created
+	// Default: false
+	OtelSamplerParentBased bool `json:"otel-sampler-parent-based"`
+
+	// MaxQueueSize specifies the max queue size for uploading traces
+	OtelMaxQueueSize int32 `json:"otel-max-queuesize"`
+
+	// ScheduleDelayMillis specifies the max delay between uploading traces
+	OtelScheduleDelayMillis int32 `json:"otel-schedule-delay-millis"`
+
+	// MaxExportBatchSize specifies the max export batch size to used when uploading traces
+	OtelMaxExportBatchSize int32 `json:"otel-max-export-batch-size"`
+
 	// ZipkinCollectorHost specifies the host to use when uploading traces
 	ZipkinCollectorHost string `json:"zipkin-collector-host"`

@ -917,6 +965,13 @@ func NewDefault() Configuration {
 		BindAddressIpv4:                        defBindAddress,
 		BindAddressIpv6:                        defBindAddress,
 		OpentracingTrustIncomingSpan:           true,
+		OpentelemetryTrustIncomingSpan:         true,
+		OpentelemetryConfig:                    "/etc/nginx/opentelemetry.toml",
+		OtlpCollectorPort:                      "4317",
+		OtelServiceName:                        "nginx",
+		OtelSampler:                            "AlwaysOff",
+		OtelSamplerRatio:                       0.01,
+		OtelSamplerParentBased:                 false,
 		ZipkinCollectorPort:                    9411,
 		ZipkinServiceName:                      "nginx",
 		ZipkinSampleRate:                       1.0,
--- a/internal/ingress/controller/controller.go
+++ b/internal/ingress/controller/controller.go
@ -1444,6 +1444,7 @@ func locationApplyAnnotations(loc *ingress.Location, anns *annotations.Ingress)
 	loc.EnableGlobalAuth = anns.EnableGlobalAuth
 	loc.HTTP2PushPreload = anns.HTTP2PushPreload
 	loc.Opentracing = anns.Opentracing
+	loc.Opentelemetry = anns.Opentelemetry
 	loc.Proxy = anns.Proxy
 	loc.ProxySSL = anns.ProxySSL
 	loc.RateLimit = anns.RateLimit
--- a/internal/ingress/controller/nginx.go
+++ b/internal/ingress/controller/nginx.go
@ -673,6 +673,11 @@ func (n *NGINXController) OnUpdate(ingressCfg ingress.Configuration) error {
 		return err
 	}

+	err = createOpentelemetryCfg(cfg)
+	if err != nil {
+		return err
+	}
+
 	err = n.testTemplate(content)
 	if err != nil {
 		return err
@ -1056,6 +1061,29 @@ const datadogTmpl = `{
  "dd.priority.sampling": {{ .DatadogPrioritySampling }}
 }`

+const otelTmpl = `
+exporter = "otlp"
+processor = "batch"
+
+[exporters.otlp]
+# Alternatively the OTEL_EXPORTER_OTLP_ENDPOINT environment variable can also be used.
+host = "{{ .OtlpCollectorHost }}"
+port = {{ .OtlpCollectorPort }}
+
+[processors.batch]
+max_queue_size = {{ .OtelMaxQueueSize }}
+schedule_delay_millis = {{ .OtelScheduleDelayMillis }}
+max_export_batch_size = {{ .OtelMaxExportBatchSize }}
+
+[service]
+name = "{{ .OtelServiceName }}" # Opentelemetry resource name
+
+[sampler]
+name = "{{ .OtelSampler }}" # Also: AlwaysOff, TraceIdRatioBased
+ratio = {{ .OtelSamplerRatio }}
+parent_based = {{ .OtelSamplerParentBased }}
+`
+
 func createOpentracingCfg(cfg ngx_config.Configuration) error {
 	var tmpl *template.Template
 	var err error
@ -1091,6 +1119,21 @@ func createOpentracingCfg(cfg ngx_config.Configuration) error {
 	return os.WriteFile("/etc/nginx/opentracing.json", []byte(expanded), file.ReadWriteByUser)
 }

+func createOpentelemetryCfg(cfg ngx_config.Configuration) error {
+
+	tmpl, err := template.New("otel").Parse(otelTmpl)
+	if err != nil {
+		return err
+	}
+	tmplBuf := bytes.NewBuffer(make([]byte, 0))
+	err = tmpl.Execute(tmplBuf, cfg)
+	if err != nil {
+		return err
+	}
+
+	return os.WriteFile(cfg.OpentelemetryConfig, tmplBuf.Bytes(), file.ReadWriteByUser)
+}
+
 func cleanTempNginxCfg() error {
 	var files []string

--- a/internal/ingress/controller/template/template.go
+++ b/internal/ingress/controller/template/template.go
@ -265,6 +265,7 @@ var (
 		"buildAuthSignURL":                   buildAuthSignURL,
 		"buildAuthSignURLLocation":           buildAuthSignURLLocation,
 		"buildOpentracing":                   buildOpentracing,
+		"buildOpentelemetry":                 buildOpentelemetry,
 		"proxySetHeader":                     proxySetHeader,
 		"buildInfluxDB":                      buildInfluxDB,
 		"enforceRegexModifier":               enforceRegexModifier,
@ -274,7 +275,9 @@ var (
 		"buildHTTPListener":                  buildHTTPListener,
 		"buildHTTPSListener":                 buildHTTPSListener,
 		"buildOpentracingForLocation":        buildOpentracingForLocation,
+		"buildOpentelemetryForLocation":      buildOpentelemetryForLocation,
 		"shouldLoadOpentracingModule":        shouldLoadOpentracingModule,
+		"shouldLoadOpentelemetryModule":      shouldLoadOpentelemetryModule,
 		"buildModSecurityForLocation":        buildModSecurityForLocation,
 		"buildMirrorLocations":               buildMirrorLocations,
 		"shouldLoadAuthDigestModule":         shouldLoadAuthDigestModule,
@ -1239,6 +1242,33 @@ func buildOpentracing(c interface{}, s interface{}) string {
 	return buf.String()
 }

+func buildOpentelemetry(c interface{}, s interface{}) string {
+	cfg, ok := c.(config.Configuration)
+	if !ok {
+		klog.Errorf("expected a 'config.Configuration' type but %T was returned", c)
+		return ""
+	}
+
+	servers, ok := s.([]*ingress.Server)
+	if !ok {
+		klog.Errorf("expected an '[]*ingress.Server' type but %T was returned", s)
+		return ""
+	}
+
+	if !shouldLoadOpentelemetryModule(cfg, servers) {
+		return ""
+	}
+
+	buf := bytes.NewBufferString("")
+
+	buf.WriteString("\r\n")
+
+	if cfg.OpentelemetryOperationName != "" {
+		buf.WriteString(fmt.Sprintf("opentelemetry_operation_name \"%s\";\n", cfg.OpentelemetryOperationName))
+	}
+	return buf.String()
+}
+
 // buildInfluxDB produces the single line configuration
 // needed by the InfluxDB module to send request's metrics
 // for the current resource
@ -1360,6 +1390,13 @@ func opentracingPropagateContext(location *ingress.Location) string {
 	return "opentracing_propagate_context;"
 }

+func opentelemetryPropagateContext(location *ingress.Location) string {
+	if location == nil {
+		return ""
+	}
+	return "opentelemetry_propagate;"
+}
+
 // shouldLoadModSecurityModule determines whether or not the ModSecurity module needs to be loaded.
 // First, it checks if `enable-modsecurity` is set in the ConfigMap. If it is not, it iterates over all locations to
 // check if ModSecurity is enabled by the annotation `nginx.ingress.kubernetes.io/enable-modsecurity`.
@ -1575,6 +1612,36 @@ func buildOpentracingForLocation(isOTEnabled bool, isOTTrustSet bool, location *
 	return opc
 }

+func buildOpentelemetryForLocation(isOTEnabled bool, isOTTrustSet bool, location *ingress.Location) string {
+	isOTEnabledInLoc := location.Opentelemetry.Enabled
+	isOTSetInLoc := location.Opentelemetry.Set
+
+	if isOTEnabled {
+		if isOTSetInLoc && !isOTEnabledInLoc {
+			return "opentelemetry off;"
+		}
+	} else if !isOTSetInLoc || !isOTEnabledInLoc {
+		return ""
+	}
+
+	opc := opentelemetryPropagateContext(location)
+	if opc != "" {
+		opc = fmt.Sprintf("opentelemetry on;\n%v", opc)
+	}
+
+	if location.Opentelemetry.OperationName != "" {
+		opc = opc + "\nopentelemetry_operation_name " + location.Opentelemetry.OperationName + ";"
+	}
+
+	if (!isOTTrustSet && !location.Opentelemetry.TrustSet) ||
+		(location.Opentelemetry.TrustSet && !location.Opentelemetry.TrustEnabled) {
+		opc = opc + "\nopentelemetry_trust_incoming_spans off;"
+	} else {
+		opc = opc + "\nopentelemetry_trust_incoming_spans on;"
+	}
+	return opc
+}
+
 // shouldLoadOpentracingModule determines whether or not the Opentracing module needs to be loaded.
 // First, it checks if `enable-opentracing` is set in the ConfigMap. If it is not, it iterates over all locations to
 // check if Opentracing is enabled by the annotation `nginx.ingress.kubernetes.io/enable-opentracing`.
@ -1606,6 +1673,35 @@ func shouldLoadOpentracingModule(c interface{}, s interface{}) bool {
 	return false
 }

+// shouldLoadOpentelemetryModule determines whether or not the Opentelemetry module needs to be loaded.
+// It checks if `enable-opentelemetry` is set in the ConfigMap.
+func shouldLoadOpentelemetryModule(c interface{}, s interface{}) bool {
+	cfg, ok := c.(config.Configuration)
+	if !ok {
+		klog.Errorf("expected a 'config.Configuration' type but %T was returned", c)
+		return false
+	}
+
+	servers, ok := s.([]*ingress.Server)
+	if !ok {
+		klog.Errorf("expected an '[]*ingress.Server' type but %T was returned", s)
+		return false
+	}
+
+	if cfg.EnableOpentelemetry {
+		return true
+	}
+
+	for _, server := range servers {
+		for _, location := range server.Locations {
+			if location.Opentelemetry.Enabled {
+				return true
+			}
+		}
+	}
+	return false
+}
+
 func buildModSecurityForLocation(cfg config.Configuration, location *ingress.Location) string {
 	isMSEnabledInLoc := location.ModSecurity.Enable
 	isMSEnableSetInLoc := location.ModSecurity.EnableSet
--- a/internal/ingress/controller/template/template_test.go
+++ b/internal/ingress/controller/template/template_test.go
@ -37,6 +37,7 @@ import (
 	"k8s.io/ingress-nginx/internal/ingress/annotations/authreq"
 	"k8s.io/ingress-nginx/internal/ingress/annotations/influxdb"
 	"k8s.io/ingress-nginx/internal/ingress/annotations/modsecurity"
+	"k8s.io/ingress-nginx/internal/ingress/annotations/opentelemetry"
 	"k8s.io/ingress-nginx/internal/ingress/annotations/opentracing"
 	"k8s.io/ingress-nginx/internal/ingress/annotations/ratelimit"
 	"k8s.io/ingress-nginx/internal/ingress/annotations/rewrite"
@ -1150,6 +1151,26 @@ func TestOpentracingPropagateContext(t *testing.T) {
 	}
 }

+func TestOpentelemetryPropagateContext(t *testing.T) {
+	tests := map[*ingress.Location]string{
+		{BackendProtocol: "HTTP"}:      "opentelemetry_propagate;",
+		{BackendProtocol: "HTTPS"}:     "opentelemetry_propagate;",
+		{BackendProtocol: "AUTO_HTTP"}: "opentelemetry_propagate;",
+		{BackendProtocol: "GRPC"}:      "opentelemetry_propagate;",
+		{BackendProtocol: "GRPCS"}:     "opentelemetry_propagate;",
+		{BackendProtocol: "AJP"}:       "opentelemetry_propagate;",
+		{BackendProtocol: "FCGI"}:      "opentelemetry_propagate;",
+		nil:                            "",
+	}
+
+	for loc, expectedDirective := range tests {
+		actualDirective := opentelemetryPropagateContext(loc)
+		if actualDirective != expectedDirective {
+			t.Errorf("Expected %v but returned %v", expectedDirective, actualDirective)
+		}
+	}
+}
+
 func TestGetIngressInformation(t *testing.T) {

 	testcases := map[string]struct {
@ -1723,6 +1744,37 @@ func TestBuildOpenTracing(t *testing.T) {

 }

+func TestBuildOpenTelemetry(t *testing.T) {
+	invalidType := &ingress.Ingress{}
+	expected := ""
+	actual := buildOpentelemetry(invalidType, []*ingress.Server{})
+
+	if expected != actual {
+		t.Errorf("Expected '%v' but returned '%v'", expected, actual)
+	}
+
+	cfgNoHost := config.Configuration{
+		EnableOpentelemetry: true,
+	}
+	expected = "\r\n"
+	actual = buildOpentelemetry(cfgNoHost, []*ingress.Server{})
+
+	if expected != actual {
+		t.Errorf("Expected '%v' but returned '%v'", expected, actual)
+	}
+
+	cfgOpenTelemetry := config.Configuration{
+		EnableOpentelemetry:        true,
+		OpentelemetryOperationName: "my-operation-name",
+	}
+	expected = "\r\n"
+	expected += "opentelemetry_operation_name \"my-operation-name\";\n"
+	actual = buildOpentelemetry(cfgOpenTelemetry, []*ingress.Server{})
+	if expected != actual {
+		t.Errorf("Expected '%v' but returned '%v'", expected, actual)
+	}
+}
+
 func TestEnforceRegexModifier(t *testing.T) {
 	invalidType := &ingress.Ingress{}
 	expected := false
@ -1903,6 +1955,107 @@ func TestShouldLoadOpentracingModule(t *testing.T) {
 	}
 }

+func TestOpentelemetryForLocation(t *testing.T) {
+	trueVal := true
+	falseVal := false
+
+	loadOT := `opentelemetry on;
+opentelemetry_propagate;
+opentelemetry_trust_incoming_spans on;`
+	loadOTUntrustedSpan := `opentelemetry on;
+opentelemetry_propagate;
+opentelemetry_trust_incoming_spans off;`
+	testCases := []struct {
+		description     string
+		globalOT        bool
+		isSetInLoc      bool
+		isOTInLoc       *bool
+		globalTrust     bool
+		isTrustSetInLoc bool
+		isTrustInLoc    *bool
+		expected        string
+	}{
+		{"globally enabled, without annotation", true, false, nil, true, false, nil, loadOT},
+		{"globally enabled and enabled in location", true, true, &trueVal, true, false, nil, loadOT},
+		{"globally disabled and not enabled in location", false, false, nil, true, false, nil, ""},
+		{"globally disabled but enabled in location", false, true, &trueVal, true, false, nil, loadOT},
+		{"globally trusted, not trusted in location", true, false, nil, true, true, &falseVal, loadOTUntrustedSpan},
+		{"not globally trusted, trust set in location", true, false, nil, false, true, &trueVal, loadOT},
+		{"not globally trusted, trust not set in location", true, false, nil, false, false, nil, loadOTUntrustedSpan},
+	}
+
+	for _, testCase := range testCases {
+		il := &ingress.Location{
+			Opentelemetry: opentelemetry.Config{Set: testCase.isSetInLoc, TrustSet: testCase.isTrustSetInLoc},
+		}
+		if il.Opentelemetry.Set {
+			il.Opentelemetry.Enabled = *testCase.isOTInLoc
+		}
+		if il.Opentelemetry.TrustSet {
+			il.Opentelemetry.TrustEnabled = *testCase.isTrustInLoc
+		}
+
+		actual := buildOpentelemetryForLocation(testCase.globalOT, testCase.globalTrust, il)
+
+		if testCase.expected != actual {
+			t.Errorf("%v: expected '%v' but returned '%v'", testCase.description, testCase.expected, actual)
+		}
+	}
+}
+
+func TestShouldLoadOpentelemetryModule(t *testing.T) {
+	// ### Invalid argument type tests ###
+	// The first tests should return false.
+	expected := false
+
+	invalidType := &ingress.Ingress{}
+	actual := shouldLoadOpentelemetryModule(config.Configuration{}, invalidType)
+	if expected != actual {
+		t.Errorf("Expected '%v' but returned '%v'", expected, actual)
+	}
+
+	actual = shouldLoadOpentelemetryModule(invalidType, []*ingress.Server{})
+	if expected != actual {
+		t.Errorf("Expected '%v' but returned '%v'", expected, actual)
+	}
+
+	// ### Functional tests ###
+	actual = shouldLoadOpentelemetryModule(config.Configuration{}, []*ingress.Server{})
+	if expected != actual {
+		t.Errorf("Expected '%v' but returned '%v'", expected, actual)
+	}
+
+	// All further tests should return true.
+	expected = true
+
+	configuration := config.Configuration{EnableOpentelemetry: true}
+	actual = shouldLoadOpentelemetryModule(configuration, []*ingress.Server{})
+	if expected != actual {
+		t.Errorf("Expected '%v' but returned '%v'", expected, actual)
+	}
+
+	servers := []*ingress.Server{
+		{
+			Locations: []*ingress.Location{
+				{
+					Opentelemetry: opentelemetry.Config{
+						Enabled: true,
+					},
+				},
+			},
+		},
+	}
+	actual = shouldLoadOpentelemetryModule(config.Configuration{}, servers)
+	if expected != actual {
+		t.Errorf("Expected '%v' but returned '%v'", expected, actual)
+	}
+
+	actual = shouldLoadOpentelemetryModule(configuration, servers)
+	if expected != actual {
+		t.Errorf("Expected '%v' but returned '%v'", expected, actual)
+	}
+}
+
 func TestModSecurityForLocation(t *testing.T) {
 	loadModule := `modsecurity on;
 `
--- a/mkdocs.yml
+++ b/mkdocs.yml
@ -101,6 +101,7 @@ nav:
      - Third party addons:
          - ModSecurity Web Application Firewall: "user-guide/third-party-addons/modsecurity.md"
          - OpenTracing: "user-guide/third-party-addons/opentracing.md"
+          - OpenTelemetry: "user-guide/third-party-addons/opentelemetry.md"
  - Examples:
      - Introduction: "examples/index.md"
      - Prerequisites: "examples/PREREQUISITES.md"
--- a/pkg/apis/ingress/types.go
+++ b/pkg/apis/ingress/types.go
@ -35,6 +35,7 @@ import (
 	"k8s.io/ingress-nginx/internal/ingress/annotations/log"
 	"k8s.io/ingress-nginx/internal/ingress/annotations/mirror"
 	"k8s.io/ingress-nginx/internal/ingress/annotations/modsecurity"
+	"k8s.io/ingress-nginx/internal/ingress/annotations/opentelemetry"
 	"k8s.io/ingress-nginx/internal/ingress/annotations/opentracing"
 	"k8s.io/ingress-nginx/internal/ingress/annotations/proxy"
 	"k8s.io/ingress-nginx/internal/ingress/annotations/proxyssl"
@ -360,6 +361,9 @@ type Location struct {
 	// Opentracing allows the global opentracing setting to be overridden for a location
 	// +optional
 	Opentracing opentracing.Config `json:"opentracing"`
+	// Opentelemetry allows the global opentelemetry setting to be overridden for a location
+	// +optional
+	Opentelemetry opentelemetry.Config `json:"opentelemetry"`
 }

 // SSLPassthroughBackend describes a SSL upstream server configured
--- a/pkg/apis/ingress/types_equals.go
+++ b/pkg/apis/ingress/types_equals.go
@ -464,6 +464,10 @@ func (l1 *Location) Equal(l2 *Location) bool {
 		return false
 	}

+	if !l1.Opentelemetry.Equal(&l2.Opentelemetry) {
+		return false
+	}
+
 	if !l1.Mirror.Equal(&l2.Mirror) {
 		return false
 	}
--- a/rootfs/etc/nginx/template/nginx.tmpl
+++ b/rootfs/etc/nginx/template/nginx.tmpl
@ -37,6 +37,10 @@ load_module /etc/nginx/modules/ngx_http_modsecurity_module.so;
 load_module /etc/nginx/modules/ngx_http_opentracing_module.so;
 {{ end }}

+{{ if (shouldLoadOpentelemetryModule $cfg $servers) }}
+load_module /modules_mount/etc/nginx/modules/otel/otel_ngx_module.so;
+{{ end }}
+
 daemon off;

 worker_processes {{ $cfg.WorkerProcesses }};
@ -64,6 +68,10 @@ events {
 }

 http {
+    {{ if (shouldLoadOpentelemetryModule $cfg $servers) }}
+    opentelemetry_config {{ $cfg.OpentelemetryConfig }};
+    {{ end }}
+
    lua_package_path "/etc/nginx/lua/?.lua;;";

    {{ buildLuaSharedDictionaries $cfg $servers }}
@ -322,6 +330,7 @@ http {
    limit_conn_status               {{ $cfg.LimitConnStatusCode }};

    {{ buildOpentracing $cfg $servers }}
+    {{ buildOpentelemetry $cfg $servers }}

    include /etc/nginx/mime.types;
    default_type {{ $cfg.DefaultType }};
@ -695,6 +704,9 @@ http {
        opentracing off;
        {{ end }}

+        {{ if $cfg.EnableOpentelemetry }}
+        opentelemetry off;
+        {{ end }}
        location {{ $healthzURI }} {
            return 200;
        }
@ -1063,6 +1075,11 @@ stream {
            opentracing_propagate_context;
            {{ end }}

+            {{ if (or $all.Cfg.EnableOpentelemetry $location.Opentelemetry.Enabled) }}
+            opentelemetry on;
+            opentelemetry_propagate;
+            {{ end }}
+
            access_log off;

            # Ensure that modsecurity will not run on an internal location as this is not accessible from outside
@ -1204,6 +1221,7 @@ stream {
            set $global_rate_limit_exceeding n;

            {{ buildOpentracingForLocation $all.Cfg.EnableOpentracing $all.Cfg.OpentracingTrustIncomingSpan $location }}
+            {{ buildOpentelemetryForLocation $all.Cfg.EnableOpentelemetry $all.Cfg.OpentelemetryTrustIncomingSpan $location }}

            {{ if $location.Mirror.Source }}
            mirror {{ $location.Mirror.Source }};
@ -1520,6 +1538,10 @@ stream {
            opentracing off;
            {{ end }}

+            {{ if $all.Cfg.EnableOpentelemetry }}
+            opentelemetry off;
+            {{ end }}
+
            access_log off;
            return 200;
        }
@ -1531,6 +1553,10 @@ stream {
            opentracing off;
            {{ end }}

+            {{ if $all.Cfg.EnableOpentelemetry }}
+            opentelemetry off;
+            {{ end }}
+
            {{ range $v := $all.NginxStatusIpv4Whitelist }}
            allow {{ $v }};
            {{ end }}
--- a/test/e2e/framework/exec.go
+++ b/test/e2e/framework/exec.go
@ -21,6 +21,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"os"
 	"os/exec"
 	"regexp"
 	"strconv"
@ -111,7 +112,11 @@ func (f *Framework) NamespaceContent() (string, error) {
 // newIngressController deploys a new NGINX Ingress controller in a namespace
 func (f *Framework) newIngressController(namespace string, namespaceOverlay string) error {
 	// Creates an nginx deployment
-	cmd := exec.Command("./wait-for-nginx.sh", namespace, namespaceOverlay)
+	isChroot, ok := os.LookupEnv("IS_CHROOT")
+	if !ok {
+		isChroot = "false"
+	}
+	cmd := exec.Command("./wait-for-nginx.sh", namespace, namespaceOverlay, isChroot)
 	out, err := cmd.CombinedOutput()
 	if err != nil {
 		return fmt.Errorf("unexpected error waiting for ingress controller deployment: %v.\nLogs:\n%v", err, string(out))
--- a/test/e2e/run-e2e-suite.sh
+++ b/test/e2e/run-e2e-suite.sh
@ -76,6 +76,7 @@ kubectl run --rm \
  --restart=Never \
  --env="E2E_NODES=${E2E_NODES}" \
  --env="FOCUS=${FOCUS}" \
+  --env="IS_CHROOT=${IS_CHROOT:-false}"\
  --env="E2E_CHECK_LEAKS=${E2E_CHECK_LEAKS}" \
  --env="NGINX_BASE_IMAGE=${NGINX_BASE_IMAGE}" \
  --overrides='{ "apiVersion": "v1", "spec":{"serviceAccountName": "ingress-nginx-e2e"}}' \
--- a/test/e2e/settings/opentelemetry.go
+++ b/test/e2e/settings/opentelemetry.go
@ -0,0 +1,119 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package settings
+
+import (
+	"strings"
+
+	"github.com/onsi/ginkgo/v2"
+
+	"k8s.io/ingress-nginx/test/e2e/framework"
+)
+
+const (
+	enableOpentelemetry            = "enable-opentelemetry"
+	opentelemetryTrustIncomingSpan = "opentelemetry-trust-incoming-span"
+
+	opentelemetryOperationName         = "opentelemetry-operation-name"
+	opentelemetryLocationOperationName = "opentelemetry-location-operation-name"
+	opentelemetryConfig                = "opentelemetry-config"
+	opentelemetryConfigPath            = "/etc/nginx/opentelemetry.toml"
+)
+
+var _ = framework.IngressNginxDescribe("Configure Opentelemetry", func() {
+	f := framework.NewDefaultFramework("enable-opentelemetry")
+
+	ginkgo.BeforeEach(func() {
+		f.NewEchoDeployment()
+	})
+
+	ginkgo.AfterEach(func() {
+	})
+
+	ginkgo.It("should not exists opentelemetry directive", func() {
+		config := map[string]string{}
+		config[enableOpentelemetry] = "false"
+		f.SetNginxConfigMapData(config)
+
+		f.EnsureIngress(framework.NewSingleIngress(enableOpentelemetry, "/", enableOpentelemetry, f.Namespace, "http-svc", 80, nil))
+
+		f.WaitForNginxConfiguration(
+			func(cfg string) bool {
+				return !strings.Contains(cfg, "opentelemetry on")
+			})
+	})
+
+	ginkgo.It("should exists opentelemetry directive when is enabled", func() {
+		config := map[string]string{}
+		config[enableOpentelemetry] = "true"
+		config[opentelemetryConfig] = opentelemetryConfigPath
+		f.SetNginxConfigMapData(config)
+
+		f.EnsureIngress(framework.NewSingleIngress(enableOpentelemetry, "/", enableOpentelemetry, f.Namespace, "http-svc", 80, nil))
+
+		f.WaitForNginxConfiguration(
+			func(cfg string) bool {
+				return strings.Contains(cfg, "opentelemetry on")
+			})
+	})
+
+	ginkgo.It("should include opentelemetry_trust_incoming_spans on directive when enabled", func() {
+		config := map[string]string{}
+		config[enableOpentelemetry] = "true"
+		config[opentelemetryConfig] = opentelemetryConfigPath
+		config[opentelemetryTrustIncomingSpan] = "true"
+		f.SetNginxConfigMapData(config)
+
+		f.EnsureIngress(framework.NewSingleIngress(enableOpentelemetry, "/", enableOpentelemetry, f.Namespace, "http-svc", 80, nil))
+
+		f.WaitForNginxConfiguration(
+			func(cfg string) bool {
+				return strings.Contains(cfg, "opentelemetry_trust_incoming_spans on")
+			})
+	})
+
+	ginkgo.It("should not exists opentelemetry_operation_name directive when is empty", func() {
+		config := map[string]string{}
+		config[enableOpentelemetry] = "true"
+		config[opentelemetryConfig] = opentelemetryConfigPath
+		config[opentelemetryOperationName] = ""
+		f.SetNginxConfigMapData(config)
+
+		f.EnsureIngress(framework.NewSingleIngress(enableOpentelemetry, "/", enableOpentelemetry, f.Namespace, "http-svc", 80, nil))
+
+		f.WaitForNginxConfiguration(
+			func(cfg string) bool {
+				return !strings.Contains(cfg, "opentelemetry_operation_name")
+			})
+	})
+
+	ginkgo.It("should exists opentelemetry_operation_name directive when is configured", func() {
+		config := map[string]string{}
+		config[enableOpentelemetry] = "true"
+		config[opentelemetryConfig] = opentelemetryConfigPath
+		config[opentelemetryOperationName] = "HTTP $request_method $uri"
+		f.SetNginxConfigMapData(config)
+
+		f.EnsureIngress(framework.NewSingleIngress(enableOpentelemetry, "/", enableOpentelemetry, f.Namespace, "http-svc", 80, nil))
+
+		f.WaitForNginxConfiguration(
+			func(cfg string) bool {
+				return strings.Contains(cfg, `opentelemetry_operation_name "HTTP $request_method $uri"`)
+			})
+	})
+
+})
--- a/test/e2e/wait-for-nginx.sh
+++ b/test/e2e/wait-for-nginx.sh
@ -23,6 +23,7 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"

 export NAMESPACE=$1
 export NAMESPACE_OVERLAY=$2
+export IS_CHROOT=$3

 echo "deploying NGINX Ingress controller in namespace $NAMESPACE"

@ -46,6 +47,16 @@ metadata:

 EOF

+OTEL_MODULE=$(cat <<EOF
+  opentelemetry:
+    enabled: true
+EOF
+)
+
+if [[ "$NAMESPACE_OVERLAY" != "enable-opentelemetry" ]]; then
+  OTEL_MODULE=""
+fi
+
 # Use the namespace overlay if it was requested
 if [[ ! -z "$NAMESPACE_OVERLAY" && -d "$DIR/namespace-overlays/$NAMESPACE_OVERLAY" ]]; then
    echo "Namespace overlay $NAMESPACE_OVERLAY is being used for namespace $NAMESPACE"
@ -59,7 +70,7 @@ fullnameOverride: nginx-ingress
 controller:
  image:
    repository: ingress-controller/controller
-    chroot: true
+    chroot: ${IS_CHROOT}
    tag: 1.0.0-dev
    digest:
    digestChroot:
@ -100,6 +111,8 @@ controller:
      hostPath:
        path: /tmp/coredump

+${OTEL_MODULE}
+
 rbac:
  create: true
  scope: true