From ae4cea0db21f139dfa004636bde593aa2b431540 Mon Sep 17 00:00:00 2001 From: Marco Ebert Date: Mon, 29 Jan 2024 23:28:55 +0100 Subject: [PATCH 01/12] Docs: Convert `opentelemetry.md` from CRLF to LF. --- .../third-party-addons/opentelemetry.md | 628 +++++++++--------- 1 file changed, 314 insertions(+), 314 deletions(-) diff --git a/docs/user-guide/third-party-addons/opentelemetry.md b/docs/user-guide/third-party-addons/opentelemetry.md index d2cf56bf5..af7fd2b14 100644 --- a/docs/user-guide/third-party-addons/opentelemetry.md +++ b/docs/user-guide/third-party-addons/opentelemetry.md @@ -1,314 +1,314 @@ -# OpenTelemetry - -Enables requests served by NGINX for distributed telemetry via The OpenTelemetry Project. - -Using the third party module [opentelemetry-cpp-contrib/nginx](https://github.com/open-telemetry/opentelemetry-cpp-contrib/tree/main/instrumentation/nginx) the Ingress-Nginx Controller can configure NGINX to enable [OpenTelemetry](http://opentelemetry.io) instrumentation. -By default this feature is disabled. - -Check out this demo showcasing OpenTelemetry in Ingress NGINX. The video provides an overview and -practical demonstration of how OpenTelemetry can be utilized in Ingress NGINX for observability -and monitoring purposes. - -

- - Video Thumbnail - -

- -

Demo: OpenTelemetry in Ingress NGINX.

- -## Usage - -To enable the instrumentation we must enable OpenTelemetry in the configuration ConfigMap: -```yaml -data: - enable-opentelemetry: "true" -``` - -To enable or disable instrumentation for a single Ingress, use -the `enable-opentelemetry` annotation: -```yaml -kind: Ingress -metadata: - annotations: - nginx.ingress.kubernetes.io/enable-opentelemetry: "true" -``` - -We must also set the host to use when uploading traces: - -```yaml -otlp-collector-host: "otel-coll-collector.otel.svc" -``` -NOTE: While the option is called `otlp-collector-host`, you will need to point this to any backend that receives otlp-grpc. - -Next you will need to deploy a distributed telemetry system which uses OpenTelemetry. -[opentelemetry-collector](https://github.com/open-telemetry/opentelemetry-collector), [Jaeger](https://www.jaegertracing.io/) -[Tempo](https://github.com/grafana/tempo), and [zipkin](https://zipkin.io/) -have been tested. - -Other optional configuration options: -```yaml -# specifies the name to use for the server span -opentelemetry-operation-name - -# sets whether or not to trust incoming telemetry spans -opentelemetry-trust-incoming-span - -# specifies the port to use when uploading traces, Default: 4317 -otlp-collector-port - -# specifies the service name to use for any traces created, Default: nginx -otel-service-name - -# The maximum queue size. After the size is reached data are dropped. -otel-max-queuesize - -# The delay interval in milliseconds between two consecutive exports. -otel-schedule-delay-millis - -# How long the export can run before it is cancelled. -otel-schedule-delay-millis - -# The maximum batch size of every export. It must be smaller or equal to maxQueueSize. -otel-max-export-batch-size - -# specifies sample rate for any traces created, Default: 0.01 -otel-sampler-ratio - -# specifies the sampler to be used when sampling traces. -# The available samplers are: AlwaysOn, AlwaysOff, TraceIdRatioBased, Default: AlwaysOff -otel-sampler - -# Uses sampler implementation which by default will take a sample if parent Activity is sampled, Default: false -otel-sampler-parent-based -``` - -Note that you can also set whether to trust incoming spans (global default is true) per-location using annotations like the following: -```yaml -kind: Ingress -metadata: - annotations: - nginx.ingress.kubernetes.io/opentelemetry-trust-incoming-span: "true" -``` - -## Examples - -The following examples show how to deploy and test different distributed telemetry systems. These example can be performed using Docker Desktop. - -In the [esigo/nginx-example](https://github.com/esigo/nginx-example) -GitHub repository is an example of a simple hello service: - -```mermaid -graph TB - subgraph Browser - start["http://esigo.dev/hello/nginx"] - end - - subgraph app - sa[service-a] - sb[service-b] - sa --> |name: nginx| sb - sb --> |hello nginx!| sa - end - - subgraph otel - otc["Otel Collector"] - end - - subgraph observability - tempo["Tempo"] - grafana["Grafana"] - backend["Jaeger"] - zipkin["Zipkin"] - end - - subgraph ingress-nginx - ngx[nginx] - end - - subgraph ngx[nginx] - ng[nginx] - om[OpenTelemetry module] - end - - subgraph Node - app - otel - observability - ingress-nginx - om --> |otlp-gRPC| otc --> |jaeger| backend - otc --> |zipkin| zipkin - otc --> |otlp-gRPC| tempo --> grafana - sa --> |otlp-gRPC| otc - sb --> |otlp-gRPC| otc - start --> ng --> sa - end -``` - -To install the example and collectors run: - -1. Enable Ingress addon with: - - ```yaml - opentelemetry: - enabled: true - image: registry.k8s.io/ingress-nginx/opentelemetry-1.25.3:v20240813-b933310d@sha256:f7604ac0547ed64d79b98d92133234e66c2c8aade3c1f4809fed5eec1fb7f922 - containerSecurityContext: - allowPrivilegeEscalation: false - ``` - -2. Enable OpenTelemetry and set the otlp-collector-host: - - ```yaml - $ echo ' - apiVersion: v1 - kind: ConfigMap - data: - enable-opentelemetry: "true" - opentelemetry-config: "/etc/nginx/opentelemetry.toml" - opentelemetry-operation-name: "HTTP $request_method $service_name $uri" - opentelemetry-trust-incoming-span: "true" - otlp-collector-host: "otel-coll-collector.otel.svc" - otlp-collector-port: "4317" - otel-max-queuesize: "2048" - otel-schedule-delay-millis: "5000" - otel-max-export-batch-size: "512" - otel-service-name: "nginx-proxy" # Opentelemetry resource name - otel-sampler: "AlwaysOn" # Also: AlwaysOff, TraceIdRatioBased - otel-sampler-ratio: "1.0" - otel-sampler-parent-based: "false" - metadata: - name: ingress-nginx-controller - namespace: ingress-nginx - ' | kubectl replace -f - - ``` - -4. Deploy otel-collector, grafana and Jaeger backend: - - ```bash - # add helm charts needed for grafana and OpenTelemetry collector - helm repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts - helm repo add grafana https://grafana.github.io/helm-charts - helm repo update - # deploy cert-manager needed for OpenTelemetry collector operator - kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.9.1/cert-manager.yaml - # create observability namespace - kubectl apply -f https://raw.githubusercontent.com/esigo/nginx-example/main/observability/namespace.yaml - # install OpenTelemetry collector operator - helm upgrade --install otel-collector-operator -n otel --create-namespace open-telemetry/opentelemetry-operator - # deploy OpenTelemetry collector - kubectl apply -f https://raw.githubusercontent.com/esigo/nginx-example/main/observability/collector.yaml - # deploy Jaeger all-in-one - kubectl apply -f https://github.com/jaegertracing/jaeger-operator/releases/download/v1.37.0/jaeger-operator.yaml -n observability - kubectl apply -f https://raw.githubusercontent.com/esigo/nginx-example/main/observability/jaeger.yaml -n observability - # deploy zipkin - kubectl apply -f https://raw.githubusercontent.com/esigo/nginx-example/main/observability/zipkin.yaml -n observability - # deploy tempo and grafana - helm upgrade --install tempo grafana/tempo --create-namespace -n observability - helm upgrade -f https://raw.githubusercontent.com/esigo/nginx-example/main/observability/grafana/grafana-values.yaml --install grafana grafana/grafana --create-namespace -n observability - ``` - -3. Build and deploy demo app: - - ```bash - # build images - make images - - # deploy demo app: - make deploy-app - ``` - -5. Make a few requests to the Service: - - ```bash - kubectl port-forward --namespace=ingress-nginx service/ingress-nginx-controller 8090:80 - curl http://esigo.dev:8090/hello/nginx - - - StatusCode : 200 - StatusDescription : OK - Content : {"v":"hello nginx!"} - - RawContent : HTTP/1.1 200 OK - Connection: keep-alive - Content-Length: 21 - Content-Type: text/plain; charset=utf-8 - Date: Mon, 10 Oct 2022 17:43:33 GMT - - {"v":"hello nginx!"} - - Forms : {} - Headers : {[Connection, keep-alive], [Content-Length, 21], [Content-Type, text/plain; charset=utf-8], [Date, - Mon, 10 Oct 2022 17:43:33 GMT]} - Images : {} - InputFields : {} - Links : {} - ParsedHtml : System.__ComObject - RawContentLength : 21 - ``` - -6. View the Grafana UI: - - ```bash - kubectl port-forward --namespace=observability service/grafana 3000:80 - ``` - In the Grafana interface we can see the details: - ![grafana screenshot](../../images/otel-grafana-demo.png "grafana screenshot") - -7. View the Jaeger UI: - - ```bash - kubectl port-forward --namespace=observability service/jaeger-all-in-one-query 16686:16686 - ``` - In the Jaeger interface we can see the details: - ![Jaeger screenshot](../../images/otel-jaeger-demo.png "Jaeger screenshot") - -8. View the Zipkin UI: - - ```bash - kubectl port-forward --namespace=observability service/zipkin 9411:9411 - ``` - In the Zipkin interface we can see the details: - ![zipkin screenshot](../../images/otel-zipkin-demo.png "zipkin screenshot") - -## Migration from OpenTracing, Jaeger, Zipkin and Datadog - -If you are migrating from OpenTracing, Jaeger, Zipkin, or Datadog to OpenTelemetry, -you may need to update various annotations and configurations. Here are the mappings -for common annotations and configurations: - -### Annotations - -| Legacy | OpenTelemetry | -|---------------------------------------------------------------|-----------------------------------------------------------------| -| `nginx.ingress.kubernetes.io/enable-opentracing` | `nginx.ingress.kubernetes.io/enable-opentelemetry` | -| `nginx.ingress.kubernetes.io/opentracing-trust-incoming-span` | `nginx.ingress.kubernetes.io/opentelemetry-trust-incoming-span` | - -### Configs - -| Legacy | OpenTelemetry | -|---------------------------------------|----------------------------------------------| -| `opentracing-operation-name` | `opentelemetry-operation-name` | -| `opentracing-location-operation-name` | `opentelemetry-operation-name` | -| `opentracing-trust-incoming-span` | `opentelemetry-trust-incoming-span` | -| `zipkin-collector-port` | `otlp-collector-port` | -| `zipkin-service-name` | `otel-service-name` | -| `zipkin-sample-rate` | `otel-sampler-ratio` | -| `jaeger-collector-port` | `otlp-collector-port` | -| `jaeger-endpoint` | `otlp-collector-port`, `otlp-collector-host` | -| `jaeger-service-name` | `otel-service-name` | -| `jaeger-propagation-format` | `N/A` | -| `jaeger-sampler-type` | `otel-sampler` | -| `jaeger-sampler-param` | `otel-sampler` | -| `jaeger-sampler-host` | `N/A` | -| `jaeger-sampler-port` | `N/A` | -| `jaeger-trace-context-header-name` | `N/A` | -| `jaeger-debug-header` | `N/A` | -| `jaeger-baggage-header` | `N/A` | -| `jaeger-tracer-baggage-header-prefix` | `N/A` | -| `datadog-collector-port` | `otlp-collector-port` | -| `datadog-service-name` | `otel-service-name` | -| `datadog-environment` | `N/A` | -| `datadog-operation-name-override` | `N/A` | -| `datadog-priority-sampling` | `otel-sampler` | -| `datadog-sample-rate` | `otel-sampler-ratio` | +# OpenTelemetry + +Enables requests served by NGINX for distributed telemetry via The OpenTelemetry Project. + +Using the third party module [opentelemetry-cpp-contrib/nginx](https://github.com/open-telemetry/opentelemetry-cpp-contrib/tree/main/instrumentation/nginx) the Ingress-Nginx Controller can configure NGINX to enable [OpenTelemetry](http://opentelemetry.io) instrumentation. +By default this feature is disabled. + +Check out this demo showcasing OpenTelemetry in Ingress NGINX. The video provides an overview and +practical demonstration of how OpenTelemetry can be utilized in Ingress NGINX for observability +and monitoring purposes. + +

+ + Video Thumbnail + +

+ +

Demo: OpenTelemetry in Ingress NGINX.

+ +## Usage + +To enable the instrumentation we must enable OpenTelemetry in the configuration ConfigMap: +```yaml +data: + enable-opentelemetry: "true" +``` + +To enable or disable instrumentation for a single Ingress, use +the `enable-opentelemetry` annotation: +```yaml +kind: Ingress +metadata: + annotations: + nginx.ingress.kubernetes.io/enable-opentelemetry: "true" +``` + +We must also set the host to use when uploading traces: + +```yaml +otlp-collector-host: "otel-coll-collector.otel.svc" +``` +NOTE: While the option is called `otlp-collector-host`, you will need to point this to any backend that receives otlp-grpc. + +Next you will need to deploy a distributed telemetry system which uses OpenTelemetry. +[opentelemetry-collector](https://github.com/open-telemetry/opentelemetry-collector), [Jaeger](https://www.jaegertracing.io/) +[Tempo](https://github.com/grafana/tempo), and [zipkin](https://zipkin.io/) +have been tested. + +Other optional configuration options: +```yaml +# specifies the name to use for the server span +opentelemetry-operation-name + +# sets whether or not to trust incoming telemetry spans +opentelemetry-trust-incoming-span + +# specifies the port to use when uploading traces, Default: 4317 +otlp-collector-port + +# specifies the service name to use for any traces created, Default: nginx +otel-service-name + +# The maximum queue size. After the size is reached data are dropped. +otel-max-queuesize + +# The delay interval in milliseconds between two consecutive exports. +otel-schedule-delay-millis + +# How long the export can run before it is cancelled. +otel-schedule-delay-millis + +# The maximum batch size of every export. It must be smaller or equal to maxQueueSize. +otel-max-export-batch-size + +# specifies sample rate for any traces created, Default: 0.01 +otel-sampler-ratio + +# specifies the sampler to be used when sampling traces. +# The available samplers are: AlwaysOn, AlwaysOff, TraceIdRatioBased, Default: AlwaysOff +otel-sampler + +# Uses sampler implementation which by default will take a sample if parent Activity is sampled, Default: false +otel-sampler-parent-based +``` + +Note that you can also set whether to trust incoming spans (global default is true) per-location using annotations like the following: +```yaml +kind: Ingress +metadata: + annotations: + nginx.ingress.kubernetes.io/opentelemetry-trust-incoming-span: "true" +``` + +## Examples + +The following examples show how to deploy and test different distributed telemetry systems. These example can be performed using Docker Desktop. + +In the [esigo/nginx-example](https://github.com/esigo/nginx-example) +GitHub repository is an example of a simple hello service: + +```mermaid +graph TB + subgraph Browser + start["http://esigo.dev/hello/nginx"] + end + + subgraph app + sa[service-a] + sb[service-b] + sa --> |name: nginx| sb + sb --> |hello nginx!| sa + end + + subgraph otel + otc["Otel Collector"] + end + + subgraph observability + tempo["Tempo"] + grafana["Grafana"] + backend["Jaeger"] + zipkin["Zipkin"] + end + + subgraph ingress-nginx + ngx[nginx] + end + + subgraph ngx[nginx] + ng[nginx] + om[OpenTelemetry module] + end + + subgraph Node + app + otel + observability + ingress-nginx + om --> |otlp-gRPC| otc --> |jaeger| backend + otc --> |zipkin| zipkin + otc --> |otlp-gRPC| tempo --> grafana + sa --> |otlp-gRPC| otc + sb --> |otlp-gRPC| otc + start --> ng --> sa + end +``` + +To install the example and collectors run: + +1. Enable Ingress addon with: + + ```yaml + opentelemetry: + enabled: true + image: registry.k8s.io/ingress-nginx/opentelemetry-1.25.3:v20240813-b933310d@sha256:f7604ac0547ed64d79b98d92133234e66c2c8aade3c1f4809fed5eec1fb7f922 + containerSecurityContext: + allowPrivilegeEscalation: false + ``` + +2. Enable OpenTelemetry and set the otlp-collector-host: + + ```yaml + $ echo ' + apiVersion: v1 + kind: ConfigMap + data: + enable-opentelemetry: "true" + opentelemetry-config: "/etc/nginx/opentelemetry.toml" + opentelemetry-operation-name: "HTTP $request_method $service_name $uri" + opentelemetry-trust-incoming-span: "true" + otlp-collector-host: "otel-coll-collector.otel.svc" + otlp-collector-port: "4317" + otel-max-queuesize: "2048" + otel-schedule-delay-millis: "5000" + otel-max-export-batch-size: "512" + otel-service-name: "nginx-proxy" # Opentelemetry resource name + otel-sampler: "AlwaysOn" # Also: AlwaysOff, TraceIdRatioBased + otel-sampler-ratio: "1.0" + otel-sampler-parent-based: "false" + metadata: + name: ingress-nginx-controller + namespace: ingress-nginx + ' | kubectl replace -f - + ``` + +4. Deploy otel-collector, grafana and Jaeger backend: + + ```bash + # add helm charts needed for grafana and OpenTelemetry collector + helm repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts + helm repo add grafana https://grafana.github.io/helm-charts + helm repo update + # deploy cert-manager needed for OpenTelemetry collector operator + kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.9.1/cert-manager.yaml + # create observability namespace + kubectl apply -f https://raw.githubusercontent.com/esigo/nginx-example/main/observability/namespace.yaml + # install OpenTelemetry collector operator + helm upgrade --install otel-collector-operator -n otel --create-namespace open-telemetry/opentelemetry-operator + # deploy OpenTelemetry collector + kubectl apply -f https://raw.githubusercontent.com/esigo/nginx-example/main/observability/collector.yaml + # deploy Jaeger all-in-one + kubectl apply -f https://github.com/jaegertracing/jaeger-operator/releases/download/v1.37.0/jaeger-operator.yaml -n observability + kubectl apply -f https://raw.githubusercontent.com/esigo/nginx-example/main/observability/jaeger.yaml -n observability + # deploy zipkin + kubectl apply -f https://raw.githubusercontent.com/esigo/nginx-example/main/observability/zipkin.yaml -n observability + # deploy tempo and grafana + helm upgrade --install tempo grafana/tempo --create-namespace -n observability + helm upgrade -f https://raw.githubusercontent.com/esigo/nginx-example/main/observability/grafana/grafana-values.yaml --install grafana grafana/grafana --create-namespace -n observability + ``` + +3. Build and deploy demo app: + + ```bash + # build images + make images + + # deploy demo app: + make deploy-app + ``` + +5. Make a few requests to the Service: + + ```bash + kubectl port-forward --namespace=ingress-nginx service/ingress-nginx-controller 8090:80 + curl http://esigo.dev:8090/hello/nginx + + + StatusCode : 200 + StatusDescription : OK + Content : {"v":"hello nginx!"} + + RawContent : HTTP/1.1 200 OK + Connection: keep-alive + Content-Length: 21 + Content-Type: text/plain; charset=utf-8 + Date: Mon, 10 Oct 2022 17:43:33 GMT + + {"v":"hello nginx!"} + + Forms : {} + Headers : {[Connection, keep-alive], [Content-Length, 21], [Content-Type, text/plain; charset=utf-8], [Date, + Mon, 10 Oct 2022 17:43:33 GMT]} + Images : {} + InputFields : {} + Links : {} + ParsedHtml : System.__ComObject + RawContentLength : 21 + ``` + +6. View the Grafana UI: + + ```bash + kubectl port-forward --namespace=observability service/grafana 3000:80 + ``` + In the Grafana interface we can see the details: + ![grafana screenshot](../../images/otel-grafana-demo.png "grafana screenshot") + +7. View the Jaeger UI: + + ```bash + kubectl port-forward --namespace=observability service/jaeger-all-in-one-query 16686:16686 + ``` + In the Jaeger interface we can see the details: + ![Jaeger screenshot](../../images/otel-jaeger-demo.png "Jaeger screenshot") + +8. View the Zipkin UI: + + ```bash + kubectl port-forward --namespace=observability service/zipkin 9411:9411 + ``` + In the Zipkin interface we can see the details: + ![zipkin screenshot](../../images/otel-zipkin-demo.png "zipkin screenshot") + +## Migration from OpenTracing, Jaeger, Zipkin and Datadog + +If you are migrating from OpenTracing, Jaeger, Zipkin, or Datadog to OpenTelemetry, +you may need to update various annotations and configurations. Here are the mappings +for common annotations and configurations: + +### Annotations + +| Legacy | OpenTelemetry | +|---------------------------------------------------------------|-----------------------------------------------------------------| +| `nginx.ingress.kubernetes.io/enable-opentracing` | `nginx.ingress.kubernetes.io/enable-opentelemetry` | +| `nginx.ingress.kubernetes.io/opentracing-trust-incoming-span` | `nginx.ingress.kubernetes.io/opentelemetry-trust-incoming-span` | + +### Configs + +| Legacy | OpenTelemetry | +|---------------------------------------|----------------------------------------------| +| `opentracing-operation-name` | `opentelemetry-operation-name` | +| `opentracing-location-operation-name` | `opentelemetry-operation-name` | +| `opentracing-trust-incoming-span` | `opentelemetry-trust-incoming-span` | +| `zipkin-collector-port` | `otlp-collector-port` | +| `zipkin-service-name` | `otel-service-name` | +| `zipkin-sample-rate` | `otel-sampler-ratio` | +| `jaeger-collector-port` | `otlp-collector-port` | +| `jaeger-endpoint` | `otlp-collector-port`, `otlp-collector-host` | +| `jaeger-service-name` | `otel-service-name` | +| `jaeger-propagation-format` | `N/A` | +| `jaeger-sampler-type` | `otel-sampler` | +| `jaeger-sampler-param` | `otel-sampler` | +| `jaeger-sampler-host` | `N/A` | +| `jaeger-sampler-port` | `N/A` | +| `jaeger-trace-context-header-name` | `N/A` | +| `jaeger-debug-header` | `N/A` | +| `jaeger-baggage-header` | `N/A` | +| `jaeger-tracer-baggage-header-prefix` | `N/A` | +| `datadog-collector-port` | `otlp-collector-port` | +| `datadog-service-name` | `otel-service-name` | +| `datadog-environment` | `N/A` | +| `datadog-operation-name-override` | `N/A` | +| `datadog-priority-sampling` | `otel-sampler` | +| `datadog-sample-rate` | `otel-sampler-ratio` | From 965ac64ec46c554f7d382ce0d05f4ef2471c582a Mon Sep 17 00:00:00 2001 From: Marco Ebert Date: Sun, 4 Feb 2024 21:26:56 +0100 Subject: [PATCH 02/12] CI: Rename `helm` to `chart`. --- .github/workflows/ci.yaml | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 05cf9269c..f9cccad4d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -192,13 +192,12 @@ jobs: path: docker.tar.gz retention-days: 5 - helm-lint: - name: Helm chart lint + chart-lint: + name: Chart / Lint runs-on: ubuntu-latest needs: - changes - if: | - (needs.changes.outputs.charts == 'true') || (needs.changes.outputs.baseimage == 'true') || ${{ github.event.workflow_dispatch.run_e2e == 'true' }} + if: fromJSON(needs.changes.outputs.charts) || fromJSON(needs.changes.outputs.baseimage) || fromJSON(github.event.workflow_dispatch.run_e2e) steps: - name: Checkout @@ -246,15 +245,14 @@ jobs: ./ah lint -p charts/ingress-nginx || exit 1 rm -f ./ah ./ah_1.5.0_linux_amd64.tar.gz - helm-test: - name: Helm chart testing + chart-test: + name: Chart / Test runs-on: ubuntu-latest needs: - changes - build - - helm-lint - if: | - (needs.changes.outputs.charts == 'true') || (needs.changes.outputs.baseimage == 'true') || ${{ github.event.workflow_dispatch.run_e2e == 'true' }} + - chart-lint + if: fromJSON(needs.changes.outputs.charts) || fromJSON(needs.changes.outputs.baseimage) || fromJSON(github.event.workflow_dispatch.run_e2e) strategy: matrix: From 9e8bf2b3ebb2ab9fdf63a5b2dec1251d5050ebe1 Mon Sep 17 00:00:00 2001 From: Marco Ebert Date: Fri, 14 Jun 2024 18:52:30 +0200 Subject: [PATCH 03/12] CI: Rework Python. --- .github/workflows/ci.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index f9cccad4d..ae03e78c2 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -205,13 +205,14 @@ jobs: with: fetch-depth: 0 + - name: Set up Python + uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 + with: + python-version: "3.x" + - name: Set up Helm uses: azure/setup-helm@fe7b79cd5ee1e45176fcad797de68ecaf3ca4814 # v4.2.0 - - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 - with: - python-version: '3.x' - - name: Set up chart-testing uses: helm/chart-testing-action@e6669bcd63d7cb57cb4380c33043eebe5d111992 # v2.6.1 From 312f816c1940e1fd874615d32d8c6a2ad57f78c2 Mon Sep 17 00:00:00 2001 From: Marco Ebert Date: Sun, 4 Feb 2024 21:42:04 +0100 Subject: [PATCH 04/12] CI: Rework unit testing. --- .github/workflows/ci.yaml | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index ae03e78c2..daa58e571 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -216,14 +216,6 @@ jobs: - name: Set up chart-testing uses: helm/chart-testing-action@e6669bcd63d7cb57cb4380c33043eebe5d111992 # v2.6.1 - - name: Install Helm Unit Test Plugin - run: | - helm plugin install https://github.com/helm-unittest/helm-unittest - - - name: Run Helm Unit Tests - run: | - helm unittest charts/ingress-nginx -d - - name: Run chart-testing (lint) run: ct lint --config ./.ct.yaml @@ -246,6 +238,11 @@ jobs: ./ah lint -p charts/ingress-nginx || exit 1 rm -f ./ah ./ah_1.5.0_linux_amd64.tar.gz + - name: Run unit tests + run: | + helm plugin install https://github.com/helm-unittest/helm-unittest + helm unittest charts/ingress-nginx + chart-test: name: Chart / Test runs-on: ubuntu-latest From f65a5d09a8b08047418a07159b55e3325534e166 Mon Sep 17 00:00:00 2001 From: Marco Ebert Date: Sun, 4 Feb 2024 21:42:44 +0100 Subject: [PATCH 05/12] CI: Rework cache loading. --- .github/workflows/ci.yaml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index daa58e571..2537821fd 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -266,11 +266,14 @@ jobs: go-version: ${{ needs.build.outputs.golangversion }} check-latest: true - - name: cache + - name: Download cache uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 with: name: docker.tar.gz + - name: Load cache + run: gzip --decompress --stdout docker.tar.gz | docker load + - name: fix permissions run: | sudo mkdir -p $HOME/.kube @@ -281,11 +284,6 @@ jobs: run: | kind create cluster --image=kindest/node:${{ matrix.k8s }} - - name: Load images from cache - run: | - echo "loading docker images..." - gzip -dc docker.tar.gz | docker load - - name: Test env: KIND_CLUSTER_NAME: kind From f6c88d66a6765014f1dc5a08faf3309ed1e973d4 Mon Sep 17 00:00:00 2001 From: Marco Ebert Date: Sun, 4 Feb 2024 22:36:48 +0100 Subject: [PATCH 06/12] CI: Rework chart linting. --- .github/workflows/ci.yaml | 21 +++++++++++---------- .github/workflows/helm.yaml | 21 +++++++++++---------- 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2537821fd..7d70e7deb 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -213,12 +213,9 @@ jobs: - name: Set up Helm uses: azure/setup-helm@fe7b79cd5ee1e45176fcad797de68ecaf3ca4814 # v4.2.0 - - name: Set up chart-testing + - name: Set up Helm Chart Testing uses: helm/chart-testing-action@e6669bcd63d7cb57cb4380c33043eebe5d111992 # v2.6.1 - - name: Run chart-testing (lint) - run: ct lint --config ./.ct.yaml - - name: Run helm-docs run: | GOBIN=$PWD GO111MODULE=on go install github.com/norwoodj/helm-docs/cmd/helm-docs@v1.11.0 @@ -230,13 +227,17 @@ jobs: git diff --exit-code rm -f ./helm-docs - - name: Run Artifact Hub lint + - name: Lint chart run: | - wget https://github.com/artifacthub/hub/releases/download/v1.5.0/ah_1.5.0_linux_amd64.tar.gz - echo 'ad0e44c6ea058ab6b85dbf582e88bad9fdbc64ded0d1dd4edbac65133e5c87da *ah_1.5.0_linux_amd64.tar.gz' | shasum -c - tar -xzvf ah_1.5.0_linux_amd64.tar.gz ah - ./ah lint -p charts/ingress-nginx || exit 1 - rm -f ./ah ./ah_1.5.0_linux_amd64.tar.gz + ct lint --config ./.ct.yaml + + curl --silent --show-error --fail --location https://github.com/artifacthub/hub/releases/download/v1.17.0/ah_1.17.0_linux_amd64.tar.gz --remote-name + echo "57c184b71a9a5c59192c2158fc08bdddca5c340fb1deeed0158383a665b38bf1 ah_1.17.0_linux_amd64.tar.gz" | shasum --check + tar xzf ah_1.17.0_linux_amd64.tar.gz ah + + ./ah lint --path charts/ingress-nginx + + rm ah_1.17.0_linux_amd64.tar.gz ah - name: Run unit tests run: | diff --git a/.github/workflows/helm.yaml b/.github/workflows/helm.yaml index f7a68af1a..60f3632dc 100644 --- a/.github/workflows/helm.yaml +++ b/.github/workflows/helm.yaml @@ -29,19 +29,20 @@ jobs: - name: Checkout uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - - name: Run Artifact Hub lint - run: | - wget https://github.com/artifacthub/hub/releases/download/v1.5.0/ah_1.5.0_linux_amd64.tar.gz - echo 'ad0e44c6ea058ab6b85dbf582e88bad9fdbc64ded0d1dd4edbac65133e5c87da *ah_1.5.0_linux_amd64.tar.gz' | shasum -c - tar -xzvf ah_1.5.0_linux_amd64.tar.gz ah - ./ah lint -p charts/ingress-nginx || exit 1 - rm -f ./ah ./ah_1.5.0_linux_amd64.tar.gz - - name: Set up chart-testing uses: helm/chart-testing-action@e6669bcd63d7cb57cb4380c33043eebe5d111992 # v2.6.1 - - name: Run chart-testing (lint) - run: ct lint --target-branch ${{ github.ref_name }} --config ./.ct.yaml + - name: Lint chart + run: | + ct lint --config ./.ct.yaml --target-branch ${{ github.ref_name }} + + curl --silent --show-error --fail --location https://github.com/artifacthub/hub/releases/download/v1.17.0/ah_1.17.0_linux_amd64.tar.gz --remote-name + echo "57c184b71a9a5c59192c2158fc08bdddca5c340fb1deeed0158383a665b38bf1 ah_1.17.0_linux_amd64.tar.gz" | shasum --check + tar xzf ah_1.17.0_linux_amd64.tar.gz ah + + ./ah lint --path charts/ingress-nginx + + rm ah_1.17.0_linux_amd64.tar.gz ah - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 id: filter From 543a2bb2c50d4e1ada92c7813f3c107bd1d0224d Mon Sep 17 00:00:00 2001 From: Marco Ebert Date: Mon, 29 Jan 2024 23:03:53 +0100 Subject: [PATCH 07/12] CI: Rework docs generation. --- .github/workflows/ci.yaml | 19 +++++++++++-------- .gitignore | 1 - MANUAL_RELEASE.md | 10 ++++------ magefiles/steps/helm.go | 4 ++-- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 7d70e7deb..4282b91aa 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -216,16 +216,19 @@ jobs: - name: Set up Helm Chart Testing uses: helm/chart-testing-action@e6669bcd63d7cb57cb4380c33043eebe5d111992 # v2.6.1 - - name: Run helm-docs + - name: Set up Helm Docs + uses: gabe565/setup-helm-docs-action@79a4cfe8dc7ae51c1113e3f3a1ae45e601d0c963 # v1.0.3 + + - name: Generate docs run: | - GOBIN=$PWD GO111MODULE=on go install github.com/norwoodj/helm-docs/cmd/helm-docs@v1.11.0 - ./helm-docs --chart-search-root=${GITHUB_WORKSPACE}/charts - DIFF=$(git diff ${GITHUB_WORKSPACE}/charts/ingress-nginx/README.md) - if [ ! -z "$DIFF" ]; then - echo "Please use helm-docs in your clone, of your fork, of the project, and commit a updated README.md for the chart. https://github.com/kubernetes/ingress-nginx/blob/main/RELEASE.md#d-edit-the-valuesyaml-and-run-helm-docs" + helm-docs --chart-search-root charts + + if ! git diff --exit-code charts/ingress-nginx/README.md + then + echo "Please use helm-docs in your fork of the project and commit an updated README.md for the chart." + echo "https://github.com/kubernetes/ingress-nginx/blob/main/MANUAL_RELEASE.md#d-edit-the-valuesyaml-and-run-helm-docs" + exit 1 fi - git diff --exit-code - rm -f ./helm-docs - name: Lint chart run: | diff --git a/.gitignore b/.gitignore index 73108f627..5eac1a800 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,3 @@ -helm-docs # OSX ._* .DS_Store diff --git a/MANUAL_RELEASE.md b/MANUAL_RELEASE.md index b1c1fd068..a6cd3e5da 100644 --- a/MANUAL_RELEASE.md +++ b/MANUAL_RELEASE.md @@ -231,14 +231,12 @@ Promoting the images basically means that images, that were pushed to staging co - tag - digest - - [helm-docs](https://github.com/norwoodj/helm-docs) is a tool that generates the README.md for a helm-chart automatically. In the CI pipeline workflow of github actions (/.github/workflows/ci.yaml), you can see how helm-docs is used. But the CI pipeline is not designed to make commits back into the project. So we need to run helm-docs manually, and check in the resulting autogenerated README.md at the path /charts/ingress-nginx/README.md + - [helm-docs](https://github.com/norwoodj/helm-docs) is a tool that generates the README.md for a Helm chart automatically. In the CI pipeline workflow of GitHub actions (.github/workflows/ci.yaml), you can see how helm-docs is used. The CI pipeline is not designed to make commits back into the project, so we need to run helm-docs manually and commit the resulting generated README.md. You can obtain a recent version of the helm-docs binary here: https://github.com/norwoodj/helm-docs/releases. ``` - GOBIN=$PWD GO111MODULE=on go install github.com/norwoodj/helm-docs/cmd/helm-docs@v1.11.0 - ./helm-docs --chart-search-root=${GITHUB_WORKSPACE}/charts - git diff --exit-code - rm -f ./helm-docs + helm-docs --chart-search-root charts + git diff charts/ingress-nginx/README.md ``` - Watchout for mistakes like leaving the helm-docs executable in your clone workspace or not checking the new README.md manually etc. + Take care of not leaving the helm-docs executable in your clone workspace or not committing the new README.md. ### e. Edit the static manifests diff --git a/magefiles/steps/helm.go b/magefiles/steps/helm.go index 245f5e1c0..3019ab225 100644 --- a/magefiles/steps/helm.go +++ b/magefiles/steps/helm.go @@ -170,7 +170,7 @@ func runHelmDocs() error { if err != nil { return err } - err = sh.RunV("helm-docs", "--chart-search-root=${PWD}/charts") + err = sh.RunV("helm-docs", "--chart-search-root", "${PWD}/charts") if err != nil { return err } @@ -181,7 +181,7 @@ func installHelmDocs() error { utils.Info("HELM Install HelmDocs") g0 := sh.RunCmd("go") - err := g0("install", "github.com/norwoodj/helm-docs/cmd/helm-docs@v1.11.0") + err := g0("install", "github.com/norwoodj/helm-docs/cmd/helm-docs@v1.11.3") if err != nil { return err } From b7046ef9d1181b7bb15e94dc71dba27f77de13f8 Mon Sep 17 00:00:00 2001 From: Marco Ebert Date: Sun, 4 Feb 2024 22:39:04 +0100 Subject: [PATCH 08/12] CI: Rework cluster creation. --- .github/workflows/ci.yaml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4282b91aa..41ea68ba5 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -278,15 +278,11 @@ jobs: - name: Load cache run: gzip --decompress --stdout docker.tar.gz | docker load - - name: fix permissions - run: | - sudo mkdir -p $HOME/.kube - sudo chmod -R 777 $HOME/.kube - - - name: Create Kubernetes ${{ matrix.k8s }} cluster + - name: Create cluster id: kind run: | - kind create cluster --image=kindest/node:${{ matrix.k8s }} + sudo chmod 777 "${HOME}/.kube" + kind create cluster --image kindest/node:${{ matrix.k8s }} --kubeconfig "${HOME}/.kube/kind-config-kind" - name: Test env: From 0381ddb096e437b0550f3f8a875f50e81b0f41dc Mon Sep 17 00:00:00 2001 From: Marco Ebert Date: Sun, 4 Feb 2024 22:45:56 +0100 Subject: [PATCH 09/12] CI: Rework E2E testing. --- .github/workflows/ci.yaml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 41ea68ba5..ae3ef3836 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -284,15 +284,13 @@ jobs: sudo chmod 777 "${HOME}/.kube" kind create cluster --image kindest/node:${{ matrix.k8s }} --kubeconfig "${HOME}/.kube/kind-config-kind" - - name: Test + - name: Run E2E tests env: KIND_CLUSTER_NAME: kind SKIP_CLUSTER_CREATION: true SKIP_IMAGE_CREATION: true SKIP_INGRESS_IMAGE_CREATION: true - run: | - kind get kubeconfig > $HOME/.kube/kind-config-kind - make kind-e2e-chart-tests + run: make kind-e2e-chart-tests kubernetes: name: Kubernetes From da8abcfc2c46dc4cf8ab91789da6574c191f9a4e Mon Sep 17 00:00:00 2001 From: Marco Ebert Date: Mon, 29 Jan 2024 23:31:08 +0100 Subject: [PATCH 10/12] CI: Rework cert-manager installation. --- .../third-party-addons/opentelemetry.md | 2 +- test/e2e/run-chart-test.sh | 23 +++++++++++-------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/docs/user-guide/third-party-addons/opentelemetry.md b/docs/user-guide/third-party-addons/opentelemetry.md index af7fd2b14..a6a3b19c7 100644 --- a/docs/user-guide/third-party-addons/opentelemetry.md +++ b/docs/user-guide/third-party-addons/opentelemetry.md @@ -191,7 +191,7 @@ To install the example and collectors run: helm repo add grafana https://grafana.github.io/helm-charts helm repo update # deploy cert-manager needed for OpenTelemetry collector operator - kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.9.1/cert-manager.yaml + kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.13.3/cert-manager.yaml # create observability namespace kubectl apply -f https://raw.githubusercontent.com/esigo/nginx-example/main/observability/namespace.yaml # install OpenTelemetry collector operator diff --git a/test/e2e/run-chart-test.sh b/test/e2e/run-chart-test.sh index 95465ee32..45794a677 100755 --- a/test/e2e/run-chart-test.sh +++ b/test/e2e/run-chart-test.sh @@ -91,16 +91,19 @@ echo "[dev-env] copying docker images to cluster..." kind load docker-image --name="${KIND_CLUSTER_NAME}" --nodes=${KIND_WORKERS} ${REGISTRY}/controller:${TAG} if [ "${SKIP_CERT_MANAGER_CREATION:-false}" = "false" ]; then - curl -fsSL -o cmctl.tar.gz https://github.com/cert-manager/cert-manager/releases/download/v1.11.1/cmctl-linux-amd64.tar.gz - tar xzf cmctl.tar.gz - chmod +x cmctl - ./cmctl help - echo "[dev-env] apply cert-manager ..." - kubectl apply --wait -f https://github.com/cert-manager/cert-manager/releases/download/v1.11.0/cert-manager.yaml - kubectl wait --timeout=30s --for=condition=available deployment/cert-manager -n cert-manager - kubectl get validatingwebhookconfigurations cert-manager-webhook -ojson | jq '.webhooks[].clientConfig' - kubectl get endpoints -n cert-manager cert-manager-webhook - ./cmctl check api --wait=2m + echo "[dev-env] deploying cert-manager..." + + # Get OS and platform for downloading cmctl. + os="$(uname -o | tr "[:upper:]" "[:lower:]")" + platform="$(uname -m | sed "s/aarch64/arm64/;s/x86_64/amd64/")" + + # Download cmctl. Cannot validate checksum since OS and platform may vary. + curl --silent --show-error --fail --location "https://github.com/cert-manager/cert-manager/releases/download/v1.13.3/cmctl-${os}-${platform}.tar.gz" | tar xz cmctl + + # Deploy cert-manager. + kubectl create --filename https://github.com/cert-manager/cert-manager/releases/download/v1.13.3/cert-manager.yaml + kubectl rollout status deployment --namespace cert-manager cert-manager --timeout 30s + ./cmctl check api --wait 2m fi echo "[dev-env] running helm chart e2e tests..." From 9f395e69ccb0bfa998980521836a50a844360b73 Mon Sep 17 00:00:00 2001 From: Marco Ebert Date: Mon, 29 Jan 2024 23:32:06 +0100 Subject: [PATCH 11/12] CI: Rework chart testing. --- test/e2e/run-chart-test.sh | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/test/e2e/run-chart-test.sh b/test/e2e/run-chart-test.sh index 45794a677..3623e0aa5 100755 --- a/test/e2e/run-chart-test.sh +++ b/test/e2e/run-chart-test.sh @@ -107,12 +107,15 @@ if [ "${SKIP_CERT_MANAGER_CREATION:-false}" = "false" ]; then fi echo "[dev-env] running helm chart e2e tests..." -docker run --rm --interactive --network host \ - --name ct \ - --volume $KUBECONFIG:/root/.kube/config \ - --volume "${DIR}/../../":/workdir \ - --workdir /workdir \ - registry.k8s.io/ingress-nginx/e2e-test-runner:v20240829-2c421762@sha256:5b7809bfe9cbd9cd6bcb8033ca27576ca704f05ce729fe4dcb574810f7a25785 \ - ct install \ - --charts charts/ingress-nginx \ - --helm-extra-args "--timeout 60s" +docker run \ + --name ct \ + --volume "${KUBECONFIG}:/root/.kube/config:ro" \ + --volume "${DIR}/../../:/workdir" \ + --network host \ + --workdir /workdir \ + --entrypoint ct \ + --rm \ + registry.k8s.io/ingress-nginx/e2e-test-runner:v20240829-2c421762@sha256:5b7809bfe9cbd9cd6bcb8033ca27576ca704f05ce729fe4dcb574810f7a25785 \ + install \ + --charts charts/ingress-nginx \ + --helm-extra-args "--timeout 60s" From 576958d1aa2b4390f810ee9e743ac9a5b84e14ea Mon Sep 17 00:00:00 2001 From: Marco Ebert Date: Sun, 4 Feb 2024 23:25:52 +0100 Subject: [PATCH 12/12] Chart: Fix CI values names. --- ...lags.yaml => controller-custom-ingressclass-flags-values.yaml} | 0 ...set-extra-modules.yaml => daemonset-extra-modules-values.yaml} | 0 ...yment-extra-modules-default-container-sec-context-values.yaml} | 0 ...ment-extra-modules-specific-container-sec-context-values.yaml} | 0 ...nt-extra-modules.yaml => deployment-extra-modules-values.yaml} | 0 5 files changed, 0 insertions(+), 0 deletions(-) rename charts/ingress-nginx/ci/{controller-custom-ingressclass-flags.yaml => controller-custom-ingressclass-flags-values.yaml} (100%) rename charts/ingress-nginx/ci/{daemonset-extra-modules.yaml => daemonset-extra-modules-values.yaml} (100%) rename charts/ingress-nginx/ci/{deployment-extra-modules-default-container-sec-context.yaml => deployment-extra-modules-default-container-sec-context-values.yaml} (100%) rename charts/ingress-nginx/ci/{deployment-extra-modules-specific-container-sec-context.yaml => deployment-extra-modules-specific-container-sec-context-values.yaml} (100%) rename charts/ingress-nginx/ci/{deployment-extra-modules.yaml => deployment-extra-modules-values.yaml} (100%) diff --git a/charts/ingress-nginx/ci/controller-custom-ingressclass-flags.yaml b/charts/ingress-nginx/ci/controller-custom-ingressclass-flags-values.yaml similarity index 100% rename from charts/ingress-nginx/ci/controller-custom-ingressclass-flags.yaml rename to charts/ingress-nginx/ci/controller-custom-ingressclass-flags-values.yaml diff --git a/charts/ingress-nginx/ci/daemonset-extra-modules.yaml b/charts/ingress-nginx/ci/daemonset-extra-modules-values.yaml similarity index 100% rename from charts/ingress-nginx/ci/daemonset-extra-modules.yaml rename to charts/ingress-nginx/ci/daemonset-extra-modules-values.yaml diff --git a/charts/ingress-nginx/ci/deployment-extra-modules-default-container-sec-context.yaml b/charts/ingress-nginx/ci/deployment-extra-modules-default-container-sec-context-values.yaml similarity index 100% rename from charts/ingress-nginx/ci/deployment-extra-modules-default-container-sec-context.yaml rename to charts/ingress-nginx/ci/deployment-extra-modules-default-container-sec-context-values.yaml diff --git a/charts/ingress-nginx/ci/deployment-extra-modules-specific-container-sec-context.yaml b/charts/ingress-nginx/ci/deployment-extra-modules-specific-container-sec-context-values.yaml similarity index 100% rename from charts/ingress-nginx/ci/deployment-extra-modules-specific-container-sec-context.yaml rename to charts/ingress-nginx/ci/deployment-extra-modules-specific-container-sec-context-values.yaml diff --git a/charts/ingress-nginx/ci/deployment-extra-modules.yaml b/charts/ingress-nginx/ci/deployment-extra-modules-values.yaml similarity index 100% rename from charts/ingress-nginx/ci/deployment-extra-modules.yaml rename to charts/ingress-nginx/ci/deployment-extra-modules-values.yaml