From 71fbdcb5e0a02dae5782d03d9f2600ed1e3d3d0a Mon Sep 17 00:00:00 2001 From: miwr Date: Wed, 12 Mar 2025 13:37:16 +0100 Subject: [PATCH 01/15] alloy implementation --- .gitignore | 1 + template/stacks/core/ingress-apps/alloy.yaml | 18 ++++++++++++ template/stacks/monitoring/alloy.yaml | 29 ++++++++++++++++++++ template/stacks/monitoring/alloy/values.yaml | 4 +++ 4 files changed, 52 insertions(+) create mode 100644 .gitignore create mode 100644 template/stacks/core/ingress-apps/alloy.yaml create mode 100644 template/stacks/monitoring/alloy.yaml create mode 100644 template/stacks/monitoring/alloy/values.yaml diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2a6a657 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +/.history \ No newline at end of file diff --git a/template/stacks/core/ingress-apps/alloy.yaml b/template/stacks/core/ingress-apps/alloy.yaml new file mode 100644 index 0000000..e939823 --- /dev/null +++ b/template/stacks/core/ingress-apps/alloy.yaml @@ -0,0 +1,18 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: alloy + namespace: monitoring +spec: + ingressClassName: nginx + rules: + - host: {{{ .Env.DOMAIN }}} + http: + paths: + - backend: + service: + name: alloy + port: + number: 12345 + path: /alloy + pathType: Prefix diff --git a/template/stacks/monitoring/alloy.yaml b/template/stacks/monitoring/alloy.yaml new file mode 100644 index 0000000..7d4d614 --- /dev/null +++ b/template/stacks/monitoring/alloy.yaml @@ -0,0 +1,29 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: alloy + namespace: argocd + labels: + env: dev + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + project: default + syncPolicy: + automated: + selfHeal: true + syncOptions: + - CreateNamespace=true + destination: + name: in-cluster + namespace: monitoring + sources: + - repoURL: https://github.com/grafana/alloy.git + path: operations/helm/charts/alloy + targetRevision: HEAD + helm: + valueFiles: + - $values/stacks/monitoring/alloy/values.yaml + - repoURL: https://{{{ .Env.DOMAIN_GITEA }}}/giteaAdmin/edfbuilder + targetRevision: HEAD + ref: values \ No newline at end of file diff --git a/template/stacks/monitoring/alloy/values.yaml b/template/stacks/monitoring/alloy/values.yaml new file mode 100644 index 0000000..411863c --- /dev/null +++ b/template/stacks/monitoring/alloy/values.yaml @@ -0,0 +1,4 @@ +alloy: + create: false + name: alloy-config + key: config.alloy \ No newline at end of file From dd7cd2fa91334390d47f9434ba020718369a4049 Mon Sep 17 00:00:00 2001 From: miwr Date: Wed, 12 Mar 2025 13:47:07 +0100 Subject: [PATCH 02/15] alloy.uiPathPrefix: "/alloy" added --- template/stacks/monitoring/alloy/values.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/template/stacks/monitoring/alloy/values.yaml b/template/stacks/monitoring/alloy/values.yaml index 411863c..df9f147 100644 --- a/template/stacks/monitoring/alloy/values.yaml +++ b/template/stacks/monitoring/alloy/values.yaml @@ -1,4 +1,6 @@ alloy: create: false name: alloy-config - key: config.alloy \ No newline at end of file + key: config.alloy + + uiPathPrefix: "/alloy" \ No newline at end of file From 81e85ff518d2987e2e71006e372c24e77111a7c1 Mon Sep 17 00:00:00 2001 From: miwr Date: Wed, 12 Mar 2025 14:22:11 +0100 Subject: [PATCH 03/15] config.alloy added to the values --- template/stacks/monitoring/alloy/values.yaml | 38 +++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/template/stacks/monitoring/alloy/values.yaml b/template/stacks/monitoring/alloy/values.yaml index df9f147..930f84b 100644 --- a/template/stacks/monitoring/alloy/values.yaml +++ b/template/stacks/monitoring/alloy/values.yaml @@ -3,4 +3,40 @@ alloy: name: alloy-config key: config.alloy - uiPathPrefix: "/alloy" \ No newline at end of file + uiPathPrefix: "/alloy" + + configMap.content: | + logging { + level = "info" + format = "logfmt" + } + + loki.write "local_loki" { + endpoint { + url = "http://loki.default.svc.cluster.local:3100/loki/api/v1/push" + } + } + + discovery.kubernetes "pod" { + role = "pod" + } + + discovery.kubernetes "nodes" { + role = "node" + } + + discovery.kubernetes "services" { + role = "service" + } + + discovery.kubernetes "endpoints" { + role = "endpoints" + } + + discovery.kubernetes "endpointslices" { + role = "endpointslice" + } + + discovery.kubernetes "ingresses" { + role = "ingress" + } \ No newline at end of file From 3a5df116045e37db74ca172534ac2fb8b70b2454 Mon Sep 17 00:00:00 2001 From: miwr Date: Wed, 12 Mar 2025 14:22:29 +0100 Subject: [PATCH 04/15] alloy implementation commented out --- template/stacks/monitoring/promtail.yaml | 58 ++++++++++++------------ 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/template/stacks/monitoring/promtail.yaml b/template/stacks/monitoring/promtail.yaml index 8f4af77..4769ead 100644 --- a/template/stacks/monitoring/promtail.yaml +++ b/template/stacks/monitoring/promtail.yaml @@ -1,29 +1,29 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Application -metadata: - name: promtail - namespace: argocd - labels: - env: dev - finalizers: - - resources-finalizer.argocd.argoproj.io -spec: - project: default - syncPolicy: - automated: - selfHeal: true - syncOptions: - - CreateNamespace=true - destination: - name: in-cluster - namespace: monitoring - sources: - - repoURL: https://github.com/grafana/helm-charts - path: charts/promtail - targetRevision: HEAD - helm: - valueFiles: - - $values/stacks/monitoring/promtail/values.yaml - - repoURL: https://{{{ .Env.DOMAIN_GITEA }}}/giteaAdmin/edfbuilder - targetRevision: HEAD - ref: values \ No newline at end of file +# apiVersion: argoproj.io/v1alpha1 +# kind: Application +# metadata: +# name: promtail +# namespace: argocd +# labels: +# env: dev +# finalizers: +# - resources-finalizer.argocd.argoproj.io +# spec: +# project: default +# syncPolicy: +# automated: +# selfHeal: true +# syncOptions: +# - CreateNamespace=true +# destination: +# name: in-cluster +# namespace: monitoring +# sources: +# - repoURL: https://github.com/grafana/helm-charts +# path: charts/promtail +# targetRevision: HEAD +# helm: +# valueFiles: +# - $values/stacks/monitoring/promtail/values.yaml +# - repoURL: https://{{{ .Env.DOMAIN_GITEA }}}/giteaAdmin/edfbuilder +# targetRevision: HEAD +# ref: values \ No newline at end of file From 180b74697a43947fb43b6405c8240cb7b8a63750 Mon Sep 17 00:00:00 2001 From: miwr Date: Wed, 12 Mar 2025 14:30:37 +0100 Subject: [PATCH 05/15] config.alloy in values.yaml adjusted --- template/stacks/monitoring/alloy/values.yaml | 58 ++++++++++---------- 1 file changed, 30 insertions(+), 28 deletions(-) diff --git a/template/stacks/monitoring/alloy/values.yaml b/template/stacks/monitoring/alloy/values.yaml index 930f84b..9441652 100644 --- a/template/stacks/monitoring/alloy/values.yaml +++ b/template/stacks/monitoring/alloy/values.yaml @@ -5,38 +5,40 @@ alloy: uiPathPrefix: "/alloy" - configMap.content: | - logging { - level = "info" - format = "logfmt" - } + configMap: + content: |- - loki.write "local_loki" { - endpoint { - url = "http://loki.default.svc.cluster.local:3100/loki/api/v1/push" - } - } + logging { + level = "info" + format = "logfmt" + } - discovery.kubernetes "pod" { - role = "pod" - } + loki.write "local_loki" { + endpoint { + url = "http://loki.default.svc.cluster.local:3100/loki/api/v1/push" + } + } - discovery.kubernetes "nodes" { - role = "node" - } + discovery.kubernetes "pod" { + role = "pod" + } - discovery.kubernetes "services" { - role = "service" - } + discovery.kubernetes "nodes" { + role = "node" + } - discovery.kubernetes "endpoints" { - role = "endpoints" - } + discovery.kubernetes "services" { + role = "service" + } - discovery.kubernetes "endpointslices" { - role = "endpointslice" - } + discovery.kubernetes "endpoints" { + role = "endpoints" + } - discovery.kubernetes "ingresses" { - role = "ingress" - } \ No newline at end of file + discovery.kubernetes "endpointslices" { + role = "endpointslice" + } + + discovery.kubernetes "ingresses" { + role = "ingress" + } \ No newline at end of file From ddaf06b29c4b99678e9979915461323e61779fd4 Mon Sep 17 00:00:00 2001 From: miwr Date: Wed, 12 Mar 2025 14:39:36 +0100 Subject: [PATCH 06/15] loki reference changes --- template/stacks/monitoring/alloy/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/template/stacks/monitoring/alloy/values.yaml b/template/stacks/monitoring/alloy/values.yaml index 9441652..3d6fd5a 100644 --- a/template/stacks/monitoring/alloy/values.yaml +++ b/template/stacks/monitoring/alloy/values.yaml @@ -15,7 +15,7 @@ alloy: loki.write "local_loki" { endpoint { - url = "http://loki.default.svc.cluster.local:3100/loki/api/v1/push" + url = "http://loki-loki-distributed-gateway/loki/api/v1/push" } } From 8f62875529a1ae3a3c261da05457afe4b71a95ae Mon Sep 17 00:00:00 2001 From: miwr Date: Wed, 12 Mar 2025 14:53:01 +0100 Subject: [PATCH 07/15] config.alloy adjusted in values.yaml --- template/stacks/monitoring/alloy/values.yaml | 51 +++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/template/stacks/monitoring/alloy/values.yaml b/template/stacks/monitoring/alloy/values.yaml index 3d6fd5a..88fe6c4 100644 --- a/template/stacks/monitoring/alloy/values.yaml +++ b/template/stacks/monitoring/alloy/values.yaml @@ -41,4 +41,53 @@ alloy: discovery.kubernetes "ingresses" { role = "ingress" - } \ No newline at end of file + } + + # Process and label logs before sending to Loki + discovery.relabel "pod_logs" { + targets = discovery.kubernetes.pod.targets + + # Assign labels for logs + rule { + source_labels = ["__meta_kubernetes_namespace"] + action = "replace" + target_label = "namespace" + } + + rule { + source_labels = ["__meta_kubernetes_pod_name"] + action = "replace" + target_label = "pod" + } + + rule { + source_labels = ["__meta_kubernetes_pod_container_name"] + action = "replace" + target_label = "container" + } + + rule { + source_labels = ["__meta_kubernetes_pod_uid", "__meta_kubernetes_pod_container_name"] + action = "replace" + target_label = "__path__" + separator = "/" + replacement = "/var/log/pods/*$1/*.log" + } + } + + # Collect logs from Kubernetes API + loki.source.kubernetes "all_pod_logs" { + targets = discovery.relabel.pod_logs.output + forward_to = [loki.process.all_logs.receiver] + } + + # Process logs before writing + loki.process "all_logs" { + stage.static_labels { + values = { + cluster = "cluster" + } + } + + forward_to = [loki.write.local_loki.receiver] + } From 1682302b69eae1e6b3e94ea42aedadbe1940535e Mon Sep 17 00:00:00 2001 From: miwr Date: Wed, 12 Mar 2025 15:04:59 +0100 Subject: [PATCH 08/15] "#" are not allowed in config.alloy in values.yaml --- template/stacks/monitoring/alloy/values.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/template/stacks/monitoring/alloy/values.yaml b/template/stacks/monitoring/alloy/values.yaml index 88fe6c4..8d8ee45 100644 --- a/template/stacks/monitoring/alloy/values.yaml +++ b/template/stacks/monitoring/alloy/values.yaml @@ -43,11 +43,9 @@ alloy: role = "ingress" } - # Process and label logs before sending to Loki discovery.relabel "pod_logs" { targets = discovery.kubernetes.pod.targets - # Assign labels for logs rule { source_labels = ["__meta_kubernetes_namespace"] action = "replace" @@ -75,13 +73,11 @@ alloy: } } - # Collect logs from Kubernetes API loki.source.kubernetes "all_pod_logs" { targets = discovery.relabel.pod_logs.output forward_to = [loki.process.all_logs.receiver] } - # Process logs before writing loki.process "all_logs" { stage.static_labels { values = { From 687322525b17bde7420ce8b0ba3c6a6f3f34b1a9 Mon Sep 17 00:00:00 2001 From: miwr Date: Wed, 12 Mar 2025 15:18:59 +0100 Subject: [PATCH 09/15] values.yaml for alloy edited --- template/stacks/monitoring/alloy/values.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/template/stacks/monitoring/alloy/values.yaml b/template/stacks/monitoring/alloy/values.yaml index 8d8ee45..e9eb0f2 100644 --- a/template/stacks/monitoring/alloy/values.yaml +++ b/template/stacks/monitoring/alloy/values.yaml @@ -79,11 +79,5 @@ alloy: } loki.process "all_logs" { - stage.static_labels { - values = { - cluster = "cluster" - } - } - forward_to = [loki.write.local_loki.receiver] } From fbb5aeb32b2765fc570cef57692cecd0dd888b88 Mon Sep 17 00:00:00 2001 From: miwr Date: Wed, 12 Mar 2025 15:20:35 +0100 Subject: [PATCH 10/15] forward_to = [loki.write.local_loki.receiver] --- template/stacks/monitoring/alloy/values.yaml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/template/stacks/monitoring/alloy/values.yaml b/template/stacks/monitoring/alloy/values.yaml index e9eb0f2..7120fab 100644 --- a/template/stacks/monitoring/alloy/values.yaml +++ b/template/stacks/monitoring/alloy/values.yaml @@ -71,13 +71,7 @@ alloy: separator = "/" replacement = "/var/log/pods/*$1/*.log" } - } - loki.source.kubernetes "all_pod_logs" { - targets = discovery.relabel.pod_logs.output - forward_to = [loki.process.all_logs.receiver] - } - - loki.process "all_logs" { forward_to = [loki.write.local_loki.receiver] - } + + } \ No newline at end of file From b462804f2999349c6c581df23eea98016d4db243 Mon Sep 17 00:00:00 2001 From: miwr Date: Wed, 12 Mar 2025 15:28:20 +0100 Subject: [PATCH 11/15] loki.source.kubernetes "all_pod_logs" { targets = discovery.relabel.pod_logs.output forward_to = [loki.write.local_loki.receiver] } --- template/stacks/monitoring/alloy/values.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/template/stacks/monitoring/alloy/values.yaml b/template/stacks/monitoring/alloy/values.yaml index 7120fab..91f606f 100644 --- a/template/stacks/monitoring/alloy/values.yaml +++ b/template/stacks/monitoring/alloy/values.yaml @@ -71,7 +71,9 @@ alloy: separator = "/" replacement = "/var/log/pods/*$1/*.log" } + } + loki.source.kubernetes "all_pod_logs" { + targets = discovery.relabel.pod_logs.output forward_to = [loki.write.local_loki.receiver] - - } \ No newline at end of file + } From 75f40e070cd1866e520701e425c590d1b165237f Mon Sep 17 00:00:00 2001 From: miwr Date: Wed, 12 Mar 2025 15:55:41 +0100 Subject: [PATCH 12/15] promtail references replaces with alloy in dashboard_loki_container.yaml --- .../dashboards/dashboard_loki_container.yaml | 4 +- template/stacks/monitoring/promtail.yaml | 29 ------------ .../stacks/monitoring/promtail/values.yaml | 45 ------------------- 3 files changed, 2 insertions(+), 76 deletions(-) delete mode 100644 template/stacks/monitoring/promtail.yaml delete mode 100644 template/stacks/monitoring/promtail/values.yaml diff --git a/template/stacks/monitoring/kube-prometheus/dashboards/dashboard_loki_container.yaml b/template/stacks/monitoring/kube-prometheus/dashboards/dashboard_loki_container.yaml index 267bd90..e38896e 100644 --- a/template/stacks/monitoring/kube-prometheus/dashboards/dashboard_loki_container.yaml +++ b/template/stacks/monitoring/kube-prometheus/dashboards/dashboard_loki_container.yaml @@ -110,12 +110,12 @@ data: "uid": "P8E80F9AEF21F6940" }, "editorMode": "builder", - "expr": "{container=\"promtail\"} |= ``", + "expr": "{container=\"alloy\"} |= ``", "queryType": "range", "refId": "A" } ], - "title": "Logs: Container promtail", + "title": "Logs: Container alloy", "type": "logs" }, { diff --git a/template/stacks/monitoring/promtail.yaml b/template/stacks/monitoring/promtail.yaml deleted file mode 100644 index 4769ead..0000000 --- a/template/stacks/monitoring/promtail.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# apiVersion: argoproj.io/v1alpha1 -# kind: Application -# metadata: -# name: promtail -# namespace: argocd -# labels: -# env: dev -# finalizers: -# - resources-finalizer.argocd.argoproj.io -# spec: -# project: default -# syncPolicy: -# automated: -# selfHeal: true -# syncOptions: -# - CreateNamespace=true -# destination: -# name: in-cluster -# namespace: monitoring -# sources: -# - repoURL: https://github.com/grafana/helm-charts -# path: charts/promtail -# targetRevision: HEAD -# helm: -# valueFiles: -# - $values/stacks/monitoring/promtail/values.yaml -# - repoURL: https://{{{ .Env.DOMAIN_GITEA }}}/giteaAdmin/edfbuilder -# targetRevision: HEAD -# ref: values \ No newline at end of file diff --git a/template/stacks/monitoring/promtail/values.yaml b/template/stacks/monitoring/promtail/values.yaml deleted file mode 100644 index 49faadc..0000000 --- a/template/stacks/monitoring/promtail/values.yaml +++ /dev/null @@ -1,45 +0,0 @@ -# -- Overrides the chart's name -nameOverride: null - -# -- Overrides the chart's computed fullname -fullnameOverride: null - -global: - # -- Allow parent charts to override registry hostname - imageRegistry: "" - # -- Allow parent charts to override registry credentials - imagePullSecrets: [] - -daemonset: - # -- Deploys Promtail as a DaemonSet - enabled: true - autoscaling: - # -- Creates a VerticalPodAutoscaler for the daemonset - enabled: false - -deployment: - # -- Deploys Promtail as a Deployment - enabled: false - -config: - enabled: true - logLevel: info - logFormat: logfmt - serverPort: 3101 - clients: - - url: http://loki-loki-distributed-gateway/loki/api/v1/push - scrape_configs: - - job_name: authlog - static_configs: - - targets: - - authlog - labels: - job: authlog - __path__: /logs/auth.log - - job_name: syslog - static_configs: - - targets: - - syslog - labels: - job: syslog - __path__: /logs/syslog \ No newline at end of file From 8f621647f5f4048ba8f47bcf0bacd987267eb6ec Mon Sep 17 00:00:00 2001 From: miwr Date: Thu, 13 Mar 2025 10:08:59 +0100 Subject: [PATCH 13/15] rule { source_labels = ["__meta_kubernetes_pod_name", "__meta_kubernetes_pod_container_name"] action = "replace" target_label = "__path__" replacement = "/var/log/containers/$1_$2.log" } --- template/stacks/monitoring/alloy/values.yaml | 24 +++----------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/template/stacks/monitoring/alloy/values.yaml b/template/stacks/monitoring/alloy/values.yaml index 91f606f..036f989 100644 --- a/template/stacks/monitoring/alloy/values.yaml +++ b/template/stacks/monitoring/alloy/values.yaml @@ -47,30 +47,12 @@ alloy: targets = discovery.kubernetes.pod.targets rule { - source_labels = ["__meta_kubernetes_namespace"] - action = "replace" - target_label = "namespace" - } - - rule { - source_labels = ["__meta_kubernetes_pod_name"] - action = "replace" - target_label = "pod" - } - - rule { - source_labels = ["__meta_kubernetes_pod_container_name"] - action = "replace" - target_label = "container" - } - - rule { - source_labels = ["__meta_kubernetes_pod_uid", "__meta_kubernetes_pod_container_name"] + source_labels = ["__meta_kubernetes_pod_name", "__meta_kubernetes_pod_container_name"] action = "replace" target_label = "__path__" - separator = "/" - replacement = "/var/log/pods/*$1/*.log" + replacement = "/var/log/containers/$1_$2.log" } + } loki.source.kubernetes "all_pod_logs" { From 1e5fa94c477333aefdaf4e54bbfef9fa9a6aca1e Mon Sep 17 00:00:00 2001 From: miwr Date: Thu, 13 Mar 2025 10:19:45 +0100 Subject: [PATCH 14/15] rules in alloy's values.yaml adjusted --- template/stacks/monitoring/alloy/values.yaml | 24 ++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/template/stacks/monitoring/alloy/values.yaml b/template/stacks/monitoring/alloy/values.yaml index 036f989..a881409 100644 --- a/template/stacks/monitoring/alloy/values.yaml +++ b/template/stacks/monitoring/alloy/values.yaml @@ -46,6 +46,30 @@ alloy: discovery.relabel "pod_logs" { targets = discovery.kubernetes.pod.targets + rule { + source_labels = ["__meta_kubernetes_namespace"] + action = "replace" + target_label = "namespace" + } + + rule { + source_labels = ["__meta_kubernetes_pod_name"] + action = "replace" + target_label = "pod" + } + + rule { + source_labels = ["__meta_kubernetes_pod_node_name"] + action = "replace" + target_label = "node" + } + + rule { + source_labels = ["__meta_kubernetes_pod_container_name"] + action = "replace" + target_label = "container" + } + rule { source_labels = ["__meta_kubernetes_pod_name", "__meta_kubernetes_pod_container_name"] action = "replace" From 415576c2cbe3baaaa6219cca147be95f1f6902b3 Mon Sep 17 00:00:00 2001 From: miwr Date: Thu, 13 Mar 2025 10:26:56 +0100 Subject: [PATCH 15/15] unnecessary rule deleted --- template/stacks/monitoring/alloy/values.yaml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/template/stacks/monitoring/alloy/values.yaml b/template/stacks/monitoring/alloy/values.yaml index a881409..a2ac67d 100644 --- a/template/stacks/monitoring/alloy/values.yaml +++ b/template/stacks/monitoring/alloy/values.yaml @@ -70,13 +70,6 @@ alloy: target_label = "container" } - rule { - source_labels = ["__meta_kubernetes_pod_name", "__meta_kubernetes_pod_container_name"] - action = "replace" - target_label = "__path__" - replacement = "/var/log/containers/$1_$2.log" - } - } loki.source.kubernetes "all_pod_logs" {