configuration added

listener {
          address = "0.0.0.0:1514"
          labels   = { component = "loki.source.syslog", protocol = "tcp" }
        }
        forward_to = [loki.write.local_loki.receiver]
      }

README.md

@@ -1,3 +1,7 @@
 # edpbuilder stacks
 
-This repository contains the building blocks to instantiate Internal Developer Platforms.
+This repository contains the building blocks to instanciate Internal Developer Platform's.
+
+### Install edpbuilder
+
+To get started, you need to install [edpbuilder](https://forgejo.edf-bootstrap.cx.fg1.ffm.osc.live/DevFW/edpbuilder).

template/edfbuilder.yaml

@@ -12,8 +12,8 @@ spec:
     name: in-cluster
     namespace: argocd
   source:
-    path: "{{{ .Env.CLIENT_REPO_ID }}}/{{{ .Env.DOMAIN }}}/registry"
-    repoURL: "https://{{{ .Env.CLIENT_REPO_DOMAIN }}}/{{{ .Env.CLIENT_REPO_ORG_NAME }}}"
+    path: registry
+    repoURL: 'https://{{{ .Env.DOMAIN_GITEA }}}/giteaAdmin/edfbuilder'
     targetRevision: HEAD
   project: default
   syncPolicy:

template/registry/core.yaml

@@ -12,8 +12,8 @@ spec:
     name: in-cluster
     namespace: argocd
   source:
-    path: "{{{ .Env.CLIENT_REPO_ID }}}/{{{ .Env.DOMAIN }}}/stacks/core"
-    repoURL: "https://{{{ .Env.CLIENT_REPO_DOMAIN }}}/{{{ .Env.CLIENT_REPO_ORG_NAME }}}"
+    path: stacks/core
+    repoURL: 'https://{{{ .Env.DOMAIN_GITEA }}}/giteaAdmin/edfbuilder'
     targetRevision: HEAD
   project: default
   syncPolicy:

template/registry/local-backup.yaml

@@ -1,7 +1,7 @@
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
-  name: otc
+  name: local-backup
   namespace: argocd
   labels:
     env: dev
@@ -12,8 +12,8 @@ spec:
     name: in-cluster
     namespace: argocd
   source:
-    path: "{{{ .Env.CLIENT_REPO_ID }}}/{{{ .Env.DOMAIN }}}/stacks/otc"
-    repoURL: "https://{{{ .Env.CLIENT_REPO_DOMAIN }}}/{{{ .Env.CLIENT_REPO_ORG_NAME }}}"
+    path: stacks/local-backup
+    repoURL: 'https://{{{ .Env.DOMAIN_GITEA }}}/giteaAdmin/edfbuilder'
     targetRevision: HEAD
   project: default
   syncPolicy:

template/registry/monitoring.yaml

@@ -1,7 +1,7 @@
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
-  name: forgejo
+  name: monitoring
   namespace: argocd
   labels:
     env: dev
@@ -12,8 +12,8 @@ spec:
     name: in-cluster
     namespace: argocd
   source:
-    path: "{{{ .Env.CLIENT_REPO_ID }}}/{{{ .Env.DOMAIN }}}/stacks/forgejo"
-    repoURL: "https://{{{ .Env.CLIENT_REPO_DOMAIN }}}/{{{ .Env.CLIENT_REPO_ORG_NAME }}}"
+    path: stacks/monitoring
+    repoURL: 'https://{{{ .Env.DOMAIN_GITEA }}}/giteaAdmin/edfbuilder'
     targetRevision: HEAD
   project: default
   syncPolicy:

template/registry/ref-implementation.yaml

@@ -1,7 +1,7 @@
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
-  name: observability
+  name: ref-implementation
   namespace: argocd
   labels:
     env: dev
@@ -12,8 +12,8 @@ spec:
     name: in-cluster
     namespace: argocd
   source:
-    path: "{{{ .Env.CLIENT_REPO_ID }}}/{{{ .Env.DOMAIN }}}/stacks/observability"
-    repoURL: "https://{{{ .Env.CLIENT_REPO_DOMAIN }}}/{{{ .Env.CLIENT_REPO_ORG_NAME }}}"
+    path: stacks/ref-implementation
+    repoURL: 'https://{{{ .Env.DOMAIN_GITEA }}}/giteaAdmin/edfbuilder'
     targetRevision: HEAD
   project: default
   syncPolicy:

template/registry/second-cluster.yaml

@@ -1,7 +1,7 @@
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
-  name: observability-client
+  name: second-cluster
   namespace: argocd
   labels:
     env: dev
@@ -12,8 +12,8 @@ spec:
     name: in-cluster
     namespace: argocd
   source:
-    path: "{{{ .Env.CLIENT_REPO_ID }}}/{{{ .Env.DOMAIN }}}/stacks/observability-client"
-    repoURL: "https://{{{ .Env.CLIENT_REPO_DOMAIN }}}/{{{ .Env.CLIENT_REPO_ORG_NAME }}}"
+    path: stacks/second-cluster
+    repoURL: 'https://{{{ .Env.DOMAIN_GITEA }}}/giteaAdmin/edfbuilder'
     targetRevision: HEAD
   project: default
   syncPolicy:

template/stacks/core/argocd.yaml

@@ -12,24 +12,19 @@ spec:
       selfHeal: true
     syncOptions:
       - CreateNamespace=true
-    retry:
-      limit: -1
   destination:
     name: in-cluster
     namespace: argocd
   sources:
-    - repoURL: https://edp.buildth.ing/DevFW-CICD/argocd-helm.git
+    - repoURL: https://github.com/argoproj/argo-helm
       path: charts/argo-cd
       # TODO: RIRE Can be updated when https://github.com/argoproj/argo-cd/issues/20790 is fixed and merged
       # As logout make problems, it is suggested to switch from path based routing to an own argocd domain,
       # similar to the CNOE amazon reference implementation and in our case, Forgejo
-      targetRevision: argo-cd-7.8.14-depends
+      targetRevision: argo-cd-7.7.5
       helm:
         valueFiles:
-          - $values/{{{ .Env.CLIENT_REPO_ID }}}/{{{ .Env.DOMAIN }}}/stacks/core/argocd/values.yaml
-    - repoURL: https://{{{ .Env.CLIENT_REPO_DOMAIN }}}/{{{ .Env.CLIENT_REPO_ORG_NAME }}}
+          - $values/stacks/core/argocd/values.yaml
+    - repoURL: https://{{{ .Env.DOMAIN_GITEA }}}/giteaAdmin/edfbuilder
       targetRevision: HEAD
       ref: values
-    - repoURL: https://{{{ .Env.CLIENT_REPO_DOMAIN }}}/{{{ .Env.CLIENT_REPO_ORG_NAME }}}
-      targetRevision: HEAD
-      path: "{{{ .Env.CLIENT_REPO_ID }}}/{{{ .Env.DOMAIN }}}/stacks/core/argocd/manifests"

template/stacks/core/argocd/values.yaml

@@ -1,9 +1,10 @@
 global:
-  domain: {{{ .Env.DOMAIN_ARGOCD }}}
+  domain: {{{ .Env.DOMAIN }}}
 
 configs:
   params:
     server.insecure: true
+    server.basehref: /argocd
   cm:
     application.resourceTrackingMethod: annotation
     timeout.reconciliation: 60s
@@ -19,7 +20,6 @@ configs:
         clusters:
         - "*"
     accounts.provider-argocd: apiKey
-    url: https://{{{ .Env.DOMAIN_ARGOCD }}}
   rbac:
     policy.csv: 'g, provider-argocd, role:admin'
 

template/stacks/core/crossplane-compositions.yaml

@@ -0,0 +1,23 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: crossplane-compositions
+  namespace: argocd
+  labels:
+    env: dev
+spec:
+  project: default
+  syncPolicy:
+    automated:
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true
+  destination:
+    name: in-cluster
+    namespace: crossplane-system
+  source:
+    path: stacks/core/crossplane-compositions
+    repoURL: https://{{{ .Env.DOMAIN_GITEA }}}/giteaAdmin/edfbuilder
+    targetRevision: HEAD
+    directory:
+      recurse: true

template/stacks/core/crossplane-compositions/edfbuilder/definition.yaml

@@ -0,0 +1,30 @@
+apiVersion: apiextensions.crossplane.io/v1
+kind: CompositeResourceDefinition
+metadata:
+  name: edfbuilders.edfbuilder.crossplane.io
+spec:
+  connectionSecretKeys:
+    - kubeconfig
+  group: edfbuilder.crossplane.io
+  names:
+    kind: EDFBuilder
+    listKind: EDFBuilderList
+    plural: edfbuilders
+    singular: edfbuilders
+  versions:
+    - name: v1alpha1
+      served: true
+      referenceable: true
+      schema:
+        openAPIV3Schema:
+          description: A EDFBuilder is a composite resource that represents a K8S Cluster with edfbuilder Installed
+          type: object
+          properties:
+            spec:
+              type: object
+              properties:
+                repoURL:
+                  type: string
+                  description: URL to ArgoCD stack of stacks repo
+              required:
+                - repoURL

template/stacks/core/crossplane-providers.yaml

@@ -0,0 +1,23 @@
+{{{ if eq .Env.CLUSTER_TYPE "kind" }}}
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: crossplane-providers
+  namespace: argocd
+  labels:
+    env: dev
+spec:
+  project: default
+  syncPolicy:
+    automated:
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true
+  destination:
+    name: in-cluster
+    namespace: crossplane-system
+  source:
+    path: stacks/core/crossplane-providers
+    repoURL: https://{{{ .Env.DOMAIN_GITEA }}}/giteaAdmin/edfbuilder
+    targetRevision: HEAD
+{{{ end }}}

template/stacks/core/crossplane-providers/function-patch-and-transform.yaml

@@ -0,0 +1,9 @@
+apiVersion: pkg.crossplane.io/v1
+kind: Function
+metadata:
+  name: crossplane-contrib-function-patch-and-transform
+spec:
+  package: xpkg.upbound.io/crossplane-contrib/function-patch-and-transform:v0.7.0
+  packagePullPolicy: IfNotPresent # Only download the package if it isn’t in the cache.
+  revisionActivationPolicy: Automatic # Otherwise our Provider never gets activate & healthy
+  revisionHistoryLimit: 1

template/stacks/core/crossplane-providers/provider-argocd-config.yaml

@@ -0,0 +1,14 @@
+apiVersion: argocd.crossplane.io/v1alpha1
+kind: ProviderConfig
+metadata:
+  name: argocd-provider
+spec:
+  serverAddr: argocd-server.argocd.svc.cluster.local:80
+  insecure: true
+  plainText: true
+  credentials:
+    source: Secret
+    secretRef:
+      namespace: crossplane-system
+      name: argocd-credentials
+      key: authToken

template/stacks/core/crossplane-providers/provider-argocd.yaml

@@ -0,0 +1,9 @@
+apiVersion: pkg.crossplane.io/v1
+kind: Provider
+metadata:
+  name: provider-argocd
+spec:
+  package: xpkg.upbound.io/crossplane-contrib/provider-argocd:v0.9.1
+  packagePullPolicy: IfNotPresent # Only download the package if it isn’t in the cache.
+  revisionActivationPolicy: Automatic # Otherwise our Provider never gets activate & healthy
+  revisionHistoryLimit: 1

template/stacks/core/crossplane-providers/provider-kind-config.yaml

@@ -0,0 +1,14 @@
+apiVersion: kind.crossplane.io/v1alpha1
+kind: ProviderConfig
+metadata:
+  name: kind-provider
+spec:
+  credentials:
+    source: Secret
+    secretRef:
+      namespace: crossplane-system
+      name: kind-credentials
+      key: credentials
+  endpoint:
+    # the url is managed by crossplane-edfbuilder
+    url: https://DOCKER_HOST:SERVER_PORT/api/v1/kindserver

template/stacks/core/crossplane-providers/provider-kind.yaml

@@ -0,0 +1,9 @@
+apiVersion: pkg.crossplane.io/v1
+kind: Provider
+metadata:
+  name: provider-kind
+spec:
+  package: forgejo.edf-bootstrap.cx.fg1.ffm.osc.live/devfw-cicd/provider-kind:v0.1.0
+  packagePullPolicy: IfNotPresent # Only download the package if it isn’t in the cache.
+  revisionActivationPolicy: Automatic # Otherwise our Provider never gets activate & healthy
+  revisionHistoryLimit: 1

template/stacks/core/crossplane-providers/provider-shell.yaml

@@ -0,0 +1,9 @@
+apiVersion: pkg.crossplane.io/v1
+kind: Provider
+metadata:
+  name: provider-shell
+spec:
+  package: forgejo.edf-bootstrap.cx.fg1.ffm.osc.live/devfw-cicd/provider-shell:v0.1.1
+  packagePullPolicy: IfNotPresent # Only download the package if it isn’t in the cache.
+  revisionActivationPolicy: Automatic # Otherwise our Provider never gets activate & healthy
+  revisionHistoryLimit: 1

template/stacks/core/crossplane.yaml

@@ -0,0 +1,23 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: crossplane
+  namespace: argocd
+  labels:
+    env: dev
+spec:
+  project: default
+  syncPolicy:
+    automated:
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true
+  destination:
+    name: in-cluster
+    namespace: crossplane-system
+  source:
+    chart: crossplane
+    repoURL: https://charts.crossplane.io/stable
+    targetRevision: 1.18.0
+    helm:
+      releaseName: crossplane

template/stacks/core/forgejo-runner.yaml

@@ -0,0 +1,22 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: forgejo-runner
+  namespace: argocd
+  labels:
+    env: dev
+  finalizers:
+    - resources-finalizer.argocd.argoproj.io
+spec:
+  project: default
+  syncPolicy:
+    automated:
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true
+  destination:
+    server: "https://kubernetes.default.svc"
+  source:
+    repoURL: https://{{{ .Env.DOMAIN_GITEA }}}/giteaAdmin/edfbuilder
+    targetRevision: HEAD
+    path: "stacks/core/forgejo-runner"

template/stacks/core/forgejo-runner/dind-docker.yaml

@@ -28,18 +28,19 @@ spec:
       # https://forgejo.org/docs/v1.21/admin/actions/#offline-registration
       initContainers:
         - name: runner-register
-          image: code.forgejo.org/forgejo/runner:6.3.1
+          image: code.forgejo.org/forgejo/runner:6.0.1
           command: 
-          - "sh"
-          - "-c"
-          - |
-            forgejo-runner \
-              register \
-              --no-interactive \
-              --token ${RUNNER_SECRET} \
-              --name ${RUNNER_NAME} \
-              --instance ${FORGEJO_INSTANCE_URL} \
-              --labels docker:docker://node:20-bookworm,ubuntu-22.04:docker://edp.buildth.ing/devfw-cicd/catthehackerubuntu:act-22.04,ubuntu-latest:docker://edp.buildth.ing/devfw-cicd/catthehackerubuntu:act-22.04
+            - "forgejo-runner"
+            - "register"
+            - "--no-interactive"
+            - "--token"
+            - $(RUNNER_SECRET)
+            - "--name"
+            - $(RUNNER_NAME)
+            - "--instance"
+            - $(FORGEJO_INSTANCE_URL)
+            - "--labels"
+            - "docker:docker://node:20-bookworm,ubuntu-22.04:docker://forgejo.edf-bootstrap.cx.fg1.ffm.osc.live/devfw-cicd/catthehackerubuntu:act-22.04,ubuntu-latest:docker://forgejo.edf-bootstrap.cx.fg1.ffm.osc.live/devfw-cicd/catthehackerubuntu:act-22.04"
           env:
             - name: RUNNER_NAME
               valueFrom:
@@ -57,7 +58,7 @@ spec:
               mountPath: /data
       containers:
       - name: runner
-        image: code.forgejo.org/forgejo/runner:6.3.1
+        image: code.forgejo.org/forgejo/runner:6.0.1
         command: 
           - "sh"
           - "-c"
@@ -93,7 +94,7 @@ spec:
         - name: runner-data
           mountPath: /data
       - name: daemon
-        image: docker:28.0.4-dind
+        image: docker:27.4.1-dind
         env:
         - name: DOCKER_TLS_CERTDIR
           value: /certs

template/stacks/core/forgejo.yaml

@@ -0,0 +1,27 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: forgejo
+  namespace: argocd
+  labels:
+    env: dev
+spec:
+  project: default
+  syncPolicy:
+    automated:
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true
+  destination:
+    name: in-cluster
+    namespace: gitea
+  sources:
+    - repoURL: https://code.forgejo.org/forgejo-helm/forgejo-helm.git
+      path: .
+      targetRevision: v10.1.1
+      helm:
+        valueFiles:
+          - $values/stacks/core/forgejo/values.yaml
+    - repoURL: https://{{{ .Env.DOMAIN_GITEA }}}/giteaAdmin/edfbuilder
+      targetRevision: HEAD
+      ref: values

template/stacks/core/forgejo/values.yaml

@@ -0,0 +1,56 @@
+redis-cluster:
+  enabled: false
+postgresql:
+  enabled: false
+postgresql-ha:
+  enabled: false
+
+persistence:
+  enabled: true
+  size: 5Gi
+
+test:
+  enabled: false
+
+gitea:
+  admin:
+    existingSecret: gitea-credential
+  config:
+    database:
+      DB_TYPE: sqlite3
+    session:
+      PROVIDER: memory
+    cache:
+      ADAPTER: memory
+    queue:
+      TYPE: level
+    server:
+      DOMAIN: '{{{ .Env.DOMAIN_GITEA }}}'
+      ROOT_URL: 'https://{{{ .Env.DOMAIN_GITEA }}}:443'
+
+service:
+  ssh:
+    type: NodePort
+    nodePort: 32222
+    externalTrafficPolicy: Local
+
+image:
+  pullPolicy: "IfNotPresent"
+  # Overrides the image tag whose default is the chart appVersion.
+  #tag: "8.0.3"
+  # Adds -rootless suffix to image name
+  rootless: true
+
+forgejo:
+  runner:
+    enabled: true
+    image:
+      tag: latest
+    # replicas: 3
+    config:
+      runner:
+        labels:
+          - docker:docker://node:16-bullseye
+          - self-hosted:docker://ghcr.io/catthehacker/ubuntu:act-22.04
+          - ubuntu-22.04:docker://ghcr.io/catthehacker/ubuntu:act-22.04
+          - ubuntu-latest:docker://ghcr.io/catthehacker/ubuntu:act-22.04

template/stacks/core/ingress-apps.yaml

@@ -0,0 +1,22 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: ingress-apps
+  namespace: argocd
+  labels:
+    example: ref-implementation
+  finalizers:
+    - resources-finalizer.argocd.argoproj.io
+spec:
+  destination:
+    server: "https://kubernetes.default.svc"
+  source:
+    repoURL: https://{{{ .Env.DOMAIN_GITEA }}}/giteaAdmin/edfbuilder
+    targetRevision: HEAD
+    path: "stacks/core/ingress-apps"
+  project: default
+  syncPolicy:
+    automated:
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true

template/stacks/core/ingress-apps/alloy.yaml

@@ -0,0 +1,18 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: alloy
+  namespace: monitoring
+spec:
+  ingressClassName: nginx
+  rules:
+  - host: {{{ .Env.DOMAIN }}}
+    http:
+      paths:
+      - backend:
+          service:
+            name: alloy
+            port:
+              number: 12345
+        path: /alloy
+        pathType: Prefix

template/stacks/core/ingress-apps/argo-workflows-ingress.yaml

@@ -0,0 +1,31 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    nginx.ingress.kubernetes.io/rewrite-target: /$2
+    nginx.ingress.kubernetes.io/use-regex: "true"
+  name: argo-workflows-ingress
+  namespace: argo
+spec:
+  ingressClassName: nginx
+  rules:
+  - host: localhost
+    http:
+      paths:
+      - backend:
+          service:
+            name: argo-server
+            port:
+              name: web
+        path: /argo-workflows(/|$)(.*)
+        pathType: ImplementationSpecific
+  - host: {{{ .Env.DOMAIN }}}
+    http:
+      paths:
+      - backend:
+          service:
+            name: argo-server
+            port:
+              name: web
+        path: /argo-workflows(/|$)(.*)
+        pathType: ImplementationSpecific

template/stacks/core/ingress-apps/argocd-server.yaml

@@ -4,10 +4,11 @@ metadata:
   annotations:
     nginx.ingress.kubernetes.io/backend-protocol: HTTP
     nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
-    cert-manager.io/cluster-issuer: main
+    nginx.ingress.kubernetes.io/rewrite-target: /$2
+    nginx.ingress.kubernetes.io/use-regex: "true"
 {{{ if eq .Env.CLUSTER_TYPE "osc" }}}
     dns.gardener.cloud/class: garden
-    dns.gardener.cloud/dnsnames: {{{ .Env.DOMAIN_ARGOCD }}}
+    dns.gardener.cloud/dnsnames: {{{ .Env.DOMAIN }}}
     dns.gardener.cloud/ttl: "600"
 {{{ end }}}
   name: argocd-server
@@ -15,7 +16,7 @@ metadata:
 spec:
   ingressClassName: nginx
   rules:
-  - host: {{{ .Env.DOMAIN_ARGOCD }}}
+  - host: {{{ .Env.DOMAIN }}}
     http:
       paths:
       - backend:
@@ -23,9 +24,9 @@ spec:
             name: argocd-server
             port:
               number: 80
-        path: /
-        pathType: Prefix
+        path: /argocd(/|$)(.*)
+        pathType: ImplementationSpecific
   tls:
   - hosts:
-    - {{{ .Env.DOMAIN_ARGOCD }}}
+    - {{{ .Env.DOMAIN }}}
     secretName: argocd-net-tls

template/stacks/core/ingress-apps/backstage.yaml

@@ -0,0 +1,28 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: backstage
+  namespace: backstage
+spec:
+  ingressClassName: nginx
+  rules:
+  - host: localhost
+    http:
+      paths:
+      - backend:
+          service:
+            name: backstage
+            port:
+              name: http
+        path: /
+        pathType: Prefix
+  - host: {{{ .Env.DOMAIN }}}
+    http:
+      paths:
+      - backend:
+          service:
+            name: backstage
+            port:
+              name: http
+        path: /
+        pathType: Prefix

template/stacks/core/ingress-apps/fibonacci-service.yaml

@@ -0,0 +1,18 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: fibonacci-service
+  namespace: fibonacci-app
+spec:
+  ingressClassName: nginx
+  rules:
+  - host: {{{ .Env.DOMAIN }}}
+    http:
+      paths:
+      - backend:
+          service:
+            name: fibonacci-service
+            port:
+              number: 9090
+        path: /fibonacci
+        pathType: Prefix

template/stacks/core/ingress-apps/forgejo.yaml

@@ -4,13 +4,12 @@ metadata:
   annotations:
     nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
     nginx.ingress.kubernetes.io/proxy-body-size: 512m
-    cert-manager.io/cluster-issuer: main
 {{{ if eq .Env.CLUSTER_TYPE "osc" }}}
     dns.gardener.cloud/class: garden
     dns.gardener.cloud/dnsnames: {{{ .Env.DOMAIN_GITEA }}}
     dns.gardener.cloud/ttl: "600"
 {{{ end }}}
-  name: forgejo-server
+  name: forgejo
   namespace: gitea
 spec:
   ingressClassName: nginx
@@ -20,7 +19,7 @@ spec:
       paths:
       - backend:
           service:
-            name: forgejo-server-http
+            name: forgejo-http
             port:
               number: 3000
         path: /

template/stacks/core/ingress-apps/keycloak-ingress-localhost.yaml

@@ -0,0 +1,28 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: keycloak-ingress-localhost
+  namespace: keycloak
+spec:
+  ingressClassName: nginx
+  rules:
+  - host: localhost
+    http:
+      paths:
+      - backend:
+          service:
+            name: keycloak
+            port:
+              name: http
+        path: /keycloak
+        pathType: ImplementationSpecific
+  - host: {{{ .Env.DOMAIN }}}
+    http:
+      paths:
+      - backend:
+          service:
+            name: keycloak
+            port:
+              name: http
+        path: /keycloak
+        pathType: ImplementationSpecific

template/stacks/core/ingress-apps/kube-prometheus-stack-grafana.yaml

@@ -0,0 +1,18 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: kube-prometheus-stack-grafana
+  namespace: monitoring
+spec:
+  ingressClassName: nginx
+  rules:
+  - host: {{{ .Env.DOMAIN }}}
+    http:
+      paths:
+      - backend:
+          service:
+            name: kube-prometheus-stack-grafana
+            port:
+              number: 80
+        path: /grafana
+        pathType: Prefix

template/stacks/core/ingress-apps/minio-console.yaml

@@ -0,0 +1,24 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: minio-console
+  namespace: minio-backup
+{{{ if eq .Env.CLUSTER_TYPE "osc" }}}
+  annotations:
+    dns.gardener.cloud/class: garden
+    dns.gardener.cloud/dnsnames: minio-backup.{{{ .Env.DOMAIN }}}
+    dns.gardener.cloud/ttl: "600"
+{{{ end }}}
+spec:
+  ingressClassName: nginx
+  rules:
+  - host: minio-backup.{{{ .Env.DOMAIN }}}
+    http:
+      paths:
+      - backend:
+          service:
+            name: minio-console
+            port:
+              number: 9001
+        path: /
+        pathType: Prefix

template/stacks/core/ingress-apps/openbao.yaml

@@ -0,0 +1,24 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: openbao
+  namespace: openbao
+{{{ if eq .Env.CLUSTER_TYPE "osc" }}}
+  annotations:
+    dns.gardener.cloud/class: garden
+    dns.gardener.cloud/dnsnames: openbao.{{{ .Env.DOMAIN }}}
+    dns.gardener.cloud/ttl: "600"
+{{{ end }}}
+spec:
+  ingressClassName: nginx
+  rules:
+  - host: openbao.{{{ .Env.DOMAIN }}}
+    http:
+      paths:
+      - backend:
+          service:
+            name: openbao
+            port:
+              number: 8200
+        path: /
+        pathType: Prefix

template/stacks/core/ingress-nginx.yaml

@@ -12,18 +12,16 @@ spec:
       selfHeal: true
     syncOptions:
       - CreateNamespace=true
-    retry:
-      limit: -1
   destination:
     name: in-cluster
     namespace: ingress-nginx
   sources:
-    - repoURL: https://edp.buildth.ing/DevFW-CICD/ingress-nginx-helm.git
+    - repoURL: https://github.com/kubernetes/ingress-nginx
       path: charts/ingress-nginx
-      targetRevision: helm-chart-4.12.1-depends
+      targetRevision: helm-chart-4.11.3
       helm:
         valueFiles:
-          - $values/{{{ .Env.CLIENT_REPO_ID }}}/{{{ .Env.DOMAIN }}}/stacks/otc/ingress-nginx/values.yaml
-    - repoURL: https://{{{ .Env.CLIENT_REPO_DOMAIN }}}/{{{ .Env.CLIENT_REPO_ORG_NAME }}}
+          - $values/stacks/core/ingress-nginx/values.yaml
+    - repoURL: https://{{{ .Env.DOMAIN_GITEA }}}/giteaAdmin/edfbuilder
       targetRevision: HEAD
       ref: values

template/stacks/core/ingress-nginx/values.yaml

@@ -0,0 +1,49 @@
+controller:
+  updateStrategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 1
+
+  ingressClassResource:
+    name: nginx
+
+  # added for idpbuilder
+  allowSnippetAnnotations: true
+
+  # added for idpbuilder
+  config:
+    proxy-buffer-size: 32k
+    use-forwarded-headers: "true"
+
+  # monitoring nginx
+  metrics:
+    enabled: true
+    serviceMonitor:
+      additionalLabels: 
+        release: "ingress-nginx"  
+      enabled: true
+
+{{{ if eq .Env.CLUSTER_TYPE "kind" }}}
+  hostPort:
+    enabled: true
+  terminationGracePeriodSeconds: 0
+  service:
+    type: NodePort
+
+  nodeSelector:
+    ingress-ready: "true"
+  tolerations:
+    - key: "node-role.kubernetes.io/master"
+      operator: "Equal"
+      effect: "NoSchedule"
+    - key: "node-role.kubernetes.io/control-plane"
+      operator: "Equal"
+      effect: "NoSchedule"
+
+  publishService:
+    enabled: false
+  extraArgs:
+    publish-status-address: localhost
+    # added for idpbuilder
+    enable-ssl-passthrough: ""
+{{{ end }}}

template/stacks/forgejo/forgejo-server.yaml

@@ -1,38 +0,0 @@
-apiVersion: argoproj.io/v1alpha1
-kind: Application
-metadata:
-  name: forgejo-server
-  namespace: argocd
-  labels:
-    env: dev
-spec:
-  project: default
-  syncPolicy:
-    automated:
-      selfHeal: true
-    syncOptions:
-      - CreateNamespace=true
-    retry:
-      limit: -1
-  destination:
-    name: in-cluster
-    namespace: gitea
-  sources:
-    - repoURL: https://edp.buildth.ing/DevFW-CICD/forgejo-helm.git
-      path: .
-      # first check out the desired version (example v9.0.0): https://code.forgejo.org/forgejo-helm/forgejo-helm/src/tag/v9.0.0/Chart.yaml
-      # (note that the chart version is not the same as the forgejo application version, which is specified in the above Chart.yaml file)
-      # then use the devops pipeline and select development, forgejo and the desired version (example v9.0.0):
-      # https://edp.buildth.ing/DevFW-CICD/devops-pipelines/actions?workflow=update-helm-depends.yaml&actor=0&status=0
-      # finally update the desired version here and include "-depends", it is created by the devops pipeline. 
-      # why do we have an added "-depends" tag? it resolves rate limitings when downloading helm OCI dependencies
-      targetRevision: v12.0.0-depends
-      helm:
-        valueFiles:
-          - $values/{{{ .Env.CLIENT_REPO_ID }}}/{{{ .Env.DOMAIN }}}/stacks/forgejo/forgejo-server/values.yaml
-    - repoURL: https://{{{ .Env.CLIENT_REPO_DOMAIN }}}/{{{ .Env.CLIENT_REPO_ORG_NAME }}}
-      targetRevision: HEAD
-      ref: values
-    - repoURL: https://{{{ .Env.CLIENT_REPO_DOMAIN }}}/{{{ .Env.CLIENT_REPO_ORG_NAME }}}
-      targetRevision: HEAD
-      path: "{{{ .Env.CLIENT_REPO_ID }}}/{{{ .Env.DOMAIN }}}/stacks/forgejo/forgejo-server/manifests"

template/stacks/forgejo/forgejo-server/values.yaml

@@ -1,180 +0,0 @@
-# We use recreate to make sure only one instance with one version is running, because Forgejo might break or data gets inconsistant. 
-strategy:
-  type: Recreate
-
-redis-cluster:
-  enabled: false
-
-redis:
-  enabled: false
-
-postgresql:
-  enabled: false
-
-postgresql-ha:
-  enabled: false
-
-persistence:
-  enabled: true
-  size: 200Gi
-  annotations:
-    everest.io/crypt-key-id: {{{ .Env.PVC_KMS_KEY_ID }}}
-
-test:
-  enabled: false
-
-deployment:
-  env:
-    - name: SSL_CERT_DIR
-      value: /etc/ssl/forgejo
-
-extraVolumeMounts:
-  - mountPath: /etc/ssl/forgejo
-    name: custom-database-certs-volume
-    readOnly: true
-
-extraVolumes:
-  - name: custom-database-certs-volume
-    secret:
-      secretName: custom-database-certs
-
-gitea:
-  additionalConfigFromEnvs:
-    - name: FORGEJO__storage__MINIO_ACCESS_KEY_ID
-      valueFrom:
-        secretKeyRef:
-          name: forgejo-cloud-credentials
-          key: access-key
-    - name: FORGEJO__storage__MINIO_SECRET_ACCESS_KEY
-      valueFrom:
-        secretKeyRef:
-          name: forgejo-cloud-credentials
-          key: secret-key
-    - name: FORGEJO__queue__CONN_STR
-      valueFrom:
-        secretKeyRef:
-          name: redis-forgejo-cloud-credentials
-          key: connection-string
-    - name: FORGEJO__session__PROVIDER_CONFIG
-      valueFrom:
-        secretKeyRef:
-          name: redis-forgejo-cloud-credentials
-          key: connection-string
-    - name: FORGEJO__cache__HOST
-      valueFrom:
-        secretKeyRef:
-          name: redis-forgejo-cloud-credentials
-          key: connection-string
-    - name: FORGEJO__database__HOST
-      valueFrom:
-        secretKeyRef:
-          name: postgres-forgejo-cloud-credentials
-          key: host_port
-    - name: FORGEJO__database__NAME
-      valueFrom:
-        secretKeyRef:
-          name: postgres-forgejo-cloud-credentials
-          key: database
-    - name: FORGEJO__database__USER
-      valueFrom:
-        secretKeyRef:
-          name: postgres-forgejo-cloud-credentials
-          key: username
-    - name: FORGEJO__database__PASSWD
-      valueFrom:
-        secretKeyRef:
-          name: postgres-forgejo-cloud-credentials
-          key: password
-    - name: FORGEJO__indexer__ISSUE_INDEXER_CONN_STR
-      valueFrom:
-        secretKeyRef:
-          name: elasticsearch-cloud-credentials
-          key: connection-string
-    - name: FORGEJO__mailer__PASSWD
-      valueFrom:
-        secretKeyRef:
-          name: email-user-credentials
-          key: connection-string
-
-  admin:
-    existingSecret: gitea-credential
-
-  config:
-    APP_NAME: 'EDP'
-    APP_SLOGAN: 'Build your thing in minutes'
-    indexer:
-      ISSUE_INDEXER_ENABLED: true
-      ISSUE_INDEXER_TYPE: elasticsearch
-      # TODO next
-      REPO_INDEXER_ENABLED: false
-      # REPO_INDEXER_TYPE: meilisearch # not yet working
-
-    storage:
-      MINIO_ENDPOINT: obs.eu-de.otc.t-systems.com:443
-      STORAGE_TYPE: minio
-      MINIO_LOCATION: eu-de
-      MINIO_BUCKET: edp-forgejo-{{{ .Env.CLUSTER_ENVIRONMENT }}}
-      MINIO_USE_SSL: true
-
-    queue:
-      TYPE: redis
-
-    session:
-      PROVIDER: redis
-
-    cache:
-      ENABLED: true
-      ADAPTER: redis
-
-    service:
-      DISABLE_REGISTRATION: true
-
-    other:
-      SHOW_FOOTER_VERSION: false
-      SHOW_FOOTER_TEMPLATE_LOAD_TIME: false
-
-    database:
-      DB_TYPE: postgres
-      SSL_MODE: verify-ca
-
-    server:
-      DOMAIN: '{{{ .Env.DOMAIN_GITEA }}}'
-      ROOT_URL: 'https://{{{ .Env.DOMAIN_GITEA }}}:443'
-
-    mailer:
-      ENABLED: true
-      USER: ipcei-cis-devfw@mms-support.de
-      PROTOCOL: smtps
-      FROM: '"IPCEI CIS DevFW" <ipcei-cis-devfw@mms-support.de>'
-      SMTP_ADDR: mail.mms-support.de
-      SMTP_PORT: 465
-
-service:
-  ssh:
-    type: LoadBalancer
-    nodePort: 32222
-    externalTrafficPolicy: Cluster
-    annotations:
-      kubernetes.io/elb.id: {{{ .Env.LOADBALANCER_ID }}}  
-
-image:
-  pullPolicy: "IfNotPresent"
-  # Overrides the image tag whose default is the chart appVersion.
-  #tag: "8.0.3"
-  # Adds -rootless suffix to image name
-  # rootless: true
-  fullOverride: forgejo.edf-bootstrap.cx.fg1.ffm.osc.live/devfw-cicd/edp-forgejo:prerelease-v11-0-1-rootless
-
-forgejo:
-  runner:
-    enabled: true
-    image:
-      tag: latest
-    # replicas: 3
-    config:
-      runner:
-        labels:
-          - docker:docker://node:16-bullseye
-          - self-hosted:docker://ghcr.io/catthehacker/ubuntu:act-22.04
-          - ubuntu-22.04:docker://ghcr.io/catthehacker/ubuntu:act-22.04
-          - ubuntu-latest:docker://ghcr.io/catthehacker/ubuntu:act-22.04

template/stacks/local-backup/README.md

@@ -0,0 +1,126 @@
+# Local Backup with Velero and Minio
+
+This is example is adapted from the original icpbuilder stack.
+
+The two significant changes from the original were made:
+
+* disabled `hostPath` mount to persist backups within kind, since backups do not work sufficiently in this example due to PVC issues, see below.
+* renamed `minio` namespace to `minio-backup` so it does not collide with other minio examples.
+
+Within kind, it can only backup kubernetes objects. Data from PVC's is skipped, see below why.
+
+[Velero](https://velero.io/) requires some compatible storage providers as its backup target. This local installation uses [MinIO](https://min.io/) as an example.
+However, MinIO is not officially supported by Velero but works due to S3 compatibility.
+
+The current setup does NOT persist backups but stores them in MinIO's PVCs. Proper backups should configure external storage, see [Supported Providers](https://velero.io/docs/main/supported-providers/).
+
+## Installation
+
+The stack is installed as part of the `./example.sh` run.
+
+In order to persist a local backup you have to mount a local directory within `main.go`:
+
+```yaml
+nodes:
+  - role: control-plane
+    extraMounts:
+      - hostPath: /some/path/backup # replace with your own path
+        containerPath: /backup
+```
+
+Kind creates the directory on the host but you might have to adjust the permissions, otherwise the minio pod fails to start.
+
+## Using it
+
+After the installation velero and minio should be visible in ArgoCD.
+
+During the installation credentials for minio are generated and shared with velero. You can access them manually:
+
+```bash
+kubectl -n minio-backup get secret root-creds -o go-template='{{ range $key, $value := .data }}{{ printf "%s: %s\n" $key ($value | base64decode) }}{{ end }}'
+# example output
+# rootPassword: aKKZzLnyry6OYZts17vMTf32H5ghFL4WYgu6bHujm
+# rootUser: ge8019yksArb7BICt3MLY9
+```
+
+A bucket in minio was created and velero uses it for its backups by default, see helm `values.yaml` files.
+
+
+### Backup and Restore
+
+Backups and subsequent restores can be scheduled by either using the velero cli or by creating CRD objects.
+
+Check the `./demo` directory for equivalent CRD manifests.
+
+Create a backup of the backstage namespace, see `schedule` task for more permanent setups:
+
+```shell
+velero backup create backstage-backup --include-namespaces backstage
+```
+
+There are more options to create a fine granular backup and to set the backup storage.
+See velero's docs for details.
+
+Check the backup with:
+```shell
+velero backup get
+```
+
+To get more details on the backup you need to be able to connect to velero's backup storage, i.e. minio.
+Using `kubefwd` here helps a lot (this is not necessary for restore).
+
+```shell
+kubefwd services -n minio-backup
+```
+
+More details with `describe` and `logs`:
+
+```shell
+velero backup describe backstage-backup --details
+velero backup logs backstage-backup
+```
+
+Restore the backup into the original namespace, you might want to delete the existing namespace beforehand:
+
+```shell
+kubectl delete namespace backstage
+velero restore create --from-backup backstage-backup
+```
+
+When restoring, velero does not replace existing objects in the backup target.
+
+ArgoCD does pickup on the changes and also validates that the backup is in sync.
+
+
+## Issues with Persistent Volumes
+
+Velero has no issue to backup kubernetes objects like Deployments, ConfigMaps, etc. since they are just yaml/json definitions.
+Volumes containing data are, however, more complex. The preferred type of backup are kubernetes' VolumeSnapshots as they consistently store the state
+of a volume at a given point in time in an atomic action. Those snapshots live within the cluster and are subsequently downloaded into one of velero's
+storage backends for safekeeping.
+
+However, VolumeSnapshots are only possible on storage backends that support them via CSI drivers.
+Backends like `nfs` or `hostPath` do NOT support them. Here, velero uses an alternative method
+called [File System Backups](https://velero.io/docs/main/file-system-backup/).
+In essence, this a simple copy operation based on the file system. Even though
+this uses more sophisticated tooling under the hood, i.e. kopia, it is not
+possible to create a backup in an atomic transaction. Thus, the resulting backup
+might be inconsistent.
+
+Furthermore, for file system backups to work velero installs a node-agent as a
+DaemonSet on each Kubernetes node. The agent is aware of the node's internal
+storage and accesses the directories on the host directly to copy the files.
+This is not supported for hostPath volumes as they mount an arbitrary path
+on the host. In theory, a backup is possible but due extra config and security
+considerations intentionally skipped. Kind's local-path provider storage uses
+a hostPath and is thus not supported for any kind of backup.
+
+## TODOs
+
+* The MinIO -backup installation is only intended as an example and must either
+be configured properly or replaced.
+
+* The current example does not automatically schedule backups.
+
+* velero chart must be properly parameterized
+

template/stacks/local-backup/demo/backup.yaml

@@ -0,0 +1,9 @@
+# velero backup create backstage-backup --include-namespaces backstage
+apiVersion: velero.io/v1
+kind: Backup
+metadata:
+  name: backstage-backup
+  namespace: velero
+spec:
+  includedNamespaces:
+    - 'backstage'

template/stacks/local-backup/demo/restore.yaml

@@ -0,0 +1,10 @@
+# velero restore create --from-backup backstage-backup
+apiVersion: velero.io/v1
+kind: Restore
+metadata:
+  name: backstage-backup
+  namespace: velero
+spec:
+  backupName: backstage-backup
+  includedNamespaces:
+    - 'backstage'

template/stacks/local-backup/minio.yaml

@@ -0,0 +1,33 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: minio
+  namespace: argocd
+  labels:
+    env: dev
+  finalizers:
+    - resources-finalizer.argocd.argoproj.io
+spec:
+  project: default
+  sources:
+    - repoURL: 'https://charts.min.io'
+      targetRevision: 5.0.15
+      helm:
+        releaseName: minio
+        valueFiles:
+          - $values/stacks/local-backup/minio/helm/values.yaml
+      chart: minio
+    - repoURL: https://{{{ .Env.DOMAIN_GITEA }}}/giteaAdmin/edfbuilder
+      targetRevision: HEAD
+      ref: values
+    - repoURL: https://{{{ .Env.DOMAIN_GITEA }}}/giteaAdmin/edfbuilder
+      targetRevision: HEAD
+      path: "stacks/local-backup/minio/manifests"
+  destination:
+    server: "https://kubernetes.default.svc"
+    namespace: minio-backup
+  syncPolicy:
+    syncOptions:
+      - CreateNamespace=true
+    automated:
+      selfHeal: true

template/stacks/local-backup/minio/helm/values.yaml

@@ -0,0 +1,17 @@
+replicas: 1
+mode: standalone
+
+resources:
+  requests:
+    memory: 128Mi
+
+persistence:
+  enabled: true
+  storageClass: standard
+  size: 512Mi
+  # volumeName: backup # re-enable this to mount a local host path, see minio-pv.yaml
+
+buckets:
+  - name: edfbuilder-backups
+
+existingSecret: root-creds

template/stacks/local-backup/minio/manifests/minio-pv.yaml

@@ -0,0 +1,13 @@
+# re-enable this config to mount a local host path, see `../helm/values.yaml`
+# apiVersion: v1
+# kind: PersistentVolume
+# metadata:
+#   name: backup
+# spec:
+#   storageClassName: standard
+#   accessModes:
+#     - ReadWriteOnce
+#   capacity:
+#     storage: 512Mi
+#   hostPath:
+#     path: /backup

template/stacks/local-backup/minio/manifests/secret-sync.yaml

@@ -0,0 +1,154 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: secret-sync
+  namespace: minio-backup
+  annotations:
+    argocd.argoproj.io/hook: Sync
+    argocd.argoproj.io/sync-wave: "-20"
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: secret-sync
+  namespace: minio-backup
+  annotations:
+    argocd.argoproj.io/hook: Sync
+    argocd.argoproj.io/sync-wave: "-20"
+rules:
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get", "create", "update", "patch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: secret-sync
+  namespace: minio-backup
+  annotations:
+    argocd.argoproj.io/hook: Sync
+    argocd.argoproj.io/sync-wave: "-20"
+subjects:
+  - kind: ServiceAccount
+    name: secret-sync
+    namespace: minio-backup
+roleRef:
+  kind: Role
+  name: secret-sync
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: secret-sync
+  namespace: velero
+  annotations:
+    argocd.argoproj.io/hook: Sync
+    argocd.argoproj.io/sync-wave: "-20"
+rules:
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get", "create", "update", "patch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: secret-sync
+  namespace: velero
+  annotations:
+    argocd.argoproj.io/hook: Sync
+    argocd.argoproj.io/sync-wave: "-20"
+subjects:
+  - kind: ServiceAccount
+    name: secret-sync
+    namespace: minio-backup
+roleRef:
+  kind: Role
+  name: secret-sync
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: secret-sync
+  namespace: minio-backup
+  annotations:
+    argocd.argoproj.io/hook: PostSync
+spec:
+  template:
+    metadata:
+      generateName: secret-sync
+    spec:
+      serviceAccountName: secret-sync
+      restartPolicy: Never
+      containers:
+        - name: kubectl
+          image: docker.io/bitnami/kubectl
+          command: ["/bin/bash", "-c"]
+          args:
+            - |
+              set -e
+              kubectl get secrets -n minio-backup root-creds -o json > /tmp/secret
+              ACCESS=$(jq -r '.data.rootUser | @base64d'  /tmp/secret)
+              SECRET=$(jq -r '.data.rootPassword | @base64d'  /tmp/secret)
+              
+              echo \
+              "apiVersion: v1
+              kind: Secret
+              metadata:
+                name: secret-key
+                namespace: velero
+              type: Opaque
+              stringData:
+                aws: |
+                  [default]
+                    aws_access_key_id=${ACCESS}
+                    aws_secret_access_key=${SECRET}
+              " > /tmp/secret.yaml
+              
+              kubectl apply -f /tmp/secret.yaml
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: minio-root-creds
+  namespace: minio-backup
+  annotations:
+    argocd.argoproj.io/hook: Sync
+    argocd.argoproj.io/sync-wave: "-10"
+spec:
+  template:
+    metadata:
+      generateName: minio-root-creds
+    spec:
+      serviceAccountName: secret-sync
+      restartPolicy: Never
+      containers:
+        - name: kubectl
+          image: docker.io/bitnami/kubectl
+          command: ["/bin/bash", "-c"]
+          args:
+            - |
+              kubectl get secrets -n minio-backup root-creds
+              if [ $? -eq 0 ]; then
+                exit 0
+              fi
+              
+              set -e
+              
+              NAME=$(openssl rand -base64 24)
+              PASS=$(openssl rand -base64 36)
+              
+              echo \
+              "apiVersion: v1
+              kind: Secret
+              metadata:
+                name: root-creds
+                namespace: minio-backup
+              type: Opaque
+              stringData:
+                rootUser: "${NAME}"
+                rootPassword: "${PASS}"
+              " > /tmp/secret.yaml
+              
+              kubectl apply -f /tmp/secret.yaml

template/stacks/local-backup/velero.yaml

@@ -0,0 +1,31 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: velero
+  namespace: argocd
+  labels:
+    env: dev
+  finalizers:
+    - resources-finalizer.argocd.argoproj.io
+spec:
+  project: default
+  sources:
+    - repoURL: 'https://vmware-tanzu.github.io/helm-charts'
+      targetRevision: 8.0.0
+      helm:
+        releaseName: velero
+        valueFiles:
+          - $values/stacks/local-backup/velero/helm/values.yaml
+      chart: velero
+    - repoURL: https://{{{ .Env.DOMAIN_GITEA }}}/giteaAdmin/edfbuilder
+      targetRevision: HEAD
+      ref: values
+  destination:
+    server: "https://kubernetes.default.svc"
+    namespace: velero
+  syncPolicy:
+    syncOptions:
+      - CreateNamespace=true
+    automated:
+      prune: true
+      selfHeal: true

template/stacks/local-backup/velero/helm/values.yaml

@@ -0,0 +1,25 @@
+resources:
+  requests:
+    memory: 128Mi
+initContainers:
+  - name: velero-plugin-for-aws
+    image: velero/velero-plugin-for-aws:v1.11.0
+    imagePullPolicy: IfNotPresent
+    volumeMounts:
+      - mountPath: /target
+        name: plugins
+# snapshotsEnabled: false # create snapshot crd?
+# deployNodeAgent: true # install node agent as daemonset for file system backups?
+configuration:
+  # defaultVolumesToFsBackup: true # backup pod volumes via fsb without explicit annotation?
+  backupStorageLocation:
+    - name: default
+      provider: aws
+      bucket: edfbuilder-backups
+      credential:
+        name: secret-key # this key is created within the minio-backup/secret-sync and injected into the velero namespace
+        key: aws
+      config:
+        region: minio
+        s3Url: http://minio.minio-backup.svc.cluster.local:9000 # internal resolution, external access for velero cli via fwd
+        s3ForcePathStyle: "true"

template/stacks/monitoring/alloy.yaml

@@ -0,0 +1,29 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: alloy
+  namespace: argocd
+  labels:
+    env: dev
+  finalizers:
+    - resources-finalizer.argocd.argoproj.io
+spec:
+  project: default
+  syncPolicy:
+    automated:
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true
+  destination:
+    name: in-cluster
+    namespace: monitoring
+  sources:
+    - repoURL: https://github.com/grafana/alloy.git
+      path: operations/helm/charts/alloy   
+      targetRevision: HEAD      
+      helm:
+        valueFiles:
+          - $values/stacks/monitoring/alloy/values.yaml
+    - repoURL: https://{{{ .Env.DOMAIN_GITEA }}}/giteaAdmin/edfbuilder
+      targetRevision: HEAD
+      ref: values

template/stacks/monitoring/alloy/values.yaml

@@ -0,0 +1,88 @@
+alloy:
+  uiPathPrefix: "/alloy"
+
+  create: false
+  name: alloy-config
+  key: config.alloy
+
+  configMap:
+    content: |-
+
+      logging {
+        level  = "info"
+        format = "logfmt"
+      }
+
+      loki.write "local_loki" {
+          endpoint {
+              url = "http://loki-loki-distributed-gateway/loki/api/v1/push"
+          }
+      }
+
+      discovery.kubernetes "pod" {
+        role = "pod"
+      }
+
+      discovery.kubernetes "nodes" {
+        role = "node"
+      }
+
+      discovery.kubernetes "services" {
+        role = "service"
+      }
+
+      discovery.kubernetes "endpoints" {
+        role = "endpoints"
+      }
+
+      discovery.kubernetes "endpointslices" {
+        role = "endpointslice"
+      }
+
+      discovery.kubernetes "ingresses" {
+        role = "ingress"
+      }
+
+      discovery.relabel "pod_logs" {
+        targets = discovery.kubernetes.pod.targets
+
+        rule {
+          source_labels = ["__meta_kubernetes_namespace"]
+          action        = "replace"
+          target_label  = "namespace"
+        }
+
+        rule {
+          source_labels = ["__meta_kubernetes_pod_name"]
+          action        = "replace"
+          target_label  = "pod"
+        }
+
+        rule {
+          source_labels = ["__meta_kubernetes_pod_node_name"]
+          action        = "replace"
+          target_label  = "node"
+        }
+
+        rule {
+          source_labels = ["__meta_kubernetes_pod_container_name"]
+          action        = "replace"
+          target_label  = "container"
+        }
+
+      }
+
+      local.file_match "openbao_file_logs" {
+        path_targets = [{"__path__" = "/openbao/logs/*"}]
+        sync_period = "5s"
+      }
+
+      loki.source.file "openbao_logs" {
+        targets    = local.file_match.openbao_file_logs.output
+        forward_to = [loki.write.local_loki.receiver]
+      }
+
+      loki.source.kubernetes "all_pod_logs" {
+        targets    = discovery.relabel.pod_logs.output
+        forward_to = [loki.write.local_loki.receiver]
+      }

template/stacks/monitoring/grafana-dashboards.yaml

@@ -0,0 +1,25 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: grafana-dashboards
+  namespace: argocd
+  labels:
+    env: dev
+  finalizers:
+    - resources-finalizer.argocd.argoproj.io
+spec:
+  project: default
+  source:
+    repoURL: https://{{{ .Env.DOMAIN_GITEA }}}/giteaAdmin/edfbuilder
+    targetRevision: HEAD
+    path: "stacks/monitoring/kube-prometheus/dashboards"
+  destination:
+    server: "https://kubernetes.default.svc"
+    namespace: monitoring
+  syncPolicy:
+    syncOptions:
+      - CreateNamespace=true
+    automated:
+      selfHeal: true
+    retry:
+      limit: -1

template/stacks/monitoring/kube-prometheus-sso.yaml

@@ -0,0 +1,29 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: kube-prometheus-sso
+  namespace: argocd
+  labels:
+    env: dev
+  finalizers:
+    - resources-finalizer.argocd.argoproj.io
+spec:
+  project: default
+  source:
+    repoURL: https://{{{ .Env.DOMAIN_GITEA }}}/giteaAdmin/edfbuilder
+    targetRevision: HEAD
+    path: "stacks/monitoring/kube-prometheus-sso"
+  destination:
+    server: "https://kubernetes.default.svc"
+    namespace: monitoring
+  syncPolicy:
+    syncOptions:
+      - CreateNamespace=true
+    automated:
+      selfHeal: true
+    retry:
+      limit: -1
+      backoff:
+        duration: 15s
+        factor: 1
+        maxDuration: 15s

template/stacks/monitoring/kube-prometheus-sso/secret-grafana.yaml

@@ -0,0 +1,21 @@
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: auth-generic-oauth-secret
+  namespace: monitoring
+spec:
+  secretStoreRef:
+    name: keycloak
+    kind: ClusterSecretStore
+  refreshInterval: "0"
+  target:
+    name: auth-generic-oauth-secret
+    template:
+      engineVersion: v2
+      data:
+        client_secret: "{{.GRAFANA_CLIENT_SECRET}}"
+  data:
+    - secretKey: GRAFANA_CLIENT_SECRET
+      remoteRef:
+        key: keycloak-clients
+        property: GRAFANA_CLIENT_SECRET

template/stacks/monitoring/kube-prometheus.yaml

@@ -0,0 +1,36 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: kube-prometheus-stack
+  namespace: argocd
+  labels:
+    env: dev
+  finalizers:
+    - resources-finalizer.argocd.argoproj.io
+spec:
+  project: default
+  syncPolicy:
+    automated:
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true
+      - ServerSideApply=true  # do not copy metdata, since (because of its large size) it can lead to sync failure
+    retry:
+      limit: -1
+      backoff:
+        duration: 15s
+        factor: 1
+        maxDuration: 15s
+  destination:
+    name: in-cluster
+    namespace: monitoring
+  sources:
+    - repoURL: https://github.com/prometheus-community/helm-charts
+      path: charts/kube-prometheus-stack
+      targetRevision: HEAD
+      helm:
+        valueFiles:
+          - $values/stacks/monitoring/kube-prometheus/values.yaml
+    - repoURL: https://{{{ .Env.DOMAIN_GITEA }}}/giteaAdmin/edfbuilder
+      targetRevision: HEAD
+      ref: values

template/stacks/monitoring/kube-prometheus/dashboards/dashboard_loki_apps.yaml

@@ -0,0 +1,268 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: grafana-dashboard-1
+  labels:
+    grafana_dashboard: "1"
+data:
+  k8s-dashboard-01.json: |
+    {
+        "annotations": {
+            "list": [
+                {
+                    "builtIn": 1,
+                    "datasource": {
+                        "type": "grafana",
+                        "uid": "-- Grafana --"
+                    },
+                    "enable": true,
+                    "hide": true,
+                    "iconColor": "rgba(0, 211, 255, 1)",
+                    "name": "Annotations & Alerts",
+                    "type": "dashboard"
+                }
+            ]
+        },
+        "editable": true,
+        "fiscalYearStartMonth": 0,
+        "graphTooltip": 0,
+        "id": 1,
+        "links": [
+        ],
+        "panels": [
+            {
+                "datasource": {
+                    "type": "loki",
+                    "uid": "P8E80F9AEF21F6940"
+                },
+                "fieldConfig": {
+                    "defaults": {
+                    },
+                    "overrides": [
+                    ]
+                },
+                "gridPos": {
+                    "h": 8,
+                    "w": 24,
+                    "x": 0,
+                    "y": 0
+                },
+                "id": 5,
+                "options": {
+                    "dedupStrategy": "none",
+                    "enableLogDetails": true,
+                    "prettifyLogMessage": false,
+                    "showCommonLabels": false,
+                    "showLabels": false,
+                    "showTime": false,
+                    "sortOrder": "Descending",
+                    "wrapLogMessage": false
+                },
+                "pluginVersion": "11.3.1",
+                "targets": [
+                    {
+                        "datasource": {
+                            "type": "loki",
+                            "uid": "P8E80F9AEF21F6940"
+                        },
+                        "expr": "{app=\"crossplane\"}",
+                        "queryType": "range",
+                        "refId": "A"
+                    }
+                ],
+                "title": "Logs: App crossplane",
+                "type": "logs"
+            },
+            {
+                "datasource": {
+                    "type": "loki",
+                    "uid": "P8E80F9AEF21F6940"
+                },
+                "fieldConfig": {
+                    "defaults": {
+                    },
+                    "overrides": [
+                    ]
+                },
+                "gridPos": {
+                    "h": 8,
+                    "w": 24,
+                    "x": 0,
+                    "y": 8
+                },
+                "id": 4,
+                "options": {
+                    "dedupStrategy": "none",
+                    "enableLogDetails": true,
+                    "prettifyLogMessage": false,
+                    "showCommonLabels": false,
+                    "showLabels": false,
+                    "showTime": false,
+                    "sortOrder": "Descending",
+                    "wrapLogMessage": false
+                },
+                "pluginVersion": "11.3.1",
+                "targets": [
+                    {
+                        "datasource": {
+                            "type": "loki",
+                            "uid": "P8E80F9AEF21F6940"
+                        },
+                        "expr": "{app=\"argo-server\"}",
+                        "queryType": "range",
+                        "refId": "A"
+                    }
+                ],
+                "title": "Logs: App argo-server",
+                "type": "logs"
+            },
+            {
+                "datasource": {
+                    "type": "loki",
+                    "uid": "P8E80F9AEF21F6940"
+                },
+                "fieldConfig": {
+                    "defaults": {
+                    },
+                    "overrides": [
+                    ]
+                },
+                "gridPos": {
+                    "h": 8,
+                    "w": 24,
+                    "x": 0,
+                    "y": 16
+                },
+                "id": 3,
+                "options": {
+                    "dedupStrategy": "none",
+                    "enableLogDetails": true,
+                    "prettifyLogMessage": false,
+                    "showCommonLabels": false,
+                    "showLabels": false,
+                    "showTime": false,
+                    "sortOrder": "Descending",
+                    "wrapLogMessage": false
+                },
+                "pluginVersion": "11.3.1",
+                "targets": [
+                    {
+                        "datasource": {
+                            "type": "loki",
+                            "uid": "P8E80F9AEF21F6940"
+                        },
+                        "expr": "{app=\"forgejo\"}",
+                        "queryType": "range",
+                        "refId": "A"
+                    }
+                ],
+                "title": "Logs: App forgejo",
+                "type": "logs"
+            },
+            {
+                "datasource": {
+                    "type": "loki",
+                    "uid": "P8E80F9AEF21F6940"
+                },
+                "fieldConfig": {
+                    "defaults": {
+                    },
+                    "overrides": [
+                    ]
+                },
+                "gridPos": {
+                    "h": 8,
+                    "w": 24,
+                    "x": 0,
+                    "y": 24
+                },
+                "id": 2,
+                "options": {
+                    "dedupStrategy": "none",
+                    "enableLogDetails": true,
+                    "prettifyLogMessage": false,
+                    "showCommonLabels": false,
+                    "showLabels": false,
+                    "showTime": false,
+                    "sortOrder": "Descending",
+                    "wrapLogMessage": false
+                },
+                "pluginVersion": "11.3.1",
+                "targets": [
+                    {
+                        "datasource": {
+                            "type": "loki",
+                            "uid": "P8E80F9AEF21F6940"
+                        },
+                        "expr": "{app=\"backstage\"}",
+                        "queryType": "range",
+                        "refId": "A"
+                    }
+                ],
+                "title": "Logs: App backstage",
+                "type": "logs"
+            },
+            {
+                "datasource": {
+                    "type": "loki",
+                    "uid": "P8E80F9AEF21F6940"
+                },
+                "fieldConfig": {
+                    "defaults": {
+                    },
+                    "overrides": [
+                    ]
+                },
+                "gridPos": {
+                    "h": 8,
+                    "w": 24,
+                    "x": 0,
+                    "y": 32
+                },
+                "id": 1,
+                "options": {
+                    "dedupStrategy": "none",
+                    "enableLogDetails": true,
+                    "prettifyLogMessage": false,
+                    "showCommonLabels": false,
+                    "showLabels": false,
+                    "showTime": false,
+                    "sortOrder": "Descending",
+                    "wrapLogMessage": false
+                },
+                "pluginVersion": "11.3.1",
+                "targets": [
+                    {
+                        "datasource": {
+                            "type": "loki",
+                            "uid": "P8E80F9AEF21F6940"
+                        },
+                        "expr": "{app=\"shoot-control-plane\"}",
+                        "queryType": "range",
+                        "refId": "A"
+                    }
+                ],
+                "title": "Logs: App shoot-control-plane",
+                "type": "logs"
+            }
+        ],
+        "preload": false,
+        "schemaVersion": 40,
+        "tags": [
+        ],
+        "templating": {
+            "list": [
+            ]
+        },
+        "time": {
+            "from": "now-6h",
+            "to": "now"
+        },
+        "timepicker": {
+        },
+        "timezone": "browser",
+        "title": "Loki Logs: Apps",
+        "uid": "ee4iuluru756of",
+        "version": 2,
+        "weekStart": ""
+    }

template/stacks/monitoring/kube-prometheus/dashboards/dashboard_loki_components.yaml

@@ -0,0 +1,845 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: grafana-dashboard-2
+  labels:
+    grafana_dashboard: "1"
+data:
+  k8s-dashboard-02.json: |
+    {
+        "annotations": {
+            "list": [
+                {
+                    "builtIn": 1,
+                    "datasource": {
+                        "type": "grafana",
+                        "uid": "-- Grafana --"
+                    },
+                    "enable": true,
+                    "hide": true,
+                    "iconColor": "rgba(0, 211, 255, 1)",
+                    "name": "Annotations & Alerts",
+                    "type": "dashboard"
+                }
+            ]
+        },
+        "editable": true,
+        "fiscalYearStartMonth": 0,
+        "graphTooltip": 0,
+        "id": 30,
+        "links": [
+        ],
+        "panels": [
+            {
+                "datasource": {
+                    "type": "loki",
+                    "uid": "P8E80F9AEF21F6940"
+                },
+                "fieldConfig": {
+                    "defaults": {
+                    },
+                    "overrides": [
+                    ]
+                },
+                "gridPos": {
+                    "h": 8,
+                    "w": 24,
+                    "x": 0,
+                    "y": 0
+                },
+                "id": 19,
+                "options": {
+                    "dedupStrategy": "none",
+                    "enableLogDetails": true,
+                    "prettifyLogMessage": false,
+                    "showCommonLabels": false,
+                    "showLabels": false,
+                    "showTime": false,
+                    "sortOrder": "Descending",
+                    "wrapLogMessage": false
+                },
+                "pluginVersion": "11.3.1",
+                "targets": [
+                    {
+                        "datasource": {
+                            "type": "loki",
+                            "uid": "P8E80F9AEF21F6940"
+                        },
+                        "editorMode": "builder",
+                        "expr": "{component=\"server\"} |= ``",
+                        "queryType": "range",
+                        "refId": "A"
+                    }
+                ],
+                "title": "Logs: Component server",
+                "type": "logs"
+            },
+            {
+                "datasource": {
+                    "type": "loki",
+                    "uid": "P8E80F9AEF21F6940"
+                },
+                "fieldConfig": {
+                    "defaults": {
+                    },
+                    "overrides": [
+                    ]
+                },
+                "gridPos": {
+                    "h": 8,
+                    "w": 24,
+                    "x": 0,
+                    "y": 8
+                },
+                "id": 17,
+                "options": {
+                    "dedupStrategy": "none",
+                    "enableLogDetails": true,
+                    "prettifyLogMessage": false,
+                    "showCommonLabels": false,
+                    "showLabels": false,
+                    "showTime": false,
+                    "sortOrder": "Descending",
+                    "wrapLogMessage": false
+                },
+                "pluginVersion": "11.3.1",
+                "targets": [
+                    {
+                        "datasource": {
+                            "type": "loki",
+                            "uid": "P8E80F9AEF21F6940"
+                        },
+                        "editorMode": "builder",
+                        "expr": "{component=\"repo-server\"} |= ``",
+                        "queryType": "range",
+                        "refId": "A"
+                    }
+                ],
+                "title": "Logs: Component repo-server",
+                "type": "logs"
+            },
+            {
+                "datasource": {
+                    "type": "loki",
+                    "uid": "P8E80F9AEF21F6940"
+                },
+                "fieldConfig": {
+                    "defaults": {
+                    },
+                    "overrides": [
+                    ]
+                },
+                "gridPos": {
+                    "h": 8,
+                    "w": 24,
+                    "x": 0,
+                    "y": 16
+                },
+                "id": 16,
+                "options": {
+                    "dedupStrategy": "none",
+                    "enableLogDetails": true,
+                    "prettifyLogMessage": false,
+                    "showCommonLabels": false,
+                    "showLabels": false,
+                    "showTime": false,
+                    "sortOrder": "Descending",
+                    "wrapLogMessage": false
+                },
+                "pluginVersion": "11.3.1",
+                "targets": [
+                    {
+                        "datasource": {
+                            "type": "loki",
+                            "uid": "P8E80F9AEF21F6940"
+                        },
+                        "editorMode": "builder",
+                        "expr": "{component=\"redis\"} |= ``",
+                        "queryType": "range",
+                        "refId": "A"
+                    }
+                ],
+                "title": "Logs: Component redis",
+                "type": "logs"
+            },
+            {
+                "datasource": {
+                    "type": "loki",
+                    "uid": "P8E80F9AEF21F6940"
+                },
+                "fieldConfig": {
+                    "defaults": {
+                    },
+                    "overrides": [
+                    ]
+                },
+                "gridPos": {
+                    "h": 8,
+                    "w": 24,
+                    "x": 0,
+                    "y": 24
+                },
+                "id": 15,
+                "options": {
+                    "dedupStrategy": "none",
+                    "enableLogDetails": true,
+                    "prettifyLogMessage": false,
+                    "showCommonLabels": false,
+                    "showLabels": false,
+                    "showTime": false,
+                    "sortOrder": "Descending",
+                    "wrapLogMessage": false
+                },
+                "pluginVersion": "11.3.1",
+                "targets": [
+                    {
+                        "datasource": {
+                            "type": "loki",
+                            "uid": "P8E80F9AEF21F6940"
+                        },
+                        "editorMode": "builder",
+                        "expr": "{component=\"query-frontend\"} |= ``",
+                        "queryType": "range",
+                        "refId": "A"
+                    }
+                ],
+                "title": "Logs: Component query-frontend",
+                "type": "logs"
+            },
+            {
+                "datasource": {
+                    "type": "loki",
+                    "uid": "P8E80F9AEF21F6940"
+                },
+                "fieldConfig": {
+                    "defaults": {
+                    },
+                    "overrides": [
+                    ]
+                },
+                "gridPos": {
+                    "h": 8,
+                    "w": 24,
+                    "x": 0,
+                    "y": 32
+                },
+                "id": 14,
+                "options": {
+                    "dedupStrategy": "none",
+                    "enableLogDetails": true,
+                    "prettifyLogMessage": false,
+                    "showCommonLabels": false,
+                    "showLabels": false,
+                    "showTime": false,
+                    "sortOrder": "Descending",
+                    "wrapLogMessage": false
+                },
+                "pluginVersion": "11.3.1",
+                "targets": [
+                    {
+                        "datasource": {
+                            "type": "loki",
+                            "uid": "P8E80F9AEF21F6940"
+                        },
+                        "editorMode": "builder",
+                        "expr": "{component=\"querier\"} |= ``",
+                        "queryType": "range",
+                        "refId": "A"
+                    }
+                ],
+                "title": "Logs: Component querier",
+                "type": "logs"
+            },
+            {
+                "datasource": {
+                    "type": "loki",
+                    "uid": "P8E80F9AEF21F6940"
+                },
+                "fieldConfig": {
+                    "defaults": {
+                    },
+                    "overrides": [
+                    ]
+                },
+                "gridPos": {
+                    "h": 8,
+                    "w": 24,
+                    "x": 0,
+                    "y": 40
+                },
+                "id": 13,
+                "options": {
+                    "dedupStrategy": "none",
+                    "enableLogDetails": true,
+                    "prettifyLogMessage": false,
+                    "showCommonLabels": false,
+                    "showLabels": false,
+                    "showTime": false,
+                    "sortOrder": "Descending",
+                    "wrapLogMessage": false
+                },
+                "pluginVersion": "11.3.1",
+                "targets": [
+                    {
+                        "datasource": {
+                            "type": "loki",
+                            "uid": "P8E80F9AEF21F6940"
+                        },
+                        "editorMode": "builder",
+                        "expr": "{component=\"prometheus-operator-webhook\"} |= ``",
+                        "queryType": "range",
+                        "refId": "A"
+                    }
+                ],
+                "title": "Logs: Component prometheus-operator-webhook",
+                "type": "logs"
+            },
+            {
+                "datasource": {
+                    "type": "loki",
+                    "uid": "P8E80F9AEF21F6940"
+                },
+                "fieldConfig": {
+                    "defaults": {
+                    },
+                    "overrides": [
+                    ]
+                },
+                "gridPos": {
+                    "h": 8,
+                    "w": 24,
+                    "x": 0,
+                    "y": 48
+                },
+                "id": 12,
+                "options": {
+                    "dedupStrategy": "none",
+                    "enableLogDetails": true,
+                    "prettifyLogMessage": false,
+                    "showCommonLabels": false,
+                    "showLabels": false,
+                    "showTime": false,
+                    "sortOrder": "Descending",
+                    "wrapLogMessage": false
+                },
+                "pluginVersion": "11.3.1",
+                "targets": [
+                    {
+                        "datasource": {
+                            "type": "loki",
+                            "uid": "P8E80F9AEF21F6940"
+                        },
+                        "editorMode": "builder",
+                        "expr": "{component=\"prometheus-operator\"} |= ``",
+                        "queryType": "range",
+                        "refId": "A"
+                    }
+                ],
+                "title": "Logs: Component prometheus-operator",
+                "type": "logs"
+            },
+            {
+                "datasource": {
+                    "type": "loki",
+                    "uid": "P8E80F9AEF21F6940"
+                },
+                "fieldConfig": {
+                    "defaults": {
+                    },
+                    "overrides": [
+                    ]
+                },
+                "gridPos": {
+                    "h": 8,
+                    "w": 24,
+                    "x": 0,
+                    "y": 56
+                },
+                "id": 11,
+                "options": {
+                    "dedupStrategy": "none",
+                    "enableLogDetails": true,
+                    "prettifyLogMessage": false,
+                    "showCommonLabels": false,
+                    "showLabels": false,
+                    "showTime": false,
+                    "sortOrder": "Descending",
+                    "wrapLogMessage": false
+                },
+                "pluginVersion": "11.3.1",
+                "targets": [
+                    {
+                        "datasource": {
+                            "type": "loki",
+                            "uid": "P8E80F9AEF21F6940"
+                        },
+                        "editorMode": "builder",
+                        "expr": "{component=\"metrics\"} |= ``",
+                        "queryType": "range",
+                        "refId": "A"
+                    }
+                ],
+                "title": "Logs: Component metrics",
+                "type": "logs"
+            },
+            {
+                "datasource": {
+                    "type": "loki",
+                    "uid": "P8E80F9AEF21F6940"
+                },
+                "fieldConfig": {
+                    "defaults": {
+                    },
+                    "overrides": [
+                    ]
+                },
+                "gridPos": {
+                    "h": 8,
+                    "w": 24,
+                    "x": 0,
+                    "y": 64
+                },
+                "id": 10,
+                "options": {
+                    "dedupStrategy": "none",
+                    "enableLogDetails": true,
+                    "prettifyLogMessage": false,
+                    "showCommonLabels": false,
+                    "showLabels": false,
+                    "showTime": false,
+                    "sortOrder": "Descending",
+                    "wrapLogMessage": false
+                },
+                "pluginVersion": "11.3.1",
+                "targets": [
+                    {
+                        "datasource": {
+                            "type": "loki",
+                            "uid": "P8E80F9AEF21F6940"
+                        },
+                        "editorMode": "builder",
+                        "expr": "{component=\"kube-scheduler\"} |= ``",
+                        "queryType": "range",
+                        "refId": "A"
+                    }
+                ],
+                "title": "Logs: Component kube-scheduler",
+                "type": "logs"
+            },
+            {
+                "datasource": {
+                    "type": "loki",
+                    "uid": "P8E80F9AEF21F6940"
+                },
+                "fieldConfig": {
+                    "defaults": {
+                    },
+                    "overrides": [
+                    ]
+                },
+                "gridPos": {
+                    "h": 8,
+                    "w": 24,
+                    "x": 0,
+                    "y": 72
+                },
+                "id": 9,
+                "options": {
+                    "dedupStrategy": "none",
+                    "enableLogDetails": true,
+                    "prettifyLogMessage": false,
+                    "showCommonLabels": false,
+                    "showLabels": false,
+                    "showTime": false,
+                    "sortOrder": "Descending",
+                    "wrapLogMessage": false
+                },
+                "pluginVersion": "11.3.1",
+                "targets": [
+                    {
+                        "datasource": {
+                            "type": "loki",
+                            "uid": "P8E80F9AEF21F6940"
+                        },
+                        "editorMode": "builder",
+                        "expr": "{component=\"kube-controller-manager\"} |= ``",
+                        "queryType": "range",
+                        "refId": "A"
+                    }
+                ],
+                "title": "Logs: Component kube-controller-manager",
+                "type": "logs"
+            },
+            {
+                "datasource": {
+                    "type": "loki",
+                    "uid": "P8E80F9AEF21F6940"
+                },
+                "fieldConfig": {
+                    "defaults": {
+                    },
+                    "overrides": [
+                    ]
+                },
+                "gridPos": {
+                    "h": 8,
+                    "w": 24,
+                    "x": 0,
+                    "y": 80
+                },
+                "id": 8,
+                "options": {
+                    "dedupStrategy": "none",
+                    "enableLogDetails": true,
+                    "prettifyLogMessage": false,
+                    "showCommonLabels": false,
+                    "showLabels": false,
+                    "showTime": false,
+                    "sortOrder": "Descending",
+                    "wrapLogMessage": false
+                },
+                "pluginVersion": "11.3.1",
+                "targets": [
+                    {
+                        "datasource": {
+                            "type": "loki",
+                            "uid": "P8E80F9AEF21F6940"
+                        },
+                        "editorMode": "builder",
+                        "expr": "{component=\"kube-apiserver\"} |= ``",
+                        "queryType": "range",
+                        "refId": "A"
+                    }
+                ],
+                "title": "Logs: Component kube-apiserver",
+                "type": "logs"
+            },
+            {
+                "datasource": {
+                    "type": "loki",
+                    "uid": "P8E80F9AEF21F6940"
+                },
+                "fieldConfig": {
+                    "defaults": {
+                    },
+                    "overrides": [
+                    ]
+                },
+                "gridPos": {
+                    "h": 8,
+                    "w": 24,
+                    "x": 0,
+                    "y": 88
+                },
+                "id": 7,
+                "options": {
+                    "dedupStrategy": "none",
+                    "enableLogDetails": true,
+                    "prettifyLogMessage": false,
+                    "showCommonLabels": false,
+                    "showLabels": false,
+                    "showTime": false,
+                    "sortOrder": "Descending",
+                    "wrapLogMessage": false
+                },
+                "pluginVersion": "11.3.1",
+                "targets": [
+                    {
+                        "datasource": {
+                            "type": "loki",
+                            "uid": "P8E80F9AEF21F6940"
+                        },
+                        "editorMode": "builder",
+                        "expr": "{component=\"ingester\"} |= ``",
+                        "queryType": "range",
+                        "refId": "A"
+                    }
+                ],
+                "title": "Logs: Component ingester",
+                "type": "logs"
+            },
+            {
+                "datasource": {
+                    "type": "loki",
+                    "uid": "P8E80F9AEF21F6940"
+                },
+                "fieldConfig": {
+                    "defaults": {
+                    },
+                    "overrides": [
+                    ]
+                },
+                "gridPos": {
+                    "h": 8,
+                    "w": 24,
+                    "x": 0,
+                    "y": 96
+                },
+                "id": 6,
+                "options": {
+                    "dedupStrategy": "none",
+                    "enableLogDetails": true,
+                    "prettifyLogMessage": false,
+                    "showCommonLabels": false,
+                    "showLabels": false,
+                    "showTime": false,
+                    "sortOrder": "Descending",
+                    "wrapLogMessage": false
+                },
+                "pluginVersion": "11.3.1",
+                "targets": [
+                    {
+                        "datasource": {
+                            "type": "loki",
+                            "uid": "P8E80F9AEF21F6940"
+                        },
+                        "editorMode": "builder",
+                        "expr": "{component=\"gateway\"} |= ``",
+                        "queryType": "range",
+                        "refId": "A"
+                    }
+                ],
+                "title": "Logs: Component gateway",
+                "type": "logs"
+            },
+            {
+                "datasource": {
+                    "type": "loki",
+                    "uid": "P8E80F9AEF21F6940"
+                },
+                "fieldConfig": {
+                    "defaults": {
+                    },
+                    "overrides": [
+                    ]
+                },
+                "gridPos": {
+                    "h": 8,
+                    "w": 24,
+                    "x": 0,
+                    "y": 104
+                },
+                "id": 5,
+                "options": {
+                    "dedupStrategy": "none",
+                    "enableLogDetails": true,
+                    "prettifyLogMessage": false,
+                    "showCommonLabels": false,
+                    "showLabels": false,
+                    "showTime": false,
+                    "sortOrder": "Descending",
+                    "wrapLogMessage": false
+                },
+                "pluginVersion": "11.3.1",
+                "targets": [
+                    {
+                        "datasource": {
+                            "type": "loki",
+                            "uid": "P8E80F9AEF21F6940"
+                        },
+                        "editorMode": "builder",
+                        "expr": "{component=\"etcd\"} |= ``",
+                        "queryType": "range",
+                        "refId": "A"
+                    }
+                ],
+                "title": "Logs: Component etcd",
+                "type": "logs"
+            },
+            {
+                "datasource": {
+                    "type": "loki",
+                    "uid": "P8E80F9AEF21F6940"
+                },
+                "fieldConfig": {
+                    "defaults": {
+                    },
+                    "overrides": [
+                    ]
+                },
+                "gridPos": {
+                    "h": 8,
+                    "w": 24,
+                    "x": 0,
+                    "y": 112
+                },
+                "id": 4,
+                "options": {
+                    "dedupStrategy": "none",
+                    "enableLogDetails": true,
+                    "prettifyLogMessage": false,
+                    "showCommonLabels": false,
+                    "showLabels": false,
+                    "showTime": false,
+                    "sortOrder": "Descending",
+                    "wrapLogMessage": false
+                },
+                "pluginVersion": "11.3.1",
+                "targets": [
+                    {
+                        "datasource": {
+                            "type": "loki",
+                            "uid": "P8E80F9AEF21F6940"
+                        },
+                        "editorMode": "builder",
+                        "expr": "{component=\"distributor\"} |= ``",
+                        "queryType": "range",
+                        "refId": "A"
+                    }
+                ],
+                "title": "Logs: Component distributor",
+                "type": "logs"
+            },
+            {
+                "datasource": {
+                    "type": "loki",
+                    "uid": "P8E80F9AEF21F6940"
+                },
+                "fieldConfig": {
+                    "defaults": {
+                    },
+                    "overrides": [
+                    ]
+                },
+                "gridPos": {
+                    "h": 8,
+                    "w": 24,
+                    "x": 0,
+                    "y": 120
+                },
+                "id": 3,
+                "options": {
+                    "dedupStrategy": "none",
+                    "enableLogDetails": true,
+                    "prettifyLogMessage": false,
+                    "showCommonLabels": false,
+                    "showLabels": false,
+                    "showTime": false,
+                    "sortOrder": "Descending",
+                    "wrapLogMessage": false
+                },
+                "pluginVersion": "11.3.1",
+                "targets": [
+                    {
+                        "datasource": {
+                            "type": "loki",
+                            "uid": "P8E80F9AEF21F6940"
+                        },
+                        "editorMode": "builder",
+                        "expr": "{component=\"controller\"} |= ``",
+                        "queryType": "range",
+                        "refId": "A"
+                    }
+                ],
+                "title": "Logs: Component controller",
+                "type": "logs"
+            },
+            {
+                "datasource": {
+                    "type": "loki",
+                    "uid": "P8E80F9AEF21F6940"
+                },
+                "fieldConfig": {
+                    "defaults": {
+                    },
+                    "overrides": [
+                    ]
+                },
+                "gridPos": {
+                    "h": 8,
+                    "w": 24,
+                    "x": 0,
+                    "y": 128
+                },
+                "id": 2,
+                "options": {
+                    "dedupStrategy": "none",
+                    "enableLogDetails": true,
+                    "prettifyLogMessage": false,
+                    "showCommonLabels": false,
+                    "showLabels": false,
+                    "showTime": false,
+                    "sortOrder": "Descending",
+                    "wrapLogMessage": false
+                },
+                "pluginVersion": "11.3.1",
+                "targets": [
+                    {
+                        "datasource": {
+                            "type": "loki",
+                            "uid": "P8E80F9AEF21F6940"
+                        },
+                        "editorMode": "builder",
+                        "expr": "{component=\"cloud-infrastructure-controller\"} |= ``",
+                        "queryType": "range",
+                        "refId": "A"
+                    }
+                ],
+                "title": "Logs: Component cloud-infrastructure-controller",
+                "type": "logs"
+            },
+            {
+                "datasource": {
+                    "type": "loki",
+                    "uid": "P8E80F9AEF21F6940"
+                },
+                "fieldConfig": {
+                    "defaults": {
+                    },
+                    "overrides": [
+                    ]
+                },
+                "gridPos": {
+                    "h": 8,
+                    "w": 24,
+                    "x": 0,
+                    "y": 136
+                },
+                "id": 1,
+                "options": {
+                    "dedupStrategy": "none",
+                    "enableLogDetails": true,
+                    "prettifyLogMessage": false,
+                    "showCommonLabels": false,
+                    "showLabels": false,
+                    "showTime": false,
+                    "sortOrder": "Descending",
+                    "wrapLogMessage": false
+                },
+                "pluginVersion": "11.3.1",
+                "targets": [
+                    {
+                        "datasource": {
+                            "type": "loki",
+                            "uid": "P8E80F9AEF21F6940"
+                        },
+                        "editorMode": "builder",
+                        "expr": "{component=\"applicationset-controller\"} |= ``",
+                        "queryType": "range",
+                        "refId": "A"
+                    }
+                ],
+                "title": "Logs: Component application-controller",
+                "type": "logs"
+            }
+        ],
+        "preload": false,
+        "schemaVersion": 40,
+        "tags": [
+        ],
+        "templating": {
+            "list": [
+            ]
+        },
+        "time": {
+            "from": "now-6h",
+            "to": "now"
+        },
+        "timepicker": {
+        },
+        "timezone": "browser",
+        "title": "Loki Logs: Components",
+        "uid": "ae4zuyp1kui9sc",
+        "version": 2,
+        "weekStart": ""
+    }

template/stacks/monitoring/kube-prometheus/dashboards/dashboard_loki_container.yaml

@@ -0,0 +1,537 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: grafana-dashboard-3
+  labels:
+    grafana_dashboard: "1"
+data:
+  k8s-dashboard-03.json: |
+    {
+        "annotations": {
+            "list": [
+                {
+                    "builtIn": 1,
+                    "datasource": {
+                        "type": "grafana",
+                        "uid": "-- Grafana --"
+                    },
+                    "enable": true,
+                    "hide": true,
+                    "iconColor": "rgba(0, 211, 255, 1)",
+                    "name": "Annotations & Alerts",
+                    "type": "dashboard"
+                }
+            ]
+        },
+        "editable": true,
+        "fiscalYearStartMonth": 0,
+        "graphTooltip": 0,
+        "id": 31,
+        "links": [
+        ],
+        "panels": [
+            {
+                "datasource": {
+                    "type": "loki",
+                    "uid": "P8E80F9AEF21F6940"
+                },
+                "fieldConfig": {
+                    "defaults": {
+                    },
+                    "overrides": [
+                    ]
+                },
+                "gridPos": {
+                    "h": 8,
+                    "w": 24,
+                    "x": 0,
+                    "y": 0
+                },
+                "id": 11,
+                "options": {
+                    "dedupStrategy": "none",
+                    "enableLogDetails": true,
+                    "prettifyLogMessage": false,
+                    "showCommonLabels": false,
+                    "showLabels": false,
+                    "showTime": false,
+                    "sortOrder": "Descending",
+                    "wrapLogMessage": false
+                },
+                "pluginVersion": "11.3.1",
+                "targets": [
+                    {
+                        "datasource": {
+                            "type": "loki",
+                            "uid": "P8E80F9AEF21F6940"
+                        },
+                        "editorMode": "builder",
+                        "expr": "{container=\"repo-server\"} |= ``",
+                        "queryType": "range",
+                        "refId": "A"
+                    }
+                ],
+                "title": "Logs: Container repo-server",
+                "type": "logs"
+            },
+            {
+                "datasource": {
+                    "type": "loki",
+                    "uid": "P8E80F9AEF21F6940"
+                },
+                "fieldConfig": {
+                    "defaults": {
+                    },
+                    "overrides": [
+                    ]
+                },
+                "gridPos": {
+                    "h": 8,
+                    "w": 24,
+                    "x": 0,
+                    "y": 8
+                },
+                "id": 10,
+                "options": {
+                    "dedupStrategy": "none",
+                    "enableLogDetails": true,
+                    "prettifyLogMessage": false,
+                    "showCommonLabels": false,
+                    "showLabels": false,
+                    "showTime": false,
+                    "sortOrder": "Descending",
+                    "wrapLogMessage": false
+                },
+                "pluginVersion": "11.3.1",
+                "targets": [
+                    {
+                        "datasource": {
+                            "type": "loki",
+                            "uid": "P8E80F9AEF21F6940"
+                        },
+                        "editorMode": "builder",
+                        "expr": "{container=\"alloy\"} |= ``",
+                        "queryType": "range",
+                        "refId": "A"
+                    }
+                ],
+                "title": "Logs: Container alloy",
+                "type": "logs"
+            },
+            {
+                "datasource": {
+                    "type": "loki",
+                    "uid": "P8E80F9AEF21F6940"
+                },
+                "fieldConfig": {
+                    "defaults": {
+                    },
+                    "overrides": [
+                    ]
+                },
+                "gridPos": {
+                    "h": 8,
+                    "w": 24,
+                    "x": 0,
+                    "y": 16
+                },
+                "id": 9,
+                "options": {
+                    "dedupStrategy": "none",
+                    "enableLogDetails": true,
+                    "prettifyLogMessage": false,
+                    "showCommonLabels": false,
+                    "showLabels": false,
+                    "showTime": false,
+                    "sortOrder": "Descending",
+                    "wrapLogMessage": false
+                },
+                "pluginVersion": "11.3.1",
+                "targets": [
+                    {
+                        "datasource": {
+                            "type": "loki",
+                            "uid": "P8E80F9AEF21F6940"
+                        },
+                        "editorMode": "builder",
+                        "expr": "{container=\"prometheus\"} |= ``",
+                        "queryType": "range",
+                        "refId": "A"
+                    }
+                ],
+                "title": "Logs: Container prometheus",
+                "type": "logs"
+            },
+            {
+                "datasource": {
+                    "type": "loki",
+                    "uid": "P8E80F9AEF21F6940"
+                },
+                "fieldConfig": {
+                    "defaults": {
+                    },
+                    "overrides": [
+                    ]
+                },
+                "gridPos": {
+                    "h": 8,
+                    "w": 24,
+                    "x": 0,
+                    "y": 24
+                },
+                "id": 8,
+                "options": {
+                    "dedupStrategy": "none",
+                    "enableLogDetails": true,
+                    "prettifyLogMessage": false,
+                    "showCommonLabels": false,
+                    "showLabels": false,
+                    "showTime": false,
+                    "sortOrder": "Descending",
+                    "wrapLogMessage": false
+                },
+                "pluginVersion": "11.3.1",
+                "targets": [
+                    {
+                        "datasource": {
+                            "type": "loki",
+                            "uid": "P8E80F9AEF21F6940"
+                        },
+                        "editorMode": "builder",
+                        "expr": "{container=\"postgres\"} |= ``",
+                        "queryType": "range",
+                        "refId": "A"
+                    }
+                ],
+                "title": "Logs: Container postgres",
+                "type": "logs"
+            },
+            {
+                "datasource": {
+                    "type": "loki",
+                    "uid": "P8E80F9AEF21F6940"
+                },
+                "fieldConfig": {
+                    "defaults": {
+                    },
+                    "overrides": [
+                    ]
+                },
+                "gridPos": {
+                    "h": 8,
+                    "w": 24,
+                    "x": 0,
+                    "y": 32
+                },
+                "id": 7,
+                "options": {
+                    "dedupStrategy": "none",
+                    "enableLogDetails": true,
+                    "prettifyLogMessage": false,
+                    "showCommonLabels": false,
+                    "showLabels": false,
+                    "showTime": false,
+                    "sortOrder": "Descending",
+                    "wrapLogMessage": false
+                },
+                "pluginVersion": "11.3.1",
+                "targets": [
+                    {
+                        "datasource": {
+                            "type": "loki",
+                            "uid": "P8E80F9AEF21F6940"
+                        },
+                        "editorMode": "builder",
+                        "expr": "{container=\"kube-prometheus-stack\"} |= ``",
+                        "queryType": "range",
+                        "refId": "A"
+                    }
+                ],
+                "title": "Logs: Container kube-prometheus-stack",
+                "type": "logs"
+            },
+            {
+                "datasource": {
+                    "type": "loki",
+                    "uid": "P8E80F9AEF21F6940"
+                },
+                "fieldConfig": {
+                    "defaults": {
+                    },
+                    "overrides": [
+                    ]
+                },
+                "gridPos": {
+                    "h": 8,
+                    "w": 24,
+                    "x": 0,
+                    "y": 40
+                },
+                "id": 6,
+                "options": {
+                    "dedupStrategy": "none",
+                    "enableLogDetails": true,
+                    "prettifyLogMessage": false,
+                    "showCommonLabels": false,
+                    "showLabels": false,
+                    "showTime": false,
+                    "sortOrder": "Descending",
+                    "wrapLogMessage": false
+                },
+                "pluginVersion": "11.3.1",
+                "targets": [
+                    {
+                        "datasource": {
+                            "type": "loki",
+                            "uid": "P8E80F9AEF21F6940"
+                        },
+                        "editorMode": "builder",
+                        "expr": "{container=\"keycloak\"} |= ``",
+                        "queryType": "range",
+                        "refId": "A"
+                    }
+                ],
+                "title": "Logs: Container keycloak",
+                "type": "logs"
+            },
+            {
+                "datasource": {
+                    "type": "loki",
+                    "uid": "P8E80F9AEF21F6940"
+                },
+                "fieldConfig": {
+                    "defaults": {
+                    },
+                    "overrides": [
+                    ]
+                },
+                "gridPos": {
+                    "h": 8,
+                    "w": 24,
+                    "x": 0,
+                    "y": 48
+                },
+                "id": 5,
+                "options": {
+                    "dedupStrategy": "none",
+                    "enableLogDetails": true,
+                    "prettifyLogMessage": false,
+                    "showCommonLabels": false,
+                    "showLabels": false,
+                    "showTime": false,
+                    "sortOrder": "Descending",
+                    "wrapLogMessage": false
+                },
+                "pluginVersion": "11.3.1",
+                "targets": [
+                    {
+                        "datasource": {
+                            "type": "loki",
+                            "uid": "P8E80F9AEF21F6940"
+                        },
+                        "editorMode": "builder",
+                        "expr": "{container=\"grafana\"} |= ``",
+                        "queryType": "range",
+                        "refId": "A"
+                    }
+                ],
+                "title": "Logs: Container grafana",
+                "type": "logs"
+            },
+            {
+                "datasource": {
+                    "type": "loki",
+                    "uid": "P8E80F9AEF21F6940"
+                },
+                "fieldConfig": {
+                    "defaults": {
+                    },
+                    "overrides": [
+                    ]
+                },
+                "gridPos": {
+                    "h": 8,
+                    "w": 24,
+                    "x": 0,
+                    "y": 56
+                },
+                "id": 4,
+                "options": {
+                    "dedupStrategy": "none",
+                    "enableLogDetails": true,
+                    "prettifyLogMessage": false,
+                    "showCommonLabels": false,
+                    "showLabels": false,
+                    "showTime": false,
+                    "sortOrder": "Descending",
+                    "wrapLogMessage": false
+                },
+                "pluginVersion": "11.3.1",
+                "targets": [
+                    {
+                        "datasource": {
+                            "type": "loki",
+                            "uid": "P8E80F9AEF21F6940"
+                        },
+                        "editorMode": "builder",
+                        "expr": "{container=\"forgejo\"} |= ``",
+                        "queryType": "range",
+                        "refId": "A"
+                    }
+                ],
+                "title": "Logs: Container forgejo",
+                "type": "logs"
+            },
+            {
+                "datasource": {
+                    "type": "loki",
+                    "uid": "P8E80F9AEF21F6940"
+                },
+                "fieldConfig": {
+                    "defaults": {
+                    },
+                    "overrides": [
+                    ]
+                },
+                "gridPos": {
+                    "h": 8,
+                    "w": 24,
+                    "x": 0,
+                    "y": 64
+                },
+                "id": 3,
+                "options": {
+                    "dedupStrategy": "none",
+                    "enableLogDetails": true,
+                    "prettifyLogMessage": false,
+                    "showCommonLabels": false,
+                    "showLabels": false,
+                    "showTime": false,
+                    "sortOrder": "Descending",
+                    "wrapLogMessage": false
+                },
+                "pluginVersion": "11.3.1",
+                "targets": [
+                    {
+                        "datasource": {
+                            "type": "loki",
+                            "uid": "P8E80F9AEF21F6940"
+                        },
+                        "editorMode": "builder",
+                        "expr": "{container=\"crossplane\"} |= ``",
+                        "queryType": "range",
+                        "refId": "A"
+                    }
+                ],
+                "title": "Logs: Container crossplane",
+                "type": "logs"
+            },
+            {
+                "datasource": {
+                    "type": "loki",
+                    "uid": "P8E80F9AEF21F6940"
+                },
+                "fieldConfig": {
+                    "defaults": {
+                    },
+                    "overrides": [
+                    ]
+                },
+                "gridPos": {
+                    "h": 8,
+                    "w": 24,
+                    "x": 0,
+                    "y": 72
+                },
+                "id": 2,
+                "options": {
+                    "dedupStrategy": "none",
+                    "enableLogDetails": true,
+                    "prettifyLogMessage": false,
+                    "showCommonLabels": false,
+                    "showLabels": false,
+                    "showTime": false,
+                    "sortOrder": "Descending",
+                    "wrapLogMessage": false
+                },
+                "pluginVersion": "11.3.1",
+                "targets": [
+                    {
+                        "datasource": {
+                            "type": "loki",
+                            "uid": "P8E80F9AEF21F6940"
+                        },
+                        "editorMode": "builder",
+                        "expr": "{container=\"backstage\"} |= ``",
+                        "queryType": "range",
+                        "refId": "A"
+                    }
+                ],
+                "title": "Logs: Container backstage",
+                "type": "logs"
+            },
+            {
+                "datasource": {
+                    "type": "loki",
+                    "uid": "P8E80F9AEF21F6940"
+                },
+                "fieldConfig": {
+                    "defaults": {
+                    },
+                    "overrides": [
+                    ]
+                },
+                "gridPos": {
+                    "h": 8,
+                    "w": 24,
+                    "x": 0,
+                    "y": 80
+                },
+                "id": 1,
+                "options": {
+                    "dedupStrategy": "none",
+                    "enableLogDetails": true,
+                    "prettifyLogMessage": false,
+                    "showCommonLabels": false,
+                    "showLabels": false,
+                    "showTime": false,
+                    "sortOrder": "Descending",
+                    "wrapLogMessage": false
+                },
+                "pluginVersion": "11.3.1",
+                "targets": [
+                    {
+                        "datasource": {
+                            "type": "loki",
+                            "uid": "P8E80F9AEF21F6940"
+                        },
+                        "editorMode": "builder",
+                        "expr": "{container=\"argo-server\"} |= ``",
+                        "queryType": "range",
+                        "refId": "A"
+                    }
+                ],
+                "title": "Logs: Container argo-server",
+                "type": "logs"
+            }
+        ],
+        "preload": false,
+        "schemaVersion": 40,
+        "tags": [
+        ],
+        "templating": {
+            "list": [
+            ]
+        },
+        "time": {
+            "from": "now-6h",
+            "to": "now"
+        },
+        "timepicker": {
+        },
+        "timezone": "browser",
+        "title": "Loki Logs: Container",
+        "uid": "ee50bcaehmv40e",
+        "version": 2,
+        "weekStart": ""
+    }

template/stacks/monitoring/kube-prometheus/values.yaml

@@ -0,0 +1,71 @@
+grafana:
+  namespaceOverride: "monitoring"
+
+  admin:
+    existingSecret: "kube-prometheus-stack-grafana-admin-password"
+    userKey: admin-user
+    passwordKey: admin-password
+
+  defaultDashboardsTimezone: Europe/Berlin
+
+  additionalDataSources:
+    - name: Loki
+      type: loki
+      url: http://loki-loki-distributed-gateway.monitoring:80
+
+  # syncPolicy:
+  #   syncOptions:
+  #   - ServerSideApply=true
+
+  sidecar:
+    dashboards:
+      enabled: true
+      label: grafana_dashboard
+      folder: /tmp/dashboards
+      updateIntervalSeconds: 10
+      folderAnnotation: grafana_folder
+      provider:
+        allowUiUpdates: true
+        foldersFromFilesStructure: true
+
+  grafana.ini:
+    server:
+      domain: {{{ .Env.DOMAIN }}}
+      root_url: "%(protocol)s://%(domain)s/grafana"
+      serve_from_sub_path: true
+    auth:
+      disable_login: true
+      disable_login_form: true
+    auth.generic_oauth:
+      enabled: true
+      name: Keycloak-OAuth
+      allow_sign_up: true
+      use_refresh_token: true
+      client_id: grafana
+      client_secret: $__file{/etc/secrets/auth_generic_oauth/client_secret}
+      scopes: openid email profile offline_access roles
+      email_attribute_path: email
+      login_attribute_path: username
+      name_attribute_path: full_name
+      auth_url: https://{{{ .Env.DOMAIN }}}/keycloak/realms/cnoe/protocol/openid-connect/auth
+      token_url: https://{{{ .Env.DOMAIN }}}/keycloak/realms/cnoe/protocol/openid-connect/token
+      api_url: https://{{{ .Env.DOMAIN }}}/keycloak/realms/cnoe/protocol/openid-connect/userinfo
+      redirect_uri: http://{{{ .Env.DOMAIN }}}/grafana/login/generic_oauth
+      role_attribute_path: "contains(groups[*], 'admin') && 'Admin' || contains(groups[*], 'editor') && 'Editor' || 'Viewer'"
+      
+  extraSecretMounts:
+    - name: auth-generic-oauth-secret-mount
+      secretName: auth-generic-oauth-secret
+      defaultMode: 0440
+      mountPath: /etc/secrets/auth_generic_oauth
+      readOnly: true
+
+  serviceMonitor:
+    # If true, a ServiceMonitor CRD is created for a prometheus operator https://github.com/coreos/prometheus-operator
+    enabled: true
+
+#monitoring nginx
+prometheus:
+  prometheusSpec:
+    podMonitorSelectorNilUsesHelmValues: false
+    serviceMonitorSelectorNilUsesHelmValues: false     

template/stacks/monitoring/loki.yaml

@@ -0,0 +1,34 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: loki
+  namespace: argocd
+  labels:
+    env: dev
+  finalizers:
+    - resources-finalizer.argocd.argoproj.io
+spec:
+  project: default
+  syncPolicy:
+    automated:
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true
+  destination:
+    name: in-cluster
+    namespace: monitoring
+  sources:
+    - repoURL: https://github.com/grafana/helm-charts
+      path:     charts/loki-distributed         
+      targetRevision: HEAD      
+      helm:
+        valueFiles:
+          - $values/stacks/monitoring/loki/values.yaml
+    - repoURL: https://{{{ .Env.DOMAIN_GITEA }}}/giteaAdmin/edfbuilder
+      targetRevision: HEAD
+      ref: values
+
+    ## consider using the following version, if it works again
+    #- repoURL: https://github.com/grafana/loki
+    #  path: production/helm/loki        
+

template/stacks/monitoring/loki/values.yaml

@@ -0,0 +1,7 @@
+loki:
+  commonConfig:
+    replication_factor: 1
+  auth_enabled: false
+
+#  storageConfig:
+#    filesystem: null

template/stacks/observability-client/metrics-server.yaml

@@ -1,29 +0,0 @@
-apiVersion: argoproj.io/v1alpha1
-kind: Application
-metadata:
-  name: metrics-server
-  namespace: argocd
-  labels:
-    env: dev
-spec:
-  project: default
-  syncPolicy:
-    automated:
-      selfHeal: true
-    syncOptions:
-      - CreateNamespace=true
-    retry:
-      limit: -1
-  destination:
-    name: in-cluster
-    namespace: observability
-  sources:
-    - chart: metrics-server
-      repoURL: https://kubernetes-sigs.github.io/metrics-server/
-      targetRevision: 3.12.2
-      helm:
-        valueFiles:
-          - $values/{{{ .Env.CLIENT_REPO_ID }}}/{{{ .Env.DOMAIN }}}/stacks/observability-client/metrics-server/values.yaml
-    - repoURL: https://{{{ .Env.CLIENT_REPO_DOMAIN }}}/{{{ .Env.CLIENT_REPO_ORG_NAME }}}
-      targetRevision: HEAD
-      ref: values

template/stacks/observability-client/metrics-server/values.yaml

@@ -1,4 +0,0 @@
-metrics:
-  enabled: true
-serviceMonitor:
-  enabled: true

template/stacks/observability-client/vector.yaml

@@ -1,29 +0,0 @@
-apiVersion: argoproj.io/v1alpha1
-kind: Application
-metadata:
-  name: vector
-  namespace: argocd
-  labels:
-    env: dev
-spec:
-  project: default
-  syncPolicy:
-    automated:
-      selfHeal: true
-    syncOptions:
-      - CreateNamespace=true
-    retry:
-      limit: -1
-  destination:
-    name: in-cluster
-    namespace: observability
-  sources:
-    - chart: vector
-      repoURL: https://helm.vector.dev
-      targetRevision: 0.43.0
-      helm:
-        valueFiles:
-          - $values/{{{ .Env.CLIENT_REPO_ID }}}/{{{ .Env.DOMAIN }}}/stacks/observability-client/vector/values.yaml
-    - repoURL: https://{{{ .Env.CLIENT_REPO_DOMAIN }}}/{{{ .Env.CLIENT_REPO_ORG_NAME }}}
-      targetRevision: HEAD
-      ref: values

template/stacks/observability-client/vector/values.yaml

@@ -1,68 +0,0 @@
-# -- Enable deployment of vector
-role: Agent
-dataDir: /vector-data-dir
-resources: {}
-args:
-  - -w
-  - --config-dir
-  - /etc/vector/
-env:
-  - name: VECTOR_USER
-    valueFrom:
-      secretKeyRef:
-        name: simple-user-secret
-        key: username
-  - name: VECTOR_PASSWORD
-    valueFrom:
-      secretKeyRef:
-        name: simple-user-secret
-        key: password
-containerPorts:
-  - name: prom-exporter
-    containerPort: 9090
-    protocol: TCP
-service:
-  enabled: false
-customConfig:
-  data_dir: /vector-data-dir
-  api:
-    enabled: false
-    address: 0.0.0.0:8686
-    playground: true
-  sources:
-    k8s:
-      type: kubernetes_logs
-    internal_metrics:
-      type: internal_metrics
-  transforms:
-    parser:
-      type: remap
-      inputs: [k8s]
-      source: |
-        ._msg = parse_json(.message) ?? .message
-        del(.message)
-        # Add the cluster environment to the log event
-        .cluster_environment = "{{{ .Env.CLUSTER_ENVIRONMENT }}}"
-  sinks:
-    vlogs:
-      type: elasticsearch
-      inputs: [parser]
-      endpoints: 
-        - https://{{{ .Env.DOMAIN_O12Y }}}/insert/elasticsearch/
-      auth:
-        strategy: basic
-        user: ${VECTOR_USER}
-        password: ${VECTOR_PASSWORD}
-      mode: bulk
-      api_version: v8
-      compression: gzip
-      healthcheck:
-        enabled: false
-      request:
-        headers:
-          AccountID: "0"
-          ProjectID: "0"
-      query:
-        _msg_field: _msg
-        _time_field: _time
-        _stream_fields: cluster_environment,kubernetes.container_name,kubernetes.namespace

template/stacks/observability-client/vm-client-stack.yaml

@@ -1,30 +0,0 @@
-apiVersion: argoproj.io/v1alpha1
-kind: Application
-metadata:
-  name: vm-client
-  namespace: argocd
-  labels:
-    env: dev
-spec:
-  project: default
-  syncPolicy:
-    automated:
-      selfHeal: true
-    syncOptions:
-      - CreateNamespace=true
-  destination:
-    name: in-cluster
-    namespace: observability
-  sources:
-    - chart: victoria-metrics-k8s-stack
-      repoURL: https://victoriametrics.github.io/helm-charts/
-      targetRevision: 0.48.1
-      helm:
-        valueFiles:
-          - $values/{{{ .Env.CLIENT_REPO_ID }}}/{{{ .Env.DOMAIN }}}/stacks/observability-client/vm-client-stack/values.yaml
-    - repoURL: https://{{{ .Env.CLIENT_REPO_DOMAIN }}}/{{{ .Env.CLIENT_REPO_ORG_NAME }}}
-      targetRevision: HEAD
-      ref: values
-    - repoURL: https://{{{ .Env.CLIENT_REPO_DOMAIN }}}/{{{ .Env.CLIENT_REPO_ORG_NAME }}}
-      targetRevision: HEAD
-      path: "{{{ .Env.CLIENT_REPO_ID }}}/{{{ .Env.DOMAIN }}}/stacks/observability-client/vm-client-stack/manifests"

template/stacks/observability-client/vm-client-stack/manifests/simple-user-secret.yaml

@@ -1,9 +0,0 @@
-apiVersion: v1
-kind: Secret
-metadata:
-  name: simple-user-secret
-  namespace: observability
-type: Opaque
-stringData:
-  username: simple-user
-  password: simple-password

template/stacks/observability/grafana-operator.yaml

@@ -1,25 +0,0 @@
-apiVersion: argoproj.io/v1alpha1
-kind: Application
-metadata:
-  name: grafana-operator
-  namespace: argocd
-  labels:
-    env: dev
-spec:
-  project: default
-  syncPolicy:
-    automated:
-      selfHeal: true
-    syncOptions:
-      - CreateNamespace=true
-      - ServerSideApply=true
-  destination:
-    name: in-cluster
-    namespace: observability
-  sources:
-    - chart: grafana-operator
-      repoURL: ghcr.io/grafana/helm-charts
-      targetRevision: v5.18.0
-    - repoURL: https://{{{ .Env.CLIENT_REPO_DOMAIN }}}/{{{ .Env.CLIENT_REPO_ORG_NAME }}}
-      targetRevision: HEAD
-      path: "{{{ .Env.CLIENT_REPO_ID }}}/{{{ .Env.DOMAIN }}}/stacks/observability/grafana-operator/manifests"

template/stacks/observability/grafana-operator/manifests/argocd.yaml

@@ -1,9 +0,0 @@
-apiVersion: grafana.integreatly.org/v1beta1
-kind: GrafanaDashboard
-metadata:
-  name: argocd
-spec:
-  instanceSelector:
-    matchLabels:
-      dashboards: "grafana"
-  url: "https://raw.githubusercontent.com/argoproj/argo-cd/refs/heads/master/examples/dashboard.json"

template/stacks/observability/grafana-operator/manifests/grafana.yaml

@@ -1,36 +0,0 @@
-apiVersion: grafana.integreatly.org/v1beta1
-kind: Grafana
-metadata:
-  name: grafana
-  labels:
-    dashboards: "grafana"
-spec:
-  persistentVolumeClaim:
-    spec:
-      accessModes:
-        - ReadWriteOnce
-      resources:
-        requests:
-          storage: 10Gi
-  ingress:
-    metadata:
-      annotations:
-        cert-manager.io/cluster-issuer: main
-        nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
-    spec:
-      ingressClassName: nginx
-      rules:
-      - host: grafana.{{{ .Env.DOMAIN }}}
-        http:
-          paths:
-          - backend:
-              service:
-                name: grafana-service
-                port:
-                  number: 3000
-            path: /
-            pathType: Prefix
-      tls:
-      - hosts:
-        - grafana.{{{ .Env.DOMAIN }}}
-        secretName: grafana-net-tls

template/stacks/observability/grafana-operator/manifests/ingress-nginx.yaml

@@ -1,9 +0,0 @@
-apiVersion: grafana.integreatly.org/v1beta1
-kind: GrafanaDashboard
-metadata:
-  name: ingress-nginx 
-spec:
-  instanceSelector:
-    matchLabels:
-      dashboards: "grafana"
-  url: "https://raw.githubusercontent.com/adinhodovic/ingress-nginx-mixin/refs/heads/main/dashboards_out/ingress-nginx-overview.json"

template/stacks/observability/grafana-operator/manifests/victoria-logs.yaml

@@ -1,9 +0,0 @@
-apiVersion: grafana.integreatly.org/v1beta1
-kind: GrafanaDashboard
-metadata:
-  name: victoria-logs
-spec:
-  instanceSelector:
-    matchLabels:
-      dashboards: "grafana"
-  url: "https://raw.githubusercontent.com/VictoriaMetrics/VictoriaMetrics/refs/heads/master/dashboards/vm/victorialogs.json"

template/stacks/observability/victoria-k8s-stack.yaml

@@ -1,31 +0,0 @@
-apiVersion: argoproj.io/v1alpha1
-kind: Application
-metadata:
-  name: o12y
-  namespace: argocd
-  labels:
-    env: dev
-spec:
-  project: default
-  syncPolicy:
-    automated:
-      selfHeal: true
-    syncOptions:
-      - CreateNamespace=true
-      - ServerSideApply=true
-  destination:
-    name: in-cluster
-    namespace: observability
-  sources:
-    - chart: victoria-metrics-k8s-stack
-      repoURL: https://victoriametrics.github.io/helm-charts/
-      targetRevision: 0.48.1
-      helm:
-        valueFiles:
-          - $values/{{{ .Env.CLIENT_REPO_ID }}}/{{{ .Env.DOMAIN }}}/stacks/observability/victoria-k8s-stack/values.yaml
-    - repoURL: https://{{{ .Env.CLIENT_REPO_DOMAIN }}}/{{{ .Env.CLIENT_REPO_ORG_NAME }}}
-      targetRevision: HEAD
-      ref: values
-    - repoURL: https://{{{ .Env.CLIENT_REPO_DOMAIN }}}/{{{ .Env.CLIENT_REPO_ORG_NAME }}}
-      targetRevision: HEAD
-      path: "{{{ .Env.CLIENT_REPO_ID }}}/{{{ .Env.DOMAIN }}}/stacks/observability/victoria-k8s-stack/manifests"

template/stacks/observability/victoria-k8s-stack/manifests/vlogs.yaml

@@ -1,24 +0,0 @@
-apiVersion: operator.victoriametrics.com/v1beta1
-kind: VLogs
-metadata:
-  name: victorialogs
-  namespace: observability
-spec:
-  retentionPeriod: "12"
-  removePvcAfterDelete: true
-  storageMetadata:
-    annotations:
-      everest.io/crypt-key-id: {{{ .Env.PVC_KMS_KEY_ID }}}
-  storage:
-    accessModes:
-      - ReadWriteOnce
-    resources:
-      requests:
-        storage: 50Gi
-  resources:
-    requests:
-      memory: 500Mi
-      cpu: 500m
-    limits:
-      memory: 10Gi
-      cpu: 2

template/stacks/observability/victoria-k8s-stack/manifests/vmauth.yaml

@@ -1,15 +0,0 @@
-apiVersion: operator.victoriametrics.com/v1beta1
-kind: VMUser
-metadata:
-  name: simple-user
-  namespace: observability
-spec:
-  username: simple-user
-  password: simple-password
-  targetRefs:
-    - static:
-        url: http://vmsingle-o12y:8429
-      paths: ["/api/v1/write"]
-    - static:
-        url: http://vlogs-victorialogs:9428
-      paths: ["/insert/elasticsearch/.*"]

template/stacks/otc/cert-manager/manifests/clusterissuer.yaml

@@ -1,14 +0,0 @@
-apiVersion: cert-manager.io/v1
-kind: ClusterIssuer
-metadata:
-  name: main
-spec:
-  acme:
-    email: admin@think-ahead.tech
-    server: https://acme-v02.api.letsencrypt.org/directory
-    privateKeySecretRef:
-      name: cluster-issuer-account-key
-    solvers:
-    - http01:
-        ingress:
-          ingressClassName: nginx

template/stacks/otc/cert-manager/values.yaml

@@ -1,4 +0,0 @@
-crds:
-  enabled: true
-
-replicaCount: 1

template/stacks/otc/cert-manger.yaml

@@ -1,32 +0,0 @@
-apiVersion: argoproj.io/v1alpha1
-kind: Application
-metadata:
-  name: cert-manager
-  namespace: argocd
-  labels:
-    env: dev
-spec:
-  project: default
-  syncPolicy:
-    automated:
-      selfHeal: true
-    syncOptions:
-      - CreateNamespace=true
-    retry:
-      limit: -1
-  destination:
-    name: in-cluster
-    namespace: cert-manager
-  sources:
-    - chart: cert-manager
-      repoURL: https://charts.jetstack.io
-      targetRevision: v1.17.2
-      helm:
-        valueFiles:
-          - $values/{{{ .Env.CLIENT_REPO_ID }}}/{{{ .Env.DOMAIN }}}/stacks/otc/cert-manager/values.yaml
-    - repoURL: https://{{{ .Env.CLIENT_REPO_DOMAIN }}}/{{{ .Env.CLIENT_REPO_ORG_NAME }}}
-      targetRevision: HEAD
-      ref: values
-    - repoURL: https://{{{ .Env.CLIENT_REPO_DOMAIN }}}/{{{ .Env.CLIENT_REPO_ORG_NAME }}}
-      targetRevision: HEAD
-      path: "{{{ .Env.CLIENT_REPO_ID }}}/{{{ .Env.DOMAIN }}}/stacks/otc/cert-manager/manifests"

template/stacks/otc/ingress-nginx/values.yaml

@@ -1,31 +0,0 @@
-controller:
-  updateStrategy:
-    type: RollingUpdate
-    rollingUpdate:
-      maxUnavailable: 1
-
-  service:
-    annotations:
-      kubernetes.io/elb.class: union
-      kubernetes.io/elb.port: '80'
-      kubernetes.io/elb.id: {{{ .Env.LOADBALANCER_ID }}}
-      kubernetes.io/elb.ip: {{{ .Env.LOADBALANCER_IP }}}
-
-  ingressClassResource:
-    name: nginx
-
-  # added for idpbuilder
-  allowSnippetAnnotations: true
-
-  # added for idpbuilder
-  config:
-    proxy-buffer-size: 32k
-    use-forwarded-headers: "true"
-
-  # monitoring nginx
-  metrics:
-    enabled: true
-    serviceMonitor:
-      additionalLabels: 
-        release: "ingress-nginx"  
-      enabled: true

template/stacks/otc/storageclass/storageclass.yaml

@@ -1,18 +0,0 @@
-apiVersion: storage.k8s.io/v1
-kind: StorageClass
-metadata:
-  annotations:
-    storageclass.beta.kubernetes.io/is-default-class: "true"
-  labels:
-    kubernetes.io/cluster-service: "true"
-  name: default
-parameters:
-  kubernetes.io/description: ""
-  kubernetes.io/hw:passthrough: "true"
-  kubernetes.io/storagetype: BS
-  kubernetes.io/volumetype: SATA
-  kubernetes.io/zone: eu-de-02
-provisioner: flexvolume-huawei.com/fuxivol
-reclaimPolicy: Delete
-volumeBindingMode: Immediate
-allowVolumeExpansion: true

template/stacks/ref-implementation/README.md

@@ -0,0 +1,146 @@
+# Reference implementation 
+
+This example creates a local version of the CNOE reference implementation.
+
+## Prerequisites
+
+Ensure you have the following tools installed on your computer.
+
+**Required**
+
+- [idpbuilder](https://github.com/cnoe-io/idpbuilder/releases/latest): version `0.3.0` or later
+- [kubectl](https://kubernetes.io/docs/tasks/tools/#kubectl): version `1.27` or later
+- Your computer should have at least 6 GB RAM allocated to Docker. If you are on Docker Desktop, see [this guide](https://docs.docker.com/desktop/settings/mac/).
+
+**Optional**
+
+- AWS credentials: Access Key and secret Key. If you want to create AWS resources in one of examples below.
+
+## Installation
+
+**_NOTE:_**
+- If you'd like to run this in your web browser through Codespaces, please follow [the instructions here](./codespaces.md) to install instead. 
+
+- _This example assumes that you run the reference implementation with the default port configguration of 8443 for the idpBuilder.
+If you happen to configure a different host or port for the idpBuilder, the manifests in the reference example need to be updated
+and be configured with the new host and port. you can use the [replace.sh](replace.sh) to change the port as desired prior to applying the manifest as instructed in the command above._
+
+```bash
+idpbuilder create --use-path-routing \
+  --package https://github.com/cnoe-io/stacks//ref-implementation
+```
+
+This will take ~6 minutes for everything to come up. To track the progress, you can go to the [ArgoCD UI](https://{{{ .Env.DOMAIN }}}:8443/argocd/applications).
+
+### What was installed?
+
+1. **Argo Workflows** to enable workflow orchestrations.
+1. **Backstage** as the UI for software catalog and templating. Source is available [here](https://github.com/cnoe-io/backstage-app).
+1. **External Secrets** to generate secrets and coordinate secrets between applications.
+1. **Keycloak** as the identity provider for applications.
+1. **Spark Operator** to demonstrate an example Spark workload through Backstage.
+
+If you don't want to install a package above, you can remove the ArgoCD Application file corresponding to the package you want to remove.
+For example, if you want to remove Spark Operator, you can delete [this file](./spark-operator.yaml).
+
+The only package that cannot be removed this way is Keycloak because other packages rely on it. 
+
+
+#### Accessing UIs
+- Argo CD: https://{{{ .Env.DOMAIN }}}:8443/argocd
+- Argo Workflows: https://{{{ .Env.DOMAIN }}}:8443/argo-workflows
+- Backstage: https://{{{ .Env.DOMAIN }}}:8443/
+- Gitea: https://{{{ .Env.DOMAIN }}}:8443/gitea
+- Keycloak: https://{{{ .Env.DOMAIN }}}:8443/keycloak/admin/master/console/
+
+# Using it
+
+For this example, we will walk through a few demonstrations. Once applications are ready, go to the [backstage URL](https://{{{ .Env.DOMAIN }}}:8443).
+
+Click on the Sign-In button, you will be asked to log into the Keycloak instance. There are two users set up in this 
+configuration, and their password can be retrieved with the following command:
+
+```bash
+idpbuilder get secrets
+```
+
+Use the username **`user1`** and the password value given by `USER_PASSWORD` field to login to the backstage instance.
+`user1` is an admin user who has access to everything in the cluster, while `user2` is a regular user with limited access.
+Both users use the same password retrieved above.
+
+If you want to create a new user or change existing users:
+
+1. Go to the [Keycloak UI](https://{{{ .Env.DOMAIN }}}:8443/keycloak/admin/master/console/). 
+Login with the username `cnoe-admin`. Password is the `KEYCLOAK_ADMIN_PASSWORD` field from the command above. 
+2. Select `cnoe` from the realms drop down menu.
+3. Select users tab.
+
+
+## Basic Deployment
+
+Let's start by deploying a simple application to the cluster through Backstage.
+
+Click on the `Create...` button on the left, then select the `Create a Basic Deployment` template.
+
+![img.png](images/backstage-templates.png)
+
+
+In the next screen, type `demo` for the name field, then click Review, then Create. 
+Once steps run, click the Open In Catalog button to go to the entity page. 
+
+![img.png](images/basic-template-flow.png)
+
+In the demo entity page, you will notice a ArgoCD overview card associated with this entity. 
+You can click on the ArgoCD Application name to see more details.
+
+![img.png](images/demo-entity.png)
+
+### What just happened?
+
+1. Backstage created [a git repository](https://{{{ .Env.DOMAIN }}}:8443/gitea/giteaAdmin/demo), then pushed templated contents to it.
+2. Backstage created [an ArgoCD Application](https://{{{ .Env.DOMAIN }}}:8443/argocd/applications/argocd/demo?) and pointed it to the git repository.
+3. Backstage registered the application as [a component](https://{{{ .Env.DOMAIN }}}:8443/gitea/giteaAdmin/demo/src/branch/main/catalog-info.yaml) in Backstage.
+4. ArgoCD deployed the manifests stored in the repo to the cluster.
+5. Backstage retrieved application health from ArgoCD API, then displayed it.
+
+![image.png](images/basic-deployment.png)
+
+
+## Argo Workflows and Spark Operator
+
+In this example, we will deploy a simple Apache Spark job through Argo Workflows.
+
+Click on the `Create...` button on the left, then select the `Basic Argo Workflow with a Spark Job` template.
+
+![img.png](images/backstage-templates-spark.png)
+
+Type `demo2` for the name field, then click create. You will notice that the Backstage templating steps are very similar to the basic example above.
+Click on the Open In Catalog button to go to the entity page.
+
+![img.png](images/demo2-entity.png)
+
+Deployment processes are the same as the first example. Instead of deploying a pod, we deployed a workflow to create a Spark job.
+
+In the entity page, there is a card for Argo Workflows, and it should say running or succeeded. 
+You can click the name in the card to go to the Argo Workflows UI to view more details about this workflow run. 
+When prompted to log in, click the login button under single sign on. Argo Workflows is configured to use SSO with Keycloak allowing you to login with the same credentials as Backstage login.
+
+Note that Argo Workflows are not usually deployed this way. This is just an example to show you how you can integrate workflows, backstage, and spark.
+
+Back in the entity page, you can view more details about Spark jobs by navigating to the Spark tab.
+
+## Application with cloud resources.
+
+To deploy cloud resources, you can follow any of the instructions below:
+
+- [Cloud resource deployments via Crossplane](../crossplane-integrations/)
+- [Cloud resource deployments via Terraform](../terraform-integrations/)
+
+## Notes
+
+- In these examples, we have used the pattern of creating a new repository for every app, then having ArgoCD deploy it.
+This is done for convenience and demonstration purposes only. There are alternative actions that you can use. 
+For example, you can create a PR to an existing repository, create a repository but not deploy them yet, etc.
+
+- If Backstage's pipelining and templating mechanisms is too simple, you can use more advanced workflow engines like Tekton or Argo Workflows. 
+  You can invoke them in Backstage templates, then track progress similar to how it was described above.  

template/stacks/ref-implementation/argo-workflows.yaml

@@ -0,0 +1,29 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: argo-workflows
+  namespace: argocd
+  labels:
+    env: dev
+  finalizers:
+    - resources-finalizer.argocd.argoproj.io
+spec:
+  project: default
+  source:
+    repoURL: https://{{{ .Env.DOMAIN_GITEA }}}/giteaAdmin/edfbuilder
+    targetRevision: HEAD
+    path: "stacks/ref-implementation/argo-workflows/manifests/dev"
+  destination:
+    server: "https://kubernetes.default.svc"
+    namespace: argo
+  syncPolicy:
+    syncOptions:
+      - CreateNamespace=true
+    automated:
+      selfHeal: true
+    retry:
+      limit: -1
+      backoff:
+        duration: 15s
+        factor: 1
+        maxDuration: 15s

template/stacks/ref-implementation/argo-workflows/manifests/base/kustomization.yaml

@@ -0,0 +1,2 @@
+resources:
+  - install.yaml

template/stacks/ref-implementation/argo-workflows/manifests/dev/external-secret.yaml

@@ -0,0 +1,20 @@
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: keycloak-oidc
+  namespace: argo
+spec:
+  secretStoreRef:
+    name: keycloak
+    kind: ClusterSecretStore
+  target:
+    name: keycloak-oidc
+  data:
+    - secretKey: client-id
+      remoteRef:
+        key: keycloak-clients
+        property: ARGO_WORKFLOWS_CLIENT_ID
+    - secretKey: secret-key
+      remoteRef:
+        key: keycloak-clients
+        property: ARGO_WORKFLOWS_CLIENT_SECRET

template/stacks/ref-implementation/argo-workflows/manifests/dev/kustomization.yaml

@@ -0,0 +1,7 @@
+resources:
+  - ../base
+  - external-secret.yaml
+  - sa-admin.yaml
+patches:
+  - path: patches/cm-argo-workflows.yaml
+  - path: patches/deployment-argo-server.yaml

template/stacks/ref-implementation/argo-workflows/manifests/dev/patches/cm-argo-workflows.yaml

@@ -0,0 +1,26 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: workflow-controller-configmap
+  namespace: argo
+data:
+  config: |
+    sso:
+      insecureSkipVerify: true 
+      issuer: https://{{{ .Env.DOMAIN }}}/keycloak/realms/cnoe
+      clientId:
+        name: keycloak-oidc
+        key: client-id
+      clientSecret:
+        name: keycloak-oidc
+        key: secret-key
+      redirectUrl: https://{{{ .Env.DOMAIN }}}:443/argo-workflows/oauth2/callback
+      rbac:
+        enabled: true
+      scopes:
+        - openid
+        - profile
+        - email
+        - groups
+    nodeEvents:
+      enabled: false

template/stacks/ref-implementation/argo-workflows/manifests/dev/patches/deployment-argo-server.yaml

@@ -0,0 +1,30 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: argo-server
+  namespace: argo
+  annotations:
+    argocd.argoproj.io/sync-wave: "20"
+spec:
+  template:
+    spec:
+      containers:
+        - name: argo-server
+          readinessProbe:
+            httpGet:
+              path: /
+              port: 2746
+              scheme: HTTP
+          env:
+            - name: BASE_HREF
+              value: "/argo-workflows/"
+          args:
+            - server
+            - --configmap=workflow-controller-configmap
+            - --auth-mode=client
+            - --auth-mode=sso
+            - "--secure=false"
+            - "--loglevel"
+            - "info"
+            - "--log-format"
+            - "text"

template/stacks/ref-implementation/argo-workflows/manifests/dev/sa-admin.yaml

@@ -0,0 +1,32 @@
+# Used by users in the admin group
+# TODO Need to tighten up permissions.
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: admin
+  namespace: argo
+  annotations:
+    workflows.argoproj.io/rbac-rule: "'admin' in groups"
+    workflows.argoproj.io/rbac-rule-precedence: "10"
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: argo-admin
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-admin
+subjects:
+  - kind: ServiceAccount
+    name: admin
+    namespace: argo
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: admin.service-account-token
+  annotations:
+    kubernetes.io/service-account.name: admin
+  namespace: argo
+type: kubernetes.io/service-account-token

template/stacks/ref-implementation/backstage-templates.yaml

@@ -1,25 +1,27 @@
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
-  name: storageclass
+  name: backstage-templates
   namespace: argocd
   labels:
-    example: otc
+    env: dev
   finalizers:
     - resources-finalizer.argocd.argoproj.io
 spec:
-  destination:
-    namespace: default
-    server: "https://kubernetes.default.svc"
-  source:
-    repoURL: https://{{{ .Env.CLIENT_REPO_DOMAIN }}}/{{{ .Env.CLIENT_REPO_ORG_NAME }}}
-    targetRevision: HEAD
-    path: "{{{ .Env.CLIENT_REPO_ID }}}/{{{ .Env.DOMAIN }}}/stacks/otc/storageclass"
   project: default
+  source:
+    repoURL: https://{{{ .Env.DOMAIN_GITEA }}}/giteaAdmin/edfbuilder
+    targetRevision: HEAD
+    path: "stacks/ref-implementation/backstage-templates/entities"
+    directory:
+      exclude: 'catalog-info.yaml'
+  destination:
+    server: "https://kubernetes.default.svc"
+    namespace: backstage
   syncPolicy:
-    automated:
-      selfHeal: true
     syncOptions:
       - CreateNamespace=true
+    automated:
+      selfHeal: true
     retry:
       limit: -1

template/stacks/ref-implementation/backstage-templates/entities/app-with-bucket/skeleton/catalog-info.yaml

@@ -0,0 +1,48 @@
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: ${{values.name}}-bucket
+  description: Stores things
+  annotations:
+    argocd/app-name: ${{values.name | dump}}
+spec:
+  type: s3-bucket
+  owner: guests
+---
+apiVersion: backstage.io/v1alpha1
+kind: Component
+metadata:
+  name: ${{values.name | dump}}
+  description: This is for testing purposes
+  annotations:
+    backstage.io/techdocs-ref: dir:.
+    backstage.io/kubernetes-label-selector: 'entity-id=${{values.name}}'
+    backstage.io/kubernetes-namespace: default
+    argocd/app-name: ${{values.name | dump}}
+  links:
+    - url: https://{{{ .Env.DOMAIN_GITEA }}}:443
+      title: Repo URL
+      icon: github
+spec:
+  owner: guests
+  lifecycle: experimental
+  type: service
+  system: ${{values.name | dump}}
+  dependsOn:
+    - resource:default/${{values.name}}-bucket
+---
+apiVersion: backstage.io/v1alpha1
+kind: System
+metadata:
+  name: ${{values.name | dump}}
+  description: An example system for demonstration purposes
+  annotations:
+    backstage.io/techdocs-ref: dir:.
+  links:
+    - url: https://github.com/cnoe-io/stacks/tree/main/ref-implementation
+      title: CNOE Repo
+      icon: github
+spec:
+  owner: guests
+  lifecycle: experimental
+  type: service

template/stacks/ref-implementation/backstage-templates/entities/app-with-bucket/skeleton/docs/idpbuilder.md

@@ -0,0 +1,46 @@
+[![Codespell][codespell-badge]][codespell-link]
+[![E2E][e2e-badge]][e2e-link]
+[![Go Report Card][report-badge]][report-link]
+[![Commit Activity][commit-activity-badge]][commit-activity-link]
+
+# IDP Builder
+
+Internal development platform binary launcher.
+
+> **WORK IN PROGRESS**: This tool is in a pre-release stage and is under active development.
+
+## About
+
+Spin up a complete internal developer platform using industry standard technologies like Kubernetes, Argo, and backstage with only Docker required as a dependency.
+
+This can be useful in several ways:
+* Create a single binary which can demonstrate an IDP reference implementation.
+* Use within CI to perform integration testing.
+* Use as a local development environment for platform engineers.
+
+## Getting Started
+
+Checkout our [documentation website](https://cnoe.io/docs/reference-implementation/installations/idpbuilder) for getting started with idpbuilder.
+
+## Community
+
+- If you have questions or concerns about this tool, please feel free to reach out to us on the [CNCF Slack Channel](https://cloud-native.slack.com/archives/C05TN9WFN5S).
+- You can also join our community meetings to meet the team and ask any questions. Checkout [this calendar](https://calendar.google.com/calendar/embed?src=064a2adfce866ccb02e61663a09f99147f22f06374e7a8994066bdc81e066986%40group.calendar.google.com&ctz=America%2FLos_Angeles) for more information.
+
+## Contribution
+
+Checkout the [contribution doc](./CONTRIBUTING.md) for contribution guidelines and more information on how to set up your local environment.
+
+
+<!-- JUST BADGES & LINKS -->
+[codespell-badge]: https://github.com/cnoe-io/idpbuilder/actions/workflows/codespell.yaml/badge.svg
+[codespell-link]: https://github.com/cnoe-io/idpbuilder/actions/workflows/codespell.yaml
+
+[e2e-badge]: https://github.com/cnoe-io/idpbuilder/actions/workflows/e2e.yaml/badge.svg
+[e2e-link]: https://github.com/cnoe-io/idpbuilder/actions/workflows/e2e.yaml
+
+[report-badge]: https://goreportcard.com/badge/github.com/cnoe-io/idpbuilder
+[report-link]: https://goreportcard.com/report/github.com/cnoe-io/idpbuilder
+
+[commit-activity-badge]: https://img.shields.io/github/commit-activity/m/cnoe-io/idpbuilder
+[commit-activity-link]: https://github.com/cnoe-io/idpbuilder/pulse

template/stacks/ref-implementation/backstage-templates/entities/app-with-bucket/skeleton/docs/index.md

@@ -0,0 +1,16 @@
+![cnoe logo](./images/cnoe-logo.png)
+
+# Example Basic Application
+
+Thanks for trying out this demo! In this example, we deployed a simple application with a S3 bucket using Crossplane.
+
+
+### idpbuilder 
+
+Checkout the idpbuilder website: https://cnoe.io/docs/reference-implementation/installations/idpbuilder
+
+Checkout the idpbuilder repository: https://github.com/cnoe-io/idpbuilder
+
+## Crossplane
+
+Checkout the Crossplane website: https://www.crossplane.io/

template/stacks/ref-implementation/backstage-templates/entities/app-with-bucket/skeleton/go.mod

@@ -0,0 +1,3 @@
+module ${{ values.name }}
+
+go 1.19

template/stacks/ref-implementation/backstage-templates/entities/app-with-bucket/skeleton/kustomize/base/kustomization.yaml

@@ -0,0 +1,3 @@
+resources:
+  - nginx.yaml
+  - ${{ values.name }}.yaml

template/stacks/ref-implementation/backstage-templates/entities/app-with-bucket/skeleton/kustomize/base/nginx.yaml

@@ -0,0 +1,34 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx
+  labels:
+    app: nginx
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      app: nginx
+  template:
+    metadata:
+      labels:
+        app: nginx
+    spec:
+      containers:
+      - name: nginx
+        image: nginx:1.14.2
+        ports:
+        - containerPort: 80
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: nginx
+  labels:
+    app: nginx
+spec:
+  ports:
+  - port: 80
+    targetPort: 80
+  selector:
+    app: nginx

template/stacks/ref-implementation/backstage-templates/entities/app-with-bucket/skeleton/kustomize/dev/kustomization.yaml

@@ -0,0 +1,2 @@
+resources:
+  - ../base

template/stacks/ref-implementation/backstage-templates/entities/app-with-bucket/skeleton/kustomize/prod/kustomization.yaml

@@ -0,0 +1,35 @@
+{%- if values.awsResources %}
+resources:
+{%- if 'Bucket' in values.awsResources %}
+  - ../base/
+{%- endif %}
+{%- if 'Table' in values.awsResources %}
+  - ../base/table.yaml
+{%- endif %}
+{%- endif %}
+namespace: default
+
+patches:
+  - target:
+      kind: Deployment
+    patch: |
+      apiVersion: apps/v1
+      kind: Deployment
+      metadata:
+        name: not-used
+        labels:
+          backstage.io/kubernetes-id: ${{values.name}}
+      spec:
+        template:
+          metadata:
+            labels:
+              backstage.io/kubernetes-id: ${{values.name}}
+  - target:
+      kind: Service
+    patch: |
+      apiVersion: apps/v1
+      kind: Service
+      metadata:
+        name: not-used
+        labels:
+          backstage.io/kubernetes-id: ${{values.name}}