DevFW-CICD/stacks-instances
12
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Initial upload

Browse source
This commit is contained in:
Christopher Hase 2025-04-24 10:23:16 +02:00
parent 4f72750283
commit 4b3aa31790
118 changed files with 19189 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
cihe/factory.c-one-infra.de/edfbuilder.yaml Normal file
View file

@ -0,0 +1,24 @@
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: edfbuilder
namespace: argocd
labels:
env: dev
finalizers:
- resources-finalizer.argocd.argoproj.io
spec:
destination:
name: in-cluster
namespace: argocd
source:
path: "cihe/factory.c-one-infra.de/registry"
repoURL: "https://forgejo.edf-bootstrap.cx.fg1.ffm.osc.live/DevFW-CICD/stacks-instances"
targetRevision: HEAD
project: default
syncPolicy:
automated:
prune: true
selfHeal: true
syncOptions:
- CreateNamespace=true

24
cihe/factory.c-one-infra.de/registry/core.yaml Normal file
View file

@ -0,0 +1,24 @@
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: core
namespace: argocd
labels:
env: dev
finalizers:
- resources-finalizer.argocd.argoproj.io
spec:
destination:
name: in-cluster
namespace: argocd
source:
path: "cihe/factory.c-one-infra.de/stacks/core"
repoURL: "https://forgejo.edf-bootstrap.cx.fg1.ffm.osc.live/DevFW-CICD/stacks-instances"
targetRevision: HEAD
project: default
syncPolicy:
automated:
prune: true
selfHeal: true
syncOptions:
- CreateNamespace=true

24
cihe/factory.c-one-infra.de/registry/local-backup.yaml Normal file
View file

@ -0,0 +1,24 @@
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: local-backup
namespace: argocd
labels:
env: dev
finalizers:
- resources-finalizer.argocd.argoproj.io
spec:
destination:
name: in-cluster
namespace: argocd
source:
path: "cihe/factory.c-one-infra.de/stacks/local-backup"
repoURL: "https://forgejo.edf-bootstrap.cx.fg1.ffm.osc.live/DevFW-CICD/stacks-instances"
targetRevision: HEAD
project: default
syncPolicy:
automated:
prune: true
selfHeal: true
syncOptions:
- CreateNamespace=true

24
cihe/factory.c-one-infra.de/registry/monitoring.yaml Normal file
View file

@ -0,0 +1,24 @@
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: monitoring
namespace: argocd
labels:
env: dev
finalizers:
- resources-finalizer.argocd.argoproj.io
spec:
destination:
name: in-cluster
namespace: argocd
source:
path: "cihe/factory.c-one-infra.de/stacks/monitoring"
repoURL: "https://forgejo.edf-bootstrap.cx.fg1.ffm.osc.live/DevFW-CICD/stacks-instances"
targetRevision: HEAD
project: default
syncPolicy:
automated:
prune: true
selfHeal: true
syncOptions:
- CreateNamespace=true

24
cihe/factory.c-one-infra.de/registry/ref-implementation.yaml Normal file
View file

@ -0,0 +1,24 @@
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: ref-implementation
namespace: argocd
labels:
env: dev
finalizers:
- resources-finalizer.argocd.argoproj.io
spec:
destination:
name: in-cluster
namespace: argocd
source:
path: "cihe/factory.c-one-infra.de/stacks/ref-implementation"
repoURL: "https://forgejo.edf-bootstrap.cx.fg1.ffm.osc.live/DevFW-CICD/stacks-instances"
targetRevision: HEAD
project: default
syncPolicy:
automated:
prune: true
selfHeal: true
syncOptions:
- CreateNamespace=true

30
cihe/factory.c-one-infra.de/stacks/core/argocd.yaml Normal file
View file

@ -0,0 +1,30 @@
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: argocd
namespace: argocd
labels:
env: dev
spec:
project: default
syncPolicy:
automated:
selfHeal: true
syncOptions:
- CreateNamespace=true
destination:
name: in-cluster
namespace: argocd
sources:
- repoURL: https://forgejo.edf-bootstrap.cx.fg1.ffm.osc.live/DevFW-CICD/argocd-helm.git
path: charts/argo-cd
# TODO: RIRE Can be updated when https://github.com/argoproj/argo-cd/issues/20790 is fixed and merged
# As logout make problems, it is suggested to switch from path based routing to an own argocd domain,
# similar to the CNOE amazon reference implementation and in our case, Forgejo
targetRevision: argo-cd-7.8.14-depends
helm:
valueFiles:
- $values/cihe/factory.c-one-infra.de/stacks/core/argocd/values.yaml
- repoURL: https://forgejo.edf-bootstrap.cx.fg1.ffm.osc.live/DevFW-CICD/stacks-instances
targetRevision: HEAD
ref: values

24
cihe/factory.c-one-infra.de/stacks/core/argocd/manifests/argocd-secret.yaml Normal file
View file

@ -0,0 +1,24 @@
apiVersion: external-secrets.io/v1beta1
kind: ExternalSecret
metadata:
name: auth-generic-oauth-secret
namespace: argocd
spec:
secretStoreRef:
name: keycloak
kind: ClusterSecretStore
refreshInterval: "0"
target:
name: auth-generic-oauth-secret
template:
engineVersion: v2
data:
client_secret: "{{.ARGOCD_CLIENT_SECRET}}"
metadata:
labels:
app.kubernetes.io/part-of: argocd
data:
- secretKey: ARGOCD_CLIENT_SECRET
remoteRef:
key: keycloak-clients
property: ARGOCD_CLIENT_SECRET

35
cihe/factory.c-one-infra.de/stacks/core/argocd/values.yaml Normal file
View file

@ -0,0 +1,35 @@
global:
domain: factory.c-one-infra.de
configs:
params:
server.insecure: true
server.basehref: /argocd
server.rootpath: /argocd
cm:
application.resourceTrackingMethod: annotation
timeout.reconciliation: 60s
resource.exclusions: |
- apiGroups:
- "*"
kinds:
- ProviderConfigUsage
- apiGroups:
- cilium.io
kinds:
- CiliumIdentity
clusters:
- "*"
accounts.provider-argocd: apiKey
url: https://factory.c-one-infra.de/argocd
rbac:
policy.csv: 'g, provider-argocd, role:admin'
tls:
certificates:
notifications:
enabled: false
dex:
enabled: false

22
cihe/factory.c-one-infra.de/stacks/core/forgejo-runner.yaml Normal file
View file

@ -0,0 +1,22 @@
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: forgejo-runner
namespace: argocd
labels:
env: dev
finalizers:
- resources-finalizer.argocd.argoproj.io
spec:
project: default
syncPolicy:
automated:
selfHeal: true
syncOptions:
- CreateNamespace=true
destination:
server: "https://kubernetes.default.svc"
source:
repoURL: https://forgejo.edf-bootstrap.cx.fg1.ffm.osc.live/DevFW-CICD/stacks-instances
targetRevision: HEAD
path: "cihe/factory.c-one-infra.de/stacks/core/forgejo-runner"

104
cihe/factory.c-one-infra.de/stacks/core/forgejo-runner/dind-docker.yaml Normal file
View file

@ -0,0 +1,104 @@
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app: forgejo-runner
name: forgejo-runner
namespace: gitea
spec:
# Two replicas means that if one is busy, the other can pick up jobs.
replicas: 1
selector:
matchLabels:
app: forgejo-runner
strategy: {}
template:
metadata:
creationTimestamp: null
labels:
app: forgejo-runner
spec:
restartPolicy: Always
volumes:
- name: docker-certs
emptyDir: {}
- name: runner-data
emptyDir: {}
# Initialise our configuration file using offline registration
# https://forgejo.org/docs/v1.21/admin/actions/#offline-registration
initContainers:
- name: runner-register
image: code.forgejo.org/forgejo/runner:6.3.1
command:
- "sh"
- "-c"
- |
forgejo-runner \
register \
--no-interactive \
--token ${RUNNER_SECRET} \
--name ${RUNNER_NAME} \
--instance ${FORGEJO_INSTANCE_URL} \
--labels docker:docker://node:20-bookworm,ubuntu-22.04:docker://forgejo.edf-bootstrap.cx.fg1.ffm.osc.live/devfw-cicd/catthehackerubuntu:act-22.04,ubuntu-latest:docker://forgejo.edf-bootstrap.cx.fg1.ffm.osc.live/devfw-cicd/catthehackerubuntu:act-22.04
env:
- name: RUNNER_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: RUNNER_SECRET
valueFrom:
secretKeyRef:
name: forgejo-runner-token
key: token
- name: FORGEJO_INSTANCE_URL
value: https://gitea.factory.c-one-infra.de
volumeMounts:
- name: runner-data
mountPath: /data
containers:
- name: runner
image: code.forgejo.org/forgejo/runner:6.3.1
command:
- "sh"
- "-c"
- |
while ! nc -z 127.0.0.1 2376 </dev/null; do
echo 'waiting for docker daemon...';
sleep 5;
done
forgejo-runner generate-config > config.yml ;
sed -i -e "s|privileged: .*|privileged: true|" config.yml
sed -i -e "s|network: .*|network: host|" config.yml ;
sed -i -e "s|^ envs:$$| envs:\n DOCKER_HOST: tcp://127.0.0.1:2376\n DOCKER_TLS_VERIFY: 1\n DOCKER_CERT_PATH: /certs/client|" config.yml ;
sed -i -e "s|^ options:| options: -v /certs/client:/certs/client|" config.yml ;
sed -i -e "s| valid_volumes: \[\]$$| valid_volumes:\n - /certs/client|" config.yml ;
/bin/forgejo-runner --config config.yml daemon
securityContext:
allowPrivilegeEscalation: true
privileged: true
readOnlyRootFilesystem: false
runAsGroup: 0
runAsNonRoot: false
runAsUser: 0
env:
- name: DOCKER_HOST
value: tcp://localhost:2376
- name: DOCKER_CERT_PATH
value: /certs/client
- name: DOCKER_TLS_VERIFY
value: "1"
volumeMounts:
- name: docker-certs
mountPath: /certs
- name: runner-data
mountPath: /data
- name: daemon
image: docker:28.0.4-dind
env:
- name: DOCKER_TLS_CERTDIR
value: /certs
securityContext:
privileged: true
volumeMounts:
- name: docker-certs
mountPath: /certs

30
cihe/factory.c-one-infra.de/stacks/core/forgejo.yaml Normal file
View file

@ -0,0 +1,30 @@
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: forgejo
namespace: argocd
labels:
env: dev
spec:
project: default
syncPolicy:
automated:
selfHeal: true
syncOptions:
- CreateNamespace=true
destination:
name: in-cluster
namespace: gitea
sources:
- repoURL: https://forgejo.edf-bootstrap.cx.fg1.ffm.osc.live/DevFW-CICD/forgejo-helm.git
path: .
targetRevision: v11.0.5-depends
helm:
valueFiles:
- $values/cihe/factory.c-one-infra.de/stacks/core/forgejo/values.yaml
- repoURL: https://forgejo.edf-bootstrap.cx.fg1.ffm.osc.live/DevFW-CICD/stacks-instances
targetRevision: HEAD
ref: values
- repoURL: https://forgejo.edf-bootstrap.cx.fg1.ffm.osc.live/DevFW-CICD/stacks-instances
targetRevision: HEAD
path: "cihe/factory.c-one-infra.de/stacks/core/forgejo/manifests"

36
cihe/factory.c-one-infra.de/stacks/core/forgejo/manifests/secret-admin-password.yaml Normal file
View file

@ -0,0 +1,36 @@
apiVersion: generators.external-secrets.io/v1alpha1
kind: Password
metadata:
name: forgejo-admin-password-generator
namespace: gitea
spec:
length: 36
digits: 5
symbols: 5
symbolCharacters: "/-+"
noUpper: false
allowRepeat: true
---
apiVersion: external-secrets.io/v1beta1
kind: ExternalSecret
metadata:
name: forgejo-admin-password-generator
namespace: gitea
spec:
refreshInterval: "0"
target:
name: gitea-credential
template:
engineVersion: v2
data:
username: giteaAdmin
password: "{{.INITIAL_ADMIN_PASSWORD}}"
dataFrom:
- sourceRef:
generatorRef:
apiVersion: generators.external-secrets.io/v1alpha1
kind: Password
name: forgejo-admin-password-generator
rewrite:
- transform:
template: "INITIAL_ADMIN_PASSWORD"

26
cihe/factory.c-one-infra.de/stacks/core/forgejo/manifests/secret-forgejo.yaml Normal file
View file

@ -0,0 +1,26 @@
apiVersion: external-secrets.io/v1beta1
kind: ExternalSecret
metadata:
name: auth-generic-oauth-secret
namespace: gitea
spec:
secretStoreRef:
name: keycloak
kind: ClusterSecretStore
refreshInterval: "0"
target:
name: auth-generic-oauth-secret
template:
engineVersion: v2
data:
key: "{{.FORGEJO_CLIENT_ID}}"
secret: "{{.FORGEJO_CLIENT_SECRET}}"
data:
- secretKey: FORGEJO_CLIENT_ID
remoteRef:
key: keycloak-clients
property: FORGEJO_CLIENT_ID
- secretKey: FORGEJO_CLIENT_SECRET
remoteRef:
key: keycloak-clients
property: FORGEJO_CLIENT_SECRET

67
cihe/factory.c-one-infra.de/stacks/core/forgejo/values.yaml Normal file
View file

@ -0,0 +1,67 @@
redis-cluster:
enabled: true
postgresql:
enabled: false
postgresql-ha:
enabled: false
persistence:
enabled: true
size: 5Gi
test:
enabled: false
gitea:
admin:
existingSecret: gitea-credential
config:
service:
DISABLE_REGISTRATION: true
other:
SHOW_FOOTER_VERSION: false
SHOW_FOOTER_TEMPLATE_LOAD_TIME: false
database:
DB_TYPE: sqlite3
session:
PROVIDER: memory
cache:
ADAPTER: memory
queue:
TYPE: level
server:
DOMAIN: 'gitea.factory.c-one-infra.de'
ROOT_URL: 'https://gitea.factory.c-one-infra.de:443'
mailer:
ENABLED: true
FROM: forgejo@gitea.factory.c-one-infra.de
PROTOCOL: smtp
SMTP_ADDR: mailhog.mailhog.svc.cluster.local
SMTP_PORT: 1025
service:
ssh:
type: NodePort
nodePort: 32222
externalTrafficPolicy: Local
image:
pullPolicy: "IfNotPresent"
# Overrides the image tag whose default is the chart appVersion.
#tag: "8.0.3"
# Adds -rootless suffix to image name
rootless: true
forgejo:
runner:
enabled: true
image:
tag: latest
# replicas: 3
config:
runner:
labels:
- docker:docker://node:16-bullseye
- self-hosted:docker://ghcr.io/catthehacker/ubuntu:act-22.04
- ubuntu-22.04:docker://ghcr.io/catthehacker/ubuntu:act-22.04
- ubuntu-latest:docker://ghcr.io/catthehacker/ubuntu:act-22.04

22
cihe/factory.c-one-infra.de/stacks/core/ingress-apps.yaml Normal file
View file

@ -0,0 +1,22 @@
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: ingress-apps
namespace: argocd
labels:
example: ref-implementation
finalizers:
- resources-finalizer.argocd.argoproj.io
spec:
destination:
server: "https://kubernetes.default.svc"
source:
repoURL: https://forgejo.edf-bootstrap.cx.fg1.ffm.osc.live/DevFW-CICD/stacks-instances
targetRevision: HEAD
path: "cihe/factory.c-one-infra.de/stacks/core/ingress-apps"
project: default
syncPolicy:
automated:
selfHeal: true
syncOptions:
- CreateNamespace=true

31
cihe/factory.c-one-infra.de/stacks/core/ingress-apps/argo-workflows-ingress.yaml Normal file
View file

@ -0,0 +1,31 @@
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
annotations:
nginx.ingress.kubernetes.io/rewrite-target: /$2
nginx.ingress.kubernetes.io/use-regex: "true"
name: argo-workflows-ingress
namespace: argo
spec:
ingressClassName: nginx
rules:
- host: localhost
http:
paths:
- backend:
service:
name: argo-server
port:
name: web
path: /argo-workflows(/|$)(.*)
pathType: ImplementationSpecific
- host: factory.c-one-infra.de
http:
paths:
- backend:
service:
name: argo-server
port:
name: web
path: /argo-workflows(/|$)(.*)
pathType: ImplementationSpecific

26
cihe/factory.c-one-infra.de/stacks/core/ingress-apps/argocd-server.yaml Normal file
View file

@ -0,0 +1,26 @@
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
annotations:
nginx.ingress.kubernetes.io/backend-protocol: HTTP
nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
name: argocd-server
namespace: argocd
spec:
ingressClassName: nginx
rules:
- host: factory.c-one-infra.de
http:
paths:
- backend:
service:
name: argocd-server
port:
number: 80
path: /argocd
pathType: Prefix
tls:
- hosts:
- factory.c-one-infra.de
secretName: argocd-net-tls

28
cihe/factory.c-one-infra.de/stacks/core/ingress-apps/backstage.yaml Normal file
View file

@ -0,0 +1,28 @@
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: backstage
namespace: backstage
spec:
ingressClassName: nginx
rules:
- host: localhost
http:
paths:
- backend:
service:
name: backstage
port:
name: http
path: /
pathType: Prefix
- host: factory.c-one-infra.de
http:
paths:
- backend:
service:
name: backstage
port:
name: http
path: /
pathType: Prefix

26
cihe/factory.c-one-infra.de/stacks/core/ingress-apps/forgejo.yaml Normal file
View file

@ -0,0 +1,26 @@
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
annotations:
nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
nginx.ingress.kubernetes.io/proxy-body-size: 512m
name: forgejo
namespace: gitea
spec:
ingressClassName: nginx
rules:
- host: gitea.factory.c-one-infra.de
http:
paths:
- backend:
service:
name: forgejo-http
port:
number: 3000
path: /
pathType: Prefix
tls:
- hosts:
- gitea.factory.c-one-infra.de
secretName: forgejo-net-tls

28
cihe/factory.c-one-infra.de/stacks/core/ingress-apps/keycloak-ingress-localhost.yaml Normal file
View file

@ -0,0 +1,28 @@
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: keycloak-ingress-localhost
namespace: keycloak
spec:
ingressClassName: nginx
rules:
- host: localhost
http:
paths:
- backend:
service:
name: keycloak
port:
name: http
path: /keycloak
pathType: ImplementationSpecific
- host: factory.c-one-infra.de
http:
paths:
- backend:
service:
name: keycloak
port:
name: http
path: /keycloak
pathType: ImplementationSpecific

18
cihe/factory.c-one-infra.de/stacks/core/ingress-apps/kube-prometheus-stack-grafana.yaml Normal file
View file

@ -0,0 +1,18 @@
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: kube-prometheus-stack-grafana
namespace: monitoring
spec:
ingressClassName: nginx
rules:
- host: factory.c-one-infra.de
http:
paths:
- backend:
service:
name: kube-prometheus-stack-grafana
port:
number: 80
path: /grafana
pathType: Prefix

18
cihe/factory.c-one-infra.de/stacks/core/ingress-apps/mailhog.yaml Normal file
View file

@ -0,0 +1,18 @@
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: mailhog
namespace: mailhog
spec:
ingressClassName: nginx
rules:
- host: factory.c-one-infra.de
http:
paths:
- backend:
service:
name: mailhog
port:
number: 8025
path: /mailhog
pathType: Prefix

19
cihe/factory.c-one-infra.de/stacks/core/ingress-apps/openbao.yaml Normal file
View file

@ -0,0 +1,19 @@
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: openbao
namespace: openbao
spec:
ingressClassName: nginx
rules:
- host: openbao.factory.c-one-infra.de
http:
paths:
- backend:
service:
name: openbao
port:
number: 8200
path: /
pathType: Prefix

27
cihe/factory.c-one-infra.de/stacks/core/ingress-nginx.yaml Normal file
View file

@ -0,0 +1,27 @@
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: ingress-nginx
namespace: argocd
labels:
env: dev
spec:
project: default
syncPolicy:
automated:
selfHeal: true
syncOptions:
- CreateNamespace=true
destination:
name: in-cluster
namespace: ingress-nginx
sources:
- repoURL: https://forgejo.edf-bootstrap.cx.fg1.ffm.osc.live/DevFW-CICD/ingress-nginx-helm.git
path: charts/ingress-nginx
targetRevision: helm-chart-4.12.1-depends
helm:
valueFiles:
- $values/cihe/factory.c-one-infra.de/stacks/core/ingress-nginx/values.yaml
- repoURL: https://forgejo.edf-bootstrap.cx.fg1.ffm.osc.live/DevFW-CICD/stacks-instances
targetRevision: HEAD
ref: values

49
cihe/factory.c-one-infra.de/stacks/core/ingress-nginx/values.yaml Normal file
View file

@ -0,0 +1,49 @@
controller:
updateStrategy:
type: RollingUpdate
rollingUpdate:
maxUnavailable: 1
ingressClassResource:
name: nginx
# added for idpbuilder
allowSnippetAnnotations: true
# added for idpbuilder
config:
proxy-buffer-size: 32k
use-forwarded-headers: "true"
# monitoring nginx
metrics:
enabled: true
serviceMonitor:
additionalLabels:
release: "ingress-nginx"
enabled: true
hostPort:
enabled: true
terminationGracePeriodSeconds: 0
service:
type: NodePort
nodeSelector:
ingress-ready: "true"
tolerations:
- key: "node-role.kubernetes.io/master"
operator: "Equal"
effect: "NoSchedule"
- key: "node-role.kubernetes.io/control-plane"
operator: "Equal"
effect: "NoSchedule"
publishService:
enabled: false
extraArgs:
publish-status-address: localhost
# added for idpbuilder
enable-ssl-passthrough: ""

126
cihe/factory.c-one-infra.de/stacks/local-backup/README.md Normal file
View file

@ -0,0 +1,126 @@
# Local Backup with Velero and Minio
This is example is adapted from the original icpbuilder stack.
The two significant changes from the original were made:
* disabled `hostPath` mount to persist backups within kind, since backups do not work sufficiently in this example due to PVC issues, see below.
* renamed `minio` namespace to `minio-backup` so it does not collide with other minio examples.
Within kind, it can only backup kubernetes objects. Data from PVC's is skipped, see below why.
[Velero](https://velero.io/) requires some compatible storage providers as its backup target. This local installation uses [MinIO](https://min.io/) as an example.
However, MinIO is not officially supported by Velero but works due to S3 compatibility.
The current setup does NOT persist backups but stores them in MinIO's PVCs. Proper backups should configure external storage, see [Supported Providers](https://velero.io/docs/main/supported-providers/).
## Installation
The stack is installed as part of the `./example.sh` run.
In order to persist a local backup you have to mount a local directory within `main.go`:
```yaml
nodes:
- role: control-plane
extraMounts:
- hostPath: /some/path/backup # replace with your own path
containerPath: /backup
```
Kind creates the directory on the host but you might have to adjust the permissions, otherwise the minio pod fails to start.
## Using it
After the installation velero and minio should be visible in ArgoCD.
During the installation credentials for minio are generated and shared with velero. You can access them manually:
```bash
kubectl -n minio-backup get secret root-creds -o go-template='{{ range $key, $value := .data }}{{ printf "%s: %s\n" $key ($value | base64decode) }}{{ end }}'
# example output
# rootPassword: aKKZzLnyry6OYZts17vMTf32H5ghFL4WYgu6bHujm
# rootUser: ge8019yksArb7BICt3MLY9
```
A bucket in minio was created and velero uses it for its backups by default, see helm `values.yaml` files.
### Backup and Restore
Backups and subsequent restores can be scheduled by either using the velero cli or by creating CRD objects.
Check the `./demo` directory for equivalent CRD manifests.
Create a backup of the backstage namespace, see `schedule` task for more permanent setups:
```shell
velero backup create backstage-backup --include-namespaces backstage
```
There are more options to create a fine granular backup and to set the backup storage.
See velero's docs for details.
Check the backup with:
```shell
velero backup get
```
To get more details on the backup you need to be able to connect to velero's backup storage, i.e. minio.
Using `kubefwd` here helps a lot (this is not necessary for restore).
```shell
kubefwd services -n minio-backup
```
More details with `describe` and `logs`:
```shell
velero backup describe backstage-backup --details
velero backup logs backstage-backup
```
Restore the backup into the original namespace, you might want to delete the existing namespace beforehand:
```shell
kubectl delete namespace backstage
velero restore create --from-backup backstage-backup
```
When restoring, velero does not replace existing objects in the backup target.
ArgoCD does pickup on the changes and also validates that the backup is in sync.
## Issues with Persistent Volumes
Velero has no issue to backup kubernetes objects like Deployments, ConfigMaps, etc. since they are just yaml/json definitions.
Volumes containing data are, however, more complex. The preferred type of backup are kubernetes' VolumeSnapshots as they consistently store the state
of a volume at a given point in time in an atomic action. Those snapshots live within the cluster and are subsequently downloaded into one of velero's
storage backends for safekeeping.
However, VolumeSnapshots are only possible on storage backends that support them via CSI drivers.
Backends like `nfs` or `hostPath` do NOT support them. Here, velero uses an alternative method
called [File System Backups](https://velero.io/docs/main/file-system-backup/).
In essence, this a simple copy operation based on the file system. Even though
this uses more sophisticated tooling under the hood, i.e. kopia, it is not
possible to create a backup in an atomic transaction. Thus, the resulting backup
might be inconsistent.
Furthermore, for file system backups to work velero installs a node-agent as a
DaemonSet on each Kubernetes node. The agent is aware of the node's internal
storage and accesses the directories on the host directly to copy the files.
This is not supported for hostPath volumes as they mount an arbitrary path
on the host. In theory, a backup is possible but due extra config and security
considerations intentionally skipped. Kind's local-path provider storage uses
a hostPath and is thus not supported for any kind of backup.
## TODOs
* The MinIO -backup installation is only intended as an example and must either
be configured properly or replaced.
* The current example does not automatically schedule backups.
* velero chart must be properly parameterized

9
cihe/factory.c-one-infra.de/stacks/local-backup/demo/backup.yaml Normal file
View file

@ -0,0 +1,9 @@
# velero backup create backstage-backup --include-namespaces backstage
apiVersion: velero.io/v1
kind: Backup
metadata:
name: backstage-backup
namespace: velero
spec:
includedNamespaces:
- 'backstage'

10
cihe/factory.c-one-infra.de/stacks/local-backup/demo/restore.yaml Normal file
View file

@ -0,0 +1,10 @@
# velero restore create --from-backup backstage-backup
apiVersion: velero.io/v1
kind: Restore
metadata:
name: backstage-backup
namespace: velero
spec:
backupName: backstage-backup
includedNamespaces:
- 'backstage'

33
cihe/factory.c-one-infra.de/stacks/local-backup/minio.yaml Normal file
View file

@ -0,0 +1,33 @@
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: minio
namespace: argocd
labels:
env: dev
finalizers:
- resources-finalizer.argocd.argoproj.io
spec:
project: default
sources:
- repoURL: "https://charts.min.io"
targetRevision: 5.0.15
helm:
releaseName: minio
valueFiles:
- $values/cihe/factory.c-one-infra.de/stacks/local-backup/minio/helm/values.yaml
chart: minio
- repoURL: https://forgejo.edf-bootstrap.cx.fg1.ffm.osc.live/DevFW-CICD/stacks-instances
targetRevision: HEAD
ref: values
- repoURL: https://forgejo.edf-bootstrap.cx.fg1.ffm.osc.live/DevFW-CICD/stacks-instances
targetRevision: HEAD
path: "cihe/factory.c-one-infra.de/stacks/local-backup/minio/manifests"
destination:
server: "https://kubernetes.default.svc"
namespace: minio-backup
syncPolicy:
syncOptions:
- CreateNamespace=true
automated:
selfHeal: true

17
cihe/factory.c-one-infra.de/stacks/local-backup/minio/helm/values.yaml Normal file
View file

@ -0,0 +1,17 @@
replicas: 1
mode: standalone
resources:
requests:
memory: 128Mi
persistence:
enabled: true
storageClass: standard
size: 512Mi
# volumeName: backup # re-enable this to mount a local host path, see minio-pv.yaml
buckets:
- name: edfbuilder-backups
existingSecret: root-creds

13
cihe/factory.c-one-infra.de/stacks/local-backup/minio/manifests/minio-pv.yaml Normal file
View file

@ -0,0 +1,13 @@
# re-enable this config to mount a local host path, see `../helm/values.yaml`
# apiVersion: v1
# kind: PersistentVolume
# metadata:
# name: backup
# spec:
# storageClassName: standard
# accessModes:
# - ReadWriteOnce
# capacity:
# storage: 512Mi
# hostPath:
# path: /backup

154
cihe/factory.c-one-infra.de/stacks/local-backup/minio/manifests/secret-sync.yaml Normal file
View file

@ -0,0 +1,154 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: secret-sync
namespace: minio-backup
annotations:
argocd.argoproj.io/hook: Sync
argocd.argoproj.io/sync-wave: "-20"
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: secret-sync
namespace: minio-backup
annotations:
argocd.argoproj.io/hook: Sync
argocd.argoproj.io/sync-wave: "-20"
rules:
- apiGroups: [""]
resources: ["secrets"]
verbs: ["get", "create", "update", "patch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: secret-sync
namespace: minio-backup
annotations:
argocd.argoproj.io/hook: Sync
argocd.argoproj.io/sync-wave: "-20"
subjects:
- kind: ServiceAccount
name: secret-sync
namespace: minio-backup
roleRef:
kind: Role
name: secret-sync
apiGroup: rbac.authorization.k8s.io
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: secret-sync
namespace: velero
annotations:
argocd.argoproj.io/hook: Sync
argocd.argoproj.io/sync-wave: "-20"
rules:
- apiGroups: [""]
resources: ["secrets"]
verbs: ["get", "create", "update", "patch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: secret-sync
namespace: velero
annotations:
argocd.argoproj.io/hook: Sync
argocd.argoproj.io/sync-wave: "-20"
subjects:
- kind: ServiceAccount
name: secret-sync
namespace: minio-backup
roleRef:
kind: Role
name: secret-sync
apiGroup: rbac.authorization.k8s.io
---
apiVersion: batch/v1
kind: Job
metadata:
name: secret-sync
namespace: minio-backup
annotations:
argocd.argoproj.io/hook: PostSync
spec:
template:
metadata:
generateName: secret-sync
spec:
serviceAccountName: secret-sync
restartPolicy: Never
containers:
- name: kubectl
image: docker.io/bitnami/kubectl
command: ["/bin/bash", "-c"]
args:
- |
set -e
kubectl get secrets -n minio-backup root-creds -o json > /tmp/secret
ACCESS=$(jq -r '.data.rootUser | @base64d' /tmp/secret)
SECRET=$(jq -r '.data.rootPassword | @base64d' /tmp/secret)
echo \
"apiVersion: v1
kind: Secret
metadata:
name: secret-key
namespace: velero
type: Opaque
stringData:
aws: |
[default]
aws_access_key_id=${ACCESS}
aws_secret_access_key=${SECRET}
" > /tmp/secret.yaml
kubectl apply -f /tmp/secret.yaml
---
apiVersion: batch/v1
kind: Job
metadata:
name: minio-root-creds
namespace: minio-backup
annotations:
argocd.argoproj.io/hook: Sync
argocd.argoproj.io/sync-wave: "-10"
spec:
template:
metadata:
generateName: minio-root-creds
spec:
serviceAccountName: secret-sync
restartPolicy: Never
containers:
- name: kubectl
image: docker.io/bitnami/kubectl
command: ["/bin/bash", "-c"]
args:
- |
kubectl get secrets -n minio-backup root-creds
if [ $? -eq 0 ]; then
exit 0
fi
set -e
NAME=$(openssl rand -base64 24)
PASS=$(openssl rand -base64 36)
echo \
"apiVersion: v1
kind: Secret
metadata:
name: root-creds
namespace: minio-backup
type: Opaque
stringData:
rootUser: "${NAME}"
rootPassword: "${PASS}"
" > /tmp/secret.yaml
kubectl apply -f /tmp/secret.yaml

31
cihe/factory.c-one-infra.de/stacks/local-backup/velero.yaml Normal file
View file

@ -0,0 +1,31 @@
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: velero
namespace: argocd
labels:
env: dev
finalizers:
- resources-finalizer.argocd.argoproj.io
spec:
project: default
sources:
- repoURL: "https://vmware-tanzu.github.io/helm-charts"
targetRevision: 8.0.0
helm:
releaseName: velero
valueFiles:
- $values/cihe/factory.c-one-infra.de/stacks/local-backup/velero/helm/values.yaml
chart: velero
- repoURL: https://forgejo.edf-bootstrap.cx.fg1.ffm.osc.live/DevFW-CICD/stacks-instances
targetRevision: HEAD
ref: values
destination:
server: "https://kubernetes.default.svc"
namespace: velero
syncPolicy:
syncOptions:
- CreateNamespace=true
automated:
prune: true
selfHeal: true

25
cihe/factory.c-one-infra.de/stacks/local-backup/velero/helm/values.yaml Normal file
View file

@ -0,0 +1,25 @@
resources:
requests:
memory: 128Mi
initContainers:
- name: velero-plugin-for-aws
image: velero/velero-plugin-for-aws:v1.11.0
imagePullPolicy: IfNotPresent
volumeMounts:
- mountPath: /target
name: plugins
# snapshotsEnabled: false # create snapshot crd?
# deployNodeAgent: true # install node agent as daemonset for file system backups?
configuration:
# defaultVolumesToFsBackup: true # backup pod volumes via fsb without explicit annotation?
backupStorageLocation:
- name: default
provider: aws
bucket: edfbuilder-backups
credential:
name: secret-key # this key is created within the minio-backup/secret-sync and injected into the velero namespace
key: aws
config:
region: minio
s3Url: http://minio.minio-backup.svc.cluster.local:9000 # internal resolution, external access for velero cli via fwd
s3ForcePathStyle: "true"

29
cihe/factory.c-one-infra.de/stacks/monitoring/alloy.yaml Normal file
View file

@ -0,0 +1,29 @@
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: alloy
namespace: argocd
labels:
env: dev
finalizers:
- resources-finalizer.argocd.argoproj.io
spec:
project: default
syncPolicy:
automated:
selfHeal: true
syncOptions:
- CreateNamespace=true
destination:
name: in-cluster
namespace: monitoring
sources:
- repoURL: https://github.com/grafana/alloy.git
path: operations/helm/charts/alloy
targetRevision: HEAD
helm:
valueFiles:
- $values/cihe/factory.c-one-infra.de/stacks/monitoring/alloy/values.yaml
- repoURL: https://forgejo.edf-bootstrap.cx.fg1.ffm.osc.live/DevFW-CICD/stacks-instances
targetRevision: HEAD
ref: values

78
cihe/factory.c-one-infra.de/stacks/monitoring/alloy/values.yaml Normal file
View file

@ -0,0 +1,78 @@
alloy:
create: false
name: alloy-config
key: config.alloy
uiPathPrefix: "/alloy"
configMap:
content: |-
logging {
level = "info"
format = "logfmt"
}
loki.write "local_loki" {
endpoint {
url = "http://loki-loki-distributed-gateway/loki/api/v1/push"
}
}
discovery.kubernetes "pod" {
role = "pod"
}
discovery.kubernetes "nodes" {
role = "node"
}
discovery.kubernetes "services" {
role = "service"
}
discovery.kubernetes "endpoints" {
role = "endpoints"
}
discovery.kubernetes "endpointslices" {
role = "endpointslice"
}
discovery.kubernetes "ingresses" {
role = "ingress"
}
discovery.relabel "pod_logs" {
targets = discovery.kubernetes.pod.targets
rule {
source_labels = ["__meta_kubernetes_namespace"]
action = "replace"
target_label = "namespace"
}
rule {
source_labels = ["__meta_kubernetes_pod_name"]
action = "replace"
target_label = "pod"
}
rule {
source_labels = ["__meta_kubernetes_pod_node_name"]
action = "replace"
target_label = "node"
}
rule {
source_labels = ["__meta_kubernetes_pod_container_name"]
action = "replace"
target_label = "container"
}
}
loki.source.kubernetes "all_pod_logs" {
targets = discovery.relabel.pod_logs.output
forward_to = [loki.write.local_loki.receiver]
}

25
cihe/factory.c-one-infra.de/stacks/monitoring/grafana-dashboards.yaml Normal file
View file

@ -0,0 +1,25 @@
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: grafana-dashboards
namespace: argocd
labels:
env: dev
finalizers:
- resources-finalizer.argocd.argoproj.io
spec:
project: default
source:
repoURL: https://forgejo.edf-bootstrap.cx.fg1.ffm.osc.live/DevFW-CICD/stacks-instances
targetRevision: HEAD
path: "cihe/factory.c-one-infra.de/stacks/monitoring/kube-prometheus/dashboards"
destination:
server: "https://kubernetes.default.svc"
namespace: monitoring
syncPolicy:
syncOptions:
- CreateNamespace=true
automated:
selfHeal: true
retry:
limit: -1

39
cihe/factory.c-one-infra.de/stacks/monitoring/kube-prometheus.yaml Normal file
View file

@ -0,0 +1,39 @@
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: kube-prometheus-stack
namespace: argocd
labels:
env: dev
finalizers:
- resources-finalizer.argocd.argoproj.io
spec:
project: default
syncPolicy:
automated:
selfHeal: true
syncOptions:
- CreateNamespace=true
- ServerSideApply=true # do not copy metdata, since (because of its large size) it can lead to sync failure
retry:
limit: -1
backoff:
duration: 15s
factor: 1
maxDuration: 15s
destination:
name: in-cluster
namespace: monitoring
sources:
- repoURL: https://github.com/prometheus-community/helm-charts
path: charts/kube-prometheus-stack
targetRevision: HEAD
helm:
valueFiles:
- $values/cihe/factory.c-one-infra.de/stacks/monitoring/kube-prometheus/values.yaml
- repoURL: https://forgejo.edf-bootstrap.cx.fg1.ffm.osc.live/DevFW-CICD/stacks-instances
targetRevision: HEAD
ref: values
- repoURL: https://forgejo.edf-bootstrap.cx.fg1.ffm.osc.live/DevFW-CICD/stacks-instances
targetRevision: HEAD
path: "cihe/factory.c-one-infra.de/stacks/monitoring/kube-prometheus/manifests"

268
cihe/factory.c-one-infra.de/stacks/monitoring/kube-prometheus/dashboards/dashboard_loki_apps.yaml Normal file
View file

@ -0,0 +1,268 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: grafana-dashboard-1
labels:
grafana_dashboard: "1"
data:
k8s-dashboard-01.json: |
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": 1,
"links": [
],
"panels": [
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"fieldConfig": {
"defaults": {
},
"overrides": [
]
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 0
},
"id": 5,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": false,
"showTime": false,
"sortOrder": "Descending",
"wrapLogMessage": false
},
"pluginVersion": "11.3.1",
"targets": [
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"expr": "{app=\"crossplane\"}",
"queryType": "range",
"refId": "A"
}
],
"title": "Logs: App crossplane",
"type": "logs"
},
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"fieldConfig": {
"defaults": {
},
"overrides": [
]
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 8
},
"id": 4,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": false,
"showTime": false,
"sortOrder": "Descending",
"wrapLogMessage": false
},
"pluginVersion": "11.3.1",
"targets": [
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"expr": "{app=\"argo-server\"}",
"queryType": "range",
"refId": "A"
}
],
"title": "Logs: App argo-server",
"type": "logs"
},
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"fieldConfig": {
"defaults": {
},
"overrides": [
]
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 16
},
"id": 3,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": false,
"showTime": false,
"sortOrder": "Descending",
"wrapLogMessage": false
},
"pluginVersion": "11.3.1",
"targets": [
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"expr": "{app=\"forgejo\"}",
"queryType": "range",
"refId": "A"
}
],
"title": "Logs: App forgejo",
"type": "logs"
},
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"fieldConfig": {
"defaults": {
},
"overrides": [
]
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 24
},
"id": 2,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": false,
"showTime": false,
"sortOrder": "Descending",
"wrapLogMessage": false
},
"pluginVersion": "11.3.1",
"targets": [
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"expr": "{app=\"backstage\"}",
"queryType": "range",
"refId": "A"
}
],
"title": "Logs: App backstage",
"type": "logs"
},
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"fieldConfig": {
"defaults": {
},
"overrides": [
]
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 32
},
"id": 1,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": false,
"showTime": false,
"sortOrder": "Descending",
"wrapLogMessage": false
},
"pluginVersion": "11.3.1",
"targets": [
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"expr": "{app=\"shoot-control-plane\"}",
"queryType": "range",
"refId": "A"
}
],
"title": "Logs: App shoot-control-plane",
"type": "logs"
}
],
"preload": false,
"schemaVersion": 40,
"tags": [
],
"templating": {
"list": [
]
},
"time": {
"from": "now-6h",
"to": "now"
},
"timepicker": {
},
"timezone": "browser",
"title": "Loki Logs: Apps",
"uid": "ee4iuluru756of",
"version": 2,
"weekStart": ""
}

845
cihe/factory.c-one-infra.de/stacks/monitoring/kube-prometheus/dashboards/dashboard_loki_components.yaml Normal file
View file

@ -0,0 +1,845 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: grafana-dashboard-2
labels:
grafana_dashboard: "1"
data:
k8s-dashboard-02.json: |
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": 30,
"links": [
],
"panels": [
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"fieldConfig": {
"defaults": {
},
"overrides": [
]
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 0
},
"id": 19,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": false,
"showTime": false,
"sortOrder": "Descending",
"wrapLogMessage": false
},
"pluginVersion": "11.3.1",
"targets": [
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"editorMode": "builder",
"expr": "{component=\"server\"} |= ``",
"queryType": "range",
"refId": "A"
}
],
"title": "Logs: Component server",
"type": "logs"
},
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"fieldConfig": {
"defaults": {
},
"overrides": [
]
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 8
},
"id": 17,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": false,
"showTime": false,
"sortOrder": "Descending",
"wrapLogMessage": false
},
"pluginVersion": "11.3.1",
"targets": [
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"editorMode": "builder",
"expr": "{component=\"repo-server\"} |= ``",
"queryType": "range",
"refId": "A"
}
],
"title": "Logs: Component repo-server",
"type": "logs"
},
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"fieldConfig": {
"defaults": {
},
"overrides": [
]
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 16
},
"id": 16,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": false,
"showTime": false,
"sortOrder": "Descending",
"wrapLogMessage": false
},
"pluginVersion": "11.3.1",
"targets": [
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"editorMode": "builder",
"expr": "{component=\"redis\"} |= ``",
"queryType": "range",
"refId": "A"
}
],
"title": "Logs: Component redis",
"type": "logs"
},
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"fieldConfig": {
"defaults": {
},
"overrides": [
]
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 24
},
"id": 15,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": false,
"showTime": false,
"sortOrder": "Descending",
"wrapLogMessage": false
},
"pluginVersion": "11.3.1",
"targets": [
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"editorMode": "builder",
"expr": "{component=\"query-frontend\"} |= ``",
"queryType": "range",
"refId": "A"
}
],
"title": "Logs: Component query-frontend",
"type": "logs"
},
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"fieldConfig": {
"defaults": {
},
"overrides": [
]
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 32
},
"id": 14,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": false,
"showTime": false,
"sortOrder": "Descending",
"wrapLogMessage": false
},
"pluginVersion": "11.3.1",
"targets": [
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"editorMode": "builder",
"expr": "{component=\"querier\"} |= ``",
"queryType": "range",
"refId": "A"
}
],
"title": "Logs: Component querier",
"type": "logs"
},
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"fieldConfig": {
"defaults": {
},
"overrides": [
]
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 40
},
"id": 13,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": false,
"showTime": false,
"sortOrder": "Descending",
"wrapLogMessage": false
},
"pluginVersion": "11.3.1",
"targets": [
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"editorMode": "builder",
"expr": "{component=\"prometheus-operator-webhook\"} |= ``",
"queryType": "range",
"refId": "A"
}
],
"title": "Logs: Component prometheus-operator-webhook",
"type": "logs"
},
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"fieldConfig": {
"defaults": {
},
"overrides": [
]
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 48
},
"id": 12,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": false,
"showTime": false,
"sortOrder": "Descending",
"wrapLogMessage": false
},
"pluginVersion": "11.3.1",
"targets": [
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"editorMode": "builder",
"expr": "{component=\"prometheus-operator\"} |= ``",
"queryType": "range",
"refId": "A"
}
],
"title": "Logs: Component prometheus-operator",
"type": "logs"
},
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"fieldConfig": {
"defaults": {
},
"overrides": [
]
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 56
},
"id": 11,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": false,
"showTime": false,
"sortOrder": "Descending",
"wrapLogMessage": false
},
"pluginVersion": "11.3.1",
"targets": [
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"editorMode": "builder",
"expr": "{component=\"metrics\"} |= ``",
"queryType": "range",
"refId": "A"
}
],
"title": "Logs: Component metrics",
"type": "logs"
},
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"fieldConfig": {
"defaults": {
},
"overrides": [
]
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 64
},
"id": 10,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": false,
"showTime": false,
"sortOrder": "Descending",
"wrapLogMessage": false
},
"pluginVersion": "11.3.1",
"targets": [
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"editorMode": "builder",
"expr": "{component=\"kube-scheduler\"} |= ``",
"queryType": "range",
"refId": "A"
}
],
"title": "Logs: Component kube-scheduler",
"type": "logs"
},
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"fieldConfig": {
"defaults": {
},
"overrides": [
]
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 72
},
"id": 9,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": false,
"showTime": false,
"sortOrder": "Descending",
"wrapLogMessage": false
},
"pluginVersion": "11.3.1",
"targets": [
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"editorMode": "builder",
"expr": "{component=\"kube-controller-manager\"} |= ``",
"queryType": "range",
"refId": "A"
}
],
"title": "Logs: Component kube-controller-manager",
"type": "logs"
},
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"fieldConfig": {
"defaults": {
},
"overrides": [
]
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 80
},
"id": 8,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": false,
"showTime": false,
"sortOrder": "Descending",
"wrapLogMessage": false
},
"pluginVersion": "11.3.1",
"targets": [
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"editorMode": "builder",
"expr": "{component=\"kube-apiserver\"} |= ``",
"queryType": "range",
"refId": "A"
}
],
"title": "Logs: Component kube-apiserver",
"type": "logs"
},
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"fieldConfig": {
"defaults": {
},
"overrides": [
]
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 88
},
"id": 7,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": false,
"showTime": false,
"sortOrder": "Descending",
"wrapLogMessage": false
},
"pluginVersion": "11.3.1",
"targets": [
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"editorMode": "builder",
"expr": "{component=\"ingester\"} |= ``",
"queryType": "range",
"refId": "A"
}
],
"title": "Logs: Component ingester",
"type": "logs"
},
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"fieldConfig": {
"defaults": {
},
"overrides": [
]
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 96
},
"id": 6,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": false,
"showTime": false,
"sortOrder": "Descending",
"wrapLogMessage": false
},
"pluginVersion": "11.3.1",
"targets": [
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"editorMode": "builder",
"expr": "{component=\"gateway\"} |= ``",
"queryType": "range",
"refId": "A"
}
],
"title": "Logs: Component gateway",
"type": "logs"
},
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"fieldConfig": {
"defaults": {
},
"overrides": [
]
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 104
},
"id": 5,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": false,
"showTime": false,
"sortOrder": "Descending",
"wrapLogMessage": false
},
"pluginVersion": "11.3.1",
"targets": [
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"editorMode": "builder",
"expr": "{component=\"etcd\"} |= ``",
"queryType": "range",
"refId": "A"
}
],
"title": "Logs: Component etcd",
"type": "logs"
},
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"fieldConfig": {
"defaults": {
},
"overrides": [
]
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 112
},
"id": 4,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": false,
"showTime": false,
"sortOrder": "Descending",
"wrapLogMessage": false
},
"pluginVersion": "11.3.1",
"targets": [
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"editorMode": "builder",
"expr": "{component=\"distributor\"} |= ``",
"queryType": "range",
"refId": "A"
}
],
"title": "Logs: Component distributor",
"type": "logs"
},
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"fieldConfig": {
"defaults": {
},
"overrides": [
]
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 120
},
"id": 3,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": false,
"showTime": false,
"sortOrder": "Descending",
"wrapLogMessage": false
},
"pluginVersion": "11.3.1",
"targets": [
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"editorMode": "builder",
"expr": "{component=\"controller\"} |= ``",
"queryType": "range",
"refId": "A"
}
],
"title": "Logs: Component controller",
"type": "logs"
},
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"fieldConfig": {
"defaults": {
},
"overrides": [
]
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 128
},
"id": 2,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": false,
"showTime": false,
"sortOrder": "Descending",
"wrapLogMessage": false
},
"pluginVersion": "11.3.1",
"targets": [
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"editorMode": "builder",
"expr": "{component=\"cloud-infrastructure-controller\"} |= ``",
"queryType": "range",
"refId": "A"
}
],
"title": "Logs: Component cloud-infrastructure-controller",
"type": "logs"
},
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"fieldConfig": {
"defaults": {
},
"overrides": [
]
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 136
},
"id": 1,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": false,
"showTime": false,
"sortOrder": "Descending",
"wrapLogMessage": false
},
"pluginVersion": "11.3.1",
"targets": [
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"editorMode": "builder",
"expr": "{component=\"applicationset-controller\"} |= ``",
"queryType": "range",
"refId": "A"
}
],
"title": "Logs: Component application-controller",
"type": "logs"
}
],
"preload": false,
"schemaVersion": 40,
"tags": [
],
"templating": {
"list": [
]
},
"time": {
"from": "now-6h",
"to": "now"
},
"timepicker": {
},
"timezone": "browser",
"title": "Loki Logs: Components",
"uid": "ae4zuyp1kui9sc",
"version": 2,
"weekStart": ""
}

537
cihe/factory.c-one-infra.de/stacks/monitoring/kube-prometheus/dashboards/dashboard_loki_container.yaml Normal file
View file

@ -0,0 +1,537 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: grafana-dashboard-3
labels:
grafana_dashboard: "1"
data:
k8s-dashboard-03.json: |
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": 31,
"links": [
],
"panels": [
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"fieldConfig": {
"defaults": {
},
"overrides": [
]
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 0
},
"id": 11,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": false,
"showTime": false,
"sortOrder": "Descending",
"wrapLogMessage": false
},
"pluginVersion": "11.3.1",
"targets": [
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"editorMode": "builder",
"expr": "{container=\"repo-server\"} |= ``",
"queryType": "range",
"refId": "A"
}
],
"title": "Logs: Container repo-server",
"type": "logs"
},
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"fieldConfig": {
"defaults": {
},
"overrides": [
]
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 8
},
"id": 10,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": false,
"showTime": false,
"sortOrder": "Descending",
"wrapLogMessage": false
},
"pluginVersion": "11.3.1",
"targets": [
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"editorMode": "builder",
"expr": "{container=\"alloy\"} |= ``",
"queryType": "range",
"refId": "A"
}
],
"title": "Logs: Container alloy",
"type": "logs"
},
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"fieldConfig": {
"defaults": {
},
"overrides": [
]
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 16
},
"id": 9,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": false,
"showTime": false,
"sortOrder": "Descending",
"wrapLogMessage": false
},
"pluginVersion": "11.3.1",
"targets": [
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"editorMode": "builder",
"expr": "{container=\"prometheus\"} |= ``",
"queryType": "range",
"refId": "A"
}
],
"title": "Logs: Container prometheus",
"type": "logs"
},
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"fieldConfig": {
"defaults": {
},
"overrides": [
]
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 24
},
"id": 8,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": false,
"showTime": false,
"sortOrder": "Descending",
"wrapLogMessage": false
},
"pluginVersion": "11.3.1",
"targets": [
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"editorMode": "builder",
"expr": "{container=\"postgres\"} |= ``",
"queryType": "range",
"refId": "A"
}
],
"title": "Logs: Container postgres",
"type": "logs"
},
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"fieldConfig": {
"defaults": {
},
"overrides": [
]
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 32
},
"id": 7,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": false,
"showTime": false,
"sortOrder": "Descending",
"wrapLogMessage": false
},
"pluginVersion": "11.3.1",
"targets": [
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"editorMode": "builder",
"expr": "{container=\"kube-prometheus-stack\"} |= ``",
"queryType": "range",
"refId": "A"
}
],
"title": "Logs: Container kube-prometheus-stack",
"type": "logs"
},
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"fieldConfig": {
"defaults": {
},
"overrides": [
]
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 40
},
"id": 6,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": false,
"showTime": false,
"sortOrder": "Descending",
"wrapLogMessage": false
},
"pluginVersion": "11.3.1",
"targets": [
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"editorMode": "builder",
"expr": "{container=\"keycloak\"} |= ``",
"queryType": "range",
"refId": "A"
}
],
"title": "Logs: Container keycloak",
"type": "logs"
},
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"fieldConfig": {
"defaults": {
},
"overrides": [
]
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 48
},
"id": 5,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": false,
"showTime": false,
"sortOrder": "Descending",
"wrapLogMessage": false
},
"pluginVersion": "11.3.1",
"targets": [
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"editorMode": "builder",
"expr": "{container=\"grafana\"} |= ``",
"queryType": "range",
"refId": "A"
}
],
"title": "Logs: Container grafana",
"type": "logs"
},
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"fieldConfig": {
"defaults": {
},
"overrides": [
]
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 56
},
"id": 4,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": false,
"showTime": false,
"sortOrder": "Descending",
"wrapLogMessage": false
},
"pluginVersion": "11.3.1",
"targets": [
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"editorMode": "builder",
"expr": "{container=\"forgejo\"} |= ``",
"queryType": "range",
"refId": "A"
}
],
"title": "Logs: Container forgejo",
"type": "logs"
},
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"fieldConfig": {
"defaults": {
},
"overrides": [
]
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 64
},
"id": 3,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": false,
"showTime": false,
"sortOrder": "Descending",
"wrapLogMessage": false
},
"pluginVersion": "11.3.1",
"targets": [
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"editorMode": "builder",
"expr": "{container=\"crossplane\"} |= ``",
"queryType": "range",
"refId": "A"
}
],
"title": "Logs: Container crossplane",
"type": "logs"
},
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"fieldConfig": {
"defaults": {
},
"overrides": [
]
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 72
},
"id": 2,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": false,
"showTime": false,
"sortOrder": "Descending",
"wrapLogMessage": false
},
"pluginVersion": "11.3.1",
"targets": [
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"editorMode": "builder",
"expr": "{container=\"backstage\"} |= ``",
"queryType": "range",
"refId": "A"
}
],
"title": "Logs: Container backstage",
"type": "logs"
},
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"fieldConfig": {
"defaults": {
},
"overrides": [
]
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 80
},
"id": 1,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": false,
"showTime": false,
"sortOrder": "Descending",
"wrapLogMessage": false
},
"pluginVersion": "11.3.1",
"targets": [
{
"datasource": {
"type": "loki",
"uid": "P8E80F9AEF21F6940"
},
"editorMode": "builder",
"expr": "{container=\"argo-server\"} |= ``",
"queryType": "range",
"refId": "A"
}
],
"title": "Logs: Container argo-server",
"type": "logs"
}
],
"preload": false,
"schemaVersion": 40,
"tags": [
],
"templating": {
"list": [
]
},
"time": {
"from": "now-6h",
"to": "now"
},
"timepicker": {
},
"timezone": "browser",
"title": "Loki Logs: Container",
"uid": "ee50bcaehmv40e",
"version": 2,
"weekStart": ""
}

1577
cihe/factory.c-one-infra.de/stacks/monitoring/kube-prometheus/dashboards/dashboard_nginx-ingress.yaml Normal file
View file

File diff suppressed because it is too large Load diff

36
cihe/factory.c-one-infra.de/stacks/monitoring/kube-prometheus/manifests/secret-admin-password.yaml Normal file
View file

@ -0,0 +1,36 @@
apiVersion: generators.external-secrets.io/v1alpha1
kind: Password
metadata:
name: grafana-admin-password-generator
namespace: monitoring
spec:
length: 36
digits: 5
symbols: 5
symbolCharacters: "/-+"
noUpper: false
allowRepeat: true
---
apiVersion: external-secrets.io/v1beta1
kind: ExternalSecret
metadata:
name: grafana-admin-password-generator
namespace: monitoring
spec:
refreshInterval: "0"
target:
name: kube-prometheus-stack-grafana-admin-password
template:
engineVersion: v2
data:
admin-user: admin
admin-password: "{{.INITIAL_ADMIN_PASSWORD}}"
dataFrom:
- sourceRef:
generatorRef:
apiVersion: generators.external-secrets.io/v1alpha1
kind: Password
name: grafana-admin-password-generator
rewrite:
- transform:
template: "INITIAL_ADMIN_PASSWORD"

21
cihe/factory.c-one-infra.de/stacks/monitoring/kube-prometheus/manifests/secret-grafana.yaml Normal file
View file

@ -0,0 +1,21 @@
apiVersion: external-secrets.io/v1beta1
kind: ExternalSecret
metadata:
name: auth-generic-oauth-secret
namespace: monitoring
spec:
secretStoreRef:
name: keycloak
kind: ClusterSecretStore
refreshInterval: "0"
target:
name: auth-generic-oauth-secret
template:
engineVersion: v2
data:
client_secret: "{{.GRAFANA_CLIENT_SECRET}}"
data:
- secretKey: GRAFANA_CLIENT_SECRET
remoteRef:
key: keycloak-clients
property: GRAFANA_CLIENT_SECRET

71
cihe/factory.c-one-infra.de/stacks/monitoring/kube-prometheus/values.yaml Normal file
View file

@ -0,0 +1,71 @@
grafana:
namespaceOverride: "monitoring"
admin:
existingSecret: "kube-prometheus-stack-grafana-admin-password"
userKey: admin-user
passwordKey: admin-password
defaultDashboardsTimezone: Europe/Berlin
additionalDataSources:
- name: Loki
type: loki
url: http://loki-loki-distributed-gateway.monitoring:80
# syncPolicy:
# syncOptions:
# - ServerSideApply=true
sidecar:
dashboards:
enabled: true
label: grafana_dashboard
folder: /tmp/dashboards
updateIntervalSeconds: 10
folderAnnotation: grafana_folder
provider:
allowUiUpdates: true
foldersFromFilesStructure: true
grafana.ini:
server:
domain: factory.c-one-infra.de
root_url: "%(protocol)s://%(domain)s/grafana"
serve_from_sub_path: true
auth:
disable_login: true
disable_login_form: true
auth.generic_oauth:
enabled: true
name: Keycloak-OAuth
allow_sign_up: true
use_refresh_token: true
client_id: grafana
client_secret: $__file{/etc/secrets/auth_generic_oauth/client_secret}
scopes: openid email profile offline_access roles
email_attribute_path: email
login_attribute_path: username
name_attribute_path: full_name
auth_url: https://factory.c-one-infra.de/keycloak/realms/cnoe/protocol/openid-connect/auth
token_url: https://factory.c-one-infra.de/keycloak/realms/cnoe/protocol/openid-connect/token
api_url: https://factory.c-one-infra.de/keycloak/realms/cnoe/protocol/openid-connect/userinfo
redirect_uri: http://factory.c-one-infra.de/grafana/login/generic_oauth
role_attribute_path: "contains(groups[*], 'admin') && 'Admin' || contains(groups[*], 'editor') && 'Editor' || 'Viewer'"
extraSecretMounts:
- name: auth-generic-oauth-secret-mount
secretName: auth-generic-oauth-secret
defaultMode: 0440
mountPath: /etc/secrets/auth_generic_oauth
readOnly: true
serviceMonitor:
# If true, a ServiceMonitor CRD is created for a prometheus operator https://github.com/coreos/prometheus-operator
enabled: true
#monitoring nginx
prometheus:
prometheusSpec:
podMonitorSelectorNilUsesHelmValues: false
serviceMonitorSelectorNilUsesHelmValues: false

33
cihe/factory.c-one-infra.de/stacks/monitoring/loki.yaml Normal file
View file

@ -0,0 +1,33 @@
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: loki
namespace: argocd
labels:
env: dev
finalizers:
- resources-finalizer.argocd.argoproj.io
spec:
project: default
syncPolicy:
automated:
selfHeal: true
syncOptions:
- CreateNamespace=true
destination:
name: in-cluster
namespace: monitoring
sources:
- repoURL: https://github.com/grafana/helm-charts
path: charts/loki-distributed
targetRevision: HEAD
helm:
valueFiles:
- $values/cihe/factory.c-one-infra.de/stacks/monitoring/loki/values.yaml
- repoURL: https://forgejo.edf-bootstrap.cx.fg1.ffm.osc.live/DevFW-CICD/stacks-instances
targetRevision: HEAD
ref: values
## consider using the following version, if it works again
#- repoURL: https://github.com/grafana/loki
# path: production/helm/loki

7
cihe/factory.c-one-infra.de/stacks/monitoring/loki/values.yaml Normal file
View file

@ -0,0 +1,7 @@
loki:
commonConfig:
replication_factor: 1
auth_enabled: false
# storageConfig:
# filesystem: null

146
cihe/factory.c-one-infra.de/stacks/ref-implementation/README.md Normal file
View file

@ -0,0 +1,146 @@
# Reference implementation
This example creates a local version of the CNOE reference implementation.
## Prerequisites
Ensure you have the following tools installed on your computer.
**Required**
- [idpbuilder](https://github.com/cnoe-io/idpbuilder/releases/latest): version `0.3.0` or later
- [kubectl](https://kubernetes.io/docs/tasks/tools/#kubectl): version `1.27` or later
- Your computer should have at least 6 GB RAM allocated to Docker. If you are on Docker Desktop, see [this guide](https://docs.docker.com/desktop/settings/mac/).
**Optional**
- AWS credentials: Access Key and secret Key. If you want to create AWS resources in one of examples below.
## Installation
**_NOTE:_**
- If you'd like to run this in your web browser through Codespaces, please follow [the instructions here](./codespaces.md) to install instead.
- _This example assumes that you run the reference implementation with the default port configguration of 8443 for the idpBuilder.
If you happen to configure a different host or port for the idpBuilder, the manifests in the reference example need to be updated
and be configured with the new host and port. you can use the [replace.sh](replace.sh) to change the port as desired prior to applying the manifest as instructed in the command above._
```bash
idpbuilder create --use-path-routing \
--package https://github.com/cnoe-io/stacks//ref-implementation
```
This will take ~6 minutes for everything to come up. To track the progress, you can go to the [ArgoCD UI](https://factory.c-one-infra.de:8443/argocd/applications).
### What was installed?
1. **Argo Workflows** to enable workflow orchestrations.
1. **Backstage** as the UI for software catalog and templating. Source is available [here](https://github.com/cnoe-io/backstage-app).
1. **External Secrets** to generate secrets and coordinate secrets between applications.
1. **Keycloak** as the identity provider for applications.
1. **Spark Operator** to demonstrate an example Spark workload through Backstage.
If you don't want to install a package above, you can remove the ArgoCD Application file corresponding to the package you want to remove.
For example, if you want to remove Spark Operator, you can delete [this file](./spark-operator.yaml).
The only package that cannot be removed this way is Keycloak because other packages rely on it.
#### Accessing UIs
- Argo CD: https://factory.c-one-infra.de:8443/argocd
- Argo Workflows: https://factory.c-one-infra.de:8443/argo-workflows
- Backstage: https://factory.c-one-infra.de:8443/
- Gitea: https://factory.c-one-infra.de:8443/gitea
- Keycloak: https://factory.c-one-infra.de:8443/keycloak/admin/master/console/
# Using it
For this example, we will walk through a few demonstrations. Once applications are ready, go to the [backstage URL](https://factory.c-one-infra.de:8443).
Click on the Sign-In button, you will be asked to log into the Keycloak instance. There are two users set up in this
configuration, and their password can be retrieved with the following command:
```bash
idpbuilder get secrets
```
Use the username **`user1`** and the password value given by `USER_PASSWORD` field to login to the backstage instance.
`user1` is an admin user who has access to everything in the cluster, while `user2` is a regular user with limited access.
Both users use the same password retrieved above.
If you want to create a new user or change existing users:
1. Go to the [Keycloak UI](https://factory.c-one-infra.de:8443/keycloak/admin/master/console/).
Login with the username `cnoe-admin`. Password is the `KEYCLOAK_ADMIN_PASSWORD` field from the command above.
2. Select `cnoe` from the realms drop down menu.
3. Select users tab.
## Basic Deployment
Let's start by deploying a simple application to the cluster through Backstage.
Click on the `Create...` button on the left, then select the `Create a Basic Deployment` template.
![img.png](images/backstage-templates.png)
In the next screen, type `demo` for the name field, then click Review, then Create.
Once steps run, click the Open In Catalog button to go to the entity page.
![img.png](images/basic-template-flow.png)
In the demo entity page, you will notice a ArgoCD overview card associated with this entity.
You can click on the ArgoCD Application name to see more details.
![img.png](images/demo-entity.png)
### What just happened?
1. Backstage created [a git repository](https://factory.c-one-infra.de:8443/gitea/giteaAdmin/demo), then pushed templated contents to it.
2. Backstage created [an ArgoCD Application](https://factory.c-one-infra.de:8443/argocd/applications/argocd/demo?) and pointed it to the git repository.
3. Backstage registered the application as [a component](https://factory.c-one-infra.de:8443/gitea/giteaAdmin/demo/src/branch/main/catalog-info.yaml) in Backstage.
4. ArgoCD deployed the manifests stored in the repo to the cluster.
5. Backstage retrieved application health from ArgoCD API, then displayed it.
![image.png](images/basic-deployment.png)
## Argo Workflows and Spark Operator
In this example, we will deploy a simple Apache Spark job through Argo Workflows.
Click on the `Create...` button on the left, then select the `Basic Argo Workflow with a Spark Job` template.
![img.png](images/backstage-templates-spark.png)
Type `demo2` for the name field, then click create. You will notice that the Backstage templating steps are very similar to the basic example above.
Click on the Open In Catalog button to go to the entity page.
![img.png](images/demo2-entity.png)
Deployment processes are the same as the first example. Instead of deploying a pod, we deployed a workflow to create a Spark job.
In the entity page, there is a card for Argo Workflows, and it should say running or succeeded.
You can click the name in the card to go to the Argo Workflows UI to view more details about this workflow run.
When prompted to log in, click the login button under single sign on. Argo Workflows is configured to use SSO with Keycloak allowing you to login with the same credentials as Backstage login.
Note that Argo Workflows are not usually deployed this way. This is just an example to show you how you can integrate workflows, backstage, and spark.
Back in the entity page, you can view more details about Spark jobs by navigating to the Spark tab.
## Application with cloud resources.
To deploy cloud resources, you can follow any of the instructions below:
- [Cloud resource deployments via Crossplane](../crossplane-integrations/)
- [Cloud resource deployments via Terraform](../terraform-integrations/)
## Notes
- In these examples, we have used the pattern of creating a new repository for every app, then having ArgoCD deploy it.
This is done for convenience and demonstration purposes only. There are alternative actions that you can use.
For example, you can create a PR to an existing repository, create a repository but not deploy them yet, etc.
- If Backstage's pipelining and templating mechanisms is too simple, you can use more advanced workflow engines like Tekton or Argo Workflows.
You can invoke them in Backstage templates, then track progress similar to how it was described above.

29
cihe/factory.c-one-infra.de/stacks/ref-implementation/argo-workflows.yaml Normal file
View file

@ -0,0 +1,29 @@
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: argo-workflows
namespace: argocd
labels:
env: dev
finalizers:
- resources-finalizer.argocd.argoproj.io
spec:
project: default
source:
repoURL: https://forgejo.edf-bootstrap.cx.fg1.ffm.osc.live/DevFW-CICD/stacks-instances
targetRevision: HEAD
path: "cihe/factory.c-one-infra.de/stacks/ref-implementation/argo-workflows/manifests/dev"
destination:
server: "https://kubernetes.default.svc"
namespace: argo
syncPolicy:
syncOptions:
- CreateNamespace=true
automated:
selfHeal: true
retry:
limit: -1
backoff:
duration: 15s
factor: 1
maxDuration: 15s

1352
cihe/factory.c-one-infra.de/stacks/ref-implementation/argo-workflows/manifests/base/install.yaml Normal file
View file

File diff suppressed because it is too large Load diff

2
cihe/factory.c-one-infra.de/stacks/ref-implementation/argo-workflows/manifests/base/kustomization.yaml Normal file
View file

@ -0,0 +1,2 @@
resources:
- install.yaml

20
cihe/factory.c-one-infra.de/stacks/ref-implementation/argo-workflows/manifests/dev/external-secret.yaml Normal file
View file

@ -0,0 +1,20 @@
apiVersion: external-secrets.io/v1beta1
kind: ExternalSecret
metadata:
name: keycloak-oidc
namespace: argo
spec:
secretStoreRef:
name: keycloak
kind: ClusterSecretStore
target:
name: keycloak-oidc
data:
- secretKey: client-id
remoteRef:
key: keycloak-clients
property: ARGO_WORKFLOWS_CLIENT_ID
- secretKey: secret-key
remoteRef:
key: keycloak-clients
property: ARGO_WORKFLOWS_CLIENT_SECRET

7
cihe/factory.c-one-infra.de/stacks/ref-implementation/argo-workflows/manifests/dev/kustomization.yaml Normal file
View file

@ -0,0 +1,7 @@
resources:
- ../base
- external-secret.yaml
- sa-admin.yaml
patches:
- path: patches/cm-argo-workflows.yaml
- path: patches/deployment-argo-server.yaml

26
cihe/factory.c-one-infra.de/stacks/ref-implementation/argo-workflows/manifests/dev/patches/cm-argo-workflows.yaml Normal file
View file

@ -0,0 +1,26 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: workflow-controller-configmap
namespace: argo
data:
config: |
sso:
insecureSkipVerify: true
issuer: https://factory.c-one-infra.de/keycloak/realms/cnoe
clientId:
name: keycloak-oidc
key: client-id
clientSecret:
name: keycloak-oidc
key: secret-key
redirectUrl: https://factory.c-one-infra.de:443/argo-workflows/oauth2/callback
rbac:
enabled: true
scopes:
- openid
- profile
- email
- groups
nodeEvents:
enabled: false

30
cihe/factory.c-one-infra.de/stacks/ref-implementation/argo-workflows/manifests/dev/patches/deployment-argo-server.yaml Normal file
View file

@ -0,0 +1,30 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: argo-server
namespace: argo
annotations:
argocd.argoproj.io/sync-wave: "20"
spec:
template:
spec:
containers:
- name: argo-server
readinessProbe:
httpGet:
path: /
port: 2746
scheme: HTTP
env:
- name: BASE_HREF
value: "/argo-workflows/"
args:
- server
- --configmap=workflow-controller-configmap
- --auth-mode=client
- --auth-mode=sso
- "--secure=false"
- "--loglevel"
- "info"
- "--log-format"
- "text"

32
cihe/factory.c-one-infra.de/stacks/ref-implementation/argo-workflows/manifests/dev/sa-admin.yaml Normal file
View file

@ -0,0 +1,32 @@
# Used by users in the admin group
# TODO Need to tighten up permissions.
apiVersion: v1
kind: ServiceAccount
metadata:
name: admin
namespace: argo
annotations:
workflows.argoproj.io/rbac-rule: "'admin' in groups"
workflows.argoproj.io/rbac-rule-precedence: "10"
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: argo-admin
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: cluster-admin
subjects:
- kind: ServiceAccount
name: admin
namespace: argo
---
apiVersion: v1
kind: Secret
metadata:
name: admin.service-account-token
annotations:
kubernetes.io/service-account.name: admin
namespace: argo
type: kubernetes.io/service-account-token

27
cihe/factory.c-one-infra.de/stacks/ref-implementation/backstage-templates.yaml Normal file
View file

@ -0,0 +1,27 @@
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: backstage-templates
namespace: argocd
labels:
env: dev
finalizers:
- resources-finalizer.argocd.argoproj.io
spec:
project: default
source:
repoURL: https://forgejo.edf-bootstrap.cx.fg1.ffm.osc.live/DevFW-CICD/stacks-instances
targetRevision: HEAD
path: "cihe/factory.c-one-infra.de/stacks/ref-implementation/backstage-templates/entities"
directory:
exclude: "catalog-info.yaml"
destination:
server: "https://kubernetes.default.svc"
namespace: backstage
syncPolicy:
syncOptions:
- CreateNamespace=true
automated:
selfHeal: true
retry:
limit: -1

48
cihe/factory.c-one-infra.de/stacks/ref-implementation/backstage-templates/entities/app-with-bucket/skeleton/catalog-info.yaml Normal file
View file

@ -0,0 +1,48 @@
apiVersion: backstage.io/v1alpha1
kind: Resource
metadata:
name: ${{values.name}}-bucket
description: Stores things
annotations:
argocd/app-name: ${{values.name | dump}}
spec:
type: s3-bucket
owner: guests
---
apiVersion: backstage.io/v1alpha1
kind: Component
metadata:
name: ${{values.name | dump}}
description: This is for testing purposes
annotations:
backstage.io/techdocs-ref: dir:.
backstage.io/kubernetes-label-selector: 'entity-id=${{values.name}}'
backstage.io/kubernetes-namespace: default
argocd/app-name: ${{values.name | dump}}
links:
- url: https://gitea.factory.c-one-infra.de:443
title: Repo URL
icon: github
spec:
owner: guests
lifecycle: experimental
type: service
system: ${{values.name | dump}}
dependsOn:
- resource:default/${{values.name}}-bucket
---
apiVersion: backstage.io/v1alpha1
kind: System
metadata:
name: ${{values.name | dump}}
description: An example system for demonstration purposes
annotations:
backstage.io/techdocs-ref: dir:.
links:
- url: https://github.com/cnoe-io/stacks/tree/main/ref-implementation
title: CNOE Repo
icon: github
spec:
owner: guests
lifecycle: experimental
type: service

46
cihe/factory.c-one-infra.de/stacks/ref-implementation/backstage-templates/entities/app-with-bucket/skeleton/docs/idpbuilder.md Normal file
View file

@ -0,0 +1,46 @@
[![Codespell][codespell-badge]][codespell-link]
[![E2E][e2e-badge]][e2e-link]
[![Go Report Card][report-badge]][report-link]
[![Commit Activity][commit-activity-badge]][commit-activity-link]
# IDP Builder
Internal development platform binary launcher.
> **WORK IN PROGRESS**: This tool is in a pre-release stage and is under active development.
## About
Spin up a complete internal developer platform using industry standard technologies like Kubernetes, Argo, and backstage with only Docker required as a dependency.
This can be useful in several ways:
* Create a single binary which can demonstrate an IDP reference implementation.
* Use within CI to perform integration testing.
* Use as a local development environment for platform engineers.
## Getting Started
Checkout our [documentation website](https://cnoe.io/docs/reference-implementation/installations/idpbuilder) for getting started with idpbuilder.
## Community
- If you have questions or concerns about this tool, please feel free to reach out to us on the [CNCF Slack Channel](https://cloud-native.slack.com/archives/C05TN9WFN5S).
- You can also join our community meetings to meet the team and ask any questions. Checkout [this calendar](https://calendar.google.com/calendar/embed?src=064a2adfce866ccb02e61663a09f99147f22f06374e7a8994066bdc81e066986%40group.calendar.google.com&ctz=America%2FLos_Angeles) for more information.
## Contribution
Checkout the [contribution doc](./CONTRIBUTING.md) for contribution guidelines and more information on how to set up your local environment.
<!-- JUST BADGES & LINKS -->
[codespell-badge]: https://github.com/cnoe-io/idpbuilder/actions/workflows/codespell.yaml/badge.svg
[codespell-link]: https://github.com/cnoe-io/idpbuilder/actions/workflows/codespell.yaml
[e2e-badge]: https://github.com/cnoe-io/idpbuilder/actions/workflows/e2e.yaml/badge.svg
[e2e-link]: https://github.com/cnoe-io/idpbuilder/actions/workflows/e2e.yaml
[report-badge]: https://goreportcard.com/badge/github.com/cnoe-io/idpbuilder
[report-link]: https://goreportcard.com/report/github.com/cnoe-io/idpbuilder
[commit-activity-badge]: https://img.shields.io/github/commit-activity/m/cnoe-io/idpbuilder
[commit-activity-link]: https://github.com/cnoe-io/idpbuilder/pulse

BIN
cihe/factory.c-one-infra.de/stacks/ref-implementation/backstage-templates/entities/app-with-bucket/skeleton/docs/images/cnoe-logo.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 57 KiB

16
cihe/factory.c-one-infra.de/stacks/ref-implementation/backstage-templates/entities/app-with-bucket/skeleton/docs/index.md Normal file
View file

@ -0,0 +1,16 @@
![cnoe logo](./images/cnoe-logo.png)
# Example Basic Application
Thanks for trying out this demo! In this example, we deployed a simple application with a S3 bucket using Crossplane.
### idpbuilder
Checkout the idpbuilder website: https://cnoe.io/docs/reference-implementation/installations/idpbuilder
Checkout the idpbuilder repository: https://github.com/cnoe-io/idpbuilder
## Crossplane
Checkout the Crossplane website: https://www.crossplane.io/

3
cihe/factory.c-one-infra.de/stacks/ref-implementation/backstage-templates/entities/app-with-bucket/skeleton/go.mod Normal file
View file

@ -0,0 +1,3 @@
module ${{ values.name }}
go 1.19

3
cihe/factory.c-one-infra.de/stacks/ref-implementation/backstage-templates/entities/app-with-bucket/skeleton/kustomize/base/kustomization.yaml Normal file
View file

@ -0,0 +1,3 @@
resources:
- nginx.yaml
- ${{ values.name }}.yaml

34
cihe/factory.c-one-infra.de/stacks/ref-implementation/backstage-templates/entities/app-with-bucket/skeleton/kustomize/base/nginx.yaml Normal file
View file

@ -0,0 +1,34 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: nginx
labels:
app: nginx
spec:
replicas: 3
selector:
matchLabels:
app: nginx
template:
metadata:
labels:
app: nginx
spec:
containers:
- name: nginx
image: nginx:1.14.2
ports:
- containerPort: 80
---
apiVersion: v1
kind: Service
metadata:
name: nginx
labels:
app: nginx
spec:
ports:
- port: 80
targetPort: 80
selector:
app: nginx

2
cihe/factory.c-one-infra.de/stacks/ref-implementation/backstage-templates/entities/app-with-bucket/skeleton/kustomize/dev/kustomization.yaml Normal file
View file

@ -0,0 +1,2 @@
resources:
- ../base

35
cihe/factory.c-one-infra.de/stacks/ref-implementation/backstage-templates/entities/app-with-bucket/skeleton/kustomize/prod/kustomization.yaml Normal file
View file

@ -0,0 +1,35 @@
{%- if values.awsResources %}
resources:
{%- if 'Bucket' in values.awsResources %}
- ../base/
{%- endif %}
{%- if 'Table' in values.awsResources %}
- ../base/table.yaml
{%- endif %}
{%- endif %}
namespace: default
patches:
- target:
kind: Deployment
patch: |
apiVersion: apps/v1
kind: Deployment
metadata:
name: not-used
labels:
backstage.io/kubernetes-id: ${{values.name}}
spec:
template:
metadata:
labels:
backstage.io/kubernetes-id: ${{values.name}}
- target:
kind: Service
patch: |
apiVersion: apps/v1
kind: Service
metadata:
name: not-used
labels:
backstage.io/kubernetes-id: ${{values.name}}

5
cihe/factory.c-one-infra.de/stacks/ref-implementation/backstage-templates/entities/app-with-bucket/skeleton/main.go Normal file
View file

@ -0,0 +1,5 @@
package main
func main() {
}

6
cihe/factory.c-one-infra.de/stacks/ref-implementation/backstage-templates/entities/app-with-bucket/skeleton/mkdocs.yml Normal file
View file

@ -0,0 +1,6 @@
site_name: 'Argo Spark Example'
nav:
- Home: index.md
- idpBuilder: idpbuilder.md
plugins:
- techdocs-core

126
cihe/factory.c-one-infra.de/stacks/ref-implementation/backstage-templates/entities/app-with-bucket/template.yaml Normal file
View file

@ -0,0 +1,126 @@
apiVersion: scaffolder.backstage.io/v1beta3
kind: Template
metadata:
description: Adds a Go application with AWS resources
name: app-with-aws-resources
title: Add a Go App with AWS resources
spec:
owner: guests
type: service
parameters:
- properties:
name:
title: Application Name
type: string
description: Unique name of the component
ui:autofocus: true
labels:
title: Labels
type: object
additionalProperties:
type: string
description: Labels to apply to the application
ui:autofocus: true
required:
- name
title: Choose your repository location
- description: Configure your bucket
properties:
apiVersion:
default: awsblueprints.io/v1alpha1
description: APIVersion for the resource
type: string
kind:
default: ObjectStorage
description: Kind for the resource
type: string
config:
description: ObjectStorageSpec defines the desired state of ObjectStorage
properties:
resourceConfig:
description: ResourceConfig defines general properties of this AWS resource.
properties:
deletionPolicy:
description: Defaults to Delete
enum:
- Delete
- Orphan
type: string
region:
type: string
providerConfigName:
type: string
default: default
tags:
items:
properties:
key:
type: string
value:
type: string
required:
- key
- value
type: object
type: array
required:
- region
type: object
required:
- resourceConfig
title: Bucket configuration options
type: object
steps:
- id: template
name: Generating component
action: fetch:template
input:
url: ./skeleton
values:
name: ${{parameters.name}}
- action: roadiehq:utils:serialize:yaml
id: serialize
input:
data:
apiVersion: awsblueprints.io/v1alpha1
kind: ${{ parameters.kind }}
metadata:
name: ${{ parameters.name }}
spec: ${{ parameters.config }}
name: serialize
- action: roadiehq:utils:fs:write
id: write
input:
content: ${{ steps['serialize'].output.serialized }}
path: kustomize/base/${{ parameters.name }}.yaml
name: write-to-file
- id: publish
name: Publishing to a gitea git repository
action: publish:gitea
input:
description: This is an example app
# Hard coded value for this demo purposes only.
repoUrl: gitea.factory.c-one-infra.de:443/?repo=${{parameters.name}}
defaultBranch: main
- id: create-argocd-app
name: Create ArgoCD App
action: cnoe:create-argocd-app
input:
appName: ${{parameters.name}}
appNamespace: default
argoInstance: in-cluster
projectName: default
# necessary until we generate our own cert
repoUrl: https://gitea.factory.c-one-infra.de:443/giteaAdmin/${{parameters.name}}
path: "kustomize/base"
- id: register
name: Register
action: catalog:register
input:
repoContentsUrl: ${{ steps['publish'].output.repoContentsUrl }}
catalogInfoPath: 'catalog-info.yaml'
output:
links:
- title: Open in catalog
icon: catalog
entityRef: ${{ steps['register'].output.entityRef }}

40
cihe/factory.c-one-infra.de/stacks/ref-implementation/backstage-templates/entities/argo-workflows/skeleton/catalog-info.yaml Normal file
View file

@ -0,0 +1,40 @@
---
apiVersion: backstage.io/v1alpha1
kind: Component
metadata:
name: ${{values.name | dump}}
description: This is an example Backstage component representing the use of Argo Workflows and Spark Operator.
annotations:
backstage.io/techdocs-ref: dir:.
backstage.io/kubernetes-label-selector: 'entity-id=${{values.name}}'
backstage.io/kubernetes-namespace: argo
argocd/app-name: ${{values.name | dump}}
argo-workflows.cnoe.io/label-selector: env=dev,entity-id=${{values.name}}
argo-workflows.cnoe.io/cluster-name: local
apache-spark.cnoe.io/label-selector: env=dev,entity-id=${{values.name}}
apache-spark.cnoe.io/cluster-name: local
links:
- url: https://gitea.factory.c-one-infra.de:443
title: Repo URL
icon: github
spec:
owner: guests
lifecycle: experimental
type: service
system: ${{values.name | dump}}
---
apiVersion: backstage.io/v1alpha1
kind: System
metadata:
name: ${{values.name | dump}}
description: An example system for demonstration purposes
annotations:
backstage.io/techdocs-ref: dir:.
links:
- url: https://github.com/cnoe-io/stacks/tree/main/ref-implementation
title: CNOE Repo
icon: github
spec:
owner: guests
lifecycle: experimental
type: service

160
cihe/factory.c-one-infra.de/stacks/ref-implementation/backstage-templates/entities/argo-workflows/skeleton/docs/argo-workflows.md Normal file
View file

@ -0,0 +1,160 @@
<!-- markdownlint-disable-next-line MD041 -->
[![Security Status](https://github.com/argoproj/argo-workflows/actions/workflows/snyk.yml/badge.svg?branch=main)](https://github.com/argoproj/argo-workflows/actions/workflows/snyk.yml?query=branch%3Amain)
[![OpenSSF Best Practices](https://bestpractices.coreinfrastructure.org/projects/3830/badge)](https://bestpractices.coreinfrastructure.org/projects/3830)
[![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/argoproj/argo-workflows/badge)](https://api.securityscorecards.dev/projects/github.com/argoproj/argo-workflows)
[![FOSSA License Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2Fargoproj%2Fargo-workflows.svg?type=shield)](https://app.fossa.com/projects/git%2Bgithub.com%2Fargoproj%2Fargo-workflows?ref=badge_shield)
[![Slack](https://img.shields.io/badge/slack-argoproj-brightgreen.svg?logo=slack)](https://argoproj.github.io/community/join-slack)
[![Twitter Follow](https://img.shields.io/twitter/follow/argoproj?style=social)](https://twitter.com/argoproj)
[![LinkedIn](https://img.shields.io/badge/LinkedIn-argoproj-blue.svg?logo=linkedin)](https://www.linkedin.com/company/argoproj/)
[![Release Version](https://img.shields.io/github/v/release/argoproj/argo-workflows?label=argo-workflows)](https://github.com/argoproj/argo-workflows/releases/latest)
[![Artifact HUB](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/argo-workflows)](https://artifacthub.io/packages/helm/argo/argo-workflows)
## What is Argo Workflows?
Argo Workflows is an open source container-native workflow engine for orchestrating parallel jobs on Kubernetes.
Argo Workflows is implemented as a Kubernetes CRD (Custom Resource Definition).
* Define workflows where each step is a container.
* Model multi-step workflows as a sequence of tasks or capture the dependencies between tasks using a directed acyclic graph (DAG).
* Easily run compute intensive jobs for machine learning or data processing in a fraction of the time using Argo Workflows on Kubernetes.
Argo is a [Cloud Native Computing Foundation (CNCF)](https://cncf.io/) graduated project.
## Use Cases
* [Machine Learning pipelines](use-cases/machine-learning.md)
* [Data and batch processing](use-cases/data-processing.md)
* [Infrastructure automation](use-cases/infrastructure-automation.md)
* [CI/CD](use-cases/ci-cd.md)
* [Other use cases](use-cases/other.md)
## Why Argo Workflows?
* Argo Workflows is the most popular workflow execution engine for Kubernetes.
* Light-weight, scalable, and easier to use.
* Designed from the ground up for containers without the overhead and limitations of legacy VM and server-based environments.
* Cloud agnostic and can run on any Kubernetes cluster.
[Read what people said in our latest survey](https://blog.argoproj.io/argo-workflows-events-2023-user-survey-results-82c53bc30543)
## Try Argo Workflows
You can try Argo Workflows via one of the following:
1. [Interactive Training Material](https://killercoda.com/argoproj/course/argo-workflows/)
1. [Access the demo environment](https://workflows.apps.argoproj.io/workflows/argo)
![Screenshot](assets/screenshot.png)
## Who uses Argo Workflows?
[About 200+ organizations are officially using Argo Workflows](https://github.com/argoproj/argo-workflows/blob/main/USERS.md)
## Ecosystem
Just some of the projects that use or rely on Argo Workflows (complete list [here](https://github.com/akuity/awesome-argo#ecosystem-projects)):
* [Argo Events](https://github.com/argoproj/argo-events)
* [Couler](https://github.com/couler-proj/couler)
* [Hera](https://github.com/argoproj-labs/hera-workflows)
* [Katib](https://github.com/kubeflow/katib)
* [Kedro](https://kedro.readthedocs.io/en/stable/)
* [Kubeflow Pipelines](https://github.com/kubeflow/pipelines)
* [Netflix Metaflow](https://metaflow.org)
* [Onepanel](https://github.com/onepanelio/onepanel)
* [Orchest](https://github.com/orchest/orchest/)
* [Piper](https://github.com/quickube/piper)
* [Ploomber](https://github.com/ploomber/ploomber)
* [Seldon](https://github.com/SeldonIO/seldon-core)
* [SQLFlow](https://github.com/sql-machine-learning/sqlflow)
## Client Libraries
Check out our [Java, Golang and Python clients](client-libraries.md).
## Quickstart
* [Get started here](quick-start.md)
* [Walk-through examples](walk-through/index.md)
## Documentation
You're here!
## Features
An incomplete list of features Argo Workflows provide:
* UI to visualize and manage Workflows
* Artifact support (S3, Artifactory, Alibaba Cloud OSS, Azure Blob Storage, HTTP, Git, GCS, raw)
* Workflow templating to store commonly used Workflows in the cluster
* Archiving Workflows after executing for later access
* Scheduled workflows using cron
* Server interface with REST API (HTTP and GRPC)
* DAG or Steps based declaration of workflows
* Step level input & outputs (artifacts/parameters)
* Loops
* Parameterization
* Conditionals
* Timeouts (step & workflow level)
* Retry (step & workflow level)
* Resubmit (memoized)
* Suspend & Resume
* Cancellation
* K8s resource orchestration
* Exit Hooks (notifications, cleanup)
* Garbage collection of completed workflow
* Scheduling (affinity/tolerations/node selectors)
* Volumes (ephemeral/existing)
* Parallelism limits
* Daemoned steps
* DinD (docker-in-docker)
* Script steps
* Event emission
* Prometheus metrics
* Multiple executors
* Multiple pod and workflow garbage collection strategies
* Automatically calculated resource usage per step
* Java/Golang/Python SDKs
* Pod Disruption Budget support
* Single-sign on (OAuth2/OIDC)
* Webhook triggering
* CLI
* Out-of-the box and custom Prometheus metrics
* Windows container support
* Embedded widgets
* Multiplex log viewer
## Community Meetings
We host monthly community meetings where we and the community showcase demos and discuss the current and future state of the project. Feel free to join us!
For Community Meeting information, minutes and recordings, please [see here](https://bit.ly/argo-wf-cmty-mtng).
Participation in Argo Workflows is governed by the [CNCF Code of Conduct](https://github.com/cncf/foundation/blob/master/code-of-conduct.md)
## Community Blogs and Presentations
* [Awesome-Argo: A Curated List of Awesome Projects and Resources Related to Argo](https://github.com/terrytangyuan/awesome-argo)
* [Automation of Everything - How To Combine Argo Events, Workflows & Pipelines, CD, and Rollouts](https://youtu.be/XNXJtxkUKeY)
* [Argo Workflows and Pipelines - CI/CD, Machine Learning, and Other Kubernetes Workflows](https://youtu.be/UMaivwrAyTA)
* [Argo Ansible role: Provisioning Argo Workflows on OpenShift](https://medium.com/@marekermk/provisioning-argo-on-openshift-with-ansible-and-kustomize-340a1fda8b50)
* [Argo Workflows vs Apache Airflow](http://bit.ly/30YNIvT)
* [CI/CD with Argo on Kubernetes](https://medium.com/@bouwe.ceunen/ci-cd-with-argo-on-kubernetes-28c1a99616a9)
* [Define Your CI/CD Pipeline with Argo Workflows](https://haque-zubair.medium.com/define-your-ci-cd-pipeline-with-argo-workflows-25aefb02fa63)
* [Distributed Machine Learning Patterns from Manning Publication](https://github.com/terrytangyuan/distributed-ml-patterns)
* [Running Argo Workflows Across Multiple Kubernetes Clusters](https://admiralty.io/blog/running-argo-workflows-across-multiple-kubernetes-clusters/)
* [Open Source Model Management Roundup: Polyaxon, Argo, and Seldon](https://www.anaconda.com/blog/developer-blog/open-source-model-management-roundup-polyaxon-argo-and-seldon/)
* [Producing 200 OpenStreetMap extracts in 35 minutes using a scalable data workflow](https://www.interline.io/blog/scaling-openstreetmap-data-workflows/)
* [Argo integration review](http://dev.matt.hillsdon.net/2018/03/24/argo-integration-review.html)
* TGI Kubernetes with Joe Beda: [Argo workflow system](https://www.youtube.com/watch?v=M_rxPPLG8pU&start=859)
## Project Resources
* [Argo Project GitHub organization](https://github.com/argoproj)
* [Argo Website](https://argoproj.github.io/)
* [Argo Slack](https://argoproj.github.io/community/join-slack)
## Security
See [Security](security.md).

BIN
cihe/factory.c-one-infra.de/stacks/ref-implementation/backstage-templates/entities/argo-workflows/skeleton/docs/images/cnoe-logo.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 57 KiB

9
cihe/factory.c-one-infra.de/stacks/ref-implementation/backstage-templates/entities/argo-workflows/skeleton/docs/index.md Normal file
View file

@ -0,0 +1,9 @@
![cnoe logo](./images/cnoe-logo.png)
# Example Spark Application
Thanks for trying out this demo! In this example, we deployed a simple Apache Spark job through Argo Workflows.
To learn more about Spark Operators, check out [this link](https://github.com/kubeflow/spark-operator)
To learn more about Argo Workflows, see [this link](https://argoproj.github.io/workflows/)

86
cihe/factory.c-one-infra.de/stacks/ref-implementation/backstage-templates/entities/argo-workflows/skeleton/docs/spark-operator.md Normal file
View file

@ -0,0 +1,86 @@
# Kubeflow Spark Operator
[![Go Report Card](https://goreportcard.com/badge/github.com/kubeflow/spark-operator)](https://goreportcard.com/report/github.com/kubeflow/spark-operator)
## What is Spark Operator?
The Kubernetes Operator for Apache Spark aims to make specifying and running [Spark](https://github.com/apache/spark) applications as easy and idiomatic as running other workloads on Kubernetes. It uses
[Kubernetes custom resources](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/) for specifying, running, and surfacing status of Spark applications.
## Overview
For a complete reference of the custom resource definitions, please refer to the [API Definition](docs/api-docs.md). For details on its design, please refer to the [Architecture](https://www.kubeflow.org/docs/components/spark-operator/overview/#architecture). It requires Spark 2.3 and above that supports Kubernetes as a native scheduler backend.
The Kubernetes Operator for Apache Spark currently supports the following list of features:
* Supports Spark 2.3 and up.
* Enables declarative application specification and management of applications through custom resources.
* Automatically runs `spark-submit` on behalf of users for each `SparkApplication` eligible for submission.
* Provides native [cron](https://en.wikipedia.org/wiki/Cron) support for running scheduled applications.
* Supports customization of Spark pods beyond what Spark natively is able to do through the mutating admission webhook, e.g., mounting ConfigMaps and volumes, and setting pod affinity/anti-affinity.
* Supports automatic application re-submission for updated `SparkApplication` objects with updated specification.
* Supports automatic application restart with a configurable restart policy.
* Supports automatic retries of failed submissions with optional linear back-off.
* Supports mounting local Hadoop configuration as a Kubernetes ConfigMap automatically via `sparkctl`.
* Supports automatically staging local application dependencies to Google Cloud Storage (GCS) via `sparkctl`.
* Supports collecting and exporting application-level metrics and driver/executor metrics to Prometheus.
## Project Status
**Project status:** *beta*
**Current API version:** *`v1beta2`*
**If you are currently using the `v1beta1` version of the APIs in your manifests, please update them to use the `v1beta2` version by changing `apiVersion: "sparkoperator.k8s.io/<version>"` to `apiVersion: "sparkoperator.k8s.io/v1beta2"`. You will also need to delete the `previous` version of the CustomResourceDefinitions named `sparkapplications.sparkoperator.k8s.io` and `scheduledsparkapplications.sparkoperator.k8s.io`, and replace them with the `v1beta2` version either by installing the latest version of the operator or by running `kubectl create -f config/crd/bases`.**
## Prerequisites
* Version >= 1.13 of Kubernetes to use the [`subresource` support for CustomResourceDefinitions](https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/#subresources), which became beta in 1.13 and is enabled by default in 1.13 and higher.
* Version >= 1.16 of Kubernetes to use the `MutatingWebhook` and `ValidatingWebhook` of `apiVersion: admissionregistration.k8s.io/v1`.
## Getting Started
For getting started with Spark operator, please refer to [Getting Started](https://www.kubeflow.org/docs/components/spark-operator/getting-started/).
## User Guide
For detailed user guide and API documentation, please refer to [User Guide](https://www.kubeflow.org/docs/components/spark-operator/user-guide/) and [API Specification](docs/api-docs.md).
If you are running Spark operator on Google Kubernetes Engine (GKE) and want to use Google Cloud Storage (GCS) and/or BigQuery for reading/writing data, also refer to the [GCP guide](https://www.kubeflow.org/docs/components/spark-operator/user-guide/gcp/).
## Version Matrix
The following table lists the most recent few versions of the operator.
| Operator Version | API Version | Kubernetes Version | Base Spark Version |
| ------------- | ------------- | ------------- | ------------- |
| `v1beta2-1.6.x-3.5.0` | `v1beta2` | 1.16+ | `3.5.0` |
| `v1beta2-1.5.x-3.5.0` | `v1beta2` | 1.16+ | `3.5.0` |
| `v1beta2-1.4.x-3.5.0` | `v1beta2` | 1.16+ | `3.5.0` |
| `v1beta2-1.3.x-3.1.1` | `v1beta2` | 1.16+ | `3.1.1` |
| `v1beta2-1.2.3-3.1.1` | `v1beta2` | 1.13+ | `3.1.1` |
| `v1beta2-1.2.2-3.0.0` | `v1beta2` | 1.13+ | `3.0.0` |
| `v1beta2-1.2.1-3.0.0` | `v1beta2` | 1.13+ | `3.0.0` |
| `v1beta2-1.2.0-3.0.0` | `v1beta2` | 1.13+ | `3.0.0` |
| `v1beta2-1.1.x-2.4.5` | `v1beta2` | 1.13+ | `2.4.5` |
| `v1beta2-1.0.x-2.4.4` | `v1beta2` | 1.13+ | `2.4.4` |
## Developer Guide
For developing with Spark Operator, please refer to [Developer Guide](https://www.kubeflow.org/docs/components/spark-operator/developer-guide/).
## Contributor Guide
For contributing to Spark Operator, please refer to [Contributor Guide](CONTRIBUTING.md).
## Community
* Join the [CNCF Slack Channel](https://www.kubeflow.org/docs/about/community/#kubeflow-slack-channels) and then join `#kubeflow-spark-operator` Channel.
* Check out our blog post [Announcing the Kubeflow Spark Operator: Building a Stronger Spark on Kubernetes Community](https://blog.kubeflow.org/operators/2024/04/15/kubeflow-spark-operator.html).
* Join our monthly community meeting [Kubeflow Spark Operator Meeting Notes](https://bit.ly/3VGzP4n).
## Adopters
Check out [adopters of Spark Operator](ADOPTERS.md).

109
cihe/factory.c-one-infra.de/stacks/ref-implementation/backstage-templates/entities/argo-workflows/skeleton/manifests/deployment.yaml Normal file
View file

@ -0,0 +1,109 @@
# apiVersion: argoproj.io/v1alpha1
# kind: Workflow
# metadata:
# name: ${{values.name}}
# namespace: argo
# labels:
# env: dev
# entity-id: ${{values.name}}
# spec:
# serviceAccountName: admin
# entrypoint: whalesay
# templates:
# - name: whalesay
# container:
# image: docker/whalesay:latest
# command: [cowsay]
# args: ["hello world"]
---
apiVersion: argoproj.io/v1alpha1
kind: Workflow
metadata:
name: ${{values.name}}
namespace: argo
labels:
env: dev
entity-id: ${{values.name}}
spec:
serviceAccountName: admin
entrypoint: main
action: create
templates:
- name: main
steps:
- - name: spark-job
template: spark-job
- - name: wait
template: wait
arguments:
parameters:
- name: spark-job-name
value: '{{steps.spark-job.outputs.parameters.spark-job-name}}'
- name: wait
inputs:
parameters:
- name: spark-job-name
resource:
action: get
successCondition: status.applicationState.state == COMPLETED
failureCondition: status.applicationState.state == FAILED
manifest: |
apiVersion: "sparkoperator.k8s.io/v1beta2"
kind: SparkApplication
metadata:
name: {{inputs.parameters.spark-job-name}}
namespace: argo
- name: spark-job
outputs:
parameters:
- name: spark-job-name
valueFrom:
jsonPath: '{.metadata.name}'
resource:
action: create
setOwnerReference: true
manifest: |
apiVersion: "sparkoperator.k8s.io/v1beta2"
kind: SparkApplication
metadata:
name: spark-pi-${{values.name}}
namespace: argo
labels:
env: dev
entity-id: ${{values.name}}
spec:
type: Scala
mode: cluster
image: "docker.io/apache/spark:v3.1.3"
imagePullPolicy: IfNotPresent
mainClass: org.apache.spark.examples.SparkPi
mainApplicationFile: "local:///opt/spark/examples/jars/spark-examples_2.12-3.1.3.jar"
sparkVersion: "3.1.1"
restartPolicy:
type: Never
volumes:
- name: "test-volume"
hostPath:
path: "/tmp"
type: Directory
driver:
cores: 1
coreLimit: "1200m"
memory: "512m"
labels:
version: 3.1.1
serviceAccount: admin
volumeMounts:
- name: "test-volume"
mountPath: "/tmp"
executor:
cores: 1
instances: 1
memory: "512m"
labels:
version: 3.1.1
volumeMounts:
- name: "test-volume"
mountPath: "/tmp"

8
cihe/factory.c-one-infra.de/stacks/ref-implementation/backstage-templates/entities/argo-workflows/skeleton/mkdocs.yml Normal file
View file

@ -0,0 +1,8 @@
site_name: 'Argo Spark Example'
nav:
- Home: index.md
- Argo-Workflows: argo-workflows.md
- Apache Spark Operator: spark-operator.md
plugins:
- techdocs-core

62
cihe/factory.c-one-infra.de/stacks/ref-implementation/backstage-templates/entities/argo-workflows/template.yaml Normal file
View file

@ -0,0 +1,62 @@
apiVersion: scaffolder.backstage.io/v1beta3
kind: Template
metadata:
description: Creates a Basic Kubernetes Deployment
name: argo-workflows-basic
title: Basic Argo Workflow with a Spark Job
spec:
owner: guests
type: service
parameters:
- title: Configuration Options
required:
- name
properties:
name:
type: string
description: name of this application
mainApplicationFile:
type: string
default: 'local:///opt/spark/examples/jars/spark-examples_2.12-3.1.3.jar'
description: Path to the main application file
steps:
- id: template
name: Generating component
action: fetch:template
input:
url: ./skeleton
values:
name: ${{parameters.name}}
- id: publish
name: Publishing to a gitea git repository
action: publish:gitea
input:
description: This is an example app
# Hard coded value for this demo purposes only.
repoUrl: gitea.factory.c-one-infra.de:443/?repo=${{parameters.name}}
defaultBranch: main
- id: create-argocd-app
name: Create ArgoCD App
action: cnoe:create-argocd-app
input:
appName: ${{parameters.name}}
appNamespace: ${{parameters.name}}
argoInstance: in-cluster
projectName: default
# necessary until we generate our own cert
repoUrl: https://gitea.factory.c-one-infra.de:443/giteaAdmin/${{parameters.name}}
path: "manifests"
- id: register
name: Register
action: catalog:register
input:
repoContentsUrl: ${{ steps['publish'].output.repoContentsUrl }}
catalogInfoPath: 'catalog-info.yaml'
output:
links:
- title: Open in catalog
icon: catalog
entityRef: ${{ steps['register'].output.entityRef }}

6
cihe/factory.c-one-infra.de/stacks/ref-implementation/backstage-templates/entities/basic/mkdocs.yml Normal file
View file

@ -0,0 +1,6 @@
site_name: 'Argo Spark Example'
nav:
- Home: index.md
- idpBuilder: idpbuilder.md
plugins:
- techdocs-core

36
cihe/factory.c-one-infra.de/stacks/ref-implementation/backstage-templates/entities/basic/skeleton/catalog-info.yaml Normal file
View file

@ -0,0 +1,36 @@
---
apiVersion: backstage.io/v1alpha1
kind: Component
metadata:
name: ${{values.name | dump}}
description: This is a basic example application
annotations:
backstage.io/techdocs-ref: dir:.
backstage.io/kubernetes-label-selector: 'entity-id=${{values.name}}'
backstage.io/kubernetes-namespace: default
argocd/app-name: ${{values.name | dump}}
links:
- url: https://gitea.factory.c-one-infra.de:443
title: Repo URL
icon: github
spec:
owner: guests
lifecycle: experimental
type: service
system: ${{values.name | dump}}
---
apiVersion: backstage.io/v1alpha1
kind: System
metadata:
name: ${{values.name | dump}}
description: An example system for demonstration purposes
annotations:
backstage.io/techdocs-ref: dir:.
links:
- url: https://github.com/cnoe-io/stacks/tree/main/ref-implementation
title: CNOE Repo
icon: github
spec:
owner: guests
lifecycle: experimental
type: service

46
cihe/factory.c-one-infra.de/stacks/ref-implementation/backstage-templates/entities/basic/skeleton/docs/idpbuilder.md Normal file
View file

@ -0,0 +1,46 @@
[![Codespell][codespell-badge]][codespell-link]
[![E2E][e2e-badge]][e2e-link]
[![Go Report Card][report-badge]][report-link]
[![Commit Activity][commit-activity-badge]][commit-activity-link]
# IDP Builder
Internal development platform binary launcher.
> **WORK IN PROGRESS**: This tool is in a pre-release stage and is under active development.
## About
Spin up a complete internal developer platform using industry standard technologies like Kubernetes, Argo, and backstage with only Docker required as a dependency.
This can be useful in several ways:
* Create a single binary which can demonstrate an IDP reference implementation.
* Use within CI to perform integration testing.
* Use as a local development environment for platform engineers.
## Getting Started
Checkout our [documentation website](https://cnoe.io/docs/reference-implementation/installations/idpbuilder) for getting started with idpbuilder.
## Community
- If you have questions or concerns about this tool, please feel free to reach out to us on the [CNCF Slack Channel](https://cloud-native.slack.com/archives/C05TN9WFN5S).
- You can also join our community meetings to meet the team and ask any questions. Checkout [this calendar](https://calendar.google.com/calendar/embed?src=064a2adfce866ccb02e61663a09f99147f22f06374e7a8994066bdc81e066986%40group.calendar.google.com&ctz=America%2FLos_Angeles) for more information.
## Contribution
Checkout the [contribution doc](./CONTRIBUTING.md) for contribution guidelines and more information on how to set up your local environment.
<!-- JUST BADGES & LINKS -->
[codespell-badge]: https://github.com/cnoe-io/idpbuilder/actions/workflows/codespell.yaml/badge.svg
[codespell-link]: https://github.com/cnoe-io/idpbuilder/actions/workflows/codespell.yaml
[e2e-badge]: https://github.com/cnoe-io/idpbuilder/actions/workflows/e2e.yaml/badge.svg
[e2e-link]: https://github.com/cnoe-io/idpbuilder/actions/workflows/e2e.yaml
[report-badge]: https://goreportcard.com/badge/github.com/cnoe-io/idpbuilder
[report-link]: https://goreportcard.com/report/github.com/cnoe-io/idpbuilder
[commit-activity-badge]: https://img.shields.io/github/commit-activity/m/cnoe-io/idpbuilder
[commit-activity-link]: https://github.com/cnoe-io/idpbuilder/pulse

BIN
cihe/factory.c-one-infra.de/stacks/ref-implementation/backstage-templates/entities/basic/skeleton/docs/images/cnoe-logo.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 57 KiB

11
cihe/factory.c-one-infra.de/stacks/ref-implementation/backstage-templates/entities/basic/skeleton/docs/index.md Normal file
View file

@ -0,0 +1,11 @@
![cnoe logo](./images/cnoe-logo.png)
# Example Basic Application
Thanks for trying out this demo! In this example, we deployed a simple application.
### idpbuilder
Checkout idpbuilder website: https://cnoe.io/docs/reference-implementation/installations/idpbuilder
Checkout idpbuilder repository: https://github.com/cnoe-io/idpbuilder

24
cihe/factory.c-one-infra.de/stacks/ref-implementation/backstage-templates/entities/basic/skeleton/manifests/deployment.yaml Normal file
View file

@ -0,0 +1,24 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: ${{values.name | dump}}
namespace: default
labels:
entity-id: ${{values.name}}
app: nginx
spec:
replicas: 1
selector:
matchLabels:
app: nginx
template:
metadata:
labels:
app: nginx
entity-id: ${{values.name}}
spec:
containers:
- name: nginx
image: nginx:1.14.2
ports:
- containerPort: 80

58
cihe/factory.c-one-infra.de/stacks/ref-implementation/backstage-templates/entities/basic/template.yaml Normal file
View file

@ -0,0 +1,58 @@
apiVersion: scaffolder.backstage.io/v1beta3
kind: Template
metadata:
description: Creates a Basic Kubernetes Deployment
name: basic
title: Create a Basic Deployment
spec:
owner: guests
type: service
parameters:
- title: Configuration Options
required:
- name
properties:
name:
type: string
description: name of this application
steps:
- id: template
name: Generating component
action: fetch:template
input:
url: ./skeleton
values:
name: ${{parameters.name}}
- id: publish
name: Publishing to a gitea git repository
action: publish:gitea
input:
description: This is an example app
# Hard coded value for this demo purposes only.
repoUrl: gitea.factory.c-one-infra.de:443/?repo=${{parameters.name}}
defaultBranch: main
- id: create-argocd-app
name: Create ArgoCD App
action: cnoe:create-argocd-app
input:
appName: ${{parameters.name}}
appNamespace: ${{parameters.name}}
argoInstance: in-cluster
projectName: default
# necessary until we generate our own cert
repoUrl: https://gitea.factory.c-one-infra.de:443/giteaAdmin/${{parameters.name}}
path: "manifests"
- id: register
name: Register
action: catalog:register
input:
repoContentsUrl: ${{ steps['publish'].output.repoContentsUrl }}
catalogInfoPath: 'catalog-info.yaml'
output:
links:
- title: Open in catalog
icon: catalog
entityRef: ${{ steps['register'].output.entityRef }}

21
cihe/factory.c-one-infra.de/stacks/ref-implementation/backstage-templates/entities/catalog-info.yaml Normal file
View file

@ -0,0 +1,21 @@
apiVersion: backstage.io/v1alpha1
kind: Location
metadata:
name: basic-example-templates
description: A collection of example templates
spec:
targets:
- ./basic/template.yaml
- ./argo-workflows/template.yaml
- ./app-with-bucket/template.yaml
- ./demo-go-hello-world/template.yaml
- ./spring-petclinic/template.yaml
---
apiVersion: backstage.io/v1alpha1
kind: Location
metadata:
name: basic-organization
description: Basic organization data
spec:
targets:
- ./organization/guests.yaml

35
cihe/factory.c-one-infra.de/stacks/ref-implementation/backstage-templates/entities/demo-go-hello-world/skeleton/catalog-info.yaml Normal file
View file

@ -0,0 +1,35 @@
---
apiVersion: backstage.io/v1alpha1
kind: Component
metadata:
name: ${{ values.name }}
description: This is a Backstage component created from the custom template that creates Hello-World example
annotations:
backstage.io/techdocs-ref: dir:.
backstage.io/kubernetes-label-selector: 'entity-id=${{ values.name }}'
backstage.io/kubernetes-namespace: gitea
links:
- url: https://gitea.factory.c-one-infra.de:443
title: Repo URL
icon: git
spec:
owner: guests
lifecycle: experimental
type: service
system: ${{ values.name | dump }}
---
apiVersion: backstage.io/v1alpha1
kind: System
metadata:
name: ${{ values.name | dump }}
description: A system for managing services created from the Gitea template.
annotations:
backstage.io/techdocs-ref: dir:.
links:
- url: https://gitea.factory.c-one-infra.de:443
title: Gitea Repo
icon: git
spec:
owner: guests
lifecycle: experimental
type: service

20
cihe/factory.c-one-infra.de/stacks/ref-implementation/backstage-templates/entities/demo-go-hello-world/skeleton/main.go Normal file
View file

@ -0,0 +1,20 @@
package main
import (
"fmt"
"net/http"
)
func helloHandler(w http.ResponseWriter, r *http.Request) {
fmt.Fprintln(w, "Hello World")
}
func main() {
http.HandleFunc("/", helloHandler)
fmt.Println("Server is running on port 8081...")
err := http.ListenAndServe(":8081", nil)
if err != nil {
fmt.Println("Error starting the server:", err)
}
}

54
cihe/factory.c-one-infra.de/stacks/ref-implementation/backstage-templates/entities/demo-go-hello-world/template.yaml Normal file
View file

@ -0,0 +1,54 @@
apiVersion: scaffolder.backstage.io/v1beta3
kind: Template
metadata:
name: example-go-template
title: Example Go template
description: An example template for the scaffolder that creates a simple go service
spec:
owner: user:guest
type: service
parameters:
- title: Fill in some steps
required:
- name
properties:
name:
title: Name
type: string
description: Unique name of the component
ui:autofocus: true
steps:
- id: fetch-template
name: Fetch Template
action: fetch:template
input:
url: ./skeleton
targetPath: ./skeleton
values:
name: ${{ parameters.name }}
- id: publish
name: Publish to Gitea
action: publish:gitea
input:
repoUrl: gitea.factory.c-one-infra.de:443/?repo=${{parameters.name}}
description: This is the repository for ${{ parameters.name }}
sourcePath: ./skeleton
defaultBranch: main
- id: register
name: Register in Catalog
action: catalog:register
input:
repoContentsUrl: ${{ steps['publish'].output.repoContentsUrl }}
catalogInfoPath: 'catalog-info.yaml'
output:
links:
- title: Repository
url: ${{ steps['publish'].output.remoteUrl }}
- title: Open in Catalog
icon: catalog
entityRef: ${{ steps['register'].output.entityRef }}

15
cihe/factory.c-one-infra.de/stacks/ref-implementation/backstage-templates/entities/organization/guests.yaml Normal file
View file

@ -0,0 +1,15 @@
---
apiVersion: backstage.io/v1alpha1
kind: User
metadata:
name: guest
spec:
memberOf: [guests]
---
apiVersion: backstage.io/v1alpha1
kind: Group
metadata:
name: guests
spec:
type: team
children: []

94
cihe/factory.c-one-infra.de/stacks/ref-implementation/backstage-templates/entities/spring-petclinic/README.md Normal file
View file

@ -0,0 +1,94 @@
## Spring PetClinic Template
This template can deploy a fork of Spring's PetClinic and showcase the entire
development workflow from version control over build and deployment into
kubernetes to monitoring the state at runtime.
The goal is to demonstrate the migration of an existing GitHub project into the
stack by applying only minimal changes for compatibility. These changes
include:
- Overwriting the CI workflows due to compatibility with forgejo actions and
container issues, thus making builds container less.
- Extending the kubernetes manifests to use Ingress and Service objects
- Integrate Prometheus monitoring endpoint
### Backstage - Forking
The Backstage template allows you to define a URL to 'fork' from. However,
the template does not actually create a git fork but downloads the latest
`HEAD` from a given branch. The source repository has to be known by backstage
so it can apply the correct download mechanism, see `backend.integrations` in
the backstage config.
In the first step, the original source code is downloaded from the given
upstream repository. Subsequently, overrides provided by the template are
copied into the codebase replacing some workflow and deployment files.
This 'merged' codebase is pushed into a new git repository in the supplied
forgejo instance. Additionally, an ArgoCD deployment is created based on this
new given repo.
### Forgejo - CI with Forgejo Actions
As soon as the patched codebase is pushed into the forgejo git repository,
forgejo actions are triggered and start executing the three existing workflows.
However, only the two build workflows are patched to work within the current
stack, the third deployment workflow fails as it cannot start a kind cluster.
In the current configuration, workflows are by default executed in a minimal
node-debian container. This suffices in most cases to run GitHub actions.
Like in the original upstream repository on GitHub, a Gradle- and a Maven-based
workflow are started. The Gradle one only executes a simple java build while
the Maven version also includes the creation of a container image and a
trivy-based security scan.
Both workflows are patched as referencing of actions differs in forgejo actions
from GitHub actions. In GitHub Actions, actions are rerefenced as paths to
github repositories. In Forgejo Actions, this mechanism similarly refers to
actions hosted on `code.forgejo.org` even on self-hosted instances. As only a
small subset of actions is ported to `code.forgejo.org` due to licensing and
compatibility (not all GitHub actions can work with forgejo actions), forgejo
actions also allow referencing actions by URL. Thus, the action
`https://github.com/actions/setup-java` instructs the forgejo runner to
download the action from GitHub. (The default actions repository can be
overwritten)
Creating the application container within the Maven workflow is accomplished
without using 'native' container tooling, i.e. docker or podman. Besides this
being favorable as it introduces less side effects, the current stack
implementation does not support running nested containers, yet.
Furthermore, as the system uses self-signed certificates, certificate checks
are disabled throughout the system for now.
After a successful build, the container image is published into the Forgejo
container registry and can be pulled for deployment.
### ArgoCD - Deployment
The PetClinic consists of two components, a persistent PostgreSQL database and
the java application that is compiled from source. On execution of the
Backstage template an application deployment is created in ArgoCD. The
resources defined in the newly created git repository are synchronized into the
kubernetes cluster. However, as the java app container has to be built by
the CI workflows, the deployment will initially fail but become successful as
soon as the container image is available.
### Prometheus & Grafana - Monitoring
Prometheus and Grafana among others are deployed as apart of the IDP monitoring
stack. To integrate with these existing components the Backstage template adds
a ServiceMonitor definition to the deployment of the PetClinic. It instructs
Prometheus to scrape the `actuator/prometheus` endpoint in specific intervals.
The data contains jvm health data and can be visualized in Grafana.
As the upstream PetClinic on GitHub does not contain the necessary dependencies
to enable the Prometheus endpoint, the app is by default bootstrapped from
a fork that contains the `micrometer-registry-prometheus` dependency.

67
cihe/factory.c-one-infra.de/stacks/ref-implementation/backstage-templates/entities/spring-petclinic/skeleton/.github/workflows/maven-build.yml vendored Normal file
View file

@ -0,0 +1,67 @@
# This workflow will build a Java project with Maven, and cache/restore any dependencies to improve the workflow execution time
# For more information see: https://docs.github.com/en/actions/use-cases-and-examples/building-and-testing/building-and-testing-java-with-maven
name: Java CI with Maven
on:
push:
branches: [ main ]
pull_request:
branches: [ main ]
jobs:
build:
runs-on: ubuntu-latest
#container:
# image: ghcr.io/catthehacker/ubuntu:act-latest # the large image
strategy:
matrix:
java: [ '17' ]
steps:
- uses: actions/checkout@v4
- name: Set up JDK {% raw %}${{matrix.java}}{% endraw %}
uses: https://github.com/actions/setup-java@v4
with:
java-version: '{% raw %}${{matrix.java}}{% endraw %}'
distribution: 'adopt'
cache: maven
- name: Build with Maven Wrapper
run: ./mvnw -B -DskipTests verify
- name: Build image
#run: ./mvnw spring-boot:build-image # the original image build
run: |
export CONTAINER_REPO=$(echo {% raw %}${{ env.GITHUB_REPOSITORY }}{% endraw %} | tr '[:upper:]' '[:lower:]')
./mvnw com.google.cloud.tools:jib-maven-plugin:3.4.4:build -Djib.allowInsecureRegistries=true -Dimage=gitea.factory.c-one-infra.de/${CONTAINER_REPO}:latest -Djib.to.auth.username={% raw %}${{ secrets.PACKAGES_USER }}{% endraw %} -Djib.to.auth.password={% raw %}${{ secrets.PACKAGES_TOKEN }}{% endraw %} -Djib.from.platforms=linux/arm64,linux/amd64
- name: Build image as tar
run: |
./mvnw com.google.cloud.tools:jib-maven-plugin:3.4.4:buildTar -Djib.allowInsecureRegistries=true
# separating the trivy scan into another job is not necessary. It, however, demonstrates forgejo's compatibility with GitHub
- uses: forgejo/upload-artifact@v4
with:
name: petclinic-image
path: target/jib-image.tar
env:
NODE_TLS_REJECT_UNAUTHORIZED: 0 # This is necessary due to self signed certs for forgejo, proper setups can skip this
securityscan:
runs-on: ubuntu-latest
#container:
# image: aquasec/trivy # the container does not contain node...
steps:
- uses: forgejo/download-artifact@v4
with:
name: petclinic-image
env:
NODE_TLS_REJECT_UNAUTHORIZED: 0 # This is necessary due to self signed certs for forgejo, proper setups can skip this
- name: install trivy from deb package
run: |
if [[ "$(uname -m)" == "x86_64" ]]; then
wget -O trivy.deb https://github.com/aquasecurity/trivy/releases/download/v0.58.0/trivy_0.58.0_Linux-64bit.deb
else
wget -O trivy.deb https://github.com/aquasecurity/trivy/releases/download/v0.58.0/trivy_0.58.0_Linux-ARM64.deb
fi
DEBIAN_FRONTEND=noninteractive dpkg -i trivy.deb
- name: scan the image
run: trivy image --input jib-image.tar

36
cihe/factory.c-one-infra.de/stacks/ref-implementation/backstage-templates/entities/spring-petclinic/skeleton/catalog-info.yaml Normal file
View file

@ -0,0 +1,36 @@
---
apiVersion: backstage.io/v1alpha1
kind: Component
metadata:
name: ${{ values.name }}
description: This is a Backstage component created from the custom template that forks the PetClinic
annotations:
backstage.io/techdocs-ref: dir:.
backstage.io/kubernetes-label-selector: 'entity-id=${{ values.name }}'
backstage.io/kubernetes-namespace: gitea
links:
- url: https://gitea.factory.c-one-infra.de:443
title: Repo URL
icon: git
spec:
owner: guests
lifecycle: experimental
type: service
system: ${{ values.name | dump }}
---
apiVersion: backstage.io/v1alpha1
kind: System
metadata:
name: ${{ values.name | dump }}
description: A system for managing services created from the Gitea template.
annotations:
backstage.io/techdocs-ref: dir:.
links:
- url: https://gitea.factory.c-one-infra.de:443
title: Gitea Repo
icon: git
spec:
owner: guests
lifecycle: experimental
type: service

76
cihe/factory.c-one-infra.de/stacks/ref-implementation/backstage-templates/entities/spring-petclinic/skeleton/k8s/db.yml Normal file
View file

@ -0,0 +1,76 @@
---
apiVersion: v1
kind: Secret
metadata:
name: demo-db
namespace: ${{ values.namespace }}
type: servicebinding.io/postgresql
stringData:
type: "postgresql"
provider: "postgresql"
host: "demo-db"
port: "5432"
database: "petclinic"
username: "user"
password: "pass"
---
apiVersion: v1
kind: Service
metadata:
name: demo-db
namespace: ${{ values.namespace }}
spec:
ports:
- port: 5432
selector:
app: demo-db
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: demo-db
namespace: ${{ values.namespace }}
labels:
app: demo-db
spec:
selector:
matchLabels:
app: demo-db
template:
metadata:
labels:
app: demo-db
spec:
containers:
- image: postgres:17
name: postgresql
env:
- name: POSTGRES_USER
valueFrom:
secretKeyRef:
name: demo-db
key: username
- name: POSTGRES_PASSWORD
valueFrom:
secretKeyRef:
name: demo-db
key: password
- name: POSTGRES_DB
valueFrom:
secretKeyRef:
name: demo-db
key: database
ports:
- containerPort: 5432
name: postgresql
livenessProbe:
tcpSocket:
port: postgresql
readinessProbe:
tcpSocket:
port: postgresql
startupProbe:
tcpSocket:
port: postgresql

125
cihe/factory.c-one-infra.de/stacks/ref-implementation/backstage-templates/entities/spring-petclinic/skeleton/k8s/petclinic.yml Normal file
View file

@ -0,0 +1,125 @@
---
apiVersion: v1
kind: Namespace
metadata:
name: ${{ values.namespace }}
---
apiVersion: v1
kind: Service
metadata:
name: petclinic
namespace: ${{ values.namespace }}
labels:
app: petclinic
spec:
type: ClusterIP
ports:
- port: 8080
targetPort: http
name: http
selector:
app: petclinic
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: ${{ values.namespace }}-petclinic
namespace: ${{ values.namespace }}
spec:
ingressClassName: nginx
rules:
- host: ${{ values.namespace }}.factory.c-one-infra.de
http:
paths:
- backend:
service:
name: petclinic
port:
name: http
path: /
pathType: Prefix
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: petclinic
namespace: ${{ values.namespace }}
labels:
app: petclinic
spec:
replicas: 1
selector:
matchLabels:
app: petclinic
template:
metadata:
labels:
app: petclinic
spec:
containers:
- name: workload
image: gitea.factory.c-one-infra.de/giteaadmin/${{ values.name }}
env:
- name: SPRING_PROFILES_ACTIVE
value: postgres
- name: POSTGRES_URL
value: jdbc:postgresql://demo-db/petclinic
- name: POSTGRES_USER
valueFrom:
secretKeyRef:
name: demo-db
key: username
- name: POSTGRES_PASS
valueFrom:
secretKeyRef:
name: demo-db
key: password
- name: SERVICE_BINDING_ROOT
value: /bindings
- name: SPRING_APPLICATION_JSON
value: |
{
"management.endpoint.health.probes.add-additional-paths": true
}
ports:
- name: http
containerPort: 8080
livenessProbe:
httpGet:
path: /livez
port: http
readinessProbe:
httpGet:
path: /readyz
port: http
volumeMounts:
- mountPath: /bindings/secret
name: binding
readOnly: true
volumes:
- name: binding
projected:
sources:
- secret:
name: demo-db
---
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: ${{ values.namespace }}-petclinic-monitor
namespace: monitoring # Namespace where Prometheus is running
labels:
release: kube-prometheus-stack # this must match Prometheus' service monitor matching
spec:
selector:
matchLabels:
app: petclinic # Match your application's labels
namespaceSelector:
matchNames:
- ${{ values.namespace }} # Namespace where your app is running
endpoints:
- port: http # Name of the port in your Service
path: /actuator/prometheus # Path to your metrics endpoint
interval: 15s # How often to scrape metrics

84
cihe/factory.c-one-infra.de/stacks/ref-implementation/backstage-templates/entities/spring-petclinic/template.yaml Normal file
View file

@ -0,0 +1,84 @@
apiVersion: scaffolder.backstage.io/v1beta3
kind: Template
metadata:
name: spring-petclinic
title: Spring PetClinic template
description: An example template for the scaffolder that creates a 'fork' of Spring's PetClinic
spec:
owner: user:guest
type: service
parameters:
- title: Fill in some steps
required:
- name
properties:
name:
title: Project Name
type: string
description: Unique name of the fork app
ui:autofocus: true
upstreamurl:
title: Repo to Fork
type: string
description: The URL of the repo to fork
default: https://forgejo.edf-bootstrap.cx.fg1.ffm.osc.live/Patrick.Sy/ipcei-petclinic/src/branch/main
ui:emptyValue: 'https://forgejo.edf-bootstrap.cx.fg1.ffm.osc.live/Patrick.Sy/ipcei-petclinic/src/branch/main'
steps:
- id: fetch-code # get the latest upstream code
name: Fetch Code
action: fetch:plain
input:
# url: https://github.com/spring-projects/spring-petclinic/tree/main
url: ${{ parameters.upstreamurl }}
- id: fetch-overrides # Apply specific overrides to add features and make modifications for compatibility
name: Fetch Overrides
action: fetch:template
input:
# url: ./skeleton/.github/workflows
# targetPath: ./.github/workflows
url: ./skeleton/
targetPath: ./
replace: true
values:
name: ${{ parameters.name }}
namespace: ${{ parameters.name }}
- id: publish
name: Publish to Gitea
action: publish:gitea
input:
repoUrl: gitea.factory.c-one-infra.de:443/?repo=${{parameters.name}}
description: This is the repository for ${{ parameters.name }}
sourcePath: ./
defaultBranch: main
- id: create-argocd-app
name: Create ArgoCD App
action: cnoe:create-argocd-app
input:
appName: ${{parameters.name}}
appNamespace: ${{parameters.name}}
argoInstance: in-cluster
projectName: default
# necessary until we generate our own cert
repoUrl: https://gitea.factory.c-one-infra.de:443/giteaAdmin/${{parameters.name}}
path: "k8s"
- id: register
name: Register in Catalog
action: catalog:register
input:
repoContentsUrl: ${{ steps['publish'].output.repoContentsUrl }}
catalogInfoPath: 'catalog-info.yaml'
output:
links:
- title: Repository
url: ${{ steps['publish'].output.remoteUrl }}
- title: Open in Catalog
icon: catalog
entityRef: ${{ steps['register'].output.entityRef }}

29
cihe/factory.c-one-infra.de/stacks/ref-implementation/backstage.yaml Normal file
View file

@ -0,0 +1,29 @@
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: backstage
namespace: argocd
labels:
env: dev
finalizers:
- resources-finalizer.argocd.argoproj.io
spec:
project: default
source:
repoURL: https://forgejo.edf-bootstrap.cx.fg1.ffm.osc.live/DevFW-CICD/stacks-instances
targetRevision: HEAD
path: "cihe/factory.c-one-infra.de/stacks/ref-implementation/backstage/manifests"
destination:
server: "https://kubernetes.default.svc"
namespace: backstage
syncPolicy:
syncOptions:
- CreateNamespace=true
automated:
selfHeal: true
retry:
limit: -1
backoff:
duration: 15s
factor: 1
maxDuration: 15s

77
cihe/factory.c-one-infra.de/stacks/ref-implementation/backstage/manifests/argocd-secrets.yaml Normal file
View file

@ -0,0 +1,77 @@
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: eso-store
namespace: argocd
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: eso-store
namespace: argocd
rules:
- apiGroups: [""]
resources:
- secrets
verbs:
- get
- list
- watch
- apiGroups:
- authorization.k8s.io
resources:
- selfsubjectrulesreviews
verbs:
- create
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: eso-store
namespace: argocd
subjects:
- kind: ServiceAccount
name: eso-store
namespace: argocd
roleRef:
kind: Role
name: eso-store
apiGroup: rbac.authorization.k8s.io
---
apiVersion: external-secrets.io/v1beta1
kind: ClusterSecretStore
metadata:
name: argocd
spec:
provider:
kubernetes:
remoteNamespace: argocd
server:
caProvider:
type: ConfigMap
name: kube-root-ca.crt
namespace: argocd
key: ca.crt
auth:
serviceAccount:
name: eso-store
namespace: argocd
---
apiVersion: external-secrets.io/v1beta1
kind: ExternalSecret
metadata:
name: argocd-credentials
namespace: backstage
spec:
secretStoreRef:
name: argocd
kind: ClusterSecretStore
refreshInterval: "0"
target:
name: argocd-credentials
data:
- secretKey: ARGOCD_ADMIN_PASSWORD
remoteRef:
key: argocd-initial-admin-secret
property: password

431
cihe/factory.c-one-infra.de/stacks/ref-implementation/backstage/manifests/install.yaml Normal file
View file

@ -0,0 +1,431 @@
apiVersion: v1
kind: Namespace
metadata:
name: backstage
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: backstage
namespace: backstage
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: backstage-argo-worfklows
rules:
- apiGroups:
- argoproj.io
resources:
- workflows
verbs:
- create
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: read-all
rules:
- apiGroups:
- '*'
resources:
- '*'
verbs:
- get
- list
- watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: backstage-argo-worfklows
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: backstage-argo-worfklows
subjects:
- kind: ServiceAccount
name: backstage
namespace: backstage
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: backstage-read-all
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: read-all
subjects:
- kind: ServiceAccount
name: backstage
namespace: backstage
---
apiVersion: v1
kind: ConfigMap
metadata:
name: backstage-config
namespace: backstage
data:
app-config.yaml: |
app:
title: CNOE Backstage
baseUrl: https://factory.c-one-infra.de:443
organization:
name: CNOE
backend:
# Used for enabling authentication, secret is shared by all backend plugins
# See https://backstage.io/docs/tutorials/backend-to-backend-auth for
# information on the format
# auth:
# keys:
# - secret: ${BACKEND_SECRET}
baseUrl: https://factory.c-one-infra.de:443
listen:
port: 7007
# Uncomment the following host directive to bind to specific interfaces
# host: 127.0.0.1
csp:
connect-src: ["'self'", 'http:', 'https:']
# Content-Security-Policy directives follow the Helmet format: https://helmetjs.github.io/#reference
# Default Helmet Content-Security-Policy values can be removed by setting the key to false
cors:
origin: https://factory.c-one-infra.de:443
methods: [GET, HEAD, PATCH, POST, PUT, DELETE]
credentials: true
database:
client: pg
connection:
host: ${POSTGRES_HOST}
port: ${POSTGRES_PORT}
user: ${POSTGRES_USER}
password: ${POSTGRES_PASSWORD}
cache:
store: memory
# workingDirectory: /tmp # Use this to configure a working directory for the scaffolder, defaults to the OS temp-dir
integrations:
gitea:
- baseUrl: https://gitea.factory.c-one-infra.de:443
host: gitea.factory.c-one-infra.de:443
username: ${GITEA_USERNAME}
password: ${GITEA_PASSWORD}
- baseUrl: https://gitea.factory.c-one-infra.de
host: gitea.factory.c-one-infra.de
username: ${GITEA_USERNAME}
password: ${GITEA_PASSWORD}
- baseUrl: https://forgejo.edf-bootstrap.cx.fg1.ffm.osc.live
host: forgejo.edf-bootstrap.cx.fg1.ffm.osc.live
# github:
# - host: github.com
# apps:
# - $include: github-integration.yaml
# - host: github.com
# # This is a Personal Access Token or PAT from GitHub. You can find out how to generate this token, and more information
# # about setting up the GitHub integration here: https://backstage.io/docs/getting-started/configuration#setting-up-a-github-integration
# token: ${GITHUB_TOKEN}
### Example for how to add your GitHub Enterprise instance using the API:
# - host: ghe.example.net
# apiBaseUrl: https://ghe.example.net/api/v3
# token: ${GHE_TOKEN}
# Reference documentation http://backstage.io/docs/features/techdocs/configuration
# Note: After experimenting with basic setup, use CI/CD to generate docs
# and an external cloud storage when deploying TechDocs for production use-case.
# https://backstage.io/docs/features/techdocs/how-to-guides#how-to-migrate-from-techdocs-basic-to-recommended-deployment-approach
techdocs:
builder: 'local' # Alternatives - 'external'
generator:
runIn: 'local'
publisher:
type: 'local' # Alternatives - 'googleGcs' or 'awsS3'. Read documentation for using alternatives.
auth:
environment: development
session:
secret: MW2sV-sIPngEl26vAzatV-6VqfsgAx4bPIz7PuE_2Lk=
providers:
keycloak-oidc:
development:
metadataUrl: ${KEYCLOAK_NAME_METADATA}
clientId: backstage
clientSecret: ${KEYCLOAK_CLIENT_SECRET}
prompt: auto
scaffolder:
# see https://backstage.io/docs/features/software-templates/configuration for software template options
defaultAuthor:
name: backstage-scaffolder
email: noreply
defaultCommitMessage: "backstage scaffolder"
catalog:
import:
entityFilename: catalog-info.yaml
pullRequestBranchName: backstage-integration
rules:
- allow: [Component, System, API, Resource, Location, Template]
locations:
# Examples from a public GitHub repository.
- type: url
target: https://forgejo.edf-bootstrap.cx.fg1.ffm.osc.live:443/DevFW-CICD/stacks-instances/raw/branch/main/cihe/factory.c-one-infra.de/stacks/ref-implementation/backstage-templates/entities/catalog-info.yaml
rules:
- allow: [Component, System, API, Resource, Location, Template, User, Group]
kubernetes:
serviceLocatorMethod:
type: 'multiTenant'
clusterLocatorMethods:
- $include: k8s-config.yaml
argocd:
username: admin
password: ${ARGOCD_ADMIN_PASSWORD}
appLocatorMethods:
- type: 'config'
instances:
- name: in-cluster
url: https://factory.c-one-infra.de:443/argocd
username: admin
password: ${ARGOCD_ADMIN_PASSWORD}
argoWorkflows:
baseUrl: ${ARGO_WORKFLOWS_URL}
---
apiVersion: v1
kind: Secret
metadata:
name: k8s-config
namespace: backstage
stringData:
k8s-config.yaml: "type: 'config'\nclusters:\n - url: https://kubernetes.default.svc.cluster.local\n
\ name: local\n authProvider: 'serviceAccount'\n skipTLSVerify: true\n
\ skipMetricsLookup: true\n serviceAccountToken: \n $file: /var/run/secrets/kubernetes.io/serviceaccount/token\n
\ caData: \n $file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt\n"
---
apiVersion: v1
kind: Service
metadata:
name: backstage
namespace: backstage
spec:
ports:
- name: http
port: 7007
targetPort: http
selector:
app: backstage
---
apiVersion: v1
kind: Service
metadata:
labels:
app: postgresql
name: postgresql
namespace: backstage
spec:
clusterIP: None
ports:
- name: postgres
port: 5432
selector:
app: postgresql
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: backstage
namespace: backstage
annotations:
argocd.argoproj.io/sync-wave: "20"
spec:
replicas: 1
selector:
matchLabels:
app: backstage
template:
metadata:
labels:
app: backstage
spec:
containers:
- command:
- node
- packages/backend
- --config
- config/app-config.yaml
env:
- name: LOG_LEVEL
value: debug
- name: NODE_TLS_REJECT_UNAUTHORIZED
value: "0"
- name: NODE_OPTIONS
value: "--no-node-snapshot"
envFrom:
- secretRef:
name: backstage-env-vars
- secretRef:
name: gitea-credentials
- secretRef:
name: argocd-credentials
image: forgejo.edf-bootstrap.cx.fg1.ffm.osc.live/devfw-cicd/backstage-edp:development
name: backstage
ports:
- containerPort: 7007
name: http
volumeMounts:
- mountPath: /app/config
name: backstage-config
readOnly: true
serviceAccountName: backstage
volumes:
- name: backstage-config
projected:
sources:
- configMap:
items:
- key: app-config.yaml
path: app-config.yaml
name: backstage-config
- secret:
items:
- key: k8s-config.yaml
path: k8s-config.yaml
name: k8s-config
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
labels:
app: postgresql
name: postgresql
namespace: backstage
annotations:
argocd.argoproj.io/sync-wave: "10"
spec:
replicas: 1
selector:
matchLabels:
app: postgresql
serviceName: service-postgresql
template:
metadata:
labels:
app: postgresql
spec:
containers:
- env:
- name: POSTGRES_DB
valueFrom:
secretKeyRef:
name: backstage-env-vars
key: POSTGRES_DB
- name: POSTGRES_USER
valueFrom:
secretKeyRef:
name: backstage-env-vars
key: POSTGRES_USER
- name: POSTGRES_PASSWORD
valueFrom:
secretKeyRef:
name: backstage-env-vars
key: POSTGRES_PASSWORD
image: docker.io/library/postgres:15.3-alpine3.18
name: postgres
ports:
- containerPort: 5432
name: postgresdb
resources:
limits:
memory: 500Mi
requests:
cpu: 100m
memory: 300Mi
volumeMounts:
- name: data
mountPath: /var/lib/postgresql/data
subPath: postgres
volumeClaimTemplates:
- metadata:
name: data
spec:
accessModes: ["ReadWriteOnce"]
resources:
requests:
storage: "500Mi"
---
apiVersion: generators.external-secrets.io/v1alpha1
kind: Password
metadata:
name: backstage
namespace: backstage
spec:
length: 36
digits: 5
symbols: 5
symbolCharacters: "/-+"
noUpper: false
allowRepeat: true
---
apiVersion: external-secrets.io/v1beta1
kind: ExternalSecret
metadata:
name: backstage-oidc
namespace: backstage
spec:
secretStoreRef:
name: keycloak
kind: ClusterSecretStore
refreshInterval: "0"
target:
name: backstage-env-vars
template:
engineVersion: v2
data:
BACKSTAGE_FRONTEND_URL: https://factory.c-one-infra.de:443/backstage
POSTGRES_HOST: postgresql.backstage.svc.cluster.local
POSTGRES_PORT: '5432'
POSTGRES_DB: backstage
POSTGRES_USER: backstage
POSTGRES_PASSWORD: "{{.POSTGRES_PASSWORD}}"
ARGO_WORKFLOWS_URL: https://factory.c-one-infra.de:443/argo-workflows
KEYCLOAK_NAME_METADATA: https://factory.c-one-infra.de:443/keycloak/realms/cnoe/.well-known/openid-configuration
KEYCLOAK_CLIENT_SECRET: "{{.BACKSTAGE_CLIENT_SECRET}}"
ARGOCD_AUTH_TOKEN: "argocd.token={{.ARGOCD_SESSION_TOKEN}}"
ARGO_CD_URL: 'https://factory.c-one-infra.de/argocd/api/v1/'
data:
- secretKey: ARGOCD_SESSION_TOKEN
remoteRef:
key: keycloak-clients
property: ARGOCD_SESSION_TOKEN
- secretKey: BACKSTAGE_CLIENT_SECRET
remoteRef:
key: keycloak-clients
property: BACKSTAGE_CLIENT_SECRET
dataFrom:
- sourceRef:
generatorRef:
apiVersion: generators.external-secrets.io/v1alpha1
kind: Password
name: backstage
rewrite:
- transform:
template: "POSTGRES_PASSWORD"
---
apiVersion: external-secrets.io/v1beta1
kind: ExternalSecret
metadata:
name: gitea-credentials
namespace: backstage
spec:
secretStoreRef:
name: gitea
kind: ClusterSecretStore
refreshInterval: "0"
target:
name: gitea-credentials
data:
- secretKey: GITEA_USERNAME
remoteRef:
key: gitea-credential
property: username
- secretKey: GITEA_PASSWORD
remoteRef:
key: gitea-credential
property: password

73
cihe/factory.c-one-infra.de/stacks/ref-implementation/codespaces.md Normal file
View file

@ -0,0 +1,73 @@
## Running idpbuilder in Codespaces in Browser
**_NOTE:_**: __Steps described below applies to running this implementation in Codespaces in **web browsers** (e.g. Firefox and Chrome).
If you are using Codespaces with GitHub CLI, steps described here do not apply to you.__
Let's create an instance of Codespaces.
![img.png](images/codespaces-create.png)
It may take a few minutes for it to be ready. Once it's ready, you can either get the latest release of idpbuilder or build from the main branch.
Build the idpbuilder binary.
- Get the latest release:
```bash
version=$(curl -Ls -o /dev/null -w %{url_effective} https://github.com/cnoe-io/idpbuilder/releases/latest)
version=${version##*/}
wget https://github.com/cnoe-io/idpbuilder/releases/download/${version}/idpbuilder-linux-amd64.tar.gz
tar xzf idpbuilder-linux-amd64.tar.gz
sudo mv ./idpbuilder /usr/local/bin/
```
- Alternatively, build from the main branch
```bash
make build
sudo mv ./idpbuilder /usr/local/bin/
```
Codespaces assigns random hostname to your specific instance. You need to make sure they are reflected correctly.
Instance host name is available as an environment variable (`CODESPACE_NAME`). Let's use it to setup our host names.
Run the following commands to update host name and ports. Port is set to 443 because this is the port used by the browser to access your instance.
Clone the [stacks](https://github.com/cnoe-io/stacks) repo.
```bash
cd ref-implementation
./replace.sh ${CODESPACE_NAME}-8080.${GITHUB_CODESPACES_PORT_FORWARDING_DOMAIN} 443
cd -
```
Now you are ready to run idpbuilder with reference implementation.
```bash
idpbuilder create --protocol http \
--host ${CODESPACE_NAME}-8080.${GITHUB_CODESPACES_PORT_FORWARDING_DOMAIN} \
--port 8080 --use-path-routing --package ref-implementation
```
Once idpbuilder finishes bootstrapping, you should have port 8080 forward in the port tab within Codespaces.
![](images/port.png)
You may get a 404 page after clicking the port 8080 forwarded address. This is completely normal because Backstage may not be ready yet.
Give it a few more minutes and it should redirect you to a Backstage page.
### Accessing UIs
If you'd like to track progress of deployment, go to `/argocd` path and login with your ArgoCD credentials.
For example run this command to get the URL for Argo CD:
```bash
echo https://${CODESPACE_NAME}-8080.${GITHUB_CODESPACES_PORT_FORWARDING_DOMAIN}/argocd
```
From here on, you can follow the instructions in the [README](./README.md) file. The only difference is that the URL to access UIs is given by:
```echo
echo https://${CODESPACE_NAME}-8080.${GITHUB_CODESPACES_PORT_FORWARDING_DOMAIN}
```
For example, if you need to access Argo Workflows UI, instead of going to `https://factory.c-one-infra.de:8443/argo`,
you go to `https://${CODESPACE_NAME}-8080.${GITHUB_CODESPACES_PORT_FORWARDING_DOMAIN}/argo`

23
cihe/factory.c-one-infra.de/stacks/ref-implementation/external-secrets.yaml Normal file
View file

@ -0,0 +1,23 @@
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: external-secrets
namespace: argocd
labels:
env: dev
finalizers:
- resources-finalizer.argocd.argoproj.io
spec:
destination:
namespace: external-secrets
server: "https://kubernetes.default.svc"
source:
repoURL: https://forgejo.edf-bootstrap.cx.fg1.ffm.osc.live/DevFW-CICD/stacks-instances
targetRevision: HEAD
path: "cihe/factory.c-one-infra.de/stacks/ref-implementation/external-secrets/manifests"
project: default
syncPolicy:
automated:
selfHeal: true
syncOptions:
- CreateNamespace=true

Some files were not shown because too many files have changed in this diff Show more