From b6588e85b8b2bff989a3861d9b21182454a9ca7a Mon Sep 17 00:00:00 2001 From: Justin Nauman Date: Thu, 16 Aug 2018 00:39:27 -0700 Subject: [PATCH] Chart Grooming (#14) * Chart Grooming - Changed the default `workflow-controller` installation to use the `ServiceAccount` that is created and bound. - Customized the instanceID logic: - No longer defaults to installed (this was very difficult to see/understand when coming from starter tutorials) - Kept logic to allow for release name or explicit mappings but changed structure a bit - Added in optional configuration for: - CRD Install hook's ServiceAccount to allow clean install if your default roles aren't privledged - Optional Pod and Service annotations - Controller logging level configuration - Minio Customizations - Changed the Secret configuration to properly represent the path of a secret instead of the actual contents - Changed the names of the secret and service that are represented to mirror that of the underlying chart * Adding in role bindings for minio secrets if installed --- charts/argo/Chart.yaml | 2 +- charts/argo/README.md | 23 ++++-- .../templates/apply-workflow-crd-job.yaml | 3 +- charts/argo/templates/ui-deployment.yaml | 8 ++ charts/argo/templates/ui-service.yaml | 3 + .../worfkflow-controller-secrets-access.yaml | 18 +++++ .../workflow-controller-config-map.yaml | 12 +-- .../workflow-controller-deployment.yaml | 12 ++- .../workflow-controller-minio-secret-crb.yaml | 25 ++++++ charts/argo/values.yaml | 78 ++++++++++++------- 10 files changed, 143 insertions(+), 41 deletions(-) create mode 100644 charts/argo/templates/worfkflow-controller-secrets-access.yaml create mode 100644 charts/argo/templates/workflow-controller-minio-secret-crb.yaml diff --git a/charts/argo/Chart.yaml b/charts/argo/Chart.yaml index cdbafc0e..0616b08c 100644 --- a/charts/argo/Chart.yaml +++ b/charts/argo/Chart.yaml @@ -1,4 +1,4 @@ apiVersion: v1 description: A Helm chart for Kubernetes name: argo -version: 0.1.1 +version: 0.2.0 diff --git a/charts/argo/README.md b/charts/argo/README.md index 8bd449b6..d9ec2be3 100644 --- a/charts/argo/README.md +++ b/charts/argo/README.md @@ -3,15 +3,26 @@ This chart is used to set up argo and it's needed dependencies through one comma If you want your deployment of this helm chart to most closely match the [argo CLI](https://github.com/argoproj/argo), you should deploy it in the `kube-system` namespace. +## Pre-Requisites +This chart uses an install hook to configure the CRD definition. Installation of CRDs is a somewhat privileged process in itself and in RBAC enabled clusters the `default` service account for namespaces does not typically have the ability to do create these. + +A few options are: +- Setup the CRD yourself manually and use the `--no-hooks` options of `helm install` +- Manually create a ServiceAccount in the Namespace which your release will be deployed w/ appropriate bindings to perform this action and set the `init.serviceAccount` attribute +- Augment the `default` ServiceAccount permissions in the Namespace in which your Release is deployed to have the appropriate permissions + +## Usage Notes: +This chart defaults to setting the `controller.instanceID.enabled` to `false` now, which means the deployed controller will act upon any workflow deployed to the cluster. If you would like to limit the behavior and deploy multiple workflow controllers, please use the `controller.instanceID.enabled` attribute along with one of it's configuration options to set the `instanceID` of the workflow controller to be properly scoped for your needs. + ## Values -The `values.yaml` contains items used to tweak a deployment of this chart. +The `values.yaml` contains items used to tweak a deployment of this chart. Fields to note: -* `controller.useReleaseAsInstanceID`: If set to true then chart set controller instance id to release name - - __Note:__ If this is set to false then `controller.instanceId` must be set +* `controller.instanceID.enabled`: If set to true, the Argo Controller will **ONLY** monitor Workflow submissions with a `--instanceid` attribute +* `controller.instanceID.useReleaseName`: If set to true then chart set controller instance id to release name +* `controller.instanceID.explicitID`: Allows customization of an instance id for the workflow controller to monitor * `controller.workflowNamespaces`: This is a list of namespaces where workflows will be ran * `ui.enableWebConsole`: Enables ability to SSH into pod using web UI -* `minio.install`: If this is true, we'll install [minio](https://github.com/kubernetes/charts/tree/master/stable/minio) and build out the artifactRepository section in workflow controller config map. -* `artifactRepository.s3.accessKeySecret` and `artifactRepository.s3.secretKeySecret` These by default have the minio default credentials in them. - +* `minio.install`: If this is true, we'll install [minio](https://github.com/kubernetes/charts/tree/master/stable/minio) and build out the artifactRepository section in workflow controller config map. +* `artifactRepository.s3.accessKeySecret` and `artifactRepository.s3.secretKeySecret` These by default link to minio default credentials stored in the secret deployed by the minio chart. diff --git a/charts/argo/templates/apply-workflow-crd-job.yaml b/charts/argo/templates/apply-workflow-crd-job.yaml index 313a05f2..eb656d4a 100644 --- a/charts/argo/templates/apply-workflow-crd-job.yaml +++ b/charts/argo/templates/apply-workflow-crd-job.yaml @@ -10,9 +10,10 @@ spec: activeDeadlineSeconds: 100 template: spec: + serviceAccountName: {{ .Values.init.serviceAccount | quote }} containers: - name: kubectl-apply image: lachlanevenson/k8s-kubectl command: ["/bin/sh"] args: ["-c", 'echo ''{{- include "workflow-crd-json" .}}'' | kubectl apply -f -'] - restartPolicy: Never \ No newline at end of file + restartPolicy: Never diff --git a/charts/argo/templates/ui-deployment.yaml b/charts/argo/templates/ui-deployment.yaml index 661ce610..0ed78888 100644 --- a/charts/argo/templates/ui-deployment.yaml +++ b/charts/argo/templates/ui-deployment.yaml @@ -13,10 +13,18 @@ spec: labels: app: {{ .Release.Name }}-{{ .Values.ui.name}} release: {{ .Release.Name }} + {{- if .Values.ui.podAnnotations }} + annotations: +{{ toYaml .Values.ui.podAnnotations | indent 8}}{{- end }} spec: + serviceAccountName: {{ .Values.ui.serviceAccount | quote }} containers: - name: ui image: "{{ .Values.images.namespace }}/{{ .Values.images.ui }}:{{ .Values.images.tag }}" env: - name: IN_CLUSTER value: "true" + {{- if .Values.ui.enableWebConsole }} + - name: ENABLE_WEB_CONSOLE + value: "true" + {{- end }} diff --git a/charts/argo/templates/ui-service.yaml b/charts/argo/templates/ui-service.yaml index 83d0b25e..fd6c40c6 100644 --- a/charts/argo/templates/ui-service.yaml +++ b/charts/argo/templates/ui-service.yaml @@ -7,6 +7,9 @@ metadata: release: {{ .Release.Name }} heritage: {{ .Release.Service }} namspace: {{ .Release.Namespace }} + {{- if .Values.ui.serviceAnnotations }} + annotations: +{{ toYaml .Values.ui.serviceAnnotations | indent 4}}{{- end }} spec: ports: - port: 80 diff --git a/charts/argo/templates/worfkflow-controller-secrets-access.yaml b/charts/argo/templates/worfkflow-controller-secrets-access.yaml new file mode 100644 index 00000000..f0d48519 --- /dev/null +++ b/charts/argo/templates/worfkflow-controller-secrets-access.yaml @@ -0,0 +1,18 @@ +{{ if .Values.minio.install }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ .Release.Name }}-{{ .Values.controller.name}}-minio-secret +rules: +- apiGroups: + - "" + resources: + - secrets + resourceNames: + - {{ .Values.artifactRepository.s3.accessKeySecret.name | default (printf "%s-%s" .Release.Name "minio") | quote }} + - {{ .Values.artifactRepository.s3.secretKeySecret.name | default (printf "%s-%s" .Release.Name "minio") | quote }} + verbs: + - get + - watch + - list +{{- end }} \ No newline at end of file diff --git a/charts/argo/templates/workflow-controller-config-map.yaml b/charts/argo/templates/workflow-controller-config-map.yaml index 081719f4..597d8e35 100644 --- a/charts/argo/templates/workflow-controller-config-map.yaml +++ b/charts/argo/templates/workflow-controller-config-map.yaml @@ -9,10 +9,12 @@ metadata: namespace: {{ .Release.Namespace }} data: config: | - {{- if .Values.controller.useReleaseAsInstanceID }} + {{- if .Values.controller.instanceID.enabled }} + {{- if .Values.controller.instanceID.useReleaseName }} instanceID: {{ .Release.Name }} {{- else }} - instanceID: {{ .Values.controller.instanceID }} + instanceID: {{ .Values.controller.instanceID.explicitID }} + {{- end }} {{- end }} artifactRepository: {{- if or .Values.minio.install .Values.useDefaultArtifactRepo }} @@ -20,13 +22,13 @@ data: {{- if .Values.useStaticCredentials }} accessKeySecret: key: {{ .Values.artifactRepository.s3.accessKeySecret.key }} - name: {{ .Values.artifactRepository.s3.accessKeySecret.name | default (printf "%s-%s" .Release.Name "minio-user") }} + name: {{ .Values.artifactRepository.s3.accessKeySecret.name | default (printf "%s-%s" .Release.Name "minio") }} secretKeySecret: key: {{ .Values.artifactRepository.s3.secretKeySecret.key }} - name: {{ .Values.artifactRepository.s3.secretKeySecret.name | default (printf "%s-%s" .Release.Name "minio-user") }} + name: {{ .Values.artifactRepository.s3.secretKeySecret.name | default (printf "%s-%s" .Release.Name "minio") }} {{- end }} bucket: {{ .Values.artifactRepository.s3.bucket | default .Values.minio.defaultBucket.name }} - endpoint: {{ .Values.artifactRepository.s3.endpoint | default (printf "%s-%s" .Release.Name "minio-svc:9000") }} + endpoint: {{ .Values.artifactRepository.s3.endpoint | default (printf "%s-%s" .Release.Name "minio:9000") }} insecure: {{ .Values.artifactRepository.s3.insecure }} {{- end}} executorImage: "{{ .Values.images.namespace }}/{{ .Values.images.executor }}:{{ .Values.images.tag }}" diff --git a/charts/argo/templates/workflow-controller-deployment.yaml b/charts/argo/templates/workflow-controller-deployment.yaml index d4291bd3..9f5d8da0 100644 --- a/charts/argo/templates/workflow-controller-deployment.yaml +++ b/charts/argo/templates/workflow-controller-deployment.yaml @@ -14,12 +14,22 @@ spec: labels: app: {{ .Release.Name }}-{{ .Values.controller.name}} release: {{ .Release.Name }} + {{- if .Values.controller.podAnnotations }} + annotations: +{{ toYaml .Values.controller.podAnnotations | indent 8}}{{- end }} spec: + serviceAccountName: {{ .Values.controller.serviceAccount | quote }} containers: - name: controller image: "{{ .Values.images.namespace }}/{{ .Values.images.controller }}:{{ .Values.images.tag }}" command: [ "workflow-controller" ] - args: ["--configmap", "{{ .Release.Name }}-{{ .Values.controller.name}}-configmap"] + args: + - "--configmap" + - "{{ .Release.Name }}-{{ .Values.controller.name}}-configmap" + - "--loglevel" + - "{{ .Values.controller.logging.level }}" + - "--gloglevel" + - "{{ .Values.controller.logging.globallevel }}" env: - name: ARGO_NAMESPACE valueFrom: diff --git a/charts/argo/templates/workflow-controller-minio-secret-crb.yaml b/charts/argo/templates/workflow-controller-minio-secret-crb.yaml new file mode 100644 index 00000000..722776a4 --- /dev/null +++ b/charts/argo/templates/workflow-controller-minio-secret-crb.yaml @@ -0,0 +1,25 @@ +{{ if .Values.minio.install }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ .Release.Name }}-{{ .Values.controller.name}}-minio-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ .Release.Name }}-{{ .Values.controller.name}}-minio-secret +subjects: + - kind: ServiceAccount + name: {{ .Values.controller.serviceAccount }} + namespace: {{ .Release.Namespace }} +{{- if .Values.controller.workflowNamespaces }} +{{- $uiServiceAccount := .Values.controller.serviceAccount }} +{{- $namespace := .Release.Namespace }} +{{- range $key := .Values.controller.workflowNamespaces }} + {{- if not (eq $key $namespace) }} + - kind: ServiceAccount + name: {{ $uiServiceAccount }} + namespace: {{ $key }} + {{- end }} +{{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/argo/values.yaml b/charts/argo/values.yaml index 7a93f926..7ce97984 100644 --- a/charts/argo/values.yaml +++ b/charts/argo/values.yaml @@ -5,40 +5,64 @@ images: executor: argoexec tag: v2.1.1 -controller: - serviceAccount: argo - name: workflow-controller - workflowNamespaces: - - default - # If set to true then chart set controller instance id to release name - useReleaseAsInstanceID: true - instanceID: - -ui: - name: ui - serviceType: LoadBalancer - serviceAccount: argo-ui - # Enables ability to SSH into pod using web UI - enableWebConsole: false - crdVersion: v1alpha1 +init: + # By default the installation will not set an explicit one, which will mean it uses `default` for the namespace the chart is + # being deployed to. In RBAC clusters, that will almost certainly fail. See the NOTES: section of the readme for more info. + serviceAccount: "" + +controller: + # podAnnotations is an optional map of annotations to be applied to the controller Pods + podAnnotations: {} + serviceAccount: argo + name: workflow-controller + workflowNamespaces: + - default + instanceID: + # `instanceID.enabled` configures the controller to filter workflow submissions + # to only those which have a matching instanceID attribute. + enabled: false + # NOTE: If `instanceID.enabled` is set to `true` then either `instanceID.userReleaseName` + # or `instanceID.explicitID` must be defined. + # useReleaseName: true + # explicitID: unique-argo-controller-identifier + logging: + level: info + globallevel: "0" + +ui: + # optional map of annotations to be applied to the ui Pods + podAnnotations: {} + name: ui + # Enables ability to SSH into pod using web UI + enableWebConsole: false + serviceType: LoadBalancer + serviceAccount: argo-ui + # Annotations to be applied to the UI Service + serviceAnnotations: {} + +# Influences the creation of the ConfigMap for the workflow-controller itself. useDefaultArtifactRepo: false useStaticCredentials: true +artifactRepository: + s3: + # Note the `key` attribute is not the actual secret, it's the PATH to + # the contents in the associated secret, as defined by the `name` attribute. + accessKeySecret: + # name: -minio (default) + key: accesskey + secretKeySecret: + # name: -minio + key: secretkey + insecure: true + # bucket: + # endpoint: -# If set to true then chart installs minio and generate according artifactRepository section in workflow controller config map +# NOTE: These are setting attributes for the `minio` optional dependency minio: - # This is not used by the subchart, but makes sense to group here. + # If set to true then chart installs minio and generate according artifactRepository section in workflow controller config map install: false defaultBucket: enabled: true name: argo-artifacts - -artifactRepository: - s3: - #default minio creds - accessKeySecret: - key: AKIAIOSFODNN7EXAMPLE - secretKeySecret: - key: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY - insecure: true