From 51ad0bc54b1475384b67bee9e8a8e41e26b18bc4 Mon Sep 17 00:00:00 2001 From: Nick Novitski Date: Tue, 30 Apr 2019 09:45:58 -0700 Subject: [PATCH] Rearrange deployment files into kustomizations --- build/dev-env.sh | 27 +- deploy/aws/l4/kustomization.yaml | 11 + .../{provider/aws => aws/l4}/service-l4.yaml | 19 +- deploy/aws/l7/kustomization.yaml | 13 + .../{provider/aws => aws/l7}/service-l7.yaml | 19 +- deploy/aws/nlb/kustomization.yaml | 6 + deploy/aws/nlb/service-nlb.yaml | 7 + deploy/baremetal/kustomization.yaml | 6 + .../baremetal/service-nodeport.yaml | 5 +- .../deployment.yaml} | 22 +- deploy/cloud-generic/kustomization.yaml | 50 ++++ deploy/cloud-generic/role-binding.yaml | 11 + deploy/cloud-generic/role.yaml | 39 +++ deploy/cloud-generic/service-account.yaml | 4 + .../service.yaml} | 10 - deploy/cluster-wide/cluster-role-binding.yaml | 11 + .../cluster-wide/cluster-role.yaml | 3 - deploy/cluster-wide/kustomization.yaml | 8 + deploy/configmap.yaml | 30 -- .../grafana.yaml => grafana/deployment.yaml} | 35 --- deploy/grafana/kustomization.yaml | 12 + deploy/grafana/service.yaml | 10 + deploy/mandatory.yaml | 265 ------------------ deploy/minikube/kustomization.yaml | 10 + deploy/monitoring/configuration.yaml | 45 --- deploy/monitoring/prometheus.yaml | 109 ------- deploy/namespace.yaml | 10 - deploy/prometheus/deployment.yaml | 28 ++ deploy/prometheus/kustomization.yaml | 19 ++ deploy/prometheus/prometheus.yaml | 29 ++ deploy/prometheus/role-binding.yaml | 11 + deploy/prometheus/role.yaml | 11 + deploy/prometheus/service-account.yaml | 4 + deploy/prometheus/service.yaml | 9 + deploy/provider/aws/patch-configmap-l4.yaml | 10 - deploy/provider/aws/patch-configmap-l7.yaml | 14 - deploy/provider/aws/service-nlb.yaml | 28 -- .../provider/baremetal/service-nodeport.yaml | 25 -- deploy/rbac.yaml | 147 ---------- docs/deploy/index.md | 108 ++++--- docs/user-guide/monitoring.md | 24 +- hack/build-single-manifest-sh | 26 -- test/e2e-image/.gitignore | 2 + test/e2e-image/Dockerfile | 5 +- test/e2e-image/Makefile | 2 + test/e2e-image/e2e.sh | 2 - test/e2e-image/manifests/mandatory.yaml | 205 -------------- test/e2e-image/overlay/deployment-e2e.yaml | 26 ++ .../deployment-extension-group-patch.yaml | 3 + .../overlay/deployment-namespace-patch.yaml | 3 + test/e2e-image/overlay/kustomization.yaml | 34 +++ .../overlay/service-cluster-patch.yaml | 4 + .../overlay/service-protocol-tcp.yaml | 14 + .../defaultbackend/custom_default_backend.go | 3 +- test/e2e/settings/pod_security_policy.go | 5 +- test/e2e/wait-for-nginx.sh | 28 +- 56 files changed, 532 insertions(+), 1094 deletions(-) create mode 100644 deploy/aws/l4/kustomization.yaml rename deploy/{provider/aws => aws/l4}/service-l4.yaml (56%) create mode 100644 deploy/aws/l7/kustomization.yaml rename deploy/{provider/aws => aws/l7}/service-l7.yaml (68%) create mode 100644 deploy/aws/nlb/kustomization.yaml create mode 100644 deploy/aws/nlb/service-nlb.yaml create mode 100644 deploy/baremetal/kustomization.yaml rename test/e2e-image/manifests/service.yaml => deploy/baremetal/service-nodeport.yaml (69%) rename deploy/{with-rbac.yaml => cloud-generic/deployment.yaml} (73%) create mode 100644 deploy/cloud-generic/kustomization.yaml create mode 100644 deploy/cloud-generic/role-binding.yaml create mode 100644 deploy/cloud-generic/role.yaml create mode 100644 deploy/cloud-generic/service-account.yaml rename deploy/{provider/cloud-generic.yaml => cloud-generic/service.yaml} (51%) create mode 100644 deploy/cluster-wide/cluster-role-binding.yaml rename test/e2e-image/manifests/rbac.yaml => deploy/cluster-wide/cluster-role.yaml (88%) create mode 100644 deploy/cluster-wide/kustomization.yaml delete mode 100644 deploy/configmap.yaml rename deploy/{monitoring/grafana.yaml => grafana/deployment.yaml} (50%) create mode 100644 deploy/grafana/kustomization.yaml create mode 100644 deploy/grafana/service.yaml delete mode 100644 deploy/mandatory.yaml create mode 100644 deploy/minikube/kustomization.yaml delete mode 100644 deploy/monitoring/configuration.yaml delete mode 100644 deploy/monitoring/prometheus.yaml delete mode 100644 deploy/namespace.yaml create mode 100644 deploy/prometheus/deployment.yaml create mode 100644 deploy/prometheus/kustomization.yaml create mode 100644 deploy/prometheus/prometheus.yaml create mode 100644 deploy/prometheus/role-binding.yaml create mode 100644 deploy/prometheus/role.yaml create mode 100644 deploy/prometheus/service-account.yaml create mode 100644 deploy/prometheus/service.yaml delete mode 100644 deploy/provider/aws/patch-configmap-l4.yaml delete mode 100644 deploy/provider/aws/patch-configmap-l7.yaml delete mode 100644 deploy/provider/aws/service-nlb.yaml delete mode 100644 deploy/provider/baremetal/service-nodeport.yaml delete mode 100644 deploy/rbac.yaml delete mode 100755 hack/build-single-manifest-sh delete mode 100644 test/e2e-image/manifests/mandatory.yaml create mode 100644 test/e2e-image/overlay/deployment-e2e.yaml create mode 100644 test/e2e-image/overlay/deployment-extension-group-patch.yaml create mode 100644 test/e2e-image/overlay/deployment-namespace-patch.yaml create mode 100644 test/e2e-image/overlay/kustomization.yaml create mode 100644 test/e2e-image/overlay/service-cluster-patch.yaml create mode 100644 test/e2e-image/overlay/service-protocol-tcp.yaml diff --git a/build/dev-env.sh b/build/dev-env.sh index e0a828ca3..59b883cdc 100755 --- a/build/dev-env.sh +++ b/build/dev-env.sh @@ -47,16 +47,21 @@ make build container docker save "${DEV_IMAGE}" | (eval $(minikube docker-env --shell bash) && docker load) || true -echo "[dev-env] installing kubectl" -kubectl version || brew install kubectl +for tool in kubectl kustomize; do + echo "[dev-env] installing $tool" + $tool version || brew install $tool +done + +if ! kubectl get namespace $NAMESPACE; then + kubectl create namespace $NAMESPACE +fi + +ROOT=./deploy/minikube + +pushd $ROOT +kustomize edit set namespace $NAMESPACE +kustomize edit set image quay.io/kubernetes-ingress-controller/nginx-ingress-controller=${DEV_IMAGE} +popd echo "[dev-env] deploying NGINX Ingress controller in namespace $NAMESPACE" -cat ./deploy/mandatory.yaml | kubectl apply --namespace=$NAMESPACE -f - -cat ./deploy/provider/baremetal/service-nodeport.yaml | kubectl apply --namespace=$NAMESPACE -f - - -echo "updating image..." -kubectl set image \ - deployments \ - --namespace ingress-nginx \ - --selector app.kubernetes.io/name=ingress-nginx \ - nginx-ingress-controller=${DEV_IMAGE} +kustomize build $ROOT | kubectl apply -f - diff --git a/deploy/aws/l4/kustomization.yaml b/deploy/aws/l4/kustomization.yaml new file mode 100644 index 000000000..a17bd9156 --- /dev/null +++ b/deploy/aws/l4/kustomization.yaml @@ -0,0 +1,11 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +bases: +- ../../cloud-generic +patchesStrategicMerge: +- service-l4.yaml +configMapGenerator: +- name: nginx-configuration + behavior: merge + literals: + - use-proxy-protocol=true diff --git a/deploy/provider/aws/service-l4.yaml b/deploy/aws/l4/service-l4.yaml similarity index 56% rename from deploy/provider/aws/service-l4.yaml rename to deploy/aws/l4/service-l4.yaml index 893b5a03d..3d9642491 100644 --- a/deploy/provider/aws/service-l4.yaml +++ b/deploy/aws/l4/service-l4.yaml @@ -2,10 +2,6 @@ kind: Service apiVersion: v1 metadata: name: ingress-nginx - namespace: ingress-nginx - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx annotations: # Enable PROXY protocol service.beta.kubernetes.io/aws-load-balancer-proxy-protocol: "*" @@ -14,17 +10,4 @@ metadata: # increased to '3600' to avoid any potential issues. service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout: "60" spec: - type: LoadBalancer - selector: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - ports: - - name: http - port: 80 - targetPort: http - - name: https - port: 443 - targetPort: https - ---- - + externalTrafficPolicy: Cluster diff --git a/deploy/aws/l7/kustomization.yaml b/deploy/aws/l7/kustomization.yaml new file mode 100644 index 000000000..35dbc67e4 --- /dev/null +++ b/deploy/aws/l7/kustomization.yaml @@ -0,0 +1,13 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +bases: +- ../../cloud-generic +patchesStrategicMerge: +- service-l7.yaml +configMapGenerator: +- name: nginx-configuration + behavior: merge + literals: + - use-proxy-protocol=false + - use-forwarded-headers=true + - proxy-real-ip-cidr=0.0.0.0/0 # restrict this to the IP addresses of ELB diff --git a/deploy/provider/aws/service-l7.yaml b/deploy/aws/l7/service-l7.yaml similarity index 68% rename from deploy/provider/aws/service-l7.yaml rename to deploy/aws/l7/service-l7.yaml index 6616108a2..b3b0b64d8 100644 --- a/deploy/provider/aws/service-l7.yaml +++ b/deploy/aws/l7/service-l7.yaml @@ -2,10 +2,6 @@ kind: Service apiVersion: v1 metadata: name: ingress-nginx - namespace: ingress-nginx - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx annotations: # replace with the correct value of the generated certificate in the AWS console service.beta.kubernetes.io/aws-load-balancer-ssl-cert: "arn:aws:acm:us-west-2:XXXXXXXX:certificate/XXXXXX-XXXXXXX-XXXXXXX-XXXXXXXX" @@ -18,17 +14,4 @@ metadata: # increased to '3600' to avoid any potential issues. service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout: "60" spec: - type: LoadBalancer - selector: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - ports: - - name: http - port: 80 - targetPort: http - - name: https - port: 443 - targetPort: http - ---- - + externalTrafficPolicy: Cluster diff --git a/deploy/aws/nlb/kustomization.yaml b/deploy/aws/nlb/kustomization.yaml new file mode 100644 index 000000000..cfffbefc4 --- /dev/null +++ b/deploy/aws/nlb/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +bases: +- ../../cloud-generic +patchesStrategicMerge: +- service-nlb.yaml diff --git a/deploy/aws/nlb/service-nlb.yaml b/deploy/aws/nlb/service-nlb.yaml new file mode 100644 index 000000000..a0438c207 --- /dev/null +++ b/deploy/aws/nlb/service-nlb.yaml @@ -0,0 +1,7 @@ +kind: Service +apiVersion: v1 +metadata: + name: ingress-nginx + annotations: + # by default the type is elb (classic load balancer). + service.beta.kubernetes.io/aws-load-balancer-type: nlb diff --git a/deploy/baremetal/kustomization.yaml b/deploy/baremetal/kustomization.yaml new file mode 100644 index 000000000..3512703b8 --- /dev/null +++ b/deploy/baremetal/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +bases: +- ../cloud-generic +patchesStrategicMerge: +- service-nodeport.yaml diff --git a/test/e2e-image/manifests/service.yaml b/deploy/baremetal/service-nodeport.yaml similarity index 69% rename from test/e2e-image/manifests/service.yaml rename to deploy/baremetal/service-nodeport.yaml index 4226bb0e0..0aadea157 100644 --- a/test/e2e-image/manifests/service.yaml +++ b/deploy/baremetal/service-nodeport.yaml @@ -3,6 +3,7 @@ kind: Service metadata: name: ingress-nginx spec: + type: NodePort ports: - name: http port: 80 @@ -12,6 +13,4 @@ spec: port: 443 targetPort: 443 protocol: TCP - selector: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx + externalTrafficPolicy: Cluster diff --git a/deploy/with-rbac.yaml b/deploy/cloud-generic/deployment.yaml similarity index 73% rename from deploy/with-rbac.yaml rename to deploy/cloud-generic/deployment.yaml index 3e3661cdb..71de9202b 100644 --- a/deploy/with-rbac.yaml +++ b/deploy/cloud-generic/deployment.yaml @@ -2,21 +2,10 @@ apiVersion: apps/v1 kind: Deployment metadata: name: nginx-ingress-controller - namespace: ingress-nginx - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx spec: replicas: 1 - selector: - matchLabels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx template: metadata: - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx annotations: prometheus.io/port: "10254" prometheus.io/scrape: "true" @@ -27,10 +16,10 @@ spec: image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.24.1 args: - /nginx-ingress-controller - - --configmap=$(POD_NAMESPACE)/nginx-configuration - - --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services - - --udp-services-configmap=$(POD_NAMESPACE)/udp-services - - --publish-service=$(POD_NAMESPACE)/ingress-nginx + - --configmap=$(POD_NAMESPACE)/$(NGINX_CONFIGMAP_NAME) + - --tcp-services-configmap=$(POD_NAMESPACE)/$(TCP_CONFIGMAP_NAME) + - --udp-services-configmap=$(POD_NAMESPACE)/$(UDP_CONFIGMAP_NAME) + - --publish-service=$(POD_NAMESPACE)/$(SERVICE_NAME) - --annotations-prefix=nginx.ingress.kubernetes.io securityContext: allowPrivilegeEscalation: true @@ -74,6 +63,3 @@ spec: periodSeconds: 10 successThreshold: 1 timeoutSeconds: 10 - ---- - diff --git a/deploy/cloud-generic/kustomization.yaml b/deploy/cloud-generic/kustomization.yaml new file mode 100644 index 000000000..c2b03ddbf --- /dev/null +++ b/deploy/cloud-generic/kustomization.yaml @@ -0,0 +1,50 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: ingress-nginx +commonLabels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx +resources: +- deployment.yaml +- role-binding.yaml +- role.yaml +- service-account.yaml +- service.yaml +images: +- name: quay.io/kubernetes-ingress-controller/nginx-ingress-controller + newTag: 0.24.1 +vars: +- fieldref: + fieldPath: metadata.name + name: NGINX_CONFIGMAP_NAME + objref: + apiVersion: v1 + kind: ConfigMap + name: nginx-configuration +- fieldref: + fieldPath: metadata.name + name: TCP_CONFIGMAP_NAME + objref: + apiVersion: v1 + kind: ConfigMap + name: tcp-services +- fieldref: + fieldPath: metadata.name + name: UDP_CONFIGMAP_NAME + objref: + apiVersion: v1 + kind: ConfigMap + name: udp-services +- fieldref: + fieldPath: metadata.name + name: SERVICE_NAME + objref: + apiVersion: v1 + kind: Service + name: ingress-nginx +configMapGenerator: +- name: nginx-configuration +- name: tcp-services +- name: udp-services +generatorOptions: + disableNameSuffixHash: true diff --git a/deploy/cloud-generic/role-binding.yaml b/deploy/cloud-generic/role-binding.yaml new file mode 100644 index 000000000..228588e6d --- /dev/null +++ b/deploy/cloud-generic/role-binding.yaml @@ -0,0 +1,11 @@ +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: RoleBinding +metadata: + name: nginx-ingress-role-nisa-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: nginx-ingress-role +subjects: + - kind: ServiceAccount + name: nginx-ingress-serviceaccount diff --git a/deploy/cloud-generic/role.yaml b/deploy/cloud-generic/role.yaml new file mode 100644 index 000000000..936b63d72 --- /dev/null +++ b/deploy/cloud-generic/role.yaml @@ -0,0 +1,39 @@ +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: Role +metadata: + name: nginx-ingress-role +rules: + - apiGroups: + - "" + resources: + - configmaps + - pods + - secrets + - namespaces + verbs: + - get + - apiGroups: + - "" + resources: + - configmaps + resourceNames: + # Defaults to "-" + # Here: "-" + # This has to be adapted if you change either parameter + # when launching the nginx-ingress-controller. + - "ingress-controller-leader-nginx" + verbs: + - get + - update + - apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - apiGroups: + - "" + resources: + - endpoints + verbs: + - get diff --git a/deploy/cloud-generic/service-account.yaml b/deploy/cloud-generic/service-account.yaml new file mode 100644 index 000000000..a52fb8ac8 --- /dev/null +++ b/deploy/cloud-generic/service-account.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: nginx-ingress-serviceaccount diff --git a/deploy/provider/cloud-generic.yaml b/deploy/cloud-generic/service.yaml similarity index 51% rename from deploy/provider/cloud-generic.yaml rename to deploy/cloud-generic/service.yaml index 8bbac569b..3a3a3e2a8 100644 --- a/deploy/provider/cloud-generic.yaml +++ b/deploy/cloud-generic/service.yaml @@ -2,16 +2,9 @@ kind: Service apiVersion: v1 metadata: name: ingress-nginx - namespace: ingress-nginx - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx spec: externalTrafficPolicy: Local type: LoadBalancer - selector: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx ports: - name: http port: 80 @@ -19,6 +12,3 @@ spec: - name: https port: 443 targetPort: https - ---- - diff --git a/deploy/cluster-wide/cluster-role-binding.yaml b/deploy/cluster-wide/cluster-role-binding.yaml new file mode 100644 index 000000000..7293fb37d --- /dev/null +++ b/deploy/cluster-wide/cluster-role-binding.yaml @@ -0,0 +1,11 @@ +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: nginx-ingress-clusterrole-nisa-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: nginx-ingress-clusterrole +subjects: + - kind: ServiceAccount + name: nginx-ingress-serviceaccount diff --git a/test/e2e-image/manifests/rbac.yaml b/deploy/cluster-wide/cluster-role.yaml similarity index 88% rename from test/e2e-image/manifests/rbac.yaml rename to deploy/cluster-wide/cluster-role.yaml index 7f4f79fe8..9e5d39ca3 100644 --- a/test/e2e-image/manifests/rbac.yaml +++ b/deploy/cluster-wide/cluster-role.yaml @@ -2,9 +2,6 @@ apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRole metadata: name: nginx-ingress-clusterrole - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx rules: - apiGroups: - "" diff --git a/deploy/cluster-wide/kustomization.yaml b/deploy/cluster-wide/kustomization.yaml new file mode 100644 index 000000000..aeef6ed6b --- /dev/null +++ b/deploy/cluster-wide/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +commonLabels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx +resources: +- cluster-role.yaml +- cluster-role-binding.yaml diff --git a/deploy/configmap.yaml b/deploy/configmap.yaml deleted file mode 100644 index 436b660a9..000000000 --- a/deploy/configmap.yaml +++ /dev/null @@ -1,30 +0,0 @@ -kind: ConfigMap -apiVersion: v1 -metadata: - name: nginx-configuration - namespace: ingress-nginx - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - ---- -kind: ConfigMap -apiVersion: v1 -metadata: - name: tcp-services - namespace: ingress-nginx - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - ---- -kind: ConfigMap -apiVersion: v1 -metadata: - name: udp-services - namespace: ingress-nginx - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - ---- diff --git a/deploy/monitoring/grafana.yaml b/deploy/grafana/deployment.yaml similarity index 50% rename from deploy/monitoring/grafana.yaml rename to deploy/grafana/deployment.yaml index 7c7b00000..765b2f64f 100644 --- a/deploy/monitoring/grafana.yaml +++ b/deploy/grafana/deployment.yaml @@ -1,27 +1,15 @@ apiVersion: apps/v1 kind: Deployment metadata: - labels: - app.kubernetes.io/name: grafana - app.kubernetes.io/part-of: ingress-nginx - name: grafana namespace: ingress-nginx spec: - selector: - matchLabels: - app.kubernetes.io/name: grafana - app.kubernetes.io/part-of: ingress-nginx strategy: rollingUpdate: maxSurge: 1 maxUnavailable: 1 type: RollingUpdate template: - metadata: - labels: - app.kubernetes.io/name: grafana - app.kubernetes.io/part-of: ingress-nginx spec: containers: - image: grafana/grafana @@ -43,26 +31,3 @@ spec: volumes: - emptyDir: {} name: data - ---- -apiVersion: v1 -kind: Service -metadata: - name: grafana - namespace: ingress-nginx - labels: - app.kubernetes.io/name: grafana - app.kubernetes.io/part-of: ingress-nginx - -spec: - ports: - - port: 3000 - protocol: TCP - targetPort: 3000 - selector: - app.kubernetes.io/name: grafana - app.kubernetes.io/part-of: ingress-nginx - type: NodePort - ---- - diff --git a/deploy/grafana/kustomization.yaml b/deploy/grafana/kustomization.yaml new file mode 100644 index 000000000..2069c1a7a --- /dev/null +++ b/deploy/grafana/kustomization.yaml @@ -0,0 +1,12 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: ingress-nginx +commonLabels: + app.kubernetes.io/name: grafana + app.kubernetes.io/part-of: ingress-nginx +resources: +- deployment.yaml +- service.yaml +images: +- name: grafana/grafana + newTag: 6.1.6 diff --git a/deploy/grafana/service.yaml b/deploy/grafana/service.yaml new file mode 100644 index 000000000..16d69ac7a --- /dev/null +++ b/deploy/grafana/service.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Service +metadata: + name: grafana +spec: + ports: + - port: 3000 + protocol: TCP + targetPort: 3000 + type: NodePort diff --git a/deploy/mandatory.yaml b/deploy/mandatory.yaml deleted file mode 100644 index 921bc482d..000000000 --- a/deploy/mandatory.yaml +++ /dev/null @@ -1,265 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: ingress-nginx - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - ---- - -kind: ConfigMap -apiVersion: v1 -metadata: - name: nginx-configuration - namespace: ingress-nginx - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - ---- -kind: ConfigMap -apiVersion: v1 -metadata: - name: tcp-services - namespace: ingress-nginx - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - ---- -kind: ConfigMap -apiVersion: v1 -metadata: - name: udp-services - namespace: ingress-nginx - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: nginx-ingress-serviceaccount - namespace: ingress-nginx - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRole -metadata: - name: nginx-ingress-clusterrole - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx -rules: - - apiGroups: - - "" - resources: - - configmaps - - endpoints - - nodes - - pods - - secrets - verbs: - - list - - watch - - apiGroups: - - "" - resources: - - nodes - verbs: - - get - - apiGroups: - - "" - resources: - - services - verbs: - - get - - list - - watch - - apiGroups: - - "extensions" - resources: - - ingresses - verbs: - - get - - list - - watch - - apiGroups: - - "" - resources: - - events - verbs: - - create - - patch - - apiGroups: - - "extensions" - resources: - - ingresses/status - verbs: - - update - ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: Role -metadata: - name: nginx-ingress-role - namespace: ingress-nginx - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx -rules: - - apiGroups: - - "" - resources: - - configmaps - - pods - - secrets - - namespaces - verbs: - - get - - apiGroups: - - "" - resources: - - configmaps - resourceNames: - # Defaults to "-" - # Here: "-" - # This has to be adapted if you change either parameter - # when launching the nginx-ingress-controller. - - "ingress-controller-leader-nginx" - verbs: - - get - - update - - apiGroups: - - "" - resources: - - configmaps - verbs: - - create - - apiGroups: - - "" - resources: - - endpoints - verbs: - - get - ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: RoleBinding -metadata: - name: nginx-ingress-role-nisa-binding - namespace: ingress-nginx - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: nginx-ingress-role -subjects: - - kind: ServiceAccount - name: nginx-ingress-serviceaccount - namespace: ingress-nginx - ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRoleBinding -metadata: - name: nginx-ingress-clusterrole-nisa-binding - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: nginx-ingress-clusterrole -subjects: - - kind: ServiceAccount - name: nginx-ingress-serviceaccount - namespace: ingress-nginx - ---- - -apiVersion: apps/v1 -kind: Deployment -metadata: - name: nginx-ingress-controller - namespace: ingress-nginx - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx -spec: - replicas: 1 - selector: - matchLabels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - template: - metadata: - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - annotations: - prometheus.io/port: "10254" - prometheus.io/scrape: "true" - spec: - serviceAccountName: nginx-ingress-serviceaccount - containers: - - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.24.1 - args: - - /nginx-ingress-controller - - --configmap=$(POD_NAMESPACE)/nginx-configuration - - --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services - - --udp-services-configmap=$(POD_NAMESPACE)/udp-services - - --publish-service=$(POD_NAMESPACE)/ingress-nginx - - --annotations-prefix=nginx.ingress.kubernetes.io - securityContext: - allowPrivilegeEscalation: true - capabilities: - drop: - - ALL - add: - - NET_BIND_SERVICE - # www-data -> 33 - runAsUser: 33 - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - ports: - - name: http - containerPort: 80 - - name: https - containerPort: 443 - livenessProbe: - failureThreshold: 3 - httpGet: - path: /healthz - port: 10254 - scheme: HTTP - initialDelaySeconds: 10 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 10 - readinessProbe: - failureThreshold: 3 - httpGet: - path: /healthz - port: 10254 - scheme: HTTP - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 10 - ---- diff --git a/deploy/minikube/kustomization.yaml b/deploy/minikube/kustomization.yaml new file mode 100644 index 000000000..34ea278ad --- /dev/null +++ b/deploy/minikube/kustomization.yaml @@ -0,0 +1,10 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: ingress-nginx +bases: +- ../baremetal +- ../cluster-wide +images: +- name: quay.io/kubernetes-ingress-controller/nginx-ingress-controller + newName: ingress-controller/nginx-ingress-controller + newTag: dev diff --git a/deploy/monitoring/configuration.yaml b/deploy/monitoring/configuration.yaml deleted file mode 100644 index 36b9d1773..000000000 --- a/deploy/monitoring/configuration.yaml +++ /dev/null @@ -1,45 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: prometheus-configuration - labels: - app.kubernetes.io/name: prometheus - app.kubernetes.io/part-of: ingress-nginx - name: prometheus-configuration - namespace: ingress-nginx -data: - prometheus.yml: |- - global: - scrape_interval: 10s - scrape_configs: - - job_name: 'ingress-nginx-endpoints' - kubernetes_sd_configs: - - role: pod - namespaces: - names: - - ingress-nginx - - relabel_configs: - - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape] - action: keep - regex: true - - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scheme] - action: replace - target_label: __scheme__ - regex: (https?) - - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path] - action: replace - target_label: __metrics_path__ - regex: (.+) - - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] - action: replace - target_label: __address__ - regex: ([^:]+)(?::\d+)?;(\d+) - replacement: $1:$2 - - - source_labels: [__meta_kubernetes_service_name] - regex: prometheus-server - action: drop - ---- - diff --git a/deploy/monitoring/prometheus.yaml b/deploy/monitoring/prometheus.yaml deleted file mode 100644 index aee13d286..000000000 --- a/deploy/monitoring/prometheus.yaml +++ /dev/null @@ -1,109 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: prometheus-server - namespace: ingress-nginx - labels: - app.kubernetes.io/name: prometheus - app.kubernetes.io/part-of: ingress-nginx -rules: - - apiGroups: [""] - resources: - - services - - endpoints - - pods - verbs: ["get", "list", "watch"] - ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: prometheus-server - namespace: ingress-nginx - labels: - app.kubernetes.io/name: prometheus - app.kubernetes.io/part-of: ingress-nginx - ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: prometheus-server - namespace: ingress-nginx - labels: - app.kubernetes.io/name: prometheus - app.kubernetes.io/part-of: ingress-nginx - -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: prometheus-server -subjects: - - kind: ServiceAccount - name: prometheus-server - namespace: ingress-nginx - ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: prometheus-server - namespace: ingress-nginx - labels: - app.kubernetes.io/name: prometheus - app.kubernetes.io/part-of: ingress-nginx - -spec: - replicas: 1 - selector: - matchLabels: - app.kubernetes.io/name: prometheus - app.kubernetes.io/part-of: ingress-nginx - template: - metadata: - labels: - app.kubernetes.io/name: prometheus - app.kubernetes.io/part-of: ingress-nginx - spec: - serviceAccountName: prometheus-server - containers: - - name: prometheus - image: prom/prometheus:v2.3.2 - args: - - "--config.file=/etc/prometheus/prometheus.yml" - - "--storage.tsdb.path=/prometheus/" - ports: - - containerPort: 9090 - volumeMounts: - - name: prometheus-config-volume - mountPath: /etc/prometheus/ - - name: prometheus-storage-volume - mountPath: /prometheus/ - volumes: - - name: prometheus-config-volume - configMap: - name: prometheus-configuration - - name: prometheus-storage-volume - emptyDir: {} - ---- -apiVersion: v1 -kind: Service -metadata: - name: prometheus-server - namespace: ingress-nginx - labels: - app.kubernetes.io/name: prometheus - app.kubernetes.io/part-of: ingress-nginx - -spec: - selector: - app.kubernetes.io/name: prometheus - app.kubernetes.io/part-of: ingress-nginx - type: NodePort - ports: - - port: 9090 - targetPort: 9090 - ---- - diff --git a/deploy/namespace.yaml b/deploy/namespace.yaml deleted file mode 100644 index 9196d6d16..000000000 --- a/deploy/namespace.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: ingress-nginx - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - ---- - diff --git a/deploy/prometheus/deployment.yaml b/deploy/prometheus/deployment.yaml new file mode 100644 index 000000000..2b15c686b --- /dev/null +++ b/deploy/prometheus/deployment.yaml @@ -0,0 +1,28 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: prometheus-server +spec: + replicas: 1 + template: + spec: + serviceAccountName: prometheus-server + containers: + - name: prometheus + image: prom/prometheus + args: + - "--config.file=/etc/prometheus/prometheus.yaml" + - "--storage.tsdb.path=/prometheus/" + ports: + - containerPort: 9090 + volumeMounts: + - name: prometheus-config-volume + mountPath: /etc/prometheus/ + - name: prometheus-storage-volume + mountPath: /prometheus/ + volumes: + - name: prometheus-config-volume + configMap: + name: prometheus-configuration + - name: prometheus-storage-volume + emptyDir: {} diff --git a/deploy/prometheus/kustomization.yaml b/deploy/prometheus/kustomization.yaml new file mode 100644 index 000000000..2d6e0a7e2 --- /dev/null +++ b/deploy/prometheus/kustomization.yaml @@ -0,0 +1,19 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: ingress-nginx +commonLabels: + app.kubernetes.io/name: prometheus + app.kubernetes.io/part-of: ingress-nginx +resources: +- role.yaml +- service-account.yaml +- role-binding.yaml +- deployment.yaml +- service.yaml +images: +- name: prom/prometheus + newTag: v2.3.2 +configMapGenerator: +- name: prometheus-configuration + files: + - prometheus.yaml diff --git a/deploy/prometheus/prometheus.yaml b/deploy/prometheus/prometheus.yaml new file mode 100644 index 000000000..fe8cefe96 --- /dev/null +++ b/deploy/prometheus/prometheus.yaml @@ -0,0 +1,29 @@ +global: + scrape_interval: 10s +scrape_configs: +- job_name: 'ingress-nginx-endpoints' + kubernetes_sd_configs: + - role: pod + namespaces: + names: + - ingress-nginx + relabel_configs: + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape] + action: keep + regex: true + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scheme] + action: replace + target_label: __scheme__ + regex: (https?) + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path] + action: replace + target_label: __metrics_path__ + regex: (.+) + - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] + action: replace + target_label: __address__ + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + - source_labels: [__meta_kubernetes_service_name] + regex: prometheus-server + action: drop diff --git a/deploy/prometheus/role-binding.yaml b/deploy/prometheus/role-binding.yaml new file mode 100644 index 000000000..b73b94f3a --- /dev/null +++ b/deploy/prometheus/role-binding.yaml @@ -0,0 +1,11 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: prometheus-server +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: prometheus-server +subjects: + - kind: ServiceAccount + name: prometheus-server diff --git a/deploy/prometheus/role.yaml b/deploy/prometheus/role.yaml new file mode 100644 index 000000000..0328f80ba --- /dev/null +++ b/deploy/prometheus/role.yaml @@ -0,0 +1,11 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: prometheus-server +rules: + - apiGroups: [""] + resources: + - services + - endpoints + - pods + verbs: ["get", "list", "watch"] diff --git a/deploy/prometheus/service-account.yaml b/deploy/prometheus/service-account.yaml new file mode 100644 index 000000000..3e2c9a770 --- /dev/null +++ b/deploy/prometheus/service-account.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: prometheus-server diff --git a/deploy/prometheus/service.yaml b/deploy/prometheus/service.yaml new file mode 100644 index 000000000..9a2de8ce4 --- /dev/null +++ b/deploy/prometheus/service.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Service +metadata: + name: prometheus-server +spec: + type: NodePort + ports: + - port: 9090 + targetPort: 9090 diff --git a/deploy/provider/aws/patch-configmap-l4.yaml b/deploy/provider/aws/patch-configmap-l4.yaml deleted file mode 100644 index 1d612289f..000000000 --- a/deploy/provider/aws/patch-configmap-l4.yaml +++ /dev/null @@ -1,10 +0,0 @@ -kind: ConfigMap -apiVersion: v1 -metadata: - name: nginx-configuration - namespace: ingress-nginx - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx -data: - use-proxy-protocol: "true" diff --git a/deploy/provider/aws/patch-configmap-l7.yaml b/deploy/provider/aws/patch-configmap-l7.yaml deleted file mode 100644 index b1bcd2a97..000000000 --- a/deploy/provider/aws/patch-configmap-l7.yaml +++ /dev/null @@ -1,14 +0,0 @@ -kind: ConfigMap -apiVersion: v1 -metadata: - name: nginx-configuration - namespace: ingress-nginx - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx -data: - use-proxy-protocol: "false" - use-forwarded-headers: "true" - proxy-real-ip-cidr: "0.0.0.0/0" # restrict this to the IP addresses of ELB ---- - diff --git a/deploy/provider/aws/service-nlb.yaml b/deploy/provider/aws/service-nlb.yaml deleted file mode 100644 index 244460b6d..000000000 --- a/deploy/provider/aws/service-nlb.yaml +++ /dev/null @@ -1,28 +0,0 @@ -kind: Service -apiVersion: v1 -metadata: - name: ingress-nginx - namespace: ingress-nginx - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - annotations: - # by default the type is elb (classic load balancer). - service.beta.kubernetes.io/aws-load-balancer-type: nlb -spec: - # this setting is to make sure the source IP address is preserved. - externalTrafficPolicy: Local - type: LoadBalancer - selector: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - ports: - - name: http - port: 80 - targetPort: http - - name: https - port: 443 - targetPort: https - ---- - diff --git a/deploy/provider/baremetal/service-nodeport.yaml b/deploy/provider/baremetal/service-nodeport.yaml deleted file mode 100644 index 24e302818..000000000 --- a/deploy/provider/baremetal/service-nodeport.yaml +++ /dev/null @@ -1,25 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: ingress-nginx - namespace: ingress-nginx - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx -spec: - type: NodePort - ports: - - name: http - port: 80 - targetPort: 80 - protocol: TCP - - name: https - port: 443 - targetPort: 443 - protocol: TCP - selector: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - ---- - diff --git a/deploy/rbac.yaml b/deploy/rbac.yaml deleted file mode 100644 index 103bd98cc..000000000 --- a/deploy/rbac.yaml +++ /dev/null @@ -1,147 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: nginx-ingress-serviceaccount - namespace: ingress-nginx - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRole -metadata: - name: nginx-ingress-clusterrole - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx -rules: - - apiGroups: - - "" - resources: - - configmaps - - endpoints - - nodes - - pods - - secrets - verbs: - - list - - watch - - apiGroups: - - "" - resources: - - nodes - verbs: - - get - - apiGroups: - - "" - resources: - - services - verbs: - - get - - list - - watch - - apiGroups: - - "extensions" - resources: - - ingresses - verbs: - - get - - list - - watch - - apiGroups: - - "" - resources: - - events - verbs: - - create - - patch - - apiGroups: - - "extensions" - resources: - - ingresses/status - verbs: - - update - ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: Role -metadata: - name: nginx-ingress-role - namespace: ingress-nginx - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx -rules: - - apiGroups: - - "" - resources: - - configmaps - - pods - - secrets - - namespaces - verbs: - - get - - apiGroups: - - "" - resources: - - configmaps - resourceNames: - # Defaults to "-" - # Here: "-" - # This has to be adapted if you change either parameter - # when launching the nginx-ingress-controller. - - "ingress-controller-leader-nginx" - verbs: - - get - - update - - apiGroups: - - "" - resources: - - configmaps - verbs: - - create - - apiGroups: - - "" - resources: - - endpoints - verbs: - - get - ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: RoleBinding -metadata: - name: nginx-ingress-role-nisa-binding - namespace: ingress-nginx - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: nginx-ingress-role -subjects: - - kind: ServiceAccount - name: nginx-ingress-serviceaccount - namespace: ingress-nginx - ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRoleBinding -metadata: - name: nginx-ingress-clusterrole-nisa-binding - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: nginx-ingress-clusterrole -subjects: - - kind: ServiceAccount - name: nginx-ingress-serviceaccount - namespace: ingress-nginx - ---- - diff --git a/docs/deploy/index.md b/docs/deploy/index.md index 9f56f6e88..786d1b4bf 100644 --- a/docs/deploy/index.md +++ b/docs/deploy/index.md @@ -18,6 +18,9 @@ The following **Mandatory Command** is required for all deployments. +!!! attention + These commands depend on having kubectl version 1.14 or newer. + !!! attention The default configuration watches Ingress object from all the namespaces. To change this behavior use the flag `--watch-namespace` to limit the scope to a particular namespace. @@ -25,17 +28,25 @@ The following **Mandatory Command** is required for all deployments. !!! warning If multiple Ingresses define different paths for the same host, the ingress controller will merge the definitions. -!!! attention - If you're using GKE you need to initialize your user as a cluster-admin with the following command: - ```kubectl create clusterrolebinding cluster-admin-binding --clusterrole cluster-admin --user $(gcloud config get-value account)``` ```console -kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/mandatory.yaml +kubectl create namespace ingress-nginx +``` + +```console +cat << EOF > kustomization.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: ingress-nginx +bases: +- github.com/kubernetes/ingress-nginx/deploy/cluster-wide +- # provider-specific, see below +EOF ``` ### Provider Specific Steps -There are cloud provider specific yaml files. +There are cloud provider specific kustomize bases. #### Docker for Mac @@ -43,11 +54,7 @@ Kubernetes is available in Docker for Mac (from [version 18.06.0-ce](https://doc [enable]: https://docs.docker.com/docker-for-mac/#kubernetes -Create a service - -```console -kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/provider/cloud-generic.yaml -``` +Add `github.com/kubernetes/ingress-nginx/deploy/cloud-generic` to the `bases` list in `kustomization.yaml` and run `kubectl apply --kustomize .`. #### minikube @@ -88,30 +95,57 @@ This setup requires to choose in which layer (L4 or L7) we want to configure the - [Layer 4](https://en.wikipedia.org/wiki/OSI_model#Layer_4:_Transport_Layer): use TCP as the listener protocol for ports 80 and 443. - [Layer 7](https://en.wikipedia.org/wiki/OSI_model#Layer_7:_Application_Layer): use HTTP as the listener protocol for port 80 and terminate TLS in the ELB + +Check that no change is necessary with regards to the ELB idle timeout. In some scenarios, users may want to modify the ELB idle timeout, so please check the [ELB Idle Timeouts section](#elb-idle-timeouts) for additional information. If a change is required, users will need to override the value of the annotation `service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout` on the service object. + +To do this, create a patch file which will replace the annotation. + +``` +cat << EOF > elb-timeout.yaml +kind: Service +apiVersion: v1 +metadata: + name: ingress-nginx + annotations: + service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout: "3600" # Recommended value for WebSockets +EOF +``` + +After creating the patch file, reference it in your `kustomization.yaml`: +```yaml +patchesStrategicMerge: +- elb-timeout.yaml +``` + For L4: -Check that no change is necessary with regards to the ELB idle timeout. In some scenarios, users may want to modify the ELB idle timeout, so please check the [ELB Idle Timeouts section](#elb-idle-timeouts) for additional information. If a change is required, users will need to update the value of `service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout` in `provider/aws/service-l4.yaml` - -Then execute: - -```console -kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/provider/aws/service-l4.yaml -kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/provider/aws/patch-configmap-l4.yaml -``` +To deploy the default example, add the base ` github.com/kubernetes/ingress-nginx/deploy/aws/l4` and then run `kubectl apply --kustomize .` For L7: -Change line of the file `provider/aws/service-l7.yaml` replacing the dummy id with a valid one `"arn:aws:acm:us-west-2:XXXXXXXX:certificate/XXXXXX-XXXXXXX-XXXXXXX-XXXXXXXX"` - -Check that no change is necessary with regards to the ELB idle timeout. In some scenarios, users may want to modify the ELB idle timeout, so please check the [ELB Idle Timeouts section](#elb-idle-timeouts) for additional information. If a change is required, users will need to update the value of `service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout` in `provider/aws/service-l7.yaml` - -Then execute: +Create a a patch that will annotate the ingress-controller's service with your ssl certificate id. ```console -kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/provider/aws/service-l7.yaml -kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/provider/aws/patch-configmap-l7.yaml +cat << EOF > elb-ssl.yaml +kind: Service +apiVersion: v1 +metadata: + name: ingress-nginx + annotations: + # replace with the correct value of the generated certificate in the AWS console + service.beta.kubernetes.io/aws-load-balancer-ssl-cert: "arn:aws:acm:us-west-2:XXXXXXXX:certificate/XXXXXX-XXXXXXX-XXXXXXX-XXXXXXXX" +EOF ``` +Reference this patch in your `kustomization.yaml`: + +```yaml +patchesStrategicMerge: +- elb-ssl.yaml +``` + +Then add the l7 base, `github.com/kubernetes/ingress-nginx/deploy/aws/l7` and execute `kubectl apply --kustomize .` + This example creates an ELB with just two listeners, one in port 80 and another in port 443 ![Listeners](../images/elb-l7-listener.png) @@ -127,35 +161,31 @@ More information with regards to idle timeouts for your Load Balancer can be fou ##### Network Load Balancer (NLB) -This type of load balancer is supported since v1.10.0 as an ALPHA feature. +This type of load balancer is supported since v1.10.0 as an ALPHA feature. Use the base `github.com/kubernetes/ingress-nginx/deploy/aws/nlb` and execute `kubectl apply --kustomize .` -```console -kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/provider/aws/service-nlb.yaml -``` #### GCE-GKE -```console -kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/provider/cloud-generic.yaml -``` +!!! attention + If you're using GKE you need to initialize your user as a cluster-admin with the following command: + ```kubectl create clusterrolebinding cluster-admin-binding --clusterrole cluster-admin --user $(gcloud config get-value account)``` + +Use the base `github.com/kubernetes/ingress-nginx/deploy/cloud-generic` and execute `kubectl apply --kustomize .` **Important Note:** proxy protocol is not supported in GCE/GKE + #### Azure - -```console -kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/provider/cloud-generic.yaml -``` +Use the base `github.com/kubernetes/ingress-nginx/deploy/cloud-generic` and execute `kubectl apply --kustomize .` #### Bare-metal Using [NodePort](https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport): -```console -kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/provider/baremetal/service-nodeport.yaml -``` + +Use the base `github.com/kubernetes/ingress-nginx/deploy/baremetal` and execute `kubectl apply --kustomize .` !!! tip For extended notes regarding deployments on bare-metal, see [Bare-metal considerations](./baremetal.md). diff --git a/docs/user-guide/monitoring.md b/docs/user-guide/monitoring.md index f295302cc..ed5de73e7 100644 --- a/docs/user-guide/monitoring.md +++ b/docs/user-guide/monitoring.md @@ -9,7 +9,7 @@ This tutorial will show you how to install [Prometheus](https://prometheus.io/) The NGINX Ingress controller should already be deployed according to the deployment instructions [here](../deploy/index.md). -Note that the yaml files used in this tutorial are stored in the [deploy/monitoring](https://github.com/kubernetes/ingress-nginx/tree/master/deploy/monitoring) folder of the GitHub repository [kubernetes/ingress-nginx](https://github.com/kubernetes/ingress-nginx). +Note that the kustomize bases used in this tutorial are stored in the [deploy](https://github.com/kubernetes/ingress-nginx/tree/master/deploy) folder of the GitHub repository [kubernetes/ingress-nginx](https://github.com/kubernetes/ingress-nginx). ## Deploy and configure Prometheus Server @@ -17,22 +17,16 @@ The Prometheus server must be configured so that it can discover endpoints of se If there is no existing Prometheus server running, the rest of this tutorial will guide you through the steps needed to deploy a properly configured Prometheus server. -Running the following command deploys the prometheus configuration in Kubernetes: - -```console -kubectl create -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/monitoring/configuration.yaml -configmap "prometheus-configuration" created -``` - Running the following command deploys prometheus in Kubernetes: ```console -kubectl create -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/monitoring/prometheus.yaml -clusterrole "prometheus-server" created -serviceaccount "prometheus-server" created -clusterrolebinding "prometheus-server" created -deployment "prometheus-server" created -service "prometheus-server" created +kubectl apply --kustomize github.com/kubernetes/ingress-nginx/deploy/prometheus/ +serviceaccount/prometheus-server created +role.rbac.authorization.k8s.io/prometheus-server created +rolebinding.rbac.authorization.k8s.io/prometheus-server created +configmap/prometheus-configuration-bc6bcg7b65 created +service/prometheus-server created +deployment.apps/prometheus-server created ``` ### Prometheus Dashboard @@ -69,7 +63,7 @@ According to the above example, this URL will be http://10.192.0.3:32630 ### Grafana ```console -kubectl create -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/monitoring/grafana.yaml +kubectl apply --kustomize github.com/kubernetes/ingress-nginx/deploy/grafana/ ``` ```console diff --git a/hack/build-single-manifest-sh b/hack/build-single-manifest-sh deleted file mode 100755 index 0e236e782..000000000 --- a/hack/build-single-manifest-sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -# Copyright 2018 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -o errexit -set -o nounset -set -o pipefail - -SCRIPT_ROOT=$(dirname ${BASH_SOURCE})/.. - -INPUT="namespace.yaml configmap.yaml rbac.yaml with-rbac.yaml" -MANIFEST=$(cd ${SCRIPT_ROOT}/deploy; cat ${INPUT}) - -echo "${MANIFEST}" > ${SCRIPT_ROOT}/deploy/mandatory.yaml diff --git a/test/e2e-image/.gitignore b/test/e2e-image/.gitignore index 811b620e0..260b7c282 100644 --- a/test/e2e-image/.gitignore +++ b/test/e2e-image/.gitignore @@ -1,3 +1,5 @@ e2e.test ginkgo kubectl +/cloud-generic/ +/cluster-wide/ diff --git a/test/e2e-image/Dockerfile b/test/e2e-image/Dockerfile index efc981311..fbdff334f 100644 --- a/test/e2e-image/Dockerfile +++ b/test/e2e-image/Dockerfile @@ -15,7 +15,10 @@ RUN curl -Lo /usr/local/bin/kubectl \ COPY --from=BASE /go/bin/ginkgo /usr/local/bin/ COPY e2e.sh /e2e.sh -COPY manifests /manifests +COPY cloud-generic /cloud-generic +COPY cluster-wide /cluster-wide +COPY overlay /overlay +RUN sed -E -i 's|^- .*deploy/cloud-generic$|- ../cloud-generic|' /overlay/kustomization.yaml COPY wait-for-nginx.sh / COPY e2e.test / diff --git a/test/e2e-image/Makefile b/test/e2e-image/Makefile index f64d2c26a..f5ea8d88f 100644 --- a/test/e2e-image/Makefile +++ b/test/e2e-image/Makefile @@ -9,6 +9,8 @@ container: cp ../e2e/e2e.test . cp ../e2e/wait-for-nginx.sh . + cp -r ../../deploy/cloud-generic . + cp -r ../../deploy/cluster-wide . docker build -t $(IMAGE) . diff --git a/test/e2e-image/e2e.sh b/test/e2e-image/e2e.sh index f185c60ec..3178c8a3b 100755 --- a/test/e2e-image/e2e.sh +++ b/test/e2e-image/e2e.sh @@ -27,8 +27,6 @@ if [ ! -f ${HOME}/.kube/config ]; then kubectl config use-context default fi -kubectl apply -f manifests/rbac.yaml - ginkgo_args=( "-randomizeSuites" "-randomizeAllSpecs" diff --git a/test/e2e-image/manifests/mandatory.yaml b/test/e2e-image/manifests/mandatory.yaml deleted file mode 100644 index 7380a67a1..000000000 --- a/test/e2e-image/manifests/mandatory.yaml +++ /dev/null @@ -1,205 +0,0 @@ -kind: ConfigMap -apiVersion: v1 -metadata: - name: nginx-configuration - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx -data: - worker-processes: "1" - ---- -kind: ConfigMap -apiVersion: v1 -metadata: - name: tcp-services - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - ---- -kind: ConfigMap -apiVersion: v1 -metadata: - name: udp-services - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: nginx-ingress-serviceaccount - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: Role -metadata: - name: nginx-ingress-role - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx -rules: - - apiGroups: - - "" - resources: - - configmaps - - pods - - secrets - - namespaces - verbs: - - get - - apiGroups: - - "" - resources: - - configmaps - resourceNames: - # Defaults to "-" - # Here: "-" - # This has to be adapted if you change either parameter - # when launching the nginx-ingress-controller. - - "ingress-controller-leader-nginx" - verbs: - - get - - update - - apiGroups: - - "" - resources: - - configmaps - verbs: - - create - - apiGroups: - - "" - resources: - - endpoints - verbs: - - get - ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: RoleBinding -metadata: - name: nginx-ingress-role-${NAMESPACE} - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: nginx-ingress-role -subjects: - - kind: ServiceAccount - name: nginx-ingress-serviceaccount - namespace: ${NAMESPACE} - ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRoleBinding -metadata: - name: nginx-ingress-clusterrole-${NAMESPACE} - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: nginx-ingress-clusterrole -subjects: - - kind: ServiceAccount - name: nginx-ingress-serviceaccount - namespace: ${NAMESPACE} - ---- -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: nginx-ingress-controller - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx -spec: - replicas: 1 - selector: - matchLabels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - template: - metadata: - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: ingress-nginx - annotations: - prometheus.io/port: "10254" - prometheus.io/scrape: "true" - spec: - terminationGracePeriodSeconds: 0 - serviceAccountName: nginx-ingress-serviceaccount - initContainers: - - name: enable-coredump - image: busybox - command: - - /bin/sh - - -c - - | - ulimit -c unlimited - echo "/tmp/core.%e.%p" > /proc/sys/kernel/core_pattern - sysctl -w fs.suid_dumpable=2 - securityContext: - privileged: true - containers: - - name: nginx-ingress-controller - image: ingress-controller/nginx-ingress-controller:dev - args: - - /nginx-ingress-controller - - --configmap=$(POD_NAMESPACE)/nginx-configuration - - --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services - - --udp-services-configmap=$(POD_NAMESPACE)/udp-services - - --publish-service=$(POD_NAMESPACE)/ingress-nginx - - --annotations-prefix=nginx.ingress.kubernetes.io - - --watch-namespace=${NAMESPACE} - securityContext: - allowPrivilegeEscalation: true - capabilities: - drop: - - ALL - add: - - NET_BIND_SERVICE - # www-data -> 33 - runAsUser: 33 - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - ports: - - name: http - containerPort: 80 - - name: https - containerPort: 443 - livenessProbe: - failureThreshold: 3 - httpGet: - path: /healthz - port: 10254 - scheme: HTTP - initialDelaySeconds: 10 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 1 - readinessProbe: - failureThreshold: 3 - httpGet: - path: /healthz - port: 10254 - scheme: HTTP - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 1 diff --git a/test/e2e-image/overlay/deployment-e2e.yaml b/test/e2e-image/overlay/deployment-e2e.yaml new file mode 100644 index 000000000..f8df465a6 --- /dev/null +++ b/test/e2e-image/overlay/deployment-e2e.yaml @@ -0,0 +1,26 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-ingress-controller +spec: + template: + spec: + terminationGracePeriodSeconds: 0 + initContainers: + - name: enable-coredump + image: busybox + command: + - /bin/sh + - -c + - | + ulimit -c unlimited + echo "/tmp/core.%e.%p" > /proc/sys/kernel/core_pattern + sysctl -w fs.suid_dumpable=2 + securityContext: + privileged: true + containers: + - name: nginx-ingress-controller + livenessProbe: + timeoutSeconds: 1 + readinessProbe: + timeoutSeconds: 1 diff --git a/test/e2e-image/overlay/deployment-extension-group-patch.yaml b/test/e2e-image/overlay/deployment-extension-group-patch.yaml new file mode 100644 index 000000000..837a5f7e1 --- /dev/null +++ b/test/e2e-image/overlay/deployment-extension-group-patch.yaml @@ -0,0 +1,3 @@ +- op: replace + path: /apiVersion + value: extensions/v1beta1 diff --git a/test/e2e-image/overlay/deployment-namespace-patch.yaml b/test/e2e-image/overlay/deployment-namespace-patch.yaml new file mode 100644 index 000000000..f0f1fddd8 --- /dev/null +++ b/test/e2e-image/overlay/deployment-namespace-patch.yaml @@ -0,0 +1,3 @@ +- op: add + path: /spec/template/spec/containers/0/args/-1 + value: "--watch-namespace=$(POD_NAMESPACE)" diff --git a/test/e2e-image/overlay/kustomization.yaml b/test/e2e-image/overlay/kustomization.yaml new file mode 100644 index 000000000..a79dfb73c --- /dev/null +++ b/test/e2e-image/overlay/kustomization.yaml @@ -0,0 +1,34 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +bases: +- ../../../deploy/cloud-generic +configMapGenerator: +- name: nginx-configuration + behavior: merge + literals: + - worker-processes=1 +patchesStrategicMerge: +- deployment-e2e.yaml +- service-protocol-tcp.yaml +patchesJson6902: +- path: deployment-namespace-patch.yaml + target: + group: apps + kind: Deployment + name: nginx-ingress-controller + version: v1 +- path: service-cluster-patch.yaml + target: + kind: Service + name: ingress-nginx + version: v1 +- path: deployment-extension-group-patch.yaml + target: + group: apps + kind: Deployment + name: nginx-ingress-controller + version: v1 +images: +- name: quay.io/kubernetes-ingress-controller/nginx-ingress-controller + newName: ingress-controller/nginx-ingress-controller + newTag: dev diff --git a/test/e2e-image/overlay/service-cluster-patch.yaml b/test/e2e-image/overlay/service-cluster-patch.yaml new file mode 100644 index 000000000..0465d3804 --- /dev/null +++ b/test/e2e-image/overlay/service-cluster-patch.yaml @@ -0,0 +1,4 @@ +- op: remove + path: /spec/externalTrafficPolicy +- op: remove + path: /spec/type diff --git a/test/e2e-image/overlay/service-protocol-tcp.yaml b/test/e2e-image/overlay/service-protocol-tcp.yaml new file mode 100644 index 000000000..c49626fcd --- /dev/null +++ b/test/e2e-image/overlay/service-protocol-tcp.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: ingress-nginx +spec: + ports: + - name: http + port: 80 + targetPort: 80 + protocol: TCP + - name: https + port: 443 + targetPort: 443 + protocol: TCP diff --git a/test/e2e/defaultbackend/custom_default_backend.go b/test/e2e/defaultbackend/custom_default_backend.go index 463d302ae..4bdb9ed17 100644 --- a/test/e2e/defaultbackend/custom_default_backend.go +++ b/test/e2e/defaultbackend/custom_default_backend.go @@ -17,7 +17,6 @@ limitations under the License. package defaultbackend import ( - "fmt" "net/http" "strings" @@ -40,7 +39,7 @@ var _ = framework.IngressNginxDescribe("Custom Default Backend", func() { framework.UpdateDeployment(f.KubeClientSet, f.Namespace, "nginx-ingress-controller", 1, func(deployment *appsv1beta1.Deployment) error { args := deployment.Spec.Template.Spec.Containers[0].Args - args = append(args, fmt.Sprintf("--default-backend-service=%s/%s", f.Namespace, "http-svc")) + args = append(args, "--default-backend-service=$(POD_NAMESPACE)/http-svc") deployment.Spec.Template.Spec.Containers[0].Args = args _, err := f.KubeClientSet.AppsV1beta1().Deployments(f.Namespace).Update(deployment) diff --git a/test/e2e/settings/pod_security_policy.go b/test/e2e/settings/pod_security_policy.go index 58cd227c0..fbf833a34 100644 --- a/test/e2e/settings/pod_security_policy.go +++ b/test/e2e/settings/pod_security_policy.go @@ -17,6 +17,7 @@ limitations under the License. package settings import ( + "fmt" "net/http" "strings" @@ -48,7 +49,7 @@ var _ = framework.IngressNginxDescribe("[Serial] Pod Security Policies", func() Expect(err).NotTo(HaveOccurred(), "creating Pod Security Policy") } - role, err := f.KubeClientSet.RbacV1().ClusterRoles().Get("nginx-ingress-clusterrole", metav1.GetOptions{}) + role, err := f.KubeClientSet.RbacV1().ClusterRoles().Get(fmt.Sprintf("nginx-ingress-clusterrole-%v", f.Namespace), metav1.GetOptions{}) Expect(err).NotTo(HaveOccurred(), "getting ingress controller cluster role") Expect(role).NotTo(BeNil()) @@ -78,7 +79,7 @@ var _ = framework.IngressNginxDescribe("[Serial] Pod Security Policies", func() }) AfterEach(func() { - role, err := f.KubeClientSet.RbacV1().ClusterRoles().Get("nginx-ingress-clusterrole", metav1.GetOptions{}) + role, err := f.KubeClientSet.RbacV1().ClusterRoles().Get(fmt.Sprintf("nginx-ingress-clusterrole-%v", f.Namespace), metav1.GetOptions{}) Expect(err).NotTo(HaveOccurred(), "getting ingress controller cluster role") Expect(role).NotTo(BeNil()) diff --git a/test/e2e/wait-for-nginx.sh b/test/e2e/wait-for-nginx.sh index f4e01d968..a9c6658bd 100755 --- a/test/e2e/wait-for-nginx.sh +++ b/test/e2e/wait-for-nginx.sh @@ -35,10 +35,32 @@ function on_exit { } trap on_exit EXIT -kubectl apply --namespace=$NAMESPACE -f $DIR/manifests/service.yaml +CLUSTER_WIDE="$DIR/cluster-wide-$NAMESPACE" -sed "s@\${NAMESPACE}@${NAMESPACE}@" $DIR/manifests/mandatory.yaml | kubectl apply --namespace=$NAMESPACE -f - -cat $DIR/manifests/service.yaml | kubectl apply --namespace=$NAMESPACE -f - +mkdir "$CLUSTER_WIDE" + +cat << EOF > "$CLUSTER_WIDE/kustomization.yaml" +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +bases: +- ../cluster-wide +nameSuffix: "-$NAMESPACE" +EOF + +OVERLAY="$DIR/overlay-$NAMESPACE" + +mkdir "$OVERLAY" + +cat << EOF > "$OVERLAY/kustomization.yaml" +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: $NAMESPACE +bases: +- ../overlay +- ../cluster-wide-$NAMESPACE +EOF + +kubectl apply --kustomize "$OVERLAY" # wait for the deployment and fail if there is an error before starting the execution of any test kubectl rollout status \