From 0365a7c1728aef7c3f2e112ef97edf98df0c3e4e Mon Sep 17 00:00:00 2001
From: Manuel Alejandro de Brito Fontes <aledbf@gmail.com>
Date: Wed, 12 Feb 2020 20:19:57 -0300
Subject: [PATCH] Remove minikube and only use kind (#5059)

---
 build/dev-env.sh                              | 89 +++++++++++--------
 build/run-in-docker.sh                        |  8 --
 build/run-ingress-controller.sh               |  6 --
 deploy/{minikube => kind}/kustomization.yaml  |  2 +
 deploy/kind/service-hostport.yaml             | 25 ++++++
 docs/deploy/index.md                          |  1 -
 docs/development.md                           |  5 --
 .../third-party-addons/opentracing.md         |  3 +-
 8 files changed, 81 insertions(+), 58 deletions(-)
 rename deploy/{minikube => kind}/kustomization.yaml (82%)
 create mode 100644 deploy/kind/service-hostport.yaml

diff --git a/build/dev-env.sh b/build/dev-env.sh
index e535af5c4..dba3a9052 100755
--- a/build/dev-env.sh
+++ b/build/dev-env.sh
@@ -22,10 +22,7 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
-NAMESPACE="${NAMESPACE:-ingress-nginx}"
-echo "NAMESPACE is set to ${NAMESPACE}"
-
-kubectl config use-context minikube
+DIR=$(cd $(dirname "${BASH_SOURCE}") && pwd -P)
 
 export TAG=dev
 export ARCH=amd64
@@ -33,46 +30,66 @@ export REGISTRY=${REGISTRY:-ingress-controller}
 
 DEV_IMAGE=${REGISTRY}/nginx-ingress-controller:${TAG}
 
-{ [ "$(minikube status | grep -c Running)" -ge 2 ] && minikube status | grep -qE ': Configured$|Correctly Configured'; } \
-  || minikube start \
-    --extra-config=kubelet.sync-frequency=1s \
-    --extra-config=apiserver.authorization-mode=RBAC
+kind --version || $(echo "Please install kind.";exit 1)
 
-# shellcheck disable=SC2046
-eval $(minikube docker-env --shell bash)
+KUBE_CLIENT_VERSION=$(kubectl version --client --short | awk '{print $3}' | cut -d. -f2) || true
+if [[ ${KUBE_CLIENT_VERSION} -lt 14 ]]; then
+  echo "[dev-env] Please update kubectl to 1.15 or higher"
+fi
 
 echo "[dev-env] building container"
 make build container
 docker tag "${REGISTRY}/nginx-ingress-controller-${ARCH}:${TAG}" "${DEV_IMAGE}"
 
-# kubectl >= 1.14 includes Kustomize via "apply -k". Makes it easier to use on Linux as well, assuming kubectl installed
-KUBE_CLIENT_VERSION=$(kubectl version --client --short | awk '{print $3}' | cut -d. -f2) || true
-if [[ ${KUBE_CLIENT_VERSION} -lt 14 ]]; then
-  for tool in kubectl kustomize; do
-	echo "[dev-env] installing $tool"
-	$tool version || brew install $tool
-  done
-fi
+export K8S_VERSION=${K8S_VERSION:-v1.17.2@sha256:59df31fc61d1da5f46e8a61ef612fa53d3f9140f82419d1ef1a6b9656c6b737c}
 
-if ! kubectl get namespace "${NAMESPACE}"; then
-  kubectl create namespace "${NAMESPACE}"
-fi
+export DOCKER_CLI_EXPERIMENTAL=enabled
 
-kubectl get deploy nginx-ingress-controller -n "${NAMESPACE}" && kubectl delete deploy nginx-ingress-controller -n "${NAMESPACE}"
+KIND_CLUSTER_NAME="ingress-nginx-dev"
 
-ROOT=./deploy/minikube
-
-if [[ ${KUBE_CLIENT_VERSION} -lt 14 ]]; then
-  pushd $ROOT
-  kustomize edit set namespace "${NAMESPACE}"
-  kustomize edit set image "quay.io/kubernetes-ingress-controller/nginx-ingress-controller=${DEV_IMAGE}"
-  popd
-
-  echo "[dev-env] deploying NGINX Ingress controller in namespace $NAMESPACE"
-  kustomize build $ROOT | kubectl apply -f -
+if ! kind get clusters -q | grep -q ${KIND_CLUSTER_NAME}; then
+echo "[dev-env] creating Kubernetes cluster with kind"
+cat <<EOF | kind create cluster --name ${KIND_CLUSTER_NAME} --config=-
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+- role: control-plane
+  kubeadmConfigPatches:
+  - |
+    kind: InitConfiguration
+    nodeRegistration:
+      kubeletExtraArgs:
+        node-labels: "ingress-ready=true"
+        authorization-mode: "AlwaysAllow"
+  extraPortMappings:
+  - containerPort: 80
+    hostPort: 80
+    protocol: TCP
+  - containerPort: 443
+    hostPort: 443
+    protocol: TCP
+EOF
 else
-  sed -i -e "s|^namespace: .*|namespace: ${NAMESPACE}|g" "${ROOT}/kustomization.yaml"
-
-  echo "[dev-env] deploying NGINX Ingress controller in namespace $NAMESPACE"
-  kubectl apply -k "${ROOT}"
+  echo "[dev-env] using existing Kubernetes kind cluster"
 fi
+
+echo "[dev-env] copying docker images to cluster..."
+kind load docker-image --name="${KIND_CLUSTER_NAME}" "${DEV_IMAGE}"
+
+echo "[dev-env] deploying NGINX Ingress controller..."
+kubectl create namespace ingress-nginx || true
+kubectl apply -k ${DIR}/../deploy/kind
+
+echo "[dev-env] deleting old ingress-nginx pods..."
+kubectl get pods --namespace ingress-nginx -o go-template \
+  --template '{{range .items}}{{.metadata.name}} {{.metadata.creationTimestamp}}{{"\n"}}{{end}}' | \
+  awk '$2 <= "'$(date -d'now-1 minute' -Ins --utc | sed 's/+0000/Z/')'" { print $1 }' | \
+  xargs --no-run-if-empty kubectl delete pod --namespace ingress-nginx
+
+cat <<EOF
+
+Kubernetes cluster ready and ingress-nginx listening in localhost using ports 80 and 443
+
+To delete the dev cluster execute: 'kind delete cluster --name ingress-nginx-dev'
+
+EOF
diff --git a/build/run-in-docker.sh b/build/run-in-docker.sh
index 3a56e8af2..b4e1b01c0 100755
--- a/build/run-in-docker.sh
+++ b/build/run-in-docker.sh
@@ -45,13 +45,6 @@ FLAGS=$@
 PKG=k8s.io/ingress-nginx
 ARCH=$(go env GOARCH)
 
-MINIKUBE_PATH=${HOME}/.minikube
-MINIKUBE_VOLUME="-v ${MINIKUBE_PATH}:${MINIKUBE_PATH}"
-if [ ! -d "${MINIKUBE_PATH}" ]; then
-  echo "Minikube directory not found! Volume will be excluded from docker build."
-  MINIKUBE_VOLUME=""
-fi
-
 # create output directory as current user to avoid problem with docker.
 mkdir -p "${KUBE_ROOT}/bin" "${KUBE_ROOT}/bin/${ARCH}"
 
@@ -67,7 +60,6 @@ docker run                                            \
   -v "${KUBE_ROOT}/bin/${ARCH}:/go/bin/linux_${ARCH}" \
   -v "/var/run/docker.sock:/var/run/docker.sock"      \
   -v "${INGRESS_VOLUME}:/etc/ingress-controller/"     \
-  ${MINIKUBE_VOLUME}                                  \
   -w "/go/src/${PKG}"                                 \
   -u $(id -u ${USER}):$(id -g ${USER})                \
   ${E2E_IMAGE} /bin/bash -c "${FLAGS}"
diff --git a/build/run-ingress-controller.sh b/build/run-ingress-controller.sh
index a30bd07f8..57292cd71 100755
--- a/build/run-ingress-controller.sh
+++ b/build/run-ingress-controller.sh
@@ -94,11 +94,6 @@ until curl --output /dev/null -fsSL http://localhost:8001/; do
   sleep 5
 done
 
-MINIKUBE_VOLUME=
-if [[ -d "${HOME}/.minikube" ]]; then
-  MINIKUBE_VOLUME=" -v ${HOME}/.minikube:${HOME}/.minikube "
-fi
-
 # if we run as user we cannot bind to port 80 and 443
 docker run \
   --rm \
@@ -109,7 +104,6 @@ docker run \
   -e POD_NAME="${POD_NAME}" \
   -v "${SSL_VOLUME}:/etc/ingress-controller/ssl/" \
   -v "${HOME}/.kube:${HOME}/.kube:ro" \
-  ${MINIKUBE_VOLUME} \
   "${IMAGE}-${ARCH}:${TAG}" /nginx-ingress-controller \
   --update-status=false \
   --v=2 \
diff --git a/deploy/minikube/kustomization.yaml b/deploy/kind/kustomization.yaml
similarity index 82%
rename from deploy/minikube/kustomization.yaml
rename to deploy/kind/kustomization.yaml
index 5f857e6e1..8230fce40 100644
--- a/deploy/minikube/kustomization.yaml
+++ b/deploy/kind/kustomization.yaml
@@ -7,3 +7,5 @@ bases:
 images:
 - name: quay.io/kubernetes-ingress-controller/nginx-ingress-controller
   newTag: dev
+patchesStrategicMerge:
+- service-hostport.yaml
diff --git a/deploy/kind/service-hostport.yaml b/deploy/kind/service-hostport.yaml
new file mode 100644
index 000000000..ff0601cc1
--- /dev/null
+++ b/deploy/kind/service-hostport.yaml
@@ -0,0 +1,25 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-ingress-controller
+spec:
+  replicas: 1
+  template:
+    metadata:
+      annotations:
+        prometheus.io/port: "10254"
+        prometheus.io/scrape: "true"
+    spec:
+      containers:
+      - name: nginx-ingress-controller
+        ports:
+        - containerPort: 80
+          hostPort: 80
+        - containerPort: 443
+          hostPort: 443
+      nodeSelector:
+        ingress-ready: "true"
+      tolerations:
+      - key: node-role.kubernetes.io/master
+        operator: Equal
+        effect: NoSchedule
diff --git a/docs/deploy/index.md b/docs/deploy/index.md
index fc99c39e5..78f7506cf 100644
--- a/docs/deploy/index.md
+++ b/docs/deploy/index.md
@@ -78,7 +78,6 @@ minikube addons disable ingress
 ```console
 $ kubectl get pods -n ingress-nginx
 NAME                                       READY     STATUS    RESTARTS   AGE
-default-http-backend-66b447d9cf-rrlf9      1/1       Running   0          12s
 nginx-ingress-controller-fdcdcd6dd-vvpgs   1/1       Running   0          11s
 ```
 
diff --git a/docs/development.md b/docs/development.md
index 9a032ebbd..34ed3f777 100644
--- a/docs/development.md
+++ b/docs/development.md
@@ -19,11 +19,6 @@ cd ingress-nginx
 
 ### Initial developer environment build
 
->**Prequisites**: Minikube must be installed.
-See [releases](https://github.com/kubernetes/minikube/releases) for installation instructions.
-
-If you are using **MacOS** and deploying to **minikube**, the following command will build the local nginx controller container image and deploy the ingress controller onto a minikube cluster with RBAC enabled in the namespace `ingress-nginx`:
-
 ```
 $ make dev-env
 ```
diff --git a/docs/user-guide/third-party-addons/opentracing.md b/docs/user-guide/third-party-addons/opentracing.md
index ca7dc9890..0ced23000 100644
--- a/docs/user-guide/third-party-addons/opentracing.md
+++ b/docs/user-guide/third-party-addons/opentracing.md
@@ -100,8 +100,7 @@ All these options (including host) allow environment variables, such as `$HOSTNA
 
 ## Examples
 
-The following examples show how to deploy and test different distributed tracing systems. These example can be performed
-using Minikube.
+The following examples show how to deploy and test different distributed tracing systems. These example can be performed using Minikube.
 
 ### Zipkin