Merge pull request #2863 from aledbf/replace-minikube

Replace minikube for e2e tests

.travis.yml

@@ -15,9 +15,10 @@ notifications:
 # New secure variables can be added using travis encrypt -r kubernetes/ingress-nginx --add K=V
 env:
   global:
-  - CHANGE_MINIKUBE_NONE_USER=true
-  - KUBERNETES_VERSION=v1.10.0
   - DOCKER=docker
+  - SKIP_SNAPSHOT=true
+  - NODE_IP=10.192.0.3
+  - E2E_NODES=6
   - GH_REF=github.com/kubernetes/ingress-nginx
   - secure: LIS2XpZufWTcJ53jiRsSZy2Gi1EUJ1XmLg7z3f2ZHeMnyG2Jhk3GW4vod1FNru+PY4PWgddLdCdIl+jqOYXndFlbdAWF3/Oy5fEkYLXdYV7tdlHcPWDkqNFrfiyZ4guChN+b2Nk6FqU7o5fsZAIR7VAbgqNRF5XMo9Mhn/vhDCQRcnbXy7uq7JTrYUkqDbQoyYvT6b480GCY5gags1zp/xZfPDNZEe936o8i5IPTyiykRyNOXN/AH6kd3pR5e1xYgcvJ9KpSVPghcwFE7kJ4fOVMRhRG5ML+IyML+xD0jX43EMNoqRKZ/HS42kIMCInFbJEcxVde7DPNBZ7Y3GAqh7HO6qrE70Dn3ha6DID6zCoH2ArW39BxG4zempjn2VxYoMRGREyZszWQb++dwGoHmo5FHt6zvIrYBG0dA0H8ja9VkZkjFwtYTGHU1ooPzUfJK4O4VBayV8LqZibyZQR+GrmyQc0aagUY7J/fe4A2PJyI4DbkeZ7GX1ELj0ciDz4urQSzUc8l/T3aU3X+FuJItjgYtMLPmqcjA5uifDCtutE8Z9L2gSpanqUdvLSOozuxPho/KNl+2YlF7fXqPW3LnRf5mHD+NbOff306pvKlHJOb2Vmth+HBQ1XDzt/Cy5+sfwS3E0Vmh6UTq/NtkUXxwH10BDMF7FMVlQ4zdHQvyZ0=
   - secure: rKDoy9IYYYy0fYBs4+9mwuBVq/TcxfFwMfE0ywYWhUUdgzrUYSJAwpoe/96EQ4YmESUefwC2nDNq4G3XzJKYOWf83PaIveb9Z//zmMrCQXjDuDBDLpwV3sXSh7evXiVDohJz4ogBCeMRUCMKYsyKBM9yWfa/iu+yI92dbphpK9peOKW6yBc0uspJlln4swN3GS2WT9LVuPY2Azv9U2UqrXufOPDKG/qEb/Vrn4yZ2lR/50r2k45e9nSvDoByvr10V8ubM5Zc0iP0vBuAUVRdByv6N53Q4gaBGapY6SxhIjIPC/h0rNnuT9EXp7MWaPT5FmBxLt9wnyleT9QhZJnFyaBYqFgcz/DKifYQkryY4M5dLMo/Rt3yATyAy8Y0df1TOoV2dKdqwOOwQ8bXB1wDfyrGxmQj9HY4Ffnphx3wPE1a+Sjuh+S5Epm7XJbPx5pZJqNO2hd4sTbk0Xp3gpPbihny2r/jtNwHl0wpFCfOM68RNrsVRlIwG3UhzbZvblbQ/M/mmWCdgzINjt07I2SGCJxfKG0e98Q49SKUoDoOgQTTRDqTC9IgOEDxyfAkT0Vr6BtlP88Nsgnf6kmboyigBrRAiaDQGTxn3SP6LnQI3CeopaRDYvFZe/rTwPXE9XlKoTn9FTWnAqF3MuWaLslDcDKYEh7OaYJjF01piu6g4Nc=
@@ -38,7 +39,6 @@ jobs:
   - stage: e2e
     before_script:
     - test/e2e/up.sh
-    - SKIP_MINIKUBE_START=true make dev-env
     script:
     - make e2e-test
   # split builds to avoid job timeouts

Makefile

@@ -26,7 +26,9 @@ GOHOSTOS ?= $(shell go env GOHOSTOS)
 # Allow limiting the scope of the e2e tests. By default run everything
 FOCUS ?= .*
 # number of parallel test
-E2E_NODES ?= 3
+E2E_NODES ?= 4
+# slow test only if takes > 40s
+SLOW_E2E_THRESHOLD ?= 40
 
 NODE_IP ?= $(shell minikube ip)
 
@@ -179,11 +181,12 @@ lua-test:
 
 .PHONY: e2e-test
 e2e-test:
-	@$(DEF_VARS)                 \
+	@$(DEF_VARS)                             \
-	FOCUS=$(FOCUS)               \
+	FOCUS=$(FOCUS)                           \
-	E2E_NODES=$(E2E_NODES)       \
+	E2E_NODES=$(E2E_NODES)                   \
-	DOCKER_OPTS="-i --net=host"  \
+	DOCKER_OPTS="-i --net=host"              \
-	NODE_IP=$(NODE_IP)           \
+	NODE_IP=$(NODE_IP)                       \
+	SLOW_E2E_THRESHOLD=$(SLOW_E2E_THRESHOLD) \
 	build/go-in-docker.sh build/e2e-tests.sh
 
 .PHONY: cover

build/build.sh

@@ -18,29 +18,25 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
-if [ -z "${PKG}" ]; then
+declare -a mandatory
-    echo "PKG must be set"
+mandatory=(
-    exit 1
+  PKG
-fi
+  ARCH
-if [ -z "${ARCH}" ]; then
+  GIT_COMMIT
-    echo "ARCH must be set"
+  REPO_INFO
-    exit 1
+  TAG
-fi
+)
-if [ -z "${GIT_COMMIT}" ]; then
+
-    echo "GIT_COMMIT must be set"
+missing=false
-    exit 1
+for var in ${mandatory[@]}; do
-fi
+  if [[ -z "${!var+x}" ]]; then
-if [ -z "${REPO_INFO}" ]; then
+    echo "Environment variable $var must be set"
-    echo "REPO_INFO must be set"
+    missing=true
-    exit 1
+  fi
-fi
+done
-if [ -z "${TAG}" ]; then
+
-    echo "TAG must be set"
+if [ "$missing" = true ];then
-    exit 1
+  exit 1
-fi
-if [ -z "${TAG}" ]; then
-    echo "TAG must be set"
-    exit 1
 fi
 
 export CGO_ENABLED=0

build/e2e-tests.sh

@@ -18,21 +18,25 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
-if [ -z "${PKG}" ]; then
+declare -a mandatory
-    echo "PKG must be set"
+mandatory=(
-    exit 1
+  NODE_IP
-fi
+  SLOW_E2E_THRESHOLD
-if [ -z "${FOCUS}" ]; then
+  PKG
-    echo "FOCUS must be set"
+  FOCUS
-    exit 1
+  E2E_NODES
-fi
+)
-if [ -z "${E2E_NODES}" ]; then
+
-    echo "E2E_NODES must be set"
+missing=false
-    exit 1
+for var in ${mandatory[@]}; do
-fi
+  if [[ -z "${!var+x}" ]]; then
-if [ -z "${NODE_IP}" ]; then
+    echo "Environment variable $var must be set"
-    echo "NODE_IP must be set"
+    missing=true
-    exit 1
+  fi
+done
+
+if [ "$missing" = true ];then
+  exit 1
 fi
 
 SCRIPT_ROOT=$(dirname ${BASH_SOURCE})/..
@@ -52,13 +56,14 @@ fi
 
 ginkgo build ./test/e2e
 
-exec --                 \
+exec --                                      \
-ginkgo                  \
+ginkgo                                       \
-    -randomizeSuites    \
+    -randomizeSuites                         \
-    -randomizeAllSpecs  \
+    -randomizeAllSpecs                       \
-    -flakeAttempts=2    \
+    -flakeAttempts=2                         \
-    --focus=${FOCUS}    \
+    --focus=${FOCUS}                         \
-    -p                  \
+    -p                                       \
-    -trace              \
+    -trace                                   \
-    -nodes=${E2E_NODES} \
+    -nodes=${E2E_NODES}                      \
+    -slowSpecThreshold=${SLOW_E2E_THRESHOLD} \
     test/e2e/e2e.test

build/go-in-docker.sh

@@ -18,29 +18,26 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
-if [ -z "${PKG}" ]; then
+declare -a mandatory
-    echo "PKG must be set"
+mandatory=(
-    exit 1
+  PKG
-fi
+  ARCH
-if [ -z "${ARCH}" ]; then
+  GIT_COMMIT
-    echo "ARCH must be set"
+  REPO_INFO
-    exit 1
+  TAG
-fi
+  HOME
-if [ -z "${GIT_COMMIT}" ]; then
+)
-    echo "GIT_COMMIT must be set"
+
-    exit 1
+missing=false
-fi
+for var in ${mandatory[@]}; do
-if [ -z "${REPO_INFO}" ]; then
+  if [[ -z "${!var+x}" ]]; then
-    echo "REPO_INFO must be set"
+    echo "Environment variable $var must be set"
-    exit 1
+    missing=true
-fi
+  fi
-if [ -z "${TAG}" ]; then
+done
-    echo "TAG must be set"
+
-    exit 1
+if [ "$missing" = true ];then
-fi
+  exit 1
-if [ -z "${HOME}" ]; then
-    echo "HOME must be set"
-    exit 1
 fi
 
 DOCKER_OPTS=${DOCKER_OPTS:-""}
@@ -53,7 +50,7 @@ tee .env << EOF
 PKG=${PKG:-""}
 ARCH=${ARCH:-""}
 GIT_COMMIT=${GIT_COMMIT:-""}
-E2E_NODES=${E2E_NODES:-3}
+E2E_NODES=${E2E_NODES:-4}
 FOCUS=${FOCUS:-.*}
 TAG=${TAG:-"0.0"}
 HOME=${HOME:-/root}
@@ -63,6 +60,7 @@ PWD=${PWD}
 BUSTED_ARGS=${BUSTED_ARGS:-""}
 REPO_INFO=${REPO_INFO:-local}
 NODE_IP=${NODE_IP:-127.0.0.1}
+SLOW_E2E_THRESHOLD=${SLOW_E2E_THRESHOLD:-40}
 EOF
 
 docker run                                       \

hack/boilerplate/boilerplate.py

@@ -50,6 +50,7 @@ args = parser.parse_args()
 
 verbose_out = sys.stderr if args.verbose else open("/dev/null", "w")
 
+
 def get_refs():
     refs = {}
 
@@ -63,6 +64,7 @@ def get_refs():
 
     return refs
 
+
 def file_passes(filename, refs, regexs):
     try:
         f = open(filename, 'r')
@@ -127,10 +129,19 @@ def file_passes(filename, refs, regexs):
 
     return True
 
+
 def file_extension(filename):
     return os.path.splitext(filename)[1].split(".")[-1].lower()
 
-skipped_dirs = ['.git', "vendor", "test/e2e/framework/framework.go", "test/e2e/generated/bindata.go", "hack/boilerplate/test", "internal/file/bindata.go"]
+
+skipped_dirs = [
+    '.git',
+    "vendor",
+    "test/e2e/framework/framework.go",
+    "hack/boilerplate/test",
+    "test/e2e/dind-cluster-v1.11.sh"
+]
+
 
 def normalize_files(files):
     newfiles = []
@@ -143,6 +154,7 @@ def normalize_files(files):
             newfiles[i] = os.path.join(args.rootdir, pathname)
     return newfiles
 
+
 def get_files(extensions):
     files = []
     if len(args.filenames) > 0:
@@ -170,19 +182,23 @@ def get_files(extensions):
             outfiles.append(pathname)
     return outfiles
 
+
 def get_regexs():
     regexs = {}
     # Search for "YEAR" which exists in the boilerplate, but shouldn't in the real thing
-    regexs["year"] = re.compile( 'YEAR' )
+    regexs["year"] = re.compile('YEAR')
     # dates can be 2014, 2015, 2016, ..., CURRENT_YEAR, company holder names can be anything
     years = range(2014, date.today().year + 1)
-    regexs["date"] = re.compile( '(%s)' % "|".join(map(lambda l: str(l), years)) )
+    regexs["date"] = re.compile(
+        '(%s)' % "|".join(map(lambda l: str(l), years)))
     # strip // +build \n\n build constraints
-    regexs["go_build_constraints"] = re.compile(r"^(// \+build.*\n)+\n", re.MULTILINE)
+    regexs["go_build_constraints"] = re.compile(
+        r"^(// \+build.*\n)+\n", re.MULTILINE)
     # strip #!.* from shell scripts
     regexs["shebang"] = re.compile(r"^(#!.*\n)\n*", re.MULTILINE)
     return regexs
 
+
 def main():
     regexs = get_regexs()
     refs = get_refs()
@@ -194,5 +210,6 @@ def main():
 
     return 0
 
+
 if __name__ == "__main__":
-  sys.exit(main())
+    sys.exit(main())

test/e2e/down.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# Copyright 2017 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+SCRIPT_ROOT=$(dirname ${BASH_SOURCE})/..
+
+export PATH="$HOME/.kubeadm-dind-cluster:$SCRIPT_ROOT/build:$PATH"
+
+dind-cluster-v1.11.sh down
+dind-cluster-v1.11.sh clean
+

test/e2e/up.sh

@@ -14,45 +14,37 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-export JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}'
+set -e
 
-echo "downloading kubectl..."
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/$KUBERNETES_VERSION/bin/linux/amd64/kubectl && \
-    chmod +x kubectl && sudo mv kubectl /usr/local/bin/
 
-echo "downloading minikube..."
+if test -e kubectl; then
-curl -Lo minikube https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64 && \
+  echo "skipping download of kubectl"
-    chmod +x minikube && \
+else
-    sudo mv minikube /usr/local/bin/
+  echo "downloading kubectl..."
+  curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/v1.11.0/bin/linux/amd64/kubectl && \
+      chmod +x kubectl && sudo mv kubectl /usr/local/bin/
+fi
 
-echo "starting minikube..."
+mkdir -p ${HOME}/.kube
-# Using a lower value for sync-frequency to speed up the tests (during the cleanup of resources inside a namespace)
+touch ${HOME}/.kube/config
+export KUBECONFIG=${HOME}/.kube/config
 
-export MINIKUBE_WANTUPDATENOTIFICATION=false
+echo "starting Kubernetes cluster..."
-export MINIKUBE_WANTREPORTERRORPROMPT=false
+$DIR/dind-cluster-v1.11.sh up
-export MINIKUBE_HOME=$HOME
-mkdir $HOME/.kube || true
-touch $HOME/.kube/config
 
-export KUBECONFIG=$HOME/.kube/config
+kubectl config use-context dind
 
-# workaround to avoid breaking the execution
+echo "Kubernetes cluster:"
-set +o errexit
+kubectl get nodes -o wide
-set +o pipefail
 
-# --vm-driver=none, use host docker (avoid docker-in-docker)
+export TAG=dev
-# --bootstrapper=localkube, works around https://github.com/kubernetes/minikube/issues/2704
+export ARCH=amd64
-sudo -E minikube start \
+export REGISTRY=${REGISTRY:-ingress-controller}
-    --vm-driver=none \
-    --bootstrapper=localkube \
-    --kubernetes-version=$KUBERNETES_VERSION \
-    --extra-config=kubelet.sync-frequency=1s \
-    --extra-config=apiserver.authorization-mode=RBAC
 
-minikube update-context
+echo "building container..."
+make -C ${DIR}/../../ build container
 
-echo "waiting for kubernetes cluster"
+echo "copying docker image to cluster..."
-until kubectl get nodes -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True";
+DEV_IMAGE=${REGISTRY}/nginx-ingress-controller:${TAG}
-do
+${DIR}/dind-cluster-v1.11.sh copy-image ${DEV_IMAGE}
-    sleep 1;
-done