diff --git a/.gitignore b/.gitignore index 562090a30..2314eb193 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,10 @@ Session.vim # coverage artifacts .coverprofile /gover.coverprofile + +# skip go tests +*/**/*.test + +# skip e2e binaries +minikube +kubectl diff --git a/.travis.yml b/.travis.yml index 0b5767d5b..941f42ecb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,7 @@ notifications: email: true go: - - 1.8.3 + - 1.9 go_import_path: k8s.io/ingress @@ -18,9 +18,10 @@ env: # to add additional secure variables: # docker run --rm caktux/travis-cli encrypt key=value -r kubernetes/ingress - RELEASE="ci-${TRAVIS_BUILD_ID}" + - DOCKER=docker before_script: - - export PATH=$PATH:$PWD/hack/e2e-internal/ + - export PATH=$PATH:$PWD/controllers/nginx/e2e/e2e-internal jobs: include: @@ -34,4 +35,6 @@ jobs: - go get github.com/modocache/gover - if ! go get github.com/golang/tools/cmd/cover; then go get golang.org/x/tools/cmd/cover; fi - make cover -#- make test-e2e + - stage: e2e + script: + - make test-e2e diff --git a/Makefile b/Makefile index 2c1188922..13d4f72a4 100644 --- a/Makefile +++ b/Makefile @@ -2,15 +2,6 @@ all: fmt lint vet BUILDTAGS= -# building inside travis generates a custom version of the -# backends in order to run e2e tests agains the build. -ifdef TRAVIS_BUILD_ID - RELEASE := ci-build-${TRAVIS_BUILD_ID} -endif - -# 0.0 shouldn't clobber any release builds -RELEASE?=0.0 - # by default build a linux version GOOS?=linux @@ -23,7 +14,7 @@ endif # base package. It contains the common and backends code PKG := "k8s.io/ingress" -GO_LIST_FILES=$(shell go list ${PKG}/... | grep -v vendor | grep -v -e "test/e2e") +GO_LIST_FILES=$(shell go list ${PKG}/... | grep -v vendor | grep -v -e "e2e") .PHONY: fmt fmt: @@ -38,8 +29,8 @@ test: @go test -v -race -tags "$(BUILDTAGS) cgo" ${GO_LIST_FILES} .PHONY: test-e2e -test-e2e: ginkgo - @go run hack/e2e.go -v --up --test --down +test-e2e: + make -C controllers/nginx test-e2e .PHONY: cover cover: @@ -70,7 +61,3 @@ docker-push: .PHONE: release release: make -C controllers/nginx release - -.PHONY: ginkgo -ginkgo: - go get github.com/onsi/ginkgo/ginkgo diff --git a/controllers/nginx/Makefile b/controllers/nginx/Makefile index 3293c1be9..4c8676ae1 100644 --- a/controllers/nginx/Makefile +++ b/controllers/nginx/Makefile @@ -10,6 +10,8 @@ DOCKER?=gcloud docker -- SED_I?=sed -i GOHOSTOS ?= $(shell go env GOHOSTOS) +SKIP_TESTS?= + ifeq ($(GOHOSTOS),darwin) SED_I=sed -i '' endif @@ -120,11 +122,14 @@ lint: test: fmt lint vet @echo "+ $@" - @go test -v -race -tags "$(BUILDTAGS) cgo" $(shell go list ${PKG}/... | grep -v vendor) + @go test -v -race -tags "$(BUILDTAGS) cgo" $(shell go list ${PKG}/... | grep -v vendor |grep -v e2e) + +test-e2e: sub-container-amd64 + @TAG=${TAG} IMAGE=$(MULTI_ARCH_IMG) go run e2e/e2e.go --files=e2e/nginx-ingress-controller.yaml,e2e/default-backend.yaml --verbose --up --test --down cover: @echo "+ $@" - @go list -f '{{if len .TestGoFiles}}"go test -coverprofile={{.Dir}}/.coverprofile {{.ImportPath}}"{{end}}' $(shell go list ${PKG}/... | grep -v vendor) | xargs -L 1 sh -c + @go list -f '{{if len .TestGoFiles}}"go test -skipTest "${SKIP_TESTS}" -coverprofile={{.Dir}}/.coverprofile {{.ImportPath}}"{{end}}' $(shell go list ${PKG}/... | grep -v vendor) | xargs -L 1 sh -c gover goveralls -coverprofile=gover.coverprofile -service travis-ci -repotoken ${COVERALLS_TOKEN} diff --git a/controllers/nginx/e2e/default-backend.yaml b/controllers/nginx/e2e/default-backend.yaml new file mode 100644 index 000000000..3c40989a3 --- /dev/null +++ b/controllers/nginx/e2e/default-backend.yaml @@ -0,0 +1,51 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: default-http-backend + labels: + k8s-app: default-http-backend + namespace: kube-system +spec: + replicas: 1 + template: + metadata: + labels: + k8s-app: default-http-backend + spec: + terminationGracePeriodSeconds: 60 + containers: + - name: default-http-backend + # Any image is permissable as long as: + # 1. It serves a 404 page at / + # 2. It serves 200 on a /healthz endpoint + image: gcr.io/google_containers/defaultbackend:1.0 + livenessProbe: + httpGet: + path: /healthz + port: 8080 + scheme: HTTP + initialDelaySeconds: 30 + timeoutSeconds: 5 + ports: + - containerPort: 8080 + resources: + limits: + cpu: 10m + memory: 20Mi + requests: + cpu: 10m + memory: 20Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: default-http-backend + namespace: kube-system + labels: + k8s-app: default-http-backend +spec: + ports: + - port: 80 + targetPort: 8080 + selector: + k8s-app: default-http-backend diff --git a/controllers/nginx/e2e/e2e-internal/e2e-down.sh b/controllers/nginx/e2e/e2e-internal/e2e-down.sh new file mode 100755 index 000000000..0df5401d4 --- /dev/null +++ b/controllers/nginx/e2e/e2e-internal/e2e-down.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +. ./e2e/e2e-internal/e2e-env.sh + +echo "Destroying running e2e cluster..." +${MINIKUBE} --profile ${MINIKUBE_PROFILE} delete || echo "Cluster already destroyed" diff --git a/controllers/nginx/e2e/e2e-internal/e2e-env.sh b/controllers/nginx/e2e/e2e-internal/e2e-env.sh new file mode 100755 index 000000000..adc0c7e98 --- /dev/null +++ b/controllers/nginx/e2e/e2e-internal/e2e-env.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +export MINIKUBE_VERSION=0.22.0 +export K8S_VERSION=v1.7.5 + +export PWD=`pwd` +export BASEDIR="$(dirname ${BASH_SOURCE})" +export KUBECTL="${BASEDIR}/kubectl" +export MINIKUBE="${BASEDIR}/minikube" +export GOOS="${GOOS:-linux}" + +export MINIKUBE_WANTUPDATENOTIFICATION=false +export MINIKUBE_WANTREPORTERRORPROMPT=false +export MINIKUBE_HOME=$HOME +export CHANGE_MINIKUBE_NONE_USER=true + +export KUBECONFIG=$HOME/.kube/config + +export MINIKUBE_PROFILE="ingress-e2e" + +export PATH=$PATH:$BASEDIR + +if [ ! -e ${KUBECTL} ]; then + echo "kubectl binary is missing. downloading..." + curl -sSL http://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/${GOOS}/amd64/kubectl -o ${KUBECTL} + chmod u+x ${KUBECTL} +fi + +if [ ! -e ${MINIKUBE} ]; then + echo "minikube binary is missing. downloading..." + curl -sSLo ${MINIKUBE} https://storage.googleapis.com/minikube/releases/v${MINIKUBE_VERSION}/minikube-linux-amd64 + chmod +x ${MINIKUBE} +fi diff --git a/controllers/nginx/e2e/e2e-internal/e2e-status.sh b/controllers/nginx/e2e/e2e-internal/e2e-status.sh new file mode 100755 index 000000000..933fc5ce3 --- /dev/null +++ b/controllers/nginx/e2e/e2e-internal/e2e-status.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +set -eof pipefail + +. ./e2e/e2e-internal/e2e-env.sh + +${MINIKUBE} --profile ${MINIKUBE_PROFILE} status diff --git a/controllers/nginx/e2e/e2e-internal/e2e-up.sh b/controllers/nginx/e2e/e2e-internal/e2e-up.sh new file mode 100755 index 000000000..00275e86c --- /dev/null +++ b/controllers/nginx/e2e/e2e-internal/e2e-up.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +set -eof pipefail + +. ./e2e/e2e-internal/e2e-env.sh + +mkdir -p $HOME/.kube +touch $KUBECONFIG + +if [ "$TRAVIS" = true ] ; then + sudo -E ${MINIKUBE} --profile ${MINIKUBE_PROFILE} start --vm-driver=none +else + ${MINIKUBE} --profile ${MINIKUBE_PROFILE} start +fi + +# this for loop waits until kubectl can access the api server that minikube has created +for i in {1..150} # timeout for 5 minutes +do + $KUBECTL get po &> /dev/null + if [ $? -ne 1 ]; then + break + fi + sleep 10 +done + +sleep 60 + +echo "Kubernetes started" diff --git a/controllers/nginx/e2e/e2e-internal/run-e2e.sh b/controllers/nginx/e2e/e2e-internal/run-e2e.sh new file mode 100755 index 000000000..8678a468b --- /dev/null +++ b/controllers/nginx/e2e/e2e-internal/run-e2e.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +set -eof pipefail + +. ./e2e/e2e-internal/e2e-env.sh + +echo "Creating test tag for image $IMAGE:$TAG" +docker tag $IMAGE:$TAG $IMAGE:test + +echo "Uploading test image to minikube" +dockerenv=$(${MINIKUBE} --profile ${MINIKUBE_PROFILE} docker-env | sed 's/export//g' | sed 's/^#.*$//g' | sed 's/"//g') +docker save $IMAGE:test | env -i $dockerenv docker load + +echo "Running tests..." +go test -v k8s.io/ingress/controllers/nginx/e2e/... -run ^TestIngressSuite$ --args --alsologtostderr --v=10 diff --git a/hack/e2e.go b/controllers/nginx/e2e/e2e.go similarity index 77% rename from hack/e2e.go rename to controllers/nginx/e2e/e2e.go index be9f2aa28..ccce1d0ea 100644 --- a/hack/e2e.go +++ b/controllers/nginx/e2e/e2e.go @@ -1,5 +1,5 @@ /* -Copyright 2014 The Kubernetes Authors. +Copyright 2017 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -31,14 +31,14 @@ import ( ) var ( - build = flag.Bool("build", true, "Build the backends images indicated by the env var BACKENDS required to run e2e tests.") up = flag.Bool("up", true, "Creates a kubernetes cluster using hyperkube (containerized kubelet).") down = flag.Bool("down", true, "destroys the created cluster.") - test = flag.Bool("test", true, "Run Ginkgo tests.") + test = flag.Bool("test", true, "Run tests.") dump = flag.String("dump", "", "If set, dump cluster logs to this location on test or cluster-up failure") - testArgs = flag.String("test-args", "", "Space-separated list of arguments to pass to Ginkgo test runner.") + testArgs = flag.String("test-args", "", "Space-separated list of arguments to pass to the test runner.") deployment = flag.String("deployment", "bash", "up/down mechanism") - verbose = flag.Bool("v", false, "If true, print all command output.") + verbose = flag.Bool("verbose", false, "If true, print all command output.") + files = flag.String("files", "", "Path to a file/S descriptor that will create an Ingress controller") ) func appendError(errs []error, err error) []error { @@ -57,7 +57,7 @@ func validWorkingDirectory() error { if err != nil { return fmt.Errorf("failed to convert %s to an absolute path: %v", cwd, err) } - if !strings.Contains(filepath.Base(acwd), "ingress") { + if !strings.Contains(filepath.Base(acwd), "nginx") { return fmt.Errorf("must run from git root directory: %v", acwd) } return nil @@ -139,14 +139,12 @@ func main() { } func run(deploy deployer) error { - if *dump != "" { - defer writeXML(time.Now()) + if *files == "" { + return fmt.Errorf("missing required flag --files") } - if *build { - if err := xmlWrap("Build", Build); err != nil { - return fmt.Errorf("error building: %s", err) - } + if *dump != "" { + defer writeXML(time.Now()) } if *up { @@ -172,7 +170,7 @@ func run(deploy deployer) error { return fmt.Errorf("starting e2e cluster: %s", err) } if *dump != "" { - cmd := exec.Command("./cluster/kubectl.sh", "--match-server-version=false", "get", "nodes", "-oyaml") + cmd := kubectlCmd("get", "nodes", "-oyaml") b, err := cmd.CombinedOutput() if *verbose { log.Printf("kubectl get nodes:\n%s", string(b)) @@ -187,17 +185,22 @@ func run(deploy deployer) error { } } + log.Printf("deploying ingress controller") + if err := deploy.SetupController(*files); err != nil { + errs = appendError(errs, err) + } + if *test { if err := xmlWrap("IsUp", deploy.IsUp); err != nil { errs = appendError(errs, err) } else { - errs = appendError(errs, Test()) + errs = appendError(errs, runTests()) } } if len(errs) > 0 && *dump != "" { errs = appendError(errs, xmlWrap("DumpClusterLogs", func() error { - return DumpClusterLogs(*dump) + return dumpClusterLogs(*dump) })) } @@ -208,24 +211,14 @@ func run(deploy deployer) error { if len(errs) != 0 { return fmt.Errorf("encountered %d errors: %v", len(errs), errs) } - return nil -} -func Build() error { - // The build-release script needs stdin to ask the user whether - // it's OK to download the docker image. - cmd := exec.Command("make", "docker-build") - cmd.Stdin = os.Stdin - if err := finishRunning("build-release", cmd); err != nil { - return fmt.Errorf("error building: %v", err) - } return nil } type deployer interface { Up() error IsUp() error - SetupKubecfg() error + SetupController(p string) error Down() error } @@ -241,31 +234,36 @@ func getDeployer() (deployer, error) { type bash struct{} func (b bash) Up() error { - return finishRunning("up", exec.Command("./hack/e2e-internal/e2e-up.sh")) + return finishRunning("up", exec.Command("./e2e/e2e-internal/e2e-up.sh")) } func (b bash) IsUp() error { - return finishRunning("get status", exec.Command("./hack/e2e-internal/e2e-status.sh")) + return finishRunning("get status", exec.Command("./e2e/e2e-internal/e2e-status.sh")) } -func (b bash) SetupKubecfg() error { +func (b bash) SetupController(p string) error { + files := strings.Split(p, ",") + for _, f := range files { + err := finishRunning("setup controller", kubectlCmd("create", "-f", f)) + if err != nil { + return err + } + } + return nil } func (b bash) Down() error { - return finishRunning("teardown", exec.Command("./hack/e2e-internal/e2e-down.sh")) + return finishRunning("teardown", exec.Command("./e2e/e2e-internal/e2e-down.sh")) } -func DumpClusterLogs(location string) error { +func dumpClusterLogs(location string) error { log.Printf("Dumping cluster logs to: %v", location) - return finishRunning("dump cluster logs", exec.Command("./hack/e2e-internal/log-dump.sh", location)) + return finishRunning("dump cluster logs", exec.Command("./e2e/e2e-internal/log-dump.sh", location)) } -func Test() error { - if *testArgs == "" { - *testArgs = "--ginkgo.focus=\\[Feature:Ingress\\]" - } - return finishRunning("Ginkgo tests", exec.Command("./hack/e2e-internal/ginkgo-e2e.sh", strings.Fields(*testArgs)...)) +func runTests() error { + return finishRunning("Ingress tests", exec.Command("./e2e/e2e-internal/run-e2e.sh", strings.Fields(*testArgs)...)) } func finishRunning(stepName string, cmd *exec.Cmd) error { @@ -283,3 +281,15 @@ func finishRunning(stepName string, cmd *exec.Cmd) error { } return nil } + +func kubectlCmd(arg ...string) *exec.Cmd { + args := []string{"--context", "ingress-e2e"} + args = append(args, arg...) + + kb := os.Getenv("KUBECTL") + if kb == "" { + kb = "./e2e/e2e-internal/kubectl" + } + + return exec.Command(kb, args...) +} diff --git a/controllers/nginx/e2e/e2e_types.go b/controllers/nginx/e2e/e2e_types.go new file mode 100644 index 000000000..3d2ea636b --- /dev/null +++ b/controllers/nginx/e2e/e2e_types.go @@ -0,0 +1,62 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations +*/ + +package main + +import ( + "time" + + apiv1 "k8s.io/api/core/v1" + extensions "k8s.io/api/extensions/v1beta1" +) + +// IngressTestCase defines a test case for Ingress +type IngressTestCase struct { + Name string `json:"name"` + Description string `json:"description"` + Ingress *extensions.Ingress `json:"ingress"` + ReplicationController *apiv1.ReplicationController `json:"replicationController,omitempty"` + Deployment *extensions.Deployment `json:"deployment,omitempty"` + Service *apiv1.Service `json:"service"` + Assert []*Assert `json:"tests"` +} + +// Assert defines a verification over the +type Assert struct { + Name string `json:"name"` + Request Request `json:"request"` + Expect []*Expect `json:"expect"` + Timeout time.Duration `json:"timeout"` +} + +// Request defines a HTTP/s request to be executed against an Ingress +type Request struct { + Method string `json:"method"` + URL string `json:"url"` + Query map[string]interface{} `json:"query"` + Form map[string]interface{} `json:"form"` + Body interface{} `json:"body"` + Headers map[string]string `json:"headers"` +} + +// Expect defines the required conditions that must be true from a request response +type Expect struct { + Body []byte `json:"body"` + ContentType string `json:"contentType"` + Header []string `json:"header"` + HeaderAndValue map[string]string `json:"headerAndValue"` + Statuscode int `json:"statusCode"` +} diff --git a/controllers/nginx/e2e/e2e_types_test.go b/controllers/nginx/e2e/e2e_types_test.go new file mode 100644 index 000000000..f6c512743 --- /dev/null +++ b/controllers/nginx/e2e/e2e_types_test.go @@ -0,0 +1,40 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations +*/ + +package main + +import ( + "testing" +) + +func TestReadYamlCase(t *testing.T) { + itc, err := parseTestCase("suite/0001.yaml") + if err != nil { + t.Fatalf("unexpected error reading test case 0001: %v", err) + } + + if itc == nil { + t.Fatal("unexpected decoding of test case 0001") + } + + if itc.ReplicationController != nil { + t.Fatal("unexpected replication controller in test case 0001") + } + + if len(itc.Assert) != 1 { + t.Fatalf("expected 1 tests but %v returned", len(itc.Assert)) + } +} diff --git a/controllers/nginx/e2e/ingress_test.go b/controllers/nginx/e2e/ingress_test.go new file mode 100644 index 000000000..bb2253244 --- /dev/null +++ b/controllers/nginx/e2e/ingress_test.go @@ -0,0 +1,161 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations +*/ + +package main + +import ( + "fmt" + "io/ioutil" + "os" + "path" + "path/filepath" + "strings" + "testing" + + "github.com/ghodss/yaml" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + + "k8s.io/ingress/controllers/nginx/e2e/util" +) + +func TestIngressSuite(t *testing.T) { + client, err := util.GetClient() + if err != nil { + t.Fatalf("unexpected error creating k8s client: %v", err) + } + + pwd, _ := os.Getwd() + filepath.Walk(path.Join(pwd, "/suite"), + func(path string, info os.FileInfo, err error) error { + if strings.HasSuffix(path, ".yaml") { + t.Log(path) + runTestCase(path, client, t) + } + return nil + }) +} + +func runTestCase(rawtc string, client kubernetes.Interface, t *testing.T) { + tc, err := parseTestCase(rawtc) + if err != nil { + t.Fatalf("unexpected error reading Ingress test case file %v: %v", rawtc, err) + } + + t.Run(tc.Name, func(t *testing.T) { + if len(tc.Assert) == 0 { + t.Fatal("test case does not contains tests") + } + + if tc.Ingress == nil { + t.Fatal("the test case does not contains an Ingress rule") + } + + t.Logf("starting deploy of requirements for test case '%v'", tc.Name) + err := tc.deploy(client) + if err != nil { + t.Fatalf("unexpected error in test case deploy process: %v", err) + } + + for _, assert := range tc.Assert { + t.Logf("running assert %v", assert.Name) + } + + err = tc.undeploy(client) + if err != nil { + t.Fatalf("unexpected error in test case deploy process: %v", err) + } + }) +} + +// parseTestCase parses a test case from a yaml file +func parseTestCase(p string) (*IngressTestCase, error) { + file, err := ioutil.ReadFile(p) + if err != nil { + return nil, err + } + + var itc IngressTestCase + err = yaml.Unmarshal(file, &itc) + if err != nil { + return nil, err + } + + return &itc, nil +} + +// deploy creates the kubernetes object specified in the test case +func (tc IngressTestCase) deploy(client kubernetes.Interface) error { + _, err := client.Extensions().Ingresses(getNamespace(tc.Ingress.Namespace)).Create(tc.Ingress) + if err != nil { + return err + } + + if tc.Service != nil { + _, err := client.CoreV1().Services(getNamespace(tc.Service.Namespace)).Create(tc.Service) + if err != nil { + return err + } + } + + if tc.ReplicationController != nil { + _, err := client.CoreV1().ReplicationControllers(getNamespace(tc.ReplicationController.Namespace)).Create(tc.ReplicationController) + return err + } else if tc.Deployment != nil { + _, err := client.Extensions().Deployments(getNamespace(tc.Deployment.Namespace)).Create(tc.Deployment) + return err + } + + return fmt.Errorf("invalid deployment option. Please check the test case") +} + +// undeploy removes the kubernetes object created by the test case +func (tc IngressTestCase) undeploy(client kubernetes.Interface) error { + err := client.Extensions().Ingresses(getNamespace(tc.Ingress.Namespace)).Delete(tc.Ingress.Name, &metav1.DeleteOptions{}) + if err != nil { + return err + } + + if tc.Service != nil { + err := client.CoreV1().Services(getNamespace(tc.Service.Namespace)).Delete(tc.Service.Name, &metav1.DeleteOptions{}) + if err != nil { + return err + } + } + + if tc.ReplicationController != nil { + err := client.CoreV1().ReplicationControllers(getNamespace(tc.ReplicationController.Namespace)).Delete(tc.ReplicationController.Name, &metav1.DeleteOptions{}) + if err != nil { + return err + } + } else if tc.Deployment != nil { + err := client.Extensions().Deployments(getNamespace(tc.Deployment.Namespace)).Delete(tc.Deployment.Name, &metav1.DeleteOptions{}) + if err != nil { + return err + } + } + + return nil +} + +func getNamespace(ns string) string { + if ns == "" { + return "default" + } + + return ns +} diff --git a/controllers/nginx/e2e/nginx-ingress-controller.yaml b/controllers/nginx/e2e/nginx-ingress-controller.yaml new file mode 100644 index 000000000..e04351e0b --- /dev/null +++ b/controllers/nginx/e2e/nginx-ingress-controller.yaml @@ -0,0 +1,47 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: nginx-ingress-controller + labels: + k8s-app: nginx-ingress-controller + namespace: kube-system +spec: + replicas: 1 + template: + metadata: + labels: + k8s-app: nginx-ingress-controller + spec: + terminationGracePeriodSeconds: 60 + containers: + - image: gcr.io/google_containers/nginx-ingress-controller-amd64:test + name: nginx-ingress-controller + readinessProbe: + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + livenessProbe: + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + initialDelaySeconds: 10 + timeoutSeconds: 1 + ports: + - containerPort: 80 + hostPort: 80 + - containerPort: 443 + hostPort: 443 + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + args: + - /nginx-ingress-controller + - --default-backend-service=$(POD_NAMESPACE)/default-http-backend diff --git a/controllers/nginx/e2e/suite/0001.yaml b/controllers/nginx/e2e/suite/0001.yaml new file mode 100644 index 000000000..645612209 --- /dev/null +++ b/controllers/nginx/e2e/suite/0001.yaml @@ -0,0 +1,62 @@ +name: "001" +description: "simple test" +ingress: + apiVersion: extensions/v1beta1 + kind: Ingress + metadata: + name: echomap + spec: + rules: + - host: foo.bar.com + http: + paths: + - path: / + backend: + serviceName: echoheaders + servicePort: 80 +deployment: + apiVersion: extensions/v1beta1 + kind: Deployment + metadata: + name: http-svc + labels: + k8s-app: http-svc + spec: + replicas: 1 + template: + metadata: + labels: + k8s-app: http-svc + spec: + containers: + - name: http-svc + image: gcr.io/google_containers/echoserver:1.8 + livenessProbe: + httpGet: + path: /healthz + port: 8080 + scheme: HTTP + initialDelaySeconds: 30 + timeoutSeconds: 5 + ports: + - containerPort: 8080 +service: + apiVersion: v1 + kind: Service + metadata: + name: http-svc + labels: + k8s-app: http-svc + spec: + ports: + - port: 80 + targetPort: 8080 + selector: + k8s-app: http-svc +tests: + - name: "get /" + expect: + - statusCode: 200 + request: + method: GET + url: / diff --git a/controllers/nginx/e2e/util/util.go b/controllers/nginx/e2e/util/util.go new file mode 100644 index 000000000..eccf7d5f5 --- /dev/null +++ b/controllers/nginx/e2e/util/util.go @@ -0,0 +1,223 @@ +/* +Copyright 2017 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "fmt" + "os" + "testing" + "time" + + "github.com/pkg/errors" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/apimachinery/pkg/watch" + + "k8s.io/api/core/v1" + apierrs "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/client-go/kubernetes" + + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/tools/clientcmd" +) + +type PodStore struct { + cache.Store + stopCh chan struct{} + Reflector *cache.Reflector +} + +func (s *PodStore) List() []*v1.Pod { + objects := s.Store.List() + pods := make([]*v1.Pod, 0) + for _, o := range objects { + pods = append(pods, o.(*v1.Pod)) + } + return pods +} + +func (s *PodStore) Stop() { + close(s.stopCh) +} + +func GetClient() (kubernetes.Interface, error) { + profile := os.Getenv("MINIKUBE_PROFILE") + + loadingRules := clientcmd.NewDefaultClientConfigLoadingRules() + configOverrides := &clientcmd.ConfigOverrides{} + configOverrides.CurrentContext = profile + kubeConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, configOverrides) + config, err := kubeConfig.ClientConfig() + if err != nil { + return nil, fmt.Errorf("Error creating kubeConfig: %s", err) + } + client, err := kubernetes.NewForConfig(config) + if err != nil { + return nil, errors.Wrap(err, "Error creating new client from kubeConfig.ClientConfig()") + } + return client, nil +} + +func NewPodStore(c kubernetes.Interface, namespace string, label labels.Selector, field fields.Selector) *PodStore { + lw := &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + options.LabelSelector = label.String() + options.FieldSelector = field.String() + obj, err := c.Core().Pods(namespace).List(options) + return runtime.Object(obj), err + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + options.LabelSelector = label.String() + options.FieldSelector = field.String() + return c.Core().Pods(namespace).Watch(options) + }, + } + store := cache.NewStore(cache.MetaNamespaceKeyFunc) + stopCh := make(chan struct{}) + reflector := cache.NewReflector(lw, &v1.Pod{}, store, 0) + reflector.Run(stopCh) + return &PodStore{Store: store, stopCh: stopCh, Reflector: reflector} +} + +func StartPods(c kubernetes.Interface, namespace string, pod v1.Pod, waitForRunning bool) error { + pod.ObjectMeta.Labels["name"] = pod.Name + if waitForRunning { + label := labels.SelectorFromSet(labels.Set(map[string]string{"name": pod.Name})) + err := WaitForPodsWithLabelRunning(c, namespace, label) + if err != nil { + return fmt.Errorf("Error waiting for pod %s to be running: %v", pod.Name, err) + } + } + return nil +} + +// Wait up to 10 minutes for all matching pods to become Running and at least one +// matching pod exists. +func WaitForPodsWithLabelRunning(c kubernetes.Interface, ns string, label labels.Selector) error { + running := false + PodStore := NewPodStore(c, ns, label, fields.Everything()) + defer PodStore.Stop() +waitLoop: + for start := time.Now(); time.Since(start) < 10*time.Minute; time.Sleep(250 * time.Millisecond) { + pods := PodStore.List() + if len(pods) == 0 { + continue waitLoop + } + for _, p := range pods { + if p.Status.Phase != v1.PodRunning { + continue waitLoop + } + } + running = true + break + } + if !running { + return fmt.Errorf("Timeout while waiting for pods with labels %q to be running", label.String()) + } + return nil +} + +// WaitForRCToStabilize waits till the RC has a matching generation/replica count between spec and status. +func WaitForRCToStabilize(t *testing.T, c kubernetes.Interface, ns, name string, timeout time.Duration) error { + options := metav1.ListOptions{FieldSelector: fields.Set{ + "metadata.name": name, + "metadata.namespace": ns, + }.AsSelector().String()} + w, err := c.Core().ReplicationControllers(ns).Watch(options) + if err != nil { + return err + } + _, err = watch.Until(timeout, w, func(event watch.Event) (bool, error) { + switch event.Type { + case watch.Deleted: + return false, apierrs.NewNotFound(schema.GroupResource{Resource: "replicationcontrollers"}, "") + } + switch rc := event.Object.(type) { + case *v1.ReplicationController: + if rc.Name == name && rc.Namespace == ns && + rc.Generation <= rc.Status.ObservedGeneration && + *(rc.Spec.Replicas) == rc.Status.Replicas { + return true, nil + } + t.Logf("Waiting for rc %s to stabilize, generation %v observed generation %v spec.replicas %d status.replicas %d", + name, rc.Generation, rc.Status.ObservedGeneration, *(rc.Spec.Replicas), rc.Status.Replicas) + } + return false, nil + }) + return err +} + +// WaitForService waits until the service appears (exist == true), or disappears (exist == false) +func WaitForService(t *testing.T, c kubernetes.Interface, namespace, name string, exist bool, interval, timeout time.Duration) error { + err := wait.PollImmediate(interval, timeout, func() (bool, error) { + _, err := c.Core().Services(namespace).Get(name, metav1.GetOptions{}) + switch { + case err == nil: + t.Logf("Service %s in namespace %s found.", name, namespace) + return exist, nil + case apierrs.IsNotFound(err): + t.Logf("Service %s in namespace %s disappeared.", name, namespace) + return !exist, nil + case !IsRetryableAPIError(err): + t.Logf("Non-retryable failure while getting service.") + return false, err + default: + t.Logf("Get service %s in namespace %s failed: %v", name, namespace, err) + return false, nil + } + }) + if err != nil { + stateMsg := map[bool]string{true: "to appear", false: "to disappear"} + return fmt.Errorf("error waiting for service %s/%s %s: %v", namespace, name, stateMsg[exist], err) + } + return nil +} + +//WaitForServiceEndpointsNum waits until the amount of endpoints that implement service to expectNum. +func WaitForServiceEndpointsNum(t *testing.T, c kubernetes.Interface, namespace, serviceName string, expectNum int, interval, timeout time.Duration) error { + return wait.Poll(interval, timeout, func() (bool, error) { + t.Logf("Waiting for amount of service:%s endpoints to be %d", serviceName, expectNum) + list, err := c.Core().Endpoints(namespace).List(metav1.ListOptions{}) + if err != nil { + return false, err + } + + for _, e := range list.Items { + if e.Name == serviceName && countEndpointsNum(&e) == expectNum { + return true, nil + } + } + return false, nil + }) +} + +func countEndpointsNum(e *v1.Endpoints) int { + num := 0 + for _, sub := range e.Subsets { + num += len(sub.Addresses) + } + return num +} + +func IsRetryableAPIError(err error) bool { + return apierrs.IsTimeout(err) || apierrs.IsServerTimeout(err) || apierrs.IsTooManyRequests(err) || apierrs.IsInternalError(err) +} diff --git a/core/pkg/ingress/controller/annotations.go b/core/pkg/ingress/controller/annotations.go index af2037dc7..11ef016fc 100644 --- a/core/pkg/ingress/controller/annotations.go +++ b/core/pkg/ingress/controller/annotations.go @@ -19,6 +19,7 @@ package controller import ( "github.com/golang/glog" extensions "k8s.io/api/extensions/v1beta1" + "k8s.io/ingress/core/pkg/ingress/annotations/alias" "k8s.io/ingress/core/pkg/ingress/annotations/auth" "k8s.io/ingress/core/pkg/ingress/annotations/authreq" diff --git a/hack/e2e-internal/e2e-down.sh b/hack/e2e-internal/e2e-down.sh deleted file mode 100755 index 62ee7aec2..000000000 --- a/hack/e2e-internal/e2e-down.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash - -[[ $DEBUG ]] && set -x - -set -eof pipefail - -# include env -. hack/e2e-internal/e2e-env.sh - -echo "Destroying running docker containers..." -# do not failt if the container is not running -docker rm -f kubelet || true -docker rm -f apiserver || true -docker rm -f etcd || true diff --git a/hack/e2e-internal/e2e-env.sh b/hack/e2e-internal/e2e-env.sh deleted file mode 100755 index d0747bb6e..000000000 --- a/hack/e2e-internal/e2e-env.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env bash - -[[ $DEBUG ]] && set -x - -export ETCD_VERSION=3.0.14 -export K8S_VERSION=1.4.5 - -export PWD=`pwd` -export BASEDIR="$(dirname ${BASH_SOURCE})" -export KUBECTL="${BASEDIR}/kubectl" -export GOOS="${GOOS:-linux}" - -if [ ! -e ${KUBECTL} ]; then - echo "kubectl binary is missing. downloading..." - curl -sSL http://storage.googleapis.com/kubernetes-release/release/v${K8S_VERSION}/bin/${GOOS}/amd64/kubectl -o ${KUBECTL} - chmod u+x ${KUBECTL} -fi - -${KUBECTL} config set-cluster travis --server=http://0.0.0.0:8080 -${KUBECTL} config set-context travis --cluster=travis -${KUBECTL} config use-context travis diff --git a/hack/e2e-internal/e2e-status.sh b/hack/e2e-internal/e2e-status.sh deleted file mode 100755 index 21a4e9b29..000000000 --- a/hack/e2e-internal/e2e-status.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash - -[[ $DEBUG ]] && set -x - -set -eof pipefail - -# include env -. hack/e2e-internal/e2e-env.sh - -echo "Kubernetes information:" -${KUBECTL} version diff --git a/hack/e2e-internal/e2e-up.sh b/hack/e2e-internal/e2e-up.sh deleted file mode 100755 index 15b2d4631..000000000 --- a/hack/e2e-internal/e2e-up.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env bash - -[[ $DEBUG ]] && set -x - -set -eof pipefail - -# include env -. hack/e2e-internal/e2e-env.sh - -echo "Starting etcd..." -docker run -d \ - --net=host \ - --name=etcd \ - quay.io/coreos/etcd:v$ETCD_VERSION - -echo "Starting kubernetes..." - -docker run -d --name=apiserver \ - --net=host \ - --pid=host \ - --privileged=true \ - gcr.io/google_containers/hyperkube:v${K8S_VERSION} \ - /hyperkube apiserver \ - --insecure-bind-address=0.0.0.0 \ - --service-cluster-ip-range=10.0.0.1/24 \ - --etcd_servers=http://127.0.0.1:4001 \ - --v=2 - -docker run -d --name=kubelet \ - --volume=/:/rootfs:ro \ - --volume=/sys:/sys:ro \ - --volume=/dev:/dev \ - --volume=/var/lib/docker/:/var/lib/docker:rw \ - --volume=/var/lib/kubelet/:/var/lib/kubelet:rw \ - --volume=/var/run:/var/run:rw \ - --net=host \ - --pid=host \ - --privileged=true \ - gcr.io/google_containers/hyperkube:v${K8S_VERSION} \ - /hyperkube kubelet \ - --containerized \ - --hostname-override="0.0.0.0" \ - --address="0.0.0.0" \ - --cluster_dns=10.0.0.10 --cluster_domain=cluster.local \ - --api-servers=http://localhost:8080 \ - --config=/etc/kubernetes/manifests-multi - -echo "waiting until api server is available..." -until curl -o /dev/null -sIf http://0.0.0.0:8080; do \ - sleep 10; -done; - -echo "Kubernetes started" -echo "Kubernetes information:" -${KUBECTL} version diff --git a/hack/e2e-internal/ginkgo-e2e.sh b/hack/e2e-internal/ginkgo-e2e.sh deleted file mode 100755 index aa3c61ce6..000000000 --- a/hack/e2e-internal/ginkgo-e2e.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash - -echo "running ginkgo" \ No newline at end of file