From 16c580054509b4b3451ae7636d17949678ebc2ae Mon Sep 17 00:00:00 2001 From: Manuel de Brito Fontes Date: Wed, 23 Nov 2016 20:22:29 -0300 Subject: [PATCH] Add e2e boilerplate --- .../rootfs/etc/nginx/template/nginx.tmpl | 2 +- core/pkg/ingress/controller/backend_ssl.go | 2 +- core/pkg/ingress/controller/controller.go | 8 +- hack/e2e-internal/e2e-down.sh | 14 + hack/e2e-internal/e2e-env.sh | 21 ++ hack/e2e-internal/e2e-status.sh | 11 + hack/e2e-internal/e2e-up.sh | 55 ++++ hack/e2e-internal/ginkgo-e2e.sh | 3 + hack/e2e.go | 285 ++++++++++++++++++ 9 files changed, 397 insertions(+), 4 deletions(-) create mode 100755 hack/e2e-internal/e2e-down.sh create mode 100755 hack/e2e-internal/e2e-env.sh create mode 100755 hack/e2e-internal/e2e-status.sh create mode 100755 hack/e2e-internal/e2e-up.sh create mode 100755 hack/e2e-internal/ginkgo-e2e.sh create mode 100644 hack/e2e.go diff --git a/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl b/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl index 30774f6f6..906afd370 100644 --- a/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl +++ b/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl @@ -188,7 +188,7 @@ http { server { server_name {{ $server.Hostname }}; listen 80{{ if $cfg.UseProxyProtocol }} proxy_protocol{{ end }}; - {{ if not (empty $server.SSLCertificate) }}listen 442 {{ if $cfg.UseProxyProtocol }}proxy_protocol{{ end }} ssl {{ if $cfg.UseHttp2 }}http2{{ end }}; + {{ if not (empty $server.SSLCertificate) }}listen 442 {{ if $cfg.UseProxyProtocol }}proxy_protocol{{ end }} ssl {{ if $cfg.UseHTTP2 }}http2{{ end }}; {{/* comment PEM sha is required to detect changes in the generated configuration and force a reload */}} # PEM sha: {{ $server.SSLPemChecksum }} ssl_certificate {{ $server.SSLCertificate }}; diff --git a/core/pkg/ingress/controller/backend_ssl.go b/core/pkg/ingress/controller/backend_ssl.go index aa17c7807..92c32fc8c 100644 --- a/core/pkg/ingress/controller/backend_ssl.go +++ b/core/pkg/ingress/controller/backend_ssl.go @@ -77,7 +77,7 @@ func (ic *GenericController) syncSecret(k interface{}) error { } sec := secObj.(*api.Secret) if !ic.secrReferenced(sec.Name, sec.Namespace) { - glog.V(2).Infof("secret %v/%v is not used in Ingress rules. skipping ", sec.Namespace, sec.Name) + glog.V(3).Infof("secret %v/%v is not used in Ingress rules. skipping ", sec.Namespace, sec.Name) return nil } diff --git a/core/pkg/ingress/controller/controller.go b/core/pkg/ingress/controller/controller.go index 655d7ea8a..1306e4e01 100644 --- a/core/pkg/ingress/controller/controller.go +++ b/core/pkg/ingress/controller/controller.go @@ -100,7 +100,8 @@ type GenericController struct { syncStatus status.Sync - // controller for SSL certificates + // local store of SSL certificates + // (only certificates used in ingress) sslCertTracker *sslCertTracker // TaskQueue in charge of keep the secrets referenced from Ingress // in sync with the files on disk @@ -922,7 +923,8 @@ func (ic *GenericController) createServers(data []interface{}, upstreams map[str } // only add a certificate if the server does not have one previously configured - if len(ing.Spec.TLS) > 0 && servers[host].SSLCertificate != "" { + // TODO: TLS without secret? + if len(ing.Spec.TLS) > 0 && servers[host].SSLCertificate == "" && ing.Spec.TLS[0].SecretName != "" { key := fmt.Sprintf("%v/%v", ing.Namespace, ing.Spec.TLS[0].SecretName) bc, exists := ic.sslCertTracker.Get(key) if exists { @@ -931,6 +933,8 @@ func (ic *GenericController) createServers(data []interface{}, upstreams map[str servers[host].SSLCertificate = cert.PemFileName servers[host].SSLPemChecksum = cert.PemSHA } + } else { + glog.Warningf("secret %v does not exists", key) } } diff --git a/hack/e2e-internal/e2e-down.sh b/hack/e2e-internal/e2e-down.sh new file mode 100755 index 000000000..62ee7aec2 --- /dev/null +++ b/hack/e2e-internal/e2e-down.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +[[ $DEBUG ]] && set -x + +set -eof pipefail + +# include env +. hack/e2e-internal/e2e-env.sh + +echo "Destroying running docker containers..." +# do not failt if the container is not running +docker rm -f kubelet || true +docker rm -f apiserver || true +docker rm -f etcd || true diff --git a/hack/e2e-internal/e2e-env.sh b/hack/e2e-internal/e2e-env.sh new file mode 100755 index 000000000..d0747bb6e --- /dev/null +++ b/hack/e2e-internal/e2e-env.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +[[ $DEBUG ]] && set -x + +export ETCD_VERSION=3.0.14 +export K8S_VERSION=1.4.5 + +export PWD=`pwd` +export BASEDIR="$(dirname ${BASH_SOURCE})" +export KUBECTL="${BASEDIR}/kubectl" +export GOOS="${GOOS:-linux}" + +if [ ! -e ${KUBECTL} ]; then + echo "kubectl binary is missing. downloading..." + curl -sSL http://storage.googleapis.com/kubernetes-release/release/v${K8S_VERSION}/bin/${GOOS}/amd64/kubectl -o ${KUBECTL} + chmod u+x ${KUBECTL} +fi + +${KUBECTL} config set-cluster travis --server=http://0.0.0.0:8080 +${KUBECTL} config set-context travis --cluster=travis +${KUBECTL} config use-context travis diff --git a/hack/e2e-internal/e2e-status.sh b/hack/e2e-internal/e2e-status.sh new file mode 100755 index 000000000..21a4e9b29 --- /dev/null +++ b/hack/e2e-internal/e2e-status.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +[[ $DEBUG ]] && set -x + +set -eof pipefail + +# include env +. hack/e2e-internal/e2e-env.sh + +echo "Kubernetes information:" +${KUBECTL} version diff --git a/hack/e2e-internal/e2e-up.sh b/hack/e2e-internal/e2e-up.sh new file mode 100755 index 000000000..15b2d4631 --- /dev/null +++ b/hack/e2e-internal/e2e-up.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash + +[[ $DEBUG ]] && set -x + +set -eof pipefail + +# include env +. hack/e2e-internal/e2e-env.sh + +echo "Starting etcd..." +docker run -d \ + --net=host \ + --name=etcd \ + quay.io/coreos/etcd:v$ETCD_VERSION + +echo "Starting kubernetes..." + +docker run -d --name=apiserver \ + --net=host \ + --pid=host \ + --privileged=true \ + gcr.io/google_containers/hyperkube:v${K8S_VERSION} \ + /hyperkube apiserver \ + --insecure-bind-address=0.0.0.0 \ + --service-cluster-ip-range=10.0.0.1/24 \ + --etcd_servers=http://127.0.0.1:4001 \ + --v=2 + +docker run -d --name=kubelet \ + --volume=/:/rootfs:ro \ + --volume=/sys:/sys:ro \ + --volume=/dev:/dev \ + --volume=/var/lib/docker/:/var/lib/docker:rw \ + --volume=/var/lib/kubelet/:/var/lib/kubelet:rw \ + --volume=/var/run:/var/run:rw \ + --net=host \ + --pid=host \ + --privileged=true \ + gcr.io/google_containers/hyperkube:v${K8S_VERSION} \ + /hyperkube kubelet \ + --containerized \ + --hostname-override="0.0.0.0" \ + --address="0.0.0.0" \ + --cluster_dns=10.0.0.10 --cluster_domain=cluster.local \ + --api-servers=http://localhost:8080 \ + --config=/etc/kubernetes/manifests-multi + +echo "waiting until api server is available..." +until curl -o /dev/null -sIf http://0.0.0.0:8080; do \ + sleep 10; +done; + +echo "Kubernetes started" +echo "Kubernetes information:" +${KUBECTL} version diff --git a/hack/e2e-internal/ginkgo-e2e.sh b/hack/e2e-internal/ginkgo-e2e.sh new file mode 100755 index 000000000..aa3c61ce6 --- /dev/null +++ b/hack/e2e-internal/ginkgo-e2e.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +echo "running ginkgo" \ No newline at end of file diff --git a/hack/e2e.go b/hack/e2e.go new file mode 100644 index 000000000..d6ef3fa06 --- /dev/null +++ b/hack/e2e.go @@ -0,0 +1,285 @@ +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// e2e.go runs the e2e test suite. No non-standard package dependencies; call with "go run". +package main + +import ( + "encoding/xml" + "flag" + "fmt" + "io/ioutil" + "log" + "os" + "os/exec" + "path/filepath" + "strings" + "time" +) + +var ( + build = flag.Bool("build", true, "Build the backends images indicated by the env var BACKENDS required to run e2e tests.") + up = flag.Bool("up", true, "Creates a kubernetes cluster using hyperkube (containerized kubelet).") + down = flag.Bool("down", true, "destroys the created cluster.") + test = flag.Bool("test", true, "Run Ginkgo tests.") + dump = flag.String("dump", "", "If set, dump cluster logs to this location on test or cluster-up failure") + testArgs = flag.String("test-args", "", "Space-separated list of arguments to pass to Ginkgo test runner.") + deployment = flag.String("deployment", "bash", "up/down mechanism") + verbose = flag.Bool("v", false, "If true, print all command output.") +) + +func appendError(errs []error, err error) []error { + if err != nil { + return append(errs, err) + } + return errs +} + +func validWorkingDirectory() error { + cwd, err := os.Getwd() + if err != nil { + return fmt.Errorf("could not get pwd: %v", err) + } + acwd, err := filepath.Abs(cwd) + if err != nil { + return fmt.Errorf("failed to convert %s to an absolute path: %v", cwd, err) + } + if !strings.Contains(filepath.Base(acwd), "ingress-controller") { + return fmt.Errorf("must run from git root directory: %v", acwd) + } + return nil +} + +type TestCase struct { + XMLName xml.Name `xml:"testcase"` + ClassName string `xml:"classname,attr"` + Name string `xml:"name,attr"` + Time float64 `xml:"time,attr"` + Failure string `xml:"failure,omitempty"` +} + +type TestSuite struct { + XMLName xml.Name `xml:"testsuite"` + Failures int `xml:"failures,attr"` + Tests int `xml:"tests,attr"` + Time float64 `xml:"time,attr"` + Cases []TestCase +} + +var suite TestSuite + +func xmlWrap(name string, f func() error) error { + start := time.Now() + err := f() + duration := time.Since(start) + c := TestCase{ + Name: name, + ClassName: "e2e.go", + Time: duration.Seconds(), + } + if err != nil { + c.Failure = err.Error() + suite.Failures++ + } + suite.Cases = append(suite.Cases, c) + suite.Tests++ + return err +} + +func writeXML(start time.Time) { + suite.Time = time.Since(start).Seconds() + out, err := xml.MarshalIndent(&suite, "", " ") + if err != nil { + log.Fatalf("Could not marshal XML: %s", err) + } + path := filepath.Join(*dump, "junit_runner.xml") + f, err := os.Create(path) + if err != nil { + log.Fatalf("Could not create file: %s", err) + } + defer f.Close() + if _, err := f.WriteString(xml.Header); err != nil { + log.Fatalf("Error writing XML header: %s", err) + } + if _, err := f.Write(out); err != nil { + log.Fatalf("Error writing XML data: %s", err) + } + log.Printf("Saved XML output to %s.", path) +} + +func main() { + log.SetFlags(log.LstdFlags | log.Lshortfile) + flag.Parse() + + if err := validWorkingDirectory(); err != nil { + log.Fatalf("Called from invalid working directory: %v", err) + } + + deploy, err := getDeployer() + if err != nil { + log.Fatalf("Error creating deployer: %v", err) + } + + if err := run(deploy); err != nil { + log.Fatalf("Something went wrong: %s", err) + } +} + +func run(deploy deployer) error { + if *dump != "" { + defer writeXML(time.Now()) + } + + if *build { + if err := xmlWrap("Build", Build); err != nil { + return fmt.Errorf("error building: %s", err) + } + } + + if *up { + if err := xmlWrap("TearDown", deploy.Down); err != nil { + return fmt.Errorf("error tearing down previous cluster: %s", err) + } + } + + var errs []error + + if *up { + // If we tried to bring the cluster up, make a courtesy + // attempt to bring it down so we're not leaving resources around. + // + // TODO: We should try calling deploy.Down exactly once. Though to + // stop the leaking resources for now, we want to be on the safe side + // and call it explictly in defer if the other one is not called. + if *down { + defer xmlWrap("Deferred TearDown", deploy.Down) + } + // Start the cluster using this version. + if err := xmlWrap("Up", deploy.Up); err != nil { + return fmt.Errorf("starting e2e cluster: %s", err) + } + if *dump != "" { + cmd := exec.Command("./cluster/kubectl.sh", "--match-server-version=false", "get", "nodes", "-oyaml") + b, err := cmd.CombinedOutput() + if *verbose { + log.Printf("kubectl get nodes:\n%s", string(b)) + } + if err == nil { + if err := ioutil.WriteFile(filepath.Join(*dump, "nodes.yaml"), b, 0644); err != nil { + errs = appendError(errs, fmt.Errorf("error writing nodes.yaml: %v", err)) + } + } else { + errs = appendError(errs, fmt.Errorf("error running get nodes: %v", err)) + } + } + } + + if *test { + if err := xmlWrap("IsUp", deploy.IsUp); err != nil { + errs = appendError(errs, err) + } else { + errs = appendError(errs, Test()) + } + } + + if len(errs) > 0 && *dump != "" { + errs = appendError(errs, xmlWrap("DumpClusterLogs", func() error { + return DumpClusterLogs(*dump) + })) + } + + if *down { + errs = appendError(errs, xmlWrap("TearDown", deploy.Down)) + } + + if len(errs) != 0 { + return fmt.Errorf("encountered %d errors: %v", len(errs), errs) + } + return nil +} + +func Build() error { + // The build-release script needs stdin to ask the user whether + // it's OK to download the docker image. + cmd := exec.Command("make", "backends", "backends-images", "backends-push") + cmd.Stdin = os.Stdin + if err := finishRunning("build-release", cmd); err != nil { + return fmt.Errorf("error building: %v", err) + } + return nil +} + +type deployer interface { + Up() error + IsUp() error + SetupKubecfg() error + Down() error +} + +func getDeployer() (deployer, error) { + switch *deployment { + case "bash": + return bash{}, nil + default: + return nil, fmt.Errorf("Unknown deployment strategy %q", *deployment) + } +} + +type bash struct{} + +func (b bash) Up() error { + return finishRunning("up", exec.Command("./hack/e2e-internal/e2e-up.sh")) +} + +func (b bash) IsUp() error { + return finishRunning("get status", exec.Command("./hack/e2e-internal/e2e-status.sh")) +} + +func (b bash) SetupKubecfg() error { + return nil +} + +func (b bash) Down() error { + return finishRunning("teardown", exec.Command("./hack/e2e-internal/e2e-down.sh")) +} + +func DumpClusterLogs(location string) error { + log.Printf("Dumping cluster logs to: %v", location) + return finishRunning("dump cluster logs", exec.Command("./hack/e2e-internal/log-dump.sh", location)) +} + +func Test() error { + if *testArgs == "" { + *testArgs = "--ginkgo.focus=\\[Feature:Ingress\\]" + } + return finishRunning("Ginkgo tests", exec.Command("./hack/e2e-internal/ginkgo-e2e.sh", strings.Fields(*testArgs)...)) +} + +func finishRunning(stepName string, cmd *exec.Cmd) error { + if *verbose { + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + } + log.Printf("Running: %v", stepName) + defer func(start time.Time) { + log.Printf("Step '%s' finished in %s", stepName, time.Since(start)) + }(time.Now()) + + if err := cmd.Run(); err != nil { + return fmt.Errorf("error running %v: %v", stepName, err) + } + return nil +}