Add fake filesystem for test to avoid temporal files on the local filesystem

.travis.yml

@@ -34,16 +34,21 @@ jobs:
     - go get github.com/golang/lint/golint
     - make fmt lint vet
   - stage: Coverage
+    before_script:
+    # start minikube 
+    - test/e2e/up.sh
     script:
     - go get github.com/mattn/goveralls
     - go get github.com/modocache/gover
-    - if ! go get github.com/golang/tools/cmd/cover; then go get golang.org/x/tools/cmd/cover;
+    - if ! go get github.com/golang/tools/cmd/cover; then go get golang.org/x/tools/cmd/cover;fi
-      fi
+    - if ! go get github.com/jteeuwen/go-bindata/...; then github.com/jteeuwen/go-bindata/...;fi
     - make cover
   - stage: e2e
     before_script:
+    - if ! go get github.com/jteeuwen/go-bindata/...; then github.com/jteeuwen/go-bindata/...;fi
     - make e2e-image
     - test/e2e/up.sh
+    - test/e2e/wait-for-nginx.sh
     script:
     - make e2e-test
   # split builds to avoid job timeouts

Makefile

@@ -133,8 +133,12 @@ endif
 clean:
 	$(DOCKER) rmi -f $(MULTI_ARCH_IMG):$(TAG) || true
 
+.PHONE: gobindata
+gobindata:
+		go-bindata -o internal/file/bindata.go -prefix="rootfs" -pkg=file -ignore=Dockerfile -ignore=".DS_Store" rootfs/...
+
 .PHONY: build
-build: clean
+build: clean gobindata
 	CGO_ENABLED=0 GOOS=${GOOS} GOARCH=${GOARCH} go build -a -installsuffix cgo \
 		-ldflags "-s -w -X ${PKG}/version.RELEASE=${TAG} -X ${PKG}/version.COMMIT=${COMMIT} -X ${PKG}/version.REPO=${REPO_INFO}" \
 		-o ${TEMP_DIR}/rootfs/nginx-ingress-controller ${PKG}/cmd/nginx
@@ -150,7 +154,7 @@ lint:
 	@go list -f '{{if len .TestGoFiles}}"golint {{.Dir}}/..."{{end}}' $(shell go list ${PKG}/... | grep -v vendor | grep -v '/test/e2e') | xargs -L 1 sh -c
 
 .PHONY: test
-test: fmt lint vet
+test: fmt lint vet gobindata
 	@echo "+ $@"
 	@go test -v -race -tags "$(BUILDTAGS) cgo" $(shell go list ${PKG}/... | grep -v vendor | grep -v '/test/e2e')
 
@@ -165,7 +169,7 @@ e2e-test:
 	@KUBECONFIG=${HOME}/.kube/config INGRESSNGINXCONFIG=${HOME}/.kube/config ./e2e-tests
 
 .PHONY: cover
-cover:
+cover: gobindata
 	@echo "+ $@"
 	@go list -f '{{if len .TestGoFiles}}"go test -coverprofile={{.Dir}}/.coverprofile {{.ImportPath}}"{{end}}' $(shell go list ${PKG}/... | grep -v vendor | grep -v '/test/e2e') | xargs -L 1 sh -c
 	gover

cmd/nginx/main.go

@@ -118,7 +118,7 @@ func main() {
 
 	// create the default SSL certificate (dummy)
 	defCert, defKey := ssl.GetFakeSSLCert()
-	c, err := ssl.AddOrUpdateCertAndKey(fakeCertificate, defCert, defKey, []byte{}, fs)
+	c, err := ssl.AddOrUpdateCertAndKey(fakeCertificate, defCert, defKey, []byte{})
 	if err != nil {
 		glog.Fatalf("Error generating self signed certificate: %v", err)
 	}

internal/file/filesystem.go

@@ -0,0 +1,127 @@
+package file
+
+import (
+	"os"
+	"path/filepath"
+
+	"github.com/golang/glog"
+
+	"k8s.io/kubernetes/pkg/util/filesystem"
+)
+
+// Filesystem is an interface that we can use to mock various filesystem operations
+type Filesystem interface {
+	filesystem.Filesystem
+}
+
+// NewLocalFS implements Filesystem using same-named functions from "os" and "io/ioutil".
+func NewLocalFS() (Filesystem, error) {
+	fs := filesystem.DefaultFs{}
+
+	err := initialize(false, fs)
+	if err != nil {
+		return nil, err
+	}
+
+	return fs, nil
+}
+
+// NewFakeFS creates an in-memory filesytem with all the required
+// paths used by the ingress controller.
+// This allows running test without polluting the local machine.
+func NewFakeFS() (Filesystem, error) {
+	fs := filesystem.NewFakeFs()
+
+	err := initialize(true, fs)
+	if err != nil {
+		return nil, err
+	}
+
+	return fs, nil
+}
+
+// initialize creates the required directory structure and when
+// runs as virtual filesystem it copies the local files to it
+func initialize(isVirtual bool, fs Filesystem) error {
+	for _, directory := range directories {
+		err := fs.MkdirAll(directory, 0655)
+		if err != nil {
+			return err
+		}
+	}
+
+	if !isVirtual {
+		return nil
+	}
+
+	for _, file := range files {
+		f, err := fs.Create(file)
+		if err != nil {
+			return err
+		}
+
+		_, err = f.Write([]byte(""))
+		if err != nil {
+			return err
+		}
+
+		err = f.Close()
+		if err != nil {
+			return err
+		}
+	}
+
+	err := fs.MkdirAll("/proc", 0655)
+	if err != nil {
+		return err
+	}
+
+	glog.Info("Restoring generated (go-bindata) assets in virtual filesystem...")
+	for _, assetName := range AssetNames() {
+		err := restoreAsset("/", assetName, fs)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// restoreAsset restores an asset under the given directory
+func restoreAsset(dir, name string, fs Filesystem) error {
+	data, err := Asset(name)
+	if err != nil {
+		return err
+	}
+	info, err := AssetInfo(name)
+	if err != nil {
+		return err
+	}
+	err = fs.MkdirAll(_filePath(dir, filepath.Dir(name)), os.FileMode(0755))
+	if err != nil {
+		return err
+	}
+
+	f, err := fs.Create(_filePath(dir, name))
+	if err != nil {
+		return err
+	}
+
+	_, err = f.Write(data)
+	if err != nil {
+		return err
+	}
+
+	err = f.Close()
+	if err != nil {
+		return err
+	}
+
+	//Missing info.Mode()
+
+	err = fs.Chtimes(_filePath(dir, name), info.ModTime(), info.ModTime())
+	if err != nil {
+		return err
+	}
+	return nil
+}

internal/file/filesystem_test.go

@@ -0,0 +1,37 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package file
+
+import (
+	"testing"
+)
+
+func TestNewFakeFS(t *testing.T) {
+	fs, err := NewFakeFS()
+	if err != nil {
+		t.Fatalf("unexpected error creating filesystem abstraction: %v", err)
+	}
+
+	if fs == nil {
+		t.Fatal("expected a filesystem but none returned")
+	}
+
+	_, err = fs.Stat("/etc/nginx/nginx.conf")
+	if err != nil {
+		t.Fatalf("unexpected error reading default nginx.conf file: %v", err)
+	}
+}

internal/file/structure.go

@@ -0,0 +1,26 @@
+package file
+
+const (
+	// AuthDirectory default directory used to store files
+	// to authenticate request
+	AuthDirectory = "/etc/ingress-controller/auth"
+
+	// DefaultSSLDirectory defines the location where the SSL certificates will be generated
+	// This directory contains all the SSL certificates that are specified in Ingress rules.
+	// The name of each file is <namespace>-<secret name>.pem. The content is the concatenated
+	// certificate and key.
+	DefaultSSLDirectory = "/ingress-controller/ssl"
+)
+
+var (
+	directories = []string{
+		"/etc/nginx/template",
+		"/run",
+		DefaultSSLDirectory,
+		AuthDirectory,
+	}
+
+	files = []string{
+		"/run/nginx.pid",
+	}
+)

internal/ingress/controller/controller.go

@@ -76,7 +76,7 @@ type Configuration struct {
 
 	ConfigMapName  string
 	DefaultService string
-	IngressClass   string
+
 	Namespace string
 
 	ForceNamespaceIsolation bool
@@ -87,7 +87,6 @@ type Configuration struct {
 	UDPConfigMapName string
 
 	DefaultHealthzURL     string
-	DefaultIngressClass   string
 	DefaultSSLCertificate string
 
 	// optional

internal/ingress/controller/nginx.go

@@ -43,6 +43,7 @@ import (
 	"k8s.io/client-go/util/flowcontrol"
 	"k8s.io/kubernetes/pkg/util/filesystem"
 
+	"k8s.io/ingress-nginx/internal/file"
 	"k8s.io/ingress-nginx/internal/ingress"
 	"k8s.io/ingress-nginx/internal/ingress/annotations"
 	"k8s.io/ingress-nginx/internal/ingress/annotations/class"
@@ -57,6 +58,7 @@ import (
 	"k8s.io/ingress-nginx/internal/net/dns"
 	"k8s.io/ingress-nginx/internal/net/ssl"
 	"k8s.io/ingress-nginx/internal/task"
+	"k8s.io/ingress-nginx/internal/watch"
 )
 
 type statusModule string
@@ -77,7 +79,7 @@ var (
 // NewNGINXController creates a new NGINX Ingress controller.
 // If the environment variable NGINX_BINARY exists it will be used
 // as source for nginx commands
-func NewNGINXController(config *Configuration) *NGINXController {
+func NewNGINXController(config *Configuration, fs file.Filesystem) *NGINXController {
 	ngx := os.Getenv("NGINX_BINARY")
 	if ngx == "" {
 		ngx = nginxBinary
@@ -114,12 +116,12 @@ func NewNGINXController(config *Configuration) *NGINXController {
 		stopCh:   make(chan struct{}),
 		stopLock: &sync.Mutex{},
 
-		fileSystem: filesystem.DefaultFs{},
+		fileSystem: fs,
 	}
 
 	n.listers, n.controllers = n.createListers(n.stopCh)
 
-	n.stats = newStatsCollector(config.Namespace, config.IngressClass, n.binary, n.cfg.ListenPorts.Status)
+	n.stats = newStatsCollector(config.Namespace, class.IngressClass, n.binary, n.cfg.ListenPorts.Status)
 
 	n.syncQueue = task.NewTaskQueue(n.syncIngress)
 
@@ -131,8 +133,8 @@ func NewNGINXController(config *Configuration) *NGINXController {
 			PublishService:         config.PublishService,
 			IngressLister:          n.listers.Ingress,
 			ElectionID:             config.ElectionID,
-			IngressClass:           config.IngressClass,
+			IngressClass:           class.IngressClass,
-			DefaultIngressClass:    config.DefaultIngressClass,
+			DefaultIngressClass:    class.DefaultClass,
 			UpdateStatusOnShutdown: config.UpdateStatusOnShutdown,
 			UseNodeInternalIP:      config.UseNodeInternalIP,
 		})
@@ -142,7 +144,7 @@ func NewNGINXController(config *Configuration) *NGINXController {
 
 	var onChange func()
 	onChange = func() {
-		template, err := ngx_template.NewTemplate(tmplPath, onChange)
+		template, err := ngx_template.NewTemplate(tmplPath, fs)
 		if err != nil {
 			// this error is different from the rest because it must be clear why nginx is not working
 			glog.Errorf(`
@@ -153,19 +155,28 @@ Error loading new template : %v
 			return
 		}
 
-		n.t.Close()
 		n.t = template
 		glog.Info("new NGINX template loaded")
 		n.SetForceReload(true)
 	}
 
-	ngxTpl, err := ngx_template.NewTemplate(tmplPath, onChange)
+	ngxTpl, err := ngx_template.NewTemplate(tmplPath, fs)
 	if err != nil {
 		glog.Fatalf("invalid NGINX template: %v", err)
 	}
 
 	n.t = ngxTpl
 
+	// TODO: refactor
+	if _, ok := fs.(filesystem.DefaultFs); !ok {
+		watch.NewDummyFileWatcher(tmplPath, onChange)
+	} else {
+		_, err = watch.NewFileWatcher(tmplPath, onChange)
+		if err != nil {
+			glog.Fatalf("unexpected error watching template %v: %v", tmplPath, err)
+		}
+	}
+
 	return n
 }
 

internal/ingress/controller/template/template.go

@@ -29,16 +29,17 @@ import (
 	text_template "text/template"
 
 	"github.com/golang/glog"
+	"github.com/pkg/errors"
 
 	"github.com/pborman/uuid"
 
 	extensions "k8s.io/api/extensions/v1beta1"
 	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/ingress-nginx/internal/file"
 	"k8s.io/ingress-nginx/internal/ingress"
 	"k8s.io/ingress-nginx/internal/ingress/annotations/ratelimit"
 	"k8s.io/ingress-nginx/internal/ingress/controller/config"
 	ing_net "k8s.io/ingress-nginx/internal/net"
-	"k8s.io/ingress-nginx/internal/watch"
 )
 
 const (
@@ -50,34 +51,29 @@ const (
 // Template ...
 type Template struct {
 	tmpl *text_template.Template
-	fw   watch.FileWatcher
+	//fw   watch.FileWatcher
 	bp *BufferPool
 }
 
 //NewTemplate returns a new Template instance or an
 //error if the specified template file contains errors
-func NewTemplate(file string, onChange func()) (*Template, error) {
+func NewTemplate(file string, fs file.Filesystem) (*Template, error) {
-	tmpl, err := text_template.New("nginx.tmpl").Funcs(funcMap).ParseFiles(file)
+	data, err := fs.ReadFile(file)
 	if err != nil {
-		return nil, err
+		return nil, errors.Wrapf(err, "unexpected error reading template %v", file)
 	}
-	fw, err := watch.NewFileWatcher(file, onChange)
+
+	tmpl, err := text_template.New("nginx.tmpl").Funcs(funcMap).Parse(string(data))
 	if err != nil {
 		return nil, err
 	}
 
 	return &Template{
 		tmpl: tmpl,
-		fw:   fw,
 		bp:   NewBufferPool(defBufferSize),
 	}, nil
 }
 
-// Close removes the file watcher
-func (t *Template) Close() {
-	t.fw.Close()
-}
-
 // Write populates a buffer using a template with NGINX configuration
 // and the servers and upstreams created by Ingress rules
 func (t *Template) Write(conf config.TemplateConfig) ([]byte, error) {

internal/ingress/controller/template/template_test.go

@@ -26,6 +26,7 @@ import (
 	"strings"
 	"testing"
 
+	"k8s.io/ingress-nginx/internal/file"
 	"k8s.io/ingress-nginx/internal/ingress"
 	"k8s.io/ingress-nginx/internal/ingress/annotations/authreq"
 	"k8s.io/ingress-nginx/internal/ingress/annotations/rewrite"
@@ -174,13 +175,13 @@ func TestTemplateWithData(t *testing.T) {
 	if dat.ListenPorts == nil {
 		dat.ListenPorts = &config.ListenPorts{}
 	}
-	tf, err := os.Open(path.Join(pwd, "../../../../rootfs/etc/nginx/template/nginx.tmpl"))
-	if err != nil {
-		t.Errorf("unexpected error reading json file: %v", err)
-	}
-	defer tf.Close()
 
-	ngxTpl, err := NewTemplate(tf.Name(), func() {})
+	fs, err := file.NewFakeFS()
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	ngxTpl, err := NewTemplate("/etc/nginx/template/nginx.tmpl", fs)
 	if err != nil {
 		t.Errorf("invalid NGINX template: %v", err)
 	}
@@ -207,13 +208,12 @@ func BenchmarkTemplateWithData(b *testing.B) {
 		b.Errorf("unexpected error unmarshalling json: %v", err)
 	}
 
-	tf, err := os.Open(path.Join(pwd, "../../../rootfs/etc/nginx/template/nginx.tmpl"))
+	fs, err := file.NewFakeFS()
 	if err != nil {
-		b.Errorf("unexpected error reading json file: %v", err)
+		b.Fatalf("unexpected error: %v", err)
 	}
-	defer tf.Close()
 
-	ngxTpl, err := NewTemplate(tf.Name(), func() {})
+	ngxTpl, err := NewTemplate("/etc/nginx/template/nginx.tmpl", fs)
 	if err != nil {
 		b.Errorf("invalid NGINX template: %v", err)
 	}

internal/watch/dummy.go

@@ -0,0 +1,29 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package watch
+
+// DummyFileWatcher noop implementation of a file watcher
+type DummyFileWatcher struct{}
+
+func NewDummyFileWatcher(file string, onEvent func()) FileWatcher {
+	return DummyFileWatcher{}
+}
+
+// Close ends the watch
+func (f DummyFileWatcher) Close() error {
+	return nil
+}

internal/watch/file_watcher.go

@@ -24,8 +24,12 @@ import (
 	"gopkg.in/fsnotify.v1"
 )
 
-// FileWatcher defines a watch over a file
+type FileWatcher interface {
-type FileWatcher struct {
+	Close() error
+}
+
+// OSFileWatcher defines a watch over a file
+type OSFileWatcher struct {
 	file    string
 	watcher *fsnotify.Watcher
 	// onEvent callback to be invoked after the file being watched changes
@@ -34,7 +38,7 @@ type FileWatcher struct {
 
 // NewFileWatcher creates a new FileWatcher
 func NewFileWatcher(file string, onEvent func()) (FileWatcher, error) {
-	fw := FileWatcher{
+	fw := OSFileWatcher{
 		file:    file,
 		onEvent: onEvent,
 	}
@@ -43,12 +47,12 @@ func NewFileWatcher(file string, onEvent func()) (FileWatcher, error) {
 }
 
 // Close ends the watch
-func (f *FileWatcher) Close() error {
+func (f OSFileWatcher) Close() error {
 	return f.watcher.Close()
 }
 
 // watch creates a fsnotify watcher for a file and create of write events
-func (f *FileWatcher) watch() error {
+func (f *OSFileWatcher) watch() error {
 	watcher, err := fsnotify.NewWatcher()
 	if err != nil {
 		return err

test/e2e/up.sh

@@ -35,47 +35,3 @@ until kubectl get nodes -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True";
 do
     sleep 1;
 done
-
-echo "deploying NGINX Ingress controller"
-cat deploy/namespace.yaml | kubectl apply -f -
-cat deploy/default-backend.yaml | kubectl apply -f -
-cat deploy/configmap.yaml | kubectl apply -f -
-cat deploy/tcp-services-configmap.yaml | kubectl apply -f -
-cat deploy/udp-services-configmap.yaml | kubectl apply -f -
-cat deploy/without-rbac.yaml | kubectl apply -f -
-cat deploy/provider/baremetal/service-nodeport.yaml | kubectl apply -f -
-
-echo "updating image..."
-kubectl set image \
-    deployments \
-    --namespace ingress-nginx \
-	--selector app=ingress-nginx \
-    nginx-ingress-controller=quay.io/kubernetes-ingress-controller/nginx-ingress-controller:test
-
-sleep 5
-
-echo "waiting NGINX ingress pod..."
-
-function waitForPod() {
-    until kubectl get pods -n ingress-nginx -l app=ingress-nginx -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True";
-    do
-        sleep 1;
-    done
-}
-
-export -f waitForPod
-
-timeout 10s bash -c waitForPod
-
-if kubectl get pods -n ingress-nginx -l app=ingress-nginx -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True";
-then
-    echo "Kubernetes deployments started"
-else
-    echo "Kubernetes deployments with issues:"
-    kubectl get pods -n ingress-nginx
-
-    echo "Reason:"
-    kubectl describe pods -n ingress-nginx
-    kubectl logs -n ingress-nginx -l app=ingress-nginx
-    exit 1
-fi

test/e2e/wait-for-nginx.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+
+# Copyright 2017 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+export JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}'
+
+echo "deploying NGINX Ingress controller"
+cat deploy/namespace.yaml | kubectl apply -f -
+cat deploy/default-backend.yaml | kubectl apply -f -
+cat deploy/configmap.yaml | kubectl apply -f -
+cat deploy/tcp-services-configmap.yaml | kubectl apply -f -
+cat deploy/udp-services-configmap.yaml | kubectl apply -f -
+cat deploy/without-rbac.yaml | kubectl apply -f -
+cat deploy/provider/baremetal/service-nodeport.yaml | kubectl apply -f -
+
+echo "updating image..."
+kubectl set image \
+    deployments \
+    --namespace ingress-nginx \
+	--selector app=ingress-nginx \
+    nginx-ingress-controller=quay.io/kubernetes-ingress-controller/nginx-ingress-controller:test
+
+sleep 5
+
+echo "waiting NGINX ingress pod..."
+
+function waitForPod() {
+    until kubectl get pods -n ingress-nginx -l app=ingress-nginx -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True";
+    do
+        sleep 1;
+    done
+}
+
+export -f waitForPod
+
+timeout 10s bash -c waitForPod
+
+if kubectl get pods -n ingress-nginx -l app=ingress-nginx -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True";
+then
+    echo "Kubernetes deployments started"
+else
+    echo "Kubernetes deployments with issues:"
+    kubectl get pods -n ingress-nginx
+
+    echo "Reason:"
+    kubectl describe pods -n ingress-nginx
+    kubectl logs -n ingress-nginx -l app=ingress-nginx
+    exit 1
+fi