From 926b02987404586593b5843ddb3946d4f2755893 Mon Sep 17 00:00:00 2001 From: Manuel de Brito Fontes Date: Mon, 20 Nov 2017 19:06:00 -0300 Subject: [PATCH] Add virtual filesystem for testing --- .travis.yml | 5 +- Makefile | 10 +- cmd/nginx/main.go | 17 +- cmd/nginx/main_test.go | 8 +- internal/file/bindata.go | 289 ++++++++++++++++++ internal/file/filesystem.go | 144 +++++++++ internal/file/structure.go | 26 ++ internal/ingress/annotations/annotations.go | 5 +- .../ingress/annotations/annotations_test.go | 30 +- internal/ingress/annotations/auth/main.go | 43 ++- .../ingress/annotations/auth/main_test.go | 26 +- internal/ingress/controller/checker_test.go | 26 +- internal/ingress/controller/controller.go | 157 +--------- internal/ingress/controller/nginx.go | 30 +- internal/ingress/controller/reload.go | 22 ++ internal/ingress/controller/stream.go | 140 +++++++++ .../ingress/controller/template/template.go | 22 +- .../controller/template/template_test.go | 20 +- internal/ingress/status/status_test.go | 4 +- internal/ingress/store/backend_ssl.go | 25 +- internal/ingress/store/backend_ssl_test.go | 12 +- internal/ingress/store/store.go | 26 +- internal/ingress/store/store_test.go | 251 +++++++++++++-- internal/ingress/types.go | 8 - internal/net/ssl/ssl.go | 66 ++-- internal/net/ssl/ssl_test.go | 34 +-- internal/watch/dummy.go | 29 ++ internal/watch/file_watcher.go | 14 +- test/e2e/framework/ssl.go | 115 +++++++ test/e2e/framework/util.go | 63 ++++ test/e2e/ssl/secret_update.go | 109 +------ 31 files changed, 1305 insertions(+), 471 deletions(-) create mode 100644 internal/file/bindata.go create mode 100644 internal/file/filesystem.go create mode 100644 internal/file/structure.go create mode 100644 internal/ingress/controller/reload.go create mode 100644 internal/ingress/controller/stream.go create mode 100644 internal/watch/dummy.go create mode 100644 test/e2e/framework/ssl.go diff --git a/.travis.yml b/.travis.yml index cd453b96a..bf40ab59c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -40,11 +40,12 @@ jobs: script: - go get github.com/mattn/goveralls - go get github.com/modocache/gover - - if ! go get github.com/golang/tools/cmd/cover; then go get golang.org/x/tools/cmd/cover; - fi + - if ! go get github.com/golang/tools/cmd/cover; then go get golang.org/x/tools/cmd/cover;fi + - if ! go get github.com/jteeuwen/go-bindata/...; then github.com/jteeuwen/go-bindata/...;fi - make cover - stage: e2e before_script: + - if ! go get github.com/jteeuwen/go-bindata/...; then github.com/jteeuwen/go-bindata/...;fi - make e2e-image - test/e2e/up.sh - test/e2e/wait-for-nginx.sh diff --git a/Makefile b/Makefile index 5b64f2530..3bf3bc0d0 100644 --- a/Makefile +++ b/Makefile @@ -133,8 +133,12 @@ endif clean: $(DOCKER) rmi -f $(MULTI_ARCH_IMG):$(TAG) || true +.PHONE: gobindata +gobindata: + go-bindata -o internal/file/bindata.go -prefix="rootfs" -pkg=file -ignore=Dockerfile -ignore=".DS_Store" rootfs/... + .PHONY: build -build: clean +build: clean gobindata CGO_ENABLED=0 GOOS=${GOOS} GOARCH=${GOARCH} go build -a -installsuffix cgo \ -ldflags "-s -w -X ${PKG}/version.RELEASE=${TAG} -X ${PKG}/version.COMMIT=${COMMIT} -X ${PKG}/version.REPO=${REPO_INFO}" \ -o ${TEMP_DIR}/rootfs/nginx-ingress-controller ${PKG}/cmd/nginx @@ -150,7 +154,7 @@ lint: @go list -f '{{if len .TestGoFiles}}"golint {{.Dir}}/..."{{end}}' $(shell go list ${PKG}/... | grep -v vendor | grep -v '/test/e2e') | xargs -L 1 sh -c .PHONY: test -test: fmt lint vet +test: fmt lint vet gobindata @echo "+ $@" @go test -v -race -tags "$(BUILDTAGS) cgo" $(shell go list ${PKG}/... | grep -v vendor | grep -v '/test/e2e') @@ -165,7 +169,7 @@ e2e-test: @KUBECONFIG=${HOME}/.kube/config INGRESSNGINXCONFIG=${HOME}/.kube/config ./e2e-tests .PHONY: cover -cover: +cover: gobindata @echo "+ $@" @go list -f '{{if len .TestGoFiles}}"go test -coverprofile={{.Dir}}/.coverprofile {{.ImportPath}}"{{end}}' $(shell go list ${PKG}/... | grep -v vendor | grep -v '/test/e2e') | xargs -L 1 sh -c gover diff --git a/cmd/nginx/main.go b/cmd/nginx/main.go index d1f0546f0..a34bfb51c 100644 --- a/cmd/nginx/main.go +++ b/cmd/nginx/main.go @@ -39,7 +39,7 @@ import ( "k8s.io/client-go/tools/clientcmd" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" - "k8s.io/ingress-nginx/internal/ingress" + "k8s.io/ingress-nginx/internal/file" "k8s.io/ingress-nginx/internal/ingress/controller" "k8s.io/ingress-nginx/internal/k8s" "k8s.io/ingress-nginx/internal/net/ssl" @@ -58,6 +58,11 @@ func main() { glog.Fatal(err) } + fs, err := file.NewLocalFS() + if err != nil { + glog.Fatal(err) + } + kubeClient, err := createApiserverClient(conf.APIServerHost, conf.KubeConfigFile) if err != nil { handleFatalInitError(err) @@ -111,15 +116,9 @@ func main() { glog.Fatalf("resync period (%vs) is too low", conf.ResyncPeriod.Seconds()) } - // create directory that will contains the SSL Certificates - err = os.MkdirAll(ingress.DefaultSSLDirectory, 0655) - if err != nil { - glog.Errorf("Failed to mkdir SSL directory: %v", err) - } - // create the default SSL certificate (dummy) defCert, defKey := ssl.GetFakeSSLCert() - c, err := ssl.AddOrUpdateCertAndKey(fakeCertificate, defCert, defKey, []byte{}) + c, err := ssl.AddOrUpdateCertAndKey(fakeCertificate, defCert, defKey, []byte{}, fs) if err != nil { glog.Fatalf("Error generating self signed certificate: %v", err) } @@ -129,7 +128,7 @@ func main() { conf.Client = kubeClient - ngx := controller.NewNGINXController(conf) + ngx := controller.NewNGINXController(conf, fs) if conf.EnableSSLPassthrough { setupSSLProxy(conf.ListenPorts.HTTPS, conf.ListenPorts.SSLProxy, ngx) diff --git a/cmd/nginx/main_test.go b/cmd/nginx/main_test.go index 7933b5df0..752e65e68 100644 --- a/cmd/nginx/main_test.go +++ b/cmd/nginx/main_test.go @@ -23,6 +23,7 @@ import ( "testing" "time" + "k8s.io/ingress-nginx/internal/file" "k8s.io/ingress-nginx/internal/ingress/controller" ) @@ -70,7 +71,12 @@ func TestHandleSigterm(t *testing.T) { } conf.Client = cli - ngx := controller.NewNGINXController(conf) + fs, err := file.NewFakeFS() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + ngx := controller.NewNGINXController(conf, fs) go handleSigterm(ngx, func(code int) { if code != 1 { diff --git a/internal/file/bindata.go b/internal/file/bindata.go new file mode 100644 index 000000000..b37e6fd58 --- /dev/null +++ b/internal/file/bindata.go @@ -0,0 +1,289 @@ +// Code generated by go-bindata. +// sources: +// rootfs/etc/nginx/nginx.conf +// rootfs/etc/nginx/template/nginx.tmpl +// rootfs/ingress-controller/clean-nginx-conf.sh +// DO NOT EDIT! + +package file + +import ( + "bytes" + "compress/gzip" + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + "strings" + "time" +) + +func bindataRead(data []byte, name string) ([]byte, error) { + gz, err := gzip.NewReader(bytes.NewBuffer(data)) + if err != nil { + return nil, fmt.Errorf("Read %q: %v", name, err) + } + + var buf bytes.Buffer + _, err = io.Copy(&buf, gz) + clErr := gz.Close() + + if err != nil { + return nil, fmt.Errorf("Read %q: %v", name, err) + } + if clErr != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +type asset struct { + bytes []byte + info os.FileInfo +} + +type bindataFileInfo struct { + name string + size int64 + mode os.FileMode + modTime time.Time +} + +func (fi bindataFileInfo) Name() string { + return fi.name +} +func (fi bindataFileInfo) Size() int64 { + return fi.size +} +func (fi bindataFileInfo) Mode() os.FileMode { + return fi.mode +} +func (fi bindataFileInfo) ModTime() time.Time { + return fi.modTime +} +func (fi bindataFileInfo) IsDir() bool { + return false +} +func (fi bindataFileInfo) Sys() interface{} { + return nil +} + +var _etcNginxNginxConf = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x2c\xca\x41\x0a\x83\x40\x0c\x85\xe1\xfd\x9c\xe2\x41\xf7\x7a\x00\x57\x3d\xca\xa0\x89\x06\x34\x19\x32\x4f\x69\x29\xbd\x7b\x69\xe9\xea\x5f\xfc\xdf\x0d\x77\x5c\x92\x4f\x74\x3b\xda\x2e\xf0\xd5\xfc\x81\x39\x5c\x6d\x3d\xb3\xd2\xc2\xa1\xb6\x0b\xb8\x55\x42\x23\x67\xe9\x7f\xc4\x40\x67\x4d\x0e\xa5\xd9\x82\x31\x4f\x1f\x7f\x63\x68\xb6\x4c\xa5\xc8\x25\xce\x8e\xd7\xbb\x6c\x64\xfb\x76\xa9\x72\x84\x23\x54\xa7\x4f\x00\x00\x00\xff\xff\x75\xb5\xe6\xb8\x77\x00\x00\x00") + +func etcNginxNginxConfBytes() ([]byte, error) { + return bindataRead( + _etcNginxNginxConf, + "etc/nginx/nginx.conf", + ) +} + +func etcNginxNginxConf() (*asset, error) { + bytes, err := etcNginxNginxConfBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "etc/nginx/nginx.conf", size: 119, mode: os.FileMode(420), modTime: time.Unix(1508444716, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + +var _etcNginxTemplateNginxTmpl = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xec\x7d\x6d\x73\xdb\x38\xd2\xe0\xf7\xfc\x0a\x94\xec\x2a\xdb\xa9\x48\x76\xb2\xd9\x79\xe6\xb1\x2b\x57\xe7\xd8\xc9\xda\x37\x4e\xe2\xb2\x9c\xcc\xd4\x5e\x5d\xa9\x20\xb2\x25\x62\x4d\x01\x5c\x00\xb4\xac\xe4\x74\xbf\xfd\x0a\x6f\x24\x08\x82\x94\x9c\xc9\x64\x66\xb6\x86\x1f\x5c\x16\xd9\x68\x34\x1a\x8d\x06\xfa\x05\xc0\x97\x2f\x68\x17\xe7\x39\x3a\x7e\x85\x46\x68\xbd\x7e\xa2\x7e\x0b\xe0\xf7\xc0\x85\x7e\x37\xb6\xff\xdb\x4f\xc9\x6c\xae\x5f\x9f\xcd\xe6\xee\xd5\xa5\xb8\xbc\xfe\xf4\xc3\x1b\x8a\xa7\x39\xa4\xfa\x63\xf3\x8d\x05\xcb\x00\xe7\x32\xfb\xfc\xf1\xe6\x52\xc3\x5c\xd4\x3f\x2d\xc0\x14\x27\x77\x40\x53\x53\xed\x6b\xf7\xc3\x7e\x2c\x38\x7b\x58\x5d\x00\x4e\x1d\x5d\xd7\xea\xc5\x18\xa4\x7b\x67\xe1\x70\x9a\xfa\x50\xa7\xf5\xcf\xf5\xfa\x89\x82\x20\x33\xdd\x88\x91\xa1\xee\x1d\x4b\x05\x24\x25\x27\x72\xa5\x00\x72\x86\xd3\xc9\x82\xa5\x65\x0e\xe8\x10\x64\x72\x48\xe7\x84\x3e\x1c\x9a\x37\xe2\x90\xce\x1f\x26\x99\x94\x85\x02\x71\xc5\x2c\xf8\x48\xb0\x13\x85\x1e\x68\x1a\xaf\xe9\x43\x01\x54\x72\x9c\x10\x3a\x7f\x4c\x4d\xac\x2e\xd6\x5b\xd3\x3e\xa6\x69\x57\x75\xfb\x14\xcc\xa7\x7f\x92\xe2\x8e\xd0\x33\x96\xe7\x90\x48\xc6\x2f\x98\x90\x68\x30\x38\x38\x78\x0c\x41\x9f\x35\x8e\x0e\x5a\x52\x0c\x0b\x46\x11\x9b\xcd\x4e\x9e\x3c\x59\x32\x7e\x07\x7c\x52\x70\x96\x80\x10\x20\x90\x95\x9f\xd1\xcf\xfa\xc3\x75\xf5\x7e\xbd\x3e\x79\x52\x90\x14\x1d\xf2\x92\x9a\x6a\x47\x05\x49\x4f\x6c\xcb\x28\xa0\xd1\x3b\xfc\xa0\x1a\xf4\x96\xe4\x20\xd0\x91\xaa\xca\x22\xe7\x39\x59\x10\x39\xa1\x6c\x46\x72\x50\x15\x34\x41\x15\x66\x43\x9e\xe1\xd4\xe1\x53\xa4\x1a\x71\x7c\x68\xab\x61\x7c\x7e\x08\xf4\x30\x65\x89\x69\x61\xc2\x38\xb8\xa6\x65\x72\x91\xef\xd8\x6a\x44\x56\xca\x94\x2d\xe9\x44\x92\x05\xb0\x52\xa2\xa7\x87\x5a\xde\x0e\x9f\x22\x7c\xcf\x48\x8a\x96\x98\x48\xc5\x6a\xc9\x18\xca\x19\x9d\xa3\xb4\xe4\xea\x37\x46\x1c\x14\x67\x4d\x81\x2e\x6c\x4d\xc6\x8c\xed\xe7\x5b\xfb\x75\xbd\x46\x27\x4f\x9e\xc0\x3d\x50\x29\xd0\x97\x27\x08\x21\xb4\x28\x73\x49\x26\x38\x49\xa0\x90\xc8\x3e\x8c\x9e\xe8\x6f\xb6\x92\x84\x51\x0a\x89\x24\x8c\x0a\x54\x55\xf0\x0e\x3f\x98\x3a\xce\xbc\xaf\x8a\x49\xaa\x60\x29\x00\x85\x0f\x14\x2c\xcf\x4f\x9e\xac\x9f\x3c\x51\x7c\xb3\xb5\xeb\x76\x2f\x41\x17\x90\x19\xa0\x7b\x9c\x97\x80\xd8\x4c\xff\xc8\xf4\x70\x43\xbf\x0c\xdf\x32\xbe\xc4\x3c\x85\x54\xfd\x87\x24\x43\x53\x40\x4a\x32\xd5\xbf\xae\xe4\x1c\xd8\x84\x14\xc8\x8a\x9d\xe6\x91\xa9\xa0\x1a\x3c\x1f\x05\xe8\x91\x7e\xcd\x99\x64\x09\xcb\x91\x05\xe1\x80\xf3\x09\x29\x26\xb6\x3a\xfd\x68\x1d\xa1\xe4\x4d\x43\x9e\x38\x54\x90\x0b\xe8\x2b\xe6\x98\x53\x11\xfc\x96\x71\xa3\x34\x2a\xd6\x78\x22\xee\x63\xe1\x4a\x07\x08\x72\x0f\x1e\xfb\xbf\x7c\x41\x1c\xd3\x39\xa0\x5d\xc9\x4b\x21\x21\x55\x0d\x3c\x7e\x65\xaa\xd0\x4d\xb9\x01\x9c\x5f\x5e\x9f\x5d\x9e\xdf\x38\xaa\x04\xc8\x89\xc3\x39\xe3\x6c\xe1\xa8\xf2\x30\x74\x50\xa2\xbb\x22\xc5\x12\x4f\xb1\x1a\x49\xa5\x80\x54\xf1\x37\x05\x09\x7c\x41\xa8\xe1\x72\xc2\x4a\x2a\xf9\x0a\xa5\x50\x00\x4d\x95\x58\x32\x6a\x3e\xe4\x04\xa8\x44\x97\xd7\x08\xa7\x29\x07\x21\xfc\x2e\xe8\x1b\x2b\xea\x43\xad\x12\xe6\xc0\x48\xe1\x0f\x9b\x00\x8d\xcc\x88\x40\x44\x20\x0e\xff\x2e\x09\xd7\x02\x90\xe0\x3c\x29\x73\x2c\x01\x49\x8e\x67\x33\x92\xa0\x19\xe3\x88\xd0\x94\xdc\x93\xb4\xc4\x79\x45\x73\x29\x14\xbd\xff\x00\x76\x79\x8d\x88\xa1\x5a\x48\x2c\x4b\x81\x0a\x3c\xf7\x44\xc6\xd0\xe0\x4a\x99\xc7\x53\x63\x1a\xc1\x28\xc5\xf2\xc4\x87\x56\x5a\xbf\x7a\x9a\xd0\x57\x44\xc2\x19\x91\xab\xb0\x8c\x91\xb1\xba\xe3\x55\xb7\x87\x32\x6b\x34\xf0\x27\x29\xc6\x86\x52\x4b\xe2\x7d\xc6\x84\x9c\xd8\xf6\x4e\x4c\x2b\x26\x9f\x19\x05\x24\x32\xcc\x21\x3d\x8e\x01\x1c\x3b\xe9\xac\xd0\xfd\x93\x51\x18\x93\xcf\x50\x89\x44\x14\xef\x8c\xe4\x12\xf8\x64\xba\x9a\x28\xe1\xba\x83\x15\xf2\x10\x9d\xc3\x0c\x97\xb9\x7c\xab\x61\x7e\x82\x55\x97\x74\x09\xa0\xa9\x56\xac\xde\x53\x35\x18\x13\xd6\x52\x17\x32\xe3\x80\x53\x71\xe2\x00\x26\x4b\x4e\x24\xc4\x4a\xcb\xa4\x98\x50\x56\x94\x22\x6b\x7e\xac\xbf\xa5\x90\xe3\x55\xbb\x60\xce\xe6\x13\x51\x4e\x95\x2c\x81\x90\xc1\x47\x0e\xaa\xb5\x4a\xad\xa6\xac\x94\x9e\x0e\xac\x41\xee\x00\x0a\x9c\x93\x7b\xa8\xb4\x6f\xc5\x99\x9f\x00\x8a\x53\xf5\x09\xad\xd7\xb6\x0d\x35\xb4\xad\x50\xb4\xa1\x6f\xdc\x17\xc5\x45\x5d\xca\x8c\x2b\xab\x64\x26\xd3\x72\x36\x53\x5a\x5f\xf5\x59\x53\xe3\x9c\x69\x38\xa3\x6b\x5e\x6b\xa8\x46\xc7\x36\xd1\x54\xe4\x56\x4f\x0c\x4d\x3d\x69\xd8\x16\xe4\x98\xcf\x61\x12\xa3\x48\x34\x90\x5c\x29\xb8\x36\x41\x22\xa4\x66\xca\xd2\x55\xa4\x49\x21\x35\xaf\x59\xba\xea\x6e\x92\x46\xd2\x6e\x50\x0c\x49\xa3\x41\x1a\x87\xd2\x39\x2f\x26\x0b\xfc\x30\x99\x11\xc8\xd3\x26\x15\x1e\x8e\x8b\xdb\xdb\xeb\x17\xef\xf0\xc3\x5b\x05\xd5\xa0\xa2\xc6\x60\x19\x12\xa0\x08\x31\x18\x86\x54\x28\x8c\x90\xae\x0a\x10\x93\x0c\x8b\x4c\x23\x0a\x89\x40\x2f\x8e\x5e\xfe\x78\x62\x47\x91\x5a\x41\x4f\x28\x5e\xc4\x0b\xb8\xda\xcc\x4a\xfb\x3d\x5e\xc0\x05\x16\xd9\x3b\xfc\xd0\xa0\xb9\x8d\x65\x5a\x26\x77\x20\x1d\xa2\x38\x96\xd7\x1a\xa6\x81\x68\x81\x8b\x58\xf9\x56\xe3\xdf\xe1\x22\x82\x41\xa3\x30\x4a\xd0\xf0\x2e\xd6\x24\x87\xe2\xda\x5b\xb4\xc7\xda\x14\xc1\xd3\x20\xaa\x0b\x4f\x8c\xa6\x7b\xcc\x89\x52\xbb\x51\xe6\x78\x34\x7d\x72\x70\x1d\xec\x09\xf0\xb4\xfa\x36\x8a\xc7\x6f\x98\x59\x4b\x51\x45\xab\x5a\x4e\x8a\x09\xa1\xae\x89\x0d\x26\x37\x67\x8b\x8f\x75\x81\x4b\x5a\x9b\x2c\x8c\xd6\x0b\x18\x36\x9b\x55\xfa\xd9\xd0\x4a\xe6\x54\x2d\x58\x09\xbd\xc7\x39\x49\x7b\x6b\xb9\xd4\xa0\x97\x06\x72\x1b\xfc\xf1\x49\x2d\xb0\x62\xb4\xf2\xf5\x5e\x79\x8b\xa0\xe6\x2a\xe5\x9b\x98\x28\x0a\x95\x35\x3f\x12\x07\x30\x51\x93\x5f\x44\x7a\x63\x78\x1c\xdb\x5a\x38\x0a\xc6\xb7\xc0\x71\xad\xa0\x02\x1c\x6a\x54\x92\x04\xf4\xb0\x8c\x2a\x32\x83\x63\x6c\xc0\xd4\xa8\xec\x9a\x6b\x09\x4d\xf2\x32\x6d\xda\x5e\x64\x01\x23\xad\x67\x4c\x89\xd4\x4c\xdb\x13\xf5\x0a\x49\x78\x90\x87\x6a\xb1\x65\xbe\xc5\xbb\xeb\x35\x67\x32\x27\x8e\x77\x53\xf3\xcb\x75\x92\xf9\x39\x49\xd8\xa2\x98\xe4\x70\x0f\x79\x45\xb4\x29\x76\xa5\xdf\x39\x72\x2d\xb4\x26\x27\x00\xbc\xd5\xef\x3a\x57\xa8\xfe\x5a\xfe\x1f\x9f\xf5\x5a\xd6\x2c\xa8\xd4\xff\x8e\x18\xf5\xc3\x27\xe5\xef\xde\x6b\xbd\xc8\xbc\x07\x2e\xd4\x44\xfe\x7c\xf4\xdc\xfb\xb4\x20\x74\x92\x03\x9d\xcb\x0c\xbd\xf8\xfb\x0f\xde\x87\x26\x9d\xaa\xda\x26\x95\x1a\x48\xe9\x1f\x02\x29\xc2\x74\xe5\xbd\xbd\xc7\x7c\xd5\x25\xc9\x3b\xe8\xac\x14\x92\x2d\x90\x1b\x6b\x6a\xd9\xca\x41\x14\x8c\x0a\x08\xd6\xff\x77\xcf\xd0\xee\xbd\x5e\xfa\xe3\x86\x17\x42\x41\xe1\xd4\x0d\x57\x4d\xe3\x9d\x32\xf2\xbc\x67\xa0\x5e\xde\xa3\xf5\x7a\xd0\xb1\x2e\xd3\x73\x81\x64\x77\x40\x85\xcf\xe1\x71\xc6\x96\x66\x02\xb8\x35\xdf\x36\x8e\xf0\x1d\x94\x12\xa1\xcd\xb2\x25\xe6\x94\xd0\xb9\xb0\xfa\x8b\x50\x22\x09\xce\xc9\x67\x48\x27\x4e\x27\x4e\x14\x8c\xb5\xee\x4d\xe1\xd3\x34\x25\x6a\x7d\x85\x73\x84\xef\x31\xc9\x35\xa6\x4a\x85\x1e\x5b\xa8\x5d\x3d\x6b\x15\x38\x01\xf7\x82\xd0\xb9\xb2\x37\xf4\xb8\x71\xef\xfc\xb1\x54\x2d\xf5\x66\x8c\x2f\xb0\x44\x65\x21\x24\x07\xbc\x20\x74\xc6\xfc\x06\x5f\xb1\xf9\x5b\x0d\xf1\x46\x24\xb8\x80\xff\x35\xfe\xf0\x1e\xad\xd7\xa0\x7f\xbc\xfa\x97\x60\xb4\x66\xdd\xde\x97\x2f\x68\x5a\x92\x3c\xad\xca\x7c\xb4\x48\x8d\x57\x6b\xbd\xde\x3b\xf1\x8c\xaa\x05\x2e\x50\xc9\x73\x81\x64\x86\x25\x12\x19\x2b\xf3\x14\x51\x26\x11\x2e\x0a\xc0\xca\x56\x41\xca\xfe\x16\x62\x94\xb3\xf9\xd7\xd9\x4d\xaa\x7d\xbe\xb3\xc1\xe0\x9b\x34\xf0\x29\x32\x76\xed\xda\x73\x52\x72\x82\x76\x73\x36\x9f\x6b\x36\x1b\x63\xbc\x29\x73\x1c\xfe\xfd\x91\x93\xca\xde\x1c\xdf\x91\xe2\x54\x63\xbd\x62\xf3\x8f\x37\x57\x95\xfc\x39\x15\x65\xe1\xd7\x6b\x74\x74\x52\x0b\x99\x83\xb0\x0a\x07\xd9\x11\xd7\x1e\xd1\xe7\x46\x76\xaa\x2a\x2a\xf1\xae\x5b\xa2\xa5\x05\xb5\x2d\x71\x0f\xc4\x8d\xd2\x0a\xcd\x35\x96\x99\xa2\xa9\xd1\xeb\x64\xf6\xaa\x6a\x7b\x38\x2a\xd4\x2f\xe0\x9c\x71\x8d\xaf\x42\xf8\x46\xbd\xf2\xf0\x85\xef\x6b\x0d\xe7\xf0\x69\x01\xb9\x01\xc1\x72\xed\xf4\xd4\xd0\xee\x67\xd3\xe8\xfe\x39\x03\x0a\xea\xad\xee\x65\x64\x54\x89\xd0\x0e\x1f\x63\x99\x2c\x89\xcc\xd4\xc2\x15\xa3\x41\xed\x6d\x19\x58\xc5\xf1\x4c\x5b\xb2\x91\x0f\xca\x4e\x16\x20\x95\x8d\x3c\x48\x72\x26\x60\x10\xc8\xd6\x32\x03\x8a\x16\xf8\x4e\x7b\x9b\x32\x40\x52\xad\xdb\xa5\xab\x75\x84\xd0\xad\xb2\xb6\x17\x80\xa9\x15\xdd\x15\x2b\x51\x82\xa9\x12\x5d\x41\x16\x45\xbe\xd2\xde\x97\x26\xd2\x81\x59\x89\x29\xe3\xc9\x92\x51\x53\x86\x76\xb5\xb0\x26\x1e\xa9\x4a\xe9\xfd\x0c\xd3\x31\x53\xab\x27\x24\xca\x42\x4f\xa0\x53\x48\xb0\x42\xad\xcd\x74\x22\x50\x82\x05\x98\x76\x06\x95\x45\x5a\xbd\xd4\xc3\x6b\x0a\x55\xdb\x07\xba\xa1\x9a\xc1\x0a\x03\xe3\x64\x4e\x94\x96\x71\xdc\x4d\x89\x19\x8d\x19\xbe\x87\x2e\x16\x87\x8c\x23\x49\x66\x2b\x52\xec\x41\x94\xf5\x93\x62\xdd\x0d\x2d\x06\x8f\x09\x4d\xc0\x34\xd1\xc0\xa6\x70\x4f\xb0\x04\xa4\x5d\x37\xcd\x4a\xa9\x52\x34\xb9\x95\x91\x29\x64\xf8\x9e\x30\x8e\x96\x60\xe8\xae\x3c\x61\x44\x28\xb5\xc2\x19\x4e\xb2\x51\x8d\x61\x07\xdd\x80\xc4\x96\x0c\x37\x16\x0d\xaa\x0c\xd3\x34\xd7\x2b\xae\x19\xaa\x2c\xd3\x5e\x89\xab\xb5\x89\xee\xce\xb2\x98\x73\x9c\xaa\x45\x57\x05\x59\xbd\xfb\xd2\x52\x00\xd5\x63\x41\x4e\x2a\x88\xbd\x3d\xd4\x7c\xb4\xd0\x36\xf4\x85\xaa\xd4\x0d\x2d\xdf\xc7\xd6\xe3\x75\x43\xbb\x32\x03\xe7\x12\xab\xbc\x8e\x1b\x9d\x82\x86\x69\xff\x80\x86\x4b\x4b\xf7\x8a\x06\x47\x0e\xbe\xbb\x7d\xbb\x4d\x1f\xe2\x44\x21\x89\xaa\xaf\x78\x69\x0e\x0b\x26\x21\x28\x55\xab\xa8\x6a\x11\xa1\x1d\x7b\x7a\x9a\x98\x3c\xa8\x29\xce\x70\xc0\x54\x5b\x2d\x2d\x12\xc6\x39\x24\x32\x5f\x69\xbf\x58\xa2\x24\x4c\x88\x5c\x29\xd4\x9c\xe1\x94\xd0\x79\xd0\xa5\x6d\x4c\xbb\x05\x16\x62\x62\x35\xad\x48\x32\x58\xf4\x76\x6e\x07\x9a\x9e\xce\xde\x35\x48\x5b\xdd\x1d\xc1\xa4\x34\x84\xa1\xc7\xae\x5e\xf4\x9b\x1e\x6a\x3a\x90\xf4\x50\xe3\x02\x57\x1e\x5c\x2f\x4d\xda\x74\xd8\x9d\xaa\x99\x55\x7f\xd3\xbf\x1f\xc5\x1f\x55\xa2\x8f\x3d\x6a\x64\x7b\x40\xcd\x09\x14\xe7\xf9\xe8\x52\x8c\xc7\x57\xd7\x58\x08\x99\x71\x56\xce\x33\x2f\x66\x66\x24\x45\x51\x6e\x38\x65\xe2\x74\xa3\x2b\x22\x24\x50\x65\x8b\x88\x91\x2a\xab\x05\x7b\xbd\x56\xba\xe4\xe5\xcb\xbf\x69\xd5\x1c\x71\xc1\x2b\xf8\x9a\x13\xad\x6e\x30\x6f\x82\x1e\xd9\x54\x63\xe3\x79\xf9\xf2\x6f\x27\x7d\xc3\x2a\xa8\xd1\xf1\x03\xb5\x47\xd6\x23\x28\x54\x0d\xfe\x96\x54\x34\x16\xfa\x1f\xa6\x5a\xf7\x2a\xf1\xd0\x23\x15\xa9\x7e\x0c\xa4\xc9\x48\x50\xd5\xcb\x9b\x85\x67\x93\xc0\x74\xc9\x8a\xf6\x89\xb1\x45\x51\x4a\x78\x5b\xe6\x79\x43\x8d\x56\xb2\xf2\x33\xa8\x89\x7e\x4f\xea\x49\xc5\x6a\x32\x65\x68\xf8\x12\xab\xe4\xc3\x4d\xd4\x6a\x5a\x51\x3a\xb6\x8a\xc0\x58\x3c\x1c\x8a\x1c\x27\x20\x2c\x40\xa5\xd2\x74\x74\x4b\x30\x46\x7b\xc6\x94\xc2\xbf\x3b\x2b\xf3\xbc\xf5\xb6\x21\x58\x4e\xfe\xcf\x36\x68\xf3\x16\x1b\x07\xd1\x2a\x9f\x45\x15\xf7\xa0\x87\xd3\x83\x0d\x05\xb6\x51\xf7\x9d\xb4\x18\xa6\x4e\x3c\xd6\xf5\x93\xd2\x0b\x5f\xc5\x2f\x3b\x65\xd5\x73\x0d\x4e\x08\x9d\x70\x48\x89\x9a\x38\xea\xa5\xb7\x92\xf5\xc6\x17\xfb\xd4\x96\x9c\x10\x79\xc5\x8a\xda\x6c\x36\xa3\xdd\xbe\xf4\x4c\x46\x59\x2a\x33\x90\x22\x01\x42\xdb\xe4\x09\x4e\x32\x13\xfd\x44\x29\xc7\x42\x92\x04\xe7\x6a\xda\x5a\x14\x9c\xdd\x03\x2a\x80\x6b\x33\x8e\x26\x10\xca\xf4\x78\x7c\x35\x36\x48\xce\x70\x92\x55\xec\x56\xe4\x58\xe4\x93\x44\x7f\x50\xab\x07\x49\xe8\xf1\xf3\xa3\xa3\x23\x17\x2f\x19\x8f\xaf\x8e\x3d\x52\x7d\x44\x4d\xd7\xa9\x87\x2d\x0c\xb9\xd6\xe5\x6a\x47\x73\x87\xe5\x8f\xf3\x9c\x2d\x51\xc2\xe8\x8c\xcc\x4d\x74\x57\x4d\xc7\x8e\x07\x92\xa8\xc5\xb0\x88\x54\xa8\xdf\xc7\x1b\x7d\x6b\x3f\x6e\xe9\x88\x53\x0b\xde\x7d\x58\x14\x72\x15\x47\xf4\x13\xac\xd0\x41\x8c\x87\x86\x08\x1d\x06\xf2\xfc\x4b\x96\xb4\xd1\x1d\xac\x3a\x9a\x2c\x72\x32\xcf\xd4\x0a\x84\x43\x5a\x26\x46\x63\x28\x0e\x0e\x25\x1b\xce\x08\x17\x72\x38\x5d\x49\xa8\xaa\xf3\xc3\x02\x1e\x83\x83\x38\x40\x7f\x83\xce\x48\x91\x01\x17\x07\xb5\x5a\x6b\xf3\x3d\x31\x3e\x18\xc5\xfe\xc4\x80\x57\x24\xd8\xdf\x68\xcf\xab\xdf\xa2\x34\x16\x7e\x2d\xed\xa0\x69\x35\x63\xc7\x15\xeb\x75\x60\x46\xa8\x3d\xbf\xb8\xc6\x1c\x2f\xda\xd4\x1a\x0a\xcf\x2f\x90\x8e\x9f\x6d\xe9\x10\x50\x74\xf9\x0e\x01\xf5\x3b\xcd\x0a\x55\x43\x45\xb7\xfd\xed\xf3\xd7\x12\xd1\xef\x81\x53\xc4\x7b\x9e\xc1\xf3\x15\xc5\x0b\x92\xdc\x5e\x8d\x6f\x20\x61\x3c\x15\xbe\xd8\xa4\x2b\xa5\x2b\x12\xdd\x91\x93\x9c\xa1\xa3\xb8\x17\x4a\xe4\x13\x48\xd2\x6c\x92\x94\xfc\xbe\xd1\xe1\x6f\xce\xce\x2f\xce\xf4\xcb\xa0\xbf\x47\xc6\x7b\xa6\xed\x6f\x51\xf3\xcc\xfa\xd4\xb4\x05\xaf\xe3\xba\xa6\x3f\x8d\x86\x26\x54\x02\x4f\xa0\x90\x13\x30\xc5\xba\xfb\xc8\x3a\x41\x80\xf3\x33\x96\x42\xe5\x05\x31\xe8\x2f\x6e\x6f\xaf\x9b\x15\x1b\x8f\x81\x8e\x23\x2b\xe2\x5d\xb1\xf5\x1a\xbd\x42\xff\xd3\xf4\xe0\xa4\xf9\xe1\x24\xa8\xd3\x9a\xcd\xde\x48\xe3\xa0\x66\xd6\x58\x54\xf8\x54\x09\x86\x4d\x6b\x32\x3e\xba\xca\xdc\xf1\x70\xe9\x25\x8a\x5d\xc0\x19\xa8\x47\x08\xa4\x6a\xe2\x98\x92\xa2\x00\x79\xd0\xe2\x6e\xa2\x9a\x20\xcc\xd7\x6a\x34\x41\xea\xcc\x5c\xf7\x06\x6b\x9b\xdf\xfc\x5a\xe0\xc2\xd5\x16\xe2\x47\xeb\xe8\x54\x54\x77\x82\x9a\x8d\x9e\xa1\x5d\xe7\xc2\xd1\x9d\x31\xf5\x92\xba\xea\x56\xc0\xbf\x6b\xb0\x91\xd5\x67\xa7\xb3\x19\xa1\x44\xae\x46\xee\x9f\xdb\x55\x01\x68\x90\x30\x76\x47\x60\xe0\xca\x57\xc8\xd5\x94\x73\xb7\x1a\x2a\x3a\x2b\x4c\xd6\xd9\xee\xad\x3b\x0c\x14\xca\xb0\xc8\x5e\x35\x40\xc3\x4a\xcf\x74\x35\xe1\xdb\x0b\x2c\xb4\x17\x49\xb5\xec\x6b\xca\x3b\x82\xb4\x36\x60\x34\x5f\x59\x19\xa9\x39\xb1\x3f\xb7\x63\xd4\x79\x26\x7f\x72\xe1\x67\x3f\x5f\xe7\xe8\xc0\x5f\x95\x54\x11\xea\xaa\x97\x7a\x0b\x3b\x1d\x11\xe9\xbb\x66\xff\x19\xc5\xa8\xbb\xad\x6a\xe7\x1b\x9a\x16\x8c\x50\x3d\x61\xd9\xef\x75\xba\xe0\xe8\xd4\x1a\xdc\xff\x17\x19\xa7\xed\xe5\x35\x5a\xaf\x8f\x3d\x00\x1b\x40\x41\x3a\x74\x8b\x49\x2e\x5e\x79\x1f\xdf\xe1\x87\xb7\xea\x9d\x02\x50\x1f\xdd\x4c\xed\xc3\x28\x80\x70\x9a\x8e\x34\xa4\x96\x45\xf7\xb6\x29\x2d\x1b\xc4\x64\x07\x5d\x31\x9c\xa2\x29\xce\xd5\x8a\x05\xe1\x7c\xce\x38\x91\xd9\xe2\x04\x99\xa1\xa6\x5d\xfe\xac\xa4\x29\xe2\x6c\x4a\xe8\x33\xeb\x59\x22\xc2\xf7\xd3\x04\x3d\xeb\xc2\x5b\x0a\xf3\x6b\x83\xf8\xd4\xe1\x45\x03\x8d\x6d\xa2\xb1\x0d\x42\x0f\x6d\x67\xa9\xcd\x3d\xa9\x34\x4f\xd5\x4e\x27\x15\x3a\xec\xb9\xf2\x6b\x51\xe3\xa1\xc9\x93\x16\xac\xd2\x07\x42\x9b\x81\x72\x8b\x4a\xff\x12\xe3\x5f\x2b\xc6\x71\x29\x46\xce\xab\xa8\x5d\x69\x5a\xda\x16\xb8\xb0\x6e\xde\x25\xc9\x73\x34\xb5\x49\x79\x0c\xe9\x38\xaf\xce\xaf\xca\x00\xfd\x9c\x11\x09\x39\x11\xb2\x91\x62\x67\x39\x44\x68\x0a\x0f\xcf\x1a\x9c\x12\x75\xce\x6f\x13\x36\x67\x89\x99\x1f\x6a\xa8\xd1\x95\x7d\xe7\x83\xef\x16\x58\x66\x0a\xc8\x06\x5c\x6c\xa9\xba\x7c\x73\x1e\x23\xc2\x81\xe8\x49\x12\xd2\x26\x64\x0d\x38\x97\x68\x3f\x07\x0f\xd1\xa8\x6a\xda\xe8\xec\xf2\xfc\xe6\xc0\x24\x87\xda\x89\xef\x1c\xa8\x19\xad\x5f\xbe\xa0\x82\x13\x2a\x2b\x9a\x2f\x98\x90\x26\x64\x6b\x28\xad\x93\xc8\x02\xcf\xa3\x75\x5b\x2a\x4c\x2e\xe0\x8f\xf6\x3b\x70\x0d\x26\x03\x8b\xef\xa0\xa9\x50\xbc\x30\x4a\x44\x42\x6d\x7a\x60\x47\x8b\x42\x75\xa0\xc3\xa7\xad\x60\x4d\x38\x17\x6f\xff\x2b\xe8\x60\xae\x13\xc1\xf7\x4d\xf6\xd8\x0d\x96\x70\x45\x16\x44\x8a\x5a\x24\xbc\x75\x85\xfa\xac\xd3\x6f\x4d\x24\x29\x77\xaa\x34\xce\xca\xdd\xa5\x6b\xd7\xc4\x82\x5f\x9e\xc7\xd9\x74\x74\xd2\xcd\x25\x9e\xd7\xfc\x89\xb3\xe6\x79\x8b\x35\xdb\x90\xab\x9d\x19\x1d\x14\xee\x9a\x1c\xe3\x0e\xaa\x8f\x6a\x2d\xad\xc0\x94\x92\xfa\x27\xa3\x50\x49\x8b\x3f\xc0\x9f\xa3\xc1\xa0\xc7\xe7\xe4\x8d\x6d\x9c\xe7\xd6\xfb\xa2\x53\x26\x53\xc4\xd5\x58\x36\xe4\x7f\x66\x14\xc4\x08\xbd\xc1\x49\x86\x74\x5c\xc7\x0c\x14\x0b\x2a\x10\x46\x29\x18\x7f\x71\xaa\x61\x83\x90\xc4\xf3\x77\xaf\xd1\xf0\x7f\xa0\xe7\x3f\x20\x99\xb1\x52\x60\x9a\xa2\x1f\x5e\x6a\x3b\x4e\x67\x56\x82\x40\x8c\x23\x3c\x55\xca\xe9\xc7\x1a\xe4\xf9\x8b\x1f\x1b\x30\x11\x45\xa2\xeb\x52\xd2\x63\x82\x68\x4e\x78\x14\x37\x6a\x01\x3a\xf0\xb5\x84\x2e\xd1\x25\x92\x87\x4f\xd1\x6b\xcd\x0b\xab\x98\x9c\xf3\x42\xa0\xfd\x19\x67\x8b\x43\xc9\xd0\x72\xb9\x3c\x88\x51\x92\xd9\x21\xf9\x0c\xed\x4a\xa6\xf7\x01\xdc\xd8\xc2\xe3\xa6\x66\x73\xda\x3f\x22\x70\x2e\x7c\xa0\xe3\xf7\xd6\x53\xf5\x9a\xd0\xd4\x4e\x0f\x97\xc5\xfd\x4b\x5f\x00\x73\x3d\x3b\x22\xbb\x05\x41\x97\xb4\xb3\x46\xe8\x42\x55\x8b\x68\xb4\x5e\x6f\x76\x82\x05\xf9\xcc\x41\xee\x4f\x6f\xa5\x9b\xdd\xcb\x9b\x9c\xbb\xed\xba\x8d\x6f\xa2\xab\x45\xe3\x5f\xd5\xa4\xea\x1f\x65\x54\xf6\x3a\xe1\xfc\x26\x7f\x0f\xc6\xfe\x07\x71\xb2\x19\xdb\xb7\x55\xb4\xb6\xea\x7c\xcd\x48\xf8\xe1\xaf\x91\xf0\xcd\xfb\x6f\x9b\x51\xf0\xbf\x8f\x8f\xff\xcf\x77\xe2\xac\xab\xea\x3f\x84\x9f\xad\xb1\x10\xbc\xf1\x5c\xea\x5a\xa8\xdc\x8c\xd2\x40\xc4\x41\x3b\xc2\xff\x76\xf4\xdc\x05\x42\x8f\x0f\x0f\xf5\xbe\x0d\x86\xd6\x6b\x3f\x6b\xa7\x77\xd2\xdf\x6e\x21\x6e\x56\x31\x3b\x6a\xfe\xe5\x12\xb5\xcd\x96\x8b\x9a\xc2\xf8\xd4\x16\x36\xa9\x5d\xae\x61\x05\xe5\x04\xb7\xcc\x2c\x09\x8b\x42\x6f\xda\x18\x8c\xdf\xdc\x7c\x7a\x73\x33\xb0\x48\xcf\xb4\xb7\xc8\xec\x23\x74\xe4\xb7\xcd\xc2\x96\xfb\x54\x03\x86\xfe\x2a\xf4\x8d\x7c\x56\xa8\x95\x13\x1d\x78\xae\x22\x7d\xd1\x6a\xe6\xd9\xc7\xf1\xed\x87\x77\x93\x37\x37\x37\x1f\x6e\xc6\x03\xd3\xc0\xc6\x92\x7b\x67\x47\x23\xe8\xef\x8e\xb8\x6f\xdd\x2d\x77\x4d\x81\x67\x66\xef\x8e\xb2\x56\xde\xff\xe3\xf2\xfd\x2f\xc8\x6c\x97\x4c\x32\x48\xee\x90\x5a\x80\x99\x28\xbe\x32\xed\x4c\x06\x88\x5a\x87\x89\x78\x4f\xef\xa0\x8f\x36\xba\xd7\x1d\x39\x76\x3b\x53\xd0\x3e\xc7\x34\x65\x0b\xbb\x89\xeb\x5f\xa5\xd0\xf9\x37\x66\x57\xdb\x1d\x65\x4b\xaa\x91\x88\x03\x84\x45\x45\xb2\x46\xab\x48\x35\x0e\x6d\xbf\xdf\x32\xac\xde\xcd\x4d\x4e\x8b\xc1\xe9\x2d\x4c\x93\x4c\x8b\x3a\xa1\xc7\x5e\x99\x4c\xca\x42\x1c\x1f\x1e\xce\x89\xcc\xca\xe9\x28\x61\x8b\xc3\xbb\x72\x0a\x9c\x82\x04\x71\x68\x33\x05\x87\x26\x64\x31\xcd\xd9\xf4\x70\x81\x85\x04\x7e\x98\x30\x2a\x39\xcb\x73\xe0\xc2\x06\x34\x8a\xbb\xf9\x61\xb2\x48\xbd\x2f\xd6\xe5\x3e\x67\xdb\x2c\x20\x6a\xa6\xb8\x4c\xdb\x6a\xe9\x59\x0a\x30\x39\x4e\x38\xb9\xcb\xd9\xfc\x95\x2b\xfe\xda\xfc\x6e\x84\x9b\x7a\x66\xd7\x4d\x7a\xfb\xd7\x53\x10\x53\x65\xd2\x45\x47\x9d\x07\xc4\x28\x81\xc1\x70\xe0\x19\xa4\x95\xc1\x1d\xec\xd5\x6d\x18\x3b\xa8\x23\xb7\x2f\xd0\x88\x2f\x8e\x3c\x1b\x6e\x1d\xa9\xc3\x74\x8b\xdd\xab\x14\xe0\xef\xa6\x57\x47\x02\x28\xce\x7d\xb2\x51\x2c\xed\xb9\xb5\xf5\xca\x3d\xd1\xad\x52\x29\x11\x45\x8e\x57\x27\x5b\x43\xba\xa4\xd4\x3a\xfb\xda\xd7\x27\xc1\x54\xbd\x89\x65\x42\x96\x53\xc7\x08\x17\xd9\xf0\xd1\x35\x3b\x33\xca\xcc\x80\x81\x3d\xa1\x16\xf7\xb4\xd2\xfd\x10\x42\xbf\x0c\x75\x90\xe3\xe5\xd1\xcb\x4d\x44\xf4\xf7\x92\xfb\x39\xb4\x22\x3c\xb4\x5e\xff\xc1\x49\x84\x84\x02\x0b\x6f\xbf\x82\x0d\x93\x75\x61\x88\xca\xd4\xf6\xfa\x7a\xfd\xe4\x89\x73\x00\x3f\x41\xcd\xf4\x62\xbd\xa9\xac\x76\x0e\x37\x32\x8b\xc7\xe6\x7d\x6c\x4f\xc4\x6f\x9c\xfe\x5a\x53\x15\x8d\x01\x6d\x4e\x78\xad\x02\xf6\xb7\x67\xd7\xc8\xa6\x57\x8b\x70\xd5\xa1\x8c\xe4\xa4\x18\x57\x8b\x8e\xd1\xed\xd9\xf5\xeb\x20\x50\x53\xb9\xce\x65\x52\xe8\x28\x4b\x55\xc2\x79\x43\x83\xb7\x16\x81\xf6\xb1\xe8\xc4\xef\x7e\x90\xee\xaf\xce\xd9\xfa\x25\x20\xfb\x5f\xcf\xd0\x2e\x58\x3f\xae\x5e\x2a\xd5\x45\x7d\xf7\x6e\xb0\xfa\x41\xad\x47\xc7\xf7\x6c\x81\xca\xf5\x6b\xed\x95\xea\x7d\x63\xe7\x47\xcc\xf3\xf6\xcd\xfd\x08\x51\x3a\x03\x73\xaa\xd5\x07\x56\x32\x23\x4c\xf4\x17\xcb\xa3\x73\x48\x6c\x98\x73\xe3\xe2\xbf\xdb\xf4\x88\x12\xf8\x9d\x28\xfa\xbe\x86\xec\x9f\xaf\x2b\xaa\xe5\xc5\xef\xd5\x1f\xc1\x1b\x83\xa4\x7b\xff\xaa\x39\xca\x43\x2b\x98\x58\x58\x24\x36\x53\x20\xb3\x4d\xf8\x77\xd1\x45\xad\x55\xde\x26\x76\xbe\xa1\x96\x9d\x61\x93\x9c\xf9\x5a\x3d\xfe\x02\x60\x9b\x68\xd0\x0e\xfa\x78\xde\xab\xd9\xcb\xd4\xd7\xec\x1f\xcf\xbb\x35\x7b\x99\x1a\x6e\x56\x25\x1a\xdc\xac\xdf\x76\x72\x33\x0e\xd2\xfd\x75\x6b\xcd\x5e\x17\xfd\xce\x9a\xfd\xf7\x53\xed\xad\x4e\x50\xdd\xf3\xd5\x6a\x79\x0b\x6c\x7f\x3c\x95\xfa\xed\x58\x50\xa9\xc3\xaf\xe1\x43\x54\x95\xb9\x0d\x7f\xb5\x42\x7a\x1e\xaa\xab\x6f\xae\xed\x7e\xaf\xf1\x19\x66\x23\x5b\x86\xb8\xe3\x74\x52\xd0\xd9\x2d\xfa\x88\x85\x59\xb5\x12\x17\xb5\x0f\x81\x43\x01\x92\x98\x00\xb1\x3d\x35\xc7\x14\x6a\x2f\xd7\x83\x78\x5e\x11\x9c\xf0\xd4\x71\xea\x53\x5b\x28\xbd\xfc\x2f\x2d\x90\x5d\xf9\x5f\xc8\xb7\xa2\xe2\x09\x5f\x81\x69\xe5\x2c\xd0\xc0\x00\xed\x4a\x54\xab\xd2\x6b\x9b\x80\x4d\xbb\xab\x36\xbd\x50\x20\x2d\x5e\xde\xd9\x76\x58\x8c\xc1\xe2\x41\x9a\x04\x65\x73\x4e\xd0\x96\x38\x3e\xd8\x9d\x56\x43\x65\xfc\x23\xb3\xa9\x25\xf0\x63\x6e\xc4\x51\xcb\x9e\xa3\xa3\xda\x00\xba\x25\x86\x4b\xeb\xf5\x79\xef\xf6\x55\x37\x76\x8c\x6e\x89\xc4\xee\xba\xf6\x90\xf8\x5b\x4c\x83\xae\xe1\x10\x9e\x55\xa2\x9f\xfd\xd1\xd3\x03\x74\x88\xa6\x1c\xf0\xdd\x66\x03\xf6\x71\x36\xac\xfb\xaf\x1e\x55\x8d\x63\xbd\x0e\x9f\xa2\xb3\x0f\x37\xe3\x6a\x7f\x9d\xde\xc8\xe4\x9c\x65\x0b\x92\x64\x04\xf2\x3b\x9c\xdf\x2d\x30\xd5\x4e\x33\xeb\x27\xb5\xce\xb0\x61\xc2\xb8\x18\xb2\x02\xe8\xb0\xe1\x26\xf5\x0e\xe8\xf1\x47\x62\x63\x00\x6a\x55\xc5\xec\x90\x3b\x63\x5c\x58\x07\xaf\xfb\xbe\x83\xd4\x4b\x74\xcd\x61\xa6\x13\x84\xd1\x02\x64\xc6\x52\x81\x28\x40\x2a\x10\xae\x37\x05\xb3\xc2\x8c\x7c\x4c\x53\x94\x92\xd9\x0c\x38\x50\x89\x6e\x8c\x9b\x48\x09\xb7\x41\x48\x66\x68\xbf\x92\x33\x83\x0c\xbd\x42\x7b\x1f\xae\x6f\x2f\x3f\xbc\x1f\xef\x1d\x78\xc3\xd0\xdb\x39\xbd\x67\x2c\xe6\xe1\x99\x71\xf7\x0d\x75\xf2\x88\x15\xe0\x3d\x9b\x01\xcc\xb8\xd0\x2d\xd0\xdf\xcc\x27\xb4\x5e\xef\x21\x9c\x2f\xf1\x4a\xb4\x96\x72\x4d\xf8\x33\x0e\x29\x50\x49\xb0\xc9\xe7\xd9\x58\xb5\x07\x1f\xad\xbf\x89\xaf\x22\x22\x32\xcf\x6c\xac\xea\x9d\xe1\x78\xb4\x1a\xfb\x2d\xda\xce\x8d\x88\xad\x66\x8d\x22\xae\xb5\xee\xa3\x10\xbf\xc3\x0f\xc3\xd3\x39\xec\xa1\xe7\xff\xf5\xe2\xc7\x23\xdf\x37\xe8\x17\x52\xd0\x40\xe5\xf0\x76\x55\xc0\x1e\xda\xd3\xe7\x1b\x14\x39\x26\x14\x25\x19\xe6\x02\xe4\xab\x8f\xb7\x6f\x87\x3f\xee\xf5\x97\xbe\xd2\x87\x01\xec\xf9\x49\x24\x95\x53\xd2\x39\xb6\x3c\xef\xd1\x5f\xd2\xf4\xc7\x92\xa6\x96\x06\x6c\xae\x2f\xcc\x62\x7c\x58\x39\xfc\xa2\xab\x8c\x25\x91\x99\x83\xc4\x3a\x8c\x15\x6a\x3b\x17\xbd\x0a\x16\x1c\xee\x18\xcc\xb7\x84\xb7\xb3\x8b\xea\xa8\xdc\x68\x0c\x09\x6b\x2d\x14\xff\xf0\x69\x23\x75\xb2\x75\x24\x71\xed\xeb\x23\x0e\x5f\xbe\xe8\xdd\x49\xbf\x5b\xd2\xc6\xf7\x6c\x56\xd4\x4a\x32\xd1\xf0\x3f\x71\x16\xc5\x6f\xc6\xc1\x3f\x44\x26\xc3\xf7\x6d\xdd\xe6\xbc\x82\xad\x82\x71\x5a\xf5\x19\x36\x20\xd6\x11\xac\xf4\x52\x2a\xdc\x86\xce\xce\x70\xaf\xcb\xa3\x40\xc4\x9e\x11\x69\x03\xe8\xb7\x57\x63\x24\x28\x71\x4e\x8e\x2a\x95\xae\xa2\x41\x1f\x26\x61\xfa\x08\x38\x5a\x94\x42\x5a\x55\x6d\x0e\x30\x08\xfc\x58\x60\x06\xc0\xb3\xc6\x0e\xd3\xf1\xfb\xcb\x1a\x83\xdd\x29\x29\xf4\xee\x41\x56\xf2\x04\x90\x3e\xd8\x71\xc6\x14\x49\x44\x8e\x42\x1a\xc2\xc4\x01\xdb\x89\xe3\xf1\xd5\x19\x70\x49\x66\x3a\xcf\xf1\xe0\xb7\xd5\xc5\xdf\x3a\xe7\x05\x7d\xbf\x24\xb0\xdf\x56\x3b\xea\x1d\x78\x6d\x32\xf5\xb1\x79\x68\xbd\xd6\xc7\xec\x3d\x4a\x0d\xfc\xc5\xec\xdf\x84\xd9\xdf\x71\xde\x7a\xc4\x88\xfd\xcf\xcb\xd9\xfb\x13\x88\x42\x30\xee\x1e\xdf\x5d\x7f\xfa\x44\xc0\x3f\x41\x27\x6d\x5a\x45\xe8\xa9\x39\x61\x8b\x05\x50\x89\xae\xdf\xbc\x43\x22\xc3\xde\x01\xcb\xd5\x11\xd0\x89\xb4\x19\x57\xc2\x4d\xf8\x73\x35\x0b\xeb\x9d\x01\xcd\xdc\x39\x4c\x75\xee\x59\x02\xc1\x89\xe5\xae\xc6\x1d\x57\xcd\xb1\x9f\xe4\xa6\x7a\x13\x16\x67\x19\x24\x77\xa2\x5c\x34\xd6\x39\x22\x9f\x24\xb5\xfc\xa0\xae\xa7\x89\xcc\x93\xb8\x06\x4f\x02\x6c\x7a\x1f\xfb\xd7\x63\xeb\x93\xf9\xb7\x65\x9e\x9f\x65\x98\x50\x5f\xf8\x83\x76\xb9\x53\xb9\xfb\xda\xd7\xa4\x24\x86\xb5\xd5\x40\x21\x71\xa1\x8f\x74\xea\x79\xfc\xf8\xa0\x5f\x66\x72\x0f\x9c\xcc\xe2\x4c\xe9\x8e\x29\xa2\xde\x4d\x7c\x4a\x24\xf6\x37\x6b\x86\x83\x5a\xba\x2f\xc6\xb7\xe3\xc6\x52\x6c\xc1\x38\x78\x8e\x59\xd1\xd5\xac\xc1\x58\x72\x92\xc8\xe1\x2d\xc7\x54\xa8\x11\x36\x1c\xdb\xdb\x15\x8e\xd1\x02\x3f\x0c\xf1\x1c\xaa\xb1\xe6\x2a\x7a\x87\x1f\x4e\xe7\xd0\x56\x09\xea\xdb\xa5\x39\x4b\x73\x5c\x4e\x53\xb6\xc0\xc4\xec\x17\x74\x27\x6c\x8e\xcb\xe9\xb9\x79\x5b\x8f\xbe\x08\x8e\x6b\x3b\x0c\xb4\x46\xd1\xff\x56\xe0\xad\xc3\x39\xea\x3d\xa6\xbd\xf2\xe5\xb1\xed\xb4\x94\xd9\xe8\xec\xf4\x2d\xc9\xf5\xa1\xa0\x41\xf6\x6b\x6c\xac\x85\x85\xaf\x61\x31\xbe\x38\x6d\x8d\x39\x73\xbe\x72\xef\xd0\xeb\x41\x5a\x53\xd4\x12\x4f\x23\x61\xb6\x82\x8e\x7e\xec\xc1\xfc\x49\x17\x37\xc7\x3a\x77\xe1\x4e\xa1\x90\x59\x97\x8c\xf4\xe1\x36\x7b\x1d\x09\xa3\xe7\x1a\xc3\x56\x83\x3d\x44\xa2\x43\x43\xd7\x78\xde\x18\xee\xde\x09\x01\x2f\xff\xfb\xef\xe8\xe5\x7f\xff\x80\x5e\xf5\x51\x52\x21\xd9\x5a\xa5\x6f\xa1\x95\x3a\xd3\xa4\x7d\x25\xf3\xb8\x0c\xe7\xc7\x6c\xe7\x74\x35\x6d\xb1\xa5\xd3\x07\xc7\xa5\xcc\xae\xfd\x22\x8a\x41\x3d\x3b\x41\xe3\x3c\xa8\x76\x45\xde\x98\xf0\xcc\xe8\xb4\x28\x6e\x18\x93\x7e\x27\xe9\xe0\x41\xc9\x09\x7a\x85\x0e\x0f\x82\x80\x5d\x95\xa2\xff\x42\x13\xd5\x85\xae\xd1\x5d\x8f\xec\x26\xd7\xd0\x83\x68\x58\xf1\x55\x93\x19\x3d\x11\x45\xff\x6d\xb7\x9b\x00\x1e\x0c\xfc\x50\xa1\x04\x2a\x89\xa9\x27\x4c\x89\xf5\x0e\x93\x70\x61\x95\x29\x4b\xcd\x04\xd1\x4a\x41\xed\x08\xa0\xe9\xa7\xe9\x57\xd7\xfb\x17\x1b\x65\x7b\xba\xec\x8d\x25\x55\x8f\x0c\xe3\x45\x3e\x88\x27\xa2\xda\x88\x4f\xf0\x34\x3a\x2c\x82\x6c\xdb\xb8\xa8\x7e\x22\x81\x4d\xfd\x3c\x3a\xba\x69\xb1\x8d\xcd\x69\x7a\xc1\x13\x39\x70\xaf\x2b\x99\x76\xeb\xba\x2e\x9a\x07\x5c\x6f\xc1\x9e\xc6\x61\xd7\x8f\x66\xce\x95\xd7\x9c\x6a\x57\x4b\x7d\x7a\xd9\xd7\x32\xac\xaa\xe1\x9d\xdf\xd9\x41\xd0\xef\x31\x08\x55\x53\x87\xf6\xee\x85\xa1\xdb\xdf\xf9\xf5\xfd\xa9\x84\xfc\xad\xbd\x7b\xa5\x7a\x06\x26\x94\xea\xf6\x20\xd4\xfb\x0a\xe2\xe3\xcd\x1c\x04\x53\x6f\xb3\x31\x4f\x98\xd5\x1d\x19\x9b\xd5\xfa\x28\x84\xb5\xf3\xf9\x02\x3f\x98\x3b\x13\xfc\xa3\xe8\x07\x0d\x09\xd0\x76\xd0\xe8\x35\x4b\x57\xd6\x0c\x19\xb4\x24\x4f\xef\xb4\xd7\x93\x65\xf4\x8e\x86\x1a\x57\xc7\x15\x0e\x31\xca\xa2\xd7\x41\x34\x08\xeb\xbd\x0f\xc2\xa3\x2e\x36\x2e\xb4\x1e\xb4\x07\xb0\x76\xcb\xbb\x92\xd9\xb8\xb8\xeb\xe8\xbc\x45\xb0\xa5\x82\x6f\xec\x83\x28\xa2\x3a\x3b\x58\x2d\xb6\xf7\x1c\x58\x80\x8a\xda\x4f\x52\xf8\x77\xad\x3c\xe6\xc6\x96\x2e\x1c\x27\x6d\x7b\x36\xc2\xba\xe8\x14\xe2\x0e\x35\x70\x07\x6c\xe8\xe5\x5e\xcb\x04\xae\x0c\x5b\x9d\x2d\xe8\xcf\xd7\x91\x29\x60\x57\xd9\x2d\xc7\xaf\xd0\xfe\x1c\xa4\x4d\xdd\xb8\xa4\x26\xb7\xbe\x31\xdb\x8f\xec\xc7\xfa\x98\x84\x00\xd3\xe1\x53\x8d\x4b\x69\x78\x9c\x62\x89\x95\x99\x2a\xf5\x0a\x5e\x59\xaf\xae\xf4\xc2\x7d\x6d\xd8\xa8\x55\xab\x69\x33\xfd\x64\x60\x09\x6c\xa4\x44\x0d\x22\x13\xae\x9f\x62\xe2\x97\xbb\x29\xf3\xae\x22\xc1\xf9\xff\x55\x11\x9b\x7a\x12\xe7\x96\x5a\xb0\x30\x1e\x59\x8d\xbc\x55\x16\xf8\x78\x7c\x55\x69\xb3\x47\x18\x61\x2d\x64\x1e\x9e\x83\x16\xa7\x77\x10\x50\x63\xef\xeb\x23\x65\xa9\x8b\x00\x08\xe2\x52\x32\xec\xa3\x17\x57\x91\x63\x64\x5f\xe9\x4c\x94\x70\xb9\x85\x9a\xbb\x22\x5d\xb2\x4a\x70\xc6\x6a\xb7\x8e\x0e\xc5\x21\x2a\xd9\xd1\xe1\x17\x5c\xe4\xe7\xc3\x7b\x97\xf5\xd5\x07\x73\x45\x3e\x4e\x78\x99\x83\x1e\x87\xe1\x0d\x78\x0e\xc2\xff\x7f\x94\x30\x3a\x8b\xa9\xd8\x80\xb0\x0f\x3f\x9f\x8e\xaf\xcf\x18\x07\x25\x45\xad\x8d\x38\x9b\xab\x67\x4b\x2c\x8a\xa1\x07\x37\x4c\xdc\x7e\xb7\xe1\x46\x72\x22\x1b\x77\x7a\x99\xba\xc5\x41\x2c\xcd\x02\x5b\x1e\xc8\x12\x4a\xd4\xaf\x38\x57\xa5\x47\xe4\x5e\x1e\xfd\xed\x2b\xc4\xa9\x75\x38\x66\xa8\xbd\x3f\x0a\xb8\x66\x5c\x5e\xd2\x9b\xea\x04\x8a\x8d\xe7\x35\x36\xb9\xb4\xd1\x5c\x40\xe6\x58\x4d\x1d\x6c\x73\xfc\xae\xb7\x4d\x36\x56\xfa\x8d\x32\xea\xd3\xa4\x71\x15\x16\x0a\xed\x8d\x93\xce\x02\x6a\x9e\x51\xef\x34\xf4\xc4\x1c\xad\x56\x9f\x92\x64\x4f\x24\x04\x69\xbf\x04\x78\xea\xc4\x8f\xea\x19\x83\x1c\x9e\x59\x2c\x1e\xca\x50\x28\x87\x55\x7e\x7a\xfa\xf0\x0c\xed\xe6\xc4\x9c\x1a\x52\xd9\x88\x37\x36\xb9\xd6\xa5\x87\xf4\x49\xa0\x29\xdd\x7a\x3f\x7c\xb4\xe0\x6f\x36\x60\xc6\x64\x4e\x09\xfd\x78\x73\xd5\xea\x38\xdf\x49\x70\xf4\xdc\x58\x7c\x55\x6b\x54\x31\xb5\x3e\xd9\x84\x75\xdb\x15\x91\x9e\x22\x89\xb9\x6a\xb1\x62\x4c\x35\x49\x62\xff\x9c\x98\xfa\x6c\x98\x67\x28\xe1\xa0\x3e\x34\x4e\x83\x69\xf2\x51\x1f\xf3\xe3\xfa\xa1\x3a\xbc\xa5\x97\xfb\xce\x8b\xa0\x01\xf5\x19\x46\x06\x4b\x17\xfe\xee\xc5\x4a\x38\xe4\x5e\x63\x41\x92\x73\x32\x07\x21\x0d\x9b\x94\x96\x83\x58\x97\x5a\x07\x7e\x67\x49\x73\x6e\xe0\x54\xbd\x1d\xb4\xb6\x65\x2a\x21\xd5\x9f\x82\xb5\x74\x88\xe4\x06\x70\xbe\x68\xaf\x00\xea\xf2\x93\x52\x00\x9f\xb8\x3b\x48\xbb\x11\xbd\x25\x79\x7c\xed\x1b\xdb\x34\xaa\xb0\xa7\xba\xec\xaf\x20\xcf\x20\xf8\xd5\xf4\xb5\x87\x53\xcb\x9c\x52\x08\x18\x27\x9f\x8d\xb4\x0c\xda\xe6\xc7\x36\x1d\x5f\x27\x9b\xea\x7f\x23\x91\x49\x5b\xca\xdb\xf3\x69\xb6\x7a\xf6\xc8\xe9\x23\x87\xbd\x53\xf3\xa3\xd8\x60\xaf\x9c\x4d\xff\xef\xa9\x6f\x21\xf4\xcc\x4a\x81\xdf\xc9\xe2\xae\xf2\xcb\xa3\x5f\x63\x0a\x61\xab\x66\xf5\xd9\x8b\xfe\xf3\xc7\xb5\x1d\x83\x7a\xbf\x8d\x1d\x69\x4e\x93\x5a\x55\x07\x18\x94\x02\x8c\x11\x86\xb0\x30\x0e\x16\xc9\xaa\xad\x50\xcf\xd0\xb4\x94\xf6\x1c\x5f\x76\x0f\x9c\x93\xd4\x3f\xf0\x6a\x0b\x09\x72\x96\xd5\x27\x55\x45\x87\xcf\xcb\x1b\x39\x1d\x1e\x9e\x66\x1f\x35\x70\x46\xbb\x28\xa6\x44\xb6\xad\x2b\x58\xa1\x77\x32\x35\x58\xb1\x5c\x2b\xfb\x5a\x4d\x45\xf0\x20\x39\x4e\x74\x5c\xd1\xb8\xf8\xfd\x08\x84\x64\x1a\xc6\x66\xc0\x6f\x60\xe2\xe3\x62\x25\x35\x8e\xce\x38\x09\x16\x42\xbd\xbb\x65\xd5\x75\x4f\x9b\x78\x24\x44\x3e\x34\xcd\x18\xaa\x66\x54\x3c\xf2\x02\x2c\x1c\x2f\x75\x90\xe5\xeb\x3a\xa1\xa3\x82\x6e\x9d\xb9\x2d\x36\x3f\x0c\xe8\x93\x6b\xde\x6f\xf0\x87\x79\x78\x52\x6f\x1b\x95\x8f\x47\x4c\xd2\xc8\x11\x05\xdf\xb0\xc9\xdb\x36\xee\x31\xe5\xfc\xc6\x6c\x39\x2d\xed\x20\x6d\xff\xa0\x25\x4c\x85\xb9\xe4\xc8\xbb\x4e\x7b\x6b\xaf\xa2\xff\x7c\xb4\x77\xeb\x78\x7c\xf5\xef\xe1\xd9\xde\x55\xe9\x3f\xde\x25\x4d\x0e\x69\xfb\x3a\x9f\x4d\x5b\x90\x7e\x19\xaa\xf5\xc3\xf0\xf2\xba\x89\xdb\x3f\xbc\xb1\xd7\xcc\xdd\x70\x2d\x45\x4f\xbd\xcd\x1b\xc2\x5d\xbd\xb1\xcb\x23\xbe\x4e\xe6\xba\x6a\xe8\x6b\xd9\x36\x03\xce\xc7\xeb\xe9\xd3\x5e\x1d\xda\x8b\xe4\xba\xbe\x7f\xb2\xbe\xe7\xe4\x31\xe5\xf5\x6d\x3f\x5e\xf9\x9e\x08\x44\x04\x51\x34\x32\xf2\x08\x47\x7a\x47\x34\x24\x4a\x4a\xd7\x04\x52\x5d\xf0\x15\x74\xda\xd6\xb4\x37\xbb\x3a\x7e\xdf\x94\x13\xd9\x8e\x9b\xde\x03\xda\x16\x44\x92\xb9\x9a\xbf\x2e\x6e\x6f\xaf\xd9\xc3\x0a\x7d\x2a\x73\x0a\x1c\x4f\x49\x4e\xe4\x2a\x00\x76\xae\xae\xe5\x72\x39\x32\x67\x0e\x25\x6c\x71\x38\xcd\xd9\xfc\xd0\xe2\x21\x74\x3e\x94\x19\x0c\xf5\xe9\xdf\x0f\xab\xe1\xbd\x8f\x6d\xb8\x24\x32\xb3\x67\x1c\xf5\x37\xd9\xa4\x4f\xb5\x9e\x56\x60\xae\x75\x53\xa5\x64\xc8\xdd\x77\x69\xa6\xca\x0e\x73\xce\xbb\xb6\x32\xcf\xfb\x36\x5c\x46\xc9\xfb\x12\xb9\xcd\xb2\xa2\xf1\x4b\xf3\x56\x4b\xaf\xee\xee\xa0\x98\x55\x68\xd1\xdb\xa2\xab\xe2\xe1\x52\xd6\x2a\xc6\xd6\x65\xd8\x21\xd9\x34\xed\xc1\x1b\x47\x3d\x06\x9a\x6e\xc0\xcb\x01\x3f\x1e\xef\x0d\xe0\xb4\x7d\xd7\xb5\x57\xc0\x7a\x8d\xf7\x1b\xe6\xae\x29\xab\xff\x3a\xe3\x41\x07\xb1\x06\x76\xa5\x3b\x38\xd8\xb6\x00\x9b\xcd\x06\x6d\x3f\xb1\x6b\x50\xf3\xde\x98\xad\x1a\xd4\xae\x63\x4b\xc3\xf7\x1b\xd7\xb9\x11\xee\x96\x3d\xca\xe4\x35\x06\x4b\x5f\xa2\x56\x47\x80\xbd\xcf\xd2\x31\x4f\xd4\x2a\xf3\x8d\x9d\xe8\xaa\xc7\xbf\xcd\x3d\xfe\xbc\xfc\x4a\xcc\x55\xea\x40\x67\x93\x63\x78\x6d\x4c\xf6\x75\x55\xa8\x1d\x04\xb1\x17\x7f\xfb\xf7\xfa\x46\x1f\x7d\xd9\x6f\x54\x29\xb0\x3b\x02\x13\x93\xdd\x15\x2d\x19\x55\x0a\xaa\x90\xc9\xfd\xea\x88\x17\x5a\xbc\xda\xa8\x8f\x3f\x9d\x78\x9b\x47\x2e\xb9\x67\x07\x5d\x52\x7d\x11\x26\x62\x33\x64\xb7\x85\x4b\xbe\xd2\x53\x1e\x85\x07\xe9\x5d\x61\x61\x02\x30\x53\x98\x31\x0e\xd6\x77\xa0\x98\x87\xa9\x29\x17\xa1\x56\x21\xa8\x82\x7b\x5d\xd4\xea\x69\xf0\x3d\x3c\x78\xb7\xdd\x06\xf4\x37\x3f\xba\x39\xf2\x06\x24\x5f\xbd\x67\xf4\x32\x85\x45\xc1\xa4\x4b\xdb\x6a\x9b\xd5\x6e\xe3\x34\xa3\xf9\x0a\x2d\x19\xbf\x13\xce\x3f\x99\x98\xdc\x11\x44\x84\xb6\xf0\x12\xb6\x28\x38\x08\x01\x69\x97\x31\x1d\xc9\xcf\x49\xd3\xd7\x58\x80\xf1\x87\x7c\xd5\x1a\xf9\x54\x6f\x7e\x1f\xea\x33\x58\x9c\x04\x6f\xed\x9c\x3a\x7c\x8a\x4e\x53\x7d\x3f\xb4\xbf\xa9\xb9\x99\x3a\x6b\x36\x15\xc6\xdb\xe4\x3b\xb6\xbc\x32\x5e\x9e\xd6\xa6\x68\x81\xed\x0d\x17\x96\x89\xa5\x81\xa1\x6f\x78\x62\x26\xaa\xb7\x42\xc6\xea\xdd\xda\xb9\x66\x9c\xd4\x4b\x40\x98\x03\x52\xf3\xac\xbb\xae\xd6\x05\x2b\x24\x43\xd8\xdd\x92\xe4\xbc\x32\xd6\x41\xf0\x4c\x97\x4b\xd3\xe6\x01\xe8\x6e\x19\xd3\x21\x3a\xfb\x19\x16\xd7\x1c\x66\xe4\xa1\xe1\xde\xd4\x08\xd1\xc0\x54\x14\xee\xc5\x1f\x0e\x36\xfb\x66\xcc\x13\x39\xa0\xe1\xef\x61\xb0\xe9\x37\x3d\x95\xe1\x4f\x72\xa2\x42\x87\x6c\x6c\xeb\x37\xb3\xfd\x15\xf3\xf0\x68\x35\x66\x72\xfd\x75\x4e\xc7\xf6\xe9\x0a\xdb\x2c\x39\x76\xd0\x7b\x86\xa0\x3a\x5d\xa8\xbe\xca\x7c\xc6\xb8\x2f\xb6\x8d\x42\xd6\xbf\xdb\x12\x84\xee\xb8\x6b\xb4\xe6\x2a\x27\x32\x05\x4a\x20\x1d\xa1\x1b\xc0\x82\xd1\xe3\xa6\xfe\x38\xd7\x1f\xc3\xd2\xdb\x93\xd0\x4c\x61\x8c\x67\x35\x76\xec\x84\x68\xe6\x26\x9b\xa3\x41\x91\x3e\x98\x56\x6f\x2b\x48\x72\x56\xa6\x4a\x64\xee\x89\x1e\xa0\x76\xc4\x6a\xb6\x95\x66\xe2\xeb\xdd\x97\x18\x4d\x9b\x74\xc0\x17\xbf\xcd\x41\xa4\x36\xd6\x1a\x6c\x99\x30\x5b\xc9\xcd\x2d\x64\x4a\x44\xf5\x39\xbb\x44\xa0\x29\x28\xed\xb5\x60\x94\x48\xc6\x21\xf5\xb0\xe8\xad\xe6\x6a\x86\xb6\x31\x3d\x24\xd8\x4c\x2e\x95\xce\xdb\xcf\xc9\x1d\x20\xb1\x12\x29\x99\x1f\xb4\xdb\xd7\x77\x00\xaa\xf1\x3c\x3f\x7f\xf1\x5f\xa3\xa3\xd1\xd1\xe8\x79\xa7\xf3\x25\xdc\xbc\x65\xca\x1d\x1f\x87\x37\x60\xb8\x27\x05\x35\x8d\xe5\xe1\xe1\x35\x5f\x73\x38\x69\x5c\x9a\xea\x7f\xff\x7f\x00\x00\x00\xff\xff\x30\x62\x16\x2a\x55\x92\x00\x00") + +func etcNginxTemplateNginxTmplBytes() ([]byte, error) { + return bindataRead( + _etcNginxTemplateNginxTmpl, + "etc/nginx/template/nginx.tmpl", + ) +} + +func etcNginxTemplateNginxTmpl() (*asset, error) { + bytes, err := etcNginxTemplateNginxTmplBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "etc/nginx/template/nginx.tmpl", size: 37461, mode: os.FileMode(420), modTime: time.Unix(1511120583, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + +var _ingressControllerCleanNginxConfSh = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x64\x92\x41\x6b\xe3\x48\x10\x85\xef\xfa\x15\x6f\x2d\x83\x77\x17\x47\x9d\x78\x0f\x0b\xc9\xc9\x9b\x64\x19\x91\x60\x43\xe4\x4c\x08\x98\x40\xbb\x55\x96\x0a\xa4\x6e\x4d\x75\x2b\xb6\x99\x99\xff\x3e\xb4\xe2\xcc\xc4\x8c\x8e\x55\x4f\xfd\xbe\x7a\x55\xe9\x1f\x6a\xc3\x56\x6d\xb4\xaf\x93\x24\xc5\xb5\xeb\x0e\xc2\x55\x1d\x30\x3b\xbf\xf8\x17\xab\x9a\x70\xd7\x6f\x48\x2c\x05\xf2\x98\xf7\xa1\x76\xe2\xb3\x24\x4d\x52\xdc\xb3\x21\xeb\xa9\x44\x6f\x4b\x12\x84\x9a\x30\xef\xb4\xa9\xe9\xbd\x33\xc5\x67\x12\xcf\xce\x62\x96\x9d\xe3\xcf\x28\x18\x1d\x5b\xa3\xbf\xae\x92\x14\x07\xd7\xa3\xd5\x07\x58\x17\xd0\x7b\x42\xa8\xd9\x63\xcb\x0d\x81\xf6\x86\xba\x00\xb6\x30\xae\xed\x1a\xd6\xd6\x10\x76\x1c\xea\xc1\xe6\xf8\x48\x96\xa4\x78\x3e\x3e\xe1\x36\x41\xb3\x85\x86\x71\xdd\x01\x6e\xfb\x51\x07\x1d\x06\xe0\xf8\xd5\x21\x74\x97\x4a\xed\x76\xbb\x4c\x0f\xb0\x99\x93\x4a\x35\x6f\x42\xaf\xee\xf3\xeb\xdb\x45\x71\x7b\x36\xcb\xce\x87\x5f\x1e\x6d\x43\xde\x43\xe8\x4b\xcf\x42\x25\x36\x07\xe8\xae\x6b\xd8\xe8\x4d\x43\x68\xf4\x0e\x4e\xa0\x2b\x21\x2a\x11\x5c\xe4\xdd\x09\x07\xb6\xd5\x14\xde\x6d\xc3\x4e\x0b\x25\x29\x4a\xf6\x41\x78\xd3\x87\x93\xb0\xde\xe9\xd8\x9f\x08\x9c\x85\xb6\x18\xcd\x0b\xe4\xc5\x08\xff\xcd\x8b\xbc\x98\x26\x29\x9e\xf2\xd5\xa7\xe5\xe3\x0a\x4f\xf3\x87\x87\xf9\x62\x95\xdf\x16\x58\x3e\xe0\x7a\xb9\xb8\xc9\x57\xf9\x72\x51\x60\xf9\x3f\xe6\x8b\x67\xdc\xe5\x8b\x9b\x29\x88\x43\x4d\x02\xda\x77\x12\xf9\x9d\x80\x63\x8c\x54\xc6\xcc\x0a\xa2\x13\x80\xad\x7b\x03\xf2\x1d\x19\xde\xb2\x41\xa3\x6d\xd5\xeb\x8a\x50\xb9\x57\x12\xcb\xb6\x42\x47\xd2\xb2\x8f\xcb\xf4\xd0\xb6\x4c\x52\x34\xdc\x72\xd0\x61\xa8\xfc\x36\x54\x16\x6f\x69\x15\xd7\xe9\x8d\x70\x17\x20\xd4\xba\x57\xf2\x30\xce\x7a\x32\x7d\xe0\x57\x02\xb5\x5d\x38\xa0\x61\x4b\x3e\x26\x67\x2b\xb6\xfb\xcc\x38\xbb\x8d\xc1\xfb\xe8\x1a\xaf\x8b\x3d\x5a\x27\x04\x1f\x27\x88\xe0\xda\xa2\x1f\xba\x1a\x95\x83\x50\x45\xfb\x64\x98\xaa\x8c\xc7\xd2\x6a\x5b\xfa\xcb\x24\xc5\x45\x76\x34\x1d\xb8\x84\x42\x2f\x16\x46\x8b\x30\x09\x4c\xad\x45\x9b\x40\xa2\x7c\x92\x62\xf6\x53\xfa\x01\x29\x49\xf1\x4f\xac\x77\x8d\x36\x84\xb6\x6f\x02\x47\xff\x8f\x8a\x88\x77\x46\x98\x78\xb5\x16\xa5\xaa\x09\xbe\xe1\x57\xe9\x05\xf8\x7b\xac\xd6\x93\xf1\x64\x6d\x4f\x7b\xea\x65\xac\xbe\x2e\xae\xd4\xcb\xda\x8e\xd5\xcd\xd5\xf7\x49\xf2\x23\x00\x00\xff\xff\x3b\xba\x15\x8d\x85\x03\x00\x00") + +func ingressControllerCleanNginxConfShBytes() ([]byte, error) { + return bindataRead( + _ingressControllerCleanNginxConfSh, + "ingress-controller/clean-nginx-conf.sh", + ) +} + +func ingressControllerCleanNginxConfSh() (*asset, error) { + bytes, err := ingressControllerCleanNginxConfShBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "ingress-controller/clean-nginx-conf.sh", size: 901, mode: os.FileMode(493), modTime: time.Unix(1509931394, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + +// Asset loads and returns the asset for the given name. +// It returns an error if the asset could not be found or +// could not be loaded. +func Asset(name string) ([]byte, error) { + cannonicalName := strings.Replace(name, "\\", "/", -1) + if f, ok := _bindata[cannonicalName]; ok { + a, err := f() + if err != nil { + return nil, fmt.Errorf("Asset %s can't read by error: %v", name, err) + } + return a.bytes, nil + } + return nil, fmt.Errorf("Asset %s not found", name) +} + +// MustAsset is like Asset but panics when Asset would return an error. +// It simplifies safe initialization of global variables. +func MustAsset(name string) []byte { + a, err := Asset(name) + if err != nil { + panic("asset: Asset(" + name + "): " + err.Error()) + } + + return a +} + +// AssetInfo loads and returns the asset info for the given name. +// It returns an error if the asset could not be found or +// could not be loaded. +func AssetInfo(name string) (os.FileInfo, error) { + cannonicalName := strings.Replace(name, "\\", "/", -1) + if f, ok := _bindata[cannonicalName]; ok { + a, err := f() + if err != nil { + return nil, fmt.Errorf("AssetInfo %s can't read by error: %v", name, err) + } + return a.info, nil + } + return nil, fmt.Errorf("AssetInfo %s not found", name) +} + +// AssetNames returns the names of the assets. +func AssetNames() []string { + names := make([]string, 0, len(_bindata)) + for name := range _bindata { + names = append(names, name) + } + return names +} + +// _bindata is a table, holding each asset generator, mapped to its name. +var _bindata = map[string]func() (*asset, error){ + "etc/nginx/nginx.conf": etcNginxNginxConf, + "etc/nginx/template/nginx.tmpl": etcNginxTemplateNginxTmpl, + "ingress-controller/clean-nginx-conf.sh": ingressControllerCleanNginxConfSh, +} + +// AssetDir returns the file names below a certain +// directory embedded in the file by go-bindata. +// For example if you run go-bindata on data/... and data contains the +// following hierarchy: +// data/ +// foo.txt +// img/ +// a.png +// b.png +// then AssetDir("data") would return []string{"foo.txt", "img"} +// AssetDir("data/img") would return []string{"a.png", "b.png"} +// AssetDir("foo.txt") and AssetDir("notexist") would return an error +// AssetDir("") will return []string{"data"}. +func AssetDir(name string) ([]string, error) { + node := _bintree + if len(name) != 0 { + cannonicalName := strings.Replace(name, "\\", "/", -1) + pathList := strings.Split(cannonicalName, "/") + for _, p := range pathList { + node = node.Children[p] + if node == nil { + return nil, fmt.Errorf("Asset %s not found", name) + } + } + } + if node.Func != nil { + return nil, fmt.Errorf("Asset %s not found", name) + } + rv := make([]string, 0, len(node.Children)) + for childName := range node.Children { + rv = append(rv, childName) + } + return rv, nil +} + +type bintree struct { + Func func() (*asset, error) + Children map[string]*bintree +} +var _bintree = &bintree{nil, map[string]*bintree{ + "etc": &bintree{nil, map[string]*bintree{ + "nginx": &bintree{nil, map[string]*bintree{ + "nginx.conf": &bintree{etcNginxNginxConf, map[string]*bintree{}}, + "template": &bintree{nil, map[string]*bintree{ + "nginx.tmpl": &bintree{etcNginxTemplateNginxTmpl, map[string]*bintree{}}, + }}, + }}, + }}, + "ingress-controller": &bintree{nil, map[string]*bintree{ + "clean-nginx-conf.sh": &bintree{ingressControllerCleanNginxConfSh, map[string]*bintree{}}, + }}, +}} + +// RestoreAsset restores an asset under the given directory +func RestoreAsset(dir, name string) error { + data, err := Asset(name) + if err != nil { + return err + } + info, err := AssetInfo(name) + if err != nil { + return err + } + err = os.MkdirAll(_filePath(dir, filepath.Dir(name)), os.FileMode(0755)) + if err != nil { + return err + } + err = ioutil.WriteFile(_filePath(dir, name), data, info.Mode()) + if err != nil { + return err + } + err = os.Chtimes(_filePath(dir, name), info.ModTime(), info.ModTime()) + if err != nil { + return err + } + return nil +} + +// RestoreAssets restores an asset under the given directory recursively +func RestoreAssets(dir, name string) error { + children, err := AssetDir(name) + // File + if err != nil { + return RestoreAsset(dir, name) + } + // Dir + for _, child := range children { + err = RestoreAssets(dir, filepath.Join(name, child)) + if err != nil { + return err + } + } + return nil +} + +func _filePath(dir, name string) string { + cannonicalName := strings.Replace(name, "\\", "/", -1) + return filepath.Join(append([]string{dir}, strings.Split(cannonicalName, "/")...)...) +} + diff --git a/internal/file/filesystem.go b/internal/file/filesystem.go new file mode 100644 index 000000000..3c8dd58b0 --- /dev/null +++ b/internal/file/filesystem.go @@ -0,0 +1,144 @@ +package file + +import ( + "os" + "path/filepath" + + "github.com/golang/glog" + + "k8s.io/kubernetes/pkg/util/filesystem" +) + +// Filesystem is an interface that we can use to mock various filesystem operations +type Filesystem interface { + filesystem.Filesystem +} + +// NewLocalFS implements Filesystem using same-named functions from "os" and "io/ioutil". +func NewLocalFS() (Filesystem, error) { + fs := filesystem.DefaultFs{} + + err := initialize(false, fs) + if err != nil { + return nil, err + } + + return fs, nil +} + +// NewFakeFS creates an in-memory filesytem with all the required +// paths used by the ingress controller. +// This allows running test without polluting the local machine. +func NewFakeFS() (Filesystem, error) { + fs := filesystem.NewFakeFs() + + err := initialize(true, fs) + if err != nil { + return nil, err + } + + return fs, nil +} + +// initialize creates the required directory structure and when +// runs as virtual filesystem it copies the local files to it +func initialize(isVirtual bool, fs Filesystem) error { + for _, directory := range directories { + err := fs.MkdirAll(directory, 0655) + if err != nil { + return err + } + } + + if !isVirtual { + return nil + } + + for _, file := range files { + f, err := fs.Create(file) + if err != nil { + return err + } + + _, err = f.Write([]byte("")) + if err != nil { + return err + } + + err = f.Close() + if err != nil { + return err + } + } + + err := fs.MkdirAll("/proc", 0655) + if err != nil { + return err + } + + glog.Info("Restoring generated (go-bindata) assets in virtual filesystem...") + for _, assetName := range AssetNames() { + err := restoreAsset("/", assetName, fs) + if err != nil { + return err + } + } + + return nil +} + +// restoreAsset restores an asset under the given directory +func restoreAsset(dir, name string, fs Filesystem) error { + data, err := Asset(name) + if err != nil { + return err + } + info, err := AssetInfo(name) + if err != nil { + return err + } + err = fs.MkdirAll(_filePath(dir, filepath.Dir(name)), os.FileMode(0755)) + if err != nil { + return err + } + + f, err := fs.Create(_filePath(dir, name)) + if err != nil { + return err + } + + _, err = f.Write(data) + if err != nil { + return err + } + + err = f.Close() + if err != nil { + return err + } + + //Missing info.Mode() + + err = fs.Chtimes(_filePath(dir, name), info.ModTime(), info.ModTime()) + if err != nil { + return err + } + return nil +} + +// restoreAssets restores an asset under the given directory recursively +func restoreAssets(dir, name string, fs Filesystem) error { + children, err := AssetDir(name) + // File + if err != nil { + return restoreAsset(dir, name, fs) + } + // Dir + for _, child := range children { + err = restoreAssets(dir, filepath.Join(name, child), fs) + if err != nil { + return err + } + } + return nil +} diff --git a/internal/file/structure.go b/internal/file/structure.go new file mode 100644 index 000000000..aa4cd74dd --- /dev/null +++ b/internal/file/structure.go @@ -0,0 +1,26 @@ +package file + +const ( + // AuthDirectory default directory used to store files + // to authenticate request + AuthDirectory = "/etc/ingress-controller/auth" + + // DefaultSSLDirectory defines the location where the SSL certificates will be generated + // This directory contains all the SSL certificates that are specified in Ingress rules. + // The name of each file is -.pem. The content is the concatenated + // certificate and key. + DefaultSSLDirectory = "/ingress-controller/ssl" +) + +var ( + directories = []string{ + "/etc/nginx/template", + "/run", + DefaultSSLDirectory, + AuthDirectory, + } + + files = []string{ + "/run/nginx.pid", + } +) diff --git a/internal/ingress/annotations/annotations.go b/internal/ingress/annotations/annotations.go index 47f709a4f..f154d566c 100644 --- a/internal/ingress/annotations/annotations.go +++ b/internal/ingress/annotations/annotations.go @@ -23,6 +23,7 @@ import ( extensions "k8s.io/api/extensions/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/ingress-nginx/internal/file" "k8s.io/ingress-nginx/internal/ingress/annotations/alias" "k8s.io/ingress-nginx/internal/ingress/annotations/auth" "k8s.io/ingress-nginx/internal/ingress/annotations/authreq" @@ -89,11 +90,11 @@ type Extractor struct { } // NewAnnotationExtractor creates a new annotations extractor -func NewAnnotationExtractor(cfg resolver.Resolver) Extractor { +func NewAnnotationExtractor(cfg resolver.Resolver, fs file.Filesystem) Extractor { return Extractor{ map[string]parser.IngressAnnotation{ "Alias": alias.NewParser(cfg), - "BasicDigestAuth": auth.NewParser(auth.AuthDirectory, cfg), + "BasicDigestAuth": auth.NewParser(file.AuthDirectory, fs, cfg), "CertificateAuth": authtls.NewParser(cfg), "ClientBodyBufferSize": clientbodybuffersize.NewParser(), "ConfigurationSnippet": snippet.NewParser(), diff --git a/internal/ingress/annotations/annotations_test.go b/internal/ingress/annotations/annotations_test.go index 4657de41e..b4c54819f 100644 --- a/internal/ingress/annotations/annotations_test.go +++ b/internal/ingress/annotations/annotations_test.go @@ -24,6 +24,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/ingress-nginx/internal/file" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" "k8s.io/ingress-nginx/internal/ingress/defaults" "k8s.io/ingress-nginx/internal/ingress/resolver" @@ -113,7 +114,8 @@ func buildIngress() *extensions.Ingress { } func TestSecureUpstream(t *testing.T) { - ec := NewAnnotationExtractor(mockCfg{}) + fs := newFS(t) + ec := NewAnnotationExtractor(mockCfg{}, fs) ing := buildIngress() fooAnns := []struct { @@ -137,6 +139,7 @@ func TestSecureUpstream(t *testing.T) { } func TestSecureVerifyCACert(t *testing.T) { + fs := newFS(t) ec := NewAnnotationExtractor(mockCfg{ MockSecrets: map[string]*apiv1.Secret{ "default/secure-verify-ca": { @@ -145,7 +148,7 @@ func TestSecureVerifyCACert(t *testing.T) { }, }, }, - }) + }, fs) anns := []struct { it int @@ -172,7 +175,8 @@ func TestSecureVerifyCACert(t *testing.T) { } func TestHealthCheck(t *testing.T) { - ec := NewAnnotationExtractor(mockCfg{}) + fs := newFS(t) + ec := NewAnnotationExtractor(mockCfg{}, fs) ing := buildIngress() fooAnns := []struct { @@ -202,7 +206,8 @@ func TestHealthCheck(t *testing.T) { } func TestSSLPassthrough(t *testing.T) { - ec := NewAnnotationExtractor(mockCfg{}) + fs := newFS(t) + ec := NewAnnotationExtractor(mockCfg{}, fs) ing := buildIngress() fooAnns := []struct { @@ -226,7 +231,8 @@ func TestSSLPassthrough(t *testing.T) { } func TestUpstreamHashBy(t *testing.T) { - ec := NewAnnotationExtractor(mockCfg{}) + fs := newFS(t) + ec := NewAnnotationExtractor(mockCfg{}, fs) ing := buildIngress() fooAnns := []struct { @@ -250,7 +256,8 @@ func TestUpstreamHashBy(t *testing.T) { } func TestAffinitySession(t *testing.T) { - ec := NewAnnotationExtractor(mockCfg{}) + fs := newFS(t) + ec := NewAnnotationExtractor(mockCfg{}, fs) ing := buildIngress() fooAnns := []struct { @@ -282,7 +289,8 @@ func TestAffinitySession(t *testing.T) { } func TestCors(t *testing.T) { - ec := NewAnnotationExtractor(mockCfg{}) + fs := newFS(t) + ec := NewAnnotationExtractor(mockCfg{}, fs) ing := buildIngress() fooAnns := []struct { @@ -372,3 +380,11 @@ func TestMergeLocationAnnotations(t *testing.T) { } } */ + +func newFS(t *testing.T) file.Filesystem { + fs, err := file.NewFakeFS() + if err != nil { + t.Fatalf("unexpected error creating filesystem: %v", err) + } + return fs +} diff --git a/internal/ingress/annotations/auth/main.go b/internal/ingress/annotations/auth/main.go index 1dd885f04..4caa13bfc 100644 --- a/internal/ingress/annotations/auth/main.go +++ b/internal/ingress/annotations/auth/main.go @@ -18,9 +18,6 @@ package auth import ( "fmt" - "io/ioutil" - "os" - "path" "regexp" "github.com/pkg/errors" @@ -35,9 +32,6 @@ import ( var ( authTypeRegex = regexp.MustCompile(`basic|digest`) - // AuthDirectory default directory used to store files - // to authenticate request - AuthDirectory = "/etc/ingress-controller/auth" ) // Config returns authentication configuration for an Ingress rule @@ -78,23 +72,13 @@ func (bd1 *Config) Equal(bd2 *Config) bool { type auth struct { r resolver.Resolver + fs file.Filesystem authDirectory string } // NewParser creates a new authentication annotation parser -func NewParser(authDirectory string, r resolver.Resolver) parser.IngressAnnotation { - os.MkdirAll(authDirectory, 0755) - - currPath := authDirectory - for currPath != "/" { - currPath = path.Dir(currPath) - err := os.Chmod(currPath, 0755) - if err != nil { - break - } - } - - return auth{r, authDirectory} +func NewParser(authDirectory string, fs file.Filesystem, r resolver.Resolver) parser.IngressAnnotation { + return auth{r, fs, authDirectory} } // Parse parses the annotations contained in the ingress @@ -129,7 +113,7 @@ func (a auth) Parse(ing *extensions.Ingress) (interface{}, error) { realm, _ := parser.GetStringAnnotation("auth-realm", ing) passFile := fmt.Sprintf("%v/%v-%v.passwd", a.authDirectory, ing.GetNamespace(), ing.GetName()) - err = dumpSecret(passFile, secret) + err = dumpSecret(passFile, secret, a.fs) if err != nil { return nil, err } @@ -145,7 +129,7 @@ func (a auth) Parse(ing *extensions.Ingress) (interface{}, error) { // dumpSecret dumps the content of a secret into a file // in the expected format for the specified authorization -func dumpSecret(filename string, secret *api.Secret) error { +func dumpSecret(filename string, secret *api.Secret, fs file.Filesystem) error { val, ok := secret.Data["auth"] if !ok { return ing_errors.LocationDenied{ @@ -153,13 +137,26 @@ func dumpSecret(filename string, secret *api.Secret) error { } } - // TODO: check permissions required - err := ioutil.WriteFile(filename, val, 0777) + f, err := fs.Create(filename) if err != nil { return ing_errors.LocationDenied{ Reason: errors.Wrap(err, "unexpected error creating password file"), } } + _, err = f.Write(val) + if err != nil { + return ing_errors.LocationDenied{ + Reason: errors.Wrap(err, "unexpected error writing password file"), + } + } + + err = f.Close() + if err != nil { + return ing_errors.LocationDenied{ + Reason: errors.Wrap(err, "unexpected error closing password file"), + } + } + return nil } diff --git a/internal/ingress/annotations/auth/main_test.go b/internal/ingress/annotations/auth/main_test.go index 3546bd025..94282fa2b 100644 --- a/internal/ingress/annotations/auth/main_test.go +++ b/internal/ingress/annotations/auth/main_test.go @@ -29,6 +29,7 @@ import ( extensions "k8s.io/api/extensions/v1beta1" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/ingress-nginx/internal/file" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" "k8s.io/ingress-nginx/internal/ingress/resolver" ) @@ -87,10 +88,11 @@ func (m mockSecret) GetSecret(name string) (*api.Secret, error) { } func TestIngressWithoutAuth(t *testing.T) { + fs := newFS(t) ing := buildIngress() _, dir, _ := dummySecretContent(t) defer os.RemoveAll(dir) - _, err := NewParser(dir, &mockSecret{}).Parse(ing) + _, err := NewParser(dir, fs, &mockSecret{}).Parse(ing) if err == nil { t.Error("Expected error with ingress without annotations") } @@ -108,7 +110,9 @@ func TestIngressAuth(t *testing.T) { _, dir, _ := dummySecretContent(t) defer os.RemoveAll(dir) - i, err := NewParser(dir, &mockSecret{}).Parse(ing) + fs := newFS(t) + + i, err := NewParser(dir, fs, &mockSecret{}).Parse(ing) if err != nil { t.Errorf("Uxpected error with ingress: %v", err) } @@ -139,7 +143,9 @@ func TestIngressAuthWithoutSecret(t *testing.T) { _, dir, _ := dummySecretContent(t) defer os.RemoveAll(dir) - _, err := NewParser(dir, mockSecret{}).Parse(ing) + fs := newFS(t) + + _, err := NewParser(dir, fs, mockSecret{}).Parse(ing) if err == nil { t.Errorf("expected an error with invalid secret name") } @@ -167,14 +173,24 @@ func TestDumpSecret(t *testing.T) { sd := s.Data s.Data = nil - err := dumpSecret(tmpfile, s) + fs := newFS(t) + + err := dumpSecret(tmpfile, s, fs) if err == nil { t.Errorf("Expected error with secret without auth") } s.Data = sd - err = dumpSecret(tmpfile, s) + err = dumpSecret(tmpfile, s, fs) if err != nil { t.Errorf("Unexpected error creating htpasswd file %v: %v", tmpfile, err) } } + +func newFS(t *testing.T) file.Filesystem { + fs, err := file.NewFakeFS() + if err != nil { + t.Fatalf("unexpected error creating filesystem: %v", err) + } + return fs +} diff --git a/internal/ingress/controller/checker_test.go b/internal/ingress/controller/checker_test.go index 51002a5ea..0aab41c49 100644 --- a/internal/ingress/controller/checker_test.go +++ b/internal/ingress/controller/checker_test.go @@ -25,8 +25,8 @@ import ( "testing" "k8s.io/apiserver/pkg/server/healthz" - "k8s.io/kubernetes/pkg/util/filesystem" + "k8s.io/ingress-nginx/internal/file" ngx_config "k8s.io/ingress-nginx/internal/ingress/controller/config" ) @@ -41,8 +41,10 @@ func TestNginxCheck(t *testing.T) { // port to be used in the check p := server.Listener.Addr().(*net.TCPAddr).Port - // mock filesystem - fs := filesystem.NewFakeFs() + fs, err := file.NewFakeFS() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } n := &NGINXController{ cfg: &Configuration{ @@ -59,13 +61,6 @@ func TestNginxCheck(t *testing.T) { } }) - // create required files - fs.MkdirAll("/run", 0655) - pidFile, err := fs.Create("/run/nginx.pid") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - t.Run("no process", func(t *testing.T) { if err := callHealthz(true, mux); err == nil { t.Errorf("expected an error but none returned") @@ -81,18 +76,9 @@ func TestNginxCheck(t *testing.T) { cmd.Wait() }() - pidFile.Write([]byte(fmt.Sprintf("%v", pid))) - pidFile.Close() - healthz.InstallHandler(mux, n) - t.Run("valid request", func(t *testing.T) { - if err := callHealthz(false, mux); err != nil { - t.Error(err) - } - }) - - pidFile, err = fs.Create("/run/nginx.pid") + pidFile, err := fs.Create("/run/nginx.pid") if err != nil { t.Fatalf("unexpected error: %v", err) } diff --git a/internal/ingress/controller/controller.go b/internal/ingress/controller/controller.go index d9cd90b7a..c8fa6e941 100644 --- a/internal/ingress/controller/controller.go +++ b/internal/ingress/controller/controller.go @@ -23,8 +23,6 @@ import ( "reflect" "sort" "strconv" - "strings" - "sync/atomic" "time" "github.com/golang/glog" @@ -40,7 +38,6 @@ import ( "k8s.io/ingress-nginx/internal/ingress/annotations/healthcheck" "k8s.io/ingress-nginx/internal/ingress/annotations/proxy" ngx_config "k8s.io/ingress-nginx/internal/ingress/controller/config" - "k8s.io/ingress-nginx/internal/k8s" ) const ( @@ -102,16 +99,6 @@ type Configuration struct { FakeCertificateSHA string } -// GetPublishService returns the configured service used to set ingress status -func (n NGINXController) GetPublishService() *apiv1.Service { - s, err := n.storeLister.GetService(n.cfg.PublishService) - if err != nil { - return nil - } - - return s -} - // sync collects all the pieces required to assemble the configuration file and // then sends the content to the backend (OnUpdate) receiving the populated // template as response reloading the backend if is required. @@ -185,131 +172,6 @@ func (n *NGINXController) syncIngress(item interface{}) error { return nil } -func (n *NGINXController) getStreamServices(configmapName string, proto apiv1.Protocol) []ingress.L4Service { - glog.V(3).Infof("obtaining information about stream services of type %v located in configmap %v", proto, configmapName) - if configmapName == "" { - // no configmap configured - return []ingress.L4Service{} - } - - _, _, err := k8s.ParseNameNS(configmapName) - if err != nil { - glog.Errorf("unexpected error reading configmap %v: %v", configmapName, err) - return []ingress.L4Service{} - } - - configmap, err := n.storeLister.GetConfigMap(configmapName) - if err != nil { - glog.Errorf("unexpected error reading configmap %v: %v", configmapName, err) - return []ingress.L4Service{} - } - - var svcs []ingress.L4Service - var svcProxyProtocol ingress.ProxyProtocol - // k -> port to expose - // v -> /: - for k, v := range configmap.Data { - externalPort, err := strconv.Atoi(k) - if err != nil { - glog.Warningf("%v is not valid as a TCP/UDP port", k) - continue - } - - rp := []int{ - n.cfg.ListenPorts.HTTP, - n.cfg.ListenPorts.HTTPS, - n.cfg.ListenPorts.SSLProxy, - n.cfg.ListenPorts.Status, - n.cfg.ListenPorts.Health, - n.cfg.ListenPorts.Default, - } - - if intInSlice(externalPort, rp) { - glog.Warningf("port %v cannot be used for TCP or UDP services. It is reserved for the Ingress controller", k) - continue - } - - nsSvcPort := strings.Split(v, ":") - if len(nsSvcPort) < 2 { - glog.Warningf("invalid format (namespace/name:port:[PROXY]:[PROXY]) '%v'", k) - continue - } - - nsName := nsSvcPort[0] - svcPort := nsSvcPort[1] - svcProxyProtocol.Decode = false - svcProxyProtocol.Encode = false - - // Proxy protocol is possible if the service is TCP - if len(nsSvcPort) >= 3 && proto == apiv1.ProtocolTCP { - if len(nsSvcPort) >= 3 && strings.ToUpper(nsSvcPort[2]) == "PROXY" { - svcProxyProtocol.Decode = true - } - if len(nsSvcPort) == 4 && strings.ToUpper(nsSvcPort[3]) == "PROXY" { - svcProxyProtocol.Encode = true - } - } - - svcNs, svcName, err := k8s.ParseNameNS(nsName) - if err != nil { - glog.Warningf("%v", err) - continue - } - - svc, err := n.storeLister.GetService(nsName) - if err != nil { - glog.Warningf("error getting service %v: %v", nsName, err) - continue - } - - var endps []ingress.Endpoint - targetPort, err := strconv.Atoi(svcPort) - if err != nil { - glog.V(3).Infof("searching service %v endpoints using the name '%v'", svcNs, svcName, svcPort) - for _, sp := range svc.Spec.Ports { - if sp.Name == svcPort { - if sp.Protocol == proto { - endps = n.getEndpoints(svc, &sp, proto, &healthcheck.Config{}) - break - } - } - } - } else { - // we need to use the TargetPort (where the endpoints are running) - glog.V(3).Infof("searching service %v/%v endpoints using the target port '%v'", svcNs, svcName, targetPort) - for _, sp := range svc.Spec.Ports { - if sp.Port == int32(targetPort) { - if sp.Protocol == proto { - endps = n.getEndpoints(svc, &sp, proto, &healthcheck.Config{}) - break - } - } - } - } - - // stream services cannot contain empty upstreams and there is no - // default backend equivalent - if len(endps) == 0 { - glog.Warningf("service %v/%v does not have any active endpoints for port %v and protocol %v", svcNs, svcName, svcPort, proto) - continue - } - - svcs = append(svcs, ingress.L4Service{ - Port: externalPort, - Backend: ingress.L4Backend{ - Name: svcName, - Namespace: svcNs, - Port: intstr.FromString(svcPort), - Protocol: proto, - ProxyProtocol: svcProxyProtocol, - }, - Endpoints: endps, - }) - } - - return svcs -} - // getDefaultUpstream returns an upstream associated with the // default backend service. In case of error retrieving information // configure the upstream to return http code 503. @@ -664,7 +526,9 @@ func (n *NGINXController) createUpstreams(data []*extensions.Ingress, du *ingres return upstreams } -func (n *NGINXController) getServiceClusterEndpoint(svcKey string, backend *extensions.IngressBackend) (endpoint ingress.Endpoint, err error) { +func (n *NGINXController) getServiceClusterEndpoint(svcKey string, + backend *extensions.IngressBackend) (endpoint ingress.Endpoint, err error) { + svc, err := n.storeLister.GetService(svcKey) if err != nil { return endpoint, err @@ -1067,18 +931,3 @@ func (n *NGINXController) getEndpoints( glog.V(3).Infof("endpoints found: %v", upsServers) return upsServers } - -func (n *NGINXController) isForceReload() bool { - return atomic.LoadInt32(&n.forceReload) != 0 -} - -// SetForceReload sets if the ingress controller should be reloaded or not -func (n *NGINXController) SetForceReload(shouldReload bool) { - if shouldReload { - atomic.StoreInt32(&n.forceReload, 1) - n.syncQueue.Enqueue(&extensions.Ingress{}) - return - } - - atomic.StoreInt32(&n.forceReload, 0) -} diff --git a/internal/ingress/controller/nginx.go b/internal/ingress/controller/nginx.go index 6ead782ff..3f1c42dc5 100644 --- a/internal/ingress/controller/nginx.go +++ b/internal/ingress/controller/nginx.go @@ -39,6 +39,7 @@ import ( "k8s.io/client-go/util/flowcontrol" "k8s.io/kubernetes/pkg/util/filesystem" + "k8s.io/ingress-nginx/internal/file" "k8s.io/ingress-nginx/internal/ingress" "k8s.io/ingress-nginx/internal/ingress/annotations/class" ngx_config "k8s.io/ingress-nginx/internal/ingress/controller/config" @@ -50,6 +51,7 @@ import ( "k8s.io/ingress-nginx/internal/net/dns" "k8s.io/ingress-nginx/internal/net/ssl" "k8s.io/ingress-nginx/internal/task" + "k8s.io/ingress-nginx/internal/watch" ) type statusModule string @@ -70,7 +72,7 @@ var ( // NewNGINXController creates a new NGINX Ingress controller. // If the environment variable NGINX_BINARY exists it will be used // as source for nginx commands -func NewNGINXController(config *Configuration) *NGINXController { +func NewNGINXController(config *Configuration, fs file.Filesystem) *NGINXController { ngx := os.Getenv("NGINX_BINARY") if ngx == "" { ngx = nginxBinary @@ -98,7 +100,7 @@ func NewNGINXController(config *Configuration) *NGINXController { stopLock: &sync.Mutex{}, - fileSystem: filesystem.DefaultFs{}, + fileSystem: fs, } n.stats = newStatsCollector(config.Namespace, class.IngressClass, n.binary, n.cfg.ListenPorts.Status) @@ -128,6 +130,7 @@ func NewNGINXController(config *Configuration) *NGINXController { n.cfg.UDPConfigMapName, n.cfg.ResyncPeriod, n.cfg.Client, + n.fileSystem, n.updateCh, ) @@ -144,9 +147,8 @@ func NewNGINXController(config *Configuration) *NGINXController { glog.Warning("Update of ingress status is disabled (flag --update-status=false was specified)") } - var onChange func() - onChange = func() { - template, err := ngx_template.NewTemplate(tmplPath, onChange) + onChange := func() { + template, err := ngx_template.NewTemplate(tmplPath, n.fileSystem) if err != nil { // this error is different from the rest because it must be clear why nginx is not working glog.Errorf(` @@ -163,7 +165,17 @@ Error loading new template : %v n.SetForceReload(true) } - ngxTpl, err := ngx_template.NewTemplate(tmplPath, onChange) + // TODO: refactor + if _, ok := fs.(filesystem.DefaultFs); !ok { + watch.NewDummyFileWatcher(tmplPath, onChange) + } else { + _, err = watch.NewFileWatcher(tmplPath, onChange) + if err != nil { + glog.Fatalf("unexpected error watching template %v: %v", tmplPath, err) + } + } + + ngxTpl, err := ngx_template.NewTemplate(tmplPath, n.fileSystem) if err != nil { glog.Fatalf("invalid NGINX template: %v", err) } @@ -563,7 +575,7 @@ func (n *NGINXController) OnUpdate(ingressCfg ingress.Configuration) error { dh, ok := secret.Data["dhparam.pem"] if ok { - pemFileName, err := ssl.AddOrUpdateDHParam(nsSecName, dh) + pemFileName, err := ssl.AddOrUpdateDHParam(nsSecName, dh, n.fileSystem) if err != nil { glog.Warningf("unexpected error adding or updating dhparam %v file: %v", nsSecName, err) } else { @@ -584,6 +596,8 @@ func (n *NGINXController) OnUpdate(ingressCfg ingress.Configuration) error { cfg.EnableBrotli = false } + svc, _ := n.storeLister.GetService(n.cfg.PublishService) + tc := ngx_config.TemplateConfig{ ProxySetHeaders: setHeaders, AddHeaders: addHeaders, @@ -601,7 +615,7 @@ func (n *NGINXController) OnUpdate(ingressCfg ingress.Configuration) error { RedirectServers: redirectServers, IsSSLPassthroughEnabled: n.isSSLPassthroughEnabled, ListenPorts: n.cfg.ListenPorts, - PublishService: n.GetPublishService(), + PublishService: svc, } content, err := n.t.Write(tc) diff --git a/internal/ingress/controller/reload.go b/internal/ingress/controller/reload.go new file mode 100644 index 000000000..a40b0e0ca --- /dev/null +++ b/internal/ingress/controller/reload.go @@ -0,0 +1,22 @@ +package controller + +import ( + "sync/atomic" + + extensions "k8s.io/api/extensions/v1beta1" +) + +func (n *NGINXController) isForceReload() bool { + return atomic.LoadInt32(&n.forceReload) != 0 +} + +// SetForceReload sets if the ingress controller should be reloaded or not +func (n *NGINXController) SetForceReload(shouldReload bool) { + if shouldReload { + atomic.StoreInt32(&n.forceReload, 1) + n.syncQueue.Enqueue(&extensions.Ingress{}) + return + } + + atomic.StoreInt32(&n.forceReload, 0) +} diff --git a/internal/ingress/controller/stream.go b/internal/ingress/controller/stream.go new file mode 100644 index 000000000..0ee797190 --- /dev/null +++ b/internal/ingress/controller/stream.go @@ -0,0 +1,140 @@ +package controller + +import ( + "strconv" + "strings" + + "github.com/golang/glog" + + apiv1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/intstr" + + "k8s.io/ingress-nginx/internal/ingress" + "k8s.io/ingress-nginx/internal/ingress/annotations/healthcheck" + "k8s.io/ingress-nginx/internal/k8s" +) + +func (n *NGINXController) getStreamServices(configmapName string, proto apiv1.Protocol) []ingress.L4Service { + glog.V(3).Infof("obtaining information about stream services of type %v located in configmap %v", proto, configmapName) + if configmapName == "" { + // no configmap configured + return []ingress.L4Service{} + } + + _, _, err := k8s.ParseNameNS(configmapName) + if err != nil { + glog.Errorf("unexpected error reading configmap %v: %v", configmapName, err) + return []ingress.L4Service{} + } + + configmap, err := n.storeLister.GetConfigMap(configmapName) + if err != nil { + glog.Errorf("unexpected error reading configmap %v: %v", configmapName, err) + return []ingress.L4Service{} + } + + var svcs []ingress.L4Service + var svcProxyProtocol ingress.ProxyProtocol + // k -> port to expose + // v -> /: + for k, v := range configmap.Data { + externalPort, err := strconv.Atoi(k) + if err != nil { + glog.Warningf("%v is not valid as a TCP/UDP port", k) + continue + } + + rp := []int{ + n.cfg.ListenPorts.HTTP, + n.cfg.ListenPorts.HTTPS, + n.cfg.ListenPorts.SSLProxy, + n.cfg.ListenPorts.Status, + n.cfg.ListenPorts.Health, + n.cfg.ListenPorts.Default, + } + + if intInSlice(externalPort, rp) { + glog.Warningf("port %v cannot be used for TCP or UDP services. It is reserved for the Ingress controller", k) + continue + } + + nsSvcPort := strings.Split(v, ":") + if len(nsSvcPort) < 2 { + glog.Warningf("invalid format (namespace/name:port:[PROXY]:[PROXY]) '%v'", k) + continue + } + + nsName := nsSvcPort[0] + svcPort := nsSvcPort[1] + svcProxyProtocol.Decode = false + svcProxyProtocol.Encode = false + + // Proxy protocol is possible if the service is TCP + if len(nsSvcPort) >= 3 && proto == apiv1.ProtocolTCP { + if len(nsSvcPort) >= 3 && strings.ToUpper(nsSvcPort[2]) == "PROXY" { + svcProxyProtocol.Decode = true + } + if len(nsSvcPort) == 4 && strings.ToUpper(nsSvcPort[3]) == "PROXY" { + svcProxyProtocol.Encode = true + } + } + + svcNs, svcName, err := k8s.ParseNameNS(nsName) + if err != nil { + glog.Warningf("%v", err) + continue + } + + svc, err := n.storeLister.GetService(nsName) + if err != nil { + glog.Warningf("error getting service %v: %v", nsName, err) + continue + } + + var endps []ingress.Endpoint + targetPort, err := strconv.Atoi(svcPort) + if err != nil { + glog.V(3).Infof("searching service %v endpoints using the name '%v'", svcNs, svcName, svcPort) + for _, sp := range svc.Spec.Ports { + if sp.Name == svcPort { + if sp.Protocol == proto { + endps = n.getEndpoints(svc, &sp, proto, &healthcheck.Config{}) + break + } + } + } + } else { + // we need to use the TargetPort (where the endpoints are running) + glog.V(3).Infof("searching service %v/%v endpoints using the target port '%v'", svcNs, svcName, targetPort) + for _, sp := range svc.Spec.Ports { + if sp.Port == int32(targetPort) { + if sp.Protocol == proto { + endps = n.getEndpoints(svc, &sp, proto, &healthcheck.Config{}) + break + } + } + } + } + + // stream services cannot contain empty upstreams and there is no + // default backend equivalent + if len(endps) == 0 { + glog.Warningf("service %v/%v does not have any active endpoints for port %v and protocol %v", svcNs, svcName, svcPort, proto) + continue + } + + svcs = append(svcs, ingress.L4Service{ + Port: externalPort, + Backend: ingress.L4Backend{ + Name: svcName, + Namespace: svcNs, + Port: intstr.FromString(svcPort), + Protocol: proto, + ProxyProtocol: svcProxyProtocol, + }, + Endpoints: endps, + }) + } + + return svcs +} diff --git a/internal/ingress/controller/template/template.go b/internal/ingress/controller/template/template.go index b5cea55e6..751b5d19a 100644 --- a/internal/ingress/controller/template/template.go +++ b/internal/ingress/controller/template/template.go @@ -29,16 +29,17 @@ import ( text_template "text/template" "github.com/golang/glog" + "github.com/pkg/errors" "github.com/pborman/uuid" extensions "k8s.io/api/extensions/v1beta1" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/ingress-nginx/internal/file" "k8s.io/ingress-nginx/internal/ingress" "k8s.io/ingress-nginx/internal/ingress/annotations/ratelimit" "k8s.io/ingress-nginx/internal/ingress/controller/config" ing_net "k8s.io/ingress-nginx/internal/net" - "k8s.io/ingress-nginx/internal/watch" ) const ( @@ -50,32 +51,33 @@ const ( // Template ... type Template struct { tmpl *text_template.Template - fw watch.FileWatcher - bp *BufferPool + //fw watch.FileWatcher + bp *BufferPool } //NewTemplate returns a new Template instance or an //error if the specified template file contains errors -func NewTemplate(file string, onChange func()) (*Template, error) { - tmpl, err := text_template.New("nginx.tmpl").Funcs(funcMap).ParseFiles(file) +func NewTemplate(file string, fs file.Filesystem) (*Template, error) { + data, err := fs.ReadFile(file) if err != nil { - return nil, err + return nil, errors.Wrapf(err, "unexpected error reading template %v", file) } - fw, err := watch.NewFileWatcher(file, onChange) + + tmpl, err := text_template.New("nginx.tmpl").Funcs(funcMap).Parse(string(data)) if err != nil { return nil, err } return &Template{ tmpl: tmpl, - fw: fw, - bp: NewBufferPool(defBufferSize), + // fw: fw, + bp: NewBufferPool(defBufferSize), }, nil } // Close removes the file watcher func (t *Template) Close() { - t.fw.Close() + //t.fw.Close() } // Write populates a buffer using a template with NGINX configuration diff --git a/internal/ingress/controller/template/template_test.go b/internal/ingress/controller/template/template_test.go index e147a2e7b..3fd0cd4e3 100644 --- a/internal/ingress/controller/template/template_test.go +++ b/internal/ingress/controller/template/template_test.go @@ -26,6 +26,7 @@ import ( "strings" "testing" + "k8s.io/ingress-nginx/internal/file" "k8s.io/ingress-nginx/internal/ingress" "k8s.io/ingress-nginx/internal/ingress/annotations/authreq" "k8s.io/ingress-nginx/internal/ingress/annotations/rewrite" @@ -174,13 +175,13 @@ func TestTemplateWithData(t *testing.T) { if dat.ListenPorts == nil { dat.ListenPorts = &config.ListenPorts{} } - tf, err := os.Open(path.Join(pwd, "../../../../rootfs/etc/nginx/template/nginx.tmpl")) - if err != nil { - t.Errorf("unexpected error reading json file: %v", err) - } - defer tf.Close() - ngxTpl, err := NewTemplate(tf.Name(), func() {}) + fs, err := file.NewFakeFS() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + ngxTpl, err := NewTemplate("/etc/nginx/template/nginx.tmpl", fs) if err != nil { t.Errorf("invalid NGINX template: %v", err) } @@ -207,13 +208,12 @@ func BenchmarkTemplateWithData(b *testing.B) { b.Errorf("unexpected error unmarshalling json: %v", err) } - tf, err := os.Open(path.Join(pwd, "../../../rootfs/etc/nginx/template/nginx.tmpl")) + fs, err := file.NewFakeFS() if err != nil { - b.Errorf("unexpected error reading json file: %v", err) + b.Fatalf("unexpected error: %v", err) } - defer tf.Close() - ngxTpl, err := NewTemplate(tf.Name(), func() {}) + ngxTpl, err := NewTemplate("/etc/nginx/template/nginx.tmpl", fs) if err != nil { b.Errorf("invalid NGINX template: %v", err) } diff --git a/internal/ingress/status/status_test.go b/internal/ingress/status/status_test.go index 6ea536945..658ecdf73 100644 --- a/internal/ingress/status/status_test.go +++ b/internal/ingress/status/status_test.go @@ -214,12 +214,12 @@ func buildExtensionsIngresses() []extensions.Ingress { func buildIngressListener() []*extensions.Ingress { return []*extensions.Ingress{ - &extensions.Ingress{ + { ObjectMeta: metav1.ObjectMeta{ Name: "foo_ingress_non_01", Namespace: apiv1.NamespaceDefault, }}, - &extensions.Ingress{ + { ObjectMeta: metav1.ObjectMeta{ Name: "foo_ingress_1", Namespace: apiv1.NamespaceDefault, diff --git a/internal/ingress/store/backend_ssl.go b/internal/ingress/store/backend_ssl.go index a86e54930..8d674d3bd 100644 --- a/internal/ingress/store/backend_ssl.go +++ b/internal/ingress/store/backend_ssl.go @@ -18,7 +18,6 @@ package store import ( "fmt" - "io/ioutil" "strings" "github.com/golang/glog" @@ -27,6 +26,7 @@ import ( apiv1 "k8s.io/api/core/v1" extensions "k8s.io/api/extensions/v1beta1" + "k8s.io/ingress-nginx/internal/file" "k8s.io/ingress-nginx/internal/ingress" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" "k8s.io/ingress-nginx/internal/k8s" @@ -57,7 +57,7 @@ func (s k8sStore) syncSecret(key string) { s.sslStore.Update(key, cert) // this update must trigger an update // (like an update event from a change in Ingress) - //ic.syncQueue.Enqueue(&extensions.Ingress{}) + s.sendDummyEvent() return } @@ -65,7 +65,7 @@ func (s k8sStore) syncSecret(key string) { s.sslStore.Add(key, cert) // this update must trigger an update // (like an update event from a change in Ingress) - //ic.syncQueue.Enqueue(&extensions.Ingress{}) + s.sendDummyEvent() } // getPemCertificate receives a secret, and creates a ingress.SSLCert as return. @@ -94,7 +94,7 @@ func (s k8sStore) getPemCertificate(secretName string) (*ingress.SSLCert, error) // If 'ca.crt' is also present, it will allow this secret to be used in the // 'nginx.ingress.kubernetes.io/auth-tls-secret' annotation - sslCert, err = ssl.AddOrUpdateCertAndKey(nsSecName, cert, key, ca) + sslCert, err = ssl.AddOrUpdateCertAndKey(nsSecName, cert, key, ca, s.filesystem) if err != nil { return nil, fmt.Errorf("unexpected error creating pem file: %v", err) } @@ -104,7 +104,7 @@ func (s k8sStore) getPemCertificate(secretName string) (*ingress.SSLCert, error) glog.V(3).Infof("found 'ca.crt', secret %v can also be used for Certificate Authentication", secretName) } } else if ca != nil { - sslCert, err = ssl.AddCertAuth(nsSecName, ca) + sslCert, err = ssl.AddCertAuth(nsSecName, ca, s.filesystem) if err != nil { return nil, fmt.Errorf("unexpected error creating pem file: %v", err) @@ -137,14 +137,21 @@ func (s k8sStore) checkSSLChainIssues() { continue } - data, err := ssl.FullChainCert(secret.PemFileName) + data, err := ssl.FullChainCert(secret.PemFileName, s.filesystem) if err != nil { glog.Errorf("unexpected error generating SSL certificate with full intermediate chain CA certs: %v", err) continue } - fullChainPemFileName := fmt.Sprintf("%v/%v-%v-full-chain.pem", ingress.DefaultSSLDirectory, secret.Namespace, secret.Name) - err = ioutil.WriteFile(fullChainPemFileName, data, 0655) + fullChainPemFileName := fmt.Sprintf("%v/%v-%v-full-chain.pem", file.DefaultSSLDirectory, secret.Namespace, secret.Name) + + file, err := s.filesystem.Create(fullChainPemFileName) + if err != nil { + glog.Errorf("unexpected error creating SSL certificate file %v: %v", fullChainPemFileName, err) + continue + } + + _, err = file.Write(data) if err != nil { glog.Errorf("unexpected error creating SSL certificate: %v", err) continue @@ -164,7 +171,7 @@ func (s k8sStore) checkSSLChainIssues() { s.sslStore.Update(secretName, dst) // this update must trigger an update // (like an update event from a change in Ingress) - //ic.syncQueue.Enqueue(&extensions.Ingress{}) + s.sendDummyEvent() } } diff --git a/internal/ingress/store/backend_ssl_test.go b/internal/ingress/store/backend_ssl_test.go index 31304bd56..9d004d02c 100644 --- a/internal/ingress/store/backend_ssl_test.go +++ b/internal/ingress/store/backend_ssl_test.go @@ -18,16 +18,12 @@ package store import ( "encoding/base64" - "fmt" - "io/ioutil" apiv1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" testclient "k8s.io/client-go/kubernetes/fake" cache_client "k8s.io/client-go/tools/cache" "k8s.io/kubernetes/pkg/api" - - "k8s.io/ingress-nginx/internal/ingress" ) const ( @@ -115,14 +111,8 @@ func buildGenericControllerForBackendSSL() *NGINXController { return gc } */ -func buildCrtKeyAndCA() ([]byte, []byte, []byte, error) { - // prepare - td, err := ioutil.TempDir("", "ssl") - if err != nil { - return nil, nil, nil, fmt.Errorf("error occurs while creating temp directory: %v", err) - } - ingress.DefaultSSLDirectory = td +func buildCrtKeyAndCA() ([]byte, []byte, []byte, error) { dCrt, err := base64.StdEncoding.DecodeString(tlsCrt) if err != nil { return nil, nil, nil, err diff --git a/internal/ingress/store/store.go b/internal/ingress/store/store.go index 70faccd2f..2264a57db 100644 --- a/internal/ingress/store/store.go +++ b/internal/ingress/store/store.go @@ -25,6 +25,7 @@ import ( apiv1 "k8s.io/api/core/v1" extensions "k8s.io/api/extensions/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" @@ -35,6 +36,7 @@ import ( cache_client "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/record" + "k8s.io/ingress-nginx/internal/file" "k8s.io/ingress-nginx/internal/ingress" "k8s.io/ingress-nginx/internal/ingress/annotations" "k8s.io/ingress-nginx/internal/ingress/annotations/class" @@ -162,6 +164,10 @@ type k8sStore struct { sslStore *SSLCertTracker annotations annotations.Extractor + + filesystem file.Filesystem + + updateCh chan Event } // New creates a new object store to be used in the ingress controller @@ -169,6 +175,7 @@ func New(checkOCSP bool, namespace, configmap, tcp, udp string, resyncPeriod time.Duration, client clientset.Interface, + fs file.Filesystem, updateCh chan Event) Storer { store := &k8sStore{ @@ -176,6 +183,8 @@ func New(checkOCSP bool, cache: &Controller{}, listers: &Lister{}, sslStore: NewSSLCertTracker(), + filesystem: fs, + updateCh: updateCh, } eventBroadcaster := record.NewBroadcaster() @@ -188,7 +197,7 @@ func New(checkOCSP bool, }) // k8sStore fulfils resolver.Resolver interface - store.annotations = annotations.NewAnnotationExtractor(store) + store.annotations = annotations.NewAnnotationExtractor(store, fs) ingEventHandler := cache.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { @@ -494,9 +503,22 @@ func (s k8sStore) Run(stopCh chan struct{}) { } // start goroutine to check for missing local secrets - go wait.Until(s.checkMissingSecrets, 30*time.Second, stopCh) + go wait.Until(s.checkMissingSecrets, 10*time.Second, stopCh) if s.isOCSPCheckEnabled { go wait.Until(s.checkSSLChainIssues, 60*time.Second, stopCh) } } + +// sendDummyEvent sends a dummy event to trigger an update +func (s *k8sStore) sendDummyEvent() { + s.updateCh <- Event{ + Type: UpdateEvent, + Obj: &extensions.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dummy", + Namespace: "dummy", + }, + }, + } +} diff --git a/internal/ingress/store/store_test.go b/internal/ingress/store/store_test.go index d0d4c2c2d..bea582577 100644 --- a/internal/ingress/store/store_test.go +++ b/internal/ingress/store/store_test.go @@ -26,13 +26,12 @@ import ( apiv1 "k8s.io/api/core/v1" "k8s.io/api/extensions/v1beta1" extensions "k8s.io/api/extensions/v1beta1" - apierrors "k8s.io/apimachinery/pkg/api/errors" k8sErrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" + "k8s.io/ingress-nginx/internal/file" "k8s.io/ingress-nginx/test/e2e/framework" ) @@ -68,6 +67,7 @@ func TestStore(t *testing.T) { } }(updateCh) + fs := newFS(t) storer := New(true, ns.Name, fmt.Sprintf("%v/config", ns.Name), @@ -75,6 +75,7 @@ func TestStore(t *testing.T) { fmt.Sprintf("%v/udp", ns.Name), 10*time.Minute, clientSet, + fs, updateCh) storer.Run(stopCh) @@ -150,6 +151,7 @@ func TestStore(t *testing.T) { } }(updateCh) + fs := newFS(t) storer := New(true, ns.Name, fmt.Sprintf("%v/config", ns.Name), @@ -157,6 +159,7 @@ func TestStore(t *testing.T) { fmt.Sprintf("%v/udp", ns.Name), 10*time.Minute, clientSet, + fs, updateCh) storer.Run(stopCh) @@ -239,7 +242,7 @@ func TestStore(t *testing.T) { t.Errorf("unexpected error creating ingress: %v", err) } - waitForNoIngressInNamespace(clientSet, ni.Namespace, ni.Name) + framework.WaitForNoIngressInNamespace(clientSet, ni.Namespace, ni.Name) if atomic.LoadUint64(&add) != 1 { t.Errorf("expected 1 event of type Create but %v ocurred", add) @@ -252,8 +255,223 @@ func TestStore(t *testing.T) { } }) - // test add secret no referenced from ingress - // test add ingress with secret it doesn't exists + t.Run("should not receive events from new secret no referenced from ingress", func(t *testing.T) { + ns := createNamespace(clientSet, t) + defer deleteNamespace(ns, clientSet, t) + + stopCh := make(chan struct{}) + defer close(stopCh) + + updateCh := make(chan Event) + defer close(updateCh) + + var add uint64 + var upd uint64 + var del uint64 + + go func(ch chan Event) { + for { + e := <-ch + if e.Obj == nil { + continue + } + switch e.Type { + case CreateEvent: + atomic.AddUint64(&add, 1) + break + case UpdateEvent: + atomic.AddUint64(&upd, 1) + break + case DeleteEvent: + atomic.AddUint64(&del, 1) + break + } + } + }(updateCh) + + fs := newFS(t) + storer := New(true, + ns.Name, + fmt.Sprintf("%v/config", ns.Name), + fmt.Sprintf("%v/tcp", ns.Name), + fmt.Sprintf("%v/udp", ns.Name), + 10*time.Minute, + clientSet, + fs, + updateCh) + + storer.Run(stopCh) + + secretName := "no-referenced" + _, _, _, err = framework.CreateIngressTLSSecret(clientSet, []string{"foo"}, secretName, ns.Name) + if err != nil { + t.Errorf("unexpected error creating secret: %v", err) + } + + time.Sleep(1 * time.Second) + + if atomic.LoadUint64(&add) != 0 { + t.Errorf("expected 0 events of type Create but %v ocurred", add) + } + if atomic.LoadUint64(&upd) != 0 { + t.Errorf("expected 0 events of type Update but %v ocurred", upd) + } + if atomic.LoadUint64(&del) != 0 { + t.Errorf("expected 0 events of type Delete but %v ocurred", del) + } + + err = clientSet.CoreV1().Secrets(ns.Name).Delete(secretName, &metav1.DeleteOptions{}) + if err != nil { + t.Errorf("unexpected error deleting secret: %v", err) + } + + if atomic.LoadUint64(&add) != 0 { + t.Errorf("expected 0 events of type Create but %v ocurred", add) + } + if atomic.LoadUint64(&upd) != 0 { + t.Errorf("expected 0 events of type Update but %v ocurred", upd) + } + if atomic.LoadUint64(&del) != 0 { + t.Errorf("expected 0 events of type Delete but %v ocurred", del) + } + }) + + t.Run("should create an ingress with a secret it doesn't exists", func(t *testing.T) { + ns := createNamespace(clientSet, t) + defer deleteNamespace(ns, clientSet, t) + + stopCh := make(chan struct{}) + defer close(stopCh) + + updateCh := make(chan Event) + defer close(updateCh) + + var add uint64 + var upd uint64 + var del uint64 + + go func(ch chan Event) { + for { + e := <-ch + if e.Obj == nil { + continue + } + switch e.Type { + case CreateEvent: + atomic.AddUint64(&add, 1) + break + case UpdateEvent: + atomic.AddUint64(&upd, 1) + break + case DeleteEvent: + atomic.AddUint64(&del, 1) + break + } + } + }(updateCh) + + fs := newFS(t) + storer := New(true, + ns.Name, + fmt.Sprintf("%v/config", ns.Name), + fmt.Sprintf("%v/tcp", ns.Name), + fmt.Sprintf("%v/udp", ns.Name), + 10*time.Minute, + clientSet, + fs, + updateCh) + + storer.Run(stopCh) + + name := "ingress-with-secret" + secretHosts := []string{name} + + // err:= createIngress(client, name, ns.Name) + _, err := ensureIngress(&v1beta1.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns.Name, + }, + Spec: v1beta1.IngressSpec{ + TLS: []v1beta1.IngressTLS{ + { + Hosts: secretHosts, + SecretName: name, + }, + }, + Rules: []v1beta1.IngressRule{ + { + Host: name, + IngressRuleValue: v1beta1.IngressRuleValue{ + HTTP: &v1beta1.HTTPIngressRuleValue{ + Paths: []v1beta1.HTTPIngressPath{ + { + Path: "/", + Backend: v1beta1.IngressBackend{ + ServiceName: "http-svc", + ServicePort: intstr.FromInt(80), + }, + }, + }, + }, + }, + }, + }, + }, + }, clientSet) + if err != nil { + t.Errorf("unexpected error creating ingress: %v", err) + } + + err = framework.WaitForIngressInNamespace(clientSet, ns.Name, name) + if err != nil { + t.Errorf("unexpected error waiting for secret: %v", err) + } + + if atomic.LoadUint64(&add) != 1 { + t.Errorf("expected 1 events of type Create but %v ocurred", add) + } + if atomic.LoadUint64(&upd) != 0 { + t.Errorf("expected 0 events of type Update but %v ocurred", upd) + } + if atomic.LoadUint64(&del) != 0 { + t.Errorf("expected 0 events of type Delete but %v ocurred", del) + } + + _, _, _, err = framework.CreateIngressTLSSecret(clientSet, secretHosts, name, ns.Name) + if err != nil { + t.Errorf("unexpected error creating secret: %v", err) + } + + t.Run("should exists a secret in the local store and filesystem", func(t *testing.T) { + err := framework.WaitForSecretInNamespace(clientSet, ns.Name, name) + if err != nil { + t.Errorf("unexpected error waiting for secret: %v", err) + } + + pemFile := fmt.Sprintf("%v/%v-%v.pem", file.DefaultSSLDirectory, ns.Name, name) + stat, err := fs.Stat(pemFile) + if err != nil { + t.Errorf("unexpected error reading secret pem file: %v", err) + } + + if stat.Size() < 1 { + t.Errorf("unexpected size of pem file (%v)", stat.Size()) + } + + secretName := fmt.Sprintf("%v/%v", ns.Name, name) + sslCert, err := storer.GetLocalSecret(secretName) + if err != nil { + t.Errorf("unexpected error reading local secret %v: %v", secretName, err) + } + + pemSHA := file.SHA1(pemFile) + if sslCert.PemSHA != pemSHA { + t.Errorf("SHA of secret on disk differs from local secret store (%v != %v)", pemSHA, sslCert.PemSHA) + } + }) + }) + // test add ingress with secret it doesn't exists and then add secret // check secret is generated on fs // check ocsp @@ -293,23 +511,10 @@ func ensureIngress(ingress *extensions.Ingress, clientSet *kubernetes.Clientset) return s, nil } -func waitForNoIngressInNamespace(c kubernetes.Interface, namespace, name string) error { - return wait.PollImmediate(1*time.Second, time.Minute*2, noIngressInNamespace(c, namespace, name)) -} - -func noIngressInNamespace(c kubernetes.Interface, namespace, name string) wait.ConditionFunc { - return func() (bool, error) { - ing, err := c.ExtensionsV1beta1().Ingresses(namespace).Get(name, metav1.GetOptions{}) - if apierrors.IsNotFound(err) { - return true, nil - } - if err != nil { - return false, err - } - - if ing == nil { - return true, nil - } - return false, nil +func newFS(t *testing.T) file.Filesystem { + fs, err := file.NewFakeFS() + if err != nil { + t.Fatalf("unexpected error creating filesystem: %v", err) } + return fs } diff --git a/internal/ingress/types.go b/internal/ingress/types.go index 6798a49d0..ccd7e291d 100644 --- a/internal/ingress/types.go +++ b/internal/ingress/types.go @@ -35,14 +35,6 @@ import ( "k8s.io/ingress-nginx/internal/ingress/resolver" ) -var ( - // DefaultSSLDirectory defines the location where the SSL certificates will be generated - // This directory contains all the SSL certificates that are specified in Ingress rules. - // The name of each file is -.pem. The content is the concatenated - // certificate and key. - DefaultSSLDirectory = "/ingress-controller/ssl" -) - // Configuration holds the definition of all the parts required to describe all // ingresses reachable by the ingress controller (using a filter by namespace) type Configuration struct { diff --git a/internal/net/ssl/ssl.go b/internal/net/ssl/ssl.go index 5efc07503..ca684b74f 100644 --- a/internal/net/ssl/ssl.go +++ b/internal/net/ssl/ssl.go @@ -26,10 +26,8 @@ import ( "encoding/pem" "errors" "fmt" - "io/ioutil" "math/big" "net" - "os" "strconv" "time" @@ -47,10 +45,12 @@ var ( ) // AddOrUpdateCertAndKey creates a .pem file wth the cert and the key with the specified name -func AddOrUpdateCertAndKey(name string, cert, key, ca []byte) (*ingress.SSLCert, error) { +func AddOrUpdateCertAndKey(name string, cert, key, ca []byte, + fs file.Filesystem) (*ingress.SSLCert, error) { + pemName := fmt.Sprintf("%v.pem", name) - pemFileName := fmt.Sprintf("%v/%v", ingress.DefaultSSLDirectory, pemName) - tempPemFile, err := ioutil.TempFile(ingress.DefaultSSLDirectory, pemName) + pemFileName := fmt.Sprintf("%v/%v", file.DefaultSSLDirectory, pemName) + tempPemFile, err := fs.TempFile(file.DefaultSSLDirectory, pemName) if err != nil { return nil, fmt.Errorf("could not create temp pem file %v: %v", pemFileName, err) @@ -74,34 +74,30 @@ func AddOrUpdateCertAndKey(name string, cert, key, ca []byte) (*ingress.SSLCert, if err != nil { return nil, fmt.Errorf("could not close temp pem file %v: %v", tempPemFile.Name(), err) } + defer fs.RemoveAll(tempPemFile.Name()) - pemCerts, err := ioutil.ReadFile(tempPemFile.Name()) + pemCerts, err := fs.ReadFile(tempPemFile.Name()) if err != nil { - _ = os.Remove(tempPemFile.Name()) return nil, err } pemBlock, _ := pem.Decode(pemCerts) if pemBlock == nil { - _ = os.Remove(tempPemFile.Name()) return nil, fmt.Errorf("no valid PEM formatted block found") } // If the file does not start with 'BEGIN CERTIFICATE' it's invalid and must not be used. if pemBlock.Type != "CERTIFICATE" { - _ = os.Remove(tempPemFile.Name()) return nil, fmt.Errorf("certificate %v contains invalid data, and must be created with 'kubectl create secret tls'", name) } pemCert, err := x509.ParseCertificate(pemBlock.Bytes) if err != nil { - _ = os.Remove(tempPemFile.Name()) return nil, err } //Ensure that certificate and private key have a matching public key if _, err := tls.X509KeyPair(cert, key); err != nil { - _ = os.Remove(tempPemFile.Name()) return nil, err } @@ -129,7 +125,7 @@ func AddOrUpdateCertAndKey(name string, cert, key, ca []byte) (*ingress.SSLCert, } } - err = os.Rename(tempPemFile.Name(), pemFileName) + err = fs.Rename(tempPemFile.Name(), pemFileName) if err != nil { return nil, fmt.Errorf("could not move temp pem file %v to destination %v: %v", tempPemFile.Name(), pemFileName, err) } @@ -147,18 +143,24 @@ func AddOrUpdateCertAndKey(name string, cert, key, ca []byte) (*ingress.SSLCert, return nil, errors.New(oe) } - caFile, err := os.OpenFile(pemFileName, os.O_RDWR|os.O_APPEND, 0600) + caData, err := fs.ReadFile(pemFileName) if err != nil { return nil, fmt.Errorf("could not open file %v for writing additional CA chains: %v", pemFileName, err) } - defer caFile.Close() + caFile, err := fs.Create(pemFileName) + _, err = caFile.Write(caData) + if err != nil { + return nil, fmt.Errorf("could not append CA to cert file %v: %v", pemFileName, err) + } + _, err = caFile.Write([]byte("\n")) if err != nil { return nil, fmt.Errorf("could not append CA to cert file %v: %v", pemFileName, err) } caFile.Write(ca) caFile.Write([]byte("\n")) + defer caFile.Close() return &ingress.SSLCert{ Certificate: pemCert, @@ -249,10 +251,10 @@ func parseSANExtension(value []byte) (dnsNames, emailAddresses []string, ipAddre // AddCertAuth creates a .pem file with the specified CAs to be used in Cert Authentication // If it's already exists, it's clobbered. -func AddCertAuth(name string, ca []byte) (*ingress.SSLCert, error) { +func AddCertAuth(name string, ca []byte, fs file.Filesystem) (*ingress.SSLCert, error) { caName := fmt.Sprintf("ca-%v.pem", name) - caFileName := fmt.Sprintf("%v/%v", ingress.DefaultSSLDirectory, caName) + caFileName := fmt.Sprintf("%v/%v", file.DefaultSSLDirectory, caName) pemCABlock, _ := pem.Decode(ca) if pemCABlock == nil { @@ -268,7 +270,13 @@ func AddCertAuth(name string, ca []byte) (*ingress.SSLCert, error) { return nil, err } - err = ioutil.WriteFile(caFileName, ca, 0644) + caFile, err := fs.Create(caFileName) + if err != nil { + return nil, fmt.Errorf("could not write CA file %v: %v", caFileName, err) + } + defer caFile.Close() + + _, err = caFile.Write(ca) if err != nil { return nil, fmt.Errorf("could not write CA file %v: %v", caFileName, err) } @@ -282,11 +290,11 @@ func AddCertAuth(name string, ca []byte) (*ingress.SSLCert, error) { } // AddOrUpdateDHParam creates a dh parameters file with the specified name -func AddOrUpdateDHParam(name string, dh []byte) (string, error) { +func AddOrUpdateDHParam(name string, dh []byte, fs file.Filesystem) (string, error) { pemName := fmt.Sprintf("%v.pem", name) - pemFileName := fmt.Sprintf("%v/%v", ingress.DefaultSSLDirectory, pemName) + pemFileName := fmt.Sprintf("%v/%v", file.DefaultSSLDirectory, pemName) - tempPemFile, err := ioutil.TempFile(ingress.DefaultSSLDirectory, pemName) + tempPemFile, err := fs.TempFile(file.DefaultSSLDirectory, pemName) glog.V(3).Infof("Creating temp file %v for DH param: %v", tempPemFile.Name(), pemName) if err != nil { @@ -303,25 +311,24 @@ func AddOrUpdateDHParam(name string, dh []byte) (string, error) { return "", fmt.Errorf("could not close temp pem file %v: %v", tempPemFile.Name(), err) } - pemCerts, err := ioutil.ReadFile(tempPemFile.Name()) + defer fs.RemoveAll(tempPemFile.Name()) + + pemCerts, err := fs.ReadFile(tempPemFile.Name()) if err != nil { - _ = os.Remove(tempPemFile.Name()) return "", err } pemBlock, _ := pem.Decode(pemCerts) if pemBlock == nil { - _ = os.Remove(tempPemFile.Name()) return "", fmt.Errorf("no valid PEM formatted block found") } // If the file does not start with 'BEGIN DH PARAMETERS' it's invalid and must not be used. if pemBlock.Type != "DH PARAMETERS" { - _ = os.Remove(tempPemFile.Name()) return "", fmt.Errorf("certificate %v contains invalid data", name) } - err = os.Rename(tempPemFile.Name(), pemFileName) + err = fs.Rename(tempPemFile.Name(), pemFileName) if err != nil { return "", fmt.Errorf("could not move temp pem file %v to destination %v: %v", tempPemFile.Name(), pemFileName, err) } @@ -382,13 +389,8 @@ func GetFakeSSLCert() ([]byte, []byte) { // FullChainCert checks if a certificate file contains issues in the intermediate CA chain // Returns a new certificate with the intermediate certificates. // If the certificate does not contains issues with the chain it return an empty byte array -func FullChainCert(in string) ([]byte, error) { - inputFile, err := os.Open(in) - if err != nil { - return nil, err - } - - data, err := ioutil.ReadAll(inputFile) +func FullChainCert(in string, fs file.Filesystem) ([]byte, error) { + data, err := fs.ReadFile(in) if err != nil { return nil, err } diff --git a/internal/net/ssl/ssl_test.go b/internal/net/ssl/ssl_test.go index 95767eeca..d6456050b 100644 --- a/internal/net/ssl/ssl_test.go +++ b/internal/net/ssl/ssl_test.go @@ -19,14 +19,13 @@ package ssl import ( "crypto/x509" "fmt" - "io/ioutil" "testing" "time" certutil "k8s.io/client-go/util/cert" "k8s.io/client-go/util/cert/triple" - "k8s.io/ingress-nginx/internal/ingress" + "k8s.io/ingress-nginx/internal/file" ) // generateRSACerts generates a self signed certificate using a self generated ca @@ -57,11 +56,7 @@ func generateRSACerts(host string) (*triple.KeyPair, *triple.KeyPair, error) { } func TestAddOrUpdateCertAndKey(t *testing.T) { - td, err := ioutil.TempDir("", "ssl") - if err != nil { - t.Fatalf("Unexpected error creating temporal directory: %v", err) - } - ingress.DefaultSSLDirectory = td + fs := newFS(t) cert, _, err := generateRSACerts("echoheaders") if err != nil { @@ -73,7 +68,7 @@ func TestAddOrUpdateCertAndKey(t *testing.T) { c := certutil.EncodeCertPEM(cert.Cert) k := certutil.EncodePrivateKeyPEM(cert.Key) - ngxCert, err := AddOrUpdateCertAndKey(name, c, k, []byte{}) + ngxCert, err := AddOrUpdateCertAndKey(name, c, k, []byte{}, fs) if err != nil { t.Fatalf("unexpected error checking SSL certificate: %v", err) } @@ -92,11 +87,7 @@ func TestAddOrUpdateCertAndKey(t *testing.T) { } func TestCACert(t *testing.T) { - td, err := ioutil.TempDir("", "ssl") - if err != nil { - t.Fatalf("Unexpected error creating temporal directory: %v", err) - } - ingress.DefaultSSLDirectory = td + fs := newFS(t) cert, CA, err := generateRSACerts("echoheaders") if err != nil { @@ -109,7 +100,7 @@ func TestCACert(t *testing.T) { k := certutil.EncodePrivateKeyPEM(cert.Key) ca := certutil.EncodeCertPEM(CA.Cert) - ngxCert, err := AddOrUpdateCertAndKey(name, c, k, ca) + ngxCert, err := AddOrUpdateCertAndKey(name, c, k, ca, fs) if err != nil { t.Fatalf("unexpected error checking SSL certificate: %v", err) } @@ -129,11 +120,10 @@ func TestGetFakeSSLCert(t *testing.T) { } func TestAddCertAuth(t *testing.T) { - td, err := ioutil.TempDir("", "ssl") + fs, err := file.NewFakeFS() if err != nil { - t.Fatalf("Unexpected error creating temporal directory: %v", err) + t.Fatalf("unexpected error creating filesystem: %v", err) } - ingress.DefaultSSLDirectory = td cn := "demo-ca" _, ca, err := generateRSACerts(cn) @@ -141,7 +131,7 @@ func TestAddCertAuth(t *testing.T) { t.Fatalf("unexpected error creating SSL certificate: %v", err) } c := certutil.EncodeCertPEM(ca.Cert) - ic, err := AddCertAuth(cn, c) + ic, err := AddCertAuth(cn, c, fs) if err != nil { t.Fatalf("unexpected error creating SSL certificate: %v", err) } @@ -149,3 +139,11 @@ func TestAddCertAuth(t *testing.T) { t.Fatalf("expected a valid CA file name") } } + +func newFS(t *testing.T) file.Filesystem { + fs, err := file.NewFakeFS() + if err != nil { + t.Fatalf("unexpected error creating filesystem: %v", err) + } + return fs +} diff --git a/internal/watch/dummy.go b/internal/watch/dummy.go new file mode 100644 index 000000000..16a607fc2 --- /dev/null +++ b/internal/watch/dummy.go @@ -0,0 +1,29 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package watch + +// DummyFileWatcher noop implementation of a file watcher +type DummyFileWatcher struct{} + +func NewDummyFileWatcher(file string, onEvent func()) FileWatcher { + return DummyFileWatcher{} +} + +// Close ends the watch +func (f DummyFileWatcher) Close() error { + return nil +} diff --git a/internal/watch/file_watcher.go b/internal/watch/file_watcher.go index 0fea1a143..91daf0620 100644 --- a/internal/watch/file_watcher.go +++ b/internal/watch/file_watcher.go @@ -24,8 +24,12 @@ import ( "gopkg.in/fsnotify.v1" ) -// FileWatcher defines a watch over a file -type FileWatcher struct { +type FileWatcher interface { + Close() error +} + +// OSFileWatcher defines a watch over a file +type OSFileWatcher struct { file string watcher *fsnotify.Watcher // onEvent callback to be invoked after the file being watched changes @@ -34,7 +38,7 @@ type FileWatcher struct { // NewFileWatcher creates a new FileWatcher func NewFileWatcher(file string, onEvent func()) (FileWatcher, error) { - fw := FileWatcher{ + fw := OSFileWatcher{ file: file, onEvent: onEvent, } @@ -43,12 +47,12 @@ func NewFileWatcher(file string, onEvent func()) (FileWatcher, error) { } // Close ends the watch -func (f *FileWatcher) Close() error { +func (f OSFileWatcher) Close() error { return f.watcher.Close() } // watch creates a fsnotify watcher for a file and create of write events -func (f *FileWatcher) watch() error { +func (f *OSFileWatcher) watch() error { watcher, err := fsnotify.NewWatcher() if err != nil { return err diff --git a/test/e2e/framework/ssl.go b/test/e2e/framework/ssl.go new file mode 100644 index 000000000..41f0dd0eb --- /dev/null +++ b/test/e2e/framework/ssl.go @@ -0,0 +1,115 @@ +package framework + +import ( + "bytes" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "io" + "math/big" + "net" + "strings" + "time" + + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +const ( + rsaBits = 2048 + validFor = 365 * 24 * time.Hour +) + +// CreateIngressTLSSecret creates a secret containing TLS certificates for the given Ingress. +// If a secret with the same name already pathExists in the namespace of the +// Ingress, it's updated. +func CreateIngressTLSSecret(client kubernetes.Interface, hosts []string, secreName, namespace string) (host string, rootCA, privKey []byte, err error) { + var k, c bytes.Buffer + host = strings.Join(hosts, ",") + if err = generateRSACerts(host, true, &k, &c); err != nil { + return + } + cert := c.Bytes() + key := k.Bytes() + secret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secreName, + }, + Data: map[string][]byte{ + v1.TLSCertKey: cert, + v1.TLSPrivateKeyKey: key, + }, + } + var s *v1.Secret + if s, err = client.CoreV1().Secrets(namespace).Get(secreName, metav1.GetOptions{}); err == nil { + s.Data = secret.Data + _, err = client.CoreV1().Secrets(namespace).Update(s) + } else { + _, err = client.CoreV1().Secrets(namespace).Create(secret) + } + return host, cert, key, err +} + +// generateRSACerts generates a basic self signed certificate using a key length +// of rsaBits, valid for validFor time. +func generateRSACerts(host string, isCA bool, keyOut, certOut io.Writer) error { + if len(host) == 0 { + return fmt.Errorf("Require a non-empty host for client hello") + } + priv, err := rsa.GenerateKey(rand.Reader, rsaBits) + if err != nil { + return fmt.Errorf("Failed to generate key: %v", err) + } + notBefore := time.Now() + notAfter := notBefore.Add(validFor) + + serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) + serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) + + if err != nil { + return fmt.Errorf("failed to generate serial number: %s", err) + } + template := x509.Certificate{ + SerialNumber: serialNumber, + Subject: pkix.Name{ + CommonName: "default", + Organization: []string{"Acme Co"}, + }, + NotBefore: notBefore, + NotAfter: notAfter, + + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + } + + hosts := strings.Split(host, ",") + for _, h := range hosts { + if ip := net.ParseIP(h); ip != nil { + template.IPAddresses = append(template.IPAddresses, ip) + } else { + template.DNSNames = append(template.DNSNames, h) + } + } + + if isCA { + template.IsCA = true + template.KeyUsage |= x509.KeyUsageCertSign + } + + derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv) + if err != nil { + return fmt.Errorf("Failed to create certificate: %s", err) + } + if err := pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}); err != nil { + return fmt.Errorf("Failed creating cert: %v", err) + } + if err := pem.Encode(keyOut, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)}); err != nil { + return fmt.Errorf("Failed creating keay: %v", err) + } + return nil +} diff --git a/test/e2e/framework/util.go b/test/e2e/framework/util.go index f661e37c9..3427321f2 100644 --- a/test/e2e/framework/util.go +++ b/test/e2e/framework/util.go @@ -199,6 +199,69 @@ func podRunning(c kubernetes.Interface, podName, namespace string) wait.Conditio } } +func WaitForSecretInNamespace(c kubernetes.Interface, namespace, name string) error { + return wait.PollImmediate(1*time.Second, time.Minute*2, secretInNamespace(c, namespace, name)) +} + +func secretInNamespace(c kubernetes.Interface, namespace, name string) wait.ConditionFunc { + return func() (bool, error) { + s, err := c.CoreV1().Secrets(namespace).Get(name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return false, err + } + if err != nil { + return false, err + } + + if s != nil { + return true, nil + } + return false, nil + } +} + +func WaitForNoIngressInNamespace(c kubernetes.Interface, namespace, name string) error { + return wait.PollImmediate(1*time.Second, time.Minute*2, noIngressInNamespace(c, namespace, name)) +} + +func noIngressInNamespace(c kubernetes.Interface, namespace, name string) wait.ConditionFunc { + return func() (bool, error) { + ing, err := c.ExtensionsV1beta1().Ingresses(namespace).Get(name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return true, nil + } + if err != nil { + return false, err + } + + if ing == nil { + return true, nil + } + return false, nil + } +} + +func WaitForIngressInNamespace(c kubernetes.Interface, namespace, name string) error { + return wait.PollImmediate(1*time.Second, time.Minute*2, ingressInNamespace(c, namespace, name)) +} + +func ingressInNamespace(c kubernetes.Interface, namespace, name string) wait.ConditionFunc { + return func() (bool, error) { + ing, err := c.ExtensionsV1beta1().Ingresses(namespace).Get(name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return false, err + } + if err != nil { + return false, err + } + + if ing != nil { + return true, nil + } + return false, nil + } +} + func NewInt32(val int32) *int32 { p := new(int32) *p = val diff --git a/test/e2e/ssl/secret_update.go b/test/e2e/ssl/secret_update.go index ab540157d..1765407f3 100644 --- a/test/e2e/ssl/secret_update.go +++ b/test/e2e/ssl/secret_update.go @@ -17,16 +17,7 @@ limitations under the License. package ssl import ( - "bytes" - "crypto/rand" - "crypto/rsa" - "crypto/x509" - "crypto/x509/pkix" - "encoding/pem" "fmt" - "io" - "math/big" - "net" "strings" "time" @@ -37,15 +28,9 @@ import ( "k8s.io/api/extensions/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/client-go/kubernetes" "k8s.io/ingress-nginx/test/e2e/framework" ) -const ( - rsaBits = 2048 - validFor = 365 * 24 * time.Hour -) - var _ = framework.IngressNginxDescribe("SSL", func() { f := framework.NewDefaultFramework("ssl") @@ -107,7 +92,8 @@ var _ = framework.IngressNginxDescribe("SSL", func() { Expect(err).ToNot(HaveOccurred()) Expect(ing).ToNot(BeNil()) - _, _, _, err = createIngressTLSSecret(f.KubeClientSet, ing) + tls := ing.Spec.TLS[0] + _, _, _, err = framework.CreateIngressTLSSecret(f.KubeClientSet, tls.Hosts, tls.SecretName, ing.Namespace) Expect(err).ToNot(HaveOccurred()) err = f.WaitForNginxServer(host, @@ -130,94 +116,3 @@ var _ = framework.IngressNginxDescribe("SSL", func() { Expect(log).ToNot(ContainSubstring(fmt.Sprintf("error obtaining PEM from secret %v/dummy", f.Namespace.Name))) }) }) - -// createIngressTLSSecret creates a secret containing TLS certificates for the given Ingress. -// If a secret with the same name already pathExists in the namespace of the -// Ingress, it's updated. -func createIngressTLSSecret(kubeClient kubernetes.Interface, ing *v1beta1.Ingress) (host string, rootCA, privKey []byte, err error) { - var k, c bytes.Buffer - tls := ing.Spec.TLS[0] - host = strings.Join(tls.Hosts, ",") - if err = generateRSACerts(host, true, &k, &c); err != nil { - return - } - cert := c.Bytes() - key := k.Bytes() - secret := &v1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: tls.SecretName, - }, - Data: map[string][]byte{ - v1.TLSCertKey: cert, - v1.TLSPrivateKeyKey: key, - }, - } - var s *v1.Secret - if s, err = kubeClient.CoreV1().Secrets(ing.Namespace).Get(tls.SecretName, metav1.GetOptions{}); err == nil { - s.Data = secret.Data - _, err = kubeClient.CoreV1().Secrets(ing.Namespace).Update(s) - } else { - _, err = kubeClient.CoreV1().Secrets(ing.Namespace).Create(secret) - } - return host, cert, key, err -} - -// generateRSACerts generates a basic self signed certificate using a key length -// of rsaBits, valid for validFor time. -func generateRSACerts(host string, isCA bool, keyOut, certOut io.Writer) error { - if len(host) == 0 { - return fmt.Errorf("Require a non-empty host for client hello") - } - priv, err := rsa.GenerateKey(rand.Reader, rsaBits) - if err != nil { - return fmt.Errorf("Failed to generate key: %v", err) - } - notBefore := time.Now() - notAfter := notBefore.Add(validFor) - - serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) - serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) - - if err != nil { - return fmt.Errorf("failed to generate serial number: %s", err) - } - template := x509.Certificate{ - SerialNumber: serialNumber, - Subject: pkix.Name{ - CommonName: "default", - Organization: []string{"Acme Co"}, - }, - NotBefore: notBefore, - NotAfter: notAfter, - - KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, - BasicConstraintsValid: true, - } - - hosts := strings.Split(host, ",") - for _, h := range hosts { - if ip := net.ParseIP(h); ip != nil { - template.IPAddresses = append(template.IPAddresses, ip) - } else { - template.DNSNames = append(template.DNSNames, h) - } - } - - if isCA { - template.IsCA = true - template.KeyUsage |= x509.KeyUsageCertSign - } - - derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv) - if err != nil { - return fmt.Errorf("Failed to create certificate: %s", err) - } - if err := pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}); err != nil { - return fmt.Errorf("Failed creating cert: %v", err) - } - if err := pem.Encode(keyOut, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)}); err != nil { - return fmt.Errorf("Failed creating keay: %v", err) - } - return nil -}