From bc464c7cb899325783e8a0fccb8ae84080aa2654 Mon Sep 17 00:00:00 2001 From: Manuel de Brito Fontes Date: Wed, 13 Sep 2017 20:04:20 -0300 Subject: [PATCH 01/15] Add example for nginx custom error pages --- images/custom-error-pages/Makefile | 110 ++++++++++++++++++ images/custom-error-pages/README.md | 2 + images/custom-error-pages/main.go | 94 +++++++++++++++ images/custom-error-pages/rootfs/Dockerfile | 21 ++++ images/custom-error-pages/rootfs/www/404.html | 1 + images/custom-error-pages/rootfs/www/404.json | 1 + images/custom-error-pages/rootfs/www/4xx.html | 1 + images/custom-error-pages/rootfs/www/4xx.json | 1 + images/custom-error-pages/rootfs/www/500.html | 1 + images/custom-error-pages/rootfs/www/500.json | 1 + images/custom-error-pages/rootfs/www/5xx.html | 1 + images/custom-error-pages/rootfs/www/5xx.json | 1 + 12 files changed, 235 insertions(+) create mode 100644 images/custom-error-pages/Makefile create mode 100644 images/custom-error-pages/README.md create mode 100644 images/custom-error-pages/main.go create mode 100755 images/custom-error-pages/rootfs/Dockerfile create mode 100644 images/custom-error-pages/rootfs/www/404.html create mode 100644 images/custom-error-pages/rootfs/www/404.json create mode 100644 images/custom-error-pages/rootfs/www/4xx.html create mode 100644 images/custom-error-pages/rootfs/www/4xx.json create mode 100644 images/custom-error-pages/rootfs/www/500.html create mode 100644 images/custom-error-pages/rootfs/www/500.json create mode 100644 images/custom-error-pages/rootfs/www/5xx.html create mode 100644 images/custom-error-pages/rootfs/www/5xx.json diff --git a/images/custom-error-pages/Makefile b/images/custom-error-pages/Makefile new file mode 100644 index 000000000..96b3d3479 --- /dev/null +++ b/images/custom-error-pages/Makefile @@ -0,0 +1,110 @@ +all: push + +BUILDTAGS= + +# Use the 0.0 tag for testing, it shouldn't clobber any release builds +TAG?=0.1 +REGISTRY?=aledbf +GOOS?=linux +DOCKER?=gcloud docker -- +SED_I?=sed -i +GOHOSTOS ?= $(shell go env GOHOSTOS) + +PKG=k8s.io/ingress/images/custom-error-pages + +ifeq ($(GOHOSTOS),darwin) + SED_I=sed -i '' +endif + +REPO_INFO=$(shell git config --get remote.origin.url) + +ifndef COMMIT + COMMIT := git-$(shell git rev-parse --short HEAD) +endif + +ARCH ?= $(shell go env GOARCH) +GOARCH = ${ARCH} +DUMB_ARCH = ${ARCH} + +BASEIMAGE?=alpine:3.6 + +ALL_ARCH = amd64 arm arm64 ppc64le + +QEMUVERSION=v2.9.1 + +IMGNAME = custom-error-pages +IMAGE = $(REGISTRY)/$(IMGNAME) +MULTI_ARCH_IMG = $(IMAGE)-$(ARCH) + +ifeq ($(ARCH),arm) + QEMUARCH=arm + GOARCH=arm +endif +ifeq ($(ARCH),arm64) + QEMUARCH=aarch64 +endif +ifeq ($(ARCH),ppc64le) + QEMUARCH=ppc64le + GOARCH=ppc64le +endif +#ifeq ($(ARCH),s390x) +# QEMUARCH=s390x +#endif + +TEMP_DIR := $(shell mktemp -d) + +DOCKERFILE := $(TEMP_DIR)/rootfs/Dockerfile + +all: all-container + +sub-container-%: + $(MAKE) ARCH=$* build container + +sub-push-%: + $(MAKE) ARCH=$* push + +all-container: $(addprefix sub-container-,$(ALL_ARCH)) + +all-push: $(addprefix sub-push-,$(ALL_ARCH)) + +container: .container-$(ARCH) +.container-$(ARCH): + cp -r ./* $(TEMP_DIR) + $(SED_I) 's|BASEIMAGE|$(BASEIMAGE)|g' $(DOCKERFILE) + $(SED_I) "s|QEMUARCH|$(QEMUARCH)|g" $(DOCKERFILE) + +ifeq ($(ARCH),amd64) + # When building "normally" for amd64, remove the whole line, it has no part in the amd64 image + $(SED_I) "/CROSS_BUILD_/d" $(DOCKERFILE) +else + # When cross-building, only the placeholder "CROSS_BUILD_" should be removed + # Register /usr/bin/qemu-ARCH-static as the handler for ARM binaries in the kernel + $(DOCKER) run --rm --privileged multiarch/qemu-user-static:register --reset + curl -sSL https://github.com/multiarch/qemu-user-static/releases/download/$(QEMUVERSION)/x86_64_qemu-$(QEMUARCH)-static.tar.gz | tar -xz -C $(TEMP_DIR)/rootfs + $(SED_I) "s/CROSS_BUILD_//g" $(DOCKERFILE) +endif + + $(DOCKER) build -t $(MULTI_ARCH_IMG):$(TAG) $(TEMP_DIR)/rootfs + +ifeq ($(ARCH), amd64) + # This is for to maintain the backward compatibility + $(DOCKER) tag $(MULTI_ARCH_IMG):$(TAG) $(IMAGE):$(TAG) +endif + +push: .push-$(ARCH) +.push-$(ARCH): + $(DOCKER) push $(MULTI_ARCH_IMG):$(TAG) +ifeq ($(ARCH), amd64) + $(DOCKER) push $(IMAGE):$(TAG) +endif + +clean: + $(DOCKER) rmi -f $(MULTI_ARCH_IMG):$(TAG) || true + +build: clean + CGO_ENABLED=0 GOOS=${GOOS} GOARCH=${GOARCH} go build -a -installsuffix cgo \ + -ldflags "-s -w" \ + -o ${TEMP_DIR}/rootfs/custom-error-pages ${PKG}/... + +release: all-container all-push + echo "done" diff --git a/images/custom-error-pages/README.md b/images/custom-error-pages/README.md new file mode 100644 index 000000000..3ee67e5d9 --- /dev/null +++ b/images/custom-error-pages/README.md @@ -0,0 +1,2 @@ + +Example of Custom error pages for the NGINX Ingress controller diff --git a/images/custom-error-pages/main.go b/images/custom-error-pages/main.go new file mode 100644 index 000000000..88b709b6c --- /dev/null +++ b/images/custom-error-pages/main.go @@ -0,0 +1,94 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "io" + "log" + "mime" + "net/http" + "os" + "strconv" +) + +const ( + FormatHeader = "X-Format" + + CodeHeader = "X-Code" + + ContentType = "Content-Type" +) + +func main() { + path := "/www" + if os.Getenv("PATH") != "" { + path = os.Getenv("PATH") + } + http.HandleFunc("/", errorHandler(path)) + http.ListenAndServe(fmt.Sprintf(":8080"), nil) +} + +func errorHandler(path string) func(http.ResponseWriter, *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { + ext := "html" + + format := r.Header.Get(FormatHeader) + if format == "" { + format = "text/html" + log.Printf("forma not specified. Using %v\n", format) + } + + mediaType, _, _ := mime.ParseMediaType(format) + cext, err := mime.ExtensionsByType(mediaType) + if err != nil { + log.Printf("unexpected error reading media type extension: %v. Using %v\n", err, ext) + } else { + ext = cext[0] + } + w.Header().Set(ContentType, format) + + errCode := r.Header.Get(CodeHeader) + code, err := strconv.Atoi(errCode) + if err != nil { + code = 404 + log.Printf("unexpected error reading return code: %v. Using %v\n", err, code) + } + w.WriteHeader(code) + + file := fmt.Sprintf("%v/%v%v", path, code, ext) + f, err := os.Open(file) + if err != nil { + log.Printf("unexpected error opening file: %v\n", err) + scode := strconv.Itoa(code) + file := fmt.Sprintf("%v/%cxx%v", path, scode[0], ext) + f, err := os.Open(file) + if err != nil { + log.Printf("unexpected error opening file: %v\n", err) + http.NotFound(w, r) + return + } + defer f.Close() + log.Printf("serving custom error response for code %v and format %v from file %v\n", code, format, file) + io.Copy(w, f) + return + } + defer f.Close() + log.Printf("serving custom error response for code %v and format %v from file %v\n", code, format, file) + io.Copy(w, f) + } +} diff --git a/images/custom-error-pages/rootfs/Dockerfile b/images/custom-error-pages/rootfs/Dockerfile new file mode 100755 index 000000000..aaf60efb1 --- /dev/null +++ b/images/custom-error-pages/rootfs/Dockerfile @@ -0,0 +1,21 @@ +# Copyright 2017 The Kubernetes Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM BASEIMAGE + +CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/ + +COPY . / + +CMD ["/custom-error-pages"] diff --git a/images/custom-error-pages/rootfs/www/404.html b/images/custom-error-pages/rootfs/www/404.html new file mode 100644 index 000000000..0f00f93f5 --- /dev/null +++ b/images/custom-error-pages/rootfs/www/404.html @@ -0,0 +1 @@ +The page you're looking for could not be found. \ No newline at end of file diff --git a/images/custom-error-pages/rootfs/www/404.json b/images/custom-error-pages/rootfs/www/404.json new file mode 100644 index 000000000..36f3e753e --- /dev/null +++ b/images/custom-error-pages/rootfs/www/404.json @@ -0,0 +1 @@ +{ "message": "The page you're looking for could not be found" } \ No newline at end of file diff --git a/images/custom-error-pages/rootfs/www/4xx.html b/images/custom-error-pages/rootfs/www/4xx.html new file mode 100644 index 000000000..05ba01769 --- /dev/null +++ b/images/custom-error-pages/rootfs/www/4xx.html @@ -0,0 +1 @@ +4xx html \ No newline at end of file diff --git a/images/custom-error-pages/rootfs/www/4xx.json b/images/custom-error-pages/rootfs/www/4xx.json new file mode 100644 index 000000000..48246d04e --- /dev/null +++ b/images/custom-error-pages/rootfs/www/4xx.json @@ -0,0 +1 @@ +4xx json \ No newline at end of file diff --git a/images/custom-error-pages/rootfs/www/500.html b/images/custom-error-pages/rootfs/www/500.html new file mode 100644 index 000000000..68d7a7b52 --- /dev/null +++ b/images/custom-error-pages/rootfs/www/500.html @@ -0,0 +1 @@ +500 html \ No newline at end of file diff --git a/images/custom-error-pages/rootfs/www/500.json b/images/custom-error-pages/rootfs/www/500.json new file mode 100644 index 000000000..9cad530c1 --- /dev/null +++ b/images/custom-error-pages/rootfs/www/500.json @@ -0,0 +1 @@ +500 json \ No newline at end of file diff --git a/images/custom-error-pages/rootfs/www/5xx.html b/images/custom-error-pages/rootfs/www/5xx.html new file mode 100644 index 000000000..1bcb87780 --- /dev/null +++ b/images/custom-error-pages/rootfs/www/5xx.html @@ -0,0 +1 @@ +5xx html \ No newline at end of file diff --git a/images/custom-error-pages/rootfs/www/5xx.json b/images/custom-error-pages/rootfs/www/5xx.json new file mode 100644 index 000000000..4703b68c6 --- /dev/null +++ b/images/custom-error-pages/rootfs/www/5xx.json @@ -0,0 +1 @@ +5xx json \ No newline at end of file From 407d8535dacabe6141b63bf0c0201cd28080ea82 Mon Sep 17 00:00:00 2001 From: decker Date: Wed, 20 Sep 2017 17:35:16 +0800 Subject: [PATCH 02/15] Surpport snippet for server section by the annotation of the ingess --- .../rootfs/etc/nginx/template/nginx.tmpl | 4 ++ .../ingress/annotations/serversnippet/main.go | 42 ++++++++++++++ .../annotations/serversnippet/main_test.go | 58 +++++++++++++++++++ core/pkg/ingress/controller/annotations.go | 8 +++ core/pkg/ingress/controller/controller.go | 22 +++++++ core/pkg/ingress/types.go | 3 + 6 files changed, 137 insertions(+) create mode 100644 core/pkg/ingress/annotations/serversnippet/main.go create mode 100644 core/pkg/ingress/annotations/serversnippet/main_test.go diff --git a/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl b/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl index 4fc6f95a6..86183ab69 100644 --- a/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl +++ b/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl @@ -591,6 +591,10 @@ stream { {{ end }} {{ end }} + {{ if not (empty $server.ServerSnippet) }} + {{ $server.ServerSnippet }} + {{ end }} + {{ range $location := $server.Locations }} {{ $path := buildLocation $location }} {{ $authPath := buildAuthLocation $location }} diff --git a/core/pkg/ingress/annotations/serversnippet/main.go b/core/pkg/ingress/annotations/serversnippet/main.go new file mode 100644 index 000000000..1712c39ce --- /dev/null +++ b/core/pkg/ingress/annotations/serversnippet/main.go @@ -0,0 +1,42 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package serversnippet + +import ( + extensions "k8s.io/api/extensions/v1beta1" + + "k8s.io/ingress/core/pkg/ingress/annotations/parser" +) + +const ( + annotation = "ingress.kubernetes.io/server-snippet" +) + +type serverSnippet struct { +} + +// NewParser creates a new CORS annotation parser +func NewParser() parser.IngressAnnotation { + return serverSnippet{} +} + +// Parse parses the annotations contained in the ingress rule +// used to indicate if the location/s contains a fragment of +// configuration to be included inside the paths of the rules +func (a serverSnippet) Parse(ing *extensions.Ingress) (interface{}, error) { + return parser.GetStringAnnotation(annotation, ing) +} diff --git a/core/pkg/ingress/annotations/serversnippet/main_test.go b/core/pkg/ingress/annotations/serversnippet/main_test.go new file mode 100644 index 000000000..89113cbc5 --- /dev/null +++ b/core/pkg/ingress/annotations/serversnippet/main_test.go @@ -0,0 +1,58 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package serversnippet + +import ( + "testing" + + meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + api "k8s.io/client-go/pkg/api/v1" + extensions "k8s.io/client-go/pkg/apis/extensions/v1beta1" +) + +func TestParse(t *testing.T) { + ap := NewParser() + if ap == nil { + t.Fatalf("expected a parser.IngressAnnotation but returned nil") + } + + testCases := []struct { + annotations map[string]string + expected string + }{ + {map[string]string{annotation: "more_headers"}, "more_headers"}, + {map[string]string{annotation: "false"}, "false"}, + {map[string]string{}, ""}, + {nil, ""}, + } + + ing := &extensions.Ingress{ + ObjectMeta: meta_v1.ObjectMeta{ + Name: "foo", + Namespace: api.NamespaceDefault, + }, + Spec: extensions.IngressSpec{}, + } + + for _, testCase := range testCases { + ing.SetAnnotations(testCase.annotations) + result, _ := ap.Parse(ing) + if result != testCase.expected { + t.Errorf("expected %v but returned %v, annotations: %s", testCase.expected, result, testCase.annotations) + } + } +} diff --git a/core/pkg/ingress/controller/annotations.go b/core/pkg/ingress/controller/annotations.go index 4ae3ea163..40f623bcf 100644 --- a/core/pkg/ingress/controller/annotations.go +++ b/core/pkg/ingress/controller/annotations.go @@ -36,6 +36,7 @@ import ( "k8s.io/ingress/core/pkg/ingress/annotations/redirect" "k8s.io/ingress/core/pkg/ingress/annotations/rewrite" "k8s.io/ingress/core/pkg/ingress/annotations/secureupstream" + "k8s.io/ingress/core/pkg/ingress/annotations/serversnippet" "k8s.io/ingress/core/pkg/ingress/annotations/serviceupstream" "k8s.io/ingress/core/pkg/ingress/annotations/sessionaffinity" "k8s.io/ingress/core/pkg/ingress/annotations/snippet" @@ -83,6 +84,7 @@ func newAnnotationExtractor(cfg extractorConfig) annotationExtractor { "DefaultBackend": defaultbackend.NewParser(cfg), "UpstreamVhost": upstreamvhost.NewParser(), "VtsFilterKey": vtsfilterkey.NewParser(), + "ServerSnippet": serversnippet.NewParser(), }, } } @@ -128,6 +130,7 @@ const ( serverAlias = "Alias" clientBodyBufferSize = "ClientBodyBufferSize" certificateAuth = "CertificateAuth" + serverSnippet = "ServerSnippet" ) func (e *annotationExtractor) ServiceUpstream(ing *extensions.Ingress) bool { @@ -181,3 +184,8 @@ func (e *annotationExtractor) CertificateAuth(ing *extensions.Ingress) *authtls. secure := val.(*authtls.AuthSSLConfig) return secure } + +func (e *annotationExtractor) ServerSnippet(ing *extensions.Ingress) string { + val, _ := e.annotations[serverSnippet].Parse(ing) + return val.(string) +} diff --git a/core/pkg/ingress/controller/controller.go b/core/pkg/ingress/controller/controller.go index 03f4c92a9..88cbdc0f1 100644 --- a/core/pkg/ingress/controller/controller.go +++ b/core/pkg/ingress/controller/controller.go @@ -1065,6 +1065,28 @@ func (ic *GenericController) createServers(data []interface{}, servers[host].Alias = "" } } + + // configure server snippet + for _, ingIf := range data { + ing := ingIf.(*extensions.Ingress) + if !class.IsValid(ing, ic.cfg.IngressClass, ic.cfg.DefaultIngressClass) { + continue + } + + for _, rule := range ing.Spec.Rules { + host := rule.Host + if host == "" { + host = defServerName + } + + srvsnippet := ic.annotations.ServerSnippet(ing) + // only add a server snippet if the server does not have one previously configured + + if servers[host].ServerSnippet == "" && srvsnippet != "" { + servers[host].ServerSnippet = srvsnippet + } + } + } return servers } diff --git a/core/pkg/ingress/types.go b/core/pkg/ingress/types.go index a24ed5c5b..99c8b44b4 100644 --- a/core/pkg/ingress/types.go +++ b/core/pkg/ingress/types.go @@ -236,6 +236,9 @@ type Server struct { // CertificateAuth indicates the this server requires mutual authentication // +optional CertificateAuth authtls.AuthSSLConfig `json:"certificateAuth"` + + // ServerSnippet returns the snippet of server + ServerSnippet string `json:"serverSnippet"` } // Location describes an URI inside a server. From 15c6a1175a0d71212c322653b797d6e67c6dd556 Mon Sep 17 00:00:00 2001 From: decker Date: Thu, 21 Sep 2017 14:50:11 +0800 Subject: [PATCH 03/15] fix test --- core/pkg/ingress/annotations/serversnippet/main_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/pkg/ingress/annotations/serversnippet/main_test.go b/core/pkg/ingress/annotations/serversnippet/main_test.go index 89113cbc5..1a4d51bed 100644 --- a/core/pkg/ingress/annotations/serversnippet/main_test.go +++ b/core/pkg/ingress/annotations/serversnippet/main_test.go @@ -19,9 +19,9 @@ package serversnippet import ( "testing" + api "k8s.io/api/core/v1" + extensions "k8s.io/api/extensions/v1beta1" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - api "k8s.io/client-go/pkg/api/v1" - extensions "k8s.io/client-go/pkg/apis/extensions/v1beta1" ) func TestParse(t *testing.T) { From 0498f2a4453684fc6c804f0ecaaa755af77ecb6c Mon Sep 17 00:00:00 2001 From: Manuel de Brito Fontes Date: Tue, 26 Sep 2017 15:05:20 -0300 Subject: [PATCH 04/15] Fix cast error --- core/pkg/ingress/controller/controller.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/pkg/ingress/controller/controller.go b/core/pkg/ingress/controller/controller.go index 4a1efbe7f..f11192785 100644 --- a/core/pkg/ingress/controller/controller.go +++ b/core/pkg/ingress/controller/controller.go @@ -255,8 +255,8 @@ func (ic *GenericController) syncIngress(key interface{}) error { // Sort ingress rules using the ResourceVersion field ings := ic.listers.Ingress.List() sort.SliceStable(ings, func(i, j int) bool { - ir := ings[i].(*ingress.SSLCert).ResourceVersion - jr := ings[j].(*ingress.SSLCert).ResourceVersion + ir := ings[i].(*extensions.Ingress).ResourceVersion + jr := ings[j].(*extensions.Ingress).ResourceVersion return ir < jr }) From ea7771ead98f8c721eb2fbff3657075666943b86 Mon Sep 17 00:00:00 2001 From: Albert Vaca Date: Tue, 26 Sep 2017 23:11:25 +0200 Subject: [PATCH 05/15] Allow overriding the tag and registry The same way it is possible in controllers/nginx/Makefile --- images/nginx-slim/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/images/nginx-slim/Makefile b/images/nginx-slim/Makefile index 0bcce6696..0a5430822 100644 --- a/images/nginx-slim/Makefile +++ b/images/nginx-slim/Makefile @@ -13,8 +13,8 @@ # limitations under the License. # 0.0.0 shouldn't clobber any released builds -TAG = 0.25 -REGISTRY = gcr.io/google_containers +TAG ?= 0.25 +REGISTRY ?= gcr.io/google_containers ARCH ?= $(shell go env GOARCH) ALL_ARCH = amd64 arm arm64 ppc64le From 5dfee7ba9a04ef19d10931e83b507c8d0ede9645 Mon Sep 17 00:00:00 2001 From: decker Date: Wed, 27 Sep 2017 14:59:10 +0800 Subject: [PATCH 06/15] changed by suggest --- .../ingress/annotations/serversnippet/main.go | 2 +- core/pkg/ingress/controller/controller.go | 33 +++++++------------ core/pkg/ingress/types.go | 1 + 3 files changed, 14 insertions(+), 22 deletions(-) diff --git a/core/pkg/ingress/annotations/serversnippet/main.go b/core/pkg/ingress/annotations/serversnippet/main.go index 1712c39ce..0062bfdd5 100644 --- a/core/pkg/ingress/annotations/serversnippet/main.go +++ b/core/pkg/ingress/annotations/serversnippet/main.go @@ -29,7 +29,7 @@ const ( type serverSnippet struct { } -// NewParser creates a new CORS annotation parser +// NewParser creates a new server snippet annotation parser func NewParser() parser.IngressAnnotation { return serverSnippet{} } diff --git a/core/pkg/ingress/controller/controller.go b/core/pkg/ingress/controller/controller.go index 88cbdc0f1..ee7e88d10 100644 --- a/core/pkg/ingress/controller/controller.go +++ b/core/pkg/ingress/controller/controller.go @@ -987,6 +987,7 @@ func (ic *GenericController) createServers(data []interface{}, // setup server-alias based on annotations aliasAnnotation := ic.annotations.Alias(ing) + srvsnippet := ic.annotations.ServerSnippet(ing) for _, rule := range ing.Spec.Rules { host := rule.Host @@ -1002,6 +1003,17 @@ func (ic *GenericController) createServers(data []interface{}, } } + //notifying the user that it has already been configured. + if servers[host].ServerSnippet != "" && srvsnippet != "" { + glog.Warningf("ingress %v/%v for host %v contains a Server Snippet section that it has already been configured.", + ing.Namespace, ing.Name, host) + } + + // only add a server snippet if the server does not have one previously configured + if servers[host].ServerSnippet == "" && srvsnippet != "" { + servers[host].ServerSnippet = srvsnippet + } + // only add a certificate if the server does not have one previously configured if servers[host].SSLCertificate != "" { continue @@ -1066,27 +1078,6 @@ func (ic *GenericController) createServers(data []interface{}, } } - // configure server snippet - for _, ingIf := range data { - ing := ingIf.(*extensions.Ingress) - if !class.IsValid(ing, ic.cfg.IngressClass, ic.cfg.DefaultIngressClass) { - continue - } - - for _, rule := range ing.Spec.Rules { - host := rule.Host - if host == "" { - host = defServerName - } - - srvsnippet := ic.annotations.ServerSnippet(ing) - // only add a server snippet if the server does not have one previously configured - - if servers[host].ServerSnippet == "" && srvsnippet != "" { - servers[host].ServerSnippet = srvsnippet - } - } - } return servers } diff --git a/core/pkg/ingress/types.go b/core/pkg/ingress/types.go index 99c8b44b4..9235121eb 100644 --- a/core/pkg/ingress/types.go +++ b/core/pkg/ingress/types.go @@ -238,6 +238,7 @@ type Server struct { CertificateAuth authtls.AuthSSLConfig `json:"certificateAuth"` // ServerSnippet returns the snippet of server + // +optional ServerSnippet string `json:"serverSnippet"` } From 1ffeb2cee1f431a22f417a9882ff13243d6f432d Mon Sep 17 00:00:00 2001 From: Fernando Diaz Date: Tue, 26 Sep 2017 22:46:22 -0500 Subject: [PATCH 07/15] Enhance Certificate Logging and Clearup Mutual Auth Docs Adds better logging to errors caused when getting a Certificate. Adds notes and updates documentation for Mutual Authentication. --- controllers/nginx/configuration.md | 4 +-- core/pkg/ingress/controller/backend_ssl.go | 34 +++++++++++++------ core/pkg/ingress/controller/controller.go | 13 +++++-- core/pkg/ingress/resolver/main.go | 2 +- core/pkg/net/ssl/ssl.go | 2 +- examples/PREREQUISITES.md | 7 +++- examples/auth/client-certs/nginx/README.md | 13 +++---- .../client-certs/nginx/nginx-tls-auth.yaml | 3 +- 8 files changed, 49 insertions(+), 29 deletions(-) diff --git a/controllers/nginx/configuration.md b/controllers/nginx/configuration.md index 17299ba9d..d94d31c16 100644 --- a/controllers/nginx/configuration.md +++ b/controllers/nginx/configuration.md @@ -135,14 +135,14 @@ Please check the [auth](/examples/auth/basic/nginx/README.md) example. ### Certificate Authentication -It's possible to enable Certificate based authentication using additional annotations in Ingress Rule. +It's possible to enable Certificate-Based Authentication (Mutual Authentication) using additional annotations in Ingress Rule. The annotations are: ``` ingress.kubernetes.io/auth-tls-secret: secretName ``` -The name of the secret that contains the full Certificate Authority chain that is enabled to authenticate against this ingress. It's composed of namespace/secretName +The name of the secret that contains the full Certificate Authority chain `ca.crt` that is enabled to authenticate against this ingress. It's composed of namespace/secretName. ``` ingress.kubernetes.io/auth-tls-verify-depth diff --git a/core/pkg/ingress/controller/backend_ssl.go b/core/pkg/ingress/controller/backend_ssl.go index 98005f340..5791bcfa1 100644 --- a/core/pkg/ingress/controller/backend_ssl.go +++ b/core/pkg/ingress/controller/backend_ssl.go @@ -74,35 +74,47 @@ func (ic *GenericController) getPemCertificate(secretName string) (*ingress.SSLC cert, okcert := secret.Data[apiv1.TLSCertKey] key, okkey := secret.Data[apiv1.TLSPrivateKeyKey] - ca := secret.Data["ca.crt"] + // namespace/secretName -> namespace-secretName nsSecName := strings.Replace(secretName, "/", "-", -1) var s *ingress.SSLCert if okcert && okkey { - if cert == nil || key == nil { - return nil, fmt.Errorf("error retrieving cert or key from secret %v: %v", secretName, err) + if cert == nil { + return nil, fmt.Errorf("secret %v has no 'tls.crt'", secretName) } + if key == nil { + return nil, fmt.Errorf("secret %v has no 'tls.key'", secretName) + } + + // If 'ca.crt' is also present, it will allow this secret to be used in the + // 'ingress.kubernetes.io/auth-tls-secret' annotation s, err = ssl.AddOrUpdateCertAndKey(nsSecName, cert, key, ca) if err != nil { - return nil, fmt.Errorf("unexpected error creating pem file %v", err) + return nil, fmt.Errorf("unexpected error creating pem file: %v", err) } - glog.V(3).Infof("found certificate and private key, configuring %v as a TLS Secret (CN: %v)", secretName, s.CN) + + glog.V(3).Infof("found 'tls.crt' and 'tls.key', configuring %v as a TLS Secret (CN: %v)", secretName, s.CN) + if ca != nil { + glog.V(3).Infof("found 'ca.crt', secret %v can also be used for Certificate Authentication", secretName) + } + } else if ca != nil { - glog.V(3).Infof("found only ca.crt, configuring %v as an Certificate Authentication secret", secretName) s, err = ssl.AddCertAuth(nsSecName, ca) + if err != nil { - return nil, fmt.Errorf("unexpected error creating pem file %v", err) + return nil, fmt.Errorf("unexpected error creating pem file: %v", err) } + + // makes this secret in 'syncSecret' to be used for Certificate Authentication + // this does not enable Certificate Authentication + glog.V(3).Infof("found only 'ca.crt', configuring %v as an Certificate Authentication Secret", secretName) + } else { return nil, fmt.Errorf("no keypair or CA cert could be found in %v", secretName) } - if err != nil { - return nil, err - } - s.Name = secret.Name s.Namespace = secret.Namespace return s, nil diff --git a/core/pkg/ingress/controller/controller.go b/core/pkg/ingress/controller/controller.go index f11192785..f6f713540 100644 --- a/core/pkg/ingress/controller/controller.go +++ b/core/pkg/ingress/controller/controller.go @@ -509,9 +509,13 @@ func (ic *GenericController) getBackendServers(ingresses []*extensions.Ingress) ca := ic.annotations.CertificateAuth(ing) if ca != nil { server.CertificateAuth = *ca + // It is possible that no CAFileName is found in the secret + if server.CertificateAuth.CAFileName == "" { + glog.V(3).Infof("secret %v does not contain 'ca.crt', mutual authentication not enabled - ingress rule %v/%v.", server.CertificateAuth.Secret, ing.Namespace, ing.Name) + } } } else { - glog.V(3).Infof("server %v already contains a muthual autentication configuration - ingress rule %v/%v", server.Hostname, ing.Namespace, ing.Name) + glog.V(3).Infof("server %v already contains a mutual authentication configuration - ingress rule %v/%v", server.Hostname, ing.Namespace, ing.Name) } for _, path := range rule.HTTP.Paths { @@ -671,7 +675,8 @@ func (ic *GenericController) getBackendServers(ingresses []*extensions.Ingress) return aUpstreams, aServers } -// GetAuthCertificate ... + +// GetAuthCertificate is used by the auth-tls annotations to get a cert from a secret func (ic GenericController) GetAuthCertificate(secretName string) (*resolver.AuthSSLCert, error) { if _, exists := ic.sslCertTracker.Get(secretName); !exists { ic.syncSecret(secretName) @@ -894,10 +899,12 @@ func (ic *GenericController) createServers(data []*extensions.Ingress, RequestBuffering: bdef.ProxyRequestBuffering, } + // generated on Start() with createDefaultSSLCertificate() defaultPemFileName := fakeCertificatePath defaultPemSHA := fakeCertificateSHA - // Tries to fetch the default Certificate. If it does not exists, generate a new self signed one. + // Tries to fetch the default Certificate from nginx configuration. + // If it does not exists, use the ones generated on Start() defaultCertificate, err := ic.getPemCertificate(ic.cfg.DefaultSSLCertificate) if err == nil { defaultPemFileName = defaultCertificate.PemFileName diff --git a/core/pkg/ingress/resolver/main.go b/core/pkg/ingress/resolver/main.go index 9bb11db40..d4672aa76 100644 --- a/core/pkg/ingress/resolver/main.go +++ b/core/pkg/ingress/resolver/main.go @@ -54,7 +54,7 @@ type AuthSSLCert struct { Secret string `json:"secret"` // CAFileName contains the path to the secrets 'ca.crt' CAFileName string `json:"caFilename"` - // PemSHA contains the SHA1 hash of the 'tls.crt' value + // PemSHA contains the SHA1 hash of the 'ca.crt' or combinations of (tls.crt, tls.key, tls.crt) depending on certs in secret PemSHA string `json:"pemSha"` } diff --git a/core/pkg/net/ssl/ssl.go b/core/pkg/net/ssl/ssl.go index e1cd5b0ff..535c5f5fe 100644 --- a/core/pkg/net/ssl/ssl.go +++ b/core/pkg/net/ssl/ssl.go @@ -271,7 +271,7 @@ func AddCertAuth(name string, ca []byte) (*ingress.SSLCert, error) { return nil, fmt.Errorf("could not write CA file %v: %v", caFileName, err) } - glog.V(3).Infof("Created CA Certificate for authentication: %v", caFileName) + glog.V(3).Infof("Created CA Certificate for Authentication: %v", caFileName) return &ingress.SSLCert{ CAFileName: caFileName, PemFileName: caFileName, diff --git a/examples/PREREQUISITES.md b/examples/PREREQUISITES.md index 03f9f74a4..7c6bf2fd6 100644 --- a/examples/PREREQUISITES.md +++ b/examples/PREREQUISITES.md @@ -132,7 +132,12 @@ The final step is to create a secret with the content of this file. This secret the TLS Auth directive: ```console -$ kubectl create secret generic caingress --namespace=default --from-file=ca.crt +$ kubectl create secret generic caingress --namespace=default --from-file=ca.crt= +``` + +Note: You can also generate the CA Authentication Secret along with the TLS Secret by using: +```console +$ kubectl create secret generic caingress --namespace=default --from-file=ca.crt= --from-file=tls.crt= --from-file=tls.key= ``` ## Test HTTP Service diff --git a/examples/auth/client-certs/nginx/README.md b/examples/auth/client-certs/nginx/README.md index e8c9e83ac..d3da9d1a6 100644 --- a/examples/auth/client-certs/nginx/README.md +++ b/examples/auth/client-certs/nginx/README.md @@ -16,13 +16,12 @@ the child, except for the root, which has Issuer == Subject. * Client Cert: Certificate used by the clients to authenticate themselves with the loadbalancer/backends. - ## Prerequisites You need a valid CA File, composed of a group of valid enabled CAs. This MUST be in PEM Format. -The instructions are described [here](../../../PREREQUISITES.md#ca-authentication) +The instructions are described [here](../../../PREREQUISITES.md) -Also your ingress must be configured as a HTTPs/TLS Ingress. +Also your ingress must be configured as a HTTPS/TLS Ingress. ## Deployment @@ -51,8 +50,7 @@ Name: nginx-test Namespace: default Address: 104.198.183.6 Default backend: default-http-backend:80 (10.180.0.4:8080,10.240.0.2:8080) -TLS: - tls-secret terminates ingress.test.com +TLS: tls-secret terminates ingress.test.com Rules: Host Path Backends ---- ---- -------- @@ -79,13 +77,12 @@ Server: nginx/1.11.9 $ curl -I -k --key ~/user.key --cert ~/user.cer https://ingress.test.com HTTP/1.1 200 OK Server: nginx/1.11.9 - ``` You must use the full DNS name while testing, as NGINX relies on the Server Name (SNI) to select the correct Ingress to be used. - The curl version used here was ``curl 7.47.0`` ## Which certificate was used for authentication? -In your backend application you might want to know which certificate was used for authentication. For this purpose, we pass the full certificate in PEM format to the backend in the `ssl-client-cert` header. +In your backend application you might want to know which certificate was used for authentication. +For this purpose, we pass the full certificate in PEM format to the backend in the `ssl-client-cert` header. diff --git a/examples/auth/client-certs/nginx/nginx-tls-auth.yaml b/examples/auth/client-certs/nginx/nginx-tls-auth.yaml index ac03d9d7c..c3ef92a8e 100644 --- a/examples/auth/client-certs/nginx/nginx-tls-auth.yaml +++ b/examples/auth/client-certs/nginx/nginx-tls-auth.yaml @@ -21,6 +21,5 @@ spec: tls: - hosts: - ingress.test.com - # Create this cert as described in 'multi-tls' example - secretName: cert + secretName: tls-secret From a24d65745efc757a76ef749b18a263cfb37d7f02 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Guerraz Date: Wed, 27 Sep 2017 16:10:16 +0200 Subject: [PATCH 08/15] Add support for proxy protocol decoding and encoding in TCP services --- controllers/nginx/README.md | 4 +-- .../rootfs/etc/nginx/template/nginx.tmpl | 11 +++++--- core/pkg/ingress/controller/controller.go | 25 +++++++++++-------- core/pkg/ingress/types.go | 8 +++++- 4 files changed, 31 insertions(+), 17 deletions(-) diff --git a/controllers/nginx/README.md b/controllers/nginx/README.md index dabfcdc9d..fea53c814 100644 --- a/controllers/nginx/README.md +++ b/controllers/nginx/README.md @@ -334,8 +334,8 @@ version to fully support Kube-Lego is nginx Ingress controller 0.8. ## Exposing TCP services -Ingress does not support TCP services (yet). For this reason this Ingress controller uses the flag `--tcp-services-configmap` to point to an existing config map where the key is the external port to use and the value is `::[PROXY]` -It is possible to use a number or the name of the port. The last field is optional. Adding `PROXY` in the last field we can enable Proxy Protocol in a TCP service. +Ingress does not support TCP services (yet). For this reason this Ingress controller uses the flag `--tcp-services-configmap` to point to an existing config map where the key is the external port to use and the value is `::[PROXY]:[PROXY]` +It is possible to use a number or the name of the port. The two last fields are optional. Adding `PROXY` in either or both of the two last fields we can use Proxy Protocol decoding (listen) and/or encoding (proxy_pass) in a TCP service (https://www.nginx.com/resources/admin-guide/proxy-protocol/). The next example shows how to expose the service `example-go` running in the namespace `default` in the port `8080` using the port `9000` ``` diff --git a/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl b/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl index c2b933cdc..8ff59ae98 100644 --- a/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl +++ b/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl @@ -434,19 +434,22 @@ stream { } server { {{ range $address := $all.Cfg.BindAddressIpv4 }} - listen {{ $address }}:{{ $tcpServer.Port }}{{ if $tcpServer.Backend.UseProxyProtocol }} proxy_protocol{{ end }}; + listen {{ $address }}:{{ $tcpServer.Port }}{{ if $tcpServer.Backend.ProxyProtocol.Decode }} proxy_protocol{{ end }}; {{ else }} - listen {{ $tcpServer.Port }}{{ if $tcpServer.Backend.UseProxyProtocol }} proxy_protocol{{ end }}; + listen {{ $tcpServer.Port }}{{ if $tcpServer.Backend.ProxyProtocol.Decode }} proxy_protocol{{ end }}; {{ end }} {{ if $IsIPV6Enabled }} {{ range $address := $all.Cfg.BindAddressIpv6 }} - listen {{ $address }}:{{ $tcpServer.Port }}{{ if $tcpServer.Backend.UseProxyProtocol }} proxy_protocol{{ end }}; + listen {{ $address }}:{{ $tcpServer.Port }}{{ if $tcpServer.Backend.ProxyProtocol.Decode }} proxy_protocol{{ end }}; {{ else }} - listen [::]:{{ $tcpServer.Port }}{{ if $tcpServer.Backend.UseProxyProtocol }} proxy_protocol{{ end }}; + listen [::]:{{ $tcpServer.Port }}{{ if $tcpServer.Backend.ProxyProtocol.Decode }} proxy_protocol{{ end }}; {{ end }} {{ end }} proxy_timeout {{ $cfg.ProxyStreamTimeout }}; proxy_pass tcp-{{ $tcpServer.Port }}-{{ $tcpServer.Backend.Namespace }}-{{ $tcpServer.Backend.Name }}-{{ $tcpServer.Backend.Port }}; + {{ if $tcpServer.Backend.ProxyProtocol.Encode }} + proxy_protocol on; + {{ end }} } {{ end }} diff --git a/core/pkg/ingress/controller/controller.go b/core/pkg/ingress/controller/controller.go index f11192785..01d8b8260 100644 --- a/core/pkg/ingress/controller/controller.go +++ b/core/pkg/ingress/controller/controller.go @@ -346,6 +346,7 @@ func (ic *GenericController) getStreamServices(configmapName string, proto apiv1 } var svcs []ingress.L4Service + var svcProxyProtocol ingress.ProxyProtocol // k -> port to expose // v -> /: for k, v := range configmap.Data { @@ -363,18 +364,22 @@ func (ic *GenericController) getStreamServices(configmapName string, proto apiv1 nsSvcPort := strings.Split(v, ":") if len(nsSvcPort) < 2 { - glog.Warningf("invalid format (namespace/name:port:[PROXY]) '%v'", k) + glog.Warningf("invalid format (namespace/name:port:[PROXY]:[PROXY]) '%v'", k) continue } nsName := nsSvcPort[0] svcPort := nsSvcPort[1] - useProxyProtocol := false + svcProxyProtocol.Decode = false + svcProxyProtocol.Encode = false // Proxy protocol is possible if the service is TCP - if len(nsSvcPort) == 3 && proto == apiv1.ProtocolTCP { - if strings.ToUpper(nsSvcPort[2]) == "PROXY" { - useProxyProtocol = true + if len(nsSvcPort) >= 3 && proto == apiv1.ProtocolTCP { + if len(nsSvcPort) >= 3 && strings.ToUpper(nsSvcPort[2]) == "PROXY" { + svcProxyProtocol.Decode = true + } + if len(nsSvcPort) == 4 && strings.ToUpper(nsSvcPort[3]) == "PROXY" { + svcProxyProtocol.Encode = true } } @@ -432,11 +437,11 @@ func (ic *GenericController) getStreamServices(configmapName string, proto apiv1 svcs = append(svcs, ingress.L4Service{ Port: externalPort, Backend: ingress.L4Backend{ - Name: svcName, - Namespace: svcNs, - Port: intstr.FromString(svcPort), - Protocol: proto, - UseProxyProtocol: useProxyProtocol, + Name: svcName, + Namespace: svcNs, + Port: intstr.FromString(svcPort), + Protocol: proto, + ProxyProtocol: svcProxyProtocol, }, Endpoints: endps, }) diff --git a/core/pkg/ingress/types.go b/core/pkg/ingress/types.go index a24ed5c5b..1a328d647 100644 --- a/core/pkg/ingress/types.go +++ b/core/pkg/ingress/types.go @@ -359,5 +359,11 @@ type L4Backend struct { Namespace string `json:"namespace"` Protocol apiv1.Protocol `json:"protocol"` // +optional - UseProxyProtocol bool `json:"useProxyProtocol"` + ProxyProtocol ProxyProtocol `json:"proxyProtocol"` +} + +// ProxyProtocol describes the proxy protocol configuration +type ProxyProtocol struct { + Decode bool `json:"decode"` + Encode bool `json:"encode"` } From 62e000ebf2bb22d411966b5654bea9aa4df71f2b Mon Sep 17 00:00:00 2001 From: Joao Morais Date: Wed, 27 Sep 2017 20:53:31 -0300 Subject: [PATCH 09/15] Fix exec of readSecrets --- core/pkg/ingress/controller/controller.go | 4 ++-- core/pkg/task/queue.go | 9 +++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/core/pkg/ingress/controller/controller.go b/core/pkg/ingress/controller/controller.go index 5d3a7bb9c..86128f66d 100644 --- a/core/pkg/ingress/controller/controller.go +++ b/core/pkg/ingress/controller/controller.go @@ -238,14 +238,14 @@ func (ic GenericController) GetService(name string) (*apiv1.Service, error) { // sync collects all the pieces required to assemble the configuration file and // then sends the content to the backend (OnUpdate) receiving the populated // template as response reloading the backend if is required. -func (ic *GenericController) syncIngress(key interface{}) error { +func (ic *GenericController) syncIngress(element interface{}) error { ic.syncRateLimiter.Accept() if ic.syncQueue.IsShuttingDown() { return nil } - if name, ok := key.(string); ok { + if name, ok := element.(task.Element).Key.(string); ok { if obj, exists, _ := ic.listers.Ingress.GetByKey(name); exists { ing := obj.(*extensions.Ingress) ic.readSecrets(ing) diff --git a/core/pkg/task/queue.go b/core/pkg/task/queue.go index 977fdc2f6..9b05f1653 100644 --- a/core/pkg/task/queue.go +++ b/core/pkg/task/queue.go @@ -48,7 +48,8 @@ type Queue struct { lastSync int64 } -type element struct { +// Element represents one item of the queue +type Element struct { Key interface{} Timestamp int64 } @@ -72,7 +73,7 @@ func (t *Queue) Enqueue(obj interface{}) { glog.Errorf("%v", err) return } - t.queue.Add(element{ + t.queue.Add(Element{ Key: key, Timestamp: ts, }) @@ -99,7 +100,7 @@ func (t *Queue) worker() { } ts := time.Now().UnixNano() - item := key.(element) + item := key.(Element) if t.lastSync > item.Timestamp { glog.V(3).Infof("skipping %v sync (%v > %v)", item.Key, t.lastSync, item.Timestamp) t.queue.Forget(key) @@ -110,7 +111,7 @@ func (t *Queue) worker() { glog.V(3).Infof("syncing %v", item.Key) if err := t.sync(key); err != nil { glog.Warningf("requeuing %v, err %v", item.Key, err) - t.queue.AddRateLimited(element{ + t.queue.AddRateLimited(Element{ Key: item.Key, Timestamp: time.Now().UnixNano(), }) From 389c7055fa29aeacc67d5fdac1b79d3de5846e2a Mon Sep 17 00:00:00 2001 From: Joao Morais Date: Wed, 27 Sep 2017 21:33:23 -0300 Subject: [PATCH 10/15] Check if item is really a task.Element --- core/pkg/ingress/controller/controller.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/core/pkg/ingress/controller/controller.go b/core/pkg/ingress/controller/controller.go index 86128f66d..37df71a03 100644 --- a/core/pkg/ingress/controller/controller.go +++ b/core/pkg/ingress/controller/controller.go @@ -238,17 +238,19 @@ func (ic GenericController) GetService(name string) (*apiv1.Service, error) { // sync collects all the pieces required to assemble the configuration file and // then sends the content to the backend (OnUpdate) receiving the populated // template as response reloading the backend if is required. -func (ic *GenericController) syncIngress(element interface{}) error { +func (ic *GenericController) syncIngress(item interface{}) error { ic.syncRateLimiter.Accept() if ic.syncQueue.IsShuttingDown() { return nil } - if name, ok := element.(task.Element).Key.(string); ok { - if obj, exists, _ := ic.listers.Ingress.GetByKey(name); exists { - ing := obj.(*extensions.Ingress) - ic.readSecrets(ing) + if element, ok := item.(task.Element); ok { + if name, ok := element.Key.(string); ok { + if obj, exists, _ := ic.listers.Ingress.GetByKey(name); exists { + ing := obj.(*extensions.Ingress) + ic.readSecrets(ing) + } } } From 8fc6101d3ba11a1117d211bdac6104dab7af8eeb Mon Sep 17 00:00:00 2001 From: Manuel de Brito Fontes Date: Wed, 27 Sep 2017 23:51:01 -0300 Subject: [PATCH 11/15] Add header to upstream server for external authentication --- controllers/nginx/pkg/template/template.go | 18 ------------------ .../nginx/pkg/template/template_test.go | 18 ------------------ .../nginx/rootfs/etc/nginx/template/nginx.tmpl | 5 ++++- 3 files changed, 4 insertions(+), 37 deletions(-) diff --git a/controllers/nginx/pkg/template/template.go b/controllers/nginx/pkg/template/template.go index 943c43631..d4778ca3c 100644 --- a/controllers/nginx/pkg/template/template.go +++ b/controllers/nginx/pkg/template/template.go @@ -22,7 +22,6 @@ import ( "encoding/json" "fmt" "net" - "net/url" "os" "os/exec" "strconv" @@ -150,7 +149,6 @@ var ( "serverConfig": func(all config.TemplateConfig, server *ingress.Server) interface{} { return struct{ First, Second interface{} }{all, server} }, - "buildAuthSignURL": buildAuthSignURL, "isValidClientBodyBufferSize": isValidClientBodyBufferSize, "buildForwardedFor": buildForwardedFor, } @@ -567,22 +565,6 @@ func buildNextUpstream(input interface{}) string { return strings.Join(nextUpstreamCodes, " ") } -func buildAuthSignURL(input interface{}) string { - s, ok := input.(string) - if !ok { - glog.Errorf("expected an 'string' type but %T was returned", input) - return "" - } - - u, _ := url.Parse(s) - q := u.Query() - if len(q) == 0 { - return fmt.Sprintf("%v?rd=$request_uri", s) - } - - return fmt.Sprintf("%v&rd=$request_uri", s) -} - // buildRandomUUID return a random string to be used in the template func buildRandomUUID() string { s := uuid.New() diff --git a/controllers/nginx/pkg/template/template_test.go b/controllers/nginx/pkg/template/template_test.go index 62d0342f4..9bed3d3ac 100644 --- a/controllers/nginx/pkg/template/template_test.go +++ b/controllers/nginx/pkg/template/template_test.go @@ -310,24 +310,6 @@ func TestBuildResolvers(t *testing.T) { } } -func TestBuildAuthSignURL(t *testing.T) { - urlOne := "http://google.com" - validUrlOne := "http://google.com?rd=$request_uri" - - urlTwo := "http://google.com?cat" - validUrlTwo := "http://google.com?cat&rd=$request_uri" - - authSignURLOne := buildAuthSignURL(urlOne) - if authSignURLOne != validUrlOne { - t.Errorf("Expected '%v' but returned '%v'", validUrlOne, authSignURLOne) - } - - authSignURLTwo := buildAuthSignURL(urlTwo) - if authSignURLTwo != validUrlTwo { - t.Errorf("Expected '%v' but returned '%v'", validUrlTwo, authSignURLTwo) - } -} - func TestBuildNextUpstream(t *testing.T) { nextUpstream := "timeout http_500 http_502 non_idempotent" validNextUpstream := "timeout http_500 http_502" diff --git a/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl b/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl index e6270da3d..19c0947c5 100644 --- a/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl +++ b/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl @@ -684,7 +684,7 @@ stream { {{ end }} {{ if not (empty $location.ExternalAuth.SigninURL) }} - error_page 401 = {{ buildAuthSignURL $location.ExternalAuth.SigninURL }}; + error_page 401 = $location.ExternalAuth.SigninURL; {{ end }} {{/* if the location contains a rate limit annotation, create one */}} @@ -743,6 +743,9 @@ stream { proxy_set_header X-Original-URI $request_uri; proxy_set_header X-Scheme $pass_access_scheme; + {{/* This header is used for external authentication */}} + proxy_set_header X-Auth-Request-Redirect $request_uri; + # mitigate HTTPoxy Vulnerability # https://www.nginx.com/blog/mitigating-the-httpoxy-vulnerability-with-nginx/ proxy_set_header Proxy ""; From 8de75c22a30c12d8f30021792a6606f10897ea73 Mon Sep 17 00:00:00 2001 From: Manuel de Brito Fontes Date: Thu, 28 Sep 2017 10:07:02 -0300 Subject: [PATCH 12/15] Do not intercept errors from the custom error service --- controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl b/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl index e6270da3d..56f918933 100644 --- a/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl +++ b/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl @@ -490,6 +490,8 @@ stream { location @custom_{{ $errCode }} { internal; + proxy_intercept_errors off; + proxy_set_header X-Code {{ $errCode }}; proxy_set_header X-Format $http_accept; proxy_set_header X-Original-URI $request_uri; @@ -497,6 +499,7 @@ stream { proxy_set_header X-Ingress-Name $ingress_name; proxy_set_header X-Service-Name $service_name; + rewrite (.*) / break; proxy_pass http://upstream-default-backend; } {{ end }} From 4f94027154cb391c193c340d0dee55cf43b1be5e Mon Sep 17 00:00:00 2001 From: Manuel de Brito Fontes Date: Thu, 28 Sep 2017 11:52:12 -0300 Subject: [PATCH 13/15] Kill worker processes to allow the restart of nginx --- controllers/nginx/pkg/cmd/controller/nginx.go | 48 +++++++++++++++++-- 1 file changed, 45 insertions(+), 3 deletions(-) diff --git a/controllers/nginx/pkg/cmd/controller/nginx.go b/controllers/nginx/pkg/cmd/controller/nginx.go index 29e647375..2a3b953a0 100644 --- a/controllers/nginx/pkg/cmd/controller/nginx.go +++ b/controllers/nginx/pkg/cmd/controller/nginx.go @@ -35,6 +35,7 @@ import ( "github.com/spf13/pflag" proxyproto "github.com/armon/go-proxyproto" + "github.com/ncabatoff/process-exporter/proc" apiv1 "k8s.io/api/core/v1" extensions "k8s.io/api/extensions/v1beta1" @@ -62,7 +63,7 @@ const ( var ( tmplPath = "/etc/nginx/template/nginx.tmpl" cfgPath = "/etc/nginx/nginx.conf" - binary = "/usr/sbin/nginx" + nginxBinary = "/usr/sbin/nginx" defIngressClass = "nginx" ) @@ -72,7 +73,7 @@ var ( func newNGINXController() *NGINXController { ngx := os.Getenv("NGINX_BINARY") if ngx == "" { - ngx = binary + ngx = nginxBinary } h, err := dns.GetSystemNameServers() @@ -200,7 +201,26 @@ NGINX master process died (%v): %v break } conn.Close() - time.Sleep(1 * time.Second) + // kill nginx worker processes + fs, err := proc.NewFS("/proc") + procs, _ := fs.FS.AllProcs() + for _, p := range procs { + pn, err := p.Comm() + if err != nil { + glog.Errorf("unexpected error obtaining process information: %v", err) + continue + } + + if pn == "nginx" { + osp, err := os.FindProcess(p.PID) + if err != nil { + glog.Errorf("unexpected error obtaining process information: %v", err) + continue + } + osp.Signal(syscall.SIGQUIT) + } + } + time.Sleep(100 * time.Millisecond) } // restart a new nginx master process if the controller // is not being stopped @@ -709,6 +729,28 @@ func (n NGINXController) Check(_ *http.Request) error { if res.StatusCode != 200 { return fmt.Errorf("ingress controller is not healthy") } + + // check the nginx master process is running + fs, err := proc.NewFS("/proc") + if err != nil { + glog.Errorf("%v", err) + return err + } + f, err := ioutil.ReadFile("/run/nginx.pid") + if err != nil { + glog.Errorf("%v", err) + return err + } + pid, err := strconv.Atoi(strings.TrimRight(string(f), "\r\n")) + if err != nil { + return err + } + _, err = fs.NewProc(int(pid)) + if err != nil { + glog.Errorf("%v", err) + return err + } + return nil } From b8fe9eb031c9b3a195c32f6a2a160d406f8cb1b5 Mon Sep 17 00:00:00 2001 From: Manuel de Brito Fontes Date: Thu, 28 Sep 2017 19:41:08 -0300 Subject: [PATCH 14/15] Improve custom error pages doc --- controllers/nginx/README.md | 19 +++++++++++++++---- .../custom-errors/nginx/README.md | 4 +--- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/controllers/nginx/README.md b/controllers/nginx/README.md index fea53c814..1ad2c1027 100644 --- a/controllers/nginx/README.md +++ b/controllers/nginx/README.md @@ -381,11 +381,22 @@ Please check the [proxy-protocol](examples/proxy-protocol/) example ### Custom errors -In case of an error in a request the body of the response is obtained from the `default backend`. Each request to the default backend includes two headers: -- `X-Code` indicates the HTTP code -- `X-Format` the value of the `Accept` header +In case of an error in a request the body of the response is obtained from the `default backend`. +Each request to the default backend includes two headers: -Using this two headers is possible to use a custom backend service like [this one](https://github.com/aledbf/contrib/tree/nginx-debug-server/Ingress/images/nginx-error-server) that inspect each request and returns a custom error page with the format expected by the client. Please check the example [custom-errors](examples/custom-errors/README.md) +- `X-Code` indicates the HTTP code to be returned to the client. +- `X-Format` the value of the `Accept` header. + +**Important:** the custom backend must return the correct HTTP status code to be returned. NGINX do not changes the reponse from the custom default backend. + +Using this two headers is possible to use a custom backend service like [this one](https://github.com/kubernetes/ingress/tree/master/examples/customization/custom-errors/nginx) that inspect each request and returns a custom error page with the format expected by the client. Please check the example [custom-errors](examples/customization/custom-errors/nginx/README.md) + +NGINX sends aditional headers that can be used to build custom response: + +- X-Original-URI +- X-Namespace +- X-Ingress-Name +- X-Service-Name ### NGINX status page diff --git a/examples/customization/custom-errors/nginx/README.md b/examples/customization/custom-errors/nginx/README.md index 2f79388d5..c37240afb 100644 --- a/examples/customization/custom-errors/nginx/README.md +++ b/examples/customization/custom-errors/nginx/README.md @@ -1,4 +1,4 @@ -This example shows how is possible to use a custom backend to render custom error pages. The code of this example is located here [nginx-debug-server](https://github.com/aledbf/contrib/tree/nginx-debug-server) +This example shows how is possible to use a custom backend to render custom error pages. The code of this example is located here [custom-error-pages](https://github.com/kubernetes/ingress/tree/master/examples/customization/custom-errors/nginx) The idea is to use the headers `X-Code` and `X-Format` that NGINX pass to the backend in case of an error to find out the best existent representation of the response to be returned. i.e. if the request contains an `Accept` header of type `json` the error should be in that format and not in `html` (the default in NGINX). @@ -78,5 +78,3 @@ $ curl -v http://172.17.4.99/ -H 'Accept: application/json' * Connection #0 to host 172.17.4.99 left intact ``` - -By default the Ingress controller provides support for `html`, `json` and `XML`. From 34394f1d2d6eb8db5bed1a8bb37586ea64f15aeb Mon Sep 17 00:00:00 2001 From: Manuel de Brito Fontes Date: Thu, 28 Sep 2017 21:04:50 -0300 Subject: [PATCH 15/15] Opentracing docs --- controllers/nginx/README.md | 39 ++++++++++++++++++ controllers/nginx/configuration.md | 11 +++++ controllers/nginx/docs/images/zipkin-demo.png | Bin 0 -> 18409 bytes 3 files changed, 50 insertions(+) create mode 100644 controllers/nginx/docs/images/zipkin-demo.png diff --git a/controllers/nginx/README.md b/controllers/nginx/README.md index 1ad2c1027..9a1e3a4fb 100644 --- a/controllers/nginx/README.md +++ b/controllers/nginx/README.md @@ -17,6 +17,7 @@ This is an nginx Ingress controller that uses [ConfigMap](https://kubernetes.io/ * [TCP Services](#exposing-tcp-services) * [UDP Services](#exposing-udp-services) * [Proxy Protocol](#proxy-protocol) +* [Opentracing](#opentracing) * [NGINX customization](configuration.md) * [Custom errors](#custom-errors) * [NGINX status page](#nginx-status-page) @@ -378,6 +379,44 @@ Amongst others [ELBs in AWS](http://docs.aws.amazon.com/ElasticLoadBalancing/lat Please check the [proxy-protocol](examples/proxy-protocol/) example +### Opentracing + +Using the third party module [rnburn/nginx-opentracing](https://github.com/rnburn/nginx-opentracing) the NGINX ingress controller can configure NGINX to enable [OpenTracing](http://opentracing.io) instrumentation. +By default this feature is disabled. + +To enable the instrumentation we just need to enable the instrumentation in the configuration configmap and set the host where we should send the traces. + +In the [aledbf/zipkin-js-example](https://github.com/aledbf/zipkin-js-example) github repository is possible to see a dockerized version of zipkin-js-example with the required Kubernetes descriptors. +To install the example and the zipkin collector we just need to run: + +``` +$ kubectl create -f https://raw.githubusercontent.com/aledbf/zipkin-js-example/kubernetes/kubernetes/zipkin.yaml +$ kubectl create -f https://raw.githubusercontent.com/aledbf/zipkin-js-example/kubernetes/kubernetes/deployment.yaml +``` + +Also we need to configure the NGINX controller configmap with the required values: + +``` +apiVersion: v1 +data: + enable-opentracing: "true" + zipkin-collector-host: zipkin.default.svc.cluster.local +kind: ConfigMap +metadata: + labels: + k8s-app: nginx-ingress-controller + name: nginx-custom-configuration +``` + +Using curl we can generate some traces: +``` +$ curl -v http://$(minikube ip)/api -H 'Host: zipkin-js-example' +$ curl -v http://$(minikube ip)/api -H 'Host: zipkin-js-example' +``` + +In the zipkin inteface we can see the details: + +![zipkin screenshot](docs/images/zipkin-demo.png "zipkin collector screenshot") ### Custom errors diff --git a/controllers/nginx/configuration.md b/controllers/nginx/configuration.md index d94d31c16..517cea608 100644 --- a/controllers/nginx/configuration.md +++ b/controllers/nginx/configuration.md @@ -487,6 +487,17 @@ The default mime type list to compress is: `application/atom+xml application/jav **bind-address:** Sets the addresses on which the server will accept requests instead of *. It should be noted that these addresses must exist in the runtime environment or the controller will crash loop. +**enable-opentracing:** enables the nginx Opentracing extension https://github.com/rnburn/nginx-opentracing +Default is "false" + +**zipkin-collector-host:** specifies the host to use when uploading traces. It must be a valid URL + +**zipkin-collector-port:** specifies the port to use when uploading traces +Default: 9411 + +**zipkin-service-name:** specifies the service name to use for any traces created +Default: nginx + ### Default configuration options The following table shows the options, the default value and a description. diff --git a/controllers/nginx/docs/images/zipkin-demo.png b/controllers/nginx/docs/images/zipkin-demo.png new file mode 100644 index 0000000000000000000000000000000000000000..c83fcf236062efe0c50c3deaebb2b169bc8cad1d GIT binary patch literal 18409 zcmb5WbyOTr6g4l>Tzdb8wyZ)-zUAJDJs=Dvi>pC5&q9l!rMUDjk0B~hxB-8+a7f1jA<>)2avxV(# z0VV)|22fGZkbHW2y12OLT|S_vrym&^@$~fE*x0zaxj8>SKRPJv}@;eEaro zd3iZ7xNd)ce|>$u69NJ9@a*mFB_<{o7Z)!tFC&r2?d|Q=m6el|lii)2UHIT0ShVP|3Lya`S+ z=`r@VD<$0WL2|B1UM>6W=Xar|BJr8^rwiQ*BJ zC!nyqs)T6U%*oXiJTWc3^@rC`COEtNFXHOb%QGr%;82d0K0GWkI4L#hr%!o9f7I~9 z+`-jRvuE7YZB=DUROiVrtIG+Wgm7S3&lAz4IGub-}$8 zi{Gq%nnm~L7F2o#CFB$~1^%8(n1p+NEBw&{t?8UAz#&Yf69w3K*pva-#d9xQAjx(;^#ZLV+JZRA7px_4@+%F!kO z0Er@52~iF2g@bHGYYHp!0eS(R`Cs{jBp1Ii7dJ5uXyn?`@f3fx(pIwOSlu zj-MkKB6R+s)+YVN#-O77QdL%NqKi}oGxCqBh}OoCq+gqF)3rtWA%f?`m9hDc&O45O zSZO0enL*!UUWU#Zv>iwMD=Hfbsv0 zF);P3`!{oMvUj1+D>{eccHY;li(2dN%AGwS3=QBtbS-rW>Hih}G2ZA)ku3kV2?LdfRndnHk&kGA75{r$kjEWsy}q0@ zviE^RHtwe%4J3KG*lOI3>|$*4WwC#LCLoK%5n-q+#6T4gVC>RzyT~^?pWCU)N?ub$ z|B_CCeRnirWl>qlY(eosv>_L%Wm+?@Wa9MXQCuo~(u4Wh+pMJsczUy$YbA7hpEKfi zo@<~@Y59(cB&HO`xxmO5iyIad&Qk40_y$Hmrgi2iZ}$%3M?PA-YNhk(p-6k8LrI|j z=nmgg{sx{0LtEQ$kv+x_c%XXdytT5}iNjoH>$1RAvNKj1TYVXD0*XGF6!mty%&Z=Z z+;+Y`^$Uf?1xQY=V-5%_Fztq>!sIogRrSG#Q;kPo89x-0ux(qo*3~zQ5BYdO6GpGd z*#BB{5&Jsw8VY5ey`|M;ol+M6Mp2&nQ}rJ;@K@D}gQgw3Ca~hL&|J#MzPV^RqmEs1 z#IfeFz3msZRwLfL{AzWR@(9;`O^SKK<6+;Tyq3@2j%vQC8bm6;FPHB0;Oi%mRBU3o z>@FzF3&fr;`}7{6$8XI7sq#!MbS8#2j6koJBCop)kCXmtMhh*{lTH0utCC{{qxoAE zLhrIsq3rj3E4g#G&X!=eXM)Xat{EG1@~g-qp_Z#T&9&D(ly!>qOJS);zMWRk__!}C zM=Z@o3*q1P%GG`=T>8M&8HLP>l{8ibPn$}Vp0~_@tQnH0XiRhbS5!I_!aP0IXl=?Z z`Hn!*qmUST2lfW6+QpH2+x+23s77x)gcck#3-=p`T46QHsnS?cBx$e}|68)rt+5)> zD&?(|k5L02Vo5bsy;GY7oZzG)OMr|S*meWz^;dK@CIX*|y)5h!{H$f;>IrXKAwJuQ zk`_Ba4li{+wvNP{oc+(`AcVyS`^3w)*$!8zBA}rmFb9>77WnjFNUWH&+^rSNpJAO& z7b+5|6=+Ia_>YxN%WMn(Ji?sl^9xk_S7W1<;Y9U6>)w}(bSqR>x%SmBxHB!J4rdcH zHD(jwMZOfmP1Beft|q|_QZgdKrIy^Z5A=3mNh&ajU?nU(B4TiGMyP&};t>Oi0kac; z^F+F_$d*MzfAyJAB!z&X|N3U<;ULK}6Pup3hI+wlq@Hfa4u|OQjwShwzNwd-xl@g* z@d1{H@n~x>_O?Qi*9A-(NPH8RMORFJ0Q7ZGlToG|W$U_g^j6n%@n=>rP#?!WE@~Dn z)RG2osB$cfnUqc0vgr<~c@=fm!|Z^F9tXNnQldi4WOo8Re7ad1cxu5n>Xka^1|g74 zqd6yr>pE$p;5*kV;U4C~1cNZVdOIb_eKn23svYZu{&;peM7r$&xRpbvKz%(@C8|-B z>J=TQYIf0Ur&0|4LAVMq6&~*UW{4;|txmNJL+&sKd2Q*fL-Epj|6i=#gtE&Q#HVOa zA6SX=ZWyma{YceHG#^Alv#9hy$ziZLN9hnNM(a*mxT-P)yYCZ3TDfNGIFcfoa@_A; z_&vlfk!vNpu$HBC;)kDN*95^tQM1_T5_WOvbcOrb2(0i0ZMw&sD(9wzBRXX-cvK3K zS9O=Hu1tkeURh5vjpyr;bkQBsz{rb9FEwf=dLwj##lzY6mjSy&StXNK5UNs};5N2; zd?cdofrMkGPwcNQ;u9ywOQlr9gd9OG+bVBq@cw%*=z6tbCp~B%1o{AKpCpl~xk~-* zrzY8e*|C*Az3Akx)oJ7pRv$-U+Wq{q9F38RoSZUMbf1jGuiSN$X>Z(wK|CW1MTW7Vd#9f8#or3wO-u<#Ijn3 zZ8F`<*0L(Bv@8;V5AbHj!^ozp&mik8SDJDBu*6>d;Wr_6?dmQ@qjM|y>XoQZDy&)@ zf#EGz2*rE%JgTRMtC_uOZK2e$&u5|fH2`kZKvLKfve>6Kg@_} zo9}xm)Y-5EgF&d9J5K~sOUp}4(ZzC7hL{E6FenpOx>_xBd~n;VjWnRJIV(}G^(wE2 zF=M@Hb(Q?WD9$#Xe?3WrfMg)8X~mm~IlQwv&YIPxRy^KE{X>lLmGAgslUXLta^E`X zB829)@!wQ_`|@#C#u}*dot2=mXh7VvH9mpGt!bGN{A5iVay(iARV4nvg#8*YaWVkp z*eE_QlvR+csc-Hwgr1p>MCr^@Nr1se()ll;U3P#LGot&R8i5FV!YdF`%;Z`?aFN`LwivhuruC$$rEI_34ns`x+|dLw^j z9%l3#$0W&F#_Y+91;-9;Tt}iT@wmL^cE*OZd}=L<|J62CK`rbN=!_o1Nk20!Od&FH zO;y&S@B*3QPxjK^Eaqt_W=-(Bpa_4*1 zs@y)a`W?18N6>h_*>PyFV<55dwirsyiqufX{ip!Zhch*+B6?!Cv83gA7Kjvw$`A#L z(OA5jzq{^~d4hSk*Ko8jI)Ie*o)!{)b2jg`>U46skM|TO0yeo0L<;sSD8r5gXe<3Q zvx1S`CnyLXaf2TgclW-Sbe;BC-4vM5is2f|SPmbsW=!*}3TjB-sfVSr-&?H$0Lr@8 zM?X0_3~n7!{?ir1f362lI6Cvho^=G^KdFNK|1DMi5B>5uqC~X_c!`Pv=nLE7&i{yV zX*@+#^sSX*V~t@Og;xgU8T9|xbzPW7L{m$qzgz8z_Hkxr99$m*kg;#j?W`;`21H({ zdcmTxX&U3Ho!=r**nF)hr)FF*PTHg5y`>^{8s+N(vsvcKD}#TtXe0-5eS(%py%sZH zLl35}WufN%)c5HF2SWvR67hjG`YcMyrvlaHpf@U!Wdsd4MchGZ*CCE2V`@|8`ixLU z8?nN2{kC>ro)?Q6eLk1x>36b$KBV`{koMf;_N$*x>ODmRCSQ1n6u>wbX8(IdRGzIC zO89O&4J)zCo+VbF*VKcZFfY28T`!E`1tTQ8w&df>0?IV=g~r4`GaD`F?fD!{!lG>~ z8wqsC0T>C3IAw6s3M;`JVcJ`SeKq47tnW%3oyn+wKk?rdSVV|B!xpvDUhtsHhA#dO zcv929S14F>Kx%R01{{}{O5k=yLJMDfG_J#vx<+2duMzWDWhQnwev=%1w7dw}jUGB$ zBQYyG)ulfosZ!4bOpDhe%N8q!`+_l!mtQ}6gBYWYOHBJ2$^>TiT(AWcz^3niBe{rZ z+Fmk18Zwuo`t@E93C1JUD&+Ljk$+w#=A^y+3~^zJV1=iwkk?QJU7WzXEr3>rJfY;Iv2=9(vW>y`Ncq6j6oC|;8>nzQtC=JsM|-rI?KATK)Qkbk=J!#Q(!hB=j_to z#IQjk4~g23n{D9tL4mI&3pnDkG`IGLVV1`LH9M@-fNB7mz_B42%lyDD=cHc2e-uML z(j{oqk|doH;V|_{h+MWQeOprfFsd9<*UTKjb0F_qXK$hFMj|>!h`{fkCegqyw`CS` z#7G>mYDekGsKm(cR|aoQ%SI%-q#NO~T#u^421j`7ltC^nf?}h&=7ea=QD&zDi+YSCf4UDo8 z#>g7W8T)MkZT+>JlQ82wR?R~^Qi-UrY9?_-@0?R!%^Q)PA4akN2b0SneJgPdqEBptc@GfMay(D;hFRaFykfDT& zkPqw4|8(plBsqDd0-42PH5MrLw~NhC8r)*&BSMzW40>u|FBQ&(Z}Fc+AdQ6QhFm1T zW-&oUUPUB4F6a#FRM!))!$l`flBrW7HYy}Nd)lQ-GbAvH2=K1@lN~02cpn$R+C?NK zp++W%Q-ni^17r4Q4Sa^ug=FXyg9mB$aDL1=mxUE$xqqw%lFW8Rs15fGhL2;H-22J*yN4(M?@Ofj;dd)+ z2P(vyETS!;4;|btyLNS@^qzSCaf2Gl@li24QG%bowAyv`a!R_U5WWXHSp$0zc2UioXMH-Q7 z_=96ZH6qu2WvAtK11$-;?JcP9K`ka`zV8$)MlceLltn*Xd=!?L%t__e)d(}7pUi6)O!qRnCGkHYQ=?d zEHp`uXGVXUh4c)FYdf3m`3iOiCRcNVm^5C=WD}|cX-NN%I8ry~CDz55(3W*98WdUb z_}u~7n9|3hQF}b+&!mTUXNHJ=jvV+*ApG`s`>T#oajpuBULR!`?OEP6tMu)eGKi4K zn>FC5zElGX6u**Qax43rVi#X2< zeC(ORg#HjYr%(qqSp(<+^bq`+b|X^$Y)9c$`zM!IZ^q5q-32yYzAaa2Jq&}bzCbiZ zNM}IbQ4}k_6)H4dW{Q9jKJ*mPYru*Ue>fEfx!d@K;0@lJFzYvkU68$z#jmoc($#ptYPp@(HHCb&|~HClL5UYl#!d&G;EM;(B@x5%g1+ zpMb9fr@i_Nvh(i=BY3(0-OLC7*U-WE8a6g})3l|5N6%m|}WXY%Al9qmR{?7~_Redcz{NY~T3 zFQs+!b3I?xaf_7=M8CJHzMA)4G=bB(xt?VcT6U_%Bk1M#&)KKM zuNt6bHjDDgWBMW`_}Xpe$v_%t$S=()em-!Qg9)yh#N$(3vl8lhQAiLhPBm;I)o7tqHmL(u$P% zIi`qwyzXUJ*q5`SF%|He<{Jtar(LFmRHj$FlwNHY0JiA>EW1diX7pU4K*~*IOyHWM z@-|%y#{zW?KwIu_((KN{GJ!Elhz2ma{UjTO9i5V4*G2Z5${)^T6zQW-TtGpM9 zEj-gW84#tdj{K{5jc&`ME1f#+c<)W|GtNWEk6`%w zEw`-hKczZ-abyF{n~RCxgW^8t#tKJ%o-;V)TP&T}b#rKqBQ#tNPR}<4FKzGcwKQ!L zgaw)Y+o_SAsB(Bq=D-|6C_%h0tIUIRT{w(uf|oYdVP}5`XfXyWc2!i$Gd^gm2UjkNOGPPk*A^ zN`Q-r1s_gOG!$>eBe_?-e4rXAA`2a%GBpN=9wQy-EPEP>%uc^JGWi@jiQDmoceKAd zT?MHSm?+~91k$0p9Ci7gd=tU1D!2V;TWgnHGj7LG$$XT!qC=v<_^s;=)lkz;B7tTJ zZ>;C>3BCJIy~(M+ z=iHZ>$fWE_PW24k{M(SJ12$JauJMWgD5oh6&O9-2zLmibz`-NC+I+{QO`vR%*t|lIPg9;a#G5qCCZ9wFEa^lk({uqgHsis?uIg632(8ipi&r*UDPnn!GDI)u;Y@7EQ-mr{}JUnN&)%QZL#>hgpx({5uD__;odF6h!wCwv1bhX5&ia)U|a-4qk{9UW& z*bRv^`LG#Eafm6z|LxGrbgz5>w|2i#i`}_;e{CKA3TvYB(D2U=Vx8i+^#J-|r(fbF z%iIM}#2r38i!OFd4hAbnOeZ_|v{O9rqeBioNDaNc)(`?=)G@o;rPk8l^Q>-t1SU!< zKb2UC*60aB4((=tElZCqLS90cC+1DT6XIfLAbj`n_Fi8ptPG)pJQ_e zI#mXR-LKQ_YASF`2gnb>1J!@xy!LH+Yy*|d=HRDh&Cx2=_`JOD=(39dg2$C=Z2r-N zEir!2HGSUi!Ef+a!e3|f9 z5#(@ZM+}h&ehoWAkhst7l-JcXd zJP`>k9t%4|imeI;Mt+<~_WJP?x_H3_1_xQO=V^{W6Q#9Q7Hq}vfgJvPrFmWjH<;(S zL?Us-@+7WZbFIsr+%&+y7@e5<^=w0m^8>i87R*LT6Z6ro^UtY@!qr>znpYtA`=i#V zJd*2=bRrGDcWvEiIU>pvXj#+%W^HYBX)WuguU2<4Zgj|BTwSg1-bd+IUZ9&%ud9)m zAN4H>wQTqxyAOpo2gW-mW_wWOfdHW7#0N z$VS?Vo3mFt>CF(H$-BQu9($hqfHdznT59x;XrKuU0!TGox-b)Hw@>msTt1V9P+5Vd5=S11uX==57e&t#_sqP022 zILsCvtp0Sr5dO3>xC*g=Yo-jNfOLj_&YCG#{>-oXTdJO9e2|@mK3k zm{&zoxs;)c6pRDX*7eQR49(vBMaPaL1zQk_*9?c%VP*C~&EUzJI1na^VJ7>x;&J)f zTI|RQdxpy2vN)hHBi(*zWnF&NpXWvfsE{71bu4bYwuyziDy9QUFYd^vdZ$#8nstpE z={@SQBaM)|D#%2|pK%xM#9^muP3Kl{!=DCzXr&!#l~)i3#gg@*pB9m4YRhjH=nDWd&`dC)7J6GqfoO+ z#NyBt3Bfql=sWnaoMpHr|Mw90iW!cw2C|g$grm$}NbpBCdB!wSGFX*-fERQ6Z7W9T zOX&pMqzSdM8X#N|TQ%(x-Mr3*qmZu|>S@S@dRYfyKYjYn!_V})yx!c)eQsboBvG}; zR5NfG8`51XMF06Uds~$c3oG%2PGUy-6TE(ks<>R5K#mz5GJ5-{)ItT56QAH1oW!9B zzF*c?giTu56v7f;fP7@3A40h7YYGQ3C{*4*7@C`ZU8pwQ>!a^m7L6l6=&m<%QO+ZQ zW&M@>ty5WZCdRhH^AjpzGBJHxlJ?T__V1O1WD)(6>kfOamol@#HgQ89u=&l-=7Ab- zv#Pp}+Pw1-3JQ7pyUbAF8{5}83+wzZ! zvvrGNH-6gf;B=w3K2iL-8rjydQL{I;e?c8l5C|k*{$U7x3OhtT;3}oq8T`OC7kmg3 z#`{Zy{cn0p&D%wI!8!44+vdwW`zCH_`3TOmWIV>SMK5-SJ&vc;sQ7hqf7%0xJ9`BA z8veWRUEoZG5)v8On_kF^S`T9Jy~O64ea-Njh9q$s&d6}z0w@<6!B?E}c?H>eYZ|($_Nlt8cp{WJGX)j!>h!EkyKym~} ztAcW%ey_%amoFKL1_T<+11d{PNoJDcKdzCa4@?md3~^$paiXwwXTIV#MhT`}{eDD2 z9uuhi^SfA40PU1b4lbGc6jf*}&3#ZaX}3(-`xnz{bl;8Yy0r0OKQKeOalU9?jm3#6 z?kT;-R$_n$mH(mBtqtI>o{Y1Xr132p6v$8_9eQN=?34f62Iox1(5GZEuSn^4&*-jX zV^1jtDP#NS!F!nyg*mr6_BpZ*)w5a+_KGe)8GWK0|REinZCRb>O{@pA!=iCtUpnyxPRaOo6J{qL~X{y zWNxG}F5Z9Tqf<4lt(y;o<@aPsGOa5+%b?v}m-?cw;pLawp}q`+ij!2zcORts2=+!+ z&vfMJR_-uA3h|kD8-oR_XBtna4T~$i`l9p05!TUoT9bF^8>steOOoeMG|qL5jN&7e zp7X-uyiz8Abn;MK^=d#oZ#6{_!&`7l!cH=4{F+I~Fujb4G&5;=n*tG@O*}wb%dD&v zU55C>-JKTDLd|2}xLFV~)W46xO?n{VG|xd&X~gpuMxs7FnBP-p!C{2RsCeu6hs`$c zJR}J1JKaYY77E1Ue%WheMc=-@mSkBL6#J$lkJgQKjWc$_pIPWg{mZ1KZ$`!+EO6qQ zMxZavxphM4=c@Do-a`%8HKsQI-0`1ZoJ>Nr^P(Fe3`F3C^j1NmSq; z9B+^(erqj+B5b12LIL5vtau$%ux0;i^A8&3T7>;g5p*N@vn3@RZ7}ssMV9CnAkDY0 zsCZ3@JkpscVKit)Lw`wiB(hR!>qrT}Nw20p6@7*y%0D?-66S~h4KJTeNv|M3@F81Q zpU#7rf3L$|0#?cmv2jCpOjIbuzsS4*kvc}ulxe?C(0JyGU#?>K-C>-pwxSbfNNX35 z7g`QH%F1H+^aPz`+DbYqLM2vUe}@*6dm#&2Y1xup!_uG7kYw$kQmdoi#Nx^;lgXvK z%!ddT@eBkaInc;P8DA@2B+LS;Ua9E3c926|ax)Y;0xZD=iP4f2DCJOh8asXaf)84t z$p{?&X-;1&r>O%+faHA2$qB$|+H{2hq_!HGrj8RrBU9q@`ScPDcxi>&Vbqnb8cJ|R zW6J^nm^3fA`*jti%U;PBjCW`Y0F9# z>#)W-{~2(0+A0_=FDPkA0Tv2}4y@j!T@Z7|~dd4_ogw_hRM_ z%d>*X|2O3rB+gKoN>bz5LDP{v=q!ZF;uh3%yAk#XCL~gWo1-NBu*g;c9z+FxZD?lC z+=mp(V`2yXMQ(|V2nEd$i8yhD93<6XusP`_7d{V73HR4^G;DB#dniah?Y4DgWW5AE z?TgEC@DjxEn+U@fqp;=07*ry}IoPeGG{Kha$<2$|@sTQ84vzjWfg{V7gUXBjIcLTm zt*r!&R@ja8GhuDpCYy4LjZKqE&`c&TInveoxGUbfB@TeFUc zPBJi=2d>D%;%VwT5hq^mte>j=I~RPGu%2Jo+sc+AK7o%%NzT}=Kpbn+K-7^p^!Fo8 zuZ&pUe>_9$YuRFQrRDMd9?QctgckcCmqbjtv1b9I=~oiQmOM6j#L`4WTv0Ks`fP%7 zVlo;6K!dN)hqYG*O*G?&UqQ|yQmmEUUrAN3V11u+RdFe)sen!^xkzJCzQKDRudCh2 zBvkjO0!Nb?`67$lMTjwrbp^fX!je&e*y1*-;8=4h!BUzGEV(hJuVqn=n!igHCTWpS z0(W;horO9-ypWF0HXmsKtGb6#TSs+;{e4EFd6p%_`z!y8b})t4>QrSs%cp+R=_DA|R#U>q#TbQKy{^3+Pe)UR{>U$s#2#@IIpg zyp<^Pb{=vkpFbl0q6Gt9tG@;TA{Zm60eyrL_yAL^0?gOFT94k6RXUMy6o7t@>0hj8 z``l%2-;cwu(K$0VW}JG*ZGAbSamAlAQA8vhp1zK%P=co3v8YW>v(!r{yRD)6V(sfj zZynWuP1R>h-thTDezx+4qlmn)H!eskXLA}l7g0q@6!P<{-E1mgK()4XfD^xP%hEJOYTjn*E<84R&jj$W$9!J=+e9n_v#zb zXc@)!bEv#T+0E z17p~qP1Yi=ooO51PY^{7dHa|@7PkQkQa{$UPWM!B4cHuxP;R$mf8u2Iw+>X2Z>(WU zh&sCtkExF9R}%NT5clg3&t`O=8D0WOY?4@seQU5zkcrmOd<$Cddo%PfJH-=~;|Gjc zsF3W1_m_>>v+8vaaMf<5DGk#{PZwESk#Ap0vIUK}I#A3TtEJ_QIVkfYY_(JOHpszloMA}x z&BqTxZ=9qJhJW8(d^Qfd=8{zYEQwswe_Q{JTh`7HC@qikYSN4>ZiS<^f*1S)_prsn zrG{V^30K%r)h^^U2Z5WHG7)23%K~f#+7qOZ5074ES>%v74A$AG+#$$GrI6)V7N zjM(+&u_s>YfwbA2UbulqY|10&)?{V_U8D-~CvWvbg7+!ZRUbY+U44<8xjm?Ut!jdIEWT$tpheP}RO& zFIm&d<{UIe2^wQ$Vg!5-zZjn#h;m#MRJt+IuB-=vz=le#7SzkU4kIJ3%Y>2Vat23Z zqL9|3C2t1sPm|Y^%tfmlD()9Cwbs!q9OUp&dT{uogmfNqz)cIi5a^ROEZBd=G_&dl zZ%~X}>7O~H$8%CFM!zpfO2>1mUWhfs0sI++S6=gg$89F%X$x=v5dIpGW&6uP4O-Ar zo|KnepJet`Sd=|m0kT{UfI4wb6KgM!?>ShdgfeD%N2Tw1+;R` zKS52?C=t3K&o&y-4{gyqckSH8DS|woz|!)24j^?_b8i8j3JoR~4;{4 z2>brL|IK%!!r4xJ$`yZlu&vmnx|`me)FX%JWTC@_yLY7){;s9U_CMRFG=3L@Y9+VE z^LKn^`=Yf)8#i}wk}|D|>4xVv(up;lSicw|b56qb?{~J?D(DL7SHWjO+((Uax0^dW zN-@;!j*M8}9d+~X)Ol|yUmXF-K-J#Sk3r8q)U^64Ogs1Guow>xnzp1=`6Z$p)S=?$zO#zV7gpDS|ZLc zpxxsvC7FkaY+BwRXHVYWc;x$VVWx^IQp3*o)79!XpeF9@Su|@v33(Ak_HuxB%%v>w z+K3QDL@yne;w~NSb1tz_{%rw-CCLM(z*{9H6EqkE(6lvt8n@Im-S9A$+K8sDn>5pf zY*c;$yf#dkPQjGMApVc$6aNRzW50#eF3Rt&U6P^!Ue@14tXv6Eg=7Fr_t*h}U)1k^{ICM^ zOF?0LrU1a#l4~+DPH`lFhM&=2D$Jt=xVk*9z-8Q^q;URgo1nj5eD-W?(Y_(ir^N22 z3iYjDgfj0JH>45B4RaDL@TJQA51hi9zmvW68ZNURp-66xE--(o)m}UM&uxyhTP#*C zd|ro@SiSw&$)l*xf#$u)Iw!|@EDNw!E|DrGl0(x z^8*NwMlY9}K%2Xi=sQs6{gi(rAwIc@-7|S&S;Ok9hs*#T-p{jD8ZRUo(H2Nox1UV~ zuv9BKu!Pby4gd$&`@7PA7PJ^ZiK*vHfAYX+GK4nKx4U6UuQb0-VpU zz4tHf@_F38C=&{Q=(TqAGH>;loFv$U5SPYCPe>>a6OwSCy3>kZl`*UA!%sC&u*s$h zy|$(APk0C+*?hUF_>N-FJ^C~FOJOY6y-&aEmx=TK6mlQ)qrt34U+XMdXihp6boR-s ztHr_sjD%6BPq&Lr`WF5%cDC5pe~$a3aW~iG3s>L`JjmsyKt1Ma&)5*)^&4zz2vWYw z*GH4RGXVvVP`ftSI|~KS{3QMePP0G<5Dy1S;GZK2ZKv`5fYKS9#| zm1%WPy5`tj8yU#ENKS34v9d$H4Rd|{nX>6pj=4IoF46EP4B;!?3$J#Hq<(GHI5Me^ zW#v!VRN&Ke!U?Y9YIc35$TVY0sTw-4NLKcX)B=*vRTk=@($!a-nUjf*X~KBG;o=8> zajrU7n9@1oN?VQovdn3X=$*9XZdS4TMy zrB}rij+9CA>Li;Es?QGYXG_-@RCwiV>uC|6zoYaD2%fB5r2%VdYXy2o?<`JMlNc&tY_NciIk{+L`OQ94 z!gY87eMVL*wv05gCDGGwQX;hM5>p~jwR;_Fu5&$3`*S=_MSez7$1@aQevE>nM7{@$ z6h!>b>afn_NKp++pB_J_6q3%#XBy3vWmv}+Coz2=z@Cz>@c(?~?EdTVqpteU#+O*O za}2ztRCXM#UK&J$%4gUQh9$?Z)i?@`h9V7-X;Noz9n*L8tUq`Q2qc6`l(b~i94yZ5 zj^=vf&Tl*3Co=mB$B&wG@acf=$47^b|6dQdojffq3jmxwkB(Q;eeeKT#aG4D9@K%H z9#4zDya4BtmmmO%#VXY89V9O+w@LgdE^2eM5c*C#YkL%XuQK<6Cc5#6nh)Tt;TyJ6 zK>YoAg9tI1!n++Wa)`5Nw%k+Isd`-6br!gE4A=&AvmcUyW4ffcLyd^_%}No;ddgUx z7J_Ns{S}L^ZXL(s3QumiFYw=}2Oev5{0srH!NUx^_hw z($0fWGg@^2aCASkcne6NJb=i9X+Ze`?7ky*XT($&7_KdL)paVi3IW}iVJk%r3TyLF z3hkUfKRbV%+QS(u2o06Q`xloudn`44?i8gS0nRQC8l+lNdbQ;VI|(G@6yScODL>Yf zKHwcWcwFuIbT~MZ(C(=BoFRS1QCt~)$qQ6N(Ut5&Rd~xP@Z`6XZpw`hn*LE}gA|8v z|JP|~PUSqD%Lsw%$bX~3#dxmAbK)s>yqH%YqG1o&v30%w>681U5T;ntY^*p-?3=OR zD9%O^s)pZPF79;tQ1ROS*H1xP@Z(?gZiIE}uaELt$RtO4q%9b|7O(v#*GUoS?^!>; zwBQzMNLg8DXN#*))z3yS5ioPNc*=zh0=`2BC{cpJu~sn-a2uDp^3@V%C0HokS7&t( zPcbLHDbvLDeeKt>&|)-;%B)87{t~M;^Zti|xO}%jUH1FXo6T3#aHc z;cNPL95J0B3EJ-gypa%bw#7V!7@=@ZHf+fKhFZDh9Ybm70`#tECHFGtphZr+Mk}s^ z!WIM=X!e3zP$XUaWV9m&-$+_f&p@`mF3ZEqGp%7*Lc*T^6kCyS>Hcuk?06K))s_5I zZce33g#WM@TbDHuFpCHX&Cz4KbNOljor1irjz7isy&{0^Pz|`;my?>~Z`+U&dNRn` zJ}JnD;F{`kES=jY1o!pTE=babQ}dt+{1bOQ`DaouZEa$|^O}^@Bhnl@NJmHh0~akO zTmNOe=h5p_vECx2354i7?&SkWxgf^ef>Sw4`Jsa$Bcrs-SlP+HYxk-F#kzGy%4yo; z2eF^i+zKp{Z+KZ#Z)f@X<+A`unPLJr9y<6TOJW~x#*{`lV(>WNyUeTposg04P`wl=V%>E1F7S%|!sfJJ%3JeJ65cMRIb? z1Xv54I_OH#Ykll5X6|Z40Z^J>b8!K9HBM}-0nQO?Q_%mF4Uq@ej6+azSox-&a(CDZ zK=wxC-@}nt;`^gE!abCjfZIO1xW#b%M*T2uoirQapgxp9q0@!U%U-A)OIjcrz;B~* zRswm@bdb~1a(jk5qZ>_16$hW1+mrNFVi$XVD1=77s%v?)t7+b<&HmDZX68LLW>dczDWl}Xe zNAAl`?^@iPK7k8eMuPLV4*mC%;C4{iNjF^Yw`NmK34b>F3S-OE_fCgubQ0$#l*6&y zehD!VeA)}mF0hJgyekkll25^Xo4rZ_0t@`=AfaevQ8AsSJv<%G@G;LeWtvFcNNyzw zEB4P2Q43ST!R&1s2drw6DrA}_dr07m%vR+8%x%W6g@S1cq;pjxKW|X6yQRk! zeXHZ8c=O%BzBkuqGsK@Z*^=z@U8u$Ca>Q@3Xt5hV+WLjbZ%75h%$@q7^qSdGYVFlQ z%M3p%YBs@0i{k+j;+W*0+!OvsDq3J=(>T)x4vj)-1~6A}=|6!^z7xaAb=h(Eib$|u zaDw8w`J~H{ME77pDIVi;Mq0MX2VyEwY3-m3cLUuQ6_bg^OI%thW zWX{-~LMwd}{aHJGX~jgO-q83PGc7&yfC)i}ID3^*?~G@V;{@0bIrq}e@22m>e52vC zScgU9;RSHQ@zMuQ6rIDrb(=9SlbCR)rK>BhhH5uiSUZE@X!DPZYYXVPBoOqCw=&3=Y(W1w-escEqfdH?#3GO_un#?Jt1gTq?tbq(kyJ} zoGxyEy$r9tnQH(Z*&DjQ2LswlJekD!3Nyh@u|^m6y_c@N!aR!kqu;lcD<%uE8zOTe zn|^JE%zPUH4mH*8u(p*-xB8e?;yiNK5D!}vv;2PB^T~(7DB}E>><48xOyo!-0q)gc zTNk7?AHk6|Da4hIijV<3BK=++u{OL)vHszo=qIqnS0082n>Gm5#n8M&AbxB*_2VyU zbCn*kA$^Dx|DX!irwin8<*;ZNkPV$qi5ISXr5J`xinz7gZoqfrWdOZ;4D)0SU4 z{ehW9O&Eo-nD6rBRRN~+HP`cc`yr`wxyJ$?+XRT@dLO#nC2%z;solN$NeO@x)PNj? z9qsPA@;a2-s&8I!eFVS>Yq;_LSEl-Zs;#}dJWAvC;Evt&_GuCTIU7m6Oc#rHbWx#f zllYu<_{XCz9^r^7k(%O}(D^;qa6y!$$If1uz6&q3nf|{*Bv=-|#)HN0;iEJ99z`?Ix)1s{>zJn+&*V8Xd7zYB>?)Yo-s9N!wVpM*v}nCT*)^m zBBipw0%ME}A!>(hwi<6Cd95e*ZqVNlJz>(%bV0DDxWo(2$iu2+a?kN#P!;E z^*{ZbaYE+O8(qL_h}c0)P6kmE5oq@_>3AEDz4CIUDmkcn^>$%5c1fs64Niw>A% z?i+w3cCXvE=l`8t7XLat`0b?b?*{}c&vJ9h)mgR+T~3@H*}cw&wvOcS_LJ|5iPIw$xN0h3WZB4C!xtPx)A(SnmVWj z{jj6=5j-WiBG#h0ir4Fw@9_Hh04kdRHVMb`Aih?|xflb`fS*9>IL}#ph%1A21K}hZ zDT^i^&Kq>O5ZZ!J69YjNydTle{~G|zt)N~XE*ZGeqIElM*Nrw;0lM>-lWum*0lY2Z z(rgd8O#l@w%(#_9o8<%Um?;<3^T@F7W?;>^PTaT*b;6{X3Sl5BB|o_Qp?N!gN)(ZzJ{wz6~l7)Mx6X4m7RJ$ zIl`F&K>Li|-~n}i0O7{eq`q*TLWmG10Wf-HXU#xUo_ZtEN2cIPvR*jxxXReL9?qRn z>)WG;-+k)G58lU2x~ONdBZl1PrAsLSZ_Ji?z)rVD99TjDhqf86-p2#U`f;*@fY{?_ zV0=!c+^L5wNhdcI-TK=LtH~+2MZh{P=)r@{oRA~1P{0w1{36YA>t6?%4p$yFlppDE z427fcwuwu!Ew1?gFj{>aS$X*0;4|Pt8Nh@K3qPyE!4gRN*ceb!+=EdYk8>rcUqr+1 zV(+Lrm>)jYjdu}GDfquNZm-c$r*Ry>Q5q@P79`Dr+;8J<$Yt448G~jTV@46ioeVNA?VtAS*?qUq>*qY@dGTC6=leWA zT@q7C=d9j6k)sm(#O&>Kmn9rF3{$%+?p7Vw2$fYKLgvaHKHeM~TZMi1W~xrbe;cMI zNdu{1I{92jVeyl&UrU-C5v`U=b@h-Mwwya7zj-MT59OP0XMI$6oktH--rq7%j6V{(BrI z&UgD!hT(%VlEA(mW5(Bd7ARIuTkmE~I!;3bdb;P5Sux^kpdQ^aX7=@XZiPRw-Z^Y{ zcTbP#@im4iSo{3wiigq8A1ZTYu{Mk4T75T=a6g;{wV!;Kho@SQqE6SlN9;+E?WpBE z_z)e0X*$7KvE!38voB`QY;p`g6s7x3zAtV_LPcaeBo-nWDcijwwzTyq z*LY5us@Jr#pa{&@Ulok!8zz|wD@{FI8cS`B{3QdOEm0~qbM=EuS$-+hDZJu&$@?}P z&0~$lk+TCcsX@ausqq$@D~2{T^Hu{abDy-}jN#qCg{%HEWFfZ>pQSX_N_w-KWe&vMY77@cvSNWprDC-FP*;A!OlmPtW>upa_1aT% z)Tn{8l<#^P7)dGBaM8B4SY-L!7qdECW|`dMPJGMdn;S3gNU@Lt=%EWz{d}cY8VH>2 zRfiR>IQf%WtUK>ecXXGANVLS7=-bL7=5DAa3@TCWWdh<+7{sF*ki$&y!O0@5zJf0> z{vaNiKOy>#GuSYMD6`MrT)Kp}lzf1MV#y>&nohH`FTJzI zy;=>4vzG&I6J42A$}K@Df;aCxVq*gWeXcbssldCr3SAni@>wB~D~i^*5i>8f!d3#i z;una_d^uhG{76Bt&;gyqLsOwS%vA%=)VBX*b2u0 literal 0 HcmV?d00001