From d637b807cf9d66421da5b17db28943050af66e6c Mon Sep 17 00:00:00 2001 From: Fernando Diaz Date: Tue, 30 Jul 2019 14:43:13 -0500 Subject: [PATCH] Allow Requests to be Mirrored to different backends Add a feature which allows traffic to be mirrored to additional backends. This is useful for testing how requests will behave on different "test" backends. See https://nginx.org/en/docs/http/ngx_http_mirror_module.html --- .../nginx-configuration/annotations.md | 33 +++++++ internal/ingress/annotations/annotations.go | 3 + internal/ingress/annotations/mirror/main.go | 58 +++++++++++++ .../ingress/annotations/mirror/main_test.go | 86 +++++++++++++++++++ internal/ingress/controller/controller.go | 1 + internal/ingress/types.go | 4 + internal/ingress/types_equals.go | 8 ++ rootfs/etc/nginx/template/nginx.tmpl | 6 +- test/e2e/annotations/mirror.go | 66 ++++++++++++++ 9 files changed, 264 insertions(+), 1 deletion(-) create mode 100644 internal/ingress/annotations/mirror/main.go create mode 100644 internal/ingress/annotations/mirror/main_test.go create mode 100644 test/e2e/annotations/mirror.go diff --git a/docs/user-guide/nginx-configuration/annotations.md b/docs/user-guide/nginx-configuration/annotations.md index 574117abe..9aff4eeb2 100755 --- a/docs/user-guide/nginx-configuration/annotations.md +++ b/docs/user-guide/nginx-configuration/annotations.md @@ -107,6 +107,8 @@ You can add these Kubernetes annotations to specific Ingress objects to customiz |[nginx.ingress.kubernetes.io/enable-owasp-core-rules](#modsecurity)|bool| |[nginx.ingress.kubernetes.io/modsecurity-transaction-id](#modsecurity)|string| |[nginx.ingress.kubernetes.io/modsecurity-snippet](#modsecurity)|string| +|[nginx.ingress.kubernetes.io/mirror-uri](#mirror)|string| +|[nginx.ingress.kubernetes.io/mirror-request-body](#mirror)|string| ### Canary @@ -797,3 +799,34 @@ By default, a request would need to satisfy all authentication requirements in o ```yaml nginx.ingress.kubernetes.io/satisfy: "any" ``` + +### Mirror + +Enables a request to be mirrored to a mirror backend. Responses by mirror backends are ignored. This feature is useful, to see how requests will react in "test" backends. + +You can mirror a request to the `/mirror` path on your ingress, by applying the below: + +```yaml +nginx.ingress.kubernetes.io/mirror-uri: "/mirror" +``` + +The mirror path can be defined as a separate ingress resource: + +``` +location = /mirror { + internal; + proxy_pass http://test_backend; +} +``` + +By default the request-body is sent to the mirror backend, but can be turned off by applying: + +```yaml +nginx.ingress.kubernetes.io/mirror-request-body: "off" +``` + +**Note:** The mirror directive will be applied to all paths within the ingress resource. + +The request sent to the mirror is linked to the orignial request. If you have a slow mirror backend, then the orignial request will throttle. + +For more information on the mirror module see https://nginx.org/en/docs/http/ngx_http_mirror_module.html diff --git a/internal/ingress/annotations/annotations.go b/internal/ingress/annotations/annotations.go index 00080c840..deac08aa4 100644 --- a/internal/ingress/annotations/annotations.go +++ b/internal/ingress/annotations/annotations.go @@ -44,6 +44,7 @@ import ( "k8s.io/ingress-nginx/internal/ingress/annotations/loadbalancing" "k8s.io/ingress-nginx/internal/ingress/annotations/log" "k8s.io/ingress-nginx/internal/ingress/annotations/luarestywaf" + "k8s.io/ingress-nginx/internal/ingress/annotations/mirror" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" "k8s.io/ingress-nginx/internal/ingress/annotations/portinredirect" "k8s.io/ingress-nginx/internal/ingress/annotations/proxy" @@ -107,6 +108,7 @@ type Ingress struct { LuaRestyWAF luarestywaf.Config InfluxDB influxdb.Config ModSecurity modsecurity.Config + Mirror mirror.Config } // Extractor defines the annotation parsers to be used in the extraction of annotations @@ -153,6 +155,7 @@ func NewAnnotationExtractor(cfg resolver.Resolver) Extractor { "InfluxDB": influxdb.NewParser(cfg), "BackendProtocol": backendprotocol.NewParser(cfg), "ModSecurity": modsecurity.NewParser(cfg), + "Mirror": mirror.NewParser(cfg), }, } } diff --git a/internal/ingress/annotations/mirror/main.go b/internal/ingress/annotations/mirror/main.go new file mode 100644 index 000000000..67a4367d9 --- /dev/null +++ b/internal/ingress/annotations/mirror/main.go @@ -0,0 +1,58 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirror + +import ( + networking "k8s.io/api/networking/v1beta1" + + "k8s.io/ingress-nginx/internal/ingress/annotations/parser" + "k8s.io/ingress-nginx/internal/ingress/resolver" +) + +// Config returns the mirror to use in a given location +type Config struct { + URI string `json:"uri"` + RequestBody string `json:"requestBody"` +} + +type mirror struct { + r resolver.Resolver +} + +// NewParser creates a new mirror configuration annotation parser +func NewParser(r resolver.Resolver) parser.IngressAnnotation { + return mirror{r} +} + +// ParseAnnotations parses the annotations contained in the ingress +// rule used to configure mirror +func (a mirror) Parse(ing *networking.Ingress) (interface{}, error) { + config := &Config{} + var err error + + config.URI, err = parser.GetStringAnnotation("mirror-uri", ing) + if err != nil { + config.URI = "" + } + + config.RequestBody, err = parser.GetStringAnnotation("mirror-request-body", ing) + if err != nil || config.RequestBody != "off" { + config.RequestBody = "on" + } + + return config, nil +} diff --git a/internal/ingress/annotations/mirror/main_test.go b/internal/ingress/annotations/mirror/main_test.go new file mode 100644 index 000000000..e9bae8786 --- /dev/null +++ b/internal/ingress/annotations/mirror/main_test.go @@ -0,0 +1,86 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirror + +import ( + "fmt" + "reflect" + "testing" + + api "k8s.io/api/core/v1" + networking "k8s.io/api/networking/v1beta1" + meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/ingress-nginx/internal/ingress/annotations/parser" + "k8s.io/ingress-nginx/internal/ingress/resolver" +) + +func TestParse(t *testing.T) { + uri := parser.GetAnnotationWithPrefix("mirror-uri") + requestBody := parser.GetAnnotationWithPrefix("mirror-request-body") + + ap := NewParser(&resolver.Mock{}) + if ap == nil { + t.Fatalf("expected a parser.IngressAnnotation but returned nil") + } + + testCases := []struct { + annotations map[string]string + expected *Config + }{ + {map[string]string{uri: "/mirror", requestBody: ""}, &Config{ + URI: "/mirror", + RequestBody: "on", + }}, + {map[string]string{uri: "/mirror", requestBody: "off"}, &Config{ + URI: "/mirror", + RequestBody: "off", + }}, + {map[string]string{uri: "", requestBody: "ahh"}, &Config{ + URI: "", + RequestBody: "on", + }}, + {map[string]string{uri: "", requestBody: ""}, &Config{ + URI: "", + RequestBody: "on", + }}, + {map[string]string{}, &Config{ + URI: "", + RequestBody: "on", + }}, + {nil, &Config{ + URI: "", + RequestBody: "on", + }}, + } + + ing := &networking.Ingress{ + ObjectMeta: meta_v1.ObjectMeta{ + Name: "foo", + Namespace: api.NamespaceDefault, + }, + Spec: networking.IngressSpec{}, + } + + for _, testCase := range testCases { + ing.SetAnnotations(testCase.annotations) + result, _ := ap.Parse(ing) + fmt.Printf("%t", result) + if !reflect.DeepEqual(result, testCase.expected) { + t.Errorf("expected %v but returned %v, annotations: %s", testCase.expected, result, testCase.annotations) + } + } +} diff --git a/internal/ingress/controller/controller.go b/internal/ingress/controller/controller.go index e37e08553..30eb6074b 100644 --- a/internal/ingress/controller/controller.go +++ b/internal/ingress/controller/controller.go @@ -1168,6 +1168,7 @@ func locationApplyAnnotations(loc *ingress.Location, anns *annotations.Ingress) loc.CustomHTTPErrors = anns.CustomHTTPErrors loc.ModSecurity = anns.ModSecurity loc.Satisfy = anns.Satisfy + loc.Mirror = anns.Mirror } // OK to merge canary ingresses iff there exists one or more ingresses to potentially merge into diff --git a/internal/ingress/types.go b/internal/ingress/types.go index accc89c59..d1f29c184 100644 --- a/internal/ingress/types.go +++ b/internal/ingress/types.go @@ -31,6 +31,7 @@ import ( "k8s.io/ingress-nginx/internal/ingress/annotations/ipwhitelist" "k8s.io/ingress-nginx/internal/ingress/annotations/log" "k8s.io/ingress-nginx/internal/ingress/annotations/luarestywaf" + "k8s.io/ingress-nginx/internal/ingress/annotations/mirror" "k8s.io/ingress-nginx/internal/ingress/annotations/modsecurity" "k8s.io/ingress-nginx/internal/ingress/annotations/proxy" "k8s.io/ingress-nginx/internal/ingress/annotations/ratelimit" @@ -316,6 +317,9 @@ type Location struct { ModSecurity modsecurity.Config `json:"modsecurity"` // Satisfy dictates allow access if any or all is set Satisfy string `json:"satisfy"` + // Mirror allows you to mirror traffic to a "test" backend + // +optional + Mirror mirror.Config `json:"mirror,omitempty"` } // SSLPassthroughBackend describes a SSL upstream server configured diff --git a/internal/ingress/types_equals.go b/internal/ingress/types_equals.go index b2813efed..b137b6f00 100644 --- a/internal/ingress/types_equals.go +++ b/internal/ingress/types_equals.go @@ -418,6 +418,14 @@ func (l1 *Location) Equal(l2 *Location) bool { return false } + if l1.Mirror.URI != l2.Mirror.URI { + return false + } + + if l1.Mirror.RequestBody != l2.Mirror.RequestBody { + return false + } + return true } diff --git a/rootfs/etc/nginx/template/nginx.tmpl b/rootfs/etc/nginx/template/nginx.tmpl index 619ca4a7f..655fadffa 100755 --- a/rootfs/etc/nginx/template/nginx.tmpl +++ b/rootfs/etc/nginx/template/nginx.tmpl @@ -999,6 +999,11 @@ stream { {{ opentracingPropagateContext $location }}; {{ end }} + {{ if $location.Mirror.URI }} + mirror {{ $location.Mirror.URI }}; + mirror_request_body {{ $location.Mirror.RequestBody }}; + {{ end }} + rewrite_by_lua_block { lua_ingress.rewrite({{ locationConfigForLua $location $server $all }}) balancer.rewrite() @@ -1091,7 +1096,6 @@ stream { } {{ end }} - {{ if not $location.Logs.Access }} access_log off; {{ end }} diff --git a/test/e2e/annotations/mirror.go b/test/e2e/annotations/mirror.go new file mode 100644 index 000000000..efb34e07e --- /dev/null +++ b/test/e2e/annotations/mirror.go @@ -0,0 +1,66 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package annotations + +import ( + "strings" + + . "github.com/onsi/ginkgo" + + "k8s.io/ingress-nginx/test/e2e/framework" +) + +var _ = framework.IngressNginxDescribe("Annotations - Mirror", func() { + f := framework.NewDefaultFramework("mirror") + host := "mirror.foo.com" + + BeforeEach(func() { + f.NewEchoDeployment() + }) + + AfterEach(func() { + }) + + It("should set mirror-uri to /mirror", func() { + annotations := map[string]string{ + "nginx.ingress.kubernetes.io/mirror-uri": "/mirror", + } + + ing := framework.NewSingleIngress(host, "/", host, f.Namespace, "http-svc", 80, &annotations) + f.EnsureIngress(ing) + + f.WaitForNginxServer(host, + func(server string) bool { + return strings.Contains(server, "mirror /mirror;") && strings.Contains(server, "mirror_request_body on;") + }) + }) + + It("should disable mirror-request-body", func() { + annotations := map[string]string{ + "nginx.ingress.kubernetes.io/mirror-uri": "/mirror", + "nginx.ingress.kubernetes.io/mirror-request-body": "off", + } + + ing := framework.NewSingleIngress(host, "/", host, f.Namespace, "http-svc", 80, &annotations) + f.EnsureIngress(ing) + + f.WaitForNginxServer(host, + func(server string) bool { + return strings.Contains(server, "mirror /mirror;") && strings.Contains(server, "mirror_request_body off;") + }) + }) +})