From 7789843bd739b8c9d8774403f1abb9103328ae1b Mon Sep 17 00:00:00 2001 From: Balazs Szekeres Date: Fri, 6 Mar 2020 11:50:39 +0100 Subject: [PATCH] Added affinity-mode tc and refactored affinity.go --- test/e2e/annotations/affinity.go | 72 +++++------ test/e2e/annotations/affinitymode.go | 173 +++++++++++++++++++++++++++ test/e2e/framework/k8s.go | 16 ++- 3 files changed, 215 insertions(+), 46 deletions(-) create mode 100644 test/e2e/annotations/affinitymode.go diff --git a/test/e2e/annotations/affinity.go b/test/e2e/annotations/affinity.go index 80808a6b8..50c8ec0ea 100644 --- a/test/e2e/annotations/affinity.go +++ b/test/e2e/annotations/affinity.go @@ -40,10 +40,9 @@ var _ = framework.DescribeAnnotation("affinity session-cookie-name", func() { ginkgo.It("should set sticky cookie SERVERID", func() { host := "sticky.foo.com" - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/affinity": "cookie", - "nginx.ingress.kubernetes.io/session-cookie-name": "SERVERID", - } + annotations := make(map[string]string) + annotations["nginx.ingress.kubernetes.io/affinity"] = "cookie" + annotations["nginx.ingress.kubernetes.io/session-cookie-name"] = "SERVERID" ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations) f.EnsureIngress(ing) @@ -63,10 +62,9 @@ var _ = framework.DescribeAnnotation("affinity session-cookie-name", func() { ginkgo.It("should change cookie name on ingress definition change", func() { host := "change.foo.com" - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/affinity": "cookie", - "nginx.ingress.kubernetes.io/session-cookie-name": "SERVERID", - } + annotations := make(map[string]string) + annotations["nginx.ingress.kubernetes.io/affinity"] = "cookie" + annotations["nginx.ingress.kubernetes.io/session-cookie-name"] = "SERVERID" ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations) f.EnsureIngress(ing) @@ -99,10 +97,9 @@ var _ = framework.DescribeAnnotation("affinity session-cookie-name", func() { ginkgo.It("should set the path to /something on the generated cookie", func() { host := "path.foo.com" - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/affinity": "cookie", - "nginx.ingress.kubernetes.io/session-cookie-name": "SERVERID", - } + annotations := make(map[string]string) + annotations["nginx.ingress.kubernetes.io/affinity"] = "cookie" + annotations["nginx.ingress.kubernetes.io/session-cookie-name"] = "SERVERID" ing := framework.NewSingleIngress(host, "/something", host, f.Namespace, framework.EchoService, 80, annotations) f.EnsureIngress(ing) @@ -122,10 +119,9 @@ var _ = framework.DescribeAnnotation("affinity session-cookie-name", func() { ginkgo.It("does not set the path to / on the generated cookie if there's more than one rule referring to the same backend", func() { host := "morethanonerule.foo.com" - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/affinity": "cookie", - "nginx.ingress.kubernetes.io/session-cookie-name": "SERVERID", - } + annotations := make(map[string]string) + annotations["nginx.ingress.kubernetes.io/affinity"] = "cookie" + annotations["nginx.ingress.kubernetes.io/session-cookie-name"] = "SERVERID" f.EnsureIngress(&networking.Ingress{ ObjectMeta: metav1.ObjectMeta{ @@ -184,12 +180,11 @@ var _ = framework.DescribeAnnotation("affinity session-cookie-name", func() { ginkgo.It("should set cookie with expires", func() { host := "cookieexpires.foo.com" - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/affinity": "cookie", - "nginx.ingress.kubernetes.io/session-cookie-name": "ExpiresCookie", - "nginx.ingress.kubernetes.io/session-cookie-expires": "172800", - "nginx.ingress.kubernetes.io/session-cookie-max-age": "259200", - } + annotations := make(map[string]string) + annotations["nginx.ingress.kubernetes.io/affinity"] = "cookie" + annotations["nginx.ingress.kubernetes.io/session-cookie-name"] = "ExpiresCookie" + annotations["nginx.ingress.kubernetes.io/session-cookie-expires"] = "172800" + annotations["nginx.ingress.kubernetes.io/session-cookie-max-age"] = "259200" ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations) f.EnsureIngress(ing) @@ -216,12 +211,11 @@ var _ = framework.DescribeAnnotation("affinity session-cookie-name", func() { ginkgo.It("should work with use-regex annotation and session-cookie-path", func() { host := "useregex.foo.com" - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/affinity": "cookie", - "nginx.ingress.kubernetes.io/session-cookie-name": "SERVERID", - "nginx.ingress.kubernetes.io/use-regex": "true", - "nginx.ingress.kubernetes.io/session-cookie-path": "/foo/bar", - } + annotations := make(map[string]string) + annotations["nginx.ingress.kubernetes.io/affinity"] = "cookie" + annotations["nginx.ingress.kubernetes.io/session-cookie-name"] = "SERVERID" + annotations["nginx.ingress.kubernetes.io/use-regex"] = "true" + annotations["nginx.ingress.kubernetes.io/session-cookie-path"] = "/foo/bar" ing := framework.NewSingleIngress(host, "/foo/.*", host, f.Namespace, framework.EchoService, 80, annotations) f.EnsureIngress(ing) @@ -241,11 +235,10 @@ var _ = framework.DescribeAnnotation("affinity session-cookie-name", func() { ginkgo.It("should warn user when use-regex is true and session-cookie-path is not set", func() { host := "useregexwarn.foo.com" - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/affinity": "cookie", - "nginx.ingress.kubernetes.io/session-cookie-name": "SERVERID", - "nginx.ingress.kubernetes.io/use-regex": "true", - } + annotations := make(map[string]string) + annotations["nginx.ingress.kubernetes.io/affinity"] = "cookie" + annotations["nginx.ingress.kubernetes.io/session-cookie-name"] = "SERVERID" + annotations["nginx.ingress.kubernetes.io/use-regex"] = "true" ing := framework.NewSingleIngress(host, "/foo/.*", host, f.Namespace, framework.EchoService, 80, annotations) f.EnsureIngress(ing) @@ -269,9 +262,9 @@ var _ = framework.DescribeAnnotation("affinity session-cookie-name", func() { ginkgo.It("should not set affinity across all server locations when using separate ingresses", func() { host := "separate.foo.com" - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/affinity": "cookie", - } + annotations := make(map[string]string) + annotations["nginx.ingress.kubernetes.io/affinity"] = "cookie" + ing1 := framework.NewSingleIngress("ingress1", "/foo/bar", host, f.Namespace, framework.EchoService, 80, annotations) f.EnsureIngress(ing1) @@ -299,10 +292,9 @@ var _ = framework.DescribeAnnotation("affinity session-cookie-name", func() { }) ginkgo.It("should set sticky cookie without host", func() { - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/affinity": "cookie", - "nginx.ingress.kubernetes.io/session-cookie-name": "SERVERID", - } + annotations := make(map[string]string) + annotations["nginx.ingress.kubernetes.io/affinity"] = "cookie" + annotations["nginx.ingress.kubernetes.io/session-cookie-name"] = "SERVERID" ing := framework.NewSingleIngress("default-no-host", "/", "", f.Namespace, framework.EchoService, 80, annotations) f.EnsureIngress(ing) diff --git a/test/e2e/annotations/affinitymode.go b/test/e2e/annotations/affinitymode.go new file mode 100644 index 000000000..548b33836 --- /dev/null +++ b/test/e2e/annotations/affinitymode.go @@ -0,0 +1,173 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package annotations + +import ( + "fmt" + "net/http" + "reflect" + "strings" + "time" + + "github.com/onsi/ginkgo" + "github.com/stretchr/testify/assert" + + "k8s.io/ingress-nginx/test/e2e/framework" +) + +var _ = framework.DescribeAnnotation("affinitymode", func() { + f := framework.NewDefaultFramework("affinity") + + ginkgo.It("Balanced affinity mode should balance", func() { + deploymentName := "affinitybalanceecho" + replicas := 5 + f.NewEchoDeploymentWithNameAndReplicas(deploymentName, replicas) + + host := "affinity-mode-balance.com" + annotations := make(map[string]string) + annotations["nginx.ingress.kubernetes.io/affinity"] = "cookie" + annotations["ginx.ingress.kubernetes.io/session-cookie-name"] = "hello-cookie" + annotations["nginx.ingress.kubernetes.io/session-cookie-expires"] = "172800" + annotations["nginx.ingress.kubernetes.io/session-cookie-max-age"] = "172800" + annotations["nginx.ingress.kubernetes.io/ssl-redirect"] = "false" + annotations["nginx.ingress.kubernetes.io/affinity-mode"] = "balanced" + annotations["nginx.ingress.kubernetes.io/session-cookie-hash"] = "sha1" + + ing := framework.NewSingleIngress(host, "/", host, f.Namespace, deploymentName, 80, annotations) + f.EnsureIngress(ing) + + f.WaitForNginxServer(host, + func(server string) bool { + return strings.Contains(server, fmt.Sprintf("server_name %s ;", host)) + }) + + // Check configuration + ingress := f.GetIngress(f.Namespace, host) + returnedAnnotations := ingress.GetAnnotations() + isItEqual := reflect.DeepEqual(annotations, returnedAnnotations) + assert.Equal(ginkgo.GinkgoT(), isItEqual, true) + }) + + ginkgo.It("Check persistent affinity mode", func() { + deploymentName := "affinitypersistentecho" + replicas := 5 + f.NewEchoDeploymentWithNameAndReplicas(deploymentName, replicas) + + host := "affinity-mode-persistent.com" + annotations := make(map[string]string) + annotations["nginx.ingress.kubernetes.io/affinity"] = "cookie" + annotations["ginx.ingress.kubernetes.io/session-cookie-name"] = "hello-cookie" + annotations["nginx.ingress.kubernetes.io/session-cookie-expires"] = "172800" + annotations["nginx.ingress.kubernetes.io/session-cookie-max-age"] = "172800" + annotations["nginx.ingress.kubernetes.io/ssl-redirect"] = "false" + annotations["nginx.ingress.kubernetes.io/affinity-mode"] = "persistent" + annotations["nginx.ingress.kubernetes.io/session-cookie-hash"] = "sha1" + + ing := framework.NewSingleIngress(host, "/", host, f.Namespace, deploymentName, 80, annotations) + f.EnsureIngress(ing) + + f.WaitForNginxServer(host, + func(server string) bool { + return strings.Contains(server, fmt.Sprintf("server_name %s ;", host)) + }) + + // Check configuration + ingress := f.GetIngress(f.Namespace, host) + returnedAnnotations := ingress.GetAnnotations() + isItEqual := reflect.DeepEqual(annotations, returnedAnnotations) + assert.Equal(ginkgo.GinkgoT(), isItEqual, true) + + // Make a request + request := f.HTTPTestClient().GET("/").WithHeader("Host", host) + response := request.Expect() + + // Get the responder host name + originalHostName := getHostnameFromResponseBody(response.Body().Raw()) + + // Send new requests and add new backends. Check which backend responded to the sent request + cookies := getCookiesFromHeader(response.Header("Set-Cookie").Raw()) + for sendRequestNumber := 0; sendRequestNumber < 10; sendRequestNumber++ { + replicas = replicas + 1 + err := framework.UpdateDeployment(f.KubeClientSet, f.Namespace, deploymentName, replicas, nil) + assert.Nil(ginkgo.GinkgoT(), err) + time.Sleep(3 * time.Second) + response = request.WithCookies(cookies).Expect() + newHostName := getHostnameFromResponseBody(response.Body().Raw()) + assert.Equal(ginkgo.GinkgoT(), originalHostName, newHostName, + fmt.Sprintf("Response number %v is not from the same host. Original host: %s, response returned: %s", sendRequestNumber, originalHostName, newHostName)) + + } + + // remove all backends + replicas = 0 + err := framework.UpdateDeployment(f.KubeClientSet, f.Namespace, deploymentName, replicas, nil) + assert.Nil(ginkgo.GinkgoT(), err) + time.Sleep(5 * time.Second) + + // validate, there is no backend to serve the request + response = request.WithCookies(cookies).Expect().Status(http.StatusServiceUnavailable) + + // create brand new backends + replicas = 2 + err = framework.UpdateDeployment(f.KubeClientSet, f.Namespace, deploymentName, replicas, nil) + assert.Nil(ginkgo.GinkgoT(), err) + time.Sleep(5 * time.Second) + + // wait brand new backends to spawn + response = request.WithCookies(cookies).Expect() + try := 0 + for (response.Raw().StatusCode == http.StatusServiceUnavailable) && (try < 30) { + time.Sleep(5 * time.Second) + response = request.WithCookies(cookies).Expect() + try++ + } + assert.LessOrEqual(ginkgo.GinkgoT(), try, 29, "Tries reached it's maximum, backends did not deployed in time.") + + // brand new backends equals new hostname + newHostName := getHostnameFromResponseBody(response.Body().Raw()) + assert.NotEqual(ginkgo.GinkgoT(), originalHostName, newHostName, + fmt.Sprintf("Response is from the same host (That should not be possible). Original host: %s, response returned: %s", originalHostName, newHostName)) + }) +}) + +func getHostnameFromResponseBody(rawResponseBody string) string { + lines := strings.Split(strings.TrimSpace(rawResponseBody), "\n") + for _, line := range lines { + if strings.Contains(line, "Hostname") { + hostnameParts := strings.Split(strings.TrimSpace(line), ":") + if len(hostnameParts) == 2 { + return strings.TrimSpace(hostnameParts[1]) + } + return "" + } + } + return "" +} + +func getCookiesFromHeader(rawheader string) map[string]string { + cookies := make(map[string]string) + parts := strings.Split(strings.TrimSpace(rawheader), ";") + for _, part := range parts { + subparts := strings.Split(strings.TrimSpace(part), "=") + if len(subparts) == 2 { + cookies[subparts[0]] = subparts[1] + } else { + cookies[subparts[0]] = "" + } + } + return cookies +} diff --git a/test/e2e/framework/k8s.go b/test/e2e/framework/k8s.go index cff5f7f2f..51322f064 100644 --- a/test/e2e/framework/k8s.go +++ b/test/e2e/framework/k8s.go @@ -61,14 +61,20 @@ func (f *Framework) EnsureConfigMap(configMap *api.ConfigMap) (*api.ConfigMap, e return cm, nil } +// GetIngress gets an Ingress object from the given namespace, name and retunrs it, throws error if it does not exists. +func (f *Framework) GetIngress(namespace string, name string) *networking.Ingress { + ing, err := f.KubeClientSet.NetworkingV1beta1().Ingresses(namespace).Get(name, metav1.GetOptions{}) + assert.Nil(ginkgo.GinkgoT(), err, "getting ingress") + assert.NotNil(ginkgo.GinkgoT(), ing, "expected an ingress but none returned") + return ing +} + // EnsureIngress creates an Ingress object and retunrs it, throws error if it already exists. func (f *Framework) EnsureIngress(ingress *networking.Ingress) *networking.Ingress { err := createIngressWithRetries(f.KubeClientSet, f.Namespace, ingress) assert.Nil(ginkgo.GinkgoT(), err, "creating ingress") - ing, err := f.KubeClientSet.NetworkingV1beta1().Ingresses(f.Namespace).Get(ingress.Name, metav1.GetOptions{}) - assert.Nil(ginkgo.GinkgoT(), err, "getting ingress") - assert.NotNil(ginkgo.GinkgoT(), ing, "expected an ingress but none returned") + ing := f.GetIngress(f.Namespace, ingress.Name) if ing.Annotations == nil { ing.Annotations = make(map[string]string) @@ -85,9 +91,7 @@ func (f *Framework) UpdateIngress(ingress *networking.Ingress) *networking.Ingre err := updateIngressWithRetries(f.KubeClientSet, f.Namespace, ingress) assert.Nil(ginkgo.GinkgoT(), err, "updating ingress") - ing, err := f.KubeClientSet.NetworkingV1beta1().Ingresses(f.Namespace).Get(ingress.Name, metav1.GetOptions{}) - assert.Nil(ginkgo.GinkgoT(), err, "getting ingress") - assert.NotNil(ginkgo.GinkgoT(), ing, "expected an ingress but none returned") + ing := f.GetIngress(f.Namespace, ingress.Name) if ing.Annotations == nil { ing.Annotations = make(map[string]string)