DevFW-CICD/ingress-nginx-helm
15
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #5217 from SzekeresB/dev/affinity

Browse source 
Added affinity-mode tc and refactored affinity.go
This commit is contained in:
Kubernetes Prow Robot 2020-03-07 15:05:34 -08:00 committed by GitHub
commit 0c9db55265
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 215 additions and 46 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

72
test/e2e/annotations/affinity.go
View file

@ -40,10 +40,9 @@ var _ = framework.DescribeAnnotation("affinity session-cookie-name", func() {
ginkgo.It("should set sticky cookie SERVERID", func() {
host := "sticky.foo.com"
annotations := map[string]string{
"nginx.ingress.kubernetes.io/affinity": "cookie",
"nginx.ingress.kubernetes.io/session-cookie-name": "SERVERID",
}
annotations := make(map[string]string)
annotations["nginx.ingress.kubernetes.io/affinity"] = "cookie"
annotations["nginx.ingress.kubernetes.io/session-cookie-name"] = "SERVERID"
ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations)
f.EnsureIngress(ing)
@ -63,10 +62,9 @@ var _ = framework.DescribeAnnotation("affinity session-cookie-name", func() {
ginkgo.It("should change cookie name on ingress definition change", func() {
host := "change.foo.com"
annotations := map[string]string{
"nginx.ingress.kubernetes.io/affinity": "cookie",
"nginx.ingress.kubernetes.io/session-cookie-name": "SERVERID",
}
annotations := make(map[string]string)
annotations["nginx.ingress.kubernetes.io/affinity"] = "cookie"
annotations["nginx.ingress.kubernetes.io/session-cookie-name"] = "SERVERID"
ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations)
f.EnsureIngress(ing)
@ -99,10 +97,9 @@ var _ = framework.DescribeAnnotation("affinity session-cookie-name", func() {
ginkgo.It("should set the path to /something on the generated cookie", func() {
host := "path.foo.com"
annotations := map[string]string{
"nginx.ingress.kubernetes.io/affinity": "cookie",
"nginx.ingress.kubernetes.io/session-cookie-name": "SERVERID",
}
annotations := make(map[string]string)
annotations["nginx.ingress.kubernetes.io/affinity"] = "cookie"
annotations["nginx.ingress.kubernetes.io/session-cookie-name"] = "SERVERID"
ing := framework.NewSingleIngress(host, "/something", host, f.Namespace, framework.EchoService, 80, annotations)
f.EnsureIngress(ing)
@ -122,10 +119,9 @@ var _ = framework.DescribeAnnotation("affinity session-cookie-name", func() {
ginkgo.It("does not set the path to / on the generated cookie if there's more than one rule referring to the same backend", func() {
host := "morethanonerule.foo.com"
annotations := map[string]string{
"nginx.ingress.kubernetes.io/affinity": "cookie",
"nginx.ingress.kubernetes.io/session-cookie-name": "SERVERID",
}
annotations := make(map[string]string)
annotations["nginx.ingress.kubernetes.io/affinity"] = "cookie"
annotations["nginx.ingress.kubernetes.io/session-cookie-name"] = "SERVERID"
f.EnsureIngress(&networking.Ingress{
ObjectMeta: metav1.ObjectMeta{
@ -184,12 +180,11 @@ var _ = framework.DescribeAnnotation("affinity session-cookie-name", func() {
ginkgo.It("should set cookie with expires", func() {
host := "cookieexpires.foo.com"
annotations := map[string]string{
"nginx.ingress.kubernetes.io/affinity": "cookie",
"nginx.ingress.kubernetes.io/session-cookie-name": "ExpiresCookie",
"nginx.ingress.kubernetes.io/session-cookie-expires": "172800",
"nginx.ingress.kubernetes.io/session-cookie-max-age": "259200",
}
annotations := make(map[string]string)
annotations["nginx.ingress.kubernetes.io/affinity"] = "cookie"
annotations["nginx.ingress.kubernetes.io/session-cookie-name"] = "ExpiresCookie"
annotations["nginx.ingress.kubernetes.io/session-cookie-expires"] = "172800"
annotations["nginx.ingress.kubernetes.io/session-cookie-max-age"] = "259200"
ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations)
f.EnsureIngress(ing)
@ -216,12 +211,11 @@ var _ = framework.DescribeAnnotation("affinity session-cookie-name", func() {
ginkgo.It("should work with use-regex annotation and session-cookie-path", func() {
host := "useregex.foo.com"
annotations := map[string]string{
"nginx.ingress.kubernetes.io/affinity": "cookie",
"nginx.ingress.kubernetes.io/session-cookie-name": "SERVERID",
"nginx.ingress.kubernetes.io/use-regex": "true",
"nginx.ingress.kubernetes.io/session-cookie-path": "/foo/bar",
}
annotations := make(map[string]string)
annotations["nginx.ingress.kubernetes.io/affinity"] = "cookie"
annotations["nginx.ingress.kubernetes.io/session-cookie-name"] = "SERVERID"
annotations["nginx.ingress.kubernetes.io/use-regex"] = "true"
annotations["nginx.ingress.kubernetes.io/session-cookie-path"] = "/foo/bar"
ing := framework.NewSingleIngress(host, "/foo/.*", host, f.Namespace, framework.EchoService, 80, annotations)
f.EnsureIngress(ing)
@ -241,11 +235,10 @@ var _ = framework.DescribeAnnotation("affinity session-cookie-name", func() {
ginkgo.It("should warn user when use-regex is true and session-cookie-path is not set", func() {
host := "useregexwarn.foo.com"
annotations := map[string]string{
"nginx.ingress.kubernetes.io/affinity": "cookie",
"nginx.ingress.kubernetes.io/session-cookie-name": "SERVERID",
"nginx.ingress.kubernetes.io/use-regex": "true",
}
annotations := make(map[string]string)
annotations["nginx.ingress.kubernetes.io/affinity"] = "cookie"
annotations["nginx.ingress.kubernetes.io/session-cookie-name"] = "SERVERID"
annotations["nginx.ingress.kubernetes.io/use-regex"] = "true"
ing := framework.NewSingleIngress(host, "/foo/.*", host, f.Namespace, framework.EchoService, 80, annotations)
f.EnsureIngress(ing)
@ -269,9 +262,9 @@ var _ = framework.DescribeAnnotation("affinity session-cookie-name", func() {
ginkgo.It("should not set affinity across all server locations when using separate ingresses", func() {
host := "separate.foo.com"
annotations := map[string]string{
"nginx.ingress.kubernetes.io/affinity": "cookie",
}
annotations := make(map[string]string)
annotations["nginx.ingress.kubernetes.io/affinity"] = "cookie"
ing1 := framework.NewSingleIngress("ingress1", "/foo/bar", host, f.Namespace, framework.EchoService, 80, annotations)
f.EnsureIngress(ing1)
@ -299,10 +292,9 @@ var _ = framework.DescribeAnnotation("affinity session-cookie-name", func() {
})
ginkgo.It("should set sticky cookie without host", func() {
annotations := map[string]string{
"nginx.ingress.kubernetes.io/affinity": "cookie",
"nginx.ingress.kubernetes.io/session-cookie-name": "SERVERID",
}
annotations := make(map[string]string)
annotations["nginx.ingress.kubernetes.io/affinity"] = "cookie"
annotations["nginx.ingress.kubernetes.io/session-cookie-name"] = "SERVERID"
ing := framework.NewSingleIngress("default-no-host", "/", "", f.Namespace, framework.EchoService, 80, annotations)
f.EnsureIngress(ing)

173
test/e2e/annotations/affinitymode.go Normal file
View file

@ -0,0 +1,173 @@
/*
Copyright 2020 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package annotations
import (
"fmt"
"net/http"
"reflect"
"strings"
"time"
"github.com/onsi/ginkgo"
"github.com/stretchr/testify/assert"
"k8s.io/ingress-nginx/test/e2e/framework"
)
var _ = framework.DescribeAnnotation("affinitymode", func() {
f := framework.NewDefaultFramework("affinity")
ginkgo.It("Balanced affinity mode should balance", func() {
deploymentName := "affinitybalanceecho"
replicas := 5
f.NewEchoDeploymentWithNameAndReplicas(deploymentName, replicas)
host := "affinity-mode-balance.com"
annotations := make(map[string]string)
annotations["nginx.ingress.kubernetes.io/affinity"] = "cookie"
annotations["ginx.ingress.kubernetes.io/session-cookie-name"] = "hello-cookie"
annotations["nginx.ingress.kubernetes.io/session-cookie-expires"] = "172800"
annotations["nginx.ingress.kubernetes.io/session-cookie-max-age"] = "172800"
annotations["nginx.ingress.kubernetes.io/ssl-redirect"] = "false"
annotations["nginx.ingress.kubernetes.io/affinity-mode"] = "balanced"
annotations["nginx.ingress.kubernetes.io/session-cookie-hash"] = "sha1"
ing := framework.NewSingleIngress(host, "/", host, f.Namespace, deploymentName, 80, annotations)
f.EnsureIngress(ing)
f.WaitForNginxServer(host,
func(server string) bool {
return strings.Contains(server, fmt.Sprintf("server_name %s ;", host))
})
// Check configuration
ingress := f.GetIngress(f.Namespace, host)
returnedAnnotations := ingress.GetAnnotations()
isItEqual := reflect.DeepEqual(annotations, returnedAnnotations)
assert.Equal(ginkgo.GinkgoT(), isItEqual, true)
})
ginkgo.It("Check persistent affinity mode", func() {
deploymentName := "affinitypersistentecho"
replicas := 5
f.NewEchoDeploymentWithNameAndReplicas(deploymentName, replicas)
host := "affinity-mode-persistent.com"
annotations := make(map[string]string)
annotations["nginx.ingress.kubernetes.io/affinity"] = "cookie"
annotations["ginx.ingress.kubernetes.io/session-cookie-name"] = "hello-cookie"
annotations["nginx.ingress.kubernetes.io/session-cookie-expires"] = "172800"
annotations["nginx.ingress.kubernetes.io/session-cookie-max-age"] = "172800"
annotations["nginx.ingress.kubernetes.io/ssl-redirect"] = "false"
annotations["nginx.ingress.kubernetes.io/affinity-mode"] = "persistent"
annotations["nginx.ingress.kubernetes.io/session-cookie-hash"] = "sha1"
ing := framework.NewSingleIngress(host, "/", host, f.Namespace, deploymentName, 80, annotations)
f.EnsureIngress(ing)
f.WaitForNginxServer(host,
func(server string) bool {
return strings.Contains(server, fmt.Sprintf("server_name %s ;", host))
})
// Check configuration
ingress := f.GetIngress(f.Namespace, host)
returnedAnnotations := ingress.GetAnnotations()
isItEqual := reflect.DeepEqual(annotations, returnedAnnotations)
assert.Equal(ginkgo.GinkgoT(), isItEqual, true)
// Make a request
request := f.HTTPTestClient().GET("/").WithHeader("Host", host)
response := request.Expect()
// Get the responder host name
originalHostName := getHostnameFromResponseBody(response.Body().Raw())
// Send new requests and add new backends. Check which backend responded to the sent request
cookies := getCookiesFromHeader(response.Header("Set-Cookie").Raw())
for sendRequestNumber := 0; sendRequestNumber < 10; sendRequestNumber++ {
replicas = replicas + 1
err := framework.UpdateDeployment(f.KubeClientSet, f.Namespace, deploymentName, replicas, nil)
assert.Nil(ginkgo.GinkgoT(), err)
time.Sleep(3 * time.Second)
response = request.WithCookies(cookies).Expect()
newHostName := getHostnameFromResponseBody(response.Body().Raw())
assert.Equal(ginkgo.GinkgoT(), originalHostName, newHostName,
fmt.Sprintf("Response number %v is not from the same host. Original host: %s, response returned: %s", sendRequestNumber, originalHostName, newHostName))
}
// remove all backends
replicas = 0
err := framework.UpdateDeployment(f.KubeClientSet, f.Namespace, deploymentName, replicas, nil)
assert.Nil(ginkgo.GinkgoT(), err)
time.Sleep(5 * time.Second)
// validate, there is no backend to serve the request
response = request.WithCookies(cookies).Expect().Status(http.StatusServiceUnavailable)
// create brand new backends
replicas = 2
err = framework.UpdateDeployment(f.KubeClientSet, f.Namespace, deploymentName, replicas, nil)
assert.Nil(ginkgo.GinkgoT(), err)
time.Sleep(5 * time.Second)
// wait brand new backends to spawn
response = request.WithCookies(cookies).Expect()
try := 0
for (response.Raw().StatusCode == http.StatusServiceUnavailable) && (try < 30) {
time.Sleep(5 * time.Second)
response = request.WithCookies(cookies).Expect()
try++
}
assert.LessOrEqual(ginkgo.GinkgoT(), try, 29, "Tries reached it's maximum, backends did not deployed in time.")
// brand new backends equals new hostname
newHostName := getHostnameFromResponseBody(response.Body().Raw())
assert.NotEqual(ginkgo.GinkgoT(), originalHostName, newHostName,
fmt.Sprintf("Response is from the same host (That should not be possible). Original host: %s, response returned: %s", originalHostName, newHostName))
})
})
func getHostnameFromResponseBody(rawResponseBody string) string {
lines := strings.Split(strings.TrimSpace(rawResponseBody), "\n")
for _, line := range lines {
if strings.Contains(line, "Hostname") {
hostnameParts := strings.Split(strings.TrimSpace(line), ":")
if len(hostnameParts) == 2 {
return strings.TrimSpace(hostnameParts[1])
}
return ""
}
}
return ""
}
func getCookiesFromHeader(rawheader string) map[string]string {
cookies := make(map[string]string)
parts := strings.Split(strings.TrimSpace(rawheader), ";")
for _, part := range parts {
subparts := strings.Split(strings.TrimSpace(part), "=")
if len(subparts) == 2 {
cookies[subparts[0]] = subparts[1]
} else {
cookies[subparts[0]] = ""
}
}
return cookies
}

16
test/e2e/framework/k8s.go
View file

@ -61,14 +61,20 @@ func (f *Framework) EnsureConfigMap(configMap *api.ConfigMap) (*api.ConfigMap, e
return cm, nil
}
// GetIngress gets an Ingress object from the given namespace, name and retunrs it, throws error if it does not exists.
func (f *Framework) GetIngress(namespace string, name string) *networking.Ingress {
ing, err := f.KubeClientSet.NetworkingV1beta1().Ingresses(namespace).Get(name, metav1.GetOptions{})
assert.Nil(ginkgo.GinkgoT(), err, "getting ingress")
assert.NotNil(ginkgo.GinkgoT(), ing, "expected an ingress but none returned")
return ing
}
// EnsureIngress creates an Ingress object and retunrs it, throws error if it already exists.
func (f *Framework) EnsureIngress(ingress *networking.Ingress) *networking.Ingress {
err := createIngressWithRetries(f.KubeClientSet, f.Namespace, ingress)
assert.Nil(ginkgo.GinkgoT(), err, "creating ingress")
ing, err := f.KubeClientSet.NetworkingV1beta1().Ingresses(f.Namespace).Get(ingress.Name, metav1.GetOptions{})
assert.Nil(ginkgo.GinkgoT(), err, "getting ingress")
assert.NotNil(ginkgo.GinkgoT(), ing, "expected an ingress but none returned")
ing := f.GetIngress(f.Namespace, ingress.Name)
if ing.Annotations == nil {
ing.Annotations = make(map[string]string)
@ -85,9 +91,7 @@ func (f *Framework) UpdateIngress(ingress *networking.Ingress) *networking.Ingre
err := updateIngressWithRetries(f.KubeClientSet, f.Namespace, ingress)
assert.Nil(ginkgo.GinkgoT(), err, "updating ingress")
ing, err := f.KubeClientSet.NetworkingV1beta1().Ingresses(f.Namespace).Get(ingress.Name, metav1.GetOptions{})
assert.Nil(ginkgo.GinkgoT(), err, "getting ingress")
assert.NotNil(ginkgo.GinkgoT(), ing, "expected an ingress but none returned")
ing := f.GetIngress(f.Namespace, ingress.Name)
if ing.Annotations == nil {
ing.Annotations = make(map[string]string)