From 1dbe65ecb62ab5e7cf5c5b884e444e84203ec514 Mon Sep 17 00:00:00 2001 From: Ricardo Pchevuzinske Katz Date: Fri, 10 Feb 2017 01:00:17 -0200 Subject: [PATCH 1/8] Initial support for sticky config in annotations --- .../ingress/annotations/stickysession/main.go | 96 +++++++++++++++++++ .../annotations/stickysession/main_test.go | 88 +++++++++++++++++ core/pkg/ingress/controller/annotations.go | 8 ++ .../ingress/controller/annotations_test.go | 37 +++++++ core/pkg/ingress/controller/controller.go | 7 ++ core/pkg/ingress/types.go | 3 + 6 files changed, 239 insertions(+) create mode 100644 core/pkg/ingress/annotations/stickysession/main.go create mode 100644 core/pkg/ingress/annotations/stickysession/main_test.go diff --git a/core/pkg/ingress/annotations/stickysession/main.go b/core/pkg/ingress/annotations/stickysession/main.go new file mode 100644 index 000000000..be4dd2f0f --- /dev/null +++ b/core/pkg/ingress/annotations/stickysession/main.go @@ -0,0 +1,96 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package stickysession + +import ( + "regexp" + + "k8s.io/kubernetes/pkg/apis/extensions" + + "k8s.io/ingress/core/pkg/ingress/annotations/parser" + ing_errors "k8s.io/ingress/core/pkg/ingress/errors" +) + +const ( + stickyEnabled = "ingress.kubernetes.io/sticky-enabled" + stickyName = "ingress.kubernetes.io/sticky-name" + stickyHash = "ingress.kubernetes.io/sticky-hash" + defaultStickyHash = "md5" + defaultStickyName = "route" +) + +var ( + stickyHashRegex = regexp.MustCompile(`index|md5|sha1`) +) + +// StickyConfig describes the per ingress sticky session config +type StickyConfig struct { + // The name of the cookie that will be used as stickness router. + Name string `json:"name"` + // If sticky must or must not be enabled + Enabled bool `json:"enabled"` + // The hash that will be used to encode the cookie + Hash string `json:"hash"` +} + +type sticky struct { +} + +// NewParser creates a new Sticky annotation parser +func NewParser() parser.IngressAnnotation { + return sticky{} +} + +// ParseAnnotations parses the annotations contained in the ingress +// rule used to configure the sticky directives +func (a sticky) Parse(ing *extensions.Ingress) (interface{}, error) { + // Check if the sticky is enabled + se, err := parser.GetBoolAnnotation(stickyEnabled, ing) + if err != nil { + return nil, err + } + + // Get the Sticky Cookie Name + sn, err := parser.GetStringAnnotation(stickyName, ing) + if err != nil { + return nil, err + } + + if sn == "" { + sn = defaultStickyName + } + + sh, err := parser.GetStringAnnotation(stickyHash, ing) + + if err != nil { + return nil, err + } + + if sh == "" { + sh = defaultStickyHash + } + + if !stickyHashRegex.MatchString(sh) { + return nil, ing_errors.NewInvalidAnnotationContent(stickyHash, sh) + } + + return &StickyConfig{ + Name: sn, + Enabled: se, + Hash: sh, + }, nil +} diff --git a/core/pkg/ingress/annotations/stickysession/main_test.go b/core/pkg/ingress/annotations/stickysession/main_test.go new file mode 100644 index 000000000..a8a98f3c4 --- /dev/null +++ b/core/pkg/ingress/annotations/stickysession/main_test.go @@ -0,0 +1,88 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package stickysession + +import ( + "testing" + + "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/apis/extensions" + "k8s.io/kubernetes/pkg/util/intstr" +) + +func buildIngress() *extensions.Ingress { + defaultBackend := extensions.IngressBackend{ + ServiceName: "default-backend", + ServicePort: intstr.FromInt(80), + } + + return &extensions.Ingress{ + ObjectMeta: api.ObjectMeta{ + Name: "foo", + Namespace: api.NamespaceDefault, + }, + Spec: extensions.IngressSpec{ + Backend: &extensions.IngressBackend{ + ServiceName: "default-backend", + ServicePort: intstr.FromInt(80), + }, + Rules: []extensions.IngressRule{ + { + Host: "foo.bar.com", + IngressRuleValue: extensions.IngressRuleValue{ + HTTP: &extensions.HTTPIngressRuleValue{ + Paths: []extensions.HTTPIngressPath{ + { + Path: "/foo", + Backend: defaultBackend, + }, + }, + }, + }, + }, + }, + }, + } +} + +func TestIngressHealthCheck(t *testing.T) { + ing := buildIngress() + + data := map[string]string{} + data[stickyEnabled] = "true" + data[stickyHash] = "md5" + data[stickyName] = "route1" + ing.SetAnnotations(data) + + sti, _ := NewParser().Parse(ing) + nginxSti, ok := sti.(*StickyConfig) + if !ok { + t.Errorf("expected a StickyConfig type") + } + + if nginxSti.Hash != "md5" { + t.Errorf("expected md5 as sticky-hash but returned %v", nginxSti.Hash) + } + + if nginxSti.Hash != "md5" { + t.Errorf("expected md5 as sticky-hash but returned %v", nginxSti.Hash) + } + + if !nginxSti.Enabled { + t.Errorf("expected sticky-enabled but returned %v", nginxSti.Enabled) + } +} diff --git a/core/pkg/ingress/controller/annotations.go b/core/pkg/ingress/controller/annotations.go index c1eb09fbd..400776866 100644 --- a/core/pkg/ingress/controller/annotations.go +++ b/core/pkg/ingress/controller/annotations.go @@ -34,6 +34,7 @@ import ( "k8s.io/ingress/core/pkg/ingress/annotations/rewrite" "k8s.io/ingress/core/pkg/ingress/annotations/secureupstream" "k8s.io/ingress/core/pkg/ingress/annotations/sslpassthrough" + "k8s.io/ingress/core/pkg/ingress/annotations/stickysession" "k8s.io/ingress/core/pkg/ingress/errors" "k8s.io/ingress/core/pkg/ingress/resolver" ) @@ -63,6 +64,7 @@ func newAnnotationExtractor(cfg extractorConfig) annotationExtractor { "Redirect": rewrite.NewParser(cfg), "SecureUpstream": secureupstream.NewParser(), "SSLPassthrough": sslpassthrough.NewParser(), + "StickySession": stickysession.NewParser(), }, } } @@ -99,6 +101,7 @@ const ( secureUpstream = "SecureUpstream" healthCheck = "HealthCheck" sslPassthrough = "SSLPassthrough" + stickySession = "StickySession" ) func (e *annotationExtractor) SecureUpstream(ing *extensions.Ingress) bool { @@ -115,3 +118,8 @@ func (e *annotationExtractor) SSLPassthrough(ing *extensions.Ingress) bool { val, _ := e.annotations[sslPassthrough].Parse(ing) return val.(bool) } + +func (e *annotationExtractor) StickySession(ing *extensions.Ingress) *stickysession.StickyConfig { + val, _ := e.annotations[stickySession].Parse(ing) + return val.(*stickysession.StickyConfig) +} diff --git a/core/pkg/ingress/controller/annotations_test.go b/core/pkg/ingress/controller/annotations_test.go index 58b0d8093..942508bab 100644 --- a/core/pkg/ingress/controller/annotations_test.go +++ b/core/pkg/ingress/controller/annotations_test.go @@ -32,6 +32,9 @@ const ( annotationUpsMaxFails = "ingress.kubernetes.io/upstream-max-fails" annotationUpsFailTimeout = "ingress.kubernetes.io/upstream-fail-timeout" annotationPassthrough = "ingress.kubernetes.io/ssl-passthrough" + annotationStickyEnabled = "ingress.kubernetes.io/sticky-enabled" + annotationStickyName = "ingress.kubernetes.io/sticky-name" + annotationStickyHash = "ingress.kubernetes.io/sticky-hash" ) type mockCfg struct { @@ -179,3 +182,37 @@ func TestSSLPassthrough(t *testing.T) { } } } + +func TestStickySession(t *testing.T) { + ec := newAnnotationExtractor(mockCfg{}) + ing := buildIngress() + + fooAnns := []struct { + annotations map[string]string + enabled bool + hash string + name string + }{ + {map[string]string{annotationStickyEnabled: "true", annotationStickyHash: "md5", annotationStickyName: "route"}, true, "md5", "route"}, + {map[string]string{annotationStickyEnabled: "true", annotationStickyHash: "", annotationStickyName: "xpto"}, true, "md5", "xpto"}, + {map[string]string{annotationStickyEnabled: "true", annotationStickyHash: "", annotationStickyName: ""}, true, "md5", "route"}, + } + + for _, foo := range fooAnns { + ing.SetAnnotations(foo.annotations) + r := ec.StickySession(ing) + + if r == nil { + t.Errorf("Returned nil but expected a StickySesion.StickyConfig") + continue + } + + if r.Hash != foo.hash { + t.Errorf("Returned %v but expected %v for Hash", r.Hash, foo.hash) + } + + if r.Name != foo.name { + t.Errorf("Returned %v but expected %v for Name", r.Name, foo.name) + } + } +} diff --git a/core/pkg/ingress/controller/controller.go b/core/pkg/ingress/controller/controller.go index 5c979dcce..ace3049b1 100644 --- a/core/pkg/ingress/controller/controller.go +++ b/core/pkg/ingress/controller/controller.go @@ -700,6 +700,7 @@ func (ic *GenericController) createUpstreams(data []interface{}) map[string]*ing secUpstream := ic.annotations.SecureUpstream(ing) hz := ic.annotations.HealthCheck(ing) + sticky := ic.annotations.StickySession(ing) var defBackend string if ing.Spec.Backend != nil { @@ -739,6 +740,12 @@ func (ic *GenericController) createUpstreams(data []interface{}) map[string]*ing if !upstreams[name].Secure { upstreams[name].Secure = secUpstream } + if !upstreams[name].StickySession.Enabled || upstreams[name].StickySession.Name == "" || upstreams[name].StickySession.Hash == "" { + upstreams[name].StickySession.Enabled = sticky.Enabled + upstreams[name].StickySession.Name = sticky.Name + upstreams[name].StickySession.Hash = sticky.Hash + } + svcKey := fmt.Sprintf("%v/%v", ing.GetNamespace(), path.Backend.ServiceName) endp, err := ic.serviceEndpoints(svcKey, path.Backend.ServicePort.String(), hz) if err != nil { diff --git a/core/pkg/ingress/types.go b/core/pkg/ingress/types.go index 49f849a57..89f426a89 100644 --- a/core/pkg/ingress/types.go +++ b/core/pkg/ingress/types.go @@ -26,6 +26,7 @@ import ( "k8s.io/ingress/core/pkg/ingress/annotations/proxy" "k8s.io/ingress/core/pkg/ingress/annotations/ratelimit" "k8s.io/ingress/core/pkg/ingress/annotations/rewrite" + "k8s.io/ingress/core/pkg/ingress/annotations/stickysession" "k8s.io/ingress/core/pkg/ingress/defaults" "k8s.io/ingress/core/pkg/ingress/resolver" ) @@ -134,6 +135,8 @@ type Backend struct { Secure bool `json:"secure"` // Endpoints contains the list of endpoints currently running Endpoints []Endpoint `json:"endpoints"` + // StickySession contains the StickyConfig object with stickness configuration + StickySession *stickysession.StickyConfig `json:"stickysession"` } // Endpoint describes a kubernetes endpoint in an backend From 79e186cb77f240d352e87938d99d29d87b9e1610 Mon Sep 17 00:00:00 2001 From: Ricardo Pchevuzinske Katz Date: Fri, 10 Feb 2017 01:33:23 -0200 Subject: [PATCH 2/8] New sticky session configuration --- .../rootfs/etc/nginx/template/nginx.tmpl | 4 ++-- .../ingress/annotations/stickysession/main.go | 23 +++++++------------ .../annotations/stickysession/main_test.go | 2 +- 3 files changed, 11 insertions(+), 18 deletions(-) diff --git a/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl b/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl index d018d4ebb..bb3b04506 100644 --- a/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl +++ b/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl @@ -185,8 +185,8 @@ http { {{range $name, $upstream := $backends}} upstream {{$upstream.Name}} { - {{ if $cfg.EnableStickySessions }} - sticky hash=sha1 httponly; + {{ if $upstream.StickySession.Enabled }} + sticky hash={{$upstream.StickySession.Hash}} route={{$upstream.StickySession.Hash}} httponly; {{ else }} least_conn; {{ end }} diff --git a/core/pkg/ingress/annotations/stickysession/main.go b/core/pkg/ingress/annotations/stickysession/main.go index be4dd2f0f..ce9609896 100644 --- a/core/pkg/ingress/annotations/stickysession/main.go +++ b/core/pkg/ingress/annotations/stickysession/main.go @@ -59,34 +59,27 @@ func NewParser() parser.IngressAnnotation { // rule used to configure the sticky directives func (a sticky) Parse(ing *extensions.Ingress) (interface{}, error) { // Check if the sticky is enabled - se, err := parser.GetBoolAnnotation(stickyEnabled, ing) - if err != nil { - return nil, err - } + se, _ := parser.GetBoolAnnotation(stickyEnabled, ing) // Get the Sticky Cookie Name - sn, err := parser.GetStringAnnotation(stickyName, ing) - if err != nil { - return nil, err - } + sn, _ := parser.GetStringAnnotation(stickyName, ing) if sn == "" { sn = defaultStickyName } - sh, err := parser.GetStringAnnotation(stickyHash, ing) - - if err != nil { - return nil, err - } + sh, _ := parser.GetStringAnnotation(stickyHash, ing) if sh == "" { sh = defaultStickyHash } if !stickyHashRegex.MatchString(sh) { - return nil, ing_errors.NewInvalidAnnotationContent(stickyHash, sh) - } + return &StickyConfig{ + Name: "", + Enabled: false, + Hash: "", + }, ing_errors.NewInvalidAnnotationContent(stickyHash, sh) return &StickyConfig{ Name: sn, diff --git a/core/pkg/ingress/annotations/stickysession/main_test.go b/core/pkg/ingress/annotations/stickysession/main_test.go index a8a98f3c4..71384fad1 100644 --- a/core/pkg/ingress/annotations/stickysession/main_test.go +++ b/core/pkg/ingress/annotations/stickysession/main_test.go @@ -59,7 +59,7 @@ func buildIngress() *extensions.Ingress { } } -func TestIngressHealthCheck(t *testing.T) { +func TestIngressStickySession(t *testing.T) { ing := buildIngress() data := map[string]string{} From 68093193184cc7d18d539dfa6876e2e98d445ec8 Mon Sep 17 00:00:00 2001 From: Ricardo Pchevuzinske Katz Date: Fri, 10 Feb 2017 12:24:16 -0200 Subject: [PATCH 3/8] Adds support for configuring stickness per Ingress --- controllers/nginx/configuration.md | 17 ++++++ .../rootfs/etc/nginx/template/nginx.tmpl | 2 +- .../ingress/annotations/stickysession/main.go | 27 ++++----- .../ingress/controller/annotations_test.go | 2 + core/pkg/ingress/types.go | 2 +- examples/stickysession/nginx/README.md | 58 +++++++++++++++++++ .../stickysession/nginx/sticky-ingress.yaml | 19 ++++++ 7 files changed, 112 insertions(+), 15 deletions(-) create mode 100644 examples/stickysession/nginx/README.md create mode 100644 examples/stickysession/nginx/sticky-ingress.yaml diff --git a/controllers/nginx/configuration.md b/controllers/nginx/configuration.md index 77c21ca73..e9e5fb8ce 100644 --- a/controllers/nginx/configuration.md +++ b/controllers/nginx/configuration.md @@ -51,6 +51,9 @@ The following annotations are supported: |[ingress.kubernetes.io/upstream-max-fails](#custom-nginx-upstream-checks)|number| |[ingress.kubernetes.io/upstream-fail-timeout](#custom-nginx-upstream-checks)|number| |[ingress.kubernetes.io/whitelist-source-range](#whitelist-source-range)|CIDR| +|[ingress.kubernetes.io/sticky-enabled](#sticky-session)|true or false| +|[ingress.kubernetes.io/sticky-name](#sticky-session)|string| +|[ingress.kubernetes.io/sticky-hash](#sticky-session)|string| @@ -177,6 +180,20 @@ To configure this setting globally for all Ingress rules, the `whitelist-source- Please check the [whitelist](examples/whitelist/README.md) example. +### Sticky Session + +The annotation `ingress.kubernetes.io/sticky-enabled` enables stickness in all Upstreams of an Ingress. This way, a request will always be directed to the same upstream server. + +You can also specify the name of the cookie that will be used to route the requests with the annotation `ingress.kubernetes.io/sticky-name`. The default is to create a cookie named 'route'. + +The annotation `ingress.kubernetes.io/sticky-hash` defines which algorithm will be used to 'hash' the used upstream. Default value is `md5` and possible values are `md5`, `sha1` and `index`. + +This feature is implemented by the third party module *nginx-sticky-module-ng* (https://bitbucket.org/nginx-goodies/nginx-sticky-module-ng). + +The workflow used to define which upstream server will be used is explained here: https://bitbucket.org/nginx-goodies/nginx-sticky-module-ng/raw/08a395c66e425540982c00482f55034e1fee67b6/docs/sticky.pdf + + + ### **Allowed parameters in configuration ConfigMap** **proxy-body-size:** Sets the maximum allowed size of the client request body. See NGINX [client_max_body_size](http://nginx.org/en/docs/http/ngx_http_core_module.html#client_max_body_size). diff --git a/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl b/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl index bb3b04506..34bfea851 100644 --- a/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl +++ b/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl @@ -186,7 +186,7 @@ http { {{range $name, $upstream := $backends}} upstream {{$upstream.Name}} { {{ if $upstream.StickySession.Enabled }} - sticky hash={{$upstream.StickySession.Hash}} route={{$upstream.StickySession.Hash}} httponly; + sticky hash={{$upstream.StickySession.Hash}} name={{$upstream.StickySession.Name}} httponly; {{ else }} least_conn; {{ end }} diff --git a/core/pkg/ingress/annotations/stickysession/main.go b/core/pkg/ingress/annotations/stickysession/main.go index ce9609896..20bae2c8e 100644 --- a/core/pkg/ingress/annotations/stickysession/main.go +++ b/core/pkg/ingress/annotations/stickysession/main.go @@ -19,10 +19,11 @@ package stickysession import ( "regexp" + "github.com/golang/glog" + "k8s.io/kubernetes/pkg/apis/extensions" "k8s.io/ingress/core/pkg/ingress/annotations/parser" - ing_errors "k8s.io/ingress/core/pkg/ingress/errors" ) const ( @@ -59,28 +60,28 @@ func NewParser() parser.IngressAnnotation { // rule used to configure the sticky directives func (a sticky) Parse(ing *extensions.Ingress) (interface{}, error) { // Check if the sticky is enabled - se, _ := parser.GetBoolAnnotation(stickyEnabled, ing) + se, err := parser.GetBoolAnnotation(stickyEnabled, ing) + if err != nil { + se = false + } + + glog.V(3).Infof("Ingress %v: Setting stickness to %v", ing.Name, se) // Get the Sticky Cookie Name - sn, _ := parser.GetStringAnnotation(stickyName, ing) + sn, err := parser.GetStringAnnotation(stickyName, ing) - if sn == "" { + if err != nil || sn == "" { + glog.V(3).Infof("Ingress %v: No value found in annotation %v. Using the default %v", ing.Name, stickyName, defaultStickyName) sn = defaultStickyName } - sh, _ := parser.GetStringAnnotation(stickyHash, ing) + sh, err := parser.GetStringAnnotation(stickyHash, ing) - if sh == "" { + if err != nil || !stickyHashRegex.MatchString(sh) { + glog.V(3).Infof("Invalid or no annotation value found in Ingress %v: %v: %v. Setting it to default %v", ing.Name, stickyHash, sh, defaultStickyHash) sh = defaultStickyHash } - if !stickyHashRegex.MatchString(sh) { - return &StickyConfig{ - Name: "", - Enabled: false, - Hash: "", - }, ing_errors.NewInvalidAnnotationContent(stickyHash, sh) - return &StickyConfig{ Name: sn, Enabled: se, diff --git a/core/pkg/ingress/controller/annotations_test.go b/core/pkg/ingress/controller/annotations_test.go index 942508bab..80f9c99ca 100644 --- a/core/pkg/ingress/controller/annotations_test.go +++ b/core/pkg/ingress/controller/annotations_test.go @@ -196,6 +196,8 @@ func TestStickySession(t *testing.T) { {map[string]string{annotationStickyEnabled: "true", annotationStickyHash: "md5", annotationStickyName: "route"}, true, "md5", "route"}, {map[string]string{annotationStickyEnabled: "true", annotationStickyHash: "", annotationStickyName: "xpto"}, true, "md5", "xpto"}, {map[string]string{annotationStickyEnabled: "true", annotationStickyHash: "", annotationStickyName: ""}, true, "md5", "route"}, + {map[string]string{}, false, "md5", "route"}, + {nil, false, "md5", "route"}, } for _, foo := range fooAnns { diff --git a/core/pkg/ingress/types.go b/core/pkg/ingress/types.go index 89f426a89..a43e1d7a3 100644 --- a/core/pkg/ingress/types.go +++ b/core/pkg/ingress/types.go @@ -136,7 +136,7 @@ type Backend struct { // Endpoints contains the list of endpoints currently running Endpoints []Endpoint `json:"endpoints"` // StickySession contains the StickyConfig object with stickness configuration - StickySession *stickysession.StickyConfig `json:"stickysession"` + StickySession stickysession.StickyConfig `json:"stickysession,omitempty"` } // Endpoint describes a kubernetes endpoint in an backend diff --git a/examples/stickysession/nginx/README.md b/examples/stickysession/nginx/README.md new file mode 100644 index 000000000..ba5bf9abc --- /dev/null +++ b/examples/stickysession/nginx/README.md @@ -0,0 +1,58 @@ +# Sticky Session + +This example demonstrates how to Stickness in a Ingress. + +## Prerequisites + +You will need to make sure you Ingress targets exactly one Ingress +controller by specifying the [ingress.class annotation](/examples/PREREQUISITES.md#ingress-class), +and that you have an ingress controller [running](/examples/deployment) in your cluster. + +Also, you need to have a deployment with replica > 1. Using a deployment with only one replica doesn't set the 'sticky' cookie. + +## Deployment + +The following command instructs the controller to set Stickness in all Upstreams of an Ingress + +```console +$ kubectl create -f sticky-ingress.yaml +``` + +## Validation + +You can confirm that the Ingress works. + +```console +$ kubectl describe ing nginx-test +Name: nginx-test +Namespace: default +Address: +Default backend: default-http-backend:80 (10.180.0.4:8080,10.240.0.2:8080) +Rules: + Host Path Backends + ---- ---- -------- + stickyingress.example.com + / nginx-service:80 () +Annotations: + sticky-enabled: true + sticky-hash: sha1 + sticky-name: route +Events: + FirstSeen LastSeen Count From SubObjectPath Type Reason Message + --------- -------- ----- ---- ------------- -------- ------ ------- + 7s 7s 1 {nginx-ingress-controller } Normal CREATE default/nginx-test + + +$ curl -I http://stickyingress.example.com +HTTP/1.1 200 OK +Server: nginx/1.11.9 +Date: Fri, 10 Feb 2017 14:11:12 GMT +Content-Type: text/html +Content-Length: 612 +Connection: keep-alive +Set-Cookie: route=a9907b79b248140b56bb13723f72b67697baac3d; Path=/; HttpOnly +Last-Modified: Tue, 24 Jan 2017 14:02:19 GMT +ETag: "58875e6b-264" +Accept-Ranges: bytes +``` +In the example avove, you can see a line containing the 'Set-Cookie: route' setting the right defined stickness cookie. \ No newline at end of file diff --git a/examples/stickysession/nginx/sticky-ingress.yaml b/examples/stickysession/nginx/sticky-ingress.yaml new file mode 100644 index 000000000..fe7dd42b3 --- /dev/null +++ b/examples/stickysession/nginx/sticky-ingress.yaml @@ -0,0 +1,19 @@ +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: nginx-test + annotations: + kubernetes.io/ingress.class: "nginx" + ingress.kubernetes.io/sticky-enabled: "true" + ingress.kubernetes.io/sticky-name: "route" + ingress.kubernetes.io/sticky-hash: "sha1" + +spec: + rules: + - host: stickyingress.example.com + http: + paths: + - backend: + serviceName: nginx-service + servicePort: 80 + path: / From a158e5fc5ab8118c7a51614a1dc43f97cf91b12e Mon Sep 17 00:00:00 2001 From: Ricardo Pchevuzinske Katz Date: Sun, 12 Feb 2017 21:13:39 -0200 Subject: [PATCH 4/8] Improve the session affinity feature --- controllers/nginx/configuration.md | 13 +- .../rootfs/etc/nginx/template/nginx.tmpl | 4 +- .../annotations/sessionaffinity/main.go | 118 ++++++++++++++++++ .../main_test.go | 28 ++--- .../ingress/annotations/stickysession/main.go | 90 ------------- core/pkg/ingress/controller/annotations.go | 18 +-- .../ingress/controller/annotations_test.go | 48 +++---- core/pkg/ingress/controller/controller.go | 12 +- core/pkg/ingress/types.go | 21 +++- examples/affinity/cookie/nginx/README.md | 73 +++++++++++ .../cookie}/nginx/sticky-ingress.yaml | 6 +- examples/stickysession/nginx/README.md | 58 --------- 12 files changed, 276 insertions(+), 213 deletions(-) create mode 100644 core/pkg/ingress/annotations/sessionaffinity/main.go rename core/pkg/ingress/annotations/{stickysession => sessionaffinity}/main_test.go (67%) delete mode 100644 core/pkg/ingress/annotations/stickysession/main.go create mode 100644 examples/affinity/cookie/nginx/README.md rename examples/{stickysession => affinity/cookie}/nginx/sticky-ingress.yaml (66%) delete mode 100644 examples/stickysession/nginx/README.md diff --git a/controllers/nginx/configuration.md b/controllers/nginx/configuration.md index e9e5fb8ce..2b04e55dd 100644 --- a/controllers/nginx/configuration.md +++ b/controllers/nginx/configuration.md @@ -177,20 +177,21 @@ To configure this setting globally for all Ingress rules, the `whitelist-source- *Note:* Adding an annotation to an Ingress rule overrides any global restriction. -Please check the [whitelist](examples/whitelist/README.md) example. +Please check the [whitelist](examples/affinity/cookie/nginx/README.md) example. ### Sticky Session -The annotation `ingress.kubernetes.io/sticky-enabled` enables stickness in all Upstreams of an Ingress. This way, a request will always be directed to the same upstream server. +The annotation `ingress.kubernetes.io/affinity` enables and sets the affinity type in all Upstreams of an Ingress. This way, a request will always be directed to the same upstream server. -You can also specify the name of the cookie that will be used to route the requests with the annotation `ingress.kubernetes.io/sticky-name`. The default is to create a cookie named 'route'. -The annotation `ingress.kubernetes.io/sticky-hash` defines which algorithm will be used to 'hash' the used upstream. Default value is `md5` and possible values are `md5`, `sha1` and `index`. +#### Cookie affinity +If you use the ``cookie`` type you can also specify the name of the cookie that will be used to route the requests with the annotation `ingress.kubernetes.io/session-cookie-name`. The default is to create a cookie named 'route'. -This feature is implemented by the third party module *nginx-sticky-module-ng* (https://bitbucket.org/nginx-goodies/nginx-sticky-module-ng). +In case of NGINX the annotation `ingress.kubernetes.io/session-cookie-hash` defines which algorithm will be used to 'hash' the used upstream. Default value is `md5` and possible values are `md5`, `sha1` and `index`. +The `index` option is not hashed, an in-memory index is used instead, it's quicker and the overhead is shorter Warning: the matching against upstream servers list is inconsistent. So, at reload, if upstreams servers has changed, index values are not guaranted to correspond to the same server as before! USE IT WITH CAUTION and only if you need to! -The workflow used to define which upstream server will be used is explained here: https://bitbucket.org/nginx-goodies/nginx-sticky-module-ng/raw/08a395c66e425540982c00482f55034e1fee67b6/docs/sticky.pdf +In NGINX this feature is implemented by the third party module [nginx-sticky-module-ng](https://bitbucket.org/nginx-goodies/nginx-sticky-module-ng). The workflow used to define which upstream server will be used is explained [here]https://bitbucket.org/nginx-goodies/nginx-sticky-module-ng/raw/08a395c66e425540982c00482f55034e1fee67b6/docs/sticky.pdf diff --git a/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl b/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl index 34bfea851..5c8164ac6 100644 --- a/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl +++ b/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl @@ -185,8 +185,8 @@ http { {{range $name, $upstream := $backends}} upstream {{$upstream.Name}} { - {{ if $upstream.StickySession.Enabled }} - sticky hash={{$upstream.StickySession.Hash}} name={{$upstream.StickySession.Name}} httponly; + {{ if eq $upstream.SessionAffinity.AffinityType "cookie" }} + sticky hash={{$upstream.SessionAffinity.CookieSessionAffinity.Hash}} name={{$upstream.SessionAffinity.CookieSessionAffinity.Name}} httponly; {{ else }} least_conn; {{ end }} diff --git a/core/pkg/ingress/annotations/sessionaffinity/main.go b/core/pkg/ingress/annotations/sessionaffinity/main.go new file mode 100644 index 000000000..551cde2a3 --- /dev/null +++ b/core/pkg/ingress/annotations/sessionaffinity/main.go @@ -0,0 +1,118 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package sessionaffinity + +import ( + "regexp" + + "github.com/golang/glog" + + "k8s.io/kubernetes/pkg/apis/extensions" + + "k8s.io/ingress/core/pkg/ingress/annotations/parser" +) + +const ( + annotationAffinityType = "ingress.kubernetes.io/affinity" + // If a cookie with this name exists, + // its value is used as an index into the list of available backends. + annotationAffinityCookieName = "ingress.kubernetes.io/session-cookie-name" + defaultAffinityCookieName = "route" + // This is the algorithm used by nginx to generate a value for the session cookie, if + // one isn't supplied and affintiy is set to "cookie". + annotationAffinityCookieHash = "ingress.kubernetes.io/session-cookie-hash" + defaultAffinityCookieHash = "md5" +) + +var ( + affinityCookieHashRegex = regexp.MustCompile(`^(index|md5|sha1)$`) +) + +// AffinityConfig describes the per ingress session affinity config +type AffinityConfig struct { + // The type of affinity that will be used + AffinityType string `json:"type"` + CookieAffinityConfig CookieAffinityConfig `json:"cookieconfig"` +} + +// CookieAffinityConfig describes the Config of cookie type affinity +type CookieAffinityConfig struct { + // The name of the cookie that will be used in case of cookie affinity type. + Name string `json:"name"` + // The hash that will be used to encode the cookie in case of cookie affinity type + Hash string `json:"hash"` +} + +type affinity struct { +} + +// CookieAffinityParse gets the annotation values related to Cookie Affinity +// It also sets default values when no value or incorrect value is found +func CookieAffinityParse(ing *extensions.Ingress) *CookieAffinityConfig { + + sn, err := parser.GetStringAnnotation(annotationAffinityCookieName, ing) + + if err != nil || sn == "" { + glog.V(3).Infof("Ingress %v: No value found in annotation %v. Using the default %v", ing.Name, annotationAffinityCookieName, defaultAffinityCookieName) + sn = defaultAffinityCookieName + } + + sh, err := parser.GetStringAnnotation(annotationAffinityCookieHash, ing) + + if err != nil || !affinityCookieHashRegex.MatchString(sh) { + glog.V(3).Infof("Invalid or no annotation value found in Ingress %v: %v. Setting it to default %v", ing.Name, annotationAffinityCookieHash, defaultAffinityCookieHash) + sh = defaultAffinityCookieHash + } + + return &CookieAffinityConfig{ + Name: sn, + Hash: sh, + } +} + +// NewParser creates a new Affinity annotation parser +func NewParser() parser.IngressAnnotation { + return affinity{} +} + +// ParseAnnotations parses the annotations contained in the ingress +// rule used to configure the affinity directives +func (a affinity) Parse(ing *extensions.Ingress) (interface{}, error) { + + var cookieAffinityConfig *CookieAffinityConfig + cookieAffinityConfig = &CookieAffinityConfig{} + + // Check the type of affinity that will be used + at, err := parser.GetStringAnnotation(annotationAffinityType, ing) + if err != nil { + at = "" + } + //cookieAffinityConfig = CookieAffinityParse(ing) + switch at { + case "cookie": + cookieAffinityConfig = CookieAffinityParse(ing) + + default: + glog.V(3).Infof("No default affinity was found for Ingress %v", ing.Name) + + } + return &AffinityConfig{ + AffinityType: at, + CookieAffinityConfig: *cookieAffinityConfig, + }, nil + +} diff --git a/core/pkg/ingress/annotations/stickysession/main_test.go b/core/pkg/ingress/annotations/sessionaffinity/main_test.go similarity index 67% rename from core/pkg/ingress/annotations/stickysession/main_test.go rename to core/pkg/ingress/annotations/sessionaffinity/main_test.go index 71384fad1..b18f86835 100644 --- a/core/pkg/ingress/annotations/stickysession/main_test.go +++ b/core/pkg/ingress/annotations/sessionaffinity/main_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package stickysession +package sessionaffinity import ( "testing" @@ -59,30 +59,30 @@ func buildIngress() *extensions.Ingress { } } -func TestIngressStickySession(t *testing.T) { +func TestIngressAffinityCookieConfig(t *testing.T) { ing := buildIngress() data := map[string]string{} - data[stickyEnabled] = "true" - data[stickyHash] = "md5" - data[stickyName] = "route1" + data[annotationAffinityType] = "cookie" + data[annotationAffinityCookieHash] = "sha123" + data[annotationAffinityCookieName] = "route" ing.SetAnnotations(data) - sti, _ := NewParser().Parse(ing) - nginxSti, ok := sti.(*StickyConfig) + affin, _ := NewParser().Parse(ing) + nginxAffinity, ok := affin.(*AffinityConfig) if !ok { - t.Errorf("expected a StickyConfig type") + t.Errorf("expected a Config type") } - if nginxSti.Hash != "md5" { - t.Errorf("expected md5 as sticky-hash but returned %v", nginxSti.Hash) + if nginxAffinity.AffinityType != "cookie" { + t.Errorf("expected cookie as sticky-type but returned %v", nginxAffinity.AffinityType) } - if nginxSti.Hash != "md5" { - t.Errorf("expected md5 as sticky-hash but returned %v", nginxSti.Hash) + if nginxAffinity.CookieAffinityConfig.Hash != "md5" { + t.Errorf("expected md5 as sticky-hash but returned %v", nginxAffinity.CookieAffinityConfig.Hash) } - if !nginxSti.Enabled { - t.Errorf("expected sticky-enabled but returned %v", nginxSti.Enabled) + if nginxAffinity.CookieAffinityConfig.Name != "route" { + t.Errorf("expected route as sticky-name but returned %v", nginxAffinity.CookieAffinityConfig.Name) } } diff --git a/core/pkg/ingress/annotations/stickysession/main.go b/core/pkg/ingress/annotations/stickysession/main.go deleted file mode 100644 index 20bae2c8e..000000000 --- a/core/pkg/ingress/annotations/stickysession/main.go +++ /dev/null @@ -1,90 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package stickysession - -import ( - "regexp" - - "github.com/golang/glog" - - "k8s.io/kubernetes/pkg/apis/extensions" - - "k8s.io/ingress/core/pkg/ingress/annotations/parser" -) - -const ( - stickyEnabled = "ingress.kubernetes.io/sticky-enabled" - stickyName = "ingress.kubernetes.io/sticky-name" - stickyHash = "ingress.kubernetes.io/sticky-hash" - defaultStickyHash = "md5" - defaultStickyName = "route" -) - -var ( - stickyHashRegex = regexp.MustCompile(`index|md5|sha1`) -) - -// StickyConfig describes the per ingress sticky session config -type StickyConfig struct { - // The name of the cookie that will be used as stickness router. - Name string `json:"name"` - // If sticky must or must not be enabled - Enabled bool `json:"enabled"` - // The hash that will be used to encode the cookie - Hash string `json:"hash"` -} - -type sticky struct { -} - -// NewParser creates a new Sticky annotation parser -func NewParser() parser.IngressAnnotation { - return sticky{} -} - -// ParseAnnotations parses the annotations contained in the ingress -// rule used to configure the sticky directives -func (a sticky) Parse(ing *extensions.Ingress) (interface{}, error) { - // Check if the sticky is enabled - se, err := parser.GetBoolAnnotation(stickyEnabled, ing) - if err != nil { - se = false - } - - glog.V(3).Infof("Ingress %v: Setting stickness to %v", ing.Name, se) - - // Get the Sticky Cookie Name - sn, err := parser.GetStringAnnotation(stickyName, ing) - - if err != nil || sn == "" { - glog.V(3).Infof("Ingress %v: No value found in annotation %v. Using the default %v", ing.Name, stickyName, defaultStickyName) - sn = defaultStickyName - } - - sh, err := parser.GetStringAnnotation(stickyHash, ing) - - if err != nil || !stickyHashRegex.MatchString(sh) { - glog.V(3).Infof("Invalid or no annotation value found in Ingress %v: %v: %v. Setting it to default %v", ing.Name, stickyHash, sh, defaultStickyHash) - sh = defaultStickyHash - } - - return &StickyConfig{ - Name: sn, - Enabled: se, - Hash: sh, - }, nil -} diff --git a/core/pkg/ingress/controller/annotations.go b/core/pkg/ingress/controller/annotations.go index 400776866..5a54e83cd 100644 --- a/core/pkg/ingress/controller/annotations.go +++ b/core/pkg/ingress/controller/annotations.go @@ -33,8 +33,8 @@ import ( "k8s.io/ingress/core/pkg/ingress/annotations/ratelimit" "k8s.io/ingress/core/pkg/ingress/annotations/rewrite" "k8s.io/ingress/core/pkg/ingress/annotations/secureupstream" + "k8s.io/ingress/core/pkg/ingress/annotations/sessionaffinity" "k8s.io/ingress/core/pkg/ingress/annotations/sslpassthrough" - "k8s.io/ingress/core/pkg/ingress/annotations/stickysession" "k8s.io/ingress/core/pkg/ingress/errors" "k8s.io/ingress/core/pkg/ingress/resolver" ) @@ -63,8 +63,8 @@ func newAnnotationExtractor(cfg extractorConfig) annotationExtractor { "RateLimit": ratelimit.NewParser(), "Redirect": rewrite.NewParser(cfg), "SecureUpstream": secureupstream.NewParser(), + "SessionAffinity": sessionaffinity.NewParser(), "SSLPassthrough": sslpassthrough.NewParser(), - "StickySession": stickysession.NewParser(), }, } } @@ -98,10 +98,10 @@ func (e *annotationExtractor) Extract(ing *extensions.Ingress) map[string]interf } const ( - secureUpstream = "SecureUpstream" - healthCheck = "HealthCheck" - sslPassthrough = "SSLPassthrough" - stickySession = "StickySession" + secureUpstream = "SecureUpstream" + healthCheck = "HealthCheck" + sslPassthrough = "SSLPassthrough" + sessionAffinity = "SessionAffinity" ) func (e *annotationExtractor) SecureUpstream(ing *extensions.Ingress) bool { @@ -119,7 +119,7 @@ func (e *annotationExtractor) SSLPassthrough(ing *extensions.Ingress) bool { return val.(bool) } -func (e *annotationExtractor) StickySession(ing *extensions.Ingress) *stickysession.StickyConfig { - val, _ := e.annotations[stickySession].Parse(ing) - return val.(*stickysession.StickyConfig) +func (e *annotationExtractor) SessionAffinity(ing *extensions.Ingress) *sessionaffinity.AffinityConfig { + val, _ := e.annotations[sessionAffinity].Parse(ing) + return val.(*sessionaffinity.AffinityConfig) } diff --git a/core/pkg/ingress/controller/annotations_test.go b/core/pkg/ingress/controller/annotations_test.go index 80f9c99ca..6927f2733 100644 --- a/core/pkg/ingress/controller/annotations_test.go +++ b/core/pkg/ingress/controller/annotations_test.go @@ -28,13 +28,13 @@ import ( ) const ( - annotationSecureUpstream = "ingress.kubernetes.io/secure-backends" - annotationUpsMaxFails = "ingress.kubernetes.io/upstream-max-fails" - annotationUpsFailTimeout = "ingress.kubernetes.io/upstream-fail-timeout" - annotationPassthrough = "ingress.kubernetes.io/ssl-passthrough" - annotationStickyEnabled = "ingress.kubernetes.io/sticky-enabled" - annotationStickyName = "ingress.kubernetes.io/sticky-name" - annotationStickyHash = "ingress.kubernetes.io/sticky-hash" + annotationSecureUpstream = "ingress.kubernetes.io/secure-backends" + annotationUpsMaxFails = "ingress.kubernetes.io/upstream-max-fails" + annotationUpsFailTimeout = "ingress.kubernetes.io/upstream-fail-timeout" + annotationPassthrough = "ingress.kubernetes.io/ssl-passthrough" + annotationAffinityType = "ingress.kubernetes.io/affinity" + annotationAffinityCookieName = "ingress.kubernetes.io/session-cookie-name" + annotationAffinityCookieHash = "ingress.kubernetes.io/session-cookie-hash" ) type mockCfg struct { @@ -183,38 +183,38 @@ func TestSSLPassthrough(t *testing.T) { } } -func TestStickySession(t *testing.T) { +func TestAffinitySession(t *testing.T) { ec := newAnnotationExtractor(mockCfg{}) ing := buildIngress() fooAnns := []struct { - annotations map[string]string - enabled bool - hash string - name string + annotations map[string]string + affinitytype string + hash string + name string }{ - {map[string]string{annotationStickyEnabled: "true", annotationStickyHash: "md5", annotationStickyName: "route"}, true, "md5", "route"}, - {map[string]string{annotationStickyEnabled: "true", annotationStickyHash: "", annotationStickyName: "xpto"}, true, "md5", "xpto"}, - {map[string]string{annotationStickyEnabled: "true", annotationStickyHash: "", annotationStickyName: ""}, true, "md5", "route"}, - {map[string]string{}, false, "md5", "route"}, - {nil, false, "md5", "route"}, + {map[string]string{annotationAffinityType: "cookie", annotationAffinityCookieHash: "md5", annotationAffinityCookieName: "route"}, "cookie", "md5", "route"}, + {map[string]string{annotationAffinityType: "cookie", annotationAffinityCookieHash: "xpto", annotationAffinityCookieName: "route1"}, "cookie", "md5", "route1"}, + {map[string]string{annotationAffinityType: "cookie", annotationAffinityCookieHash: "", annotationAffinityCookieName: ""}, "cookie", "md5", "route"}, + {map[string]string{}, "", "", ""}, + {nil, "", "", ""}, } for _, foo := range fooAnns { ing.SetAnnotations(foo.annotations) - r := ec.StickySession(ing) - + r := ec.SessionAffinity(ing) + t.Logf("Testing pass %v %v %v", foo.affinitytype, foo.hash, foo.name) if r == nil { - t.Errorf("Returned nil but expected a StickySesion.StickyConfig") + t.Errorf("Returned nil but expected a SessionAffinity.AffinityConfig") continue } - if r.Hash != foo.hash { - t.Errorf("Returned %v but expected %v for Hash", r.Hash, foo.hash) + if r.CookieAffinityConfig.Hash != foo.hash { + t.Errorf("Returned %v but expected %v for Hash", r.CookieAffinityConfig.Hash, foo.hash) } - if r.Name != foo.name { - t.Errorf("Returned %v but expected %v for Name", r.Name, foo.name) + if r.CookieAffinityConfig.Name != foo.name { + t.Errorf("Returned %v but expected %v for Name", r.CookieAffinityConfig.Name, foo.name) } } } diff --git a/core/pkg/ingress/controller/controller.go b/core/pkg/ingress/controller/controller.go index ace3049b1..1eb0b6f2c 100644 --- a/core/pkg/ingress/controller/controller.go +++ b/core/pkg/ingress/controller/controller.go @@ -700,7 +700,7 @@ func (ic *GenericController) createUpstreams(data []interface{}) map[string]*ing secUpstream := ic.annotations.SecureUpstream(ing) hz := ic.annotations.HealthCheck(ing) - sticky := ic.annotations.StickySession(ing) + affinity := ic.annotations.SessionAffinity(ing) var defBackend string if ing.Spec.Backend != nil { @@ -740,10 +740,12 @@ func (ic *GenericController) createUpstreams(data []interface{}) map[string]*ing if !upstreams[name].Secure { upstreams[name].Secure = secUpstream } - if !upstreams[name].StickySession.Enabled || upstreams[name].StickySession.Name == "" || upstreams[name].StickySession.Hash == "" { - upstreams[name].StickySession.Enabled = sticky.Enabled - upstreams[name].StickySession.Name = sticky.Name - upstreams[name].StickySession.Hash = sticky.Hash + if upstreams[name].SessionAffinity.AffinityType == "" { + upstreams[name].SessionAffinity.AffinityType = affinity.AffinityType + if affinity.AffinityType == "cookie" { + upstreams[name].SessionAffinity.CookieSessionAffinity.Name = affinity.CookieAffinityConfig.Name + upstreams[name].SessionAffinity.CookieSessionAffinity.Hash = affinity.CookieAffinityConfig.Hash + } } svcKey := fmt.Sprintf("%v/%v", ing.GetNamespace(), path.Backend.ServiceName) diff --git a/core/pkg/ingress/types.go b/core/pkg/ingress/types.go index a43e1d7a3..9363b7830 100644 --- a/core/pkg/ingress/types.go +++ b/core/pkg/ingress/types.go @@ -26,7 +26,6 @@ import ( "k8s.io/ingress/core/pkg/ingress/annotations/proxy" "k8s.io/ingress/core/pkg/ingress/annotations/ratelimit" "k8s.io/ingress/core/pkg/ingress/annotations/rewrite" - "k8s.io/ingress/core/pkg/ingress/annotations/stickysession" "k8s.io/ingress/core/pkg/ingress/defaults" "k8s.io/ingress/core/pkg/ingress/resolver" ) @@ -136,7 +135,25 @@ type Backend struct { // Endpoints contains the list of endpoints currently running Endpoints []Endpoint `json:"endpoints"` // StickySession contains the StickyConfig object with stickness configuration - StickySession stickysession.StickyConfig `json:"stickysession,omitempty"` + + SessionAffinity SessionAffinityConfig +} + +// SessionAffinityConfig describes different affinity configurations for new sessions. +// Once a session is mapped to a backend based on some affinity setting, it +// retains that mapping till the backend goes down, or the ingress controller +// restarts. Exactly one of these values will be set on the upstream, since multiple +// affinity values are incompatible. Once set, the backend makes no guarantees +// about honoring updates. +type SessionAffinityConfig struct { + AffinityType string `json:"name"` + CookieSessionAffinity CookieSessionAffinity +} + +// CookieSessionAffinity defines the structure used in Affinity configured by Cookies. +type CookieSessionAffinity struct { + Name string `json:"name"` + Hash string `json:"hash"` } // Endpoint describes a kubernetes endpoint in an backend diff --git a/examples/affinity/cookie/nginx/README.md b/examples/affinity/cookie/nginx/README.md new file mode 100644 index 000000000..2a18f5f66 --- /dev/null +++ b/examples/affinity/cookie/nginx/README.md @@ -0,0 +1,73 @@ +# Sticky Session + +This example demonstrates how to achieve session affinity using cookies + +## Prerequisites + +You will need to make sure you Ingress targets exactly one Ingress +controller by specifying the [ingress.class annotation](/examples/PREREQUISITES.md#ingress-class), +and that you have an ingress controller [running](/examples/deployment) in your cluster. + +You will also need to deploy multiple replicas of your application that show up as endpoints for the Service referenced in the Ingress object, to test session stickyness. +Using a deployment with only one replica doesn't set the 'sticky' cookie. + +## Deployment + +Session stickyness is achieved through 3 annotations on the Ingress, as shown in the [example](sticky-ingress.yaml). + +|Name|Description|Values| +|ingress.kubernetes.io/affinity|Sets the affinity type|string (in NGINX only ``cookie`` is possible| +|ingress.kubernetes.io/session-cookie-name|Name of the cookie that will be used|string (default to route)| +|ingress.kubernetes.io/session-cookie-hash|Type of hash that will be used in cookie value|sha1/md5/index| + +You can create the ingress to test this + +```console +$ kubectl create -f sticky-ingress.yaml +``` + +## Validation + +You can confirm that the Ingress works. + +```console +$ kubectl describe ing nginx-test +Name: nginx-test +Namespace: default +Address: +Default backend: default-http-backend:80 (10.180.0.4:8080,10.240.0.2:8080) +Rules: + Host Path Backends + ---- ---- -------- + stickyingress.example.com + / nginx-service:80 () +Annotations: + affinity: cookie + session-cookie-hash: sha1 + session-cookie-name: route +Events: + FirstSeen LastSeen Count From SubObjectPath Type Reason Message + --------- -------- ----- ---- ------------- -------- ------ ------- + 7s 7s 1 {nginx-ingress-controller } Normal CREATE default/nginx-test + + +$ curl -I http://stickyingress.example.com +HTTP/1.1 200 OK +Server: nginx/1.11.9 +Date: Fri, 10 Feb 2017 14:11:12 GMT +Content-Type: text/html +Content-Length: 612 +Connection: keep-alive +Set-Cookie: route=a9907b79b248140b56bb13723f72b67697baac3d; Path=/; HttpOnly +Last-Modified: Tue, 24 Jan 2017 14:02:19 GMT +ETag: "58875e6b-264" +Accept-Ranges: bytes +``` +In the example above, you can see a line containing the 'Set-Cookie: route' setting the right defined stickness cookie. +This cookie is created by NGINX containing the hash of the used upstream in that request. +If the user changes this cookie, NGINX creates a new one and redirect the user to another upstream. + +If the backend pool grows up NGINX will keep sending the requests through the same server of the first request, even if it's overloaded. + +When the backend server is removed, the requests are then re-routed to another upstream server and NGINX creates a new cookie, as the previous hash became invalid. + diff --git a/examples/stickysession/nginx/sticky-ingress.yaml b/examples/affinity/cookie/nginx/sticky-ingress.yaml similarity index 66% rename from examples/stickysession/nginx/sticky-ingress.yaml rename to examples/affinity/cookie/nginx/sticky-ingress.yaml index fe7dd42b3..69beea75e 100644 --- a/examples/stickysession/nginx/sticky-ingress.yaml +++ b/examples/affinity/cookie/nginx/sticky-ingress.yaml @@ -4,9 +4,9 @@ metadata: name: nginx-test annotations: kubernetes.io/ingress.class: "nginx" - ingress.kubernetes.io/sticky-enabled: "true" - ingress.kubernetes.io/sticky-name: "route" - ingress.kubernetes.io/sticky-hash: "sha1" + ingress.kubernetes.io/affinity: "cookie" + ingress.kubernetes.io/session-cookie-name: "route" + ingress.kubernetes.io/session-cookie-hash: "sha1" spec: rules: diff --git a/examples/stickysession/nginx/README.md b/examples/stickysession/nginx/README.md deleted file mode 100644 index ba5bf9abc..000000000 --- a/examples/stickysession/nginx/README.md +++ /dev/null @@ -1,58 +0,0 @@ -# Sticky Session - -This example demonstrates how to Stickness in a Ingress. - -## Prerequisites - -You will need to make sure you Ingress targets exactly one Ingress -controller by specifying the [ingress.class annotation](/examples/PREREQUISITES.md#ingress-class), -and that you have an ingress controller [running](/examples/deployment) in your cluster. - -Also, you need to have a deployment with replica > 1. Using a deployment with only one replica doesn't set the 'sticky' cookie. - -## Deployment - -The following command instructs the controller to set Stickness in all Upstreams of an Ingress - -```console -$ kubectl create -f sticky-ingress.yaml -``` - -## Validation - -You can confirm that the Ingress works. - -```console -$ kubectl describe ing nginx-test -Name: nginx-test -Namespace: default -Address: -Default backend: default-http-backend:80 (10.180.0.4:8080,10.240.0.2:8080) -Rules: - Host Path Backends - ---- ---- -------- - stickyingress.example.com - / nginx-service:80 () -Annotations: - sticky-enabled: true - sticky-hash: sha1 - sticky-name: route -Events: - FirstSeen LastSeen Count From SubObjectPath Type Reason Message - --------- -------- ----- ---- ------------- -------- ------ ------- - 7s 7s 1 {nginx-ingress-controller } Normal CREATE default/nginx-test - - -$ curl -I http://stickyingress.example.com -HTTP/1.1 200 OK -Server: nginx/1.11.9 -Date: Fri, 10 Feb 2017 14:11:12 GMT -Content-Type: text/html -Content-Length: 612 -Connection: keep-alive -Set-Cookie: route=a9907b79b248140b56bb13723f72b67697baac3d; Path=/; HttpOnly -Last-Modified: Tue, 24 Jan 2017 14:02:19 GMT -ETag: "58875e6b-264" -Accept-Ranges: bytes -``` -In the example avove, you can see a line containing the 'Set-Cookie: route' setting the right defined stickness cookie. \ No newline at end of file From 0161ae43d958035c2d79873cdb738532f466dbda Mon Sep 17 00:00:00 2001 From: Ricardo Pchevuzinske Katz Date: Tue, 14 Feb 2017 08:49:10 -0200 Subject: [PATCH 5/8] Improve in documentation and naming case --- .../annotations/sessionaffinity/main.go | 30 +++++++++---------- .../annotations/sessionaffinity/main_test.go | 4 +-- .../ingress/controller/annotations_test.go | 10 +++---- core/pkg/ingress/controller/controller.go | 4 +-- examples/affinity/cookie/nginx/README.md | 1 + 5 files changed, 25 insertions(+), 24 deletions(-) diff --git a/core/pkg/ingress/annotations/sessionaffinity/main.go b/core/pkg/ingress/annotations/sessionaffinity/main.go index 551cde2a3..3cf5181e9 100644 --- a/core/pkg/ingress/annotations/sessionaffinity/main.go +++ b/core/pkg/ingress/annotations/sessionaffinity/main.go @@ -31,7 +31,7 @@ const ( // If a cookie with this name exists, // its value is used as an index into the list of available backends. annotationAffinityCookieName = "ingress.kubernetes.io/session-cookie-name" - defaultAffinityCookieName = "route" + defaultAffinityCookieName = "INGRESSCOOKIE" // This is the algorithm used by nginx to generate a value for the session cookie, if // one isn't supplied and affintiy is set to "cookie". annotationAffinityCookieHash = "ingress.kubernetes.io/session-cookie-hash" @@ -45,24 +45,21 @@ var ( // AffinityConfig describes the per ingress session affinity config type AffinityConfig struct { // The type of affinity that will be used - AffinityType string `json:"type"` - CookieAffinityConfig CookieAffinityConfig `json:"cookieconfig"` + AffinityType string `json:"type"` + CookieConfig } -// CookieAffinityConfig describes the Config of cookie type affinity -type CookieAffinityConfig struct { +// CookieConfig describes the Config of cookie type affinity +type CookieConfig struct { // The name of the cookie that will be used in case of cookie affinity type. Name string `json:"name"` // The hash that will be used to encode the cookie in case of cookie affinity type Hash string `json:"hash"` } -type affinity struct { -} - // CookieAffinityParse gets the annotation values related to Cookie Affinity // It also sets default values when no value or incorrect value is found -func CookieAffinityParse(ing *extensions.Ingress) *CookieAffinityConfig { +func CookieAffinityParse(ing *extensions.Ingress) *CookieConfig { sn, err := parser.GetStringAnnotation(annotationAffinityCookieName, ing) @@ -78,7 +75,7 @@ func CookieAffinityParse(ing *extensions.Ingress) *CookieAffinityConfig { sh = defaultAffinityCookieHash } - return &CookieAffinityConfig{ + return &CookieConfig{ Name: sn, Hash: sh, } @@ -89,19 +86,22 @@ func NewParser() parser.IngressAnnotation { return affinity{} } +type affinity struct { +} + // ParseAnnotations parses the annotations contained in the ingress // rule used to configure the affinity directives func (a affinity) Parse(ing *extensions.Ingress) (interface{}, error) { - var cookieAffinityConfig *CookieAffinityConfig - cookieAffinityConfig = &CookieAffinityConfig{} + var cookieAffinityConfig *CookieConfig + cookieAffinityConfig = &CookieConfig{} // Check the type of affinity that will be used at, err := parser.GetStringAnnotation(annotationAffinityType, ing) if err != nil { at = "" } - //cookieAffinityConfig = CookieAffinityParse(ing) + switch at { case "cookie": cookieAffinityConfig = CookieAffinityParse(ing) @@ -111,8 +111,8 @@ func (a affinity) Parse(ing *extensions.Ingress) (interface{}, error) { } return &AffinityConfig{ - AffinityType: at, - CookieAffinityConfig: *cookieAffinityConfig, + AffinityType: at, + CookieConfig: *cookieAffinityConfig, }, nil } diff --git a/core/pkg/ingress/annotations/sessionaffinity/main_test.go b/core/pkg/ingress/annotations/sessionaffinity/main_test.go index b18f86835..b898813ec 100644 --- a/core/pkg/ingress/annotations/sessionaffinity/main_test.go +++ b/core/pkg/ingress/annotations/sessionaffinity/main_test.go @@ -65,7 +65,7 @@ func TestIngressAffinityCookieConfig(t *testing.T) { data := map[string]string{} data[annotationAffinityType] = "cookie" data[annotationAffinityCookieHash] = "sha123" - data[annotationAffinityCookieName] = "route" + data[annotationAffinityCookieName] = "INGRESSCOOKIE" ing.SetAnnotations(data) affin, _ := NewParser().Parse(ing) @@ -82,7 +82,7 @@ func TestIngressAffinityCookieConfig(t *testing.T) { t.Errorf("expected md5 as sticky-hash but returned %v", nginxAffinity.CookieAffinityConfig.Hash) } - if nginxAffinity.CookieAffinityConfig.Name != "route" { + if nginxAffinity.CookieAffinityConfig.Name != "INGRESSCOOKIE" { t.Errorf("expected route as sticky-name but returned %v", nginxAffinity.CookieAffinityConfig.Name) } } diff --git a/core/pkg/ingress/controller/annotations_test.go b/core/pkg/ingress/controller/annotations_test.go index 6927f2733..1b3b9b08d 100644 --- a/core/pkg/ingress/controller/annotations_test.go +++ b/core/pkg/ingress/controller/annotations_test.go @@ -195,7 +195,7 @@ func TestAffinitySession(t *testing.T) { }{ {map[string]string{annotationAffinityType: "cookie", annotationAffinityCookieHash: "md5", annotationAffinityCookieName: "route"}, "cookie", "md5", "route"}, {map[string]string{annotationAffinityType: "cookie", annotationAffinityCookieHash: "xpto", annotationAffinityCookieName: "route1"}, "cookie", "md5", "route1"}, - {map[string]string{annotationAffinityType: "cookie", annotationAffinityCookieHash: "", annotationAffinityCookieName: ""}, "cookie", "md5", "route"}, + {map[string]string{annotationAffinityType: "cookie", annotationAffinityCookieHash: "", annotationAffinityCookieName: ""}, "cookie", "md5", "INGRESSCOOKIE"}, {map[string]string{}, "", "", ""}, {nil, "", "", ""}, } @@ -209,12 +209,12 @@ func TestAffinitySession(t *testing.T) { continue } - if r.CookieAffinityConfig.Hash != foo.hash { - t.Errorf("Returned %v but expected %v for Hash", r.CookieAffinityConfig.Hash, foo.hash) + if r.CookieConfig.Hash != foo.hash { + t.Errorf("Returned %v but expected %v for Hash", r.CookieConfig.Hash, foo.hash) } - if r.CookieAffinityConfig.Name != foo.name { - t.Errorf("Returned %v but expected %v for Name", r.CookieAffinityConfig.Name, foo.name) + if r.CookieConfig.Name != foo.name { + t.Errorf("Returned %v but expected %v for Name", r.CookieConfig.Name, foo.name) } } } diff --git a/core/pkg/ingress/controller/controller.go b/core/pkg/ingress/controller/controller.go index 1eb0b6f2c..2a91640fc 100644 --- a/core/pkg/ingress/controller/controller.go +++ b/core/pkg/ingress/controller/controller.go @@ -743,8 +743,8 @@ func (ic *GenericController) createUpstreams(data []interface{}) map[string]*ing if upstreams[name].SessionAffinity.AffinityType == "" { upstreams[name].SessionAffinity.AffinityType = affinity.AffinityType if affinity.AffinityType == "cookie" { - upstreams[name].SessionAffinity.CookieSessionAffinity.Name = affinity.CookieAffinityConfig.Name - upstreams[name].SessionAffinity.CookieSessionAffinity.Hash = affinity.CookieAffinityConfig.Hash + upstreams[name].SessionAffinity.CookieSessionAffinity.Name = affinity.CookieConfig.Name + upstreams[name].SessionAffinity.CookieSessionAffinity.Hash = affinity.CookieConfig.Hash } } diff --git a/examples/affinity/cookie/nginx/README.md b/examples/affinity/cookie/nginx/README.md index 2a18f5f66..ea5a3d3b2 100644 --- a/examples/affinity/cookie/nginx/README.md +++ b/examples/affinity/cookie/nginx/README.md @@ -16,6 +16,7 @@ Using a deployment with only one replica doesn't set the 'sticky' cookie. Session stickyness is achieved through 3 annotations on the Ingress, as shown in the [example](sticky-ingress.yaml). |Name|Description|Values| +| --- | --- | --- | |ingress.kubernetes.io/affinity|Sets the affinity type|string (in NGINX only ``cookie`` is possible| |ingress.kubernetes.io/session-cookie-name|Name of the cookie that will be used|string (default to route)| |ingress.kubernetes.io/session-cookie-hash|Type of hash that will be used in cookie value|sha1/md5/index| From b06ead1ea3da8f005bda50d14806ac44a5399755 Mon Sep 17 00:00:00 2001 From: Ricardo Pchevuzinske Katz Date: Wed, 15 Feb 2017 11:50:10 -0200 Subject: [PATCH 6/8] Corrects the affinity test --- core/pkg/ingress/annotations/sessionaffinity/main_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/pkg/ingress/annotations/sessionaffinity/main_test.go b/core/pkg/ingress/annotations/sessionaffinity/main_test.go index b898813ec..3a3a17200 100644 --- a/core/pkg/ingress/annotations/sessionaffinity/main_test.go +++ b/core/pkg/ingress/annotations/sessionaffinity/main_test.go @@ -78,11 +78,11 @@ func TestIngressAffinityCookieConfig(t *testing.T) { t.Errorf("expected cookie as sticky-type but returned %v", nginxAffinity.AffinityType) } - if nginxAffinity.CookieAffinityConfig.Hash != "md5" { - t.Errorf("expected md5 as sticky-hash but returned %v", nginxAffinity.CookieAffinityConfig.Hash) + if nginxAffinity.CookieConfig.Hash != "md5" { + t.Errorf("expected md5 as sticky-hash but returned %v", nginxAffinity.CookieConfig.Hash) } - if nginxAffinity.CookieAffinityConfig.Name != "INGRESSCOOKIE" { - t.Errorf("expected route as sticky-name but returned %v", nginxAffinity.CookieAffinityConfig.Name) + if nginxAffinity.CookieConfig.Name != "INGRESSCOOKIE" { + t.Errorf("expected route as sticky-name but returned %v", nginxAffinity.CookieConfig.Name) } } From e5c9c788a5588772775a095dd00a376b3b3b8f65 Mon Sep 17 00:00:00 2001 From: Ricardo Pchevuzinske Katz Date: Thu, 16 Feb 2017 08:31:01 -0200 Subject: [PATCH 7/8] Correct the configuration.md reference to annotations --- controllers/nginx/configuration.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/controllers/nginx/configuration.md b/controllers/nginx/configuration.md index 2b04e55dd..53640f94f 100644 --- a/controllers/nginx/configuration.md +++ b/controllers/nginx/configuration.md @@ -51,9 +51,9 @@ The following annotations are supported: |[ingress.kubernetes.io/upstream-max-fails](#custom-nginx-upstream-checks)|number| |[ingress.kubernetes.io/upstream-fail-timeout](#custom-nginx-upstream-checks)|number| |[ingress.kubernetes.io/whitelist-source-range](#whitelist-source-range)|CIDR| -|[ingress.kubernetes.io/sticky-enabled](#sticky-session)|true or false| -|[ingress.kubernetes.io/sticky-name](#sticky-session)|string| -|[ingress.kubernetes.io/sticky-hash](#sticky-session)|string| +|[ingress.kubernetes.io/affinity](#session-affinity)|true or false| +|[ingress.kubernetes.io/session-cookie-name](#cookie-affinity)|string| +|[ingress.kubernetes.io/session-cookie-hash](#cookie-affinity)|string| @@ -180,7 +180,7 @@ To configure this setting globally for all Ingress rules, the `whitelist-source- Please check the [whitelist](examples/affinity/cookie/nginx/README.md) example. -### Sticky Session +### Session Affinity The annotation `ingress.kubernetes.io/affinity` enables and sets the affinity type in all Upstreams of an Ingress. This way, a request will always be directed to the same upstream server. From 4be502efaa9e81f3dcba180e3f7db6bdb174b696 Mon Sep 17 00:00:00 2001 From: Ricardo Pchevuzinske Katz Date: Thu, 16 Feb 2017 08:40:35 -0200 Subject: [PATCH 8/8] Adds a new situation in affinity config example --- examples/affinity/cookie/nginx/README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/examples/affinity/cookie/nginx/README.md b/examples/affinity/cookie/nginx/README.md index ea5a3d3b2..51aeec310 100644 --- a/examples/affinity/cookie/nginx/README.md +++ b/examples/affinity/cookie/nginx/README.md @@ -72,3 +72,6 @@ If the backend pool grows up NGINX will keep sending the requests through the sa When the backend server is removed, the requests are then re-routed to another upstream server and NGINX creates a new cookie, as the previous hash became invalid. +When you have more than one Ingress Object pointing to the same Service, but one containing affinity configuration and other don't, the first created Ingress will be used. +This means that you can face the situation that you've configured Session Affinity in one Ingress and it doesn't reflects in NGINX configuration, because there is another Ingress Object pointing to the same service that doesn't configure this. +