configuring load balancing per ingress (#2167)

* configure load balancing through a ingress annotation

* update docs

docs/user-guide/annotations.md

@@ -58,6 +58,7 @@ The following annotations are supported:
 |[nginx.ingress.kubernetes.io/upstream-max-fails](#custom-nginx-upstream-checks)|number|
 |[nginx.ingress.kubernetes.io/upstream-fail-timeout](#custom-nginx-upstream-checks)|number|
 |[nginx.ingress.kubernetes.io/upstream-hash-by](#custom-nginx-upstream-hashing)|string|
+|[nginx.ingress.kubernetes.io/load-balance](#custom-nginx-load-balancing)|string|
 |[nginx.ingress.kubernetes.io/upstream-vhost](#custom-nginx-upstream-vhost)|string|
 |[nginx.ingress.kubernetes.io/whitelist-source-range](#whitelist-source-range)|CIDR|
 |[nginx.ingress.kubernetes.io/proxy-buffering](#proxy-buffering)|string|
@@ -137,6 +138,11 @@ To enable consistent hashing for a backend:
 
 `nginx.ingress.kubernetes.io/upstream-hash-by`: the nginx variable, text value or any combination thereof to use for consistent hashing. For example `nginx.ingress.kubernetes.io/upstream-hash-by: "$request_uri"` to consistently hash upstream requests by the current request URI.
 
+### Custom NGINX load balancing
+
+This is similar to https://github.com/kubernetes/ingress-nginx/blob/master/docs/user-guide/configmap.md#load-balance but configures load balancing algorithm per ingress.
+Note that `nginx.ingress.kubernetes.io/upstream-hash-by` takes preference over this. If this and `nginx.ingress.kubernetes.io/upstream-hash-by` are not set then we fallback to using globally configured load balancing algorithm.
+
 ### Custom NGINX upstream vhost
 
 This configuration setting allows you to control the value for host in the following statement: `proxy_set_header Host $host`, which forms part of the location block.  This is useful if you need to call the upstream server by something other than `$host`.

internal/ingress/annotations/annotations.go

@@ -34,6 +34,7 @@ import (
 	"k8s.io/ingress-nginx/internal/ingress/annotations/defaultbackend"
 	"k8s.io/ingress-nginx/internal/ingress/annotations/healthcheck"
 	"k8s.io/ingress-nginx/internal/ingress/annotations/ipwhitelist"
+	"k8s.io/ingress-nginx/internal/ingress/annotations/loadbalancing"
 	"k8s.io/ingress-nginx/internal/ingress/annotations/log"
 	"k8s.io/ingress-nginx/internal/ingress/annotations/parser"
 	"k8s.io/ingress-nginx/internal/ingress/annotations/portinredirect"
@@ -83,6 +84,7 @@ type Ingress struct {
 	SSLPassthrough       bool
 	UsePortInRedirects   bool
 	UpstreamHashBy       string
+	LoadBalancing        string
 	UpstreamVhost        string
 	VtsFilterKey         string
 	Whitelist            ipwhitelist.SourceRange
@@ -121,6 +123,7 @@ func NewAnnotationExtractor(cfg resolver.Resolver) Extractor {
 			"SSLPassthrough":       sslpassthrough.NewParser(cfg),
 			"UsePortInRedirects":   portinredirect.NewParser(cfg),
 			"UpstreamHashBy":       upstreamhashby.NewParser(cfg),
+			"LoadBalancing":        loadbalancing.NewParser(cfg),
 			"UpstreamVhost":        upstreamvhost.NewParser(cfg),
 			"VtsFilterKey":         vtsfilterkey.NewParser(cfg),
 			"Whitelist":            ipwhitelist.NewParser(cfg),

internal/ingress/annotations/loadbalancing/main.go

@@ -0,0 +1,40 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package loadbalancing
+
+import (
+	extensions "k8s.io/api/extensions/v1beta1"
+
+	"k8s.io/ingress-nginx/internal/ingress/annotations/parser"
+	"k8s.io/ingress-nginx/internal/ingress/resolver"
+)
+
+type loadbalancing struct {
+	r resolver.Resolver
+}
+
+// NewParser creates a new CORS annotation parser
+func NewParser(r resolver.Resolver) parser.IngressAnnotation {
+	return loadbalancing{r}
+}
+
+// Parse parses the annotations contained in the ingress rule
+// used to indicate if the location/s contains a fragment of
+// configuration to be included inside the paths of the rules
+func (a loadbalancing) Parse(ing *extensions.Ingress) (interface{}, error) {
+	return parser.GetStringAnnotation("load-balance", ing)
+}

internal/ingress/annotations/loadbalancing/main_test.go

@@ -0,0 +1,61 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package loadbalancing
+
+import (
+	"testing"
+
+	api "k8s.io/api/core/v1"
+	extensions "k8s.io/api/extensions/v1beta1"
+	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/ingress-nginx/internal/ingress/annotations/parser"
+	"k8s.io/ingress-nginx/internal/ingress/resolver"
+)
+
+func TestParse(t *testing.T) {
+	annotation := parser.GetAnnotationWithPrefix("load-balance")
+
+	ap := NewParser(&resolver.Mock{})
+	if ap == nil {
+		t.Fatalf("expected a parser.IngressAnnotation but returned nil")
+	}
+
+	testCases := []struct {
+		annotations map[string]string
+		expected    string
+	}{
+		{map[string]string{annotation: "ip_hash"}, "ip_hash"},
+		{map[string]string{}, ""},
+		{nil, ""},
+	}
+
+	ing := &extensions.Ingress{
+		ObjectMeta: meta_v1.ObjectMeta{
+			Name:      "foo",
+			Namespace: api.NamespaceDefault,
+		},
+		Spec: extensions.IngressSpec{},
+	}
+
+	for _, testCase := range testCases {
+		ing.SetAnnotations(testCase.annotations)
+		result, _ := ap.Parse(ing)
+		if result != testCase.expected {
+			t.Errorf("expected %v but returned %v, annotations: %s", testCase.expected, result, testCase.annotations)
+		}
+	}
+}

internal/ingress/controller/controller.go

@@ -599,6 +599,9 @@ func (n *NGINXController) createUpstreams(data []*extensions.Ingress, du *ingres
 			if upstreams[defBackend].UpstreamHashBy == "" {
 				upstreams[defBackend].UpstreamHashBy = anns.UpstreamHashBy
 			}
+			if upstreams[defBackend].LoadBalancing == "" {
+				upstreams[defBackend].LoadBalancing = anns.LoadBalancing
+			}
 
 			svcKey := fmt.Sprintf("%v/%v", ing.GetNamespace(), ing.Spec.Backend.ServiceName)
 
@@ -654,6 +657,10 @@ func (n *NGINXController) createUpstreams(data []*extensions.Ingress, du *ingres
 					upstreams[name].UpstreamHashBy = anns.UpstreamHashBy
 				}
 
+				if upstreams[name].LoadBalancing == "" {
+					upstreams[name].LoadBalancing = anns.LoadBalancing
+				}
+
 				svcKey := fmt.Sprintf("%v/%v", ing.GetNamespace(), path.Backend.ServiceName)
 
 				// Add the service cluster endpoint as the upstream instead of individual endpoints

internal/ingress/controller/template/template.go

@@ -122,6 +122,7 @@ var (
 		"buildLocation":            buildLocation,
 		"buildAuthLocation":        buildAuthLocation,
 		"buildAuthResponseHeaders": buildAuthResponseHeaders,
+		"buildLoadBalancingConfig": buildLoadBalancingConfig,
 		"buildProxyPass":           buildProxyPass,
 		"filterRateLimits":         filterRateLimits,
 		"buildRateLimitZones":      buildRateLimitZones,
@@ -277,6 +278,31 @@ func buildLogFormatUpstream(input interface{}) string {
 	return cfg.BuildLogFormatUpstream()
 }
 
+func buildLoadBalancingConfig(b interface{}, fallbackLoadBalancing string) string {
+	backend, ok := b.(*ingress.Backend)
+	if !ok {
+		glog.Errorf("expected an '*ingress.Backend' type but %T was returned", b)
+		return ""
+	}
+
+	if backend.UpstreamHashBy != "" {
+		return "hash {{ $upstream.UpstreamHashBy }} consistent;"
+	}
+
+	if backend.LoadBalancing != "" {
+		if backend.LoadBalancing == "round_robin" {
+			return ""
+		}
+		return fmt.Sprintf("%s;", backend.LoadBalancing)
+	}
+
+	if fallbackLoadBalancing == "round_robin" {
+		return ""
+	}
+
+	return fmt.Sprintf("%s;", fallbackLoadBalancing)
+}
+
 // buildProxyPass produces the proxy pass string, if the ingress has redirects
 // (specified through the nginx.ingress.kubernetes.io/rewrite-to annotation)
 // If the annotation nginx.ingress.kubernetes.io/add-base-url:"true" is specified it will

internal/ingress/defaults/main.go

@@ -121,6 +121,9 @@ type Backend struct {
 	// http://nginx.org/en/docs/http/ngx_http_upstream_module.html#hash
 	UpstreamHashBy string `json:"upstream-hash-by"`
 
+	// Let's us choose a load balancing algorithm per ingress
+	LoadBalancing string `json:"load-balance"`
+
 	// WhitelistSourceRange allows limiting access to certain client addresses
 	// http://nginx.org/en/docs/http/ngx_http_access_module.html
 	WhitelistSourceRange []string `json:"whitelist-source-range,-"`

internal/ingress/types.go

@@ -88,6 +88,8 @@ type Backend struct {
 	SessionAffinity SessionAffinityConfig `json:"sessionAffinityConfig"`
 	// Consistent hashing by NGINX variable
 	UpstreamHashBy string `json:"upstream-hash-by,omitempty"`
+	// LB algorithm configuration per ingress
+	LoadBalancing string `json:"load-balance,omitempty"`
 }
 
 // SessionAffinityConfig describes different affinity configurations for new sessions.

internal/ingress/types_equals.go

@@ -152,6 +152,9 @@ func (b1 *Backend) Equal(b2 *Backend) bool {
 	if b1.UpstreamHashBy != b2.UpstreamHashBy {
 		return false
 	}
+	if b1.LoadBalancing != b2.LoadBalancing {
+		return false
+	}
 
 	if len(b1.Endpoints) != len(b2.Endpoints) {
 		return false

rootfs/etc/nginx/template/nginx.tmpl

@@ -321,12 +321,7 @@ http {
     {{ end }}
 
     upstream {{ $upstream.Name }} {
-        {{ if $upstream.UpstreamHashBy }}
+        {{ buildLoadBalancingConfig $upstream $cfg.LoadBalanceAlgorithm }}
-        hash {{ $upstream.UpstreamHashBy }} consistent;
-        {{ else }}
-        # Load balance algorithm; empty for round robin, which is the default
-        {{ if ne $cfg.LoadBalanceAlgorithm "round_robin" }}{{ $cfg.LoadBalanceAlgorithm }};{{ end }}
-        {{ end }}
 
         {{ if (gt $cfg.UpstreamKeepaliveConnections 0) }}
         keepalive {{ $cfg.UpstreamKeepaliveConnections }};