From 0278034bcf18bfd103f9a278688791d2d948f087 Mon Sep 17 00:00:00 2001
From: Tony Li <tonyli.lives@gmail.com>
Date: Thu, 26 Jan 2017 15:17:12 -0500
Subject: [PATCH 01/73] unittesting -> unit testing

---
 controllers/gce/controller/cluster_manager.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/controllers/gce/controller/cluster_manager.go b/controllers/gce/controller/cluster_manager.go
index c16ddb709..993644413 100644
--- a/controllers/gce/controller/cluster_manager.go
+++ b/controllers/gce/controller/cluster_manager.go
@@ -243,7 +243,7 @@ func NewClusterManager(
 	defaultHealthCheckPath string) (*ClusterManager, error) {
 
 	// TODO: Make this more resilient. Currently we create the cloud client
-	// and pass it through to all the pools. This makes unittesting easier.
+	// and pass it through to all the pools. This makes unit testing easier.
 	// However if the cloud client suddenly fails, we should try to re-create it
 	// and continue.
 	var cloud *gce.GCECloud

From 881ddba90d80c3783a955e529b8bec9720703ddb Mon Sep 17 00:00:00 2001
From: Tony Li <tonyli.lives@gmail.com>
Date: Fri, 27 Jan 2017 12:52:15 -0500
Subject: [PATCH 02/73] change arg ordering in log message

---
 controllers/gce/loadbalancers/loadbalancers.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/controllers/gce/loadbalancers/loadbalancers.go b/controllers/gce/loadbalancers/loadbalancers.go
index 948718f7f..e79e7a86e 100644
--- a/controllers/gce/loadbalancers/loadbalancers.go
+++ b/controllers/gce/loadbalancers/loadbalancers.go
@@ -383,7 +383,7 @@ func (l *L7) checkSSLCert() (err error) {
 			}
 		}
 
-		glog.Infof("Creating new sslCertificates %v for %v", l.Name, certName)
+		glog.Infof("Creating new sslCertificates %v for %v", certName, l.Name)
 		cert, err = l.cloud.CreateSslCertificate(&compute.SslCertificate{
 			Name:        certName,
 			Certificate: ingCert,

From 404e0712db30d0dd68b24fb1a34020f7ea2ba626 Mon Sep 17 00:00:00 2001
From: Tony Li <tony.li@nytimes.com>
Date: Fri, 3 Feb 2017 17:24:24 -0500
Subject: [PATCH 03/73] check for error getting cert

---
 controllers/gce/loadbalancers/loadbalancers.go | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/controllers/gce/loadbalancers/loadbalancers.go b/controllers/gce/loadbalancers/loadbalancers.go
index 948718f7f..726d6ff3b 100644
--- a/controllers/gce/loadbalancers/loadbalancers.go
+++ b/controllers/gce/loadbalancers/loadbalancers.go
@@ -367,7 +367,10 @@ func (l *L7) checkSSLCert() (err error) {
 	if l.sslCert != nil {
 		certName = l.sslCert.Name
 	}
-	cert, _ := l.cloud.GetSslCertificate(certName)
+	cert, err := l.cloud.GetSslCertificate(certName)
+	if err != nil {
+		return err
+	}
 
 	// PrivateKey is write only, so compare certs alone. We're assuming that
 	// no one will change just the key. We can remember the key and compare,

From fbdacb2a67c32cd8440fcf709ea8fac8445b6db2 Mon Sep 17 00:00:00 2001
From: Tony Li <tony.li@nytimes.com>
Date: Fri, 3 Feb 2017 18:23:07 -0500
Subject: [PATCH 04/73] comment on skipping the error check

---
 controllers/gce/loadbalancers/loadbalancers.go | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/controllers/gce/loadbalancers/loadbalancers.go b/controllers/gce/loadbalancers/loadbalancers.go
index 726d6ff3b..75d9240ab 100644
--- a/controllers/gce/loadbalancers/loadbalancers.go
+++ b/controllers/gce/loadbalancers/loadbalancers.go
@@ -367,10 +367,10 @@ func (l *L7) checkSSLCert() (err error) {
 	if l.sslCert != nil {
 		certName = l.sslCert.Name
 	}
-	cert, err := l.cloud.GetSslCertificate(certName)
-	if err != nil {
-		return err
-	}
+
+	// Skip error checking because error-ing out will retry and loop, when we
+	// should create/update the cert if there is an error or does not exist.
+	cert, _ := l.cloud.GetSslCertificate(certName)
 
 	// PrivateKey is write only, so compare certs alone. We're assuming that
 	// no one will change just the key. We can remember the key and compare,

From c83d46ef8622e2d10e25ee4b6efdbd3422758735 Mon Sep 17 00:00:00 2001
From: Manuel de Brito Fontes <aledbf@gmail.com>
Date: Tue, 7 Feb 2017 11:17:25 -0300
Subject: [PATCH 05/73] Add information about cors annotation

---
 controllers/nginx/configuration.md | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/controllers/nginx/configuration.md b/controllers/nginx/configuration.md
index 776644b6f..74d9a54fe 100644
--- a/controllers/nginx/configuration.md
+++ b/controllers/nginx/configuration.md
@@ -43,6 +43,7 @@ The following annotations are supported:
 |[ingress.kubernetes.io/auth-secret](#authentication)|string|
 |[ingress.kubernetes.io/auth-type](#authentication)|basic or digest|
 |[ingress.kubernetes.io/auth-url](#external-authentication)|string|
+|[ingress.kubernetes.io/enable-cors](#enable-cors)|true or false|
 |[ingress.kubernetes.io/limit-connections](#rate-limiting)|number|
 |[ingress.kubernetes.io/limit-rps](#rate-limiting)|number|
 |[ingress.kubernetes.io/rewrite-target](#rewrite)|URI|
@@ -120,6 +121,10 @@ ingress.kubernetes.io/auth-realm: "realm string"
 
 Please check the [auth](examples/auth/README.md) example.
 
+### Enable CORS
+
+To enable Cross-Origin Resource Sharing (CORS) in an Ingress rule add the annotation `ingress.kubernetes.io/enable-cors: "true"`. This will add a section in the server location enabling this functionality.
+For more information please check https://enable-cors.org/server_nginx.html
 
 ### External Authentication
 

From 5cc5669938108ab7429bc7eee40c18a6ba18150a Mon Sep 17 00:00:00 2001
From: Manuel de Brito Fontes <aledbf@gmail.com>
Date: Tue, 7 Feb 2017 15:13:08 -0300
Subject: [PATCH 06/73] Add support for custom proxy headers using a ConfigMap

---
 controllers/nginx/pkg/cmd/controller/nginx.go | 22 ++++++++++++++++++-
 controllers/nginx/pkg/config/config.go        |  4 ++++
 .../rootfs/etc/nginx/template/nginx.tmpl      |  7 +++++-
 core/pkg/ingress/controller/controller.go     |  8 +++++++
 core/pkg/ingress/types.go                     | 15 +++++++++++++
 5 files changed, 54 insertions(+), 2 deletions(-)

diff --git a/controllers/nginx/pkg/cmd/controller/nginx.go b/controllers/nginx/pkg/cmd/controller/nginx.go
index 9e2bb64d2..685f65dc8 100644
--- a/controllers/nginx/pkg/cmd/controller/nginx.go
+++ b/controllers/nginx/pkg/cmd/controller/nginx.go
@@ -101,6 +101,8 @@ type NGINXController struct {
 
 	configmap *api.ConfigMap
 
+	storeLister ingress.StoreLister
+
 	binary string
 }
 
@@ -276,11 +278,16 @@ Error: %v
 	return nil
 }
 
-// SetConfig ...
+// SetConfig sets the configured configmap
 func (n *NGINXController) SetConfig(cmap *api.ConfigMap) {
 	n.configmap = cmap
 }
 
+// SetListers sets the configured store listers in the generic ingress controller
+func (n *NGINXController) SetListers(lister ingress.StoreLister) {
+	n.storeLister = lister
+}
+
 // OnUpdate is called by syncQueue in https://github.com/aledbf/ingress-controller/blob/master/pkg/ingress/controller/controller.go#L82
 // periodically to keep the configuration in sync.
 //
@@ -324,7 +331,20 @@ func (n *NGINXController) OnUpdate(ingressCfg ingress.Configuration) ([]byte, er
 	// and we leave some room to avoid consuming all the FDs available
 	maxOpenFiles := (sysctlFSFileMax() / cfg.WorkerProcesses) - 1024
 
+	setHeaders := map[string]string{}
+	if cfg.ProxySetHeaders != "" {
+		cmap, exists, err := n.storeLister.ConfigMap.GetByKey(cfg.ProxySetHeaders)
+		if err != nil {
+			glog.Warningf("unexpected error reading configmap %v: %v", cfg.ProxySetHeaders, err)
+		}
+
+		if exists {
+			setHeaders = cmap.(*api.ConfigMap).Data
+		}
+	}
+
 	return n.t.Write(config.TemplateConfig{
+		ProxySetHeaders:     setHeaders,
 		MaxOpenFiles:        maxOpenFiles,
 		BacklogSize:         sysctlSomaxconn(),
 		Backends:            ingressCfg.Backends,
diff --git a/controllers/nginx/pkg/config/config.go b/controllers/nginx/pkg/config/config.go
index c3dc11331..86971edcf 100644
--- a/controllers/nginx/pkg/config/config.go
+++ b/controllers/nginx/pkg/config/config.go
@@ -152,6 +152,9 @@ type Configuration struct {
 	// of your external load balancer
 	ProxyRealIPCIDR string `json:"proxy-real-ip-cidr,omitempty"`
 
+	// Sets the name of the configmap that contains the headers to pass to the backend
+	ProxySetHeaders string `json:"proxy-set-headers,omitempty"`
+
 	// Maximum size of the server names hash tables used in server names, map directive’s values,
 	// MIME types, names of request header strings, etcd.
 	// http://nginx.org/en/docs/hash.html
@@ -283,6 +286,7 @@ func NewDefault() Configuration {
 
 // TemplateConfig contains the nginx configuration to render the file nginx.conf
 type TemplateConfig struct {
+	ProxySetHeaders     map[string]string
 	MaxOpenFiles        int
 	BacklogSize         int
 	Backends            []*ingress.Backend
diff --git a/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl b/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl
index c4f4c497f..f55a5de75 100644
--- a/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl
+++ b/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl
@@ -1,4 +1,4 @@
-{{ $cfg := .Cfg }}{{ $healthzURI := .HealthzURI }}{{ $backends := .Backends }}
+{{ $cfg := .Cfg }}{{ $healthzURI := .HealthzURI }}{{ $backends := .Backends }}{{ $proxyHeaders := .ProxySetHeaders }}
 daemon off;
 
 worker_processes {{ $cfg.WorkerProcesses }};
@@ -307,6 +307,11 @@ http {
             # https://www.nginx.com/blog/mitigating-the-httpoxy-vulnerability-with-nginx/
             proxy_set_header Proxy                  "";
 
+            # Custom headers
+            {{ range $k, $v := $proxyHeaders }}
+            proxy_set_header {{ $k }}                    "{{ $v }}";
+            {{ end }}
+
             proxy_connect_timeout                   {{ $location.Proxy.ConnectTimeout }}s;
             proxy_send_timeout                      {{ $location.Proxy.SendTimeout }}s;
             proxy_read_timeout                      {{ $location.Proxy.ReadTimeout }}s;
diff --git a/core/pkg/ingress/controller/controller.go b/core/pkg/ingress/controller/controller.go
index 0fecef961..d12639aed 100644
--- a/core/pkg/ingress/controller/controller.go
+++ b/core/pkg/ingress/controller/controller.go
@@ -304,6 +304,14 @@ func newIngressController(config *Configuration) *GenericController {
 
 	ic.annotations = newAnnotationExtractor(ic)
 
+	ic.cfg.Backend.SetListers(ingress.StoreLister{
+		Ingress:   ic.ingLister,
+		Service:   ic.svcLister,
+		Endpoint:  ic.endpLister,
+		Secret:    ic.secrLister,
+		ConfigMap: ic.mapLister,
+	})
+
 	return &ic
 }
 
diff --git a/core/pkg/ingress/types.go b/core/pkg/ingress/types.go
index 4891995e7..3b7413831 100644
--- a/core/pkg/ingress/types.go
+++ b/core/pkg/ingress/types.go
@@ -18,8 +18,10 @@ package ingress
 
 import (
 	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/client/cache"
 	"k8s.io/kubernetes/pkg/healthz"
 
+	cache_store "k8s.io/ingress/core/pkg/cache"
 	"k8s.io/ingress/core/pkg/ingress/annotations/auth"
 	"k8s.io/ingress/core/pkg/ingress/annotations/authreq"
 	"k8s.io/ingress/core/pkg/ingress/annotations/ipwhitelist"
@@ -81,6 +83,9 @@ type Controller interface {
 	OnUpdate(Configuration) ([]byte, error)
 	// ConfigMap content of --configmap
 	SetConfig(*api.ConfigMap)
+	// SetListers allows the access of store listers present in the generic controller
+	// This avoid the use of the kubernetes client.
+	SetListers(StoreLister)
 	// BackendDefaults returns the minimum settings required to configure the
 	// communication to endpoints
 	BackendDefaults() defaults.Backend
@@ -88,6 +93,16 @@ type Controller interface {
 	Info() *BackendInfo
 }
 
+// StoreLister returns the configured stores for ingresses, services,
+// endpoints, secrets and configmaps.
+type StoreLister struct {
+	Ingress   cache_store.StoreToIngressLister
+	Service   cache.StoreToServiceLister
+	Endpoint  cache.StoreToEndpointsLister
+	Secret    cache_store.StoreToSecretsLister
+	ConfigMap cache_store.StoreToConfigmapLister
+}
+
 // BackendInfo returns information about the backend.
 // This fields contains information that helps to track issues or to
 // map the running ingress controller to source code

From 8e0985e6355060a64a8b5ddc19d286bfb1d004e6 Mon Sep 17 00:00:00 2001
From: Manuel de Brito Fontes <aledbf@gmail.com>
Date: Tue, 7 Feb 2017 18:04:15 -0300
Subject: [PATCH 07/73] Add example using custom headers

---
 examples/README.md                            |  4 ++
 .../custom-headers/nginx/README.md            | 40 ++++++++++++++
 .../custom-headers/nginx/default-backend.yaml | 51 ++++++++++++++++++
 .../nginx/nginx-ingress-controller.yaml       | 52 +++++++++++++++++++
 4 files changed, 147 insertions(+)
 create mode 100644 examples/customization/custom-headers/nginx/README.md
 create mode 100644 examples/customization/custom-headers/nginx/default-backend.yaml
 create mode 100644 examples/customization/custom-headers/nginx/nginx-ingress-controller.yaml

diff --git a/examples/README.md b/examples/README.md
index 01d842eb2..aa2490a9b 100644
--- a/examples/README.md
+++ b/examples/README.md
@@ -75,4 +75,8 @@ Name | Description | Platform   | Complexity Level
 -----| ----------- | ---------- | ----------------
 Dummy  | A simple dummy controller that logs updates | * | Advanced
 
+## Custommization
 
+Name | Description | Platform   | Complexity Level
+-----| ----------- | ---------- | ----------------
+custom-headers  | set custom headers before send traffic to backends  | nginx | Advanced
diff --git a/examples/customization/custom-headers/nginx/README.md b/examples/customization/custom-headers/nginx/README.md
new file mode 100644
index 000000000..497545781
--- /dev/null
+++ b/examples/customization/custom-headers/nginx/README.md
@@ -0,0 +1,40 @@
+# Deploying the Nginx Ingress controller
+
+This example aims to demonstrate the deployment of an nginx ingress controller and
+use a ConfigMap to configure a custom list of headers to be passed to the upstream
+server
+
+## Default Backend
+
+The default backend is a Service capable of handling all url paths and hosts the
+nginx controller doesn't understand. This most basic implementation just returns
+a 404 page:
+
+```console
+$ kubectl apply -f default-backend.yaml
+deployment "default-http-backend" created
+service "default-http-backend" created
+
+$ kubectl -n kube-system get po
+NAME                                    READY     STATUS    RESTARTS   AGE
+default-http-backend-2657704409-qgwdd   1/1       Running   0          28s
+```
+
+## Controller
+
+You can deploy the controller as follows:
+
+```console
+$ kubectl apply -f nginx-ingress-controller.yaml
+deployment "nginx-ingress-controller" created
+
+$ kubectl -n kube-system get po
+NAME                                       READY     STATUS    RESTARTS   AGE
+default-http-backend-2657704409-qgwdd      1/1       Running   0          2m
+nginx-ingress-controller-873061567-4n3k2   1/1       Running   0          42s
+```
+
+Note the default settings of this controller:
+* serves a `/healthz` url on port 10254, as both a liveness and readiness probe
+* takes a `--default-backend-service` argument pointing to the Service created above
+
diff --git a/examples/customization/custom-headers/nginx/default-backend.yaml b/examples/customization/custom-headers/nginx/default-backend.yaml
new file mode 100644
index 000000000..3c40989a3
--- /dev/null
+++ b/examples/customization/custom-headers/nginx/default-backend.yaml
@@ -0,0 +1,51 @@
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: default-http-backend
+  labels:
+    k8s-app: default-http-backend
+  namespace: kube-system
+spec:
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        k8s-app: default-http-backend
+    spec:
+      terminationGracePeriodSeconds: 60
+      containers:
+      - name: default-http-backend
+        # Any image is permissable as long as:
+        # 1. It serves a 404 page at /
+        # 2. It serves 200 on a /healthz endpoint
+        image: gcr.io/google_containers/defaultbackend:1.0
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 8080
+            scheme: HTTP
+          initialDelaySeconds: 30
+          timeoutSeconds: 5
+        ports:
+        - containerPort: 8080
+        resources:
+          limits:
+            cpu: 10m
+            memory: 20Mi
+          requests:
+            cpu: 10m
+            memory: 20Mi
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: default-http-backend
+  namespace: kube-system
+  labels:
+    k8s-app: default-http-backend
+spec:
+  ports:
+  - port: 80
+    targetPort: 8080
+  selector:
+    k8s-app: default-http-backend
diff --git a/examples/customization/custom-headers/nginx/nginx-ingress-controller.yaml b/examples/customization/custom-headers/nginx/nginx-ingress-controller.yaml
new file mode 100644
index 000000000..7ee5e797e
--- /dev/null
+++ b/examples/customization/custom-headers/nginx/nginx-ingress-controller.yaml
@@ -0,0 +1,52 @@
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: nginx-ingress-controller
+  labels:
+    k8s-app: nginx-ingress-controller
+  namespace: kube-system
+spec:
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        k8s-app: nginx-ingress-controller
+    spec:
+      # hostNetwork makes it possible to use ipv6 and to preserve the source IP correctly regardless of docker configuration
+      # however, it is not a hard dependency of the nginx-ingress-controller itself and it may cause issues if port 10254 already is taken on the host
+      # that said, since hostPort is broken on CNI (https://github.com/kubernetes/kubernetes/issues/31307) we have to use hostNetwork where CNI is used
+      # like with kubeadm
+      # hostNetwork: true
+      terminationGracePeriodSeconds: 60
+      containers:
+      - image: gcr.io/google_containers/nginx-ingress-controller:0.9.0-beta.1
+        name: nginx-ingress-controller
+        readinessProbe:
+          httpGet:
+            path: /healthz
+            port: 10254
+            scheme: HTTP
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 10254
+            scheme: HTTP
+          initialDelaySeconds: 10
+          timeoutSeconds: 1
+        ports:
+        - containerPort: 80
+          hostPort: 80
+        - containerPort: 443
+          hostPort: 443
+        env:
+          - name: POD_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.name
+          - name: POD_NAMESPACE
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.namespace
+        args:
+        - /nginx-ingress-controller
+        - --default-backend-service=$(POD_NAMESPACE)/default-http-backend

From 8cb6a6693e42e72f60a06f633cf892ae2c6d0f47 Mon Sep 17 00:00:00 2001
From: Joao Morais <jcmoraisjr@gmail.com>
Date: Wed, 8 Feb 2017 07:24:17 -0200
Subject: [PATCH 08/73] Fix servicePort on HAProxy Ingress docs

---
 examples/deployment/haproxy/README.md                     | 2 +-
 examples/tls-termination/haproxy/ingress-tls-default.yaml | 2 +-
 examples/tls-termination/haproxy/ingress-tls-foobar.yaml  | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/examples/deployment/haproxy/README.md b/examples/deployment/haproxy/README.md
index f33769362..ca408b539 100644
--- a/examples/deployment/haproxy/README.md
+++ b/examples/deployment/haproxy/README.md
@@ -89,7 +89,7 @@ spec:
       - path: /
         backend:
           serviceName: http-svc
-          servicePort: 80
+          servicePort: 8080
 EOF
 ```
 
diff --git a/examples/tls-termination/haproxy/ingress-tls-default.yaml b/examples/tls-termination/haproxy/ingress-tls-default.yaml
index f546fbd83..7c5034645 100644
--- a/examples/tls-termination/haproxy/ingress-tls-default.yaml
+++ b/examples/tls-termination/haproxy/ingress-tls-default.yaml
@@ -13,4 +13,4 @@ spec:
       - path: /
         backend:
           serviceName: http-svc
-          servicePort: 80
+          servicePort: 8080
diff --git a/examples/tls-termination/haproxy/ingress-tls-foobar.yaml b/examples/tls-termination/haproxy/ingress-tls-foobar.yaml
index 24cb527c7..e15f9428b 100644
--- a/examples/tls-termination/haproxy/ingress-tls-foobar.yaml
+++ b/examples/tls-termination/haproxy/ingress-tls-foobar.yaml
@@ -14,4 +14,4 @@ spec:
       - path: /
         backend:
           serviceName: http-svc
-          servicePort: 80
+          servicePort: 8080

From 67f1e77b085b41f21b0eda5e78701cbb71b0ae56 Mon Sep 17 00:00:00 2001
From: Joao Morais <jcmoraisjr@gmail.com>
Date: Wed, 8 Feb 2017 07:26:31 -0200
Subject: [PATCH 09/73] Minor fix on console cmd of HAProxy docs

---
 examples/deployment/haproxy/README.md      | 2 +-
 examples/tls-termination/haproxy/README.md | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/examples/deployment/haproxy/README.md b/examples/deployment/haproxy/README.md
index ca408b539..b03541e66 100644
--- a/examples/deployment/haproxy/README.md
+++ b/examples/deployment/haproxy/README.md
@@ -147,5 +147,5 @@ haproxy-ingress-2556761959-tv20k           1/1       Running   0          9m
 ...
 
 $ kubectl logs haproxy-ingress-2556761959-tv20k
-$ kubectl describe haproxy-ingress-2556761959-tv20k
+$ kubectl describe pod/haproxy-ingress-2556761959-tv20k
 ```
diff --git a/examples/tls-termination/haproxy/README.md b/examples/tls-termination/haproxy/README.md
index 2393ef2c2..e019a00ee 100644
--- a/examples/tls-termination/haproxy/README.md
+++ b/examples/tls-termination/haproxy/README.md
@@ -1,4 +1,4 @@
-# TLS termination
+# HAProxy Ingress TLS termination
 
 ## Prerequisites
 
@@ -102,13 +102,13 @@ Here is the difference:
 Now `foo.bar` certificate should be used to terminate TLS:
 
 ```console
-openssl s_client -connect 172.17.4.99:31692
+$ openssl s_client -connect 172.17.4.99:31692
 ...
 subject=/CN=localhost
 issuer=/CN=localhost
 ---
 
-openssl s_client -connect 172.17.4.99:31692 -servername foo.bar
+$ openssl s_client -connect 172.17.4.99:31692 -servername foo.bar
 ...
 subject=/CN=foo.bar
 issuer=/CN=foo.bar

From 24d9aada1171d8f2cca163c37fdb797b69002731 Mon Sep 17 00:00:00 2001
From: bprashanth <beeps@google.com>
Date: Wed, 8 Feb 2017 13:54:27 -0800
Subject: [PATCH 10/73] Set balancing mode

---
 controllers/gce/backends/backends.go | 77 +++++++++++++++++++++++-----
 1 file changed, 64 insertions(+), 13 deletions(-)

diff --git a/controllers/gce/backends/backends.go b/controllers/gce/backends/backends.go
index 18686b968..3f2e9ca35 100644
--- a/controllers/gce/backends/backends.go
+++ b/controllers/gce/backends/backends.go
@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"net/http"
 	"strconv"
+	"strings"
 	"time"
 
 	"k8s.io/kubernetes/pkg/util/sets"
@@ -32,6 +33,41 @@ import (
 	"k8s.io/ingress/controllers/gce/utils"
 )
 
+// BalancingMode represents the loadbalancing configuration of an individual
+// Backend in a BackendService. This is *effectively* a cluster wide setting
+// since you can't mix modes across Backends pointing to the same IG, and you
+// can't have a single node in more than 1 loadbalanced IG.
+type BalancingMode string
+
+const (
+	// Rate balances incoming requests based on observed RPS.
+	// As of this writing, it's the only balancing mode supported by GCE's
+	// internal LB. This setting doesn't make sense for Kubernets clusters
+	// because requests can get proxied between instance groups in different
+	// zones by kube-proxy without GCE even knowing it. Setting equal RPS on
+	// all IGs should achieve roughly equal distribution of requests.
+	Rate BalancingMode = "RATE"
+	// Utilization balances incoming requests based on observed utilization.
+	// This mode is only useful if you want to divert traffic away from IGs
+	// running other compute intensive workloads. Utilization statistics are
+	// aggregated per instances, not per container, and requests can get proxied
+	// between instance groups in different zones by kube-proxy without GCE even
+	// knowing about it.
+	Utilization BalancingMode = "UTILIZATION"
+	// Connections balances incoming requests based on a connection counter.
+	// This setting currently doesn't make sense for Kubernetes clusters,
+	// because we use NodePort Services as HTTP LB backends, so GCE's connection
+	// counters don't accurately represent connections per container.
+	Connections BalancingMode = "CONNECTION"
+)
+
+// maxRPS is the RPS setting for all Backends with BalancingMode RATE. The exact
+// value doesn't matter, as long as it's the same for all Backends. Requests
+// received by GCLB above this RPS are NOT dropped, GCLB continues to distribute
+// them across IGs.
+// TODO: Should this be math.MaxInt64?
+const maxRPS = 1
+
 // Backends implements BackendPool.
 type Backends struct {
 	cloud         BackendServices
@@ -116,20 +152,35 @@ func (b *Backends) create(igs []*compute.InstanceGroup, namedPort *compute.Named
 	if err != nil {
 		return nil, err
 	}
-	// Create a new backend
-	backend := &compute.BackendService{
-		Name:     name,
-		Protocol: "HTTP",
-		Backends: getBackendsForIGs(igs),
-		// Api expects one, means little to kubernetes.
-		HealthChecks: []string{hc.SelfLink},
-		Port:         namedPort.Port,
-		PortName:     namedPort.Name,
+	errs := []string{}
+	for _, bm := range []BalancingMode{Rate, Utilization} {
+		backends := getBackendsForIGs(igs)
+		for _, b := range backends {
+			switch bm {
+			case Rate:
+				b.MaxRate = maxRPS
+			default:
+				// TODO: Set utilization and connection limits when we accept them
+				// as valid fields.
+			}
+			b.BalancingMode = string(bm)
+		}
+		// Create a new backend
+		backend := &compute.BackendService{
+			Name:         name,
+			Protocol:     "HTTP",
+			Backends:     backends,
+			HealthChecks: []string{hc.SelfLink},
+			Port:         namedPort.Port,
+			PortName:     namedPort.Name,
+		}
+		if err := b.cloud.CreateBackendService(backend); err != nil {
+			glog.Infof("Error creating backend service with balancing mode %v", bm)
+			errs = append(errs, fmt.Sprintf("%v", err))
+		}
+		return b.Get(namedPort.Port)
 	}
-	if err := b.cloud.CreateBackendService(backend); err != nil {
-		return nil, err
-	}
-	return b.Get(namedPort.Port)
+	return nil, fmt.Errorf("%v", strings.Join(errs, "\n"))
 }
 
 // Add will get or create a Backend for the given port.

From bc8b658a5c72b0074fd2eb0b308f414b987ea822 Mon Sep 17 00:00:00 2001
From: bprashanth <beeps@google.com>
Date: Wed, 8 Feb 2017 14:38:51 -0800
Subject: [PATCH 11/73] Be more specific about the type of error to retry on

---
 controllers/gce/backends/backends.go | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/controllers/gce/backends/backends.go b/controllers/gce/backends/backends.go
index 3f2e9ca35..07f0b3fae 100644
--- a/controllers/gce/backends/backends.go
+++ b/controllers/gce/backends/backends.go
@@ -153,6 +153,12 @@ func (b *Backends) create(igs []*compute.InstanceGroup, namedPort *compute.Named
 		return nil, err
 	}
 	errs := []string{}
+	// We first try to create the backend with balancingMode=RATE.  If this
+	// fails, it's mostly likely because there are existing backends with
+	// balancingMode=UTILIZATION. This failure mode throws a googleapi error
+	// which wraps a HTTP 400 status code. We handle it in the loop below
+	// and come around to retry with the right balancing mode. The goal is to
+	// switch everyone to using RATE.
 	for _, bm := range []BalancingMode{Rate, Utilization} {
 		backends := getBackendsForIGs(igs)
 		for _, b := range backends {
@@ -175,8 +181,16 @@ func (b *Backends) create(igs []*compute.InstanceGroup, namedPort *compute.Named
 			PortName:     namedPort.Name,
 		}
 		if err := b.cloud.CreateBackendService(backend); err != nil {
-			glog.Infof("Error creating backend service with balancing mode %v", bm)
-			errs = append(errs, fmt.Sprintf("%v", err))
+			// This is probably a failure because we tried to create the backend
+			// with balancingMode=RATE when there are already backends with
+			// balancingMode=UTILIZATION. Just ignore it and retry setting
+			// balancingMode=UTILIZATION (b/35102911).
+			if utils.IsHTTPErrorCode(err, http.StatusBadRequest) {
+				glog.Infof("Error creating backend service with balancing mode %v:%v", bm, err)
+				errs = append(errs, fmt.Sprintf("%v", err))
+				continue
+			}
+			return nil, err
 		}
 		return b.Get(namedPort.Port)
 	}

From 3f618d7dca85fee9eac0de00a1dcaa998719d4f2 Mon Sep 17 00:00:00 2001
From: bprashanth <beeps@google.com>
Date: Wed, 8 Feb 2017 17:23:15 -0800
Subject: [PATCH 12/73] Add unittest

---
 controllers/gce/backends/backends_test.go     | 50 +++++++++++++++++--
 controllers/gce/backends/fakes.go             |  9 +++-
 controllers/gce/controller/fakes.go           |  3 +-
 .../gce/loadbalancers/loadbalancers_test.go   |  2 +-
 4 files changed, 57 insertions(+), 7 deletions(-)

diff --git a/controllers/gce/backends/backends_test.go b/controllers/gce/backends/backends_test.go
index 951cbb1cd..08afd35f6 100644
--- a/controllers/gce/backends/backends_test.go
+++ b/controllers/gce/backends/backends_test.go
@@ -17,6 +17,7 @@ limitations under the License.
 package backends
 
 import (
+	"net/http"
 	"testing"
 
 	compute "google.golang.org/api/compute/v1"
@@ -25,10 +26,14 @@ import (
 	"k8s.io/ingress/controllers/gce/storage"
 	"k8s.io/ingress/controllers/gce/utils"
 	"k8s.io/kubernetes/pkg/util/sets"
+
+	"google.golang.org/api/googleapi"
 )
 
 const defaultZone = "zone-a"
 
+var noOpErrFunc = func(op int, be *compute.BackendService) error { return nil }
+
 func newBackendPool(f BackendServices, fakeIGs instances.InstanceGroups, syncWithCloud bool) BackendPool {
 	namer := &utils.Namer{}
 	nodePool := instances.NewNodePool(fakeIGs)
@@ -40,7 +45,7 @@ func newBackendPool(f BackendServices, fakeIGs instances.InstanceGroups, syncWit
 }
 
 func TestBackendPoolAdd(t *testing.T) {
-	f := NewFakeBackendServices()
+	f := NewFakeBackendServices(noOpErrFunc)
 	fakeIGs := instances.NewFakeInstanceGroups(sets.NewString())
 	pool := newBackendPool(f, fakeIGs, false)
 	namer := utils.Namer{}
@@ -110,7 +115,7 @@ func TestBackendPoolSync(t *testing.T) {
 	// Call sync on a backend pool with a list of ports, make sure the pool
 	// creates/deletes required ports.
 	svcNodePorts := []int64{81, 82, 83}
-	f := NewFakeBackendServices()
+	f := NewFakeBackendServices(noOpErrFunc)
 	fakeIGs := instances.NewFakeInstanceGroups(sets.NewString())
 	pool := newBackendPool(f, fakeIGs, true)
 	pool.Add(81)
@@ -174,7 +179,7 @@ func TestBackendPoolSync(t *testing.T) {
 }
 
 func TestBackendPoolShutdown(t *testing.T) {
-	f := NewFakeBackendServices()
+	f := NewFakeBackendServices(noOpErrFunc)
 	fakeIGs := instances.NewFakeInstanceGroups(sets.NewString())
 	pool := newBackendPool(f, fakeIGs, false)
 	namer := utils.Namer{}
@@ -187,7 +192,7 @@ func TestBackendPoolShutdown(t *testing.T) {
 }
 
 func TestBackendInstanceGroupClobbering(t *testing.T) {
-	f := NewFakeBackendServices()
+	f := NewFakeBackendServices(noOpErrFunc)
 	fakeIGs := instances.NewFakeInstanceGroups(sets.NewString())
 	pool := newBackendPool(f, fakeIGs, false)
 	namer := utils.Namer{}
@@ -230,3 +235,40 @@ func TestBackendInstanceGroupClobbering(t *testing.T) {
 		t.Fatalf("Expected %v Got %v", expectedGroups, gotGroups)
 	}
 }
+
+func TestBackendCreateBalancingMode(t *testing.T) {
+	f := NewFakeBackendServices(noOpErrFunc)
+
+	fakeIGs := instances.NewFakeInstanceGroups(sets.NewString())
+	pool := newBackendPool(f, fakeIGs, false)
+	namer := utils.Namer{}
+	nodePort := int64(8080)
+	modes := []BalancingMode{Rate, Utilization}
+
+	// block the creation of Backends with the given balancingMode
+	// and verify that a backend with the other balancingMode is
+	// created
+	for i, bm := range modes {
+		f.errFunc = func(op int, be *compute.BackendService) error {
+			for _, b := range be.Backends {
+				if b.BalancingMode == string(bm) {
+					return &googleapi.Error{Code: http.StatusBadRequest}
+				}
+			}
+			return nil
+		}
+
+		pool.Add(nodePort)
+		be, err := f.GetBackendService(namer.BeName(nodePort))
+		if err != nil {
+			t.Fatalf("%v", err)
+		}
+
+		for _, b := range be.Backends {
+			if b.BalancingMode != string(modes[(i+1)%len(modes)]) {
+				t.Fatalf("Wrong balancing mode, expected %v got %v", modes[(i+1)%len(modes)], b.BalancingMode)
+			}
+		}
+		pool.GC([]int64{})
+	}
+}
diff --git a/controllers/gce/backends/fakes.go b/controllers/gce/backends/fakes.go
index a5eb1d006..bb2b031f0 100644
--- a/controllers/gce/backends/fakes.go
+++ b/controllers/gce/backends/fakes.go
@@ -25,8 +25,9 @@ import (
 )
 
 // NewFakeBackendServices creates a new fake backend services manager.
-func NewFakeBackendServices() *FakeBackendServices {
+func NewFakeBackendServices(ef func(op int, be *compute.BackendService) error) *FakeBackendServices {
 	return &FakeBackendServices{
+		errFunc: ef,
 		backendServices: cache.NewStore(func(obj interface{}) (string, error) {
 			svc := obj.(*compute.BackendService)
 			return svc.Name, nil
@@ -38,6 +39,7 @@ func NewFakeBackendServices() *FakeBackendServices {
 type FakeBackendServices struct {
 	backendServices cache.Store
 	calls           []int
+	errFunc         func(op int, be *compute.BackendService) error
 }
 
 // GetBackendService fakes getting a backend service from the cloud.
@@ -60,6 +62,11 @@ func (f *FakeBackendServices) GetBackendService(name string) (*compute.BackendSe
 
 // CreateBackendService fakes backend service creation.
 func (f *FakeBackendServices) CreateBackendService(be *compute.BackendService) error {
+	if f.errFunc != nil {
+		if err := f.errFunc(utils.Create, be); err != nil {
+			return err
+		}
+	}
 	f.calls = append(f.calls, utils.Create)
 	be.SelfLink = be.Name
 	return f.backendServices.Update(be)
diff --git a/controllers/gce/controller/fakes.go b/controllers/gce/controller/fakes.go
index cfa3ed08f..ae97f0d9c 100644
--- a/controllers/gce/controller/fakes.go
+++ b/controllers/gce/controller/fakes.go
@@ -20,6 +20,7 @@ import (
 	"k8s.io/kubernetes/pkg/util/intstr"
 	"k8s.io/kubernetes/pkg/util/sets"
 
+	compute "google.golang.org/api/compute/v1"
 	"k8s.io/ingress/controllers/gce/backends"
 	"k8s.io/ingress/controllers/gce/firewalls"
 	"k8s.io/ingress/controllers/gce/healthchecks"
@@ -45,7 +46,7 @@ type fakeClusterManager struct {
 // NewFakeClusterManager creates a new fake ClusterManager.
 func NewFakeClusterManager(clusterName string) *fakeClusterManager {
 	fakeLbs := loadbalancers.NewFakeLoadBalancers(clusterName)
-	fakeBackends := backends.NewFakeBackendServices()
+	fakeBackends := backends.NewFakeBackendServices(func(op int, be *compute.BackendService) error { return nil })
 	fakeIGs := instances.NewFakeInstanceGroups(sets.NewString())
 	fakeHCs := healthchecks.NewFakeHealthChecks()
 	namer := utils.NewNamer(clusterName)
diff --git a/controllers/gce/loadbalancers/loadbalancers_test.go b/controllers/gce/loadbalancers/loadbalancers_test.go
index 4d6fe133b..f1373a933 100644
--- a/controllers/gce/loadbalancers/loadbalancers_test.go
+++ b/controllers/gce/loadbalancers/loadbalancers_test.go
@@ -34,7 +34,7 @@ const (
 )
 
 func newFakeLoadBalancerPool(f LoadBalancers, t *testing.T) LoadBalancerPool {
-	fakeBackends := backends.NewFakeBackendServices()
+	fakeBackends := backends.NewFakeBackendServices(func(op int, be *compute.BackendService) error { return nil })
 	fakeIGs := instances.NewFakeInstanceGroups(sets.NewString())
 	fakeHCs := healthchecks.NewFakeHealthChecks()
 	namer := &utils.Namer{}

From 9b305f19540ea249b7b11ca7e44d0731ab13952e Mon Sep 17 00:00:00 2001
From: bprashanth <beeps@google.com>
Date: Wed, 8 Feb 2017 15:14:19 -0800
Subject: [PATCH 13/73] Flip version to 0.9.1

---
 controllers/gce/Makefile  | 2 +-
 controllers/gce/README.md | 2 +-
 controllers/gce/main.go   | 2 +-
 controllers/gce/rc.yaml   | 8 ++++----
 4 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/controllers/gce/Makefile b/controllers/gce/Makefile
index 649658af5..10ca7ca6e 100644
--- a/controllers/gce/Makefile
+++ b/controllers/gce/Makefile
@@ -1,7 +1,7 @@
 all: push
 
 # 0.0 shouldn't clobber any released builds
-TAG = 0.9.0
+TAG = 0.9.1
 PREFIX = gcr.io/google_containers/glbc
 
 server:
diff --git a/controllers/gce/README.md b/controllers/gce/README.md
index aa1d6bd8d..84d1706f0 100644
--- a/controllers/gce/README.md
+++ b/controllers/gce/README.md
@@ -327,7 +327,7 @@ So simply delete the replication controller:
 $ kubectl get rc glbc
 CONTROLLER   CONTAINER(S)           IMAGE(S)                                      SELECTOR                    REPLICAS   AGE
 glbc         default-http-backend   gcr.io/google_containers/defaultbackend:1.0   k8s-app=glbc,version=v0.5   1          2m
-             l7-lb-controller       gcr.io/google_containers/glbc:0.9.0
+             l7-lb-controller       gcr.io/google_containers/glbc:0.9.1
 
 $ kubectl delete rc glbc
 replicationcontroller "glbc" deleted
diff --git a/controllers/gce/main.go b/controllers/gce/main.go
index 2cc35751c..0f1f8e981 100644
--- a/controllers/gce/main.go
+++ b/controllers/gce/main.go
@@ -61,7 +61,7 @@ const (
 	alphaNumericChar = "0"
 
 	// Current docker image version. Only used in debug logging.
-	imageVersion = "glbc:0.9.0"
+	imageVersion = "glbc:0.9.1"
 
 	// Key used to persist UIDs to configmaps.
 	uidConfigMapName = "ingress-uid"
diff --git a/controllers/gce/rc.yaml b/controllers/gce/rc.yaml
index 3023d0207..753733808 100644
--- a/controllers/gce/rc.yaml
+++ b/controllers/gce/rc.yaml
@@ -24,18 +24,18 @@ metadata:
   name: l7-lb-controller
   labels:
     k8s-app: glbc
-    version: v0.9.0
+    version: v0.9.1
 spec:
   # There should never be more than 1 controller alive simultaneously.
   replicas: 1
   selector:
     k8s-app: glbc
-    version: v0.9.0
+    version: v0.9.1
   template:
     metadata:
       labels:
         k8s-app: glbc
-        version: v0.9.0
+        version: v0.9.1
         name: glbc
     spec:
       terminationGracePeriodSeconds: 600
@@ -61,7 +61,7 @@ spec:
           requests:
             cpu: 10m
             memory: 20Mi
-      - image: gcr.io/google_containers/glbc:0.9.0
+      - image: gcr.io/google_containers/glbc:0.9.1
         livenessProbe:
           httpGet:
             path: /healthz

From d0c4e0d713306daa75b269c21fd1b8da752496d1 Mon Sep 17 00:00:00 2001
From: Ricardo Pchevuzinske Katz <ricardo.katz@serpro.gov.br>
Date: Thu, 9 Feb 2017 21:20:12 -0200
Subject: [PATCH 14/73] Adds support for disabling the entire access_log

---
 controllers/nginx/configuration.md                     | 3 +++
 controllers/nginx/pkg/config/config.go                 | 5 +++++
 controllers/nginx/pkg/template/configmap_test.go       | 2 ++
 controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl | 9 +++++++++
 4 files changed, 19 insertions(+)

diff --git a/controllers/nginx/configuration.md b/controllers/nginx/configuration.md
index c494d1aab..77c21ca73 100644
--- a/controllers/nginx/configuration.md
+++ b/controllers/nginx/configuration.md
@@ -188,6 +188,9 @@ Setting at least one code also enables [proxy_intercept_errors](http://nginx.org
 Example usage: `custom-http-errors: 404,415`
 
 
+**disable-access-log:** Disables the Access Log from the entire Ingress Controller. This is 'false' by default.
+
+
 **enable-dynamic-tls-records:** Enables dynamically sized TLS records to improve time-to-first-byte. Enabled by default. See [CloudFlare's blog](https://blog.cloudflare.com/optimizing-tls-over-tcp-to-reduce-latency) for more information.
 
 
diff --git a/controllers/nginx/pkg/config/config.go b/controllers/nginx/pkg/config/config.go
index c3dc11331..11e2ad879 100644
--- a/controllers/nginx/pkg/config/config.go
+++ b/controllers/nginx/pkg/config/config.go
@@ -88,6 +88,10 @@ type Configuration struct {
 	// http://nginx.org/en/docs/http/ngx_http_core_module.html#client_header_buffer_size
 	ClientHeaderBufferSize string `json:"client-header-buffer-size"`
 
+	// DisableAccessLog disables the Access Log globally from NGINX ingress controller
+	//http://nginx.org/en/docs/http/ngx_http_log_module.html
+	DisableAccessLog bool `json:"disable-access-log,omitempty"`
+
 	// EnableSPDY enables spdy and use ALPN and NPN to advertise the availability of the two protocols
 	// https://blog.cloudflare.com/open-sourcing-our-nginx-http-2-spdy-code
 	// By default this is enabled
@@ -233,6 +237,7 @@ type Configuration struct {
 func NewDefault() Configuration {
 	cfg := Configuration{
 		ClientHeaderBufferSize:  "1k",
+		DisableAccessLog:        false,
 		EnableDynamicTLSRecords: true,
 		EnableSPDY:              false,
 		ErrorLogLevel:           errorLevel,
diff --git a/controllers/nginx/pkg/template/configmap_test.go b/controllers/nginx/pkg/template/configmap_test.go
index 2e4c43af2..ff2c60203 100644
--- a/controllers/nginx/pkg/template/configmap_test.go
+++ b/controllers/nginx/pkg/template/configmap_test.go
@@ -39,12 +39,14 @@ func TestMergeConfigMapToStruct(t *testing.T) {
 		"proxy-send-timeout":         "2",
 		"skip-access-log-urls":       "/log,/demo,/test",
 		"use-proxy-protocol":         "true",
+		"disable-access-log":         "true",
 		"use-gzip":                   "true",
 		"enable-dynamic-tls-records": "false",
 		"gzip-types":                 "text/html",
 	}
 	def := config.NewDefault()
 	def.CustomHTTPErrors = []int{300, 400}
+	def.DisableAccessLog = true
 	def.SkipAccessLogURLs = []string{"/log", "/demo", "/test"}
 	def.ProxyReadTimeout = 1
 	def.ProxySendTimeout = 2
diff --git a/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl b/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl
index c4f4c497f..d018d4ebb 100644
--- a/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl
+++ b/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl
@@ -87,7 +87,11 @@ http {
         default 1;
     }
 
+    {{ if $cfg.DisableAccessLog }}
+    access_log off;
+    {{ else }}
     access_log /var/log/nginx/access.log upstreaminfo if=$loggable;
+    {{ end }}
     error_log  /var/log/nginx/error.log {{ $cfg.ErrorLogLevel }};
 
     {{ buildResolvers $cfg.Resolver }}
@@ -424,7 +428,12 @@ stream {
 
     log_format log_stream '$remote_addr [$time_local] $protocol [$ssl_preread_server_name] [$stream_upstream] $status $bytes_sent $bytes_received $session_time';
 
+    {{ if $cfg.DisableAccessLog }}
+    access_log off;
+    {{ else }}
     access_log /var/log/nginx/access.log log_stream;
+    {{ end }}
+
     error_log  /var/log/nginx/error.log;
 
     # configure default backend for SSL

From a77dd5dfd03345b77a2706fc17f928e90b7b0e73 Mon Sep 17 00:00:00 2001
From: chentao1596 <chen.tao3@zte.com.cn>
Date: Fri, 20 Jan 2017 16:38:30 +0800
Subject: [PATCH 15/73] remove redudant alias

---
 core/pkg/ingress/status/status.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/pkg/ingress/status/status.go b/core/pkg/ingress/status/status.go
index 9d8bc6b5b..b1618bf58 100644
--- a/core/pkg/ingress/status/status.go
+++ b/core/pkg/ingress/status/status.go
@@ -33,7 +33,7 @@ import (
 
 	cache_store "k8s.io/ingress/core/pkg/cache"
 	"k8s.io/ingress/core/pkg/k8s"
-	strings "k8s.io/ingress/core/pkg/strings"
+	"k8s.io/ingress/core/pkg/strings"
 	"k8s.io/ingress/core/pkg/task"
 )
 

From a72e94f29707fcdffdab940038e904f5c67ef23a Mon Sep 17 00:00:00 2001
From: chentao1596 <chen.tao3@zte.com.cn>
Date: Fri, 10 Feb 2017 10:51:11 +0800
Subject: [PATCH 16/73] modify to get the right content when updating ingress

---
 core/pkg/ingress/status/status.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/pkg/ingress/status/status.go b/core/pkg/ingress/status/status.go
index b1618bf58..09ddb702d 100644
--- a/core/pkg/ingress/status/status.go
+++ b/core/pkg/ingress/status/status.go
@@ -251,7 +251,7 @@ func (s *statusSync) updateStatus(newIPs []api.LoadBalancerIngress) {
 				return
 			}
 
-			curIPs := ing.Status.LoadBalancer.Ingress
+			curIPs := currIng.Status.LoadBalancer.Ingress
 			sort.Sort(loadBalancerIngressByIP(curIPs))
 			if ingressSliceEqual(newIPs, curIPs) {
 				glog.V(3).Infof("skipping update of Ingress %v/%v (there is no change)", currIng.Namespace, currIng.Name)

From 1dbe65ecb62ab5e7cf5c5b884e444e84203ec514 Mon Sep 17 00:00:00 2001
From: Ricardo Pchevuzinske Katz <ricardo.katz@serpro.gov.br>
Date: Fri, 10 Feb 2017 01:00:17 -0200
Subject: [PATCH 17/73] Initial support for sticky config in annotations

---
 .../ingress/annotations/stickysession/main.go | 96 +++++++++++++++++++
 .../annotations/stickysession/main_test.go    | 88 +++++++++++++++++
 core/pkg/ingress/controller/annotations.go    |  8 ++
 .../ingress/controller/annotations_test.go    | 37 +++++++
 core/pkg/ingress/controller/controller.go     |  7 ++
 core/pkg/ingress/types.go                     |  3 +
 6 files changed, 239 insertions(+)
 create mode 100644 core/pkg/ingress/annotations/stickysession/main.go
 create mode 100644 core/pkg/ingress/annotations/stickysession/main_test.go

diff --git a/core/pkg/ingress/annotations/stickysession/main.go b/core/pkg/ingress/annotations/stickysession/main.go
new file mode 100644
index 000000000..be4dd2f0f
--- /dev/null
+++ b/core/pkg/ingress/annotations/stickysession/main.go
@@ -0,0 +1,96 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package stickysession
+
+import (
+	"regexp"
+
+	"k8s.io/kubernetes/pkg/apis/extensions"
+
+	"k8s.io/ingress/core/pkg/ingress/annotations/parser"
+	ing_errors "k8s.io/ingress/core/pkg/ingress/errors"
+)
+
+const (
+	stickyEnabled     = "ingress.kubernetes.io/sticky-enabled"
+	stickyName        = "ingress.kubernetes.io/sticky-name"
+	stickyHash        = "ingress.kubernetes.io/sticky-hash"
+	defaultStickyHash = "md5"
+	defaultStickyName = "route"
+)
+
+var (
+	stickyHashRegex = regexp.MustCompile(`index|md5|sha1`)
+)
+
+// StickyConfig describes the per ingress sticky session config
+type StickyConfig struct {
+	// The name of the cookie that will be used as stickness router.
+	Name string `json:"name"`
+	// If sticky must or must not be enabled
+	Enabled bool `json:"enabled"`
+	// The hash that will be used to encode the cookie
+	Hash string `json:"hash"`
+}
+
+type sticky struct {
+}
+
+// NewParser creates a new Sticky annotation parser
+func NewParser() parser.IngressAnnotation {
+	return sticky{}
+}
+
+// ParseAnnotations parses the annotations contained in the ingress
+// rule used to configure the sticky directives
+func (a sticky) Parse(ing *extensions.Ingress) (interface{}, error) {
+	// Check if the sticky is enabled
+	se, err := parser.GetBoolAnnotation(stickyEnabled, ing)
+	if err != nil {
+		return nil, err
+	}
+
+	// Get the Sticky Cookie Name
+	sn, err := parser.GetStringAnnotation(stickyName, ing)
+	if err != nil {
+		return nil, err
+	}
+
+	if sn == "" {
+		sn = defaultStickyName
+	}
+
+	sh, err := parser.GetStringAnnotation(stickyHash, ing)
+
+	if err != nil {
+		return nil, err
+	}
+
+	if sh == "" {
+		sh = defaultStickyHash
+	}
+
+	if !stickyHashRegex.MatchString(sh) {
+		return nil, ing_errors.NewInvalidAnnotationContent(stickyHash, sh)
+	}
+
+	return &StickyConfig{
+		Name:    sn,
+		Enabled: se,
+		Hash:    sh,
+	}, nil
+}
diff --git a/core/pkg/ingress/annotations/stickysession/main_test.go b/core/pkg/ingress/annotations/stickysession/main_test.go
new file mode 100644
index 000000000..a8a98f3c4
--- /dev/null
+++ b/core/pkg/ingress/annotations/stickysession/main_test.go
@@ -0,0 +1,88 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package stickysession
+
+import (
+	"testing"
+
+	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/apis/extensions"
+	"k8s.io/kubernetes/pkg/util/intstr"
+)
+
+func buildIngress() *extensions.Ingress {
+	defaultBackend := extensions.IngressBackend{
+		ServiceName: "default-backend",
+		ServicePort: intstr.FromInt(80),
+	}
+
+	return &extensions.Ingress{
+		ObjectMeta: api.ObjectMeta{
+			Name:      "foo",
+			Namespace: api.NamespaceDefault,
+		},
+		Spec: extensions.IngressSpec{
+			Backend: &extensions.IngressBackend{
+				ServiceName: "default-backend",
+				ServicePort: intstr.FromInt(80),
+			},
+			Rules: []extensions.IngressRule{
+				{
+					Host: "foo.bar.com",
+					IngressRuleValue: extensions.IngressRuleValue{
+						HTTP: &extensions.HTTPIngressRuleValue{
+							Paths: []extensions.HTTPIngressPath{
+								{
+									Path:    "/foo",
+									Backend: defaultBackend,
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+}
+
+func TestIngressHealthCheck(t *testing.T) {
+	ing := buildIngress()
+
+	data := map[string]string{}
+	data[stickyEnabled] = "true"
+	data[stickyHash] = "md5"
+	data[stickyName] = "route1"
+	ing.SetAnnotations(data)
+
+	sti, _ := NewParser().Parse(ing)
+	nginxSti, ok := sti.(*StickyConfig)
+	if !ok {
+		t.Errorf("expected a StickyConfig type")
+	}
+
+	if nginxSti.Hash != "md5" {
+		t.Errorf("expected md5 as sticky-hash but returned %v", nginxSti.Hash)
+	}
+
+	if nginxSti.Hash != "md5" {
+		t.Errorf("expected md5 as sticky-hash but returned %v", nginxSti.Hash)
+	}
+
+	if !nginxSti.Enabled {
+		t.Errorf("expected sticky-enabled  but returned %v", nginxSti.Enabled)
+	}
+}
diff --git a/core/pkg/ingress/controller/annotations.go b/core/pkg/ingress/controller/annotations.go
index c1eb09fbd..400776866 100644
--- a/core/pkg/ingress/controller/annotations.go
+++ b/core/pkg/ingress/controller/annotations.go
@@ -34,6 +34,7 @@ import (
 	"k8s.io/ingress/core/pkg/ingress/annotations/rewrite"
 	"k8s.io/ingress/core/pkg/ingress/annotations/secureupstream"
 	"k8s.io/ingress/core/pkg/ingress/annotations/sslpassthrough"
+	"k8s.io/ingress/core/pkg/ingress/annotations/stickysession"
 	"k8s.io/ingress/core/pkg/ingress/errors"
 	"k8s.io/ingress/core/pkg/ingress/resolver"
 )
@@ -63,6 +64,7 @@ func newAnnotationExtractor(cfg extractorConfig) annotationExtractor {
 			"Redirect":           rewrite.NewParser(cfg),
 			"SecureUpstream":     secureupstream.NewParser(),
 			"SSLPassthrough":     sslpassthrough.NewParser(),
+			"StickySession":      stickysession.NewParser(),
 		},
 	}
 }
@@ -99,6 +101,7 @@ const (
 	secureUpstream = "SecureUpstream"
 	healthCheck    = "HealthCheck"
 	sslPassthrough = "SSLPassthrough"
+	stickySession  = "StickySession"
 )
 
 func (e *annotationExtractor) SecureUpstream(ing *extensions.Ingress) bool {
@@ -115,3 +118,8 @@ func (e *annotationExtractor) SSLPassthrough(ing *extensions.Ingress) bool {
 	val, _ := e.annotations[sslPassthrough].Parse(ing)
 	return val.(bool)
 }
+
+func (e *annotationExtractor) StickySession(ing *extensions.Ingress) *stickysession.StickyConfig {
+	val, _ := e.annotations[stickySession].Parse(ing)
+	return val.(*stickysession.StickyConfig)
+}
diff --git a/core/pkg/ingress/controller/annotations_test.go b/core/pkg/ingress/controller/annotations_test.go
index 58b0d8093..942508bab 100644
--- a/core/pkg/ingress/controller/annotations_test.go
+++ b/core/pkg/ingress/controller/annotations_test.go
@@ -32,6 +32,9 @@ const (
 	annotationUpsMaxFails    = "ingress.kubernetes.io/upstream-max-fails"
 	annotationUpsFailTimeout = "ingress.kubernetes.io/upstream-fail-timeout"
 	annotationPassthrough    = "ingress.kubernetes.io/ssl-passthrough"
+	annotationStickyEnabled  = "ingress.kubernetes.io/sticky-enabled"
+	annotationStickyName     = "ingress.kubernetes.io/sticky-name"
+	annotationStickyHash     = "ingress.kubernetes.io/sticky-hash"
 )
 
 type mockCfg struct {
@@ -179,3 +182,37 @@ func TestSSLPassthrough(t *testing.T) {
 		}
 	}
 }
+
+func TestStickySession(t *testing.T) {
+	ec := newAnnotationExtractor(mockCfg{})
+	ing := buildIngress()
+
+	fooAnns := []struct {
+		annotations map[string]string
+		enabled     bool
+		hash        string
+		name        string
+	}{
+		{map[string]string{annotationStickyEnabled: "true", annotationStickyHash: "md5", annotationStickyName: "route"}, true, "md5", "route"},
+		{map[string]string{annotationStickyEnabled: "true", annotationStickyHash: "", annotationStickyName: "xpto"}, true, "md5", "xpto"},
+		{map[string]string{annotationStickyEnabled: "true", annotationStickyHash: "", annotationStickyName: ""}, true, "md5", "route"},
+	}
+
+	for _, foo := range fooAnns {
+		ing.SetAnnotations(foo.annotations)
+		r := ec.StickySession(ing)
+
+		if r == nil {
+			t.Errorf("Returned nil but expected a StickySesion.StickyConfig")
+			continue
+		}
+
+		if r.Hash != foo.hash {
+			t.Errorf("Returned %v but expected %v for Hash", r.Hash, foo.hash)
+		}
+
+		if r.Name != foo.name {
+			t.Errorf("Returned %v but expected %v for Name", r.Name, foo.name)
+		}
+	}
+}
diff --git a/core/pkg/ingress/controller/controller.go b/core/pkg/ingress/controller/controller.go
index 5c979dcce..ace3049b1 100644
--- a/core/pkg/ingress/controller/controller.go
+++ b/core/pkg/ingress/controller/controller.go
@@ -700,6 +700,7 @@ func (ic *GenericController) createUpstreams(data []interface{}) map[string]*ing
 
 		secUpstream := ic.annotations.SecureUpstream(ing)
 		hz := ic.annotations.HealthCheck(ing)
+		sticky := ic.annotations.StickySession(ing)
 
 		var defBackend string
 		if ing.Spec.Backend != nil {
@@ -739,6 +740,12 @@ func (ic *GenericController) createUpstreams(data []interface{}) map[string]*ing
 				if !upstreams[name].Secure {
 					upstreams[name].Secure = secUpstream
 				}
+				if !upstreams[name].StickySession.Enabled || upstreams[name].StickySession.Name == "" || upstreams[name].StickySession.Hash == "" {
+					upstreams[name].StickySession.Enabled = sticky.Enabled
+					upstreams[name].StickySession.Name = sticky.Name
+					upstreams[name].StickySession.Hash = sticky.Hash
+				}
+
 				svcKey := fmt.Sprintf("%v/%v", ing.GetNamespace(), path.Backend.ServiceName)
 				endp, err := ic.serviceEndpoints(svcKey, path.Backend.ServicePort.String(), hz)
 				if err != nil {
diff --git a/core/pkg/ingress/types.go b/core/pkg/ingress/types.go
index 49f849a57..89f426a89 100644
--- a/core/pkg/ingress/types.go
+++ b/core/pkg/ingress/types.go
@@ -26,6 +26,7 @@ import (
 	"k8s.io/ingress/core/pkg/ingress/annotations/proxy"
 	"k8s.io/ingress/core/pkg/ingress/annotations/ratelimit"
 	"k8s.io/ingress/core/pkg/ingress/annotations/rewrite"
+	"k8s.io/ingress/core/pkg/ingress/annotations/stickysession"
 	"k8s.io/ingress/core/pkg/ingress/defaults"
 	"k8s.io/ingress/core/pkg/ingress/resolver"
 )
@@ -134,6 +135,8 @@ type Backend struct {
 	Secure bool `json:"secure"`
 	// Endpoints contains the list of endpoints currently running
 	Endpoints []Endpoint `json:"endpoints"`
+	// StickySession contains the StickyConfig object with stickness configuration
+	StickySession *stickysession.StickyConfig `json:"stickysession"`
 }
 
 // Endpoint describes a kubernetes endpoint in an backend

From 79e186cb77f240d352e87938d99d29d87b9e1610 Mon Sep 17 00:00:00 2001
From: Ricardo Pchevuzinske Katz <ricardo.katz@serpro.gov.br>
Date: Fri, 10 Feb 2017 01:33:23 -0200
Subject: [PATCH 18/73] New sticky session configuration

---
 .../rootfs/etc/nginx/template/nginx.tmpl      |  4 ++--
 .../ingress/annotations/stickysession/main.go | 23 +++++++------------
 .../annotations/stickysession/main_test.go    |  2 +-
 3 files changed, 11 insertions(+), 18 deletions(-)

diff --git a/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl b/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl
index d018d4ebb..bb3b04506 100644
--- a/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl
+++ b/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl
@@ -185,8 +185,8 @@ http {
 
     {{range $name, $upstream := $backends}}
     upstream {{$upstream.Name}} {
-        {{ if $cfg.EnableStickySessions }}
-        sticky hash=sha1 httponly;
+        {{ if $upstream.StickySession.Enabled }}
+        sticky hash={{$upstream.StickySession.Hash}} route={{$upstream.StickySession.Hash}}  httponly;
         {{ else }}
         least_conn;
         {{ end }}
diff --git a/core/pkg/ingress/annotations/stickysession/main.go b/core/pkg/ingress/annotations/stickysession/main.go
index be4dd2f0f..ce9609896 100644
--- a/core/pkg/ingress/annotations/stickysession/main.go
+++ b/core/pkg/ingress/annotations/stickysession/main.go
@@ -59,34 +59,27 @@ func NewParser() parser.IngressAnnotation {
 // rule used to configure the sticky directives
 func (a sticky) Parse(ing *extensions.Ingress) (interface{}, error) {
 	// Check if the sticky is enabled
-	se, err := parser.GetBoolAnnotation(stickyEnabled, ing)
-	if err != nil {
-		return nil, err
-	}
+	se, _ := parser.GetBoolAnnotation(stickyEnabled, ing)
 
 	// Get the Sticky Cookie Name
-	sn, err := parser.GetStringAnnotation(stickyName, ing)
-	if err != nil {
-		return nil, err
-	}
+	sn, _ := parser.GetStringAnnotation(stickyName, ing)
 
 	if sn == "" {
 		sn = defaultStickyName
 	}
 
-	sh, err := parser.GetStringAnnotation(stickyHash, ing)
-
-	if err != nil {
-		return nil, err
-	}
+	sh, _ := parser.GetStringAnnotation(stickyHash, ing)
 
 	if sh == "" {
 		sh = defaultStickyHash
 	}
 
 	if !stickyHashRegex.MatchString(sh) {
-		return nil, ing_errors.NewInvalidAnnotationContent(stickyHash, sh)
-	}
+		return &StickyConfig{
+		Name:    "",
+		Enabled: false,
+		Hash:    "",
+	}, ing_errors.NewInvalidAnnotationContent(stickyHash, sh)
 
 	return &StickyConfig{
 		Name:    sn,
diff --git a/core/pkg/ingress/annotations/stickysession/main_test.go b/core/pkg/ingress/annotations/stickysession/main_test.go
index a8a98f3c4..71384fad1 100644
--- a/core/pkg/ingress/annotations/stickysession/main_test.go
+++ b/core/pkg/ingress/annotations/stickysession/main_test.go
@@ -59,7 +59,7 @@ func buildIngress() *extensions.Ingress {
 	}
 }
 
-func TestIngressHealthCheck(t *testing.T) {
+func TestIngressStickySession(t *testing.T) {
 	ing := buildIngress()
 
 	data := map[string]string{}

From f33a925e81edc2b2873f5b976bb6e7fe17964b96 Mon Sep 17 00:00:00 2001
From: chentao1596 <chen.tao3@zte.com.cn>
Date: Fri, 10 Feb 2017 16:03:12 +0800
Subject: [PATCH 19/73] add unit test cases for core.pkg.ingress.status.status

---
 core/pkg/ingress/status/status_test.go | 487 +++++++++++++++++++++++++
 1 file changed, 487 insertions(+)
 create mode 100644 core/pkg/ingress/status/status_test.go

diff --git a/core/pkg/ingress/status/status_test.go b/core/pkg/ingress/status/status_test.go
new file mode 100644
index 000000000..5f41623c9
--- /dev/null
+++ b/core/pkg/ingress/status/status_test.go
@@ -0,0 +1,487 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package status
+
+import (
+	"os"
+	"sort"
+	"sync"
+	"testing"
+	"time"
+
+	cache_store "k8s.io/ingress/core/pkg/cache"
+	"k8s.io/ingress/core/pkg/k8s"
+	"k8s.io/ingress/core/pkg/task"
+	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/apis/extensions"
+	"k8s.io/kubernetes/pkg/client/cache"
+	testclient "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/fake"
+	"k8s.io/kubernetes/pkg/util/sets"
+)
+
+func buildLoadBalancerIngressByIP() loadBalancerIngressByIP {
+	return []api.LoadBalancerIngress{
+		{
+			IP:       "10.0.0.1",
+			Hostname: "foo1",
+		},
+		{
+			IP:       "10.0.0.2",
+			Hostname: "foo2",
+		},
+		{
+			IP:       "10.0.0.3",
+			Hostname: "",
+		},
+		{
+			IP:       "",
+			Hostname: "foo4",
+		},
+	}
+}
+
+func buildSimpleClientSet() *testclient.Clientset {
+	return testclient.NewSimpleClientset(
+		&api.PodList{Items: []api.Pod{
+			{
+				ObjectMeta: api.ObjectMeta{
+					Name:      "foo1",
+					Namespace: api.NamespaceDefault,
+					Labels: map[string]string{
+						"lable_sig": "foo_pod",
+					},
+				},
+				Spec: api.PodSpec{
+					NodeName: "foo_node_2",
+				},
+			},
+			{
+				ObjectMeta: api.ObjectMeta{
+					Name:      "foo2",
+					Namespace: api.NamespaceDefault,
+					Labels: map[string]string{
+						"lable_sig": "foo_no",
+					},
+				},
+			},
+			{
+				ObjectMeta: api.ObjectMeta{
+					Name:      "foo3",
+					Namespace: api.NamespaceSystem,
+					Labels: map[string]string{
+						"lable_sig": "foo_pod",
+					},
+				},
+				Spec: api.PodSpec{
+					NodeName: "foo_node_2",
+				},
+			},
+		}},
+		&api.ServiceList{Items: []api.Service{
+			{
+				ObjectMeta: api.ObjectMeta{
+					Name:      "foo",
+					Namespace: api.NamespaceDefault,
+				},
+				Status: api.ServiceStatus{
+					LoadBalancer: api.LoadBalancerStatus{
+						Ingress: buildLoadBalancerIngressByIP(),
+					},
+				},
+			},
+			{
+				ObjectMeta: api.ObjectMeta{
+					Name:      "foo_non_exist",
+					Namespace: api.NamespaceDefault,
+				},
+			},
+		}},
+		&api.NodeList{Items: []api.Node{
+			{
+				ObjectMeta: api.ObjectMeta{
+					Name: "foo_node_1",
+				},
+				Status: api.NodeStatus{
+					Addresses: []api.NodeAddress{
+						{
+							Type:    api.NodeLegacyHostIP,
+							Address: "10.0.0.1",
+						}, {
+							Type:    api.NodeExternalIP,
+							Address: "10.0.0.2",
+						},
+					},
+				},
+			},
+			{
+				ObjectMeta: api.ObjectMeta{
+					Name: "foo_node_2",
+				},
+				Status: api.NodeStatus{
+					Addresses: []api.NodeAddress{
+						{
+							Type:    api.NodeLegacyHostIP,
+							Address: "11.0.0.1",
+						},
+						{
+							Type:    api.NodeExternalIP,
+							Address: "11.0.0.2",
+						},
+					},
+				},
+			},
+		}},
+		&api.EndpointsList{Items: []api.Endpoints{
+			{
+				ObjectMeta: api.ObjectMeta{
+					Name:      "ingress-controller-leader",
+					Namespace: api.NamespaceDefault,
+				},
+			}}},
+		&extensions.IngressList{Items: buildExtensionsIngresses()},
+	)
+}
+
+func fakeSynFn(interface{}) error {
+	return nil
+}
+
+func buildExtensionsIngresses() []extensions.Ingress {
+	return []extensions.Ingress{
+		{
+			ObjectMeta: api.ObjectMeta{
+				Name:      "foo_ingress_1",
+				Namespace: api.NamespaceDefault,
+			},
+			Status: extensions.IngressStatus{
+				LoadBalancer: api.LoadBalancerStatus{
+					Ingress: []api.LoadBalancerIngress{
+						{
+							IP:       "10.0.0.1",
+							Hostname: "foo1",
+						},
+					},
+				},
+			},
+		},
+		{
+			ObjectMeta: api.ObjectMeta{
+				Name:      "foo_ingress_2",
+				Namespace: api.NamespaceDefault,
+			},
+			Status: extensions.IngressStatus{
+				LoadBalancer: api.LoadBalancerStatus{
+					Ingress: []api.LoadBalancerIngress{},
+				},
+			},
+		},
+	}
+}
+
+func buildIngressLIstener() cache_store.StoreToIngressLister {
+	store := cache.NewStore(cache.MetaNamespaceKeyFunc)
+	ids := sets.NewString("foo_ingress_non_01")
+	for id := range ids {
+		store.Add(&extensions.Ingress{
+			ObjectMeta: api.ObjectMeta{
+				Name:      id,
+				Namespace: api.NamespaceDefault,
+			}})
+	}
+	store.Add(&extensions.Ingress{
+		ObjectMeta: api.ObjectMeta{
+			Name:      "foo_ingress_1",
+			Namespace: api.NamespaceDefault,
+		},
+		Status: extensions.IngressStatus{
+			LoadBalancer: api.LoadBalancerStatus{
+				Ingress: buildLoadBalancerIngressByIP(),
+			},
+		},
+	})
+	return cache_store.StoreToIngressLister{store}
+}
+
+func buildStatusSync() statusSync {
+	return statusSync{
+		pod: &k8s.PodInfo{
+			Name:      "foo_base_pod",
+			Namespace: api.NamespaceDefault,
+			Labels: map[string]string{
+				"lable_sig": "foo_pod",
+			},
+		},
+		runLock:   &sync.Mutex{},
+		syncQueue: task.NewTaskQueue(fakeSynFn),
+		Config: Config{
+			Client:         buildSimpleClientSet(),
+			PublishService: api.NamespaceDefault + "/" + "foo",
+			IngressLister:  buildIngressLIstener(),
+		},
+	}
+}
+
+func TestStatusActions(t *testing.T) {
+	// make sure election can be created
+	os.Setenv("POD_NAME", "foo1")
+	os.Setenv("POD_NAMESPACE", api.NamespaceDefault)
+	c := Config{
+		Client:         buildSimpleClientSet(),
+		PublishService: "",
+		IngressLister:  buildIngressLIstener(),
+	}
+	// create object
+	fkSync := NewStatusSyncer(c)
+	if fkSync == nil {
+		t.Fatalf("expected a valid Sync")
+	}
+
+	fk := fkSync.(statusSync)
+
+	ns := make(chan struct{})
+	// start it and wait for the election and syn actions
+	go fk.Run(ns)
+	//  wait for the election
+	time.Sleep(100 * time.Millisecond)
+	// execute sync
+	fk.sync("just-test")
+	// PublishService is empty, so the running address is: ["11.0.0.2"]
+	// after updated, the ingress's ip should only be "11.0.0.2"
+	newIPs := []api.LoadBalancerIngress{{
+		IP: "11.0.0.2",
+	}}
+	fooIngress1, err1 := fk.Client.Extensions().Ingresses(api.NamespaceDefault).Get("foo_ingress_1")
+	if err1 != nil {
+		t.Fatalf("unexpected error")
+	}
+	fooIngress1CurIPs := fooIngress1.Status.LoadBalancer.Ingress
+	if !ingressSliceEqual(fooIngress1CurIPs, newIPs) {
+		t.Fatalf("returned %v but expected %v", fooIngress1CurIPs, newIPs)
+	}
+
+	// execute shutdown
+	fk.Shutdown()
+	// ingress should be empty
+	newIPs2 := []api.LoadBalancerIngress{}
+	fooIngress2, err2 := fk.Client.Extensions().Ingresses(api.NamespaceDefault).Get("foo_ingress_1")
+	if err2 != nil {
+		t.Fatalf("unexpected error")
+	}
+	fooIngress2CurIPs := fooIngress2.Status.LoadBalancer.Ingress
+	if !ingressSliceEqual(fooIngress2CurIPs, newIPs2) {
+		t.Fatalf("returned %v but expected %v", fooIngress2CurIPs, newIPs2)
+	}
+
+	// end test
+	ns <- struct{}{}
+}
+
+func TestCallback(t *testing.T) {
+	fk := buildStatusSync()
+	//  do nothing
+	fk.callback("foo_base_pod")
+}
+
+func TestKeyfunc(t *testing.T) {
+	fk := buildStatusSync()
+	i := "foo_base_pod"
+	r, err := fk.keyfunc(i)
+
+	if err != nil {
+		t.Fatalf("unexpected error")
+	}
+	if r != i {
+		t.Errorf("returned %v but expected %v", r, i)
+	}
+}
+
+func TestRunningAddresessWithPublishService(t *testing.T) {
+	fk := buildStatusSync()
+
+	r, _ := fk.runningAddresess()
+	if r == nil {
+		t.Fatalf("returned nil but expected valid []string")
+	}
+	rl := len(r)
+	if len(r) != 4 {
+		t.Errorf("returned %v but expected %v", rl, 4)
+	}
+}
+
+func TestRunningAddresessWithPods(t *testing.T) {
+	fk := buildStatusSync()
+	fk.PublishService = ""
+
+	r, _ := fk.runningAddresess()
+	if r == nil {
+		t.Fatalf("returned nil but expected valid []string")
+	}
+	rl := len(r)
+	if len(r) != 1 {
+		t.Fatalf("returned %v but expected %v", rl, 1)
+	}
+	rv := r[0]
+	if rv != "11.0.0.2" {
+		t.Errorf("returned %v but expected %v", rv, "11.0.0.2")
+	}
+}
+
+func TestUpdateStatus(t *testing.T) {
+	fk := buildStatusSync()
+	newIPs := buildLoadBalancerIngressByIP()
+	sort.Sort(loadBalancerIngressByIP(newIPs))
+	fk.updateStatus(newIPs)
+
+	fooIngress1, err1 := fk.Client.Extensions().Ingresses(api.NamespaceDefault).Get("foo_ingress_1")
+	if err1 != nil {
+		t.Fatalf("unexpected error")
+	}
+	fooIngress1CurIPs := fooIngress1.Status.LoadBalancer.Ingress
+	if !ingressSliceEqual(fooIngress1CurIPs, newIPs) {
+		t.Fatalf("returned %v but expected %v", fooIngress1CurIPs, newIPs)
+	}
+
+	fooIngress2, err2 := fk.Client.Extensions().Ingresses(api.NamespaceDefault).Get("foo_ingress_2")
+	if err2 != nil {
+		t.Fatalf("unexpected error")
+	}
+	fooIngress2CurIPs := fooIngress2.Status.LoadBalancer.Ingress
+	if !ingressSliceEqual(fooIngress2CurIPs, []api.LoadBalancerIngress{}) {
+		t.Fatalf("returned %v but expected %v", fooIngress2CurIPs, []api.LoadBalancerIngress{})
+	}
+}
+
+func TestSliceToStatus(t *testing.T) {
+	fkEndpoints := []string{
+		"10.0.0.1",
+		"2001:db8::68",
+		"opensource-k8s-ingress",
+	}
+
+	r := sliceToStatus(fkEndpoints)
+
+	if r == nil {
+		t.Fatalf("returned nil but expected a valid []api.LoadBalancerIngress")
+	}
+	rl := len(r)
+	if rl != 3 {
+		t.Fatalf("returned %v but expected %v", rl, 3)
+	}
+	re1 := r[0]
+	if re1.Hostname != "opensource-k8s-ingress" {
+		t.Fatalf("returned %v but expected %v", re1, api.LoadBalancerIngress{Hostname: "opensource-k8s-ingress"})
+	}
+	re2 := r[1]
+	if re2.IP != "10.0.0.1" {
+		t.Fatalf("returned %v but expected %v", re2, api.LoadBalancerIngress{IP: "10.0.0.1"})
+	}
+	re3 := r[2]
+	if re3.IP != "2001:db8::68" {
+		t.Fatalf("returned %v but expected %v", re3, api.LoadBalancerIngress{IP: "2001:db8::68"})
+	}
+}
+
+func TestIngressSliceEqual(t *testing.T) {
+	fk1 := buildLoadBalancerIngressByIP()
+	fk2 := append(buildLoadBalancerIngressByIP(), api.LoadBalancerIngress{
+		IP:       "10.0.0.5",
+		Hostname: "foo5",
+	})
+	fk3 := buildLoadBalancerIngressByIP()
+	fk3[0].Hostname = "foo_no_01"
+	fk4 := buildLoadBalancerIngressByIP()
+	fk4[2].IP = "11.0.0.3"
+
+	fooTests := []struct {
+		lhs []api.LoadBalancerIngress
+		rhs []api.LoadBalancerIngress
+		er  bool
+	}{
+		{fk1, fk1, true},
+		{fk2, fk1, false},
+		{fk3, fk1, false},
+		{fk4, fk1, false},
+		{fk1, nil, false},
+		{nil, nil, true},
+		{[]api.LoadBalancerIngress{}, []api.LoadBalancerIngress{}, true},
+	}
+
+	for _, fooTest := range fooTests {
+		r := ingressSliceEqual(fooTest.lhs, fooTest.rhs)
+		if r != fooTest.er {
+			t.Errorf("returned %v but expected %v", r, fooTest.er)
+		}
+	}
+}
+
+func TestLoadBalancerIngressByIPLen(t *testing.T) {
+	fooTests := []struct {
+		ips loadBalancerIngressByIP
+		el  int
+	}{
+		{[]api.LoadBalancerIngress{}, 0},
+		{buildLoadBalancerIngressByIP(), 4},
+		{nil, 0},
+	}
+
+	for _, fooTest := range fooTests {
+		r := fooTest.ips.Len()
+		if r != fooTest.el {
+			t.Errorf("returned %v but expected %v ", r, fooTest.el)
+		}
+	}
+}
+
+func TestLoadBalancerIngressByIPSwap(t *testing.T) {
+	fooTests := []struct {
+		ips loadBalancerIngressByIP
+		i   int
+		j   int
+	}{
+		{buildLoadBalancerIngressByIP(), 0, 1},
+		{buildLoadBalancerIngressByIP(), 2, 1},
+	}
+
+	for _, fooTest := range fooTests {
+		fooi := fooTest.ips[fooTest.i]
+		fooj := fooTest.ips[fooTest.j]
+		fooTest.ips.Swap(fooTest.i, fooTest.j)
+		if fooi.IP != fooTest.ips[fooTest.j].IP ||
+			fooj.IP != fooTest.ips[fooTest.i].IP {
+			t.Errorf("failed to swap for loadBalancerIngressByIP")
+		}
+	}
+}
+
+func TestLoadBalancerIngressByIPLess(t *testing.T) {
+	fooTests := []struct {
+		ips loadBalancerIngressByIP
+		i   int
+		j   int
+		er  bool
+	}{
+		{buildLoadBalancerIngressByIP(), 0, 1, true},
+		{buildLoadBalancerIngressByIP(), 2, 1, false},
+	}
+
+	for _, fooTest := range fooTests {
+		r := fooTest.ips.Less(fooTest.i, fooTest.j)
+		if r != fooTest.er {
+			t.Errorf("returned %v but expected %v ", r, fooTest.er)
+		}
+	}
+}

From 9aa601f8df2d25ad3f4d1d6a340dc922c93e31f9 Mon Sep 17 00:00:00 2001
From: chentao1596 <chen.tao3@zte.com.cn>
Date: Fri, 10 Feb 2017 16:08:31 +0800
Subject: [PATCH 20/73] add unit test cases for
 core.pkg.ingress.status.election

---
 core/pkg/ingress/status/election_test.go | 130 +++++++++++++++++++++++
 1 file changed, 130 insertions(+)
 create mode 100644 core/pkg/ingress/status/election_test.go

diff --git a/core/pkg/ingress/status/election_test.go b/core/pkg/ingress/status/election_test.go
new file mode 100644
index 000000000..4726aa8af
--- /dev/null
+++ b/core/pkg/ingress/status/election_test.go
@@ -0,0 +1,130 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package status
+
+import (
+	"encoding/json"
+	"testing"
+	"time"
+
+	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/api/unversioned"
+	tc "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/fake"
+	"k8s.io/kubernetes/pkg/client/leaderelection/resourcelock"
+)
+
+func TestGetCurrentLeaderLeaderExist(t *testing.T) {
+	fkER := resourcelock.LeaderElectionRecord{
+		HolderIdentity:       "currentLeader",
+		LeaseDurationSeconds: 30,
+		AcquireTime:          unversioned.Now(),
+		RenewTime:            unversioned.Now(),
+		LeaderTransitions:    3,
+	}
+	leaderInfo, _ := json.Marshal(fkER)
+	fkEndpoints := api.Endpoints{
+		ObjectMeta: api.ObjectMeta{
+			Name:      "ingress-controller-test",
+			Namespace: api.NamespaceSystem,
+			Annotations: map[string]string{
+				resourcelock.LeaderElectionRecordAnnotationKey: string(leaderInfo),
+			},
+		},
+	}
+	fk := tc.NewSimpleClientset(&api.EndpointsList{Items: []api.Endpoints{fkEndpoints}})
+	identity, endpoints, err := getCurrentLeader("ingress-controller-test", api.NamespaceSystem, fk)
+	if err != nil {
+		t.Fatalf("expected identitiy and endpoints but returned error %s", err)
+	}
+
+	if endpoints == nil {
+		t.Fatalf("returned nil but expected an endpoints")
+	}
+
+	if identity != "currentLeader" {
+		t.Fatalf("returned %v but expected %v", identity, "currentLeader")
+	}
+}
+
+func TestGetCurrentLeaderLeaderNotExist(t *testing.T) {
+	fkEndpoints := api.Endpoints{
+		ObjectMeta: api.ObjectMeta{
+			Name:        "ingress-controller-test",
+			Namespace:   api.NamespaceSystem,
+			Annotations: map[string]string{},
+		},
+	}
+	fk := tc.NewSimpleClientset(&api.EndpointsList{Items: []api.Endpoints{fkEndpoints}})
+	identity, endpoints, err := getCurrentLeader("ingress-controller-test", api.NamespaceSystem, fk)
+	if err != nil {
+		t.Fatalf("unexpeted error: %v", err)
+	}
+
+	if endpoints == nil {
+		t.Fatalf("returned nil but expected an endpoints")
+	}
+
+	if identity != "" {
+		t.Fatalf("returned %s but expected %s", identity, "")
+	}
+}
+
+func TestGetCurrentLeaderAnnotationError(t *testing.T) {
+	fkEndpoints := api.Endpoints{
+		ObjectMeta: api.ObjectMeta{
+			Name:      "ingress-controller-test",
+			Namespace: api.NamespaceSystem,
+			Annotations: map[string]string{
+				resourcelock.LeaderElectionRecordAnnotationKey: "just-test-error-leader-annotation",
+			},
+		},
+	}
+	fk := tc.NewSimpleClientset(&api.EndpointsList{Items: []api.Endpoints{fkEndpoints}})
+	_, _, err := getCurrentLeader("ingress-controller-test", api.NamespaceSystem, fk)
+	if err == nil {
+		t.Errorf("expected error")
+	}
+}
+
+func TestNewElection(t *testing.T) {
+	fk := tc.NewSimpleClientset(&api.EndpointsList{Items: []api.Endpoints{
+		{
+			ObjectMeta: api.ObjectMeta{
+				Name:      "ingress-controller-test",
+				Namespace: api.NamespaceSystem,
+			},
+		},
+		{
+			ObjectMeta: api.ObjectMeta{
+				Name:      "ingress-controller-test-020",
+				Namespace: api.NamespaceSystem,
+			},
+		},
+	}})
+
+	ne, err := NewElection("ingress-controller-test", "startLeader", api.NamespaceSystem, 4*time.Second, func(leader string) {
+		// do nothing
+		go t.Logf("execute callback fun, leader is: %s", leader)
+	}, fk)
+	if err != nil {
+		t.Fatalf("unexpected error %v", err)
+	}
+
+	if ne == nil {
+		t.Fatalf("unexpected nil")
+	}
+}

From 68093193184cc7d18d539dfa6876e2e98d445ec8 Mon Sep 17 00:00:00 2001
From: Ricardo Pchevuzinske Katz <ricardo.katz@serpro.gov.br>
Date: Fri, 10 Feb 2017 12:24:16 -0200
Subject: [PATCH 21/73] Adds support for configuring stickness per Ingress

---
 controllers/nginx/configuration.md            | 17 ++++++
 .../rootfs/etc/nginx/template/nginx.tmpl      |  2 +-
 .../ingress/annotations/stickysession/main.go | 27 ++++-----
 .../ingress/controller/annotations_test.go    |  2 +
 core/pkg/ingress/types.go                     |  2 +-
 examples/stickysession/nginx/README.md        | 58 +++++++++++++++++++
 .../stickysession/nginx/sticky-ingress.yaml   | 19 ++++++
 7 files changed, 112 insertions(+), 15 deletions(-)
 create mode 100644 examples/stickysession/nginx/README.md
 create mode 100644 examples/stickysession/nginx/sticky-ingress.yaml

diff --git a/controllers/nginx/configuration.md b/controllers/nginx/configuration.md
index 77c21ca73..e9e5fb8ce 100644
--- a/controllers/nginx/configuration.md
+++ b/controllers/nginx/configuration.md
@@ -51,6 +51,9 @@ The following annotations are supported:
 |[ingress.kubernetes.io/upstream-max-fails](#custom-nginx-upstream-checks)|number|
 |[ingress.kubernetes.io/upstream-fail-timeout](#custom-nginx-upstream-checks)|number|
 |[ingress.kubernetes.io/whitelist-source-range](#whitelist-source-range)|CIDR|
+|[ingress.kubernetes.io/sticky-enabled](#sticky-session)|true or false|
+|[ingress.kubernetes.io/sticky-name](#sticky-session)|string|
+|[ingress.kubernetes.io/sticky-hash](#sticky-session)|string|
 
 
 
@@ -177,6 +180,20 @@ To configure this setting globally for all Ingress rules, the `whitelist-source-
 Please check the [whitelist](examples/whitelist/README.md) example.
 
 
+### Sticky Session
+
+The annotation `ingress.kubernetes.io/sticky-enabled` enables stickness in all Upstreams of an Ingress. This way, a request will always be directed to the same upstream server. 
+
+You can also specify the name of the cookie that will be used to route the requests with the annotation `ingress.kubernetes.io/sticky-name`. The default is to create a cookie named 'route'.
+
+The annotation `ingress.kubernetes.io/sticky-hash` defines which algorithm will be used to 'hash' the used upstream. Default value is `md5` and possible values are `md5`, `sha1` and `index`.
+
+This feature is implemented by the third party module *nginx-sticky-module-ng* (https://bitbucket.org/nginx-goodies/nginx-sticky-module-ng).
+
+The workflow used to define which upstream server will be used is explained here: https://bitbucket.org/nginx-goodies/nginx-sticky-module-ng/raw/08a395c66e425540982c00482f55034e1fee67b6/docs/sticky.pdf
+
+
+
 ### **Allowed parameters in configuration ConfigMap**
 
 **proxy-body-size:** Sets the maximum allowed size of the client request body. See NGINX [client_max_body_size](http://nginx.org/en/docs/http/ngx_http_core_module.html#client_max_body_size).
diff --git a/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl b/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl
index bb3b04506..34bfea851 100644
--- a/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl
+++ b/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl
@@ -186,7 +186,7 @@ http {
     {{range $name, $upstream := $backends}}
     upstream {{$upstream.Name}} {
         {{ if $upstream.StickySession.Enabled }}
-        sticky hash={{$upstream.StickySession.Hash}} route={{$upstream.StickySession.Hash}}  httponly;
+        sticky hash={{$upstream.StickySession.Hash}} name={{$upstream.StickySession.Name}}  httponly;
         {{ else }}
         least_conn;
         {{ end }}
diff --git a/core/pkg/ingress/annotations/stickysession/main.go b/core/pkg/ingress/annotations/stickysession/main.go
index ce9609896..20bae2c8e 100644
--- a/core/pkg/ingress/annotations/stickysession/main.go
+++ b/core/pkg/ingress/annotations/stickysession/main.go
@@ -19,10 +19,11 @@ package stickysession
 import (
 	"regexp"
 
+	"github.com/golang/glog"
+
 	"k8s.io/kubernetes/pkg/apis/extensions"
 
 	"k8s.io/ingress/core/pkg/ingress/annotations/parser"
-	ing_errors "k8s.io/ingress/core/pkg/ingress/errors"
 )
 
 const (
@@ -59,28 +60,28 @@ func NewParser() parser.IngressAnnotation {
 // rule used to configure the sticky directives
 func (a sticky) Parse(ing *extensions.Ingress) (interface{}, error) {
 	// Check if the sticky is enabled
-	se, _ := parser.GetBoolAnnotation(stickyEnabled, ing)
+	se, err := parser.GetBoolAnnotation(stickyEnabled, ing)
+	if err != nil {
+		se = false
+	}
+
+	glog.V(3).Infof("Ingress %v: Setting stickness to %v", ing.Name, se)
 
 	// Get the Sticky Cookie Name
-	sn, _ := parser.GetStringAnnotation(stickyName, ing)
+	sn, err := parser.GetStringAnnotation(stickyName, ing)
 
-	if sn == "" {
+	if err != nil || sn == "" {
+		glog.V(3).Infof("Ingress %v: No value found in annotation %v. Using the default %v", ing.Name, stickyName, defaultStickyName)
 		sn = defaultStickyName
 	}
 
-	sh, _ := parser.GetStringAnnotation(stickyHash, ing)
+	sh, err := parser.GetStringAnnotation(stickyHash, ing)
 
-	if sh == "" {
+	if err != nil || !stickyHashRegex.MatchString(sh) {
+		glog.V(3).Infof("Invalid or no annotation value found in Ingress %v: %v: %v. Setting it to default %v", ing.Name, stickyHash, sh, defaultStickyHash)
 		sh = defaultStickyHash
 	}
 
-	if !stickyHashRegex.MatchString(sh) {
-		return &StickyConfig{
-		Name:    "",
-		Enabled: false,
-		Hash:    "",
-	}, ing_errors.NewInvalidAnnotationContent(stickyHash, sh)
-
 	return &StickyConfig{
 		Name:    sn,
 		Enabled: se,
diff --git a/core/pkg/ingress/controller/annotations_test.go b/core/pkg/ingress/controller/annotations_test.go
index 942508bab..80f9c99ca 100644
--- a/core/pkg/ingress/controller/annotations_test.go
+++ b/core/pkg/ingress/controller/annotations_test.go
@@ -196,6 +196,8 @@ func TestStickySession(t *testing.T) {
 		{map[string]string{annotationStickyEnabled: "true", annotationStickyHash: "md5", annotationStickyName: "route"}, true, "md5", "route"},
 		{map[string]string{annotationStickyEnabled: "true", annotationStickyHash: "", annotationStickyName: "xpto"}, true, "md5", "xpto"},
 		{map[string]string{annotationStickyEnabled: "true", annotationStickyHash: "", annotationStickyName: ""}, true, "md5", "route"},
+		{map[string]string{}, false, "md5", "route"},
+		{nil, false, "md5", "route"},
 	}
 
 	for _, foo := range fooAnns {
diff --git a/core/pkg/ingress/types.go b/core/pkg/ingress/types.go
index 89f426a89..a43e1d7a3 100644
--- a/core/pkg/ingress/types.go
+++ b/core/pkg/ingress/types.go
@@ -136,7 +136,7 @@ type Backend struct {
 	// Endpoints contains the list of endpoints currently running
 	Endpoints []Endpoint `json:"endpoints"`
 	// StickySession contains the StickyConfig object with stickness configuration
-	StickySession *stickysession.StickyConfig `json:"stickysession"`
+	StickySession stickysession.StickyConfig `json:"stickysession,omitempty"`
 }
 
 // Endpoint describes a kubernetes endpoint in an backend
diff --git a/examples/stickysession/nginx/README.md b/examples/stickysession/nginx/README.md
new file mode 100644
index 000000000..ba5bf9abc
--- /dev/null
+++ b/examples/stickysession/nginx/README.md
@@ -0,0 +1,58 @@
+# Sticky Session
+
+This example demonstrates how to Stickness in a Ingress.
+
+## Prerequisites
+
+You will need to make sure you Ingress targets exactly one Ingress
+controller by specifying the [ingress.class annotation](/examples/PREREQUISITES.md#ingress-class),
+and that you have an ingress controller [running](/examples/deployment) in your cluster.
+
+Also, you need to have a deployment with replica > 1. Using a deployment with only one replica doesn't set the 'sticky' cookie.
+
+## Deployment
+
+The following command instructs the controller to set Stickness in all Upstreams of an Ingress
+
+```console
+$ kubectl create -f sticky-ingress.yaml
+```
+
+## Validation
+
+You can confirm that the Ingress works.
+
+```console
+$ kubectl describe ing nginx-test
+Name:			nginx-test
+Namespace:		default
+Address:		
+Default backend:	default-http-backend:80 (10.180.0.4:8080,10.240.0.2:8080)
+Rules:
+  Host	                        Path	Backends
+  ----	                        ----	--------
+  stickyingress.example.com     
+                                /   	 nginx-service:80 (<none>)
+Annotations:
+  sticky-enabled:	true
+  sticky-hash:		sha1
+  sticky-name:		route
+Events:
+  FirstSeen	LastSeen	Count	From				SubObjectPath	Type		Reason	Message
+  ---------	--------	-----	----				-------------	--------	------	-------
+  7s		7s		1	{nginx-ingress-controller }			Normal		CREATE	default/nginx-test
+  
+
+$ curl -I http://stickyingress.example.com
+HTTP/1.1 200 OK
+Server: nginx/1.11.9
+Date: Fri, 10 Feb 2017 14:11:12 GMT
+Content-Type: text/html
+Content-Length: 612
+Connection: keep-alive
+Set-Cookie: route=a9907b79b248140b56bb13723f72b67697baac3d; Path=/; HttpOnly
+Last-Modified: Tue, 24 Jan 2017 14:02:19 GMT
+ETag: "58875e6b-264"
+Accept-Ranges: bytes
+```
+In the example avove, you can see a line containing the 'Set-Cookie: route' setting the right defined stickness cookie.
\ No newline at end of file
diff --git a/examples/stickysession/nginx/sticky-ingress.yaml b/examples/stickysession/nginx/sticky-ingress.yaml
new file mode 100644
index 000000000..fe7dd42b3
--- /dev/null
+++ b/examples/stickysession/nginx/sticky-ingress.yaml
@@ -0,0 +1,19 @@
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+  name: nginx-test
+  annotations:
+    kubernetes.io/ingress.class: "nginx"
+    ingress.kubernetes.io/sticky-enabled: "true"
+    ingress.kubernetes.io/sticky-name: "route"
+    ingress.kubernetes.io/sticky-hash: "sha1"
+
+spec:
+  rules:
+  - host: stickyingress.example.com
+    http:
+      paths:
+      - backend:
+          serviceName: nginx-service
+          servicePort: 80
+        path: /

From e247fdb7b6f2a712e251fb7fe46a217eeeadc1d2 Mon Sep 17 00:00:00 2001
From: joshrosso <joshrosso@gmail.com>
Date: Fri, 10 Feb 2017 10:16:43 -0700
Subject: [PATCH 22/73] types.go: fix typo in godoc

in an backend -> in a backend
---
 core/pkg/ingress/types.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/pkg/ingress/types.go b/core/pkg/ingress/types.go
index 49f849a57..e619b5dc6 100644
--- a/core/pkg/ingress/types.go
+++ b/core/pkg/ingress/types.go
@@ -136,7 +136,7 @@ type Backend struct {
 	Endpoints []Endpoint `json:"endpoints"`
 }
 
-// Endpoint describes a kubernetes endpoint in an backend
+// Endpoint describes a kubernetes endpoint in a backend
 type Endpoint struct {
 	// Address IP address of the endpoint
 	Address string `json:"address"`

From 8ea814264da9209aa97369afb45965d2026ee1d3 Mon Sep 17 00:00:00 2001
From: bprashanth <beeps@google.com>
Date: Fri, 10 Feb 2017 10:58:46 -0800
Subject: [PATCH 23/73] Add nginx README and configuration docs back

---
 controllers/nginx/README.md | 462 +++++++++++++++++++++++++++++++++++-
 docs/README.md              |  18 ++
 2 files changed, 479 insertions(+), 1 deletion(-)

diff --git a/controllers/nginx/README.md b/controllers/nginx/README.md
index 43127725d..8796b932c 100644
--- a/controllers/nginx/README.md
+++ b/controllers/nginx/README.md
@@ -1,2 +1,462 @@
-
 # Nginx Ingress Controller
+
+This is an nginx Ingress controller that uses [ConfigMap](https://github.com/kubernetes/kubernetes/blob/master/docs/design/configmap.md) to store the nginx configuration. See [Ingress controller documentation](../README.md) for details on how it works.
+
+## Contents
+* [Conventions](#conventions)
+* [Requirements](#what-it-provides)
+* [Dry running](#dry-running-the-ingress-controller)
+* [Deployment](#deployment)
+* [HTTP](#http)
+* [HTTPS](#https)
+  * [Default SSL Certificate](#default-ssl-certificate)
+  * [HTTPS enforcement](#server-side-https-enforcement)
+  * [HSTS](#http-strict-transport-security)
+  * [Kube-Lego](#automated-certificate-management-with-kube-lego)
+* [TCP Services](#exposing-tcp-services)
+* [UDP Services](#exposing-udp-services)
+* [Proxy Protocol](#proxy-protocol)
+* [NGINX customization](configuration.md)
+* [NGINX status page](#nginx-status-page)
+* [Running multiple ingress controllers](#running-multiple-ingress-controllers)
+* [Running on Cloudproviders](#running-on-cloudproviders)
+* [Disabling NGINX ingress controller](#disabling-nginx-ingress-controller)
+* [Log format](#log-format)
+* [Local cluster](#local-cluster)
+* [Debug & Troubleshooting](#troubleshooting)
+* [Why endpoints and not services?](#why-endpoints-and-not-services)
+* [Limitations](#limitations)
+* [NGINX Notes](#nginx-notes)
+
+## Conventions
+
+Anytime we reference a tls secret, we mean (x509, pem encoded, RSA 2048, etc). You can generate such a certificate with: 
+ `openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout ${KEY_FILE} -out ${CERT_FILE} -subj "/CN=${HOST}/O=${HOST}"`
+ and create the secret via `kubectl create secret tls ${CERT_NAME} --key ${KEY_FILE} --cert ${CERT_FILE}`
+
+
+
+## Requirements
+- Default backend [404-server](https://github.com/kubernetes/contrib/tree/master/404-server)
+
+
+## Dry running the Ingress controller
+
+Before deploying the controller to production you might want to run it outside the cluster and observe it.
+
+```console
+$ make controller
+$ mkdir /etc/nginx-ssl
+$ ./nginx-ingress-controller --running-in-cluster=false --default-backend-service=kube-system/default-http-backend
+```
+
+## Deployment
+
+First create a default backend:
+```
+$ kubectl create -f examples/default-backend.yaml
+$ kubectl expose rc default-http-backend --port=80 --target-port=8080 --name=default-http-backend
+```
+
+Loadbalancers are created via a ReplicationController or Daemonset:
+
+```
+$ kubectl create -f examples/default/rc-default.yaml
+```
+
+## HTTP
+
+First we need to deploy some application to publish. To keep this simple we will use the [echoheaders app](https://github.com/kubernetes/contrib/blob/master/ingress/echoheaders/echo-app.yaml) that just returns information about the http request as output
+```
+kubectl run echoheaders --image=gcr.io/google_containers/echoserver:1.4 --replicas=1 --port=8080
+```
+
+Now we expose the same application in two different services (so we can create different Ingress rules)
+```
+kubectl expose deployment echoheaders --port=80 --target-port=8080 --name=echoheaders-x
+kubectl expose deployment echoheaders --port=80 --target-port=8080 --name=echoheaders-y
+```
+
+Next we create a couple of Ingress rules
+```
+kubectl create -f examples/ingress.yaml
+```
+
+we check that ingress rules are defined:
+```
+$ kubectl get ing
+NAME      RULE          BACKEND   ADDRESS
+echomap   -
+          foo.bar.com
+          /foo          echoheaders-x:80
+          bar.baz.com
+          /bar          echoheaders-y:80
+          /foo          echoheaders-x:80
+```
+
+Before the deploy of the Ingress controller we need a default backend [404-server](https://github.com/kubernetes/contrib/tree/master/404-server)
+```
+kubectl create -f examples/default-backend.yaml
+kubectl expose rc default-http-backend --port=80 --target-port=8080 --name=default-http-backend
+```
+
+Check NGINX it is running with the defined Ingress rules:
+
+```
+$ LBIP=$(kubectl get node `kubectl get po -l name=nginx-ingress-lb --template '{{range .items}}{{.spec.nodeName}}{{end}}'` --template '{{range $i, $n := .status.addresses}}{{if eq $n.type "ExternalIP"}}{{$n.address}}{{end}}{{end}}')
+$ curl $LBIP/foo -H 'Host: foo.bar.com'
+```
+
+## HTTPS
+
+You can secure an Ingress by specifying a secret that contains a TLS private key and certificate. Currently the Ingress only supports a single TLS port, 443, and assumes TLS termination. This controller supports SNI. The TLS secret must contain keys named tls.crt and tls.key that contain the certificate and private key to use for TLS, eg:
+
+```
+apiVersion: v1
+data:
+  tls.crt: base64 encoded cert
+  tls.key: base64 encoded key
+kind: Secret
+metadata:
+  name: foo-secret
+  namespace: default
+type: kubernetes.io/tls
+```
+
+Referencing this secret in an Ingress will tell the Ingress controller to secure the channel from the client to the loadbalancer using TLS:
+
+```
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+  name: no-rules-map
+spec:
+  tls:
+    secretName: foo-secret
+  backend:
+    serviceName: s1
+    servicePort: 80
+```
+Please follow [test.sh](https://github.com/bprashanth/Ingress/blob/master/examples/sni/nginx/test.sh) as a guide on how to generate secrets containing SSL certificates. The name of the secret can be different than the name of the certificate.
+
+Check the [example](examples/tls/README.md)
+
+### Default SSL Certificate
+
+NGINX provides the option [server name](http://nginx.org/en/docs/http/server_names.html) as a catch-all in case of requests that do not match one of the configured server names. This configuration works without issues for HTTP traffic. In case of HTTPS NGINX requires a certificate. For this reason the Ingress controller provides the flag `--default-ssl-certificate`. The secret behind this flag contains the default certificate to be used in the mentioned case.
+If this flag is not provided NGINX will use a self signed certificate.
+
+Running without the flag `--default-ssl-certificate`:
+
+```
+$ curl -v https://10.2.78.7:443 -k
+* Rebuilt URL to: https://10.2.78.7:443/
+*   Trying 10.2.78.4...
+* Connected to 10.2.78.7 (10.2.78.7) port 443 (#0)
+* ALPN, offering http/1.1
+* Cipher selection: ALL:!EXPORT:!EXPORT40:!EXPORT56:!aNULL:!LOW:!RC4:@STRENGTH
+* successfully set certificate verify locations:
+*   CAfile: /etc/ssl/certs/ca-certificates.crt
+  CApath: /etc/ssl/certs
+* TLSv1.2 (OUT), TLS header, Certificate Status (22):
+* TLSv1.2 (OUT), TLS handshake, Client hello (1):
+* TLSv1.2 (IN), TLS handshake, Server hello (2):
+* TLSv1.2 (IN), TLS handshake, Certificate (11):
+* TLSv1.2 (IN), TLS handshake, Server key exchange (12):
+* TLSv1.2 (IN), TLS handshake, Server finished (14):
+* TLSv1.2 (OUT), TLS handshake, Client key exchange (16):
+* TLSv1.2 (OUT), TLS change cipher, Client hello (1):
+* TLSv1.2 (OUT), TLS handshake, Finished (20):
+* TLSv1.2 (IN), TLS change cipher, Client hello (1):
+* TLSv1.2 (IN), TLS handshake, Finished (20):
+* SSL connection using TLSv1.2 / ECDHE-RSA-AES128-GCM-SHA256
+* ALPN, server accepted to use http/1.1
+* Server certificate:
+*    subject: CN=foo.bar.com
+*    start date: Apr 13 00:50:56 2016 GMT
+*    expire date: Apr 13 00:50:56 2017 GMT
+*    issuer: CN=foo.bar.com
+*    SSL certificate verify result: self signed certificate (18), continuing anyway.
+> GET / HTTP/1.1
+> Host: 10.2.78.7
+> User-Agent: curl/7.47.1
+> Accept: */*
+>
+< HTTP/1.1 404 Not Found
+< Server: nginx/1.11.1
+< Date: Thu, 21 Jul 2016 15:38:46 GMT
+< Content-Type: text/html
+< Transfer-Encoding: chunked
+< Connection: keep-alive
+< Strict-Transport-Security: max-age=15724800; includeSubDomains; preload
+<
+<span>The page you're looking for could not be found.</span>
+
+* Connection #0 to host 10.2.78.7 left intact
+```
+
+Specifying `--default-ssl-certificate=default/foo-tls`:
+
+```
+core@localhost ~ $ curl -v https://10.2.78.7:443 -k
+* Rebuilt URL to: https://10.2.78.7:443/
+*   Trying 10.2.78.7...
+* Connected to 10.2.78.7 (10.2.78.7) port 443 (#0)
+* ALPN, offering http/1.1
+* Cipher selection: ALL:!EXPORT:!EXPORT40:!EXPORT56:!aNULL:!LOW:!RC4:@STRENGTH
+* successfully set certificate verify locations:
+*   CAfile: /etc/ssl/certs/ca-certificates.crt
+  CApath: /etc/ssl/certs
+* TLSv1.2 (OUT), TLS header, Certificate Status (22):
+* TLSv1.2 (OUT), TLS handshake, Client hello (1):
+* TLSv1.2 (IN), TLS handshake, Server hello (2):
+* TLSv1.2 (IN), TLS handshake, Certificate (11):
+* TLSv1.2 (IN), TLS handshake, Server key exchange (12):
+* TLSv1.2 (IN), TLS handshake, Server finished (14):
+* TLSv1.2 (OUT), TLS handshake, Client key exchange (16):
+* TLSv1.2 (OUT), TLS change cipher, Client hello (1):
+* TLSv1.2 (OUT), TLS handshake, Finished (20):
+* TLSv1.2 (IN), TLS change cipher, Client hello (1):
+* TLSv1.2 (IN), TLS handshake, Finished (20):
+* SSL connection using TLSv1.2 / ECDHE-RSA-AES128-GCM-SHA256
+* ALPN, server accepted to use http/1.1
+* Server certificate:
+*    subject: CN=foo.bar.com
+*    start date: Apr 13 00:50:56 2016 GMT
+*    expire date: Apr 13 00:50:56 2017 GMT
+*    issuer: CN=foo.bar.com
+*    SSL certificate verify result: self signed certificate (18), continuing anyway.
+> GET / HTTP/1.1
+> Host: 10.2.78.7
+> User-Agent: curl/7.47.1
+> Accept: */*
+>
+< HTTP/1.1 404 Not Found
+< Server: nginx/1.11.1
+< Date: Mon, 18 Jul 2016 21:02:59 GMT
+< Content-Type: text/html
+< Transfer-Encoding: chunked
+< Connection: keep-alive
+< Strict-Transport-Security: max-age=15724800; includeSubDomains; preload
+<
+<span>The page you're looking for could not be found.</span>
+
+* Connection #0 to host 10.2.78.7 left intact
+```
+
+
+### Server-side HTTPS enforcement
+
+By default the controller redirects (301) to HTTPS if TLS is enabled for that ingress . If you want to disable that behaviour globally, you can use `ssl-redirect: "false"` in the NGINX config map.
+
+To configure this feature for specific ingress resources, you can use the `ingress.kubernetes.io/ssl-redirect: "false"` annotation in the particular resource.
+
+
+### HTTP Strict Transport Security
+
+HTTP Strict Transport Security (HSTS) is an opt-in security enhancement specified through the use of a special response header. Once a supported browser receives this header that browser will prevent any communications from being sent over HTTP to the specified domain and will instead send all communications over HTTPS.
+
+By default the controller redirects (301) to HTTPS if there is a TLS Ingress rule. 
+
+To disable this behavior use `hsts=false` in the NGINX config map.
+
+
+### Automated Certificate Management with Kube-Lego
+
+[Kube-Lego] automatically requests missing certificates or expired from
+[Let's Encrypt] by monitoring ingress resources and its referenced secrets. To
+enable this for an ingress resource you have to add an annotation:
+
+```
+kubectl annotate ing ingress-demo kubernetes.io/tls-acme="true"
+```
+
+To setup Kube-Lego you can take a look at this [full example]. The first
+version to fully support Kube-Lego is nginx Ingress controller 0.8.
+
+[full example]:https://github.com/jetstack/kube-lego/tree/master/examples
+[Kube-Lego]:https://github.com/jetstack/kube-lego
+[Let's Encrypt]:https://letsencrypt.org
+
+## Exposing TCP services
+
+Ingress does not support TCP services (yet). For this reason this Ingress controller uses the flag `--tcp-services-configmap` to point to an existing config map where the key is the external port to use and the value is `<namespace/service name>:<service port>`
+It is possible to use a number or the name of the port.
+
+The next example shows how to expose the service `example-go` running in the namespace `default` in the port `8080` using the port `9000`
+```
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: tcp-configmap-example
+data:
+  9000: "default/example-go:8080"
+```
+
+
+Please check the [tcp services](examples/tcp/README.md) example
+
+## Exposing UDP services
+
+Since 1.9.13 NGINX provides [UDP Load Balancing](https://www.nginx.com/blog/announcing-udp-load-balancing/).
+
+Ingress does not support UDP services (yet). For this reason this Ingress controller uses the flag `--udp-services-configmap` to point to an existing config map where the key is the external port to use and the value is `<namespace/service name>:<service port>`
+It is possible to use a number or the name of the port.
+
+The next example shows how to expose the service `kube-dns` running in the namespace `kube-system` in the port `53` using the port `53`
+```
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: udp-configmap-example
+data:
+  53: "kube-system/kube-dns:53"
+```
+
+
+Please check the [udp services](examples/udp/README.md) example
+
+## Proxy Protocol
+
+If you are using a L4 proxy to forward the traffic to the NGINX pods and terminate HTTP/HTTPS there, you will lose the remote endpoint's IP addresses. To prevent this you could use the [Proxy Protocol](http://www.haproxy.org/download/1.5/doc/proxy-protocol.txt) for forwarding traffic, this will send the connection details before forwarding the actual TCP connection itself.
+
+Amongst others [ELBs in AWS](http://docs.aws.amazon.com/ElasticLoadBalancing/latest/DeveloperGuide/enable-proxy-protocol.html) and [HAProxy](http://www.haproxy.org/) support Proxy Protocol.
+
+Please check the [proxy-protocol](examples/proxy-protocol/) example
+
+
+### Custom errors
+
+In case of an error in a request the body of the response is obtained from the `default backend`. Each request to the default backend includes two headers: 
+- `X-Code` indicates the HTTP code
+- `X-Format` the value of the `Accept` header
+
+Using this two headers is possible to use a custom backend service like [this one](https://github.com/aledbf/contrib/tree/nginx-debug-server/Ingress/images/nginx-error-server) that inspect each request and returns a custom error page with the format expected by the client. Please check the example [custom-errors](examples/custom-errors/README.md)
+
+### NGINX status page
+
+The ngx_http_stub_status_module module provides access to basic status information. This is the default module active in the url `/nginx_status`.
+This controller provides an alternative to this module using [nginx-module-vts](https://github.com/vozlt/nginx-module-vts) third party module.
+To use this module just provide a config map with the key `enable-vts-status=true`. The URL is exposed in the port 8080.
+Please check the example `example/rc-default.yaml`
+
+![nginx-module-vts screenshot](https://cloud.githubusercontent.com/assets/3648408/10876811/77a67b70-8183-11e5-9924-6a6d0c5dc73a.png "screenshot with filter")
+
+To extract the information in JSON format the module provides a custom URL: `/nginx_status/format/json`
+
+### Running multiple ingress controllers
+
+If you're running multiple ingress controllers, or running on a cloudprovider that natively handles 
+ingress, you need to specify the annotation `kubernetes.io/ingress.class: "nginx"` in all ingresses 
+that you would like this controller to claim. Not specifying the annotation will lead to multiple 
+ingress controllers claiming the same ingress. Specifying the wrong value will result in all ingress 
+controllers ignoring the ingress. Multiple ingress controllers running in the same cluster was not 
+supported in Kubernetes versions < 1.3.
+
+### Running on Cloudproviders
+
+If you're running this ingress controller on a cloudprovider, you should assume the provider also has a native 
+Ingress controller and specify the ingress.class annotation as indicated in this section.
+In addition to this, you will need to add a firewall rule for each port this controller is listening on, i.e :80 and :443.
+
+### Disabling NGINX ingress controller
+
+Setting the annotation `kubernetes.io/ingress.class` to any value other than "nginx" or the empty string, will force the NGINX Ingress controller to ignore your Ingress. Do this if you wish to use one of the other Ingress controllers at the same time as the NGINX controller.
+
+### Log format
+
+The default configuration uses a custom logging format to add additional information about upstreams
+
+```
+    log_format upstreaminfo '{{ if $cfg.useProxyProtocol }}$proxy_protocol_addr{{ else }}$remote_addr{{ end }} - '
+        '[$proxy_add_x_forwarded_for] - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" '
+        '$request_length $request_time [$proxy_upstream_name] $upstream_addr $upstream_response_length $upstream_response_time $upstream_status';
+```
+
+Sources:
+  - [upstream variables](http://nginx.org/en/docs/http/ngx_http_upstream_module.html#variables)
+  - [embedded variables](http://nginx.org/en/docs/http/ngx_http_core_module.html#variables)
+
+Description:
+- `$proxy_protocol_addr`: if PROXY protocol is enabled
+- `$remote_addr`: if PROXY protocol is disabled (default)
+- `$proxy_add_x_forwarded_for`: the `X-Forwarded-For` client request header field with the $remote_addr variable appended to it, separated by a comma
+- `$remote_user`: user name supplied with the Basic authentication
+- `$time_local`: local time in the Common Log Format
+- `$request`: full original request line
+- `$status`: response status
+- `$body_bytes_sent`: number of bytes sent to a client, not counting the response header
+- `$http_referer`: value of the Referer header
+- `$http_user_agent`: value of User-Agent header
+- `$request_length`: request length (including request line, header, and request body)
+- `$request_time`: time elapsed since the first bytes were read from the client
+- `$proxy_upstream_name`: name of the upstream. The format is `upstream-<namespace>-<service name>-<service port>`
+- `$upstream_addr`: keeps the IP address and port, or the path to the UNIX-domain socket of the upstream server. If several servers were contacted during request processing, their addresses are separated by commas
+- `$upstream_response_length`: keeps the length of the response obtained from the upstream server
+- `$upstream_response_time`: keeps time spent on receiving the response from the upstream server; the time is kept in seconds with millisecond resolution
+- `$upstream_status`: keeps status code of the response obtained from the upstream server
+
+### Local cluster
+
+Using [`hack/local-up-cluster.sh`](https://github.com/kubernetes/kubernetes/blob/master/hack/local-up-cluster.sh) is possible to start a local kubernetes cluster consisting of a master and a single node. Please read [running-locally.md](https://github.com/kubernetes/kubernetes/blob/master/docs/devel/running-locally.md) for more details.
+
+Use of `hostNetwork: true` in the ingress controller is required to falls back at localhost:8080 for the apiserver if every other client creation check fails (eg: service account not present, kubeconfig doesn't exist, no master env vars...)
+
+
+### Debug & Troubleshooting
+
+Using the flag `--v=XX` it is possible to increase the level of logging.
+In particular:
+- `--v=2` shows details using `diff` about the changes in the configuration in nginx
+
+```
+I0316 12:24:37.581267       1 utils.go:148] NGINX configuration diff a//etc/nginx/nginx.conf b//etc/nginx/nginx.conf
+I0316 12:24:37.581356       1 utils.go:149] --- /tmp/922554809  2016-03-16 12:24:37.000000000 +0000
++++ /tmp/079811012  2016-03-16 12:24:37.000000000 +0000
+@@ -235,7 +235,6 @@
+
+     upstream default-echoheadersx {
+         least_conn;
+-        server 10.2.112.124:5000;
+         server 10.2.208.50:5000;
+
+     }
+I0316 12:24:37.610073       1 command.go:69] change in configuration detected. Reloading...
+```
+
+- `--v=3` shows details about the service, Ingress rule, endpoint changes and it dumps the nginx configuration in JSON format
+- `--v=5` configures NGINX in [debug mode](http://nginx.org/en/docs/debugging_log.html)
+
+
+
+*These issues were encountered in past versions of Kubernetes:*
+
+[1.2.0-alpha7 deployment](https://github.com/kubernetes/kubernetes/blob/master/docs/getting-started-guides/docker.md):
+
+* make setup-files.sh file in hypercube does not provide 10.0.0.1 IP to make-ca-certs, resulting in CA certs that are issued to the external cluster IP address rather then 10.0.0.1 -> this results in nginx-third-party-lb appearing to get stuck at "Utils.go:177 - Waiting for default/default-http-backend" in the docker logs.  Kubernetes will eventually kill the container before nginx-third-party-lb times out with a message indicating that the CA certificate issuer is invalid (wrong ip), to verify this add zeros to the end of initialDelaySeconds and timeoutSeconds and reload the RC, and docker will log this error before kubernetes kills the container.
+  * To fix the above, setup-files.sh must be patched before the cluster is inited (refer to https://github.com/kubernetes/kubernetes/pull/21504)
+
+
+### Limitations
+
+- Ingress rules for TLS require the definition of the field `host`
+
+
+### Why endpoints and not services
+
+The NGINX ingress controller does not uses [Services](http://kubernetes.io/docs/user-guide/services) to route traffic to the pods. Instead it uses the Endpoints API in order to bypass [kube-proxy](http://kubernetes.io/docs/admin/kube-proxy/) to allow NGINX features like session affinity and custom load balancing algorithms. It also removes some overhead, such as conntrack entries for iptables DNAT.
+
+
+### NGINX notes
+
+Since `gcr.io/google_containers/nginx-slim:0.8` NGINX contains the next patches:
+- Dynamic TLS record size [nginx__dynamic_tls_records.patch](https://blog.cloudflare.com/optimizing-tls-over-tcp-to-reduce-latency/)
+NGINX provides the parameter `ssl_buffer_size` to adjust the size of the buffer. Default value in NGINX is 16KB. The ingress controller changes the default to 4KB. This improves the [TLS Time To First Byte (TTTFB)](https://www.igvita.com/2013/12/16/optimizing-nginx-tls-time-to-first-byte/) but the size is fixed. This patches adapts the size of the buffer to the content is being served helping to improve the perceived latency.
+
+- Add SPDY support back to Nginx with HTTP/2 [nginx_1_9_15_http2_spdy.patch](https://github.com/cloudflare/sslconfig/pull/36)
+At the same NGINX introduced HTTP/2 support for SPDY was removed. This patch add support for SPDY without compromising HTTP/2 support using the Application-Layer Protocol Negotiation (ALPN) or Next Protocol Negotiation (NPN) Transport Layer Security (TLS) extension to negotiate what protocol the server and client support
+```
+openssl s_client -servername www.my-site.com -connect www.my-site.com:443 -nextprotoneg ''
+CONNECTED(00000003)
+Protocols advertised by server: h2, spdy/3.1, http/1.1
+```
diff --git a/docs/README.md b/docs/README.md
index cb8241d07..721336b0b 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -1,3 +1,21 @@
 # Ingress documentation and examples
 
 This directory contains documentation.
+
+## File naming convention
+
+Try to create a README file in every directory containing documentation and index
+out from there, that's what readers will notice first. Use lower case for other
+file names unless you have a reason to draw someones attention to it.
+Avoid CamelCase.
+
+Rationale:
+
+* Files that are common to all controllers, or heavily index other files, are
+named using ALL CAPS. This is done to indicate to the user that they should
+visit these files first. Examples include PREREQUISITES and README.
+
+* Files specific to a controller, or files that contain information about
+various controllers, are named using all lower case. Examples include
+configuration and catalog files.
+

From f7a27ebcc2c87d6b7437bfbb3058bbd114168c00 Mon Sep 17 00:00:00 2001
From: Manuel de Brito Fontes <aledbf@gmail.com>
Date: Sat, 11 Feb 2017 17:19:43 -0300
Subject: [PATCH 24/73] Add example

---
 .../custom-headers/nginx/README.md            | 42 +++++++++++++++++--
 .../custom-headers/nginx/custom-headers.yaml  |  9 ++++
 .../nginx/nginx-ingress-controller.yaml       |  1 +
 .../nginx/nginx-load-balancer-conf.yaml       |  7 ++++
 4 files changed, 56 insertions(+), 3 deletions(-)
 create mode 100644 examples/customization/custom-headers/nginx/custom-headers.yaml
 create mode 100644 examples/customization/custom-headers/nginx/nginx-load-balancer-conf.yaml

diff --git a/examples/customization/custom-headers/nginx/README.md b/examples/customization/custom-headers/nginx/README.md
index 497545781..459b43b1e 100644
--- a/examples/customization/custom-headers/nginx/README.md
+++ b/examples/customization/custom-headers/nginx/README.md
@@ -20,6 +20,42 @@ NAME                                    READY     STATUS    RESTARTS   AGE
 default-http-backend-2657704409-qgwdd   1/1       Running   0          28s
 ```
 
+## Custom configuration
+
+```console
+$ cat nginx-load-balancer-conf.yaml
+apiVersion: v1
+data:
+  proxy-set-headers: "default/custom-headers"
+kind: ConfigMap
+metadata:
+  name: nginx-load-balancer-conf
+```
+
+```console
+$ kubectl create -f nginx-load-balancer-conf.yaml
+```
+
+## Custom headers
+
+```console
+$ cat custom-headers.yaml
+apiVersion: v1
+data:
+  X-Different-Name: "true"
+  X-Request-Start: t=${msec}
+  X-Using-Nginx-Controller: "true"
+kind: ConfigMap
+metadata:
+  name: proxy-headers
+  namespace: default
+
+```
+
+```console
+$ kubectl create -f custom-headers.yaml
+```
+
 ## Controller
 
 You can deploy the controller as follows:
@@ -34,7 +70,7 @@ default-http-backend-2657704409-qgwdd      1/1       Running   0          2m
 nginx-ingress-controller-873061567-4n3k2   1/1       Running   0          42s
 ```
 
-Note the default settings of this controller:
-* serves a `/healthz` url on port 10254, as both a liveness and readiness probe
-* takes a `--default-backend-service` argument pointing to the Service created above
+## Test
 
+Check the contents of the configmap is present in the nginx.conf file using:
+`kubectl exec nginx-ingress-controller-873061567-4n3k2 -n kube-system cat /etc/nginx/nginx.conf`
diff --git a/examples/customization/custom-headers/nginx/custom-headers.yaml b/examples/customization/custom-headers/nginx/custom-headers.yaml
new file mode 100644
index 000000000..beeefc8a4
--- /dev/null
+++ b/examples/customization/custom-headers/nginx/custom-headers.yaml
@@ -0,0 +1,9 @@
+apiVersion: v1
+data:
+  X-Different-Name: "true"
+  X-Request-Start: t=${msec}
+  X-Using-Nginx-Controller: "true"
+kind: ConfigMap
+metadata:
+  name: proxy-headers
+  namespace: kube-system
diff --git a/examples/customization/custom-headers/nginx/nginx-ingress-controller.yaml b/examples/customization/custom-headers/nginx/nginx-ingress-controller.yaml
index 7ee5e797e..0d3824cb2 100644
--- a/examples/customization/custom-headers/nginx/nginx-ingress-controller.yaml
+++ b/examples/customization/custom-headers/nginx/nginx-ingress-controller.yaml
@@ -50,3 +50,4 @@ spec:
         args:
         - /nginx-ingress-controller
         - --default-backend-service=$(POD_NAMESPACE)/default-http-backend
+        - --configmap=$(POD_NAMESPACE)/nginx-load-balancer-conf
diff --git a/examples/customization/custom-headers/nginx/nginx-load-balancer-conf.yaml b/examples/customization/custom-headers/nginx/nginx-load-balancer-conf.yaml
new file mode 100644
index 000000000..239918267
--- /dev/null
+++ b/examples/customization/custom-headers/nginx/nginx-load-balancer-conf.yaml
@@ -0,0 +1,7 @@
+apiVersion: v1
+data:
+  proxy-set-headers: "kube-system/custom-headers"
+kind: ConfigMap
+metadata:
+  name: nginx-load-balancer-conf
+  namespace: kube-system

From eb69a1d011cb75ec03446c31ab99f5a32d31759c Mon Sep 17 00:00:00 2001
From: Manuel de Brito Fontes <aledbf@gmail.com>
Date: Sat, 11 Feb 2017 18:34:50 -0300
Subject: [PATCH 25/73] Fix rewrite annotation parser

---
 core/pkg/ingress/annotations/rewrite/main.go      | 6 +-----
 core/pkg/ingress/annotations/rewrite/main_test.go | 4 ++--
 2 files changed, 3 insertions(+), 7 deletions(-)

diff --git a/core/pkg/ingress/annotations/rewrite/main.go b/core/pkg/ingress/annotations/rewrite/main.go
index c7e57c253..14dec6616 100644
--- a/core/pkg/ingress/annotations/rewrite/main.go
+++ b/core/pkg/ingress/annotations/rewrite/main.go
@@ -52,11 +52,7 @@ func NewParser(br resolver.DefaultBackend) parser.IngressAnnotation {
 // ParseAnnotations parses the annotations contained in the ingress
 // rule used to rewrite the defined paths
 func (a rewrite) Parse(ing *extensions.Ingress) (interface{}, error) {
-	rt, err := parser.GetStringAnnotation(rewriteTo, ing)
-	if err != nil {
-		return nil, err
-	}
-
+	rt, _ := parser.GetStringAnnotation(rewriteTo, ing)
 	sslRe, err := parser.GetBoolAnnotation(sslRedirect, ing)
 	if err != nil {
 		sslRe = a.backendResolver.GetDefaultBackend().SSLRedirect
diff --git a/core/pkg/ingress/annotations/rewrite/main_test.go b/core/pkg/ingress/annotations/rewrite/main_test.go
index 56ba6a9b7..f4f0ed973 100644
--- a/core/pkg/ingress/annotations/rewrite/main_test.go
+++ b/core/pkg/ingress/annotations/rewrite/main_test.go
@@ -76,8 +76,8 @@ func (m mockBackend) GetDefaultBackend() defaults.Backend {
 func TestWithoutAnnotations(t *testing.T) {
 	ing := buildIngress()
 	_, err := NewParser(mockBackend{}).Parse(ing)
-	if err == nil {
-		t.Error("Expected error with ingress without annotations")
+	if err != nil {
+		t.Errorf("unexpected error with ingress without annotations: %v", err)
 	}
 }
 

From a158e5fc5ab8118c7a51614a1dc43f97cf91b12e Mon Sep 17 00:00:00 2001
From: Ricardo Pchevuzinske Katz <ricardo.katz@serpro.gov.br>
Date: Sun, 12 Feb 2017 21:13:39 -0200
Subject: [PATCH 26/73] Improve the session affinity feature

---
 controllers/nginx/configuration.md            |  13 +-
 .../rootfs/etc/nginx/template/nginx.tmpl      |   4 +-
 .../annotations/sessionaffinity/main.go       | 118 ++++++++++++++++++
 .../main_test.go                              |  28 ++---
 .../ingress/annotations/stickysession/main.go |  90 -------------
 core/pkg/ingress/controller/annotations.go    |  18 +--
 .../ingress/controller/annotations_test.go    |  48 +++----
 core/pkg/ingress/controller/controller.go     |  12 +-
 core/pkg/ingress/types.go                     |  21 +++-
 examples/affinity/cookie/nginx/README.md      |  73 +++++++++++
 .../cookie}/nginx/sticky-ingress.yaml         |   6 +-
 examples/stickysession/nginx/README.md        |  58 ---------
 12 files changed, 276 insertions(+), 213 deletions(-)
 create mode 100644 core/pkg/ingress/annotations/sessionaffinity/main.go
 rename core/pkg/ingress/annotations/{stickysession => sessionaffinity}/main_test.go (67%)
 delete mode 100644 core/pkg/ingress/annotations/stickysession/main.go
 create mode 100644 examples/affinity/cookie/nginx/README.md
 rename examples/{stickysession => affinity/cookie}/nginx/sticky-ingress.yaml (66%)
 delete mode 100644 examples/stickysession/nginx/README.md

diff --git a/controllers/nginx/configuration.md b/controllers/nginx/configuration.md
index e9e5fb8ce..2b04e55dd 100644
--- a/controllers/nginx/configuration.md
+++ b/controllers/nginx/configuration.md
@@ -177,20 +177,21 @@ To configure this setting globally for all Ingress rules, the `whitelist-source-
 
 *Note:* Adding an annotation to an Ingress rule overrides any global restriction.
 
-Please check the [whitelist](examples/whitelist/README.md) example.
+Please check the [whitelist](examples/affinity/cookie/nginx/README.md) example.
 
 
 ### Sticky Session
 
-The annotation `ingress.kubernetes.io/sticky-enabled` enables stickness in all Upstreams of an Ingress. This way, a request will always be directed to the same upstream server. 
+The annotation `ingress.kubernetes.io/affinity` enables and sets the affinity type in all Upstreams of an Ingress. This way, a request will always be directed to the same upstream server. 
 
-You can also specify the name of the cookie that will be used to route the requests with the annotation `ingress.kubernetes.io/sticky-name`. The default is to create a cookie named 'route'.
 
-The annotation `ingress.kubernetes.io/sticky-hash` defines which algorithm will be used to 'hash' the used upstream. Default value is `md5` and possible values are `md5`, `sha1` and `index`.
+#### Cookie affinity
+If you use the ``cookie`` type you can also specify the name of the cookie that will be used to route the requests with the annotation `ingress.kubernetes.io/session-cookie-name`. The default is to create a cookie named 'route'.
 
-This feature is implemented by the third party module *nginx-sticky-module-ng* (https://bitbucket.org/nginx-goodies/nginx-sticky-module-ng).
+In case of NGINX the annotation `ingress.kubernetes.io/session-cookie-hash` defines which algorithm will be used to 'hash' the used upstream. Default value is `md5` and possible values are `md5`, `sha1` and `index`.
+The `index` option  is not hashed, an in-memory index is used instead, it's quicker and the overhead is shorter Warning: the matching against upstream servers list is inconsistent. So, at reload, if upstreams servers has changed, index values are not guaranted to correspond to the same server as before! USE IT WITH CAUTION and only if you need to!
 
-The workflow used to define which upstream server will be used is explained here: https://bitbucket.org/nginx-goodies/nginx-sticky-module-ng/raw/08a395c66e425540982c00482f55034e1fee67b6/docs/sticky.pdf
+In NGINX this feature is implemented by the third party module [nginx-sticky-module-ng](https://bitbucket.org/nginx-goodies/nginx-sticky-module-ng). The workflow used to define which upstream server will be used is explained [here]https://bitbucket.org/nginx-goodies/nginx-sticky-module-ng/raw/08a395c66e425540982c00482f55034e1fee67b6/docs/sticky.pdf
 
 
 
diff --git a/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl b/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl
index 34bfea851..5c8164ac6 100644
--- a/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl
+++ b/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl
@@ -185,8 +185,8 @@ http {
 
     {{range $name, $upstream := $backends}}
     upstream {{$upstream.Name}} {
-        {{ if $upstream.StickySession.Enabled }}
-        sticky hash={{$upstream.StickySession.Hash}} name={{$upstream.StickySession.Name}}  httponly;
+        {{ if eq $upstream.SessionAffinity.AffinityType "cookie" }}
+        sticky hash={{$upstream.SessionAffinity.CookieSessionAffinity.Hash}} name={{$upstream.SessionAffinity.CookieSessionAffinity.Name}}  httponly;
         {{ else }}
         least_conn;
         {{ end }}
diff --git a/core/pkg/ingress/annotations/sessionaffinity/main.go b/core/pkg/ingress/annotations/sessionaffinity/main.go
new file mode 100644
index 000000000..551cde2a3
--- /dev/null
+++ b/core/pkg/ingress/annotations/sessionaffinity/main.go
@@ -0,0 +1,118 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package sessionaffinity
+
+import (
+	"regexp"
+
+	"github.com/golang/glog"
+
+	"k8s.io/kubernetes/pkg/apis/extensions"
+
+	"k8s.io/ingress/core/pkg/ingress/annotations/parser"
+)
+
+const (
+	annotationAffinityType = "ingress.kubernetes.io/affinity"
+	// If a cookie with this name exists,
+	// its value is used as an index into the list of available backends.
+	annotationAffinityCookieName = "ingress.kubernetes.io/session-cookie-name"
+	defaultAffinityCookieName    = "route"
+	// This is the algorithm used by nginx to generate a value for the session cookie, if
+	// one isn't supplied and affintiy is set to "cookie".
+	annotationAffinityCookieHash = "ingress.kubernetes.io/session-cookie-hash"
+	defaultAffinityCookieHash    = "md5"
+)
+
+var (
+	affinityCookieHashRegex = regexp.MustCompile(`^(index|md5|sha1)$`)
+)
+
+// AffinityConfig describes the per ingress session affinity config
+type AffinityConfig struct {
+	// The type of affinity that will be used
+	AffinityType         string               `json:"type"`
+	CookieAffinityConfig CookieAffinityConfig `json:"cookieconfig"`
+}
+
+// CookieAffinityConfig describes the Config of cookie type affinity
+type CookieAffinityConfig struct {
+	// The name of the cookie that will be used in case of cookie affinity type.
+	Name string `json:"name"`
+	// The hash that will be used to encode the cookie in case of cookie affinity type
+	Hash string `json:"hash"`
+}
+
+type affinity struct {
+}
+
+// CookieAffinityParse gets the annotation values related to Cookie Affinity
+// It also sets default values when no value or incorrect value is found
+func CookieAffinityParse(ing *extensions.Ingress) *CookieAffinityConfig {
+
+	sn, err := parser.GetStringAnnotation(annotationAffinityCookieName, ing)
+
+	if err != nil || sn == "" {
+		glog.V(3).Infof("Ingress %v: No value found in annotation %v. Using the default %v", ing.Name, annotationAffinityCookieName, defaultAffinityCookieName)
+		sn = defaultAffinityCookieName
+	}
+
+	sh, err := parser.GetStringAnnotation(annotationAffinityCookieHash, ing)
+
+	if err != nil || !affinityCookieHashRegex.MatchString(sh) {
+		glog.V(3).Infof("Invalid or no annotation value found in Ingress %v: %v. Setting it to default %v", ing.Name, annotationAffinityCookieHash, defaultAffinityCookieHash)
+		sh = defaultAffinityCookieHash
+	}
+
+	return &CookieAffinityConfig{
+		Name: sn,
+		Hash: sh,
+	}
+}
+
+// NewParser creates a new Affinity annotation parser
+func NewParser() parser.IngressAnnotation {
+	return affinity{}
+}
+
+// ParseAnnotations parses the annotations contained in the ingress
+// rule used to configure the affinity directives
+func (a affinity) Parse(ing *extensions.Ingress) (interface{}, error) {
+
+	var cookieAffinityConfig *CookieAffinityConfig
+	cookieAffinityConfig = &CookieAffinityConfig{}
+
+	// Check the type of affinity that will be used
+	at, err := parser.GetStringAnnotation(annotationAffinityType, ing)
+	if err != nil {
+		at = ""
+	}
+	//cookieAffinityConfig = CookieAffinityParse(ing)
+	switch at {
+	case "cookie":
+		cookieAffinityConfig = CookieAffinityParse(ing)
+
+	default:
+		glog.V(3).Infof("No default affinity was found for Ingress %v", ing.Name)
+
+	}
+	return &AffinityConfig{
+		AffinityType:         at,
+		CookieAffinityConfig: *cookieAffinityConfig,
+	}, nil
+
+}
diff --git a/core/pkg/ingress/annotations/stickysession/main_test.go b/core/pkg/ingress/annotations/sessionaffinity/main_test.go
similarity index 67%
rename from core/pkg/ingress/annotations/stickysession/main_test.go
rename to core/pkg/ingress/annotations/sessionaffinity/main_test.go
index 71384fad1..b18f86835 100644
--- a/core/pkg/ingress/annotations/stickysession/main_test.go
+++ b/core/pkg/ingress/annotations/sessionaffinity/main_test.go
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package stickysession
+package sessionaffinity
 
 import (
 	"testing"
@@ -59,30 +59,30 @@ func buildIngress() *extensions.Ingress {
 	}
 }
 
-func TestIngressStickySession(t *testing.T) {
+func TestIngressAffinityCookieConfig(t *testing.T) {
 	ing := buildIngress()
 
 	data := map[string]string{}
-	data[stickyEnabled] = "true"
-	data[stickyHash] = "md5"
-	data[stickyName] = "route1"
+	data[annotationAffinityType] = "cookie"
+	data[annotationAffinityCookieHash] = "sha123"
+	data[annotationAffinityCookieName] = "route"
 	ing.SetAnnotations(data)
 
-	sti, _ := NewParser().Parse(ing)
-	nginxSti, ok := sti.(*StickyConfig)
+	affin, _ := NewParser().Parse(ing)
+	nginxAffinity, ok := affin.(*AffinityConfig)
 	if !ok {
-		t.Errorf("expected a StickyConfig type")
+		t.Errorf("expected a Config type")
 	}
 
-	if nginxSti.Hash != "md5" {
-		t.Errorf("expected md5 as sticky-hash but returned %v", nginxSti.Hash)
+	if nginxAffinity.AffinityType != "cookie" {
+		t.Errorf("expected cookie as sticky-type but returned %v", nginxAffinity.AffinityType)
 	}
 
-	if nginxSti.Hash != "md5" {
-		t.Errorf("expected md5 as sticky-hash but returned %v", nginxSti.Hash)
+	if nginxAffinity.CookieAffinityConfig.Hash != "md5" {
+		t.Errorf("expected md5 as sticky-hash but returned %v", nginxAffinity.CookieAffinityConfig.Hash)
 	}
 
-	if !nginxSti.Enabled {
-		t.Errorf("expected sticky-enabled  but returned %v", nginxSti.Enabled)
+	if nginxAffinity.CookieAffinityConfig.Name != "route" {
+		t.Errorf("expected route as sticky-name but returned %v", nginxAffinity.CookieAffinityConfig.Name)
 	}
 }
diff --git a/core/pkg/ingress/annotations/stickysession/main.go b/core/pkg/ingress/annotations/stickysession/main.go
deleted file mode 100644
index 20bae2c8e..000000000
--- a/core/pkg/ingress/annotations/stickysession/main.go
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
-Copyright 2016 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package stickysession
-
-import (
-	"regexp"
-
-	"github.com/golang/glog"
-
-	"k8s.io/kubernetes/pkg/apis/extensions"
-
-	"k8s.io/ingress/core/pkg/ingress/annotations/parser"
-)
-
-const (
-	stickyEnabled     = "ingress.kubernetes.io/sticky-enabled"
-	stickyName        = "ingress.kubernetes.io/sticky-name"
-	stickyHash        = "ingress.kubernetes.io/sticky-hash"
-	defaultStickyHash = "md5"
-	defaultStickyName = "route"
-)
-
-var (
-	stickyHashRegex = regexp.MustCompile(`index|md5|sha1`)
-)
-
-// StickyConfig describes the per ingress sticky session config
-type StickyConfig struct {
-	// The name of the cookie that will be used as stickness router.
-	Name string `json:"name"`
-	// If sticky must or must not be enabled
-	Enabled bool `json:"enabled"`
-	// The hash that will be used to encode the cookie
-	Hash string `json:"hash"`
-}
-
-type sticky struct {
-}
-
-// NewParser creates a new Sticky annotation parser
-func NewParser() parser.IngressAnnotation {
-	return sticky{}
-}
-
-// ParseAnnotations parses the annotations contained in the ingress
-// rule used to configure the sticky directives
-func (a sticky) Parse(ing *extensions.Ingress) (interface{}, error) {
-	// Check if the sticky is enabled
-	se, err := parser.GetBoolAnnotation(stickyEnabled, ing)
-	if err != nil {
-		se = false
-	}
-
-	glog.V(3).Infof("Ingress %v: Setting stickness to %v", ing.Name, se)
-
-	// Get the Sticky Cookie Name
-	sn, err := parser.GetStringAnnotation(stickyName, ing)
-
-	if err != nil || sn == "" {
-		glog.V(3).Infof("Ingress %v: No value found in annotation %v. Using the default %v", ing.Name, stickyName, defaultStickyName)
-		sn = defaultStickyName
-	}
-
-	sh, err := parser.GetStringAnnotation(stickyHash, ing)
-
-	if err != nil || !stickyHashRegex.MatchString(sh) {
-		glog.V(3).Infof("Invalid or no annotation value found in Ingress %v: %v: %v. Setting it to default %v", ing.Name, stickyHash, sh, defaultStickyHash)
-		sh = defaultStickyHash
-	}
-
-	return &StickyConfig{
-		Name:    sn,
-		Enabled: se,
-		Hash:    sh,
-	}, nil
-}
diff --git a/core/pkg/ingress/controller/annotations.go b/core/pkg/ingress/controller/annotations.go
index 400776866..5a54e83cd 100644
--- a/core/pkg/ingress/controller/annotations.go
+++ b/core/pkg/ingress/controller/annotations.go
@@ -33,8 +33,8 @@ import (
 	"k8s.io/ingress/core/pkg/ingress/annotations/ratelimit"
 	"k8s.io/ingress/core/pkg/ingress/annotations/rewrite"
 	"k8s.io/ingress/core/pkg/ingress/annotations/secureupstream"
+	"k8s.io/ingress/core/pkg/ingress/annotations/sessionaffinity"
 	"k8s.io/ingress/core/pkg/ingress/annotations/sslpassthrough"
-	"k8s.io/ingress/core/pkg/ingress/annotations/stickysession"
 	"k8s.io/ingress/core/pkg/ingress/errors"
 	"k8s.io/ingress/core/pkg/ingress/resolver"
 )
@@ -63,8 +63,8 @@ func newAnnotationExtractor(cfg extractorConfig) annotationExtractor {
 			"RateLimit":          ratelimit.NewParser(),
 			"Redirect":           rewrite.NewParser(cfg),
 			"SecureUpstream":     secureupstream.NewParser(),
+			"SessionAffinity":    sessionaffinity.NewParser(),
 			"SSLPassthrough":     sslpassthrough.NewParser(),
-			"StickySession":      stickysession.NewParser(),
 		},
 	}
 }
@@ -98,10 +98,10 @@ func (e *annotationExtractor) Extract(ing *extensions.Ingress) map[string]interf
 }
 
 const (
-	secureUpstream = "SecureUpstream"
-	healthCheck    = "HealthCheck"
-	sslPassthrough = "SSLPassthrough"
-	stickySession  = "StickySession"
+	secureUpstream  = "SecureUpstream"
+	healthCheck     = "HealthCheck"
+	sslPassthrough  = "SSLPassthrough"
+	sessionAffinity = "SessionAffinity"
 )
 
 func (e *annotationExtractor) SecureUpstream(ing *extensions.Ingress) bool {
@@ -119,7 +119,7 @@ func (e *annotationExtractor) SSLPassthrough(ing *extensions.Ingress) bool {
 	return val.(bool)
 }
 
-func (e *annotationExtractor) StickySession(ing *extensions.Ingress) *stickysession.StickyConfig {
-	val, _ := e.annotations[stickySession].Parse(ing)
-	return val.(*stickysession.StickyConfig)
+func (e *annotationExtractor) SessionAffinity(ing *extensions.Ingress) *sessionaffinity.AffinityConfig {
+	val, _ := e.annotations[sessionAffinity].Parse(ing)
+	return val.(*sessionaffinity.AffinityConfig)
 }
diff --git a/core/pkg/ingress/controller/annotations_test.go b/core/pkg/ingress/controller/annotations_test.go
index 80f9c99ca..6927f2733 100644
--- a/core/pkg/ingress/controller/annotations_test.go
+++ b/core/pkg/ingress/controller/annotations_test.go
@@ -28,13 +28,13 @@ import (
 )
 
 const (
-	annotationSecureUpstream = "ingress.kubernetes.io/secure-backends"
-	annotationUpsMaxFails    = "ingress.kubernetes.io/upstream-max-fails"
-	annotationUpsFailTimeout = "ingress.kubernetes.io/upstream-fail-timeout"
-	annotationPassthrough    = "ingress.kubernetes.io/ssl-passthrough"
-	annotationStickyEnabled  = "ingress.kubernetes.io/sticky-enabled"
-	annotationStickyName     = "ingress.kubernetes.io/sticky-name"
-	annotationStickyHash     = "ingress.kubernetes.io/sticky-hash"
+	annotationSecureUpstream     = "ingress.kubernetes.io/secure-backends"
+	annotationUpsMaxFails        = "ingress.kubernetes.io/upstream-max-fails"
+	annotationUpsFailTimeout     = "ingress.kubernetes.io/upstream-fail-timeout"
+	annotationPassthrough        = "ingress.kubernetes.io/ssl-passthrough"
+	annotationAffinityType       = "ingress.kubernetes.io/affinity"
+	annotationAffinityCookieName = "ingress.kubernetes.io/session-cookie-name"
+	annotationAffinityCookieHash = "ingress.kubernetes.io/session-cookie-hash"
 )
 
 type mockCfg struct {
@@ -183,38 +183,38 @@ func TestSSLPassthrough(t *testing.T) {
 	}
 }
 
-func TestStickySession(t *testing.T) {
+func TestAffinitySession(t *testing.T) {
 	ec := newAnnotationExtractor(mockCfg{})
 	ing := buildIngress()
 
 	fooAnns := []struct {
-		annotations map[string]string
-		enabled     bool
-		hash        string
-		name        string
+		annotations  map[string]string
+		affinitytype string
+		hash         string
+		name         string
 	}{
-		{map[string]string{annotationStickyEnabled: "true", annotationStickyHash: "md5", annotationStickyName: "route"}, true, "md5", "route"},
-		{map[string]string{annotationStickyEnabled: "true", annotationStickyHash: "", annotationStickyName: "xpto"}, true, "md5", "xpto"},
-		{map[string]string{annotationStickyEnabled: "true", annotationStickyHash: "", annotationStickyName: ""}, true, "md5", "route"},
-		{map[string]string{}, false, "md5", "route"},
-		{nil, false, "md5", "route"},
+		{map[string]string{annotationAffinityType: "cookie", annotationAffinityCookieHash: "md5", annotationAffinityCookieName: "route"}, "cookie", "md5", "route"},
+		{map[string]string{annotationAffinityType: "cookie", annotationAffinityCookieHash: "xpto", annotationAffinityCookieName: "route1"}, "cookie", "md5", "route1"},
+		{map[string]string{annotationAffinityType: "cookie", annotationAffinityCookieHash: "", annotationAffinityCookieName: ""}, "cookie", "md5", "route"},
+		{map[string]string{}, "", "", ""},
+		{nil, "", "", ""},
 	}
 
 	for _, foo := range fooAnns {
 		ing.SetAnnotations(foo.annotations)
-		r := ec.StickySession(ing)
-
+		r := ec.SessionAffinity(ing)
+		t.Logf("Testing pass %v %v %v", foo.affinitytype, foo.hash, foo.name)
 		if r == nil {
-			t.Errorf("Returned nil but expected a StickySesion.StickyConfig")
+			t.Errorf("Returned nil but expected a SessionAffinity.AffinityConfig")
 			continue
 		}
 
-		if r.Hash != foo.hash {
-			t.Errorf("Returned %v but expected %v for Hash", r.Hash, foo.hash)
+		if r.CookieAffinityConfig.Hash != foo.hash {
+			t.Errorf("Returned %v but expected %v for Hash", r.CookieAffinityConfig.Hash, foo.hash)
 		}
 
-		if r.Name != foo.name {
-			t.Errorf("Returned %v but expected %v for Name", r.Name, foo.name)
+		if r.CookieAffinityConfig.Name != foo.name {
+			t.Errorf("Returned %v but expected %v for Name", r.CookieAffinityConfig.Name, foo.name)
 		}
 	}
 }
diff --git a/core/pkg/ingress/controller/controller.go b/core/pkg/ingress/controller/controller.go
index ace3049b1..1eb0b6f2c 100644
--- a/core/pkg/ingress/controller/controller.go
+++ b/core/pkg/ingress/controller/controller.go
@@ -700,7 +700,7 @@ func (ic *GenericController) createUpstreams(data []interface{}) map[string]*ing
 
 		secUpstream := ic.annotations.SecureUpstream(ing)
 		hz := ic.annotations.HealthCheck(ing)
-		sticky := ic.annotations.StickySession(ing)
+		affinity := ic.annotations.SessionAffinity(ing)
 
 		var defBackend string
 		if ing.Spec.Backend != nil {
@@ -740,10 +740,12 @@ func (ic *GenericController) createUpstreams(data []interface{}) map[string]*ing
 				if !upstreams[name].Secure {
 					upstreams[name].Secure = secUpstream
 				}
-				if !upstreams[name].StickySession.Enabled || upstreams[name].StickySession.Name == "" || upstreams[name].StickySession.Hash == "" {
-					upstreams[name].StickySession.Enabled = sticky.Enabled
-					upstreams[name].StickySession.Name = sticky.Name
-					upstreams[name].StickySession.Hash = sticky.Hash
+				if upstreams[name].SessionAffinity.AffinityType == "" {
+					upstreams[name].SessionAffinity.AffinityType = affinity.AffinityType
+					if affinity.AffinityType == "cookie" {
+						upstreams[name].SessionAffinity.CookieSessionAffinity.Name = affinity.CookieAffinityConfig.Name
+						upstreams[name].SessionAffinity.CookieSessionAffinity.Hash = affinity.CookieAffinityConfig.Hash
+					}
 				}
 
 				svcKey := fmt.Sprintf("%v/%v", ing.GetNamespace(), path.Backend.ServiceName)
diff --git a/core/pkg/ingress/types.go b/core/pkg/ingress/types.go
index a43e1d7a3..9363b7830 100644
--- a/core/pkg/ingress/types.go
+++ b/core/pkg/ingress/types.go
@@ -26,7 +26,6 @@ import (
 	"k8s.io/ingress/core/pkg/ingress/annotations/proxy"
 	"k8s.io/ingress/core/pkg/ingress/annotations/ratelimit"
 	"k8s.io/ingress/core/pkg/ingress/annotations/rewrite"
-	"k8s.io/ingress/core/pkg/ingress/annotations/stickysession"
 	"k8s.io/ingress/core/pkg/ingress/defaults"
 	"k8s.io/ingress/core/pkg/ingress/resolver"
 )
@@ -136,7 +135,25 @@ type Backend struct {
 	// Endpoints contains the list of endpoints currently running
 	Endpoints []Endpoint `json:"endpoints"`
 	// StickySession contains the StickyConfig object with stickness configuration
-	StickySession stickysession.StickyConfig `json:"stickysession,omitempty"`
+
+	SessionAffinity SessionAffinityConfig
+}
+
+// SessionAffinityConfig describes different affinity configurations for new sessions.
+// Once a session is mapped to a backend based on some affinity setting, it
+// retains that mapping till the backend goes down, or the ingress controller
+// restarts. Exactly one of these values will be set on the upstream, since multiple
+// affinity values are incompatible. Once set, the backend makes no guarantees
+// about honoring updates.
+type SessionAffinityConfig struct {
+	AffinityType          string `json:"name"`
+	CookieSessionAffinity CookieSessionAffinity
+}
+
+// CookieSessionAffinity defines the structure used in Affinity configured by Cookies.
+type CookieSessionAffinity struct {
+	Name string `json:"name"`
+	Hash string `json:"hash"`
 }
 
 // Endpoint describes a kubernetes endpoint in an backend
diff --git a/examples/affinity/cookie/nginx/README.md b/examples/affinity/cookie/nginx/README.md
new file mode 100644
index 000000000..2a18f5f66
--- /dev/null
+++ b/examples/affinity/cookie/nginx/README.md
@@ -0,0 +1,73 @@
+# Sticky Session
+
+This example demonstrates how to achieve session affinity using cookies 
+
+## Prerequisites
+
+You will need to make sure you Ingress targets exactly one Ingress
+controller by specifying the [ingress.class annotation](/examples/PREREQUISITES.md#ingress-class),
+and that you have an ingress controller [running](/examples/deployment) in your cluster.
+
+You will also need to deploy multiple replicas of your application that show up as endpoints for the Service referenced in the Ingress object, to test session stickyness.
+Using a deployment with only one replica doesn't set the 'sticky' cookie.
+
+## Deployment
+
+Session stickyness is achieved through 3 annotations on the Ingress, as shown in the [example](sticky-ingress.yaml).
+
+|Name|Description|Values|
+|ingress.kubernetes.io/affinity|Sets the affinity type|string (in NGINX only ``cookie`` is possible|
+|ingress.kubernetes.io/session-cookie-name|Name of the cookie that will be used|string (default to route)|
+|ingress.kubernetes.io/session-cookie-hash|Type of hash that will be used in cookie value|sha1/md5/index|
+
+You can create the ingress to test this
+
+```console
+$ kubectl create -f sticky-ingress.yaml
+```
+
+## Validation
+
+You can confirm that the Ingress works.
+
+```console
+$ kubectl describe ing nginx-test
+Name:			nginx-test
+Namespace:		default
+Address:		
+Default backend:	default-http-backend:80 (10.180.0.4:8080,10.240.0.2:8080)
+Rules:
+  Host	                        Path	Backends
+  ----	                        ----	--------
+  stickyingress.example.com     
+                                /   	 nginx-service:80 (<none>)
+Annotations:
+  affinity:	cookie
+  session-cookie-hash:		sha1
+  session-cookie-name:		route
+Events:
+  FirstSeen	LastSeen	Count	From				SubObjectPath	Type		Reason	Message
+  ---------	--------	-----	----				-------------	--------	------	-------
+  7s		7s		1	{nginx-ingress-controller }			Normal		CREATE	default/nginx-test
+  
+
+$ curl -I http://stickyingress.example.com
+HTTP/1.1 200 OK
+Server: nginx/1.11.9
+Date: Fri, 10 Feb 2017 14:11:12 GMT
+Content-Type: text/html
+Content-Length: 612
+Connection: keep-alive
+Set-Cookie: route=a9907b79b248140b56bb13723f72b67697baac3d; Path=/; HttpOnly
+Last-Modified: Tue, 24 Jan 2017 14:02:19 GMT
+ETag: "58875e6b-264"
+Accept-Ranges: bytes
+```
+In the example above, you can see a line containing the 'Set-Cookie: route' setting the right defined stickness cookie.
+This cookie is created by NGINX containing the hash of the used upstream in that request. 
+If the user changes this cookie, NGINX creates a new one and redirect the user to another upstream.
+
+If the backend pool grows up NGINX will keep sending the requests through the same server of the first request, even if it's overloaded.
+
+When the backend server is removed, the requests are then re-routed to another upstream server and NGINX creates a new cookie, as the previous hash became invalid.
+
diff --git a/examples/stickysession/nginx/sticky-ingress.yaml b/examples/affinity/cookie/nginx/sticky-ingress.yaml
similarity index 66%
rename from examples/stickysession/nginx/sticky-ingress.yaml
rename to examples/affinity/cookie/nginx/sticky-ingress.yaml
index fe7dd42b3..69beea75e 100644
--- a/examples/stickysession/nginx/sticky-ingress.yaml
+++ b/examples/affinity/cookie/nginx/sticky-ingress.yaml
@@ -4,9 +4,9 @@ metadata:
   name: nginx-test
   annotations:
     kubernetes.io/ingress.class: "nginx"
-    ingress.kubernetes.io/sticky-enabled: "true"
-    ingress.kubernetes.io/sticky-name: "route"
-    ingress.kubernetes.io/sticky-hash: "sha1"
+    ingress.kubernetes.io/affinity: "cookie"
+    ingress.kubernetes.io/session-cookie-name: "route"
+    ingress.kubernetes.io/session-cookie-hash: "sha1"
 
 spec:
   rules:
diff --git a/examples/stickysession/nginx/README.md b/examples/stickysession/nginx/README.md
deleted file mode 100644
index ba5bf9abc..000000000
--- a/examples/stickysession/nginx/README.md
+++ /dev/null
@@ -1,58 +0,0 @@
-# Sticky Session
-
-This example demonstrates how to Stickness in a Ingress.
-
-## Prerequisites
-
-You will need to make sure you Ingress targets exactly one Ingress
-controller by specifying the [ingress.class annotation](/examples/PREREQUISITES.md#ingress-class),
-and that you have an ingress controller [running](/examples/deployment) in your cluster.
-
-Also, you need to have a deployment with replica > 1. Using a deployment with only one replica doesn't set the 'sticky' cookie.
-
-## Deployment
-
-The following command instructs the controller to set Stickness in all Upstreams of an Ingress
-
-```console
-$ kubectl create -f sticky-ingress.yaml
-```
-
-## Validation
-
-You can confirm that the Ingress works.
-
-```console
-$ kubectl describe ing nginx-test
-Name:			nginx-test
-Namespace:		default
-Address:		
-Default backend:	default-http-backend:80 (10.180.0.4:8080,10.240.0.2:8080)
-Rules:
-  Host	                        Path	Backends
-  ----	                        ----	--------
-  stickyingress.example.com     
-                                /   	 nginx-service:80 (<none>)
-Annotations:
-  sticky-enabled:	true
-  sticky-hash:		sha1
-  sticky-name:		route
-Events:
-  FirstSeen	LastSeen	Count	From				SubObjectPath	Type		Reason	Message
-  ---------	--------	-----	----				-------------	--------	------	-------
-  7s		7s		1	{nginx-ingress-controller }			Normal		CREATE	default/nginx-test
-  
-
-$ curl -I http://stickyingress.example.com
-HTTP/1.1 200 OK
-Server: nginx/1.11.9
-Date: Fri, 10 Feb 2017 14:11:12 GMT
-Content-Type: text/html
-Content-Length: 612
-Connection: keep-alive
-Set-Cookie: route=a9907b79b248140b56bb13723f72b67697baac3d; Path=/; HttpOnly
-Last-Modified: Tue, 24 Jan 2017 14:02:19 GMT
-ETag: "58875e6b-264"
-Accept-Ranges: bytes
-```
-In the example avove, you can see a line containing the 'Set-Cookie: route' setting the right defined stickness cookie.
\ No newline at end of file

From f46aedd7a24dc4d8199e6cd686e8c80b488412d4 Mon Sep 17 00:00:00 2001
From: Arnd Hannemann <arnd.hannemann@credativ.de>
Date: Tue, 14 Feb 2017 10:06:44 +0100
Subject: [PATCH 27/73] Fix typo in nginx README

---
 controllers/nginx/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/controllers/nginx/README.md b/controllers/nginx/README.md
index 8796b932c..f55cfd32d 100644
--- a/controllers/nginx/README.md
+++ b/controllers/nginx/README.md
@@ -40,7 +40,7 @@ Anytime we reference a tls secret, we mean (x509, pem encoded, RSA 2048, etc). Y
 - Default backend [404-server](https://github.com/kubernetes/contrib/tree/master/404-server)
 
 
-## Dry running the Ingress controller
+## Try running the Ingress controller
 
 Before deploying the controller to production you might want to run it outside the cluster and observe it.
 

From 0161ae43d958035c2d79873cdb738532f466dbda Mon Sep 17 00:00:00 2001
From: Ricardo Pchevuzinske Katz <ricardo.katz@serpro.gov.br>
Date: Tue, 14 Feb 2017 08:49:10 -0200
Subject: [PATCH 28/73] Improve in documentation and naming case

---
 .../annotations/sessionaffinity/main.go       | 30 +++++++++----------
 .../annotations/sessionaffinity/main_test.go  |  4 +--
 .../ingress/controller/annotations_test.go    | 10 +++----
 core/pkg/ingress/controller/controller.go     |  4 +--
 examples/affinity/cookie/nginx/README.md      |  1 +
 5 files changed, 25 insertions(+), 24 deletions(-)

diff --git a/core/pkg/ingress/annotations/sessionaffinity/main.go b/core/pkg/ingress/annotations/sessionaffinity/main.go
index 551cde2a3..3cf5181e9 100644
--- a/core/pkg/ingress/annotations/sessionaffinity/main.go
+++ b/core/pkg/ingress/annotations/sessionaffinity/main.go
@@ -31,7 +31,7 @@ const (
 	// If a cookie with this name exists,
 	// its value is used as an index into the list of available backends.
 	annotationAffinityCookieName = "ingress.kubernetes.io/session-cookie-name"
-	defaultAffinityCookieName    = "route"
+	defaultAffinityCookieName    = "INGRESSCOOKIE"
 	// This is the algorithm used by nginx to generate a value for the session cookie, if
 	// one isn't supplied and affintiy is set to "cookie".
 	annotationAffinityCookieHash = "ingress.kubernetes.io/session-cookie-hash"
@@ -45,24 +45,21 @@ var (
 // AffinityConfig describes the per ingress session affinity config
 type AffinityConfig struct {
 	// The type of affinity that will be used
-	AffinityType         string               `json:"type"`
-	CookieAffinityConfig CookieAffinityConfig `json:"cookieconfig"`
+	AffinityType string `json:"type"`
+	CookieConfig
 }
 
-// CookieAffinityConfig describes the Config of cookie type affinity
-type CookieAffinityConfig struct {
+// CookieConfig describes the Config of cookie type affinity
+type CookieConfig struct {
 	// The name of the cookie that will be used in case of cookie affinity type.
 	Name string `json:"name"`
 	// The hash that will be used to encode the cookie in case of cookie affinity type
 	Hash string `json:"hash"`
 }
 
-type affinity struct {
-}
-
 // CookieAffinityParse gets the annotation values related to Cookie Affinity
 // It also sets default values when no value or incorrect value is found
-func CookieAffinityParse(ing *extensions.Ingress) *CookieAffinityConfig {
+func CookieAffinityParse(ing *extensions.Ingress) *CookieConfig {
 
 	sn, err := parser.GetStringAnnotation(annotationAffinityCookieName, ing)
 
@@ -78,7 +75,7 @@ func CookieAffinityParse(ing *extensions.Ingress) *CookieAffinityConfig {
 		sh = defaultAffinityCookieHash
 	}
 
-	return &CookieAffinityConfig{
+	return &CookieConfig{
 		Name: sn,
 		Hash: sh,
 	}
@@ -89,19 +86,22 @@ func NewParser() parser.IngressAnnotation {
 	return affinity{}
 }
 
+type affinity struct {
+}
+
 // ParseAnnotations parses the annotations contained in the ingress
 // rule used to configure the affinity directives
 func (a affinity) Parse(ing *extensions.Ingress) (interface{}, error) {
 
-	var cookieAffinityConfig *CookieAffinityConfig
-	cookieAffinityConfig = &CookieAffinityConfig{}
+	var cookieAffinityConfig *CookieConfig
+	cookieAffinityConfig = &CookieConfig{}
 
 	// Check the type of affinity that will be used
 	at, err := parser.GetStringAnnotation(annotationAffinityType, ing)
 	if err != nil {
 		at = ""
 	}
-	//cookieAffinityConfig = CookieAffinityParse(ing)
+
 	switch at {
 	case "cookie":
 		cookieAffinityConfig = CookieAffinityParse(ing)
@@ -111,8 +111,8 @@ func (a affinity) Parse(ing *extensions.Ingress) (interface{}, error) {
 
 	}
 	return &AffinityConfig{
-		AffinityType:         at,
-		CookieAffinityConfig: *cookieAffinityConfig,
+		AffinityType: at,
+		CookieConfig: *cookieAffinityConfig,
 	}, nil
 
 }
diff --git a/core/pkg/ingress/annotations/sessionaffinity/main_test.go b/core/pkg/ingress/annotations/sessionaffinity/main_test.go
index b18f86835..b898813ec 100644
--- a/core/pkg/ingress/annotations/sessionaffinity/main_test.go
+++ b/core/pkg/ingress/annotations/sessionaffinity/main_test.go
@@ -65,7 +65,7 @@ func TestIngressAffinityCookieConfig(t *testing.T) {
 	data := map[string]string{}
 	data[annotationAffinityType] = "cookie"
 	data[annotationAffinityCookieHash] = "sha123"
-	data[annotationAffinityCookieName] = "route"
+	data[annotationAffinityCookieName] = "INGRESSCOOKIE"
 	ing.SetAnnotations(data)
 
 	affin, _ := NewParser().Parse(ing)
@@ -82,7 +82,7 @@ func TestIngressAffinityCookieConfig(t *testing.T) {
 		t.Errorf("expected md5 as sticky-hash but returned %v", nginxAffinity.CookieAffinityConfig.Hash)
 	}
 
-	if nginxAffinity.CookieAffinityConfig.Name != "route" {
+	if nginxAffinity.CookieAffinityConfig.Name != "INGRESSCOOKIE" {
 		t.Errorf("expected route as sticky-name but returned %v", nginxAffinity.CookieAffinityConfig.Name)
 	}
 }
diff --git a/core/pkg/ingress/controller/annotations_test.go b/core/pkg/ingress/controller/annotations_test.go
index 6927f2733..1b3b9b08d 100644
--- a/core/pkg/ingress/controller/annotations_test.go
+++ b/core/pkg/ingress/controller/annotations_test.go
@@ -195,7 +195,7 @@ func TestAffinitySession(t *testing.T) {
 	}{
 		{map[string]string{annotationAffinityType: "cookie", annotationAffinityCookieHash: "md5", annotationAffinityCookieName: "route"}, "cookie", "md5", "route"},
 		{map[string]string{annotationAffinityType: "cookie", annotationAffinityCookieHash: "xpto", annotationAffinityCookieName: "route1"}, "cookie", "md5", "route1"},
-		{map[string]string{annotationAffinityType: "cookie", annotationAffinityCookieHash: "", annotationAffinityCookieName: ""}, "cookie", "md5", "route"},
+		{map[string]string{annotationAffinityType: "cookie", annotationAffinityCookieHash: "", annotationAffinityCookieName: ""}, "cookie", "md5", "INGRESSCOOKIE"},
 		{map[string]string{}, "", "", ""},
 		{nil, "", "", ""},
 	}
@@ -209,12 +209,12 @@ func TestAffinitySession(t *testing.T) {
 			continue
 		}
 
-		if r.CookieAffinityConfig.Hash != foo.hash {
-			t.Errorf("Returned %v but expected %v for Hash", r.CookieAffinityConfig.Hash, foo.hash)
+		if r.CookieConfig.Hash != foo.hash {
+			t.Errorf("Returned %v but expected %v for Hash", r.CookieConfig.Hash, foo.hash)
 		}
 
-		if r.CookieAffinityConfig.Name != foo.name {
-			t.Errorf("Returned %v but expected %v for Name", r.CookieAffinityConfig.Name, foo.name)
+		if r.CookieConfig.Name != foo.name {
+			t.Errorf("Returned %v but expected %v for Name", r.CookieConfig.Name, foo.name)
 		}
 	}
 }
diff --git a/core/pkg/ingress/controller/controller.go b/core/pkg/ingress/controller/controller.go
index 1eb0b6f2c..2a91640fc 100644
--- a/core/pkg/ingress/controller/controller.go
+++ b/core/pkg/ingress/controller/controller.go
@@ -743,8 +743,8 @@ func (ic *GenericController) createUpstreams(data []interface{}) map[string]*ing
 				if upstreams[name].SessionAffinity.AffinityType == "" {
 					upstreams[name].SessionAffinity.AffinityType = affinity.AffinityType
 					if affinity.AffinityType == "cookie" {
-						upstreams[name].SessionAffinity.CookieSessionAffinity.Name = affinity.CookieAffinityConfig.Name
-						upstreams[name].SessionAffinity.CookieSessionAffinity.Hash = affinity.CookieAffinityConfig.Hash
+						upstreams[name].SessionAffinity.CookieSessionAffinity.Name = affinity.CookieConfig.Name
+						upstreams[name].SessionAffinity.CookieSessionAffinity.Hash = affinity.CookieConfig.Hash
 					}
 				}
 
diff --git a/examples/affinity/cookie/nginx/README.md b/examples/affinity/cookie/nginx/README.md
index 2a18f5f66..ea5a3d3b2 100644
--- a/examples/affinity/cookie/nginx/README.md
+++ b/examples/affinity/cookie/nginx/README.md
@@ -16,6 +16,7 @@ Using a deployment with only one replica doesn't set the 'sticky' cookie.
 Session stickyness is achieved through 3 annotations on the Ingress, as shown in the [example](sticky-ingress.yaml).
 
 |Name|Description|Values|
+| --- | --- | --- |
 |ingress.kubernetes.io/affinity|Sets the affinity type|string (in NGINX only ``cookie`` is possible|
 |ingress.kubernetes.io/session-cookie-name|Name of the cookie that will be used|string (default to route)|
 |ingress.kubernetes.io/session-cookie-hash|Type of hash that will be used in cookie value|sha1/md5/index|

From 5c9bf126487bc43cede6eec3c5a5f30b72ec6deb Mon Sep 17 00:00:00 2001
From: Manuel de Brito Fontes <aledbf@gmail.com>
Date: Tue, 14 Feb 2017 11:02:23 -0300
Subject: [PATCH 29/73] Fix error getting class information from Ingress
 annotations

---
 controllers/nginx/pkg/cmd/controller/nginx.go |  6 ++++++
 core/pkg/ingress/annotations/authreq/main.go  |  6 +++++-
 core/pkg/ingress/controller/controller.go     | 19 +++++++++++++++++--
 core/pkg/ingress/controller/launch.go         |  2 ++
 core/pkg/ingress/controller/util.go           |  6 +++++-
 core/pkg/ingress/controller/util_test.go      |  8 +++++++-
 core/pkg/ingress/types.go                     |  4 ++++
 7 files changed, 46 insertions(+), 5 deletions(-)

diff --git a/controllers/nginx/pkg/cmd/controller/nginx.go b/controllers/nginx/pkg/cmd/controller/nginx.go
index 24a9a7046..ffc04e21b 100644
--- a/controllers/nginx/pkg/cmd/controller/nginx.go
+++ b/controllers/nginx/pkg/cmd/controller/nginx.go
@@ -30,6 +30,7 @@ import (
 
 	"github.com/golang/glog"
 	"github.com/mitchellh/mapstructure"
+	"github.com/spf13/pflag"
 
 	"k8s.io/kubernetes/pkg/api"
 
@@ -251,6 +252,11 @@ func (n NGINXController) Info() *ingress.BackendInfo {
 	}
 }
 
+// OverrideFlags customize NGINX controller flags
+func (n NGINXController) OverrideFlags(flags *pflag.FlagSet) {
+	flags.Set("ingress-class", "nginx")
+}
+
 // testTemplate checks if the NGINX configuration inside the byte array is valid
 // running the command "nginx -t" using a temporal file.
 func (n NGINXController) testTemplate(cfg []byte) error {
diff --git a/core/pkg/ingress/annotations/authreq/main.go b/core/pkg/ingress/annotations/authreq/main.go
index 560a73868..31c208507 100644
--- a/core/pkg/ingress/annotations/authreq/main.go
+++ b/core/pkg/ingress/annotations/authreq/main.go
@@ -92,7 +92,11 @@ func (a authReq) Parse(ing *extensions.Ingress) (interface{}, error) {
 		return nil, ing_errors.NewLocationDenied("invalid url host")
 	}
 
-	m, _ := parser.GetStringAnnotation(authMethod, ing)
+	m, err := parser.GetStringAnnotation(authMethod, ing)
+	if err != nil {
+		return nil, err
+	}
+
 	if len(m) != 0 && !validMethod(m) {
 		return nil, ing_errors.NewLocationDenied("invalid HTTP method")
 	}
diff --git a/core/pkg/ingress/controller/controller.go b/core/pkg/ingress/controller/controller.go
index 5c979dcce..87fbdbaa4 100644
--- a/core/pkg/ingress/controller/controller.go
+++ b/core/pkg/ingress/controller/controller.go
@@ -173,7 +173,7 @@ func newIngressController(config *Configuration) *GenericController {
 		DeleteFunc: func(obj interface{}) {
 			delIng := obj.(*extensions.Ingress)
 			if !IsValidClass(delIng, config.IngressClass) {
-				glog.Infof("ignoring add for ingress %v based on annotation %v", delIng.Name, ingressClassKey)
+				glog.Infof("ignoring delete for ingress %v based on annotation %v", delIng.Name, ingressClassKey)
 				return
 			}
 			ic.recorder.Eventf(delIng, api.EventTypeNormal, "DELETE", fmt.Sprintf("Ingress %s/%s", delIng.Namespace, delIng.Name))
@@ -182,7 +182,7 @@ func newIngressController(config *Configuration) *GenericController {
 		UpdateFunc: func(old, cur interface{}) {
 			oldIng := old.(*extensions.Ingress)
 			curIng := cur.(*extensions.Ingress)
-			if !IsValidClass(curIng, config.IngressClass) {
+			if !IsValidClass(curIng, config.IngressClass) && !IsValidClass(oldIng, config.IngressClass) {
 				return
 			}
 
@@ -564,6 +564,10 @@ func (ic *GenericController) getBackendServers() ([]*ingress.Backend, []*ingress
 	for _, ingIf := range ings {
 		ing := ingIf.(*extensions.Ingress)
 
+		if !IsValidClass(ing, ic.cfg.IngressClass) {
+			continue
+		}
+
 		anns := ic.annotations.Extract(ing)
 
 		for _, rule := range ing.Spec.Rules {
@@ -698,6 +702,10 @@ func (ic *GenericController) createUpstreams(data []interface{}) map[string]*ing
 	for _, ingIf := range data {
 		ing := ingIf.(*extensions.Ingress)
 
+		if !IsValidClass(ing, ic.cfg.IngressClass) {
+			continue
+		}
+
 		secUpstream := ic.annotations.SecureUpstream(ing)
 		hz := ic.annotations.HealthCheck(ing)
 
@@ -840,6 +848,10 @@ func (ic *GenericController) createServers(data []interface{}, upstreams map[str
 	// initialize all the servers
 	for _, ingIf := range data {
 		ing := ingIf.(*extensions.Ingress)
+		if !IsValidClass(ing, ic.cfg.IngressClass) {
+			continue
+		}
+
 		// check if ssl passthrough is configured
 		sslpt := ic.annotations.SSLPassthrough(ing)
 
@@ -868,6 +880,9 @@ func (ic *GenericController) createServers(data []interface{}, upstreams map[str
 	// configure default location and SSL
 	for _, ingIf := range data {
 		ing := ingIf.(*extensions.Ingress)
+		if !IsValidClass(ing, ic.cfg.IngressClass) {
+			continue
+		}
 
 		for _, rule := range ing.Spec.Rules {
 			host := rule.Host
diff --git a/core/pkg/ingress/controller/launch.go b/core/pkg/ingress/controller/launch.go
index a1f325d4f..ce23f9a5c 100644
--- a/core/pkg/ingress/controller/launch.go
+++ b/core/pkg/ingress/controller/launch.go
@@ -84,6 +84,8 @@ func NewIngressController(backend ingress.Controller) *GenericController {
 		ingress controller should update the Ingress status IP/hostname. Default is true`)
 	)
 
+	backend.OverrideFlags(flags)
+
 	flags.AddGoFlagSet(flag.CommandLine)
 	flags.Parse(os.Args)
 
diff --git a/core/pkg/ingress/controller/util.go b/core/pkg/ingress/controller/util.go
index 439526cba..7ff154d8f 100644
--- a/core/pkg/ingress/controller/util.go
+++ b/core/pkg/ingress/controller/util.go
@@ -26,6 +26,7 @@ import (
 
 	"k8s.io/ingress/core/pkg/ingress"
 	"k8s.io/ingress/core/pkg/ingress/annotations/parser"
+	"k8s.io/ingress/core/pkg/ingress/errors"
 )
 
 // DeniedKeyName name of the key that contains the reason to deny a location
@@ -92,7 +93,10 @@ func IsValidClass(ing *extensions.Ingress, class string) bool {
 		return true
 	}
 
-	cc, _ := parser.GetStringAnnotation(ingressClassKey, ing)
+	cc, err := parser.GetStringAnnotation(ingressClassKey, ing)
+	if err != nil && !errors.IsMissingAnnotations(err) {
+		glog.Warningf("unexpected error reading ingress annotation: %v", err)
+	}
 	if cc == "" {
 		return true
 	}
diff --git a/core/pkg/ingress/controller/util_test.go b/core/pkg/ingress/controller/util_test.go
index 370714533..8ec84785a 100644
--- a/core/pkg/ingress/controller/util_test.go
+++ b/core/pkg/ingress/controller/util_test.go
@@ -19,6 +19,8 @@ package controller
 import (
 	"testing"
 
+	"reflect"
+
 	"k8s.io/ingress/core/pkg/ingress"
 	"k8s.io/ingress/core/pkg/ingress/annotations/auth"
 	"k8s.io/ingress/core/pkg/ingress/annotations/authreq"
@@ -29,7 +31,6 @@ import (
 	"k8s.io/ingress/core/pkg/ingress/resolver"
 	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/apis/extensions"
-	"reflect"
 )
 
 type fakeError struct{}
@@ -54,6 +55,7 @@ func TestIsValidClass(t *testing.T) {
 	data := map[string]string{}
 	data[ingressClassKey] = "custom"
 	ing.SetAnnotations(data)
+
 	b = IsValidClass(ing, "custom")
 	if !b {
 		t.Errorf("Expected valid class but %v returned", b)
@@ -62,6 +64,10 @@ func TestIsValidClass(t *testing.T) {
 	if b {
 		t.Errorf("Expected invalid class but %v returned", b)
 	}
+	b = IsValidClass(ing, "")
+	if !b {
+		t.Errorf("Expected invalid class but %v returned", b)
+	}
 }
 
 func TestIsHostValid(t *testing.T) {
diff --git a/core/pkg/ingress/types.go b/core/pkg/ingress/types.go
index e619b5dc6..f9aad07ab 100644
--- a/core/pkg/ingress/types.go
+++ b/core/pkg/ingress/types.go
@@ -17,6 +17,8 @@ limitations under the License.
 package ingress
 
 import (
+	"github.com/spf13/pflag"
+
 	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/healthz"
 
@@ -86,6 +88,8 @@ type Controller interface {
 	BackendDefaults() defaults.Backend
 	// Info returns information about the ingress controller
 	Info() *BackendInfo
+	// OverrideFlags allow the customization of the flags in the backend
+	OverrideFlags(*pflag.FlagSet)
 }
 
 // BackendInfo returns information about the backend.

From 0cdc4bd8baf4d29e594af849ac1a96ab3cbc4a15 Mon Sep 17 00:00:00 2001
From: Manuel de Brito Fontes <aledbf@gmail.com>
Date: Tue, 14 Feb 2017 14:24:25 -0300
Subject: [PATCH 30/73] Pass headers to custom error backend

---
 .../nginx/rootfs/etc/nginx/lua/error_page.lua      | 14 +++++++-------
 .../nginx/rootfs/etc/nginx/template/nginx.tmpl     |  4 ++--
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/controllers/nginx/rootfs/etc/nginx/lua/error_page.lua b/controllers/nginx/rootfs/etc/nginx/lua/error_page.lua
index e8a26ee9e..2b9178a56 100644
--- a/controllers/nginx/rootfs/etc/nginx/lua/error_page.lua
+++ b/controllers/nginx/rootfs/etc/nginx/lua/error_page.lua
@@ -8,17 +8,17 @@ local get_upstreams = upstream.get_upstreams
 local random = math.random
 local us = get_upstreams()
 
-function openURL(status)
+function openURL(original_headers, status)
     local httpc = http.new()
 
+    original_headers["X-Code"] = status or "404"
+    original_headers["X-Format"] = original_headers["Accept"] or "text/html"
+
     local random_backend = get_destination()
     local res, err = httpc:request_uri(random_backend, {
         path = "/",
         method = "GET",
-        headers = {
-            ["X-Code"] = status or "404",
-            ["X-Format"] = ngx.var.httpAccept or "html",
-        }
+        headers = original_headers,
     })
 
     if not res then
@@ -26,8 +26,8 @@ function openURL(status)
         ngx.exit(500)
     end
 
-    if ngx.var.http_cookie then
-        ngx.header["Cookie"] = ngx.var.http_cookie
+    for k,v in pairs(res.headers) do
+        ngx.header[k] = v
     end
 
     ngx.status = tonumber(status)
diff --git a/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl b/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl
index d018d4ebb..c3da80de8 100644
--- a/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl
+++ b/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl
@@ -407,7 +407,7 @@ http {
         location / {
             {{ if .CustomErrors }}
             content_by_lua_block {
-                openURL(503)
+                openURL(ngx.req.get_headers(0), 503)
             }
             {{ else }}
             return 503;
@@ -477,7 +477,7 @@ stream {
         location @custom_{{ $errCode }} {
             internal;
             content_by_lua_block {
-                openURL({{ $errCode }})
+                openURL(ngx.req.get_headers(0), {{ $errCode }})
             }
         }    
         {{ end }}

From 6c35adbfd2e30403c84f10fe519f446ecbae7e14 Mon Sep 17 00:00:00 2001
From: chentao1596 <chen.tao3@zte.com.cn>
Date: Wed, 15 Feb 2017 08:53:45 +0800
Subject: [PATCH 31/73] fix wrong link in the file of examples/README.md

---
 examples/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/README.md b/examples/README.md
index 01d842eb2..aa738249d 100644
--- a/examples/README.md
+++ b/examples/README.md
@@ -1,7 +1,7 @@
 # Ingress examples
 
 This directory contains a catalog of examples on how to run, configure and
-scale Ingress. Please review the [prerequisities](prerequisites.md) before
+scale Ingress. Please review the [prerequisities](PREREQUISITES.md) before
 trying them.
 
 ## Basic cross platform

From b06ead1ea3da8f005bda50d14806ac44a5399755 Mon Sep 17 00:00:00 2001
From: Ricardo Pchevuzinske Katz <ricardo.katz@serpro.gov.br>
Date: Wed, 15 Feb 2017 11:50:10 -0200
Subject: [PATCH 32/73] Corrects the affinity test

---
 core/pkg/ingress/annotations/sessionaffinity/main_test.go | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/core/pkg/ingress/annotations/sessionaffinity/main_test.go b/core/pkg/ingress/annotations/sessionaffinity/main_test.go
index b898813ec..3a3a17200 100644
--- a/core/pkg/ingress/annotations/sessionaffinity/main_test.go
+++ b/core/pkg/ingress/annotations/sessionaffinity/main_test.go
@@ -78,11 +78,11 @@ func TestIngressAffinityCookieConfig(t *testing.T) {
 		t.Errorf("expected cookie as sticky-type but returned %v", nginxAffinity.AffinityType)
 	}
 
-	if nginxAffinity.CookieAffinityConfig.Hash != "md5" {
-		t.Errorf("expected md5 as sticky-hash but returned %v", nginxAffinity.CookieAffinityConfig.Hash)
+	if nginxAffinity.CookieConfig.Hash != "md5" {
+		t.Errorf("expected md5 as sticky-hash but returned %v", nginxAffinity.CookieConfig.Hash)
 	}
 
-	if nginxAffinity.CookieAffinityConfig.Name != "INGRESSCOOKIE" {
-		t.Errorf("expected route as sticky-name but returned %v", nginxAffinity.CookieAffinityConfig.Name)
+	if nginxAffinity.CookieConfig.Name != "INGRESSCOOKIE" {
+		t.Errorf("expected route as sticky-name but returned %v", nginxAffinity.CookieConfig.Name)
 	}
 }

From 2b02ea6530eff56172cbf14338b5e86b4522d273 Mon Sep 17 00:00:00 2001
From: Andrew Stuart <andrew.stuart2@gmail.com>
Date: Wed, 15 Feb 2017 12:29:27 -0700
Subject: [PATCH 33/73] Add chmod up directory tree for world read/execute on
 directories

---
 core/pkg/ingress/annotations/auth/main.go | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/core/pkg/ingress/annotations/auth/main.go b/core/pkg/ingress/annotations/auth/main.go
index 2921a550c..fe2167f03 100644
--- a/core/pkg/ingress/annotations/auth/main.go
+++ b/core/pkg/ingress/annotations/auth/main.go
@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"path"
 	"regexp"
 
 	"github.com/pkg/errors"
@@ -59,8 +60,17 @@ type auth struct {
 
 // NewParser creates a new authentication annotation parser
 func NewParser(authDirectory string, sr resolver.Secret) parser.IngressAnnotation {
-	// TODO: check permissions required
-	os.MkdirAll(authDirectory, 0655)
+	os.MkdirAll(authDirectory, 0755)
+
+	currPath := authDirectory
+	for currPath != "/" {
+		currPath = path.Dir(currPath)
+		err := os.Chmod(currPath, 0755)
+		if err != nil {
+			break
+		}
+	}
+
 	return auth{sr, authDirectory}
 }
 

From e5c9c788a5588772775a095dd00a376b3b3b8f65 Mon Sep 17 00:00:00 2001
From: Ricardo Pchevuzinske Katz <ricardo.katz@serpro.gov.br>
Date: Thu, 16 Feb 2017 08:31:01 -0200
Subject: [PATCH 34/73] Correct the configuration.md reference to annotations

---
 controllers/nginx/configuration.md | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/controllers/nginx/configuration.md b/controllers/nginx/configuration.md
index 2b04e55dd..53640f94f 100644
--- a/controllers/nginx/configuration.md
+++ b/controllers/nginx/configuration.md
@@ -51,9 +51,9 @@ The following annotations are supported:
 |[ingress.kubernetes.io/upstream-max-fails](#custom-nginx-upstream-checks)|number|
 |[ingress.kubernetes.io/upstream-fail-timeout](#custom-nginx-upstream-checks)|number|
 |[ingress.kubernetes.io/whitelist-source-range](#whitelist-source-range)|CIDR|
-|[ingress.kubernetes.io/sticky-enabled](#sticky-session)|true or false|
-|[ingress.kubernetes.io/sticky-name](#sticky-session)|string|
-|[ingress.kubernetes.io/sticky-hash](#sticky-session)|string|
+|[ingress.kubernetes.io/affinity](#session-affinity)|true or false|
+|[ingress.kubernetes.io/session-cookie-name](#cookie-affinity)|string|
+|[ingress.kubernetes.io/session-cookie-hash](#cookie-affinity)|string|
 
 
 
@@ -180,7 +180,7 @@ To configure this setting globally for all Ingress rules, the `whitelist-source-
 Please check the [whitelist](examples/affinity/cookie/nginx/README.md) example.
 
 
-### Sticky Session
+### Session Affinity
 
 The annotation `ingress.kubernetes.io/affinity` enables and sets the affinity type in all Upstreams of an Ingress. This way, a request will always be directed to the same upstream server. 
 

From 4be502efaa9e81f3dcba180e3f7db6bdb174b696 Mon Sep 17 00:00:00 2001
From: Ricardo Pchevuzinske Katz <ricardo.katz@serpro.gov.br>
Date: Thu, 16 Feb 2017 08:40:35 -0200
Subject: [PATCH 35/73] Adds a new situation in affinity config example

---
 examples/affinity/cookie/nginx/README.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/examples/affinity/cookie/nginx/README.md b/examples/affinity/cookie/nginx/README.md
index ea5a3d3b2..51aeec310 100644
--- a/examples/affinity/cookie/nginx/README.md
+++ b/examples/affinity/cookie/nginx/README.md
@@ -72,3 +72,6 @@ If the backend pool grows up NGINX will keep sending the requests through the sa
 
 When the backend server is removed, the requests are then re-routed to another upstream server and NGINX creates a new cookie, as the previous hash became invalid.
 
+When you have more than one Ingress Object pointing to the same Service, but one containing affinity configuration and other don't, the first created Ingress will be used. 
+This means that you can face the situation that you've configured Session Affinity in one Ingress and it doesn't reflects in NGINX configuration, because there is another Ingress Object pointing to the same service that doesn't configure this.
+

From b22c19309c768dd3b351aea8e68536a2e050ea74 Mon Sep 17 00:00:00 2001
From: Manuel de Brito Fontes <aledbf@gmail.com>
Date: Thu, 16 Feb 2017 10:27:35 -0300
Subject: [PATCH 36/73] Release ubuntu-slim 0.7

---
 images/ubuntu-slim/Dockerfile.build |  8 ++++++++
 images/ubuntu-slim/Makefile         |  2 +-
 images/ubuntu-slim/README.md        | 10 +++++++---
 3 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/images/ubuntu-slim/Dockerfile.build b/images/ubuntu-slim/Dockerfile.build
index 1a1844e27..20949f7d6 100644
--- a/images/ubuntu-slim/Dockerfile.build
+++ b/images/ubuntu-slim/Dockerfile.build
@@ -7,6 +7,7 @@ COPY excludes /etc/dpkg/dpkg.cfg.d/excludes
 RUN apt-get update \
     && apt-get dist-upgrade -y
 
+# no-op script removes the need for systemd-sysv
 COPY runlevel /sbin/runlevel
 
 # hold required packages to avoid breaking the installation of packages
@@ -14,10 +15,17 @@ RUN apt-mark hold apt gnupg adduser passwd libsemanage1
 
 # dpkg --get-selections | grep -v deinstall
 RUN echo "Yes, do as I say!" | apt-get purge \
+    e2fslibs \
     libcap2-bin \
     libkmod2 \
+    libmount1 \
+    libncursesw5 \
+    libprocps4 \
     libsmartcols1 \
     libudev1 \
+    ncurses-base \
+    ncurses-bin \
+    locales \
     tzdata
 
 # cleanup
diff --git a/images/ubuntu-slim/Makefile b/images/ubuntu-slim/Makefile
index bf093cec1..e65c44022 100755
--- a/images/ubuntu-slim/Makefile
+++ b/images/ubuntu-slim/Makefile
@@ -1,6 +1,6 @@
 all: push
 
-TAG ?= 0.6
+TAG ?= 0.7
 PREFIX ?= gcr.io/google_containers/ubuntu-slim
 BUILD_IMAGE ?= ubuntu-build
 TAR_FILE ?= rootfs.tar
diff --git a/images/ubuntu-slim/README.md b/images/ubuntu-slim/README.md
index b905af43b..c41e70f72 100644
--- a/images/ubuntu-slim/README.md
+++ b/images/ubuntu-slim/README.md
@@ -1,9 +1,9 @@
 
 Small Ubuntu 16.04 docker image
 
-The size of this image is ~56MB (less than half than `ubuntu:16.04).
+The size of this image is ~44MB (less than half than `ubuntu:16.04).
 This is possible by the removal of packages that are not required in a container:
-- dmsetup
+- e2fslibs
 - e2fsprogs
 - init
 - initscripts
@@ -11,12 +11,16 @@ This is possible by the removal of packages that are not required in a container
 - libcryptsetup4
 - libdevmapper1.02.1
 - libkmod2
+- libmount1
+- libncursesw5
+- libprocps4
 - libsmartcols1
 - libudev1
 - mount
+- ncurses-base
+- ncurses-bin
 - procps
 - systemd
 - systemd-sysv
 - tzdata
-- udev
 - util-linux

From a7b6c70ad1525ab28bf336c462e45f2c13b55276 Mon Sep 17 00:00:00 2001
From: Manuel de Brito Fontes <aledbf@gmail.com>
Date: Thu, 16 Feb 2017 13:11:20 -0300
Subject: [PATCH 37/73] Update nginx-slim to 1.11.10

---
 images/nginx-slim/Dockerfile | 2 +-
 images/nginx-slim/Makefile   | 2 +-
 images/nginx-slim/build.sh   | 4 ++--
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/images/nginx-slim/Dockerfile b/images/nginx-slim/Dockerfile
index 089243ef0..5efd6b40b 100644
--- a/images/nginx-slim/Dockerfile
+++ b/images/nginx-slim/Dockerfile
@@ -13,7 +13,7 @@
 # limitations under the License.
 
 
-FROM gcr.io/google_containers/ubuntu-slim:0.6
+FROM gcr.io/google_containers/ubuntu-slim:0.7
 
 COPY build.sh /tmp
 
diff --git a/images/nginx-slim/Makefile b/images/nginx-slim/Makefile
index 2c783c60d..a40a3ea4e 100644
--- a/images/nginx-slim/Makefile
+++ b/images/nginx-slim/Makefile
@@ -1,7 +1,7 @@
 all: push
 
 # 0.0.0 shouldn't clobber any released builds
-TAG = 0.13
+TAG = 0.14
 PREFIX = gcr.io/google_containers/nginx-slim
 
 container:
diff --git a/images/nginx-slim/build.sh b/images/nginx-slim/build.sh
index f05ae633a..2236e1369 100755
--- a/images/nginx-slim/build.sh
+++ b/images/nginx-slim/build.sh
@@ -17,7 +17,7 @@
 
 set -e
 
-export NGINX_VERSION=1.11.9
+export NGINX_VERSION=1.11.10
 export NDK_VERSION=0.3.0
 export VTS_VERSION=0.1.11
 export SETMISC_VERSION=0.31
@@ -69,7 +69,7 @@ apt-get update && apt-get install --no-install-recommends -y \
   linux-headers-generic || exit 1
 
 # download, verify and extract the source files
-get_src dc22b71f16b551705930544dc042f1ad1af2f9715f565187ec22c7a4b2625748 \
+get_src 778b3cabb07633f754cd9dee32fc8e22582bce22bfa407be76a806abd935533d \
         "http://nginx.org/download/nginx-$NGINX_VERSION.tar.gz"
 
 get_src 88e05a99a8a7419066f5ae75966fb1efc409bad4522d14986da074554ae61619 \

From 0d05db2d66d2e0cc76fbc213ef8f2b9a94067bab Mon Sep 17 00:00:00 2001
From: Kraig Amador <Kraig.Amador@ticketmaster.com>
Date: Fri, 10 Feb 2017 21:07:28 -0800
Subject: [PATCH 38/73] Added a Node StoreLister type to support writing
 ingress controllers that use NodePorts

---
 core/pkg/ingress/controller/controller.go | 8 ++++++++
 core/pkg/ingress/types.go                 | 1 +
 2 files changed, 9 insertions(+)

diff --git a/core/pkg/ingress/controller/controller.go b/core/pkg/ingress/controller/controller.go
index 2c4d7c23d..80c700d35 100644
--- a/core/pkg/ingress/controller/controller.go
+++ b/core/pkg/ingress/controller/controller.go
@@ -76,11 +76,13 @@ type GenericController struct {
 	ingController  *cache.Controller
 	endpController *cache.Controller
 	svcController  *cache.Controller
+	nodeController *cache.Controller
 	secrController *cache.Controller
 	mapController  *cache.Controller
 
 	ingLister  cache_store.StoreToIngressLister
 	svcLister  cache.StoreToServiceLister
+	nodeLister cache.StoreToNodeLister
 	endpLister cache.StoreToEndpointsLister
 	secrLister cache_store.StoreToSecretsLister
 	mapLister  cache_store.StoreToConfigmapLister
@@ -292,6 +294,10 @@ func newIngressController(config *Configuration) *GenericController {
 		cache.ResourceEventHandlerFuncs{},
 		cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})
 
+	ic.nodeLister.Store, ic.nodeController = cache.NewInformer(
+		cache.NewListWatchFromClient(ic.cfg.Client.Core().RESTClient(), "nodes", ic.cfg.Namespace, fields.Everything()),
+		&api.Node{}, ic.cfg.ResyncPeriod, eventHandler)
+
 	if config.UpdateStatus {
 		ic.syncStatus = status.NewStatusSyncer(status.Config{
 			Client:         config.Client,
@@ -307,6 +313,7 @@ func newIngressController(config *Configuration) *GenericController {
 	ic.cfg.Backend.SetListers(ingress.StoreLister{
 		Ingress:   ic.ingLister,
 		Service:   ic.svcLister,
+		Node:      ic.nodeLister,
 		Endpoint:  ic.endpLister,
 		Secret:    ic.secrLister,
 		ConfigMap: ic.mapLister,
@@ -1039,6 +1046,7 @@ func (ic GenericController) Start() {
 	go ic.ingController.Run(ic.stopCh)
 	go ic.endpController.Run(ic.stopCh)
 	go ic.svcController.Run(ic.stopCh)
+	go ic.nodeController.Run(ic.stopCh)
 	go ic.secrController.Run(ic.stopCh)
 	go ic.mapController.Run(ic.stopCh)
 
diff --git a/core/pkg/ingress/types.go b/core/pkg/ingress/types.go
index 2ce5bc17d..92357084f 100644
--- a/core/pkg/ingress/types.go
+++ b/core/pkg/ingress/types.go
@@ -102,6 +102,7 @@ type Controller interface {
 type StoreLister struct {
 	Ingress   cache_store.StoreToIngressLister
 	Service   cache.StoreToServiceLister
+	Node      cache.StoreToNodeLister
 	Endpoint  cache.StoreToEndpointsLister
 	Secret    cache_store.StoreToSecretsLister
 	ConfigMap cache_store.StoreToConfigmapLister

From 8bebfbecafd4ef34d0e14f43e39f6a3bec480ac4 Mon Sep 17 00:00:00 2001
From: Manuel de Brito Fontes <aledbf@gmail.com>
Date: Thu, 16 Feb 2017 14:26:58 -0300
Subject: [PATCH 39/73] Add logs to help debugging and simplify default
 upstream configuration

---
 core/pkg/ingress/controller/controller.go | 65 ++++++++---------------
 1 file changed, 22 insertions(+), 43 deletions(-)

diff --git a/core/pkg/ingress/controller/controller.go b/core/pkg/ingress/controller/controller.go
index 2c4d7c23d..0ffe412da 100644
--- a/core/pkg/ingress/controller/controller.go
+++ b/core/pkg/ingress/controller/controller.go
@@ -420,6 +420,7 @@ func (ic *GenericController) sync(key interface{}) error {
 }
 
 func (ic *GenericController) getStreamServices(configmapName string, proto api.Protocol) []*ingress.Location {
+	glog.V(3).Infof("obtaining information about stream services of type %v located in configmap %v", proto, configmapName)
 	if configmapName == "" {
 		// no configmap configured
 		return []*ingress.Location{}
@@ -484,6 +485,7 @@ func (ic *GenericController) getStreamServices(configmapName string, proto api.P
 		var endps []ingress.Endpoint
 		targetPort, err := strconv.Atoi(svcPort)
 		if err != nil {
+			glog.V(3).Infof("searching service %v/%v endpoints using the name '%v'", svcNs, svcName, svcPort)
 			for _, sp := range svc.Spec.Ports {
 				if sp.Name == svcPort {
 					endps = ic.getEndpoints(svc, sp.TargetPort, proto, &healthcheck.Upstream{})
@@ -492,6 +494,7 @@ func (ic *GenericController) getStreamServices(configmapName string, proto api.P
 			}
 		} else {
 			// we need to use the TargetPort (where the endpoints are running)
+			glog.V(3).Infof("searching service %v/%v endpoints using the target port '%v'", svcNs, svcName, targetPort)
 			for _, sp := range svc.Spec.Ports {
 				if sp.Port == int32(targetPort) {
 					endps = ic.getEndpoints(svc, sp.TargetPort, proto, &healthcheck.Upstream{})
@@ -500,12 +503,10 @@ func (ic *GenericController) getStreamServices(configmapName string, proto api.P
 			}
 		}
 
-		sort.Sort(ingress.EndpointByAddrPort(endps))
-
-		// tcp upstreams cannot contain empty upstreams and there is no
-		// default backend equivalent for TCP
+		// stream services cannot contain empty upstreams and there is no
+		// default backend equivalent
 		if len(endps) == 0 {
-			glog.Warningf("service %v/%v does not have any active endpoints", svcNs, svcName)
+			glog.Warningf("service %v/%v does not have any active endpoints for port %v and protocol %v", svcNs, svcName, svcPort, proto)
 			continue
 		}
 
@@ -588,30 +589,9 @@ func (ic *GenericController) getBackendServers() ([]*ingress.Backend, []*ingress
 				server = servers[defServerName]
 			}
 
-			// use default upstream
-			defBackend := upstreams[defUpstreamName]
-			// we need to check if the spec contains the default backend
-			if ing.Spec.Backend != nil {
-				glog.V(3).Infof("ingress rule %v/%v defines a default Backend %v/%v",
-					ing.Namespace,
-					ing.Name,
-					ing.Spec.Backend.ServiceName,
-					ing.Spec.Backend.ServicePort.String())
-
-				name := fmt.Sprintf("%v-%v-%v",
-					ing.GetNamespace(),
-					ing.Spec.Backend.ServiceName,
-					ing.Spec.Backend.ServicePort.String())
-
-				if defUps, ok := upstreams[name]; ok {
-					defBackend = defUps
-				}
-			}
-
 			if rule.HTTP == nil &&
 				host != defServerName {
 				glog.V(3).Infof("ingress rule %v/%v does not contains HTTP rules. using default backend", ing.Namespace, ing.Name)
-				server.Locations[0].Backend = defBackend.Name
 				continue
 			}
 
@@ -806,7 +786,12 @@ func (ic *GenericController) serviceEndpoints(svcKey, backendPort string,
 	return upstreams, nil
 }
 
-func (ic *GenericController) createServers(data []interface{}, upstreams map[string]*ingress.Backend) map[string]*ingress.Server {
+// createServers initializes a map that contains information about the list of
+// FDQN referenced by ingress rules and the common name field in the referenced
+// SSL certificates. Each server is configured with location / using a default
+// backend specified by the user or the one inside the ingress spec.
+func (ic *GenericController) createServers(data []interface{},
+	upstreams map[string]*ingress.Backend) map[string]*ingress.Server {
 	servers := make(map[string]*ingress.Server)
 
 	bdef := ic.GetDefaultBackend()
@@ -818,8 +803,6 @@ func (ic *GenericController) createServers(data []interface{}, upstreams map[str
 		BufferSize:     bdef.ProxyBufferSize,
 	}
 
-	dun := ic.getDefaultUpstream().Name
-
 	// This adds the Default Certificate to Default Backend and also for vhosts missing the secret
 	var defaultPemFileName, defaultPemSHA string
 	defaultCertificate, err := ic.getPemCertificate(ic.cfg.DefaultSSLCertificate)
@@ -839,7 +822,7 @@ func (ic *GenericController) createServers(data []interface{}, upstreams map[str
 		defaultPemSHA = defaultCertificate.PemSHA
 	}
 
-	// default server
+	// initialize the default server
 	servers[defServerName] = &ingress.Server{
 		Hostname:       defServerName,
 		SSLCertificate: defaultPemFileName,
@@ -848,7 +831,7 @@ func (ic *GenericController) createServers(data []interface{}, upstreams map[str
 			{
 				Path:         rootLocation,
 				IsDefBackend: true,
-				Backend:      dun,
+				Backend:      ic.getDefaultUpstream().Name,
 				Proxy:        ngxProxy,
 			},
 		}}
@@ -862,6 +845,14 @@ func (ic *GenericController) createServers(data []interface{}, upstreams map[str
 
 		// check if ssl passthrough is configured
 		sslpt := ic.annotations.SSLPassthrough(ing)
+		dun := ic.getDefaultUpstream().Name
+		if ing.Spec.Backend != nil {
+			// replace default backend
+			defUpstream := fmt.Sprintf("%v-%v-%v", ing.GetNamespace(), ing.Spec.Backend.ServiceName, ing.Spec.Backend.ServicePort.String())
+			if backendUpstream, ok := upstreams[defUpstream]; ok {
+				dun = backendUpstream.Name
+			}
+		}
 
 		for _, rule := range ing.Spec.Rules {
 			host := rule.Host
@@ -920,22 +911,10 @@ func (ic *GenericController) createServers(data []interface{}, upstreams map[str
 						servers[host].SSLPemChecksum = cert.PemSHA
 					}
 				} else {
-
 					servers[host].SSLCertificate = defaultPemFileName
 					servers[host].SSLPemChecksum = defaultPemSHA
 				}
 			}
-
-			if ing.Spec.Backend != nil {
-				defUpstream := fmt.Sprintf("%v-%v-%v", ing.GetNamespace(), ing.Spec.Backend.ServiceName, ing.Spec.Backend.ServicePort.String())
-				if backendUpstream, ok := upstreams[defUpstream]; ok {
-					if host == "" || host == defServerName {
-						ic.recorder.Eventf(ing, api.EventTypeWarning, "MAPPING", "error: rules with Spec.Backend are allowed only with hostnames")
-						continue
-					}
-					servers[host].Locations[0].Backend = backendUpstream.Name
-				}
-			}
 		}
 	}
 

From 2d0971d6b0af44213736fa96d223ab4f4b9b06b2 Mon Sep 17 00:00:00 2001
From: Manuel de Brito Fontes <aledbf@gmail.com>
Date: Thu, 16 Feb 2017 15:10:14 -0300
Subject: [PATCH 40/73] Update nginx version in ingress controller to 1.11.10

---
 controllers/nginx/rootfs/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/controllers/nginx/rootfs/Dockerfile b/controllers/nginx/rootfs/Dockerfile
index 74ee4f493..6959d6bbc 100644
--- a/controllers/nginx/rootfs/Dockerfile
+++ b/controllers/nginx/rootfs/Dockerfile
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM gcr.io/google_containers/nginx-slim:0.13
+FROM gcr.io/google_containers/nginx-slim:0.14
 
 RUN DEBIAN_FRONTEND=noninteractive apt-get update && apt-get install -y \
   diffutils \

From 5f8a40d39239a6dc6dc0db87138e94e7d8aa63dd Mon Sep 17 00:00:00 2001
From: Tony Li <tony.li@nytimes.com>
Date: Thu, 16 Feb 2017 14:49:35 -0800
Subject: [PATCH 41/73] update makefile docker command

---
 controllers/gce/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/controllers/gce/Makefile b/controllers/gce/Makefile
index 10ca7ca6e..d908af0cf 100644
--- a/controllers/gce/Makefile
+++ b/controllers/gce/Makefile
@@ -11,7 +11,7 @@ container: server
 	docker build --pull -t $(PREFIX):$(TAG) .
 
 push: container
-	gcloud docker push $(PREFIX):$(TAG)
+	gcloud docker -- push $(PREFIX):$(TAG)
 
 clean:
 	rm -f glbc

From f32ef3248951ca383db6dfd5e3f4f41e9c4ac0d7 Mon Sep 17 00:00:00 2001
From: Tony Li <tony.li@nytimes.com>
Date: Thu, 16 Feb 2017 17:12:33 -0800
Subject: [PATCH 42/73] do the same for nginx

---
 controllers/nginx/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/controllers/nginx/Makefile b/controllers/nginx/Makefile
index 8878ecb4f..1f98d5492 100644
--- a/controllers/nginx/Makefile
+++ b/controllers/nginx/Makefile
@@ -24,7 +24,7 @@ container: build
 	docker build --pull -t $(PREFIX):$(RELEASE) rootfs
 
 push: container
-	gcloud docker push $(PREFIX):$(RELEASE)
+	gcloud docker -- push $(PREFIX):$(RELEASE)
 
 fmt:
 	@echo "+ $@"

From bba7213128883d5b6f4df0d442309173650fb48e Mon Sep 17 00:00:00 2001
From: Tony Li <tony.li@nytimes.com>
Date: Thu, 16 Feb 2017 17:13:36 -0800
Subject: [PATCH 43/73] do the same for other makefiles

---
 examples/custom-controller/Makefile | 2 +-
 images/nginx-slim/Makefile          | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/examples/custom-controller/Makefile b/examples/custom-controller/Makefile
index a66cf01f6..2e6783689 100644
--- a/examples/custom-controller/Makefile
+++ b/examples/custom-controller/Makefile
@@ -32,7 +32,7 @@ container: server
 	docker build --pull -t $(PREFIX)-$(ARCH):$(TAG) .
 
 push: container
-	gcloud docker push $(PREFIX)-$(ARCH):$(TAG)
+	gcloud docker -- push $(PREFIX)-$(ARCH):$(TAG)
 
 clean:
 	rm -f server
diff --git a/images/nginx-slim/Makefile b/images/nginx-slim/Makefile
index 2c783c60d..1f9cbd939 100644
--- a/images/nginx-slim/Makefile
+++ b/images/nginx-slim/Makefile
@@ -8,7 +8,7 @@ container:
 	docker build --pull -t $(PREFIX):$(TAG) .
 
 push: container
-	gcloud docker push $(PREFIX):$(TAG)
+	gcloud docker -- push $(PREFIX):$(TAG)
 
 clean:
 	docker rmi -f $(PREFIX):$(TAG) || true

From e26efd0b084bb741dccafb30c4a6f1e85e05913c Mon Sep 17 00:00:00 2001
From: Tang Le <at28997146@163.com>
Date: Fri, 17 Feb 2017 19:11:56 +0800
Subject: [PATCH 44/73] We need check content, when cmd failed.

Signed-off-by: Tang Le <at28997146@163.com>
---
 controllers/nginx/pkg/template/template.go | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/controllers/nginx/pkg/template/template.go b/controllers/nginx/pkg/template/template.go
index cc21dccac..280b4860c 100644
--- a/controllers/nginx/pkg/template/template.go
+++ b/controllers/nginx/pkg/template/template.go
@@ -111,7 +111,13 @@ func (t *Template) Write(conf config.TemplateConfig, isValidTemplate func([]byte
 	cmd.Stdout = t.outCmdBuf
 	if err := cmd.Run(); err != nil {
 		glog.Warningf("unexpected error cleaning template: %v", err)
-		return t.tmplBuf.Bytes(), nil
+		content := t.tmplBuf.Bytes()
+		err = isValidTemplate(content)
+		if err != nil {
+			return nil, err
+		}
+
+		return content, nil
 	}
 
 	content := t.outCmdBuf.Bytes()

From 77221b3555d50a1e442c5d24259cf662e3e4a34d Mon Sep 17 00:00:00 2001
From: Manuel de Brito Fontes <aledbf@gmail.com>
Date: Fri, 17 Feb 2017 10:38:52 -0300
Subject: [PATCH 45/73] Fix rewrite regex to match the start of the URL and not
 a substring

---
 controllers/nginx/pkg/template/template.go      | 5 ++++-
 controllers/nginx/pkg/template/template_test.go | 8 ++++----
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/controllers/nginx/pkg/template/template.go b/controllers/nginx/pkg/template/template.go
index cc21dccac..5afaf9a7f 100644
--- a/controllers/nginx/pkg/template/template.go
+++ b/controllers/nginx/pkg/template/template.go
@@ -237,7 +237,10 @@ func buildLocation(input interface{}) string {
 
 	path := location.Path
 	if len(location.Redirect.Target) > 0 && location.Redirect.Target != path {
-		return fmt.Sprintf("~* %s", path)
+		if path == "/" {
+			return fmt.Sprintf("~* %s", path)
+		}
+		return fmt.Sprintf("~* ^%s", path)
 	}
 
 	return path
diff --git a/controllers/nginx/pkg/template/template_test.go b/controllers/nginx/pkg/template/template_test.go
index fde02e3ef..244232306 100644
--- a/controllers/nginx/pkg/template/template_test.go
+++ b/controllers/nginx/pkg/template/template_test.go
@@ -45,12 +45,12 @@ var (
 	rewrite /(.*) /jenkins/$1 break;
 	proxy_pass http://upstream-name;
 	`, false},
-		"redirect /something to /": {"/something", "/", "~* /something", `
+		"redirect /something to /": {"/something", "/", "~* ^/something", `
 	rewrite /something/(.*) /$1 break;
 	rewrite /something / break;
 	proxy_pass http://upstream-name;
 	`, false},
-		"redirect /something-complex to /not-root": {"/something-complex", "/not-root", "~* /something-complex", `
+		"redirect /something-complex to /not-root": {"/something-complex", "/not-root", "~* ^/something-complex", `
 	rewrite /something-complex/(.*) /not-root/$1 break;
 	proxy_pass http://upstream-name;
 	`, false},
@@ -60,14 +60,14 @@ var (
 	subs_filter '<head(.*)>' '<head$1><base href="$scheme://$server_name/jenkins/">' r;
 	subs_filter '<HEAD(.*)>' '<HEAD$1><base href="$scheme://$server_name/jenkins/">' r;
 	`, true},
-		"redirect /something to / and rewrite": {"/something", "/", "~* /something", `
+		"redirect /something to / and rewrite": {"/something", "/", "~* ^/something", `
 	rewrite /something/(.*) /$1 break;
 	rewrite /something / break;
 	proxy_pass http://upstream-name;
 	subs_filter '<head(.*)>' '<head$1><base href="$scheme://$server_name/">' r;
 	subs_filter '<HEAD(.*)>' '<HEAD$1><base href="$scheme://$server_name/">' r;
 	`, true},
-		"redirect /something-complex to /not-root and rewrite": {"/something-complex", "/not-root", "~* /something-complex", `
+		"redirect /something-complex to /not-root and rewrite": {"/something-complex", "/not-root", "~* ^/something-complex", `
 	rewrite /something-complex/(.*) /not-root/$1 break;
 	proxy_pass http://upstream-name;
 	subs_filter '<head(.*)>' '<head$1><base href="$scheme://$server_name/not-root/">' r;

From bb489633b9c2fad71a249afeb4d4b5d20a4a1c02 Mon Sep 17 00:00:00 2001
From: bprashanth <beeps@google.com>
Date: Fri, 17 Feb 2017 10:20:34 -0800
Subject: [PATCH 46/73] Add PUSH_TOOL to ubuntu Makefile

---
 images/ubuntu-slim/Makefile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/images/ubuntu-slim/Makefile b/images/ubuntu-slim/Makefile
index e65c44022..07fd5c88f 100755
--- a/images/ubuntu-slim/Makefile
+++ b/images/ubuntu-slim/Makefile
@@ -4,6 +4,7 @@ TAG ?= 0.7
 PREFIX ?= gcr.io/google_containers/ubuntu-slim
 BUILD_IMAGE ?= ubuntu-build
 TAR_FILE ?= rootfs.tar
+PUSH_TOOL ?= gcloud
 
 container: clean
 	docker build --pull -t $(BUILD_IMAGE) -f Dockerfile.build .
@@ -12,7 +13,7 @@ container: clean
 	docker build --pull -t $(PREFIX):$(TAG) .
 
 push: container
-	docker push $(PREFIX):$(TAG)
+	$(PUSH_TOOL) docker push $(PREFIX):$(TAG)
 
 clean:
 	docker rmi -f $(PREFIX):$(TAG) || true

From cd355dc3e02f7dee8dbb7182c1cec19261eab386 Mon Sep 17 00:00:00 2001
From: bprashanth <beeps@google.com>
Date: Fri, 17 Feb 2017 09:13:53 -0800
Subject: [PATCH 47/73] Add more GCE FAQ

---
 docs/faq/gce.md | 89 ++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 88 insertions(+), 1 deletion(-)

diff --git a/docs/faq/gce.md b/docs/faq/gce.md
index 7f29e9452..f42afd15d 100644
--- a/docs/faq/gce.md
+++ b/docs/faq/gce.md
@@ -23,6 +23,8 @@ Table of Contents
 * [How does Ingress work across 2 GCE clusters?](#how-does-ingress-work-across-2-gce-clusters)
 * [I shutdown a cluster without deleting all Ingresses, how do I manually cleanup?](#i-shutdown-a-cluster-without-deleting-all-ingresses-how-do-i-manually-cleanup)
 * [How do I disable the GCE Ingress controller?](#how-do-i-disable-the-gce-ingress-controller)
+* [What GCE resources are shared between Ingresses?](#what-gce-resources-are-shared-between-ingresses)
+* [How do I debug a controller spin loop?](#host-do-i-debug-a-controller-spinloop)
 
 
 ## How do I deploy an Ingress controller?
@@ -30,6 +32,9 @@ Table of Contents
 On GCP (either GCE or GKE), every Kubernetes cluster has an Ingress controller
 running on the master, no deployment necessary. You can deploy a second,
 different (i.e non-GCE) controller, like [this](README.md#how-do-i-deploy-an-ingress-controller).
+If you wish to deploy a GCE controller as a pod in your cluster, make sure to
+turn down the existing auto-deployed Ingress controller as shown in this
+[example](/examples/deployment/gce/).
 
 ## I created an Ingress and nothing happens, now what?
 
@@ -87,7 +92,24 @@ for how to request more.
 ## Why does the Ingress need a different instance group then the GKE cluster?
 
 The controller adds/removes Kubernets nodes that are `NotReady` from the lb
-instance group.
+instance group. We cannot simply rely on health checks to achieve this for
+a few reasons.
+
+First, older Kubernetes versions (<=1.3) did not mark
+endpoints on unreachable nodes as NotReady. Meaning if the Kubelet didn't
+heart beat for 10s, the node was marked NotReady, but there was no other signal
+at the Service level to stop routing requests to endpoints on that node. In
+later Kubernetes version this is handled a little better, if the Kubelet
+doesn't heart beat for 10s it's marked NotReady, if it stays in NotReady
+for 40s all endpoints are marked NotReady. So it is still advantageous
+to pull the node out of the GCE LB Instance Group in 10s, because we
+save 30s of bad requests.
+
+Second, continuing to send requests to NotReady nodes is not a great idea.
+The NotReady condition is an aggregate of various factors. For example,
+a NotReady node might still pass health checks but have the wrong
+nodePort to endpoint mappings. The health check will pass as long as *something*
+returns a HTTP 200.
 
 ## Why does the cloud console show 0/N healthy instances?
 
@@ -228,6 +250,17 @@ controller will inject the default-http-backend Service that runs in the
 `kube-system` namespace as the default backend for the GCE HTTP lb allocated
 for that Ingress resource.
 
+Some caveats concerning the default backend:
+
+* It is the only Backend Service that doesn't directly map to a user specified
+NodePort Service
+* It's created when the first Ingress is created, and deleted when the last
+Ingress is deleted, since we don't want to waste quota if the user is not going
+to need L7 loadbalancing through Ingress
+* It has a http health check pointing at `/healthz`, not the default `/`, because
+`/` serves a 404 by design
+
+
 ## How does Ingress work across 2 GCE clusters?
 
 See federation [documentation](http://kubernetes.io/docs/user-guide/federation/federated-ingress/).
@@ -259,4 +292,58 @@ $ gcloud container clusters create mycluster --network "default" --num-nodes 1 \
 --disk-size 50 --scopes storage-full
 ```
 
+## What GCE resources are shared between Ingresses?
+
+Every Ingress creates a pipeline of GCE cloud resources behind an IP. Some of
+these are shared between Ingresses out of necessity, while some are shared
+because there was no perceived need for duplication (all resources consume
+quota and usually cost money).
+
+Shared:
+
+* Backend Services: because of low quota and high reuse. A single Service in a
+Kubernetes cluster has one NodePort, common throughout the cluster. GCE has
+a hard limit of the number of allowed BackendServices, so if multiple Ingresses
+all point to a single Service, that creates a single BackendService in GCE
+pointing to that Service's NodePort.
+
+* Instance Group: since an instance can only be part of a single loadbalanced
+Instance Group, these must be shared. There is 1 Ingress Instance Group per
+zone containing Kubernetes nodes.
+
+* HTTP Health Checks: currently the http health checks point at  the NodePort
+of a BackendService. They don't *need* to be shared, but they are since
+BackendServices are shared.
+
+* Firewall rule: In a non-federated cluster there is a single firewall rule
+that covers HTTP health check traffic from the range of [GCE loadbalancer IPs](https://cloud.google.com/compute/docs/load-balancing/http/#troubleshooting)
+to Service nodePorts.
+
+Unique:
+
+Currently, a single Ingress on GCE creates a unique IP and url map. In this
+model the following resources cannot be shared:
+* Url Map
+* Target HTTP(S) Proxies
+* SSL Certificates
+* Static-ip
+* Forwarding rules
+
+
+## How do I debug a controller spinloop?
+
+The most likely cause of a controller spin loop is some form of GCE validation
+failure, eg:
+* It's trying to delete a BackendService already in use, say in a UrlMap
+* It's trying to add an Instance to more than 1 loadbalanced InstanceGroups
+* It's trying to flip the loadbalancing algorithm on a BackendService to RATE,
+when some other BackendService is pointing at the same InstanceGroup and asking
+for UTILIZATION
+
+In all such cases, the work queue will put a single key (ingress namespace/name)
+that's getting continuously requeued into exponential backoff. However, currently
+the Informers that watch the Kubernetes api are setup to periodically resync,
+so even though a particular key is in backoff, we might end up syncing all other
+keys every, say, 10m, which might trigger the same validation-error-condition
+when syncing a shared resource.
 

From 7e02e9adaa9afc568ce4b30c2267cc32c9a8b6d7 Mon Sep 17 00:00:00 2001
From: bprashanth <beeps@google.com>
Date: Fri, 17 Feb 2017 10:45:21 -0800
Subject: [PATCH 48/73] Add more assignees and approvers

---
 OWNERS                 | 1 +
 controllers/gce/OWNERS | 6 ++++++
 2 files changed, 7 insertions(+)
 create mode 100644 controllers/gce/OWNERS

diff --git a/OWNERS b/OWNERS
index 163563012..a394ff383 100644
--- a/OWNERS
+++ b/OWNERS
@@ -3,3 +3,4 @@ assignees:
 - justinsb
 - bprashanth
 - thockin
+- nicksardo
diff --git a/controllers/gce/OWNERS b/controllers/gce/OWNERS
new file mode 100644
index 000000000..5b12a649d
--- /dev/null
+++ b/controllers/gce/OWNERS
@@ -0,0 +1,6 @@
+approvers:
+- nicksardo
+- thockin
+- freehan
+- csbell
+- bprashanth

From 8fd12b26ba4cd5c9360d2ec466a601a9d1f8e317 Mon Sep 17 00:00:00 2001
From: Manuel de Brito Fontes <aledbf@gmail.com>
Date: Fri, 17 Feb 2017 18:21:46 -0300
Subject: [PATCH 49/73] Change nginx variable to use in filter of access_log

---
 controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl b/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl
index 7853bc3c8..2f7c62082 100644
--- a/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl
+++ b/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl
@@ -81,7 +81,7 @@ http {
 
     {{/* map urls that should not appear in access.log */}}
     {{/* http://nginx.org/en/docs/http/ngx_http_log_module.html#access_log */}}
-    map $request $loggable {
+    map $request_uri $loggable {
         {{ range $reqUri := $cfg.SkipAccessLogURLs }}
         {{ $reqUri }} 0;{{ end }}
         default 1;

From 801d256e1e4f47dc05609b4541bb30db4a8b1b03 Mon Sep 17 00:00:00 2001
From: bprashanth <beeps@google.com>
Date: Fri, 17 Feb 2017 15:28:00 -0800
Subject: [PATCH 50/73] Add a GCE health checks example

---
 examples/health-checks/gce.md                 |  3 -
 examples/health-checks/gce/README.md          | 74 +++++++++++++++++
 .../health-checks/gce/health_check_app.yaml   | 82 +++++++++++++++++++
 3 files changed, 156 insertions(+), 3 deletions(-)
 delete mode 100644 examples/health-checks/gce.md
 create mode 100644 examples/health-checks/gce/README.md
 create mode 100644 examples/health-checks/gce/health_check_app.yaml

diff --git a/examples/health-checks/gce.md b/examples/health-checks/gce.md
deleted file mode 100644
index 9d3831a6f..000000000
--- a/examples/health-checks/gce.md
+++ /dev/null
@@ -1,3 +0,0 @@
-# Health checks for the GCE controller
-
-Placeholder
diff --git a/examples/health-checks/gce/README.md b/examples/health-checks/gce/README.md
new file mode 100644
index 000000000..25d2049dc
--- /dev/null
+++ b/examples/health-checks/gce/README.md
@@ -0,0 +1,74 @@
+# Simple HTTP health check example
+
+The GCE Ingress controller adopts the readiness probe from the matching endpoints, provided the readiness probe doesn't require HTTPS or special headers.
+
+Create the following app:
+```console
+$ kubectl create -f health_check_app.yaml
+replicationcontroller "echoheaders" created
+You have exposed your service on an external port on all nodes in your
+cluster.  If you want to expose this service to the external internet, you may
+need to set up firewall rules for the service port(s) (tcp:31165) to serve traffic.
+
+See http://releases.k8s.io/HEAD/docs/user-guide/services-firewalls.md for more details.
+service "echoheadersx" created
+You have exposed your service on an external port on all nodes in your
+cluster.  If you want to expose this service to the external internet, you may
+need to set up firewall rules for the service port(s) (tcp:31020) to serve traffic.
+
+See http://releases.k8s.io/HEAD/docs/user-guide/services-firewalls.md for more details.
+service "echoheadersy" created
+ingress "echomap" created
+```
+
+You should soon find an Ingress that is backed by a GCE Loadbalancer.
+
+```console
+$ kubectl describe ing echomap
+Name:			echomap
+Namespace:		default
+Address:		107.178.255.228
+Default backend:	default-http-backend:80 (10.180.0.9:8080,10.240.0.2:8080)
+Rules:
+  Host		Path	Backends
+  ----		----	--------
+  foo.bar.com
+    		/foo 	echoheadersx:80 (<none>)
+  bar.baz.com
+    		/bar 	echoheadersy:80 (<none>)
+    		/foo 	echoheadersx:80 (<none>)
+Annotations:
+  target-proxy:		k8s-tp-default-echomap--a9d60e8176d933ee
+  url-map:		k8s-um-default-echomap--a9d60e8176d933ee
+  backends:		{"k8s-be-31020--a9d60e8176d933ee":"HEALTHY","k8s-be-31165--a9d60e8176d933ee":"HEALTHY","k8s-be-31686--a9d60e8176d933ee":"HEALTHY"}
+  forwarding-rule:	k8s-fw-default-echomap--a9d60e8176d933ee
+Events:
+  FirstSeen	LastSeen	Count	From				SubobjectPath	Type		Reason	Message
+  ---------	--------	-----	----				-------------	--------	------	-------
+  17m		17m		1	{loadbalancer-controller }			Normal		ADD	default/echomap
+  15m		15m		1	{loadbalancer-controller }			Normal		CREATE	ip: 107.178.255.228
+
+$ curl 107.178.255.228/foo -H 'Host:foo.bar.com'
+CLIENT VALUES:
+client_address=10.240.0.5
+command=GET
+real path=/foo
+query=nil
+request_version=1.1
+request_uri=http://foo.bar.com:8080/foo
+...
+```
+
+You can confirm the health check endpoint point it's using one of 2 ways:
+* Through the cloud console: compute > health checks > lookup your health check. It takes the form k8s-be-nodePort-hash, where nodePort in the example above is 31165 and 31020, as shown by the kubectl output.
+* Through gcloud: Run `gcloud compute http-health-checks list`
+
+## Limitations
+
+A few points to note:
+* The readiness probe must be exposed on the port matching the `servicePort` specified in the Ingress
+* The readiness probe cannot have special requirements, like headers or HTTPS
+* The probe timeouts are translated to GCE health check timeouts
+* You must create the pods backing the endpoints with the given readiness probe. This *will not* work if you update the replication controller with a different readiness probe.
+
+
diff --git a/examples/health-checks/gce/health_check_app.yaml b/examples/health-checks/gce/health_check_app.yaml
new file mode 100644
index 000000000..b6f79c641
--- /dev/null
+++ b/examples/health-checks/gce/health_check_app.yaml
@@ -0,0 +1,82 @@
+apiVersion: v1
+kind: ReplicationController
+metadata:
+  name: echoheaders
+spec:
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: echoheaders
+    spec:
+      containers:
+      - name: echoheaders
+        image: gcr.io/google_containers/echoserver:1.4
+        ports:
+        - containerPort: 8080
+        readinessProbe:
+          httpGet:
+            path: /healthz
+            port: 8080
+          periodSeconds: 1
+          timeoutSeconds: 1
+          successThreshold: 1
+          failureThreshold: 10
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: echoheadersx
+  labels:
+    app: echoheaders
+spec:
+  type: NodePort
+  ports:
+  - port: 80
+    targetPort: 8080
+    protocol: TCP
+    name: http
+  selector:
+    app: echoheaders
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: echoheadersy
+  labels:
+    app: echoheaders
+spec:
+  type: NodePort
+  ports:
+  - port: 80
+    targetPort: 8080
+    protocol: TCP
+    name: http
+  selector:
+    app: echoheaders
+---
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+  name: echomap
+spec:
+  rules:
+  - host: foo.bar.com
+    http:
+      paths:
+      - path: /foo
+        backend:
+          serviceName: echoheadersx
+          servicePort: 80
+  - host: bar.baz.com
+    http:
+      paths:
+      - path: /bar
+        backend:
+          serviceName: echoheadersy
+          servicePort: 80
+      - path: /foo
+        backend:
+          serviceName: echoheadersx
+          servicePort: 80
+

From e68abf067b483022843278904f6f38eb98ccba88 Mon Sep 17 00:00:00 2001
From: caiyixiang <Cai.yixiang@zte.com.cn>
Date: Mon, 20 Feb 2017 10:30:37 +0800
Subject: [PATCH 51/73] change 'buildSSPassthrouthUpstreams' to
 'buildSSLPassthroughUpstreams'

---
 controllers/nginx/pkg/template/template.go             | 4 ++--
 controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/controllers/nginx/pkg/template/template.go b/controllers/nginx/pkg/template/template.go
index 5afaf9a7f..a2bbddfe8 100644
--- a/controllers/nginx/pkg/template/template.go
+++ b/controllers/nginx/pkg/template/template.go
@@ -137,7 +137,7 @@ var (
 		"buildProxyPass":              buildProxyPass,
 		"buildRateLimitZones":         buildRateLimitZones,
 		"buildRateLimit":              buildRateLimit,
-		"buildSSPassthroughUpstreams": buildSSPassthroughUpstreams,
+		"buildSSLPassthroughUpstreams": buildSSLPassthroughUpstreams,
 		"buildResolvers":              buildResolvers,
 		"isLocationAllowed":           isLocationAllowed,
 		"buildStreamUpstreams":        buildStreamUpstreams,
@@ -171,7 +171,7 @@ func buildResolvers(a interface{}) string {
 	return strings.Join(r, " ")
 }
 
-func buildSSPassthroughUpstreams(b interface{}, sslb interface{}) string {
+func buildSSLPassthroughUpstreams(b interface{}, sslb interface{}) string {
 	backends := b.([]*ingress.Backend)
 	sslBackends := sslb.([]*ingress.SSLPassthroughBackend)
 	buf := bytes.NewBuffer(make([]byte, 0, 10))
diff --git a/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl b/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl
index 2f7c62082..101038dc3 100644
--- a/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl
+++ b/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl
@@ -446,7 +446,7 @@ stream {
         server 127.0.0.1:442;
     }
 
-    {{ buildSSPassthroughUpstreams $backends .PassthroughBackends }}
+    {{ buildSSLPassthroughUpstreams $backends .PassthroughBackends }}
 
     server {
         listen                  [::]:443 ipv6only=off{{ if $cfg.UseProxyProtocol }} proxy_protocol{{ end }};

From c0f0cb2ff7918dd0b10de7ac0c904c19d8cd6494 Mon Sep 17 00:00:00 2001
From: Tang Le <at28997146@163.com>
Date: Mon, 20 Feb 2017 10:34:05 +0800
Subject: [PATCH 52/73] Check content when cmd failed

Signed-off-by: Tang Le <at28997146@163.com>
---
 controllers/nginx/pkg/cmd/controller/nginx.go  | 13 +++++++++++--
 controllers/nginx/pkg/template/template.go     | 18 +++---------------
 .../nginx/pkg/template/template_test.go        |  4 ++--
 3 files changed, 16 insertions(+), 19 deletions(-)

diff --git a/controllers/nginx/pkg/cmd/controller/nginx.go b/controllers/nginx/pkg/cmd/controller/nginx.go
index c6e8475ff..458f9f6c3 100644
--- a/controllers/nginx/pkg/cmd/controller/nginx.go
+++ b/controllers/nginx/pkg/cmd/controller/nginx.go
@@ -349,7 +349,7 @@ func (n *NGINXController) OnUpdate(ingressCfg ingress.Configuration) ([]byte, er
 		}
 	}
 
-	return n.t.Write(config.TemplateConfig{
+	content, err := n.t.Write(config.TemplateConfig{
 		ProxySetHeaders:     setHeaders,
 		MaxOpenFiles:        maxOpenFiles,
 		BacklogSize:         sysctlSomaxconn(),
@@ -361,7 +361,16 @@ func (n *NGINXController) OnUpdate(ingressCfg ingress.Configuration) ([]byte, er
 		HealthzURI:          ngxHealthPath,
 		CustomErrors:        len(cfg.CustomHTTPErrors) > 0,
 		Cfg:                 cfg,
-	}, n.testTemplate)
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	if err := n.testTemplate(content); err != nil {
+		return nil, err
+	}
+
+	return content, nil
 }
 
 // Name returns the healthcheck name
diff --git a/controllers/nginx/pkg/template/template.go b/controllers/nginx/pkg/template/template.go
index 280b4860c..a083814f5 100644
--- a/controllers/nginx/pkg/template/template.go
+++ b/controllers/nginx/pkg/template/template.go
@@ -78,7 +78,7 @@ func (t *Template) Close() {
 
 // Write populates a buffer using a template with NGINX configuration
 // and the servers and upstreams created by Ingress rules
-func (t *Template) Write(conf config.TemplateConfig, isValidTemplate func([]byte) error) ([]byte, error) {
+func (t *Template) Write(conf config.TemplateConfig) ([]byte, error) {
 	defer t.tmplBuf.Reset()
 	defer t.outCmdBuf.Reset()
 
@@ -111,22 +111,10 @@ func (t *Template) Write(conf config.TemplateConfig, isValidTemplate func([]byte
 	cmd.Stdout = t.outCmdBuf
 	if err := cmd.Run(); err != nil {
 		glog.Warningf("unexpected error cleaning template: %v", err)
-		content := t.tmplBuf.Bytes()
-		err = isValidTemplate(content)
-		if err != nil {
-			return nil, err
-		}
-
-		return content, nil
+		return t.tmplBuf.Bytes(), nil
 	}
 
-	content := t.outCmdBuf.Bytes()
-	err = isValidTemplate(content)
-	if err != nil {
-		return nil, err
-	}
-
-	return content, nil
+	return t.outCmdBuf.Bytes(), nil
 }
 
 var (
diff --git a/controllers/nginx/pkg/template/template_test.go b/controllers/nginx/pkg/template/template_test.go
index fde02e3ef..bd026f746 100644
--- a/controllers/nginx/pkg/template/template_test.go
+++ b/controllers/nginx/pkg/template/template_test.go
@@ -132,7 +132,7 @@ func TestTemplateWithData(t *testing.T) {
 		t.Errorf("invalid NGINX template: %v", err)
 	}
 
-	_, err = ngxTpl.Write(dat, func(b []byte) error { return nil })
+	_, err = ngxTpl.Write(dat)
 	if err != nil {
 		t.Errorf("invalid NGINX template: %v", err)
 	}
@@ -166,6 +166,6 @@ func BenchmarkTemplateWithData(b *testing.B) {
 	}
 
 	for i := 0; i < b.N; i++ {
-		ngxTpl.Write(dat, func(b []byte) error { return nil })
+		ngxTpl.Write(dat)
 	}
 }

From 488d89db184596e1eb0565fd475cb9d991e278a9 Mon Sep 17 00:00:00 2001
From: caiyixiang <Cai.yixiang@zte.com.cn>
Date: Mon, 20 Feb 2017 15:58:16 +0800
Subject: [PATCH 53/73] modify nginx readme

---
 controllers/nginx/README.md | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/controllers/nginx/README.md b/controllers/nginx/README.md
index f55cfd32d..1d46a1fd6 100644
--- a/controllers/nginx/README.md
+++ b/controllers/nginx/README.md
@@ -4,8 +4,8 @@ This is an nginx Ingress controller that uses [ConfigMap](https://github.com/kub
 
 ## Contents
 * [Conventions](#conventions)
-* [Requirements](#what-it-provides)
-* [Dry running](#dry-running-the-ingress-controller)
+* [Requirements](#requirements)
+* [Dry running](#try-running-the-ingress-controller)
 * [Deployment](#deployment)
 * [HTTP](#http)
 * [HTTPS](#https)
@@ -17,15 +17,16 @@ This is an nginx Ingress controller that uses [ConfigMap](https://github.com/kub
 * [UDP Services](#exposing-udp-services)
 * [Proxy Protocol](#proxy-protocol)
 * [NGINX customization](configuration.md)
+* [Custom errors](#custom-errors)
 * [NGINX status page](#nginx-status-page)
 * [Running multiple ingress controllers](#running-multiple-ingress-controllers)
 * [Running on Cloudproviders](#running-on-cloudproviders)
 * [Disabling NGINX ingress controller](#disabling-nginx-ingress-controller)
 * [Log format](#log-format)
 * [Local cluster](#local-cluster)
-* [Debug & Troubleshooting](#troubleshooting)
-* [Why endpoints and not services?](#why-endpoints-and-not-services)
+* [Debug & Troubleshooting](#debug--troubleshooting)
 * [Limitations](#limitations)
+* [Why endpoints and not services?](#why-endpoints-and-not-services)
 * [NGINX Notes](#nginx-notes)
 
 ## Conventions

From a74fe3426a53e2e675d2a297eed14ce6316e8913 Mon Sep 17 00:00:00 2001
From: chentao1596 <chen.tao3@zte.com.cn>
Date: Fri, 10 Feb 2017 17:11:12 +0800
Subject: [PATCH 54/73] Add docs for body-size annotation

---
 controllers/nginx/configuration.md | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/controllers/nginx/configuration.md b/controllers/nginx/configuration.md
index 69f27882a..85dbcb0d2 100644
--- a/controllers/nginx/configuration.md
+++ b/controllers/nginx/configuration.md
@@ -15,6 +15,7 @@
 * [Websockets](#websockets)
 * [Optimizing TLS Time To First Byte (TTTFB)](#optimizing-tls-time-to-first-byte-tttfb)
 * [Retries in non-idempotent methods](#retries-in-non-idempotent-methods)
+* [Custom max body size](#custom-max-body-size)
 
 
 ### Customizing NGINX
@@ -46,6 +47,7 @@ The following annotations are supported:
 |[ingress.kubernetes.io/enable-cors](#enable-cors)|true or false|
 |[ingress.kubernetes.io/limit-connections](#rate-limiting)|number|
 |[ingress.kubernetes.io/limit-rps](#rate-limiting)|number|
+|[ingress.kubernetes.io/proxy-body-size](#custom-max-body-size)|string|
 |[ingress.kubernetes.io/rewrite-target](#rewrite)|URI|
 |[ingress.kubernetes.io/secure-backends](#secure-backends)|true or false|
 |[ingress.kubernetes.io/ssl-redirect](#server-side-https-enforcement-through-redirect)|true or false|
@@ -414,3 +416,15 @@ NGINX provides the configuration option [ssl_buffer_size](http://nginx.org/en/do
 
 Since 1.9.13 NGINX will not retry non-idempotent requests (POST, LOCK, PATCH) in case of an error.
 The previous behavior can be restored using `retry-non-idempotent=true` in the configuration ConfigMap.
+
+
+### Custom max body size
+For NGINX, 413 error will be returned to the client when the size in a request exceeds the maximum allowed size of the client request body. This size can be configured by the parameter [`client_max_body_size`](http://nginx.org/en/docs/http/ngx_http_core_module.html#client_max_body_size).
+
+To configure this setting globally for all Ingress rules, the `proxy-body-size` value may be set in the NGINX ConfigMap.
+
+To use custom values in an Ingress rule define these annotation:
+
+```
+ingress.kubernetes.io/proxy-body-size: 8m
+```

From 38cc353f4ef3f468b0c363b227f9817993deddcf Mon Sep 17 00:00:00 2001
From: Manuel de Brito Fontes <aledbf@gmail.com>
Date: Tue, 21 Feb 2017 15:27:17 -0300
Subject: [PATCH 55/73] Add missing method in custom-controller

---
 examples/custom-controller/server.go | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/examples/custom-controller/server.go b/examples/custom-controller/server.go
index 6f1dcb840..5c6eea4b7 100644
--- a/examples/custom-controller/server.go
+++ b/examples/custom-controller/server.go
@@ -6,6 +6,8 @@ import (
 	"os/exec"
 	"strings"
 
+	"github.com/spf13/pflag"
+
 	nginxconfig "k8s.io/ingress/controllers/nginx/pkg/config"
 	"k8s.io/ingress/core/pkg/ingress"
 	"k8s.io/ingress/core/pkg/ingress/controller"
@@ -80,3 +82,10 @@ func (dc DummyController) Info() *ingress.BackendInfo {
 		Repository: "git://foo.bar.com",
 	}
 }
+
+func (n DummyController) OverrideFlags(*pflag.FlagSet) {
+}
+
+func (n DummyController) SetListers(lister ingress.StoreLister) {
+
+}

From 7d57a0dee2844156ad9299941141bc1b00db4ec2 Mon Sep 17 00:00:00 2001
From: chentao1596 <chen.tao3@zte.com.cn>
Date: Wed, 22 Feb 2017 10:12:22 +0800
Subject: [PATCH 56/73] update the description of 'Test HTTP Service'

---
 examples/PREREQUISITES.md | 50 +++++++++++++++++++--------------------
 1 file changed, 25 insertions(+), 25 deletions(-)

diff --git a/examples/PREREQUISITES.md b/examples/PREREQUISITES.md
index 1def0910d..a2d78c6b6 100644
--- a/examples/PREREQUISITES.md
+++ b/examples/PREREQUISITES.md
@@ -27,8 +27,8 @@ key/cert pair with an arbitrarily chosen hostname, created as follows
 ```console
 $ openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout tls.key -out tls.crt -subj "/CN=nginxsvc/O=nginxsvc"
 Generating a 2048 bit RSA private key
-......................................................................................................................................+++
-....................................................................+++
+................+++
+................+++
 writing new private key to 'tls.key'
 -----
 
@@ -38,7 +38,7 @@ secret "tls-secret" created
 
 ## Test HTTP Service
 
-All examples that require a test HTTP Service use the standard echoheaders pod,
+All examples that require a test HTTP Service use the standard http-svc pod,
 which you can deploy as follows
 
 ```console
@@ -47,35 +47,35 @@ service "http-svc" created
 replicationcontroller "http-svc" created
 
 $ kubectl get po
-NAME                READY     STATUS    RESTARTS   AGE
-echoheaders-p1t3t   1/1       Running   0          1d
+NAME             READY     STATUS    RESTARTS   AGE
+http-svc-p1t3t   1/1       Running   0          1d
 
 $ kubectl get svc
-NAME          CLUSTER-IP     EXTERNAL-IP   PORT(S)                      AGE
-echoheaders   10.0.122.116   <none>        80/TCP                       1d
+NAME             CLUSTER-IP     EXTERNAL-IP   PORT(S)            AGE
+http-svc         10.0.122.116   <pending>     80:30301/TCP       1d
 ```
 
 You can test that the HTTP Service works by exposing it temporarily
 ```console
-$ kubectl patch svc echoheaders -p '{"spec":{"type": "LoadBalancer"}}'
-"echoheaders" patched
+$ kubectl patch svc http-svc -p '{"spec":{"type": "LoadBalancer"}}'
+"http-svc" patched
 
-$ kubectl get svc echoheaders
-NAME          CLUSTER-IP     EXTERNAL-IP   PORT(S)        AGE
-echoheaders   10.0.122.116   <pending>     80:32100/TCP   1d
+$ kubectl get svc http-svc
+NAME             CLUSTER-IP     EXTERNAL-IP   PORT(S)            AGE
+http-svc         10.0.122.116   <pending>     80:30301/TCP       1d
 
-$ kubectl describe svc echoheaders
-Name:			echoheaders
-Namespace:		default
-Labels:			app=echoheaders
-Selector:		app=echoheaders
-Type:			LoadBalancer
-IP:			10.0.122.116
+$ kubectl describe svc http-svc
+Name:				    http-svc
+Namespace:			    default
+Labels:			        app=http-svc
+Selector:		        app=http-svc
+Type:			        LoadBalancer
+IP:			            10.0.122.116
 LoadBalancer Ingress:	108.59.87.136
-Port:			http	80/TCP
-NodePort:		http	32100/TCP
-Endpoints:		10.180.1.6:8080
-Session Affinity:	None
+Port:			        http	80/TCP
+NodePort:		        http	30301/TCP
+Endpoints:		        10.180.1.6:8080
+Session Affinity:	    None
 Events:
   FirstSeen	LastSeen	Count	From			SubObjectPath	Type		Reason			Message
   ---------	--------	-----	----			-------------	--------	------			-------
@@ -102,8 +102,8 @@ user-agent=curl/7.46.0
 BODY:
 -no body in request-
 
-$ kubectl patch svc echoheaders -p '{"spec":{"type": "NodePort"}}'
-"echoheaders" patched
+$ kubectl patch svc http-svc -p '{"spec":{"type": "NodePort"}}'
+"http-svc" patched
 ```
 
 ## Ingress Class

From 036892fb9643634f59d103b96d0076eac577f454 Mon Sep 17 00:00:00 2001
From: Manuel de Brito Fontes <aledbf@gmail.com>
Date: Sun, 19 Feb 2017 14:11:58 -0300
Subject: [PATCH 57/73] Release 0.9.0-beta.2

---
 controllers/nginx/Changelog.md                | 46 +++++++++++++++++++
 .../nginx/nginx-ingress-controller.yaml       |  2 +-
 .../kubeadm/nginx-ingress-controller.yaml     |  2 +-
 .../nginx/nginx-ingress-controller.yaml       |  2 +-
 .../nginx/nginx-ingress-controller.yaml       |  2 +-
 5 files changed, 50 insertions(+), 4 deletions(-)

diff --git a/controllers/nginx/Changelog.md b/controllers/nginx/Changelog.md
index ef43ba96f..761cda127 100644
--- a/controllers/nginx/Changelog.md
+++ b/controllers/nginx/Changelog.md
@@ -1,5 +1,51 @@
 Changelog
 
+### 0.9-beta.2
+
+**Image:**  `gcr.io/google_containers/nginx-ingress-controller:0.9.0-beta.2`
+
+*New Features:*
+
+- New configuration flag `proxy-set-headers` to allow set custom headers before send traffic to backends. [Example here](https://github.com/kubernetes/ingress/tree/master/examples/customization/custom-headers/nginx)
+- Disable directive access_log globally using `disable-access-log: "true"` in the configuration ConfigMap.
+- Sticky session per Ingress rule using the annotation `ingress.kubernetes.io/affinity`. [Example here](https://github.com/kubernetes/ingress/tree/master/examples/affinity/cookie/nginx)
+
+*Changes:*
+
+- [X] [#300](https://github.com/kubernetes/ingress/pull/300) Change nginx variable to use in filter of access_log
+- [X] [#296](https://github.com/kubernetes/ingress/pull/296) Fix rewrite regex to match the start of the URL and not a substring
+- [X] [#293](https://github.com/kubernetes/ingress/pull/293) Update makefile gcloud docker command
+- [X] [#290](https://github.com/kubernetes/ingress/pull/290) Update nginx version in ingress controller to 1.11.10
+- [X] [#286](https://github.com/kubernetes/ingress/pull/286) Add logs to help debugging and simplify default upstream configuration
+- [X] [#285](https://github.com/kubernetes/ingress/pull/285) Added a Node StoreLister type
+- [X] [#281](https://github.com/kubernetes/ingress/pull/281) Add chmod up directory tree for world read/execute on directories
+- [X] [#279](https://github.com/kubernetes/ingress/pull/279) fix wrong link in the file of examples/README.md
+- [X] [#275](https://github.com/kubernetes/ingress/pull/275) Pass headers to custom error backend
+- [X] [#272](https://github.com/kubernetes/ingress/pull/272) Fix error getting class information from Ingress annotations
+- [X] [#268](https://github.com/kubernetes/ingress/pull/268) minor: Fix typo in nginx README
+- [X] [#265](https://github.com/kubernetes/ingress/pull/265) Fix rewrite annotation parser
+- [X] [#262](https://github.com/kubernetes/ingress/pull/262) Add nginx README and configuration docs back
+- [X] [#261](https://github.com/kubernetes/ingress/pull/261) types.go: fix typo in godoc
+- [X] [#258](https://github.com/kubernetes/ingress/pull/258) Nginx sticky annotations
+- [X] [#255](https://github.com/kubernetes/ingress/pull/255) Adds support for disabling access_log globally
+- [X] [#247](https://github.com/kubernetes/ingress/pull/247) Fix wrong URL in nginx ingress configuration
+- [X] [#246](https://github.com/kubernetes/ingress/pull/246) Add support for custom proxy headers using a ConfigMap
+- [X] [#244](https://github.com/kubernetes/ingress/pull/244) Add information about cors annotation
+- [X] [#241](https://github.com/kubernetes/ingress/pull/241) correct a spell mistake
+- [X] [#232](https://github.com/kubernetes/ingress/pull/232) Change searchs with searches
+- [X] [#231](https://github.com/kubernetes/ingress/pull/231) Add information about proxy_protocol in port 442
+- [X] [#228](https://github.com/kubernetes/ingress/pull/228) Fix worker check issue
+- [X] [#227](https://github.com/kubernetes/ingress/pull/227) proxy_protocol on ssl_passthrough listener
+- [X] [#223](https://github.com/kubernetes/ingress/pull/223) Fix panic if a tempfile cannot be created
+- [X] [#220](https://github.com/kubernetes/ingress/pull/220) Fixes for minikube usage instructions.
+- [X] [#219](https://github.com/kubernetes/ingress/pull/219) Fix typo, add a couple of links. 
+- [X] [#218](https://github.com/kubernetes/ingress/pull/218) Improve links from CONTRIBUTING.
+- [X] [#217](https://github.com/kubernetes/ingress/pull/217) Fix an e2e link. 
+- [X] [#212](https://github.com/kubernetes/ingress/pull/212) Simplify code to obtain TCP or UDP services
+- [X] [#208](https://github.com/kubernetes/ingress/pull/208) Fix nil HTTP field
+- [X] [#198](https://github.com/kubernetes/ingress/pull/198) Add an example for static-ip and deployment
+
+
 ### 0.9-beta.1
 
 **Image:**  `gcr.io/google_containers/nginx-ingress-controller:0.9.0-beta.1`
diff --git a/examples/customization/custom-headers/nginx/nginx-ingress-controller.yaml b/examples/customization/custom-headers/nginx/nginx-ingress-controller.yaml
index 0d3824cb2..c4065804a 100644
--- a/examples/customization/custom-headers/nginx/nginx-ingress-controller.yaml
+++ b/examples/customization/custom-headers/nginx/nginx-ingress-controller.yaml
@@ -19,7 +19,7 @@ spec:
       # hostNetwork: true
       terminationGracePeriodSeconds: 60
       containers:
-      - image: gcr.io/google_containers/nginx-ingress-controller:0.9.0-beta.1
+      - image: gcr.io/google_containers/nginx-ingress-controller:0.9.0-beta.2
         name: nginx-ingress-controller
         readinessProbe:
           httpGet:
diff --git a/examples/deployment/nginx/kubeadm/nginx-ingress-controller.yaml b/examples/deployment/nginx/kubeadm/nginx-ingress-controller.yaml
index d25c7731d..f2ca1072e 100644
--- a/examples/deployment/nginx/kubeadm/nginx-ingress-controller.yaml
+++ b/examples/deployment/nginx/kubeadm/nginx-ingress-controller.yaml
@@ -71,7 +71,7 @@ spec:
       hostNetwork: true
       terminationGracePeriodSeconds: 60
       containers:
-      - image: gcr.io/google_containers/nginx-ingress-controller:0.9.0-beta.1
+      - image: gcr.io/google_containers/nginx-ingress-controller:0.9.0-beta.2
         name: nginx-ingress-controller
         readinessProbe:
           httpGet:
diff --git a/examples/deployment/nginx/nginx-ingress-controller.yaml b/examples/deployment/nginx/nginx-ingress-controller.yaml
index 7ee5e797e..b610d58f8 100644
--- a/examples/deployment/nginx/nginx-ingress-controller.yaml
+++ b/examples/deployment/nginx/nginx-ingress-controller.yaml
@@ -19,7 +19,7 @@ spec:
       # hostNetwork: true
       terminationGracePeriodSeconds: 60
       containers:
-      - image: gcr.io/google_containers/nginx-ingress-controller:0.9.0-beta.1
+      - image: gcr.io/google_containers/nginx-ingress-controller:0.9.0-beta.2
         name: nginx-ingress-controller
         readinessProbe:
           httpGet:
diff --git a/examples/static-ip/nginx/nginx-ingress-controller.yaml b/examples/static-ip/nginx/nginx-ingress-controller.yaml
index a7a78971e..d6eb1d512 100644
--- a/examples/static-ip/nginx/nginx-ingress-controller.yaml
+++ b/examples/static-ip/nginx/nginx-ingress-controller.yaml
@@ -18,7 +18,7 @@ spec:
       # hostNetwork: true
       terminationGracePeriodSeconds: 60
       containers:
-      - image: gcr.io/google_containers/nginx-ingress-controller:0.9.0-beta.1
+      - image: gcr.io/google_containers/nginx-ingress-controller:0.9.0-beta.2
         name: nginx-ingress-controller
         readinessProbe:
           httpGet:

From 7013a52ee5fd77cfc52f89f629d3dc10e3749c71 Mon Sep 17 00:00:00 2001
From: Manuel de Brito Fontes <aledbf@gmail.com>
Date: Tue, 21 Feb 2017 11:04:08 -0300
Subject: [PATCH 58/73] Return sorted endpoints

---
 controllers/nginx/Makefile                | 2 +-
 core/pkg/ingress/controller/controller.go | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/controllers/nginx/Makefile b/controllers/nginx/Makefile
index 1f98d5492..8bee35692 100644
--- a/controllers/nginx/Makefile
+++ b/controllers/nginx/Makefile
@@ -3,7 +3,7 @@ all: push
 BUILDTAGS=
 
 # Use the 0.0 tag for testing, it shouldn't clobber any release builds
-RELEASE?=0.9.0-beta.1
+RELEASE?=0.9.0-beta.2
 PREFIX?=gcr.io/google_containers/nginx-ingress-controller
 GOOS?=linux
 
diff --git a/core/pkg/ingress/controller/controller.go b/core/pkg/ingress/controller/controller.go
index 09c24a9e8..0523d3063 100644
--- a/core/pkg/ingress/controller/controller.go
+++ b/core/pkg/ingress/controller/controller.go
@@ -658,7 +658,6 @@ func (ic *GenericController) getBackendServers() ([]*ingress.Backend, []*ingress
 			glog.V(3).Infof("upstream %v does not have any active endpoints. Using default backend", value.Name)
 			value.Endpoints = append(value.Endpoints, newDefaultServer())
 		}
-		sort.Sort(ingress.EndpointByAddrPort(value.Endpoints))
 		aUpstreams = append(aUpstreams, value)
 	}
 	sort.Sort(ingress.BackendByNameServers(aUpstreams))
@@ -794,6 +793,7 @@ func (ic *GenericController) serviceEndpoints(svcKey, backendPort string,
 				glog.Warningf("service %v does not have any active endpoints", svcKey)
 			}
 
+			sort.Sort(ingress.EndpointByAddrPort(endps))
 			upstreams = append(upstreams, endps...)
 			break
 		}

From e5f6a31a9118465c423821f558beefa7c4d8bac2 Mon Sep 17 00:00:00 2001
From: chentao1596 <chen.tao3@zte.com.cn>
Date: Thu, 23 Feb 2017 09:17:03 +0800
Subject: [PATCH 59/73] add nginx daemonst example

---
 examples/daemonset/nginx/README.md            | 40 ++++++++++++++++
 .../nginx/nginx-ingress-daemonset.yaml        | 47 +++++++++++++++++++
 2 files changed, 87 insertions(+)
 create mode 100644 examples/daemonset/nginx/README.md
 create mode 100644 examples/daemonset/nginx/nginx-ingress-daemonset.yaml

diff --git a/examples/daemonset/nginx/README.md b/examples/daemonset/nginx/README.md
new file mode 100644
index 000000000..04ee2a443
--- /dev/null
+++ b/examples/daemonset/nginx/README.md
@@ -0,0 +1,40 @@
+# Nginx Ingress DaemonSet
+
+In some cases, the Ingress controller will be required to be run at all the nodes in cluster. Using [DaemonSet](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/daemon.md) can achieve this requirement.
+
+## Default Backend
+
+The default backend is a service of handling all url paths and hosts the nginx controller doesn't understand. Deploy the default-http-backend as follow:
+
+```console
+$ kubectl apply -f ../../deployment/nginx/default-backend.yaml 
+deployment "default-http-backend" configured
+service "default-http-backend" configured
+
+$ kubectl -n kube-system get svc
+NAME                   CLUSTER-IP    EXTERNAL-IP   PORT(S)   AGE
+default-http-backend   192.168.3.6   <none>        80/TCP    1h
+
+$ kubectl -n kube-system get po
+NAME                                    READY     STATUS    RESTARTS   AGE
+default-http-backend-2657704409-6b47n   1/1       Running   0          1h
+```
+
+## Ingress DaemonSet
+
+Deploy the daemonset as follows:
+
+```console
+$ kubectl apply -f nginx-ingress-daemonset.yaml
+daemonset "nginx-ingress-lb" created
+
+$ kubectl -n kube-system get ds
+NAME               DESIRED   CURRENT   READY     NODE-SELECTOR   AGE
+nginx-ingress-lb   2         2         2         <none>          21s
+
+$ kubectl -n kube-system get po
+NAME                                    READY     STATUS    RESTARTS   AGE
+default-http-backend-2657704409-6b47n   1/1       Running   0          2h
+nginx-ingress-lb-8381i                  1/1       Running   0          56s
+nginx-ingress-lb-h54gf                  1/1       Running   0          56s
+```
diff --git a/examples/daemonset/nginx/nginx-ingress-daemonset.yaml b/examples/daemonset/nginx/nginx-ingress-daemonset.yaml
new file mode 100644
index 000000000..1b476d670
--- /dev/null
+++ b/examples/daemonset/nginx/nginx-ingress-daemonset.yaml
@@ -0,0 +1,47 @@
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  name: nginx-ingress-lb
+  labels:
+    name: nginx-ingress-lb
+  namespace: kube-system
+spec:
+  template:
+    metadata:
+      labels:
+        name: nginx-ingress-lb
+    spec:
+      terminationGracePeriodSeconds: 60
+      containers:
+      - image: gcr.io/google_containers/nginx-ingress-controller:0.9.0-beta.2
+        name: nginx-ingress-lb
+        readinessProbe:
+          httpGet:
+            path: /healthz
+            port: 10254
+            scheme: HTTP
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 10254
+            scheme: HTTP
+          initialDelaySeconds: 10
+          timeoutSeconds: 1
+        ports:
+        - containerPort: 80
+          hostPort: 80
+        - containerPort: 443
+          hostPort: 443
+        env:
+          - name: POD_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.name
+          - name: POD_NAMESPACE
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.namespace
+        args:
+        - /nginx-ingress-controller
+        - --default-backend-service=$(POD_NAMESPACE)/default-http-backend
+

From 3d0e374f9eb2b4dc96d581f0c5fd99ccf2e60bab Mon Sep 17 00:00:00 2001
From: fate-grand-order <chenjg@harmonycloud.cn>
Date: Thu, 23 Feb 2017 21:59:09 +0800
Subject: [PATCH 60/73] fix misspell "affinity" in main.go

---
 core/pkg/ingress/annotations/sessionaffinity/main.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/pkg/ingress/annotations/sessionaffinity/main.go b/core/pkg/ingress/annotations/sessionaffinity/main.go
index 3cf5181e9..d506e844e 100644
--- a/core/pkg/ingress/annotations/sessionaffinity/main.go
+++ b/core/pkg/ingress/annotations/sessionaffinity/main.go
@@ -33,7 +33,7 @@ const (
 	annotationAffinityCookieName = "ingress.kubernetes.io/session-cookie-name"
 	defaultAffinityCookieName    = "INGRESSCOOKIE"
 	// This is the algorithm used by nginx to generate a value for the session cookie, if
-	// one isn't supplied and affintiy is set to "cookie".
+	// one isn't supplied and affinity is set to "cookie".
 	annotationAffinityCookieHash = "ingress.kubernetes.io/session-cookie-hash"
 	defaultAffinityCookieHash    = "md5"
 )

From 63573b39580f32fc1045e7000b8a74d9edef72df Mon Sep 17 00:00:00 2001
From: Jo Geraerts <jo@umask.net>
Date: Thu, 23 Feb 2017 16:52:10 +0100
Subject: [PATCH 61/73] Correct spelling mistake

---
 examples/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/README.md b/examples/README.md
index dbcf4a8cc..38c4372b7 100644
--- a/examples/README.md
+++ b/examples/README.md
@@ -75,7 +75,7 @@ Name | Description | Platform   | Complexity Level
 -----| ----------- | ---------- | ----------------
 Dummy  | A simple dummy controller that logs updates | * | Advanced
 
-## Custommization
+## Customization
 
 Name | Description | Platform   | Complexity Level
 -----| ----------- | ---------- | ----------------

From a20c287614e06ef5f0e6b8c8be4b15fd6ad76839 Mon Sep 17 00:00:00 2001
From: Manuel de Brito Fontes <aledbf@gmail.com>
Date: Thu, 23 Feb 2017 15:10:32 -0300
Subject: [PATCH 62/73] Add annotation to customize nginx location
 configuration

---
 .../rootfs/etc/nginx/template/nginx.tmpl      |  3 +
 core/pkg/ingress/annotations/snippet/main.go  | 42 ++++++++++++++
 .../ingress/annotations/snippet/main_test.go  | 57 +++++++++++++++++++
 core/pkg/ingress/controller/annotations.go    | 28 ++++-----
 core/pkg/ingress/types.go                     |  3 +
 examples/README.md                            |  1 +
 .../configuration-snippets/nginx/README.md    | 44 ++++++++++++++
 .../nginx/default-backend.yaml                | 51 +++++++++++++++++
 .../configuration-snippets/nginx/ingress.yaml | 18 ++++++
 .../nginx/nginx-ingress-controller.yaml       | 53 +++++++++++++++++
 .../nginx/nginx-load-balancer-conf.yaml       |  7 +++
 11 files changed, 294 insertions(+), 13 deletions(-)
 create mode 100644 core/pkg/ingress/annotations/snippet/main.go
 create mode 100644 core/pkg/ingress/annotations/snippet/main_test.go
 create mode 100644 examples/customization/configuration-snippets/nginx/README.md
 create mode 100644 examples/customization/configuration-snippets/nginx/default-backend.yaml
 create mode 100644 examples/customization/configuration-snippets/nginx/ingress.yaml
 create mode 100644 examples/customization/configuration-snippets/nginx/nginx-ingress-controller.yaml
 create mode 100644 examples/customization/configuration-snippets/nginx/nginx-load-balancer-conf.yaml

diff --git a/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl b/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl
index 101038dc3..6b8ec00eb 100644
--- a/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl
+++ b/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl
@@ -331,6 +331,9 @@ http {
             proxy_set_header                        Accept-Encoding     "";
             {{ end }}
 
+            {{/* Add any additional configuration defined */}}
+            {{ $location.ConfigurationSnippet }}
+
             {{ buildProxyPass $backends $location }}
             {{ else }}
             #{{ $location.Denied }}
diff --git a/core/pkg/ingress/annotations/snippet/main.go b/core/pkg/ingress/annotations/snippet/main.go
new file mode 100644
index 000000000..f05a665e3
--- /dev/null
+++ b/core/pkg/ingress/annotations/snippet/main.go
@@ -0,0 +1,42 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package snippet
+
+import (
+	"k8s.io/kubernetes/pkg/apis/extensions"
+
+	"k8s.io/ingress/core/pkg/ingress/annotations/parser"
+)
+
+const (
+	annotation = "ingress.kubernetes.io/configuration-snippet"
+)
+
+type snippet struct {
+}
+
+// NewParser creates a new CORS annotation parser
+func NewParser() parser.IngressAnnotation {
+	return snippet{}
+}
+
+// Parse parses the annotations contained in the ingress rule
+// used to indicate if the location/s contains a fragment of
+// configuration to be included inside the paths of the rules
+func (a snippet) Parse(ing *extensions.Ingress) (interface{}, error) {
+	return  parser.GetStringAnnotation(annotation, ing)
+}
diff --git a/core/pkg/ingress/annotations/snippet/main_test.go b/core/pkg/ingress/annotations/snippet/main_test.go
new file mode 100644
index 000000000..269996a95
--- /dev/null
+++ b/core/pkg/ingress/annotations/snippet/main_test.go
@@ -0,0 +1,57 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package snippet
+
+import (
+	"testing"
+
+	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/apis/extensions"
+)
+
+func TestParse(t *testing.T) {
+	ap := NewParser()
+	if ap == nil {
+		t.Fatalf("expected a parser.IngressAnnotation but returned nil")
+	}
+
+	testCases := []struct {
+		annotations map[string]string
+		expected    string
+	}{
+		{map[string]string{annotation: "more_headers"}, "more_headers"},
+		{map[string]string{annotation: "false"}, "false"},
+		{map[string]string{}, ""},
+		{nil, ""},
+	}
+
+	ing := &extensions.Ingress{
+		ObjectMeta: api.ObjectMeta{
+			Name:      "foo",
+			Namespace: api.NamespaceDefault,
+		},
+		Spec: extensions.IngressSpec{},
+	}
+
+	for _, testCase := range testCases {
+		ing.SetAnnotations(testCase.annotations)
+		result, _ := ap.Parse(ing)
+		if result != testCase.expected {
+			t.Errorf("expected %v but returned %v, annotations: %s", testCase.expected, result, testCase.annotations)
+		}
+	}
+}
diff --git a/core/pkg/ingress/controller/annotations.go b/core/pkg/ingress/controller/annotations.go
index 5a54e83cd..c0727dd87 100644
--- a/core/pkg/ingress/controller/annotations.go
+++ b/core/pkg/ingress/controller/annotations.go
@@ -34,6 +34,7 @@ import (
 	"k8s.io/ingress/core/pkg/ingress/annotations/rewrite"
 	"k8s.io/ingress/core/pkg/ingress/annotations/secureupstream"
 	"k8s.io/ingress/core/pkg/ingress/annotations/sessionaffinity"
+	"k8s.io/ingress/core/pkg/ingress/annotations/snippet"
 	"k8s.io/ingress/core/pkg/ingress/annotations/sslpassthrough"
 	"k8s.io/ingress/core/pkg/ingress/errors"
 	"k8s.io/ingress/core/pkg/ingress/resolver"
@@ -52,19 +53,20 @@ type annotationExtractor struct {
 func newAnnotationExtractor(cfg extractorConfig) annotationExtractor {
 	return annotationExtractor{
 		map[string]parser.IngressAnnotation{
-			"BasicDigestAuth":    auth.NewParser(auth.AuthDirectory, cfg),
-			"ExternalAuth":       authreq.NewParser(),
-			"CertificateAuth":    authtls.NewParser(cfg),
-			"EnableCORS":         cors.NewParser(),
-			"HealthCheck":        healthcheck.NewParser(cfg),
-			"Whitelist":          ipwhitelist.NewParser(cfg),
-			"UsePortInRedirects": portinredirect.NewParser(cfg),
-			"Proxy":              proxy.NewParser(cfg),
-			"RateLimit":          ratelimit.NewParser(),
-			"Redirect":           rewrite.NewParser(cfg),
-			"SecureUpstream":     secureupstream.NewParser(),
-			"SessionAffinity":    sessionaffinity.NewParser(),
-			"SSLPassthrough":     sslpassthrough.NewParser(),
+			"BasicDigestAuth":      auth.NewParser(auth.AuthDirectory, cfg),
+			"ExternalAuth":         authreq.NewParser(),
+			"CertificateAuth":      authtls.NewParser(cfg),
+			"EnableCORS":           cors.NewParser(),
+			"HealthCheck":          healthcheck.NewParser(cfg),
+			"Whitelist":            ipwhitelist.NewParser(cfg),
+			"UsePortInRedirects":   portinredirect.NewParser(cfg),
+			"Proxy":                proxy.NewParser(cfg),
+			"RateLimit":            ratelimit.NewParser(),
+			"Redirect":             rewrite.NewParser(cfg),
+			"SecureUpstream":       secureupstream.NewParser(),
+			"SessionAffinity":      sessionaffinity.NewParser(),
+			"SSLPassthrough":       sslpassthrough.NewParser(),
+			"ConfigurationSnippet": snippet.NewParser(),
 		},
 	}
 }
diff --git a/core/pkg/ingress/types.go b/core/pkg/ingress/types.go
index 488945574..ef7fc4d83 100644
--- a/core/pkg/ingress/types.go
+++ b/core/pkg/ingress/types.go
@@ -277,6 +277,9 @@ type Location struct {
 	// UsePortInRedirects indicates if redirects must specify the port
 	// +optional
 	UsePortInRedirects bool `json:"use-port-in-redirects"`
+	// ConfigurationSnippet contains additional configuration for the backend
+	// to be considered in the configuration of the location
+	ConfigurationSnippet string `json:"configuration-snippet"`
 }
 
 // SSLPassthroughBackend describes a SSL upstream server configured
diff --git a/examples/README.md b/examples/README.md
index dbcf4a8cc..236fb12d0 100644
--- a/examples/README.md
+++ b/examples/README.md
@@ -80,3 +80,4 @@ Dummy  | A simple dummy controller that logs updates | * | Advanced
 Name | Description | Platform   | Complexity Level
 -----| ----------- | ---------- | ----------------
 custom-headers  | set custom headers before send traffic to backends  | nginx | Advanced
+configuration-snippets | customize nginx location configuration using annotations | nginx | Advanced
diff --git a/examples/customization/configuration-snippets/nginx/README.md b/examples/customization/configuration-snippets/nginx/README.md
new file mode 100644
index 000000000..9adfa78a0
--- /dev/null
+++ b/examples/customization/configuration-snippets/nginx/README.md
@@ -0,0 +1,44 @@
+# Deploying the Nginx Ingress controller
+
+This example aims to demonstrate the deployment of an nginx ingress controller and
+with the use of an annotation in the Ingress rule be able to customize the nginx 
+configuration.
+
+## Default Backend
+
+The default backend is a Service capable of handling all url paths and hosts the
+nginx controller doesn't understand. This most basic implementation just returns
+a 404 page:
+
+```console
+$ kubectl apply -f default-backend.yaml
+deployment "default-http-backend" created
+service "default-http-backend" created
+
+$ kubectl -n kube-system get po
+NAME                                    READY     STATUS    RESTARTS   AGE
+default-http-backend-2657704409-qgwdd   1/1       Running   0          28s
+```
+
+```console
+$ kubectl create -f nginx-load-balancer-conf.yaml
+```
+
+## Controller
+
+You can deploy the controller as follows:
+
+```console
+$ kubectl apply -f nginx-ingress-controller.yaml
+deployment "nginx-ingress-controller" created
+
+$ kubectl -n kube-system get po
+NAME                                       READY     STATUS    RESTARTS   AGE
+default-http-backend-2657704409-qgwdd      1/1       Running   0          2m
+nginx-ingress-controller-873061567-4n3k2   1/1       Running   0          42s
+```
+
+## Test
+
+Check the contents of the annotation is present in the nginx.conf file using:
+`kubectl exec nginx-ingress-controller-873061567-4n3k2 -n kube-system cat /etc/nginx/nginx.conf`
diff --git a/examples/customization/configuration-snippets/nginx/default-backend.yaml b/examples/customization/configuration-snippets/nginx/default-backend.yaml
new file mode 100644
index 000000000..3c40989a3
--- /dev/null
+++ b/examples/customization/configuration-snippets/nginx/default-backend.yaml
@@ -0,0 +1,51 @@
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: default-http-backend
+  labels:
+    k8s-app: default-http-backend
+  namespace: kube-system
+spec:
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        k8s-app: default-http-backend
+    spec:
+      terminationGracePeriodSeconds: 60
+      containers:
+      - name: default-http-backend
+        # Any image is permissable as long as:
+        # 1. It serves a 404 page at /
+        # 2. It serves 200 on a /healthz endpoint
+        image: gcr.io/google_containers/defaultbackend:1.0
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 8080
+            scheme: HTTP
+          initialDelaySeconds: 30
+          timeoutSeconds: 5
+        ports:
+        - containerPort: 8080
+        resources:
+          limits:
+            cpu: 10m
+            memory: 20Mi
+          requests:
+            cpu: 10m
+            memory: 20Mi
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: default-http-backend
+  namespace: kube-system
+  labels:
+    k8s-app: default-http-backend
+spec:
+  ports:
+  - port: 80
+    targetPort: 8080
+  selector:
+    k8s-app: default-http-backend
diff --git a/examples/customization/configuration-snippets/nginx/ingress.yaml b/examples/customization/configuration-snippets/nginx/ingress.yaml
new file mode 100644
index 000000000..e60d75f90
--- /dev/null
+++ b/examples/customization/configuration-snippets/nginx/ingress.yaml
@@ -0,0 +1,18 @@
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+  name: nginx-configuration-snippet
+  annotations:
+    kubernetes.io/ingress.class: "nginx"
+    ingress.kubernetes.io/configuration-snippet: |
+      more_set_headers "Request-Id: $request_id";
+
+spec:
+  rules:
+  - host: custom.configuration.com
+    http:
+      paths:
+      - backend:
+          serviceName: http-svc
+          servicePort: 80
+        path: /
diff --git a/examples/customization/configuration-snippets/nginx/nginx-ingress-controller.yaml b/examples/customization/configuration-snippets/nginx/nginx-ingress-controller.yaml
new file mode 100644
index 000000000..c4065804a
--- /dev/null
+++ b/examples/customization/configuration-snippets/nginx/nginx-ingress-controller.yaml
@@ -0,0 +1,53 @@
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: nginx-ingress-controller
+  labels:
+    k8s-app: nginx-ingress-controller
+  namespace: kube-system
+spec:
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        k8s-app: nginx-ingress-controller
+    spec:
+      # hostNetwork makes it possible to use ipv6 and to preserve the source IP correctly regardless of docker configuration
+      # however, it is not a hard dependency of the nginx-ingress-controller itself and it may cause issues if port 10254 already is taken on the host
+      # that said, since hostPort is broken on CNI (https://github.com/kubernetes/kubernetes/issues/31307) we have to use hostNetwork where CNI is used
+      # like with kubeadm
+      # hostNetwork: true
+      terminationGracePeriodSeconds: 60
+      containers:
+      - image: gcr.io/google_containers/nginx-ingress-controller:0.9.0-beta.2
+        name: nginx-ingress-controller
+        readinessProbe:
+          httpGet:
+            path: /healthz
+            port: 10254
+            scheme: HTTP
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 10254
+            scheme: HTTP
+          initialDelaySeconds: 10
+          timeoutSeconds: 1
+        ports:
+        - containerPort: 80
+          hostPort: 80
+        - containerPort: 443
+          hostPort: 443
+        env:
+          - name: POD_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.name
+          - name: POD_NAMESPACE
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.namespace
+        args:
+        - /nginx-ingress-controller
+        - --default-backend-service=$(POD_NAMESPACE)/default-http-backend
+        - --configmap=$(POD_NAMESPACE)/nginx-load-balancer-conf
diff --git a/examples/customization/configuration-snippets/nginx/nginx-load-balancer-conf.yaml b/examples/customization/configuration-snippets/nginx/nginx-load-balancer-conf.yaml
new file mode 100644
index 000000000..239918267
--- /dev/null
+++ b/examples/customization/configuration-snippets/nginx/nginx-load-balancer-conf.yaml
@@ -0,0 +1,7 @@
+apiVersion: v1
+data:
+  proxy-set-headers: "kube-system/custom-headers"
+kind: ConfigMap
+metadata:
+  name: nginx-load-balancer-conf
+  namespace: kube-system

From 5159afad2ab672a4a764ce065bf1ffc5f2ca9838 Mon Sep 17 00:00:00 2001
From: Marcin Owsiany <marcin@owsiany.pl>
Date: Fri, 24 Feb 2017 11:56:31 +0100
Subject: [PATCH 63/73] Fix for vet complaints: #335.

---
 core/pkg/ingress/status/status_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/pkg/ingress/status/status_test.go b/core/pkg/ingress/status/status_test.go
index 5f41623c9..57e9f1e84 100644
--- a/core/pkg/ingress/status/status_test.go
+++ b/core/pkg/ingress/status/status_test.go
@@ -213,7 +213,7 @@ func buildIngressLIstener() cache_store.StoreToIngressLister {
 			},
 		},
 	})
-	return cache_store.StoreToIngressLister{store}
+	return cache_store.StoreToIngressLister{Store: store}
 }
 
 func buildStatusSync() statusSync {

From 192c551abb45ee92c3ec7da213f5a86291d2976c Mon Sep 17 00:00:00 2001
From: Marcin Owsiany <marcin@owsiany.pl>
Date: Fri, 24 Feb 2017 12:05:28 +0100
Subject: [PATCH 64/73] Fix for formatting error introduced in #304.

Why don't we fail the travis build when go fmt is unhappy?
---
 controllers/nginx/pkg/template/template.go | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/controllers/nginx/pkg/template/template.go b/controllers/nginx/pkg/template/template.go
index 4e35b3103..a4ec4b4e4 100644
--- a/controllers/nginx/pkg/template/template.go
+++ b/controllers/nginx/pkg/template/template.go
@@ -126,15 +126,15 @@ var (
 			}
 			return true
 		},
-		"buildLocation":               buildLocation,
-		"buildAuthLocation":           buildAuthLocation,
-		"buildProxyPass":              buildProxyPass,
-		"buildRateLimitZones":         buildRateLimitZones,
-		"buildRateLimit":              buildRateLimit,
+		"buildLocation":                buildLocation,
+		"buildAuthLocation":            buildAuthLocation,
+		"buildProxyPass":               buildProxyPass,
+		"buildRateLimitZones":          buildRateLimitZones,
+		"buildRateLimit":               buildRateLimit,
 		"buildSSLPassthroughUpstreams": buildSSLPassthroughUpstreams,
-		"buildResolvers":              buildResolvers,
-		"isLocationAllowed":           isLocationAllowed,
-		"buildStreamUpstreams":        buildStreamUpstreams,
+		"buildResolvers":               buildResolvers,
+		"isLocationAllowed":            isLocationAllowed,
+		"buildStreamUpstreams":         buildStreamUpstreams,
 
 		"contains":  strings.Contains,
 		"hasPrefix": strings.HasPrefix,

From 704a18cec9975d8ca7616df336bb4343afdabdf2 Mon Sep 17 00:00:00 2001
From: Giancarlo Rubio <gianrubio@gmail.com>
Date: Tue, 27 Dec 2016 10:52:04 +0100
Subject: [PATCH 65/73] Add support for proxy cookie path/proxy cookie domain

---
 controllers/nginx/configuration.md            | 10 +++++++
 controllers/nginx/pkg/config/config.go        |  2 ++
 .../rootfs/etc/nginx/template/nginx.tmpl      |  3 +++
 core/pkg/ingress/annotations/proxy/main.go    | 27 ++++++++++++++-----
 core/pkg/ingress/defaults/main.go             | 10 +++++++
 5 files changed, 46 insertions(+), 6 deletions(-)

diff --git a/controllers/nginx/configuration.md b/controllers/nginx/configuration.md
index 85dbcb0d2..f37ec71e7 100644
--- a/controllers/nginx/configuration.md
+++ b/controllers/nginx/configuration.md
@@ -259,6 +259,12 @@ http://nginx.org/en/docs/http/ngx_http_core_module.html#keepalive_timeout
 **proxy-connect-timeout:** Sets the timeout for [establishing a connection with a proxied server](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_connect_timeout). It should be noted that this timeout cannot usually exceed 75 seconds.
 
 
+**proxy-cookie-domain:** Sets a text that [should be changed in the domain attribute](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_cookie_domain) of the “Set-Cookie” header fields of a proxied server response.
+
+
+**proxy-cookie-path:** Sets a text that [should be changed in the path attribute](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_cookie_path) of the “Set-Cookie” header fields of a proxied server response.
+ 
+
 **proxy-read-timeout:** Sets the timeout in seconds for [reading a response from the proxied server](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_read_timeout). The timeout is set only between two successive read operations, not for the transmission of the whole response.
 
 
@@ -376,7 +382,11 @@ The following table shows the options, the default value and a description.
 |keep-alive|"75"|
 |map-hash-bucket-size|"64"|
 |max-worker-connections|"16384"|
+|proxy-body-size|same as body-size|
+|proxy-buffer-size|"4k"|
 |proxy-connect-timeout|"5"|
+|proxy-cookie-domain|"off"|
+|proxy-cookie-path|"off"|
 |proxy-read-timeout|"60"|
 |proxy-real-ip-cidr|0.0.0.0/0|
 |proxy-send-timeout|"60"|
diff --git a/controllers/nginx/pkg/config/config.go b/controllers/nginx/pkg/config/config.go
index 2e7fe7d73..c4ac3dfd6 100644
--- a/controllers/nginx/pkg/config/config.go
+++ b/controllers/nginx/pkg/config/config.go
@@ -274,6 +274,8 @@ func NewDefault() Configuration {
 			ProxyReadTimeout:     60,
 			ProxySendTimeout:     60,
 			ProxyBufferSize:      "4k",
+			ProxyCookieDomain:    "off",
+			ProxyCookiePath:      "off",
 			SSLRedirect:          true,
 			CustomHTTPErrors:     []int{},
 			WhitelistSourceRange: []string{},
diff --git a/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl b/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl
index 101038dc3..f7e36e070 100644
--- a/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl
+++ b/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl
@@ -326,6 +326,9 @@ http {
 
             proxy_http_version                      1.1;
 
+            proxy_cookie_domain                     {{ $location.Proxy.CookiePath }};
+            proxy_cookie_path                       {{ $location.Proxy.CookieDomain }};
+
             {{/* rewrite only works if the content is not compressed */}}
             {{ if $location.Redirect.AddBaseURL }}
             proxy_set_header                        Accept-Encoding     "";
diff --git a/core/pkg/ingress/annotations/proxy/main.go b/core/pkg/ingress/annotations/proxy/main.go
index c1ee7e47b..93f83eb63 100644
--- a/core/pkg/ingress/annotations/proxy/main.go
+++ b/core/pkg/ingress/annotations/proxy/main.go
@@ -24,11 +24,13 @@ import (
 )
 
 const (
-	bodySize   = "ingress.kubernetes.io/proxy-body-size"
-	connect    = "ingress.kubernetes.io/proxy-connect-timeout"
-	send       = "ingress.kubernetes.io/proxy-send-timeout"
-	read       = "ingress.kubernetes.io/proxy-read-timeout"
-	bufferSize = "ingress.kubernetes.io/proxy-buffer-size"
+	bodySize     = "ingress.kubernetes.io/proxy-body-size"
+	connect      = "ingress.kubernetes.io/proxy-connect-timeout"
+	send         = "ingress.kubernetes.io/proxy-send-timeout"
+	read         = "ingress.kubernetes.io/proxy-read-timeout"
+	bufferSize   = "ingress.kubernetes.io/proxy-buffer-size"
+	cookiePath   = "ingress.kubernetes.io/proxy-cookie-path"
+	cookieDomain = "ingress.kubernetes.io/proxy-cookie-domain"
 )
 
 // Configuration returns the proxy timeout to use in the upstream server/s
@@ -38,6 +40,8 @@ type Configuration struct {
 	SendTimeout    int    `json:"sendTimeout"`
 	ReadTimeout    int    `json:"readTimeout"`
 	BufferSize     string `json:"bufferSize"`
+	CookieDomain   string `json:"proxyCookieDomain"`
+	CookiePath     string `json:"proxyCookiePath"`
 }
 
 type proxy struct {
@@ -73,10 +77,21 @@ func (a proxy) Parse(ing *extensions.Ingress) (interface{}, error) {
 		bufs = defBackend.ProxyBufferSize
 	}
 
+	cp, err := parser.GetStringAnnotation(cookiePath, ing)
+	if err != nil || cp == "" {
+		cp = defBackend.ProxyCookiePath
+	}
+
+	cd, err := parser.GetStringAnnotation(cookieDomain, ing)
+	if err != nil || cp == "" {
+		cp = defBackend.ProxyCookieDomain
+	}
+
 	bs, err := parser.GetStringAnnotation(bodySize, ing)
 	if err != nil || bs == "" {
 		bs = defBackend.ProxyBodySize
 	}
 
-	return &Configuration{bs, ct, st, rt, bufs}, nil
+	return &Configuration{bs, ct, st, rt, bufs,
+		cd, cp}, nil
 }
diff --git a/core/pkg/ingress/defaults/main.go b/core/pkg/ingress/defaults/main.go
index ba56bc7c9..19b6c110b 100644
--- a/core/pkg/ingress/defaults/main.go
+++ b/core/pkg/ingress/defaults/main.go
@@ -37,6 +37,16 @@ type Backend struct {
 	// http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffer_size)
 	ProxyBufferSize string `json:"proxy-buffer-size"`
 
+	// Sets a text that should be changed in the path attribute of the “Set-Cookie” header fields of
+	// a proxied server response.
+	// http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_cookie_path
+	ProxyCookiePath string `json:"proxy-cookie-path"`
+
+	// Sets a text that should be changed in the domain attribute of the “Set-Cookie” header fields
+	// of a proxied server response.
+	// http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_cookie_domain
+	ProxyCookieDomain string `json:"proxy-cookie-domain"`
+
 	// Name server/s used to resolve names of upstream servers into IP addresses.
 	// The file /etc/resolv.conf is used as DNS resolution configuration.
 	Resolver []net.IP

From 2e302de2641cae09a29349a81f11c989de12b53f Mon Sep 17 00:00:00 2001
From: Marcin Owsiany <marcin@owsiany.pl>
Date: Fri, 24 Feb 2017 16:50:10 +0100
Subject: [PATCH 66/73] Do not run coverage check in the default target.

Reasons:
- this takes a lot longer than the other steps
- its results are harder to interpret at a single glance, compared to other
  steps
- it requires special privileges to succeed, and should probably not be ran
  from a random developer's checkout, since it pushes results to a shared
  service
---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 6f3e1f5e6..12e57452d 100644
--- a/Makefile
+++ b/Makefile
@@ -1,4 +1,4 @@
-all: fmt lint vet cover
+all: fmt lint vet
 
 BUILDTAGS=
 

From c6dd2db55033e30ac9fe5654f32b29d2fa69d33a Mon Sep 17 00:00:00 2001
From: Manuel de Brito Fontes <aledbf@gmail.com>
Date: Fri, 24 Feb 2017 14:18:10 -0300
Subject: [PATCH 67/73] Fix node lister when --watch-namespace is used

---
 core/pkg/ingress/controller/controller.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/pkg/ingress/controller/controller.go b/core/pkg/ingress/controller/controller.go
index 0523d3063..5a315dbfd 100644
--- a/core/pkg/ingress/controller/controller.go
+++ b/core/pkg/ingress/controller/controller.go
@@ -295,7 +295,7 @@ func newIngressController(config *Configuration) *GenericController {
 		cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})
 
 	ic.nodeLister.Store, ic.nodeController = cache.NewInformer(
-		cache.NewListWatchFromClient(ic.cfg.Client.Core().RESTClient(), "nodes", ic.cfg.Namespace, fields.Everything()),
+		cache.NewListWatchFromClient(ic.cfg.Client.Core().RESTClient(), "nodes", api.NamespaceAll, fields.Everything()),
 		&api.Node{}, ic.cfg.ResyncPeriod, eventHandler)
 
 	if config.UpdateStatus {

From 84324af14075b6bed206755149deacd9e9136cd9 Mon Sep 17 00:00:00 2001
From: Manuel de Brito Fontes <aledbf@gmail.com>
Date: Fri, 24 Feb 2017 18:46:39 -0300
Subject: [PATCH 68/73] Refactoring of TCP and UDP services

---
 controllers/nginx/pkg/config/config.go        |  4 +-
 controllers/nginx/pkg/template/template.go    | 29 -----------
 .../rootfs/etc/nginx/template/nginx.tmpl      | 42 +++++++++------
 controllers/nginx/test/data/config.json       | 52 +------------------
 core/pkg/ingress/annotations/proxy/main.go    | 11 ++--
 core/pkg/ingress/controller/controller.go     | 26 ++++++----
 core/pkg/ingress/types.go                     | 23 +++++++-
 7 files changed, 71 insertions(+), 116 deletions(-)

diff --git a/controllers/nginx/pkg/config/config.go b/controllers/nginx/pkg/config/config.go
index c4ac3dfd6..bfe4cc79e 100644
--- a/controllers/nginx/pkg/config/config.go
+++ b/controllers/nginx/pkg/config/config.go
@@ -299,8 +299,8 @@ type TemplateConfig struct {
 	Backends            []*ingress.Backend
 	PassthroughBackends []*ingress.SSLPassthroughBackend
 	Servers             []*ingress.Server
-	TCPBackends         []*ingress.Location
-	UDPBackends         []*ingress.Location
+	TCPBackends         []ingress.L4Service
+	UDPBackends         []ingress.L4Service
 	HealthzURI          string
 	CustomErrors        bool
 	Cfg                 Configuration
diff --git a/controllers/nginx/pkg/template/template.go b/controllers/nginx/pkg/template/template.go
index a4ec4b4e4..1945bc6cf 100644
--- a/controllers/nginx/pkg/template/template.go
+++ b/controllers/nginx/pkg/template/template.go
@@ -134,7 +134,6 @@ var (
 		"buildSSLPassthroughUpstreams": buildSSLPassthroughUpstreams,
 		"buildResolvers":               buildResolvers,
 		"isLocationAllowed":            isLocationAllowed,
-		"buildStreamUpstreams":         buildStreamUpstreams,
 
 		"contains":  strings.Contains,
 		"hasPrefix": strings.HasPrefix,
@@ -193,34 +192,6 @@ func buildSSLPassthroughUpstreams(b interface{}, sslb interface{}) string {
 	return buf.String()
 }
 
-func buildStreamUpstreams(proto string, b interface{}, s interface{}) string {
-	backends := b.([]*ingress.Backend)
-	streams := s.([]*ingress.Location)
-	buf := bytes.NewBuffer(make([]byte, 0, 10))
-	// multiple services can use the same upstream.
-	// avoid duplications using a map[name]=true
-	u := make(map[string]bool)
-	for _, stream := range streams {
-		if u[stream.Backend] {
-			continue
-		}
-		u[stream.Backend] = true
-		fmt.Fprintf(buf, "upstream %v-%v {\n", proto, stream.Backend)
-		// TODO: find a better way to avoid empty stream upstreams
-		fmt.Fprintf(buf, "\t\tserver 127.0.0.1:8181 down;\n")
-		for _, backend := range backends {
-			if backend.Name == stream.Backend {
-				for _, server := range backend.Endpoints {
-					fmt.Fprintf(buf, "\t\tserver %v:%v;\n", server.Address, server.Port)
-				}
-				break
-			}
-		}
-		fmt.Fprint(buf, "\t}\n\n")
-	}
-	return buf.String()
-}
-
 // buildLocation produces the location string, if the ingress has redirects
 // (specified through the ingress.kubernetes.io/rewrite-to annotation)
 func buildLocation(input interface{}) string {
diff --git a/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl b/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl
index 7de347406..11440b195 100644
--- a/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl
+++ b/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl
@@ -326,8 +326,8 @@ http {
 
             proxy_http_version                      1.1;
 
-            proxy_cookie_domain                     {{ $location.Proxy.CookiePath }};
-            proxy_cookie_path                       {{ $location.Proxy.CookieDomain }};
+            proxy_cookie_domain                     {{ $location.Proxy.CookieDomain }};
+            proxy_cookie_path                       {{ $location.Proxy.CookiePath }};
 
             {{/* rewrite only works if the content is not compressed */}}
             {{ if $location.Redirect.AddBaseURL }}
@@ -460,25 +460,33 @@ stream {
         ssl_preread             on;
     }
     
-    {{ buildStreamUpstreams "tcp" $backends .TCPBackends }}
-    
-    {{ buildStreamUpstreams "udp" $backends .UDPBackends }}
-
     # TCP services		
     {{ range $i, $tcpServer := .TCPBackends }}
-        server {
-            listen {{ $tcpServer.Path }};
-            proxy_pass             tcp-{{ $tcpServer.Backend }};		
-        }
+    upstream {{ $tcpServer.Backend.Namespace }}-{{ $tcpServer.Backend.Name }}-{{ $tcpServer.Backend.Port }} {
+    {{ range $j, $endpoint := $tcpServer.Endpoints }}
+        server                  {{ $endpoint.Address }}:{{ $endpoint.Port }};
+    {{ end }}
+    }
+    
+    server {
+        listen                  {{ $tcpServer.Port }};
+        proxy_pass              {{ $tcpServer.Backend.Namespace }}-{{ $tcpServer.Backend.Name }}-{{ $tcpServer.Backend.Port }};
+    }
     {{ end }}
 
-    # UDP services		
-    {{ range $i, $udpServer := .UDPBackends }}		
-        server {		
-            listen {{ $udpServer.Path }} udp;
-            proxy_responses        1;
-            proxy_pass             udp-{{ $udpServer.Backend }};
-        }		
+    # UDP services
+    {{ range $i, $udpServer := .UDPBackends }}
+    upstream {{ $udpServer.Backend.Namespace }}-{{ $udpServer.Backend.Name }}-{{ $udpServer.Backend.Port }} {
+    {{ range $j, $endpoint := $udpServer.Endpoints }}
+        server                  {{ $endpoint.Address }}:{{ $endpoint.Port }};
+    {{ end }}
+    }
+    
+    server {
+        listen                  {{ $udpServer.Port }};
+        proxy_responses         1;
+        proxy_pass              {{ $udpServer.Backend.Namespace }}-{{ $udpServer.Backend.Name }}-{{ $udpServer.Backend.Port }};		
+    }
     {{ end }}
 }
 
diff --git a/controllers/nginx/test/data/config.json b/controllers/nginx/test/data/config.json
index 4aaa2dd11..087a8fe83 100644
--- a/controllers/nginx/test/data/config.json
+++ b/controllers/nginx/test/data/config.json
@@ -57134,57 +57134,7 @@
 		}]
 	}],
 	"sslDHParam": "",
-	"tcpBackends": [{
-		"path": "2222",
-		"isDefBackend": false,
-		"backend": "default-echoheaders-2222",
-		"basicDigestAuth": {
-			"type": "",
-			"realm": "",
-			"file": "",
-			"secured": false
-		},
-		"externalAuth": {
-			"url": "",
-			"method": "",
-			"sendBody": false
-		},
-		"rateLimit": {
-			"connections": {
-				"name": "",
-				"limit": 0,
-				"burst": 0,
-				"sharedSize": 0
-			},
-			"rps": {
-				"name": "",
-				"limit": 0,
-				"burst": 0,
-				"sharedSize": 0
-			}
-		},
-		"redirect": {
-			"target": "",
-			"addBaseUrl": false,
-			"sslRedirect": false
-		},
-		"whitelist": {
-			"cidr": null
-		},
-		"proxy": {
-			"conectTimeout": 0,
-			"sendTimeout": 0,
-			"readTimeout": 0,
-			"bufferSize": ""
-		},
-		"certificateAuth": {
-			"secret": "",
-			"certFilename": "",
-			"keyFilename": "",
-			"caFilename": "",
-			"pemSha": ""
-		}
-	}],
+	"tcpBackends": [],
 	"udpBackends": [],
 	"backends": [{
 		"name": "default-echoheaders-80",
diff --git a/core/pkg/ingress/annotations/proxy/main.go b/core/pkg/ingress/annotations/proxy/main.go
index 93f83eb63..ab7d2e07a 100644
--- a/core/pkg/ingress/annotations/proxy/main.go
+++ b/core/pkg/ingress/annotations/proxy/main.go
@@ -40,8 +40,8 @@ type Configuration struct {
 	SendTimeout    int    `json:"sendTimeout"`
 	ReadTimeout    int    `json:"readTimeout"`
 	BufferSize     string `json:"bufferSize"`
-	CookieDomain   string `json:"proxyCookieDomain"`
-	CookiePath     string `json:"proxyCookiePath"`
+	CookieDomain   string `json:"cookieDomain"`
+	CookiePath     string `json:"cookiePath"`
 }
 
 type proxy struct {
@@ -83,8 +83,8 @@ func (a proxy) Parse(ing *extensions.Ingress) (interface{}, error) {
 	}
 
 	cd, err := parser.GetStringAnnotation(cookieDomain, ing)
-	if err != nil || cp == "" {
-		cp = defBackend.ProxyCookieDomain
+	if err != nil || cd == "" {
+		cd = defBackend.ProxyCookieDomain
 	}
 
 	bs, err := parser.GetStringAnnotation(bodySize, ing)
@@ -92,6 +92,5 @@ func (a proxy) Parse(ing *extensions.Ingress) (interface{}, error) {
 		bs = defBackend.ProxyBodySize
 	}
 
-	return &Configuration{bs, ct, st, rt, bufs,
-		cd, cp}, nil
+	return &Configuration{bs, ct, st, rt, bufs, cd, cp}, nil
 }
diff --git a/core/pkg/ingress/controller/controller.go b/core/pkg/ingress/controller/controller.go
index 5a315dbfd..58e647f9e 100644
--- a/core/pkg/ingress/controller/controller.go
+++ b/core/pkg/ingress/controller/controller.go
@@ -426,30 +426,30 @@ func (ic *GenericController) sync(key interface{}) error {
 	return nil
 }
 
-func (ic *GenericController) getStreamServices(configmapName string, proto api.Protocol) []*ingress.Location {
+func (ic *GenericController) getStreamServices(configmapName string, proto api.Protocol) []ingress.L4Service {
 	glog.V(3).Infof("obtaining information about stream services of type %v located in configmap %v", proto, configmapName)
 	if configmapName == "" {
 		// no configmap configured
-		return []*ingress.Location{}
+		return []ingress.L4Service{}
 	}
 
 	ns, name, err := k8s.ParseNameNS(configmapName)
 	if err != nil {
 		glog.Errorf("unexpected error reading configmap %v: %v", name, err)
-		return []*ingress.Location{}
+		return []ingress.L4Service{}
 	}
 
 	configmap, err := ic.getConfigMap(ns, name)
 	if err != nil {
 		glog.Errorf("unexpected error reading configmap %v: %v", name, err)
-		return []*ingress.Location{}
+		return []ingress.L4Service{}
 	}
 
-	var svcs []*ingress.Location
+	var svcs []ingress.L4Service
 	// k -> port to expose
 	// v -> <namespace>/<service name>:<port from service to be used>
 	for k, v := range configmap.Data {
-		_, err := strconv.Atoi(k)
+		externalPort, err := strconv.Atoi(k)
 		if err != nil {
 			glog.Warningf("%v is not valid as a TCP/UDP port", k)
 			continue
@@ -517,9 +517,15 @@ func (ic *GenericController) getStreamServices(configmapName string, proto api.P
 			continue
 		}
 
-		svcs = append(svcs, &ingress.Location{
-			Path:    k,
-			Backend: fmt.Sprintf("%v-%v-%v", svcNs, svcName, svcPort),
+		svcs = append(svcs, ingress.L4Service{
+			Port: externalPort,
+			Backend: ingress.L4Backend{
+				Name:      svcName,
+				Namespace: svcNs,
+				Port:      intstr.FromString(svcPort),
+				Protocol:  proto,
+			},
+			Endpoints: endps,
 		})
 	}
 
@@ -817,6 +823,8 @@ func (ic *GenericController) createServers(data []interface{},
 		SendTimeout:    bdef.ProxySendTimeout,
 		ReadTimeout:    bdef.ProxyReadTimeout,
 		BufferSize:     bdef.ProxyBufferSize,
+		CookieDomain:   bdef.ProxyCookieDomain,
+		CookiePath:     bdef.ProxyCookiePath,
 	}
 
 	// This adds the Default Certificate to Default Backend and also for vhosts missing the secret
diff --git a/core/pkg/ingress/types.go b/core/pkg/ingress/types.go
index ef7fc4d83..0d8a8b735 100644
--- a/core/pkg/ingress/types.go
+++ b/core/pkg/ingress/types.go
@@ -22,6 +22,7 @@ import (
 	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/client/cache"
 	"k8s.io/kubernetes/pkg/healthz"
+	"k8s.io/kubernetes/pkg/util/intstr"
 
 	cache_store "k8s.io/ingress/core/pkg/cache"
 	"k8s.io/ingress/core/pkg/ingress/annotations/auth"
@@ -132,10 +133,10 @@ type Configuration struct {
 	Servers []*Server `json:"servers"`
 	// TCPEndpoints contain endpoints for tcp streams handled by this backend
 	// +optional
-	TCPEndpoints []*Location `json:"tcpEndpoints,omitempty"`
+	TCPEndpoints []L4Service `json:"tcpEndpoints,omitempty"`
 	// UDPEndpoints contain endpoints for udp streams handled by this backend
 	// +optional
-	UDPEndpoints []*Location `json:"udpEndpoints,omitempty"`
+	UDPEndpoints []L4Service `json:"udpEndpoints,omitempty"`
 	// PassthroughBackend contains the backends used for SSL passthrough.
 	// It contains information about the associated Server Name Indication (SNI).
 	// +optional
@@ -292,3 +293,21 @@ type SSLPassthroughBackend struct {
 	// Hostname returns the FQDN of the server
 	Hostname string `json:"hostname"`
 }
+
+// L4Service describes a L4 Ingress service.
+type L4Service struct {
+	// Port external port to expose
+	Port int `json:"port"`
+	// Backend of the service
+	Backend L4Backend `json:"backend"`
+	// Endpoints active endpoints of the service
+	Endpoints []Endpoint `json:"endpoins"`
+}
+
+// L4Backend describes the kubernetes service behind L4 Ingress service
+type L4Backend struct {
+	Port      intstr.IntOrString `json:"port"`
+	Name      string             `json:"name"`
+	Namespace string             `json:"namespace"`
+	Protocol  api.Protocol       `json:"protocol"`
+}

From a44130f7f55eb82b95c84e2c6085c8259be61631 Mon Sep 17 00:00:00 2001
From: Manuel de Brito Fontes <aledbf@gmail.com>
Date: Fri, 24 Feb 2017 21:54:47 -0300
Subject: [PATCH 69/73] Fix lint error

---
 core/pkg/ingress/annotations/snippet/main.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/pkg/ingress/annotations/snippet/main.go b/core/pkg/ingress/annotations/snippet/main.go
index f05a665e3..d88cfbc32 100644
--- a/core/pkg/ingress/annotations/snippet/main.go
+++ b/core/pkg/ingress/annotations/snippet/main.go
@@ -38,5 +38,5 @@ func NewParser() parser.IngressAnnotation {
 // used to indicate if the location/s contains a fragment of
 // configuration to be included inside the paths of the rules
 func (a snippet) Parse(ing *extensions.Ingress) (interface{}, error) {
-	return  parser.GetStringAnnotation(annotation, ing)
+	return parser.GetStringAnnotation(annotation, ing)
 }

From a342c0bce378bd7089601f2eaa3c3d84ca0f0d27 Mon Sep 17 00:00:00 2001
From: Ricardo Pchevuzinske Katz <ricardo.katz@serpro.gov.br>
Date: Mon, 6 Feb 2017 16:16:36 -0200
Subject: [PATCH 70/73] Adds correct support for TLS Muthual autentication and
 depth verification

	modified:   controllers/nginx/configuration.md
	modified:   controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl
	modified:   core/pkg/ingress/annotations/authtls/main.go
	modified:   core/pkg/ingress/controller/backend_ssl.go
	modified:   core/pkg/ingress/controller/controller.go
	modified:   core/pkg/ingress/controller/util_test.go
	modified:   core/pkg/ingress/resolver/main.go
	modified:   core/pkg/ingress/types.go
	modified:   core/pkg/net/ssl/ssl.go
	modified:   examples/PREREQUISITES.md
	new file:   examples/auth/client-certs/nginx/README.md
	new file:   examples/auth/client-certs/nginx/nginx-tls-auth.yaml
---
 controllers/nginx/configuration.md            | 23 +++++
 .../rootfs/etc/nginx/template/nginx.tmpl      | 12 ++-
 core/pkg/ingress/annotations/authtls/main.go  | 46 ++++++---
 core/pkg/ingress/controller/backend_ssl.go    | 25 +++--
 core/pkg/ingress/controller/controller.go     | 15 ++-
 core/pkg/ingress/controller/util_test.go      |  4 +-
 core/pkg/ingress/resolver/main.go             |  6 --
 core/pkg/ingress/types.go                     |  4 +-
 core/pkg/net/ssl/ssl.go                       | 56 ++++++++---
 examples/PREREQUISITES.md                     | 99 +++++++++++++++++++
 examples/auth/client-certs/nginx/README.md    | 86 ++++++++++++++++
 .../client-certs/nginx/nginx-tls-auth.yaml    | 25 +++++
 12 files changed, 349 insertions(+), 52 deletions(-)
 create mode 100644 examples/auth/client-certs/nginx/README.md
 create mode 100644 examples/auth/client-certs/nginx/nginx-tls-auth.yaml

diff --git a/controllers/nginx/configuration.md b/controllers/nginx/configuration.md
index f37ec71e7..5885f587b 100644
--- a/controllers/nginx/configuration.md
+++ b/controllers/nginx/configuration.md
@@ -44,6 +44,8 @@ The following annotations are supported:
 |[ingress.kubernetes.io/auth-secret](#authentication)|string|
 |[ingress.kubernetes.io/auth-type](#authentication)|basic or digest|
 |[ingress.kubernetes.io/auth-url](#external-authentication)|string|
+|[ingress.kubernetes.io/auth-tls-secret](#Certificate Authentication)|string|
+|[ingress.kubernetes.io/auth-tls-verify-depth](#Certificate Authentication)|number|
 |[ingress.kubernetes.io/enable-cors](#enable-cors)|true or false|
 |[ingress.kubernetes.io/limit-connections](#rate-limiting)|number|
 |[ingress.kubernetes.io/limit-rps](#rate-limiting)|number|
@@ -126,6 +128,27 @@ ingress.kubernetes.io/auth-realm: "realm string"
 
 Please check the [auth](examples/auth/README.md) example.
 
+### Certificate Authentication
+
+It's possible to enable Certificate based authentication using additional annotations in Ingres Rule.
+
+The annotations are:
+
+```
+ingress.kubernetes.io/auth-tls-secret: secretName
+```
+
+The name of the secret that contains the full Certificate Authority chain that is enabled to authenticate against this ingress. It's composed of namespace/secretName
+
+```
+ingress.kubernetes.io/auth-tls-verify-depth
+```
+
+The validation depth between the provided client certificate and the Certification Authority chain.
+
+Please check the [tls-auth](examples/auth/client-certs/README.md) example.
+
+
 ### Enable CORS
 
 To enable Cross-Origin Resource Sharing (CORS) in an Ingress rule add the annotation `ingress.kubernetes.io/enable-cors: "true"`. This will add a section in the server location enabling this functionality.
diff --git a/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl b/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl
index 11440b195..07b6f5782 100644
--- a/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl
+++ b/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl
@@ -225,10 +225,11 @@ http {
         {{ $path := buildLocation $location }}
         {{ $authPath := buildAuthLocation $location }}
 
-        {{ if not (empty $location.CertificateAuth.CertFileName) }}
-        # PEM sha: {{ $location.CertificateAuth.PemSHA }}
-        ssl_client_certificate              {{ $location.CertificateAuth.CAFileName }};
+        {{ if not (empty $location.CertificateAuth.AuthSSLCert.CAFileName) }}
+        # PEM sha: {{ $location.CertificateAuth.AuthSSLCert.PemSHA }}
+        ssl_client_certificate              {{ $location.CertificateAuth.AuthSSLCert.CAFileName }};
         ssl_verify_client on;
+        ssl_verify_depth {{ $location.CertificateAuth.ValidationDepth }};
         {{ end }}
 
         {{ if not (empty $authPath) }}
@@ -295,6 +296,11 @@ http {
 
             proxy_set_header Host                   $host;
 
+            # Pass the extracted client certificate to the backend
+            {{ if not (empty $location.CertificateAuth.AuthSSLCert.CAFileName) }}
+            proxy_set_header ssl-client-cert        $ssl_client_cert;
+            {{ end }}
+
             # Pass Real IP
             proxy_set_header X-Real-IP              $remote_addr;
 
diff --git a/core/pkg/ingress/annotations/authtls/main.go b/core/pkg/ingress/annotations/authtls/main.go
index 143353249..c4172e51c 100644
--- a/core/pkg/ingress/annotations/authtls/main.go
+++ b/core/pkg/ingress/annotations/authtls/main.go
@@ -28,11 +28,16 @@ import (
 
 const (
 	// name of the secret
-	authTLSSecret = "ingress.kubernetes.io/auth-tls-secret"
+	annotationAuthTLSSecret = "ingress.kubernetes.io/auth-tls-secret"
+	annotationAuthTLSDepth  = "ingress.kubernetes.io/auth-tls-verify-depth"
+	defaultAuthTLSDepth     = 1
 )
 
-type authTLS struct {
-	certResolver resolver.AuthCertificate
+// AuthSSLConfig contains the AuthSSLCert used for muthual autentication
+// and the configured ValidationDepth
+type AuthSSLConfig struct {
+	AuthSSLCert     resolver.AuthSSLCert
+	ValidationDepth int `json:"validationDepth"`
 }
 
 // NewParser creates a new TLS authentication annotation parser
@@ -40,29 +45,42 @@ func NewParser(resolver resolver.AuthCertificate) parser.IngressAnnotation {
 	return authTLS{resolver}
 }
 
-// ParseAnnotations parses the annotations contained in the ingress
-// rule used to use an external URL as source for authentication
+type authTLS struct {
+	certResolver resolver.AuthCertificate
+}
+
+// Parse parses the annotations contained in the ingress
+// rule used to use a Certificate as authentication method
 func (a authTLS) Parse(ing *extensions.Ingress) (interface{}, error) {
-	str, err := parser.GetStringAnnotation(authTLSSecret, ing)
+
+	tlsauthsecret, err := parser.GetStringAnnotation(annotationAuthTLSSecret, ing)
 	if err != nil {
-		return nil, err
+		return &AuthSSLConfig{}, err
 	}
 
-	if str == "" {
-		return nil, ing_errors.NewLocationDenied("an empty string is not a valid secret name")
+	if tlsauthsecret == "" {
+		return &AuthSSLConfig{}, ing_errors.NewLocationDenied("an empty string is not a valid secret name")
 	}
 
-	_, _, err = k8s.ParseNameNS(str)
+	_, _, err = k8s.ParseNameNS(tlsauthsecret)
 	if err != nil {
-		return nil, ing_errors.NewLocationDenied("an empty string is not a valid secret name")
+		return &AuthSSLConfig{}, ing_errors.NewLocationDenied("an empty string is not a valid secret name")
 	}
 
-	authCert, err := a.certResolver.GetAuthCertificate(str)
+	tlsdepth, err := parser.GetIntAnnotation(annotationAuthTLSDepth, ing)
+	if err != nil || tlsdepth == 0 {
+		tlsdepth = defaultAuthTLSDepth
+	}
+
+	authCert, err := a.certResolver.GetAuthCertificate(tlsauthsecret)
 	if err != nil {
-		return nil, ing_errors.LocationDenied{
+		return &AuthSSLConfig{}, ing_errors.LocationDenied{
 			Reason: errors.Wrap(err, "error obtaining certificate"),
 		}
 	}
 
-	return authCert, nil
+	return &AuthSSLConfig{
+		AuthSSLCert:     *authCert,
+		ValidationDepth: tlsdepth,
+	}, nil
 }
diff --git a/core/pkg/ingress/controller/backend_ssl.go b/core/pkg/ingress/controller/backend_ssl.go
index 92c32fc8c..7a84a3d4f 100644
--- a/core/pkg/ingress/controller/backend_ssl.go
+++ b/core/pkg/ingress/controller/backend_ssl.go
@@ -98,6 +98,8 @@ func (ic *GenericController) syncSecret(k interface{}) error {
 	return nil
 }
 
+// getPemCertificate receives a secret, and creates a ingress.SSLCert as return.
+// It parses the secret and verifies if it's a keypair, or a 'ca.crt' secret only.
 func (ic *GenericController) getPemCertificate(secretName string) (*ingress.SSLCert, error) {
 	secretInterface, exists, err := ic.secrLister.Store.GetByKey(secretName)
 	if err != nil {
@@ -108,19 +110,24 @@ func (ic *GenericController) getPemCertificate(secretName string) (*ingress.SSLC
 	}
 
 	secret := secretInterface.(*api.Secret)
-	cert, ok := secret.Data[api.TLSCertKey]
-	if !ok {
-		return nil, fmt.Errorf("secret named %v has no private key", secretName)
-	}
-	key, ok := secret.Data[api.TLSPrivateKeyKey]
-	if !ok {
-		return nil, fmt.Errorf("secret named %v has no cert", secretName)
-	}
+	cert, okcert := secret.Data[api.TLSCertKey]
+	key, okkey := secret.Data[api.TLSPrivateKeyKey]
 
 	ca := secret.Data["ca.crt"]
 
 	nsSecName := strings.Replace(secretName, "/", "-", -1)
-	s, err := ssl.AddOrUpdateCertAndKey(nsSecName, cert, key, ca)
+
+	var s *ingress.SSLCert
+	if okcert && okkey {
+		glog.V(3).Infof("Found certificate and private key, configuring %v as a TLS Secret", secretName)
+		s, err = ssl.AddOrUpdateCertAndKey(nsSecName, cert, key, ca)
+	} else if ca != nil {
+		glog.V(3).Infof("Found only ca.crt, configuring %v as an Certificate Authentication secret", secretName)
+		s, err = ssl.AddCertAuth(nsSecName, ca)
+	} else {
+		return nil, fmt.Errorf("No keypair or CA cert could be found in %v", secretName)
+	}
+
 	if err != nil {
 		return nil, err
 	}
diff --git a/core/pkg/ingress/controller/controller.go b/core/pkg/ingress/controller/controller.go
index 58e647f9e..9b7d9512f 100644
--- a/core/pkg/ingress/controller/controller.go
+++ b/core/pkg/ingress/controller/controller.go
@@ -680,16 +680,23 @@ func (ic *GenericController) getBackendServers() ([]*ingress.Backend, []*ingress
 
 // GetAuthCertificate ...
 func (ic GenericController) GetAuthCertificate(secretName string) (*resolver.AuthSSLCert, error) {
+	key, err := ic.GetSecret(secretName)
+	if err != nil {
+		return &resolver.AuthSSLCert{}, fmt.Errorf("unexpected error: %v", err)
+	}
+	if key != nil {
+		ic.secretQueue.Enqueue(key)
+	}
+
 	bc, exists := ic.sslCertTracker.Get(secretName)
 	if !exists {
 		return &resolver.AuthSSLCert{}, fmt.Errorf("secret %v does not exists", secretName)
 	}
 	cert := bc.(*ingress.SSLCert)
 	return &resolver.AuthSSLCert{
-		Secret:       secretName,
-		CertFileName: cert.PemFileName,
-		CAFileName:   cert.CAFileName,
-		PemSHA:       cert.PemSHA,
+		Secret:     secretName,
+		CAFileName: cert.CAFileName,
+		PemSHA:     cert.PemSHA,
 	}, nil
 }
 
diff --git a/core/pkg/ingress/controller/util_test.go b/core/pkg/ingress/controller/util_test.go
index 8ec84785a..96d46000d 100644
--- a/core/pkg/ingress/controller/util_test.go
+++ b/core/pkg/ingress/controller/util_test.go
@@ -24,11 +24,11 @@ import (
 	"k8s.io/ingress/core/pkg/ingress"
 	"k8s.io/ingress/core/pkg/ingress/annotations/auth"
 	"k8s.io/ingress/core/pkg/ingress/annotations/authreq"
+	"k8s.io/ingress/core/pkg/ingress/annotations/authtls"
 	"k8s.io/ingress/core/pkg/ingress/annotations/ipwhitelist"
 	"k8s.io/ingress/core/pkg/ingress/annotations/proxy"
 	"k8s.io/ingress/core/pkg/ingress/annotations/ratelimit"
 	"k8s.io/ingress/core/pkg/ingress/annotations/rewrite"
-	"k8s.io/ingress/core/pkg/ingress/resolver"
 	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/apis/extensions"
 )
@@ -136,7 +136,7 @@ func TestMergeLocationAnnotations(t *testing.T) {
 		"Redirect":           rewrite.Redirect{},
 		"Whitelist":          ipwhitelist.SourceRange{},
 		"Proxy":              proxy.Configuration{},
-		"CertificateAuth":    resolver.AuthSSLCert{},
+		"CertificateAuth":    authtls.AuthSSLConfig{},
 		"UsePortInRedirects": true,
 	}
 
diff --git a/core/pkg/ingress/resolver/main.go b/core/pkg/ingress/resolver/main.go
index 6017c8cb5..a11b35f58 100644
--- a/core/pkg/ingress/resolver/main.go
+++ b/core/pkg/ingress/resolver/main.go
@@ -37,8 +37,6 @@ type Secret interface {
 // AuthCertificate resolves a given secret name into an SSL certificate.
 // The secret must contain 3 keys named:
 //   ca.crt: contains the certificate chain used for authentication
-//   tls.crt: (ignored) contains the tls certificate chain, or any other valid base64 data
-//   tls.key: (ignored) contains the tls secret key, or any other valid base64 data
 type AuthCertificate interface {
 	GetAuthCertificate(string) (*AuthSSLCert, error)
 }
@@ -48,10 +46,6 @@ type AuthCertificate interface {
 type AuthSSLCert struct {
 	// Secret contains the name of the secret this was fetched from
 	Secret string `json:"secret"`
-	// CertFileName contains the filename the secret's 'tls.crt' was saved to
-	CertFileName string `json:"certFilename"`
-	// KeyFileName contains the path the secret's 'tls.key'
-	KeyFileName string `json:"keyFilename"`
 	// CAFileName contains the path to the secrets 'ca.crt'
 	CAFileName string `json:"caFilename"`
 	// PemSHA contains the SHA1 hash of the 'tls.crt' value
diff --git a/core/pkg/ingress/types.go b/core/pkg/ingress/types.go
index 0d8a8b735..82ddcf0d3 100644
--- a/core/pkg/ingress/types.go
+++ b/core/pkg/ingress/types.go
@@ -27,12 +27,12 @@ import (
 	cache_store "k8s.io/ingress/core/pkg/cache"
 	"k8s.io/ingress/core/pkg/ingress/annotations/auth"
 	"k8s.io/ingress/core/pkg/ingress/annotations/authreq"
+	"k8s.io/ingress/core/pkg/ingress/annotations/authtls"
 	"k8s.io/ingress/core/pkg/ingress/annotations/ipwhitelist"
 	"k8s.io/ingress/core/pkg/ingress/annotations/proxy"
 	"k8s.io/ingress/core/pkg/ingress/annotations/ratelimit"
 	"k8s.io/ingress/core/pkg/ingress/annotations/rewrite"
 	"k8s.io/ingress/core/pkg/ingress/defaults"
-	"k8s.io/ingress/core/pkg/ingress/resolver"
 )
 
 var (
@@ -274,7 +274,7 @@ type Location struct {
 	// CertificateAuth indicates the access to this location requires
 	// external authentication
 	// +optional
-	CertificateAuth resolver.AuthSSLCert `json:"certificateAuth,omitempty"`
+	CertificateAuth authtls.AuthSSLConfig `json:"certificateAuth,omitempty"`
 	// UsePortInRedirects indicates if redirects must specify the port
 	// +optional
 	UsePortInRedirects bool `json:"use-port-in-redirects"`
diff --git a/core/pkg/net/ssl/ssl.go b/core/pkg/net/ssl/ssl.go
index 5907d5f5e..62d2f6b7e 100644
--- a/core/pkg/net/ssl/ssl.go
+++ b/core/pkg/net/ssl/ssl.go
@@ -37,6 +37,8 @@ func AddOrUpdateCertAndKey(name string, cert, key, ca []byte) (*ingress.SSLCert,
 	pemFileName := fmt.Sprintf("%v/%v", ingress.DefaultSSLDirectory, pemName)
 
 	tempPemFile, err := ioutil.TempFile(ingress.DefaultSSLDirectory, pemName)
+
+	glog.V(3).Infof("Creating temp file %v for Keypair: %v", tempPemFile.Name(), pemName)
 	if err != nil {
 		return nil, fmt.Errorf("could not create temp pem file %v: %v", pemFileName, err)
 	}
@@ -64,12 +66,12 @@ func AddOrUpdateCertAndKey(name string, cert, key, ca []byte) (*ingress.SSLCert,
 		return nil, err
 	}
 
-	pembBock, _ := pem.Decode(pemCerts)
-	if pembBock == nil {
+	pemBlock, _ := pem.Decode(pemCerts)
+	if pemBlock == nil {
 		return nil, fmt.Errorf("No valid PEM formatted block found")
 	}
 
-	pemCert, err := x509.ParseCertificate(pembBock.Bytes)
+	pemCert, err := x509.ParseCertificate(pemBlock.Bytes)
 	if err != nil {
 		return nil, err
 	}
@@ -97,21 +99,21 @@ func AddOrUpdateCertAndKey(name string, cert, key, ca []byte) (*ingress.SSLCert,
 			return nil, errors.New(oe)
 		}
 
-		caName := fmt.Sprintf("ca-%v.pem", name)
-		caFileName := fmt.Sprintf("%v/%v", ingress.DefaultSSLDirectory, caName)
-		f, err := os.Create(caFileName)
+		caFile, err := os.OpenFile(pemFileName, os.O_RDWR|os.O_APPEND, 0600)
 		if err != nil {
-			return nil, fmt.Errorf("could not create ca pem file %v: %v", caFileName, err)
+			return nil, fmt.Errorf("Could not open file %v for writing additional CA chains: %v", pemFileName, err)
 		}
-		defer f.Close()
-		_, err = f.Write(ca)
+
+		defer caFile.Close()
+		_, err = caFile.Write([]byte("\n"))
 		if err != nil {
-			return nil, fmt.Errorf("could not create ca pem file %v: %v", caFileName, err)
+			return nil, fmt.Errorf("could not append CA to cert file %v: %v", pemFileName, err)
 		}
-		f.Write([]byte("\n"))
+		caFile.Write(ca)
+		caFile.Write([]byte("\n"))
 
 		return &ingress.SSLCert{
-			CAFileName:  caFileName,
+			CAFileName:  pemFileName,
 			PemFileName: pemFileName,
 			PemSHA:      pemSHA1(pemFileName),
 			CN:          cn,
@@ -125,6 +127,36 @@ func AddOrUpdateCertAndKey(name string, cert, key, ca []byte) (*ingress.SSLCert,
 	}, nil
 }
 
+// AddCertAuth creates a .pem file with the specified CAs to be used in Cert Authentication
+// If it's already exists, it's clobbered.
+func AddCertAuth(name string, ca []byte) (*ingress.SSLCert, error) {
+
+	caName := fmt.Sprintf("ca-%v.pem", name)
+	caFileName := fmt.Sprintf("%v/%v", ingress.DefaultSSLDirectory, caName)
+
+	pemCABlock, _ := pem.Decode(ca)
+	if pemCABlock == nil {
+		return nil, fmt.Errorf("No valid PEM formatted block found")
+	}
+
+	_, err := x509.ParseCertificate(pemCABlock.Bytes)
+	if err != nil {
+		return nil, err
+	}
+
+	err = ioutil.WriteFile(caFileName, ca, 0644)
+	if err != nil {
+		return nil, fmt.Errorf("could not write CA file %v: %v", caFileName, err)
+	}
+
+	glog.V(3).Infof("Created CA Certificate for authentication: %v", caFileName)
+	return &ingress.SSLCert{
+		CAFileName:  caFileName,
+		PemFileName: caFileName,
+		PemSHA:      pemSHA1(caFileName),
+	}, nil
+}
+
 // SearchDHParamFile iterates all the secrets mounted inside the /etc/nginx-ssl directory
 // in order to find a file with the name dhparam.pem. If such file exists it will
 // returns the path. If not it just returns an empty string
diff --git a/examples/PREREQUISITES.md b/examples/PREREQUISITES.md
index a2d78c6b6..5881f9045 100644
--- a/examples/PREREQUISITES.md
+++ b/examples/PREREQUISITES.md
@@ -36,6 +36,105 @@ $ kubectl create secret tls tls-secret --key tls.key --cert tls.crt
 secret "tls-secret" created
 ```
 
+## CA Authentication
+You can act as your very own CA, or use an existing one. As an exercise / learning, we're going to generate our
+own CA, and also generate a client certificate.
+
+These instructions are based in CoreOS OpenSSL [instructions](https://coreos.com/kubernetes/docs/latest/openssl.html)
+
+### Generating a CA
+
+First of all, you've to generate a CA. This is going to be the one who will sign your client certificates.
+In real production world, you may face CAs with intermediate certificates, as the following:
+
+```console
+$ openssl s_client -connect www.google.com:443
+[...]
+---
+Certificate chain
+ 0 s:/C=US/ST=California/L=Mountain View/O=Google Inc/CN=www.google.com
+   i:/C=US/O=Google Inc/CN=Google Internet Authority G2
+ 1 s:/C=US/O=Google Inc/CN=Google Internet Authority G2
+   i:/C=US/O=GeoTrust Inc./CN=GeoTrust Global CA
+ 2 s:/C=US/O=GeoTrust Inc./CN=GeoTrust Global CA
+   i:/C=US/O=Equifax/OU=Equifax Secure Certificate Authority
+
+```
+
+To generate our CA Certificate, we've to run the following commands:
+
+```console
+$ openssl genrsa -out ca.key 2048
+$ openssl req -x509 -new -nodes -key ca.key -days 10000 -out ca.crt -subj "/CN=example-ca"
+```
+
+This will generate two files: A private key (ca.key) and a public key (ca.crt). This CA is valid for 10000 days.
+The ca.crt can be used later in the step of creation of CA authentication secret.
+
+### Generating the client certificate
+The following steps generates a client certificate signed by the CA generated above. This client can be
+used to authenticate in a tls-auth configured ingress.
+
+First, we need to generate an 'openssl.cnf' file that will be used while signing the keys:
+
+```
+[req]
+req_extensions = v3_req
+distinguished_name = req_distinguished_name
+[req_distinguished_name]
+[ v3_req ]
+basicConstraints = CA:FALSE
+keyUsage = nonRepudiation, digitalSignature, keyEncipherment
+```
+
+Then, a user generates his very own private key (that he needs to keep secret)
+and a CSR (Certificate Signing Request) that will be sent to the CA to sign and generate a certificate.
+
+```console
+$ openssl genrsa -out client1.key 2048
+$ openssl req -new -key client1.key -out client1.csr -subj "/CN=client1" -config openssl.cnf
+```
+
+As the CA receives the generated 'client1.csr' file, it signs it and generates a client.crt certificate:
+
+```console
+$ openssl x509 -req -in client1.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out client1.crt -days 365 -extensions v3_req -extfile openssl.cnf
+```
+
+Then, you'll have 3 files: the client.key (user's private key), client.crt (user's public key) and client.csr (disposable CSR).
+
+
+### Creating the CA Authentication secret
+If you're using the CA Authentication feature, you need to generate a secret containing 
+all the authorized CAs. You must download them from your CA site in PEM format (like the following):
+
+```
+-----BEGIN CERTIFICATE-----
+[....]
+-----END CERTIFICATE-----
+``` 
+
+You can have as many certificates as you want. If they're in the binary DER format, 
+you can convert them as the following:
+
+```console
+$ openssl x509 -in certificate.der -inform der -out certificate.crt -outform pem
+```
+
+Then, you've to concatenate them all in only one file, named 'ca.crt' as the following:
+
+
+```console
+$ cat certificate1.crt certificate2.crt certificate3.crt >> ca.crt
+```
+
+The final step is to create a secret with the content of this file. This secret is going to be used in 
+the TLS Auth directive:
+
+```console
+$ kubectl create secret generic caingress --namespace=default --from-file=ca.crt
+```
+
 ## Test HTTP Service
 
 All examples that require a test HTTP Service use the standard http-svc pod,
diff --git a/examples/auth/client-certs/nginx/README.md b/examples/auth/client-certs/nginx/README.md
new file mode 100644
index 000000000..fed88598a
--- /dev/null
+++ b/examples/auth/client-certs/nginx/README.md
@@ -0,0 +1,86 @@
+# TLS authentication 
+
+This example demonstrates how to enable the TLS Authentication through the nginx Ingress controller.
+
+## Terminology
+
+* CA: Certificate authority signing the client cert, in this example we will play the role of a CA. 
+You can generate a CA cert as show in this doc.
+
+* CA Certificate(s) - Certificate Authority public key. Client certs must chain back to this cert, 
+meaning the Issuer field of some certificate in the chain leading up to the client cert must contain 
+the name of this CA. For purposes of this example, this is a self signed certificate.
+
+* CA chains: A chain of certificates where the parent has a Subject field matching the Issuer field of 
+the child, except for the root, which has Issuer == Subject.
+
+* Client Cert: Certificate used by the clients to authenticate themselves with the loadbalancer/backends.
+
+
+## Prerequisites
+
+You need a valid CA File, composed of a group of valid enabled CAs. This MUST be in PEM Format.
+The instructions are described [here](../../../PREREQUISITES.md#ca-authentication)
+
+Also your ingress must be configured as a HTTPs/TLS Ingress.
+
+## Deployment
+
+Certificate Authentication is achieved through 2 annotations on the Ingress, as shown in the [example](nginx-tls-auth.yaml).
+
+|Name|Description|Values|
+| --- | --- | --- |
+|ingress.kubernetes.io/auth-tls-secret|Sets the secret that contains the authorized CA Chain|string|
+|ingress.kubernetes.io/auth-tls-verify-depth|The verification depth Certificate Authentication will make|number (default to 1)|
+
+
+The following command instructs the controller to enable TLS authentication using the secret from the ``ingress.kubernetes.io/auth-tls-secret``
+annotation on the Ingress. Clients must present this cert to the loadbalancer, or they will receive a HTTP 400 response
+
+```console
+$ kubectl create -f nginx-tls-auth.yaml
+```
+
+## Validation
+
+You can confirm that the Ingress works. 
+
+```console
+$ kubectl describe ing nginx-test
+Name:			nginx-test
+Namespace:		default
+Address:		104.198.183.6
+Default backend:	default-http-backend:80 (10.180.0.4:8080,10.240.0.2:8080)
+TLS:
+  tls-secret terminates ingress.test.com
+Rules:
+  Host	Path	Backends
+  ----	----	--------
+  *
+    	 	http-svc:80 (<none>)
+Annotations:
+  auth-tls-secret:	default/caingress
+  auth-tls-verify-depth: 3
+
+Events:
+  FirstSeen	LastSeen	Count	From				SubObjectPath	Type		Reason	Message
+  ---------	--------	-----	----				-------------	--------	------	-------
+  7s		7s		1	{nginx-ingress-controller }			Normal		CREATE	default/nginx-test
+  7s		7s		1	{nginx-ingress-controller }			Normal		UPDATE	default/nginx-test
+  7s		7s		1	{nginx-ingress-controller }			Normal		CREATE	ip: 104.198.183.6
+  7s		7s		1	{nginx-ingress-controller }			Warning		MAPPING	Ingress rule 'default/nginx-test' contains no path definition. Assuming /
+
+
+$ curl -k https://ingress.test.com
+HTTP/1.1 400 Bad Request
+Server: nginx/1.11.9
+
+$ curl -I -k --key ~/user.key --cert ~/user.cer https://ingress.test.com 
+HTTP/1.1 200 OK
+Server: nginx/1.11.9
+
+```
+
+You must use the full DNS name while testing, as NGINX relies on the Server Name (SNI) to select the correct Ingress to be used.
+
+The curl version used here was ``curl 7.47.0``
diff --git a/examples/auth/client-certs/nginx/nginx-tls-auth.yaml b/examples/auth/client-certs/nginx/nginx-tls-auth.yaml
new file mode 100644
index 000000000..d4f18bd0c
--- /dev/null
+++ b/examples/auth/client-certs/nginx/nginx-tls-auth.yaml
@@ -0,0 +1,25 @@
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+  annotations:
+    # Create this with kubectl create secret generic caingress --from-file=ca.crt --namespace=default
+    ingress.kubernetes.io/auth-tls-secret: "default/caingress"
+    ingress.kubernetes.io/auth-tls-verify-depth: "3"
+    kubernetes.io/ingress.class: "nginx"
+  name: nginx-test 
+  namespace: default
+spec:
+  rules:
+  - host: ingress.test.com
+    http:
+      paths:
+      - backend:
+          serviceName: http-svc:80
+          servicePort: 80
+        path: /
+  tls:
+  - hosts:
+    - ingress.test.com
+    # Create this cert as described in 'multi-tls' example
+    secretName: cert
+

From fd7990de6743588fb7a68dc71bd51ee05b0829f6 Mon Sep 17 00:00:00 2001
From: Piotr Szczesniak <pszczesniak@google.com>
Date: Sat, 25 Feb 2017 18:30:00 +0100
Subject: [PATCH 71/73] Expose Prometheus metrics in glbc controller

---
 controllers/gce/main.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/controllers/gce/main.go b/controllers/gce/main.go
index 0f1f8e981..db2eebfff 100644
--- a/controllers/gce/main.go
+++ b/controllers/gce/main.go
@@ -39,6 +39,7 @@ import (
 	"k8s.io/kubernetes/pkg/util/wait"
 
 	"github.com/golang/glog"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
 // Entrypoint of GLBC. Example invocation:
@@ -132,6 +133,7 @@ func registerHandlers(lbc *controller.LoadBalancerController) {
 		w.WriteHeader(200)
 		w.Write([]byte("ok"))
 	})
+	http.Handle("/metrics", promhttp.Handler())
 	http.HandleFunc("/delete-all-and-quit", func(w http.ResponseWriter, r *http.Request) {
 		// TODO: Retry failures during shutdown.
 		lbc.Stop(true)

From 02d44ccbaae729f9d1d7438f4d6221be3fc481f6 Mon Sep 17 00:00:00 2001
From: Manuel de Brito Fontes <aledbf@gmail.com>
Date: Sun, 26 Feb 2017 19:01:07 -0300
Subject: [PATCH 72/73] Fix client source IP address

---
 controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl b/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl
index 07b6f5782..4684a497c 100644
--- a/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl
+++ b/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl
@@ -1,4 +1,8 @@
-{{ $cfg := .Cfg }}{{ $healthzURI := .HealthzURI }}{{ $backends := .Backends }}{{ $proxyHeaders := .ProxySetHeaders }}
+{{ $cfg := .Cfg }}
+{{ $healthzURI := .HealthzURI }}
+{{ $backends := .Backends }}
+{{ $proxyHeaders := .ProxySetHeaders }}
+{{ $passthroughBackends := .PassthroughBackends }}
 daemon off;
 
 worker_processes {{ $cfg.WorkerProcesses }};
@@ -208,7 +212,7 @@ http {
         listen [::]:80{{ if $cfg.UseProxyProtocol }} proxy_protocol{{ end }}{{ if eq $index 0 }} ipv6only=off{{end}}{{ if eq $server.Hostname "_"}} default_server reuseport backlog={{ $backlogSize }}{{end}};
         {{/* Listen on 442 because port 443 is used in the stream section */}}
         {{/* This listen cannot contains proxy_protocol directive because port 443 is in charge of decoding the protocol */}}
-        {{ if not (empty $server.SSLCertificate) }}listen 442 {{ if eq $server.Hostname "_"}} default_server reuseport backlog={{ $backlogSize }}{{end}} ssl {{ if $cfg.UseHTTP2 }}http2{{ end }};
+        {{ if not (empty $server.SSLCertificate) }}listen {{ if gt (len $passthroughBackends) 0 }}442{{ else }}[::]:443 {{ end }}{{ if eq $server.Hostname "_"}} default_server reuseport backlog={{ $backlogSize }}{{end}} ssl {{ if $cfg.UseHTTP2 }}http2{{ end }};
         {{/* comment PEM sha is required to detect changes in the generated configuration and force a reload */}}
         # PEM sha: {{ $server.SSLPemChecksum }}
         ssl_certificate                         {{ $server.SSLCertificate }};
@@ -434,6 +438,7 @@ http {
 }
 
 stream {
+    {{ if gt (len $passthroughBackends) 0 }}
     # map FQDN that requires SSL passthrough
     map $ssl_preread_server_name $stream_upstream {
         {{ range $i, $passthrough := .PassthroughBackends }}
@@ -465,6 +470,7 @@ stream {
         proxy_pass              $stream_upstream;
         ssl_preread             on;
     }
+    {{ end }}
     
     # TCP services		
     {{ range $i, $tcpServer := .TCPBackends }}

From 2d526b213c4f3a406e00377e1afdc4f9d6e7ac15 Mon Sep 17 00:00:00 2001
From: shijunqian <qsj.daniel@gmail.com>
Date: Mon, 27 Feb 2017 15:35:04 +0800
Subject: [PATCH 73/73] Enable custom election id for status sync.

---
 core/pkg/ingress/controller/controller.go | 2 ++
 core/pkg/ingress/controller/launch.go     | 3 +++
 core/pkg/ingress/status/status.go         | 3 ++-
 3 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/core/pkg/ingress/controller/controller.go b/core/pkg/ingress/controller/controller.go
index 0523d3063..1052d8ffe 100644
--- a/core/pkg/ingress/controller/controller.go
+++ b/core/pkg/ingress/controller/controller.go
@@ -134,6 +134,7 @@ type Configuration struct {
 	Backend ingress.Controller
 
 	UpdateStatus bool
+	ElectionID   string
 }
 
 // newIngressController creates an Ingress controller
@@ -303,6 +304,7 @@ func newIngressController(config *Configuration) *GenericController {
 			Client:         config.Client,
 			PublishService: ic.cfg.PublishService,
 			IngressLister:  ic.ingLister,
+			ElectionID:     config.ElectionID,
 		})
 	} else {
 		glog.Warning("Update of ingress status is disabled (flag --update-status=false was specified)")
diff --git a/core/pkg/ingress/controller/launch.go b/core/pkg/ingress/controller/launch.go
index ce23f9a5c..216682bdd 100644
--- a/core/pkg/ingress/controller/launch.go
+++ b/core/pkg/ingress/controller/launch.go
@@ -82,6 +82,8 @@ func NewIngressController(backend ingress.Controller) *GenericController {
 
 		updateStatus = flags.Bool("update-status", true, `Indicates if the 
 		ingress controller should update the Ingress status IP/hostname. Default is true`)
+
+		electionID = flags.String("election-id", "ingress-controller-leader", `Election id to use for status update.`)
 	)
 
 	backend.OverrideFlags(flags)
@@ -137,6 +139,7 @@ func NewIngressController(backend ingress.Controller) *GenericController {
 
 	config := &Configuration{
 		UpdateStatus:          *updateStatus,
+		ElectionID:            *electionID,
 		Client:                kubeClient,
 		ResyncPeriod:          *resyncPeriod,
 		DefaultService:        *defaultSvc,
diff --git a/core/pkg/ingress/status/status.go b/core/pkg/ingress/status/status.go
index 09ddb702d..6000c7f6b 100644
--- a/core/pkg/ingress/status/status.go
+++ b/core/pkg/ingress/status/status.go
@@ -52,6 +52,7 @@ type Config struct {
 	Client         clientset.Interface
 	PublishService string
 	IngressLister  cache_store.StoreToIngressLister
+	ElectionID     string
 }
 
 // statusSync keeps the status IP in each Ingress rule updated executing a periodic check
@@ -171,7 +172,7 @@ func NewStatusSyncer(config Config) Sync {
 	}
 	st.syncQueue = task.NewCustomTaskQueue(st.sync, st.keyfunc)
 
-	le, err := NewElection("ingress-controller-leader",
+	le, err := NewElection(config.ElectionID,
 		pod.Name, pod.Namespace, 30*time.Second,
 		st.callback, config.Client)
 	if err != nil {