From 04a89ce2343be1ce91e80d4c5fa597014961afad Mon Sep 17 00:00:00 2001
From: Desmond Ho <desmond.ho@lalamove.com>
Date: Fri, 5 Oct 2018 12:24:37 +0800
Subject: [PATCH 01/31] UPT: annotation enhancement for resty-lua-waf

---
 .../nginx-configuration/annotations.md        | 40 ++++++++++++++++
 .../ingress/annotations/luarestywaf/main.go   | 37 ++++++++++----
 .../annotations/luarestywaf/main_test.go      | 11 ++++-
 rootfs/etc/nginx/template/nginx.tmpl          | 14 ++++++
 test/e2e/annotations/luarestywaf.go           | 48 +++++++++++++++++++
 5 files changed, 140 insertions(+), 10 deletions(-)

diff --git a/docs/user-guide/nginx-configuration/annotations.md b/docs/user-guide/nginx-configuration/annotations.md
index f1eb69e1b..6a61d21be 100644
--- a/docs/user-guide/nginx-configuration/annotations.md
+++ b/docs/user-guide/nginx-configuration/annotations.md
@@ -80,6 +80,9 @@ You can add these Kubernetes annotations to specific Ingress objects to customiz
 |[nginx.ingress.kubernetes.io/lua-resty-waf-debug](#lua-resty-waf)|"true" or "false"|
 |[nginx.ingress.kubernetes.io/lua-resty-waf-ignore-rulesets](#lua-resty-waf)|string|
 |[nginx.ingress.kubernetes.io/lua-resty-waf-extra-rules](#lua-resty-waf)|string|
+|[nginx.ingress.kubernetes.io/lua-resty-waf-allow-unknown-content](#lua-resty-waf)|"true" or "false"|
+|[nginx.ingress.kubernetes.io/lua-resty-waf-score](#lua-resty-waf)|number|
+|[nginx.ingress.kubernetes.io/lua-resty-waf-disable-multipart-body](#lua-resty-waf)|"true" or "false"|
 |[nginx.ingress.kubernetes.io/enable-influxdb](#influxdb)|"true" or "false"|
 |[nginx.ingress.kubernetes.io/influxdb-measurement](#influxdb)|string|
 |[nginx.ingress.kubernetes.io/influxdb-port](#influxdb)|string|
@@ -558,6 +561,43 @@ It is also possible to configure custom WAF rules per ingress using the `nginx.i
 nginx.ingress.kubernetes.io/lua-resty-waf-extra-rules: '[=[ { "access": [ { "actions": { "disrupt" : "DENY" }, "id": 10001, "msg": "my custom rule", "operator": "STR_CONTAINS", "pattern": "foo", "vars": [ { "parse": [ "values", 1 ], "type": "REQUEST_ARGS" } ] } ], "body_filter": [], "header_filter":[] } ]=]'
 ```
 
+Since the default allowed contents were `"text/html", "text/json", "application/json"`
+We can enable the following annotation for allow all contents type:
+
+
+```yaml
+nginx.ingress.kubernetes.io/lua-resty-waf-allow-unknown-content: "true"
+```
+
+The default score of lua-resty-waf is 5, which usually triggered if hitting 2 default rules, you can modify the score threshold with following annotation:
+
+
+```yaml
+nginx.ingress.kubernetes.io/lua-resty-waf-score: "10"
+```
+
+When you enabled HTTPS in the endpoint and since resty-lua will return 500 error when processing "multipart" contents
+Reference for this [issue](https://github.com/p0pr0ck5/lua-resty-waf/issues/166)
+You may enable the following annotation for work around:
+
+```yaml
+nginx.ingress.kubernetes.io/lua-resty-waf-disable-multipart-body: "true"
+```
+
+For details on how to write WAF rules, please refer to [https://github.com/p0pr0ck5/lua-resty-waf](https://github.com/p0pr0ck5/lua-resty-waf).
+
+
+```yaml
+nginx.ingress.kubernetes.io/lua-resty-waf-allow-unknown-content: "true"
+```
+
+The default score of lua-resty-waf is 5, which usually triggered if hitting 2 default rules, you can modify the score threshold with following annotation:
+
+
+```yaml
+nginx.ingress.kubernetes.io/lua-resty-waf-score: "10"
+```
+
 For details on how to write WAF rules, please refer to [https://github.com/p0pr0ck5/lua-resty-waf](https://github.com/p0pr0ck5/lua-resty-waf).
 
 [configmap]: ./configmap.md
diff --git a/internal/ingress/annotations/luarestywaf/main.go b/internal/ingress/annotations/luarestywaf/main.go
index ca7f4a8be..4873851ab 100644
--- a/internal/ingress/annotations/luarestywaf/main.go
+++ b/internal/ingress/annotations/luarestywaf/main.go
@@ -31,10 +31,13 @@ var luaRestyWAFModes = map[string]bool{"ACTIVE": true, "INACTIVE": true, "SIMULA
 
 // Config returns lua-resty-waf configuration for an Ingress rule
 type Config struct {
-	Mode               string   `json:"mode"`
-	Debug              bool     `json:"debug"`
-	IgnoredRuleSets    []string `json:"ignored-rulesets"`
-	ExtraRulesetString string   `json:"extra-ruleset-string"`
+	Mode                 string   `json:"mode"`
+	Debug                bool     `json:"debug"`
+	IgnoredRuleSets      []string `json:"ignored-rulesets"`
+	ExtraRulesetString   string   `json:"extra-ruleset-string"`
+	Score                int      `json:"score"`
+	AllowUnknownContent  bool     `json:"allow-unknown-content"`
+	DisableMultipartBody bool     `json:"disable-multipart-body"`
 }
 
 // Equal tests for equality between two Config types
@@ -57,6 +60,15 @@ func (e1 *Config) Equal(e2 *Config) bool {
 	if e1.ExtraRulesetString != e2.ExtraRulesetString {
 		return false
 	}
+	if e1.Score != e2.Score {
+		return false
+	}
+	if e1.AllowUnknownContent != e2.AllowUnknownContent {
+		return false
+	}
+	if e1.DisableMultipartBody != e2.DisableMultipartBody {
+		return false
+	}
 
 	return true
 }
@@ -95,10 +107,19 @@ func (a luarestywaf) Parse(ing *extensions.Ingress) (interface{}, error) {
 	// TODO(elvinefendi) maybe validate the ruleset string here
 	extraRulesetString, _ := parser.GetStringAnnotation("lua-resty-waf-extra-rules", ing)
 
+	score, _ := parser.GetIntAnnotation("lua-resty-waf-score", ing)
+
+	allowUnknownContent, _ := parser.GetBoolAnnotation("lua-resty-waf-allow-unknown-content", ing)
+
+	disableMultipartBody, _ := parser.GetBoolAnnotation("lua-resty-waf-disable-multipart-body", ing)
+
 	return &Config{
-		Mode:               mode,
-		Debug:              debug,
-		IgnoredRuleSets:    ignoredRuleSets,
-		ExtraRulesetString: extraRulesetString,
+		Mode:                 mode,
+		Debug:                debug,
+		IgnoredRuleSets:      ignoredRuleSets,
+		ExtraRulesetString:   extraRulesetString,
+		Score:                score,
+		AllowUnknownContent:  allowUnknownContent,
+		DisableMultipartBody: disableMultipartBody,
 	}, nil
 }
diff --git a/internal/ingress/annotations/luarestywaf/main_test.go b/internal/ingress/annotations/luarestywaf/main_test.go
index d71191f12..e9b50c53f 100644
--- a/internal/ingress/annotations/luarestywaf/main_test.go
+++ b/internal/ingress/annotations/luarestywaf/main_test.go
@@ -30,6 +30,9 @@ func TestParse(t *testing.T) {
 	luaRestyWAFAnnotation := parser.GetAnnotationWithPrefix("lua-resty-waf")
 	luaRestyWAFDebugAnnotation := parser.GetAnnotationWithPrefix("lua-resty-waf-debug")
 	luaRestyWAFIgnoredRuleSetsAnnotation := parser.GetAnnotationWithPrefix("lua-resty-waf-ignore-rulesets")
+	luaRestyWAFScoreAnnotation := parser.GetAnnotationWithPrefix("lua-resty-waf-score")
+	luaRestyWAFAllowUnknownAnnotation := parser.GetAnnotationWithPrefix("lua-resty-waf-allow-unknown-content")
+	luaRestyWAFDisableMultipartBody := parser.GetAnnotationWithPrefix("lua-resty-waf-disable-multipart-body")
 
 	ap := NewParser(&resolver.Mock{})
 	if ap == nil {
@@ -53,11 +56,15 @@ func TestParse(t *testing.T) {
 		{map[string]string{
 			luaRestyWAFAnnotation:                "active",
 			luaRestyWAFDebugAnnotation:           "true",
-			luaRestyWAFIgnoredRuleSetsAnnotation: "ruleset1, ruleset2 ruleset3,   another.ruleset"},
-			&Config{Mode: "ACTIVE", Debug: true, IgnoredRuleSets: []string{"ruleset1", "ruleset2", "ruleset3", "another.ruleset"}}},
+			luaRestyWAFIgnoredRuleSetsAnnotation: "ruleset1, ruleset2 ruleset3,   another.ruleset",
+			luaRestyWAFScoreAnnotation:           "10",
+			luaRestyWAFAllowUnknownAnnotation:    "true"},
+			&Config{Mode: "ACTIVE", Debug: true, IgnoredRuleSets: []string{"ruleset1", "ruleset2", "ruleset3", "another.ruleset"}, Score: 10, AllowUnknownContent: true}},
 
 		{map[string]string{luaRestyWAFAnnotation: "siMulate", luaRestyWAFDebugAnnotation: "true"}, &Config{Mode: "SIMULATE", Debug: true, IgnoredRuleSets: []string{}}},
 		{map[string]string{luaRestyWAFAnnotation: "siMulateX", luaRestyWAFDebugAnnotation: "true"}, &Config{Debug: false}},
+
+		{map[string]string{luaRestyWAFAnnotation: "active", luaRestyWAFDisableMultipartBody: "false"}, &Config{Mode: "ACTIVE", DisableMultipartBody: false, IgnoredRuleSets: []string{}}},
 	}
 
 	ing := &extensions.Ingress{
diff --git a/rootfs/etc/nginx/template/nginx.tmpl b/rootfs/etc/nginx/template/nginx.tmpl
index 22db89b4d..16ebf9bb4 100644
--- a/rootfs/etc/nginx/template/nginx.tmpl
+++ b/rootfs/etc/nginx/template/nginx.tmpl
@@ -891,9 +891,23 @@ stream {
 
                 waf:set_option("mode", "{{ $location.LuaRestyWAF.Mode }}")
                 waf:set_option("storage_zone", "waf_storage")
+
+                {{ if $location.LuaRestyWAF.AllowUnknownContent }} 
+                waf:set_option("allow_unknown_content_types", true)
+                {{ else }}
                 waf:set_option("allowed_content_types", { "text/html", "text/json", "application/json" })
+                {{ end }}
+
                 waf:set_option("event_log_level", ngx.WARN)
 
+                {{ if gt $location.LuaRestyWAF.Score 0 }}
+                waf:set_option("score_threshold", {{ $location.LuaRestyWAF.Score }})
+                {{ end }}
+
+                {{ if $location.LuaRestyWAF.DisableMultipartBody }}
+                waf:set_option("process_multipart_body", false)
+                {{ end }}
+
                 {{ if $location.LuaRestyWAF.Debug }}
                 waf:set_option("debug", true)
                 waf:set_option("event_log_request_arguments", true)
diff --git a/test/e2e/annotations/luarestywaf.go b/test/e2e/annotations/luarestywaf.go
index ed332db04..e96727419 100644
--- a/test/e2e/annotations/luarestywaf.go
+++ b/test/e2e/annotations/luarestywaf.go
@@ -65,6 +65,54 @@ var _ = framework.IngressNginxDescribe("Annotations - lua-resty-waf", func() {
 			Expect(len(errs)).Should(Equal(0))
 			Expect(resp.StatusCode).Should(Equal(http.StatusOK))
 		})
+		It("should apply the score threshold", func() {
+			host := "foo"
+			createIngress(f, host, "http-svc", 80, map[string]string{
+				"nginx.ingress.kubernetes.io/lua-resty-waf":       "active",
+				"nginx.ingress.kubernetes.io/lua-resty-waf-score": "20"})
+
+			url := fmt.Sprintf("%s?msg=<A href=\"http://mysite.com/\">XSS</A>", f.IngressController.HTTPURL)
+			resp, _, errs := gorequest.New().
+				Get(url).
+				Set("Host", host).
+				End()
+
+			Expect(len(errs)).Should(Equal(0))
+			Expect(resp.StatusCode).Should(Equal(http.StatusOK))
+		})
+		It("should reject the invaild content", func() {
+			host := "foo"
+			contenttype := "application/octet-stream"
+			createIngress(f, host, "http-svc", 80, map[string]string{
+				"nginx.ingress.kubernetes.io/lua-resty-waf": "active"})
+
+			url := fmt.Sprintf("%s?msg=my-message", f.IngressController.HTTPURL)
+			resp, _, errs := gorequest.New().
+				Get(url).
+				Set("Host", host).
+				Set("Content-Type", contenttype).
+				End()
+
+			Expect(len(errs)).Should(Equal(0))
+			Expect(resp.StatusCode).Should(Equal(http.StatusForbidden))
+		})
+		It("should allow the multipart content type", func() {
+			host := "foo"
+			contenttype := "multipart/form-data; boundary=alamofire.boundary.3fc2e849279e18fc"
+			createIngress(f, host, "http-svc", 80, map[string]string{
+				"nginx.ingress.kubernetes.io/lua-resty-waf-disable-multipart-body": "true",
+				"nginx.ingress.kubernetes.io/lua-resty-waf":                        "active"})
+
+			url := fmt.Sprintf("%s?msg=my-message", f.IngressController.HTTPURL)
+			resp, _, errs := gorequest.New().
+				Get(url).
+				Set("Host", host).
+				Set("Content-Type", contenttype).
+				End()
+
+			Expect(len(errs)).Should(Equal(0))
+			Expect(resp.StatusCode).Should(Equal(http.StatusOK))
+		})
 		It("should apply configured extra rules", func() {
 			host := "foo"
 			createIngress(f, host, "http-svc", 80, map[string]string{

From bab521e81abc03252d4cdde6118aec7bf9e103c6 Mon Sep 17 00:00:00 2001
From: Desmond Ho <desmond.ho@lalamove.com>
Date: Sat, 20 Oct 2018 12:06:05 +0800
Subject: [PATCH 02/31] UPT: align waf options

---
 .../nginx-configuration/annotations.md        | 26 +++---------
 .../ingress/annotations/luarestywaf/main.go   | 40 +++++++++----------
 .../annotations/luarestywaf/main_test.go      | 20 +++++-----
 rootfs/etc/nginx/template/nginx.tmpl          | 10 +++--
 test/e2e/annotations/luarestywaf.go           | 11 ++---
 5 files changed, 48 insertions(+), 59 deletions(-)

diff --git a/docs/user-guide/nginx-configuration/annotations.md b/docs/user-guide/nginx-configuration/annotations.md
index 6a61d21be..82ed90a8f 100644
--- a/docs/user-guide/nginx-configuration/annotations.md
+++ b/docs/user-guide/nginx-configuration/annotations.md
@@ -80,9 +80,9 @@ You can add these Kubernetes annotations to specific Ingress objects to customiz
 |[nginx.ingress.kubernetes.io/lua-resty-waf-debug](#lua-resty-waf)|"true" or "false"|
 |[nginx.ingress.kubernetes.io/lua-resty-waf-ignore-rulesets](#lua-resty-waf)|string|
 |[nginx.ingress.kubernetes.io/lua-resty-waf-extra-rules](#lua-resty-waf)|string|
-|[nginx.ingress.kubernetes.io/lua-resty-waf-allow-unknown-content](#lua-resty-waf)|"true" or "false"|
-|[nginx.ingress.kubernetes.io/lua-resty-waf-score](#lua-resty-waf)|number|
-|[nginx.ingress.kubernetes.io/lua-resty-waf-disable-multipart-body](#lua-resty-waf)|"true" or "false"|
+|[nginx.ingress.kubernetes.io/lua-resty-waf-allow-unknown-content-types](#lua-resty-waf)|"true" or "false"|
+|[nginx.ingress.kubernetes.io/lua-resty-waf-score-threshold](#lua-resty-waf)|number|
+|[nginx.ingress.kubernetes.io/lua-resty-waf-process-multipart-body](#lua-resty-waf)|"true" or "false"|
 |[nginx.ingress.kubernetes.io/enable-influxdb](#influxdb)|"true" or "false"|
 |[nginx.ingress.kubernetes.io/influxdb-measurement](#influxdb)|string|
 |[nginx.ingress.kubernetes.io/influxdb-port](#influxdb)|string|
@@ -566,14 +566,14 @@ We can enable the following annotation for allow all contents type:
 
 
 ```yaml
-nginx.ingress.kubernetes.io/lua-resty-waf-allow-unknown-content: "true"
+nginx.ingress.kubernetes.io/lua-resty-waf-allow-unknown-content-types: "true"
 ```
 
 The default score of lua-resty-waf is 5, which usually triggered if hitting 2 default rules, you can modify the score threshold with following annotation:
 
 
 ```yaml
-nginx.ingress.kubernetes.io/lua-resty-waf-score: "10"
+nginx.ingress.kubernetes.io/lua-resty-waf-score-threshold: "10"
 ```
 
 When you enabled HTTPS in the endpoint and since resty-lua will return 500 error when processing "multipart" contents
@@ -581,21 +581,7 @@ Reference for this [issue](https://github.com/p0pr0ck5/lua-resty-waf/issues/166)
 You may enable the following annotation for work around:
 
 ```yaml
-nginx.ingress.kubernetes.io/lua-resty-waf-disable-multipart-body: "true"
-```
-
-For details on how to write WAF rules, please refer to [https://github.com/p0pr0ck5/lua-resty-waf](https://github.com/p0pr0ck5/lua-resty-waf).
-
-
-```yaml
-nginx.ingress.kubernetes.io/lua-resty-waf-allow-unknown-content: "true"
-```
-
-The default score of lua-resty-waf is 5, which usually triggered if hitting 2 default rules, you can modify the score threshold with following annotation:
-
-
-```yaml
-nginx.ingress.kubernetes.io/lua-resty-waf-score: "10"
+nginx.ingress.kubernetes.io/lua-resty-waf-process-multipart-body: "false"
 ```
 
 For details on how to write WAF rules, please refer to [https://github.com/p0pr0ck5/lua-resty-waf](https://github.com/p0pr0ck5/lua-resty-waf).
diff --git a/internal/ingress/annotations/luarestywaf/main.go b/internal/ingress/annotations/luarestywaf/main.go
index 4873851ab..ab0207003 100644
--- a/internal/ingress/annotations/luarestywaf/main.go
+++ b/internal/ingress/annotations/luarestywaf/main.go
@@ -31,13 +31,13 @@ var luaRestyWAFModes = map[string]bool{"ACTIVE": true, "INACTIVE": true, "SIMULA
 
 // Config returns lua-resty-waf configuration for an Ingress rule
 type Config struct {
-	Mode                 string   `json:"mode"`
-	Debug                bool     `json:"debug"`
-	IgnoredRuleSets      []string `json:"ignored-rulesets"`
-	ExtraRulesetString   string   `json:"extra-ruleset-string"`
-	Score                int      `json:"score"`
-	AllowUnknownContent  bool     `json:"allow-unknown-content"`
-	DisableMultipartBody bool     `json:"disable-multipart-body"`
+	Mode                     string   `json:"mode"`
+	Debug                    bool     `json:"debug"`
+	IgnoredRuleSets          []string `json:"ignored-rulesets"`
+	ExtraRulesetString       string   `json:"extra-ruleset-string"`
+	ScoreThreshold           int      `json:"score-threshold"`
+	AllowUnknownContentTypes bool     `json:"allow-unknown-content-types"`
+	ProcessMultipartBody     bool     `json:"process-multipart-body"`
 }
 
 // Equal tests for equality between two Config types
@@ -60,13 +60,13 @@ func (e1 *Config) Equal(e2 *Config) bool {
 	if e1.ExtraRulesetString != e2.ExtraRulesetString {
 		return false
 	}
-	if e1.Score != e2.Score {
+	if e1.ScoreThreshold != e2.ScoreThreshold {
 		return false
 	}
-	if e1.AllowUnknownContent != e2.AllowUnknownContent {
+	if e1.AllowUnknownContentTypes != e2.AllowUnknownContentTypes {
 		return false
 	}
-	if e1.DisableMultipartBody != e2.DisableMultipartBody {
+	if e1.ProcessMultipartBody != e2.ProcessMultipartBody {
 		return false
 	}
 
@@ -107,19 +107,19 @@ func (a luarestywaf) Parse(ing *extensions.Ingress) (interface{}, error) {
 	// TODO(elvinefendi) maybe validate the ruleset string here
 	extraRulesetString, _ := parser.GetStringAnnotation("lua-resty-waf-extra-rules", ing)
 
-	score, _ := parser.GetIntAnnotation("lua-resty-waf-score", ing)
+	scoreThreshold, _ := parser.GetIntAnnotation("lua-resty-waf-score-threshold", ing)
 
-	allowUnknownContent, _ := parser.GetBoolAnnotation("lua-resty-waf-allow-unknown-content", ing)
+	allowUnknownContentTypes, _ := parser.GetBoolAnnotation("lua-resty-waf-allow-unknown-content-types", ing)
 
-	disableMultipartBody, _ := parser.GetBoolAnnotation("lua-resty-waf-disable-multipart-body", ing)
+	processMultipartBody, _ := parser.GetBoolAnnotation("lua-resty-waf-process-multipart-body", ing)
 
 	return &Config{
-		Mode:                 mode,
-		Debug:                debug,
-		IgnoredRuleSets:      ignoredRuleSets,
-		ExtraRulesetString:   extraRulesetString,
-		Score:                score,
-		AllowUnknownContent:  allowUnknownContent,
-		DisableMultipartBody: disableMultipartBody,
+		Mode:                     mode,
+		Debug:                    debug,
+		IgnoredRuleSets:          ignoredRuleSets,
+		ExtraRulesetString:       extraRulesetString,
+		ScoreThreshold:           scoreThreshold,
+		AllowUnknownContentTypes: allowUnknownContentTypes,
+		ProcessMultipartBody:     processMultipartBody,
 	}, nil
 }
diff --git a/internal/ingress/annotations/luarestywaf/main_test.go b/internal/ingress/annotations/luarestywaf/main_test.go
index e9b50c53f..f6151378d 100644
--- a/internal/ingress/annotations/luarestywaf/main_test.go
+++ b/internal/ingress/annotations/luarestywaf/main_test.go
@@ -30,9 +30,9 @@ func TestParse(t *testing.T) {
 	luaRestyWAFAnnotation := parser.GetAnnotationWithPrefix("lua-resty-waf")
 	luaRestyWAFDebugAnnotation := parser.GetAnnotationWithPrefix("lua-resty-waf-debug")
 	luaRestyWAFIgnoredRuleSetsAnnotation := parser.GetAnnotationWithPrefix("lua-resty-waf-ignore-rulesets")
-	luaRestyWAFScoreAnnotation := parser.GetAnnotationWithPrefix("lua-resty-waf-score")
-	luaRestyWAFAllowUnknownAnnotation := parser.GetAnnotationWithPrefix("lua-resty-waf-allow-unknown-content")
-	luaRestyWAFDisableMultipartBody := parser.GetAnnotationWithPrefix("lua-resty-waf-disable-multipart-body")
+	luaRestyWAFScoreThresholdAnnotation := parser.GetAnnotationWithPrefix("lua-resty-waf-score-threshold")
+	luaRestyWAFAllowUnknownContentTypesAnnotation := parser.GetAnnotationWithPrefix("lua-resty-waf-allow-unknown-content-types")
+	luaRestyWAFProcessMultipartBody := parser.GetAnnotationWithPrefix("lua-resty-waf-process-multipart-body")
 
 	ap := NewParser(&resolver.Mock{})
 	if ap == nil {
@@ -54,17 +54,17 @@ func TestParse(t *testing.T) {
 		{map[string]string{luaRestyWAFAnnotation: "inactive", luaRestyWAFDebugAnnotation: "true"}, &Config{Mode: "INACTIVE", Debug: true, IgnoredRuleSets: []string{}}},
 
 		{map[string]string{
-			luaRestyWAFAnnotation:                "active",
-			luaRestyWAFDebugAnnotation:           "true",
-			luaRestyWAFIgnoredRuleSetsAnnotation: "ruleset1, ruleset2 ruleset3,   another.ruleset",
-			luaRestyWAFScoreAnnotation:           "10",
-			luaRestyWAFAllowUnknownAnnotation:    "true"},
-			&Config{Mode: "ACTIVE", Debug: true, IgnoredRuleSets: []string{"ruleset1", "ruleset2", "ruleset3", "another.ruleset"}, Score: 10, AllowUnknownContent: true}},
+			luaRestyWAFAnnotation:                         "active",
+			luaRestyWAFDebugAnnotation:                    "true",
+			luaRestyWAFIgnoredRuleSetsAnnotation:          "ruleset1, ruleset2 ruleset3,   another.ruleset",
+			luaRestyWAFScoreThresholdAnnotation:           "10",
+			luaRestyWAFAllowUnknownContentTypesAnnotation: "true"},
+			&Config{Mode: "ACTIVE", Debug: true, IgnoredRuleSets: []string{"ruleset1", "ruleset2", "ruleset3", "another.ruleset"}, ScoreThreshold: 10, AllowUnknownContentTypes: true}},
 
 		{map[string]string{luaRestyWAFAnnotation: "siMulate", luaRestyWAFDebugAnnotation: "true"}, &Config{Mode: "SIMULATE", Debug: true, IgnoredRuleSets: []string{}}},
 		{map[string]string{luaRestyWAFAnnotation: "siMulateX", luaRestyWAFDebugAnnotation: "true"}, &Config{Debug: false}},
 
-		{map[string]string{luaRestyWAFAnnotation: "active", luaRestyWAFDisableMultipartBody: "false"}, &Config{Mode: "ACTIVE", DisableMultipartBody: false, IgnoredRuleSets: []string{}}},
+		{map[string]string{luaRestyWAFAnnotation: "active", luaRestyWAFProcessMultipartBody: "false"}, &Config{Mode: "ACTIVE", ProcessMultipartBody: false, IgnoredRuleSets: []string{}}},
 	}
 
 	ing := &extensions.Ingress{
diff --git a/rootfs/etc/nginx/template/nginx.tmpl b/rootfs/etc/nginx/template/nginx.tmpl
index 16ebf9bb4..e17305c64 100644
--- a/rootfs/etc/nginx/template/nginx.tmpl
+++ b/rootfs/etc/nginx/template/nginx.tmpl
@@ -892,7 +892,7 @@ stream {
                 waf:set_option("mode", "{{ $location.LuaRestyWAF.Mode }}")
                 waf:set_option("storage_zone", "waf_storage")
 
-                {{ if $location.LuaRestyWAF.AllowUnknownContent }} 
+                {{ if $location.LuaRestyWAF.AllowUnknownContentTypes }} 
                 waf:set_option("allow_unknown_content_types", true)
                 {{ else }}
                 waf:set_option("allowed_content_types", { "text/html", "text/json", "application/json" })
@@ -900,12 +900,14 @@ stream {
 
                 waf:set_option("event_log_level", ngx.WARN)
 
-                {{ if gt $location.LuaRestyWAF.Score 0 }}
-                waf:set_option("score_threshold", {{ $location.LuaRestyWAF.Score }})
+                {{ if gt $location.LuaRestyWAF.ScoreThreshold 0 }}
+                waf:set_option("score_threshold", {{ $location.LuaRestyWAF.ScoreThreshold }})
                 {{ end }}
 
-                {{ if $location.LuaRestyWAF.DisableMultipartBody }}
+                {{ if not $location.LuaRestyWAF.ProcessMultipartBody }}
                 waf:set_option("process_multipart_body", false)
+                {{ else }}
+                waf:set_option("process_multipart_body", true)
                 {{ end }}
 
                 {{ if $location.LuaRestyWAF.Debug }}
diff --git a/test/e2e/annotations/luarestywaf.go b/test/e2e/annotations/luarestywaf.go
index e96727419..d097be7c0 100644
--- a/test/e2e/annotations/luarestywaf.go
+++ b/test/e2e/annotations/luarestywaf.go
@@ -68,8 +68,8 @@ var _ = framework.IngressNginxDescribe("Annotations - lua-resty-waf", func() {
 		It("should apply the score threshold", func() {
 			host := "foo"
 			createIngress(f, host, "http-svc", 80, map[string]string{
-				"nginx.ingress.kubernetes.io/lua-resty-waf":       "active",
-				"nginx.ingress.kubernetes.io/lua-resty-waf-score": "20"})
+				"nginx.ingress.kubernetes.io/lua-resty-waf":                 "active",
+				"nginx.ingress.kubernetes.io/lua-resty-waf-score-threshold": "20"})
 
 			url := fmt.Sprintf("%s?msg=<A href=\"http://mysite.com/\">XSS</A>", f.IngressController.HTTPURL)
 			resp, _, errs := gorequest.New().
@@ -84,7 +84,8 @@ var _ = framework.IngressNginxDescribe("Annotations - lua-resty-waf", func() {
 			host := "foo"
 			contenttype := "application/octet-stream"
 			createIngress(f, host, "http-svc", 80, map[string]string{
-				"nginx.ingress.kubernetes.io/lua-resty-waf": "active"})
+				"nginx.ingress.kubernetes.io/lua-resty-waf-allow-unknown-content-types": "true",
+				"nginx.ingress.kubernetes.io/lua-resty-waf":                             "active"})
 
 			url := fmt.Sprintf("%s?msg=my-message", f.IngressController.HTTPURL)
 			resp, _, errs := gorequest.New().
@@ -94,13 +95,13 @@ var _ = framework.IngressNginxDescribe("Annotations - lua-resty-waf", func() {
 				End()
 
 			Expect(len(errs)).Should(Equal(0))
-			Expect(resp.StatusCode).Should(Equal(http.StatusForbidden))
+			Expect(resp.StatusCode).Should(Equal(http.StatusOK))
 		})
 		It("should allow the multipart content type", func() {
 			host := "foo"
 			contenttype := "multipart/form-data; boundary=alamofire.boundary.3fc2e849279e18fc"
 			createIngress(f, host, "http-svc", 80, map[string]string{
-				"nginx.ingress.kubernetes.io/lua-resty-waf-disable-multipart-body": "true",
+				"nginx.ingress.kubernetes.io/lua-resty-waf-process-multipart-body": "false",
 				"nginx.ingress.kubernetes.io/lua-resty-waf":                        "active"})
 
 			url := fmt.Sprintf("%s?msg=my-message", f.IngressController.HTTPURL)

From 3c2c0d085872aac1e36168fb5f8a7b50643ab113 Mon Sep 17 00:00:00 2001
From: Desmond Ho <desmond.ho@lalamove.com>
Date: Wed, 24 Oct 2018 18:30:43 +0800
Subject: [PATCH 03/31] UPT: updated e2e testing title for lua test

---
 test/e2e/annotations/luarestywaf.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/e2e/annotations/luarestywaf.go b/test/e2e/annotations/luarestywaf.go
index d097be7c0..a3585d304 100644
--- a/test/e2e/annotations/luarestywaf.go
+++ b/test/e2e/annotations/luarestywaf.go
@@ -80,7 +80,7 @@ var _ = framework.IngressNginxDescribe("Annotations - lua-resty-waf", func() {
 			Expect(len(errs)).Should(Equal(0))
 			Expect(resp.StatusCode).Should(Equal(http.StatusOK))
 		})
-		It("should reject the invaild content", func() {
+		It("should not reject request with an unknown content type", func() {
 			host := "foo"
 			contenttype := "application/octet-stream"
 			createIngress(f, host, "http-svc", 80, map[string]string{

From 9f2a0b63631df24ebff2a29c3ef99f8548e61f24 Mon Sep 17 00:00:00 2001
From: Zenara Daley <zenara.daley@shopify.com>
Date: Wed, 24 Oct 2018 16:02:28 -0400
Subject: [PATCH 04/31] Add healthcheck timeout as CLA

---
 cmd/nginx/flags.go                        | 3 +++
 internal/ingress/controller/checker.go    | 9 +++++----
 internal/ingress/controller/controller.go | 1 +
 3 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/cmd/nginx/flags.go b/cmd/nginx/flags.go
index f26947e52..25633ef54 100644
--- a/cmd/nginx/flags.go
+++ b/cmd/nginx/flags.go
@@ -84,6 +84,8 @@ Takes the form "namespace/name".`)
 Configured inside the NGINX status server. All requests received on the port
 defined by the healthz-port parameter are forwarded internally to this path.`)
 
+		healthCheckTimeout = flags.Duration("health-check-timeout", 10, `Time limit, in seconds, for a probe to health-check-path to succeed.`)
+
 		updateStatus = flags.Bool("update-status", true,
 			`Update the load-balancer status of Ingress objects this controller satisfies.
 Requires setting the publish-service parameter to a valid Service reference.`)
@@ -217,6 +219,7 @@ Feature backed by OpenResty Lua libraries. Requires that OCSP stapling is not en
 		ConfigMapName:              *configMap,
 		DefaultSSLCertificate:      *defSSLCertificate,
 		DefaultHealthzURL:          *defHealthzURL,
+		HealthCheckTimeout:         *healthCheckTimeout,
 		PublishService:             *publishSvc,
 		PublishStatusAddress:       *publishStatusAddress,
 		ForceNamespaceIsolation:    *forceIsolation,
diff --git a/internal/ingress/controller/checker.go b/internal/ingress/controller/checker.go
index b595dd532..6dd6b3be0 100644
--- a/internal/ingress/controller/checker.go
+++ b/internal/ingress/controller/checker.go
@@ -38,7 +38,8 @@ func (n NGINXController) Name() string {
 func (n *NGINXController) Check(_ *http.Request) error {
 
 	url := fmt.Sprintf("http://127.0.0.1:%v%v", n.cfg.ListenPorts.Status, ngxHealthPath)
-	statusCode, err := simpleGet(url)
+	timeout := n.cfg.HealthCheckTimeout
+	statusCode, err := simpleGet(url, timeout)
 	if err != nil {
 		return err
 	}
@@ -48,7 +49,7 @@ func (n *NGINXController) Check(_ *http.Request) error {
 	}
 
 	url = fmt.Sprintf("http://127.0.0.1:%v/is-dynamic-lb-initialized", n.cfg.ListenPorts.Status)
-	statusCode, err = simpleGet(url)
+	statusCode, err = simpleGet(url, timeout)
 	if err != nil {
 		return err
 	}
@@ -75,9 +76,9 @@ func (n *NGINXController) Check(_ *http.Request) error {
 	return err
 }
 
-func simpleGet(url string) (int, error) {
+func simpleGet(url string, timeout time.Duration) (int, error) {
 	client := &http.Client{
-		Timeout:   10 * time.Second,
+		Timeout:   timeout * time.Second,
 		Transport: &http.Transport{DisableKeepAlives: true},
 	}
 
diff --git a/internal/ingress/controller/controller.go b/internal/ingress/controller/controller.go
index d070c5e12..770166dda 100644
--- a/internal/ingress/controller/controller.go
+++ b/internal/ingress/controller/controller.go
@@ -61,6 +61,7 @@ type Configuration struct {
 	ForceNamespaceIsolation bool
 
 	DefaultHealthzURL     string
+	HealthCheckTimeout    time.Duration
 	DefaultSSLCertificate string
 
 	// +optional

From bf03046a80e42f968f61e3e7727f4979971826ea Mon Sep 17 00:00:00 2001
From: Desmond Ho <desmond.ho@lalamove.com>
Date: Thu, 25 Oct 2018 10:43:26 +0800
Subject: [PATCH 05/31] UPT: updated e2e test and default true for
 process-multipart-body annotation

---
 .../nginx-configuration/annotations.md        |  3 +++
 .../ingress/annotations/luarestywaf/main.go   |  5 ++++-
 .../annotations/luarestywaf/main_test.go      | 12 +++++------
 rootfs/etc/nginx/template/nginx.tmpl          |  2 --
 test/e2e/annotations/luarestywaf.go           | 20 +++++++++++++++++--
 5 files changed, 31 insertions(+), 11 deletions(-)

diff --git a/docs/user-guide/nginx-configuration/annotations.md b/docs/user-guide/nginx-configuration/annotations.md
index 82ed90a8f..a9e4f9f4e 100644
--- a/docs/user-guide/nginx-configuration/annotations.md
+++ b/docs/user-guide/nginx-configuration/annotations.md
@@ -578,6 +578,9 @@ nginx.ingress.kubernetes.io/lua-resty-waf-score-threshold: "10"
 
 When you enabled HTTPS in the endpoint and since resty-lua will return 500 error when processing "multipart" contents
 Reference for this [issue](https://github.com/p0pr0ck5/lua-resty-waf/issues/166)
+
+By default, it will be "true"
+
 You may enable the following annotation for work around:
 
 ```yaml
diff --git a/internal/ingress/annotations/luarestywaf/main.go b/internal/ingress/annotations/luarestywaf/main.go
index ab0207003..c29a97e9c 100644
--- a/internal/ingress/annotations/luarestywaf/main.go
+++ b/internal/ingress/annotations/luarestywaf/main.go
@@ -111,7 +111,10 @@ func (a luarestywaf) Parse(ing *extensions.Ingress) (interface{}, error) {
 
 	allowUnknownContentTypes, _ := parser.GetBoolAnnotation("lua-resty-waf-allow-unknown-content-types", ing)
 
-	processMultipartBody, _ := parser.GetBoolAnnotation("lua-resty-waf-process-multipart-body", ing)
+	processMultipartBody, err := parser.GetBoolAnnotation("lua-resty-waf-process-multipart-body", ing)
+	if err != nil {
+		processMultipartBody = true
+	}
 
 	return &Config{
 		Mode:                     mode,
diff --git a/internal/ingress/annotations/luarestywaf/main_test.go b/internal/ingress/annotations/luarestywaf/main_test.go
index f6151378d..60b80e22f 100644
--- a/internal/ingress/annotations/luarestywaf/main_test.go
+++ b/internal/ingress/annotations/luarestywaf/main_test.go
@@ -46,12 +46,12 @@ func TestParse(t *testing.T) {
 		{nil, &Config{}},
 		{map[string]string{}, &Config{}},
 
-		{map[string]string{luaRestyWAFAnnotation: "active"}, &Config{Mode: "ACTIVE", Debug: false, IgnoredRuleSets: []string{}}},
+		{map[string]string{luaRestyWAFAnnotation: "active"}, &Config{Mode: "ACTIVE", Debug: false, IgnoredRuleSets: []string{}, ProcessMultipartBody: true}},
 		{map[string]string{luaRestyWAFDebugAnnotation: "true"}, &Config{Debug: false}},
 
-		{map[string]string{luaRestyWAFAnnotation: "active", luaRestyWAFDebugAnnotation: "true"}, &Config{Mode: "ACTIVE", Debug: true, IgnoredRuleSets: []string{}}},
-		{map[string]string{luaRestyWAFAnnotation: "active", luaRestyWAFDebugAnnotation: "false"}, &Config{Mode: "ACTIVE", Debug: false, IgnoredRuleSets: []string{}}},
-		{map[string]string{luaRestyWAFAnnotation: "inactive", luaRestyWAFDebugAnnotation: "true"}, &Config{Mode: "INACTIVE", Debug: true, IgnoredRuleSets: []string{}}},
+		{map[string]string{luaRestyWAFAnnotation: "active", luaRestyWAFDebugAnnotation: "true"}, &Config{Mode: "ACTIVE", Debug: true, IgnoredRuleSets: []string{}, ProcessMultipartBody: true}},
+		{map[string]string{luaRestyWAFAnnotation: "active", luaRestyWAFDebugAnnotation: "false"}, &Config{Mode: "ACTIVE", Debug: false, IgnoredRuleSets: []string{}, ProcessMultipartBody: true}},
+		{map[string]string{luaRestyWAFAnnotation: "inactive", luaRestyWAFDebugAnnotation: "true"}, &Config{Mode: "INACTIVE", Debug: true, IgnoredRuleSets: []string{}, ProcessMultipartBody: true}},
 
 		{map[string]string{
 			luaRestyWAFAnnotation:                         "active",
@@ -59,9 +59,9 @@ func TestParse(t *testing.T) {
 			luaRestyWAFIgnoredRuleSetsAnnotation:          "ruleset1, ruleset2 ruleset3,   another.ruleset",
 			luaRestyWAFScoreThresholdAnnotation:           "10",
 			luaRestyWAFAllowUnknownContentTypesAnnotation: "true"},
-			&Config{Mode: "ACTIVE", Debug: true, IgnoredRuleSets: []string{"ruleset1", "ruleset2", "ruleset3", "another.ruleset"}, ScoreThreshold: 10, AllowUnknownContentTypes: true}},
+			&Config{Mode: "ACTIVE", Debug: true, IgnoredRuleSets: []string{"ruleset1", "ruleset2", "ruleset3", "another.ruleset"}, ScoreThreshold: 10, AllowUnknownContentTypes: true, ProcessMultipartBody: true}},
 
-		{map[string]string{luaRestyWAFAnnotation: "siMulate", luaRestyWAFDebugAnnotation: "true"}, &Config{Mode: "SIMULATE", Debug: true, IgnoredRuleSets: []string{}}},
+		{map[string]string{luaRestyWAFAnnotation: "siMulate", luaRestyWAFDebugAnnotation: "true"}, &Config{Mode: "SIMULATE", Debug: true, IgnoredRuleSets: []string{}, ProcessMultipartBody: true}},
 		{map[string]string{luaRestyWAFAnnotation: "siMulateX", luaRestyWAFDebugAnnotation: "true"}, &Config{Debug: false}},
 
 		{map[string]string{luaRestyWAFAnnotation: "active", luaRestyWAFProcessMultipartBody: "false"}, &Config{Mode: "ACTIVE", ProcessMultipartBody: false, IgnoredRuleSets: []string{}}},
diff --git a/rootfs/etc/nginx/template/nginx.tmpl b/rootfs/etc/nginx/template/nginx.tmpl
index e17305c64..2a7537c8e 100644
--- a/rootfs/etc/nginx/template/nginx.tmpl
+++ b/rootfs/etc/nginx/template/nginx.tmpl
@@ -906,8 +906,6 @@ stream {
 
                 {{ if not $location.LuaRestyWAF.ProcessMultipartBody }}
                 waf:set_option("process_multipart_body", false)
-                {{ else }}
-                waf:set_option("process_multipart_body", true)
                 {{ end }}
 
                 {{ if $location.LuaRestyWAF.Debug }}
diff --git a/test/e2e/annotations/luarestywaf.go b/test/e2e/annotations/luarestywaf.go
index a3585d304..a20fbdc49 100644
--- a/test/e2e/annotations/luarestywaf.go
+++ b/test/e2e/annotations/luarestywaf.go
@@ -97,9 +97,9 @@ var _ = framework.IngressNginxDescribe("Annotations - lua-resty-waf", func() {
 			Expect(len(errs)).Should(Equal(0))
 			Expect(resp.StatusCode).Should(Equal(http.StatusOK))
 		})
-		It("should allow the multipart content type", func() {
-			host := "foo"
+		It("should not fail a request with multipart content type when multipart body processing disabled", func() {
 			contenttype := "multipart/form-data; boundary=alamofire.boundary.3fc2e849279e18fc"
+			host := "foo"
 			createIngress(f, host, "http-svc", 80, map[string]string{
 				"nginx.ingress.kubernetes.io/lua-resty-waf-process-multipart-body": "false",
 				"nginx.ingress.kubernetes.io/lua-resty-waf":                        "active"})
@@ -114,6 +114,22 @@ var _ = framework.IngressNginxDescribe("Annotations - lua-resty-waf", func() {
 			Expect(len(errs)).Should(Equal(0))
 			Expect(resp.StatusCode).Should(Equal(http.StatusOK))
 		})
+		It("should fail a request with multipart content type when multipart body processing enabled by default", func() {
+			contenttype := "multipart/form-data; boundary=alamofire.boundary.3fc2e849279e18fc"
+			host := "foo"
+			createIngress(f, host, "http-svc", 80, map[string]string{
+				"nginx.ingress.kubernetes.io/lua-resty-waf": "active"})
+
+			url := fmt.Sprintf("%s?msg=my-message", f.IngressController.HTTPURL)
+			resp, _, errs := gorequest.New().
+				Get(url).
+				Set("Host", host).
+				Set("Content-Type", contenttype).
+				End()
+
+			Expect(len(errs)).Should(Equal(0))
+			Expect(resp.StatusCode).Should(Equal(http.StatusBadRequest))
+		})
 		It("should apply configured extra rules", func() {
 			host := "foo"
 			createIngress(f, host, "http-svc", 80, map[string]string{

From 3cbfd639923c8698e6fcb29f9728b171f1211e6d Mon Sep 17 00:00:00 2001
From: Henry Tran <henry.tran@shopify.com>
Date: Mon, 30 Jul 2018 15:55:47 -0400
Subject: [PATCH 06/31] Refactor EWMA to not use shared dictionaries

---
 .../ingress/controller/template/template.go   |  2 -
 rootfs/etc/nginx/lua/balancer/ewma.lua        | 59 +++----------------
 .../etc/nginx/lua/test/balancer/ewma_test.lua | 16 -----
 3 files changed, 8 insertions(+), 69 deletions(-)

diff --git a/internal/ingress/controller/template/template.go b/internal/ingress/controller/template/template.go
index e6df56282..4dd3461e0 100644
--- a/internal/ingress/controller/template/template.go
+++ b/internal/ingress/controller/template/template.go
@@ -209,8 +209,6 @@ func buildLuaSharedDictionaries(s interface{}, disableLuaRestyWAF bool) string {
 		"lua_shared_dict configuration_data 5M",
 		"lua_shared_dict certificate_data 16M",
 		"lua_shared_dict locks 512k",
-		"lua_shared_dict balancer_ewma 1M",
-		"lua_shared_dict balancer_ewma_last_touched_at 1M",
 		"lua_shared_dict sticky_sessions 1M",
 	}
 
diff --git a/rootfs/etc/nginx/lua/balancer/ewma.lua b/rootfs/etc/nginx/lua/balancer/ewma.lua
index 270e990ca..352b654f7 100644
--- a/rootfs/etc/nginx/lua/balancer/ewma.lua
+++ b/rootfs/etc/nginx/lua/balancer/ewma.lua
@@ -5,38 +5,17 @@
 --   /finagle-core/src/main/scala/com/twitter/finagle/loadbalancer/PeakEwma.scala
 
 
-local resty_lock = require("resty.lock")
 local util = require("util")
 local split = require("util.split")
 
 local DECAY_TIME = 10 -- this value is in seconds
-local LOCK_KEY = ":ewma_key"
 local PICK_SET_SIZE = 2
 
-local ewma_lock = resty_lock:new("locks", {timeout = 0, exptime = 0.1})
+local balancer_ewma = {}
+local balancer_ewma_last_touched_at = {}
 
 local _M = { name = "ewma" }
 
-local function lock(upstream)
-  local _, err = ewma_lock:lock(upstream .. LOCK_KEY)
-  if err then
-    if err ~= "timeout" then
-      ngx.log(ngx.ERR, string.format("EWMA Balancer failed to lock: %s", tostring(err)))
-    end
-  end
-
-  return err
-end
-
-local function unlock()
-  local ok, err = ewma_lock:unlock()
-  if not ok then
-    ngx.log(ngx.ERR, string.format("EWMA Balancer failed to unlock: %s", tostring(err)))
-  end
-
-  return err
-end
-
 local function decay_ewma(ewma, last_touched_at, rtt, now)
   local td = now - last_touched_at
   td = (td > 0) and td or 0
@@ -47,40 +26,18 @@ local function decay_ewma(ewma, last_touched_at, rtt, now)
 end
 
 local function get_or_update_ewma(upstream, rtt, update)
-  local lock_err = nil
-  if update then
-    lock_err = lock(upstream)
-  end
-  local ewma = ngx.shared.balancer_ewma:get(upstream) or 0
-  if lock_err ~= nil then
-    return ewma, lock_err
-  end
+  local ewma = balancer_ewma[upstream] or 0
 
   local now = ngx.now()
-  local last_touched_at = ngx.shared.balancer_ewma_last_touched_at:get(upstream) or 0
+  local last_touched_at = balancer_ewma_last_touched_at[upstream] or 0
   ewma = decay_ewma(ewma, last_touched_at, rtt, now)
 
   if not update then
     return ewma, nil
   end
 
-  local success, err, forcible = ngx.shared.balancer_ewma_last_touched_at:set(upstream, now)
-  if not success then
-    ngx.log(ngx.WARN, "balancer_ewma_last_touched_at:set failed " .. err)
-  end
-  if forcible then
-    ngx.log(ngx.WARN, "balancer_ewma_last_touched_at:set valid items forcibly overwritten")
-  end
-
-  success, err, forcible = ngx.shared.balancer_ewma:set(upstream, ewma)
-  if not success then
-    ngx.log(ngx.WARN, "balancer_ewma:set failed " .. err)
-  end
-  if forcible then
-    ngx.log(ngx.WARN, "balancer_ewma:set valid items forcibly overwritten")
-  end
-
-  unlock()
+  balancer_ewma_last_touched_at[upstream] = now
+  balancer_ewma[upstream] = ewma
   return ewma, nil
 end
 
@@ -153,8 +110,8 @@ function _M.sync(self, backend)
   self.peers = backend.endpoints
 
   -- TODO: Reset state of EWMA per backend
-  ngx.shared.balancer_ewma:flush_all()
-  ngx.shared.balancer_ewma_last_touched_at:flush_all()
+  balancer_ewma = {}
+  balancer_ewma_last_touched_at = {}
 end
 
 function _M.new(self, backend)
diff --git a/rootfs/etc/nginx/lua/test/balancer/ewma_test.lua b/rootfs/etc/nginx/lua/test/balancer/ewma_test.lua
index 920df1a92..d7cb8cab6 100644
--- a/rootfs/etc/nginx/lua/test/balancer/ewma_test.lua
+++ b/rootfs/etc/nginx/lua/test/balancer/ewma_test.lua
@@ -26,10 +26,6 @@ describe("Balancer ewma", function()
       local instance = balancer_ewma:new(backend)
       
       local stats = { ["10.184.7.40:8080"] = 0.5, ["10.184.97.100:8080"] = 0.3 }
-      ngx.shared.balancer_ewma.get = function(self, key) return stats[key] end
-      local t = ngx.now()-10
-      ngx.shared.balancer_ewma_last_touched_at.get = function(self, key) return t end
-
 
       local peer = instance:balance()
       assert.equal("10.184.97.100:8080", peer)
@@ -52,13 +48,7 @@ describe("Balancer ewma", function()
         endpoints = { { address = "10.184.7.40", port = "8080", maxFails = 0, failTimeout = 0 } }
       }
 
-      local s1 = spy.on(ngx.shared.balancer_ewma, "flush_all")
-      local s2 = spy.on(ngx.shared.balancer_ewma_last_touched_at, "flush_all")
-
       instance:sync(new_backend)
-
-      assert.spy(s1).was_not_called()
-      assert.spy(s2).was_not_called()
     end)
 
     it("updates endpoints", function()
@@ -77,13 +67,7 @@ describe("Balancer ewma", function()
       local new_backend = util.deepcopy(backend)
       new_backend.endpoints[1].maxFails = 3
 
-      local s1 = spy.on(ngx.shared.balancer_ewma, "flush_all")
-      local s2 = spy.on(ngx.shared.balancer_ewma_last_touched_at, "flush_all")
-
       instance:sync(new_backend)
-
-      assert.spy(s1).was_called()
-      assert.spy(s2).was_called()
     end)
   end)
 end)

From abeb17619871572584f05c5c042470ac22076cc4 Mon Sep 17 00:00:00 2001
From: Manuel Alejandro de Brito Fontes <aledbf@gmail.com>
Date: Fri, 26 Oct 2018 08:52:36 -0300
Subject: [PATCH 07/31] Update kubeadm-dind-cluster (#3304)

---
 test/e2e/dind-cluster-v1.11.sh | 2347 --------------------------------
 test/e2e/up.sh                 |    5 +-
 2 files changed, 4 insertions(+), 2348 deletions(-)
 delete mode 100755 test/e2e/dind-cluster-v1.11.sh

diff --git a/test/e2e/dind-cluster-v1.11.sh b/test/e2e/dind-cluster-v1.11.sh
deleted file mode 100755
index e642839cc..000000000
--- a/test/e2e/dind-cluster-v1.11.sh
+++ /dev/null
@@ -1,2347 +0,0 @@
-#!/bin/bash
-# Copyright 2017 Mirantis
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-set -o errexit
-set -o nounset
-set -o pipefail
-set -o errtrace
-
-if [ $(uname) = Darwin ]; then
-  readlinkf(){ perl -MCwd -e 'print Cwd::abs_path shift' "$1";}
-else
-  readlinkf(){ readlink -f "$1"; }
-fi
-DIND_ROOT="$(cd $(dirname "$(readlinkf "${BASH_SOURCE}")"); pwd)"
-
-RUN_ON_BTRFS_ANYWAY="${RUN_ON_BTRFS_ANYWAY:-}"
-if [[ ! ${RUN_ON_BTRFS_ANYWAY} ]] && docker info| grep -q '^Storage Driver: btrfs'; then
-  echo "ERROR: Docker is using btrfs storage driver which is unsupported by kubeadm-dind-cluster" >&2
-  echo "Please refer to the documentation for more info." >&2
-  echo "Set RUN_ON_BTRFS_ANYWAY to non-empty string to continue anyway." >&2
-  exit 1
-fi
-
-# In case of linuxkit / moby linux, -v will not work so we can't
-# mount /lib/modules and /boot. Also we'll be using localhost
-# to access the apiserver.
-using_linuxkit=
-if ! docker info|grep -s '^Operating System: .*Docker for Windows' > /dev/null 2>&1 ; then
-    if docker info|grep -s '^Kernel Version: .*-moby$' >/dev/null 2>&1 ||
-         docker info|grep -s '^Kernel Version: .*-linuxkit-' > /dev/null 2>&1 ; then
-        using_linuxkit=1
-    fi
-fi
-
-# Determine when using Linux and docker daemon running locally
-using_local_linuxdocker=
-if [[ $(uname) == Linux && -z ${DOCKER_HOST:-} ]]; then
-    using_local_linuxdocker=1
-fi
-
-EMBEDDED_CONFIG=y;DIND_IMAGE=mirantis/kubeadm-dind-cluster:v1.11
-
-# dind::localhost provides the local host IP based on the address family used for service subnet.
-function dind::localhost() {
-  if [[ ${SERVICE_NET_MODE} = "ipv6" ]]; then
-    echo '[::1]'
-  else
-    echo '127.0.0.1'
-  fi
-}
-
-# dind::family-for indicates whether the CIDR or IP is for an IPv6 or IPv4 family.
-function dind::family-for {
-    local addr=$1
-    if [[ "$addr" = *":"* ]]; then
-	echo "ipv6"
-    else
-	echo "ipv4"
-    fi
-}
-
-# dind::cluster-suffix builds a suffix used for resources, based on the DIND_LABEL.
-function dind::cluster-suffix {
-  if [ "$DIND_LABEL" != "$DEFAULT_DIND_LABEL" ]; then
-    echo "-${DIND_LABEL}"
-  else
-    echo ''
-  fi
-}
-
-function dind::net-name {
-  echo "kubeadm-dind-net$( dind::cluster-suffix )"
-}
-
-# dind::add-cluster will inject the cluster ID to the IP address. For IPv4, it is
-# assumed that the IP is a /24 with the third part of the address available for cluster ID.
-# For IPv6, it is assumed that there is enough space for the cluster to be added, and the
-# cluster ID will be added to the 16 bits before the double colon. For example:
-#
-# 10.192.0.0/24 => 10.192.5.0/24
-# fd00:77:20::/64 => fd00:77:20:5::/64
-#
-# This function is intended to be used for management networks.
-#
-# TODO: Validate that there is enough space for cluster ID.
-# TODO: For IPv6 could get fancy and handle case where cluster ID is placed in upper 8 bits of hextet
-# TODO: Consider if want to do /16 for IPv4 management subnet.
-#
-function dind::add-cluster {
-  local cidr=$1
-  local ip_mode=$2
-
-  if [[ ${ip_mode} = "ipv4" ]]; then
-      echo ${cidr} | sed "s/^\([0-9]*\.[0-9]*\.\).*\/24$/\1${CLUSTER_ID}.0\/24/"
-  else  # IPv6
-      echo ${cidr} | sed "s/^\(.*\)\(\:\:\/[0-9]*\)$/\1:${CLUSTER_ID}\2/"
-  fi
-}
-
-# dind::get-and-validate-cidrs takes a list of CIDRs and validates them based on the ip
-# mode, returning them. For IPv4 only and IPv6 only modes, only one CIDR is expected. For
-# dual stack, two CIDRS are expected. It verifies that the CIDRs are the right family and
-# will use the provided defaults, when CIDRs are missing. For dual-stack, the IPv4 address
-# will be first.
-#
-# For the management network, the cluster ID will be injected into the CIDR. Also, if no
-# MGMT_CIDRS value is specified, but the legacy DIND_SUBNET/DIND_SUBNET_SIZE is provided,
-# that will be used for the (first) CIDR.
-#
-# NOTE: It is expected that the CIDR size is /24 for IPv4 management networks.
-#
-# For pod CIDRs, the size will be increased by 8, to leave room for the node ID to be
-# injected into the address.
-#
-# NOTE: For IPv4, the pod size is expected to be /16 -> /24 in usage.
-#
-function dind::get-and-validate-cidrs {
-  IFS=', ' read -r -a cidrs <<< "$1"
-  IFS=', ' read -r -a defaults <<< "$2"
-  local is_mgmt=$3
-  case ${IP_MODE} in
-    ipv4)
-      case ${#cidrs[@]} in
-        0)
-	  cidrs[0]="${defaults[0]}"
-          ;;
-        1)
-          ;;
-        *)
-          echo "ERROR! More than one CIDR provided '$1'"
-          exit 1
-          ;;
-      esac
-      if [[ $( dind::family-for "${cidrs[0]}" ) != "ipv4" ]]; then
-	echo "ERROR! CIDR must be IPv4 value"
-	exit 1
-      fi
-      if [[ ${is_mgmt} = true ]]; then
-        cidrs[0]="$( dind::add-cluster "${cidrs[0]}" "${IP_MODE}" )"
-      fi
-      ;;
-
-    ipv6)
-      case ${#cidrs[@]} in
-        0)
-          cidrs[0]="${defaults[0]}"
-	  ;;
-        1)
-	  ;;
-        *)
-          echo "ERROR! More than one CIDR provided '$1'"
-          exit 1
-          ;;
-      esac
-      if [[ $( dind::family-for "${cidrs[0]}" ) != "ipv6" ]]; then
-	echo "ERROR! CIDR must be IPv6 value"
-	exit 1
-      fi
-      if [[ ${is_mgmt} = true ]]; then
-        cidrs[0]="$( dind::add-cluster "${cidrs[0]}" "${IP_MODE}" )"
-      fi
-      ;;
-
-    dual-stack)
-      case ${#cidrs[@]} in
-        0)
-          cidrs[0]="${defaults[0]}"
-          cidrs[1]="${defaults[1]}"
-          ;;
-        1)
-          if [[ $( dind::family-for "${cidrs[0]}" ) = "ipv6" ]]; then
-	    cidrs[1]=${cidrs[0]}
-	    cidrs[0]="${defaults[0]}"  # Assuming first default is a V4 address
-	  else
-	    cidrs[1]="${defaults[1]}"
-	  fi
-          ;;
-        2)
-	  # Force ordering to have V4 address first
-          if [[ $( dind::family-for "${cidrs[0]}" ) = "ipv6" ]]; then
-	    local temp=${cidrs[0]}
-	    cidrs[0]=${cidrs[1]}
-	    cidrs[1]=${temp}
-	  fi
-          ;;
-        *)
-          echo "ERROR! More than two CIDRs provided '$1'"
-          exit 1
-          ;;
-      esac
-      local have_v4=""
-      local have_v6=""
-      for cidr in ${cidrs[@]}; do
-        if [[ $( dind::family-for "${cidr}" ) = "ipv6" ]]; then
-	  have_v6=1
-	else
-	  have_v4=1
-	fi
-      done
-      if [[ -z ${have_v4} ]]; then
-        echo "ERROR! Missing IPv4 CIDR in '$1'"
-        exit 1
-      fi
-      if [[ -z ${have_v6} ]]; then
-        echo "ERROR! Missing IPv6 CIDR in '$1'"
-        exit 1
-      fi
-      if [[ ${is_mgmt} = true ]]; then
-        cidrs[0]="$( dind::add-cluster "${cidrs[0]}" "${IP_MODE}" )"
-        cidrs[1]="$( dind::add-cluster "${cidrs[1]}" "${IP_MODE}" )"
-      fi
-      ;;
-  esac
-  echo "${cidrs[@]}"
-}
-
-# dind::make-ip-from-cidr  strips off the slash and size, and appends the
-# interface part to the prefix to form an IP. For IPv4, it strips off the
-# fourth part of the prefix, so that it can be replaced. It assumes that the
-# resulting prefix will be of sufficient size. It also will use hex for the
-# appended part for IPv6, and decimal for IPv4.
-#
-# fd00:20::/64 -> fd00:20::a
-# 10.96.0.0/12 -> 10.96.0.10
-#
-function dind::make-ip-from-cidr {
-  prefix="$(echo $1 | sed 's,/.*,,')"
-  if [[ $( dind::family-for ${prefix} ) == "ipv4" ]]; then
-    printf "%s%d" $( echo ${prefix} | sed 's/0$//' ) $2
-  else
-    printf "%s%x" ${prefix} $2
-  fi
-}
-
-# dind::add-cluster-id-and-validate-nat64-prefix will modify the IPv4 mapping
-# subnet prefix, by adding the cluster ID (default 0) to the second octet.
-# It will produce an error, if the prefix is not in the 10.0.0.0/8 or
-# 172.16.0.0/12 private networks.
-function dind::add-cluster-id-and-validate-nat64-prefix {
-  local parts
-  IFS="." read -a parts <<<${NAT64_V4_SUBNET_PREFIX}
-  if [[ ${#parts[@]} -ne 2 ]]; then
-    echo "ERROR! NAT64_V4_SUBNET_PREFIX must be two octets (have '${NAT64_V4_SUBNET_PREFIX}')"
-    exit 1
-  fi
-  (( parts[1]+=${CLUSTER_ID} ))
-  NAT64_V4_SUBNET_PREFIX="${parts[0]}.${parts[1]}"
-  echo "Added cluster ID offset (${CLUSTER_ID}) to NAT64_V4_SUBNET_PREFIX giving prefix '${NAT64_V4_SUBNET_PREFIX}'"
-  if [[ ${parts[0]} -eq 10 ]]; then
-    if [[ ${parts[1]} > 253 ]]; then
-      echo "ERROR! NAT64_V4_SUBNET_PREFIX is too large for 10.0.0.0/8 private net"
-      exit 1
-    fi
-  elif [[ ${parts[0]} -eq 172 ]]; then
-    if [[ ${parts[1]} -lt 16 || ${parts[1]} -gt 31 ]]; then
-      echo "ERROR! NAT64_V4_SUBNET_PREFIX is outside of range for 172.16.0.0/12 private net"
-      exit 1
-    fi
-  else
-      echo "ERROR! NAT64_V4_SUBNET_PREFIX is not in 10.0.0.0/8 or 172.16.0.0/12 private networks"
-      exit 1
-  fi
-  echo "Using NAT64 V4 mapping network prefix: ${NAT64_V4_SUBNET_PREFIX}"
-}
-
-
-# START OF PROCESSING...
-
-IP_MODE="${IP_MODE:-ipv4}"  # ipv4, ipv6, dual-stack
-# FUTURE: Once dual-stack support is released, check K8s version, and reject for older versions.
-if [[ ! ${EMBEDDED_CONFIG:-} ]]; then
-  source "${DIND_ROOT}/config.sh"
-fi
-
-# Multicluster support
-# Users can specify a cluster ID number from 1..254, represented as a string.
-# This will be used to form resource names "cluster-#", and will be used in the
-# management subnet to give unique networks for each cluster. If the cluster ID
-# is not specified, or zero, it will be considered a single cluster or the first
-# in the multi-cluster. This is the recommended usage.
-#
-# For legacy support, the user can specify DIND_LABEL, which will be used in the
-# resource names. If a cluster ID is specified (a hybrid case, where people are
-# using the new method, but want custom names), the resourse name will have the
-# suffix "-#" with the cluster ID. If no cluster ID is specified (for backward
-# compatibility), then the resource name will be just the DIND_LABEL, and a pseudo-
-# random number from 1..13 will be generated for the cluster ID to be used in
-# management network. The range is limited, because, in IPv6 mode, the cluster ID
-# is used in the NAT64 V4 subnet prefix, which must be in a private network.
-# The default is 172.18, so the cluster ID cannot be larger than 13 to guarantee
-# a valid value.
-#
-# To get around that limitation, you can set the cluster ID, in addition to the
-# DIND_LABEL, and optionally, change the NAT64_V4_SUBNET_PREFIX value.
-#
-DEFAULT_DIND_LABEL='mirantis.kubeadm_dind_cluster_runtime'
-if [[ -z ${DIND_LABEL+x} ]]; then  # No legacy DIND_LABEL set
-  if [[ -z ${CLUSTER_ID+x} ]]; then  # No cluster ID set
-    DIND_LABEL=${DEFAULT_DIND_LABEL}  # Single cluster mode
-    CLUSTER_ID="0"
-  else  # Have cluster ID
-    if [[ ${CLUSTER_ID} = "0" ]]; then
-      DIND_LABEL=${DEFAULT_DIND_LABEL}  # Single cluster mode or first cluster of multi-cluster
-    else
-      DIND_LABEL="cluster-${CLUSTER_ID}"  # Multi-cluster
-    fi
-  fi
-else  # Legacy DIND_LABEL set for multi-cluster
-  if [[ -z ${CLUSTER_ID+x} ]]; then  # No cluster ID set, make one from 1..13, but don't use in resource names
-    CLUSTER_ID="$(( ($RANDOM % 12) + 1 ))"
-  else
-    if [[ ${CLUSTER_ID} = "0" ]]; then
-      CLUSTER_ID="$(( ($RANDOM % 12) + 1 ))"  # Force a pseudo-random cluster for additional legacy cluster
-    else
-      DIND_LABEL="${DIND_LABEL}-${CLUSTER_ID}"
-    fi
-  fi
-fi
-
-CNI_PLUGIN="${CNI_PLUGIN:-bridge}"
-GCE_HOSTED="${GCE_HOSTED:-}"
-DIND_ALLOW_AAAA_USE="${DIND_ALLOW_AAAA_USE:-}"  # Default is to use DNS64 always for IPv6 mode
-KUBE_ROUTER_VERSION="${KUBE_ROUTER_VERSION:-v0.2.0}"
-
-# Use legacy DIND_SUBNET/DIND_SUBNET_SIZE, only if MGMT_CIDRS is not set.
-legacy_mgmt_cidr=""
-if [[ ${DIND_SUBNET:-} && ${DIND_SUBNET_SIZE:-} ]]; then
-  legacy_mgmt_cidr="${DIND_SUBNET}/${DIND_SUBNET_SIZE}"
-fi
-
-if [[ ${IP_MODE} = "dual-stack" ]]; then
-  mgmt_net_defaults="10.192.0.0/24, fd00:20::/64"
-
-  KUBE_RSYNC_ADDR="${KUBE_RSYNC_ADDR:-::1}"
-  SERVICE_CIDR="${SERVICE_CIDR:-fd00:30::/110}"  # Will default to IPv6 service net family
-
-  pod_net_defaults="10.244.0.0/16, fd00:40::/72"
-
-  USE_HAIRPIN="${USE_HAIRPIN:-true}"  # Default is to use hairpin for dual-stack
-  DIND_ALLOW_AAAA_USE=true  # Forced, so can access external hosts via IPv6
-  if [[ ${DIND_ALLOW_AAAA_USE} && ${GCE_HOSTED} ]]; then
-    echo "ERROR! GCE does not support use of IPv6 for external addresses - aborting."
-    exit 1
-  fi
-elif [[ ${IP_MODE} = "ipv6" ]]; then
-  mgmt_net_defaults="fd00:20::/64"
-
-  KUBE_RSYNC_ADDR="${KUBE_RSYNC_ADDR:-::1}"
-  SERVICE_CIDR="${SERVICE_CIDR:-fd00:30::/110}"
-
-  pod_net_defaults="fd00:40::/72"
-
-  USE_HAIRPIN="${USE_HAIRPIN:-true}"  # Default is to use hairpin for IPv6
-  if [[ ${DIND_ALLOW_AAAA_USE} && ${GCE_HOSTED} ]]; then
-    echo "ERROR! GCE does not support use of IPv6 for external addresses - aborting."
-    exit 1
-  fi
-else  # IPv4 mode
-  mgmt_net_defaults="10.192.0.0/24"
-
-  KUBE_RSYNC_ADDR="${KUBE_RSYNC_ADDR:-127.0.0.1}"
-  SERVICE_CIDR="${SERVICE_CIDR:-10.96.0.0/12}"
-
-  pod_net_defaults="10.244.0.0/16"
-
-  USE_HAIRPIN="${USE_HAIRPIN:-false}"  # Disabled for IPv4, as issue with Virtlet networking
-  if [[ ${DIND_ALLOW_AAAA_USE} ]]; then
-    echo "WARNING! The DIND_ALLOW_AAAA_USE option is for IPv6 mode - ignoring setting."
-    DIND_ALLOW_AAAA_USE=
-  fi
-  if [[ ${CNI_PLUGIN} = "calico" || ${CNI_PLUGIN} = "calico-kdd" ]]; then
-    pod_net_defaults="192.168.0.0/16"
-  fi
-fi
-
-IFS=' ' read -r -a mgmt_net_cidrs <<<$( dind::get-and-validate-cidrs "${MGMT_CIDRS:-${legacy_mgmt_cidr}}" "${mgmt_net_defaults[@]}" true )
-
-REMOTE_DNS64_V4SERVER="${REMOTE_DNS64_V4SERVER:-8.8.8.8}"
-if [[ ${IP_MODE} == "ipv6" ]]; then
-  # Uses local DNS64 container
-  dns_server="$( dind::make-ip-from-cidr ${mgmt_net_cidrs[0]} 0x100 )"
-  DNS64_PREFIX="${DNS64_PREFIX:-fd00:10:64:ff9b::}"
-  DNS64_PREFIX_SIZE="${DNS64_PREFIX_SIZE:-96}"
-  DNS64_PREFIX_CIDR="${DNS64_PREFIX}/${DNS64_PREFIX_SIZE}"
-
-  LOCAL_NAT64_SERVER="$( dind::make-ip-from-cidr ${mgmt_net_cidrs[0]} 0x200 )"
-  NAT64_V4_SUBNET_PREFIX="${NAT64_V4_SUBNET_PREFIX:-172.18}"
-  dind::add-cluster-id-and-validate-nat64-prefix
-else
-  dns_server="${REMOTE_DNS64_V4SERVER}"
-fi
-
-SERVICE_NET_MODE="$( dind::family-for ${SERVICE_CIDR} )"
-DNS_SVC_IP="$( dind::make-ip-from-cidr ${SERVICE_CIDR} 10 )"
-
-ETCD_HOST="${ETCD_HOST:-$( dind::localhost )}"
-
-IFS=' ' read -r -a pod_net_cidrs <<<$( dind::get-and-validate-cidrs "${POD_NETWORK_CIDR:-}" "${pod_net_defaults[@]}" false )
-
-declare -a pod_prefixes
-declare -a pod_sizes
-# Extract the prefix and size from the provided pod CIDR(s), based on the IP mode of each. The
-# size will be increased by 8, to make room for the node ID to be added to the prefix later.
-# Bridge and PTP plugins can process IPv4 and IPv6 pod CIDRs, other plugins must be IPv4 only.
-for pod_cidr in "${pod_net_cidrs[@]}"; do
-  if [[ $( dind::family-for "${pod_cidr}" ) = "ipv4" ]]; then
-    actual_size=$( echo ${pod_cidr} | sed 's,.*/,,' )
-    if [[ ${actual_size} -ne 16 ]]; then
-	echo "ERROR! For IPv4 CIDRs, the size must be /16. Have '${pod_cidr}'"
-	exit 1
-    fi
-    pod_sizes+=( 24 )
-    pod_prefixes+=( "$(echo ${pod_cidr} | sed 's/^\([0-9]*\.[0-9]*\.\).*/\1/')" )
-  else  # IPv6
-    if [[ ${CNI_PLUGIN} != "bridge" && ${CNI_PLUGIN} != "ptp" ]]; then
-      echo "ERROR! IPv6 pod networks are only supported by bridge and PTP CNI plugins"
-      exit 1
-    fi
-    # There are several cases to address. First, is normal split of prefix and size:
-    #   fd00:10:20:30::/64  --->  fd00:10:20:30:  /72
-    #
-    # Second, is when the prefix needs to be padded, so that node ID can be added later:
-    #   fd00:10::/64  --->  fd00:10:0:0:  /72
-    #
-    # Third, is when the low order part of the address, must be removed for the prefix,
-    # as the node ID will be placed in the lower byte:
-    #   fd00:10:20:30:4000::/72  --->  fd00:10:20:30:40  /80
-    #
-    # We will attempt to check for three error cases. One is when the address part is
-    # way too big for the size specified:
-    #   fd00:10:20:30:40::/48  --->  fd00:10:20:  /56 desired, but conflict with 30:40:
-    #
-    # Another is when the address part, once trimmed for the size, would loose info:
-    #   fd00:10:20:1234::/56  --->  fd00:10:20:12  /64, but lost 34:, which conflicts
-    #
-    # Lastly, again, trimming would leave high byte in hextet, conflicting with
-    # the node ID:
-    #   fd00:10:20:30:1200::/64  --->  fd00:10:20:30:12  /72, but 12 conflicts
-    #
-    # Note: later, the node ID will be appended to the prefix generated.
-    #
-    cluster_size="$(echo ${pod_cidr} | sed 's,.*::/,,')"
-    pod_sizes+=( $((${cluster_size}+8)) )
-
-    pod_prefix="$(echo ${pod_cidr} | sed 's,::/.*,:,')"
-    num_colons="$(grep -o ":" <<< "${pod_prefix}" | wc -l)"
-    need_zero_pads=$((${cluster_size}/16))
-
-    if [[ ${num_colons} -gt $((need_zero_pads + 1)) ]]; then
-	echo "ERROR! Address part of CIDR (${pod_prefix}) is too large for /${cluster_size}"
-	exit 1
-    fi
-    if [[ ${num_colons} -gt ${need_zero_pads} ]]; then
-      # Will be replacing lowest byte with node ID, so pull off lower byte and colon
-	if [[ ${pod_prefix: -3} != "00:" ]]; then   # last byte is not zero
-	  echo "ERROR! Cannot trim address part of CIDR (${pod_prefix}) to fit in /${cluster_size}"
-	  exit 1
-	fi
-	pod_prefix=${pod_prefix::-3}
-	if [[ $(( ${cluster_size} % 16 )) -eq 0 && $( ${pod_prefix: -1} ) != ":" ]]; then  # should not be upper byte for this size CIDR
-	  echo "ERROR! Trimmed address part of CIDR (${pod_prefix}) is still too large for /${cluster_size}"
-	  exit 1
-	fi
-    fi
-    # Add in zeros to pad 16 bits at a time, up to the padding needed, which is
-    # need_zero_pads - num_colons.
-    while [ ${num_colons} -lt ${need_zero_pads} ]; do
-	pod_prefix+="0:"
-      ((num_colons++))
-    done
-    pod_prefixes+=( "${pod_prefix}" )
-  fi
-done
-
-DIND_IMAGE="${DIND_IMAGE:-}"
-BUILD_KUBEADM="${BUILD_KUBEADM:-}"
-BUILD_HYPERKUBE="${BUILD_HYPERKUBE:-}"
-KUBEADM_SOURCE="${KUBEADM_SOURCE-}"
-HYPERKUBE_SOURCE="${HYPERKUBE_SOURCE-}"
-NUM_NODES=${NUM_NODES:-2}
-EXTRA_PORTS="${EXTRA_PORTS:-}"
-LOCAL_KUBECTL_VERSION=${LOCAL_KUBECTL_VERSION:-}
-KUBECTL_DIR="${KUBECTL_DIR:-${HOME}/.kubeadm-dind-cluster}"
-DASHBOARD_URL="${DASHBOARD_URL:-https://rawgit.com/kubernetes/dashboard/bfab10151f012d1acc5dfb1979f3172e2400aa3c/src/deploy/kubernetes-dashboard.yaml}"
-SKIP_SNAPSHOT="${SKIP_SNAPSHOT:-}"
-E2E_REPORT_DIR="${E2E_REPORT_DIR:-}"
-DIND_NO_PARALLEL_E2E="${DIND_NO_PARALLEL_E2E:-}"
-DNS_SERVICE="${DNS_SERVICE:-kube-dns}"
-DIND_STORAGE_DRIVER="${DIND_STORAGE_DRIVER:-overlay2}"
-
-DIND_CA_CERT_URL="${DIND_CA_CERT_URL:-}"
-DIND_PROPAGATE_HTTP_PROXY="${DIND_PROPAGATE_HTTP_PROXY:-}"
-DIND_HTTP_PROXY="${DIND_HTTP_PROXY:-}"
-DIND_HTTPS_PROXY="${DIND_HTTPS_PROXY:-}"
-DIND_NO_PROXY="${DIND_NO_PROXY:-}"
-
-DIND_DAEMON_JSON_FILE="${DIND_DAEMON_JSON_FILE:-/etc/docker/daemon.json}"  # can be set to /dev/null
-DIND_REGISTRY_MIRROR="${DIND_REGISTRY_MIRROR:-}"  # plain string format
-DIND_INSECURE_REGISTRIES="${DIND_INSECURE_REGISTRIES:-}"  # json list format
-
-FEATURE_GATES="${FEATURE_GATES:-MountPropagation=true}"
-# you can set special value 'none' not to set any kubelet's feature gates.
-KUBELET_FEATURE_GATES="${KUBELET_FEATURE_GATES:-MountPropagation=true,DynamicKubeletConfig=true}"
-
-KUBELET_EXTRA_ARGS="--sync-frequency=30s"
-
-if [[ ! ${LOCAL_KUBECTL_VERSION:-} && ${DIND_IMAGE:-} =~ :(v[0-9]+\.[0-9]+)$ ]]; then
-  LOCAL_KUBECTL_VERSION="${BASH_REMATCH[1]}"
-fi
-
-ENABLE_CEPH="${ENABLE_CEPH:-}"
-
-# TODO: Test multi-cluster for IPv6, before enabling
-if [[ "${DIND_LABEL}" != "${DEFAULT_DIND_LABEL}"  && "${IP_MODE}" == 'dual-stack' ]]; then
-    echo "Multiple parallel clusters currently not supported for dual-stack mode" >&2
-    exit 1
-fi
-
-# not configurable for now, would need to setup context for kubectl _inside_ the cluster
-readonly INTERNAL_APISERVER_PORT=8080
-
-function dind::need-source {
-  if [[ ! -f cluster/kubectl.sh ]]; then
-    echo "$0 must be called from the Kubernetes repository root directory" 1>&2
-    exit 1
-  fi
-}
-
-build_tools_dir="build"
-use_k8s_source=y
-if [[ ! ${BUILD_KUBEADM} && ! ${BUILD_HYPERKUBE} ]]; then
-  use_k8s_source=
-fi
-if [[ ${use_k8s_source} ]]; then
-  dind::need-source
-  kubectl=cluster/kubectl.sh
-  if [[ ! -f ${build_tools_dir}/common.sh ]]; then
-    build_tools_dir="build-tools"
-  fi
-else
-  if [[ ! ${LOCAL_KUBECTL_VERSION:-} ]] && ! hash kubectl 2>/dev/null; then
-    echo "You need kubectl binary in your PATH to use prebuilt DIND image" 1>&2
-    exit 1
-  fi
-  kubectl=kubectl
-fi
-
-function dind::retry {
-  # based on retry function in hack/jenkins/ scripts in k8s source
-  for i in {1..10}; do
-    "$@" && return 0 || sleep ${i}
-  done
-  "$@"
-}
-
-busybox_image="busybox:1.26.2"
-e2e_base_image="golang:1.9.2"
-sys_volume_args=()
-build_volume_args=()
-
-function dind::set-build-volume-args {
-  if [ ${#build_volume_args[@]} -gt 0 ]; then
-    return 0
-  fi
-  build_container_name=
-  if [ -n "${KUBEADM_DIND_LOCAL:-}" ]; then
-    build_volume_args=(-v "$PWD:/go/src/k8s.io/kubernetes")
-  else
-    build_container_name="$(KUBE_ROOT=${PWD} ETCD_HOST=${ETCD_HOST} &&
-                            . ${build_tools_dir}/common.sh &&
-                            kube::build::verify_prereqs >&2 &&
-                            echo "${KUBE_DATA_CONTAINER_NAME:-${KUBE_BUILD_DATA_CONTAINER_NAME}}")"
-    build_volume_args=(--volumes-from "${build_container_name}")
-  fi
-}
-
-function dind::volume-exists {
-  local name="$1"
-  if docker volume inspect "${name}" >& /dev/null; then
-    return 0
-  fi
-  return 1
-}
-
-function dind::create-volume {
-  local name="$1"
-  docker volume create --label "${DIND_LABEL}" --name "${name}" >/dev/null
-}
-
-# We mount /boot and /lib/modules into the container
-# below to in case some of the workloads need them.
-# This includes virtlet, for instance. Also this may be
-# useful in future if we want DIND nodes to pass
-# preflight checks.
-# Unfortunately we can't do this when using Mac Docker
-# (unless a remote docker daemon on Linux is used)
-# NB: there's no /boot on recent Mac dockers
-function dind::prepare-sys-mounts {
-  if [[ ! ${using_linuxkit} ]]; then
-    sys_volume_args=()
-    if [[ -d /boot ]]; then
-      sys_volume_args+=(-v /boot:/boot)
-    fi
-    if [[ -d /lib/modules ]]; then
-      sys_volume_args+=(-v /lib/modules:/lib/modules)
-    fi
-    return 0
-  fi
-  local dind_sys_vol_name
-  dind_sys_vol_name="kubeadm-dind-sys$( dind::cluster-suffix )"
-  if ! dind::volume-exists "$dind_sys_vol_name"; then
-    dind::step "Saving a copy of docker host's /lib/modules"
-    dind::create-volume "$dind_sys_vol_name"
-    # Use a dirty nsenter trick to fool Docker on Mac and grab system
-    # /lib/modules into sys.tar file on kubeadm-dind-sys volume.
-    local nsenter="nsenter --mount=/proc/1/ns/mnt --"
-    docker run \
-           --rm \
-           --privileged \
-           -v "$dind_sys_vol_name":/dest \
-           --pid=host \
-           "${busybox_image}" \
-           /bin/sh -c \
-           "if ${nsenter} test -d /lib/modules; then ${nsenter} tar -C / -c lib/modules >/dest/sys.tar; fi"
-  fi
-  sys_volume_args=(-v "$dind_sys_vol_name":/dind-sys)
-}
-
-tmp_containers=()
-
-function dind::cleanup {
-  if [ ${#tmp_containers[@]} -gt 0 ]; then
-    for name in "${tmp_containers[@]}"; do
-      docker rm -vf "${name}" 2>/dev/null
-    done
-  fi
-}
-
-trap dind::cleanup EXIT
-
-function dind::check-image {
-  local name="$1"
-  if docker inspect --format 'x' "${name}" >&/dev/null; then
-    return 0
-  else
-    return 1
-  fi
-}
-
-function dind::filter-make-output {
-  # these messages make output too long and make Travis CI choke
-  egrep -v --line-buffered 'I[0-9][0-9][0-9][0-9] .*(parse|conversion|defaulter|deepcopy)\.go:[0-9]+\]'
-}
-
-function dind::run-build-command {
-    # this is like build/run.sh, but it doesn't rsync back the binaries,
-    # only the generated files.
-    local cmd=("$@")
-    (
-        # The following is taken from build/run.sh and build/common.sh
-        # of Kubernetes source tree. It differs in
-        # --filter='+ /_output/dockerized/bin/**'
-        # being removed from rsync
-        . ${build_tools_dir}/common.sh
-        kube::build::verify_prereqs
-        kube::build::build_image
-        kube::build::run_build_command "$@"
-
-        kube::log::status "Syncing out of container"
-
-        kube::build::start_rsyncd_container
-
-        local rsync_extra=""
-        if (( ${KUBE_VERBOSE} >= 6 )); then
-            rsync_extra="-iv"
-        fi
-
-        # The filter syntax for rsync is a little obscure. It filters on files and
-        # directories.  If you don't go in to a directory you won't find any files
-        # there.  Rules are evaluated in order.  The last two rules are a little
-        # magic. '+ */' says to go in to every directory and '- /**' says to ignore
-        # any file or directory that isn't already specifically allowed.
-        #
-        # We are looking to copy out all of the built binaries along with various
-        # generated files.
-        kube::build::rsync \
-            --filter='- /vendor/' \
-            --filter='- /_temp/' \
-            --filter='+ zz_generated.*' \
-            --filter='+ generated.proto' \
-            --filter='+ *.pb.go' \
-            --filter='+ types.go' \
-            --filter='+ */' \
-            --filter='- /**' \
-            "rsync://k8s@${KUBE_RSYNC_ADDR}/k8s/" "${KUBE_ROOT}"
-
-        kube::build::stop_rsyncd_container
-    )
-}
-
-function dind::make-for-linux {
-  local copy="$1"
-  shift
-  dind::step "Building binaries:" "$*"
-  if [ -n "${KUBEADM_DIND_LOCAL:-}" ]; then
-    dind::step "+ make WHAT=\"$*\""
-    make WHAT="$*" 2>&1 | dind::filter-make-output
-  elif [ "${copy}" = "y" ]; then
-    dind::step "+ ${build_tools_dir}/run.sh make WHAT=\"$*\""
-    "${build_tools_dir}/run.sh" make WHAT="$*" 2>&1 | dind::filter-make-output
-  else
-    dind::step "+ [using the build container] make WHAT=\"$*\""
-    dind::run-build-command make WHAT="$*" 2>&1 | dind::filter-make-output
-  fi
-}
-
-function dind::check-binary {
-  local filename="$1"
-  local dockerized="_output/dockerized/bin/linux/amd64/${filename}"
-  local plain="_output/local/bin/linux/amd64/${filename}"
-  dind::set-build-volume-args
-  # FIXME: don't hardcode amd64 arch
-  if [ -n "${KUBEADM_DIND_LOCAL:-${force_local:-}}" ]; then
-    if [ -f "${dockerized}" -o -f "${plain}" ]; then
-      return 0
-    fi
-  elif docker run --rm "${build_volume_args[@]}" \
-              "${busybox_image}" \
-              test -f "/go/src/k8s.io/kubernetes/${dockerized}" >&/dev/null; then
-    return 0
-  fi
-  return 1
-}
-
-function dind::ensure-downloaded-kubectl {
-  local kubectl_url
-  local kubectl_sha1
-  local kubectl_sha1_linux
-  local kubectl_sha1_darwin
-  local kubectl_link
-  local kubectl_os
-  local full_kubectl_version
-
-  case "${LOCAL_KUBECTL_VERSION}" in
-    v1.8)
-      full_kubectl_version=v1.8.15
-      kubectl_sha1_linux=52a1ee321e1e8c0ecfd6e83c38bf972c2c60adf2
-      kubectl_sha1_darwin=ac3f823d7aa104237929a1e35ea400c6aa3cc356
-      ;;
-    v1.9)
-      full_kubectl_version=v1.9.9
-      kubectl_sha1_linux=c8163a6360119c56d163fbd8cef8727e9841e712
-      kubectl_sha1_darwin=09585552eb7616954481789489ec382c633a0162
-      ;;
-    v1.10)
-      full_kubectl_version=v1.10.5
-      kubectl_sha1_linux=dbe431b2684f8ff4188335b3b3cea185d5a9ec44
-      kubectl_sha1_darwin=08e58440949c71053b45bfadf80532ea3d752d12
-      ;;
-    v1.11)
-      full_kubectl_version=v1.11.0
-      kubectl_sha1_linux=e23f251ca0cb848802f3cb0f69a4ba297d07bfc6
-      kubectl_sha1_darwin=6eff29a328c4bc00879fd6a0c8b33690c6f75908
-      ;;
-    "")
-      return 0
-      ;;
-    *)
-      echo "Invalid kubectl version" >&2
-      exit 1
-  esac
-
-  export PATH="${KUBECTL_DIR}:$PATH"
-
-  if [ $(uname) = Darwin ]; then
-    kubectl_sha1="${kubectl_sha1_darwin}"
-    kubectl_os=darwin
-  else
-    kubectl_sha1="${kubectl_sha1_linux}"
-    kubectl_os=linux
-  fi
-  local link_target="kubectl-${full_kubectl_version}"
-  local link_name="${KUBECTL_DIR}"/kubectl
-  if [[ -h "${link_name}" && "$(readlink "${link_name}")" = "${link_target}" ]]; then
-    return 0
-  fi
-
-  local path="${KUBECTL_DIR}/${link_target}"
-  if [[ ! -f "${path}" ]]; then
-    mkdir -p "${KUBECTL_DIR}"
-    curl -sSLo "${path}" "https://storage.googleapis.com/kubernetes-release/release/${full_kubectl_version}/bin/${kubectl_os}/amd64/kubectl"
-    echo "${kubectl_sha1}  ${path}" | sha1sum -c
-    chmod +x "${path}"
-  fi
-
-  ln -fs "${link_target}" "${KUBECTL_DIR}/kubectl"
-}
-
-function dind::ensure-kubectl {
-  if [[ ! ${use_k8s_source} ]]; then
-    # already checked on startup
-    dind::ensure-downloaded-kubectl
-    return 0
-  fi
-  if [ $(uname) = Darwin ]; then
-    if [ ! -f _output/local/bin/darwin/amd64/kubectl ]; then
-      dind::step "Building kubectl"
-      dind::step "+ make WHAT=cmd/kubectl"
-      make WHAT=cmd/kubectl 2>&1 | dind::filter-make-output
-    fi
-  elif ! force_local=y dind::check-binary kubectl; then
-    dind::make-for-linux y cmd/kubectl
-  fi
-}
-
-function dind::ensure-binaries {
-  local -a to_build=()
-  for name in "$@"; do
-    if ! dind::check-binary "$(basename "${name}")"; then
-      to_build+=("${name}")
-    fi
-  done
-  if [ "${#to_build[@]}" -gt 0 ]; then
-    dind::make-for-linux n "${to_build[@]}"
-  fi
-  return 0
-}
-
-# dind::ensure-network creates the management network for the cluster. For IPv4
-# only it will have the management network CIDR. For IPv6 only, it will have
-# the IPv6 management network CIDR and the NAT64 V4 mapping network CIDR. For
-# dual stack, it will have the IPv4 and IPv6 management CIDRs. Each of the
-# management networks (not the NAT64 network) will have a gateway specified.
-#
-function dind::ensure-network {
-  if ! docker network inspect $(dind::net-name) >&/dev/null; then
-    local -a args
-    for cidr in "${mgmt_net_cidrs[@]}"; do
-      if [[ $( dind::family-for ${cidr} ) = "ipv6" ]]; then
-                args+=(--ipv6)
-      fi
-      args+=(--subnet="${cidr}")
-      local gw=$( dind::make-ip-from-cidr ${cidr} 1 )
-      args+=(--gateway="${gw}")
-    done
-    if [[ ${IP_MODE} = "ipv6" ]]; then
-      # Need second network for NAT64 V4 mapping network
-      args+=(--subnet=${NAT64_V4_SUBNET_PREFIX}.0.0/16)
-    fi
-    docker network create ${args[@]} $(dind::net-name) >/dev/null
-  fi
-}
-
-function dind::ensure-volume {
-  local reuse_volume=
-  if [[ $1 = -r ]]; then
-    reuse_volume=1
-    shift
-  fi
-  local name="$1"
-  if dind::volume-exists "${name}"; then
-    if [[ ! ${reuse_volume} ]]; then
-      docker volume rm "${name}" >/dev/null
-    fi
-  fi
-  dind::create-volume "${name}"
-}
-
-function dind::ensure-dns {
-    if [[ ${IP_MODE} = "ipv6" ]]; then
-	local dns64_name="bind9$( dind::cluster-suffix )"
-        if ! docker inspect ${dns64_name} >&/dev/null; then
-            local force_dns64_for=""
-            if [[ ! ${DIND_ALLOW_AAAA_USE} ]]; then
-                # Normally, if have an AAAA record, it is used. This clause tells
-                # bind9 to do ignore AAAA records for the specified networks
-                # and/or addresses and lookup A records and synthesize new AAAA
-                # records. In this case, we select "any" networks that have AAAA
-                # records meaning we ALWAYS use A records and do NAT64.
-                force_dns64_for="exclude { any; };"
-            fi
-            read -r -d '' bind9_conf <<BIND9_EOF
-options {
-    directory "/var/bind";
-    allow-query { any; };
-    forwarders {
-        ${DNS64_PREFIX}${REMOTE_DNS64_V4SERVER};
-    };
-    auth-nxdomain no;    # conform to RFC1035
-    listen-on-v6 { any; };
-    dns64 ${DNS64_PREFIX_CIDR} {
-        ${force_dns64_for}
-    };
-};
-BIND9_EOF
-            docker run -d --name ${dns64_name} --hostname ${dns64_name} --net "$(dind::net-name)" --label "dind-support$( dind::cluster-suffix )" \
-               --sysctl net.ipv6.conf.all.disable_ipv6=0 --sysctl net.ipv6.conf.all.forwarding=1 \
-               --privileged=true --ip6 ${dns_server} --dns ${dns_server} \
-               -e bind9_conf="${bind9_conf}" \
-               diverdane/bind9:latest /bin/sh -c 'echo "${bind9_conf}" >/named.conf && named -c /named.conf -g -u named' >/dev/null
-            ipv4_addr="$(docker exec ${dns64_name} ip addr list eth0 | grep "inet" | awk '$1 == "inet" {print $2}')"
-            docker exec ${dns64_name} ip addr del ${ipv4_addr} dev eth0
-            docker exec ${dns64_name} ip -6 route add ${DNS64_PREFIX_CIDR} via ${LOCAL_NAT64_SERVER}
-        fi
-    fi
-}
-
-function dind::ensure-nat {
-    if [[  ${IP_MODE} = "ipv6" ]]; then
-	local nat64_name="tayga$( dind::cluster-suffix )"
-        if ! docker ps | grep ${nat64_name} >&/dev/null; then
-            docker run -d --name ${nat64_name} --hostname ${nat64_name} --net "$(dind::net-name)" --label "dind-support$( dind::cluster-suffix )" \
-		   --sysctl net.ipv6.conf.all.disable_ipv6=0 --sysctl net.ipv6.conf.all.forwarding=1 \
-		   --privileged=true --ip ${NAT64_V4_SUBNET_PREFIX}.0.200 --ip6 ${LOCAL_NAT64_SERVER} --dns ${REMOTE_DNS64_V4SERVER} --dns ${dns_server} \
-		   -e TAYGA_CONF_PREFIX=${DNS64_PREFIX_CIDR} -e TAYGA_CONF_IPV4_ADDR=${NAT64_V4_SUBNET_PREFIX}.0.200 \
-		   -e TAYGA_CONF_DYNAMIC_POOL=${NAT64_V4_SUBNET_PREFIX}.0.128/25 danehans/tayga:latest >/dev/null
-	    # Need to check/create, as "clean" may remove route
-	    local route="$(ip route | egrep "^${NAT64_V4_SUBNET_PREFIX}.0.128/25")"
-	    if [[ -z "${route}" ]]; then
-	        docker run --net=host --rm --privileged ${busybox_image} ip route add ${NAT64_V4_SUBNET_PREFIX}.0.128/25 via ${NAT64_V4_SUBNET_PREFIX}.0.200
-	    fi
-	fi
-    fi
-}
-
-function dind::run {
-  local reuse_volume=
-  if [[ $1 = -r ]]; then
-    reuse_volume="-r"
-    shift
-  fi
-  local container_name="${1:-}"
-  local node_id=${2:-0}
-  local portforward="${3:-}"
-  if [[ $# -gt 3 ]]; then
-    shift 3
-  else
-    shift $#
-  fi
-
-  local -a opts=("$@")
-  local ip_mode="--ip"
-  for cidr in "${mgmt_net_cidrs[@]}"; do
-    if [[ $( dind::family-for ${cidr} ) = "ipv6" ]]; then
-      ip_mode="--ip6"
-    fi
-    opts+=("${ip_mode}" "$( dind::make-ip-from-cidr ${cidr} $((${node_id}+1)) )")
-  done
-  opts+=("$@")
-  
-  local -a args=("systemd.setenv=CNI_PLUGIN=${CNI_PLUGIN}")
-  args+=("systemd.setenv=IP_MODE=${IP_MODE}")
-  args+=("systemd.setenv=DIND_STORAGE_DRIVER=${DIND_STORAGE_DRIVER}")
-
-  if [[ ${IP_MODE} != "ipv4" ]]; then
-    opts+=(--sysctl net.ipv6.conf.all.disable_ipv6=0)
-    opts+=(--sysctl net.ipv6.conf.all.forwarding=1)
-  fi
-
-  if [[ ${IP_MODE} = "ipv6" ]]; then
-    opts+=(--dns ${dns_server})
-    args+=("systemd.setenv=DNS64_PREFIX_CIDR=${DNS64_PREFIX_CIDR}")
-    args+=("systemd.setenv=LOCAL_NAT64_SERVER=${LOCAL_NAT64_SERVER}")
-  fi
-
-  declare -a pod_nets
-  local i=0
-  if [[ ${IP_MODE} = "ipv4" || ${IP_MODE} = "dual-stack" ]]; then
-    pod_nets+=("${pod_prefixes[$i]}${node_id}")
-    i=$((i+1))
-  fi
-  if [[ ${IP_MODE} = "ipv6" || ${IP_MODE} = "dual-stack" ]]; then
-    # For prefix, if node ID will be in the upper byte, push it over
-    if [[ $((${pod_sizes[$i]} % 16)) -ne 0 ]]; then
-      n_id=$(printf "%02x00\n" "${node_id}")
-    else
-      if [[ "${pod_prefixes[$i]: -1}" = ":" ]]; then
-	n_id=$(printf "%x\n" "${node_id}")
-      else
-        n_id=$(printf "%02x\n" "${node_id}")  # In lower byte, so ensure two chars
-      fi
-    fi
-    pod_nets+=("${pod_prefixes[$i]}${n_id}")
-  fi
-
-  args+=("systemd.setenv=POD_NET_PREFIX=\"${pod_nets[0]}\"")
-  args+=("systemd.setenv=POD_NET_SIZE=\"${pod_sizes[0]}\"")
-  args+=("systemd.setenv=POD_NET2_PREFIX=\"${pod_nets[1]:-}\"")
-  args+=("systemd.setenv=POD_NET2_SIZE=\"${pod_sizes[1]:-}\"")
-  args+=("systemd.setenv=SERVICE_NET_MODE=${SERVICE_NET_MODE}")
-  args+=("systemd.setenv=USE_HAIRPIN=${USE_HAIRPIN}")
-  args+=("systemd.setenv=DNS_SVC_IP=${DNS_SVC_IP}")
-  args+=("systemd.setenv=DNS_SERVICE=${DNS_SERVICE}")
-  if [[ ! "${container_name}" ]]; then
-    echo >&2 "Must specify container name"
-    exit 1
-  fi
-
-  # remove any previously created containers with the same name
-  docker rm -vf "${container_name}" >&/dev/null || true
-
-  if [[ "${portforward}" ]]; then
-    IFS=';' read -ra array <<< "${portforward}"
-    for element in "${array[@]}"; do
-      opts+=(-p "${element}")
-    done
-  fi
-
-  opts+=(${sys_volume_args[@]+"${sys_volume_args[@]}"})
-
-  dind::step "Starting DIND container:" "${container_name}"
-
-  if [[ ! ${using_linuxkit} ]]; then
-    opts+=(-v /boot:/boot -v /lib/modules:/lib/modules)
-  fi
-
-  if [[ ${ENABLE_CEPH} ]]; then
-    opts+=(-v /dev:/dev
-           -v /sys/bus:/sys/bus
-           -v /var/run/docker.sock:/opt/outer-docker.sock)
-  fi
-
-  local volume_name="kubeadm-dind-${container_name}"
-  dind::ensure-network
-  dind::ensure-volume ${reuse_volume} "${volume_name}"
-  dind::ensure-nat
-  dind::ensure-dns
-
-  # TODO: create named volume for binaries and mount it to /k8s
-  # in case of the source build
-
-  # Start the new container.
-  docker run \
-	 -e IP_MODE="${IP_MODE}" \
-         -e KUBEADM_SOURCE="${KUBEADM_SOURCE}" \
-         -e HYPERKUBE_SOURCE="${HYPERKUBE_SOURCE}" \
-         -d --privileged \
-         --net "$(dind::net-name)" \
-         --name "${container_name}" \
-         --hostname "${container_name}" \
-         -l "${DIND_LABEL}" \
-         -v "${volume_name}:/dind" \
-         ${opts[@]+"${opts[@]}"} \
-         "${DIND_IMAGE}" \
-         ${args[@]+"${args[@]}"}
-}
-
-function dind::kubeadm {
-  local container_id="$1"
-  shift
-  dind::step "Running kubeadm:" "$*"
-  status=0
-  # See image/bare/wrapkubeadm.
-  # Capturing output is necessary to grab flags for 'kubeadm join'
-  kubelet_feature_gates="-e KUBELET_FEATURE_GATES=${KUBELET_FEATURE_GATES}"
-  if ! docker exec ${kubelet_feature_gates} "${container_id}" /usr/local/bin/wrapkubeadm "$@" 2>&1 | tee /dev/fd/2; then
-    echo "*** kubeadm failed" >&2
-    return 1
-  fi
-  return ${status}
-}
-
-# function dind::bare {
-#   local container_name="${1:-}"
-#   if [[ ! "${container_name}" ]]; then
-#     echo >&2 "Must specify container name"
-#     exit 1
-#   fi
-#   shift
-#   run_opts=(${@+"$@"})
-#   dind::run "${container_name}"
-# }
-
-function dind::configure-kubectl {
-  dind::step "Setting cluster config"
-  local host="$(dind::localhost)"
-  if [[ -z "$using_local_linuxdocker" ]]; then
-    host="127.0.0.1"
-  fi
-  local context_name cluster_name
-  context_name="$(dind::context-name)"
-  cluster_name="$(dind::context-name)"
-  "${kubectl}" config set-cluster "$cluster_name" \
-    --server="http://${host}:$(dind::apiserver-port)" \
-    --insecure-skip-tls-verify=true
-  "${kubectl}" config set-context "$context_name" --cluster="$cluster_name"
-  if [[ ${DIND_LABEL} = ${DEFAULT_DIND_LABEL} ]]; then
-      # Single cluster mode
-      "${kubectl}" config use-context "$context_name"
-  fi
-}
-
-force_make_binaries=
-function dind::set-master-opts {
-  master_opts=()
-  if [[ ${BUILD_KUBEADM} || ${BUILD_HYPERKUBE} ]]; then
-    # share binaries pulled from the build container between nodes
-    local dind_k8s_bin_vol_name
-    dind_k8s_bin_vol_name="dind-k8s-binaries$(dind::cluster-suffix)"
-    dind::ensure-volume -r "${dind_k8s_bin_vol_name}"
-    dind::set-build-volume-args
-    master_opts+=("${build_volume_args[@]}" -v "${dind_k8s_bin_vol_name}:/k8s")
-    local -a bins
-    if [[ ${BUILD_KUBEADM} ]]; then
-      master_opts+=(-e KUBEADM_SOURCE=build://)
-      bins+=(cmd/kubeadm)
-    else
-      master_opts+=(-e ${KUBEADM_SOURCE})
-    fi
-    if [[ ${BUILD_HYPERKUBE} ]]; then
-      master_opts+=(-e HYPERKUBE_SOURCE=build://)
-      bins+=(cmd/hyperkube)
-    fi
-    if [[ ${force_make_binaries} ]]; then
-      dind::make-for-linux n "${bins[@]}"
-    else
-      dind::ensure-binaries "${bins[@]}"
-    fi
-  fi
-  if [[ ${MASTER_EXTRA_OPTS:-} ]]; then
-    master_opts+=( ${MASTER_EXTRA_OPTS} )
-  fi
-}
-
-function dind::ensure-dashboard-clusterrolebinding {
-  local ctx
-  ctx="$(dind::context-name)"
-  # 'create' may cause etcd timeout, yet create the clusterrolebinding.
-  # So use 'apply' to actually create it
-  "${kubectl}" --context "$ctx" create clusterrolebinding add-on-cluster-admin \
-               --clusterrole=cluster-admin \
-               --serviceaccount=kube-system:default \
-               -o json --dry-run |
-    docker exec -i "$(dind::master-name)" jq '.apiVersion="rbac.authorization.k8s.io/v1beta1"|.kind|="ClusterRoleBinding"' |
-    "${kubectl}" --context "$ctx" apply -f -
-}
-
-function dind::deploy-dashboard {
-  dind::step "Deploying k8s dashboard"
-  dind::retry "${kubectl}" --context "$(dind::context-name)" apply -f "${DASHBOARD_URL}"
-  # https://kubernetes-io-vnext-staging.netlify.com/docs/admin/authorization/rbac/#service-account-permissions
-  # Thanks @liggitt for the hint
-  dind::retry dind::ensure-dashboard-clusterrolebinding
-}
-
-function dind::kubeadm-version {
-  if [[ ${use_k8s_source} ]]; then
-    (cluster/kubectl.sh version --short 2>/dev/null || true) |
-      grep Client |
-      sed 's/^.*: v\([0-9.]*\).*/\1/'
-  else
-    docker exec "$(dind::master-name)" \
-           /bin/bash -c 'kubeadm version -o json | jq -r .clientVersion.gitVersion' |
-      sed 's/^v\([0-9.]*\).*/\1/'
-  fi
-}
-
-function dind::kubeadm-skip-checks-flag {
-  kubeadm_version="$(dind::kubeadm-version)"
-  if [[ ${kubeadm_version} =~ 1\.8\. ]]; then
-    echo -n "--skip-preflight-checks"
-  else
-    echo -n "--ignore-preflight-errors=all"
-  fi
-}
-
-function dind::verify-image-compatibility {
-  # We can't tell in advance, if the image selected supports dual-stack,
-  # but will do the next best thing, and check as soon as start up kube-master
-  local master_name=$1
-  if [[ ${IP_MODE} = "dual-stack" ]]; then
-    local dual_stack_support="$(docker exec ${master_name} cat /node-info 2>/dev/null | grep "dual-stack-support" | wc -l)"
-    if [[ ${dual_stack_support} -eq 0 ]]; then
-      echo "ERROR! DinD image (${DIND_IMAGE}) does not support dual-stack mode - aborting!"
-      dind::remove-images "${DIND_LABEL}"
-      exit 1
-    fi
-  fi
-}
-
-function dind::init {
-  local -a opts
-  dind::set-master-opts
-  local local_host master_name container_id
-  master_name="$(dind::master-name)"
-  local_host="$( dind::localhost )"
-  container_id=$(dind::run "${master_name}" 1 "${local_host}:$(dind::apiserver-port):${INTERNAL_APISERVER_PORT}" ${master_opts[@]+"${master_opts[@]}"})
-
-  dind::verify-image-compatibility ${master_name}
-
-  # FIXME: I tried using custom tokens with 'kubeadm ex token create' but join failed with:
-  # 'failed to parse response as JWS object [square/go-jose: compact JWS format must have three parts]'
-  # So we just pick the line from 'kubeadm init' output
-  # Using a template file in the image to build a kubeadm.conf file and to customize
-  # it based on CNI plugin, IP mode, and environment settings. User can add additional
-  # customizations to template and then rebuild the image used (build/build-local.sh).
-  local pod_subnet_disable="# "
-  # TODO: May want to specify each of the plugins that require --pod-network-cidr
-  if [[ ${CNI_PLUGIN} != "bridge" && ${CNI_PLUGIN} != "ptp" ]]; then
-    pod_subnet_disable=""
-  fi
-  local bind_address="0.0.0.0"
-  if [[ ${SERVICE_NET_MODE} = "ipv6" ]]; then
-    bind_address="::"
-  fi
-  dind::proxy "$master_name"
-  dind::custom-docker-opts "$master_name"
-
-  # HACK: Indicating mode, so that wrapkubeadm will not set a cluster CIDR for kube-proxy
-  # in IPv6 (only) mode.
-  if [[ ${SERVICE_NET_MODE} = "ipv6" ]]; then
-    docker exec --privileged -i "$master_name" touch /v6-mode
-  fi
-
-  feature_gates="{}"
-  if [[ ${DNS_SERVICE} == "coredns" ]]; then
-    # can't just use 'CoreDNS: false' because
-    # it'll break k8s 1.8. FIXME: simplify
-    # after 1.8 support is removed
-    feature_gates="{CoreDNS: true}"
-  elif docker exec "$master_name" kubeadm init --help 2>&1 | grep -q CoreDNS; then
-    # FIXME: CoreDNS should be the default in 1.11
-    feature_gates="{CoreDNS: false}"
-  fi
-
-  component_feature_gates=""
-  if [ "${FEATURE_GATES}" != "none" ]; then
-    component_feature_gates="feature-gates: \\\"${FEATURE_GATES}\\\""
-  fi
-
-  apiserver_extra_args=""
-  for e in $(set -o posix ; set | grep -E "^APISERVER_[a-z_]+=" | cut -d'=' -f 1); do
-    opt_name=$(echo ${e#APISERVER_} | sed 's/_/-/g')
-    apiserver_extra_args+="  ${opt_name}: \\\"$(eval echo \$$e)\\\"\\n"
-  done
-
-  controller_manager_extra_args=""
-  for e in $(set -o posix ; set | grep -E "^CONTROLLER_MANAGER_[a-z_]+=" | cut -d'=' -f 1); do
-    opt_name=$(echo ${e#CONTROLLER_MANAGER_} | sed 's/_/-/g')
-    controller_manager_extra_args+="  ${opt_name}: \\\"$(eval echo \$$e)\\\"\\n"
-  done
-
-  scheduler_extra_args=""
-  for e in $(set -o posix ; set | grep -E "^SCHEDULER_[a-z_]+=" | cut -d'=' -f 1); do
-    opt_name=$(echo ${e#SCHEDULER_} | sed 's/_/-/g')
-    scheduler_extra_args+="  ${opt_name}: \\\"$(eval echo \$$e)\\\"\\n"
-  done
-
-  kubeadm_version="$(dind::kubeadm-version)"
-  api_version="kubeadm.k8s.io/v1alpha3"
-  kind="ClusterConfiguration"
-  api_endpoint="apiEndpoint:"
-  if [[ ${kubeadm_version} =~ 1\.(8|9|10)\. ]]; then
-    api_version="kubeadm.k8s.io/v1alpha1"
-    kind="MasterConfiguration"
-    api_endpoint="api:"
-  elif [[ ${kubeadm_version} =~ 1\.(11|12)\. ]]; then
-    api_version="kubeadm.k8s.io/v1alpha2"
-    kind="MasterConfiguration"
-    api_endpoint="api:"
-  fi
-  local mgmt_cidr=${mgmt_net_cidrs[0]}
-  if [[ ${IP_MODE} = "dual-stack" && ${SERVICE_NET_MODE} = "ipv6" ]]; then
-      mgmt_cidr=${mgmt_net_cidrs[1]}
-  fi
-  local master_ip=$( dind::make-ip-from-cidr ${mgmt_cidr} 2 )
-  docker exec -i "$master_name" bash <<EOF
-sed -e "s|{{API_VERSION}}|${api_version}|" \
-    -e "s|{{ADV_ADDR}}|${master_ip}|" \
-    -e "s|{{POD_SUBNET_DISABLE}}|${pod_subnet_disable}|" \
-    -e "s|{{POD_NETWORK_CIDR}}|${pod_net_cidrs[0]}|" \
-    -e "s|{{SVC_SUBNET}}|${SERVICE_CIDR}|" \
-    -e "s|{{BIND_ADDR}}|${bind_address}|" \
-    -e "s|{{BIND_PORT}}|${INTERNAL_APISERVER_PORT}|" \
-    -e "s|{{FEATURE_GATES}}|${feature_gates}|" \
-    -e "s|{{KUBEADM_VERSION}}|${kubeadm_version}|" \
-    -e "s|{{COMPONENT_FEATURE_GATES}}|${component_feature_gates}|" \
-    -e "s|{{APISERVER_EXTRA_ARGS}}|${apiserver_extra_args}|" \
-    -e "s|{{CONTROLLER_MANAGER_EXTRA_ARGS}}|${controller_manager_extra_args}|" \
-    -e "s|{{SCHEDULER_EXTRA_ARGS}}|${scheduler_extra_args}|" \
-    -e "s|{{KUBE_MASTER_NAME}}|${master_name}|" \
-    -e "s|{{KUBEADM_KIND}}|${kind}|" \
-    -e "s|{{API_ENDPOINT}}|${api_endpoint}|" \
-    /etc/kubeadm.conf.tmpl > /etc/kubeadm.conf
-EOF
-  init_args=(--config /etc/kubeadm.conf)
-  skip_preflight_arg="$(dind::kubeadm-skip-checks-flag)"
-  # required when building from source
-  if [[ ${BUILD_KUBEADM} || ${BUILD_HYPERKUBE} ]]; then
-    docker exec "$master_name" mount --make-shared /k8s
-  fi
-  kubeadm_join_flags="$(dind::kubeadm "${container_id}" init "${init_args[@]}" "${skip_preflight_arg}" "$@" | grep '^ *kubeadm join' | sed 's/^ *kubeadm join //')"
-  dind::configure-kubectl
-  dind::start-port-forwarder
-}
-
-function dind::create-node-container {
-  local reuse_volume next_node_index node_name
-  reuse_volume=''
-  if [[ ${1:-} = -r ]]; then
-    reuse_volume="-r"
-    shift
-  fi
-  # if there's just one node currently, it's master, thus we need to use
-  # kube-node-1 hostname, if there are two nodes, we should pick
-  # kube-node-2 and so on
-  next_node_index=${1:-$(docker ps -q --filter=label="${DIND_LABEL}" | wc -l | sed 's/^ *//g')}
-  local -a opts
-  if [[ ${BUILD_KUBEADM} || ${BUILD_HYPERKUBE} ]]; then
-    opts+=(-v "dind-k8s-binaries$(dind::cluster-suffix)":/k8s)
-    if [[ ${BUILD_KUBEADM} ]]; then
-      opts+=(-e KUBEADM_SOURCE=build://)
-    fi
-    if [[ ${BUILD_HYPERKUBE} ]]; then
-      opts+=(-e HYPERKUBE_SOURCE=build://)
-    fi
-  fi
-  node_name="$(dind::node-name ${next_node_index})"
-  dind::run ${reuse_volume} "$node_name" $((next_node_index + 1)) "${EXTRA_PORTS}" ${opts[@]+"${opts[@]}"}
-}
-
-function dind::join {
-  local container_id="$1"
-  shift
-  dind::proxy "${container_id}"
-  dind::custom-docker-opts "${container_id}"
-  skip_preflight_arg="$(dind::kubeadm-skip-checks-flag)"
-  dind::kubeadm "${container_id}" join "${skip_preflight_arg}" "$@" >/dev/null
-}
-
-function dind::escape-e2e-name {
-    sed 's/[]\$*.^()[]/\\&/g; s/\s\+/\\s+/g' <<< "$1" | tr -d '\n'
-}
-
-function dind::accelerate-kube-dns {
-  if [[ ${DNS_SERVICE} == "kube-dns" ]]; then
-     dind::step "Patching kube-dns deployment to make it start faster"
-     # Could do this on the host, too, but we don't want to require jq here
-     # TODO: do this in wrapkubeadm
-     docker exec "$(dind::master-name)" /bin/bash -c \
-        "kubectl get deployment kube-dns -n kube-system -o json | jq '.spec.template.spec.containers[0].readinessProbe.initialDelaySeconds = 3|.spec.template.spec.containers[0].readinessProbe.periodSeconds = 3' | kubectl apply --force -f -"
- fi
-}
-
-function dind::component-ready {
-  local label="$1"
-  local out
-  if ! out="$("${kubectl}" --context "$(dind::context-name)" get pod -l "${label}" -n kube-system \
-                           -o jsonpath='{ .items[*].status.conditions[?(@.type == "Ready")].status }' 2>/dev/null)"; then
-    return 1
-  fi
-  if ! grep -v False <<<"${out}" | grep -q True; then
-    return 1
-  fi
-  return 0
-}
-
-function dind::kill-failed-pods {
-  local pods ctx
-  ctx="$(dind::context-name)"
-  # workaround for https://github.com/kubernetes/kubernetes/issues/36482
-  if ! pods="$(kubectl --context "$ctx" get pod -n kube-system -o jsonpath='{ .items[?(@.status.phase == "Failed")].metadata.name }' 2>/dev/null)"; then
-    return
-  fi
-  for name in ${pods}; do
-    kubectl --context "$ctx" delete pod --now -n kube-system "${name}" >&/dev/null || true
-  done
-}
-
-function dind::create-static-routes {
-  echo "Creating static routes for bridge/PTP plugin"
-  for ((i=0; i <= NUM_NODES; i++)); do
-    if [[ ${i} -eq 0 ]]; then
-      node="$(dind::master-name)"
-    else
-      node="$(dind::node-name $i)"
-    fi
-    for ((j=0; j <= NUM_NODES; j++)); do
-      if [[ ${i} -eq ${j} ]]; then
-	continue
-      fi
-      if [[ ${j} -eq 0 ]]; then
-        dest_node="$(dind::master-name)"
-      else
-        dest_node="$(dind::node-name $j)"
-      fi
-      id=$((${j}+1))
-      if [[ ${IP_MODE} = "ipv4" || ${IP_MODE} = "dual-stack" ]]; then
-	# Assuming pod subnets will all be /24
-        dest="${pod_prefixes[0]}${id}.0/24"
-        gw=`docker exec ${dest_node} ip addr show eth0 | grep -w inet | awk '{ print $2 }' | sed 's,/.*,,'`
-        docker exec "${node}" ip route add "${dest}" via "${gw}"
-      fi
-      if [[ ${IP_MODE} = "ipv6" || ${IP_MODE} = "dual-stack" ]]; then
-	local position=0
-	if [[ ${IP_MODE} = "dual-stack" ]]; then
-	    position=1
-	fi
-	instance=$(printf "%02x" ${id})
-	if [[ $((${pod_sizes[$position]} % 16)) -ne 0 ]]; then
-	  instance+="00" # Move node ID to upper byte
-	fi
-	dest="${pod_prefixes[$position]}${instance}::/${pod_sizes[$position]}"
-        gw=`docker exec ${dest_node} ip addr show eth0 | grep -w inet6 | grep -i global | head -1 | awk '{ print $2 }' | sed 's,/.*,,'`
-        docker exec "${node}" ip route add "${dest}" via "${gw}"
-      fi
-    done
-  done
-}
-
-# If we are allowing AAAA record use, then provide SNAT for IPv6 packets from
-# node containers, and forward packets to bridge used for $(dind::net-name).
-# This gives pods access to external IPv6 sites, when using IPv6 addresses.
-function dind::setup_external_access_on_host {
-  if [[ ! ${DIND_ALLOW_AAAA_USE} ]]; then
-    return
-  fi
-  local main_if=`ip route | grep default | awk '{print $5}'`
-  dind::ip6tables-on-hostnet -t nat -A POSTROUTING -o $main_if -j MASQUERADE
-  if [[ ${IP_MODE} = "dual-stack" ]]; then
-    return
-  fi
-  local bridge_if=`ip route | grep ${NAT64_V4_SUBNET_PREFIX}.0.0 | awk '{print $3}'`
-  if [[ -n "$bridge_if" ]]; then
-    dind::ip6tables-on-hostnet -A FORWARD -i $bridge_if -j ACCEPT
-  else
-    echo "WARNING! No $(dind::net-name) bridge with NAT64 - unable to setup forwarding/SNAT"
-  fi
-}
-
-# Remove ip6tables rules for SNAT and forwarding, if they exist.
-function dind::remove_external_access_on_host {
-  if [[ ! ${DIND_ALLOW_AAAA_USE} ]]; then
-    return
-  fi
-  local have_rule
-  local main_if="$(ip route | grep default | awk '{print $5}')"
-  have_rule="$(dind::ip6tables-on-hostnet -S -t nat | grep "\-o $main_if" || true)"
-  if [[ -n "$have_rule" ]]; then
-    dind::ip6tables-on-hostnet -t nat -D POSTROUTING -o $main_if -j MASQUERADE
-  else
-    echo "Skipping delete of ip6tables rule for SNAT, as rule non-existent"
-  fi
-
-  if [[ ${IP_MODE} = "dual-stack" ]]; then
-    return
-  fi
-  local bridge_if="$(ip route | grep ${NAT64_V4_SUBNET_PREFIX}.0.0 | awk '{print $3}')"
-  if [[ -n "$bridge_if" ]]; then
-    have_rule="$(dind::ip6tables-on-hostnet -S | grep "\-i $bridge_if" || true)"
-    if [[ -n "$have_rule" ]]; then
-      dind::ip6tables-on-hostnet -D FORWARD -i $bridge_if -j ACCEPT
-    else
-      echo "Skipping delete of ip6tables rule for forwarding, as rule non-existent"
-    fi
-  else
-    echo "Skipping delete of ip6tables rule for forwarding, as no bridge interface using NAT64"
-  fi
-}
-
-function dind::ip6tables-on-hostnet {
-  local mod_path='/lib/modules'
-  docker run -v "${mod_path}:${mod_path}" --entrypoint /sbin/ip6tables --net=host --rm --privileged "${DIND_IMAGE}" "$@"
-}
-
-function dind::wait-for-ready {
-  local app="kube-proxy"
-  if [[ ${CNI_PLUGIN} = "kube-router" ]]; then
-    app=kube-router
-  fi
-  dind::step "Waiting for ${app} and the nodes"
-  local app_ready
-  local nodes_ready
-  local n=3
-  local ntries=200
-  local ctx
-  ctx="$(dind::context-name)"
-  while true; do
-    dind::kill-failed-pods
-    if "${kubectl}" --context "$ctx" get nodes 2>/dev/null | grep -q NotReady; then
-      nodes_ready=
-    else
-      nodes_ready=y
-    fi
-    if dind::component-ready k8s-app=${app}; then
-      app_ready=y
-    else
-      app_ready=
-    fi
-    if [[ ${nodes_ready} && ${app_ready} ]]; then
-      if ((--n == 0)); then
-        echo "[done]" >&2
-        break
-      fi
-    else
-      n=3
-    fi
-    if ((--ntries == 0)); then
-      echo "Error waiting for ${app} and the nodes" >&2
-      exit 1
-    fi
-    echo -n "." >&2
-    sleep 1
-  done
-
-  dind::step "Bringing up ${DNS_SERVICE} and kubernetes-dashboard"
-  # on Travis 'scale' sometimes fails with 'error: Scaling the resource failed with: etcdserver: request timed out; Current resource version 442' here
-  dind::retry "${kubectl}" --context "$ctx" scale deployment --replicas=1 -n kube-system ${DNS_SERVICE}
-  dind::retry "${kubectl}" --context "$ctx" scale deployment --replicas=1 -n kube-system kubernetes-dashboard
-
-  ntries=200
-  while ! dind::component-ready k8s-app=kube-dns || ! dind::component-ready app=kubernetes-dashboard; do
-    if ((--ntries == 0)); then
-      echo "Error bringing up ${DNS_SERVICE} and kubernetes-dashboard" >&2
-      exit 1
-    fi
-    echo -n "." >&2
-    dind::kill-failed-pods
-    sleep 1
-  done
-  echo "[done]" >&2
-
-  dind::retry "${kubectl}" --context "$ctx" get nodes >&2
-
-  local local_host
-  local_host="$( dind::localhost )"
-  dind::step "Access dashboard at:" "http://${local_host}:$(dind::apiserver-port)/api/v1/namespaces/kube-system/services/kubernetes-dashboard:/proxy"
-}
-
-# dind::make-kube-router-yaml creates a temp file with contents of the configuration needed for the kube-router CNI
-# plugin at a specific version, instead of using the publically available file, which uses the latest version. This
-# allows us to control the version used. If/when updating, be sure to update the KUBE_ROUTER_VERSION env variable
-# ensure the YAML contents below, reflect the configuration in:
-#
-# https://raw.githubusercontent.com/cloudnativelabs/kube-router/master/daemonset/kubeadm-kuberouter-all-feature.yaml
-#
-# FUTURE: May be able to remove this, if/when kube-router "latest" is stable, and use the public YAML file instead.
-function dind::make-kube-router-yaml {
-  tmp_yaml=$(mktemp /tmp/kube-router-yaml.XXXXXX)
-  cat >${tmp_yaml} <<KUBE_ROUTER_YAML
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: kube-router-cfg
-  namespace: kube-system
-  labels:
-    tier: node
-    k8s-app: kube-router
-data:
-  cni-conf.json: |
-    {
-      "name":"kubernetes",
-      "type":"bridge",
-      "bridge":"kube-bridge",
-      "isDefaultGateway":true,
-      "ipam": {
-        "type":"host-local"
-      }
-    }
----
-apiVersion: extensions/v1beta1
-kind: DaemonSet
-metadata:
-  labels:
-    k8s-app: kube-router
-    tier: node
-  name: kube-router
-  namespace: kube-system
-spec:
-  template:
-    metadata:
-      labels:
-        k8s-app: kube-router
-        tier: node
-      annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
-    spec:
-      serviceAccountName: kube-router
-      serviceAccount: kube-router
-      containers:
-      - name: kube-router
-        image: cloudnativelabs/kube-router:${KUBE_ROUTER_VERSION}
-        imagePullPolicy: Always
-        args:
-        - --run-router=true
-        - --run-firewall=true
-        - --run-service-proxy=true
-        - --kubeconfig=/var/lib/kube-router/kubeconfig
-        env:
-        - name: NODE_NAME
-          valueFrom:
-            fieldRef:
-              fieldPath: spec.nodeName
-        livenessProbe:
-          httpGet:
-            path: /healthz
-            port: 20244
-          initialDelaySeconds: 10
-          periodSeconds: 3
-        resources:
-          requests:
-            cpu: 250m
-            memory: 250Mi
-        securityContext:
-          privileged: true
-        volumeMounts:
-        - name: lib-modules
-          mountPath: /lib/modules
-          readOnly: true
-        - name: cni-conf-dir
-          mountPath: /etc/cni/net.d
-        - name: kubeconfig
-          mountPath: /var/lib/kube-router
-          readOnly: true
-      initContainers:
-      - name: install-cni
-        image: busybox
-        imagePullPolicy: Always
-        command:
-        - /bin/sh
-        - -c
-        - set -e -x;
-          if [ ! -f /etc/cni/net.d/10-kuberouter.conf ]; then
-            TMP=/etc/cni/net.d/.tmp-kuberouter-cfg;
-            cp /etc/kube-router/cni-conf.json \${TMP};
-            mv \${TMP} /etc/cni/net.d/10-kuberouter.conf;
-          fi
-        volumeMounts:
-        - name: cni-conf-dir
-          mountPath: /etc/cni/net.d
-        - name: kube-router-cfg
-          mountPath: /etc/kube-router
-      hostNetwork: true
-      tolerations:
-      - key: CriticalAddonsOnly
-        operator: Exists
-      - effect: NoSchedule
-        key: node-role.kubernetes.io/master
-        operator: Exists
-      volumes:
-      - name: lib-modules
-        hostPath:
-          path: /lib/modules
-      - name: cni-conf-dir
-        hostPath:
-          path: /etc/cni/net.d
-      - name: kube-router-cfg
-        configMap:
-          name: kube-router-cfg
-      - name: kubeconfig
-        configMap:
-          name: kube-proxy
-          items:
-          - key: kubeconfig.conf
-            path: kubeconfig
----
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: kube-router
-  namespace: kube-system
----
-kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1beta1
-metadata:
-  name: kube-router
-  namespace: kube-system
-rules:
-  - apiGroups:
-    - ""
-    resources:
-      - namespaces
-      - pods
-      - services
-      - nodes
-      - endpoints
-    verbs:
-      - list
-      - get
-      - watch
-  - apiGroups:
-    - "networking.k8s.io"
-    resources:
-      - networkpolicies
-    verbs:
-      - list
-      - get
-      - watch
-  - apiGroups:
-    - extensions
-    resources:
-      - networkpolicies
-    verbs:
-      - get
-      - list
-      - watch
----
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1beta1
-metadata:
-  name: kube-router
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: kube-router
-subjects:
-- kind: ServiceAccount
-  name: kube-router
-  namespace: kube-system
-KUBE_ROUTER_YAML
-  echo $tmp_yaml
-}
-
-function dind::up {
-  dind::down
-  dind::init
-  local ctx
-  ctx="$(dind::context-name)"
-  # pre-create node containers sequentially so they get predictable IPs
-  local -a node_containers
-  for ((n=1; n <= NUM_NODES; n++)); do
-    dind::step "Starting node container:" ${n}
-    if ! container_id="$(dind::create-node-container ${n})"; then
-      echo >&2 "*** Failed to start node container ${n}"
-      exit 1
-    else
-      node_containers+=(${container_id})
-      dind::step "Node container started:" ${n}
-    fi
-  done
-  dind::fix-mounts
-  status=0
-  local -a pids
-  for ((n=1; n <= NUM_NODES; n++)); do
-    (
-      dind::step "Joining node:" ${n}
-      container_id="${node_containers[${n}-1]}"
-      if ! dind::join ${container_id} ${kubeadm_join_flags}; then
-        echo >&2 "*** Failed to start node container ${n}"
-        exit 1
-      else
-        dind::step "Node joined:" ${n}
-      fi
-    )&
-    pids[${n}]=$!
-  done
-  if ((NUM_NODES > 0)); then
-    for pid in ${pids[*]}; do
-      wait ${pid}
-    done
-  else
-    # FIXME: this may fail depending on k8s/kubeadm version
-    # FIXME: check for taint & retry if it's there
-    "${kubectl}" --context "$ctx" taint nodes $(dind::master-name) node-role.kubernetes.io/master- || true
-  fi
-  case "${CNI_PLUGIN}" in
-    bridge | ptp)
-      dind::create-static-routes
-      dind::setup_external_access_on_host
-      ;;
-    flannel)
-      # without --validate=false this will fail on older k8s versions
-      dind::retry "${kubectl}" --context "$ctx" apply --validate=false -f "https://github.com/coreos/flannel/blob/master/Documentation/kube-flannel.yml?raw=true"
-      ;;
-    calico)
-      dind::retry "${kubectl}" --context "$ctx" apply -f https://docs.projectcalico.org/v2.6/getting-started/kubernetes/installation/hosted/kubeadm/1.6/calico.yaml
-      ;;
-    calico-kdd)
-      dind::retry "${kubectl}" --context "$ctx" apply -f https://docs.projectcalico.org/v2.6/getting-started/kubernetes/installation/hosted/rbac-kdd.yaml
-      dind::retry "${kubectl}" --context "$ctx" apply -f https://docs.projectcalico.org/v2.6/getting-started/kubernetes/installation/hosted/kubernetes-datastore/calico-networking/1.7/calico.yaml
-      ;;
-    weave)
-      dind::retry "${kubectl}" --context "$ctx" apply -f "https://github.com/weaveworks/weave/blob/master/prog/weave-kube/weave-daemonset-k8s-1.6.yaml?raw=true"
-      ;;
-    kube-router)
-      kube_router_config="$( dind::make-kube-router-yaml )"
-      dind::retry "${kubectl}" --context "$ctx" apply -f ${kube_router_config}
-      rm "${kube_router_config}"
-      dind::retry "${kubectl}" --context "$ctx" -n kube-system delete ds kube-proxy
-      docker run --privileged --net=host k8s.gcr.io/kube-proxy-amd64:v1.10.2 kube-proxy --cleanup
-      ;;
-    *)
-      echo "Unsupported CNI plugin '${CNI_PLUGIN}'" >&2
-      ;;
-  esac
-  dind::deploy-dashboard
-  dind::accelerate-kube-dns
-  if [[ (${CNI_PLUGIN} != "bridge" && ${CNI_PLUGIN} != "ptp") || ${SKIP_SNAPSHOT} ]]; then
-    # This is especially important in case of Calico -
-    # the cluster will not recover after snapshotting
-    # (at least not after restarting from the snapshot)
-    # if Calico installation is interrupted
-    dind::wait-for-ready
-  fi
-  dind::step "Cluster Info"
-  echo "Network Mode: ${IP_MODE}"
-  echo "Cluster context: $( dind::context-name )"
-  echo "Cluster ID: ${CLUSTER_ID}"
-  echo "Management CIDR(s): ${mgmt_net_cidrs[@]}"
-  echo "Service CIDR/mode: ${SERVICE_CIDR}/${SERVICE_NET_MODE}"
-  echo "Pod CIDR(s): ${pod_net_cidrs[@]}"
-}
-
-function dind::fix-mounts {
-  local node_name
-  for ((n=0; n <= NUM_NODES; n++)); do
-    node_name="$(dind::master-name)"
-    if ((n > 0)); then
-      node_name="$(dind::node-name $n)"
-    fi
-    docker exec "${node_name}" mount --make-shared /run
-    if [[ ! ${using_linuxkit} ]]; then
-      docker exec "${node_name}" mount --make-shared /lib/modules/
-    fi
-    # required when building from source
-    if [[ ${BUILD_KUBEADM} || ${BUILD_HYPERKUBE} ]]; then
-      docker exec "${node_name}" mount --make-shared /k8s
-    fi
-    # docker exec "${node_name}" mount --make-shared /sys/kernel/debug
-  done
-}
-
-function dind::snapshot_container {
-  local container_name="$1"
-  docker exec -i ${container_name} /usr/local/bin/snapshot prepare
-  # remove the hidden *plnk directories
-  docker diff ${container_name} | grep -v plnk | docker exec -i ${container_name} /usr/local/bin/snapshot save
-}
-
-function dind::snapshot {
-  dind::step "Taking snapshot of the cluster"
-  dind::snapshot_container "$(dind::master-name)"
-  for ((n=1; n <= NUM_NODES; n++)); do
-    dind::snapshot_container "$(dind::node-name $n)"
-  done
-  dind::wait-for-ready
-}
-
-restore_cmd=restore
-function dind::restore_container {
-  local container_id="$1"
-  docker exec ${container_id} /usr/local/bin/snapshot "${restore_cmd}"
-}
-
-function dind::restore {
-  local apiserver_port local_host pid pids
-  dind::down
-  dind::step "Restoring containers"
-  dind::set-master-opts
-  local_host="$( dind::localhost )"
-  apiserver_port="$( dind::apiserver-port )"
-  for ((n=0; n <= NUM_NODES; n++)); do
-    (
-      if [[ n -eq 0 ]]; then
-        dind::step "Restoring master container"
-        dind::restore_container "$(dind::run -r "$(dind::master-name)" 1 "${local_host}:${apiserver_port}:${INTERNAL_APISERVER_PORT}" ${master_opts[@]+"${master_opts[@]}"})"
-        dind::verify-image-compatibility "$(dind::master-name)"
-        dind::step "Master container restored"
-      else
-        dind::step "Restoring node container:" ${n}
-        if ! container_id="$(dind::create-node-container -r ${n})"; then
-          echo >&2 "*** Failed to start node container ${n}"
-          exit 1
-        else
-          dind::restore_container "${container_id}"
-          dind::step "Node container restored:" ${n}
-        fi
-      fi
-    )&
-    pids[${n}]=$!
-  done
-  for pid in ${pids[*]}; do
-    wait ${pid}
-  done
-  if [[ ${CNI_PLUGIN} = "bridge" || ${CNI_PLUGIN} = "ptp" ]]; then
-    dind::create-static-routes
-    dind::setup_external_access_on_host
-  fi
-  dind::fix-mounts
-  # Recheck kubectl config. It's possible that the cluster was started
-  # on this docker from different host
-  dind::configure-kubectl
-  dind::start-port-forwarder
-  dind::wait-for-ready
-}
-
-function dind::down {
-  dind::remove-images "${DIND_LABEL}"
-  if [[ ${CNI_PLUGIN} = "bridge" || ${CNI_PLUGIN} = "ptp" ]]; then
-    dind::remove_external_access_on_host
-  elif [[ "${CNI_PLUGIN}" = "kube-router" ]]; then
-    if [[ ${COMMAND} = "down" || ${COMMAND} = "clean" ]]; then
-      # FUTURE: Updated pinned version, after verifying operation 
-      docker run --privileged --net=host cloudnativelabs/kube-router:${KUBE_ROUTER_VERSION} --cleanup-config
-    fi
-  fi
-}
-
-function dind::apiserver-port {
-  # APISERVER_PORT is explicitely set
-  if [ -n "${APISERVER_PORT:-}" ]
-  then
-    echo "$APISERVER_PORT"
-    return
-  fi
-
-  # Get the port from the master
-  local master port
-  master="$(dind::master-name)"
-  # 8080/tcp -> 127.0.0.1:8082  =>  8082
-  port="$( docker port "$master" 2>/dev/null | awk -F: "/^${INTERNAL_APISERVER_PORT}/{ print \$NF }" )"
-  if [ -n "$port" ]
-  then
-    APISERVER_PORT="$port"
-    echo "$APISERVER_PORT"
-    return
-  fi
-
-  # get a random free port
-  APISERVER_PORT=0
-  echo "$APISERVER_PORT"
-}
-
-function dind::master-name {
-  echo "kube-master$( dind::cluster-suffix )"
-}
-
-function dind::node-name {
-  local nr="$1"
-  echo "kube-node-${nr}$( dind::cluster-suffix )"
-}
-
-function dind::context-name {
-  echo "dind$( dind::cluster-suffix )"
-}
-
-function dind::remove-volumes {
-  # docker 1.13+: docker volume ls -q -f label="${DIND_LABEL}"
-  local nameRE
-  nameRE="^kubeadm-dind-(sys|kube-master|kube-node-[0-9]+)$(dind::cluster-suffix)$"
-  docker volume ls -q | (grep -E "$nameRE" || true) | while read -r volume_id; do
-    dind::step "Removing volume:" "${volume_id}"
-    docker volume rm "${volume_id}"
-  done
-}
-
-function dind::remove-images {
-  local which=$1
-  docker ps -a -q --filter=label="${which}" | while read container_id; do
-    dind::step "Removing container:" "${container_id}"
-    docker rm -fv "${container_id}"
-  done
-}
-
-function dind::remove-cluster {
-  cluster_name="dind$(dind::cluster-suffix)"
-  if ${kubectl} config get-clusters | grep -qE "^${cluster_name}$"; then
-    dind::step "Removing cluster from config:" "${cluster_name}"
-    ${kubectl} config delete-cluster ${cluster_name} 2>/dev/null || true
-  fi
-}
-
-function dind::remove-context {
-  context_name="$(dind::context-name)"
-  if ${kubectl} config get-contexts | grep -qE "${context_name}\\s"; then
-    dind::step "Removing context from config:" "${context_name}"
-    ${kubectl} config delete-context ${context_name} 2>/dev/null || true
-  fi
-}
-
-function dind::start-port-forwarder {
-  local fwdr port
-  fwdr="${DIND_PORT_FORWARDER:-}"
-
-  [ -n "$fwdr" ] || return 0
-
-  [ -x "$fwdr" ] || {
-    echo "'${fwdr}' is not executable." >&2
-    return 1
-  }
-
-  port="$( dind::apiserver-port )"
-  dind::step "+ Setting up port-forwarding for :${port}"
-  "$fwdr" "$port"
-}
-
-function dind::check-for-snapshot {
-  if ! dind::volume-exists "kubeadm-dind-$(dind::master-name)"; then
-    return 1
-  fi
-  for ((n=1; n <= NUM_NODES; n++)); do
-    if ! dind::volume-exists "kubeadm-dind-$(dind::node-name ${n})"; then
-      return 1
-    fi
-  done
-}
-
-function dind::do-run-e2e {
-  local parallel="${1:-}"
-  local focus="${2:-}"
-  local skip="${3:-}"
-  local host="$(dind::localhost)"
-  if [[ -z "$using_local_linuxdocker" ]]; then
-    host="127.0.0.1"
-  fi  
-  dind::need-source
-  local kubeapi test_args term=
-  local -a e2e_volume_opts=()
-  kubeapi="http://${host}:$(dind::apiserver-port)"
-  test_args="--host=${kubeapi}"
-  if [[ ${focus} ]]; then
-    test_args="--ginkgo.focus=${focus} ${test_args}"
-  fi
-  if [[ ${skip} ]]; then
-    test_args="--ginkgo.skip=${skip} ${test_args}"
-  fi
-  if [[ ${E2E_REPORT_DIR} ]]; then
-    test_args="--report-dir=/report ${test_args}"
-    e2e_volume_opts=(-v "${E2E_REPORT_DIR}:/report")
-  fi
-  dind::make-for-linux n "cmd/kubectl test/e2e/e2e.test vendor/github.com/onsi/ginkgo/ginkgo"
-  dind::step "Running e2e tests with args:" "${test_args}"
-  dind::set-build-volume-args
-  if [ -t 1 ] ; then
-    term="-it"
-    test_args="--ginkgo.noColor --num-nodes=2 ${test_args}"
-  fi
-  docker run \
-         --rm ${term} \
-         --net=host \
-         "${build_volume_args[@]}" \
-         -e KUBERNETES_PROVIDER=dind \
-         -e KUBE_MASTER_IP="${kubeapi}" \
-         -e KUBE_MASTER=local \
-         -e KUBERNETES_CONFORMANCE_TEST=y \
-         -e GINKGO_PARALLEL=${parallel} \
-         ${e2e_volume_opts[@]+"${e2e_volume_opts[@]}"} \
-         -w /go/src/k8s.io/kubernetes \
-         "${e2e_base_image}" \
-         bash -c "cluster/kubectl.sh config set-cluster dind --server='${kubeapi}' --insecure-skip-tls-verify=true &&
-         cluster/kubectl.sh config set-context dind --cluster=dind &&
-         cluster/kubectl.sh config use-context dind &&
-         go run hack/e2e.go -- --v 6 --test --check-version-skew=false --test_args='${test_args}'"
-}
-
-function dind::clean {
-  dind::down
-  dind::remove-images "dind-support$( dind::cluster-suffix )"
-  dind::remove-volumes
-  local net_name
-  net_name="$(dind::net-name)"
-  if docker network inspect "$net_name" >&/dev/null; then
-    docker network rm "$net_name"
-  fi
-  dind::remove-cluster
-  dind::remove-context
-}
-
-function dind::copy-image {
-  local image="${2:-}"
-  local image_path="/tmp/save_${image//\//_}"
-  if [[ -f "${image_path}" ]]; then
-    rm -fr "${image_path}"
-  fi
-  docker save "${image}" -o "${image_path}"
-  docker ps -a -q --filter=label="${DIND_LABEL}" | while read container_id; do
-    cat "${image_path}" | docker exec -i "${container_id}" docker load
-  done
-  rm -fr "${image_path}"
-}
-
-function dind::run-e2e {
-  local focus="${1:-}"
-  local skip="${2:-[Serial]}"
-  skip="$(dind::escape-e2e-name "${skip}")"
-  if [[ "$focus" ]]; then
-    focus="$(dind::escape-e2e-name "${focus}")"
-  else
-    focus="\[Conformance\]"
-  fi
-  local parallel=y
-  if [[ ${DIND_NO_PARALLEL_E2E} ]]; then
-    parallel=
-  fi
-  dind::do-run-e2e "${parallel}" "${focus}" "${skip}"
-}
-
-function dind::run-e2e-serial {
-  local focus="${1:-}"
-  local skip="${2:-}"
-  skip="$(dind::escape-e2e-name "${skip}")"
-  dind::need-source
-  if [[ "$focus" ]]; then
-    focus="$(dind::escape-e2e-name "${focus}")"
-  else
-    focus="\[Serial\].*\[Conformance\]"
-  fi
-  dind::do-run-e2e n "${focus}" "${skip}"
-  # TBD: specify filter
-}
-
-function dind::step {
-  local OPTS=""
-  if [ "$1" = "-n" ]; then
-    shift
-    OPTS+="-n"
-  fi
-  GREEN="$1"
-  shift
-  if [ -t 2 ] ; then
-    echo -e ${OPTS} "\x1B[97m* \x1B[92m${GREEN}\x1B[39m $*" 1>&2
-  else
-    echo ${OPTS} "* ${GREEN} $*" 1>&2
-  fi
-}
-
-function dind::dump {
-  set +e
-  echo "*** Dumping cluster state ***"
-  for node in $(docker ps --format '{{.Names}}' --filter label="${DIND_LABEL}"); do
-    for service in kubelet.service dindnet.service criproxy.service dockershim.service; do
-      if docker exec "${node}" systemctl is-enabled "${service}" >&/dev/null; then
-        echo "@@@ service-${node}-${service}.log @@@"
-        docker exec "${node}" systemctl status "${service}"
-        docker exec "${node}" journalctl -xe -n all -u "${service}"
-      fi
-    done
-    echo "@@@ psaux-${node}.txt @@@"
-    docker exec "${node}" ps auxww
-    echo "@@@ dockerps-a-${node}.txt @@@"
-    docker exec "${node}" docker ps -a
-    echo "@@@ ip-a-${node}.txt @@@"
-    docker exec "${node}" ip a
-    echo "@@@ ip-r-${node}.txt @@@"
-    docker exec "${node}" ip r
-  done
-  local ctx master_name
-  master_name="$(dind::master-name)"
-  ctx="$(dind::context-name)"
-  docker exec "$master_name" kubectl get pods --all-namespaces \
-          -o go-template='{{range $x := .items}}{{range $x.spec.containers}}{{$x.spec.nodeName}}{{" "}}{{$x.metadata.namespace}}{{" "}}{{$x.metadata.name}}{{" "}}{{.name}}{{"\n"}}{{end}}{{end}}' |
-    while read node ns pod container; do
-      echo "@@@ pod-${node}-${ns}-${pod}--${container}.log @@@"
-      docker exec "$master_name" kubectl logs -n "${ns}" -c "${container}" "${pod}"
-    done
-  echo "@@@ kubectl-all.txt @@@"
-  docker exec "$master_name" kubectl get all --all-namespaces -o wide
-  echo "@@@ describe-all.txt @@@"
-  docker exec "$master_name" kubectl describe all --all-namespaces
-  echo "@@@ nodes.txt @@@"
-  docker exec "$master_name" kubectl get nodes -o wide
-}
-
-function dind::dump64 {
-  echo "%%% start-base64 %%%"
-  dind::dump | docker exec -i "$(dind::master-name)" /bin/sh -c "lzma | base64 -w 100"
-  echo "%%% end-base64 %%%"
-}
-
-function dind::split-dump {
-  mkdir -p cluster-dump
-  cd cluster-dump
-  awk '!/^@@@ .* @@@$/{print >out}; /^@@@ .* @@@$/{out=$2}' out=/dev/null
-  ls -l
-}
-
-function dind::split-dump64 {
-  decode_opt=-d
-  if base64 --help | grep -q '^ *-D'; then
-    # Mac OS X
-    decode_opt=-D
-  fi
-  sed -n '/^%%% start-base64 %%%$/,/^%%% end-base64 %%%$/p' |
-    sed '1d;$d' |
-    base64 "${decode_opt}" |
-    lzma -dc |
-    dind::split-dump
-}
-
-function dind::proxy {
-  local container_id="$1"
-  if [[ ${DIND_CA_CERT_URL} ]] ; then
-    dind::step "+ Adding certificate on ${container_id}"
-    docker exec ${container_id} /bin/sh -c "cd /usr/local/share/ca-certificates; curl -sSO ${DIND_CA_CERT_URL}"
-    docker exec ${container_id} update-ca-certificates
-  fi
-  if [[ "${DIND_PROPAGATE_HTTP_PROXY}" || "${DIND_HTTP_PROXY}" || "${DIND_HTTPS_PROXY}" || "${DIND_NO_PROXY}" ]]; then
-    dind::step "+ Setting *_PROXY for docker service on ${container_id}"
-    local proxy_env="[Service]"$'\n'"Environment="
-    if [[ "${DIND_PROPAGATE_HTTP_PROXY}" ]]; then
-      # take *_PROXY values from container environment
-      proxy_env+=$(docker exec ${container_id} env | grep -i _proxy | awk '{ print "\""$0"\""}' | xargs -d'\n')
-    else
-      if [[ "${DIND_HTTP_PROXY}" ]] ;  then proxy_env+="\"HTTP_PROXY=${DIND_HTTP_PROXY}\" "; fi
-      if [[ "${DIND_HTTPS_PROXY}" ]] ; then proxy_env+="\"HTTPS_PROXY=${DIND_HTTPS_PROXY}\" "; fi
-      if [[ "${DIND_NO_PROXY}" ]] ;    then proxy_env+="\"NO_PROXY=${DIND_NO_PROXY}\" "; fi
-    fi
-    docker exec -i ${container_id} /bin/sh -c "cat > /etc/systemd/system/docker.service.d/30-proxy.conf" <<< "${proxy_env}"
-    docker exec ${container_id} systemctl daemon-reload
-    docker exec ${container_id} systemctl restart docker
-  fi
-}
-
-function dind::custom-docker-opts {
-  local container_id="$1"
-  local -a jq=()
-  if [[ ! -f ${DIND_DAEMON_JSON_FILE} ]] ; then
-    jq[0]="{}"
-  else
-    jq+=("$(cat ${DIND_DAEMON_JSON_FILE})")
-  fi
-  if [[ ${DIND_REGISTRY_MIRROR} ]] ; then
-    dind::step "+ Setting up registry mirror on ${container_id}"
-    jq+=("{\"registry-mirrors\": [\"${DIND_REGISTRY_MIRROR}\"]}")
-  fi
-  if [[ ${DIND_INSECURE_REGISTRIES} ]] ; then
-    dind::step "+ Setting up insecure-registries on ${container_id}"
-    jq+=("{\"insecure-registries\": ${DIND_INSECURE_REGISTRIES}}")
-  fi
-  if [[ ${jq} ]] ; then
-    local json=$(IFS="+"; echo "${jq[*]}")
-    docker exec -i ${container_id} /bin/sh -c "mkdir -p /etc/docker && jq -n '${json}' > /etc/docker/daemon.json"
-    docker exec ${container_id} systemctl daemon-reload
-    docker exec ${container_id} systemctl restart docker
-  fi
-}
-
-COMMAND="${1:-}"
-
-case ${COMMAND} in
-  up)
-    if [[ ! ( ${DIND_IMAGE} =~ local ) && ! ${DIND_SKIP_PULL:-} ]]; then
-      dind::step "Making sure DIND image is up to date"
-      docker pull "${DIND_IMAGE}" >&2
-    fi
-
-    dind::prepare-sys-mounts
-    dind::ensure-kubectl
-    if [[ ${SKIP_SNAPSHOT} ]]; then
-      force_make_binaries=y dind::up
-    elif ! dind::check-for-snapshot; then
-      force_make_binaries=y dind::up
-      dind::snapshot
-    else
-      dind::restore
-    fi
-    ;;
-  reup)
-    dind::prepare-sys-mounts
-    dind::ensure-kubectl
-    if [[ ${SKIP_SNAPSHOT} ]]; then
-      force_make_binaries=y dind::up
-    elif ! dind::check-for-snapshot; then
-      force_make_binaries=y dind::up
-      dind::snapshot
-    else
-      force_make_binaries=y
-      restore_cmd=update_and_restore
-      dind::restore
-    fi
-    ;;
-  down)
-    dind::down
-    ;;
-  init)
-    shift
-    dind::prepare-sys-mounts
-    dind::ensure-kubectl
-    dind::init "$@"
-    ;;
-  join)
-    shift
-    dind::prepare-sys-mounts
-    dind::ensure-kubectl
-    dind::join "$(dind::create-node-container)" "$@"
-    ;;
-  # bare)
-  #   shift
-  #   dind::bare "$@"
-  #   ;;
-  snapshot)
-    shift
-    dind::snapshot
-    ;;
-  restore)
-    shift
-    dind::restore
-    ;;
-  clean)
-    dind::clean
-    ;;
-  copy-image)
-    dind::copy-image "$@"
-    ;;
-  e2e)
-    shift
-    dind::run-e2e "$@"
-    ;;
-  e2e-serial)
-    shift
-    dind::run-e2e-serial "$@"
-    ;;
-  dump)
-    dind::dump
-    ;;
-  dump64)
-    dind::dump64
-    ;;
-  split-dump)
-    dind::split-dump
-    ;;
-  split-dump64)
-    dind::split-dump64
-    ;;
-  apiserver-port)
-    dind::apiserver-port
-    ;;
-  *)
-    echo "usage:" >&2
-    echo "  $0 up" >&2
-    echo "  $0 reup" >&2
-    echo "  $0 down" >&2
-    echo "  $0 init kubeadm-args..." >&2
-    echo "  $0 join kubeadm-args..." >&2
-    # echo "  $0 bare container_name [docker_options...]"
-    echo "  $0 clean"
-    echo "  $0 copy-image [image_name]" >&2
-    echo "  $0 e2e [test-name-substring]" >&2
-    echo "  $0 e2e-serial [test-name-substring]" >&2
-    echo "  $0 dump" >&2
-    echo "  $0 dump64" >&2
-    echo "  $0 split-dump" >&2
-    echo "  $0 split-dump64" >&2
-    exit 1
-    ;;
-esac
\ No newline at end of file
diff --git a/test/e2e/up.sh b/test/e2e/up.sh
index 470dde0e9..a16b5c73c 100755
--- a/test/e2e/up.sh
+++ b/test/e2e/up.sh
@@ -22,7 +22,7 @@ if test -e kubectl; then
   echo "skipping download of kubectl"
 else
   echo "downloading kubectl..."
-  curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/v1.11.0/bin/linux/amd64/kubectl && \
+  curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/v1.12.0/bin/linux/amd64/kubectl && \
       chmod +x kubectl && sudo mv kubectl /usr/local/bin/
 fi
 
@@ -31,6 +31,9 @@ touch ${HOME}/.kube/config
 export KUBECONFIG=${HOME}/.kube/config
 
 echo "starting Kubernetes cluster..."
+curl -Lo $DIR/dind-cluster-v1.11.sh https://raw.githubusercontent.com/kubernetes-sigs/kubeadm-dind-cluster/master/fixed/dind-cluster-v1.11.sh && \
+  chmod +x $DIR/dind-cluster-v1.11.sh
+
 $DIR/dind-cluster-v1.11.sh up
 
 kubectl config use-context dind

From 04e6f7bd8228a64a4862f5e8d628b618751b1b3b Mon Sep 17 00:00:00 2001
From: Anish Ramasekar <ramasekar@us.ibm.com>
Date: Thu, 25 Oct 2018 16:42:08 -0500
Subject: [PATCH 08/31] Add e2e test for force-ssl-redirect

---
 test/e2e/annotations/forcesslredirect.go | 75 ++++++++++++++++++++++++
 1 file changed, 75 insertions(+)
 create mode 100644 test/e2e/annotations/forcesslredirect.go

diff --git a/test/e2e/annotations/forcesslredirect.go b/test/e2e/annotations/forcesslredirect.go
new file mode 100644
index 000000000..7b542b286
--- /dev/null
+++ b/test/e2e/annotations/forcesslredirect.go
@@ -0,0 +1,75 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package annotations
+
+import (
+	"fmt"
+	"net/http"
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/parnurzeal/gorequest"
+	"k8s.io/ingress-nginx/test/e2e/framework"
+)
+
+var _ = framework.IngressNginxDescribe("Annotations - Forcesslredirect", func() {
+	f := framework.NewDefaultFramework("forcesslredirect")
+
+	BeforeEach(func() {
+		err := f.NewEchoDeploymentWithReplicas(2)
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	AfterEach(func() {
+	})
+
+	It("should redirect to https", func() {
+		host := "forcesslredirect.bar.com"
+
+		annotations := map[string]string{
+			"nginx.ingress.kubernetes.io/force-ssl-redirect": "true",
+		}
+
+		ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations)
+		_, err := f.EnsureIngress(ing)
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(ing).NotTo(BeNil())
+
+		err = f.WaitForNginxServer(host,
+			func(server string) bool {
+				return Expect(server).Should(ContainSubstring(`if ($redirect_to_https) {`)) &&
+					Expect(server).Should(ContainSubstring(`return 308 https://$best_http_host$request_uri;`))
+			})
+		Expect(err).NotTo(HaveOccurred())
+
+		_, _, errs := gorequest.New().
+			Get(f.IngressController.HTTPURL).
+			Retry(10, 1*time.Second, http.StatusNotFound).
+			Set("Host", host).
+			End()
+
+		Expect(errs[0].Error()).Should(ContainSubstring(`https://forcesslredirect.bar.com/`))
+
+		log, err := f.NginxLogs()
+		Expect(err).ToNot(HaveOccurred())
+		Expect(log).ToNot(BeEmpty())
+
+		Expect(log).To(ContainSubstring(fmt.Sprintf(` "GET / HTTP/1.1" 308 171 "-" "Go-http-client/1.1"`)))
+	})
+})

From 519c30ea70c9d209e08f5f946177aa0558538b4a Mon Sep 17 00:00:00 2001
From: Anish Ramasekar <ramasekar@us.ibm.com>
Date: Fri, 26 Oct 2018 11:12:44 -0500
Subject: [PATCH 09/31] Add e2e test for app-root

---
 test/e2e/annotations/approot.go | 71 +++++++++++++++++++++++++++++++++
 1 file changed, 71 insertions(+)
 create mode 100644 test/e2e/annotations/approot.go

diff --git a/test/e2e/annotations/approot.go b/test/e2e/annotations/approot.go
new file mode 100644
index 000000000..b0daa5007
--- /dev/null
+++ b/test/e2e/annotations/approot.go
@@ -0,0 +1,71 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package annotations
+
+import (
+	"net/http"
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/parnurzeal/gorequest"
+	"k8s.io/ingress-nginx/test/e2e/framework"
+)
+
+var _ = framework.IngressNginxDescribe("Annotations - Approot", func() {
+	f := framework.NewDefaultFramework("approot")
+
+	BeforeEach(func() {
+		err := f.NewEchoDeploymentWithReplicas(2)
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	AfterEach(func() {
+	})
+
+	It("should redirect to /foo", func() {
+		host := "approot.bar.com"
+
+		annotations := map[string]string{
+			"nginx.ingress.kubernetes.io/app-root": "/foo",
+		}
+
+		ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations)
+		_, err := f.EnsureIngress(ing)
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(ing).NotTo(BeNil())
+
+		err = f.WaitForNginxServer(host,
+			func(server string) bool {
+				return Expect(server).Should(ContainSubstring(`if ($uri = /) {`)) &&
+					Expect(server).Should(ContainSubstring(`return 302 /foo;`))
+			})
+		Expect(err).NotTo(HaveOccurred())
+
+		resp, _, errs := gorequest.New().
+			Get(f.IngressController.HTTPURL).
+			Retry(10, 1*time.Second, http.StatusNotFound).
+			RedirectPolicy(noRedirectPolicyFunc).
+			Set("Host", host).
+			End()
+
+		Expect(len(errs)).Should(BeNumerically("==", 0))
+		Expect(resp.StatusCode).Should(Equal(http.StatusFound))
+		Expect(resp.Header.Get("Location")).Should(Equal("http://approot.bar.com/foo"))
+	})
+})

From d3a82f7b04d97780b38104413dddf4d989035c53 Mon Sep 17 00:00:00 2001
From: Anish Ramasekar <ramasekar@us.ibm.com>
Date: Fri, 26 Oct 2018 11:21:44 -0500
Subject: [PATCH 10/31] enhance test logic

---
 test/e2e/annotations/forcesslredirect.go  | 14 +++++---------
 test/e2e/annotations/fromtowwwredirect.go | 11 +++++------
 2 files changed, 10 insertions(+), 15 deletions(-)

diff --git a/test/e2e/annotations/forcesslredirect.go b/test/e2e/annotations/forcesslredirect.go
index 7b542b286..dd3f963fe 100644
--- a/test/e2e/annotations/forcesslredirect.go
+++ b/test/e2e/annotations/forcesslredirect.go
@@ -17,7 +17,6 @@ limitations under the License.
 package annotations
 
 import (
-	"fmt"
 	"net/http"
 	"time"
 
@@ -58,18 +57,15 @@ var _ = framework.IngressNginxDescribe("Annotations - Forcesslredirect", func()
 			})
 		Expect(err).NotTo(HaveOccurred())
 
-		_, _, errs := gorequest.New().
+		resp, _, errs := gorequest.New().
 			Get(f.IngressController.HTTPURL).
 			Retry(10, 1*time.Second, http.StatusNotFound).
+			RedirectPolicy(noRedirectPolicyFunc).
 			Set("Host", host).
 			End()
 
-		Expect(errs[0].Error()).Should(ContainSubstring(`https://forcesslredirect.bar.com/`))
-
-		log, err := f.NginxLogs()
-		Expect(err).ToNot(HaveOccurred())
-		Expect(log).ToNot(BeEmpty())
-
-		Expect(log).To(ContainSubstring(fmt.Sprintf(` "GET / HTTP/1.1" 308 171 "-" "Go-http-client/1.1"`)))
+		Expect(len(errs)).Should(BeNumerically("==", 0))
+		Expect(resp.StatusCode).Should(Equal(http.StatusPermanentRedirect))
+		Expect(resp.Header.Get("Location")).Should(Equal("https://forcesslredirect.bar.com/"))
 	})
 })
diff --git a/test/e2e/annotations/fromtowwwredirect.go b/test/e2e/annotations/fromtowwwredirect.go
index 7ae32e36f..b5b292577 100644
--- a/test/e2e/annotations/fromtowwwredirect.go
+++ b/test/e2e/annotations/fromtowwwredirect.go
@@ -61,16 +61,15 @@ var _ = framework.IngressNginxDescribe("Annotations - Fromtowwwredirect", func()
 
 		By("sending request to www.fromtowwwredirect.bar.com")
 
-		gorequest.New().
+		resp, _, errs := gorequest.New().
 			Get(fmt.Sprintf("%s/%s", f.IngressController.HTTPURL, "foo")).
 			Retry(10, 1*time.Second, http.StatusNotFound).
+			RedirectPolicy(noRedirectPolicyFunc).
 			Set("Host", fmt.Sprintf("%s.%s", "www", host)).
 			End()
 
-		log, err := f.NginxLogs()
-		Expect(err).ToNot(HaveOccurred())
-		Expect(log).ToNot(BeEmpty())
-
-		Expect(log).To(ContainSubstring(fmt.Sprintf(` "GET /foo HTTP/1.1" 308 171 "-" "Go-http-client/1.1"`)))
+		Expect(len(errs)).Should(BeNumerically("==", 0))
+		Expect(resp.StatusCode).Should(Equal(http.StatusPermanentRedirect))
+		Expect(resp.Header.Get("Location")).Should(Equal("http://fromtowwwredirect.bar.com/foo"))
 	})
 })

From 213be6103f8e22298606ed970b19478c15f40bd4 Mon Sep 17 00:00:00 2001
From: Fernando Diaz <diazjf@us.ibm.com>
Date: Thu, 25 Oct 2018 21:56:54 -0500
Subject: [PATCH 11/31] Add e2e test for server snippet

Adds the e2e test for server snippet. Similar test to the
configuration snippet. Part of the #3124 effort.
---
 test/e2e/annotations/serversnippet.go | 57 +++++++++++++++++++++++++++
 1 file changed, 57 insertions(+)
 create mode 100644 test/e2e/annotations/serversnippet.go

diff --git a/test/e2e/annotations/serversnippet.go b/test/e2e/annotations/serversnippet.go
new file mode 100644
index 000000000..e96483f9f
--- /dev/null
+++ b/test/e2e/annotations/serversnippet.go
@@ -0,0 +1,57 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package annotations
+
+import (
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"k8s.io/ingress-nginx/test/e2e/framework"
+	"strings"
+)
+
+var _ = framework.IngressNginxDescribe("Annotations - ServerSnippet", func() {
+	f := framework.NewDefaultFramework("serversnippet")
+
+	BeforeEach(func() {
+		err := f.NewEchoDeploymentWithReplicas(2)
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	AfterEach(func() {
+	})
+
+	It(`add valid directives to server via server snippet"`, func() {
+		host := "serversnippet.foo.com"
+		annotations := map[string]string{
+			"nginx.ingress.kubernetes.io/server-snippet": `
+				more_set_headers "Content-Length: $content_length";
+				more_set_headers "Content-Type: $content_type";`,
+		}
+
+		ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations)
+		_, err := f.EnsureIngress(ing)
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(ing).NotTo(BeNil())
+
+		err = f.WaitForNginxServer(host,
+			func(server string) bool {
+				return strings.Contains(server, `more_set_headers "Content-Length: $content_length`) && strings.Contains(server, `more_set_headers "Content-Type: $content_type";`)
+			})
+		Expect(err).NotTo(HaveOccurred())
+	})
+})

From 49ac77e01bd4a0e5dbf7012c9fa7d4aa61d18646 Mon Sep 17 00:00:00 2001
From: Fernando Diaz <diazjf@us.ibm.com>
Date: Thu, 25 Oct 2018 21:30:39 -0500
Subject: [PATCH 12/31] Add e2e tests for IP Whitelist

Adds e2e test for the whitelist-source-range annotation.
Part of the #3124 effort.
---
 test/e2e/annotations/ipwhitelist.go | 85 +++++++++++++++++++++++++++++
 1 file changed, 85 insertions(+)
 create mode 100644 test/e2e/annotations/ipwhitelist.go

diff --git a/test/e2e/annotations/ipwhitelist.go b/test/e2e/annotations/ipwhitelist.go
new file mode 100644
index 000000000..f5ac47e63
--- /dev/null
+++ b/test/e2e/annotations/ipwhitelist.go
@@ -0,0 +1,85 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package annotations
+
+import (
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"k8s.io/ingress-nginx/test/e2e/framework"
+	"regexp"
+	"strings"
+)
+
+var _ = framework.IngressNginxDescribe("Annotations - IPWhiteList", func() {
+	f := framework.NewDefaultFramework("ipwhitelist")
+
+	BeforeEach(func() {
+		err := f.NewEchoDeploymentWithReplicas(2)
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	AfterEach(func() {
+	})
+
+	It("should set valid ip whitelist range", func() {
+		host := "ipwhitelist.foo.com"
+		nameSpace := f.IngressController.Namespace
+
+		annotations := map[string]string{
+			"nginx.ingress.kubernetes.io/whitelist-source-range": "18.0.0.0/8, 56.0.0.0/8",
+		}
+
+		ing := framework.NewSingleIngress(host, "/", host, nameSpace, "http-svc", 80, &annotations)
+		_, err := f.EnsureIngress(ing)
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(ing).NotTo(BeNil())
+
+		denyRegex := regexp.MustCompile("geo \\$the_real_ip \\$deny_[A-Za-z]{32}")
+		denyString := ""
+
+		err = f.WaitForNginxConfiguration(
+			func(conf string) bool {
+
+				match := denyRegex.FindStringSubmatch(conf)
+				// If no match found, return false
+				if !(len(match) > 0) {
+					return false
+				}
+
+				denyString = strings.Replace(match[0], "geo $the_real_ip ", "", -1)
+				return strings.Contains(conf, match[0])
+			})
+		Expect(err).NotTo(HaveOccurred())
+
+		ipOne := "18.0.0.0/8 0;"
+		ipTwo := "56.0.0.0/8 0;"
+
+		err = f.WaitForNginxConfiguration(
+			func(conf string) bool {
+				return strings.Contains(conf, ipOne) && strings.Contains(conf, ipTwo)
+			})
+		Expect(err).NotTo(HaveOccurred())
+
+		denyStatement := "if (" + denyString + ")"
+		err = f.WaitForNginxServer(host,
+			func(server string) bool {
+				return strings.Contains(server, denyStatement)
+			})
+		Expect(err).NotTo(HaveOccurred())
+	})
+})

From 9d227ab62db6068d1bbec84a1ad2e04ce5ad947c Mon Sep 17 00:00:00 2001
From: mikeweiwei <466462983@qq.com>
Date: Sat, 27 Oct 2018 19:16:34 +0800
Subject: [PATCH 13/31] fix logging calls (#3260)

---
 cmd/nginx/main.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/cmd/nginx/main.go b/cmd/nginx/main.go
index c76b27c4a..667035517 100644
--- a/cmd/nginx/main.go
+++ b/cmd/nginx/main.go
@@ -94,7 +94,7 @@ func main() {
 		if err != nil {
 			// TODO (antoineco): compare with error types from k8s.io/apimachinery/pkg/api/errors
 			if strings.Contains(err.Error(), "cannot get services in the namespace") {
-				glog.Fatalf("✖ The cluster seems to be running with a restrictive Authorization mode and the Ingress controller does not have the required permissions to operate normally.")
+				glog.Fatal("✖ The cluster seems to be running with a restrictive Authorization mode and the Ingress controller does not have the required permissions to operate normally.")
 			}
 			glog.Fatalf("No service with name %v found: %v", conf.DefaultService, err)
 		}
@@ -160,7 +160,7 @@ func handleSigterm(ngx *controller.NGINXController, exit exiter) {
 	signalChan := make(chan os.Signal, 1)
 	signal.Notify(signalChan, syscall.SIGTERM)
 	<-signalChan
-	glog.Infof("Received SIGTERM, shutting down")
+	glog.Info("Received SIGTERM, shutting down")
 
 	exitCode := 0
 	if err := ngx.Stop(); err != nil {
@@ -168,7 +168,7 @@ func handleSigterm(ngx *controller.NGINXController, exit exiter) {
 		exitCode = 1
 	}
 
-	glog.Infof("Handled quit, awaiting Pod deletion")
+	glog.Info("Handled quit, awaiting Pod deletion")
 	time.Sleep(10 * time.Second)
 
 	glog.Infof("Exiting with %v", exitCode)

From decdf72f26f3cff9b8a6ab867a866e0a5a2ff8ac Mon Sep 17 00:00:00 2001
From: samuela <skainsworth@gmail.com>
Date: Sat, 27 Oct 2018 04:17:45 -0700
Subject: [PATCH 14/31] "diretly" typo (#3263)

* "diretly" typo
---
 docs/user-guide/cli-arguments.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/user-guide/cli-arguments.md b/docs/user-guide/cli-arguments.md
index dbd2f4ff0..70c7549d7 100644
--- a/docs/user-guide/cli-arguments.md
+++ b/docs/user-guide/cli-arguments.md
@@ -10,7 +10,7 @@ They are set in the container spec of the `nginx-ingress-controller` Deployment
 | `--annotations-prefix string`     | Prefix of the Ingress annotations specific to the NGINX controller. (default "nginx.ingress.kubernetes.io") |
 | `--apiserver-host string`         | Address of the Kubernetes API server. Takes the form "protocol://address:port". If not specified, it is assumed the program runs inside a Kubernetes cluster and local discovery is attempted. |
 | `--configmap string`              | Name of the ConfigMap containing custom global configurations for the controller. |
-| `--default-backend-service string` | Service used to serve HTTP requests not matching any known server name (catch-all). Takes the form "namespace/name". The controller configures NGINX to forward requests to the first port of this Service. If not specified, 404 page will be returned diretly from Nginx.|
+| `--default-backend-service string` | Service used to serve HTTP requests not matching any known server name (catch-all). Takes the form "namespace/name". The controller configures NGINX to forward requests to the first port of this Service. If not specified, a 404 page will be returned directly from NGINX.|
 | `--default-server-port int`       | When `default-backend-service` is not specified or specified service does not have any endpoint, a local endpoint with this port will be used to serve 404 page from inside Nginx. |
 | `--default-ssl-certificate string` | Secret containing a SSL certificate to be used by the default HTTPS server (catch-all). Takes the form "namespace/name". |
 | `--election-id string`            | Election id to use for Ingress status updates. (default "ingress-controller-leader") |

From 0ab6d4a6886003ff3594fa629a9c1f08841f5439 Mon Sep 17 00:00:00 2001
From: Maximilian Bode <maximilian.bode@tngtech.com>
Date: Sun, 28 Oct 2018 21:26:31 +0100
Subject: [PATCH 15/31] Bump geoip2 module version 3.0 -> 3.2

---
 images/nginx/README.md       | 1 +
 images/nginx/rootfs/build.sh | 4 ++--
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/images/nginx/README.md b/images/nginx/README.md
index 1f3588f9a..ca0d889f6 100644
--- a/images/nginx/README.md
+++ b/images/nginx/README.md
@@ -14,6 +14,7 @@ This custom nginx image contains:
 - [zipkin-cpp-opentracing](https://github.com/rnburn/zipkin-cpp-opentracing)
 - [ModSecurity-nginx](https://github.com/SpiderLabs/ModSecurity-nginx) (only supported in x86_64)
 - [brotli](https://github.com/google/brotli)
+- [geoip2](https://github.com/leev/ngx_http_geoip2_module)
 
 **How to use this image:**
 This image provides a default configuration file with no backend servers.
diff --git a/images/nginx/rootfs/build.sh b/images/nginx/rootfs/build.sh
index b459c15d9..f52ef543c 100755
--- a/images/nginx/rootfs/build.sh
+++ b/images/nginx/rootfs/build.sh
@@ -33,7 +33,7 @@ export MODSECURITY_VERSION=37b76e88df4bce8a9846345c27271d7e6ce1acfb
 export LUA_NGX_VERSION=e94f2e5d64daa45ff396e262d8dab8e56f5f10e0
 export LUA_UPSTREAM_VERSION=0.07
 export NGINX_INFLUXDB_VERSION=f20cfb2458c338f162132f5a21eb021e2cbe6383
-export GEOIP2_VERSION=3.0
+export GEOIP2_VERSION=3.2
 export NGINX_AJP_VERSION=bf6cd93f2098b59260de8d494f0f4b1f11a84627
 export LUAJIT_VERSION=8e35a1932250b0313c06393061f332c760efdf40
 
@@ -208,7 +208,7 @@ get_src 5a4485be0031d285f2bdf59afb1f7b8f3cef4c476595ed66f1258206e1b5c3ac \
 get_src 1897d7677d99c1cedeb95b2eb00652a4a7e8e604304c3053a93bd3ba7dd82884 \
         "https://github.com/influxdata/nginx-influxdb-module/archive/$NGINX_INFLUXDB_VERSION.tar.gz"
 
-get_src 65a191688348a05d8d92b2e7ce9c6eb8cb8322205c34637da582a1205864133d \
+get_src 15bd1005228cf2c869a6f09e8c41a6aaa6846e4936c473106786ae8ac860fab7 \
         "https://github.com/leev/ngx_http_geoip2_module/archive/$GEOIP2_VERSION.tar.gz"
 
 get_src 5f629a50ba22347c441421091da70fdc2ac14586619934534e5a0f8a1390a950 \

From 77fed7bf68e2d06a9c5c71f6a2114af33fe547ed Mon Sep 17 00:00:00 2001
From: Maximilian Bode <maximilian.bode@tngtech.com>
Date: Sun, 28 Oct 2018 21:27:18 +0100
Subject: [PATCH 16/31] Fix download of geoip2 databases

---
 images/nginx/Makefile        |  2 +-
 images/nginx/rootfs/build.sh | 14 +++++++++-----
 2 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/images/nginx/Makefile b/images/nginx/Makefile
index 500c961d3..f90f13247 100644
--- a/images/nginx/Makefile
+++ b/images/nginx/Makefile
@@ -13,7 +13,7 @@
 # limitations under the License.
 
 # 0.0.0 shouldn't clobber any released builds
-TAG ?= 0.65
+TAG ?= 0.66
 REGISTRY ?= quay.io/kubernetes-ingress-controller
 ARCH ?= $(shell go env GOARCH)
 DOCKER ?= docker
diff --git a/images/nginx/rootfs/build.sh b/images/nginx/rootfs/build.sh
index f52ef543c..60cb5ecc3 100755
--- a/images/nginx/rootfs/build.sh
+++ b/images/nginx/rootfs/build.sh
@@ -125,12 +125,16 @@ function geoip_get {
   wget -O $GEOIP_FOLDER/$1 $2 || { echo "Could not download $1, exiting." ; exit 1; }
   gunzip $GEOIP_FOLDER/$1
 }
+function geoip2_get {
+  wget -O $GEOIP_FOLDER/$1.tar.gz $2 || { echo "Could not download $1, exiting." ; exit 1; }
+  mkdir $GEOIP_FOLDER/$1 && tar xf $GEOIP_FOLDER/$1.tar.gz -C $GEOIP_FOLDER/$1 --strip-components 1 && mv $GEOIP_FOLDER/$1/$1.mmdb $GEOIP_FOLDER/$1.mmdb && rm -rf $GEOIP_FOLDER/$1
+}
 
-geoip_get "GeoIPASNum.dat.gz"     "http://download.maxmind.com/download/geoip/database/asnum/GeoIPASNum.dat.gz"
-geoip_get "GeoIP.dat.gz"          "https://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz"
-geoip_get "GeoLite2-City.mmdb.gz" "http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz"
-geoip_get "GeoLite2-ASN.mmdb.gz"  "http://geolite.maxmind.com/download/geoip/database/GeoLite2-ASN.tar.gz"
-geoip_get "GeoLiteCity.dat.gz"    "https://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz"
+geoip_get "GeoIPASNum.dat.gz"  "http://download.maxmind.com/download/geoip/database/asnum/GeoIPASNum.dat.gz"
+geoip_get "GeoIP.dat.gz"       "https://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz"
+geoip_get "GeoLiteCity.dat.gz" "https://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz"
+geoip2_get "GeoLite2-City"     "http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz"
+geoip2_get "GeoLite2-ASN"      "http://geolite.maxmind.com/download/geoip/database/GeoLite2-ASN.tar.gz"
 
 mkdir --verbose -p "$BUILD_PATH"
 cd "$BUILD_PATH"

From ed107a489aa48f8b43568aeaef0da374fcc84f36 Mon Sep 17 00:00:00 2001
From: xichengliudui <liuduidui@beyondcent.com>
Date: Mon, 29 Oct 2018 02:48:56 -0400
Subject: [PATCH 17/31] Delete some extra words

---
 docs/examples/auth/client-certs/README.md    | 2 +-
 docs/user-guide/tls.md                       | 2 +-
 internal/ingress/metric/collectors/socket.go | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/examples/auth/client-certs/README.md b/docs/examples/auth/client-certs/README.md
index baa91eccd..072e1d69f 100644
--- a/docs/examples/auth/client-certs/README.md
+++ b/docs/examples/auth/client-certs/README.md
@@ -4,7 +4,7 @@ It is possible to enable Client-Certificate Authentication by adding additional
 Before getting started you must have the following Certificates Setup:
 
 1. CA certificate and Key(Intermediate Certs need to be in CA)
-2. Server Certificate(Signed by CA) and Key (CN should be equal the the hostname you will use)
+2. Server Certificate(Signed by CA) and Key (CN should be equal the hostname you will use)
 3. Client Certificate(Signed by CA) and Key
 
 ## Creating Certificate Secrets
diff --git a/docs/user-guide/tls.md b/docs/user-guide/tls.md
index 9b6961423..c468fa8ab 100644
--- a/docs/user-guide/tls.md
+++ b/docs/user-guide/tls.md
@@ -4,7 +4,7 @@
 
 Anytime we reference a TLS secret, we mean a PEM-encoded X.509, RSA (2048) secret.
 
-You can generate a self-signed certificate and private key with with:
+You can generate a self-signed certificate and private key with:
 
 ```bash
 $ openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout ${KEY_FILE} -out ${CERT_FILE} -subj "/CN=${HOST}/O=${HOST}"`
diff --git a/internal/ingress/metric/collectors/socket.go b/internal/ingress/metric/collectors/socket.go
index 1f30c9b72..904921d12 100644
--- a/internal/ingress/metric/collectors/socket.go
+++ b/internal/ingress/metric/collectors/socket.go
@@ -171,7 +171,7 @@ func NewSocketCollector(pod, namespace, class string) (*SocketCollector, error)
 		bytesSent: prometheus.NewHistogramVec(
 			prometheus.HistogramOpts{
 				Name:        "bytes_sent",
-				Help:        "The the number of bytes sent to a client",
+				Help:        "The number of bytes sent to a client",
 				Namespace:   PrometheusNamespace,
 				Buckets:     prometheus.ExponentialBuckets(10, 10, 7), // 7 buckets, exponential factor of 10.
 				ConstLabels: constLabels,

From 4e124675fa11de17aae613ad8f4b0968b9a6d6e7 Mon Sep 17 00:00:00 2001
From: Manuel Alejandro de Brito Fontes <aledbf@gmail.com>
Date: Mon, 29 Oct 2018 09:20:35 -0300
Subject: [PATCH 18/31] Update nginx and add support for TLSv1.3

---
 images/nginx/rootfs/build.sh | 86 +++++++++++++++++++++++++-----------
 1 file changed, 59 insertions(+), 27 deletions(-)

diff --git a/images/nginx/rootfs/build.sh b/images/nginx/rootfs/build.sh
index 60cb5ecc3..153bb6e80 100755
--- a/images/nginx/rootfs/build.sh
+++ b/images/nginx/rootfs/build.sh
@@ -53,10 +53,6 @@ get_src()
   rm -rf "$f"
 }
 
-if [[ ${ARCH} == "ppc64le" ]]; then
-  clean-install software-properties-common
-fi
-
 apt-get update && apt-get dist-upgrade -y
 
 # install required packages to build
@@ -94,8 +90,15 @@ clean-install \
   dumb-init \
   gdb \
   valgrind \
+  bc \
   || exit 1
 
+if [[ ${ARCH} == "ppc64le" ]]; then
+  wget http://ftp.us.debian.org/debian/pool/main/a/apt/libapt-pkg5.0_1.7.0_ppc64el.deb
+  dpkg -i libapt-pkg5.0_1.7.0_ppc64el.deb
+  clean-install python3-apt python3-software-properties software-properties-common
+fi
+
 if [[ ${ARCH} == "x86_64" ]]; then
   ln -s /usr/lib/x86_64-linux-gnu/liblua5.1.so /usr/lib/liblua.so
   ln -s /usr/lib/x86_64-linux-gnu /usr/lib/lua-platform-path
@@ -136,6 +139,13 @@ geoip_get "GeoLiteCity.dat.gz" "https://geolite.maxmind.com/download/geoip/datab
 geoip2_get "GeoLite2-City"     "http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz"
 geoip2_get "GeoLite2-ASN"      "http://geolite.maxmind.com/download/geoip/database/GeoLite2-ASN.tar.gz"
 
+if [[ (${ARCH} == "ppc64le") ]]; then
+  echo "deb http://deb.debian.org/debian experimental main" >> /etc/apt/sources.list
+  apt-get update
+  apt-get -t experimental install -y luajit
+fi
+
+
 mkdir --verbose -p "$BUILD_PATH"
 cd "$BUILD_PATH"
 
@@ -167,7 +177,7 @@ get_src 4455ca507936bc4b658ded10a90d8ebbbd61c58f06207be565a4ffdc885687b5 \
 get_src 30affaf0f3a84193f7127cc0135da91773ce45d902414082273dae78914f73df \
         "https://github.com/rnburn/zipkin-cpp-opentracing/archive/v$ZIPKIN_CPP_VERSION.tar.gz"
 
-get_src fe7d3188e097d68f1942d46c4adba262d9ddcf433409ebc15bb5355bfb001a4a \
+get_src a75e3c0249c8ce4313d21b43d3cf3dcd89518dd6582ef7c6697cb7fe6ef5a84e \
         "https://github.com/SpiderLabs/ModSecurity-nginx/archive/$MODSECURITY_VERSION.tar.gz"
 
 get_src b68286966f292fb552511b71bd8bc11af8f12c8aa760372d1437ac8760cb2f25 \
@@ -206,10 +216,10 @@ get_src a77bf0d7cf6a9ba017d0dc973b1a58f13e48242dd3849c5e99c07d250667c44c \
 get_src d81b33129c6fb5203b571fa4d8394823bf473d8872c0357a1d0f14420b1483bd \
         "https://github.com/cloudflare/lua-resty-cookie/archive/v0.1.0.tar.gz"
 
-get_src 5a4485be0031d285f2bdf59afb1f7b8f3cef4c476595ed66f1258206e1b5c3ac \
+get_src 21dab7625a028d4560d0215c4bc3b82f6153344f933abb99dc9fd5f0d19519ab \
         "https://github.com/openresty/luajit2/archive/$LUAJIT_VERSION.tar.gz"
 
-get_src 1897d7677d99c1cedeb95b2eb00652a4a7e8e604304c3053a93bd3ba7dd82884 \
+get_src c673fcee37c1c4794f921b6710b09e8a0e1e58117aa788f798507d033f737192 \
         "https://github.com/influxdata/nginx-influxdb-module/archive/$NGINX_INFLUXDB_VERSION.tar.gz"
 
 get_src 15bd1005228cf2c869a6f09e8c41a6aaa6846e4936c473106786ae8ac860fab7 \
@@ -225,21 +235,44 @@ export MAKEFLAGS=-j${CORES}
 export CTEST_BUILD_FLAGS=${MAKEFLAGS}
 export HUNTER_JOBS_NUMBER=${CORES}
 
+OPENSSL_DIR="$BUILD_PATH/openssl"
+mkdir -p $OPENSSL_DIR
+cd $OPENSSL_DIR
+
+# Install Openssl 1.1.1 from source
+wget http://http.debian.net/debian/pool/main/o/openssl/openssl_1.1.1-1.dsc
+wget http://http.debian.net/debian/pool/main/o/openssl/openssl_1.1.1.orig.tar.gz
+wget http://http.debian.net/debian/pool/main/o/openssl/openssl_1.1.1.orig.tar.gz.asc
+wget http://http.debian.net/debian/pool/main/o/openssl/openssl_1.1.1-1.debian.tar.xz
+
+tar zxpvf openssl_1.1.1.orig.tar.gz
+cd openssl-1.1.1/
+tar xpvf ../openssl_1.1.1-1.debian.tar.xz
+
+dpkg-buildpackage -rfakeroot
+
+cd ..
+
+dpkg -i openssl_1.1.1-1_amd64.deb libssl1.1_1.1.1-1_amd64.deb libssl-dev_1.1.1-1_amd64.deb
+
+# Install luajit from openresty fork
 export LUAJIT_LIB=/usr/local/lib
+export LUA_LIB_DIR="$LUAJIT_LIB/lua"
 
 # luajit is available only as deb package on ppc64le
-if [[ (${ARCH} == "ppc64le") ]]; then
-  clean-install luajit
-else
+if [[ (${ARCH} != "ppc64le") ]]; then
   cd "$BUILD_PATH/luajit2-$LUAJIT_VERSION"
   make CCDEBUG=-g
   make install
 
   export LUAJIT_INC=/usr/local/include/luajit-2.1
-  export LUA_LIB_DIR="$LUAJIT_LIB/lua"
 fi
 
 # Installing luarocks packages
+if [[ ${ARCH} == "x86_64" ]]; then
+  export PCRE_DIR=/usr/lib/x86_64-linux-gnu
+fi
+
 if [[ ${ARCH} == "armv7l" ]]; then
   export PCRE_DIR=/usr/lib/armhf-linux-gnu
 fi
@@ -252,7 +285,8 @@ if [[ ${ARCH} == "ppc64le" ]]; then
   export PCRE_DIR=/usr/lib/powerpc64le-linux-gnu
 fi
 
-luarocks install lrexlib-pcre 2.7.2-1
+cd "$BUILD_PATH"
+luarocks install lrexlib-pcre 2.7.2-1 PCRE_LIBDIR=${PCRE_DIR}
 
 cd "$BUILD_PATH/lua-resty-core-0.1.15"
 make install
@@ -330,14 +364,14 @@ EOF
 mkdir .build
 cd .build
 
-cmake   -DCMAKE_BUILD_TYPE=Release \
-        -DBUILD_TESTING=OFF \
-        -DJAEGERTRACING_BUILD_EXAMPLES=OFF \
-        -DJAEGERTRACING_BUILD_CROSSDOCK=OFF \
-        -DJAEGERTRACING_COVERAGE=OFF \
-        -DJAEGERTRACING_PLUGIN=ON \
-        -DHUNTER_CONFIGURATION_TYPES=Release \
-        -DJAEGERTRACING_WITH_YAML_CPP=ON ..
+cmake -DCMAKE_BUILD_TYPE=Release \
+      -DBUILD_TESTING=OFF \
+      -DJAEGERTRACING_BUILD_EXAMPLES=OFF \
+      -DJAEGERTRACING_BUILD_CROSSDOCK=OFF \
+      -DJAEGERTRACING_COVERAGE=OFF \
+      -DJAEGERTRACING_PLUGIN=ON \
+      -DHUNTER_CONFIGURATION_TYPES=Release \
+      -DJAEGERTRACING_WITH_YAML_CPP=ON ..
 
 make
 make install
@@ -360,10 +394,10 @@ EOF
 mkdir .build
 cd .build
 
-cmake   -DCMAKE_BUILD_TYPE=Release \
-        -DBUILD_SHARED_LIBS=ON \
-        -DBUILD_PLUGIN=ON \
-        -DBUILD_TESTING=OFF ..
+cmake -DCMAKE_BUILD_TYPE=Release \
+      -DBUILD_SHARED_LIBS=ON \
+      -DBUILD_PLUGIN=ON \
+      -DBUILD_TESTING=OFF ..
 
 make
 make install
@@ -379,9 +413,7 @@ git submodule update
 cd "$BUILD_PATH"
 git clone -b v3/master --single-branch https://github.com/SpiderLabs/ModSecurity
 cd ModSecurity/
-# TODO: use a tag once 3.0.3 is released
-# checkout v3.0.3
-# git checkout
+git checkout 973c1f1028429452308bcbce7df8a6283dc59ffe
 git submodule init
 git submodule update
 sh build.sh

From 4c25bfe75a99d24148bf6fb748c887761d160aa4 Mon Sep 17 00:00:00 2001
From: Sebastiaan van Steenis <mail@superseb.nl>
Date: Mon, 29 Oct 2018 14:07:54 +0100
Subject: [PATCH 19/31] Fix links in deploy index docs

---
 docs/deploy/index.md | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/docs/deploy/index.md b/docs/deploy/index.md
index 49d97b429..c1749061b 100644
--- a/docs/deploy/index.md
+++ b/docs/deploy/index.md
@@ -2,8 +2,7 @@
 
 ## Contents
 
-- [Generic Deployment](#generic-deployment)
-  - [Mandatory command](#mandatory-command)
+- [Prerequisite Generic Deployment Command](#prerequisite-generic-deployment-command)
   - [Provider Specific Steps](#provider-specific-steps)
     - [Docker for Mac](#docker-for-mac)
     - [minikube](#minikube)

From a42f181d739309e139a2b91f0ead0012da8b256e Mon Sep 17 00:00:00 2001
From: Manuel Alejandro de Brito Fontes <aledbf@gmail.com>
Date: Mon, 29 Oct 2018 10:58:06 -0300
Subject: [PATCH 20/31] Update nginx image and QEMU version

---
 Makefile                     | 4 ++--
 build/go-in-docker.sh        | 2 +-
 images/nginx/Makefile        | 2 +-
 images/nginx/rootfs/build.sh | 8 ++++----
 4 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/Makefile b/Makefile
index 36d67c0c1..37e38ed06 100644
--- a/Makefile
+++ b/Makefile
@@ -52,7 +52,7 @@ GOBUILD_FLAGS :=
 
 ALL_ARCH = amd64 arm arm64 ppc64le
 
-QEMUVERSION = v2.12.0-1
+QEMUVERSION = v3.0.0
 
 BUSTED_ARGS =-v --pattern=_test
 
@@ -61,7 +61,7 @@ IMAGE = $(REGISTRY)/$(IMGNAME)
 MULTI_ARCH_IMG = $(IMAGE)-$(ARCH)
 
 # Set default base image dynamically for each arch
-BASEIMAGE?=quay.io/kubernetes-ingress-controller/nginx-$(ARCH):0.65
+BASEIMAGE?=quay.io/kubernetes-ingress-controller/nginx-$(ARCH):0.66
 
 ifeq ($(ARCH),arm)
 	QEMUARCH=arm
diff --git a/build/go-in-docker.sh b/build/go-in-docker.sh
index 39deb8683..7a83b1961 100755
--- a/build/go-in-docker.sh
+++ b/build/go-in-docker.sh
@@ -40,7 +40,7 @@ if [ "$missing" = true ];then
   exit 1
 fi
 
-E2E_IMAGE=quay.io/kubernetes-ingress-controller/e2e:v10102018-dcc6495
+E2E_IMAGE=quay.io/kubernetes-ingress-controller/e2e:v10292018-5d42f0e
 
 DOCKER_OPTS=${DOCKER_OPTS:-""}
 
diff --git a/images/nginx/Makefile b/images/nginx/Makefile
index f90f13247..6bc3979c2 100644
--- a/images/nginx/Makefile
+++ b/images/nginx/Makefile
@@ -26,7 +26,7 @@ ifeq ($(GOHOSTOS),darwin)
   SED_I=sed -i ''
 endif
 
-QEMUVERSION=v2.12.0-1
+QEMUVERSION=v3.0.0
 
 IMGNAME = nginx
 IMAGE = $(REGISTRY)/$(IMGNAME)
diff --git a/images/nginx/rootfs/build.sh b/images/nginx/rootfs/build.sh
index 153bb6e80..97ebe9eff 100755
--- a/images/nginx/rootfs/build.sh
+++ b/images/nginx/rootfs/build.sh
@@ -29,13 +29,13 @@ export NGINX_OPENTRACING_VERSION=0.6.0
 export OPENTRACING_CPP_VERSION=1.5.0
 export ZIPKIN_CPP_VERSION=0.5.2
 export JAEGER_VERSION=ba0fa3fa6dbb01995d996f988a897e272100bf95
-export MODSECURITY_VERSION=37b76e88df4bce8a9846345c27271d7e6ce1acfb
+export MODSECURITY_VERSION=56cfa4e4805bb6134b97143052a9b48919cc294f
 export LUA_NGX_VERSION=e94f2e5d64daa45ff396e262d8dab8e56f5f10e0
 export LUA_UPSTREAM_VERSION=0.07
-export NGINX_INFLUXDB_VERSION=f20cfb2458c338f162132f5a21eb021e2cbe6383
+export NGINX_INFLUXDB_VERSION=0e2cb6cbf850a29c81e44be9e33d9a15d45c50e8
 export GEOIP2_VERSION=3.2
 export NGINX_AJP_VERSION=bf6cd93f2098b59260de8d494f0f4b1f11a84627
-export LUAJIT_VERSION=8e35a1932250b0313c06393061f332c760efdf40
+export LUAJIT_VERSION=c58fe79b870f1934479bf14fe8035fc3d9fdfde2
 
 export BUILD_PATH=/tmp/build
 
@@ -253,7 +253,7 @@ dpkg-buildpackage -rfakeroot
 
 cd ..
 
-dpkg -i openssl_1.1.1-1_amd64.deb libssl1.1_1.1.1-1_amd64.deb libssl-dev_1.1.1-1_amd64.deb
+dpkg -i openssl_1.1.1-1_*.deb libssl1.1_1.1.1-1_*.deb libssl-dev_1.1.1-1_*.deb
 
 # Install luajit from openresty fork
 export LUAJIT_LIB=/usr/local/lib

From fed013ab6fca95a1ae542861baf883af40c8c881 Mon Sep 17 00:00:00 2001
From: Manuel Alejandro de Brito Fontes <aledbf@gmail.com>
Date: Mon, 29 Oct 2018 13:01:41 -0300
Subject: [PATCH 21/31] Fix status update in case of connection errors

---
 build/e2e-tests.sh                     |  13 --
 images/e2e/Dockerfile                  |   6 +-
 internal/ingress/controller/nginx.go   |   3 +-
 internal/ingress/status/status.go      | 163 ++++++++++++++-----------
 internal/ingress/status/status_test.go |   3 +-
 test/e2e/annotations/influxdb.go       |   5 +-
 test/e2e/e2e.go                        |  11 +-
 test/e2e/framework/exec.go             |  58 ++++++++-
 test/e2e/framework/framework.go        |  19 ++-
 test/e2e/framework/logs.go             |   7 +-
 test/e2e/status/update.go              | 145 ++++++++++++++++++++++
 11 files changed, 326 insertions(+), 107 deletions(-)
 create mode 100644 test/e2e/status/update.go

diff --git a/build/e2e-tests.sh b/build/e2e-tests.sh
index 83c34c668..74af35714 100755
--- a/build/e2e-tests.sh
+++ b/build/e2e-tests.sh
@@ -41,19 +41,6 @@ fi
 
 SCRIPT_ROOT=$(dirname ${BASH_SOURCE})/..
 
-mkdir -p ${SCRIPT_ROOT}/test/binaries
-
-TEST_BINARIES=$( cd "${SCRIPT_ROOT}/test/binaries" ; pwd -P )
-
-export PATH=${TEST_BINARIES}:$PATH
-
-if ! [ -x "$(command -v kubectl)" ]; then
-    echo "downloading kubectl..."
-    curl -sSLo ${TEST_BINARIES}/kubectl \
-        https://storage.googleapis.com/kubernetes-release/release/v1.11.0/bin/linux/amd64/kubectl
-    chmod +x ${TEST_BINARIES}/kubectl
-fi
-
 ginkgo build ./test/e2e
 
 exec --                                      \
diff --git a/images/e2e/Dockerfile b/images/e2e/Dockerfile
index 42373160c..18e0835fd 100644
--- a/images/e2e/Dockerfile
+++ b/images/e2e/Dockerfile
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM quay.io/kubernetes-ingress-controller/nginx-amd64:0.63
+FROM quay.io/kubernetes-ingress-controller/nginx-amd64:0.64
 
 RUN clean-install \
   g++ \
@@ -61,3 +61,7 @@ RUN  luarocks install luacheck \
 
 RUN  go get github.com/onsi/ginkgo/ginkgo \
   && go get golang.org/x/lint/golint
+
+RUN  curl -Lo /usr/local/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/v1.12.0/bin/linux/amd64/kubectl \
+  && chmod +x /usr/local/bin/kubectl
+
diff --git a/internal/ingress/controller/nginx.go b/internal/ingress/controller/nginx.go
index 9c5b398be..e5ab10063 100644
--- a/internal/ingress/controller/nginx.go
+++ b/internal/ingress/controller/nginx.go
@@ -18,7 +18,6 @@ package controller
 
 import (
 	"bytes"
-	"context"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -257,7 +256,7 @@ func (n *NGINXController) Start() {
 	n.store.Run(n.stopCh)
 
 	if n.syncStatus != nil {
-		go n.syncStatus.Run(context.Background())
+		go n.syncStatus.Run()
 	}
 
 	cmd := nginxExecCommand()
diff --git a/internal/ingress/status/status.go b/internal/ingress/status/status.go
index 6b152c80a..c09128130 100644
--- a/internal/ingress/status/status.go
+++ b/internal/ingress/status/status.go
@@ -51,7 +51,7 @@ const (
 
 // Sync ...
 type Sync interface {
-	Run(ctx context.Context)
+	Run()
 	Shutdown()
 }
 
@@ -93,22 +93,97 @@ type statusSync struct {
 	pod *k8s.PodInfo
 
 	elector *leaderelection.LeaderElector
+
 	// workqueue used to keep in sync the status IP/s
 	// in the Ingress rules
 	syncQueue *task.Queue
 }
 
 // Run starts the loop to keep the status in sync
-func (s statusSync) Run(ctx context.Context) {
-	s.elector.Run(ctx)
+func (s statusSync) Run() {
+	// we need to use the defined ingress class to allow multiple leaders
+	// in order to update information about ingress status
+	electionID := fmt.Sprintf("%v-%v", s.Config.ElectionID, s.Config.DefaultIngressClass)
+	if s.Config.IngressClass != "" {
+		electionID = fmt.Sprintf("%v-%v", s.Config.ElectionID, s.Config.IngressClass)
+	}
+
+	// start a new context
+	ctx := context.Background()
+	// allow to cancel the context in case we stop being the leader
+	leaderCtx, cancel := context.WithCancel(ctx)
+
+	var stopCh chan struct{}
+	callbacks := leaderelection.LeaderCallbacks{
+		OnStartedLeading: func(ctx context.Context) {
+			glog.V(2).Infof("I am the new status update leader")
+			stopCh = make(chan struct{})
+			go s.syncQueue.Run(time.Second, stopCh)
+			// trigger initial sync
+			s.syncQueue.EnqueueTask(task.GetDummyObject("sync status"))
+			// when this instance is the leader we need to enqueue
+			// an item to trigger the update of the Ingress status.
+			wait.PollUntil(updateInterval, func() (bool, error) {
+				s.syncQueue.EnqueueTask(task.GetDummyObject("sync status"))
+				return false, nil
+			}, stopCh)
+		},
+		OnStoppedLeading: func() {
+			glog.V(2).Infof("I am not status update leader anymore")
+			close(stopCh)
+
+			// cancel the context
+			cancel()
+
+			// start a new context and run the elector
+			leaderCtx, cancel = context.WithCancel(ctx)
+			go s.elector.Run(leaderCtx)
+		},
+		OnNewLeader: func(identity string) {
+			glog.Infof("new leader elected: %v", identity)
+		},
+	}
+
+	broadcaster := record.NewBroadcaster()
+	hostname, _ := os.Hostname()
+
+	recorder := broadcaster.NewRecorder(scheme.Scheme, apiv1.EventSource{
+		Component: "ingress-leader-elector",
+		Host:      hostname,
+	})
+
+	lock := resourcelock.ConfigMapLock{
+		ConfigMapMeta: metav1.ObjectMeta{Namespace: s.pod.Namespace, Name: electionID},
+		Client:        s.Config.Client.CoreV1(),
+		LockConfig: resourcelock.ResourceLockConfig{
+			Identity:      s.pod.Name,
+			EventRecorder: recorder,
+		},
+	}
+
+	ttl := 30 * time.Second
+	le, err := leaderelection.NewLeaderElector(leaderelection.LeaderElectionConfig{
+		Lock:          &lock,
+		LeaseDuration: ttl,
+		RenewDeadline: ttl / 2,
+		RetryPeriod:   ttl / 4,
+		Callbacks:     callbacks,
+	})
+	if err != nil {
+		glog.Fatalf("unexpected error starting leader election: %v", err)
+	}
+	s.elector = le
+
+	go le.Run(leaderCtx)
 }
 
 // Shutdown stop the sync. In case the instance is the leader it will remove the current IP
 // if there is no other instances running.
 func (s statusSync) Shutdown() {
 	go s.syncQueue.Shutdown()
+
 	// remove IP from Ingress
-	if !s.elector.IsLeader() {
+	if s.elector != nil && !s.elector.IsLeader() {
 		return
 	}
 
@@ -146,6 +221,10 @@ func (s *statusSync) sync(key interface{}) error {
 		return nil
 	}
 
+	if s.elector != nil && !s.elector.IsLeader() {
+		return fmt.Errorf("i am not the current leader. Skiping status update")
+	}
+
 	addrs, err := s.runningAddresses()
 	if err != nil {
 		return err
@@ -173,66 +252,6 @@ func NewStatusSyncer(config Config) Sync {
 	}
 	st.syncQueue = task.NewCustomTaskQueue(st.sync, st.keyfunc)
 
-	// we need to use the defined ingress class to allow multiple leaders
-	// in order to update information about ingress status
-	electionID := fmt.Sprintf("%v-%v", config.ElectionID, config.DefaultIngressClass)
-	if config.IngressClass != "" {
-		electionID = fmt.Sprintf("%v-%v", config.ElectionID, config.IngressClass)
-	}
-
-	var stopCh chan struct{}
-	callbacks := leaderelection.LeaderCallbacks{
-		OnStartedLeading: func(ctx context.Context) {
-			glog.V(2).Infof("I am the new status update leader")
-			stopCh = make(chan struct{})
-			go st.syncQueue.Run(time.Second, stopCh)
-			// when this instance is the leader we need to enqueue
-			// an item to trigger the update of the Ingress status.
-			wait.PollUntil(updateInterval, func() (bool, error) {
-				st.syncQueue.EnqueueTask(task.GetDummyObject("sync status"))
-				return false, nil
-			}, stopCh)
-		},
-		OnStoppedLeading: func() {
-			glog.V(2).Infof("I am not status update leader anymore")
-			close(stopCh)
-		},
-		OnNewLeader: func(identity string) {
-			glog.Infof("new leader elected: %v", identity)
-		},
-	}
-
-	broadcaster := record.NewBroadcaster()
-	hostname, _ := os.Hostname()
-
-	recorder := broadcaster.NewRecorder(scheme.Scheme, apiv1.EventSource{
-		Component: "ingress-leader-elector",
-		Host:      hostname,
-	})
-
-	lock := resourcelock.ConfigMapLock{
-		ConfigMapMeta: metav1.ObjectMeta{Namespace: pod.Namespace, Name: electionID},
-		Client:        config.Client.CoreV1(),
-		LockConfig: resourcelock.ResourceLockConfig{
-			Identity:      pod.Name,
-			EventRecorder: recorder,
-		},
-	}
-
-	ttl := 30 * time.Second
-	le, err := leaderelection.NewLeaderElector(leaderelection.LeaderElectionConfig{
-		Lock:          &lock,
-		LeaseDuration: ttl,
-		RenewDeadline: ttl / 2,
-		RetryPeriod:   ttl / 4,
-		Callbacks:     callbacks,
-	})
-
-	if err != nil {
-		glog.Fatalf("unexpected error starting leader election: %v", err)
-	}
-
-	st.elector = le
 	return st
 }
 
@@ -333,6 +352,13 @@ func (s *statusSync) updateStatus(newIngressPoint []apiv1.LoadBalancerIngress) {
 	sort.SliceStable(newIngressPoint, lessLoadBalancerIngress(newIngressPoint))
 
 	for _, ing := range ings {
+		curIPs := ing.Status.LoadBalancer.Ingress
+		sort.SliceStable(curIPs, lessLoadBalancerIngress(curIPs))
+		if ingressSliceEqual(curIPs, newIngressPoint) {
+			glog.V(3).Infof("skipping update of Ingress %v/%v (no change)", ing.Namespace, ing.Name)
+			continue
+		}
+
 		batch.Queue(runUpdate(ing, newIngressPoint, s.Client))
 	}
 
@@ -347,14 +373,6 @@ func runUpdate(ing *extensions.Ingress, status []apiv1.LoadBalancerIngress,
 			return nil, nil
 		}
 
-		curIPs := ing.Status.LoadBalancer.Ingress
-		sort.SliceStable(curIPs, lessLoadBalancerIngress(curIPs))
-
-		if ingressSliceEqual(status, curIPs) {
-			glog.V(3).Infof("skipping update of Ingress %v/%v (no change)", ing.Namespace, ing.Name)
-			return true, nil
-		}
-
 		ingClient := client.ExtensionsV1beta1().Ingresses(ing.Namespace)
 
 		currIng, err := ingClient.Get(ing.Name, metav1.GetOptions{})
@@ -398,5 +416,6 @@ func ingressSliceEqual(lhs, rhs []apiv1.LoadBalancerIngress) bool {
 			return false
 		}
 	}
+
 	return true
 }
diff --git a/internal/ingress/status/status_test.go b/internal/ingress/status/status_test.go
index 7234193ff..c4f223ec5 100644
--- a/internal/ingress/status/status_test.go
+++ b/internal/ingress/status/status_test.go
@@ -17,7 +17,6 @@ limitations under the License.
 package status
 
 import (
-	"context"
 	"os"
 	"testing"
 	"time"
@@ -298,7 +297,7 @@ func TestStatusActions(t *testing.T) {
 	fk := fkSync.(statusSync)
 
 	// start it and wait for the election and syn actions
-	go fk.Run(context.Background())
+	go fk.Run()
 	//  wait for the election
 	time.Sleep(100 * time.Millisecond)
 	// execute sync
diff --git a/test/e2e/annotations/influxdb.go b/test/e2e/annotations/influxdb.go
index 1286bdc93..6ea3bc4f6 100644
--- a/test/e2e/annotations/influxdb.go
+++ b/test/e2e/annotations/influxdb.go
@@ -183,8 +183,7 @@ func execInfluxDBCommand(pod *corev1.Pod, command string) (string, error) {
 		execErr bytes.Buffer
 	)
 
-	args := fmt.Sprintf("kubectl exec --namespace %v %v -- %v", pod.Namespace, pod.Name, command)
-	cmd := exec.Command("/bin/bash", "-c", args)
+	cmd := exec.Command("/bin/bash", "-c", fmt.Sprintf("%v exec --namespace %s %s -- %s", framework.KubectlPath, pod.Namespace, pod.Name, command))
 	cmd.Stdout = &execOut
 	cmd.Stderr = &execErr
 
@@ -195,7 +194,7 @@ func execInfluxDBCommand(pod *corev1.Pod, command string) (string, error) {
 	}
 
 	if err != nil {
-		return "", fmt.Errorf("could not execute: %v", err)
+		return "", fmt.Errorf("could not execute '%s %s': %v", cmd.Path, cmd.Args, err)
 	}
 
 	return execOut.String(), nil
diff --git a/test/e2e/e2e.go b/test/e2e/e2e.go
index d42336254..773635cf5 100644
--- a/test/e2e/e2e.go
+++ b/test/e2e/e2e.go
@@ -17,13 +17,14 @@ limitations under the License.
 package e2e
 
 import (
+	"os"
 	"testing"
 
-	"github.com/golang/glog"
 	"github.com/onsi/ginkgo"
 	"github.com/onsi/ginkgo/config"
 	"github.com/onsi/gomega"
 	"k8s.io/apiserver/pkg/util/logs"
+
 	// required
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
 
@@ -36,6 +37,7 @@ import (
 	_ "k8s.io/ingress-nginx/test/e2e/servicebackend"
 	_ "k8s.io/ingress-nginx/test/e2e/settings"
 	_ "k8s.io/ingress-nginx/test/e2e/ssl"
+	_ "k8s.io/ingress-nginx/test/e2e/status"
 )
 
 // RunE2ETests checks configuration parameters (specified through flags) and then runs
@@ -50,7 +52,12 @@ func RunE2ETests(t *testing.T) {
 		config.GinkgoConfig.SkipString = `\[Flaky\]|\[Feature:.+\]`
 	}
 
-	glog.Infof("Starting e2e run %q on Ginkgo node %d", framework.RunID, config.GinkgoConfig.ParallelNode)
+	if os.Getenv("KUBECTL_PATH") != "" {
+		framework.KubectlPath = os.Getenv("KUBECTL_PATH")
+		framework.Logf("Using kubectl path '%s'", framework.KubectlPath)
+	}
+
+	framework.Logf("Starting e2e run %q on Ginkgo node %d", framework.RunID, config.GinkgoConfig.ParallelNode)
 	ginkgo.RunSpecs(t, "nginx-ingress-controller e2e suite")
 }
 
diff --git a/test/e2e/framework/exec.go b/test/e2e/framework/exec.go
index f6f82911f..893e343de 100644
--- a/test/e2e/framework/exec.go
+++ b/test/e2e/framework/exec.go
@@ -19,7 +19,11 @@ package framework
 import (
 	"bytes"
 	"fmt"
+	"io"
 	"os/exec"
+	"regexp"
+	"strconv"
+	"strings"
 
 	"k8s.io/api/core/v1"
 )
@@ -31,14 +35,14 @@ func (f *Framework) ExecCommand(pod *v1.Pod, command string) (string, error) {
 		execErr bytes.Buffer
 	)
 
-	args := fmt.Sprintf("kubectl exec --namespace %v %v --container nginx-ingress-controller -- %v", pod.Namespace, pod.Name, command)
-	cmd := exec.Command("/bin/bash", "-c", args)
+	cmd := exec.Command("/bin/bash", "-c", fmt.Sprintf("%v exec --namespace %s %s --container nginx-ingress-controller -- %s", KubectlPath, pod.Namespace, pod.Name, command))
 	cmd.Stdout = &execOut
 	cmd.Stderr = &execErr
 
 	err := cmd.Run()
 	if err != nil {
-		return "", fmt.Errorf("could not execute: %v", err)
+		return "", fmt.Errorf("could not execute '%s %s': %v", cmd.Path, cmd.Args, err)
+
 	}
 
 	if execErr.Len() > 0 {
@@ -59,3 +63,51 @@ func (f *Framework) NewIngressController(namespace string) error {
 
 	return nil
 }
+
+var (
+	proxyRegexp = regexp.MustCompile("Starting to serve on .*:([0-9]+)")
+)
+
+// KubectlProxy creates a proxy to kubernetes apiserver
+func (f *Framework) KubectlProxy(port int) (int, *exec.Cmd, error) {
+	cmd := exec.Command("/bin/bash", "-c", fmt.Sprintf("%s proxy --accept-hosts=.* --address=0.0.0.0 --port=%d", KubectlPath, port))
+	stdout, stderr, err := startCmdAndStreamOutput(cmd)
+	if err != nil {
+		return -1, nil, err
+	}
+
+	defer stdout.Close()
+	defer stderr.Close()
+
+	buf := make([]byte, 128)
+	var n int
+	if n, err = stdout.Read(buf); err != nil {
+		return -1, cmd, fmt.Errorf("Failed to read from kubectl proxy stdout: %v", err)
+	}
+
+	output := string(buf[:n])
+	match := proxyRegexp.FindStringSubmatch(output)
+	if len(match) == 2 {
+		if port, err := strconv.Atoi(match[1]); err == nil {
+			return port, cmd, nil
+		}
+	}
+
+	return -1, cmd, fmt.Errorf("Failed to parse port from proxy stdout: %s", output)
+}
+
+func startCmdAndStreamOutput(cmd *exec.Cmd) (stdout, stderr io.ReadCloser, err error) {
+	stdout, err = cmd.StdoutPipe()
+	if err != nil {
+		return
+	}
+
+	stderr, err = cmd.StderrPipe()
+	if err != nil {
+		return
+	}
+
+	Logf("Asynchronously running '%s'", strings.Join(cmd.Args, " "))
+	err = cmd.Start()
+	return
+}
diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go
index 10234a719..cbb635c58 100644
--- a/test/e2e/framework/framework.go
+++ b/test/e2e/framework/framework.go
@@ -46,6 +46,11 @@ const (
 	HTTPS RequestScheme = "https"
 )
 
+var (
+	// KubectlPath defines the full path of the kubectl binary
+	KubectlPath = "/usr/local/bin/kubectl"
+)
+
 // Framework supports common operations used by e2e tests; it will keep a client & a namespace for you.
 type Framework struct {
 	BaseName string
@@ -197,9 +202,8 @@ func (f *Framework) WaitForNginxConfiguration(matcher func(cfg string) bool) err
 	return wait.Poll(Poll, time.Minute*5, f.matchNginxConditions("", matcher))
 }
 
-// NginxLogs returns the logs of the nginx ingress controller pod running
-func (f *Framework) NginxLogs() (string, error) {
-	l, err := f.KubeClientSet.CoreV1().Pods(f.IngressController.Namespace).List(metav1.ListOptions{
+func nginxLogs(client kubernetes.Interface, namespace string) (string, error) {
+	l, err := client.CoreV1().Pods(namespace).List(metav1.ListOptions{
 		LabelSelector: "app.kubernetes.io/name=ingress-nginx",
 	})
 	if err != nil {
@@ -209,7 +213,7 @@ func (f *Framework) NginxLogs() (string, error) {
 	for _, pod := range l.Items {
 		if strings.HasPrefix(pod.GetName(), "nginx-ingress-controller") {
 			if isRunning, err := podRunningReady(&pod); err == nil && isRunning {
-				return f.Logs(&pod)
+				return Logs(&pod)
 			}
 		}
 	}
@@ -217,6 +221,11 @@ func (f *Framework) NginxLogs() (string, error) {
 	return "", fmt.Errorf("no nginx ingress controller pod is running (logs)")
 }
 
+// NginxLogs returns the logs of the nginx ingress controller pod running
+func (f *Framework) NginxLogs() (string, error) {
+	return nginxLogs(f.KubeClientSet, f.IngressController.Namespace)
+}
+
 func (f *Framework) matchNginxConditions(name string, matcher func(cfg string) bool) wait.ConditionFunc {
 	return func() (bool, error) {
 		l, err := f.KubeClientSet.CoreV1().Pods(f.IngressController.Namespace).List(metav1.ListOptions{
@@ -380,7 +389,7 @@ func UpdateDeployment(kubeClientSet kubernetes.Interface, namespace string, name
 		LabelSelector: fields.SelectorFromSet(fields.Set(deployment.Spec.Template.ObjectMeta.Labels)).String(),
 	})
 	if err != nil {
-		return errors.Wrapf(err, "failed to wait for nginx-ingress-controller replica count to be %v", replicas)
+		return errors.Wrapf(err, "waiting for nginx-ingress-controller replica count to be %v", replicas)
 	}
 
 	return nil
diff --git a/test/e2e/framework/logs.go b/test/e2e/framework/logs.go
index f1a08b82d..21a93e569 100644
--- a/test/e2e/framework/logs.go
+++ b/test/e2e/framework/logs.go
@@ -25,7 +25,7 @@ import (
 )
 
 // Logs returns the log entries of a given Pod.
-func (f *Framework) Logs(pod *v1.Pod) (string, error) {
+func Logs(pod *v1.Pod) (string, error) {
 	var (
 		execOut bytes.Buffer
 		execErr bytes.Buffer
@@ -35,14 +35,13 @@ func (f *Framework) Logs(pod *v1.Pod) (string, error) {
 		return "", fmt.Errorf("could not determine which container to use")
 	}
 
-	args := fmt.Sprintf("kubectl logs -n %v %v", pod.Namespace, pod.Name)
-	cmd := exec.Command("/bin/bash", "-c", args)
+	cmd := exec.Command("/bin/bash", "-c", fmt.Sprintf("%v logs --namespace %s %s", KubectlPath, pod.Namespace, pod.Name))
 	cmd.Stdout = &execOut
 	cmd.Stderr = &execErr
 
 	err := cmd.Run()
 	if err != nil {
-		return "", fmt.Errorf("could not execute: %v", err)
+		return "", fmt.Errorf("could not execute '%s %s': %v", cmd.Path, cmd.Args, err)
 	}
 
 	if execErr.Len() > 0 {
diff --git a/test/e2e/status/update.go b/test/e2e/status/update.go
new file mode 100644
index 000000000..171d62232
--- /dev/null
+++ b/test/e2e/status/update.go
@@ -0,0 +1,145 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package settings
+
+import (
+	"fmt"
+	"log"
+	"net"
+	"strings"
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	appsv1beta1 "k8s.io/api/apps/v1beta1"
+	apiv1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+
+	"k8s.io/ingress-nginx/test/e2e/framework"
+)
+
+var _ = framework.IngressNginxDescribe("Status Update [Status]", func() {
+	f := framework.NewDefaultFramework("status-update")
+	host := "status-update"
+	address := getHostIP()
+
+	BeforeEach(func() {
+	})
+
+	AfterEach(func() {
+	})
+
+	It("should update status field after client-go reconnection", func() {
+		port, cmd, err := f.KubectlProxy(0)
+		Expect(err).NotTo(HaveOccurred(), "starting kubectl proxy")
+
+		err = framework.UpdateDeployment(f.KubeClientSet, f.IngressController.Namespace, "nginx-ingress-controller", 1,
+			func(deployment *appsv1beta1.Deployment) error {
+				args := deployment.Spec.Template.Spec.Containers[0].Args
+				args = append(args, fmt.Sprintf("--apiserver-host=http://%s:%d", address.String(), port))
+				args = append(args, "--publish-status-address=1.1.0.0")
+				// flags --publish-service and --publish-status-address are mutually exclusive
+				var index int
+				for k, v := range args {
+					if strings.Index(v, "--publish-service") != -1 {
+						index = k
+						break
+					}
+				}
+				if index > -1 {
+					args[index] = ""
+				}
+
+				deployment.Spec.Template.Spec.Containers[0].Args = args
+				_, err := f.KubeClientSet.AppsV1beta1().Deployments(f.IngressController.Namespace).Update(deployment)
+				return err
+			})
+		Expect(err).NotTo(HaveOccurred(), "updating ingress controller deployment flags")
+
+		err = f.NewEchoDeploymentWithReplicas(1)
+		Expect(err).NotTo(HaveOccurred(), "waiting one replicaset in echoserver deployment")
+
+		ing, err := f.EnsureIngress(framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, nil))
+		Expect(err).NotTo(HaveOccurred(), "waiting Ingress creation for hostname %v", host)
+		Expect(ing).NotTo(BeNil())
+
+		err = f.WaitForNginxConfiguration(
+			func(cfg string) bool {
+				return strings.Contains(cfg, fmt.Sprintf("server_name %s", host))
+			})
+		Expect(err).NotTo(HaveOccurred(), "waiting for nginx server section with server_name %v", host)
+
+		framework.Logf("waiting for leader election and initial status update")
+		time.Sleep(30 * time.Second)
+
+		err = cmd.Process.Kill()
+		Expect(err).NotTo(HaveOccurred(), "terminating kubectl proxy")
+
+		ing, err = f.KubeClientSet.Extensions().Ingresses(f.IngressController.Namespace).Get(host, metav1.GetOptions{})
+		Expect(err).NotTo(HaveOccurred(), "getting %s/%v Ingress", f.IngressController.Namespace, host)
+
+		ing.Status.LoadBalancer.Ingress = []apiv1.LoadBalancerIngress{}
+		_, err = f.KubeClientSet.Extensions().Ingresses(f.IngressController.Namespace).UpdateStatus(ing)
+		Expect(err).NotTo(HaveOccurred(), "cleaning Ingress status")
+		time.Sleep(10 * time.Second)
+
+		err = f.KubeClientSet.CoreV1().
+			ConfigMaps(f.IngressController.Namespace).
+			Delete("ingress-controller-leader-nginx", &metav1.DeleteOptions{})
+		Expect(err).NotTo(HaveOccurred(), "deleting leader election configmap")
+
+		_, cmd, err = f.KubectlProxy(port)
+		Expect(err).NotTo(HaveOccurred(), "starting kubectl proxy")
+		defer func() {
+			if cmd != nil {
+				err := cmd.Process.Kill()
+				Expect(err).NotTo(HaveOccurred(), "terminating kubectl proxy")
+			}
+		}()
+
+		err = wait.Poll(10*time.Second, time.Minute*3, func() (done bool, err error) {
+			ing, err = f.KubeClientSet.Extensions().Ingresses(f.IngressController.Namespace).Get(host, metav1.GetOptions{})
+			if err != nil {
+				return false, err
+			}
+
+			if len(ing.Status.LoadBalancer.Ingress) != 1 {
+				return false, nil
+			}
+
+			return true, nil
+		})
+		Expect(err).NotTo(HaveOccurred(), "waiting for ingress status")
+		Expect(ing.Status.LoadBalancer.Ingress).Should(Equal([]apiv1.LoadBalancerIngress{
+			{IP: "1.1.0.0"},
+		}))
+	})
+})
+
+func getHostIP() net.IP {
+	conn, err := net.Dial("udp", "8.8.8.8:80")
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer conn.Close()
+
+	localAddr := conn.LocalAddr().(*net.UDPAddr)
+
+	return localAddr.IP
+}

From d869aab61dff29268074ad07b9d3bd35a42e1cfa Mon Sep 17 00:00:00 2001
From: Manuel Alejandro de Brito Fontes <aledbf@gmail.com>
Date: Mon, 29 Oct 2018 13:14:57 -0300
Subject: [PATCH 22/31] Update e2e image

---
 build/go-in-docker.sh | 2 +-
 images/e2e/Dockerfile | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/build/go-in-docker.sh b/build/go-in-docker.sh
index 7a83b1961..ff081d90e 100755
--- a/build/go-in-docker.sh
+++ b/build/go-in-docker.sh
@@ -40,7 +40,7 @@ if [ "$missing" = true ];then
   exit 1
 fi
 
-E2E_IMAGE=quay.io/kubernetes-ingress-controller/e2e:v10292018-5d42f0e
+E2E_IMAGE=quay.io/kubernetes-ingress-controller/e2e:v10292018-240c7274b
 
 DOCKER_OPTS=${DOCKER_OPTS:-""}
 
diff --git a/images/e2e/Dockerfile b/images/e2e/Dockerfile
index 18e0835fd..17e6c18df 100644
--- a/images/e2e/Dockerfile
+++ b/images/e2e/Dockerfile
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM quay.io/kubernetes-ingress-controller/nginx-amd64:0.64
+FROM quay.io/kubernetes-ingress-controller/nginx-amd64:0.66
 
 RUN clean-install \
   g++ \

From c27c57dc8b11e8129db00e4cf77c4d3255a1f24a Mon Sep 17 00:00:00 2001
From: Maximilian Bode <maximilian.bode@tngtech.com>
Date: Mon, 29 Oct 2018 21:25:23 +0100
Subject: [PATCH 23/31] Add configuration for geoip2 module

Based on closed PRs #2551, #2755
---
 .../nginx-configuration/configmap.md          |  8 ++
 internal/ingress/controller/config/config.go  |  5 +
 rootfs/etc/nginx/template/nginx.tmpl          | 24 +++++
 test/e2e/settings/geoip2.go                   | 96 +++++++++++++++++++
 4 files changed, 133 insertions(+)
 create mode 100644 test/e2e/settings/geoip2.go

diff --git a/docs/user-guide/nginx-configuration/configmap.md b/docs/user-guide/nginx-configuration/configmap.md
index d4cdc16c2..85615f26b 100644
--- a/docs/user-guide/nginx-configuration/configmap.md
+++ b/docs/user-guide/nginx-configuration/configmap.md
@@ -86,6 +86,7 @@ The following table shows a configuration option's name, type, and the default v
 |[proxy-protocol-header-timeout](#proxy-protocol-header-timeout)|string|"5s"|
 |[use-gzip](#use-gzip)|bool|"true"|
 |[use-geoip](#use-geoip)|bool|"true"|
+|[use-geoip2](#use-geoip2)|bool|"false"|
 |[enable-brotli](#enable-brotli)|bool|"false"|
 |[brotli-level](#brotli-level)|int|4|
 |[brotli-types](#brotli-types)|string|"application/xml+rss application/atom+xml application/javascript application/x-javascript application/json application/rss+xml application/vnd.ms-fontobject application/x-font-ttf application/x-web-app-manifest+json application/xhtml+xml application/xml font/opentype image/svg+xml image/x-icon text/css text/plain text/x-component"|
@@ -498,6 +499,13 @@ The default mime type list to compress is: `application/atom+xml application/jav
 Enables or disables ["geoip" module](http://nginx.org/en/docs/http/ngx_http_geoip_module.html) that creates variables with values depending on the client IP address, using the precompiled MaxMind databases.
 _**default:**_ true
 
+> __Note:__ MaxMind legacy databases are discontinued and will not receive updates after 2019-01-02, cf. [discontinuation notice](https://support.maxmind.com/geolite-legacy-discontinuation-notice/). Consider [use-geoip2](#use-geoip2) below.
+
+## use-geoip2
+
+Enables the [geoip2 module](https://github.com/leev/ngx_http_geoip2_module) for NGINX.
+_**default:**_ false
+
 ## enable-brotli
 
 Enables or disables compression of HTTP responses using the ["brotli" module](https://github.com/google/ngx_brotli).
diff --git a/internal/ingress/controller/config/config.go b/internal/ingress/controller/config/config.go
index 74cfac4cf..d6520dd5e 100644
--- a/internal/ingress/controller/config/config.go
+++ b/internal/ingress/controller/config/config.go
@@ -347,6 +347,10 @@ type Configuration struct {
 	// http://nginx.org/en/docs/http/ngx_http_geoip_module.html
 	UseGeoIP bool `json:"use-geoip,omitempty"`
 
+	// UseGeoIP2 enables the geoip2 module for NGINX
+	// By default this is disabled
+	UseGeoIP2 bool `json:"use-geoip2,omitempty"`
+
 	// Enables or disables the use of the NGINX Brotli Module for compression
 	// https://github.com/google/ngx_brotli
 	EnableBrotli bool `json:"enable-brotli,omitempty"`
@@ -630,6 +634,7 @@ func NewDefault() Configuration {
 		EnableBrotli:               false,
 		UseGzip:                    true,
 		UseGeoIP:                   true,
+		UseGeoIP2:                  false,
 		WorkerProcesses:            strconv.Itoa(runtime.NumCPU()),
 		WorkerShutdownTimeout:      "10s",
 		LoadBalanceAlgorithm:       defaultLoadBalancerAlgorithm,
diff --git a/rootfs/etc/nginx/template/nginx.tmpl b/rootfs/etc/nginx/template/nginx.tmpl
index ca62f1672..0e855953d 100644
--- a/rootfs/etc/nginx/template/nginx.tmpl
+++ b/rootfs/etc/nginx/template/nginx.tmpl
@@ -12,6 +12,10 @@
 # setup custom paths that do not require root access
 pid /tmp/nginx.pid;
 
+{{ if $cfg.UseGeoIP2 }}
+load_module /etc/nginx/modules/ngx_http_geoip2_module.so;
+{{ end }}
+
 {{ if $cfg.EnableModsecurity }}
 load_module /etc/nginx/modules/ngx_http_modsecurity_module.so;
 {{ end }}
@@ -123,6 +127,26 @@ http {
     geoip_proxy_recursive on;
     {{ end }}
 
+    {{ if $cfg.UseGeoIP2 }}
+    # https://github.com/leev/ngx_http_geoip2_module#example-usage
+
+    geoip2 /etc/nginx/geoip/GeoLite2-City.mmdb {
+        $geoip2_city_country_code source=$the_real_ip country iso_code;
+        $geoip2_city_country_name source=$the_real_ip country names en;
+        $geoip2_city source=$the_real_ip city names en;
+        $geoip2_postal_code source=$the_real_ip postal code;
+        $geoip2_dma_code source=$the_real_ip location metro_code;
+        $geoip2_latitude source=$the_real_ip location latitude;
+        $geoip2_longitude source=$the_real_ip location longitude;
+        $geoip2_region_code source=$the_real_ip subdivisions 0 iso_code;
+        $geoip2_region_name source=$the_real_ip subdivisions 0 names en;
+    }
+
+    geoip2 /etc/nginx/geoip/GeoLite2-ASN.mmdb {
+        $geoip2_asn source=$the_real_ip autonomous_system_number;
+    }
+    {{ end }}
+
     aio                 threads;
     aio_write           on;
 
diff --git a/test/e2e/settings/geoip2.go b/test/e2e/settings/geoip2.go
new file mode 100644
index 000000000..f802fd575
--- /dev/null
+++ b/test/e2e/settings/geoip2.go
@@ -0,0 +1,96 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package settings
+
+import (
+	"strings"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/parnurzeal/gorequest"
+	"k8s.io/ingress-nginx/test/e2e/framework"
+	"net/http"
+)
+
+var _ = framework.IngressNginxDescribe("Geoip2", func() {
+	f := framework.NewDefaultFramework("geoip2")
+
+	host := "geoip2"
+
+	BeforeEach(func() {
+		err := f.NewEchoDeployment()
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	It("should only allow requests from specific countries", func() {
+		err := f.UpdateNginxConfigMapData("use-geoip2", "true")
+		Expect(err).NotTo(HaveOccurred())
+
+		httpSnippetAllowingOnlyAustralia :=
+			`map $geoip2_city_country_code $blocked_country {
+  default 1;
+  AU 0;
+}`
+		err = f.UpdateNginxConfigMapData("http-snippet", httpSnippetAllowingOnlyAustralia)
+		Expect(err).NotTo(HaveOccurred())
+
+		err = f.WaitForNginxConfiguration(
+			func(cfg string) bool {
+				return strings.Contains(cfg, "map $geoip2_city_country_code $blocked_country")
+			})
+		Expect(err).NotTo(HaveOccurred())
+
+		configSnippet :=
+			`if ($blocked_country) {
+  return 403;
+}`
+
+		annotations := map[string]string{
+			"nginx.ingress.kubernetes.io/configuration-snippet": configSnippet,
+		}
+
+		ing, err := f.EnsureIngress(framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations))
+		Expect(err).NotTo(HaveOccurred())
+		Expect(ing).NotTo(BeNil())
+
+		err = f.WaitForNginxConfiguration(
+			func(cfg string) bool {
+				return strings.Contains(cfg, "if ($blocked_country)")
+			})
+		Expect(err).NotTo(HaveOccurred())
+
+		// Should be blocked
+		usIp := "8.8.8.8"
+		resp, _, errs := gorequest.New().
+			Get(f.IngressController.HTTPURL).
+			Set("Host", host).
+			Set("X-Forwarded-For", usIp).
+			End()
+		Expect(errs).To(BeNil())
+		Expect(resp.StatusCode).Should(Equal(http.StatusForbidden))
+
+		// Shouldn't be blocked
+		australianIp := "1.1.1.1"
+		resp, _, errs = gorequest.New().
+			Get(f.IngressController.HTTPURL).
+			Set("Host", host).
+			Set("X-Forwarded-For", australianIp).
+			End()
+		Expect(errs).To(BeNil())
+		Expect(resp.StatusCode).Should(Equal(http.StatusOK))
+	})
+})

From 83dc4607c50df1695d5307add9beb6532d64802d Mon Sep 17 00:00:00 2001
From: Manuel Alejandro de Brito Fontes <aledbf@gmail.com>
Date: Mon, 29 Oct 2018 18:39:04 -0300
Subject: [PATCH 24/31] Remove e2e boilerplate

---
 test/e2e/annotations/alias.go                 | 19 +---
 test/e2e/annotations/approot.go               | 11 +--
 test/e2e/annotations/auth.go                  | 85 +++++------------
 test/e2e/annotations/authtls.go               | 45 +++++----
 test/e2e/annotations/backendprotocol.go       | 35 ++-----
 test/e2e/annotations/clientbodybuffersize.go  | 51 +++-------
 test/e2e/annotations/connection.go            | 11 +--
 test/e2e/annotations/cors.go                  | 66 ++++---------
 test/e2e/annotations/default_backend.go       | 11 +--
 test/e2e/annotations/forcesslredirect.go      | 11 +--
 test/e2e/annotations/fromtowwwredirect.go     | 11 +--
 test/e2e/annotations/grpc.go                  | 14 +--
 test/e2e/annotations/influxdb.go              | 36 +++----
 test/e2e/annotations/ipwhitelist.go           | 24 ++---
 test/e2e/annotations/log.go                   | 20 ++--
 test/e2e/annotations/luarestywaf.go           | 11 +--
 test/e2e/annotations/proxy.go                 | 95 +++++--------------
 test/e2e/annotations/redirect.go              | 16 +---
 test/e2e/annotations/rewrite.go               | 59 ++++--------
 test/e2e/annotations/serversnippet.go         | 18 ++--
 test/e2e/annotations/snippet.go               | 11 +--
 test/e2e/annotations/sslciphers.go            | 11 +--
 test/e2e/annotations/upstreamvhost.go         | 11 +--
 .../defaultbackend/custom_default_backend.go  |  7 +-
 test/e2e/framework/deployment.go              | 42 +++-----
 test/e2e/framework/framework.go               | 63 +++++-------
 test/e2e/framework/grpc_fortune_teller.go     | 33 ++-----
 test/e2e/framework/influxdb.go                | 27 ++----
 test/e2e/framework/k8s.go                     | 57 +++++++----
 test/e2e/framework/ssl.go                     |  7 +-
 test/e2e/lua/dynamic_certificates.go          | 63 +++++-------
 test/e2e/lua/dynamic_configuration.go         | 36 +++----
 test/e2e/servicebackend/service_backend.go    | 17 +---
 test/e2e/settings/configmap_change.go         | 21 ++--
 test/e2e/settings/forwarded_headers.go        | 28 ++----
 test/e2e/settings/global_access_block.go      | 26 ++---
 test/e2e/settings/main_snippet.go             |  7 +-
 test/e2e/settings/multi_accept.go             | 16 +---
 test/e2e/settings/no_auth_locations.go        | 27 ++----
 test/e2e/settings/proxy_protocol.go           | 26 ++---
 test/e2e/settings/server_tokens.go            | 25 ++---
 test/e2e/settings/tls.go                      | 50 +++-------
 test/e2e/ssl/secret_update.go                 | 15 +--
 test/e2e/status/update.go                     | 28 +++---
 44 files changed, 432 insertions(+), 871 deletions(-)

diff --git a/test/e2e/annotations/alias.go b/test/e2e/annotations/alias.go
index 6f67f3f75..2ebda7b5e 100644
--- a/test/e2e/annotations/alias.go
+++ b/test/e2e/annotations/alias.go
@@ -31,8 +31,7 @@ var _ = framework.IngressNginxDescribe("Annotations - Alias", func() {
 	f := framework.NewDefaultFramework("alias")
 
 	BeforeEach(func() {
-		err := f.NewEchoDeployment()
-		Expect(err).NotTo(HaveOccurred())
+		f.NewEchoDeployment()
 	})
 
 	AfterEach(func() {
@@ -43,17 +42,13 @@ var _ = framework.IngressNginxDescribe("Annotations - Alias", func() {
 		annotations := map[string]string{}
 
 		ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations)
-		_, err := f.EnsureIngress(ing)
+		f.EnsureIngress(ing)
 
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
-
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return Expect(server).Should(ContainSubstring("server_name foo")) &&
 					Expect(server).ShouldNot(ContainSubstring("return 503"))
 			})
-		Expect(err).NotTo(HaveOccurred())
 
 		resp, body, errs := gorequest.New().
 			Get(f.IngressController.HTTPURL).
@@ -81,17 +76,13 @@ var _ = framework.IngressNginxDescribe("Annotations - Alias", func() {
 		}
 
 		ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations)
-		_, err := f.EnsureIngress(ing)
+		f.EnsureIngress(ing)
 
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
-
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return Expect(server).Should(ContainSubstring("server_name foo")) &&
 					Expect(server).ShouldNot(ContainSubstring("return 503"))
 			})
-		Expect(err).NotTo(HaveOccurred())
 
 		hosts := []string{"foo", "bar"}
 		for _, host := range hosts {
diff --git a/test/e2e/annotations/approot.go b/test/e2e/annotations/approot.go
index b0daa5007..5c6d2942a 100644
--- a/test/e2e/annotations/approot.go
+++ b/test/e2e/annotations/approot.go
@@ -30,8 +30,7 @@ var _ = framework.IngressNginxDescribe("Annotations - Approot", func() {
 	f := framework.NewDefaultFramework("approot")
 
 	BeforeEach(func() {
-		err := f.NewEchoDeploymentWithReplicas(2)
-		Expect(err).NotTo(HaveOccurred())
+		f.NewEchoDeploymentWithReplicas(2)
 	})
 
 	AfterEach(func() {
@@ -45,17 +44,13 @@ var _ = framework.IngressNginxDescribe("Annotations - Approot", func() {
 		}
 
 		ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations)
-		_, err := f.EnsureIngress(ing)
+		f.EnsureIngress(ing)
 
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
-
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return Expect(server).Should(ContainSubstring(`if ($uri = /) {`)) &&
 					Expect(server).Should(ContainSubstring(`return 302 /foo;`))
 			})
-		Expect(err).NotTo(HaveOccurred())
 
 		resp, _, errs := gorequest.New().
 			Get(f.IngressController.HTTPURL).
diff --git a/test/e2e/annotations/auth.go b/test/e2e/annotations/auth.go
index 720c76f64..20939be24 100644
--- a/test/e2e/annotations/auth.go
+++ b/test/e2e/annotations/auth.go
@@ -25,6 +25,7 @@ import (
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+
 	"github.com/parnurzeal/gorequest"
 
 	corev1 "k8s.io/api/core/v1"
@@ -39,8 +40,7 @@ var _ = framework.IngressNginxDescribe("Annotations - Auth", func() {
 	f := framework.NewDefaultFramework("auth")
 
 	BeforeEach(func() {
-		err := f.NewEchoDeployment()
-		Expect(err).NotTo(HaveOccurred())
+		f.NewEchoDeployment()
 	})
 
 	AfterEach(func() {
@@ -50,17 +50,13 @@ var _ = framework.IngressNginxDescribe("Annotations - Auth", func() {
 		host := "auth"
 
 		ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, nil)
-		_, err := f.EnsureIngress(ing)
+		f.EnsureIngress(ing)
 
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
-
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return Expect(server).Should(ContainSubstring("server_name auth")) &&
 					Expect(server).ShouldNot(ContainSubstring("return 503"))
 			})
-		Expect(err).NotTo(HaveOccurred())
 
 		resp, body, errs := gorequest.New().
 			Get(f.IngressController.HTTPURL).
@@ -82,17 +78,13 @@ var _ = framework.IngressNginxDescribe("Annotations - Auth", func() {
 		}
 
 		ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations)
-		_, err := f.EnsureIngress(ing)
+		f.EnsureIngress(ing)
 
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
-
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return Expect(server).Should(ContainSubstring("server_name auth")) &&
 					Expect(server).Should(ContainSubstring("return 503"))
 			})
-		Expect(err).NotTo(HaveOccurred())
 
 		resp, body, errs := gorequest.New().
 			Get(f.IngressController.HTTPURL).
@@ -108,10 +100,7 @@ var _ = framework.IngressNginxDescribe("Annotations - Auth", func() {
 	It("should return status code 401 when authentication is configured but Authorization header is not configured", func() {
 		host := "auth"
 
-		s, err := f.EnsureSecret(buildSecret("foo", "bar", "test", f.IngressController.Namespace))
-		Expect(err).NotTo(HaveOccurred())
-		Expect(s).NotTo(BeNil())
-		Expect(s.ObjectMeta).NotTo(BeNil())
+		s := f.EnsureSecret(buildSecret("foo", "bar", "test", f.IngressController.Namespace))
 
 		annotations := map[string]string{
 			"nginx.ingress.kubernetes.io/auth-type":   "basic",
@@ -120,17 +109,13 @@ var _ = framework.IngressNginxDescribe("Annotations - Auth", func() {
 		}
 
 		ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations)
-		_, err = f.EnsureIngress(ing)
+		f.EnsureIngress(ing)
 
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
-
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return Expect(server).Should(ContainSubstring("server_name auth")) &&
 					Expect(server).ShouldNot(ContainSubstring("return 503"))
 			})
-		Expect(err).NotTo(HaveOccurred())
 
 		resp, body, errs := gorequest.New().
 			Get(f.IngressController.HTTPURL).
@@ -146,10 +131,7 @@ var _ = framework.IngressNginxDescribe("Annotations - Auth", func() {
 	It("should return status code 401 when authentication is configured and Authorization header is sent with invalid credentials", func() {
 		host := "auth"
 
-		s, err := f.EnsureSecret(buildSecret("foo", "bar", "test", f.IngressController.Namespace))
-		Expect(err).NotTo(HaveOccurred())
-		Expect(s).NotTo(BeNil())
-		Expect(s.ObjectMeta).NotTo(BeNil())
+		s := f.EnsureSecret(buildSecret("foo", "bar", "test", f.IngressController.Namespace))
 
 		annotations := map[string]string{
 			"nginx.ingress.kubernetes.io/auth-type":   "basic",
@@ -158,17 +140,13 @@ var _ = framework.IngressNginxDescribe("Annotations - Auth", func() {
 		}
 
 		ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations)
-		_, err = f.EnsureIngress(ing)
+		f.EnsureIngress(ing)
 
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
-
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return Expect(server).Should(ContainSubstring("server_name auth")) &&
 					Expect(server).ShouldNot(ContainSubstring("return 503"))
 			})
-		Expect(err).NotTo(HaveOccurred())
 
 		resp, body, errs := gorequest.New().
 			Get(f.IngressController.HTTPURL).
@@ -185,10 +163,7 @@ var _ = framework.IngressNginxDescribe("Annotations - Auth", func() {
 	It("should return status code 200 when authentication is configured and Authorization header is sent", func() {
 		host := "auth"
 
-		s, err := f.EnsureSecret(buildSecret("foo", "bar", "test", f.IngressController.Namespace))
-		Expect(err).NotTo(HaveOccurred())
-		Expect(s).NotTo(BeNil())
-		Expect(s.ObjectMeta).NotTo(BeNil())
+		s := f.EnsureSecret(buildSecret("foo", "bar", "test", f.IngressController.Namespace))
 
 		annotations := map[string]string{
 			"nginx.ingress.kubernetes.io/auth-type":   "basic",
@@ -197,17 +172,13 @@ var _ = framework.IngressNginxDescribe("Annotations - Auth", func() {
 		}
 
 		ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations)
-		_, err = f.EnsureIngress(ing)
+		f.EnsureIngress(ing)
 
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
-
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return Expect(server).Should(ContainSubstring("server_name auth")) &&
 					Expect(server).ShouldNot(ContainSubstring("return 503"))
 			})
-		Expect(err).NotTo(HaveOccurred())
 
 		resp, _, errs := gorequest.New().
 			Get(f.IngressController.HTTPURL).
@@ -223,7 +194,7 @@ var _ = framework.IngressNginxDescribe("Annotations - Auth", func() {
 	It("should return status code 500 when authentication is configured with invalid content and Authorization header is sent", func() {
 		host := "auth"
 
-		s, err := f.EnsureSecret(
+		s := f.EnsureSecret(
 			&corev1.Secret{
 				ObjectMeta: metav1.ObjectMeta{
 					Name:      "test",
@@ -236,9 +207,6 @@ var _ = framework.IngressNginxDescribe("Annotations - Auth", func() {
 				Type: corev1.SecretTypeOpaque,
 			},
 		)
-		Expect(err).NotTo(HaveOccurred())
-		Expect(s).NotTo(BeNil())
-		Expect(s.ObjectMeta).NotTo(BeNil())
 
 		annotations := map[string]string{
 			"nginx.ingress.kubernetes.io/auth-type":   "basic",
@@ -247,17 +215,13 @@ var _ = framework.IngressNginxDescribe("Annotations - Auth", func() {
 		}
 
 		ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations)
-		_, err = f.EnsureIngress(ing)
+		f.EnsureIngress(ing)
 
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
-
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return Expect(server).Should(ContainSubstring("server_name auth")) &&
 					Expect(server).ShouldNot(ContainSubstring("return 503"))
 			})
-		Expect(err).NotTo(HaveOccurred())
 
 		resp, _, errs := gorequest.New().
 			Get(f.IngressController.HTTPURL).
@@ -274,11 +238,10 @@ var _ = framework.IngressNginxDescribe("Annotations - Auth", func() {
 		host := "auth"
 
 		BeforeEach(func() {
-			err := f.NewHttpbinDeployment()
-			Expect(err).NotTo(HaveOccurred())
+			f.NewHttpbinDeployment()
 
 			var httpbinIP string
-			err = wait.PollImmediate(time.Second, time.Minute, func() (bool, error) {
+			err := wait.PollImmediate(time.Second, time.Minute, func() (bool, error) {
 				e, err := f.KubeClientSet.CoreV1().Endpoints(f.IngressController.Namespace).Get("httpbin", metav1.GetOptions{})
 				if errors.IsNotFound(err) {
 					return false, nil
@@ -300,15 +263,11 @@ var _ = framework.IngressNginxDescribe("Annotations - Auth", func() {
 			}
 
 			ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations)
-			_, err = f.EnsureIngress(ing)
+			f.EnsureIngress(ing)
 
-			Expect(err).NotTo(HaveOccurred())
-			Expect(ing).NotTo(BeNil())
-
-			err = f.WaitForNginxServer(host, func(server string) bool {
+			f.WaitForNginxServer(host, func(server string) bool {
 				return Expect(server).ShouldNot(ContainSubstring("return 503"))
 			})
-			Expect(err).NotTo(HaveOccurred())
 		})
 
 		It("should return status code 200 when signed in", func() {
diff --git a/test/e2e/annotations/authtls.go b/test/e2e/annotations/authtls.go
index 73ab53ca0..89f1de4a6 100644
--- a/test/e2e/annotations/authtls.go
+++ b/test/e2e/annotations/authtls.go
@@ -19,20 +19,20 @@ package annotations
 import (
 	"crypto/tls"
 	"fmt"
+	"net/http"
+	"strings"
+
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	"github.com/parnurzeal/gorequest"
 	"k8s.io/ingress-nginx/test/e2e/framework"
-	"net/http"
-	"strings"
 )
 
 var _ = framework.IngressNginxDescribe("Annotations - AuthTLS", func() {
 	f := framework.NewDefaultFramework("authtls")
 
 	BeforeEach(func() {
-		err := f.NewEchoDeploymentWithReplicas(2)
-		Expect(err).NotTo(HaveOccurred())
+		f.NewEchoDeploymentWithReplicas(2)
 	})
 
 	AfterEach(func() {
@@ -54,10 +54,7 @@ var _ = framework.IngressNginxDescribe("Annotations - AuthTLS", func() {
 		}
 
 		ing := framework.NewSingleIngressWithTLS(host, "/", host, nameSpace, "http-svc", 80, &annotations)
-		_, err = f.EnsureIngress(ing)
-
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
+		f.EnsureIngress(ing)
 
 		// Since we can use the same certificate-chain for tls as well as mutual-auth, we will check all values
 		sslCertDirective := fmt.Sprintf("ssl_certificate /etc/ingress-controller/ssl/%s-%s.pem;", nameSpace, host)
@@ -67,11 +64,14 @@ var _ = framework.IngressNginxDescribe("Annotations - AuthTLS", func() {
 		sslVerify := "ssl_verify_client on;"
 		sslVerifyDepth := "ssl_verify_depth 1;"
 
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
-				return strings.Contains(server, sslCertDirective) && strings.Contains(server, sslKeyDirective) && strings.Contains(server, sslClientCertDirective) && strings.Contains(server, sslVerify) && strings.Contains(server, sslVerifyDepth)
+				return strings.Contains(server, sslCertDirective) &&
+					strings.Contains(server, sslKeyDirective) &&
+					strings.Contains(server, sslClientCertDirective) &&
+					strings.Contains(server, sslVerify) &&
+					strings.Contains(server, sslVerifyDepth)
 			})
-		Expect(err).NotTo(HaveOccurred())
 
 		// Send Request without Client Certs
 		req := gorequest.New()
@@ -112,10 +112,7 @@ var _ = framework.IngressNginxDescribe("Annotations - AuthTLS", func() {
 		}
 
 		ing := framework.NewSingleIngressWithTLS(host, "/", host, nameSpace, "http-svc", 80, &annotations)
-		_, err = f.EnsureIngress(ing)
-
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
+		f.EnsureIngress(ing)
 
 		// Since we can use the same certificate-chain for tls as well as mutual-auth, we will check all values
 		sslCertDirective := fmt.Sprintf("ssl_certificate /etc/ingress-controller/ssl/%s-%s.pem;", nameSpace, host)
@@ -125,11 +122,10 @@ var _ = framework.IngressNginxDescribe("Annotations - AuthTLS", func() {
 		sslVerify := "ssl_verify_client off;"
 		sslVerifyDepth := "ssl_verify_depth 2;"
 
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return strings.Contains(server, sslCertDirective) && strings.Contains(server, sslKeyDirective) && strings.Contains(server, sslClientCertDirective) && strings.Contains(server, sslVerify) && strings.Contains(server, sslVerifyDepth)
 			})
-		Expect(err).NotTo(HaveOccurred())
 
 		// Send Request without Client Certs
 		req := gorequest.New()
@@ -163,9 +159,7 @@ var _ = framework.IngressNginxDescribe("Annotations - AuthTLS", func() {
 		}
 
 		ing := framework.NewSingleIngressWithTLS(host, "/", host, nameSpace, "http-svc", 80, &annotations)
-		_, err = f.EnsureIngress(ing)
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
+		f.EnsureIngress(ing)
 
 		// Since we can use the same certificate-chain for tls as well as mutual-auth, we will check all values
 		sslCertDirective := fmt.Sprintf("ssl_certificate /etc/ingress-controller/ssl/%s-%s.pem;", nameSpace, host)
@@ -177,11 +171,16 @@ var _ = framework.IngressNginxDescribe("Annotations - AuthTLS", func() {
 		sslErrorPage := fmt.Sprintf("error_page 495 496 = %s;", f.IngressController.HTTPURL+errorPath)
 		sslUpstreamClientCert := "proxy_set_header ssl-client-cert $ssl_client_escaped_cert;"
 
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
-				return strings.Contains(server, sslCertDirective) && strings.Contains(server, sslKeyDirective) && strings.Contains(server, sslClientCertDirective) && strings.Contains(server, sslVerify) && strings.Contains(server, sslVerifyDepth) && strings.Contains(server, sslErrorPage) && strings.Contains(server, sslUpstreamClientCert)
+				return strings.Contains(server, sslCertDirective) &&
+					strings.Contains(server, sslKeyDirective) &&
+					strings.Contains(server, sslClientCertDirective) &&
+					strings.Contains(server, sslVerify) &&
+					strings.Contains(server, sslVerifyDepth) &&
+					strings.Contains(server, sslErrorPage) &&
+					strings.Contains(server, sslUpstreamClientCert)
 			})
-		Expect(err).NotTo(HaveOccurred())
 
 		// Send Request without Client Certs
 		req := gorequest.New()
diff --git a/test/e2e/annotations/backendprotocol.go b/test/e2e/annotations/backendprotocol.go
index 30ebddfb0..1723e3b87 100644
--- a/test/e2e/annotations/backendprotocol.go
+++ b/test/e2e/annotations/backendprotocol.go
@@ -26,8 +26,7 @@ var _ = framework.IngressNginxDescribe("Annotations - Backendprotocol", func() {
 	f := framework.NewDefaultFramework("backendprotocol")
 
 	BeforeEach(func() {
-		err := f.NewEchoDeploymentWithReplicas(2)
-		Expect(err).NotTo(HaveOccurred())
+		f.NewEchoDeploymentWithReplicas(2)
 	})
 
 	AfterEach(func() {
@@ -40,16 +39,12 @@ var _ = framework.IngressNginxDescribe("Annotations - Backendprotocol", func() {
 		}
 
 		ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations)
-		_, err := f.EnsureIngress(ing)
+		f.EnsureIngress(ing)
 
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
-
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return Expect(server).Should(ContainSubstring("proxy_pass https://upstream_balancer;"))
 			})
-		Expect(err).NotTo(HaveOccurred())
 	})
 
 	It("should set backend protocol to grpc://", func() {
@@ -59,16 +54,12 @@ var _ = framework.IngressNginxDescribe("Annotations - Backendprotocol", func() {
 		}
 
 		ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations)
-		_, err := f.EnsureIngress(ing)
+		f.EnsureIngress(ing)
 
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
-
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return Expect(server).Should(ContainSubstring("grpc_pass grpc://upstream_balancer;"))
 			})
-		Expect(err).NotTo(HaveOccurred())
 	})
 
 	It("should set backend protocol to grpcs://", func() {
@@ -78,16 +69,12 @@ var _ = framework.IngressNginxDescribe("Annotations - Backendprotocol", func() {
 		}
 
 		ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations)
-		_, err := f.EnsureIngress(ing)
+		f.EnsureIngress(ing)
 
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
-
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return Expect(server).Should(ContainSubstring("grpc_pass grpcs://upstream_balancer;"))
 			})
-		Expect(err).NotTo(HaveOccurred())
 	})
 
 	It("should set backend protocol to ''", func() {
@@ -97,15 +84,11 @@ var _ = framework.IngressNginxDescribe("Annotations - Backendprotocol", func() {
 		}
 
 		ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations)
-		_, err := f.EnsureIngress(ing)
+		f.EnsureIngress(ing)
 
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
-
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return Expect(server).Should(ContainSubstring("ajp_pass upstream_balancer;"))
 			})
-		Expect(err).NotTo(HaveOccurred())
 	})
 })
diff --git a/test/e2e/annotations/clientbodybuffersize.go b/test/e2e/annotations/clientbodybuffersize.go
index 80ef8330b..621deb65d 100644
--- a/test/e2e/annotations/clientbodybuffersize.go
+++ b/test/e2e/annotations/clientbodybuffersize.go
@@ -26,8 +26,7 @@ var _ = framework.IngressNginxDescribe("Annotations - Client-Body-Buffer-Size",
 	f := framework.NewDefaultFramework("clientbodybuffersize")
 
 	BeforeEach(func() {
-		err := f.NewEchoDeploymentWithReplicas(2)
-		Expect(err).NotTo(HaveOccurred())
+		f.NewEchoDeploymentWithReplicas(2)
 	})
 
 	AfterEach(func() {
@@ -40,16 +39,12 @@ var _ = framework.IngressNginxDescribe("Annotations - Client-Body-Buffer-Size",
 		}
 
 		ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations)
-		_, err := f.EnsureIngress(ing)
+		f.EnsureIngress(ing)
 
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
-
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return Expect(server).Should(ContainSubstring("client_body_buffer_size 1000;"))
 			})
-		Expect(err).NotTo(HaveOccurred())
 	})
 
 	It("should set client_body_buffer_size to 1K", func() {
@@ -59,16 +54,12 @@ var _ = framework.IngressNginxDescribe("Annotations - Client-Body-Buffer-Size",
 		}
 
 		ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations)
-		_, err := f.EnsureIngress(ing)
+		f.EnsureIngress(ing)
 
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
-
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return Expect(server).Should(ContainSubstring("client_body_buffer_size 1K;"))
 			})
-		Expect(err).NotTo(HaveOccurred())
 	})
 
 	It("should set client_body_buffer_size to 1k", func() {
@@ -78,16 +69,12 @@ var _ = framework.IngressNginxDescribe("Annotations - Client-Body-Buffer-Size",
 		}
 
 		ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations)
-		_, err := f.EnsureIngress(ing)
+		f.EnsureIngress(ing)
 
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
-
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return Expect(server).Should(ContainSubstring("client_body_buffer_size 1k;"))
 			})
-		Expect(err).NotTo(HaveOccurred())
 	})
 
 	It("should set client_body_buffer_size to 1m", func() {
@@ -97,16 +84,12 @@ var _ = framework.IngressNginxDescribe("Annotations - Client-Body-Buffer-Size",
 		}
 
 		ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations)
-		_, err := f.EnsureIngress(ing)
+		f.EnsureIngress(ing)
 
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
-
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return Expect(server).Should(ContainSubstring("client_body_buffer_size 1m;"))
 			})
-		Expect(err).NotTo(HaveOccurred())
 	})
 
 	It("should set client_body_buffer_size to 1M", func() {
@@ -116,16 +99,12 @@ var _ = framework.IngressNginxDescribe("Annotations - Client-Body-Buffer-Size",
 		}
 
 		ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations)
-		_, err := f.EnsureIngress(ing)
+		f.EnsureIngress(ing)
 
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
-
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return Expect(server).Should(ContainSubstring("client_body_buffer_size 1M;"))
 			})
-		Expect(err).NotTo(HaveOccurred())
 	})
 
 	It("should not set client_body_buffer_size to invalid 1b", func() {
@@ -135,15 +114,11 @@ var _ = framework.IngressNginxDescribe("Annotations - Client-Body-Buffer-Size",
 		}
 
 		ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations)
-		_, err := f.EnsureIngress(ing)
+		f.EnsureIngress(ing)
 
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
-
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return Expect(server).ShouldNot(ContainSubstring("client_body_buffer_size 1b;"))
 			})
-		Expect(err).NotTo(HaveOccurred())
 	})
 })
diff --git a/test/e2e/annotations/connection.go b/test/e2e/annotations/connection.go
index ce14ccd3c..c281a2a69 100644
--- a/test/e2e/annotations/connection.go
+++ b/test/e2e/annotations/connection.go
@@ -31,8 +31,7 @@ var _ = framework.IngressNginxDescribe("Annotations - Connection", func() {
 	f := framework.NewDefaultFramework("connection")
 
 	BeforeEach(func() {
-		err := f.NewEchoDeploymentWithReplicas(2)
-		Expect(err).NotTo(HaveOccurred())
+		f.NewEchoDeploymentWithReplicas(2)
 	})
 
 	AfterEach(func() {
@@ -45,16 +44,12 @@ var _ = framework.IngressNginxDescribe("Annotations - Connection", func() {
 		}
 
 		ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations)
-		_, err := f.EnsureIngress(ing)
+		f.EnsureIngress(ing)
 
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
-
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return Expect(server).Should(ContainSubstring(`proxy_set_header Connection keep-alive;`))
 			})
-		Expect(err).NotTo(HaveOccurred())
 
 		resp, body, errs := gorequest.New().
 			Get(f.IngressController.HTTPURL).
diff --git a/test/e2e/annotations/cors.go b/test/e2e/annotations/cors.go
index e1abd7ada..8da735c61 100644
--- a/test/e2e/annotations/cors.go
+++ b/test/e2e/annotations/cors.go
@@ -17,10 +17,11 @@ limitations under the License.
 package annotations
 
 import (
+	"net/http"
+
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	"github.com/parnurzeal/gorequest"
-	"net/http"
 
 	"k8s.io/ingress-nginx/test/e2e/framework"
 )
@@ -29,8 +30,7 @@ var _ = framework.IngressNginxDescribe("Annotations - CORS", func() {
 	f := framework.NewDefaultFramework("cors")
 
 	BeforeEach(func() {
-		err := f.NewEchoDeploymentWithReplicas(2)
-		Expect(err).NotTo(HaveOccurred())
+		f.NewEchoDeploymentWithReplicas(2)
 	})
 
 	AfterEach(func() {
@@ -43,40 +43,32 @@ var _ = framework.IngressNginxDescribe("Annotations - CORS", func() {
 		}
 
 		ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations)
-		_, err := f.EnsureIngress(ing)
+		f.EnsureIngress(ing)
 
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
-
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return Expect(server).Should(ContainSubstring("more_set_headers 'Access-Control-Allow-Methods: GET, PUT, POST, DELETE, PATCH, OPTIONS';"))
 			})
-		Expect(err).NotTo(HaveOccurred())
 
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return Expect(server).Should(ContainSubstring("more_set_headers 'Access-Control-Allow-Origin: *';"))
 			})
-		Expect(err).NotTo(HaveOccurred())
 
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return Expect(server).Should(ContainSubstring("more_set_headers 'Access-Control-Allow-Headers: DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Authorization';"))
 			})
-		Expect(err).NotTo(HaveOccurred())
 
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return Expect(server).Should(ContainSubstring("more_set_headers 'Access-Control-Max-Age: 1728000';"))
 			})
-		Expect(err).NotTo(HaveOccurred())
 
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return Expect(server).Should(ContainSubstring("more_set_headers 'Access-Control-Allow-Credentials: true';"))
 			})
-		Expect(err).NotTo(HaveOccurred())
 
 		uri := "/"
 		resp, _, errs := gorequest.New().
@@ -95,16 +87,12 @@ var _ = framework.IngressNginxDescribe("Annotations - CORS", func() {
 		}
 
 		ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations)
-		_, err := f.EnsureIngress(ing)
+		f.EnsureIngress(ing)
 
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
-
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return Expect(server).Should(ContainSubstring("more_set_headers 'Access-Control-Allow-Methods: POST, GET';"))
 			})
-		Expect(err).NotTo(HaveOccurred())
 	})
 
 	It("should set cors max-age", func() {
@@ -115,16 +103,12 @@ var _ = framework.IngressNginxDescribe("Annotations - CORS", func() {
 		}
 
 		ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations)
-		_, err := f.EnsureIngress(ing)
+		f.EnsureIngress(ing)
 
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
-
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return Expect(server).Should(ContainSubstring("more_set_headers 'Access-Control-Max-Age: 200';"))
 			})
-		Expect(err).NotTo(HaveOccurred())
 	})
 
 	It("should disable cors allow credentials", func() {
@@ -135,16 +119,12 @@ var _ = framework.IngressNginxDescribe("Annotations - CORS", func() {
 		}
 
 		ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations)
-		_, err := f.EnsureIngress(ing)
+		f.EnsureIngress(ing)
 
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
-
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return Expect(server).ShouldNot(ContainSubstring("more_set_headers 'Access-Control-Allow-Credentials: true';"))
 			})
-		Expect(err).NotTo(HaveOccurred())
 	})
 
 	It("should allow origin for cors", func() {
@@ -155,16 +135,12 @@ var _ = framework.IngressNginxDescribe("Annotations - CORS", func() {
 		}
 
 		ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations)
-		_, err := f.EnsureIngress(ing)
+		f.EnsureIngress(ing)
 
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
-
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return Expect(server).Should(ContainSubstring("more_set_headers 'Access-Control-Allow-Origin: https://origin.cors.com:8080';"))
 			})
-		Expect(err).NotTo(HaveOccurred())
 	})
 
 	It("should allow headers for cors", func() {
@@ -175,15 +151,11 @@ var _ = framework.IngressNginxDescribe("Annotations - CORS", func() {
 		}
 
 		ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations)
-		_, err := f.EnsureIngress(ing)
+		f.EnsureIngress(ing)
 
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
-
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return Expect(server).Should(ContainSubstring("more_set_headers 'Access-Control-Allow-Headers: DNT, User-Agent';"))
 			})
-		Expect(err).NotTo(HaveOccurred())
 	})
 })
diff --git a/test/e2e/annotations/default_backend.go b/test/e2e/annotations/default_backend.go
index 1cec44c7e..a1d7967f9 100644
--- a/test/e2e/annotations/default_backend.go
+++ b/test/e2e/annotations/default_backend.go
@@ -32,8 +32,7 @@ var _ = framework.IngressNginxDescribe("Annotations - custom default-backend", f
 	f := framework.NewDefaultFramework("default-backend")
 
 	BeforeEach(func() {
-		err := f.NewEchoDeployment()
-		Expect(err).NotTo(HaveOccurred())
+		f.NewEchoDeployment()
 	})
 
 	Context("when default backend annotation is enabled", func() {
@@ -44,18 +43,14 @@ var _ = framework.IngressNginxDescribe("Annotations - custom default-backend", f
 			}
 
 			ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "invalid", 80, &annotations)
-			_, err := f.EnsureIngress(ing)
-
-			Expect(err).NotTo(HaveOccurred())
-			Expect(ing).NotTo(BeNil())
+			f.EnsureIngress(ing)
 
 			time.Sleep(5 * time.Second)
 
-			err = f.WaitForNginxServer(host,
+			f.WaitForNginxServer(host,
 				func(server string) bool {
 					return Expect(server).Should(ContainSubstring(fmt.Sprintf("server_name %v", host)))
 				})
-			Expect(err).NotTo(HaveOccurred())
 
 			uri := "/alma/armud"
 			resp, body, errs := gorequest.New().
diff --git a/test/e2e/annotations/forcesslredirect.go b/test/e2e/annotations/forcesslredirect.go
index dd3f963fe..590bf3c31 100644
--- a/test/e2e/annotations/forcesslredirect.go
+++ b/test/e2e/annotations/forcesslredirect.go
@@ -30,8 +30,7 @@ var _ = framework.IngressNginxDescribe("Annotations - Forcesslredirect", func()
 	f := framework.NewDefaultFramework("forcesslredirect")
 
 	BeforeEach(func() {
-		err := f.NewEchoDeploymentWithReplicas(2)
-		Expect(err).NotTo(HaveOccurred())
+		f.NewEchoDeploymentWithReplicas(2)
 	})
 
 	AfterEach(func() {
@@ -45,17 +44,13 @@ var _ = framework.IngressNginxDescribe("Annotations - Forcesslredirect", func()
 		}
 
 		ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations)
-		_, err := f.EnsureIngress(ing)
+		f.EnsureIngress(ing)
 
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
-
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return Expect(server).Should(ContainSubstring(`if ($redirect_to_https) {`)) &&
 					Expect(server).Should(ContainSubstring(`return 308 https://$best_http_host$request_uri;`))
 			})
-		Expect(err).NotTo(HaveOccurred())
 
 		resp, _, errs := gorequest.New().
 			Get(f.IngressController.HTTPURL).
diff --git a/test/e2e/annotations/fromtowwwredirect.go b/test/e2e/annotations/fromtowwwredirect.go
index b5b292577..6d1e597ed 100644
--- a/test/e2e/annotations/fromtowwwredirect.go
+++ b/test/e2e/annotations/fromtowwwredirect.go
@@ -31,8 +31,7 @@ var _ = framework.IngressNginxDescribe("Annotations - Fromtowwwredirect", func()
 	f := framework.NewDefaultFramework("fromtowwwredirect")
 
 	BeforeEach(func() {
-		err := f.NewEchoDeploymentWithReplicas(2)
-		Expect(err).NotTo(HaveOccurred())
+		f.NewEchoDeploymentWithReplicas(2)
 	})
 
 	AfterEach(func() {
@@ -47,17 +46,13 @@ var _ = framework.IngressNginxDescribe("Annotations - Fromtowwwredirect", func()
 		}
 
 		ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations)
-		_, err := f.EnsureIngress(ing)
+		f.EnsureIngress(ing)
 
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
-
-		err = f.WaitForNginxConfiguration(
+		f.WaitForNginxConfiguration(
 			func(cfg string) bool {
 				return Expect(cfg).Should(ContainSubstring(`server_name www.fromtowwwredirect.bar.com;`)) &&
 					Expect(cfg).Should(ContainSubstring(`return 308 $scheme://fromtowwwredirect.bar.com$request_uri;`))
 			})
-		Expect(err).NotTo(HaveOccurred())
 
 		By("sending request to www.fromtowwwredirect.bar.com")
 
diff --git a/test/e2e/annotations/grpc.go b/test/e2e/annotations/grpc.go
index 9bface49b..38ae3dc92 100644
--- a/test/e2e/annotations/grpc.go
+++ b/test/e2e/annotations/grpc.go
@@ -29,8 +29,7 @@ var _ = framework.IngressNginxDescribe("Annotations - grpc", func() {
 	f := framework.NewDefaultFramework("grpc")
 
 	BeforeEach(func() {
-		err := f.NewGRPCFortuneTellerDeployment()
-		Expect(err).NotTo(HaveOccurred())
+		f.NewGRPCFortuneTellerDeployment()
 	})
 
 	Context("when grpc is enabled", func() {
@@ -42,25 +41,20 @@ var _ = framework.IngressNginxDescribe("Annotations - grpc", func() {
 			}
 
 			ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "fortune-teller", 50051, &annotations)
-			_, err := f.EnsureIngress(ing)
+			f.EnsureIngress(ing)
 
-			Expect(err).NotTo(HaveOccurred())
-			Expect(ing).NotTo(BeNil())
-
-			err = f.WaitForNginxServer(host,
+			f.WaitForNginxServer(host,
 				func(server string) bool {
 					return Expect(server).Should(ContainSubstring(fmt.Sprintf("server_name %v", host))) &&
 						Expect(server).ShouldNot(ContainSubstring("return 503"))
 				})
-			Expect(err).NotTo(HaveOccurred())
 
-			err = f.WaitForNginxServer(host,
+			f.WaitForNginxServer(host,
 				func(server string) bool {
 					return Expect(server).Should(ContainSubstring("grpc_pass")) &&
 						Expect(server).Should(ContainSubstring("grpc_set_header")) &&
 						Expect(server).ShouldNot(ContainSubstring("proxy_pass"))
 				})
-			Expect(err).NotTo(HaveOccurred())
 		})
 	})
 })
diff --git a/test/e2e/annotations/influxdb.go b/test/e2e/annotations/influxdb.go
index 6ea3bc4f6..45c1957d0 100644
--- a/test/e2e/annotations/influxdb.go
+++ b/test/e2e/annotations/influxdb.go
@@ -23,9 +23,10 @@ import (
 	"os/exec"
 	"time"
 
-	jsoniter "github.com/json-iterator/go"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+
+	jsoniter "github.com/json-iterator/go"
 	"github.com/parnurzeal/gorequest"
 
 	corev1 "k8s.io/api/core/v1"
@@ -39,17 +40,13 @@ var _ = framework.IngressNginxDescribe("Annotations - influxdb", func() {
 	f := framework.NewDefaultFramework("influxdb")
 
 	BeforeEach(func() {
-		err := f.NewInfluxDBDeployment()
-		Expect(err).NotTo(HaveOccurred())
-		err = f.NewEchoDeployment()
-		Expect(err).NotTo(HaveOccurred())
+		f.NewInfluxDBDeployment()
+		f.NewEchoDeployment()
 	})
 
 	Context("when influxdb is enabled", func() {
 		It("should send the request metric to the influxdb server", func() {
-			ifs, err := createInfluxDBService(f)
-
-			Expect(err).NotTo(HaveOccurred())
+			ifs := createInfluxDBService(f)
 
 			// Ingress configured with InfluxDB annotations
 			host := "influxdb.e2e.local"
@@ -80,6 +77,8 @@ var _ = framework.IngressNginxDescribe("Annotations - influxdb", func() {
 			time.Sleep(5 * time.Second)
 
 			var measurements string
+			var err error
+
 			err = wait.PollImmediate(time.Second, time.Minute, func() (bool, error) {
 				measurements, err = extractInfluxDBMeasurements(f)
 				if err != nil {
@@ -100,7 +99,7 @@ var _ = framework.IngressNginxDescribe("Annotations - influxdb", func() {
 	})
 })
 
-func createInfluxDBService(f *framework.Framework) (*corev1.Service, error) {
+func createInfluxDBService(f *framework.Framework) *corev1.Service {
 	service := &corev1.Service{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "inflxudb-svc",
@@ -120,29 +119,18 @@ func createInfluxDBService(f *framework.Framework) (*corev1.Service, error) {
 		},
 	}
 
-	s, err := f.EnsureService(service)
-	if err != nil {
-		return nil, err
-	}
-
-	if s == nil {
-		return nil, fmt.Errorf("unexpected error creating service for influxdb deployment")
-	}
-
-	return s, nil
+	return f.EnsureService(service)
 }
 
 func createInfluxDBIngress(f *framework.Framework, host, service string, port int, annotations map[string]string) {
-	ing, err := f.EnsureIngress(framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, service, port, &annotations))
-	Expect(err).NotTo(HaveOccurred())
-	Expect(ing).NotTo(BeNil())
+	ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, service, port, &annotations)
+	f.EnsureIngress(ing)
 
-	err = f.WaitForNginxServer(host,
+	f.WaitForNginxServer(host,
 		func(server string) bool {
 			return Expect(server).Should(ContainSubstring(fmt.Sprintf("server_name %v", host))) &&
 				Expect(server).ShouldNot(ContainSubstring("return 503"))
 		})
-	Expect(err).NotTo(HaveOccurred())
 }
 
 func extractInfluxDBMeasurements(f *framework.Framework) (string, error) {
diff --git a/test/e2e/annotations/ipwhitelist.go b/test/e2e/annotations/ipwhitelist.go
index f5ac47e63..79125cff8 100644
--- a/test/e2e/annotations/ipwhitelist.go
+++ b/test/e2e/annotations/ipwhitelist.go
@@ -17,19 +17,19 @@ limitations under the License.
 package annotations
 
 import (
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-	"k8s.io/ingress-nginx/test/e2e/framework"
 	"regexp"
 	"strings"
+
+	. "github.com/onsi/ginkgo"
+
+	"k8s.io/ingress-nginx/test/e2e/framework"
 )
 
 var _ = framework.IngressNginxDescribe("Annotations - IPWhiteList", func() {
 	f := framework.NewDefaultFramework("ipwhitelist")
 
 	BeforeEach(func() {
-		err := f.NewEchoDeploymentWithReplicas(2)
-		Expect(err).NotTo(HaveOccurred())
+		f.NewEchoDeploymentWithReplicas(2)
 	})
 
 	AfterEach(func() {
@@ -44,15 +44,12 @@ var _ = framework.IngressNginxDescribe("Annotations - IPWhiteList", func() {
 		}
 
 		ing := framework.NewSingleIngress(host, "/", host, nameSpace, "http-svc", 80, &annotations)
-		_, err := f.EnsureIngress(ing)
-
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
+		f.EnsureIngress(ing)
 
 		denyRegex := regexp.MustCompile("geo \\$the_real_ip \\$deny_[A-Za-z]{32}")
 		denyString := ""
 
-		err = f.WaitForNginxConfiguration(
+		f.WaitForNginxConfiguration(
 			func(conf string) bool {
 
 				match := denyRegex.FindStringSubmatch(conf)
@@ -64,22 +61,19 @@ var _ = framework.IngressNginxDescribe("Annotations - IPWhiteList", func() {
 				denyString = strings.Replace(match[0], "geo $the_real_ip ", "", -1)
 				return strings.Contains(conf, match[0])
 			})
-		Expect(err).NotTo(HaveOccurred())
 
 		ipOne := "18.0.0.0/8 0;"
 		ipTwo := "56.0.0.0/8 0;"
 
-		err = f.WaitForNginxConfiguration(
+		f.WaitForNginxConfiguration(
 			func(conf string) bool {
 				return strings.Contains(conf, ipOne) && strings.Contains(conf, ipTwo)
 			})
-		Expect(err).NotTo(HaveOccurred())
 
 		denyStatement := "if (" + denyString + ")"
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return strings.Contains(server, denyStatement)
 			})
-		Expect(err).NotTo(HaveOccurred())
 	})
 })
diff --git a/test/e2e/annotations/log.go b/test/e2e/annotations/log.go
index 884ffd524..9ae77ad51 100644
--- a/test/e2e/annotations/log.go
+++ b/test/e2e/annotations/log.go
@@ -19,6 +19,7 @@ package annotations
 import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+
 	"k8s.io/ingress-nginx/test/e2e/framework"
 )
 
@@ -26,8 +27,7 @@ var _ = framework.IngressNginxDescribe("Annotations - Log", func() {
 	f := framework.NewDefaultFramework("log")
 
 	BeforeEach(func() {
-		err := f.NewEchoDeploymentWithReplicas(2)
-		Expect(err).NotTo(HaveOccurred())
+		f.NewEchoDeploymentWithReplicas(2)
 	})
 
 	AfterEach(func() {
@@ -40,16 +40,12 @@ var _ = framework.IngressNginxDescribe("Annotations - Log", func() {
 		}
 
 		ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations)
-		_, err := f.EnsureIngress(ing)
+		f.EnsureIngress(ing)
 
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
-
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return Expect(server).Should(ContainSubstring(`access_log off;`))
 			})
-		Expect(err).NotTo(HaveOccurred())
 	})
 
 	It("set rewrite_log on", func() {
@@ -59,15 +55,11 @@ var _ = framework.IngressNginxDescribe("Annotations - Log", func() {
 		}
 
 		ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations)
-		_, err := f.EnsureIngress(ing)
+		f.EnsureIngress(ing)
 
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
-
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return Expect(server).Should(ContainSubstring(`rewrite_log on;`))
 			})
-		Expect(err).NotTo(HaveOccurred())
 	})
 })
diff --git a/test/e2e/annotations/luarestywaf.go b/test/e2e/annotations/luarestywaf.go
index a20fbdc49..0b95fb6e8 100644
--- a/test/e2e/annotations/luarestywaf.go
+++ b/test/e2e/annotations/luarestywaf.go
@@ -32,8 +32,7 @@ var _ = framework.IngressNginxDescribe("Annotations - lua-resty-waf", func() {
 	f := framework.NewDefaultFramework("luarestywaf")
 
 	BeforeEach(func() {
-		err := f.NewEchoDeployment()
-		Expect(err).NotTo(HaveOccurred())
+		f.NewEchoDeployment()
 	})
 
 	Context("when lua-resty-waf is enabled", func() {
@@ -204,16 +203,14 @@ var _ = framework.IngressNginxDescribe("Annotations - lua-resty-waf", func() {
 })
 
 func createIngress(f *framework.Framework, host, service string, port int, annotations map[string]string) {
-	ing, err := f.EnsureIngress(framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, service, port, &annotations))
-	Expect(err).NotTo(HaveOccurred())
-	Expect(ing).NotTo(BeNil())
+	ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, service, port, &annotations)
+	f.EnsureIngress(ing)
 
-	err = f.WaitForNginxServer(host,
+	f.WaitForNginxServer(host,
 		func(server string) bool {
 			return Expect(server).Should(ContainSubstring(fmt.Sprintf("server_name %v", host))) &&
 				Expect(server).ShouldNot(ContainSubstring("return 503"))
 		})
-	Expect(err).NotTo(HaveOccurred())
 
 	time.Sleep(1 * time.Second)
 
diff --git a/test/e2e/annotations/proxy.go b/test/e2e/annotations/proxy.go
index b1a116d7f..83f8df1b9 100644
--- a/test/e2e/annotations/proxy.go
+++ b/test/e2e/annotations/proxy.go
@@ -17,9 +17,10 @@ limitations under the License.
 package annotations
 
 import (
+	"strings"
+
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
-	"strings"
 
 	"k8s.io/ingress-nginx/test/e2e/framework"
 )
@@ -28,8 +29,7 @@ var _ = framework.IngressNginxDescribe("Annotations - Proxy", func() {
 	f := framework.NewDefaultFramework("proxy")
 
 	BeforeEach(func() {
-		err := f.NewEchoDeploymentWithReplicas(2)
-		Expect(err).NotTo(HaveOccurred())
+		f.NewEchoDeploymentWithReplicas(2)
 	})
 
 	AfterEach(func() {
@@ -43,16 +43,12 @@ var _ = framework.IngressNginxDescribe("Annotations - Proxy", func() {
 		}
 
 		ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations)
-		_, err := f.EnsureIngress(ing)
+		f.EnsureIngress(ing)
 
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
-
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return Expect(server).Should(ContainSubstring("proxy_redirect off;"))
 			})
-		Expect(err).NotTo(HaveOccurred())
 	})
 
 	It("should set proxy_redirect to default", func() {
@@ -63,16 +59,12 @@ var _ = framework.IngressNginxDescribe("Annotations - Proxy", func() {
 		}
 
 		ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations)
-		_, err := f.EnsureIngress(ing)
+		f.EnsureIngress(ing)
 
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
-
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return Expect(server).Should(ContainSubstring("proxy_redirect default;"))
 			})
-		Expect(err).NotTo(HaveOccurred())
 	})
 
 	It("should set proxy_redirect to hello.com goodbye.com", func() {
@@ -83,16 +75,12 @@ var _ = framework.IngressNginxDescribe("Annotations - Proxy", func() {
 		}
 
 		ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations)
-		_, err := f.EnsureIngress(ing)
+		f.EnsureIngress(ing)
 
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
-
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return Expect(server).Should(ContainSubstring("proxy_redirect hello.com goodbye.com;"))
 			})
-		Expect(err).NotTo(HaveOccurred())
 	})
 
 	It("should set proxy client-max-body-size to 8m", func() {
@@ -102,16 +90,12 @@ var _ = framework.IngressNginxDescribe("Annotations - Proxy", func() {
 		}
 
 		ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations)
-		_, err := f.EnsureIngress(ing)
+		f.EnsureIngress(ing)
 
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
-
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return Expect(server).Should(ContainSubstring("client_max_body_size 8m;"))
 			})
-		Expect(err).NotTo(HaveOccurred())
 	})
 
 	It("should not set proxy client-max-body-size to incorrect value", func() {
@@ -121,16 +105,12 @@ var _ = framework.IngressNginxDescribe("Annotations - Proxy", func() {
 		}
 
 		ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations)
-		_, err := f.EnsureIngress(ing)
+		f.EnsureIngress(ing)
 
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
-
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return Expect(server).ShouldNot(ContainSubstring("client_max_body_size 15r;"))
 			})
-		Expect(err).NotTo(HaveOccurred())
 	})
 
 	It("should set valid proxy timeouts", func() {
@@ -142,16 +122,12 @@ var _ = framework.IngressNginxDescribe("Annotations - Proxy", func() {
 		}
 
 		ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations)
-		_, err := f.EnsureIngress(ing)
+		f.EnsureIngress(ing)
 
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
-
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return strings.Contains(server, "proxy_connect_timeout 50s;") && strings.Contains(server, "proxy_send_timeout 20s;") && strings.Contains(server, "proxy_read_timeout 20s;")
 			})
-		Expect(err).NotTo(HaveOccurred())
 	})
 
 	It("should not set invalid proxy timeouts", func() {
@@ -163,16 +139,12 @@ var _ = framework.IngressNginxDescribe("Annotations - Proxy", func() {
 		}
 
 		ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations)
-		_, err := f.EnsureIngress(ing)
+		f.EnsureIngress(ing)
 
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
-
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return !strings.Contains(server, "proxy_connect_timeout 50ks;") && !strings.Contains(server, "proxy_send_timeout 20ks;") && !strings.Contains(server, "proxy_read_timeout 60s;")
 			})
-		Expect(err).NotTo(HaveOccurred())
 	})
 
 	It("should turn on proxy-buffering", func() {
@@ -183,16 +155,12 @@ var _ = framework.IngressNginxDescribe("Annotations - Proxy", func() {
 		}
 
 		ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations)
-		_, err := f.EnsureIngress(ing)
+		f.EnsureIngress(ing)
 
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
-
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return strings.Contains(server, "proxy_buffering on;") && strings.Contains(server, "proxy_buffer_size 8k;") && strings.Contains(server, "proxy_buffers 4 8k;") && strings.Contains(server, "proxy_request_buffering on;")
 			})
-		Expect(err).NotTo(HaveOccurred())
 	})
 
 	It("should turn off proxy-request-buffering", func() {
@@ -202,16 +170,12 @@ var _ = framework.IngressNginxDescribe("Annotations - Proxy", func() {
 		}
 
 		ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations)
-		_, err := f.EnsureIngress(ing)
+		f.EnsureIngress(ing)
 
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
-
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return Expect(server).Should(ContainSubstring("proxy_request_buffering off;"))
 			})
-		Expect(err).NotTo(HaveOccurred())
 	})
 
 	It("should build proxy next upstream", func() {
@@ -222,16 +186,12 @@ var _ = framework.IngressNginxDescribe("Annotations - Proxy", func() {
 		}
 
 		ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations)
-		_, err := f.EnsureIngress(ing)
+		f.EnsureIngress(ing)
 
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
-
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return strings.Contains(server, "proxy_next_upstream error timeout http_502;") && strings.Contains(server, "proxy_next_upstream_tries 5;")
 			})
-		Expect(err).NotTo(HaveOccurred())
 	})
 
 	It("should setup proxy cookies", func() {
@@ -242,16 +202,11 @@ var _ = framework.IngressNginxDescribe("Annotations - Proxy", func() {
 		}
 
 		ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations)
-		_, err := f.EnsureIngress(ing)
+		f.EnsureIngress(ing)
 
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
-
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return strings.Contains(server, "proxy_cookie_domain localhost example.org;") && strings.Contains(server, "proxy_cookie_path /one/ /;")
 			})
-		Expect(err).NotTo(HaveOccurred())
 	})
-
 })
diff --git a/test/e2e/annotations/redirect.go b/test/e2e/annotations/redirect.go
index a2ce10d25..563c24778 100644
--- a/test/e2e/annotations/redirect.go
+++ b/test/e2e/annotations/redirect.go
@@ -53,17 +53,13 @@ var _ = framework.IngressNginxDescribe("Annotations - Redirect", func() {
 		annotations := map[string]string{"nginx.ingress.kubernetes.io/permanent-redirect": redirectURL}
 
 		ing := framework.NewSingleIngress(host, redirectPath, host, f.IngressController.Namespace, "http-svc", 80, &annotations)
-		_, err := f.EnsureIngress(ing)
+		f.EnsureIngress(ing)
 
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
-
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return strings.Contains(server, fmt.Sprintf("if ($uri ~* %s) {", redirectPath)) &&
 					strings.Contains(server, fmt.Sprintf("return 301 %s;", redirectURL))
 			})
-		Expect(err).NotTo(HaveOccurred())
 
 		By("sending request to redirected URL path")
 
@@ -93,17 +89,13 @@ var _ = framework.IngressNginxDescribe("Annotations - Redirect", func() {
 		}
 
 		ing := framework.NewSingleIngress(host, redirectPath, host, f.IngressController.Namespace, "http-svc", 80, &annotations)
-		_, err := f.EnsureIngress(ing)
+		f.EnsureIngress(ing)
 
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
-
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return strings.Contains(server, fmt.Sprintf("if ($uri ~* %s) {", redirectPath)) &&
 					strings.Contains(server, fmt.Sprintf("return %d %s;", redirectCode, redirectURL))
 			})
-		Expect(err).NotTo(HaveOccurred())
 
 		By("sending request to redirected URL path")
 
diff --git a/test/e2e/annotations/rewrite.go b/test/e2e/annotations/rewrite.go
index 70c89db58..5f6c89f98 100644
--- a/test/e2e/annotations/rewrite.go
+++ b/test/e2e/annotations/rewrite.go
@@ -33,8 +33,7 @@ var _ = framework.IngressNginxDescribe("Annotations - Rewrite", func() {
 	f := framework.NewDefaultFramework("rewrite")
 
 	BeforeEach(func() {
-		err := f.NewEchoDeploymentWithReplicas(1)
-		Expect(err).NotTo(HaveOccurred())
+		f.NewEchoDeploymentWithReplicas(1)
 	})
 
 	AfterEach(func() {
@@ -48,17 +47,13 @@ var _ = framework.IngressNginxDescribe("Annotations - Rewrite", func() {
 		expectBodyRequestURI := fmt.Sprintf("request_uri=http://%v:8080/", host)
 
 		ing := framework.NewSingleIngress(host, "/something", host, f.IngressController.Namespace, "http-svc", 80, &annotations)
-		_, err := f.EnsureIngress(ing)
+		f.EnsureIngress(ing)
 
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
-
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return strings.Contains(server, `rewrite "(?i)/something/(.*)" /$1 break;`) &&
 					strings.Contains(server, `rewrite "(?i)/something$" / break;`)
 			})
-		Expect(err).NotTo(HaveOccurred())
 
 		By("sending request to Ingress rule path (lowercase)")
 
@@ -93,16 +88,12 @@ var _ = framework.IngressNginxDescribe("Annotations - Rewrite", func() {
 		}
 
 		ing := framework.NewSingleIngress(host, "/something", host, f.IngressController.Namespace, "http-svc", 80, &annotations)
-		_, err := f.EnsureIngress(ing)
+		f.EnsureIngress(ing)
 
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
-
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return strings.Contains(server, "rewrite_log on;")
 			})
-		Expect(err).NotTo(HaveOccurred())
 
 		resp, _, errs := gorequest.New().
 			Get(f.IngressController.HTTPURL+"/something").
@@ -123,15 +114,12 @@ var _ = framework.IngressNginxDescribe("Annotations - Rewrite", func() {
 
 		By("creating a regular ingress definition")
 		ing := framework.NewSingleIngress("kube-lego", "/.well-known/acme/challenge", host, f.IngressController.Namespace, "http-svc", 80, &map[string]string{})
-		_, err := f.EnsureIngress(ing)
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
+		f.EnsureIngress(ing)
 
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return strings.Contains(server, "/.well-known/acme/challenge")
 			})
-		Expect(err).NotTo(HaveOccurred())
 
 		By("making a request to the non-rewritten location")
 		resp, body, errs := gorequest.New().
@@ -148,15 +136,13 @@ var _ = framework.IngressNginxDescribe("Annotations - Rewrite", func() {
 			"nginx.ingress.kubernetes.io/rewrite-target": "/new/backend",
 		}
 		rewriteIng := framework.NewSingleIngress("rewrite-index", "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations)
-		_, err = f.EnsureIngress(rewriteIng)
-		Expect(err).NotTo(HaveOccurred())
-		Expect(rewriteIng).NotTo(BeNil())
 
-		err = f.WaitForNginxServer(host,
+		f.EnsureIngress(rewriteIng)
+
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return strings.Contains(server, "location ~* ^/ {") && strings.Contains(server, `location ~* "^/.well-known/acme/challenge" {`)
 			})
-		Expect(err).NotTo(HaveOccurred())
 
 		By("making a second request to the non-rewritten location")
 		resp, body, errs = gorequest.New().
@@ -173,15 +159,12 @@ var _ = framework.IngressNginxDescribe("Annotations - Rewrite", func() {
 
 		By("creating a regular ingress definition")
 		ing := framework.NewSingleIngress("foo", "/foo", host, f.IngressController.Namespace, "http-svc", 80, &map[string]string{})
-		_, err := f.EnsureIngress(ing)
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
+		f.EnsureIngress(ing)
 
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return strings.Contains(server, "location /foo {")
 			})
-		Expect(err).NotTo(HaveOccurred())
 
 		By(`creating an ingress definition with the use-regex amd rewrite-target annotation`)
 		annotations := map[string]string{
@@ -189,15 +172,12 @@ var _ = framework.IngressNginxDescribe("Annotations - Rewrite", func() {
 			"nginx.ingress.kubernetes.io/rewrite-target": "/new/backend",
 		}
 		ing = framework.NewSingleIngress("regex", "/foo.+", host, f.IngressController.Namespace, "http-svc", 80, &annotations)
-		_, err = f.EnsureIngress(ing)
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
+		f.EnsureIngress(ing)
 
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return strings.Contains(server, `location ~* "^/foo" {`) && strings.Contains(server, `location ~* "^/foo.+\/?(?<baseuri>.*)" {`)
 			})
-		Expect(err).NotTo(HaveOccurred())
 
 		By("ensuring '/foo' matches '~* ^/foo'")
 		resp, body, errs := gorequest.New().
@@ -225,9 +205,7 @@ var _ = framework.IngressNginxDescribe("Annotations - Rewrite", func() {
 
 		By("creating a regular ingress definition")
 		ing := framework.NewSingleIngress("foo", "/foo/bar/bar", host, f.IngressController.Namespace, "http-svc", 80, &map[string]string{})
-		_, err := f.EnsureIngress(ing)
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
+		f.EnsureIngress(ing)
 
 		By(`creating an ingress definition with the use-regex annotation`)
 		annotations := map[string]string{
@@ -235,15 +213,12 @@ var _ = framework.IngressNginxDescribe("Annotations - Rewrite", func() {
 			"nginx.ingress.kubernetes.io/rewrite-target": "/new/backend",
 		}
 		ing = framework.NewSingleIngress("regex", "/foo/bar/[a-z]{3}", host, f.IngressController.Namespace, "http-svc", 80, &annotations)
-		_, err = f.EnsureIngress(ing)
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
+		f.EnsureIngress(ing)
 
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return strings.Contains(server, `location ~* "^/foo/bar/bar" {`) && strings.Contains(server, `location ~* "^/foo/bar/[a-z]{3}\/?(?<baseuri>.*)" {`)
 			})
-		Expect(err).NotTo(HaveOccurred())
 
 		By("check that '/foo/bar/bar' does not match the longest exact path")
 		resp, body, errs := gorequest.New().
diff --git a/test/e2e/annotations/serversnippet.go b/test/e2e/annotations/serversnippet.go
index e96483f9f..90bb36a25 100644
--- a/test/e2e/annotations/serversnippet.go
+++ b/test/e2e/annotations/serversnippet.go
@@ -17,18 +17,18 @@ limitations under the License.
 package annotations
 
 import (
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-	"k8s.io/ingress-nginx/test/e2e/framework"
 	"strings"
+
+	. "github.com/onsi/ginkgo"
+
+	"k8s.io/ingress-nginx/test/e2e/framework"
 )
 
 var _ = framework.IngressNginxDescribe("Annotations - ServerSnippet", func() {
 	f := framework.NewDefaultFramework("serversnippet")
 
 	BeforeEach(func() {
-		err := f.NewEchoDeploymentWithReplicas(2)
-		Expect(err).NotTo(HaveOccurred())
+		f.NewEchoDeploymentWithReplicas(2)
 	})
 
 	AfterEach(func() {
@@ -43,15 +43,11 @@ var _ = framework.IngressNginxDescribe("Annotations - ServerSnippet", func() {
 		}
 
 		ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations)
-		_, err := f.EnsureIngress(ing)
+		f.EnsureIngress(ing)
 
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
-
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return strings.Contains(server, `more_set_headers "Content-Length: $content_length`) && strings.Contains(server, `more_set_headers "Content-Type: $content_type";`)
 			})
-		Expect(err).NotTo(HaveOccurred())
 	})
 })
diff --git a/test/e2e/annotations/snippet.go b/test/e2e/annotations/snippet.go
index 81ee9724e..1686fb37b 100644
--- a/test/e2e/annotations/snippet.go
+++ b/test/e2e/annotations/snippet.go
@@ -26,8 +26,7 @@ var _ = framework.IngressNginxDescribe("Annotations - Configurationsnippet", fun
 	f := framework.NewDefaultFramework("configurationsnippet")
 
 	BeforeEach(func() {
-		err := f.NewEchoDeploymentWithReplicas(2)
-		Expect(err).NotTo(HaveOccurred())
+		f.NewEchoDeploymentWithReplicas(2)
 	})
 
 	AfterEach(func() {
@@ -41,15 +40,11 @@ var _ = framework.IngressNginxDescribe("Annotations - Configurationsnippet", fun
 		}
 
 		ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations)
-		_, err := f.EnsureIngress(ing)
+		f.EnsureIngress(ing)
 
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
-
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return Expect(server).Should(ContainSubstring(`more_set_headers "Request-Id: $req_id";`))
 			})
-		Expect(err).NotTo(HaveOccurred())
 	})
 })
diff --git a/test/e2e/annotations/sslciphers.go b/test/e2e/annotations/sslciphers.go
index b71a39cfe..859444022 100644
--- a/test/e2e/annotations/sslciphers.go
+++ b/test/e2e/annotations/sslciphers.go
@@ -27,8 +27,7 @@ var _ = framework.IngressNginxDescribe("Annotations - SSL CIPHERS", func() {
 	f := framework.NewDefaultFramework("sslciphers")
 
 	BeforeEach(func() {
-		err := f.NewEchoDeploymentWithReplicas(2)
-		Expect(err).NotTo(HaveOccurred())
+		f.NewEchoDeploymentWithReplicas(2)
 	})
 
 	AfterEach(func() {
@@ -41,15 +40,11 @@ var _ = framework.IngressNginxDescribe("Annotations - SSL CIPHERS", func() {
 		}
 
 		ing := framework.NewSingleIngress(host, "/something", host, f.IngressController.Namespace, "http-svc", 80, &annotations)
-		_, err := f.EnsureIngress(ing)
+		f.EnsureIngress(ing)
 
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
-
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return Expect(server).Should(ContainSubstring("ssl_ciphers ALL:!aNULL:!EXPORT56:RC4+RSA:+HIGH:+MEDIUM:+LOW:+SSLv2:+EXP;"))
 			})
-		Expect(err).NotTo(HaveOccurred())
 	})
 })
diff --git a/test/e2e/annotations/upstreamvhost.go b/test/e2e/annotations/upstreamvhost.go
index ff57396a5..ae5789425 100644
--- a/test/e2e/annotations/upstreamvhost.go
+++ b/test/e2e/annotations/upstreamvhost.go
@@ -26,8 +26,7 @@ var _ = framework.IngressNginxDescribe("Annotations - Upstreamvhost", func() {
 	f := framework.NewDefaultFramework("upstreamvhost")
 
 	BeforeEach(func() {
-		err := f.NewEchoDeploymentWithReplicas(2)
-		Expect(err).NotTo(HaveOccurred())
+		f.NewEchoDeploymentWithReplicas(2)
 	})
 
 	AfterEach(func() {
@@ -40,15 +39,11 @@ var _ = framework.IngressNginxDescribe("Annotations - Upstreamvhost", func() {
 		}
 
 		ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations)
-		_, err := f.EnsureIngress(ing)
+		f.EnsureIngress(ing)
 
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
-
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return Expect(server).Should(ContainSubstring(`proxy_set_header Host "upstreamvhost.bar.com";`))
 			})
-		Expect(err).NotTo(HaveOccurred())
 	})
 })
diff --git a/test/e2e/defaultbackend/custom_default_backend.go b/test/e2e/defaultbackend/custom_default_backend.go
index ea9e9f5dd..c2104b415 100644
--- a/test/e2e/defaultbackend/custom_default_backend.go
+++ b/test/e2e/defaultbackend/custom_default_backend.go
@@ -23,6 +23,7 @@ import (
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+
 	"github.com/parnurzeal/gorequest"
 
 	appsv1beta1 "k8s.io/api/apps/v1beta1"
@@ -34,8 +35,7 @@ var _ = framework.IngressNginxDescribe("Dynamic Certificate", func() {
 	f := framework.NewDefaultFramework("custom-default-backend")
 
 	BeforeEach(func() {
-		err := f.NewEchoDeploymentWithReplicas(1)
-		Expect(err).NotTo(HaveOccurred())
+		f.NewEchoDeploymentWithReplicas(1)
 
 		framework.UpdateDeployment(f.KubeClientSet, f.IngressController.Namespace, "nginx-ingress-controller", 1,
 			func(deployment *appsv1beta1.Deployment) error {
@@ -47,11 +47,10 @@ var _ = framework.IngressNginxDescribe("Dynamic Certificate", func() {
 				return err
 			})
 
-		err = f.WaitForNginxServer("_",
+		f.WaitForNginxServer("_",
 			func(server string) bool {
 				return strings.Contains(server, "set $proxy_upstream_name \"upstream-default-backend\"")
 			})
-		Expect(err).ToNot(HaveOccurred())
 	})
 
 	It("uses custom default backend", func() {
diff --git a/test/e2e/framework/deployment.go b/test/e2e/framework/deployment.go
index 122c20f1a..a01e3d333 100644
--- a/test/e2e/framework/deployment.go
+++ b/test/e2e/framework/deployment.go
@@ -17,10 +17,9 @@ limitations under the License.
 package framework
 
 import (
-	"fmt"
 	"time"
 
-	"github.com/pkg/errors"
+	. "github.com/onsi/gomega"
 
 	corev1 "k8s.io/api/core/v1"
 	extensions "k8s.io/api/extensions/v1beta1"
@@ -30,23 +29,23 @@ import (
 )
 
 // NewEchoDeployment creates a new single replica deployment of the echoserver image in a particular namespace
-func (f *Framework) NewEchoDeployment() error {
-	return f.NewEchoDeploymentWithReplicas(1)
+func (f *Framework) NewEchoDeployment() {
+	f.NewEchoDeploymentWithReplicas(1)
 }
 
 // NewEchoDeploymentWithReplicas creates a new deployment of the echoserver image in a particular namespace. Number of
 // replicas is configurable
-func (f *Framework) NewEchoDeploymentWithReplicas(replicas int32) error {
-	return f.NewDeployment("http-svc", "gcr.io/kubernetes-e2e-test-images/echoserver:2.1", 8080, replicas)
+func (f *Framework) NewEchoDeploymentWithReplicas(replicas int32) {
+	f.NewDeployment("http-svc", "gcr.io/kubernetes-e2e-test-images/echoserver:2.1", 8080, replicas)
 }
 
 // NewHttpbinDeployment creates a new single replica deployment of the httpbin image in a particular namespace.
-func (f *Framework) NewHttpbinDeployment() error {
-	return f.NewDeployment("httpbin", "kennethreitz/httpbin", 80, 1)
+func (f *Framework) NewHttpbinDeployment() {
+	f.NewDeployment("httpbin", "kennethreitz/httpbin", 80, 1)
 }
 
 // NewDeployment creates a new deployment in a particular namespace.
-func (f *Framework) NewDeployment(name, image string, port int32, replicas int32) error {
+func (f *Framework) NewDeployment(name, image string, port int32, replicas int32) {
 	deployment := &extensions.Deployment{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      name,
@@ -86,20 +85,13 @@ func (f *Framework) NewDeployment(name, image string, port int32, replicas int32
 	}
 
 	d, err := f.EnsureDeployment(deployment)
-	if err != nil {
-		return err
-	}
-
-	if d == nil {
-		return fmt.Errorf("unexpected error creating deployement %s", name)
-	}
+	Expect(err).NotTo(HaveOccurred(), "failed to create a deployment")
+	Expect(d).NotTo(BeNil(), "expected a deployement but none returned")
 
 	err = WaitForPodsReady(f.KubeClientSet, 5*time.Minute, int(replicas), f.IngressController.Namespace, metav1.ListOptions{
 		LabelSelector: fields.SelectorFromSet(fields.Set(d.Spec.Template.ObjectMeta.Labels)).String(),
 	})
-	if err != nil {
-		return errors.Wrap(err, "failed to wait for to become ready")
-	}
+	Expect(err).NotTo(HaveOccurred(), "failed to wait for to become ready")
 
 	service := &corev1.Service{
 		ObjectMeta: metav1.ObjectMeta{
@@ -121,14 +113,6 @@ func (f *Framework) NewDeployment(name, image string, port int32, replicas int32
 		},
 	}
 
-	s, err := f.EnsureService(service)
-	if err != nil {
-		return err
-	}
-
-	if s == nil {
-		return fmt.Errorf("unexpected error creating service %s", name)
-	}
-
-	return nil
+	s := f.EnsureService(service)
+	Expect(s).NotTo(BeNil(), "expected a service but none returned")
 }
diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go
index cbb635c58..d2ec36035 100644
--- a/test/e2e/framework/framework.go
+++ b/test/e2e/framework/framework.go
@@ -117,14 +117,10 @@ func (f *Framework) BeforeEach() {
 	})
 	Expect(err).NotTo(HaveOccurred())
 
-	HTTPURL, err := f.GetNginxURL(HTTP)
-	Expect(err).NotTo(HaveOccurred())
-
+	HTTPURL := f.GetNginxURL(HTTP)
 	f.IngressController.HTTPURL = HTTPURL
 
-	HTTPSURL, err := f.GetNginxURL(HTTPS)
-	Expect(err).NotTo(HaveOccurred())
-
+	HTTPSURL := f.GetNginxURL(HTTPS)
 	f.IngressController.HTTPSURL = HTTPSURL
 
 	// we wait for any change in the informers and SSL certificate generation
@@ -154,8 +150,10 @@ func IngressNginxDescribe(text string, body func()) bool {
 
 // GetNginxIP returns the IP address of the minikube cluster
 // where the NGINX ingress controller is running
-func (f *Framework) GetNginxIP() (string, error) {
-	return os.Getenv("NODE_IP"), nil
+func (f *Framework) GetNginxIP() string {
+	nodeIP := os.Getenv("NODE_IP")
+	Expect(nodeIP).NotTo(BeEmpty(), "env variable NODE_IP is empty")
+	return nodeIP
 }
 
 // GetNginxPort returns the number of TCP port where NGINX is running
@@ -178,28 +176,24 @@ func (f *Framework) GetNginxPort(name string) (int, error) {
 }
 
 // GetNginxURL returns the URL should be used to make a request to NGINX
-func (f *Framework) GetNginxURL(scheme RequestScheme) (string, error) {
-	ip, err := f.GetNginxIP()
-	if err != nil {
-		return "", err
-	}
-
+func (f *Framework) GetNginxURL(scheme RequestScheme) string {
+	ip := f.GetNginxIP()
 	port, err := f.GetNginxPort(fmt.Sprintf("%v", scheme))
-	if err != nil {
-		return "", err
-	}
+	Expect(err).NotTo(HaveOccurred(), "unexpected error obtaning NGINX Port")
 
-	return fmt.Sprintf("%v://%v:%v", scheme, ip, port), nil
+	return fmt.Sprintf("%v://%v:%v", scheme, ip, port)
 }
 
 // WaitForNginxServer waits until the nginx configuration contains a particular server section
-func (f *Framework) WaitForNginxServer(name string, matcher func(cfg string) bool) error {
-	return wait.Poll(Poll, time.Minute*5, f.matchNginxConditions(name, matcher))
+func (f *Framework) WaitForNginxServer(name string, matcher func(cfg string) bool) {
+	err := wait.Poll(Poll, time.Minute*5, f.matchNginxConditions(name, matcher))
+	Expect(err).NotTo(HaveOccurred(), "unexpected error waiting for nginx server condition/s")
 }
 
 // WaitForNginxConfiguration waits until the nginx configuration contains a particular configuration
-func (f *Framework) WaitForNginxConfiguration(matcher func(cfg string) bool) error {
-	return wait.Poll(Poll, time.Minute*5, f.matchNginxConditions("", matcher))
+func (f *Framework) WaitForNginxConfiguration(matcher func(cfg string) bool) {
+	err := wait.Poll(Poll, time.Minute*5, f.matchNginxConditions("", matcher))
+	Expect(err).NotTo(HaveOccurred(), "unexpected error waiting for nginx server condition/s")
 }
 
 func nginxLogs(client kubernetes.Interface, namespace string) (string, error) {
@@ -320,16 +314,12 @@ func (f *Framework) GetNginxConfigMapData() (map[string]string, error) {
 }
 
 // SetNginxConfigMapData sets ingress-nginx's nginx-configuration configMap data
-func (f *Framework) SetNginxConfigMapData(cmData map[string]string) error {
+func (f *Framework) SetNginxConfigMapData(cmData map[string]string) {
 	// Needs to do a Get and Set, Update will not take just the Data field
 	// or a configMap that is not the very last revision
 	config, err := f.getNginxConfigMap()
-	if err != nil {
-		return err
-	}
-	if config == nil {
-		return fmt.Errorf("Unable to get nginx-configuration configMap")
-	}
+	Expect(err).NotTo(HaveOccurred())
+	Expect(config).NotTo(BeNil(), "expected a configmap but none returned")
 
 	config.Data = cmData
 
@@ -337,25 +327,19 @@ func (f *Framework) SetNginxConfigMapData(cmData map[string]string) error {
 		CoreV1().
 		ConfigMaps(f.IngressController.Namespace).
 		Update(config)
-	if err != nil {
-		return err
-	}
+	Expect(err).NotTo(HaveOccurred())
 
 	time.Sleep(5 * time.Second)
-
-	return err
 }
 
 // UpdateNginxConfigMapData updates single field in ingress-nginx's nginx-configuration map data
-func (f *Framework) UpdateNginxConfigMapData(key string, value string) error {
+func (f *Framework) UpdateNginxConfigMapData(key string, value string) {
 	config, err := f.GetNginxConfigMapData()
-	if err != nil {
-		return err
-	}
+	Expect(err).NotTo(HaveOccurred(), "unexpected error reading configmap")
 
 	config[key] = value
 
-	return f.SetNginxConfigMapData(config)
+	f.SetNginxConfigMapData(config)
 }
 
 // UpdateDeployment runs the given updateFunc on the deployment and waits for it to be updated
@@ -437,6 +421,7 @@ func newSingleIngress(name, path, host, ns, service string, port int, annotation
 			},
 		},
 	}
+
 	if withTLS {
 		ing.Spec.TLS = []extensions.IngressTLS{
 			{
diff --git a/test/e2e/framework/grpc_fortune_teller.go b/test/e2e/framework/grpc_fortune_teller.go
index 7a1e2c77e..d2075e017 100644
--- a/test/e2e/framework/grpc_fortune_teller.go
+++ b/test/e2e/framework/grpc_fortune_teller.go
@@ -17,10 +17,9 @@ limitations under the License.
 package framework
 
 import (
-	"fmt"
 	"time"
 
-	"github.com/pkg/errors"
+	. "github.com/onsi/gomega"
 
 	corev1 "k8s.io/api/core/v1"
 	extensions "k8s.io/api/extensions/v1beta1"
@@ -31,13 +30,13 @@ import (
 
 // NewGRPCFortuneTellerDeployment creates a new single replica
 // deployment of the fortune teller image in a particular namespace
-func (f *Framework) NewGRPCFortuneTellerDeployment() error {
-	return f.NewNewGRPCFortuneTellerDeploymentWithReplicas(1)
+func (f *Framework) NewGRPCFortuneTellerDeployment() {
+	f.NewNewGRPCFortuneTellerDeploymentWithReplicas(1)
 }
 
 // NewNewGRPCFortuneTellerDeploymentWithReplicas creates a new deployment of the
 // fortune teller image in a particular namespace. Number of replicas is configurable
-func (f *Framework) NewNewGRPCFortuneTellerDeploymentWithReplicas(replicas int32) error {
+func (f *Framework) NewNewGRPCFortuneTellerDeploymentWithReplicas(replicas int32) {
 	deployment := &extensions.Deployment{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "fortune-teller",
@@ -77,20 +76,13 @@ func (f *Framework) NewNewGRPCFortuneTellerDeploymentWithReplicas(replicas int32
 	}
 
 	d, err := f.EnsureDeployment(deployment)
-	if err != nil {
-		return err
-	}
-
-	if d == nil {
-		return fmt.Errorf("unexpected error creating deployement for fortune-teller")
-	}
+	Expect(err).NotTo(HaveOccurred())
+	Expect(d).NotTo(BeNil(), "expected a fortune-teller deployment")
 
 	err = WaitForPodsReady(f.KubeClientSet, 5*time.Minute, int(replicas), f.IngressController.Namespace, metav1.ListOptions{
 		LabelSelector: fields.SelectorFromSet(fields.Set(d.Spec.Template.ObjectMeta.Labels)).String(),
 	})
-	if err != nil {
-		return errors.Wrap(err, "failed to wait for to become ready")
-	}
+	Expect(err).NotTo(HaveOccurred(), "failed to wait for to become ready")
 
 	service := &corev1.Service{
 		ObjectMeta: metav1.ObjectMeta{
@@ -112,14 +104,5 @@ func (f *Framework) NewNewGRPCFortuneTellerDeploymentWithReplicas(replicas int32
 		},
 	}
 
-	s, err := f.EnsureService(service)
-	if err != nil {
-		return err
-	}
-
-	if s == nil {
-		return fmt.Errorf("unexpected error creating service for fortune-teller deployment")
-	}
-
-	return nil
+	f.EnsureService(service)
 }
diff --git a/test/e2e/framework/influxdb.go b/test/e2e/framework/influxdb.go
index 82ef69b80..0ad895e72 100644
--- a/test/e2e/framework/influxdb.go
+++ b/test/e2e/framework/influxdb.go
@@ -17,10 +17,9 @@ limitations under the License.
 package framework
 
 import (
-	"fmt"
 	"time"
 
-	"github.com/pkg/errors"
+	. "github.com/onsi/gomega"
 
 	corev1 "k8s.io/api/core/v1"
 	extensions "k8s.io/api/extensions/v1beta1"
@@ -60,7 +59,7 @@ bind-address = "0.0.0.0:8088"
 
 // NewInfluxDBDeployment creates an InfluxDB server configured to reply
 // on 8086/tcp and 8089/udp
-func (f *Framework) NewInfluxDBDeployment() error {
+func (f *Framework) NewInfluxDBDeployment() {
 	configuration := &corev1.ConfigMap{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "influxdb-config",
@@ -72,13 +71,9 @@ func (f *Framework) NewInfluxDBDeployment() error {
 	}
 
 	cm, err := f.EnsureConfigMap(configuration)
-	if err != nil {
-		return err
-	}
+	Expect(err).NotTo(HaveOccurred(), "failed to create an Influxdb deployment")
 
-	if cm == nil {
-		return fmt.Errorf("unexpected error creating configmap for influxdb")
-	}
+	Expect(cm).NotTo(BeNil(), "expected a configmap but none returned")
 
 	deployment := &extensions.Deployment{
 		ObjectMeta: metav1.ObjectMeta{
@@ -143,20 +138,12 @@ func (f *Framework) NewInfluxDBDeployment() error {
 	}
 
 	d, err := f.EnsureDeployment(deployment)
-	if err != nil {
-		return err
-	}
+	Expect(err).NotTo(HaveOccurred(), "failed to create an Influxdb deployment")
 
-	if d == nil {
-		return fmt.Errorf("unexpected error creating deployement for influxdb")
-	}
+	Expect(d).NotTo(BeNil(), "unexpected error creating deployement for influxdb")
 
 	err = WaitForPodsReady(f.KubeClientSet, 5*time.Minute, 1, f.IngressController.Namespace, metav1.ListOptions{
 		LabelSelector: fields.SelectorFromSet(fields.Set(d.Spec.Template.ObjectMeta.Labels)).String(),
 	})
-	if err != nil {
-		return errors.Wrap(err, "failed to wait for influxdb to become ready")
-	}
-
-	return nil
+	Expect(err).NotTo(HaveOccurred(), "failed to wait for influxdb to become ready")
 }
diff --git a/test/e2e/framework/k8s.go b/test/e2e/framework/k8s.go
index e5594497f..5a40fe100 100644
--- a/test/e2e/framework/k8s.go
+++ b/test/e2e/framework/k8s.go
@@ -20,6 +20,8 @@ import (
 	"fmt"
 	"time"
 
+	. "github.com/onsi/gomega"
+
 	api "k8s.io/api/core/v1"
 	core "k8s.io/api/core/v1"
 	extensions "k8s.io/api/extensions/v1beta1"
@@ -31,15 +33,23 @@ import (
 )
 
 // EnsureSecret creates a Secret object or returns it if it already exists.
-func (f *Framework) EnsureSecret(secret *api.Secret) (*api.Secret, error) {
+func (f *Framework) EnsureSecret(secret *api.Secret) *api.Secret {
 	s, err := f.KubeClientSet.CoreV1().Secrets(secret.Namespace).Create(secret)
 	if err != nil {
 		if k8sErrors.IsAlreadyExists(err) {
-			return f.KubeClientSet.CoreV1().Secrets(secret.Namespace).Update(secret)
+			s, err := f.KubeClientSet.CoreV1().Secrets(secret.Namespace).Update(secret)
+			Expect(err).NotTo(HaveOccurred(), "unexpected error updating secret")
+
+			return s
 		}
-		return nil, err
+
+		Expect(err).NotTo(HaveOccurred(), "unexpected error creating secret")
 	}
-	return s, nil
+
+	Expect(s).NotTo(BeNil())
+	Expect(s.ObjectMeta).NotTo(BeNil())
+
+	return s
 }
 
 // EnsureConfigMap creates a ConfigMap object or returns it if it already exists.
@@ -51,40 +61,49 @@ func (f *Framework) EnsureConfigMap(configMap *api.ConfigMap) (*api.ConfigMap, e
 		}
 		return nil, err
 	}
+
 	return cm, nil
 }
 
 // EnsureIngress creates an Ingress object or returns it if it already exists.
-func (f *Framework) EnsureIngress(ingress *extensions.Ingress) (*extensions.Ingress, error) {
-	s, err := f.KubeClientSet.ExtensionsV1beta1().Ingresses(ingress.Namespace).Update(ingress)
+func (f *Framework) EnsureIngress(ingress *extensions.Ingress) *extensions.Ingress {
+	ing, err := f.KubeClientSet.ExtensionsV1beta1().Ingresses(ingress.Namespace).Update(ingress)
 	if err != nil {
 		if k8sErrors.IsNotFound(err) {
-			s, err = f.KubeClientSet.ExtensionsV1beta1().Ingresses(ingress.Namespace).Create(ingress)
-			if err != nil {
-				return nil, err
-			}
-		} else {
-			return nil, err
+			ing, err = f.KubeClientSet.ExtensionsV1beta1().Ingresses(ingress.Namespace).Create(ingress)
+			Expect(err).NotTo(HaveOccurred(), "unexpected error creating ingress")
+			return ing
 		}
+
+		Expect(err).NotTo(HaveOccurred())
 	}
 
-	if s.Annotations == nil {
-		s.Annotations = make(map[string]string)
+	Expect(ing).NotTo(BeNil())
+
+	if ing.Annotations == nil {
+		ing.Annotations = make(map[string]string)
 	}
 
-	return s, nil
+	return ing
 }
 
 // EnsureService creates a Service object or returns it if it already exists.
-func (f *Framework) EnsureService(service *core.Service) (*core.Service, error) {
+func (f *Framework) EnsureService(service *core.Service) *core.Service {
 	s, err := f.KubeClientSet.CoreV1().Services(service.Namespace).Update(service)
 	if err != nil {
 		if k8sErrors.IsNotFound(err) {
-			return f.KubeClientSet.CoreV1().Services(service.Namespace).Create(service)
+			s, err := f.KubeClientSet.CoreV1().Services(service.Namespace).Create(service)
+			Expect(err).NotTo(HaveOccurred(), "unexpected error creating service")
+			return s
+
 		}
-		return nil, err
+
+		Expect(err).NotTo(HaveOccurred())
 	}
-	return s, nil
+
+	Expect(s).NotTo(BeNil(), "expected a service but none returned")
+
+	return s
 }
 
 // EnsureDeployment creates a Deployment object or returns it if it already exists.
diff --git a/test/e2e/framework/ssl.go b/test/e2e/framework/ssl.go
index 9b6055e2c..803f5f034 100644
--- a/test/e2e/framework/ssl.go
+++ b/test/e2e/framework/ssl.go
@@ -32,6 +32,8 @@ import (
 	"strings"
 	"time"
 
+	. "github.com/onsi/gomega"
+
 	"k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
@@ -140,8 +142,9 @@ func CreateIngressMASecret(client kubernetes.Interface, host string, secretName,
 }
 
 // WaitForTLS waits until the TLS handshake with a given server completes successfully.
-func WaitForTLS(url string, tlsConfig *tls.Config) error {
-	return wait.Poll(Poll, 30*time.Second, matchTLSServerName(url, tlsConfig))
+func WaitForTLS(url string, tlsConfig *tls.Config) {
+	err := wait.Poll(Poll, 30*time.Second, matchTLSServerName(url, tlsConfig))
+	Expect(err).NotTo(HaveOccurred(), "timeout waiting for TLS configuration in URL %s", url)
 }
 
 // generateRSACert generates a basic self signed certificate using a key length
diff --git a/test/e2e/lua/dynamic_certificates.go b/test/e2e/lua/dynamic_certificates.go
index 143db355f..46cf79585 100644
--- a/test/e2e/lua/dynamic_certificates.go
+++ b/test/e2e/lua/dynamic_certificates.go
@@ -27,7 +27,6 @@ import (
 	appsv1beta1 "k8s.io/api/apps/v1beta1"
 	extensions "k8s.io/api/extensions/v1beta1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/client-go/kubernetes"
 
 	"k8s.io/ingress-nginx/test/e2e/framework"
 )
@@ -37,17 +36,24 @@ var _ = framework.IngressNginxDescribe("Dynamic Certificate", func() {
 	host := "foo.com"
 
 	BeforeEach(func() {
-		err := enableDynamicCertificates(f.IngressController.Namespace, f.KubeClientSet)
-		Expect(err).NotTo(HaveOccurred())
+		err := framework.UpdateDeployment(f.KubeClientSet, f.IngressController.Namespace, "nginx-ingress-controller", 1,
+			func(deployment *appsv1beta1.Deployment) error {
+				args := deployment.Spec.Template.Spec.Containers[0].Args
+				args = append(args, "--enable-dynamic-certificates")
+				args = append(args, "--enable-ssl-chain-completion=false")
+				deployment.Spec.Template.Spec.Containers[0].Args = args
+				_, err := f.KubeClientSet.AppsV1beta1().Deployments(f.IngressController.Namespace).Update(deployment)
 
-		err = f.WaitForNginxConfiguration(
-			func(cfg string) bool {
-				return strings.Contains(cfg, "ok, res = pcall(require, \"certificate\")")
+				return err
 			})
 		Expect(err).NotTo(HaveOccurred())
 
-		err = f.NewEchoDeploymentWithReplicas(1)
-		Expect(err).NotTo(HaveOccurred())
+		f.WaitForNginxConfiguration(
+			func(cfg string) bool {
+				return strings.Contains(cfg, "ok, res = pcall(require, \"certificate\")")
+			})
+
+		f.NewEchoDeploymentWithReplicas(1)
 	})
 
 	It("picks up the certificate when we add TLS spec to existing ingress", func() {
@@ -74,27 +80,27 @@ var _ = framework.IngressNginxDescribe("Dynamic Certificate", func() {
 	})
 
 	It("picks up the previously missing secret for a given ingress without reloading", func() {
-		ing, err := f.EnsureIngress(framework.NewSingleIngressWithTLS(host, "/", host, f.IngressController.Namespace, "http-svc", 80, nil))
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
+		ing := framework.NewSingleIngressWithTLS(host, "/", host, f.IngressController.Namespace, "http-svc", 80, nil)
+		f.EnsureIngress(ing)
+
 		time.Sleep(waitForLuaSync)
+
 		ensureHTTPSRequest(fmt.Sprintf("%s?id=dummy_log_splitter_foo_bar", f.IngressController.HTTPSURL), host, "ingress.local")
 
-		_, err = framework.CreateIngressTLSSecret(f.KubeClientSet,
+		_, err := framework.CreateIngressTLSSecret(f.KubeClientSet,
 			ing.Spec.TLS[0].Hosts,
 			ing.Spec.TLS[0].SecretName,
 			ing.Namespace)
 		Expect(err).ToNot(HaveOccurred())
 
 		By("configuring certificate_by_lua and skipping Nginx configuration of the new certificate")
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return strings.Contains(server, "ssl_certificate_by_lua_block") &&
 					!strings.Contains(server, fmt.Sprintf("ssl_certificate /etc/ingress-controller/ssl/%s-%s.pem;", ing.Namespace, host)) &&
 					!strings.Contains(server, fmt.Sprintf("ssl_certificate_key /etc/ingress-controller/ssl/%s-%s.pem;", ing.Namespace, host)) &&
 					strings.Contains(server, "listen 443")
 			})
-		Expect(err).ToNot(HaveOccurred())
 
 		time.Sleep(waitForLuaSync)
 
@@ -114,14 +120,13 @@ var _ = framework.IngressNginxDescribe("Dynamic Certificate", func() {
 
 	Context("given an ingress with TLS correctly configured", func() {
 		BeforeEach(func() {
-			ing, err := f.EnsureIngress(framework.NewSingleIngressWithTLS(host, "/", host, f.IngressController.Namespace, "http-svc", 80, nil))
-			Expect(err).NotTo(HaveOccurred())
-			Expect(ing).NotTo(BeNil())
+			ing := f.EnsureIngress(framework.NewSingleIngressWithTLS(host, "/", host, f.IngressController.Namespace, "http-svc", 80, nil))
+
 			time.Sleep(waitForLuaSync)
 
 			ensureHTTPSRequest(f.IngressController.HTTPSURL, host, "ingress.local")
 
-			_, err = framework.CreateIngressTLSSecret(f.KubeClientSet,
+			_, err := framework.CreateIngressTLSSecret(f.KubeClientSet,
 				ing.Spec.TLS[0].Hosts,
 				ing.Spec.TLS[0].SecretName,
 				ing.Namespace)
@@ -129,14 +134,13 @@ var _ = framework.IngressNginxDescribe("Dynamic Certificate", func() {
 			time.Sleep(waitForLuaSync)
 
 			By("configuring certificate_by_lua and skipping Nginx configuration of the new certificate")
-			err = f.WaitForNginxServer(ing.Spec.TLS[0].Hosts[0],
+			f.WaitForNginxServer(ing.Spec.TLS[0].Hosts[0],
 				func(server string) bool {
 					return strings.Contains(server, "ssl_certificate_by_lua_block") &&
 						!strings.Contains(server, fmt.Sprintf("ssl_certificate /etc/ingress-controller/ssl/%s-%s.pem;", ing.Namespace, host)) &&
 						!strings.Contains(server, fmt.Sprintf("ssl_certificate_key /etc/ingress-controller/ssl/%s-%s.pem;", ing.Namespace, host)) &&
 						strings.Contains(server, "listen 443")
 				})
-			Expect(err).ToNot(HaveOccurred())
 
 			time.Sleep(waitForLuaSync)
 
@@ -157,14 +161,13 @@ var _ = framework.IngressNginxDescribe("Dynamic Certificate", func() {
 			time.Sleep(waitForLuaSync)
 
 			By("configuring certificate_by_lua and skipping Nginx configuration of the new certificate")
-			err = f.WaitForNginxServer(ing.Spec.TLS[0].Hosts[0],
+			f.WaitForNginxServer(ing.Spec.TLS[0].Hosts[0],
 				func(server string) bool {
 					return strings.Contains(server, "ssl_certificate_by_lua_block") &&
 						!strings.Contains(server, fmt.Sprintf("ssl_certificate /etc/ingress-controller/ssl/%s-%s.pem;", ing.Namespace, host)) &&
 						!strings.Contains(server, fmt.Sprintf("ssl_certificate_key /etc/ingress-controller/ssl/%s-%s.pem;", ing.Namespace, host)) &&
 						strings.Contains(server, "listen 443")
 				})
-			Expect(err).ToNot(HaveOccurred())
 
 			time.Sleep(waitForLuaSync)
 
@@ -192,14 +195,13 @@ var _ = framework.IngressNginxDescribe("Dynamic Certificate", func() {
 			time.Sleep(waitForLuaSync)
 
 			By("configuring certificate_by_lua and skipping Nginx configuration of the new certificate")
-			err = f.WaitForNginxServer(ing.Spec.TLS[0].Hosts[0],
+			f.WaitForNginxServer(ing.Spec.TLS[0].Hosts[0],
 				func(server string) bool {
 					return strings.Contains(server, "ssl_certificate_by_lua_block") &&
 						strings.Contains(server, "ssl_certificate /etc/ingress-controller/ssl/default-fake-certificate.pem;") &&
 						strings.Contains(server, "ssl_certificate_key /etc/ingress-controller/ssl/default-fake-certificate.pem;") &&
 						strings.Contains(server, "listen 443")
 				})
-			Expect(err).ToNot(HaveOccurred())
 
 			time.Sleep(waitForLuaSync)
 
@@ -242,16 +244,3 @@ var _ = framework.IngressNginxDescribe("Dynamic Certificate", func() {
 		})
 	})
 })
-
-func enableDynamicCertificates(namespace string, kubeClientSet kubernetes.Interface) error {
-	return framework.UpdateDeployment(kubeClientSet, namespace, "nginx-ingress-controller", 1,
-		func(deployment *appsv1beta1.Deployment) error {
-			args := deployment.Spec.Template.Spec.Containers[0].Args
-			args = append(args, "--enable-dynamic-certificates")
-			args = append(args, "--enable-ssl-chain-completion=false")
-			deployment.Spec.Template.Spec.Containers[0].Args = args
-			_, err := kubeClientSet.AppsV1beta1().Deployments(namespace).Update(deployment)
-
-			return err
-		})
-}
diff --git a/test/e2e/lua/dynamic_configuration.go b/test/e2e/lua/dynamic_configuration.go
index a3f8c7037..38bb9ffd2 100644
--- a/test/e2e/lua/dynamic_configuration.go
+++ b/test/e2e/lua/dynamic_configuration.go
@@ -47,60 +47,54 @@ var _ = framework.IngressNginxDescribe("Dynamic Configuration", func() {
 	f := framework.NewDefaultFramework("dynamic-configuration")
 
 	BeforeEach(func() {
-		err := f.NewEchoDeploymentWithReplicas(1)
-		Expect(err).NotTo(HaveOccurred())
-
+		f.NewEchoDeploymentWithReplicas(1)
 		ensureIngress(f, "foo.com")
 	})
 
 	It("configures balancer Lua middleware correctly", func() {
-		err := f.WaitForNginxConfiguration(func(cfg string) bool {
+		f.WaitForNginxConfiguration(func(cfg string) bool {
 			return strings.Contains(cfg, "balancer.init_worker()") && strings.Contains(cfg, "balancer.balance()")
 		})
-		Expect(err).NotTo(HaveOccurred())
 
 		host := "foo.com"
-		err = f.WaitForNginxServer(host, func(server string) bool {
+		f.WaitForNginxServer(host, func(server string) bool {
 			return strings.Contains(server, "balancer.rewrite()") && strings.Contains(server, "balancer.log()")
 		})
-		Expect(err).NotTo(HaveOccurred())
 	})
 
 	It("sets nameservers for Lua", func() {
-		err := f.WaitForNginxConfiguration(func(cfg string) bool {
+		f.WaitForNginxConfiguration(func(cfg string) bool {
 			r := regexp.MustCompile(`configuration.nameservers = { [".,0-9a-zA-Z]+ }`)
 			return r.MatchString(cfg)
 		})
-		Expect(err).NotTo(HaveOccurred())
 	})
 
 	Context("when only backends change", func() {
 		It("handles endpoints only changes", func() {
 			var nginxConfig string
-			err := f.WaitForNginxConfiguration(func(cfg string) bool {
+			f.WaitForNginxConfiguration(func(cfg string) bool {
 				nginxConfig = cfg
 				return true
 			})
 
 			replicas := 2
-			err = framework.UpdateDeployment(f.KubeClientSet, f.IngressController.Namespace, "http-svc", replicas, nil)
+			err := framework.UpdateDeployment(f.KubeClientSet, f.IngressController.Namespace, "http-svc", replicas, nil)
 			Expect(err).NotTo(HaveOccurred())
 			time.Sleep(waitForLuaSync)
 
 			ensureRequest(f, "foo.com")
 
 			var newNginxConfig string
-			err = f.WaitForNginxConfiguration(func(cfg string) bool {
+			f.WaitForNginxConfiguration(func(cfg string) bool {
 				newNginxConfig = cfg
 				return true
 			})
-
 			Expect(nginxConfig).Should(Equal(newNginxConfig))
 		})
 
 		It("handles an annotation change", func() {
 			var nginxConfig string
-			err := f.WaitForNginxConfiguration(func(cfg string) bool {
+			f.WaitForNginxConfiguration(func(cfg string) bool {
 				nginxConfig = cfg
 				return true
 			})
@@ -116,7 +110,7 @@ var _ = framework.IngressNginxDescribe("Dynamic Configuration", func() {
 			ensureRequest(f, "foo.com")
 
 			var newNginxConfig string
-			err = f.WaitForNginxConfiguration(func(cfg string) bool {
+			f.WaitForNginxConfiguration(func(cfg string) bool {
 				newNginxConfig = cfg
 				return true
 			})
@@ -127,7 +121,7 @@ var _ = framework.IngressNginxDescribe("Dynamic Configuration", func() {
 
 	It("handles a non backend update", func() {
 		var nginxConfig string
-		err := f.WaitForNginxConfiguration(func(cfg string) bool {
+		f.WaitForNginxConfiguration(func(cfg string) bool {
 			nginxConfig = cfg
 			return true
 		})
@@ -149,21 +143,19 @@ var _ = framework.IngressNginxDescribe("Dynamic Configuration", func() {
 		Expect(err).ToNot(HaveOccurred())
 
 		var newNginxConfig string
-		err = f.WaitForNginxConfiguration(func(cfg string) bool {
+		f.WaitForNginxConfiguration(func(cfg string) bool {
 			newNginxConfig = cfg
 			return true
 		})
-
 		Expect(nginxConfig).ShouldNot(Equal(newNginxConfig))
 	})
 })
 
 func ensureIngress(f *framework.Framework, host string) *extensions.Ingress {
-	ing, err := f.EnsureIngress(framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80,
+	ing := f.EnsureIngress(framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80,
 		&map[string]string{"nginx.ingress.kubernetes.io/load-balance": "ewma"}))
-	Expect(err).NotTo(HaveOccurred())
-	Expect(ing).NotTo(BeNil())
-	err = f.WaitForNginxServer(host,
+
+	f.WaitForNginxServer(host,
 		func(server string) bool {
 			return strings.Contains(server, fmt.Sprintf("server_name %s ;", host)) &&
 				strings.Contains(server, "proxy_pass http://upstream_balancer;")
diff --git a/test/e2e/servicebackend/service_backend.go b/test/e2e/servicebackend/service_backend.go
index e60319084..180327d67 100644
--- a/test/e2e/servicebackend/service_backend.go
+++ b/test/e2e/servicebackend/service_backend.go
@@ -45,15 +45,12 @@ var _ = framework.IngressNginxDescribe("Service backend - 503", func() {
 		host := "nonexistent.svc.com"
 
 		bi := buildIngressWithNonexistentService(host, f.IngressController.Namespace, "/")
-		ing, err := f.EnsureIngress(bi)
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
+		f.EnsureIngress(bi)
 
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return strings.Contains(server, "return 503;")
 			})
-		Expect(err).NotTo(HaveOccurred())
 
 		resp, _, errs := gorequest.New().
 			Get(f.IngressController.HTTPURL).
@@ -68,19 +65,15 @@ var _ = framework.IngressNginxDescribe("Service backend - 503", func() {
 
 		bi, bs := buildIngressWithUnavailableServiceEndpoints(host, f.IngressController.Namespace, "/")
 
-		svc, err := f.EnsureService(bs)
-		Expect(err).NotTo(HaveOccurred())
+		svc := f.EnsureService(bs)
 		Expect(svc).NotTo(BeNil())
 
-		ing, err := f.EnsureIngress(bi)
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
+		f.EnsureIngress(bi)
 
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return strings.Contains(server, "return 503;")
 			})
-		Expect(err).NotTo(HaveOccurred())
 
 		resp, _, errs := gorequest.New().
 			Get(f.IngressController.HTTPURL).
diff --git a/test/e2e/settings/configmap_change.go b/test/e2e/settings/configmap_change.go
index ff88687f7..4e19ca90d 100644
--- a/test/e2e/settings/configmap_change.go
+++ b/test/e2e/settings/configmap_change.go
@@ -30,8 +30,7 @@ var _ = framework.IngressNginxDescribe("Configmap change", func() {
 	f := framework.NewDefaultFramework("configmap-change")
 
 	BeforeEach(func() {
-		err := f.NewEchoDeployment()
-		Expect(err).NotTo(HaveOccurred())
+		f.NewEchoDeployment()
 	})
 
 	AfterEach(func() {
@@ -40,22 +39,20 @@ var _ = framework.IngressNginxDescribe("Configmap change", func() {
 	It("should reload after an update in the configuration", func() {
 		host := "configmap-change"
 
-		ing, err := f.EnsureIngress(framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, nil))
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
+		ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, nil)
+		f.EnsureIngress(ing)
 
 		wlKey := "whitelist-source-range"
 		wlValue := "1.1.1.1"
 
 		By("adding a whitelist-source-range")
 
-		err = f.UpdateNginxConfigMapData(wlKey, wlValue)
-		Expect(err).NotTo(HaveOccurred())
+		f.UpdateNginxConfigMapData(wlKey, wlValue)
 
 		checksumRegex := regexp.MustCompile("Configuration checksum:\\s+(\\d+)")
 		checksum := ""
 
-		err = f.WaitForNginxConfiguration(
+		f.WaitForNginxConfiguration(
 			func(cfg string) bool {
 				// before returning, extract the current checksum
 				match := checksumRegex.FindStringSubmatch(cfg)
@@ -66,16 +63,14 @@ var _ = framework.IngressNginxDescribe("Configmap change", func() {
 				return strings.Contains(cfg, "geo $the_real_ip $deny_") &&
 					strings.Contains(cfg, "1.1.1.1 0")
 			})
-		Expect(err).NotTo(HaveOccurred())
 		Expect(checksum).NotTo(BeEmpty())
 
 		By("changing error-log-level")
 
-		err = f.UpdateNginxConfigMapData("error-log-level", "debug")
-		Expect(err).NotTo(HaveOccurred())
+		f.UpdateNginxConfigMapData("error-log-level", "debug")
 
 		newChecksum := ""
-		err = f.WaitForNginxConfiguration(
+		f.WaitForNginxConfiguration(
 			func(cfg string) bool {
 				match := checksumRegex.FindStringSubmatch(cfg)
 				if len(match) > 0 {
@@ -84,8 +79,6 @@ var _ = framework.IngressNginxDescribe("Configmap change", func() {
 
 				return strings.ContainsAny(cfg, "error_log  /var/log/nginx/error.log debug;")
 			})
-		Expect(err).NotTo(HaveOccurred())
-
 		Expect(checksum).NotTo(BeEquivalentTo(newChecksum))
 	})
 })
diff --git a/test/e2e/settings/forwarded_headers.go b/test/e2e/settings/forwarded_headers.go
index 35f9a5fff..fe451655e 100644
--- a/test/e2e/settings/forwarded_headers.go
+++ b/test/e2e/settings/forwarded_headers.go
@@ -34,11 +34,8 @@ var _ = framework.IngressNginxDescribe("X-Forwarded headers", func() {
 	setting := "use-forwarded-headers"
 
 	BeforeEach(func() {
-		err := f.NewEchoDeployment()
-		Expect(err).NotTo(HaveOccurred())
-
-		err = f.UpdateNginxConfigMapData(setting, "false")
-		Expect(err).NotTo(HaveOccurred())
+		f.NewEchoDeployment()
+		f.UpdateNginxConfigMapData(setting, "false")
 	})
 
 	AfterEach(func() {
@@ -47,18 +44,15 @@ var _ = framework.IngressNginxDescribe("X-Forwarded headers", func() {
 	It("should trust X-Forwarded headers when setting is true", func() {
 		host := "forwarded-headers"
 
-		err := f.UpdateNginxConfigMapData(setting, "true")
-		Expect(err).NotTo(HaveOccurred())
+		f.UpdateNginxConfigMapData(setting, "true")
 
-		ing, err := f.EnsureIngress(framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, nil))
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
+		ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, nil)
+		f.EnsureIngress(ing)
 
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return strings.Contains(server, "server_name forwarded-headers")
 			})
-		Expect(err).NotTo(HaveOccurred())
 
 		resp, body, errs := gorequest.New().
 			Get(f.IngressController.HTTPURL).
@@ -80,18 +74,14 @@ var _ = framework.IngressNginxDescribe("X-Forwarded headers", func() {
 	It("should not trust X-Forwarded headers when setting is false", func() {
 		host := "forwarded-headers"
 
-		err := f.UpdateNginxConfigMapData(setting, "false")
-		Expect(err).NotTo(HaveOccurred())
+		f.UpdateNginxConfigMapData(setting, "false")
 
-		ing, err := f.EnsureIngress(framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, nil))
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
+		f.EnsureIngress(framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, nil))
 
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return strings.Contains(server, "server_name forwarded-headers")
 			})
-		Expect(err).NotTo(HaveOccurred())
 
 		resp, body, errs := gorequest.New().
 			Get(f.IngressController.HTTPURL).
diff --git a/test/e2e/settings/global_access_block.go b/test/e2e/settings/global_access_block.go
index 3caec7328..924283b18 100644
--- a/test/e2e/settings/global_access_block.go
+++ b/test/e2e/settings/global_access_block.go
@@ -33,28 +33,22 @@ var _ = framework.IngressNginxDescribe("Global access block", func() {
 	host := "global-access-block"
 
 	BeforeEach(func() {
-		err := f.NewEchoDeploymentWithReplicas(1)
-		Expect(err).NotTo(HaveOccurred())
-
-		ing, err := f.EnsureIngress(framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, nil))
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
+		f.NewEchoDeploymentWithReplicas(1)
+		f.EnsureIngress(framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, nil))
 	})
 
 	AfterEach(func() {
 	})
 
 	It("should block CIDRs defined in the ConfigMap", func() {
-		err := f.UpdateNginxConfigMapData("block-cidrs", "172.16.0.0/12,192.168.0.0/16,10.0.0.0/8")
-		Expect(err).NotTo(HaveOccurred())
+		f.UpdateNginxConfigMapData("block-cidrs", "172.16.0.0/12,192.168.0.0/16,10.0.0.0/8")
 
-		err = f.WaitForNginxConfiguration(
+		f.WaitForNginxConfiguration(
 			func(cfg string) bool {
 				return strings.Contains(cfg, "deny 172.16.0.0/12;") &&
 					strings.Contains(cfg, "deny 192.168.0.0/16;") &&
 					strings.Contains(cfg, "deny 10.0.0.0/8;")
 			})
-		Expect(err).NotTo(HaveOccurred())
 
 		// This test works for minikube, but may have problems with real kubernetes clusters,
 		// especially if connection is done via Internet. In this case, the test should be disabled/removed.
@@ -67,15 +61,13 @@ var _ = framework.IngressNginxDescribe("Global access block", func() {
 	})
 
 	It("should block User-Agents defined in the ConfigMap", func() {
-		err := f.UpdateNginxConfigMapData("block-user-agents", "~*chrome\\/68\\.0\\.3440\\.106\\ safari\\/537\\.36,AlphaBot")
-		Expect(err).NotTo(HaveOccurred())
+		f.UpdateNginxConfigMapData("block-user-agents", "~*chrome\\/68\\.0\\.3440\\.106\\ safari\\/537\\.36,AlphaBot")
 
-		err = f.WaitForNginxConfiguration(
+		f.WaitForNginxConfiguration(
 			func(cfg string) bool {
 				return strings.Contains(cfg, "~*chrome\\/68\\.0\\.3440\\.106\\ safari\\/537\\.36 1;") &&
 					strings.Contains(cfg, "AlphaBot 1;")
 			})
-		Expect(err).NotTo(HaveOccurred())
 
 		// Should be blocked
 		resp, _, errs := gorequest.New().
@@ -105,15 +97,13 @@ var _ = framework.IngressNginxDescribe("Global access block", func() {
 	})
 
 	It("should block Referers defined in the ConfigMap", func() {
-		err := f.UpdateNginxConfigMapData("block-referers", "~*example\\.com,qwerty")
-		Expect(err).NotTo(HaveOccurred())
+		f.UpdateNginxConfigMapData("block-referers", "~*example\\.com,qwerty")
 
-		err = f.WaitForNginxConfiguration(
+		f.WaitForNginxConfiguration(
 			func(cfg string) bool {
 				return strings.Contains(cfg, "~*example\\.com 1;") &&
 					strings.Contains(cfg, "qwerty 1;")
 			})
-		Expect(err).NotTo(HaveOccurred())
 
 		// Should be blocked
 		resp, _, errs := gorequest.New().
diff --git a/test/e2e/settings/main_snippet.go b/test/e2e/settings/main_snippet.go
index d71ec5c91..0a7d4e4cc 100644
--- a/test/e2e/settings/main_snippet.go
+++ b/test/e2e/settings/main_snippet.go
@@ -20,7 +20,6 @@ import (
 	"strings"
 
 	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
 
 	"k8s.io/ingress-nginx/test/e2e/framework"
 )
@@ -31,13 +30,11 @@ var _ = framework.IngressNginxDescribe("Main Snippet", func() {
 
 	It("should add value of main-snippet setting to nginx config", func() {
 		expectedComment := "# main snippet"
-		err := f.UpdateNginxConfigMapData(mainSnippet, expectedComment)
-		Expect(err).NotTo(HaveOccurred())
+		f.UpdateNginxConfigMapData(mainSnippet, expectedComment)
 
-		err = f.WaitForNginxConfiguration(
+		f.WaitForNginxConfiguration(
 			func(cfg string) bool {
 				return strings.Contains(cfg, expectedComment)
 			})
-		Expect(err).NotTo(HaveOccurred())
 	})
 })
diff --git a/test/e2e/settings/multi_accept.go b/test/e2e/settings/multi_accept.go
index d067204e9..94d1349fe 100644
--- a/test/e2e/settings/multi_accept.go
+++ b/test/e2e/settings/multi_accept.go
@@ -20,7 +20,6 @@ import (
 	"strings"
 
 	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
 
 	"k8s.io/ingress-nginx/test/e2e/framework"
 )
@@ -31,34 +30,29 @@ var _ = framework.IngressNginxDescribe("Multi Accept", func() {
 
 	It("should be enabled by default", func() {
 		expectedDirective := "multi_accept on;"
-		err := f.WaitForNginxConfiguration(
+		f.WaitForNginxConfiguration(
 			func(cfg string) bool {
 				return strings.Contains(cfg, expectedDirective)
 			})
-		Expect(err).NotTo(HaveOccurred())
 	})
 
 	It("should be enabled when set to true", func() {
 		expectedDirective := "multi_accept on;"
-		err := f.UpdateNginxConfigMapData(multiAccept, "true")
-		Expect(err).NotTo(HaveOccurred())
+		f.UpdateNginxConfigMapData(multiAccept, "true")
 
-		err = f.WaitForNginxConfiguration(
+		f.WaitForNginxConfiguration(
 			func(cfg string) bool {
 				return strings.Contains(cfg, expectedDirective)
 			})
-		Expect(err).NotTo(HaveOccurred())
 	})
 
 	It("should be disabled when set to false", func() {
 		expectedDirective := "multi_accept off;"
-		err := f.UpdateNginxConfigMapData(multiAccept, "false")
-		Expect(err).NotTo(HaveOccurred())
+		f.UpdateNginxConfigMapData(multiAccept, "false")
 
-		err = f.WaitForNginxConfiguration(
+		f.WaitForNginxConfiguration(
 			func(cfg string) bool {
 				return strings.Contains(cfg, expectedDirective)
 			})
-		Expect(err).NotTo(HaveOccurred())
 	})
 })
diff --git a/test/e2e/settings/no_auth_locations.go b/test/e2e/settings/no_auth_locations.go
index a91c473f2..a1eb741fc 100644
--- a/test/e2e/settings/no_auth_locations.go
+++ b/test/e2e/settings/no_auth_locations.go
@@ -43,32 +43,24 @@ var _ = framework.IngressNginxDescribe("No Auth locations", func() {
 	noAuthPath := "/noauth"
 
 	BeforeEach(func() {
-		err := f.NewEchoDeployment()
-		Expect(err).NotTo(HaveOccurred())
+		f.NewEchoDeployment()
 
-		s, err := f.EnsureSecret(buildSecret(username, password, secretName, f.IngressController.Namespace))
-		Expect(err).NotTo(HaveOccurred())
-		Expect(s).NotTo(BeNil())
-		Expect(s.ObjectMeta).NotTo(BeNil())
+		s := f.EnsureSecret(buildSecret(username, password, secretName, f.IngressController.Namespace))
 
-		err = f.UpdateNginxConfigMapData(setting, noAuthPath)
-		Expect(err).NotTo(HaveOccurred())
+		f.UpdateNginxConfigMapData(setting, noAuthPath)
 
 		bi := buildBasicAuthIngressWithSecondPath(host, f.IngressController.Namespace, s.Name, noAuthPath)
-		ing, err := f.EnsureIngress(bi)
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
+		f.EnsureIngress(bi)
 	})
 
 	AfterEach(func() {
 	})
 
 	It("should return status code 401 when accessing '/' unauthentication", func() {
-		err := f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return Expect(server).Should(ContainSubstring("test auth"))
 			})
-		Expect(err).NotTo(HaveOccurred())
 
 		resp, body, errs := gorequest.New().
 			Get(f.IngressController.HTTPURL).
@@ -81,11 +73,10 @@ var _ = framework.IngressNginxDescribe("No Auth locations", func() {
 	})
 
 	It("should return status code 200 when accessing '/'  authentication", func() {
-		err := f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return Expect(server).Should(ContainSubstring("test auth"))
 			})
-		Expect(err).NotTo(HaveOccurred())
 
 		resp, _, errs := gorequest.New().
 			Get(f.IngressController.HTTPURL).
@@ -98,11 +89,10 @@ var _ = framework.IngressNginxDescribe("No Auth locations", func() {
 	})
 
 	It("should return status code 200 when accessing '/noauth' unauthenticated", func() {
-		err := f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return Expect(server).Should(ContainSubstring("test auth"))
 			})
-		Expect(err).NotTo(HaveOccurred())
 
 		resp, _, errs := gorequest.New().
 			Get(fmt.Sprintf("%s/noauth", f.IngressController.HTTPURL)).
@@ -156,8 +146,9 @@ func buildBasicAuthIngressWithSecondPath(host, namespace, secretName, pathName s
 
 func buildSecret(username, password, name, namespace string) *corev1.Secret {
 	out, err := exec.Command("openssl", "passwd", "-crypt", password).CombinedOutput()
+	Expect(err).NotTo(HaveOccurred(), "creating password")
+
 	encpass := fmt.Sprintf("%v:%s\n", username, out)
-	Expect(err).NotTo(HaveOccurred())
 
 	return &corev1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
diff --git a/test/e2e/settings/proxy_protocol.go b/test/e2e/settings/proxy_protocol.go
index acf10d718..8a2956515 100644
--- a/test/e2e/settings/proxy_protocol.go
+++ b/test/e2e/settings/proxy_protocol.go
@@ -35,11 +35,8 @@ var _ = framework.IngressNginxDescribe("Proxy Protocol", func() {
 	setting := "use-proxy-protocol"
 
 	BeforeEach(func() {
-		err := f.NewEchoDeployment()
-		Expect(err).NotTo(HaveOccurred())
-
-		err = f.UpdateNginxConfigMapData(setting, "false")
-		Expect(err).NotTo(HaveOccurred())
+		f.NewEchoDeployment()
+		f.UpdateNginxConfigMapData(setting, "false")
 	})
 
 	AfterEach(func() {
@@ -48,27 +45,22 @@ var _ = framework.IngressNginxDescribe("Proxy Protocol", func() {
 	It("should respect port passed by the PROXY Protocol", func() {
 		host := "proxy-protocol"
 
-		err := f.UpdateNginxConfigMapData(setting, "true")
-		Expect(err).NotTo(HaveOccurred())
+		f.UpdateNginxConfigMapData(setting, "true")
 
-		ing, err := f.EnsureIngress(framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, nil))
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
+		f.EnsureIngress(framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, nil))
 
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return strings.Contains(server, "server_name proxy-protocol") &&
 					strings.Contains(server, "listen 80 proxy_protocol")
 			})
-		Expect(err).NotTo(HaveOccurred())
 
-		ip, err := f.GetNginxIP()
-		Expect(err).NotTo(HaveOccurred())
+		ip := f.GetNginxIP()
 		port, err := f.GetNginxPort("http")
-		Expect(err).NotTo(HaveOccurred())
+		Expect(err).NotTo(HaveOccurred(), "unexpected error obtaning NGINX Port")
 
 		conn, err := net.Dial("tcp", net.JoinHostPort(ip, strconv.Itoa(port)))
-		Expect(err).NotTo(HaveOccurred())
+		Expect(err).NotTo(HaveOccurred(), "unexpected error creating connection to %s:%d", ip, port)
 		defer conn.Close()
 
 		header := "PROXY TCP4 192.168.0.1 192.168.0.11 56324 1234\r\n"
@@ -76,7 +68,7 @@ var _ = framework.IngressNginxDescribe("Proxy Protocol", func() {
 		conn.Write([]byte("GET / HTTP/1.1\r\nHost: proxy-protocol\r\n\r\n"))
 
 		data, err := ioutil.ReadAll(conn)
-		Expect(err).NotTo(HaveOccurred())
+		Expect(err).NotTo(HaveOccurred(), "unexpected error reading connection data")
 		body := string(data)
 		Expect(body).Should(ContainSubstring(fmt.Sprintf("host=%v", "proxy-protocol")))
 		Expect(body).Should(ContainSubstring(fmt.Sprintf("x-forwarded-port=80")))
diff --git a/test/e2e/settings/server_tokens.go b/test/e2e/settings/server_tokens.go
index a5aecd6b8..2189f2f79 100644
--- a/test/e2e/settings/server_tokens.go
+++ b/test/e2e/settings/server_tokens.go
@@ -20,7 +20,6 @@ import (
 	"strings"
 
 	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
 
 	"k8s.io/api/extensions/v1beta1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -33,34 +32,28 @@ var _ = framework.IngressNginxDescribe("Server Tokens", func() {
 	serverTokens := "server-tokens"
 
 	BeforeEach(func() {
-		err := f.NewEchoDeployment()
-		Expect(err).NotTo(HaveOccurred())
+		f.NewEchoDeployment()
 	})
 
 	AfterEach(func() {
 	})
 
 	It("should not exists Server header in the response", func() {
-		err := f.UpdateNginxConfigMapData(serverTokens, "false")
-		Expect(err).NotTo(HaveOccurred())
+		f.UpdateNginxConfigMapData(serverTokens, "false")
 
-		ing, err := f.EnsureIngress(framework.NewSingleIngress(serverTokens, "/", serverTokens, f.IngressController.Namespace, "http-svc", 80, nil))
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
+		f.EnsureIngress(framework.NewSingleIngress(serverTokens, "/", serverTokens, f.IngressController.Namespace, "http-svc", 80, nil))
 
-		err = f.WaitForNginxConfiguration(
+		f.WaitForNginxConfiguration(
 			func(cfg string) bool {
 				return strings.Contains(cfg, "server_tokens off") &&
 					strings.Contains(cfg, "more_clear_headers Server;")
 			})
-		Expect(err).NotTo(HaveOccurred())
 	})
 
 	It("should exists Server header in the response when is enabled", func() {
-		err := f.UpdateNginxConfigMapData(serverTokens, "true")
-		Expect(err).NotTo(HaveOccurred())
+		f.UpdateNginxConfigMapData(serverTokens, "true")
 
-		ing, err := f.EnsureIngress(&v1beta1.Ingress{
+		f.EnsureIngress(&v1beta1.Ingress{
 			ObjectMeta: metav1.ObjectMeta{
 				Name:        serverTokens,
 				Namespace:   f.IngressController.Namespace,
@@ -88,13 +81,9 @@ var _ = framework.IngressNginxDescribe("Server Tokens", func() {
 			},
 		})
 
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
-
-		err = f.WaitForNginxConfiguration(
+		f.WaitForNginxConfiguration(
 			func(cfg string) bool {
 				return strings.Contains(cfg, "server_tokens on")
 			})
-		Expect(err).NotTo(HaveOccurred())
 	})
 })
diff --git a/test/e2e/settings/tls.go b/test/e2e/settings/tls.go
index 0e7333ae9..2247ebff9 100644
--- a/test/e2e/settings/tls.go
+++ b/test/e2e/settings/tls.go
@@ -34,8 +34,7 @@ var _ = framework.IngressNginxDescribe("Settings - TLS)", func() {
 	host := "settings-tls"
 
 	BeforeEach(func() {
-		err := f.NewEchoDeployment()
-		Expect(err).NotTo(HaveOccurred())
+		f.NewEchoDeployment()
 	})
 
 	AfterEach(func() {
@@ -52,19 +51,15 @@ var _ = framework.IngressNginxDescribe("Settings - TLS)", func() {
 		tlsConfig, err := tlsEndpoint(f, host)
 		Expect(err).NotTo(HaveOccurred())
 
-		err = framework.WaitForTLS(f.IngressController.HTTPSURL, tlsConfig)
-		Expect(err).NotTo(HaveOccurred())
+		framework.WaitForTLS(f.IngressController.HTTPSURL, tlsConfig)
 
 		By("setting cipher suite")
+		f.UpdateNginxConfigMapData(sslCiphers, testCiphers)
 
-		err = f.UpdateNginxConfigMapData(sslCiphers, testCiphers)
-		Expect(err).NotTo(HaveOccurred())
-
-		err = f.WaitForNginxConfiguration(
+		f.WaitForNginxConfiguration(
 			func(cfg string) bool {
 				return strings.Contains(cfg, fmt.Sprintf("ssl_ciphers '%s';", testCiphers))
 			})
-		Expect(err).NotTo(HaveOccurred())
 
 		resp, _, errs := gorequest.New().
 			Get(f.IngressController.HTTPSURL).
@@ -78,15 +73,12 @@ var _ = framework.IngressNginxDescribe("Settings - TLS)", func() {
 		Expect(resp.TLS.CipherSuite).Should(BeNumerically("==", tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384))
 
 		By("enforcing TLS v1.0")
+		f.UpdateNginxConfigMapData(sslProtocols, "TLSv1")
 
-		err = f.UpdateNginxConfigMapData(sslProtocols, "TLSv1")
-		Expect(err).NotTo(HaveOccurred())
-
-		err = f.WaitForNginxConfiguration(
+		f.WaitForNginxConfiguration(
 			func(cfg string) bool {
 				return strings.Contains(cfg, "ssl_protocols TLSv1;")
 			})
-		Expect(err).NotTo(HaveOccurred())
 
 		resp, _, errs = gorequest.New().
 			Get(f.IngressController.HTTPSURL).
@@ -108,19 +100,15 @@ var _ = framework.IngressNginxDescribe("Settings - TLS)", func() {
 		tlsConfig, err := tlsEndpoint(f, host)
 		Expect(err).NotTo(HaveOccurred())
 
-		err = framework.WaitForTLS(f.IngressController.HTTPSURL, tlsConfig)
-		Expect(err).NotTo(HaveOccurred())
+		framework.WaitForTLS(f.IngressController.HTTPSURL, tlsConfig)
 
 		By("setting max-age parameter")
+		f.UpdateNginxConfigMapData(hstsMaxAge, "86400")
 
-		err = f.UpdateNginxConfigMapData(hstsMaxAge, "86400")
-		Expect(err).NotTo(HaveOccurred())
-
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return strings.Contains(server, "Strict-Transport-Security: max-age=86400; includeSubDomains\"")
 			})
-		Expect(err).NotTo(HaveOccurred())
 
 		resp, _, errs := gorequest.New().
 			Get(f.IngressController.HTTPSURL).
@@ -133,15 +121,12 @@ var _ = framework.IngressNginxDescribe("Settings - TLS)", func() {
 		Expect(resp.Header.Get("Strict-Transport-Security")).Should(ContainSubstring("max-age=86400"))
 
 		By("setting includeSubDomains parameter")
+		f.UpdateNginxConfigMapData(hstsIncludeSubdomains, "false")
 
-		err = f.UpdateNginxConfigMapData(hstsIncludeSubdomains, "false")
-		Expect(err).NotTo(HaveOccurred())
-
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return strings.Contains(server, "Strict-Transport-Security: max-age=86400\"")
 			})
-		Expect(err).NotTo(HaveOccurred())
 
 		resp, _, errs = gorequest.New().
 			Get(f.IngressController.HTTPSURL).
@@ -154,15 +139,12 @@ var _ = framework.IngressNginxDescribe("Settings - TLS)", func() {
 		Expect(resp.Header.Get("Strict-Transport-Security")).ShouldNot(ContainSubstring("includeSubDomains"))
 
 		By("setting preload parameter")
+		f.UpdateNginxConfigMapData(hstsPreload, "true")
 
-		err = f.UpdateNginxConfigMapData(hstsPreload, "true")
-		Expect(err).NotTo(HaveOccurred())
-
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return strings.Contains(server, "Strict-Transport-Security: max-age=86400; preload\"")
 			})
-		Expect(err).NotTo(HaveOccurred())
 
 		resp, _, errs = gorequest.New().
 			Get(f.IngressController.HTTPSURL).
@@ -177,11 +159,7 @@ var _ = framework.IngressNginxDescribe("Settings - TLS)", func() {
 })
 
 func tlsEndpoint(f *framework.Framework, host string) (*tls.Config, error) {
-	ing, err := f.EnsureIngress(framework.NewSingleIngressWithTLS(host, "/", host, f.IngressController.Namespace, "http-svc", 80, nil))
-	if err != nil {
-		return nil, err
-	}
-
+	ing := f.EnsureIngress(framework.NewSingleIngressWithTLS(host, "/", host, f.IngressController.Namespace, "http-svc", 80, nil))
 	return framework.CreateIngressTLSSecret(f.KubeClientSet,
 		ing.Spec.TLS[0].Hosts,
 		ing.Spec.TLS[0].SecretName,
diff --git a/test/e2e/ssl/secret_update.go b/test/e2e/ssl/secret_update.go
index 9050d37ad..8dded8a45 100644
--- a/test/e2e/ssl/secret_update.go
+++ b/test/e2e/ssl/secret_update.go
@@ -33,8 +33,7 @@ var _ = framework.IngressNginxDescribe("SSL", func() {
 	f := framework.NewDefaultFramework("ssl")
 
 	BeforeEach(func() {
-		err := f.NewEchoDeployment()
-		Expect(err).NotTo(HaveOccurred())
+		f.NewEchoDeployment()
 	})
 
 	AfterEach(func() {
@@ -43,7 +42,7 @@ var _ = framework.IngressNginxDescribe("SSL", func() {
 	It("should not appear references to secret updates not used in ingress rules", func() {
 		host := "ssl-update"
 
-		dummySecret, err := f.EnsureSecret(&v1.Secret{
+		dummySecret := f.EnsureSecret(&v1.Secret{
 			ObjectMeta: metav1.ObjectMeta{
 				Name:      "dummy",
 				Namespace: f.IngressController.Namespace,
@@ -52,24 +51,20 @@ var _ = framework.IngressNginxDescribe("SSL", func() {
 				"key": []byte("value"),
 			},
 		})
-		Expect(err).NotTo(HaveOccurred())
 
-		ing, err := f.EnsureIngress(framework.NewSingleIngressWithTLS(host, "/", host, f.IngressController.Namespace, "http-svc", 80, nil))
-		Expect(err).ToNot(HaveOccurred())
-		Expect(ing).ToNot(BeNil())
+		ing := f.EnsureIngress(framework.NewSingleIngressWithTLS(host, "/", host, f.IngressController.Namespace, "http-svc", 80, nil))
 
-		_, err = framework.CreateIngressTLSSecret(f.KubeClientSet,
+		_, err := framework.CreateIngressTLSSecret(f.KubeClientSet,
 			ing.Spec.TLS[0].Hosts,
 			ing.Spec.TLS[0].SecretName,
 			ing.Namespace)
 		Expect(err).ToNot(HaveOccurred())
 
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return strings.Contains(server, "server_name ssl-update") &&
 					strings.Contains(server, "listen 443")
 			})
-		Expect(err).ToNot(HaveOccurred())
 
 		log, err := f.NginxLogs()
 		Expect(err).ToNot(HaveOccurred())
diff --git a/test/e2e/status/update.go b/test/e2e/status/update.go
index 171d62232..728dbf2fa 100644
--- a/test/e2e/status/update.go
+++ b/test/e2e/status/update.go
@@ -47,7 +47,7 @@ var _ = framework.IngressNginxDescribe("Status Update [Status]", func() {
 
 	It("should update status field after client-go reconnection", func() {
 		port, cmd, err := f.KubectlProxy(0)
-		Expect(err).NotTo(HaveOccurred(), "starting kubectl proxy")
+		Expect(err).NotTo(HaveOccurred(), "unexpected error starting kubectl proxy")
 
 		err = framework.UpdateDeployment(f.KubeClientSet, f.IngressController.Namespace, "nginx-ingress-controller", 1,
 			func(deployment *appsv1beta1.Deployment) error {
@@ -70,46 +70,42 @@ var _ = framework.IngressNginxDescribe("Status Update [Status]", func() {
 				_, err := f.KubeClientSet.AppsV1beta1().Deployments(f.IngressController.Namespace).Update(deployment)
 				return err
 			})
-		Expect(err).NotTo(HaveOccurred(), "updating ingress controller deployment flags")
+		Expect(err).NotTo(HaveOccurred(), "unexpected error updating ingress controller deployment flags")
 
-		err = f.NewEchoDeploymentWithReplicas(1)
-		Expect(err).NotTo(HaveOccurred(), "waiting one replicaset in echoserver deployment")
+		f.NewEchoDeploymentWithReplicas(1)
 
-		ing, err := f.EnsureIngress(framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, nil))
-		Expect(err).NotTo(HaveOccurred(), "waiting Ingress creation for hostname %v", host)
-		Expect(ing).NotTo(BeNil())
+		ing := f.EnsureIngress(framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, nil))
 
-		err = f.WaitForNginxConfiguration(
+		f.WaitForNginxConfiguration(
 			func(cfg string) bool {
 				return strings.Contains(cfg, fmt.Sprintf("server_name %s", host))
 			})
-		Expect(err).NotTo(HaveOccurred(), "waiting for nginx server section with server_name %v", host)
 
 		framework.Logf("waiting for leader election and initial status update")
 		time.Sleep(30 * time.Second)
 
 		err = cmd.Process.Kill()
-		Expect(err).NotTo(HaveOccurred(), "terminating kubectl proxy")
+		Expect(err).NotTo(HaveOccurred(), "unexpected error terminating kubectl proxy")
 
 		ing, err = f.KubeClientSet.Extensions().Ingresses(f.IngressController.Namespace).Get(host, metav1.GetOptions{})
-		Expect(err).NotTo(HaveOccurred(), "getting %s/%v Ingress", f.IngressController.Namespace, host)
+		Expect(err).NotTo(HaveOccurred(), "unexpected error getting %s/%v Ingress", f.IngressController.Namespace, host)
 
 		ing.Status.LoadBalancer.Ingress = []apiv1.LoadBalancerIngress{}
 		_, err = f.KubeClientSet.Extensions().Ingresses(f.IngressController.Namespace).UpdateStatus(ing)
-		Expect(err).NotTo(HaveOccurred(), "cleaning Ingress status")
+		Expect(err).NotTo(HaveOccurred(), "unexpected error cleaning Ingress status")
 		time.Sleep(10 * time.Second)
 
 		err = f.KubeClientSet.CoreV1().
 			ConfigMaps(f.IngressController.Namespace).
 			Delete("ingress-controller-leader-nginx", &metav1.DeleteOptions{})
-		Expect(err).NotTo(HaveOccurred(), "deleting leader election configmap")
+		Expect(err).NotTo(HaveOccurred(), "unexpected error deleting leader election configmap")
 
 		_, cmd, err = f.KubectlProxy(port)
-		Expect(err).NotTo(HaveOccurred(), "starting kubectl proxy")
+		Expect(err).NotTo(HaveOccurred(), "unexpected error starting kubectl proxy")
 		defer func() {
 			if cmd != nil {
 				err := cmd.Process.Kill()
-				Expect(err).NotTo(HaveOccurred(), "terminating kubectl proxy")
+				Expect(err).NotTo(HaveOccurred(), "unexpected error terminating kubectl proxy")
 			}
 		}()
 
@@ -125,7 +121,7 @@ var _ = framework.IngressNginxDescribe("Status Update [Status]", func() {
 
 			return true, nil
 		})
-		Expect(err).NotTo(HaveOccurred(), "waiting for ingress status")
+		Expect(err).NotTo(HaveOccurred(), "unexpected error waiting for ingress status")
 		Expect(ing.Status.LoadBalancer.Ingress).Should(Equal([]apiv1.LoadBalancerIngress{
 			{IP: "1.1.0.0"},
 		}))

From 91a12ffd526edb92c95e0f53b7fff7f4b44e373c Mon Sep 17 00:00:00 2001
From: Elvin Efendi <elvin.efendiyev@gmail.com>
Date: Tue, 30 Oct 2018 16:02:15 +0400
Subject: [PATCH 25/31] sticky session e2e test

---
 test/e2e/annotations/affinity.go | 71 ++++++++++++--------------------
 1 file changed, 27 insertions(+), 44 deletions(-)

diff --git a/test/e2e/annotations/affinity.go b/test/e2e/annotations/affinity.go
index 10d0ece50..2cd389bfe 100644
--- a/test/e2e/annotations/affinity.go
+++ b/test/e2e/annotations/affinity.go
@@ -16,22 +16,27 @@ limitations under the License.
 
 package annotations
 
-/*
 import (
+	"fmt"
+	"net/http"
+	"strings"
+
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+	"github.com/parnurzeal/gorequest"
+
+	"k8s.io/api/extensions/v1beta1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
+
+	"k8s.io/ingress-nginx/test/e2e/framework"
 )
 
-
-// TODO(elvinefendi) merge this with Affinity tests in test/e2e/lua/dynamic_configuration.go
-var _ = framework.IngressNginxDescribe("Annotations - Affinity", func() {
+var _ = framework.IngressNginxDescribe("Annotations - Affinity/Sticky Sessions", func() {
 	f := framework.NewDefaultFramework("affinity")
 
 	BeforeEach(func() {
-		err := f.DisableDynamicConfiguration()
-		Expect(err).NotTo(HaveOccurred())
-
-		err = f.NewEchoDeploymentWithReplicas(2)
+		err := f.NewEchoDeploymentWithReplicas(2)
 		Expect(err).NotTo(HaveOccurred())
 	})
 
@@ -53,7 +58,7 @@ var _ = framework.IngressNginxDescribe("Annotations - Affinity", func() {
 
 		err = f.WaitForNginxServer(host,
 			func(server string) bool {
-				return strings.Contains(server, "proxy_pass http://sticky-"+f.IngressController.Namespace+"-http-svc-80;")
+				return strings.Contains(server, fmt.Sprintf("server_name %s ;", host))
 			})
 		Expect(err).NotTo(HaveOccurred())
 
@@ -67,36 +72,6 @@ var _ = framework.IngressNginxDescribe("Annotations - Affinity", func() {
 		Expect(resp.Header.Get("Set-Cookie")).Should(ContainSubstring("SERVERID="))
 	})
 
-	It("should redirect to '/something' with enabled affinity", func() {
-		host := "example.com"
-		annotations := map[string]string{
-			"nginx.ingress.kubernetes.io/affinity":            "cookie",
-			"nginx.ingress.kubernetes.io/session-cookie-name": "SERVERID",
-		}
-
-		ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations)
-		_, err := f.EnsureIngress(ing)
-
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
-
-		err = f.WaitForNginxServer(host,
-			func(server string) bool {
-				return strings.Contains(server, "proxy_pass http://sticky-"+f.IngressController.Namespace+"-http-svc-80;")
-			})
-		Expect(err).NotTo(HaveOccurred())
-
-		resp, body, errs := gorequest.New().
-			Get(f.IngressController.HTTPURL).
-			Set("Host", host).
-			End()
-
-		Expect(len(errs)).Should(BeNumerically("==", 0))
-		Expect(resp.StatusCode).Should(Equal(http.StatusOK))
-		Expect(body).Should(ContainSubstring(fmt.Sprintf("request_uri=http://%v:8080/", host)))
-		Expect(resp.Header.Get("Set-Cookie")).Should(ContainSubstring("SERVERID="))
-	})
-
 	It("should set the path to /something on the generated cookie", func() {
 		host := "example.com"
 		annotations := map[string]string{
@@ -112,7 +87,7 @@ var _ = framework.IngressNginxDescribe("Annotations - Affinity", func() {
 
 		err = f.WaitForNginxServer(host,
 			func(server string) bool {
-				return strings.Contains(server, "proxy_pass http://sticky-"+f.IngressController.Namespace+"-http-svc-80;")
+				return strings.Contains(server, fmt.Sprintf("server_name %s ;", host))
 			})
 		Expect(err).NotTo(HaveOccurred())
 
@@ -126,7 +101,7 @@ var _ = framework.IngressNginxDescribe("Annotations - Affinity", func() {
 		Expect(resp.Header.Get("Set-Cookie")).Should(ContainSubstring("Path=/something"))
 	})
 
-	It("should set the path to / on the generated cookie if there's more than one rule referring to the same backend", func() {
+	It("does not set the path to / on the generated cookie if there's more than one rule referring to the same backend", func() {
 		host := "example.com"
 		annotations := map[string]string{
 			"nginx.ingress.kubernetes.io/affinity":            "cookie",
@@ -173,7 +148,7 @@ var _ = framework.IngressNginxDescribe("Annotations - Affinity", func() {
 
 		err = f.WaitForNginxServer(host,
 			func(server string) bool {
-				return strings.Contains(server, "proxy_pass http://sticky-"+f.IngressController.Namespace+"-http-svc-80;")
+				return strings.Contains(server, fmt.Sprintf("server_name %s ;", host))
 			})
 		Expect(err).NotTo(HaveOccurred())
 
@@ -184,7 +159,15 @@ var _ = framework.IngressNginxDescribe("Annotations - Affinity", func() {
 
 		Expect(len(errs)).Should(BeNumerically("==", 0))
 		Expect(resp.StatusCode).Should(Equal(http.StatusOK))
-		Expect(resp.Header.Get("Set-Cookie")).Should(ContainSubstring("Path=/;"))
+		Expect(resp.Header.Get("Set-Cookie")).Should(ContainSubstring("Path=/something;"))
+
+		resp, _, errs = gorequest.New().
+			Get(f.IngressController.HTTPURL+"/somewhereelese").
+			Set("Host", host).
+			End()
+
+		Expect(len(errs)).Should(BeNumerically("==", 0))
+		Expect(resp.StatusCode).Should(Equal(http.StatusOK))
+		Expect(resp.Header.Get("Set-Cookie")).Should(ContainSubstring("Path=/somewhereelese;"))
 	})
 })
-*/

From 9e639f97888ead927ab918210f410e9b787cc270 Mon Sep 17 00:00:00 2001
From: Elvin Efendi <elvin.efendiyev@gmail.com>
Date: Tue, 30 Oct 2018 16:23:08 +0400
Subject: [PATCH 26/31] fix sticky session implementation

---
 rootfs/etc/nginx/lua/balancer/sticky.lua           | 13 ++++---------
 rootfs/etc/nginx/lua/test/balancer/sticky_test.lua |  3 ++-
 2 files changed, 6 insertions(+), 10 deletions(-)

diff --git a/rootfs/etc/nginx/lua/balancer/sticky.lua b/rootfs/etc/nginx/lua/balancer/sticky.lua
index 4d4684146..5eb550ba5 100644
--- a/rootfs/etc/nginx/lua/balancer/sticky.lua
+++ b/rootfs/etc/nginx/lua/balancer/sticky.lua
@@ -50,22 +50,17 @@ local function set_cookie(self, value)
   end
 end
 
-local function pick_random(instance)
-  local index = math.random(instance.npoints)
-  return instance:next(index)
-end
-
 function _M.balance(self)
   local cookie, err = ck:new()
   if not cookie then
-    ngx.log(ngx.ERR, err)
-    return pick_random(self.instance)
+    ngx.log(ngx.ERR, "error while initializing cookie: " .. tostring(err))
+    return
   end
 
   local key = cookie:get(self.cookie_name)
   if not key then
-    local tmp_endpoint_string = pick_random(self.instance)
-    key = encrypted_endpoint_string(self, tmp_endpoint_string)
+    local random_str = string.format("%s.%s", ngx.now(), ngx.worker.pid())
+    key = encrypted_endpoint_string(self, random_str)
     set_cookie(self, key)
   end
 
diff --git a/rootfs/etc/nginx/lua/test/balancer/sticky_test.lua b/rootfs/etc/nginx/lua/test/balancer/sticky_test.lua
index d5c402236..9c022d9c1 100644
--- a/rootfs/etc/nginx/lua/test/balancer/sticky_test.lua
+++ b/rootfs/etc/nginx/lua/test/balancer/sticky_test.lua
@@ -116,7 +116,8 @@ describe("Sticky", function()
           local cookie_instance = {
             set = function(self, payload)
               assert.equal(payload.key, test_backend.sessionAffinityConfig.cookieSessionAffinity.name)
-              assert.equal(payload.value, util[test_backend_hash_fn .. "_digest"](test_backend_endpoint))
+              local expected_len = #util[test_backend_hash_fn .. "_digest"]("anything")
+              assert.equal(#payload.value, expected_len)
               assert.equal(payload.path, ngx.var.location_path)
               assert.equal(payload.domain, nil)
               assert.equal(payload.httponly, true)

From c9668dd40bddc28d4a5cee96c5899927c6570ea5 Mon Sep 17 00:00:00 2001
From: Manuel Alejandro de Brito Fontes <aledbf@gmail.com>
Date: Tue, 30 Oct 2018 13:31:07 -0300
Subject: [PATCH 27/31] Fix e2e tests

---
 internal/ingress/annotations/rewrite/main.go |  2 +-
 test/e2e/annotations/affinity.go             | 27 +++++-------------
 test/e2e/lua/dynamic_certificates.go         |  1 +
 test/e2e/settings/geoip2.go                  | 30 ++++++++------------
 4 files changed, 21 insertions(+), 39 deletions(-)

diff --git a/internal/ingress/annotations/rewrite/main.go b/internal/ingress/annotations/rewrite/main.go
index 270bdb1a7..b1e63bc37 100644
--- a/internal/ingress/annotations/rewrite/main.go
+++ b/internal/ingress/annotations/rewrite/main.go
@@ -39,7 +39,7 @@ type Config struct {
 	// AppRoot defines the Application Root that the Controller must redirect if it's in '/' context
 	AppRoot string `json:"appRoot"`
 	// UseRegex indicates whether or not the locations use regex paths
-	UseRegex bool `json:useRegex`
+	UseRegex bool `json:"useRegex"`
 }
 
 // Equal tests for equality between two Redirect types
diff --git a/test/e2e/annotations/affinity.go b/test/e2e/annotations/affinity.go
index 2cd389bfe..5903f7e15 100644
--- a/test/e2e/annotations/affinity.go
+++ b/test/e2e/annotations/affinity.go
@@ -36,8 +36,7 @@ var _ = framework.IngressNginxDescribe("Annotations - Affinity/Sticky Sessions",
 	f := framework.NewDefaultFramework("affinity")
 
 	BeforeEach(func() {
-		err := f.NewEchoDeploymentWithReplicas(2)
-		Expect(err).NotTo(HaveOccurred())
+		f.NewEchoDeploymentWithReplicas(2)
 	})
 
 	AfterEach(func() {
@@ -51,16 +50,12 @@ var _ = framework.IngressNginxDescribe("Annotations - Affinity/Sticky Sessions",
 		}
 
 		ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations)
-		_, err := f.EnsureIngress(ing)
+		f.EnsureIngress(ing)
 
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
-
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return strings.Contains(server, fmt.Sprintf("server_name %s ;", host))
 			})
-		Expect(err).NotTo(HaveOccurred())
 
 		resp, _, errs := gorequest.New().
 			Get(f.IngressController.HTTPURL).
@@ -80,16 +75,12 @@ var _ = framework.IngressNginxDescribe("Annotations - Affinity/Sticky Sessions",
 		}
 
 		ing := framework.NewSingleIngress(host, "/something", host, f.IngressController.Namespace, "http-svc", 80, &annotations)
-		_, err := f.EnsureIngress(ing)
+		f.EnsureIngress(ing)
 
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
-
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return strings.Contains(server, fmt.Sprintf("server_name %s ;", host))
 			})
-		Expect(err).NotTo(HaveOccurred())
 
 		resp, _, errs := gorequest.New().
 			Get(f.IngressController.HTTPURL+"/something").
@@ -108,7 +99,7 @@ var _ = framework.IngressNginxDescribe("Annotations - Affinity/Sticky Sessions",
 			"nginx.ingress.kubernetes.io/session-cookie-name": "SERVERID",
 		}
 
-		ing, err := f.EnsureIngress(&v1beta1.Ingress{
+		f.EnsureIngress(&v1beta1.Ingress{
 			ObjectMeta: metav1.ObjectMeta{
 				Name:        host,
 				Namespace:   f.IngressController.Namespace,
@@ -143,14 +134,10 @@ var _ = framework.IngressNginxDescribe("Annotations - Affinity/Sticky Sessions",
 			},
 		})
 
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
-
-		err = f.WaitForNginxServer(host,
+		f.WaitForNginxServer(host,
 			func(server string) bool {
 				return strings.Contains(server, fmt.Sprintf("server_name %s ;", host))
 			})
-		Expect(err).NotTo(HaveOccurred())
 
 		resp, _, errs := gorequest.New().
 			Get(f.IngressController.HTTPURL+"/something").
diff --git a/test/e2e/lua/dynamic_certificates.go b/test/e2e/lua/dynamic_certificates.go
index 46cf79585..3541997df 100644
--- a/test/e2e/lua/dynamic_certificates.go
+++ b/test/e2e/lua/dynamic_certificates.go
@@ -150,6 +150,7 @@ var _ = framework.IngressNginxDescribe("Dynamic Certificate", func() {
 
 		It("picks up the updated certificate without reloading", func() {
 			ing, err := f.KubeClientSet.ExtensionsV1beta1().Ingresses(f.IngressController.Namespace).Get(host, metav1.GetOptions{})
+			Expect(err).ToNot(HaveOccurred())
 
 			ensureHTTPSRequest(fmt.Sprintf("%s?id=dummy_log_splitter_foo_bar", f.IngressController.HTTPSURL), host, host)
 
diff --git a/test/e2e/settings/geoip2.go b/test/e2e/settings/geoip2.go
index f802fd575..325691b51 100644
--- a/test/e2e/settings/geoip2.go
+++ b/test/e2e/settings/geoip2.go
@@ -19,11 +19,12 @@ package settings
 import (
 	"strings"
 
+	"net/http"
+
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	"github.com/parnurzeal/gorequest"
 	"k8s.io/ingress-nginx/test/e2e/framework"
-	"net/http"
 )
 
 var _ = framework.IngressNginxDescribe("Geoip2", func() {
@@ -32,27 +33,23 @@ var _ = framework.IngressNginxDescribe("Geoip2", func() {
 	host := "geoip2"
 
 	BeforeEach(func() {
-		err := f.NewEchoDeployment()
-		Expect(err).NotTo(HaveOccurred())
+		f.NewEchoDeployment()
 	})
 
 	It("should only allow requests from specific countries", func() {
-		err := f.UpdateNginxConfigMapData("use-geoip2", "true")
-		Expect(err).NotTo(HaveOccurred())
+		f.UpdateNginxConfigMapData("use-geoip2", "true")
 
 		httpSnippetAllowingOnlyAustralia :=
 			`map $geoip2_city_country_code $blocked_country {
   default 1;
   AU 0;
 }`
-		err = f.UpdateNginxConfigMapData("http-snippet", httpSnippetAllowingOnlyAustralia)
-		Expect(err).NotTo(HaveOccurred())
+		f.UpdateNginxConfigMapData("http-snippet", httpSnippetAllowingOnlyAustralia)
 
-		err = f.WaitForNginxConfiguration(
+		f.WaitForNginxConfiguration(
 			func(cfg string) bool {
 				return strings.Contains(cfg, "map $geoip2_city_country_code $blocked_country")
 			})
-		Expect(err).NotTo(HaveOccurred())
 
 		configSnippet :=
 			`if ($blocked_country) {
@@ -63,32 +60,29 @@ var _ = framework.IngressNginxDescribe("Geoip2", func() {
 			"nginx.ingress.kubernetes.io/configuration-snippet": configSnippet,
 		}
 
-		ing, err := f.EnsureIngress(framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations))
-		Expect(err).NotTo(HaveOccurred())
-		Expect(ing).NotTo(BeNil())
+		f.EnsureIngress(framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations))
 
-		err = f.WaitForNginxConfiguration(
+		f.WaitForNginxConfiguration(
 			func(cfg string) bool {
 				return strings.Contains(cfg, "if ($blocked_country)")
 			})
-		Expect(err).NotTo(HaveOccurred())
 
 		// Should be blocked
-		usIp := "8.8.8.8"
+		usIP := "8.8.8.8"
 		resp, _, errs := gorequest.New().
 			Get(f.IngressController.HTTPURL).
 			Set("Host", host).
-			Set("X-Forwarded-For", usIp).
+			Set("X-Forwarded-For", usIP).
 			End()
 		Expect(errs).To(BeNil())
 		Expect(resp.StatusCode).Should(Equal(http.StatusForbidden))
 
 		// Shouldn't be blocked
-		australianIp := "1.1.1.1"
+		australianIP := "1.1.1.1"
 		resp, _, errs = gorequest.New().
 			Get(f.IngressController.HTTPURL).
 			Set("Host", host).
-			Set("X-Forwarded-For", australianIp).
+			Set("X-Forwarded-For", australianIP).
 			End()
 		Expect(errs).To(BeNil())
 		Expect(resp.StatusCode).Should(Equal(http.StatusOK))

From c4834e5063b16186b3d46186a674422ee0ea5684 Mon Sep 17 00:00:00 2001
From: Angga Lanuma <lanuma.angga@gmail.com>
Date: Wed, 31 Oct 2018 00:13:21 +0700
Subject: [PATCH 28/31] fix baremetal.md link

---
 docs/deploy/index.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/deploy/index.md b/docs/deploy/index.md
index c1749061b..7ee1864dc 100644
--- a/docs/deploy/index.md
+++ b/docs/deploy/index.md
@@ -158,7 +158,7 @@ kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/mast
 ```
 
 !!! tip
-    For extended notes regarding deployments on bare-metal, see [Bare-metal considerations](./baremetal/).
+    For extended notes regarding deployments on bare-metal, see [Bare-metal considerations](./baremetal.md/).
 
 ### Verify installation
 

From 71ebe1cba5862103749c77614b57b33610a473ae Mon Sep 17 00:00:00 2001
From: Manuel Alejandro de Brito Fontes <aledbf@gmail.com>
Date: Tue, 30 Oct 2018 20:46:48 -0300
Subject: [PATCH 29/31] Code linting

---
 .../annotations/backendprotocol/main.go       |  9 +++++---
 .../ingress/annotations/rewrite/main_test.go  |  2 +-
 internal/ingress/controller/config/config.go  | 14 +++++------
 internal/ingress/controller/nginx.go          |  7 +-----
 .../controller/template/configmap_test.go     |  2 +-
 .../ingress/controller/template/template.go   | 13 -----------
 internal/ingress/status/status.go             | 23 +++++++++++--------
 rootfs/etc/nginx/template/nginx.tmpl          |  8 +++----
 8 files changed, 34 insertions(+), 44 deletions(-)

diff --git a/internal/ingress/annotations/backendprotocol/main.go b/internal/ingress/annotations/backendprotocol/main.go
index 66dbae783..4f9417a2f 100644
--- a/internal/ingress/annotations/backendprotocol/main.go
+++ b/internal/ingress/annotations/backendprotocol/main.go
@@ -27,6 +27,9 @@ import (
 	"k8s.io/ingress-nginx/internal/ingress/resolver"
 )
 
+// HTTP protocol
+const HTTP = "HTTP"
+
 var (
 	validProtocols = regexp.MustCompile(`^(HTTP|HTTPS|AJP|GRPC|GRPCS)$`)
 )
@@ -44,18 +47,18 @@ func NewParser(r resolver.Resolver) parser.IngressAnnotation {
 // rule used to indicate the backend protocol.
 func (a backendProtocol) Parse(ing *extensions.Ingress) (interface{}, error) {
 	if ing.GetAnnotations() == nil {
-		return "HTTP", nil
+		return HTTP, nil
 	}
 
 	proto, err := parser.GetStringAnnotation("backend-protocol", ing)
 	if err != nil {
-		return "HTTP", nil
+		return HTTP, nil
 	}
 
 	proto = strings.TrimSpace(strings.ToUpper(proto))
 	if !validProtocols.MatchString(proto) {
 		glog.Warningf("Protocol %v is not a valid value for the backend-protocol annotation. Using HTTP as protocol", proto)
-		return "HTTP", nil
+		return HTTP, nil
 	}
 
 	return proto, nil
diff --git a/internal/ingress/annotations/rewrite/main_test.go b/internal/ingress/annotations/rewrite/main_test.go
index 71721ad93..899379826 100644
--- a/internal/ingress/annotations/rewrite/main_test.go
+++ b/internal/ingress/annotations/rewrite/main_test.go
@@ -191,7 +191,7 @@ func TestUseRegex(t *testing.T) {
 	if !ok {
 		t.Errorf("expected a App Context")
 	}
-	if redirect.UseRegex != true {
+	if !redirect.UseRegex {
 		t.Errorf("Unexpected value got in UseRegex")
 	}
 }
diff --git a/internal/ingress/controller/config/config.go b/internal/ingress/controller/config/config.go
index d6520dd5e..2089ec995 100644
--- a/internal/ingress/controller/config/config.go
+++ b/internal/ingress/controller/config/config.go
@@ -102,10 +102,10 @@ type Configuration struct {
 	// By default access logs go to /var/log/nginx/access.log
 	AccessLogPath string `json:"access-log-path,omitempty"`
 
-	// WorkerCpuAffinity bind nginx worker processes to CPUs this will improve response latency
+	// WorkerCPUAffinity bind nginx worker processes to CPUs this will improve response latency
 	// http://nginx.org/en/docs/ngx_core_module.html#worker_cpu_affinity
 	// By default this is disabled
-	WorkerCpuAffinity string `json:"worker-cpu-affinity,omitempty"`
+	WorkerCPUAffinity string `json:"worker-cpu-affinity,omitempty"`
 	// ErrorLogPath sets the path of the error logs
 	// http://nginx.org/en/docs/ngx_core_module.html#error_log
 	// By default error logs go to /var/log/nginx/error.log
@@ -442,11 +442,11 @@ type Configuration struct {
 
 	// If the request does not have a request-id, should we generate a random value?
 	// Default: true
-	GenerateRequestId bool `json:"generate-request-id,omitempty"`
+	GenerateRequestID bool `json:"generate-request-id,omitempty"`
 
 	// Adds an X-Original-Uri header with the original request URI to the backend request
 	// Default: true
-	ProxyAddOriginalUriHeader bool `json:"proxy-add-original-uri-header"`
+	ProxyAddOriginalURIHeader bool `json:"proxy-add-original-uri-header"`
 
 	// EnableOpentracing enables the nginx Opentracing extension
 	// https://github.com/opentracing-contrib/nginx-opentracing
@@ -574,7 +574,7 @@ func NewDefault() Configuration {
 	cfg := Configuration{
 		AllowBackendServerHeader:   false,
 		AccessLogPath:              "/var/log/nginx/access.log",
-		WorkerCpuAffinity:          "",
+		WorkerCPUAffinity:          "",
 		ErrorLogPath:               "/var/log/nginx/error.log",
 		BlockCIDRs:                 defBlockEntity,
 		BlockUserAgents:            defBlockEntity,
@@ -591,8 +591,8 @@ func NewDefault() Configuration {
 		UseForwardedHeaders:        true,
 		ForwardedForHeader:         "X-Forwarded-For",
 		ComputeFullForwardedFor:    false,
-		ProxyAddOriginalUriHeader:  true,
-		GenerateRequestId:          true,
+		ProxyAddOriginalURIHeader:  true,
+		GenerateRequestID:          true,
 		HTTP2MaxFieldSize:          "4k",
 		HTTP2MaxHeaderSize:         "16k",
 		HTTP2MaxRequests:           1000,
diff --git a/internal/ingress/controller/nginx.go b/internal/ingress/controller/nginx.go
index e5ab10063..f99b8d58e 100644
--- a/internal/ingress/controller/nginx.go
+++ b/internal/ingress/controller/nginx.go
@@ -811,12 +811,7 @@ func configureCertificates(pcfg *ingress.Configuration, port int) error {
 	}
 
 	url := fmt.Sprintf("http://localhost:%d/configuration/servers", port)
-	err := post(url, servers)
-	if err != nil {
-		return err
-	}
-
-	return nil
+	return post(url, servers)
 }
 
 func post(url string, data interface{}) error {
diff --git a/internal/ingress/controller/template/configmap_test.go b/internal/ingress/controller/template/configmap_test.go
index 1bb7b773b..535f105a9 100644
--- a/internal/ingress/controller/template/configmap_test.go
+++ b/internal/ingress/controller/template/configmap_test.go
@@ -90,7 +90,7 @@ func TestMergeConfigMapToStruct(t *testing.T) {
 	def.WorkerShutdownTimeout = "99s"
 	def.NginxStatusIpv4Whitelist = []string{"127.0.0.1", "10.0.0.0/24"}
 	def.NginxStatusIpv6Whitelist = []string{"::1", "2001::/16"}
-	def.ProxyAddOriginalUriHeader = false
+	def.ProxyAddOriginalURIHeader = false
 
 	hash, err := hashstructure.Hash(def, &hashstructure.HashOptions{
 		TagName: "json",
diff --git a/internal/ingress/controller/template/template.go b/internal/ingress/controller/template/template.go
index 4dd3461e0..0f8b3ee65 100644
--- a/internal/ingress/controller/template/template.go
+++ b/internal/ingress/controller/template/template.go
@@ -722,19 +722,6 @@ func buildUpstreamName(loc interface{}) string {
 	return upstreamName
 }
 
-// TODO: Needs Unit Tests
-func isSticky(host string, loc *ingress.Location, stickyLocations map[string][]string) bool {
-	if _, ok := stickyLocations[host]; ok {
-		for _, sl := range stickyLocations[host] {
-			if sl == loc.Path {
-				return true
-			}
-		}
-	}
-
-	return false
-}
-
 func buildNextUpstream(i, r interface{}) string {
 	nextUpstream, ok := i.(string)
 	if !ok {
diff --git a/internal/ingress/status/status.go b/internal/ingress/status/status.go
index c09128130..320ad7c0a 100644
--- a/internal/ingress/status/status.go
+++ b/internal/ingress/status/status.go
@@ -110,8 +110,15 @@ func (s statusSync) Run() {
 
 	// start a new context
 	ctx := context.Background()
-	// allow to cancel the context in case we stop being the leader
-	leaderCtx, cancel := context.WithCancel(ctx)
+
+	var cancelContext context.CancelFunc
+
+	var newLeaderCtx = func(ctx context.Context) context.CancelFunc {
+		// allow to cancel the context in case we stop being the leader
+		leaderCtx, cancel := context.WithCancel(ctx)
+		go s.elector.Run(leaderCtx)
+		return cancel
+	}
 
 	var stopCh chan struct{}
 	callbacks := leaderelection.LeaderCallbacks{
@@ -133,11 +140,9 @@ func (s statusSync) Run() {
 			close(stopCh)
 
 			// cancel the context
-			cancel()
+			cancelContext()
 
-			// start a new context and run the elector
-			leaderCtx, cancel = context.WithCancel(ctx)
-			go s.elector.Run(leaderCtx)
+			cancelContext = newLeaderCtx(ctx)
 		},
 		OnNewLeader: func(identity string) {
 			glog.Infof("new leader elected: %v", identity)
@@ -162,7 +167,8 @@ func (s statusSync) Run() {
 	}
 
 	ttl := 30 * time.Second
-	le, err := leaderelection.NewLeaderElector(leaderelection.LeaderElectionConfig{
+	var err error
+	s.elector, err = leaderelection.NewLeaderElector(leaderelection.LeaderElectionConfig{
 		Lock:          &lock,
 		LeaseDuration: ttl,
 		RenewDeadline: ttl / 2,
@@ -172,9 +178,8 @@ func (s statusSync) Run() {
 	if err != nil {
 		glog.Fatalf("unexpected error starting leader election: %v", err)
 	}
-	s.elector = le
 
-	go le.Run(leaderCtx)
+	cancelContext = newLeaderCtx(ctx)
 }
 
 // Shutdown stop the sync. In case the instance is the leader it will remove the current IP
diff --git a/rootfs/etc/nginx/template/nginx.tmpl b/rootfs/etc/nginx/template/nginx.tmpl
index 0e855953d..22c7f893d 100644
--- a/rootfs/etc/nginx/template/nginx.tmpl
+++ b/rootfs/etc/nginx/template/nginx.tmpl
@@ -27,8 +27,8 @@ load_module /etc/nginx/modules/ngx_http_opentracing_module.so;
 daemon off;
 
 worker_processes {{ $cfg.WorkerProcesses }};
-{{ if gt (len $cfg.WorkerCpuAffinity) 0 }}
-worker_cpu_affinity {{ $cfg.WorkerCpuAffinity }};
+{{ if gt (len $cfg.WorkerCPUAffinity) 0 }}
+worker_cpu_affinity {{ $cfg.WorkerCPUAffinity }};
 {{ end }}
 
 {{ if ne .MaxOpenFiles 0 }}
@@ -346,7 +346,7 @@ http {
     # If no such header is provided, it can provide a random value.
     map $http_x_request_id $req_id {
         default   $http_x_request_id;
-        {{ if $cfg.GenerateRequestId }}
+        {{ if $cfg.GenerateRequestID }}
         ""        $request_id;
         {{ end }}
     }
@@ -1115,7 +1115,7 @@ stream {
             {{ $proxySetHeader }} X-Forwarded-Host       $best_http_host;
             {{ $proxySetHeader }} X-Forwarded-Port       $pass_port;
             {{ $proxySetHeader }} X-Forwarded-Proto      $pass_access_scheme;
-            {{ if $all.Cfg.ProxyAddOriginalUriHeader }}
+            {{ if $all.Cfg.ProxyAddOriginalURIHeader }}
             {{ $proxySetHeader }} X-Original-URI         $request_uri;
             {{ end }}
             {{ $proxySetHeader }} X-Scheme               $pass_access_scheme;

From 9d951101f6b397720e8b3b27ee4f976740af1034 Mon Sep 17 00:00:00 2001
From: crystaljade <xiaodadeyuer@163.com>
Date: Wed, 31 Oct 2018 17:30:59 +0800
Subject: [PATCH 30/31] Update build-single-manifest-sh

---
 hack/build-single-manifest-sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hack/build-single-manifest-sh b/hack/build-single-manifest-sh
index dd29ab55c..0e236e782 100755
--- a/hack/build-single-manifest-sh
+++ b/hack/build-single-manifest-sh
@@ -20,7 +20,7 @@ set -o pipefail
 
 SCRIPT_ROOT=$(dirname ${BASH_SOURCE})/..
 
-INPUT="namespace.yaml configmap.yaml tcp-services-configmap.yaml udp-services-configmap.yaml rbac.yaml with-rbac.yaml"
+INPUT="namespace.yaml configmap.yaml rbac.yaml with-rbac.yaml"
 MANIFEST=$(cd ${SCRIPT_ROOT}/deploy; cat ${INPUT})
 
 echo "${MANIFEST}" > ${SCRIPT_ROOT}/deploy/mandatory.yaml

From 36aceded3200f9b32d371ebc33f5905b77a55364 Mon Sep 17 00:00:00 2001
From: Manuel Alejandro de Brito Fontes <aledbf@gmail.com>
Date: Thu, 1 Nov 2018 09:29:46 -0300
Subject: [PATCH 31/31] Avoid reloads when endpoints are not available

---
 rootfs/etc/nginx/template/nginx.tmpl       | 5 -----
 test/e2e/servicebackend/service_backend.go | 4 ++--
 2 files changed, 2 insertions(+), 7 deletions(-)

diff --git a/rootfs/etc/nginx/template/nginx.tmpl b/rootfs/etc/nginx/template/nginx.tmpl
index 22c7f893d..82370bba7 100644
--- a/rootfs/etc/nginx/template/nginx.tmpl
+++ b/rootfs/etc/nginx/template/nginx.tmpl
@@ -1173,7 +1173,6 @@ stream {
             proxy_set_header       X-Service-Port     $service_port;
             {{ end }}
 
-            {{ if not (empty $location.Backend) }}
             {{ buildProxyPass $server.Hostname $all.Backends $location }}
             {{ if (or (eq $location.Proxy.ProxyRedirectFrom "default") (eq $location.Proxy.ProxyRedirectFrom "off")) }}
             proxy_redirect                          {{ $location.Proxy.ProxyRedirectFrom }};
@@ -1181,10 +1180,6 @@ stream {
             proxy_redirect                          {{ $location.Proxy.ProxyRedirectFrom }} {{ $location.Proxy.ProxyRedirectTo }};
             {{ end }}
             {{ else }}
-            # No endpoints available for the request
-            return 503;
-            {{ end }}
-            {{ else }}
             # Location denied. Reason: {{ $location.Denied | printf "%q" }}
             return 503;
             {{ end }}
diff --git a/test/e2e/servicebackend/service_backend.go b/test/e2e/servicebackend/service_backend.go
index 180327d67..7e28d0315 100644
--- a/test/e2e/servicebackend/service_backend.go
+++ b/test/e2e/servicebackend/service_backend.go
@@ -49,7 +49,7 @@ var _ = framework.IngressNginxDescribe("Service backend - 503", func() {
 
 		f.WaitForNginxServer(host,
 			func(server string) bool {
-				return strings.Contains(server, "return 503;")
+				return strings.Contains(server, "proxy_pass http://upstream_balancer;")
 			})
 
 		resp, _, errs := gorequest.New().
@@ -72,7 +72,7 @@ var _ = framework.IngressNginxDescribe("Service backend - 503", func() {
 
 		f.WaitForNginxServer(host,
 			func(server string) bool {
-				return strings.Contains(server, "return 503;")
+				return strings.Contains(server, "proxy_pass http://upstream_balancer;")
 			})
 
 		resp, _, errs := gorequest.New().