Sync master

controllers/nginx/Makefile

@@ -3,8 +3,8 @@ all: push
 BUILDTAGS=
 
 # Use the 0.0 tag for testing, it shouldn't clobber any release builds
-RELEASE?=0.9.0-beta.9
+TAG?=0.9.0-beta.8
-PREFIX?=gcr.io/google_containers/nginx-ingress-controller
+REGISTRY?=gcr.io/google_containers
 GOOS?=linux
 DOCKER?=gcloud docker --
 
@@ -16,16 +16,91 @@ endif
 
 PKG=k8s.io/ingress/controllers/nginx
 
+ARCH ?= $(shell go env GOARCH)
+GOARCH = ${ARCH}
+DUMB_ARCH = ${ARCH}
+
+ALL_ARCH = amd64 arm ppc64le
+
+QEMUVERSION=v2.7.0
+
+IMGNAME = nginx-ingress-controller
+IMAGE = $(REGISTRY)/$(IMGNAME)
+MULTI_ARCH_IMG = $(IMAGE)-$(ARCH)
+
+# Set default base image dynamically for each arch
+BASEIMAGE?=gcr.io/google_containers/nginx-slim-$(ARCH):0.20
+
+ifeq ($(ARCH),arm)
+    QEMUARCH=arm
+	GOARCH=arm
+	DUMB_ARCH=armhf
+endif
+#ifeq ($(ARCH),arm64)
+#        QEMUARCH=aarch64
+#endif
+ifeq ($(ARCH),ppc64le)
+	QEMUARCH=ppc64le
+	GOARCH=ppc64le
+	DUMB_ARCH=ppc64el
+endif
+#ifeq ($(ARCH),s390x)
+#        QEMUARCH=s390x
+#endif
+
+TEMP_DIR := $(shell mktemp -d)
+
+all: all-container
+
+sub-container-%:
+	$(MAKE) ARCH=$* build container
+
+sub-push-%:
+	$(MAKE) ARCH=$* push
+
+all-container: $(addprefix sub-container-,$(ALL_ARCH))
+
+all-push: $(addprefix sub-push-,$(ALL_ARCH))
+
+container: .container-$(ARCH)
+.container-$(ARCH):
+	cp -r ./* $(TEMP_DIR)
+	cd $(TEMP_DIR) && sed -i 's|BASEIMAGE|$(BASEIMAGE)|g' rootfs/Dockerfile
+	cd $(TEMP_DIR) && sed -i "s|QEMUARCH|$(QEMUARCH)|g" rootfs/Dockerfile
+	cd $(TEMP_DIR) && sed -i "s|DUMB_ARCH|$(DUMB_ARCH)|g" rootfs/Dockerfile
+
+ifeq ($(ARCH),amd64)
+	# When building "normally" for amd64, remove the whole line, it has no part in the amd64 image
+	cd $(TEMP_DIR) && sed -i "/CROSS_BUILD_/d" rootfs/Dockerfile
+else
+	# When cross-building, only the placeholder "CROSS_BUILD_" should be removed
+	# Register /usr/bin/qemu-ARCH-static as the handler for ARM binaries in the kernel
+	$(DOCKER) run --rm --privileged multiarch/qemu-user-static:register --reset
+	curl -sSL https://github.com/multiarch/qemu-user-static/releases/download/$(QEMUVERSION)/x86_64_qemu-$(QEMUARCH)-static.tar.gz | tar -xz -C $(TEMP_DIR)/rootfs
+	cd $(TEMP_DIR) && sed -i "s/CROSS_BUILD_//g" rootfs/Dockerfile
+endif
+
+	$(DOCKER) build -t $(MULTI_ARCH_IMG):$(TAG) $(TEMP_DIR)/rootfs
+
+ifeq ($(ARCH), amd64)
+	# This is for to maintain the backward compatibility
+	$(DOCKER) tag $(MULTI_ARCH_IMG):$(TAG) $(IMAGE):$(TAG)
+endif
+
+push: .push-$(ARCH)
+.push-$(ARCH): .container-$(ARCH)
+	$(DOCKER) push $(MULTI_ARCH_IMG):$(TAG)
+ifeq ($(ARCH), amd64)
+	$(DOCKER) push $(IMAGE):$(TAG)
+endif
+
+clean: 
+	$(DOCKER) rmi -f $(MULTI_ARCH_IMG):$(TAG) || true
+
 build: clean
-	CGO_ENABLED=0 GOOS=${GOOS} go build -a -installsuffix cgo \
+	CGO_ENABLED=0 GOOS=${GOOS} GOARCH=${GOARCH} go build -a -installsuffix cgo \
-		-ldflags "-s -w -X ${PKG}/pkg/version.RELEASE=${RELEASE} -X ${PKG}/pkg/version.COMMIT=${COMMIT} -X ${PKG}/pkg/version.REPO=${REPO_INFO}" \
+		-ldflags "-s -w -X ${PKG}/pkg/version.RELEASE=${TAG} -X ${PKG}/pkg/version.COMMIT=${COMMIT} -X ${PKG}/pkg/version.REPO=${REPO_INFO}" \
-		-o rootfs/nginx-ingress-controller ${PKG}/pkg/cmd/controller
+		-o $(TEMP_DIR)/rootfs/nginx-ingress-controller ${PKG}/pkg/cmd/controller
-
-container: build
-	$(DOCKER) build --pull -t $(PREFIX):$(RELEASE) rootfs
-
-push: container
-	$(DOCKER) push $(PREFIX):$(RELEASE)
 
 fmt:
 	@echo "+ $@"
@@ -48,6 +123,3 @@ cover:
 vet:
 	@echo "+ $@"
 	@go vet $(shell go list ${PKG}/... | grep -v vendor)
-
-clean:
-	rm -f rootfs/nginx-ingress-controller

controllers/nginx/configuration.md

@@ -349,6 +349,9 @@ log-format-upstream: '{ "time": "$time_iso8601", "remote_addr": "$proxy_protocol
 **proxy-send-timeout:** Sets the timeout in seconds for [transmitting a request to the proxied server](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_send_timeout). The timeout is set only between two successive write operations, not for the transmission of the whole request.
 
 
+**proxy-next-upstream:** Specifies in [which cases](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_next_upstream) a request should be passed to the next server.
+
+
 **retry-non-idempotent:** Since 1.9.13 NGINX will not retry non-idempotent requests (POST, LOCK, PATCH) in case of an error in the upstream server.
 
 The previous behavior can be restored using the value "true".

controllers/nginx/pkg/config/config.go

@@ -354,6 +354,7 @@ func NewDefault() Configuration {
 			ProxyBufferSize:      "4k",
 			ProxyCookieDomain:    "off",
 			ProxyCookiePath:      "off",
+			ProxyNextUpstream:    "error timeout invalid_header http_502 http_503 http_504",
 			SSLRedirect:          true,
 			CustomHTTPErrors:     []int{},
 			WhitelistSourceRange: []string{},

controllers/nginx/pkg/template/template.go

@@ -146,6 +146,7 @@ var (
 		"toUpper":                  strings.ToUpper,
 		"toLower":                  strings.ToLower,
 		"formatIP":                 formatIP,
+		"buildNextUpstream":        buildNextUpstream,
 	}
 )
 
@@ -450,3 +451,21 @@ func isSticky(host string, loc *ingress.Location, stickyLocations map[string][]s
 
 	return false
 }
+
+func buildNextUpstream(input interface{}) string {
+	nextUpstream, ok := input.(string)
+	if !ok {
+		glog.Errorf("expected an string type but %T was returned", input)
+	}
+
+	parts := strings.Split(nextUpstream, " ")
+
+	nextUpstreamCodes := make([]string, 0, len(parts))
+	for _, v := range parts {
+		if v != "" && v != "non_idempotent" {
+			nextUpstreamCodes = append(nextUpstreamCodes, v)
+		}
+	}
+
+	return strings.Join(nextUpstreamCodes, " ")
+}

controllers/nginx/rootfs/Dockerfile

@@ -12,17 +12,20 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM gcr.io/google_containers/nginx-slim-amd64:0.19
+FROM BASEIMAGE
+
+CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/
 
 RUN DEBIAN_FRONTEND=noninteractive apt-get update && apt-get install -y \
   diffutils \
   --no-install-recommends \
   && rm -rf /var/lib/apt/lists/*
 
-RUN curl -sSL -o /sbin/tini https://github.com/krallin/tini/releases/download/v0.14.0/tini-amd64 && \
+RUN curl -sSL -o /tmp/dumb-init.deb http://ftp.us.debian.org/debian/pool/main/d/dumb-init/dumb-init_1.2.0-1_DUMB_ARCH.deb && \
-  chmod +x /sbin/tini
+  dpkg -i /tmp/dumb-init.deb && \
+  rm /tmp/dumb-init.deb
 
-ENTRYPOINT ["/sbin/tini", "--"]
+ENTRYPOINT ["/usr/bin/dumb-init", "-v"]
 
 COPY . /
 

controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl

@@ -221,9 +221,6 @@ http {
     {{ range $errCode := $cfg.CustomHTTPErrors }}
     error_page {{ $errCode }} = @custom_{{ $errCode }};{{ end }}
 
-    # In case of errors try the next upstream server before returning an error
-    proxy_next_upstream                     error timeout invalid_header http_502 http_503 http_504{{ if $cfg.RetryNonIdempotent }} non_idempotent{{ end }};
-
     proxy_ssl_session_reuse on;
 
     {{ if $cfg.AllowBackendServerHeader }}
@@ -318,13 +315,6 @@ http {
         ssl_verify_depth {{ $location.CertificateAuth.ValidationDepth }};
         {{ end }}
 
-        {{ if (or $location.Redirect.ForceSSLRedirect (and (not (empty $server.SSLCertificate)) $location.Redirect.SSLRedirect)) }}
-        # enforce ssl on server side
-        if ($pass_access_scheme = http) {
-            return 301 https://$best_http_host$request_uri;
-        }
-        {{ end }}
-
         {{ if not (empty $location.Redirect.AppRoot)}}
         if ($uri = /) {
             return 302 {{ $location.Redirect.AppRoot }};
@@ -358,6 +348,14 @@ http {
 
         location {{ $path }} {
             set $proxy_upstream_name "{{ buildUpstreamName $server.Hostname $backends $location }}";
+
+            {{ if (or $location.Redirect.ForceSSLRedirect (and (not (empty $server.SSLCertificate)) $location.Redirect.SSLRedirect)) }}
+            # enforce ssl on server side
+            if ($pass_access_scheme = http) {
+                return 301 https://$best_http_host$request_uri;
+            }
+            {{ end }}
+
             {{ if isLocationAllowed $location }}
             {{ if gt (len $location.Whitelist.CIDR) 0 }}
             if ({{ buildDenyVariable (print $server.Hostname "_"  $path) }}) {
@@ -444,6 +442,9 @@ http {
             proxy_cookie_domain                     {{ $location.Proxy.CookieDomain }};
             proxy_cookie_path                       {{ $location.Proxy.CookiePath }};
 
+            # In case of errors try the next upstream server before returning an error
+            proxy_next_upstream                     {{ buildNextUpstream $location.Proxy.NextUpstream }}{{ if $cfg.RetryNonIdempotent }} non_idempotent{{ end }};
+
             {{/* rewrite only works if the content is not compressed */}}
             {{ if $location.Redirect.AddBaseURL }}
             proxy_set_header                        Accept-Encoding     "";

core/pkg/ingress/annotations/proxy/main.go

@@ -31,6 +31,7 @@ const (
 	bufferSize   = "ingress.kubernetes.io/proxy-buffer-size"
 	cookiePath   = "ingress.kubernetes.io/proxy-cookie-path"
 	cookieDomain = "ingress.kubernetes.io/proxy-cookie-domain"
+	nextUpstream = "ingress.kubernetes.io/proxy-next-upstream"
 )
 
 // Configuration returns the proxy timeout to use in the upstream server/s
@@ -42,6 +43,7 @@ type Configuration struct {
 	BufferSize     string `json:"bufferSize"`
 	CookieDomain   string `json:"cookieDomain"`
 	CookiePath     string `json:"cookiePath"`
+	NextUpstream   string `json:"nextUpstream"`
 }
 
 func (l1 *Configuration) Equal(l2 *Configuration) bool {
@@ -124,5 +126,10 @@ func (a proxy) Parse(ing *extensions.Ingress) (interface{}, error) {
 		bs = defBackend.ProxyBodySize
 	}
 
-	return &Configuration{bs, ct, st, rt, bufs, cd, cp}, nil
+	nu, err := parser.GetStringAnnotation(nextUpstream, ing)
+	if err != nil || nu == "" {
+		nu = defBackend.ProxyNextUpstream
+	}
+
+	return &Configuration{bs, ct, st, rt, bufs, cd, cp, nu}, nil
 }

core/pkg/ingress/annotations/proxy/main_test.go

@@ -73,6 +73,7 @@ func (m mockBackend) GetDefaultBackend() defaults.Backend {
 		ProxyReadTimeout:    20,
 		ProxyBufferSize:     "10k",
 		ProxyBodySize:       "3k",
+		ProxyNextUpstream:   "error",
 	}
 }
 
@@ -85,6 +86,7 @@ func TestProxy(t *testing.T) {
 	data[read] = "3"
 	data[bufferSize] = "1k"
 	data[bodySize] = "2k"
+	data[nextUpstream] = "off"
 	ing.SetAnnotations(data)
 
 	i, err := NewParser(mockBackend{}).Parse(ing)
@@ -110,6 +112,9 @@ func TestProxy(t *testing.T) {
 	if p.BodySize != "2k" {
 		t.Errorf("expected 2k as body-size but returned %v", p.BodySize)
 	}
+	if p.NextUpstream != "off" {
+		t.Errorf("expected off as next-upstream but returned %v", p.NextUpstream)
+	}
 }
 
 func TestProxyWithNoAnnotation(t *testing.T) {
@@ -141,4 +146,7 @@ func TestProxyWithNoAnnotation(t *testing.T) {
 	if p.BodySize != "3k" {
 		t.Errorf("expected 3k as body-size but returned %v", p.BodySize)
 	}
+	if p.NextUpstream != "error" {
+		t.Errorf("expected error as next-upstream but returned %v", p.NextUpstream)
+	}
 }

core/pkg/ingress/controller/controller.go

@@ -907,6 +907,7 @@ func (ic *GenericController) createServers(data []interface{},
 		BufferSize:     bdef.ProxyBufferSize,
 		CookieDomain:   bdef.ProxyCookieDomain,
 		CookiePath:     bdef.ProxyCookiePath,
+		NextUpstream:   bdef.ProxyNextUpstream,
 	}
 
 	// This adds the Default Certificate to Default Backend (or generates a new self signed one)

core/pkg/ingress/defaults/main.go

@@ -49,6 +49,10 @@ type Backend struct {
 	// http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_cookie_domain
 	ProxyCookieDomain string `json:"proxy-cookie-domain"`
 
+	// Specifies in which cases a request should be passed to the next server.
+	// http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_next_upstream
+	ProxyNextUpstream string `json:"proxy-next-upstream"`
+
 	// Name server/s used to resolve names of upstream servers into IP addresses.
 	// The file /etc/resolv.conf is used as DNS resolution configuration.
 	Resolver []net.IP

core/pkg/task/queue.go

@@ -39,7 +39,7 @@ type Queue struct {
 	// sync is called for each item in the queue
 	sync func(interface{}) error
 	// workerDone is closed when the worker exits
-	workerDone chan struct{}
+	workerDone chan bool
 
 	fn func(obj interface{}) (interface{}, error)
 }
@@ -79,7 +79,9 @@ func (t *Queue) worker() {
 	for {
 		key, quit := t.queue.Get()
 		if quit {
+			if !isClosed(t.workerDone) {
 				close(t.workerDone)
+			}
 			return
 		}
 
@@ -95,6 +97,16 @@ func (t *Queue) worker() {
 	}
 }
 
+func isClosed(ch <-chan bool) bool {
+	select {
+	case <-ch:
+		return true
+	default:
+	}
+
+	return false
+}
+
 // Shutdown shuts down the work queue and waits for the worker to ACK
 func (t *Queue) Shutdown() {
 	t.queue.ShutDown()
@@ -117,7 +129,7 @@ func NewCustomTaskQueue(syncFn func(interface{}) error, fn func(interface{}) (in
 	q := &Queue{
 		queue:      workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()),
 		sync:       syncFn,
-		workerDone: make(chan struct{}),
+		workerDone: make(chan bool),
 		fn:         fn,
 	}
 

examples/tls-termination/elb-nginx/README.md

@@ -0,0 +1,15 @@
+
+### Elastic Load Balancer for TLS termination
+
+This example shows the required steps to use classic Elastic Load Balancer for TLS termination. 
+
+Change line of the file `elb-tls-nginx-ingress-controller.yaml` replacing the dummy id with a valid one `"arn:aws:acm:us-west-2:XXXXXXXX:certificate/XXXXXX-XXXXXXX-XXXXXXX-XXXXXXXX"`
+
+Then execute:
+```
+$ kubectl create -f elb-tls-nginx-ingress-controller.yaml
+```
+
+This example creates an ELB with just two listeners, one in port 80 and another in port 443
+
+![Listeners](images/listener.png)

examples/tls-termination/elb-nginx/nginx-ingress-controller.yaml

@@ -0,0 +1,135 @@
+kind: Service
+apiVersion: v1
+metadata:
+  name: nginx-default-backend
+  labels:
+    k8s-addon: ingress-nginx.addons.k8s.io
+spec:
+  ports:
+  - port: 80
+    targetPort: http
+  selector:
+    app: nginx-default-backend
+
+---
+
+kind: Deployment
+apiVersion: extensions/v1beta1
+metadata:
+  name: nginx-default-backend
+  labels:
+    k8s-addon: ingress-nginx.addons.k8s.io
+spec:
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        k8s-addon: ingress-nginx.addons.k8s.io
+        app: nginx-default-backend
+    spec:
+      terminationGracePeriodSeconds: 60
+      containers:
+      - name: default-http-backend
+        image: gcr.io/google_containers/defaultbackend:1.0
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 8080
+            scheme: HTTP
+          initialDelaySeconds: 30
+          timeoutSeconds: 5
+        resources:
+          limits:
+            cpu: 10m
+            memory: 20Mi
+          requests:
+            cpu: 10m
+            memory: 20Mi
+        ports:
+        - name: http
+          containerPort: 8080
+          protocol: TCP
+
+---
+
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: ingress-nginx
+  labels:
+    k8s-addon: ingress-nginx.addons.k8s.io
+
+---
+
+kind: Service
+apiVersion: v1
+metadata:
+  name: ingress-nginx
+  labels:
+    k8s-addon: ingress-nginx.addons.k8s.io
+  annotations:
+    # replace with the correct value of the generated certifcate in the AWS console
+    service.beta.kubernetes.io/aws-load-balancer-ssl-cert: "arn:aws:acm:us-west-2:XXXXXXXX:certificate/XXXXXX-XXXXXXX-XXXXXXX-XXXXXXXX" 
+    # the backend instances are HTTP
+    service.beta.kubernetes.io/aws-load-balancer-backend-protocol: "http"
+    # Map port 443
+    service.beta.kubernetes.io/aws-load-balancer-ssl-ports: "https"
+
+spec:
+  type: LoadBalancer
+  selector:
+    app: ingress-nginx
+  ports:
+  - name: http
+    port: 80
+    targetPort: http
+  - name: https
+    port: 443
+    targetPort: http
+
+---
+
+kind: Deployment
+apiVersion: extensions/v1beta1
+metadata:
+  name: ingress-nginx
+  labels:
+    k8s-addon: ingress-nginx.addons.k8s.io
+spec:
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: ingress-nginx
+        k8s-addon: ingress-nginx.addons.k8s.io
+    spec:
+      terminationGracePeriodSeconds: 60
+      containers:
+      - image: quay.io/aledbf/nginx-ingress-controller:0.154
+        name: ingress-nginx
+        imagePullPolicy: Always
+        ports:
+          - name: http
+            containerPort: 80
+            protocol: TCP
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 10254
+            scheme: HTTP
+          initialDelaySeconds: 30
+          timeoutSeconds: 5
+        env:
+          - name: POD_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.name
+          - name: POD_NAMESPACE
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.namespace
+        args:
+        - /nginx-ingress-controller
+        - --default-backend-service=$(POD_NAMESPACE)/nginx-default-backend
+        - --configmap=$(POD_NAMESPACE)/ingress-nginx
+        - --publish-service=$(POD_NAMESPACE)/ingress-nginx

images/nginx-slim/Makefile

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 # 0.0.0 shouldn't clobber any released builds
-TAG = 0.19
+TAG = 0.20
 REGISTRY = gcr.io/google_containers
 ARCH ?= $(shell go env GOARCH)
 ALL_ARCH = amd64 arm ppc64le

images/nginx-slim/README.md

@@ -15,7 +15,7 @@ This image does provides a default configuration file with no backend servers.
 
 *Using docker*
 ```
-$ docker run -v /some/nginx.con:/etc/nginx/nginx.conf:ro gcr.io/google_containers/nginx-slim:0.19
+$ docker run -v /some/nginx.con:/etc/nginx/nginx.conf:ro gcr.io/google_containers/nginx-slim:0.20
 ```
 
 *Creating a replication controller*

images/nginx-slim/build.sh

@@ -17,9 +17,9 @@
 
 set -e
 
-export NGINX_VERSION=1.13.1
+export NGINX_VERSION=1.13.2
 export NDK_VERSION=0.3.0
-export VTS_VERSION=0.1.14
+export VTS_VERSION=0.1.15
 export SETMISC_VERSION=0.31
 export LUA_VERSION=0.10.8
 export STICKY_SESSIONS_VERSION=08a395c66e42
@@ -77,7 +77,7 @@ apt-get update && apt-get install --no-install-recommends -y \
   linux-headers-generic || exit 1
 
 # download, verify and extract the source files
-get_src a5856c72a6609a4dc68c88a7f3c33b79e6693343b62952e021e043fe347b6776 \
+get_src d77f234d14989d273a363f570e1d892395c006fef2ec04789be90f41a1919b70 \
         "http://nginx.org/download/nginx-$NGINX_VERSION.tar.gz"
 
 get_src 88e05a99a8a7419066f5ae75966fb1efc409bad4522d14986da074554ae61619 \
@@ -86,7 +86,7 @@ get_src 88e05a99a8a7419066f5ae75966fb1efc409bad4522d14986da074554ae61619 \
 get_src 97946a68937b50ab8637e1a90a13198fe376d801dc3e7447052e43c28e9ee7de \
         "https://github.com/openresty/set-misc-nginx-module/archive/v$SETMISC_VERSION.tar.gz"
 
-get_src e3b0018959ac899b73d3843e07351023f02be0ff421214426e3fe32193138963 \
+get_src 5112a054b1b1edb4c0042a9a840ef45f22abb3c05c68174e28ebf483164fb7e1 \
         "https://github.com/vozlt/nginx-module-vts/archive/v$VTS_VERSION.tar.gz"
 
 get_src d67449c71051b3cc2d6dd60df0ae0d21fca08aa19c9b30c5b95ee21ff38ef8dd \

images/nginx-slim/rc.yaml

@@ -29,6 +29,6 @@ spec:
     spec:
       containers:
       - name: nginxslim
-        image: gcr.io/google_containers/nginx-slim:0.19
+        image: gcr.io/google_containers/nginx-slim:0.20
         ports:
         - containerPort: 80