From 5cc5669938108ab7429bc7eee40c18a6ba18150a Mon Sep 17 00:00:00 2001 From: Manuel de Brito Fontes Date: Tue, 7 Feb 2017 15:13:08 -0300 Subject: [PATCH 1/3] Add support for custom proxy headers using a ConfigMap --- controllers/nginx/pkg/cmd/controller/nginx.go | 22 ++++++++++++++++++- controllers/nginx/pkg/config/config.go | 4 ++++ .../rootfs/etc/nginx/template/nginx.tmpl | 7 +++++- core/pkg/ingress/controller/controller.go | 8 +++++++ core/pkg/ingress/types.go | 15 +++++++++++++ 5 files changed, 54 insertions(+), 2 deletions(-) diff --git a/controllers/nginx/pkg/cmd/controller/nginx.go b/controllers/nginx/pkg/cmd/controller/nginx.go index 9e2bb64d2..685f65dc8 100644 --- a/controllers/nginx/pkg/cmd/controller/nginx.go +++ b/controllers/nginx/pkg/cmd/controller/nginx.go @@ -101,6 +101,8 @@ type NGINXController struct { configmap *api.ConfigMap + storeLister ingress.StoreLister + binary string } @@ -276,11 +278,16 @@ Error: %v return nil } -// SetConfig ... +// SetConfig sets the configured configmap func (n *NGINXController) SetConfig(cmap *api.ConfigMap) { n.configmap = cmap } +// SetListers sets the configured store listers in the generic ingress controller +func (n *NGINXController) SetListers(lister ingress.StoreLister) { + n.storeLister = lister +} + // OnUpdate is called by syncQueue in https://github.com/aledbf/ingress-controller/blob/master/pkg/ingress/controller/controller.go#L82 // periodically to keep the configuration in sync. // @@ -324,7 +331,20 @@ func (n *NGINXController) OnUpdate(ingressCfg ingress.Configuration) ([]byte, er // and we leave some room to avoid consuming all the FDs available maxOpenFiles := (sysctlFSFileMax() / cfg.WorkerProcesses) - 1024 + setHeaders := map[string]string{} + if cfg.ProxySetHeaders != "" { + cmap, exists, err := n.storeLister.ConfigMap.GetByKey(cfg.ProxySetHeaders) + if err != nil { + glog.Warningf("unexpected error reading configmap %v: %v", cfg.ProxySetHeaders, err) + } + + if exists { + setHeaders = cmap.(*api.ConfigMap).Data + } + } + return n.t.Write(config.TemplateConfig{ + ProxySetHeaders: setHeaders, MaxOpenFiles: maxOpenFiles, BacklogSize: sysctlSomaxconn(), Backends: ingressCfg.Backends, diff --git a/controllers/nginx/pkg/config/config.go b/controllers/nginx/pkg/config/config.go index c3dc11331..86971edcf 100644 --- a/controllers/nginx/pkg/config/config.go +++ b/controllers/nginx/pkg/config/config.go @@ -152,6 +152,9 @@ type Configuration struct { // of your external load balancer ProxyRealIPCIDR string `json:"proxy-real-ip-cidr,omitempty"` + // Sets the name of the configmap that contains the headers to pass to the backend + ProxySetHeaders string `json:"proxy-set-headers,omitempty"` + // Maximum size of the server names hash tables used in server names, map directive’s values, // MIME types, names of request header strings, etcd. // http://nginx.org/en/docs/hash.html @@ -283,6 +286,7 @@ func NewDefault() Configuration { // TemplateConfig contains the nginx configuration to render the file nginx.conf type TemplateConfig struct { + ProxySetHeaders map[string]string MaxOpenFiles int BacklogSize int Backends []*ingress.Backend diff --git a/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl b/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl index c4f4c497f..f55a5de75 100644 --- a/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl +++ b/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl @@ -1,4 +1,4 @@ -{{ $cfg := .Cfg }}{{ $healthzURI := .HealthzURI }}{{ $backends := .Backends }} +{{ $cfg := .Cfg }}{{ $healthzURI := .HealthzURI }}{{ $backends := .Backends }}{{ $proxyHeaders := .ProxySetHeaders }} daemon off; worker_processes {{ $cfg.WorkerProcesses }}; @@ -307,6 +307,11 @@ http { # https://www.nginx.com/blog/mitigating-the-httpoxy-vulnerability-with-nginx/ proxy_set_header Proxy ""; + # Custom headers + {{ range $k, $v := $proxyHeaders }} + proxy_set_header {{ $k }} "{{ $v }}"; + {{ end }} + proxy_connect_timeout {{ $location.Proxy.ConnectTimeout }}s; proxy_send_timeout {{ $location.Proxy.SendTimeout }}s; proxy_read_timeout {{ $location.Proxy.ReadTimeout }}s; diff --git a/core/pkg/ingress/controller/controller.go b/core/pkg/ingress/controller/controller.go index 0fecef961..d12639aed 100644 --- a/core/pkg/ingress/controller/controller.go +++ b/core/pkg/ingress/controller/controller.go @@ -304,6 +304,14 @@ func newIngressController(config *Configuration) *GenericController { ic.annotations = newAnnotationExtractor(ic) + ic.cfg.Backend.SetListers(ingress.StoreLister{ + Ingress: ic.ingLister, + Service: ic.svcLister, + Endpoint: ic.endpLister, + Secret: ic.secrLister, + ConfigMap: ic.mapLister, + }) + return &ic } diff --git a/core/pkg/ingress/types.go b/core/pkg/ingress/types.go index 4891995e7..3b7413831 100644 --- a/core/pkg/ingress/types.go +++ b/core/pkg/ingress/types.go @@ -18,8 +18,10 @@ package ingress import ( "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/client/cache" "k8s.io/kubernetes/pkg/healthz" + cache_store "k8s.io/ingress/core/pkg/cache" "k8s.io/ingress/core/pkg/ingress/annotations/auth" "k8s.io/ingress/core/pkg/ingress/annotations/authreq" "k8s.io/ingress/core/pkg/ingress/annotations/ipwhitelist" @@ -81,6 +83,9 @@ type Controller interface { OnUpdate(Configuration) ([]byte, error) // ConfigMap content of --configmap SetConfig(*api.ConfigMap) + // SetListers allows the access of store listers present in the generic controller + // This avoid the use of the kubernetes client. + SetListers(StoreLister) // BackendDefaults returns the minimum settings required to configure the // communication to endpoints BackendDefaults() defaults.Backend @@ -88,6 +93,16 @@ type Controller interface { Info() *BackendInfo } +// StoreLister returns the configured stores for ingresses, services, +// endpoints, secrets and configmaps. +type StoreLister struct { + Ingress cache_store.StoreToIngressLister + Service cache.StoreToServiceLister + Endpoint cache.StoreToEndpointsLister + Secret cache_store.StoreToSecretsLister + ConfigMap cache_store.StoreToConfigmapLister +} + // BackendInfo returns information about the backend. // This fields contains information that helps to track issues or to // map the running ingress controller to source code From 8e0985e6355060a64a8b5ddc19d286bfb1d004e6 Mon Sep 17 00:00:00 2001 From: Manuel de Brito Fontes Date: Tue, 7 Feb 2017 18:04:15 -0300 Subject: [PATCH 2/3] Add example using custom headers --- examples/README.md | 4 ++ .../custom-headers/nginx/README.md | 40 ++++++++++++++ .../custom-headers/nginx/default-backend.yaml | 51 ++++++++++++++++++ .../nginx/nginx-ingress-controller.yaml | 52 +++++++++++++++++++ 4 files changed, 147 insertions(+) create mode 100644 examples/customization/custom-headers/nginx/README.md create mode 100644 examples/customization/custom-headers/nginx/default-backend.yaml create mode 100644 examples/customization/custom-headers/nginx/nginx-ingress-controller.yaml diff --git a/examples/README.md b/examples/README.md index 01d842eb2..aa2490a9b 100644 --- a/examples/README.md +++ b/examples/README.md @@ -75,4 +75,8 @@ Name | Description | Platform | Complexity Level -----| ----------- | ---------- | ---------------- Dummy | A simple dummy controller that logs updates | * | Advanced +## Custommization +Name | Description | Platform | Complexity Level +-----| ----------- | ---------- | ---------------- +custom-headers | set custom headers before send traffic to backends | nginx | Advanced diff --git a/examples/customization/custom-headers/nginx/README.md b/examples/customization/custom-headers/nginx/README.md new file mode 100644 index 000000000..497545781 --- /dev/null +++ b/examples/customization/custom-headers/nginx/README.md @@ -0,0 +1,40 @@ +# Deploying the Nginx Ingress controller + +This example aims to demonstrate the deployment of an nginx ingress controller and +use a ConfigMap to configure a custom list of headers to be passed to the upstream +server + +## Default Backend + +The default backend is a Service capable of handling all url paths and hosts the +nginx controller doesn't understand. This most basic implementation just returns +a 404 page: + +```console +$ kubectl apply -f default-backend.yaml +deployment "default-http-backend" created +service "default-http-backend" created + +$ kubectl -n kube-system get po +NAME READY STATUS RESTARTS AGE +default-http-backend-2657704409-qgwdd 1/1 Running 0 28s +``` + +## Controller + +You can deploy the controller as follows: + +```console +$ kubectl apply -f nginx-ingress-controller.yaml +deployment "nginx-ingress-controller" created + +$ kubectl -n kube-system get po +NAME READY STATUS RESTARTS AGE +default-http-backend-2657704409-qgwdd 1/1 Running 0 2m +nginx-ingress-controller-873061567-4n3k2 1/1 Running 0 42s +``` + +Note the default settings of this controller: +* serves a `/healthz` url on port 10254, as both a liveness and readiness probe +* takes a `--default-backend-service` argument pointing to the Service created above + diff --git a/examples/customization/custom-headers/nginx/default-backend.yaml b/examples/customization/custom-headers/nginx/default-backend.yaml new file mode 100644 index 000000000..3c40989a3 --- /dev/null +++ b/examples/customization/custom-headers/nginx/default-backend.yaml @@ -0,0 +1,51 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: default-http-backend + labels: + k8s-app: default-http-backend + namespace: kube-system +spec: + replicas: 1 + template: + metadata: + labels: + k8s-app: default-http-backend + spec: + terminationGracePeriodSeconds: 60 + containers: + - name: default-http-backend + # Any image is permissable as long as: + # 1. It serves a 404 page at / + # 2. It serves 200 on a /healthz endpoint + image: gcr.io/google_containers/defaultbackend:1.0 + livenessProbe: + httpGet: + path: /healthz + port: 8080 + scheme: HTTP + initialDelaySeconds: 30 + timeoutSeconds: 5 + ports: + - containerPort: 8080 + resources: + limits: + cpu: 10m + memory: 20Mi + requests: + cpu: 10m + memory: 20Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: default-http-backend + namespace: kube-system + labels: + k8s-app: default-http-backend +spec: + ports: + - port: 80 + targetPort: 8080 + selector: + k8s-app: default-http-backend diff --git a/examples/customization/custom-headers/nginx/nginx-ingress-controller.yaml b/examples/customization/custom-headers/nginx/nginx-ingress-controller.yaml new file mode 100644 index 000000000..7ee5e797e --- /dev/null +++ b/examples/customization/custom-headers/nginx/nginx-ingress-controller.yaml @@ -0,0 +1,52 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: nginx-ingress-controller + labels: + k8s-app: nginx-ingress-controller + namespace: kube-system +spec: + replicas: 1 + template: + metadata: + labels: + k8s-app: nginx-ingress-controller + spec: + # hostNetwork makes it possible to use ipv6 and to preserve the source IP correctly regardless of docker configuration + # however, it is not a hard dependency of the nginx-ingress-controller itself and it may cause issues if port 10254 already is taken on the host + # that said, since hostPort is broken on CNI (https://github.com/kubernetes/kubernetes/issues/31307) we have to use hostNetwork where CNI is used + # like with kubeadm + # hostNetwork: true + terminationGracePeriodSeconds: 60 + containers: + - image: gcr.io/google_containers/nginx-ingress-controller:0.9.0-beta.1 + name: nginx-ingress-controller + readinessProbe: + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + livenessProbe: + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + initialDelaySeconds: 10 + timeoutSeconds: 1 + ports: + - containerPort: 80 + hostPort: 80 + - containerPort: 443 + hostPort: 443 + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + args: + - /nginx-ingress-controller + - --default-backend-service=$(POD_NAMESPACE)/default-http-backend From f7a27ebcc2c87d6b7437bfbb3058bbd114168c00 Mon Sep 17 00:00:00 2001 From: Manuel de Brito Fontes Date: Sat, 11 Feb 2017 17:19:43 -0300 Subject: [PATCH 3/3] Add example --- .../custom-headers/nginx/README.md | 42 +++++++++++++++++-- .../custom-headers/nginx/custom-headers.yaml | 9 ++++ .../nginx/nginx-ingress-controller.yaml | 1 + .../nginx/nginx-load-balancer-conf.yaml | 7 ++++ 4 files changed, 56 insertions(+), 3 deletions(-) create mode 100644 examples/customization/custom-headers/nginx/custom-headers.yaml create mode 100644 examples/customization/custom-headers/nginx/nginx-load-balancer-conf.yaml diff --git a/examples/customization/custom-headers/nginx/README.md b/examples/customization/custom-headers/nginx/README.md index 497545781..459b43b1e 100644 --- a/examples/customization/custom-headers/nginx/README.md +++ b/examples/customization/custom-headers/nginx/README.md @@ -20,6 +20,42 @@ NAME READY STATUS RESTARTS AGE default-http-backend-2657704409-qgwdd 1/1 Running 0 28s ``` +## Custom configuration + +```console +$ cat nginx-load-balancer-conf.yaml +apiVersion: v1 +data: + proxy-set-headers: "default/custom-headers" +kind: ConfigMap +metadata: + name: nginx-load-balancer-conf +``` + +```console +$ kubectl create -f nginx-load-balancer-conf.yaml +``` + +## Custom headers + +```console +$ cat custom-headers.yaml +apiVersion: v1 +data: + X-Different-Name: "true" + X-Request-Start: t=${msec} + X-Using-Nginx-Controller: "true" +kind: ConfigMap +metadata: + name: proxy-headers + namespace: default + +``` + +```console +$ kubectl create -f custom-headers.yaml +``` + ## Controller You can deploy the controller as follows: @@ -34,7 +70,7 @@ default-http-backend-2657704409-qgwdd 1/1 Running 0 2m nginx-ingress-controller-873061567-4n3k2 1/1 Running 0 42s ``` -Note the default settings of this controller: -* serves a `/healthz` url on port 10254, as both a liveness and readiness probe -* takes a `--default-backend-service` argument pointing to the Service created above +## Test +Check the contents of the configmap is present in the nginx.conf file using: +`kubectl exec nginx-ingress-controller-873061567-4n3k2 -n kube-system cat /etc/nginx/nginx.conf` diff --git a/examples/customization/custom-headers/nginx/custom-headers.yaml b/examples/customization/custom-headers/nginx/custom-headers.yaml new file mode 100644 index 000000000..beeefc8a4 --- /dev/null +++ b/examples/customization/custom-headers/nginx/custom-headers.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +data: + X-Different-Name: "true" + X-Request-Start: t=${msec} + X-Using-Nginx-Controller: "true" +kind: ConfigMap +metadata: + name: proxy-headers + namespace: kube-system diff --git a/examples/customization/custom-headers/nginx/nginx-ingress-controller.yaml b/examples/customization/custom-headers/nginx/nginx-ingress-controller.yaml index 7ee5e797e..0d3824cb2 100644 --- a/examples/customization/custom-headers/nginx/nginx-ingress-controller.yaml +++ b/examples/customization/custom-headers/nginx/nginx-ingress-controller.yaml @@ -50,3 +50,4 @@ spec: args: - /nginx-ingress-controller - --default-backend-service=$(POD_NAMESPACE)/default-http-backend + - --configmap=$(POD_NAMESPACE)/nginx-load-balancer-conf diff --git a/examples/customization/custom-headers/nginx/nginx-load-balancer-conf.yaml b/examples/customization/custom-headers/nginx/nginx-load-balancer-conf.yaml new file mode 100644 index 000000000..239918267 --- /dev/null +++ b/examples/customization/custom-headers/nginx/nginx-load-balancer-conf.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +data: + proxy-set-headers: "kube-system/custom-headers" +kind: ConfigMap +metadata: + name: nginx-load-balancer-conf + namespace: kube-system