From 9d4dfe76090d85ca36d71123de5620ed5d8cb455 Mon Sep 17 00:00:00 2001 From: Manuel de Brito Fontes Date: Fri, 16 Sep 2016 18:23:52 -0300 Subject: [PATCH] Change readiness probe for nginx ingress that works behind a CP lb --- controllers/nginx/controller.go | 45 +++++++++++-------- .../rc-custom-configuration.yaml | 13 ++++-- .../custom-errors/rc-custom-errors.yaml | 13 ++++-- .../custom-template/custom-template.yaml | 13 ++++-- .../examples/daemonset/as-daemonset.yaml | 13 ++++-- .../nginx/examples/default/rc-default.yaml | 13 ++++-- controllers/nginx/examples/full/rc-full.yaml | 13 ++++-- .../examples/proxy-protocol/nginx-rc.yaml | 13 ++++-- .../sysctl/change-proc-values-rc.yaml | 13 ++++-- controllers/nginx/examples/tcp/rc-tcp.yaml | 13 ++++-- controllers/nginx/examples/tls/rc-ssl.yaml | 13 ++++-- controllers/nginx/examples/udp/rc-udp.yaml | 13 ++++-- controllers/nginx/main.go | 5 ++- controllers/nginx/nginx.tmpl | 6 +++ controllers/nginx/nginx/config/config.go | 3 ++ controllers/nginx/rc.yaml | 13 ++++-- 16 files changed, 148 insertions(+), 67 deletions(-) diff --git a/controllers/nginx/controller.go b/controllers/nginx/controller.go index 2250dde43..1a08e8d95 100644 --- a/controllers/nginx/controller.go +++ b/controllers/nginx/controller.go @@ -89,24 +89,31 @@ func (npm namedPortMapping) getPortMappings() map[string]string { // loadBalancerController watches the kubernetes api and adds/removes services // from the loadbalancer type loadBalancerController struct { - client *client.Client - ingController *framework.Controller - endpController *framework.Controller - svcController *framework.Controller - secrController *framework.Controller - mapController *framework.Controller - ingLister StoreToIngressLister - svcLister cache.StoreToServiceLister - endpLister cache.StoreToEndpointsLister - secrLister StoreToSecretsLister - mapLister StoreToConfigmapLister - nginx *nginx.Manager - podInfo *podInfo - defaultSvc string - nxgConfigMap string - tcpConfigMap string - udpConfigMap string + client *client.Client + + ingController *framework.Controller + endpController *framework.Controller + svcController *framework.Controller + secrController *framework.Controller + mapController *framework.Controller + + ingLister StoreToIngressLister + svcLister cache.StoreToServiceLister + endpLister cache.StoreToEndpointsLister + secrLister StoreToSecretsLister + mapLister StoreToConfigmapLister + + nginx *nginx.Manager + podInfo *podInfo + + defaultSvc string + + nxgConfigMap string + tcpConfigMap string + udpConfigMap string + defSSLCertificate string + defHealthzURL string recorder record.EventRecorder @@ -127,7 +134,7 @@ type loadBalancerController struct { // newLoadBalancerController creates a controller for nginx loadbalancer func newLoadBalancerController(kubeClient *client.Client, resyncPeriod time.Duration, defaultSvc, namespace, nxgConfigMapName, tcpConfigMapName, udpConfigMapName, - defSSLCertificate string, runtimeInfo *podInfo) (*loadBalancerController, error) { + defSSLCertificate, defHealthzURL string, runtimeInfo *podInfo) (*loadBalancerController, error) { eventBroadcaster := record.NewBroadcaster() eventBroadcaster.StartLogging(glog.Infof) @@ -143,6 +150,7 @@ func newLoadBalancerController(kubeClient *client.Client, resyncPeriod time.Dura udpConfigMap: udpConfigMapName, defSSLCertificate: defSSLCertificate, defaultSvc: defaultSvc, + defHealthzURL: defHealthzURL, recorder: eventBroadcaster.NewRecorder(api.EventSource{ Component: "nginx-ingress-controller", }), @@ -450,6 +458,7 @@ func (lbc *loadBalancerController) sync(key string) error { } ngxConfig := lbc.nginx.ReadConfig(cfg) + ngxConfig.HealthzURL = lbc.defHealthzURL ings := lbc.ingLister.Store.List() upstreams, servers := lbc.getUpstreamServers(ngxConfig, ings) diff --git a/controllers/nginx/examples/custom-configuration/rc-custom-configuration.yaml b/controllers/nginx/examples/custom-configuration/rc-custom-configuration.yaml index 4e5179875..90fcd43e3 100644 --- a/controllers/nginx/examples/custom-configuration/rc-custom-configuration.yaml +++ b/controllers/nginx/examples/custom-configuration/rc-custom-configuration.yaml @@ -19,13 +19,18 @@ spec: - image: gcr.io/google_containers/nginx-ingress-controller:0.8.3 name: nginx-ingress-lb imagePullPolicy: Always + readinessProbe: + httpGet: + path: /ingress-controller-healthz + port: 80 + scheme: HTTP livenessProbe: httpGet: - path: /healthz - port: 10254 + path: /ingress-controller-healthz + port: 80 scheme: HTTP - initialDelaySeconds: 30 - timeoutSeconds: 5 + initialDelaySeconds: 10 + timeoutSeconds: 1 # use downward API env: - name: POD_NAME diff --git a/controllers/nginx/examples/custom-errors/rc-custom-errors.yaml b/controllers/nginx/examples/custom-errors/rc-custom-errors.yaml index c5c8c1183..63dfbfd35 100644 --- a/controllers/nginx/examples/custom-errors/rc-custom-errors.yaml +++ b/controllers/nginx/examples/custom-errors/rc-custom-errors.yaml @@ -19,13 +19,18 @@ spec: - image: gcr.io/google_containers/nginx-ingress-controller:0.8.3 name: nginx-ingress-lb imagePullPolicy: Always + readinessProbe: + httpGet: + path: /ingress-controller-healthz + port: 80 + scheme: HTTP livenessProbe: httpGet: - path: /healthz - port: 10254 + path: /ingress-controller-healthz + port: 80 scheme: HTTP - initialDelaySeconds: 30 - timeoutSeconds: 5 + initialDelaySeconds: 10 + timeoutSeconds: 1 # use downward API env: - name: POD_NAME diff --git a/controllers/nginx/examples/custom-template/custom-template.yaml b/controllers/nginx/examples/custom-template/custom-template.yaml index 020c95a28..ccf2809a5 100644 --- a/controllers/nginx/examples/custom-template/custom-template.yaml +++ b/controllers/nginx/examples/custom-template/custom-template.yaml @@ -19,13 +19,18 @@ spec: - image: gcr.io/google_containers/nginx-ingress-controller:0.8.3 name: nginx-ingress-lb imagePullPolicy: Always + readinessProbe: + httpGet: + path: /ingress-controller-healthz + port: 80 + scheme: HTTP livenessProbe: httpGet: - path: /healthz - port: 10254 + path: /ingress-controller-healthz + port: 80 scheme: HTTP - initialDelaySeconds: 30 - timeoutSeconds: 5 + initialDelaySeconds: 10 + timeoutSeconds: 1 # use downward API env: - name: POD_NAME diff --git a/controllers/nginx/examples/daemonset/as-daemonset.yaml b/controllers/nginx/examples/daemonset/as-daemonset.yaml index 220cde04f..c48ef5db4 100644 --- a/controllers/nginx/examples/daemonset/as-daemonset.yaml +++ b/controllers/nginx/examples/daemonset/as-daemonset.yaml @@ -13,13 +13,18 @@ spec: - image: gcr.io/google_containers/nginx-ingress-controller:0.8.3 name: nginx-ingress-lb imagePullPolicy: Always + readinessProbe: + httpGet: + path: /ingress-controller-healthz + port: 80 + scheme: HTTP livenessProbe: httpGet: - path: /healthz - port: 10254 + path: /ingress-controller-healthz + port: 80 scheme: HTTP - initialDelaySeconds: 30 - timeoutSeconds: 5 + initialDelaySeconds: 10 + timeoutSeconds: 1 # use downward API env: - name: POD_NAME diff --git a/controllers/nginx/examples/default/rc-default.yaml b/controllers/nginx/examples/default/rc-default.yaml index a79780c8d..0f1a28ea4 100644 --- a/controllers/nginx/examples/default/rc-default.yaml +++ b/controllers/nginx/examples/default/rc-default.yaml @@ -19,13 +19,18 @@ spec: - image: gcr.io/google_containers/nginx-ingress-controller:0.8.3 name: nginx-ingress-lb imagePullPolicy: Always + readinessProbe: + httpGet: + path: /ingress-controller-healthz + port: 80 + scheme: HTTP livenessProbe: httpGet: - path: /healthz - port: 10254 + path: /ingress-controller-healthz + port: 80 scheme: HTTP - initialDelaySeconds: 30 - timeoutSeconds: 5 + initialDelaySeconds: 10 + timeoutSeconds: 1 # use downward API env: - name: POD_NAME diff --git a/controllers/nginx/examples/full/rc-full.yaml b/controllers/nginx/examples/full/rc-full.yaml index 6ee5705df..71c1bd29f 100644 --- a/controllers/nginx/examples/full/rc-full.yaml +++ b/controllers/nginx/examples/full/rc-full.yaml @@ -24,13 +24,18 @@ spec: - image: gcr.io/google_containers/nginx-ingress-controller:0.8.3 name: nginx-ingress-lb imagePullPolicy: Always + readinessProbe: + httpGet: + path: /ingress-controller-healthz + port: 80 + scheme: HTTP livenessProbe: httpGet: - path: /healthz - port: 10254 + path: /ingress-controller-healthz + port: 80 scheme: HTTP - initialDelaySeconds: 30 - timeoutSeconds: 5 + initialDelaySeconds: 10 + timeoutSeconds: 1 # use downward API env: - name: POD_NAME diff --git a/controllers/nginx/examples/proxy-protocol/nginx-rc.yaml b/controllers/nginx/examples/proxy-protocol/nginx-rc.yaml index 45263397a..f67bb6d62 100644 --- a/controllers/nginx/examples/proxy-protocol/nginx-rc.yaml +++ b/controllers/nginx/examples/proxy-protocol/nginx-rc.yaml @@ -19,13 +19,18 @@ spec: - image: gcr.io/google_containers/nginx-ingress-controller:0.8.3 name: nginx-ingress-lb imagePullPolicy: Always + readinessProbe: + httpGet: + path: /ingress-controller-healthz + port: 80 + scheme: HTTP livenessProbe: httpGet: - path: /healthz - port: 10254 + path: /ingress-controller-healthz + port: 80 scheme: HTTP - initialDelaySeconds: 30 - timeoutSeconds: 5 + initialDelaySeconds: 10 + timeoutSeconds: 1 # use downward API env: - name: POD_NAME diff --git a/controllers/nginx/examples/sysctl/change-proc-values-rc.yaml b/controllers/nginx/examples/sysctl/change-proc-values-rc.yaml index e85c59336..a4c010ea1 100644 --- a/controllers/nginx/examples/sysctl/change-proc-values-rc.yaml +++ b/controllers/nginx/examples/sysctl/change-proc-values-rc.yaml @@ -92,13 +92,18 @@ spec: - image: gcr.io/google_containers/nginx-ingress-controller:0.8.3 name: nginx-ingress-lb imagePullPolicy: Always + readinessProbe: + httpGet: + path: /ingress-controller-healthz + port: 80 + scheme: HTTP livenessProbe: httpGet: - path: /healthz - port: 10254 + path: /ingress-controller-healthz + port: 80 scheme: HTTP - initialDelaySeconds: 30 - timeoutSeconds: 5 + initialDelaySeconds: 10 + timeoutSeconds: 1 # use downward API env: - name: POD_NAME diff --git a/controllers/nginx/examples/tcp/rc-tcp.yaml b/controllers/nginx/examples/tcp/rc-tcp.yaml index 32377e316..76560c55f 100644 --- a/controllers/nginx/examples/tcp/rc-tcp.yaml +++ b/controllers/nginx/examples/tcp/rc-tcp.yaml @@ -19,13 +19,18 @@ spec: - image: gcr.io/google_containers/nginx-ingress-controller:0.8.3 name: nginx-ingress-lb imagePullPolicy: Always + readinessProbe: + httpGet: + path: /ingress-controller-healthz + port: 80 + scheme: HTTP livenessProbe: httpGet: - path: /healthz - port: 10254 + path: /ingress-controller-healthz + port: 80 scheme: HTTP - initialDelaySeconds: 30 - timeoutSeconds: 5 + initialDelaySeconds: 10 + timeoutSeconds: 1 # use downward API env: - name: POD_NAME diff --git a/controllers/nginx/examples/tls/rc-ssl.yaml b/controllers/nginx/examples/tls/rc-ssl.yaml index a79780c8d..0f1a28ea4 100644 --- a/controllers/nginx/examples/tls/rc-ssl.yaml +++ b/controllers/nginx/examples/tls/rc-ssl.yaml @@ -19,13 +19,18 @@ spec: - image: gcr.io/google_containers/nginx-ingress-controller:0.8.3 name: nginx-ingress-lb imagePullPolicy: Always + readinessProbe: + httpGet: + path: /ingress-controller-healthz + port: 80 + scheme: HTTP livenessProbe: httpGet: - path: /healthz - port: 10254 + path: /ingress-controller-healthz + port: 80 scheme: HTTP - initialDelaySeconds: 30 - timeoutSeconds: 5 + initialDelaySeconds: 10 + timeoutSeconds: 1 # use downward API env: - name: POD_NAME diff --git a/controllers/nginx/examples/udp/rc-udp.yaml b/controllers/nginx/examples/udp/rc-udp.yaml index 4ba8cef7b..34f935587 100644 --- a/controllers/nginx/examples/udp/rc-udp.yaml +++ b/controllers/nginx/examples/udp/rc-udp.yaml @@ -19,13 +19,18 @@ spec: - image: gcr.io/google_containers/nginx-ingress-controller:0.8.3 name: nginx-ingress-lb imagePullPolicy: Always + readinessProbe: + httpGet: + path: /ingress-controller-healthz + port: 80 + scheme: HTTP livenessProbe: httpGet: - path: /healthz - port: 10254 + path: /ingress-controller-healthz + port: 80 scheme: HTTP - initialDelaySeconds: 30 - timeoutSeconds: 5 + initialDelaySeconds: 10 + timeoutSeconds: 1 # use downward API env: - name: POD_NAME diff --git a/controllers/nginx/main.go b/controllers/nginx/main.go index 06943e305..5b0bc50c4 100644 --- a/controllers/nginx/main.go +++ b/controllers/nginx/main.go @@ -79,6 +79,9 @@ var ( defSSLCertificate = flags.String("default-ssl-certificate", "", `Name of the secret that contains a SSL certificate to be used as default for a HTTPS catch-all server`) + + defHealthzURL = flags.String("health-check-path", "/ingress-controller-healthz", `Defines the URL to + be used as health check inside in the default server in NGINX.`) ) func main() { @@ -121,7 +124,7 @@ func main() { lbc, err := newLoadBalancerController(kubeClient, *resyncPeriod, *defaultSvc, *watchNamespace, *nxgConfigMap, *tcpConfigMapName, - *udpConfigMapName, *defSSLCertificate, runtimePodInfo) + *udpConfigMapName, *defSSLCertificate, *defHealthzURL, runtimePodInfo) if err != nil { glog.Fatalf("%v", err) } diff --git a/controllers/nginx/nginx.tmpl b/controllers/nginx/nginx.tmpl index fb1ca360c..bd00de9af 100644 --- a/controllers/nginx/nginx.tmpl +++ b/controllers/nginx/nginx.tmpl @@ -265,6 +265,12 @@ http { {{ end }} {{ if eq $server.Name "_" }} + # health checks in cloud providers require the use of port 80 + location {{ $cfg.healthzUrl }} { + access_log off; + return 200; + } + # this is required to avoid error if nginx is being monitored # with an external software (like sysdig) location /nginx_status { diff --git a/controllers/nginx/nginx/config/config.go b/controllers/nginx/nginx/config/config.go index 684c55d91..cb9eb370a 100644 --- a/controllers/nginx/nginx/config/config.go +++ b/controllers/nginx/nginx/config/config.go @@ -78,6 +78,9 @@ type Configuration struct { // Sets the maximum allowed size of the client request body BodySize string `structs:"body-size,omitempty"` + // HealthzURL defines the URL should be used in probes + HealthzURL string + // EnableDynamicTLSRecords enables dynamic TLS record sizes // https://blog.cloudflare.com/optimizing-tls-over-tcp-to-reduce-latency // By default this is enabled diff --git a/controllers/nginx/rc.yaml b/controllers/nginx/rc.yaml index 3a8e9a5af..2b979385d 100644 --- a/controllers/nginx/rc.yaml +++ b/controllers/nginx/rc.yaml @@ -71,13 +71,18 @@ spec: - image: gcr.io/google_containers/nginx-ingress-controller:0.8.3 name: nginx-ingress-lb imagePullPolicy: Always + readinessProbe: + httpGet: + path: /ingress-controller-healthz + port: 80 + scheme: HTTP livenessProbe: httpGet: - path: /healthz - port: 10254 + path: /ingress-controller-healthz + port: 80 scheme: HTTP - initialDelaySeconds: 30 - timeoutSeconds: 5 + initialDelaySeconds: 10 + timeoutSeconds: 1 # use downward API env: - name: POD_NAME