Merge pull request #1749 from aledbf/cloud-health-check

Automatic merge from submit-queue [nginx-ingress-controller] Readiness probe that works behind a CP lb fixes #1507
2016-09-26 15:44:12 -07:00 · 2016-09-26 15:44:12 -07:00 · 741ac710cd
commit 741ac710cd
parent d90ceb7f3c 9d4dfe7609
16 changed files with 148 additions and 67 deletions
--- a/controllers/nginx/controller.go
+++ b/controllers/nginx/controller.go
@ -89,24 +89,31 @@ func (npm namedPortMapping) getPortMappings() map[string]string {
 // loadBalancerController watches the kubernetes api and adds/removes services
 // from the loadbalancer
 type loadBalancerController struct {
-	client            *client.Client
-	ingController     *framework.Controller
-	endpController    *framework.Controller
-	svcController     *framework.Controller
-	secrController    *framework.Controller
-	mapController     *framework.Controller
-	ingLister         StoreToIngressLister
-	svcLister         cache.StoreToServiceLister
-	endpLister        cache.StoreToEndpointsLister
-	secrLister        StoreToSecretsLister
-	mapLister         StoreToConfigmapLister
-	nginx             *nginx.Manager
-	podInfo           *podInfo
-	defaultSvc        string
-	nxgConfigMap      string
-	tcpConfigMap      string
-	udpConfigMap      string
+	client *client.Client
+
+	ingController  *framework.Controller
+	endpController *framework.Controller
+	svcController  *framework.Controller
+	secrController *framework.Controller
+	mapController  *framework.Controller
+
+	ingLister  StoreToIngressLister
+	svcLister  cache.StoreToServiceLister
+	endpLister cache.StoreToEndpointsLister
+	secrLister StoreToSecretsLister
+	mapLister  StoreToConfigmapLister
+
+	nginx   *nginx.Manager
+	podInfo *podInfo
+
+	defaultSvc string
+
+	nxgConfigMap string
+	tcpConfigMap string
+	udpConfigMap string
+
 	defSSLCertificate string
+	defHealthzURL     string

 	recorder record.EventRecorder

@ -127,7 +134,7 @@ type loadBalancerController struct {
 // newLoadBalancerController creates a controller for nginx loadbalancer
 func newLoadBalancerController(kubeClient *client.Client, resyncPeriod time.Duration,
 	defaultSvc, namespace, nxgConfigMapName, tcpConfigMapName, udpConfigMapName,
-	defSSLCertificate string, runtimeInfo *podInfo) (*loadBalancerController, error) {
+	defSSLCertificate, defHealthzURL string, runtimeInfo *podInfo) (*loadBalancerController, error) {

 	eventBroadcaster := record.NewBroadcaster()
 	eventBroadcaster.StartLogging(glog.Infof)
@ -143,6 +150,7 @@ func newLoadBalancerController(kubeClient *client.Client, resyncPeriod time.Dura
 		udpConfigMap:      udpConfigMapName,
 		defSSLCertificate: defSSLCertificate,
 		defaultSvc:        defaultSvc,
+		defHealthzURL:     defHealthzURL,
 		recorder: eventBroadcaster.NewRecorder(api.EventSource{
 			Component: "nginx-ingress-controller",
 		}),
@ -450,6 +458,7 @@ func (lbc *loadBalancerController) sync(key string) error {
 	}

 	ngxConfig := lbc.nginx.ReadConfig(cfg)
+	ngxConfig.HealthzURL = lbc.defHealthzURL

 	ings := lbc.ingLister.Store.List()
 	upstreams, servers := lbc.getUpstreamServers(ngxConfig, ings)
--- a/controllers/nginx/examples/custom-configuration/rc-custom-configuration.yaml
+++ b/controllers/nginx/examples/custom-configuration/rc-custom-configuration.yaml
@ -19,13 +19,18 @@ spec:
      - image: gcr.io/google_containers/nginx-ingress-controller:0.8.3
        name: nginx-ingress-lb
        imagePullPolicy: Always
+        readinessProbe:
+          httpGet:
+            path: /ingress-controller-healthz
+            port: 80
+            scheme: HTTP
        livenessProbe:
          httpGet:
-            path: /healthz
-            port: 10254
+            path: /ingress-controller-healthz
+            port: 80
            scheme: HTTP
-          initialDelaySeconds: 30
-          timeoutSeconds: 5
+          initialDelaySeconds: 10
+          timeoutSeconds: 1
        # use downward API
        env:
          - name: POD_NAME
--- a/controllers/nginx/examples/custom-errors/rc-custom-errors.yaml
+++ b/controllers/nginx/examples/custom-errors/rc-custom-errors.yaml
@ -19,13 +19,18 @@ spec:
      - image: gcr.io/google_containers/nginx-ingress-controller:0.8.3
        name: nginx-ingress-lb
        imagePullPolicy: Always
+        readinessProbe:
+          httpGet:
+            path: /ingress-controller-healthz
+            port: 80
+            scheme: HTTP
        livenessProbe:
          httpGet:
-            path: /healthz
-            port: 10254
+            path: /ingress-controller-healthz
+            port: 80
            scheme: HTTP
-          initialDelaySeconds: 30
-          timeoutSeconds: 5
+          initialDelaySeconds: 10
+          timeoutSeconds: 1
        # use downward API
        env:
          - name: POD_NAME
--- a/controllers/nginx/examples/custom-template/custom-template.yaml
+++ b/controllers/nginx/examples/custom-template/custom-template.yaml
@ -19,13 +19,18 @@ spec:
      - image: gcr.io/google_containers/nginx-ingress-controller:0.8.3
        name: nginx-ingress-lb
        imagePullPolicy: Always
+        readinessProbe:
+          httpGet:
+            path: /ingress-controller-healthz
+            port: 80
+            scheme: HTTP
        livenessProbe:
          httpGet:
-            path: /healthz
-            port: 10254
+            path: /ingress-controller-healthz
+            port: 80
            scheme: HTTP
-          initialDelaySeconds: 30
-          timeoutSeconds: 5
+          initialDelaySeconds: 10
+          timeoutSeconds: 1
        # use downward API
        env:
          - name: POD_NAME
--- a/controllers/nginx/examples/daemonset/as-daemonset.yaml
+++ b/controllers/nginx/examples/daemonset/as-daemonset.yaml
@ -13,13 +13,18 @@ spec:
      - image: gcr.io/google_containers/nginx-ingress-controller:0.8.3
        name: nginx-ingress-lb
        imagePullPolicy: Always
+        readinessProbe:
+          httpGet:
+            path: /ingress-controller-healthz
+            port: 80
+            scheme: HTTP
        livenessProbe:
          httpGet:
-            path: /healthz
-            port: 10254
+            path: /ingress-controller-healthz
+            port: 80
            scheme: HTTP
-          initialDelaySeconds: 30
-          timeoutSeconds: 5
+          initialDelaySeconds: 10
+          timeoutSeconds: 1
        # use downward API
        env:
          - name: POD_NAME
--- a/controllers/nginx/examples/default/rc-default.yaml
+++ b/controllers/nginx/examples/default/rc-default.yaml
@ -19,13 +19,18 @@ spec:
      - image: gcr.io/google_containers/nginx-ingress-controller:0.8.3
        name: nginx-ingress-lb
        imagePullPolicy: Always
+        readinessProbe:
+          httpGet:
+            path: /ingress-controller-healthz
+            port: 80
+            scheme: HTTP
        livenessProbe:
          httpGet:
-            path: /healthz
-            port: 10254
+            path: /ingress-controller-healthz
+            port: 80
            scheme: HTTP
-          initialDelaySeconds: 30
-          timeoutSeconds: 5
+          initialDelaySeconds: 10
+          timeoutSeconds: 1
        # use downward API
        env:
          - name: POD_NAME
--- a/controllers/nginx/examples/full/rc-full.yaml
+++ b/controllers/nginx/examples/full/rc-full.yaml
@ -24,13 +24,18 @@ spec:
      - image: gcr.io/google_containers/nginx-ingress-controller:0.8.3
        name: nginx-ingress-lb
        imagePullPolicy: Always
+        readinessProbe:
+          httpGet:
+            path: /ingress-controller-healthz
+            port: 80
+            scheme: HTTP
        livenessProbe:
          httpGet:
-            path: /healthz
-            port: 10254
+            path: /ingress-controller-healthz
+            port: 80
            scheme: HTTP
-          initialDelaySeconds: 30
-          timeoutSeconds: 5
+          initialDelaySeconds: 10
+          timeoutSeconds: 1
        # use downward API
        env:
          - name: POD_NAME
--- a/controllers/nginx/examples/proxy-protocol/nginx-rc.yaml
+++ b/controllers/nginx/examples/proxy-protocol/nginx-rc.yaml
@ -19,13 +19,18 @@ spec:
      - image: gcr.io/google_containers/nginx-ingress-controller:0.8.3
        name: nginx-ingress-lb
        imagePullPolicy: Always
+        readinessProbe:
+          httpGet:
+            path: /ingress-controller-healthz
+            port: 80
+            scheme: HTTP
        livenessProbe:
          httpGet:
-            path: /healthz
-            port: 10254
+            path: /ingress-controller-healthz
+            port: 80
            scheme: HTTP
-          initialDelaySeconds: 30
-          timeoutSeconds: 5
+          initialDelaySeconds: 10
+          timeoutSeconds: 1
        # use downward API
        env:
          - name: POD_NAME
--- a/controllers/nginx/examples/sysctl/change-proc-values-rc.yaml
+++ b/controllers/nginx/examples/sysctl/change-proc-values-rc.yaml
@ -92,13 +92,18 @@ spec:
      - image: gcr.io/google_containers/nginx-ingress-controller:0.8.3
        name: nginx-ingress-lb
        imagePullPolicy: Always
+        readinessProbe:
+          httpGet:
+            path: /ingress-controller-healthz
+            port: 80
+            scheme: HTTP
        livenessProbe:
          httpGet:
-            path: /healthz
-            port: 10254
+            path: /ingress-controller-healthz
+            port: 80
            scheme: HTTP
-          initialDelaySeconds: 30
-          timeoutSeconds: 5
+          initialDelaySeconds: 10
+          timeoutSeconds: 1
        # use downward API
        env:
          - name: POD_NAME
--- a/controllers/nginx/examples/tcp/rc-tcp.yaml
+++ b/controllers/nginx/examples/tcp/rc-tcp.yaml
@ -19,13 +19,18 @@ spec:
      - image: gcr.io/google_containers/nginx-ingress-controller:0.8.3
        name: nginx-ingress-lb
        imagePullPolicy: Always
+        readinessProbe:
+          httpGet:
+            path: /ingress-controller-healthz
+            port: 80
+            scheme: HTTP
        livenessProbe:
          httpGet:
-            path: /healthz
-            port: 10254
+            path: /ingress-controller-healthz
+            port: 80
            scheme: HTTP
-          initialDelaySeconds: 30
-          timeoutSeconds: 5
+          initialDelaySeconds: 10
+          timeoutSeconds: 1
        # use downward API
        env:
          - name: POD_NAME
--- a/controllers/nginx/examples/tls/rc-ssl.yaml
+++ b/controllers/nginx/examples/tls/rc-ssl.yaml
@ -19,13 +19,18 @@ spec:
      - image: gcr.io/google_containers/nginx-ingress-controller:0.8.3
        name: nginx-ingress-lb
        imagePullPolicy: Always
+        readinessProbe:
+          httpGet:
+            path: /ingress-controller-healthz
+            port: 80
+            scheme: HTTP
        livenessProbe:
          httpGet:
-            path: /healthz
-            port: 10254
+            path: /ingress-controller-healthz
+            port: 80
            scheme: HTTP
-          initialDelaySeconds: 30
-          timeoutSeconds: 5
+          initialDelaySeconds: 10
+          timeoutSeconds: 1
        # use downward API
        env:
          - name: POD_NAME
--- a/controllers/nginx/examples/udp/rc-udp.yaml
+++ b/controllers/nginx/examples/udp/rc-udp.yaml
@ -19,13 +19,18 @@ spec:
      - image: gcr.io/google_containers/nginx-ingress-controller:0.8.3
        name: nginx-ingress-lb
        imagePullPolicy: Always
+        readinessProbe:
+          httpGet:
+            path: /ingress-controller-healthz
+            port: 80
+            scheme: HTTP
        livenessProbe:
          httpGet:
-            path: /healthz
-            port: 10254
+            path: /ingress-controller-healthz
+            port: 80
            scheme: HTTP
-          initialDelaySeconds: 30
-          timeoutSeconds: 5
+          initialDelaySeconds: 10
+          timeoutSeconds: 1
        # use downward API
        env:
          - name: POD_NAME
--- a/controllers/nginx/main.go
+++ b/controllers/nginx/main.go
@ -79,6 +79,9 @@ var (

 	defSSLCertificate = flags.String("default-ssl-certificate", "", `Name of the secret that contains a SSL 
 		certificate to be used as default for a HTTPS catch-all server`)
+
+	defHealthzURL = flags.String("health-check-path", "/ingress-controller-healthz", `Defines the URL to
+		be used as health check inside in the default server in NGINX.`)
 )

 func main() {
@ -121,7 +124,7 @@ func main() {

 	lbc, err := newLoadBalancerController(kubeClient, *resyncPeriod,
 		*defaultSvc, *watchNamespace, *nxgConfigMap, *tcpConfigMapName,
-		*udpConfigMapName, *defSSLCertificate, runtimePodInfo)
+		*udpConfigMapName, *defSSLCertificate, *defHealthzURL, runtimePodInfo)
 	if err != nil {
 		glog.Fatalf("%v", err)
 	}
--- a/controllers/nginx/nginx.tmpl
+++ b/controllers/nginx/nginx.tmpl
@ -267,6 +267,12 @@ http {
        {{ end }}

        {{ if eq $server.Name "_" }}
+        # health checks in cloud providers require the use of port 80
+        location {{ $cfg.healthzUrl }} {
+            access_log off;
+            return 200;
+        }
+
        # this is required to avoid error if nginx is being monitored
        # with an external software (like sysdig)
        location /nginx_status {
--- a/controllers/nginx/nginx/config/config.go
+++ b/controllers/nginx/nginx/config/config.go
@ -78,6 +78,9 @@ type Configuration struct {
 	// Sets the maximum allowed size of the client request body
 	BodySize string `structs:"body-size,omitempty"`

+	// HealthzURL defines the URL should be used in probes
+	HealthzURL string
+
 	// EnableDynamicTLSRecords enables dynamic TLS record sizes
 	// https://blog.cloudflare.com/optimizing-tls-over-tcp-to-reduce-latency
 	// By default this is enabled
--- a/controllers/nginx/rc.yaml
+++ b/controllers/nginx/rc.yaml
@ -71,13 +71,18 @@ spec:
      - image: gcr.io/google_containers/nginx-ingress-controller:0.8.3
        name: nginx-ingress-lb
        imagePullPolicy: Always
+        readinessProbe:
+          httpGet:
+            path: /ingress-controller-healthz
+            port: 80
+            scheme: HTTP
        livenessProbe:
          httpGet:
-            path: /healthz
-            port: 10254
+            path: /ingress-controller-healthz
+            port: 80
            scheme: HTTP
-          initialDelaySeconds: 30
-          timeoutSeconds: 5
+          initialDelaySeconds: 10
+          timeoutSeconds: 1
        # use downward API
        env:
          - name: POD_NAME