Merge pull request #1749 from aledbf/cloud-health-check

Automatic merge from submit-queue

[nginx-ingress-controller] Readiness probe that works behind a CP lb

fixes #1507

controllers/nginx/controller.go

@@ -89,24 +89,31 @@ func (npm namedPortMapping) getPortMappings() map[string]string {
 // loadBalancerController watches the kubernetes api and adds/removes services
 // from the loadbalancer
 type loadBalancerController struct {
-	client            *client.Client
+	client *client.Client
-	ingController     *framework.Controller
+
-	endpController    *framework.Controller
+	ingController  *framework.Controller
-	svcController     *framework.Controller
+	endpController *framework.Controller
-	secrController    *framework.Controller
+	svcController  *framework.Controller
-	mapController     *framework.Controller
+	secrController *framework.Controller
-	ingLister         StoreToIngressLister
+	mapController  *framework.Controller
-	svcLister         cache.StoreToServiceLister
+
-	endpLister        cache.StoreToEndpointsLister
+	ingLister  StoreToIngressLister
-	secrLister        StoreToSecretsLister
+	svcLister  cache.StoreToServiceLister
-	mapLister         StoreToConfigmapLister
+	endpLister cache.StoreToEndpointsLister
-	nginx             *nginx.Manager
+	secrLister StoreToSecretsLister
-	podInfo           *podInfo
+	mapLister  StoreToConfigmapLister
-	defaultSvc        string
+
-	nxgConfigMap      string
+	nginx   *nginx.Manager
-	tcpConfigMap      string
+	podInfo *podInfo
-	udpConfigMap      string
+
+	defaultSvc string
+
+	nxgConfigMap string
+	tcpConfigMap string
+	udpConfigMap string
+
 	defSSLCertificate string
+	defHealthzURL     string
 
 	recorder record.EventRecorder
 
@@ -127,7 +134,7 @@ type loadBalancerController struct {
 // newLoadBalancerController creates a controller for nginx loadbalancer
 func newLoadBalancerController(kubeClient *client.Client, resyncPeriod time.Duration,
 	defaultSvc, namespace, nxgConfigMapName, tcpConfigMapName, udpConfigMapName,
-	defSSLCertificate string, runtimeInfo *podInfo) (*loadBalancerController, error) {
+	defSSLCertificate, defHealthzURL string, runtimeInfo *podInfo) (*loadBalancerController, error) {
 
 	eventBroadcaster := record.NewBroadcaster()
 	eventBroadcaster.StartLogging(glog.Infof)
@@ -143,6 +150,7 @@ func newLoadBalancerController(kubeClient *client.Client, resyncPeriod time.Dura
 		udpConfigMap:      udpConfigMapName,
 		defSSLCertificate: defSSLCertificate,
 		defaultSvc:        defaultSvc,
+		defHealthzURL:     defHealthzURL,
 		recorder: eventBroadcaster.NewRecorder(api.EventSource{
 			Component: "nginx-ingress-controller",
 		}),
@@ -450,6 +458,7 @@ func (lbc *loadBalancerController) sync(key string) error {
 	}
 
 	ngxConfig := lbc.nginx.ReadConfig(cfg)
+	ngxConfig.HealthzURL = lbc.defHealthzURL
 
 	ings := lbc.ingLister.Store.List()
 	upstreams, servers := lbc.getUpstreamServers(ngxConfig, ings)

controllers/nginx/examples/custom-configuration/rc-custom-configuration.yaml

@@ -19,13 +19,18 @@ spec:
       - image: gcr.io/google_containers/nginx-ingress-controller:0.8.3
         name: nginx-ingress-lb
         imagePullPolicy: Always
+        readinessProbe:
+          httpGet:
+            path: /ingress-controller-healthz
+            port: 80
+            scheme: HTTP
         livenessProbe:
           httpGet:
-            path: /healthz
+            path: /ingress-controller-healthz
-            port: 10254
+            port: 80
             scheme: HTTP
-          initialDelaySeconds: 30
+          initialDelaySeconds: 10
-          timeoutSeconds: 5
+          timeoutSeconds: 1
         # use downward API
         env:
           - name: POD_NAME

controllers/nginx/examples/custom-errors/rc-custom-errors.yaml

@@ -19,13 +19,18 @@ spec:
       - image: gcr.io/google_containers/nginx-ingress-controller:0.8.3
         name: nginx-ingress-lb
         imagePullPolicy: Always
+        readinessProbe:
+          httpGet:
+            path: /ingress-controller-healthz
+            port: 80
+            scheme: HTTP
         livenessProbe:
           httpGet:
-            path: /healthz
+            path: /ingress-controller-healthz
-            port: 10254
+            port: 80
             scheme: HTTP
-          initialDelaySeconds: 30
+          initialDelaySeconds: 10
-          timeoutSeconds: 5
+          timeoutSeconds: 1
         # use downward API
         env:
           - name: POD_NAME

controllers/nginx/examples/custom-template/custom-template.yaml

@@ -19,13 +19,18 @@ spec:
       - image: gcr.io/google_containers/nginx-ingress-controller:0.8.3
         name: nginx-ingress-lb
         imagePullPolicy: Always
+        readinessProbe:
+          httpGet:
+            path: /ingress-controller-healthz
+            port: 80
+            scheme: HTTP
         livenessProbe:
           httpGet:
-            path: /healthz
+            path: /ingress-controller-healthz
-            port: 10254
+            port: 80
             scheme: HTTP
-          initialDelaySeconds: 30
+          initialDelaySeconds: 10
-          timeoutSeconds: 5
+          timeoutSeconds: 1
         # use downward API
         env:
           - name: POD_NAME

controllers/nginx/examples/daemonset/as-daemonset.yaml

@@ -13,13 +13,18 @@ spec:
       - image: gcr.io/google_containers/nginx-ingress-controller:0.8.3
         name: nginx-ingress-lb
         imagePullPolicy: Always
+        readinessProbe:
+          httpGet:
+            path: /ingress-controller-healthz
+            port: 80
+            scheme: HTTP
         livenessProbe:
           httpGet:
-            path: /healthz
+            path: /ingress-controller-healthz
-            port: 10254
+            port: 80
             scheme: HTTP
-          initialDelaySeconds: 30
+          initialDelaySeconds: 10
-          timeoutSeconds: 5
+          timeoutSeconds: 1
         # use downward API
         env:
           - name: POD_NAME

controllers/nginx/examples/default/rc-default.yaml

@@ -19,13 +19,18 @@ spec:
       - image: gcr.io/google_containers/nginx-ingress-controller:0.8.3
         name: nginx-ingress-lb
         imagePullPolicy: Always
+        readinessProbe:
+          httpGet:
+            path: /ingress-controller-healthz
+            port: 80
+            scheme: HTTP
         livenessProbe:
           httpGet:
-            path: /healthz
+            path: /ingress-controller-healthz
-            port: 10254
+            port: 80
             scheme: HTTP
-          initialDelaySeconds: 30
+          initialDelaySeconds: 10
-          timeoutSeconds: 5
+          timeoutSeconds: 1
         # use downward API
         env:
           - name: POD_NAME

controllers/nginx/examples/full/rc-full.yaml

@@ -24,13 +24,18 @@ spec:
       - image: gcr.io/google_containers/nginx-ingress-controller:0.8.3
         name: nginx-ingress-lb
         imagePullPolicy: Always
+        readinessProbe:
+          httpGet:
+            path: /ingress-controller-healthz
+            port: 80
+            scheme: HTTP
         livenessProbe:
           httpGet:
-            path: /healthz
+            path: /ingress-controller-healthz
-            port: 10254
+            port: 80
             scheme: HTTP
-          initialDelaySeconds: 30
+          initialDelaySeconds: 10
-          timeoutSeconds: 5
+          timeoutSeconds: 1
         # use downward API
         env:
           - name: POD_NAME

controllers/nginx/examples/proxy-protocol/nginx-rc.yaml

@@ -19,13 +19,18 @@ spec:
       - image: gcr.io/google_containers/nginx-ingress-controller:0.8.3
         name: nginx-ingress-lb
         imagePullPolicy: Always
+        readinessProbe:
+          httpGet:
+            path: /ingress-controller-healthz
+            port: 80
+            scheme: HTTP
         livenessProbe:
           httpGet:
-            path: /healthz
+            path: /ingress-controller-healthz
-            port: 10254
+            port: 80
             scheme: HTTP
-          initialDelaySeconds: 30
+          initialDelaySeconds: 10
-          timeoutSeconds: 5
+          timeoutSeconds: 1
         # use downward API
         env:
           - name: POD_NAME

controllers/nginx/examples/sysctl/change-proc-values-rc.yaml

@@ -92,13 +92,18 @@ spec:
       - image: gcr.io/google_containers/nginx-ingress-controller:0.8.3
         name: nginx-ingress-lb
         imagePullPolicy: Always
+        readinessProbe:
+          httpGet:
+            path: /ingress-controller-healthz
+            port: 80
+            scheme: HTTP
         livenessProbe:
           httpGet:
-            path: /healthz
+            path: /ingress-controller-healthz
-            port: 10254
+            port: 80
             scheme: HTTP
-          initialDelaySeconds: 30
+          initialDelaySeconds: 10
-          timeoutSeconds: 5
+          timeoutSeconds: 1
         # use downward API
         env:
           - name: POD_NAME

controllers/nginx/examples/tcp/rc-tcp.yaml

@@ -19,13 +19,18 @@ spec:
       - image: gcr.io/google_containers/nginx-ingress-controller:0.8.3
         name: nginx-ingress-lb
         imagePullPolicy: Always
+        readinessProbe:
+          httpGet:
+            path: /ingress-controller-healthz
+            port: 80
+            scheme: HTTP
         livenessProbe:
           httpGet:
-            path: /healthz
+            path: /ingress-controller-healthz
-            port: 10254
+            port: 80
             scheme: HTTP
-          initialDelaySeconds: 30
+          initialDelaySeconds: 10
-          timeoutSeconds: 5
+          timeoutSeconds: 1
         # use downward API
         env:
           - name: POD_NAME

controllers/nginx/examples/tls/rc-ssl.yaml

@@ -19,13 +19,18 @@ spec:
       - image: gcr.io/google_containers/nginx-ingress-controller:0.8.3
         name: nginx-ingress-lb
         imagePullPolicy: Always
+        readinessProbe:
+          httpGet:
+            path: /ingress-controller-healthz
+            port: 80
+            scheme: HTTP
         livenessProbe:
           httpGet:
-            path: /healthz
+            path: /ingress-controller-healthz
-            port: 10254
+            port: 80
             scheme: HTTP
-          initialDelaySeconds: 30
+          initialDelaySeconds: 10
-          timeoutSeconds: 5
+          timeoutSeconds: 1
         # use downward API
         env:
           - name: POD_NAME

controllers/nginx/examples/udp/rc-udp.yaml

@@ -19,13 +19,18 @@ spec:
       - image: gcr.io/google_containers/nginx-ingress-controller:0.8.3
         name: nginx-ingress-lb
         imagePullPolicy: Always
+        readinessProbe:
+          httpGet:
+            path: /ingress-controller-healthz
+            port: 80
+            scheme: HTTP
         livenessProbe:
           httpGet:
-            path: /healthz
+            path: /ingress-controller-healthz
-            port: 10254
+            port: 80
             scheme: HTTP
-          initialDelaySeconds: 30
+          initialDelaySeconds: 10
-          timeoutSeconds: 5
+          timeoutSeconds: 1
         # use downward API
         env:
           - name: POD_NAME

controllers/nginx/main.go

@@ -79,6 +79,9 @@ var (
 
 	defSSLCertificate = flags.String("default-ssl-certificate", "", `Name of the secret that contains a SSL 
 		certificate to be used as default for a HTTPS catch-all server`)
+
+	defHealthzURL = flags.String("health-check-path", "/ingress-controller-healthz", `Defines the URL to
+		be used as health check inside in the default server in NGINX.`)
 )
 
 func main() {
@@ -121,7 +124,7 @@ func main() {
 
 	lbc, err := newLoadBalancerController(kubeClient, *resyncPeriod,
 		*defaultSvc, *watchNamespace, *nxgConfigMap, *tcpConfigMapName,
-		*udpConfigMapName, *defSSLCertificate, runtimePodInfo)
+		*udpConfigMapName, *defSSLCertificate, *defHealthzURL, runtimePodInfo)
 	if err != nil {
 		glog.Fatalf("%v", err)
 	}

controllers/nginx/nginx.tmpl

@@ -267,6 +267,12 @@ http {
         {{ end }}
 
         {{ if eq $server.Name "_" }}
+        # health checks in cloud providers require the use of port 80
+        location {{ $cfg.healthzUrl }} {
+            access_log off;
+            return 200;
+        }
+
         # this is required to avoid error if nginx is being monitored
         # with an external software (like sysdig)
         location /nginx_status {

controllers/nginx/nginx/config/config.go

@@ -78,6 +78,9 @@ type Configuration struct {
 	// Sets the maximum allowed size of the client request body
 	BodySize string `structs:"body-size,omitempty"`
 
+	// HealthzURL defines the URL should be used in probes
+	HealthzURL string
+
 	// EnableDynamicTLSRecords enables dynamic TLS record sizes
 	// https://blog.cloudflare.com/optimizing-tls-over-tcp-to-reduce-latency
 	// By default this is enabled

controllers/nginx/rc.yaml

@@ -71,13 +71,18 @@ spec:
       - image: gcr.io/google_containers/nginx-ingress-controller:0.8.3
         name: nginx-ingress-lb
         imagePullPolicy: Always
+        readinessProbe:
+          httpGet:
+            path: /ingress-controller-healthz
+            port: 80
+            scheme: HTTP
         livenessProbe:
           httpGet:
-            path: /healthz
+            path: /ingress-controller-healthz
-            port: 10254
+            port: 80
             scheme: HTTP
-          initialDelaySeconds: 30
+          initialDelaySeconds: 10
-          timeoutSeconds: 5
+          timeoutSeconds: 1
         # use downward API
         env:
           - name: POD_NAME