Improve shutdown logic: wait until no requests are made

Pods in Kubernetes endpoints are expected to shut-down 'gracefully' after receiving SIGTERM -
we should keep accepting new connections for a while. This is because Kubernetes updates Service endpoints
and sends SIGTERM to pods *in parallel*.

See https://github.com/kubernetes/kubernetes/issues/106476 for more detail.

internal/ingress/controller/nginx.go

@@ -23,6 +23,7 @@ import (
 	"errors"
 	"fmt"
 	"io/fs"
+	"k8s.io/ingress-nginx/internal/ingress/metric/collectors"
 	"net"
 	"net/http"
 	"os"
@@ -377,6 +378,63 @@ func (n *NGINXController) Start() {
 	}
 }
 
+// stopWait waits until no more connections are made to nginx.
+//
+// This waits until all of following conditions are met:
+//   - No more requests are made to nginx for the last 5 seconds.
+//   - 'shutdown-grace-period' seconds have passed after calling this method.
+//
+// Pods in Kubernetes endpoints are expected to shut-down 'gracefully' after receiving SIGTERM -
+// we should keep accepting new connections for a while. This is because Kubernetes updates Service endpoints
+// and sends SIGTERM to pods *in parallel*.
+// If we don't see new requests for 5 seconds, then we assume that this pod was removed from the upstream endpoints
+// (AWS ALB endpoints for example), and proceed with shutdown.
+//
+// See https://github.com/kubernetes/kubernetes/issues/106476 for more detail on this issue.
+func (n *NGINXController) stopWait() {
+	const checkFrequency = time.Second
+	const waitUntilNoConnectionsFor = int((5 * time.Second) / checkFrequency)
+	waitAtLeastUntil := time.Now().Add(time.Duration(n.cfg.ShutdownGracePeriod) * time.Second)
+
+	var scraper collectors.NginxStatusScraper
+	lastRequests := 0
+	noChangeTimes := 0
+
+	for ; ; time.Sleep(checkFrequency) {
+		st, err := scraper.Scrape()
+		if err != nil {
+			klog.Warningf("failed to scrape nginx status: %v", err)
+			noChangeTimes = 0
+			continue
+		}
+
+		diff := st.Requests - lastRequests
+		// We assume that there were no client requests to nginx, if and only if
+		// there were 0 to 2 increase in handled requests from the last scrape.
+		// 1 is to account for our own stub_status request from this method,
+		// and the other 1 is to account for the readinessProbe.
+		// Note that readinessProbe DO happen even when the pod is terminating.
+		// See: https://github.com/kubernetes/kubernetes/issues/122824#issuecomment-1899224434
+		noChange := 0 <= diff && diff <= 2
+		if noChange {
+			noChangeTimes++
+			if noChangeTimes >= waitUntilNoConnectionsFor {
+				// Safe to proceed shutdown, we are seeing no more client request.
+				break
+			}
+		} else {
+			noChangeTimes = 0
+		}
+		lastRequests = st.Requests
+	}
+
+	// Wait at least for the configured duration, if any
+	delay := waitAtLeastUntil.Sub(time.Now())
+	if delay > 0 {
+		time.Sleep(delay)
+	}
+}
+
 // Stop gracefully stops the NGINX master process.
 func (n *NGINXController) Stop() error {
 	n.isShuttingDown = true
@@ -388,7 +446,8 @@ func (n *NGINXController) Stop() error {
 		return fmt.Errorf("shutdown already in progress")
 	}
 
-	time.Sleep(time.Duration(n.cfg.ShutdownGracePeriod) * time.Second)
+	klog.InfoS("Graceful shutdown - waiting until no more requests are made")
+	n.stopWait()
 
 	klog.InfoS("Shutting down controller queues")
 	close(n.stopCh)

test/e2e/framework/deployment.go

@@ -624,6 +624,10 @@ func (f *Framework) ScaleDeploymentToZero(name string) {
 	assert.Nil(ginkgo.GinkgoT(), err, "getting deployment")
 	assert.NotNil(ginkgo.GinkgoT(), d, "expected a deployment but none returned")
 
+	err = waitForPodsDeleted(f.KubeClientSet, 2*time.Minute, f.Namespace, &metav1.ListOptions{
+		LabelSelector: labelSelectorToString(d.Spec.Selector.MatchLabels),
+	})
+	assert.Nil(ginkgo.GinkgoT(), err, "waiting for no pods")
 	err = WaitForEndpoints(f.KubeClientSet, DefaultTimeout, name, f.Namespace, 0)
 	assert.Nil(ginkgo.GinkgoT(), err, "waiting for no endpoints")
 }

test/e2e/gracefulshutdown/k8s_async_shutdown.go

@@ -0,0 +1,87 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package gracefulshutdown
+
+import (
+	"context"
+	"fmt"
+	"github.com/onsi/ginkgo/v2"
+	"github.com/stretchr/testify/assert"
+	appsv1 "k8s.io/api/apps/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/ingress-nginx/test/e2e/framework"
+	"net/http"
+	"strings"
+	"time"
+)
+
+var _ = framework.IngressNginxDescribe("[Shutdown] Asynchronous shutdown", func() {
+	f := framework.NewDefaultFramework("k8s-async-shutdown", func(f *framework.Framework) {
+		f.Namespace = "k8s-async-shutdown"
+	})
+
+	host := "async-shutdown"
+
+	ginkgo.BeforeEach(func() {
+		f.NewSlowEchoDeployment()
+	})
+
+	ginkgo.It("should not shut down while still receiving traffic", func() {
+		defer ginkgo.GinkgoRecover()
+
+		err := f.UpdateIngressControllerDeployment(func(deployment *appsv1.Deployment) error {
+			// Note: e2e's default terminationGracePeriodSeconds is 1 for some reason, so extend it
+			grace := int64(300)
+			deployment.Spec.Template.Spec.TerminationGracePeriodSeconds = &grace
+			_, err := f.KubeClientSet.AppsV1().Deployments(f.Namespace).Update(context.TODO(), deployment, metav1.UpdateOptions{})
+			return err
+		})
+		assert.Nil(ginkgo.GinkgoT(), err, "updating ingress controller deployment")
+
+		f.EnsureIngress(framework.NewSingleIngress(host, "/", host, f.Namespace, framework.SlowEchoService, 80, nil))
+
+		f.WaitForNginxServer(host,
+			func(server string) bool {
+				return strings.Contains(server, "server_name "+host)
+			})
+
+		// We need to get pod IP first because after the pod becomes terminating,
+		// it is removed from Service endpoints, and becomes unable to be discovered by "f.HTTPTestClient()".
+		ip := f.GetNginxPodIP()
+
+		// Assume that the upstream takes 30 seconds to update its endpoints,
+		// therefore we are still receiving traffic while shutting down
+		go func() {
+			defer ginkgo.GinkgoRecover()
+			for i := 0; i < 120; i++ {
+				f.HTTPDumbTestClient().
+					GET("/").
+					WithURL(fmt.Sprintf("http://%s/", ip)).
+					WithHeader("Host", host).
+					Expect().
+					Status(http.StatusOK)
+
+				framework.Sleep(250 * time.Millisecond)
+			}
+		}()
+
+		start := time.Now()
+		f.ScaleDeploymentToZero("nginx-ingress-controller")
+		assert.GreaterOrEqualf(ginkgo.GinkgoT(), int(time.Since(start).Seconds()), 35,
+			"should take more than 30 + 5 seconds for graceful shutdown")
+	})
+})