Merge pull request #3548 from breunigs/integration-test-graceful-shutdown-clean

e2e test to ensure graceful shutdown does not lose requests

test/e2e/e2e.go

@@ -33,6 +33,7 @@ import (
 	// tests to run
 	_ "k8s.io/ingress-nginx/test/e2e/annotations"
 	_ "k8s.io/ingress-nginx/test/e2e/defaultbackend"
+	_ "k8s.io/ingress-nginx/test/e2e/gracefulshutdown"
 	_ "k8s.io/ingress-nginx/test/e2e/loadbalance"
 	_ "k8s.io/ingress-nginx/test/e2e/lua"
 	_ "k8s.io/ingress-nginx/test/e2e/servicebackend"

test/e2e/framework/deployment.go

@@ -36,6 +36,11 @@ func (f *Framework) NewEchoDeploymentWithReplicas(replicas int32) {
 	f.NewDeployment("http-svc", "gcr.io/kubernetes-e2e-test-images/echoserver:2.2", 8080, replicas)
 }
 
+// NewSlowEchoDeployment creates a new deployment of the slow echo server image in a particular namespace.
+func (f *Framework) NewSlowEchoDeployment() {
+	f.NewDeployment("slowecho", "breunigs/slowechoserver", 8080, 1)
+}
+
 // NewHttpbinDeployment creates a new single replica deployment of the httpbin image in a particular namespace.
 func (f *Framework) NewHttpbinDeployment() {
 	f.NewDeployment("httpbin", "kennethreitz/httpbin", 80, 1)

test/e2e/framework/framework.go

@@ -315,6 +315,26 @@ func (f *Framework) UpdateNginxConfigMapData(key string, value string) {
 	f.SetNginxConfigMapData(config)
 }
 
+// DeleteNGINXPod deletes the currently running pod. It waits for the replacement pod to be up.
+// Grace period to wait for pod shutdown is in seconds.
+func (f *Framework) DeleteNGINXPod(grace int64) {
+	ns := f.IngressController.Namespace
+	pod, err := getIngressNGINXPod(ns, f.KubeClientSet)
+	Expect(err).NotTo(HaveOccurred(), "expected ingress nginx pod to be running")
+
+	err = f.KubeClientSet.CoreV1().Pods(ns).Delete(pod.GetName(), metav1.NewDeleteOptions(grace))
+	Expect(err).NotTo(HaveOccurred(), "unexpected error deleting ingress nginx pod")
+
+	err = wait.Poll(Poll, time.Minute*5, func() (bool, error) {
+		pod, err := getIngressNGINXPod(ns, f.KubeClientSet)
+		if err != nil || pod == nil {
+			return false, nil
+		}
+		return pod.GetName() != "", nil
+	})
+	Expect(err).NotTo(HaveOccurred(), "unexpected error while waiting for ingress nginx pod to come up again")
+}
+
 // UpdateDeployment runs the given updateFunc on the deployment and waits for it to be updated
 func UpdateDeployment(kubeClientSet kubernetes.Interface, namespace string, name string, replicas int, updateFunc func(d *appsv1beta1.Deployment) error) error {
 	deployment, err := kubeClientSet.AppsV1beta1().Deployments(namespace).Get(name, metav1.GetOptions{})

test/e2e/gracefulshutdown/slow_requests.go

@@ -0,0 +1,63 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package gracefulshutdown
+
+import (
+	"net/http"
+	"strings"
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/parnurzeal/gorequest"
+	"k8s.io/ingress-nginx/test/e2e/framework"
+)
+
+var _ = framework.IngressNginxDescribe("Graceful Shutdown - Slow Requests", func() {
+	f := framework.NewDefaultFramework("shutdown-slow-requests")
+
+	BeforeEach(func() {
+		f.NewSlowEchoDeployment()
+		f.UpdateNginxConfigMapData("worker-shutdown-timeout", "50s")
+	})
+
+	It("should let slow requests finish before shutting down", func() {
+		host := "graceful-shutdown"
+
+		f.EnsureIngress(framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "slowecho", 8080, nil))
+		f.WaitForNginxConfiguration(
+			func(conf string) bool {
+				return strings.Contains(conf, "worker_shutdown_timeout")
+			})
+
+		done := make(chan bool)
+		go func() {
+			defer func() { done <- true }()
+			defer GinkgoRecover()
+			resp, _, errs := gorequest.New().
+				Get(f.IngressController.HTTPURL+"/sleep/30").
+				Set("Host", host).
+				End()
+			Expect(errs).To(BeNil())
+			Expect(resp.StatusCode).Should(Equal(http.StatusOK))
+		}()
+
+		time.Sleep(1 * time.Second)
+		f.DeleteNGINXPod(60)
+		<-done
+	})
+})