From 54b13bd2167a0ffa77c3d1bf0baecfcc95f183a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Carlsson?= Date: Mon, 6 Jul 2020 21:05:16 +0200 Subject: [PATCH] Add flag to allow setting a shutdown grace period --- cmd/nginx/flags.go | 3 + docs/user-guide/cli-arguments.md | 1 + internal/ingress/controller/controller.go | 2 + internal/ingress/controller/nginx.go | 2 + test/e2e/framework/healthz.go | 44 ++++++++++++ test/e2e/gracefulshutdown/grace_period.go | 88 +++++++++++++++++++++++ 6 files changed, 140 insertions(+) create mode 100644 test/e2e/framework/healthz.go create mode 100644 test/e2e/gracefulshutdown/grace_period.go diff --git a/cmd/nginx/flags.go b/cmd/nginx/flags.go index 0bb44c483..55249679a 100644 --- a/cmd/nginx/flags.go +++ b/cmd/nginx/flags.go @@ -171,6 +171,8 @@ Takes the form ":port". If not provided, no admission controller is starte profilerPort = flags.Int("profiler-port", 10245, "Port to use for expose the ingress controller Go profiler when it is enabled.") statusUpdateInterval = flags.Int("status-update-interval", status.UpdateInterval, "Time interval in seconds in which the status should check if an update is required. Default is 60 seconds") + + shutdownGracePeriod = flags.Int("shutdown-grace-period", 0, "Seconds to wait after receiving the shutdown signal, before stopping the nginx process.") ) flags.StringVar(&nginx.MaxmindMirror, "maxmind-mirror", "", `Maxmind mirror url (example: http://geoip.local/databases`) @@ -281,6 +283,7 @@ https://blog.maxmind.com/2019/12/18/significant-changes-to-accessing-and-using-g PublishService: *publishSvc, PublishStatusAddress: *publishStatusAddress, UpdateStatusOnShutdown: *updateStatusOnShutdown, + ShutdownGracePeriod: *shutdownGracePeriod, UseNodeInternalIP: *useNodeInternalIP, SyncRateLimit: *syncRateLimit, ListenPorts: &ngx_config.ListenPorts{ diff --git a/docs/user-guide/cli-arguments.md b/docs/user-guide/cli-arguments.md index 8b55e41ac..fc5634697 100644 --- a/docs/user-guide/cli-arguments.md +++ b/docs/user-guide/cli-arguments.md @@ -53,6 +53,7 @@ They are set in the container spec of the `nginx-ingress-controller` Deployment | `--udp-services-configmap` | Name of the ConfigMap containing the definition of the UDP services to expose. The key in the map indicates the external port to be used. The value is a reference to a Service in the form "namespace/name:port", where "port" can either be a port name or number. | | `--update-status` | Update the load-balancer status of Ingress objects this controller satisfies. Requires setting the publish-service parameter to a valid Service reference. (default true) | | `--update-status-on-shutdown` | Update the load-balancer status of Ingress objects when the controller shuts down. Requires the update-status parameter. (default true) | +| `--shutdown-grace-period` | Seconds to wait after receiving the shutdown signal, before stopping the nginx process. | | `-v, --v Level` | number for the log level verbosity | | `--validating-webhook` | The address to start an admission controller on to validate incoming ingresses. Takes the form ":port". If not provided, no admission controller is started. | | `--validating-webhook-certificate` | The path of the validating webhook certificate PEM. | diff --git a/internal/ingress/controller/controller.go b/internal/ingress/controller/controller.go index 8e3111180..a29b0b14b 100644 --- a/internal/ingress/controller/controller.go +++ b/internal/ingress/controller/controller.go @@ -106,6 +106,8 @@ type Configuration struct { MaxmindEditionFiles []string MonitorMaxBatchSize int + + ShutdownGracePeriod int } // GetPublishService returns the Service used to set the load-balancer status of Ingresses. diff --git a/internal/ingress/controller/nginx.go b/internal/ingress/controller/nginx.go index 56112bbb5..afa79919b 100644 --- a/internal/ingress/controller/nginx.go +++ b/internal/ingress/controller/nginx.go @@ -367,6 +367,8 @@ func (n *NGINXController) Stop() error { return fmt.Errorf("shutdown already in progress") } + time.Sleep(time.Duration(n.cfg.ShutdownGracePeriod) * time.Second) + klog.InfoS("Shutting down controller queues") close(n.stopCh) go n.syncQueue.Shutdown() diff --git a/test/e2e/framework/healthz.go b/test/e2e/framework/healthz.go new file mode 100644 index 000000000..bef093355 --- /dev/null +++ b/test/e2e/framework/healthz.go @@ -0,0 +1,44 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +import ( + "fmt" + "net/http" +) + +// VerifyHealthz verifies the status code of the healthz endpoint +func (f *Framework) VerifyHealthz(ip string, statusCode int) error { + url := fmt.Sprintf("http://%v:10254/healthz", ip) + + client := &http.Client{} + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return fmt.Errorf("creating GET request for URL %q failed: %v", url, err) + } + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("executing GET request for URL %q failed: %v", url, err) + } + defer resp.Body.Close() + + if resp.StatusCode != statusCode { + return fmt.Errorf("GET request for URL %q returned HTTP status %s", url, resp.Status) + } + + return nil +} diff --git a/test/e2e/gracefulshutdown/grace_period.go b/test/e2e/gracefulshutdown/grace_period.go new file mode 100644 index 000000000..95c3729a1 --- /dev/null +++ b/test/e2e/gracefulshutdown/grace_period.go @@ -0,0 +1,88 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gracefulshutdown + +import ( + "context" + "net/http" + "strings" + "time" + + "github.com/onsi/ginkgo" + "github.com/stretchr/testify/assert" + appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/ingress-nginx/test/e2e/framework" +) + +var _ = framework.IngressNginxDescribe("[Shutdown] Grace period shutdown", func() { + f := framework.NewDefaultFramework("shutdown-grace-period") + + ginkgo.It("/healthz should return status code 500 during shutdown grace period", func() { + + f.NewSlowEchoDeployment() + + err := f.UpdateIngressControllerDeployment(func(deployment *appsv1.Deployment) error { + args := []string{} + for _, v := range deployment.Spec.Template.Spec.Containers[0].Args { + if strings.Contains(v, "--shutdown-grace-period") { + continue + } + + args = append(args, v) + } + + args = append(args, "--shutdown-grace-period=90") + deployment.Spec.Template.Spec.Containers[0].Args = args + cmds := []string{"/wait-shutdown"} + deployment.Spec.Template.Spec.Containers[0].Lifecycle.PreStop.Exec.Command = cmds + grace := int64(3600) + deployment.Spec.Template.Spec.TerminationGracePeriodSeconds = &grace + _, err := f.KubeClientSet.AppsV1().Deployments(f.Namespace).Update(context.TODO(), deployment, metav1.UpdateOptions{}) + return err + }) + + assert.Nil(ginkgo.GinkgoT(), err, "updating ingress controller deployment flags") + + ip := f.GetNginxPodIP() + + err = f.VerifyHealthz(ip, http.StatusOK) + assert.Nil(ginkgo.GinkgoT(), err) + + result := make(chan []error) + go func(c chan []error) { + defer ginkgo.GinkgoRecover() + errors := []error{} + + framework.Sleep(60 * time.Second) + + err = f.VerifyHealthz(ip, http.StatusInternalServerError) + if err != nil { + errors = append(errors, err) + } + + c <- errors + }(result) + + f.ScaleDeploymentToZero("nginx-ingress-controller") + + for _, err := range <-result { + assert.Nil(ginkgo.GinkgoT(), err) + } + + }) +})