Merge pull request #4236 from aledbf/leaks

Add e2e test suite to detect memory leaks in lua

Makefile

@@ -31,6 +31,8 @@ E2E_NODES ?= 10
 SLOW_E2E_THRESHOLD ?= 50
 K8S_VERSION ?= v1.14.1
 
+E2E_CHECK_LEAKS ?=
+
 ifeq ($(GOHOSTOS),darwin)
   SED_I=sed -i ''
 endif
@@ -70,6 +72,9 @@ export GOBUILD_FLAGS
 export REPO_INFO
 export BUSTED_ARGS
 export IMAGE
+export E2E_NODES
+export E2E_CHECK_LEAKS
+export SLOW_E2E_THRESHOLD
 
 # Set default base image dynamically for each arch
 BASEIMAGE?=quay.io/kubernetes-ingress-controller/nginx-$(ARCH):0.90
@@ -174,28 +179,7 @@ lua-test:
 
 .PHONY: e2e-test
 e2e-test:
-	echo "Granting permissions to ingress-nginx e2e service account..."
+	@build/run-e2e-suite.sh
-	kubectl create serviceaccount ingress-nginx-e2e || true
-	kubectl create clusterrolebinding permissive-binding \
-		--clusterrole=cluster-admin \
-		--user=admin \
-		--user=kubelet \
-		--serviceaccount=default:ingress-nginx-e2e || true
-
-	until kubectl get secret | grep -q ^ingress-nginx-e2e-token; do \
-		echo "waiting for api token"; \
-		sleep 3; \
-	done
-
-	kubectl run --rm \
-		--attach \
-		--restart=Never \
-		--generator=run-pod/v1 \
-		--env="E2E_NODES=$(E2E_NODES)" \
-		--env="FOCUS=$(FOCUS)" \
-		--env="SLOW_E2E_THRESHOLD=$(SLOW_E2E_THRESHOLD)" \
-		--overrides='{ "apiVersion": "v1", "spec":{"serviceAccountName": "ingress-nginx-e2e"}}' \
-		e2e --image=nginx-ingress-controller:e2e
 
 .PHONY: e2e-test-image
 e2e-test-image: e2e-test-binary

build/run-e2e-suite.sh

@@ -0,0 +1,80 @@
+#!/bin/bash
+
+# Copyright 2018 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if ! [ -z "$DEBUG" ]; then
+	set -x
+fi
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+RED='\e[35m'
+NC='\e[0m'
+BGREEN='\e[32m'
+
+declare -a mandatory
+mandatory=(
+  E2E_NODES
+  SLOW_E2E_THRESHOLD
+)
+
+missing=false
+for var in "${mandatory[@]}"; do
+  if [[ -z "${!var:-}" ]]; then
+    echo -e "${RED}Environment variable $var must be set${NC}"
+    missing=true
+  fi
+done
+
+if [ "$missing" = true ]; then
+  exit 1
+fi
+
+function cleanup {
+  kubectl delete pod e2e 2>/dev/null || true
+}
+trap cleanup EXIT
+
+E2E_CHECK_LEAKS=${E2E_CHECK_LEAKS:-}
+FOCUS=${FOCUS:-.*}
+
+export E2E_CHECK_LEAKS
+export FOCUS
+
+echo -e "${BGREEN}Granting permissions to ingress-nginx e2e service account...${NC}"
+kubectl create serviceaccount ingress-nginx-e2e || true
+kubectl create clusterrolebinding permissive-binding \
+  --clusterrole=cluster-admin \
+  --user=admin \
+  --user=kubelet \
+  --serviceaccount=default:ingress-nginx-e2e || true
+
+until kubectl get secret | grep -q ^ingress-nginx-e2e-token; do \
+  echo -e "waiting for api token"; \
+  sleep 3; \
+done
+
+kubectl run --rm \
+  --attach \
+  --restart=Never \
+  --generator=run-pod/v1 \
+  --env="E2E_NODES=${E2E_NODES}" \
+  --env="FOCUS=${FOCUS}" \
+  --env="E2E_CHECK_LEAKS=${E2E_CHECK_LEAKS}" \
+  --env="SLOW_E2E_THRESHOLD=${SLOW_E2E_THRESHOLD}" \
+  --overrides='{ "apiVersion": "v1", "spec":{"serviceAccountName": "ingress-nginx-e2e"}}' \
+  e2e --image=nginx-ingress-controller:e2e

rootfs/etc/nginx/template/nginx.tmpl

@@ -54,7 +54,6 @@ http {
     {{ buildLuaSharedDictionaries $servers $all.Cfg.DisableLuaRestyWAF }}
 
     init_by_lua_block {
-        require("resty.core")
         collectgarbage("collect")
 
         {{ if not $all.Cfg.DisableLuaRestyWAF }}
@@ -632,7 +631,6 @@ stream {
     lua_shared_dict tcp_udp_configuration_data 5M;
 
     init_by_lua_block {
-        require("resty.core")
         collectgarbage("collect")
 
         -- init modules

test/e2e-image/e2e.sh

@@ -16,36 +16,50 @@
 
 set -e
 
+NC='\e[0m'
+BGREEN='\e[32m'
+
 SLOW_E2E_THRESHOLD=${SLOW_E2E_THRESHOLD:-50}
 FOCUS=${FOCUS:-.*}
 E2E_NODES=${E2E_NODES:-5}
+E2E_CHECK_LEAKS=${E2E_CHECK_LEAKS:-""}
 
-if [ ! -f ${HOME}/.kube/config ]; then
+if [ ! -f "${HOME}/.kube/config" ]; then
-    kubectl config set-cluster dev --certificate-authority=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt --embed-certs=true --server="https://kubernetes.default/"
+  kubectl config set-cluster dev --certificate-authority=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt --embed-certs=true --server="https://kubernetes.default/"
-    kubectl config set-credentials user --token="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)"
+  kubectl config set-credentials user --token="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)"
-    kubectl config set-context default --cluster=dev --user=user
+  kubectl config set-context default --cluster=dev --user=user
-    kubectl config use-context default
+  kubectl config use-context default
 fi
 
 ginkgo_args=(
-    "-randomizeSuites"
+  "-randomizeSuites"
-    "-randomizeAllSpecs"
+  "-randomizeAllSpecs"
-    "-flakeAttempts=2"
+  "-flakeAttempts=2"
-    "-p"
+  "-p"
-    "-trace"
+  "-trace"
-    "--noColor=true"
+  "-slowSpecThreshold=${SLOW_E2E_THRESHOLD}"
-    "-slowSpecThreshold=${SLOW_E2E_THRESHOLD}"
+  "-r"
 )
 
-echo "Running e2e test suite..."
+echo -e "${BGREEN}Running e2e test suite (FOCUS=${FOCUS})...${NC}"
-ginkgo "${ginkgo_args[@]}"                   \
+ginkgo "${ginkgo_args[@]}"               \
-    -focus=${FOCUS}                          \
+  -focus="${FOCUS}"                      \
-    -skip="\[Serial\]"                       \
+  -skip="\[Serial\]|\[MemoryLeak\]"      \
-    -nodes=${E2E_NODES}                      \
+  -nodes="${E2E_NODES}"                  \
-    /e2e.test
+  /e2e.test
 
-echo "Running e2e test suite with tests that require serial execution..."
+echo -e "${BGREEN}Running e2e test suite with tests that require serial execution...${NC}"
-ginkgo "${ginkgo_args[@]}"                   \
+ginkgo "${ginkgo_args[@]}"               \
-    -focus="\[Serial\]"                      \
+  -focus="\[Serial\]"                    \
-    -nodes=1                                 \
+  -skip="\[MemoryLeak\]"                 \
+  -nodes=1                               \
+  /e2e.test
+
+if [[ ${E2E_CHECK_LEAKS} != "" ]]; then
+  echo -e "${BGREEN}Running e2e test suite with tests that check for memory leaks...${NC}"
+  ginkgo "${ginkgo_args[@]}"             \
+    -focus="\[MemoryLeak\]"              \
+    -skip="\[Serial\]"                   \
+    -nodes=1                             \
     /e2e.test
+fi

test/e2e/e2e.go

@@ -35,6 +35,7 @@ import (
 	_ "k8s.io/ingress-nginx/test/e2e/dbg"
 	_ "k8s.io/ingress-nginx/test/e2e/defaultbackend"
 	_ "k8s.io/ingress-nginx/test/e2e/gracefulshutdown"
+	_ "k8s.io/ingress-nginx/test/e2e/leaks"
 	_ "k8s.io/ingress-nginx/test/e2e/loadbalance"
 	_ "k8s.io/ingress-nginx/test/e2e/lua"
 	_ "k8s.io/ingress-nginx/test/e2e/servicebackend"

test/e2e/framework/framework.go

@@ -147,7 +147,12 @@ func (f *Framework) AfterEach() {
 
 // IngressNginxDescribe wrapper function for ginkgo describe. Adds namespacing.
 func IngressNginxDescribe(text string, body func()) bool {
-	return Describe("[nginx-ingress] "+text, body)
+	return Describe("[ingress-nginx] "+text, body)
+}
+
+// MemoryLeakIt is wrapper function for ginkgo It.  Adds "[MemoryLeak]" tag and makes static analysis easier.
+func MemoryLeakIt(text string, body interface{}, timeout ...float64) bool {
+	return It(text+" [MemoryLeak]", body, timeout...)
 }
 
 // GetNginxIP returns the number of TCP port where NGINX is running

test/e2e/leaks/lua_ssl.go

@@ -0,0 +1,130 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package leaks
+
+import (
+	"crypto/tls"
+	"fmt"
+	"net/http"
+	"strings"
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/parnurzeal/gorequest"
+	pool "gopkg.in/go-playground/pool.v3"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"k8s.io/ingress-nginx/test/e2e/framework"
+)
+
+var _ = framework.IngressNginxDescribe("DynamicCertificates", func() {
+	f := framework.NewDefaultFramework("lua-dynamic-certificates")
+
+	BeforeEach(func() {
+		f.NewEchoDeployment()
+	})
+
+	AfterEach(func() {
+	})
+
+	framework.MemoryLeakIt("should not leak memory from ingress SSL certificates or configuration updates", func() {
+		hostCount := 1000
+		iterations := 10
+
+		By("Waiting a minute before starting the test")
+		time.Sleep(1 * time.Minute)
+
+		for iteration := 1; iteration <= iterations; iteration++ {
+			By(fmt.Sprintf("Running iteration %v", iteration))
+
+			p := pool.NewLimited(200)
+
+			batch := p.Batch()
+
+			for index := 1; index <= hostCount; index++ {
+				host := fmt.Sprintf("hostname-%v", index)
+				batch.Queue(run(host, f))
+			}
+
+			batch.QueueComplete()
+			batch.WaitAll()
+
+			p.Close()
+
+			By("waiting one minute before next iteration")
+			time.Sleep(1 * time.Minute)
+		}
+	})
+})
+
+func privisionIngress(hostname string, f *framework.Framework) {
+	ing := f.EnsureIngress(framework.NewSingleIngressWithTLS(hostname, "/", hostname, []string{hostname}, f.Namespace, "http-svc", 80, nil))
+	_, err := framework.CreateIngressTLSSecret(f.KubeClientSet,
+		ing.Spec.TLS[0].Hosts,
+		ing.Spec.TLS[0].SecretName,
+		ing.Namespace)
+	Expect(err).NotTo(HaveOccurred())
+
+	f.WaitForNginxServer(hostname,
+		func(server string) bool {
+			return strings.Contains(server, fmt.Sprintf("server_name %v", hostname)) &&
+				strings.Contains(server, "listen 443")
+		})
+}
+
+func checkIngress(hostname string, f *framework.Framework) {
+	req := gorequest.New()
+	resp, _, errs := req.
+		Get(f.GetURL(framework.HTTPS)).
+		TLSClientConfig(&tls.Config{ServerName: hostname, InsecureSkipVerify: true}).
+		Set("Host", hostname).
+		End()
+	Expect(errs).Should(BeEmpty())
+	Expect(resp.StatusCode).Should(Equal(http.StatusOK))
+
+	// check the returned secret is not the fake one
+	cert := resp.TLS.PeerCertificates[0]
+	Expect(cert.DNSNames[0]).Should(Equal(hostname))
+}
+
+func deleteIngress(hostname string, f *framework.Framework) {
+	err := f.KubeClientSet.ExtensionsV1beta1().Ingresses(f.Namespace).Delete(hostname, &metav1.DeleteOptions{})
+	Expect(err).NotTo(HaveOccurred(), "unexpected error deleting ingress")
+}
+
+func run(host string, f *framework.Framework) pool.WorkFunc {
+	return func(wu pool.WorkUnit) (interface{}, error) {
+		if wu.IsCancelled() {
+			return nil, nil
+		}
+
+		By(fmt.Sprintf("\tcreating ingress for host %v", host))
+		privisionIngress(host, f)
+
+		time.Sleep(100 * time.Millisecond)
+
+		By(fmt.Sprintf("\tchecking ingress for host %v", host))
+		checkIngress(host, f)
+
+		By(fmt.Sprintf("\tdestroying ingress for host %v", host))
+		deleteIngress(host, f)
+
+		return true, nil
+	}
+}