Merge pull request #4236 from aledbf/leaks
Add e2e test suite to detect memory leaks in lua
This commit is contained in:
Kubernetes Prow Robot
GitHub
commit
d9d8ce7f13
7 changed files with 259 additions and 47 deletions
28
Makefile
28
Makefile
|
|
@ -31,6 +31,8 @@ E2E_NODES ?= 10
|
|||
SLOW_E2E_THRESHOLD ?= 50
|
||||
K8S_VERSION ?= v1.14.1
|
||||
|
||||
E2E_CHECK_LEAKS ?=
|
||||
|
||||
ifeq ($(GOHOSTOS),darwin)
|
||||
SED_I=sed -i ''
|
||||
endif
|
||||
|
|
@ -70,6 +72,9 @@ export GOBUILD_FLAGS
|
|||
export REPO_INFO
|
||||
export BUSTED_ARGS
|
||||
export IMAGE
|
||||
export E2E_NODES
|
||||
export E2E_CHECK_LEAKS
|
||||
export SLOW_E2E_THRESHOLD
|
||||
|
||||
# Set default base image dynamically for each arch
|
||||
BASEIMAGE?=quay.io/kubernetes-ingress-controller/nginx-$(ARCH):0.90
|
||||
|
|
@ -174,28 +179,7 @@ lua-test:
|
|||
|
||||
.PHONY: e2e-test
|
||||
e2e-test:
|
||||
echo "Granting permissions to ingress-nginx e2e service account..."
|
||||
kubectl create serviceaccount ingress-nginx-e2e || true
|
||||
kubectl create clusterrolebinding permissive-binding \
|
||||
--clusterrole=cluster-admin \
|
||||
--user=admin \
|
||||
--user=kubelet \
|
||||
--serviceaccount=default:ingress-nginx-e2e || true
|
||||
|
||||
until kubectl get secret | grep -q ^ingress-nginx-e2e-token; do \
|
||||
echo "waiting for api token"; \
|
||||
sleep 3; \
|
||||
done
|
||||
|
||||
kubectl run --rm \
|
||||
--attach \
|
||||
--restart=Never \
|
||||
--generator=run-pod/v1 \
|
||||
--env="E2E_NODES=$(E2E_NODES)" \
|
||||
--env="FOCUS=$(FOCUS)" \
|
||||
--env="SLOW_E2E_THRESHOLD=$(SLOW_E2E_THRESHOLD)" \
|
||||
--overrides='{ "apiVersion": "v1", "spec":{"serviceAccountName": "ingress-nginx-e2e"}}' \
|
||||
e2e --image=nginx-ingress-controller:e2e
|
||||
@build/run-e2e-suite.sh
|
||||
|
||||
.PHONY: e2e-test-image
|
||||
e2e-test-image: e2e-test-binary
|
||||
|
|
|
|||
80
build/run-e2e-suite.sh
Executable file
80
build/run-e2e-suite.sh
Executable file
|
|
@ -0,0 +1,80 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Copyright 2018 The Kubernetes Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
if ! [ -z "$DEBUG" ]; then
|
||||
set -x
|
||||
fi
|
||||
|
||||
set -o errexit
|
||||
set -o nounset
|
||||
set -o pipefail
|
||||
|
||||
RED='\e[35m'
|
||||
NC='\e[0m'
|
||||
BGREEN='\e[32m'
|
||||
|
||||
declare -a mandatory
|
||||
mandatory=(
|
||||
E2E_NODES
|
||||
SLOW_E2E_THRESHOLD
|
||||
)
|
||||
|
||||
missing=false
|
||||
for var in "${mandatory[@]}"; do
|
||||
if [[ -z "${!var:-}" ]]; then
|
||||
echo -e "${RED}Environment variable $var must be set${NC}"
|
||||
missing=true
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$missing" = true ]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
function cleanup {
|
||||
kubectl delete pod e2e 2>/dev/null || true
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
E2E_CHECK_LEAKS=${E2E_CHECK_LEAKS:-}
|
||||
FOCUS=${FOCUS:-.*}
|
||||
|
||||
export E2E_CHECK_LEAKS
|
||||
export FOCUS
|
||||
|
||||
echo -e "${BGREEN}Granting permissions to ingress-nginx e2e service account...${NC}"
|
||||
kubectl create serviceaccount ingress-nginx-e2e || true
|
||||
kubectl create clusterrolebinding permissive-binding \
|
||||
--clusterrole=cluster-admin \
|
||||
--user=admin \
|
||||
--user=kubelet \
|
||||
--serviceaccount=default:ingress-nginx-e2e || true
|
||||
|
||||
until kubectl get secret | grep -q ^ingress-nginx-e2e-token; do \
|
||||
echo -e "waiting for api token"; \
|
||||
sleep 3; \
|
||||
done
|
||||
|
||||
kubectl run --rm \
|
||||
--attach \
|
||||
--restart=Never \
|
||||
--generator=run-pod/v1 \
|
||||
--env="E2E_NODES=${E2E_NODES}" \
|
||||
--env="FOCUS=${FOCUS}" \
|
||||
--env="E2E_CHECK_LEAKS=${E2E_CHECK_LEAKS}" \
|
||||
--env="SLOW_E2E_THRESHOLD=${SLOW_E2E_THRESHOLD}" \
|
||||
--overrides='{ "apiVersion": "v1", "spec":{"serviceAccountName": "ingress-nginx-e2e"}}' \
|
||||
e2e --image=nginx-ingress-controller:e2e
|
||||
|
|
@ -54,7 +54,6 @@ http {
|
|||
{{ buildLuaSharedDictionaries $servers $all.Cfg.DisableLuaRestyWAF }}
|
||||
|
||||
init_by_lua_block {
|
||||
require("resty.core")
|
||||
collectgarbage("collect")
|
||||
|
||||
{{ if not $all.Cfg.DisableLuaRestyWAF }}
|
||||
|
|
@ -632,7 +631,6 @@ stream {
|
|||
lua_shared_dict tcp_udp_configuration_data 5M;
|
||||
|
||||
init_by_lua_block {
|
||||
require("resty.core")
|
||||
collectgarbage("collect")
|
||||
|
||||
-- init modules
|
||||
|
|
|
|||
|
|
@ -16,36 +16,50 @@
|
|||
|
||||
set -e
|
||||
|
||||
NC='\e[0m'
|
||||
BGREEN='\e[32m'
|
||||
|
||||
SLOW_E2E_THRESHOLD=${SLOW_E2E_THRESHOLD:-50}
|
||||
FOCUS=${FOCUS:-.*}
|
||||
E2E_NODES=${E2E_NODES:-5}
|
||||
E2E_CHECK_LEAKS=${E2E_CHECK_LEAKS:-""}
|
||||
|
||||
if [ ! -f ${HOME}/.kube/config ]; then
|
||||
kubectl config set-cluster dev --certificate-authority=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt --embed-certs=true --server="https://kubernetes.default/"
|
||||
kubectl config set-credentials user --token="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)"
|
||||
kubectl config set-context default --cluster=dev --user=user
|
||||
kubectl config use-context default
|
||||
if [ ! -f "${HOME}/.kube/config" ]; then
|
||||
kubectl config set-cluster dev --certificate-authority=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt --embed-certs=true --server="https://kubernetes.default/"
|
||||
kubectl config set-credentials user --token="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)"
|
||||
kubectl config set-context default --cluster=dev --user=user
|
||||
kubectl config use-context default
|
||||
fi
|
||||
|
||||
ginkgo_args=(
|
||||
"-randomizeSuites"
|
||||
"-randomizeAllSpecs"
|
||||
"-flakeAttempts=2"
|
||||
"-p"
|
||||
"-trace"
|
||||
"--noColor=true"
|
||||
"-slowSpecThreshold=${SLOW_E2E_THRESHOLD}"
|
||||
"-randomizeSuites"
|
||||
"-randomizeAllSpecs"
|
||||
"-flakeAttempts=2"
|
||||
"-p"
|
||||
"-trace"
|
||||
"-slowSpecThreshold=${SLOW_E2E_THRESHOLD}"
|
||||
"-r"
|
||||
)
|
||||
|
||||
echo "Running e2e test suite..."
|
||||
ginkgo "${ginkgo_args[@]}" \
|
||||
-focus=${FOCUS} \
|
||||
-skip="\[Serial\]" \
|
||||
-nodes=${E2E_NODES} \
|
||||
/e2e.test
|
||||
echo -e "${BGREEN}Running e2e test suite (FOCUS=${FOCUS})...${NC}"
|
||||
ginkgo "${ginkgo_args[@]}" \
|
||||
-focus="${FOCUS}" \
|
||||
-skip="\[Serial\]|\[MemoryLeak\]" \
|
||||
-nodes="${E2E_NODES}" \
|
||||
/e2e.test
|
||||
|
||||
echo "Running e2e test suite with tests that require serial execution..."
|
||||
ginkgo "${ginkgo_args[@]}" \
|
||||
-focus="\[Serial\]" \
|
||||
-nodes=1 \
|
||||
echo -e "${BGREEN}Running e2e test suite with tests that require serial execution...${NC}"
|
||||
ginkgo "${ginkgo_args[@]}" \
|
||||
-focus="\[Serial\]" \
|
||||
-skip="\[MemoryLeak\]" \
|
||||
-nodes=1 \
|
||||
/e2e.test
|
||||
|
||||
if [[ ${E2E_CHECK_LEAKS} != "" ]]; then
|
||||
echo -e "${BGREEN}Running e2e test suite with tests that check for memory leaks...${NC}"
|
||||
ginkgo "${ginkgo_args[@]}" \
|
||||
-focus="\[MemoryLeak\]" \
|
||||
-skip="\[Serial\]" \
|
||||
-nodes=1 \
|
||||
/e2e.test
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -35,6 +35,7 @@ import (
|
|||
_ "k8s.io/ingress-nginx/test/e2e/dbg"
|
||||
_ "k8s.io/ingress-nginx/test/e2e/defaultbackend"
|
||||
_ "k8s.io/ingress-nginx/test/e2e/gracefulshutdown"
|
||||
_ "k8s.io/ingress-nginx/test/e2e/leaks"
|
||||
_ "k8s.io/ingress-nginx/test/e2e/loadbalance"
|
||||
_ "k8s.io/ingress-nginx/test/e2e/lua"
|
||||
_ "k8s.io/ingress-nginx/test/e2e/servicebackend"
|
||||
|
|
|
|||
|
|
@ -147,7 +147,12 @@ func (f *Framework) AfterEach() {
|
|||
|
||||
// IngressNginxDescribe wrapper function for ginkgo describe. Adds namespacing.
|
||||
func IngressNginxDescribe(text string, body func()) bool {
|
||||
return Describe("[nginx-ingress] "+text, body)
|
||||
return Describe("[ingress-nginx] "+text, body)
|
||||
}
|
||||
|
||||
// MemoryLeakIt is wrapper function for ginkgo It. Adds "[MemoryLeak]" tag and makes static analysis easier.
|
||||
func MemoryLeakIt(text string, body interface{}, timeout ...float64) bool {
|
||||
return It(text+" [MemoryLeak]", body, timeout...)
|
||||
}
|
||||
|
||||
// GetNginxIP returns the number of TCP port where NGINX is running
|
||||
|
|
|
|||
130
test/e2e/leaks/lua_ssl.go
Normal file
130
test/e2e/leaks/lua_ssl.go
Normal file
|
|
@ -0,0 +1,130 @@
|
|||
/*
|
||||
Copyright 2019 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package leaks
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
"github.com/parnurzeal/gorequest"
|
||||
pool "gopkg.in/go-playground/pool.v3"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
|
||||
"k8s.io/ingress-nginx/test/e2e/framework"
|
||||
)
|
||||
|
||||
var _ = framework.IngressNginxDescribe("DynamicCertificates", func() {
|
||||
f := framework.NewDefaultFramework("lua-dynamic-certificates")
|
||||
|
||||
BeforeEach(func() {
|
||||
f.NewEchoDeployment()
|
||||
})
|
||||
|
||||
AfterEach(func() {
|
||||
})
|
||||
|
||||
framework.MemoryLeakIt("should not leak memory from ingress SSL certificates or configuration updates", func() {
|
||||
hostCount := 1000
|
||||
iterations := 10
|
||||
|
||||
By("Waiting a minute before starting the test")
|
||||
time.Sleep(1 * time.Minute)
|
||||
|
||||
for iteration := 1; iteration <= iterations; iteration++ {
|
||||
By(fmt.Sprintf("Running iteration %v", iteration))
|
||||
|
||||
p := pool.NewLimited(200)
|
||||
|
||||
batch := p.Batch()
|
||||
|
||||
for index := 1; index <= hostCount; index++ {
|
||||
host := fmt.Sprintf("hostname-%v", index)
|
||||
batch.Queue(run(host, f))
|
||||
}
|
||||
|
||||
batch.QueueComplete()
|
||||
batch.WaitAll()
|
||||
|
||||
p.Close()
|
||||
|
||||
By("waiting one minute before next iteration")
|
||||
time.Sleep(1 * time.Minute)
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
func privisionIngress(hostname string, f *framework.Framework) {
|
||||
ing := f.EnsureIngress(framework.NewSingleIngressWithTLS(hostname, "/", hostname, []string{hostname}, f.Namespace, "http-svc", 80, nil))
|
||||
_, err := framework.CreateIngressTLSSecret(f.KubeClientSet,
|
||||
ing.Spec.TLS[0].Hosts,
|
||||
ing.Spec.TLS[0].SecretName,
|
||||
ing.Namespace)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
f.WaitForNginxServer(hostname,
|
||||
func(server string) bool {
|
||||
return strings.Contains(server, fmt.Sprintf("server_name %v", hostname)) &&
|
||||
strings.Contains(server, "listen 443")
|
||||
})
|
||||
}
|
||||
|
||||
func checkIngress(hostname string, f *framework.Framework) {
|
||||
req := gorequest.New()
|
||||
resp, _, errs := req.
|
||||
Get(f.GetURL(framework.HTTPS)).
|
||||
TLSClientConfig(&tls.Config{ServerName: hostname, InsecureSkipVerify: true}).
|
||||
Set("Host", hostname).
|
||||
End()
|
||||
Expect(errs).Should(BeEmpty())
|
||||
Expect(resp.StatusCode).Should(Equal(http.StatusOK))
|
||||
|
||||
// check the returned secret is not the fake one
|
||||
cert := resp.TLS.PeerCertificates[0]
|
||||
Expect(cert.DNSNames[0]).Should(Equal(hostname))
|
||||
}
|
||||
|
||||
func deleteIngress(hostname string, f *framework.Framework) {
|
||||
err := f.KubeClientSet.ExtensionsV1beta1().Ingresses(f.Namespace).Delete(hostname, &metav1.DeleteOptions{})
|
||||
Expect(err).NotTo(HaveOccurred(), "unexpected error deleting ingress")
|
||||
}
|
||||
|
||||
func run(host string, f *framework.Framework) pool.WorkFunc {
|
||||
return func(wu pool.WorkUnit) (interface{}, error) {
|
||||
if wu.IsCancelled() {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
By(fmt.Sprintf("\tcreating ingress for host %v", host))
|
||||
privisionIngress(host, f)
|
||||
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
|
||||
By(fmt.Sprintf("\tchecking ingress for host %v", host))
|
||||
checkIngress(host, f)
|
||||
|
||||
By(fmt.Sprintf("\tdestroying ingress for host %v", host))
|
||||
deleteIngress(host, f)
|
||||
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue