diff --git a/Makefile b/Makefile index 36d67c0c1..37e38ed06 100644 --- a/Makefile +++ b/Makefile @@ -52,7 +52,7 @@ GOBUILD_FLAGS := ALL_ARCH = amd64 arm arm64 ppc64le -QEMUVERSION = v2.12.0-1 +QEMUVERSION = v3.0.0 BUSTED_ARGS =-v --pattern=_test @@ -61,7 +61,7 @@ IMAGE = $(REGISTRY)/$(IMGNAME) MULTI_ARCH_IMG = $(IMAGE)-$(ARCH) # Set default base image dynamically for each arch -BASEIMAGE?=quay.io/kubernetes-ingress-controller/nginx-$(ARCH):0.65 +BASEIMAGE?=quay.io/kubernetes-ingress-controller/nginx-$(ARCH):0.66 ifeq ($(ARCH),arm) QEMUARCH=arm diff --git a/build/e2e-tests.sh b/build/e2e-tests.sh index 83c34c668..74af35714 100755 --- a/build/e2e-tests.sh +++ b/build/e2e-tests.sh @@ -41,19 +41,6 @@ fi SCRIPT_ROOT=$(dirname ${BASH_SOURCE})/.. -mkdir -p ${SCRIPT_ROOT}/test/binaries - -TEST_BINARIES=$( cd "${SCRIPT_ROOT}/test/binaries" ; pwd -P ) - -export PATH=${TEST_BINARIES}:$PATH - -if ! [ -x "$(command -v kubectl)" ]; then - echo "downloading kubectl..." - curl -sSLo ${TEST_BINARIES}/kubectl \ - https://storage.googleapis.com/kubernetes-release/release/v1.11.0/bin/linux/amd64/kubectl - chmod +x ${TEST_BINARIES}/kubectl -fi - ginkgo build ./test/e2e exec -- \ diff --git a/build/go-in-docker.sh b/build/go-in-docker.sh index 39deb8683..ff081d90e 100755 --- a/build/go-in-docker.sh +++ b/build/go-in-docker.sh @@ -40,7 +40,7 @@ if [ "$missing" = true ];then exit 1 fi -E2E_IMAGE=quay.io/kubernetes-ingress-controller/e2e:v10102018-dcc6495 +E2E_IMAGE=quay.io/kubernetes-ingress-controller/e2e:v10292018-240c7274b DOCKER_OPTS=${DOCKER_OPTS:-""} diff --git a/cmd/nginx/flags.go b/cmd/nginx/flags.go index f26947e52..25633ef54 100644 --- a/cmd/nginx/flags.go +++ b/cmd/nginx/flags.go @@ -84,6 +84,8 @@ Takes the form "namespace/name".`) Configured inside the NGINX status server. All requests received on the port defined by the healthz-port parameter are forwarded internally to this path.`) + healthCheckTimeout = flags.Duration("health-check-timeout", 10, `Time limit, in seconds, for a probe to health-check-path to succeed.`) + updateStatus = flags.Bool("update-status", true, `Update the load-balancer status of Ingress objects this controller satisfies. Requires setting the publish-service parameter to a valid Service reference.`) @@ -217,6 +219,7 @@ Feature backed by OpenResty Lua libraries. Requires that OCSP stapling is not en ConfigMapName: *configMap, DefaultSSLCertificate: *defSSLCertificate, DefaultHealthzURL: *defHealthzURL, + HealthCheckTimeout: *healthCheckTimeout, PublishService: *publishSvc, PublishStatusAddress: *publishStatusAddress, ForceNamespaceIsolation: *forceIsolation, diff --git a/cmd/nginx/main.go b/cmd/nginx/main.go index c76b27c4a..667035517 100644 --- a/cmd/nginx/main.go +++ b/cmd/nginx/main.go @@ -94,7 +94,7 @@ func main() { if err != nil { // TODO (antoineco): compare with error types from k8s.io/apimachinery/pkg/api/errors if strings.Contains(err.Error(), "cannot get services in the namespace") { - glog.Fatalf("✖ The cluster seems to be running with a restrictive Authorization mode and the Ingress controller does not have the required permissions to operate normally.") + glog.Fatal("✖ The cluster seems to be running with a restrictive Authorization mode and the Ingress controller does not have the required permissions to operate normally.") } glog.Fatalf("No service with name %v found: %v", conf.DefaultService, err) } @@ -160,7 +160,7 @@ func handleSigterm(ngx *controller.NGINXController, exit exiter) { signalChan := make(chan os.Signal, 1) signal.Notify(signalChan, syscall.SIGTERM) <-signalChan - glog.Infof("Received SIGTERM, shutting down") + glog.Info("Received SIGTERM, shutting down") exitCode := 0 if err := ngx.Stop(); err != nil { @@ -168,7 +168,7 @@ func handleSigterm(ngx *controller.NGINXController, exit exiter) { exitCode = 1 } - glog.Infof("Handled quit, awaiting Pod deletion") + glog.Info("Handled quit, awaiting Pod deletion") time.Sleep(10 * time.Second) glog.Infof("Exiting with %v", exitCode) diff --git a/docs/deploy/index.md b/docs/deploy/index.md index 49d97b429..7ee1864dc 100644 --- a/docs/deploy/index.md +++ b/docs/deploy/index.md @@ -2,8 +2,7 @@ ## Contents -- [Generic Deployment](#generic-deployment) - - [Mandatory command](#mandatory-command) +- [Prerequisite Generic Deployment Command](#prerequisite-generic-deployment-command) - [Provider Specific Steps](#provider-specific-steps) - [Docker for Mac](#docker-for-mac) - [minikube](#minikube) @@ -159,7 +158,7 @@ kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/mast ``` !!! tip - For extended notes regarding deployments on bare-metal, see [Bare-metal considerations](./baremetal/). + For extended notes regarding deployments on bare-metal, see [Bare-metal considerations](./baremetal.md/). ### Verify installation diff --git a/docs/examples/auth/client-certs/README.md b/docs/examples/auth/client-certs/README.md index baa91eccd..072e1d69f 100644 --- a/docs/examples/auth/client-certs/README.md +++ b/docs/examples/auth/client-certs/README.md @@ -4,7 +4,7 @@ It is possible to enable Client-Certificate Authentication by adding additional Before getting started you must have the following Certificates Setup: 1. CA certificate and Key(Intermediate Certs need to be in CA) -2. Server Certificate(Signed by CA) and Key (CN should be equal the the hostname you will use) +2. Server Certificate(Signed by CA) and Key (CN should be equal the hostname you will use) 3. Client Certificate(Signed by CA) and Key ## Creating Certificate Secrets diff --git a/docs/user-guide/cli-arguments.md b/docs/user-guide/cli-arguments.md index dbd2f4ff0..70c7549d7 100644 --- a/docs/user-guide/cli-arguments.md +++ b/docs/user-guide/cli-arguments.md @@ -10,7 +10,7 @@ They are set in the container spec of the `nginx-ingress-controller` Deployment | `--annotations-prefix string` | Prefix of the Ingress annotations specific to the NGINX controller. (default "nginx.ingress.kubernetes.io") | | `--apiserver-host string` | Address of the Kubernetes API server. Takes the form "protocol://address:port". If not specified, it is assumed the program runs inside a Kubernetes cluster and local discovery is attempted. | | `--configmap string` | Name of the ConfigMap containing custom global configurations for the controller. | -| `--default-backend-service string` | Service used to serve HTTP requests not matching any known server name (catch-all). Takes the form "namespace/name". The controller configures NGINX to forward requests to the first port of this Service. If not specified, 404 page will be returned diretly from Nginx.| +| `--default-backend-service string` | Service used to serve HTTP requests not matching any known server name (catch-all). Takes the form "namespace/name". The controller configures NGINX to forward requests to the first port of this Service. If not specified, a 404 page will be returned directly from NGINX.| | `--default-server-port int` | When `default-backend-service` is not specified or specified service does not have any endpoint, a local endpoint with this port will be used to serve 404 page from inside Nginx. | | `--default-ssl-certificate string` | Secret containing a SSL certificate to be used by the default HTTPS server (catch-all). Takes the form "namespace/name". | | `--election-id string` | Election id to use for Ingress status updates. (default "ingress-controller-leader") | diff --git a/docs/user-guide/nginx-configuration/annotations.md b/docs/user-guide/nginx-configuration/annotations.md index f1eb69e1b..a9e4f9f4e 100644 --- a/docs/user-guide/nginx-configuration/annotations.md +++ b/docs/user-guide/nginx-configuration/annotations.md @@ -80,6 +80,9 @@ You can add these Kubernetes annotations to specific Ingress objects to customiz |[nginx.ingress.kubernetes.io/lua-resty-waf-debug](#lua-resty-waf)|"true" or "false"| |[nginx.ingress.kubernetes.io/lua-resty-waf-ignore-rulesets](#lua-resty-waf)|string| |[nginx.ingress.kubernetes.io/lua-resty-waf-extra-rules](#lua-resty-waf)|string| +|[nginx.ingress.kubernetes.io/lua-resty-waf-allow-unknown-content-types](#lua-resty-waf)|"true" or "false"| +|[nginx.ingress.kubernetes.io/lua-resty-waf-score-threshold](#lua-resty-waf)|number| +|[nginx.ingress.kubernetes.io/lua-resty-waf-process-multipart-body](#lua-resty-waf)|"true" or "false"| |[nginx.ingress.kubernetes.io/enable-influxdb](#influxdb)|"true" or "false"| |[nginx.ingress.kubernetes.io/influxdb-measurement](#influxdb)|string| |[nginx.ingress.kubernetes.io/influxdb-port](#influxdb)|string| @@ -558,6 +561,32 @@ It is also possible to configure custom WAF rules per ingress using the `nginx.i nginx.ingress.kubernetes.io/lua-resty-waf-extra-rules: '[=[ { "access": [ { "actions": { "disrupt" : "DENY" }, "id": 10001, "msg": "my custom rule", "operator": "STR_CONTAINS", "pattern": "foo", "vars": [ { "parse": [ "values", 1 ], "type": "REQUEST_ARGS" } ] } ], "body_filter": [], "header_filter":[] } ]=]' ``` +Since the default allowed contents were `"text/html", "text/json", "application/json"` +We can enable the following annotation for allow all contents type: + + +```yaml +nginx.ingress.kubernetes.io/lua-resty-waf-allow-unknown-content-types: "true" +``` + +The default score of lua-resty-waf is 5, which usually triggered if hitting 2 default rules, you can modify the score threshold with following annotation: + + +```yaml +nginx.ingress.kubernetes.io/lua-resty-waf-score-threshold: "10" +``` + +When you enabled HTTPS in the endpoint and since resty-lua will return 500 error when processing "multipart" contents +Reference for this [issue](https://github.com/p0pr0ck5/lua-resty-waf/issues/166) + +By default, it will be "true" + +You may enable the following annotation for work around: + +```yaml +nginx.ingress.kubernetes.io/lua-resty-waf-process-multipart-body: "false" +``` + For details on how to write WAF rules, please refer to [https://github.com/p0pr0ck5/lua-resty-waf](https://github.com/p0pr0ck5/lua-resty-waf). [configmap]: ./configmap.md diff --git a/docs/user-guide/nginx-configuration/configmap.md b/docs/user-guide/nginx-configuration/configmap.md index d4cdc16c2..85615f26b 100644 --- a/docs/user-guide/nginx-configuration/configmap.md +++ b/docs/user-guide/nginx-configuration/configmap.md @@ -86,6 +86,7 @@ The following table shows a configuration option's name, type, and the default v |[proxy-protocol-header-timeout](#proxy-protocol-header-timeout)|string|"5s"| |[use-gzip](#use-gzip)|bool|"true"| |[use-geoip](#use-geoip)|bool|"true"| +|[use-geoip2](#use-geoip2)|bool|"false"| |[enable-brotli](#enable-brotli)|bool|"false"| |[brotli-level](#brotli-level)|int|4| |[brotli-types](#brotli-types)|string|"application/xml+rss application/atom+xml application/javascript application/x-javascript application/json application/rss+xml application/vnd.ms-fontobject application/x-font-ttf application/x-web-app-manifest+json application/xhtml+xml application/xml font/opentype image/svg+xml image/x-icon text/css text/plain text/x-component"| @@ -498,6 +499,13 @@ The default mime type list to compress is: `application/atom+xml application/jav Enables or disables ["geoip" module](http://nginx.org/en/docs/http/ngx_http_geoip_module.html) that creates variables with values depending on the client IP address, using the precompiled MaxMind databases. _**default:**_ true +> __Note:__ MaxMind legacy databases are discontinued and will not receive updates after 2019-01-02, cf. [discontinuation notice](https://support.maxmind.com/geolite-legacy-discontinuation-notice/). Consider [use-geoip2](#use-geoip2) below. + +## use-geoip2 + +Enables the [geoip2 module](https://github.com/leev/ngx_http_geoip2_module) for NGINX. +_**default:**_ false + ## enable-brotli Enables or disables compression of HTTP responses using the ["brotli" module](https://github.com/google/ngx_brotli). diff --git a/docs/user-guide/tls.md b/docs/user-guide/tls.md index 9b6961423..c468fa8ab 100644 --- a/docs/user-guide/tls.md +++ b/docs/user-guide/tls.md @@ -4,7 +4,7 @@ Anytime we reference a TLS secret, we mean a PEM-encoded X.509, RSA (2048) secret. -You can generate a self-signed certificate and private key with with: +You can generate a self-signed certificate and private key with: ```bash $ openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout ${KEY_FILE} -out ${CERT_FILE} -subj "/CN=${HOST}/O=${HOST}"` diff --git a/hack/build-single-manifest-sh b/hack/build-single-manifest-sh index dd29ab55c..0e236e782 100755 --- a/hack/build-single-manifest-sh +++ b/hack/build-single-manifest-sh @@ -20,7 +20,7 @@ set -o pipefail SCRIPT_ROOT=$(dirname ${BASH_SOURCE})/.. -INPUT="namespace.yaml configmap.yaml tcp-services-configmap.yaml udp-services-configmap.yaml rbac.yaml with-rbac.yaml" +INPUT="namespace.yaml configmap.yaml rbac.yaml with-rbac.yaml" MANIFEST=$(cd ${SCRIPT_ROOT}/deploy; cat ${INPUT}) echo "${MANIFEST}" > ${SCRIPT_ROOT}/deploy/mandatory.yaml diff --git a/images/e2e/Dockerfile b/images/e2e/Dockerfile index 42373160c..17e6c18df 100644 --- a/images/e2e/Dockerfile +++ b/images/e2e/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM quay.io/kubernetes-ingress-controller/nginx-amd64:0.63 +FROM quay.io/kubernetes-ingress-controller/nginx-amd64:0.66 RUN clean-install \ g++ \ @@ -61,3 +61,7 @@ RUN luarocks install luacheck \ RUN go get github.com/onsi/ginkgo/ginkgo \ && go get golang.org/x/lint/golint + +RUN curl -Lo /usr/local/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/v1.12.0/bin/linux/amd64/kubectl \ + && chmod +x /usr/local/bin/kubectl + diff --git a/images/nginx/Makefile b/images/nginx/Makefile index 500c961d3..6bc3979c2 100644 --- a/images/nginx/Makefile +++ b/images/nginx/Makefile @@ -13,7 +13,7 @@ # limitations under the License. # 0.0.0 shouldn't clobber any released builds -TAG ?= 0.65 +TAG ?= 0.66 REGISTRY ?= quay.io/kubernetes-ingress-controller ARCH ?= $(shell go env GOARCH) DOCKER ?= docker @@ -26,7 +26,7 @@ ifeq ($(GOHOSTOS),darwin) SED_I=sed -i '' endif -QEMUVERSION=v2.12.0-1 +QEMUVERSION=v3.0.0 IMGNAME = nginx IMAGE = $(REGISTRY)/$(IMGNAME) diff --git a/images/nginx/README.md b/images/nginx/README.md index 1f3588f9a..ca0d889f6 100644 --- a/images/nginx/README.md +++ b/images/nginx/README.md @@ -14,6 +14,7 @@ This custom nginx image contains: - [zipkin-cpp-opentracing](https://github.com/rnburn/zipkin-cpp-opentracing) - [ModSecurity-nginx](https://github.com/SpiderLabs/ModSecurity-nginx) (only supported in x86_64) - [brotli](https://github.com/google/brotli) +- [geoip2](https://github.com/leev/ngx_http_geoip2_module) **How to use this image:** This image provides a default configuration file with no backend servers. diff --git a/images/nginx/rootfs/build.sh b/images/nginx/rootfs/build.sh index b459c15d9..97ebe9eff 100755 --- a/images/nginx/rootfs/build.sh +++ b/images/nginx/rootfs/build.sh @@ -29,13 +29,13 @@ export NGINX_OPENTRACING_VERSION=0.6.0 export OPENTRACING_CPP_VERSION=1.5.0 export ZIPKIN_CPP_VERSION=0.5.2 export JAEGER_VERSION=ba0fa3fa6dbb01995d996f988a897e272100bf95 -export MODSECURITY_VERSION=37b76e88df4bce8a9846345c27271d7e6ce1acfb +export MODSECURITY_VERSION=56cfa4e4805bb6134b97143052a9b48919cc294f export LUA_NGX_VERSION=e94f2e5d64daa45ff396e262d8dab8e56f5f10e0 export LUA_UPSTREAM_VERSION=0.07 -export NGINX_INFLUXDB_VERSION=f20cfb2458c338f162132f5a21eb021e2cbe6383 -export GEOIP2_VERSION=3.0 +export NGINX_INFLUXDB_VERSION=0e2cb6cbf850a29c81e44be9e33d9a15d45c50e8 +export GEOIP2_VERSION=3.2 export NGINX_AJP_VERSION=bf6cd93f2098b59260de8d494f0f4b1f11a84627 -export LUAJIT_VERSION=8e35a1932250b0313c06393061f332c760efdf40 +export LUAJIT_VERSION=c58fe79b870f1934479bf14fe8035fc3d9fdfde2 export BUILD_PATH=/tmp/build @@ -53,10 +53,6 @@ get_src() rm -rf "$f" } -if [[ ${ARCH} == "ppc64le" ]]; then - clean-install software-properties-common -fi - apt-get update && apt-get dist-upgrade -y # install required packages to build @@ -94,8 +90,15 @@ clean-install \ dumb-init \ gdb \ valgrind \ + bc \ || exit 1 +if [[ ${ARCH} == "ppc64le" ]]; then + wget http://ftp.us.debian.org/debian/pool/main/a/apt/libapt-pkg5.0_1.7.0_ppc64el.deb + dpkg -i libapt-pkg5.0_1.7.0_ppc64el.deb + clean-install python3-apt python3-software-properties software-properties-common +fi + if [[ ${ARCH} == "x86_64" ]]; then ln -s /usr/lib/x86_64-linux-gnu/liblua5.1.so /usr/lib/liblua.so ln -s /usr/lib/x86_64-linux-gnu /usr/lib/lua-platform-path @@ -125,12 +128,23 @@ function geoip_get { wget -O $GEOIP_FOLDER/$1 $2 || { echo "Could not download $1, exiting." ; exit 1; } gunzip $GEOIP_FOLDER/$1 } +function geoip2_get { + wget -O $GEOIP_FOLDER/$1.tar.gz $2 || { echo "Could not download $1, exiting." ; exit 1; } + mkdir $GEOIP_FOLDER/$1 && tar xf $GEOIP_FOLDER/$1.tar.gz -C $GEOIP_FOLDER/$1 --strip-components 1 && mv $GEOIP_FOLDER/$1/$1.mmdb $GEOIP_FOLDER/$1.mmdb && rm -rf $GEOIP_FOLDER/$1 +} + +geoip_get "GeoIPASNum.dat.gz" "http://download.maxmind.com/download/geoip/database/asnum/GeoIPASNum.dat.gz" +geoip_get "GeoIP.dat.gz" "https://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz" +geoip_get "GeoLiteCity.dat.gz" "https://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz" +geoip2_get "GeoLite2-City" "http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz" +geoip2_get "GeoLite2-ASN" "http://geolite.maxmind.com/download/geoip/database/GeoLite2-ASN.tar.gz" + +if [[ (${ARCH} == "ppc64le") ]]; then + echo "deb http://deb.debian.org/debian experimental main" >> /etc/apt/sources.list + apt-get update + apt-get -t experimental install -y luajit +fi -geoip_get "GeoIPASNum.dat.gz" "http://download.maxmind.com/download/geoip/database/asnum/GeoIPASNum.dat.gz" -geoip_get "GeoIP.dat.gz" "https://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz" -geoip_get "GeoLite2-City.mmdb.gz" "http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz" -geoip_get "GeoLite2-ASN.mmdb.gz" "http://geolite.maxmind.com/download/geoip/database/GeoLite2-ASN.tar.gz" -geoip_get "GeoLiteCity.dat.gz" "https://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz" mkdir --verbose -p "$BUILD_PATH" cd "$BUILD_PATH" @@ -163,7 +177,7 @@ get_src 4455ca507936bc4b658ded10a90d8ebbbd61c58f06207be565a4ffdc885687b5 \ get_src 30affaf0f3a84193f7127cc0135da91773ce45d902414082273dae78914f73df \ "https://github.com/rnburn/zipkin-cpp-opentracing/archive/v$ZIPKIN_CPP_VERSION.tar.gz" -get_src fe7d3188e097d68f1942d46c4adba262d9ddcf433409ebc15bb5355bfb001a4a \ +get_src a75e3c0249c8ce4313d21b43d3cf3dcd89518dd6582ef7c6697cb7fe6ef5a84e \ "https://github.com/SpiderLabs/ModSecurity-nginx/archive/$MODSECURITY_VERSION.tar.gz" get_src b68286966f292fb552511b71bd8bc11af8f12c8aa760372d1437ac8760cb2f25 \ @@ -202,13 +216,13 @@ get_src a77bf0d7cf6a9ba017d0dc973b1a58f13e48242dd3849c5e99c07d250667c44c \ get_src d81b33129c6fb5203b571fa4d8394823bf473d8872c0357a1d0f14420b1483bd \ "https://github.com/cloudflare/lua-resty-cookie/archive/v0.1.0.tar.gz" -get_src 5a4485be0031d285f2bdf59afb1f7b8f3cef4c476595ed66f1258206e1b5c3ac \ +get_src 21dab7625a028d4560d0215c4bc3b82f6153344f933abb99dc9fd5f0d19519ab \ "https://github.com/openresty/luajit2/archive/$LUAJIT_VERSION.tar.gz" -get_src 1897d7677d99c1cedeb95b2eb00652a4a7e8e604304c3053a93bd3ba7dd82884 \ +get_src c673fcee37c1c4794f921b6710b09e8a0e1e58117aa788f798507d033f737192 \ "https://github.com/influxdata/nginx-influxdb-module/archive/$NGINX_INFLUXDB_VERSION.tar.gz" -get_src 65a191688348a05d8d92b2e7ce9c6eb8cb8322205c34637da582a1205864133d \ +get_src 15bd1005228cf2c869a6f09e8c41a6aaa6846e4936c473106786ae8ac860fab7 \ "https://github.com/leev/ngx_http_geoip2_module/archive/$GEOIP2_VERSION.tar.gz" get_src 5f629a50ba22347c441421091da70fdc2ac14586619934534e5a0f8a1390a950 \ @@ -221,21 +235,44 @@ export MAKEFLAGS=-j${CORES} export CTEST_BUILD_FLAGS=${MAKEFLAGS} export HUNTER_JOBS_NUMBER=${CORES} +OPENSSL_DIR="$BUILD_PATH/openssl" +mkdir -p $OPENSSL_DIR +cd $OPENSSL_DIR + +# Install Openssl 1.1.1 from source +wget http://http.debian.net/debian/pool/main/o/openssl/openssl_1.1.1-1.dsc +wget http://http.debian.net/debian/pool/main/o/openssl/openssl_1.1.1.orig.tar.gz +wget http://http.debian.net/debian/pool/main/o/openssl/openssl_1.1.1.orig.tar.gz.asc +wget http://http.debian.net/debian/pool/main/o/openssl/openssl_1.1.1-1.debian.tar.xz + +tar zxpvf openssl_1.1.1.orig.tar.gz +cd openssl-1.1.1/ +tar xpvf ../openssl_1.1.1-1.debian.tar.xz + +dpkg-buildpackage -rfakeroot + +cd .. + +dpkg -i openssl_1.1.1-1_*.deb libssl1.1_1.1.1-1_*.deb libssl-dev_1.1.1-1_*.deb + +# Install luajit from openresty fork export LUAJIT_LIB=/usr/local/lib +export LUA_LIB_DIR="$LUAJIT_LIB/lua" # luajit is available only as deb package on ppc64le -if [[ (${ARCH} == "ppc64le") ]]; then - clean-install luajit -else +if [[ (${ARCH} != "ppc64le") ]]; then cd "$BUILD_PATH/luajit2-$LUAJIT_VERSION" make CCDEBUG=-g make install export LUAJIT_INC=/usr/local/include/luajit-2.1 - export LUA_LIB_DIR="$LUAJIT_LIB/lua" fi # Installing luarocks packages +if [[ ${ARCH} == "x86_64" ]]; then + export PCRE_DIR=/usr/lib/x86_64-linux-gnu +fi + if [[ ${ARCH} == "armv7l" ]]; then export PCRE_DIR=/usr/lib/armhf-linux-gnu fi @@ -248,7 +285,8 @@ if [[ ${ARCH} == "ppc64le" ]]; then export PCRE_DIR=/usr/lib/powerpc64le-linux-gnu fi -luarocks install lrexlib-pcre 2.7.2-1 +cd "$BUILD_PATH" +luarocks install lrexlib-pcre 2.7.2-1 PCRE_LIBDIR=${PCRE_DIR} cd "$BUILD_PATH/lua-resty-core-0.1.15" make install @@ -326,14 +364,14 @@ EOF mkdir .build cd .build -cmake -DCMAKE_BUILD_TYPE=Release \ - -DBUILD_TESTING=OFF \ - -DJAEGERTRACING_BUILD_EXAMPLES=OFF \ - -DJAEGERTRACING_BUILD_CROSSDOCK=OFF \ - -DJAEGERTRACING_COVERAGE=OFF \ - -DJAEGERTRACING_PLUGIN=ON \ - -DHUNTER_CONFIGURATION_TYPES=Release \ - -DJAEGERTRACING_WITH_YAML_CPP=ON .. +cmake -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_TESTING=OFF \ + -DJAEGERTRACING_BUILD_EXAMPLES=OFF \ + -DJAEGERTRACING_BUILD_CROSSDOCK=OFF \ + -DJAEGERTRACING_COVERAGE=OFF \ + -DJAEGERTRACING_PLUGIN=ON \ + -DHUNTER_CONFIGURATION_TYPES=Release \ + -DJAEGERTRACING_WITH_YAML_CPP=ON .. make make install @@ -356,10 +394,10 @@ EOF mkdir .build cd .build -cmake -DCMAKE_BUILD_TYPE=Release \ - -DBUILD_SHARED_LIBS=ON \ - -DBUILD_PLUGIN=ON \ - -DBUILD_TESTING=OFF .. +cmake -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_SHARED_LIBS=ON \ + -DBUILD_PLUGIN=ON \ + -DBUILD_TESTING=OFF .. make make install @@ -375,9 +413,7 @@ git submodule update cd "$BUILD_PATH" git clone -b v3/master --single-branch https://github.com/SpiderLabs/ModSecurity cd ModSecurity/ -# TODO: use a tag once 3.0.3 is released -# checkout v3.0.3 -# git checkout +git checkout 973c1f1028429452308bcbce7df8a6283dc59ffe git submodule init git submodule update sh build.sh diff --git a/internal/ingress/annotations/backendprotocol/main.go b/internal/ingress/annotations/backendprotocol/main.go index 66dbae783..4f9417a2f 100644 --- a/internal/ingress/annotations/backendprotocol/main.go +++ b/internal/ingress/annotations/backendprotocol/main.go @@ -27,6 +27,9 @@ import ( "k8s.io/ingress-nginx/internal/ingress/resolver" ) +// HTTP protocol +const HTTP = "HTTP" + var ( validProtocols = regexp.MustCompile(`^(HTTP|HTTPS|AJP|GRPC|GRPCS)$`) ) @@ -44,18 +47,18 @@ func NewParser(r resolver.Resolver) parser.IngressAnnotation { // rule used to indicate the backend protocol. func (a backendProtocol) Parse(ing *extensions.Ingress) (interface{}, error) { if ing.GetAnnotations() == nil { - return "HTTP", nil + return HTTP, nil } proto, err := parser.GetStringAnnotation("backend-protocol", ing) if err != nil { - return "HTTP", nil + return HTTP, nil } proto = strings.TrimSpace(strings.ToUpper(proto)) if !validProtocols.MatchString(proto) { glog.Warningf("Protocol %v is not a valid value for the backend-protocol annotation. Using HTTP as protocol", proto) - return "HTTP", nil + return HTTP, nil } return proto, nil diff --git a/internal/ingress/annotations/luarestywaf/main.go b/internal/ingress/annotations/luarestywaf/main.go index ca7f4a8be..c29a97e9c 100644 --- a/internal/ingress/annotations/luarestywaf/main.go +++ b/internal/ingress/annotations/luarestywaf/main.go @@ -31,10 +31,13 @@ var luaRestyWAFModes = map[string]bool{"ACTIVE": true, "INACTIVE": true, "SIMULA // Config returns lua-resty-waf configuration for an Ingress rule type Config struct { - Mode string `json:"mode"` - Debug bool `json:"debug"` - IgnoredRuleSets []string `json:"ignored-rulesets"` - ExtraRulesetString string `json:"extra-ruleset-string"` + Mode string `json:"mode"` + Debug bool `json:"debug"` + IgnoredRuleSets []string `json:"ignored-rulesets"` + ExtraRulesetString string `json:"extra-ruleset-string"` + ScoreThreshold int `json:"score-threshold"` + AllowUnknownContentTypes bool `json:"allow-unknown-content-types"` + ProcessMultipartBody bool `json:"process-multipart-body"` } // Equal tests for equality between two Config types @@ -57,6 +60,15 @@ func (e1 *Config) Equal(e2 *Config) bool { if e1.ExtraRulesetString != e2.ExtraRulesetString { return false } + if e1.ScoreThreshold != e2.ScoreThreshold { + return false + } + if e1.AllowUnknownContentTypes != e2.AllowUnknownContentTypes { + return false + } + if e1.ProcessMultipartBody != e2.ProcessMultipartBody { + return false + } return true } @@ -95,10 +107,22 @@ func (a luarestywaf) Parse(ing *extensions.Ingress) (interface{}, error) { // TODO(elvinefendi) maybe validate the ruleset string here extraRulesetString, _ := parser.GetStringAnnotation("lua-resty-waf-extra-rules", ing) + scoreThreshold, _ := parser.GetIntAnnotation("lua-resty-waf-score-threshold", ing) + + allowUnknownContentTypes, _ := parser.GetBoolAnnotation("lua-resty-waf-allow-unknown-content-types", ing) + + processMultipartBody, err := parser.GetBoolAnnotation("lua-resty-waf-process-multipart-body", ing) + if err != nil { + processMultipartBody = true + } + return &Config{ - Mode: mode, - Debug: debug, - IgnoredRuleSets: ignoredRuleSets, - ExtraRulesetString: extraRulesetString, + Mode: mode, + Debug: debug, + IgnoredRuleSets: ignoredRuleSets, + ExtraRulesetString: extraRulesetString, + ScoreThreshold: scoreThreshold, + AllowUnknownContentTypes: allowUnknownContentTypes, + ProcessMultipartBody: processMultipartBody, }, nil } diff --git a/internal/ingress/annotations/luarestywaf/main_test.go b/internal/ingress/annotations/luarestywaf/main_test.go index d71191f12..60b80e22f 100644 --- a/internal/ingress/annotations/luarestywaf/main_test.go +++ b/internal/ingress/annotations/luarestywaf/main_test.go @@ -30,6 +30,9 @@ func TestParse(t *testing.T) { luaRestyWAFAnnotation := parser.GetAnnotationWithPrefix("lua-resty-waf") luaRestyWAFDebugAnnotation := parser.GetAnnotationWithPrefix("lua-resty-waf-debug") luaRestyWAFIgnoredRuleSetsAnnotation := parser.GetAnnotationWithPrefix("lua-resty-waf-ignore-rulesets") + luaRestyWAFScoreThresholdAnnotation := parser.GetAnnotationWithPrefix("lua-resty-waf-score-threshold") + luaRestyWAFAllowUnknownContentTypesAnnotation := parser.GetAnnotationWithPrefix("lua-resty-waf-allow-unknown-content-types") + luaRestyWAFProcessMultipartBody := parser.GetAnnotationWithPrefix("lua-resty-waf-process-multipart-body") ap := NewParser(&resolver.Mock{}) if ap == nil { @@ -43,21 +46,25 @@ func TestParse(t *testing.T) { {nil, &Config{}}, {map[string]string{}, &Config{}}, - {map[string]string{luaRestyWAFAnnotation: "active"}, &Config{Mode: "ACTIVE", Debug: false, IgnoredRuleSets: []string{}}}, + {map[string]string{luaRestyWAFAnnotation: "active"}, &Config{Mode: "ACTIVE", Debug: false, IgnoredRuleSets: []string{}, ProcessMultipartBody: true}}, {map[string]string{luaRestyWAFDebugAnnotation: "true"}, &Config{Debug: false}}, - {map[string]string{luaRestyWAFAnnotation: "active", luaRestyWAFDebugAnnotation: "true"}, &Config{Mode: "ACTIVE", Debug: true, IgnoredRuleSets: []string{}}}, - {map[string]string{luaRestyWAFAnnotation: "active", luaRestyWAFDebugAnnotation: "false"}, &Config{Mode: "ACTIVE", Debug: false, IgnoredRuleSets: []string{}}}, - {map[string]string{luaRestyWAFAnnotation: "inactive", luaRestyWAFDebugAnnotation: "true"}, &Config{Mode: "INACTIVE", Debug: true, IgnoredRuleSets: []string{}}}, + {map[string]string{luaRestyWAFAnnotation: "active", luaRestyWAFDebugAnnotation: "true"}, &Config{Mode: "ACTIVE", Debug: true, IgnoredRuleSets: []string{}, ProcessMultipartBody: true}}, + {map[string]string{luaRestyWAFAnnotation: "active", luaRestyWAFDebugAnnotation: "false"}, &Config{Mode: "ACTIVE", Debug: false, IgnoredRuleSets: []string{}, ProcessMultipartBody: true}}, + {map[string]string{luaRestyWAFAnnotation: "inactive", luaRestyWAFDebugAnnotation: "true"}, &Config{Mode: "INACTIVE", Debug: true, IgnoredRuleSets: []string{}, ProcessMultipartBody: true}}, {map[string]string{ - luaRestyWAFAnnotation: "active", - luaRestyWAFDebugAnnotation: "true", - luaRestyWAFIgnoredRuleSetsAnnotation: "ruleset1, ruleset2 ruleset3, another.ruleset"}, - &Config{Mode: "ACTIVE", Debug: true, IgnoredRuleSets: []string{"ruleset1", "ruleset2", "ruleset3", "another.ruleset"}}}, + luaRestyWAFAnnotation: "active", + luaRestyWAFDebugAnnotation: "true", + luaRestyWAFIgnoredRuleSetsAnnotation: "ruleset1, ruleset2 ruleset3, another.ruleset", + luaRestyWAFScoreThresholdAnnotation: "10", + luaRestyWAFAllowUnknownContentTypesAnnotation: "true"}, + &Config{Mode: "ACTIVE", Debug: true, IgnoredRuleSets: []string{"ruleset1", "ruleset2", "ruleset3", "another.ruleset"}, ScoreThreshold: 10, AllowUnknownContentTypes: true, ProcessMultipartBody: true}}, - {map[string]string{luaRestyWAFAnnotation: "siMulate", luaRestyWAFDebugAnnotation: "true"}, &Config{Mode: "SIMULATE", Debug: true, IgnoredRuleSets: []string{}}}, + {map[string]string{luaRestyWAFAnnotation: "siMulate", luaRestyWAFDebugAnnotation: "true"}, &Config{Mode: "SIMULATE", Debug: true, IgnoredRuleSets: []string{}, ProcessMultipartBody: true}}, {map[string]string{luaRestyWAFAnnotation: "siMulateX", luaRestyWAFDebugAnnotation: "true"}, &Config{Debug: false}}, + + {map[string]string{luaRestyWAFAnnotation: "active", luaRestyWAFProcessMultipartBody: "false"}, &Config{Mode: "ACTIVE", ProcessMultipartBody: false, IgnoredRuleSets: []string{}}}, } ing := &extensions.Ingress{ diff --git a/internal/ingress/annotations/rewrite/main.go b/internal/ingress/annotations/rewrite/main.go index 270bdb1a7..b1e63bc37 100644 --- a/internal/ingress/annotations/rewrite/main.go +++ b/internal/ingress/annotations/rewrite/main.go @@ -39,7 +39,7 @@ type Config struct { // AppRoot defines the Application Root that the Controller must redirect if it's in '/' context AppRoot string `json:"appRoot"` // UseRegex indicates whether or not the locations use regex paths - UseRegex bool `json:useRegex` + UseRegex bool `json:"useRegex"` } // Equal tests for equality between two Redirect types diff --git a/internal/ingress/annotations/rewrite/main_test.go b/internal/ingress/annotations/rewrite/main_test.go index 71721ad93..899379826 100644 --- a/internal/ingress/annotations/rewrite/main_test.go +++ b/internal/ingress/annotations/rewrite/main_test.go @@ -191,7 +191,7 @@ func TestUseRegex(t *testing.T) { if !ok { t.Errorf("expected a App Context") } - if redirect.UseRegex != true { + if !redirect.UseRegex { t.Errorf("Unexpected value got in UseRegex") } } diff --git a/internal/ingress/controller/checker.go b/internal/ingress/controller/checker.go index b595dd532..6dd6b3be0 100644 --- a/internal/ingress/controller/checker.go +++ b/internal/ingress/controller/checker.go @@ -38,7 +38,8 @@ func (n NGINXController) Name() string { func (n *NGINXController) Check(_ *http.Request) error { url := fmt.Sprintf("http://127.0.0.1:%v%v", n.cfg.ListenPorts.Status, ngxHealthPath) - statusCode, err := simpleGet(url) + timeout := n.cfg.HealthCheckTimeout + statusCode, err := simpleGet(url, timeout) if err != nil { return err } @@ -48,7 +49,7 @@ func (n *NGINXController) Check(_ *http.Request) error { } url = fmt.Sprintf("http://127.0.0.1:%v/is-dynamic-lb-initialized", n.cfg.ListenPorts.Status) - statusCode, err = simpleGet(url) + statusCode, err = simpleGet(url, timeout) if err != nil { return err } @@ -75,9 +76,9 @@ func (n *NGINXController) Check(_ *http.Request) error { return err } -func simpleGet(url string) (int, error) { +func simpleGet(url string, timeout time.Duration) (int, error) { client := &http.Client{ - Timeout: 10 * time.Second, + Timeout: timeout * time.Second, Transport: &http.Transport{DisableKeepAlives: true}, } diff --git a/internal/ingress/controller/config/config.go b/internal/ingress/controller/config/config.go index 74cfac4cf..2089ec995 100644 --- a/internal/ingress/controller/config/config.go +++ b/internal/ingress/controller/config/config.go @@ -102,10 +102,10 @@ type Configuration struct { // By default access logs go to /var/log/nginx/access.log AccessLogPath string `json:"access-log-path,omitempty"` - // WorkerCpuAffinity bind nginx worker processes to CPUs this will improve response latency + // WorkerCPUAffinity bind nginx worker processes to CPUs this will improve response latency // http://nginx.org/en/docs/ngx_core_module.html#worker_cpu_affinity // By default this is disabled - WorkerCpuAffinity string `json:"worker-cpu-affinity,omitempty"` + WorkerCPUAffinity string `json:"worker-cpu-affinity,omitempty"` // ErrorLogPath sets the path of the error logs // http://nginx.org/en/docs/ngx_core_module.html#error_log // By default error logs go to /var/log/nginx/error.log @@ -347,6 +347,10 @@ type Configuration struct { // http://nginx.org/en/docs/http/ngx_http_geoip_module.html UseGeoIP bool `json:"use-geoip,omitempty"` + // UseGeoIP2 enables the geoip2 module for NGINX + // By default this is disabled + UseGeoIP2 bool `json:"use-geoip2,omitempty"` + // Enables or disables the use of the NGINX Brotli Module for compression // https://github.com/google/ngx_brotli EnableBrotli bool `json:"enable-brotli,omitempty"` @@ -438,11 +442,11 @@ type Configuration struct { // If the request does not have a request-id, should we generate a random value? // Default: true - GenerateRequestId bool `json:"generate-request-id,omitempty"` + GenerateRequestID bool `json:"generate-request-id,omitempty"` // Adds an X-Original-Uri header with the original request URI to the backend request // Default: true - ProxyAddOriginalUriHeader bool `json:"proxy-add-original-uri-header"` + ProxyAddOriginalURIHeader bool `json:"proxy-add-original-uri-header"` // EnableOpentracing enables the nginx Opentracing extension // https://github.com/opentracing-contrib/nginx-opentracing @@ -570,7 +574,7 @@ func NewDefault() Configuration { cfg := Configuration{ AllowBackendServerHeader: false, AccessLogPath: "/var/log/nginx/access.log", - WorkerCpuAffinity: "", + WorkerCPUAffinity: "", ErrorLogPath: "/var/log/nginx/error.log", BlockCIDRs: defBlockEntity, BlockUserAgents: defBlockEntity, @@ -587,8 +591,8 @@ func NewDefault() Configuration { UseForwardedHeaders: true, ForwardedForHeader: "X-Forwarded-For", ComputeFullForwardedFor: false, - ProxyAddOriginalUriHeader: true, - GenerateRequestId: true, + ProxyAddOriginalURIHeader: true, + GenerateRequestID: true, HTTP2MaxFieldSize: "4k", HTTP2MaxHeaderSize: "16k", HTTP2MaxRequests: 1000, @@ -630,6 +634,7 @@ func NewDefault() Configuration { EnableBrotli: false, UseGzip: true, UseGeoIP: true, + UseGeoIP2: false, WorkerProcesses: strconv.Itoa(runtime.NumCPU()), WorkerShutdownTimeout: "10s", LoadBalanceAlgorithm: defaultLoadBalancerAlgorithm, diff --git a/internal/ingress/controller/controller.go b/internal/ingress/controller/controller.go index e211db233..5739f805b 100644 --- a/internal/ingress/controller/controller.go +++ b/internal/ingress/controller/controller.go @@ -61,6 +61,7 @@ type Configuration struct { ForceNamespaceIsolation bool DefaultHealthzURL string + HealthCheckTimeout time.Duration DefaultSSLCertificate string // +optional diff --git a/internal/ingress/controller/nginx.go b/internal/ingress/controller/nginx.go index 9c5b398be..f99b8d58e 100644 --- a/internal/ingress/controller/nginx.go +++ b/internal/ingress/controller/nginx.go @@ -18,7 +18,6 @@ package controller import ( "bytes" - "context" "encoding/json" "errors" "fmt" @@ -257,7 +256,7 @@ func (n *NGINXController) Start() { n.store.Run(n.stopCh) if n.syncStatus != nil { - go n.syncStatus.Run(context.Background()) + go n.syncStatus.Run() } cmd := nginxExecCommand() @@ -812,12 +811,7 @@ func configureCertificates(pcfg *ingress.Configuration, port int) error { } url := fmt.Sprintf("http://localhost:%d/configuration/servers", port) - err := post(url, servers) - if err != nil { - return err - } - - return nil + return post(url, servers) } func post(url string, data interface{}) error { diff --git a/internal/ingress/controller/template/configmap_test.go b/internal/ingress/controller/template/configmap_test.go index 1bb7b773b..535f105a9 100644 --- a/internal/ingress/controller/template/configmap_test.go +++ b/internal/ingress/controller/template/configmap_test.go @@ -90,7 +90,7 @@ func TestMergeConfigMapToStruct(t *testing.T) { def.WorkerShutdownTimeout = "99s" def.NginxStatusIpv4Whitelist = []string{"127.0.0.1", "10.0.0.0/24"} def.NginxStatusIpv6Whitelist = []string{"::1", "2001::/16"} - def.ProxyAddOriginalUriHeader = false + def.ProxyAddOriginalURIHeader = false hash, err := hashstructure.Hash(def, &hashstructure.HashOptions{ TagName: "json", diff --git a/internal/ingress/controller/template/template.go b/internal/ingress/controller/template/template.go index e6df56282..0f8b3ee65 100644 --- a/internal/ingress/controller/template/template.go +++ b/internal/ingress/controller/template/template.go @@ -209,8 +209,6 @@ func buildLuaSharedDictionaries(s interface{}, disableLuaRestyWAF bool) string { "lua_shared_dict configuration_data 5M", "lua_shared_dict certificate_data 16M", "lua_shared_dict locks 512k", - "lua_shared_dict balancer_ewma 1M", - "lua_shared_dict balancer_ewma_last_touched_at 1M", "lua_shared_dict sticky_sessions 1M", } @@ -724,19 +722,6 @@ func buildUpstreamName(loc interface{}) string { return upstreamName } -// TODO: Needs Unit Tests -func isSticky(host string, loc *ingress.Location, stickyLocations map[string][]string) bool { - if _, ok := stickyLocations[host]; ok { - for _, sl := range stickyLocations[host] { - if sl == loc.Path { - return true - } - } - } - - return false -} - func buildNextUpstream(i, r interface{}) string { nextUpstream, ok := i.(string) if !ok { diff --git a/internal/ingress/metric/collectors/socket.go b/internal/ingress/metric/collectors/socket.go index 1f30c9b72..904921d12 100644 --- a/internal/ingress/metric/collectors/socket.go +++ b/internal/ingress/metric/collectors/socket.go @@ -171,7 +171,7 @@ func NewSocketCollector(pod, namespace, class string) (*SocketCollector, error) bytesSent: prometheus.NewHistogramVec( prometheus.HistogramOpts{ Name: "bytes_sent", - Help: "The the number of bytes sent to a client", + Help: "The number of bytes sent to a client", Namespace: PrometheusNamespace, Buckets: prometheus.ExponentialBuckets(10, 10, 7), // 7 buckets, exponential factor of 10. ConstLabels: constLabels, diff --git a/internal/ingress/status/status.go b/internal/ingress/status/status.go index 6b152c80a..320ad7c0a 100644 --- a/internal/ingress/status/status.go +++ b/internal/ingress/status/status.go @@ -51,7 +51,7 @@ const ( // Sync ... type Sync interface { - Run(ctx context.Context) + Run() Shutdown() } @@ -93,22 +93,102 @@ type statusSync struct { pod *k8s.PodInfo elector *leaderelection.LeaderElector + // workqueue used to keep in sync the status IP/s // in the Ingress rules syncQueue *task.Queue } // Run starts the loop to keep the status in sync -func (s statusSync) Run(ctx context.Context) { - s.elector.Run(ctx) +func (s statusSync) Run() { + // we need to use the defined ingress class to allow multiple leaders + // in order to update information about ingress status + electionID := fmt.Sprintf("%v-%v", s.Config.ElectionID, s.Config.DefaultIngressClass) + if s.Config.IngressClass != "" { + electionID = fmt.Sprintf("%v-%v", s.Config.ElectionID, s.Config.IngressClass) + } + + // start a new context + ctx := context.Background() + + var cancelContext context.CancelFunc + + var newLeaderCtx = func(ctx context.Context) context.CancelFunc { + // allow to cancel the context in case we stop being the leader + leaderCtx, cancel := context.WithCancel(ctx) + go s.elector.Run(leaderCtx) + return cancel + } + + var stopCh chan struct{} + callbacks := leaderelection.LeaderCallbacks{ + OnStartedLeading: func(ctx context.Context) { + glog.V(2).Infof("I am the new status update leader") + stopCh = make(chan struct{}) + go s.syncQueue.Run(time.Second, stopCh) + // trigger initial sync + s.syncQueue.EnqueueTask(task.GetDummyObject("sync status")) + // when this instance is the leader we need to enqueue + // an item to trigger the update of the Ingress status. + wait.PollUntil(updateInterval, func() (bool, error) { + s.syncQueue.EnqueueTask(task.GetDummyObject("sync status")) + return false, nil + }, stopCh) + }, + OnStoppedLeading: func() { + glog.V(2).Infof("I am not status update leader anymore") + close(stopCh) + + // cancel the context + cancelContext() + + cancelContext = newLeaderCtx(ctx) + }, + OnNewLeader: func(identity string) { + glog.Infof("new leader elected: %v", identity) + }, + } + + broadcaster := record.NewBroadcaster() + hostname, _ := os.Hostname() + + recorder := broadcaster.NewRecorder(scheme.Scheme, apiv1.EventSource{ + Component: "ingress-leader-elector", + Host: hostname, + }) + + lock := resourcelock.ConfigMapLock{ + ConfigMapMeta: metav1.ObjectMeta{Namespace: s.pod.Namespace, Name: electionID}, + Client: s.Config.Client.CoreV1(), + LockConfig: resourcelock.ResourceLockConfig{ + Identity: s.pod.Name, + EventRecorder: recorder, + }, + } + + ttl := 30 * time.Second + var err error + s.elector, err = leaderelection.NewLeaderElector(leaderelection.LeaderElectionConfig{ + Lock: &lock, + LeaseDuration: ttl, + RenewDeadline: ttl / 2, + RetryPeriod: ttl / 4, + Callbacks: callbacks, + }) + if err != nil { + glog.Fatalf("unexpected error starting leader election: %v", err) + } + + cancelContext = newLeaderCtx(ctx) } // Shutdown stop the sync. In case the instance is the leader it will remove the current IP // if there is no other instances running. func (s statusSync) Shutdown() { go s.syncQueue.Shutdown() + // remove IP from Ingress - if !s.elector.IsLeader() { + if s.elector != nil && !s.elector.IsLeader() { return } @@ -146,6 +226,10 @@ func (s *statusSync) sync(key interface{}) error { return nil } + if s.elector != nil && !s.elector.IsLeader() { + return fmt.Errorf("i am not the current leader. Skiping status update") + } + addrs, err := s.runningAddresses() if err != nil { return err @@ -173,66 +257,6 @@ func NewStatusSyncer(config Config) Sync { } st.syncQueue = task.NewCustomTaskQueue(st.sync, st.keyfunc) - // we need to use the defined ingress class to allow multiple leaders - // in order to update information about ingress status - electionID := fmt.Sprintf("%v-%v", config.ElectionID, config.DefaultIngressClass) - if config.IngressClass != "" { - electionID = fmt.Sprintf("%v-%v", config.ElectionID, config.IngressClass) - } - - var stopCh chan struct{} - callbacks := leaderelection.LeaderCallbacks{ - OnStartedLeading: func(ctx context.Context) { - glog.V(2).Infof("I am the new status update leader") - stopCh = make(chan struct{}) - go st.syncQueue.Run(time.Second, stopCh) - // when this instance is the leader we need to enqueue - // an item to trigger the update of the Ingress status. - wait.PollUntil(updateInterval, func() (bool, error) { - st.syncQueue.EnqueueTask(task.GetDummyObject("sync status")) - return false, nil - }, stopCh) - }, - OnStoppedLeading: func() { - glog.V(2).Infof("I am not status update leader anymore") - close(stopCh) - }, - OnNewLeader: func(identity string) { - glog.Infof("new leader elected: %v", identity) - }, - } - - broadcaster := record.NewBroadcaster() - hostname, _ := os.Hostname() - - recorder := broadcaster.NewRecorder(scheme.Scheme, apiv1.EventSource{ - Component: "ingress-leader-elector", - Host: hostname, - }) - - lock := resourcelock.ConfigMapLock{ - ConfigMapMeta: metav1.ObjectMeta{Namespace: pod.Namespace, Name: electionID}, - Client: config.Client.CoreV1(), - LockConfig: resourcelock.ResourceLockConfig{ - Identity: pod.Name, - EventRecorder: recorder, - }, - } - - ttl := 30 * time.Second - le, err := leaderelection.NewLeaderElector(leaderelection.LeaderElectionConfig{ - Lock: &lock, - LeaseDuration: ttl, - RenewDeadline: ttl / 2, - RetryPeriod: ttl / 4, - Callbacks: callbacks, - }) - - if err != nil { - glog.Fatalf("unexpected error starting leader election: %v", err) - } - - st.elector = le return st } @@ -333,6 +357,13 @@ func (s *statusSync) updateStatus(newIngressPoint []apiv1.LoadBalancerIngress) { sort.SliceStable(newIngressPoint, lessLoadBalancerIngress(newIngressPoint)) for _, ing := range ings { + curIPs := ing.Status.LoadBalancer.Ingress + sort.SliceStable(curIPs, lessLoadBalancerIngress(curIPs)) + if ingressSliceEqual(curIPs, newIngressPoint) { + glog.V(3).Infof("skipping update of Ingress %v/%v (no change)", ing.Namespace, ing.Name) + continue + } + batch.Queue(runUpdate(ing, newIngressPoint, s.Client)) } @@ -347,14 +378,6 @@ func runUpdate(ing *extensions.Ingress, status []apiv1.LoadBalancerIngress, return nil, nil } - curIPs := ing.Status.LoadBalancer.Ingress - sort.SliceStable(curIPs, lessLoadBalancerIngress(curIPs)) - - if ingressSliceEqual(status, curIPs) { - glog.V(3).Infof("skipping update of Ingress %v/%v (no change)", ing.Namespace, ing.Name) - return true, nil - } - ingClient := client.ExtensionsV1beta1().Ingresses(ing.Namespace) currIng, err := ingClient.Get(ing.Name, metav1.GetOptions{}) @@ -398,5 +421,6 @@ func ingressSliceEqual(lhs, rhs []apiv1.LoadBalancerIngress) bool { return false } } + return true } diff --git a/internal/ingress/status/status_test.go b/internal/ingress/status/status_test.go index 7234193ff..c4f223ec5 100644 --- a/internal/ingress/status/status_test.go +++ b/internal/ingress/status/status_test.go @@ -17,7 +17,6 @@ limitations under the License. package status import ( - "context" "os" "testing" "time" @@ -298,7 +297,7 @@ func TestStatusActions(t *testing.T) { fk := fkSync.(statusSync) // start it and wait for the election and syn actions - go fk.Run(context.Background()) + go fk.Run() // wait for the election time.Sleep(100 * time.Millisecond) // execute sync diff --git a/rootfs/etc/nginx/lua/balancer/ewma.lua b/rootfs/etc/nginx/lua/balancer/ewma.lua index 270e990ca..352b654f7 100644 --- a/rootfs/etc/nginx/lua/balancer/ewma.lua +++ b/rootfs/etc/nginx/lua/balancer/ewma.lua @@ -5,38 +5,17 @@ -- /finagle-core/src/main/scala/com/twitter/finagle/loadbalancer/PeakEwma.scala -local resty_lock = require("resty.lock") local util = require("util") local split = require("util.split") local DECAY_TIME = 10 -- this value is in seconds -local LOCK_KEY = ":ewma_key" local PICK_SET_SIZE = 2 -local ewma_lock = resty_lock:new("locks", {timeout = 0, exptime = 0.1}) +local balancer_ewma = {} +local balancer_ewma_last_touched_at = {} local _M = { name = "ewma" } -local function lock(upstream) - local _, err = ewma_lock:lock(upstream .. LOCK_KEY) - if err then - if err ~= "timeout" then - ngx.log(ngx.ERR, string.format("EWMA Balancer failed to lock: %s", tostring(err))) - end - end - - return err -end - -local function unlock() - local ok, err = ewma_lock:unlock() - if not ok then - ngx.log(ngx.ERR, string.format("EWMA Balancer failed to unlock: %s", tostring(err))) - end - - return err -end - local function decay_ewma(ewma, last_touched_at, rtt, now) local td = now - last_touched_at td = (td > 0) and td or 0 @@ -47,40 +26,18 @@ local function decay_ewma(ewma, last_touched_at, rtt, now) end local function get_or_update_ewma(upstream, rtt, update) - local lock_err = nil - if update then - lock_err = lock(upstream) - end - local ewma = ngx.shared.balancer_ewma:get(upstream) or 0 - if lock_err ~= nil then - return ewma, lock_err - end + local ewma = balancer_ewma[upstream] or 0 local now = ngx.now() - local last_touched_at = ngx.shared.balancer_ewma_last_touched_at:get(upstream) or 0 + local last_touched_at = balancer_ewma_last_touched_at[upstream] or 0 ewma = decay_ewma(ewma, last_touched_at, rtt, now) if not update then return ewma, nil end - local success, err, forcible = ngx.shared.balancer_ewma_last_touched_at:set(upstream, now) - if not success then - ngx.log(ngx.WARN, "balancer_ewma_last_touched_at:set failed " .. err) - end - if forcible then - ngx.log(ngx.WARN, "balancer_ewma_last_touched_at:set valid items forcibly overwritten") - end - - success, err, forcible = ngx.shared.balancer_ewma:set(upstream, ewma) - if not success then - ngx.log(ngx.WARN, "balancer_ewma:set failed " .. err) - end - if forcible then - ngx.log(ngx.WARN, "balancer_ewma:set valid items forcibly overwritten") - end - - unlock() + balancer_ewma_last_touched_at[upstream] = now + balancer_ewma[upstream] = ewma return ewma, nil end @@ -153,8 +110,8 @@ function _M.sync(self, backend) self.peers = backend.endpoints -- TODO: Reset state of EWMA per backend - ngx.shared.balancer_ewma:flush_all() - ngx.shared.balancer_ewma_last_touched_at:flush_all() + balancer_ewma = {} + balancer_ewma_last_touched_at = {} end function _M.new(self, backend) diff --git a/rootfs/etc/nginx/lua/balancer/sticky.lua b/rootfs/etc/nginx/lua/balancer/sticky.lua index 1ba695258..fd1af8ed3 100644 --- a/rootfs/etc/nginx/lua/balancer/sticky.lua +++ b/rootfs/etc/nginx/lua/balancer/sticky.lua @@ -94,22 +94,17 @@ local function set_cookie(self, value) end end -local function pick_random(instance) - local index = math.random(instance.npoints) - return instance:next(index) -end - function _M.balance(self) local cookie, err = ck:new() if not cookie then - ngx.log(ngx.ERR, err) - return pick_random(self.instance) + ngx.log(ngx.ERR, "error while initializing cookie: " .. tostring(err)) + return end local key = cookie:get(self.cookie_name) if not key then - local tmp_endpoint_string = pick_random(self.instance) - key = encrypted_endpoint_string(self, tmp_endpoint_string) + local random_str = string.format("%s.%s", ngx.now(), ngx.worker.pid()) + key = encrypted_endpoint_string(self, random_str) set_cookie(self, key) end diff --git a/rootfs/etc/nginx/lua/test/balancer/ewma_test.lua b/rootfs/etc/nginx/lua/test/balancer/ewma_test.lua index 920df1a92..d7cb8cab6 100644 --- a/rootfs/etc/nginx/lua/test/balancer/ewma_test.lua +++ b/rootfs/etc/nginx/lua/test/balancer/ewma_test.lua @@ -26,10 +26,6 @@ describe("Balancer ewma", function() local instance = balancer_ewma:new(backend) local stats = { ["10.184.7.40:8080"] = 0.5, ["10.184.97.100:8080"] = 0.3 } - ngx.shared.balancer_ewma.get = function(self, key) return stats[key] end - local t = ngx.now()-10 - ngx.shared.balancer_ewma_last_touched_at.get = function(self, key) return t end - local peer = instance:balance() assert.equal("10.184.97.100:8080", peer) @@ -52,13 +48,7 @@ describe("Balancer ewma", function() endpoints = { { address = "10.184.7.40", port = "8080", maxFails = 0, failTimeout = 0 } } } - local s1 = spy.on(ngx.shared.balancer_ewma, "flush_all") - local s2 = spy.on(ngx.shared.balancer_ewma_last_touched_at, "flush_all") - instance:sync(new_backend) - - assert.spy(s1).was_not_called() - assert.spy(s2).was_not_called() end) it("updates endpoints", function() @@ -77,13 +67,7 @@ describe("Balancer ewma", function() local new_backend = util.deepcopy(backend) new_backend.endpoints[1].maxFails = 3 - local s1 = spy.on(ngx.shared.balancer_ewma, "flush_all") - local s2 = spy.on(ngx.shared.balancer_ewma_last_touched_at, "flush_all") - instance:sync(new_backend) - - assert.spy(s1).was_called() - assert.spy(s2).was_called() end) end) end) diff --git a/rootfs/etc/nginx/lua/test/balancer/sticky_test.lua b/rootfs/etc/nginx/lua/test/balancer/sticky_test.lua index 87f0fc186..a6b4652ad 100644 --- a/rootfs/etc/nginx/lua/test/balancer/sticky_test.lua +++ b/rootfs/etc/nginx/lua/test/balancer/sticky_test.lua @@ -116,7 +116,8 @@ describe("Sticky", function() local cookie_instance = { set = function(self, payload) assert.equal(payload.key, test_backend.sessionAffinityConfig.cookieSessionAffinity.name) - assert.equal(payload.value, util[test_backend_hash_fn .. "_digest"](test_backend_endpoint)) + local expected_len = #util[test_backend_hash_fn .. "_digest"]("anything") + assert.equal(#payload.value, expected_len) assert.equal(payload.path, ngx.var.location_path) assert.equal(payload.domain, nil) assert.equal(payload.httponly, true) diff --git a/rootfs/etc/nginx/template/nginx.tmpl b/rootfs/etc/nginx/template/nginx.tmpl index a7c2c7287..82370bba7 100644 --- a/rootfs/etc/nginx/template/nginx.tmpl +++ b/rootfs/etc/nginx/template/nginx.tmpl @@ -12,6 +12,10 @@ # setup custom paths that do not require root access pid /tmp/nginx.pid; +{{ if $cfg.UseGeoIP2 }} +load_module /etc/nginx/modules/ngx_http_geoip2_module.so; +{{ end }} + {{ if $cfg.EnableModsecurity }} load_module /etc/nginx/modules/ngx_http_modsecurity_module.so; {{ end }} @@ -23,8 +27,8 @@ load_module /etc/nginx/modules/ngx_http_opentracing_module.so; daemon off; worker_processes {{ $cfg.WorkerProcesses }}; -{{ if gt (len $cfg.WorkerCpuAffinity) 0 }} -worker_cpu_affinity {{ $cfg.WorkerCpuAffinity }}; +{{ if gt (len $cfg.WorkerCPUAffinity) 0 }} +worker_cpu_affinity {{ $cfg.WorkerCPUAffinity }}; {{ end }} {{ if ne .MaxOpenFiles 0 }} @@ -123,6 +127,26 @@ http { geoip_proxy_recursive on; {{ end }} + {{ if $cfg.UseGeoIP2 }} + # https://github.com/leev/ngx_http_geoip2_module#example-usage + + geoip2 /etc/nginx/geoip/GeoLite2-City.mmdb { + $geoip2_city_country_code source=$the_real_ip country iso_code; + $geoip2_city_country_name source=$the_real_ip country names en; + $geoip2_city source=$the_real_ip city names en; + $geoip2_postal_code source=$the_real_ip postal code; + $geoip2_dma_code source=$the_real_ip location metro_code; + $geoip2_latitude source=$the_real_ip location latitude; + $geoip2_longitude source=$the_real_ip location longitude; + $geoip2_region_code source=$the_real_ip subdivisions 0 iso_code; + $geoip2_region_name source=$the_real_ip subdivisions 0 names en; + } + + geoip2 /etc/nginx/geoip/GeoLite2-ASN.mmdb { + $geoip2_asn source=$the_real_ip autonomous_system_number; + } + {{ end }} + aio threads; aio_write on; @@ -322,7 +346,7 @@ http { # If no such header is provided, it can provide a random value. map $http_x_request_id $req_id { default $http_x_request_id; - {{ if $cfg.GenerateRequestId }} + {{ if $cfg.GenerateRequestID }} "" $request_id; {{ end }} } @@ -891,9 +915,23 @@ stream { waf:set_option("mode", "{{ $location.LuaRestyWAF.Mode }}") waf:set_option("storage_zone", "waf_storage") + + {{ if $location.LuaRestyWAF.AllowUnknownContentTypes }} + waf:set_option("allow_unknown_content_types", true) + {{ else }} waf:set_option("allowed_content_types", { "text/html", "text/json", "application/json" }) + {{ end }} + waf:set_option("event_log_level", ngx.WARN) + {{ if gt $location.LuaRestyWAF.ScoreThreshold 0 }} + waf:set_option("score_threshold", {{ $location.LuaRestyWAF.ScoreThreshold }}) + {{ end }} + + {{ if not $location.LuaRestyWAF.ProcessMultipartBody }} + waf:set_option("process_multipart_body", false) + {{ end }} + {{ if $location.LuaRestyWAF.Debug }} waf:set_option("debug", true) waf:set_option("event_log_request_arguments", true) @@ -1077,7 +1115,7 @@ stream { {{ $proxySetHeader }} X-Forwarded-Host $best_http_host; {{ $proxySetHeader }} X-Forwarded-Port $pass_port; {{ $proxySetHeader }} X-Forwarded-Proto $pass_access_scheme; - {{ if $all.Cfg.ProxyAddOriginalUriHeader }} + {{ if $all.Cfg.ProxyAddOriginalURIHeader }} {{ $proxySetHeader }} X-Original-URI $request_uri; {{ end }} {{ $proxySetHeader }} X-Scheme $pass_access_scheme; @@ -1135,7 +1173,6 @@ stream { proxy_set_header X-Service-Port $service_port; {{ end }} - {{ if not (empty $location.Backend) }} {{ buildProxyPass $server.Hostname $all.Backends $location }} {{ if (or (eq $location.Proxy.ProxyRedirectFrom "default") (eq $location.Proxy.ProxyRedirectFrom "off")) }} proxy_redirect {{ $location.Proxy.ProxyRedirectFrom }}; @@ -1143,10 +1180,6 @@ stream { proxy_redirect {{ $location.Proxy.ProxyRedirectFrom }} {{ $location.Proxy.ProxyRedirectTo }}; {{ end }} {{ else }} - # No endpoints available for the request - return 503; - {{ end }} - {{ else }} # Location denied. Reason: {{ $location.Denied | printf "%q" }} return 503; {{ end }} diff --git a/test/e2e/annotations/affinity.go b/test/e2e/annotations/affinity.go index 10d0ece50..5903f7e15 100644 --- a/test/e2e/annotations/affinity.go +++ b/test/e2e/annotations/affinity.go @@ -16,23 +16,27 @@ limitations under the License. package annotations -/* import ( + "fmt" + "net/http" + "strings" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/parnurzeal/gorequest" + + "k8s.io/api/extensions/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + + "k8s.io/ingress-nginx/test/e2e/framework" ) - -// TODO(elvinefendi) merge this with Affinity tests in test/e2e/lua/dynamic_configuration.go -var _ = framework.IngressNginxDescribe("Annotations - Affinity", func() { +var _ = framework.IngressNginxDescribe("Annotations - Affinity/Sticky Sessions", func() { f := framework.NewDefaultFramework("affinity") BeforeEach(func() { - err := f.DisableDynamicConfiguration() - Expect(err).NotTo(HaveOccurred()) - - err = f.NewEchoDeploymentWithReplicas(2) - Expect(err).NotTo(HaveOccurred()) + f.NewEchoDeploymentWithReplicas(2) }) AfterEach(func() { @@ -46,16 +50,12 @@ var _ = framework.IngressNginxDescribe("Annotations - Affinity", func() { } ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations) - _, err := f.EnsureIngress(ing) + f.EnsureIngress(ing) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) - - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { - return strings.Contains(server, "proxy_pass http://sticky-"+f.IngressController.Namespace+"-http-svc-80;") + return strings.Contains(server, fmt.Sprintf("server_name %s ;", host)) }) - Expect(err).NotTo(HaveOccurred()) resp, _, errs := gorequest.New(). Get(f.IngressController.HTTPURL). @@ -67,36 +67,6 @@ var _ = framework.IngressNginxDescribe("Annotations - Affinity", func() { Expect(resp.Header.Get("Set-Cookie")).Should(ContainSubstring("SERVERID=")) }) - It("should redirect to '/something' with enabled affinity", func() { - host := "example.com" - annotations := map[string]string{ - "nginx.ingress.kubernetes.io/affinity": "cookie", - "nginx.ingress.kubernetes.io/session-cookie-name": "SERVERID", - } - - ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations) - _, err := f.EnsureIngress(ing) - - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) - - err = f.WaitForNginxServer(host, - func(server string) bool { - return strings.Contains(server, "proxy_pass http://sticky-"+f.IngressController.Namespace+"-http-svc-80;") - }) - Expect(err).NotTo(HaveOccurred()) - - resp, body, errs := gorequest.New(). - Get(f.IngressController.HTTPURL). - Set("Host", host). - End() - - Expect(len(errs)).Should(BeNumerically("==", 0)) - Expect(resp.StatusCode).Should(Equal(http.StatusOK)) - Expect(body).Should(ContainSubstring(fmt.Sprintf("request_uri=http://%v:8080/", host))) - Expect(resp.Header.Get("Set-Cookie")).Should(ContainSubstring("SERVERID=")) - }) - It("should set the path to /something on the generated cookie", func() { host := "example.com" annotations := map[string]string{ @@ -105,16 +75,12 @@ var _ = framework.IngressNginxDescribe("Annotations - Affinity", func() { } ing := framework.NewSingleIngress(host, "/something", host, f.IngressController.Namespace, "http-svc", 80, &annotations) - _, err := f.EnsureIngress(ing) + f.EnsureIngress(ing) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) - - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { - return strings.Contains(server, "proxy_pass http://sticky-"+f.IngressController.Namespace+"-http-svc-80;") + return strings.Contains(server, fmt.Sprintf("server_name %s ;", host)) }) - Expect(err).NotTo(HaveOccurred()) resp, _, errs := gorequest.New(). Get(f.IngressController.HTTPURL+"/something"). @@ -126,14 +92,14 @@ var _ = framework.IngressNginxDescribe("Annotations - Affinity", func() { Expect(resp.Header.Get("Set-Cookie")).Should(ContainSubstring("Path=/something")) }) - It("should set the path to / on the generated cookie if there's more than one rule referring to the same backend", func() { + It("does not set the path to / on the generated cookie if there's more than one rule referring to the same backend", func() { host := "example.com" annotations := map[string]string{ "nginx.ingress.kubernetes.io/affinity": "cookie", "nginx.ingress.kubernetes.io/session-cookie-name": "SERVERID", } - ing, err := f.EnsureIngress(&v1beta1.Ingress{ + f.EnsureIngress(&v1beta1.Ingress{ ObjectMeta: metav1.ObjectMeta{ Name: host, Namespace: f.IngressController.Namespace, @@ -168,14 +134,10 @@ var _ = framework.IngressNginxDescribe("Annotations - Affinity", func() { }, }) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) - - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { - return strings.Contains(server, "proxy_pass http://sticky-"+f.IngressController.Namespace+"-http-svc-80;") + return strings.Contains(server, fmt.Sprintf("server_name %s ;", host)) }) - Expect(err).NotTo(HaveOccurred()) resp, _, errs := gorequest.New(). Get(f.IngressController.HTTPURL+"/something"). @@ -184,7 +146,15 @@ var _ = framework.IngressNginxDescribe("Annotations - Affinity", func() { Expect(len(errs)).Should(BeNumerically("==", 0)) Expect(resp.StatusCode).Should(Equal(http.StatusOK)) - Expect(resp.Header.Get("Set-Cookie")).Should(ContainSubstring("Path=/;")) + Expect(resp.Header.Get("Set-Cookie")).Should(ContainSubstring("Path=/something;")) + + resp, _, errs = gorequest.New(). + Get(f.IngressController.HTTPURL+"/somewhereelese"). + Set("Host", host). + End() + + Expect(len(errs)).Should(BeNumerically("==", 0)) + Expect(resp.StatusCode).Should(Equal(http.StatusOK)) + Expect(resp.Header.Get("Set-Cookie")).Should(ContainSubstring("Path=/somewhereelese;")) }) }) -*/ diff --git a/test/e2e/annotations/alias.go b/test/e2e/annotations/alias.go index 6f67f3f75..2ebda7b5e 100644 --- a/test/e2e/annotations/alias.go +++ b/test/e2e/annotations/alias.go @@ -31,8 +31,7 @@ var _ = framework.IngressNginxDescribe("Annotations - Alias", func() { f := framework.NewDefaultFramework("alias") BeforeEach(func() { - err := f.NewEchoDeployment() - Expect(err).NotTo(HaveOccurred()) + f.NewEchoDeployment() }) AfterEach(func() { @@ -43,17 +42,13 @@ var _ = framework.IngressNginxDescribe("Annotations - Alias", func() { annotations := map[string]string{} ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations) - _, err := f.EnsureIngress(ing) + f.EnsureIngress(ing) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) - - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return Expect(server).Should(ContainSubstring("server_name foo")) && Expect(server).ShouldNot(ContainSubstring("return 503")) }) - Expect(err).NotTo(HaveOccurred()) resp, body, errs := gorequest.New(). Get(f.IngressController.HTTPURL). @@ -81,17 +76,13 @@ var _ = framework.IngressNginxDescribe("Annotations - Alias", func() { } ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations) - _, err := f.EnsureIngress(ing) + f.EnsureIngress(ing) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) - - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return Expect(server).Should(ContainSubstring("server_name foo")) && Expect(server).ShouldNot(ContainSubstring("return 503")) }) - Expect(err).NotTo(HaveOccurred()) hosts := []string{"foo", "bar"} for _, host := range hosts { diff --git a/test/e2e/annotations/approot.go b/test/e2e/annotations/approot.go new file mode 100644 index 000000000..5c6d2942a --- /dev/null +++ b/test/e2e/annotations/approot.go @@ -0,0 +1,66 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package annotations + +import ( + "net/http" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/parnurzeal/gorequest" + "k8s.io/ingress-nginx/test/e2e/framework" +) + +var _ = framework.IngressNginxDescribe("Annotations - Approot", func() { + f := framework.NewDefaultFramework("approot") + + BeforeEach(func() { + f.NewEchoDeploymentWithReplicas(2) + }) + + AfterEach(func() { + }) + + It("should redirect to /foo", func() { + host := "approot.bar.com" + + annotations := map[string]string{ + "nginx.ingress.kubernetes.io/app-root": "/foo", + } + + ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations) + f.EnsureIngress(ing) + + f.WaitForNginxServer(host, + func(server string) bool { + return Expect(server).Should(ContainSubstring(`if ($uri = /) {`)) && + Expect(server).Should(ContainSubstring(`return 302 /foo;`)) + }) + + resp, _, errs := gorequest.New(). + Get(f.IngressController.HTTPURL). + Retry(10, 1*time.Second, http.StatusNotFound). + RedirectPolicy(noRedirectPolicyFunc). + Set("Host", host). + End() + + Expect(len(errs)).Should(BeNumerically("==", 0)) + Expect(resp.StatusCode).Should(Equal(http.StatusFound)) + Expect(resp.Header.Get("Location")).Should(Equal("http://approot.bar.com/foo")) + }) +}) diff --git a/test/e2e/annotations/auth.go b/test/e2e/annotations/auth.go index 720c76f64..20939be24 100644 --- a/test/e2e/annotations/auth.go +++ b/test/e2e/annotations/auth.go @@ -25,6 +25,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/parnurzeal/gorequest" corev1 "k8s.io/api/core/v1" @@ -39,8 +40,7 @@ var _ = framework.IngressNginxDescribe("Annotations - Auth", func() { f := framework.NewDefaultFramework("auth") BeforeEach(func() { - err := f.NewEchoDeployment() - Expect(err).NotTo(HaveOccurred()) + f.NewEchoDeployment() }) AfterEach(func() { @@ -50,17 +50,13 @@ var _ = framework.IngressNginxDescribe("Annotations - Auth", func() { host := "auth" ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, nil) - _, err := f.EnsureIngress(ing) + f.EnsureIngress(ing) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) - - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return Expect(server).Should(ContainSubstring("server_name auth")) && Expect(server).ShouldNot(ContainSubstring("return 503")) }) - Expect(err).NotTo(HaveOccurred()) resp, body, errs := gorequest.New(). Get(f.IngressController.HTTPURL). @@ -82,17 +78,13 @@ var _ = framework.IngressNginxDescribe("Annotations - Auth", func() { } ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations) - _, err := f.EnsureIngress(ing) + f.EnsureIngress(ing) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) - - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return Expect(server).Should(ContainSubstring("server_name auth")) && Expect(server).Should(ContainSubstring("return 503")) }) - Expect(err).NotTo(HaveOccurred()) resp, body, errs := gorequest.New(). Get(f.IngressController.HTTPURL). @@ -108,10 +100,7 @@ var _ = framework.IngressNginxDescribe("Annotations - Auth", func() { It("should return status code 401 when authentication is configured but Authorization header is not configured", func() { host := "auth" - s, err := f.EnsureSecret(buildSecret("foo", "bar", "test", f.IngressController.Namespace)) - Expect(err).NotTo(HaveOccurred()) - Expect(s).NotTo(BeNil()) - Expect(s.ObjectMeta).NotTo(BeNil()) + s := f.EnsureSecret(buildSecret("foo", "bar", "test", f.IngressController.Namespace)) annotations := map[string]string{ "nginx.ingress.kubernetes.io/auth-type": "basic", @@ -120,17 +109,13 @@ var _ = framework.IngressNginxDescribe("Annotations - Auth", func() { } ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations) - _, err = f.EnsureIngress(ing) + f.EnsureIngress(ing) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) - - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return Expect(server).Should(ContainSubstring("server_name auth")) && Expect(server).ShouldNot(ContainSubstring("return 503")) }) - Expect(err).NotTo(HaveOccurred()) resp, body, errs := gorequest.New(). Get(f.IngressController.HTTPURL). @@ -146,10 +131,7 @@ var _ = framework.IngressNginxDescribe("Annotations - Auth", func() { It("should return status code 401 when authentication is configured and Authorization header is sent with invalid credentials", func() { host := "auth" - s, err := f.EnsureSecret(buildSecret("foo", "bar", "test", f.IngressController.Namespace)) - Expect(err).NotTo(HaveOccurred()) - Expect(s).NotTo(BeNil()) - Expect(s.ObjectMeta).NotTo(BeNil()) + s := f.EnsureSecret(buildSecret("foo", "bar", "test", f.IngressController.Namespace)) annotations := map[string]string{ "nginx.ingress.kubernetes.io/auth-type": "basic", @@ -158,17 +140,13 @@ var _ = framework.IngressNginxDescribe("Annotations - Auth", func() { } ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations) - _, err = f.EnsureIngress(ing) + f.EnsureIngress(ing) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) - - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return Expect(server).Should(ContainSubstring("server_name auth")) && Expect(server).ShouldNot(ContainSubstring("return 503")) }) - Expect(err).NotTo(HaveOccurred()) resp, body, errs := gorequest.New(). Get(f.IngressController.HTTPURL). @@ -185,10 +163,7 @@ var _ = framework.IngressNginxDescribe("Annotations - Auth", func() { It("should return status code 200 when authentication is configured and Authorization header is sent", func() { host := "auth" - s, err := f.EnsureSecret(buildSecret("foo", "bar", "test", f.IngressController.Namespace)) - Expect(err).NotTo(HaveOccurred()) - Expect(s).NotTo(BeNil()) - Expect(s.ObjectMeta).NotTo(BeNil()) + s := f.EnsureSecret(buildSecret("foo", "bar", "test", f.IngressController.Namespace)) annotations := map[string]string{ "nginx.ingress.kubernetes.io/auth-type": "basic", @@ -197,17 +172,13 @@ var _ = framework.IngressNginxDescribe("Annotations - Auth", func() { } ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations) - _, err = f.EnsureIngress(ing) + f.EnsureIngress(ing) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) - - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return Expect(server).Should(ContainSubstring("server_name auth")) && Expect(server).ShouldNot(ContainSubstring("return 503")) }) - Expect(err).NotTo(HaveOccurred()) resp, _, errs := gorequest.New(). Get(f.IngressController.HTTPURL). @@ -223,7 +194,7 @@ var _ = framework.IngressNginxDescribe("Annotations - Auth", func() { It("should return status code 500 when authentication is configured with invalid content and Authorization header is sent", func() { host := "auth" - s, err := f.EnsureSecret( + s := f.EnsureSecret( &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: "test", @@ -236,9 +207,6 @@ var _ = framework.IngressNginxDescribe("Annotations - Auth", func() { Type: corev1.SecretTypeOpaque, }, ) - Expect(err).NotTo(HaveOccurred()) - Expect(s).NotTo(BeNil()) - Expect(s.ObjectMeta).NotTo(BeNil()) annotations := map[string]string{ "nginx.ingress.kubernetes.io/auth-type": "basic", @@ -247,17 +215,13 @@ var _ = framework.IngressNginxDescribe("Annotations - Auth", func() { } ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations) - _, err = f.EnsureIngress(ing) + f.EnsureIngress(ing) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) - - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return Expect(server).Should(ContainSubstring("server_name auth")) && Expect(server).ShouldNot(ContainSubstring("return 503")) }) - Expect(err).NotTo(HaveOccurred()) resp, _, errs := gorequest.New(). Get(f.IngressController.HTTPURL). @@ -274,11 +238,10 @@ var _ = framework.IngressNginxDescribe("Annotations - Auth", func() { host := "auth" BeforeEach(func() { - err := f.NewHttpbinDeployment() - Expect(err).NotTo(HaveOccurred()) + f.NewHttpbinDeployment() var httpbinIP string - err = wait.PollImmediate(time.Second, time.Minute, func() (bool, error) { + err := wait.PollImmediate(time.Second, time.Minute, func() (bool, error) { e, err := f.KubeClientSet.CoreV1().Endpoints(f.IngressController.Namespace).Get("httpbin", metav1.GetOptions{}) if errors.IsNotFound(err) { return false, nil @@ -300,15 +263,11 @@ var _ = framework.IngressNginxDescribe("Annotations - Auth", func() { } ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations) - _, err = f.EnsureIngress(ing) + f.EnsureIngress(ing) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) - - err = f.WaitForNginxServer(host, func(server string) bool { + f.WaitForNginxServer(host, func(server string) bool { return Expect(server).ShouldNot(ContainSubstring("return 503")) }) - Expect(err).NotTo(HaveOccurred()) }) It("should return status code 200 when signed in", func() { diff --git a/test/e2e/annotations/authtls.go b/test/e2e/annotations/authtls.go index 73ab53ca0..89f1de4a6 100644 --- a/test/e2e/annotations/authtls.go +++ b/test/e2e/annotations/authtls.go @@ -19,20 +19,20 @@ package annotations import ( "crypto/tls" "fmt" + "net/http" + "strings" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/parnurzeal/gorequest" "k8s.io/ingress-nginx/test/e2e/framework" - "net/http" - "strings" ) var _ = framework.IngressNginxDescribe("Annotations - AuthTLS", func() { f := framework.NewDefaultFramework("authtls") BeforeEach(func() { - err := f.NewEchoDeploymentWithReplicas(2) - Expect(err).NotTo(HaveOccurred()) + f.NewEchoDeploymentWithReplicas(2) }) AfterEach(func() { @@ -54,10 +54,7 @@ var _ = framework.IngressNginxDescribe("Annotations - AuthTLS", func() { } ing := framework.NewSingleIngressWithTLS(host, "/", host, nameSpace, "http-svc", 80, &annotations) - _, err = f.EnsureIngress(ing) - - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) + f.EnsureIngress(ing) // Since we can use the same certificate-chain for tls as well as mutual-auth, we will check all values sslCertDirective := fmt.Sprintf("ssl_certificate /etc/ingress-controller/ssl/%s-%s.pem;", nameSpace, host) @@ -67,11 +64,14 @@ var _ = framework.IngressNginxDescribe("Annotations - AuthTLS", func() { sslVerify := "ssl_verify_client on;" sslVerifyDepth := "ssl_verify_depth 1;" - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { - return strings.Contains(server, sslCertDirective) && strings.Contains(server, sslKeyDirective) && strings.Contains(server, sslClientCertDirective) && strings.Contains(server, sslVerify) && strings.Contains(server, sslVerifyDepth) + return strings.Contains(server, sslCertDirective) && + strings.Contains(server, sslKeyDirective) && + strings.Contains(server, sslClientCertDirective) && + strings.Contains(server, sslVerify) && + strings.Contains(server, sslVerifyDepth) }) - Expect(err).NotTo(HaveOccurred()) // Send Request without Client Certs req := gorequest.New() @@ -112,10 +112,7 @@ var _ = framework.IngressNginxDescribe("Annotations - AuthTLS", func() { } ing := framework.NewSingleIngressWithTLS(host, "/", host, nameSpace, "http-svc", 80, &annotations) - _, err = f.EnsureIngress(ing) - - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) + f.EnsureIngress(ing) // Since we can use the same certificate-chain for tls as well as mutual-auth, we will check all values sslCertDirective := fmt.Sprintf("ssl_certificate /etc/ingress-controller/ssl/%s-%s.pem;", nameSpace, host) @@ -125,11 +122,10 @@ var _ = framework.IngressNginxDescribe("Annotations - AuthTLS", func() { sslVerify := "ssl_verify_client off;" sslVerifyDepth := "ssl_verify_depth 2;" - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return strings.Contains(server, sslCertDirective) && strings.Contains(server, sslKeyDirective) && strings.Contains(server, sslClientCertDirective) && strings.Contains(server, sslVerify) && strings.Contains(server, sslVerifyDepth) }) - Expect(err).NotTo(HaveOccurred()) // Send Request without Client Certs req := gorequest.New() @@ -163,9 +159,7 @@ var _ = framework.IngressNginxDescribe("Annotations - AuthTLS", func() { } ing := framework.NewSingleIngressWithTLS(host, "/", host, nameSpace, "http-svc", 80, &annotations) - _, err = f.EnsureIngress(ing) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) + f.EnsureIngress(ing) // Since we can use the same certificate-chain for tls as well as mutual-auth, we will check all values sslCertDirective := fmt.Sprintf("ssl_certificate /etc/ingress-controller/ssl/%s-%s.pem;", nameSpace, host) @@ -177,11 +171,16 @@ var _ = framework.IngressNginxDescribe("Annotations - AuthTLS", func() { sslErrorPage := fmt.Sprintf("error_page 495 496 = %s;", f.IngressController.HTTPURL+errorPath) sslUpstreamClientCert := "proxy_set_header ssl-client-cert $ssl_client_escaped_cert;" - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { - return strings.Contains(server, sslCertDirective) && strings.Contains(server, sslKeyDirective) && strings.Contains(server, sslClientCertDirective) && strings.Contains(server, sslVerify) && strings.Contains(server, sslVerifyDepth) && strings.Contains(server, sslErrorPage) && strings.Contains(server, sslUpstreamClientCert) + return strings.Contains(server, sslCertDirective) && + strings.Contains(server, sslKeyDirective) && + strings.Contains(server, sslClientCertDirective) && + strings.Contains(server, sslVerify) && + strings.Contains(server, sslVerifyDepth) && + strings.Contains(server, sslErrorPage) && + strings.Contains(server, sslUpstreamClientCert) }) - Expect(err).NotTo(HaveOccurred()) // Send Request without Client Certs req := gorequest.New() diff --git a/test/e2e/annotations/backendprotocol.go b/test/e2e/annotations/backendprotocol.go index 30ebddfb0..1723e3b87 100644 --- a/test/e2e/annotations/backendprotocol.go +++ b/test/e2e/annotations/backendprotocol.go @@ -26,8 +26,7 @@ var _ = framework.IngressNginxDescribe("Annotations - Backendprotocol", func() { f := framework.NewDefaultFramework("backendprotocol") BeforeEach(func() { - err := f.NewEchoDeploymentWithReplicas(2) - Expect(err).NotTo(HaveOccurred()) + f.NewEchoDeploymentWithReplicas(2) }) AfterEach(func() { @@ -40,16 +39,12 @@ var _ = framework.IngressNginxDescribe("Annotations - Backendprotocol", func() { } ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations) - _, err := f.EnsureIngress(ing) + f.EnsureIngress(ing) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) - - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return Expect(server).Should(ContainSubstring("proxy_pass https://upstream_balancer;")) }) - Expect(err).NotTo(HaveOccurred()) }) It("should set backend protocol to grpc://", func() { @@ -59,16 +54,12 @@ var _ = framework.IngressNginxDescribe("Annotations - Backendprotocol", func() { } ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations) - _, err := f.EnsureIngress(ing) + f.EnsureIngress(ing) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) - - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return Expect(server).Should(ContainSubstring("grpc_pass grpc://upstream_balancer;")) }) - Expect(err).NotTo(HaveOccurred()) }) It("should set backend protocol to grpcs://", func() { @@ -78,16 +69,12 @@ var _ = framework.IngressNginxDescribe("Annotations - Backendprotocol", func() { } ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations) - _, err := f.EnsureIngress(ing) + f.EnsureIngress(ing) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) - - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return Expect(server).Should(ContainSubstring("grpc_pass grpcs://upstream_balancer;")) }) - Expect(err).NotTo(HaveOccurred()) }) It("should set backend protocol to ''", func() { @@ -97,15 +84,11 @@ var _ = framework.IngressNginxDescribe("Annotations - Backendprotocol", func() { } ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations) - _, err := f.EnsureIngress(ing) + f.EnsureIngress(ing) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) - - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return Expect(server).Should(ContainSubstring("ajp_pass upstream_balancer;")) }) - Expect(err).NotTo(HaveOccurred()) }) }) diff --git a/test/e2e/annotations/clientbodybuffersize.go b/test/e2e/annotations/clientbodybuffersize.go index 80ef8330b..621deb65d 100644 --- a/test/e2e/annotations/clientbodybuffersize.go +++ b/test/e2e/annotations/clientbodybuffersize.go @@ -26,8 +26,7 @@ var _ = framework.IngressNginxDescribe("Annotations - Client-Body-Buffer-Size", f := framework.NewDefaultFramework("clientbodybuffersize") BeforeEach(func() { - err := f.NewEchoDeploymentWithReplicas(2) - Expect(err).NotTo(HaveOccurred()) + f.NewEchoDeploymentWithReplicas(2) }) AfterEach(func() { @@ -40,16 +39,12 @@ var _ = framework.IngressNginxDescribe("Annotations - Client-Body-Buffer-Size", } ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations) - _, err := f.EnsureIngress(ing) + f.EnsureIngress(ing) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) - - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return Expect(server).Should(ContainSubstring("client_body_buffer_size 1000;")) }) - Expect(err).NotTo(HaveOccurred()) }) It("should set client_body_buffer_size to 1K", func() { @@ -59,16 +54,12 @@ var _ = framework.IngressNginxDescribe("Annotations - Client-Body-Buffer-Size", } ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations) - _, err := f.EnsureIngress(ing) + f.EnsureIngress(ing) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) - - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return Expect(server).Should(ContainSubstring("client_body_buffer_size 1K;")) }) - Expect(err).NotTo(HaveOccurred()) }) It("should set client_body_buffer_size to 1k", func() { @@ -78,16 +69,12 @@ var _ = framework.IngressNginxDescribe("Annotations - Client-Body-Buffer-Size", } ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations) - _, err := f.EnsureIngress(ing) + f.EnsureIngress(ing) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) - - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return Expect(server).Should(ContainSubstring("client_body_buffer_size 1k;")) }) - Expect(err).NotTo(HaveOccurred()) }) It("should set client_body_buffer_size to 1m", func() { @@ -97,16 +84,12 @@ var _ = framework.IngressNginxDescribe("Annotations - Client-Body-Buffer-Size", } ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations) - _, err := f.EnsureIngress(ing) + f.EnsureIngress(ing) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) - - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return Expect(server).Should(ContainSubstring("client_body_buffer_size 1m;")) }) - Expect(err).NotTo(HaveOccurred()) }) It("should set client_body_buffer_size to 1M", func() { @@ -116,16 +99,12 @@ var _ = framework.IngressNginxDescribe("Annotations - Client-Body-Buffer-Size", } ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations) - _, err := f.EnsureIngress(ing) + f.EnsureIngress(ing) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) - - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return Expect(server).Should(ContainSubstring("client_body_buffer_size 1M;")) }) - Expect(err).NotTo(HaveOccurred()) }) It("should not set client_body_buffer_size to invalid 1b", func() { @@ -135,15 +114,11 @@ var _ = framework.IngressNginxDescribe("Annotations - Client-Body-Buffer-Size", } ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations) - _, err := f.EnsureIngress(ing) + f.EnsureIngress(ing) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) - - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return Expect(server).ShouldNot(ContainSubstring("client_body_buffer_size 1b;")) }) - Expect(err).NotTo(HaveOccurred()) }) }) diff --git a/test/e2e/annotations/connection.go b/test/e2e/annotations/connection.go index ce14ccd3c..c281a2a69 100644 --- a/test/e2e/annotations/connection.go +++ b/test/e2e/annotations/connection.go @@ -31,8 +31,7 @@ var _ = framework.IngressNginxDescribe("Annotations - Connection", func() { f := framework.NewDefaultFramework("connection") BeforeEach(func() { - err := f.NewEchoDeploymentWithReplicas(2) - Expect(err).NotTo(HaveOccurred()) + f.NewEchoDeploymentWithReplicas(2) }) AfterEach(func() { @@ -45,16 +44,12 @@ var _ = framework.IngressNginxDescribe("Annotations - Connection", func() { } ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations) - _, err := f.EnsureIngress(ing) + f.EnsureIngress(ing) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) - - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return Expect(server).Should(ContainSubstring(`proxy_set_header Connection keep-alive;`)) }) - Expect(err).NotTo(HaveOccurred()) resp, body, errs := gorequest.New(). Get(f.IngressController.HTTPURL). diff --git a/test/e2e/annotations/cors.go b/test/e2e/annotations/cors.go index e1abd7ada..8da735c61 100644 --- a/test/e2e/annotations/cors.go +++ b/test/e2e/annotations/cors.go @@ -17,10 +17,11 @@ limitations under the License. package annotations import ( + "net/http" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/parnurzeal/gorequest" - "net/http" "k8s.io/ingress-nginx/test/e2e/framework" ) @@ -29,8 +30,7 @@ var _ = framework.IngressNginxDescribe("Annotations - CORS", func() { f := framework.NewDefaultFramework("cors") BeforeEach(func() { - err := f.NewEchoDeploymentWithReplicas(2) - Expect(err).NotTo(HaveOccurred()) + f.NewEchoDeploymentWithReplicas(2) }) AfterEach(func() { @@ -43,40 +43,32 @@ var _ = framework.IngressNginxDescribe("Annotations - CORS", func() { } ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations) - _, err := f.EnsureIngress(ing) + f.EnsureIngress(ing) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) - - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return Expect(server).Should(ContainSubstring("more_set_headers 'Access-Control-Allow-Methods: GET, PUT, POST, DELETE, PATCH, OPTIONS';")) }) - Expect(err).NotTo(HaveOccurred()) - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return Expect(server).Should(ContainSubstring("more_set_headers 'Access-Control-Allow-Origin: *';")) }) - Expect(err).NotTo(HaveOccurred()) - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return Expect(server).Should(ContainSubstring("more_set_headers 'Access-Control-Allow-Headers: DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Authorization';")) }) - Expect(err).NotTo(HaveOccurred()) - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return Expect(server).Should(ContainSubstring("more_set_headers 'Access-Control-Max-Age: 1728000';")) }) - Expect(err).NotTo(HaveOccurred()) - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return Expect(server).Should(ContainSubstring("more_set_headers 'Access-Control-Allow-Credentials: true';")) }) - Expect(err).NotTo(HaveOccurred()) uri := "/" resp, _, errs := gorequest.New(). @@ -95,16 +87,12 @@ var _ = framework.IngressNginxDescribe("Annotations - CORS", func() { } ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations) - _, err := f.EnsureIngress(ing) + f.EnsureIngress(ing) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) - - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return Expect(server).Should(ContainSubstring("more_set_headers 'Access-Control-Allow-Methods: POST, GET';")) }) - Expect(err).NotTo(HaveOccurred()) }) It("should set cors max-age", func() { @@ -115,16 +103,12 @@ var _ = framework.IngressNginxDescribe("Annotations - CORS", func() { } ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations) - _, err := f.EnsureIngress(ing) + f.EnsureIngress(ing) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) - - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return Expect(server).Should(ContainSubstring("more_set_headers 'Access-Control-Max-Age: 200';")) }) - Expect(err).NotTo(HaveOccurred()) }) It("should disable cors allow credentials", func() { @@ -135,16 +119,12 @@ var _ = framework.IngressNginxDescribe("Annotations - CORS", func() { } ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations) - _, err := f.EnsureIngress(ing) + f.EnsureIngress(ing) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) - - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return Expect(server).ShouldNot(ContainSubstring("more_set_headers 'Access-Control-Allow-Credentials: true';")) }) - Expect(err).NotTo(HaveOccurred()) }) It("should allow origin for cors", func() { @@ -155,16 +135,12 @@ var _ = framework.IngressNginxDescribe("Annotations - CORS", func() { } ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations) - _, err := f.EnsureIngress(ing) + f.EnsureIngress(ing) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) - - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return Expect(server).Should(ContainSubstring("more_set_headers 'Access-Control-Allow-Origin: https://origin.cors.com:8080';")) }) - Expect(err).NotTo(HaveOccurred()) }) It("should allow headers for cors", func() { @@ -175,15 +151,11 @@ var _ = framework.IngressNginxDescribe("Annotations - CORS", func() { } ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations) - _, err := f.EnsureIngress(ing) + f.EnsureIngress(ing) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) - - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return Expect(server).Should(ContainSubstring("more_set_headers 'Access-Control-Allow-Headers: DNT, User-Agent';")) }) - Expect(err).NotTo(HaveOccurred()) }) }) diff --git a/test/e2e/annotations/default_backend.go b/test/e2e/annotations/default_backend.go index 1cec44c7e..a1d7967f9 100644 --- a/test/e2e/annotations/default_backend.go +++ b/test/e2e/annotations/default_backend.go @@ -32,8 +32,7 @@ var _ = framework.IngressNginxDescribe("Annotations - custom default-backend", f f := framework.NewDefaultFramework("default-backend") BeforeEach(func() { - err := f.NewEchoDeployment() - Expect(err).NotTo(HaveOccurred()) + f.NewEchoDeployment() }) Context("when default backend annotation is enabled", func() { @@ -44,18 +43,14 @@ var _ = framework.IngressNginxDescribe("Annotations - custom default-backend", f } ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "invalid", 80, &annotations) - _, err := f.EnsureIngress(ing) - - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) + f.EnsureIngress(ing) time.Sleep(5 * time.Second) - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return Expect(server).Should(ContainSubstring(fmt.Sprintf("server_name %v", host))) }) - Expect(err).NotTo(HaveOccurred()) uri := "/alma/armud" resp, body, errs := gorequest.New(). diff --git a/test/e2e/annotations/forcesslredirect.go b/test/e2e/annotations/forcesslredirect.go new file mode 100644 index 000000000..590bf3c31 --- /dev/null +++ b/test/e2e/annotations/forcesslredirect.go @@ -0,0 +1,66 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package annotations + +import ( + "net/http" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/parnurzeal/gorequest" + "k8s.io/ingress-nginx/test/e2e/framework" +) + +var _ = framework.IngressNginxDescribe("Annotations - Forcesslredirect", func() { + f := framework.NewDefaultFramework("forcesslredirect") + + BeforeEach(func() { + f.NewEchoDeploymentWithReplicas(2) + }) + + AfterEach(func() { + }) + + It("should redirect to https", func() { + host := "forcesslredirect.bar.com" + + annotations := map[string]string{ + "nginx.ingress.kubernetes.io/force-ssl-redirect": "true", + } + + ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations) + f.EnsureIngress(ing) + + f.WaitForNginxServer(host, + func(server string) bool { + return Expect(server).Should(ContainSubstring(`if ($redirect_to_https) {`)) && + Expect(server).Should(ContainSubstring(`return 308 https://$best_http_host$request_uri;`)) + }) + + resp, _, errs := gorequest.New(). + Get(f.IngressController.HTTPURL). + Retry(10, 1*time.Second, http.StatusNotFound). + RedirectPolicy(noRedirectPolicyFunc). + Set("Host", host). + End() + + Expect(len(errs)).Should(BeNumerically("==", 0)) + Expect(resp.StatusCode).Should(Equal(http.StatusPermanentRedirect)) + Expect(resp.Header.Get("Location")).Should(Equal("https://forcesslredirect.bar.com/")) + }) +}) diff --git a/test/e2e/annotations/fromtowwwredirect.go b/test/e2e/annotations/fromtowwwredirect.go index 7ae32e36f..6d1e597ed 100644 --- a/test/e2e/annotations/fromtowwwredirect.go +++ b/test/e2e/annotations/fromtowwwredirect.go @@ -31,8 +31,7 @@ var _ = framework.IngressNginxDescribe("Annotations - Fromtowwwredirect", func() f := framework.NewDefaultFramework("fromtowwwredirect") BeforeEach(func() { - err := f.NewEchoDeploymentWithReplicas(2) - Expect(err).NotTo(HaveOccurred()) + f.NewEchoDeploymentWithReplicas(2) }) AfterEach(func() { @@ -47,30 +46,25 @@ var _ = framework.IngressNginxDescribe("Annotations - Fromtowwwredirect", func() } ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations) - _, err := f.EnsureIngress(ing) + f.EnsureIngress(ing) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) - - err = f.WaitForNginxConfiguration( + f.WaitForNginxConfiguration( func(cfg string) bool { return Expect(cfg).Should(ContainSubstring(`server_name www.fromtowwwredirect.bar.com;`)) && Expect(cfg).Should(ContainSubstring(`return 308 $scheme://fromtowwwredirect.bar.com$request_uri;`)) }) - Expect(err).NotTo(HaveOccurred()) By("sending request to www.fromtowwwredirect.bar.com") - gorequest.New(). + resp, _, errs := gorequest.New(). Get(fmt.Sprintf("%s/%s", f.IngressController.HTTPURL, "foo")). Retry(10, 1*time.Second, http.StatusNotFound). + RedirectPolicy(noRedirectPolicyFunc). Set("Host", fmt.Sprintf("%s.%s", "www", host)). End() - log, err := f.NginxLogs() - Expect(err).ToNot(HaveOccurred()) - Expect(log).ToNot(BeEmpty()) - - Expect(log).To(ContainSubstring(fmt.Sprintf(` "GET /foo HTTP/1.1" 308 171 "-" "Go-http-client/1.1"`))) + Expect(len(errs)).Should(BeNumerically("==", 0)) + Expect(resp.StatusCode).Should(Equal(http.StatusPermanentRedirect)) + Expect(resp.Header.Get("Location")).Should(Equal("http://fromtowwwredirect.bar.com/foo")) }) }) diff --git a/test/e2e/annotations/grpc.go b/test/e2e/annotations/grpc.go index 9bface49b..38ae3dc92 100644 --- a/test/e2e/annotations/grpc.go +++ b/test/e2e/annotations/grpc.go @@ -29,8 +29,7 @@ var _ = framework.IngressNginxDescribe("Annotations - grpc", func() { f := framework.NewDefaultFramework("grpc") BeforeEach(func() { - err := f.NewGRPCFortuneTellerDeployment() - Expect(err).NotTo(HaveOccurred()) + f.NewGRPCFortuneTellerDeployment() }) Context("when grpc is enabled", func() { @@ -42,25 +41,20 @@ var _ = framework.IngressNginxDescribe("Annotations - grpc", func() { } ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "fortune-teller", 50051, &annotations) - _, err := f.EnsureIngress(ing) + f.EnsureIngress(ing) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) - - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return Expect(server).Should(ContainSubstring(fmt.Sprintf("server_name %v", host))) && Expect(server).ShouldNot(ContainSubstring("return 503")) }) - Expect(err).NotTo(HaveOccurred()) - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return Expect(server).Should(ContainSubstring("grpc_pass")) && Expect(server).Should(ContainSubstring("grpc_set_header")) && Expect(server).ShouldNot(ContainSubstring("proxy_pass")) }) - Expect(err).NotTo(HaveOccurred()) }) }) }) diff --git a/test/e2e/annotations/influxdb.go b/test/e2e/annotations/influxdb.go index 1286bdc93..45c1957d0 100644 --- a/test/e2e/annotations/influxdb.go +++ b/test/e2e/annotations/influxdb.go @@ -23,9 +23,10 @@ import ( "os/exec" "time" - jsoniter "github.com/json-iterator/go" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + + jsoniter "github.com/json-iterator/go" "github.com/parnurzeal/gorequest" corev1 "k8s.io/api/core/v1" @@ -39,17 +40,13 @@ var _ = framework.IngressNginxDescribe("Annotations - influxdb", func() { f := framework.NewDefaultFramework("influxdb") BeforeEach(func() { - err := f.NewInfluxDBDeployment() - Expect(err).NotTo(HaveOccurred()) - err = f.NewEchoDeployment() - Expect(err).NotTo(HaveOccurred()) + f.NewInfluxDBDeployment() + f.NewEchoDeployment() }) Context("when influxdb is enabled", func() { It("should send the request metric to the influxdb server", func() { - ifs, err := createInfluxDBService(f) - - Expect(err).NotTo(HaveOccurred()) + ifs := createInfluxDBService(f) // Ingress configured with InfluxDB annotations host := "influxdb.e2e.local" @@ -80,6 +77,8 @@ var _ = framework.IngressNginxDescribe("Annotations - influxdb", func() { time.Sleep(5 * time.Second) var measurements string + var err error + err = wait.PollImmediate(time.Second, time.Minute, func() (bool, error) { measurements, err = extractInfluxDBMeasurements(f) if err != nil { @@ -100,7 +99,7 @@ var _ = framework.IngressNginxDescribe("Annotations - influxdb", func() { }) }) -func createInfluxDBService(f *framework.Framework) (*corev1.Service, error) { +func createInfluxDBService(f *framework.Framework) *corev1.Service { service := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "inflxudb-svc", @@ -120,29 +119,18 @@ func createInfluxDBService(f *framework.Framework) (*corev1.Service, error) { }, } - s, err := f.EnsureService(service) - if err != nil { - return nil, err - } - - if s == nil { - return nil, fmt.Errorf("unexpected error creating service for influxdb deployment") - } - - return s, nil + return f.EnsureService(service) } func createInfluxDBIngress(f *framework.Framework, host, service string, port int, annotations map[string]string) { - ing, err := f.EnsureIngress(framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, service, port, &annotations)) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) + ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, service, port, &annotations) + f.EnsureIngress(ing) - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return Expect(server).Should(ContainSubstring(fmt.Sprintf("server_name %v", host))) && Expect(server).ShouldNot(ContainSubstring("return 503")) }) - Expect(err).NotTo(HaveOccurred()) } func extractInfluxDBMeasurements(f *framework.Framework) (string, error) { @@ -183,8 +171,7 @@ func execInfluxDBCommand(pod *corev1.Pod, command string) (string, error) { execErr bytes.Buffer ) - args := fmt.Sprintf("kubectl exec --namespace %v %v -- %v", pod.Namespace, pod.Name, command) - cmd := exec.Command("/bin/bash", "-c", args) + cmd := exec.Command("/bin/bash", "-c", fmt.Sprintf("%v exec --namespace %s %s -- %s", framework.KubectlPath, pod.Namespace, pod.Name, command)) cmd.Stdout = &execOut cmd.Stderr = &execErr @@ -195,7 +182,7 @@ func execInfluxDBCommand(pod *corev1.Pod, command string) (string, error) { } if err != nil { - return "", fmt.Errorf("could not execute: %v", err) + return "", fmt.Errorf("could not execute '%s %s': %v", cmd.Path, cmd.Args, err) } return execOut.String(), nil diff --git a/test/e2e/annotations/ipwhitelist.go b/test/e2e/annotations/ipwhitelist.go new file mode 100644 index 000000000..79125cff8 --- /dev/null +++ b/test/e2e/annotations/ipwhitelist.go @@ -0,0 +1,79 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package annotations + +import ( + "regexp" + "strings" + + . "github.com/onsi/ginkgo" + + "k8s.io/ingress-nginx/test/e2e/framework" +) + +var _ = framework.IngressNginxDescribe("Annotations - IPWhiteList", func() { + f := framework.NewDefaultFramework("ipwhitelist") + + BeforeEach(func() { + f.NewEchoDeploymentWithReplicas(2) + }) + + AfterEach(func() { + }) + + It("should set valid ip whitelist range", func() { + host := "ipwhitelist.foo.com" + nameSpace := f.IngressController.Namespace + + annotations := map[string]string{ + "nginx.ingress.kubernetes.io/whitelist-source-range": "18.0.0.0/8, 56.0.0.0/8", + } + + ing := framework.NewSingleIngress(host, "/", host, nameSpace, "http-svc", 80, &annotations) + f.EnsureIngress(ing) + + denyRegex := regexp.MustCompile("geo \\$the_real_ip \\$deny_[A-Za-z]{32}") + denyString := "" + + f.WaitForNginxConfiguration( + func(conf string) bool { + + match := denyRegex.FindStringSubmatch(conf) + // If no match found, return false + if !(len(match) > 0) { + return false + } + + denyString = strings.Replace(match[0], "geo $the_real_ip ", "", -1) + return strings.Contains(conf, match[0]) + }) + + ipOne := "18.0.0.0/8 0;" + ipTwo := "56.0.0.0/8 0;" + + f.WaitForNginxConfiguration( + func(conf string) bool { + return strings.Contains(conf, ipOne) && strings.Contains(conf, ipTwo) + }) + + denyStatement := "if (" + denyString + ")" + f.WaitForNginxServer(host, + func(server string) bool { + return strings.Contains(server, denyStatement) + }) + }) +}) diff --git a/test/e2e/annotations/log.go b/test/e2e/annotations/log.go index 884ffd524..9ae77ad51 100644 --- a/test/e2e/annotations/log.go +++ b/test/e2e/annotations/log.go @@ -19,6 +19,7 @@ package annotations import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "k8s.io/ingress-nginx/test/e2e/framework" ) @@ -26,8 +27,7 @@ var _ = framework.IngressNginxDescribe("Annotations - Log", func() { f := framework.NewDefaultFramework("log") BeforeEach(func() { - err := f.NewEchoDeploymentWithReplicas(2) - Expect(err).NotTo(HaveOccurred()) + f.NewEchoDeploymentWithReplicas(2) }) AfterEach(func() { @@ -40,16 +40,12 @@ var _ = framework.IngressNginxDescribe("Annotations - Log", func() { } ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations) - _, err := f.EnsureIngress(ing) + f.EnsureIngress(ing) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) - - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return Expect(server).Should(ContainSubstring(`access_log off;`)) }) - Expect(err).NotTo(HaveOccurred()) }) It("set rewrite_log on", func() { @@ -59,15 +55,11 @@ var _ = framework.IngressNginxDescribe("Annotations - Log", func() { } ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations) - _, err := f.EnsureIngress(ing) + f.EnsureIngress(ing) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) - - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return Expect(server).Should(ContainSubstring(`rewrite_log on;`)) }) - Expect(err).NotTo(HaveOccurred()) }) }) diff --git a/test/e2e/annotations/luarestywaf.go b/test/e2e/annotations/luarestywaf.go index ed332db04..0b95fb6e8 100644 --- a/test/e2e/annotations/luarestywaf.go +++ b/test/e2e/annotations/luarestywaf.go @@ -32,8 +32,7 @@ var _ = framework.IngressNginxDescribe("Annotations - lua-resty-waf", func() { f := framework.NewDefaultFramework("luarestywaf") BeforeEach(func() { - err := f.NewEchoDeployment() - Expect(err).NotTo(HaveOccurred()) + f.NewEchoDeployment() }) Context("when lua-resty-waf is enabled", func() { @@ -65,6 +64,71 @@ var _ = framework.IngressNginxDescribe("Annotations - lua-resty-waf", func() { Expect(len(errs)).Should(Equal(0)) Expect(resp.StatusCode).Should(Equal(http.StatusOK)) }) + It("should apply the score threshold", func() { + host := "foo" + createIngress(f, host, "http-svc", 80, map[string]string{ + "nginx.ingress.kubernetes.io/lua-resty-waf": "active", + "nginx.ingress.kubernetes.io/lua-resty-waf-score-threshold": "20"}) + + url := fmt.Sprintf("%s?msg=XSS", f.IngressController.HTTPURL) + resp, _, errs := gorequest.New(). + Get(url). + Set("Host", host). + End() + + Expect(len(errs)).Should(Equal(0)) + Expect(resp.StatusCode).Should(Equal(http.StatusOK)) + }) + It("should not reject request with an unknown content type", func() { + host := "foo" + contenttype := "application/octet-stream" + createIngress(f, host, "http-svc", 80, map[string]string{ + "nginx.ingress.kubernetes.io/lua-resty-waf-allow-unknown-content-types": "true", + "nginx.ingress.kubernetes.io/lua-resty-waf": "active"}) + + url := fmt.Sprintf("%s?msg=my-message", f.IngressController.HTTPURL) + resp, _, errs := gorequest.New(). + Get(url). + Set("Host", host). + Set("Content-Type", contenttype). + End() + + Expect(len(errs)).Should(Equal(0)) + Expect(resp.StatusCode).Should(Equal(http.StatusOK)) + }) + It("should not fail a request with multipart content type when multipart body processing disabled", func() { + contenttype := "multipart/form-data; boundary=alamofire.boundary.3fc2e849279e18fc" + host := "foo" + createIngress(f, host, "http-svc", 80, map[string]string{ + "nginx.ingress.kubernetes.io/lua-resty-waf-process-multipart-body": "false", + "nginx.ingress.kubernetes.io/lua-resty-waf": "active"}) + + url := fmt.Sprintf("%s?msg=my-message", f.IngressController.HTTPURL) + resp, _, errs := gorequest.New(). + Get(url). + Set("Host", host). + Set("Content-Type", contenttype). + End() + + Expect(len(errs)).Should(Equal(0)) + Expect(resp.StatusCode).Should(Equal(http.StatusOK)) + }) + It("should fail a request with multipart content type when multipart body processing enabled by default", func() { + contenttype := "multipart/form-data; boundary=alamofire.boundary.3fc2e849279e18fc" + host := "foo" + createIngress(f, host, "http-svc", 80, map[string]string{ + "nginx.ingress.kubernetes.io/lua-resty-waf": "active"}) + + url := fmt.Sprintf("%s?msg=my-message", f.IngressController.HTTPURL) + resp, _, errs := gorequest.New(). + Get(url). + Set("Host", host). + Set("Content-Type", contenttype). + End() + + Expect(len(errs)).Should(Equal(0)) + Expect(resp.StatusCode).Should(Equal(http.StatusBadRequest)) + }) It("should apply configured extra rules", func() { host := "foo" createIngress(f, host, "http-svc", 80, map[string]string{ @@ -139,16 +203,14 @@ var _ = framework.IngressNginxDescribe("Annotations - lua-resty-waf", func() { }) func createIngress(f *framework.Framework, host, service string, port int, annotations map[string]string) { - ing, err := f.EnsureIngress(framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, service, port, &annotations)) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) + ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, service, port, &annotations) + f.EnsureIngress(ing) - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return Expect(server).Should(ContainSubstring(fmt.Sprintf("server_name %v", host))) && Expect(server).ShouldNot(ContainSubstring("return 503")) }) - Expect(err).NotTo(HaveOccurred()) time.Sleep(1 * time.Second) diff --git a/test/e2e/annotations/proxy.go b/test/e2e/annotations/proxy.go index b1a116d7f..83f8df1b9 100644 --- a/test/e2e/annotations/proxy.go +++ b/test/e2e/annotations/proxy.go @@ -17,9 +17,10 @@ limitations under the License. package annotations import ( + "strings" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "strings" "k8s.io/ingress-nginx/test/e2e/framework" ) @@ -28,8 +29,7 @@ var _ = framework.IngressNginxDescribe("Annotations - Proxy", func() { f := framework.NewDefaultFramework("proxy") BeforeEach(func() { - err := f.NewEchoDeploymentWithReplicas(2) - Expect(err).NotTo(HaveOccurred()) + f.NewEchoDeploymentWithReplicas(2) }) AfterEach(func() { @@ -43,16 +43,12 @@ var _ = framework.IngressNginxDescribe("Annotations - Proxy", func() { } ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations) - _, err := f.EnsureIngress(ing) + f.EnsureIngress(ing) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) - - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return Expect(server).Should(ContainSubstring("proxy_redirect off;")) }) - Expect(err).NotTo(HaveOccurred()) }) It("should set proxy_redirect to default", func() { @@ -63,16 +59,12 @@ var _ = framework.IngressNginxDescribe("Annotations - Proxy", func() { } ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations) - _, err := f.EnsureIngress(ing) + f.EnsureIngress(ing) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) - - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return Expect(server).Should(ContainSubstring("proxy_redirect default;")) }) - Expect(err).NotTo(HaveOccurred()) }) It("should set proxy_redirect to hello.com goodbye.com", func() { @@ -83,16 +75,12 @@ var _ = framework.IngressNginxDescribe("Annotations - Proxy", func() { } ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations) - _, err := f.EnsureIngress(ing) + f.EnsureIngress(ing) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) - - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return Expect(server).Should(ContainSubstring("proxy_redirect hello.com goodbye.com;")) }) - Expect(err).NotTo(HaveOccurred()) }) It("should set proxy client-max-body-size to 8m", func() { @@ -102,16 +90,12 @@ var _ = framework.IngressNginxDescribe("Annotations - Proxy", func() { } ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations) - _, err := f.EnsureIngress(ing) + f.EnsureIngress(ing) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) - - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return Expect(server).Should(ContainSubstring("client_max_body_size 8m;")) }) - Expect(err).NotTo(HaveOccurred()) }) It("should not set proxy client-max-body-size to incorrect value", func() { @@ -121,16 +105,12 @@ var _ = framework.IngressNginxDescribe("Annotations - Proxy", func() { } ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations) - _, err := f.EnsureIngress(ing) + f.EnsureIngress(ing) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) - - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return Expect(server).ShouldNot(ContainSubstring("client_max_body_size 15r;")) }) - Expect(err).NotTo(HaveOccurred()) }) It("should set valid proxy timeouts", func() { @@ -142,16 +122,12 @@ var _ = framework.IngressNginxDescribe("Annotations - Proxy", func() { } ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations) - _, err := f.EnsureIngress(ing) + f.EnsureIngress(ing) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) - - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return strings.Contains(server, "proxy_connect_timeout 50s;") && strings.Contains(server, "proxy_send_timeout 20s;") && strings.Contains(server, "proxy_read_timeout 20s;") }) - Expect(err).NotTo(HaveOccurred()) }) It("should not set invalid proxy timeouts", func() { @@ -163,16 +139,12 @@ var _ = framework.IngressNginxDescribe("Annotations - Proxy", func() { } ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations) - _, err := f.EnsureIngress(ing) + f.EnsureIngress(ing) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) - - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return !strings.Contains(server, "proxy_connect_timeout 50ks;") && !strings.Contains(server, "proxy_send_timeout 20ks;") && !strings.Contains(server, "proxy_read_timeout 60s;") }) - Expect(err).NotTo(HaveOccurred()) }) It("should turn on proxy-buffering", func() { @@ -183,16 +155,12 @@ var _ = framework.IngressNginxDescribe("Annotations - Proxy", func() { } ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations) - _, err := f.EnsureIngress(ing) + f.EnsureIngress(ing) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) - - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return strings.Contains(server, "proxy_buffering on;") && strings.Contains(server, "proxy_buffer_size 8k;") && strings.Contains(server, "proxy_buffers 4 8k;") && strings.Contains(server, "proxy_request_buffering on;") }) - Expect(err).NotTo(HaveOccurred()) }) It("should turn off proxy-request-buffering", func() { @@ -202,16 +170,12 @@ var _ = framework.IngressNginxDescribe("Annotations - Proxy", func() { } ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations) - _, err := f.EnsureIngress(ing) + f.EnsureIngress(ing) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) - - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return Expect(server).Should(ContainSubstring("proxy_request_buffering off;")) }) - Expect(err).NotTo(HaveOccurred()) }) It("should build proxy next upstream", func() { @@ -222,16 +186,12 @@ var _ = framework.IngressNginxDescribe("Annotations - Proxy", func() { } ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations) - _, err := f.EnsureIngress(ing) + f.EnsureIngress(ing) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) - - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return strings.Contains(server, "proxy_next_upstream error timeout http_502;") && strings.Contains(server, "proxy_next_upstream_tries 5;") }) - Expect(err).NotTo(HaveOccurred()) }) It("should setup proxy cookies", func() { @@ -242,16 +202,11 @@ var _ = framework.IngressNginxDescribe("Annotations - Proxy", func() { } ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations) - _, err := f.EnsureIngress(ing) + f.EnsureIngress(ing) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) - - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return strings.Contains(server, "proxy_cookie_domain localhost example.org;") && strings.Contains(server, "proxy_cookie_path /one/ /;") }) - Expect(err).NotTo(HaveOccurred()) }) - }) diff --git a/test/e2e/annotations/redirect.go b/test/e2e/annotations/redirect.go index a2ce10d25..563c24778 100644 --- a/test/e2e/annotations/redirect.go +++ b/test/e2e/annotations/redirect.go @@ -53,17 +53,13 @@ var _ = framework.IngressNginxDescribe("Annotations - Redirect", func() { annotations := map[string]string{"nginx.ingress.kubernetes.io/permanent-redirect": redirectURL} ing := framework.NewSingleIngress(host, redirectPath, host, f.IngressController.Namespace, "http-svc", 80, &annotations) - _, err := f.EnsureIngress(ing) + f.EnsureIngress(ing) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) - - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return strings.Contains(server, fmt.Sprintf("if ($uri ~* %s) {", redirectPath)) && strings.Contains(server, fmt.Sprintf("return 301 %s;", redirectURL)) }) - Expect(err).NotTo(HaveOccurred()) By("sending request to redirected URL path") @@ -93,17 +89,13 @@ var _ = framework.IngressNginxDescribe("Annotations - Redirect", func() { } ing := framework.NewSingleIngress(host, redirectPath, host, f.IngressController.Namespace, "http-svc", 80, &annotations) - _, err := f.EnsureIngress(ing) + f.EnsureIngress(ing) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) - - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return strings.Contains(server, fmt.Sprintf("if ($uri ~* %s) {", redirectPath)) && strings.Contains(server, fmt.Sprintf("return %d %s;", redirectCode, redirectURL)) }) - Expect(err).NotTo(HaveOccurred()) By("sending request to redirected URL path") diff --git a/test/e2e/annotations/rewrite.go b/test/e2e/annotations/rewrite.go index 70c89db58..5f6c89f98 100644 --- a/test/e2e/annotations/rewrite.go +++ b/test/e2e/annotations/rewrite.go @@ -33,8 +33,7 @@ var _ = framework.IngressNginxDescribe("Annotations - Rewrite", func() { f := framework.NewDefaultFramework("rewrite") BeforeEach(func() { - err := f.NewEchoDeploymentWithReplicas(1) - Expect(err).NotTo(HaveOccurred()) + f.NewEchoDeploymentWithReplicas(1) }) AfterEach(func() { @@ -48,17 +47,13 @@ var _ = framework.IngressNginxDescribe("Annotations - Rewrite", func() { expectBodyRequestURI := fmt.Sprintf("request_uri=http://%v:8080/", host) ing := framework.NewSingleIngress(host, "/something", host, f.IngressController.Namespace, "http-svc", 80, &annotations) - _, err := f.EnsureIngress(ing) + f.EnsureIngress(ing) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) - - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return strings.Contains(server, `rewrite "(?i)/something/(.*)" /$1 break;`) && strings.Contains(server, `rewrite "(?i)/something$" / break;`) }) - Expect(err).NotTo(HaveOccurred()) By("sending request to Ingress rule path (lowercase)") @@ -93,16 +88,12 @@ var _ = framework.IngressNginxDescribe("Annotations - Rewrite", func() { } ing := framework.NewSingleIngress(host, "/something", host, f.IngressController.Namespace, "http-svc", 80, &annotations) - _, err := f.EnsureIngress(ing) + f.EnsureIngress(ing) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) - - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return strings.Contains(server, "rewrite_log on;") }) - Expect(err).NotTo(HaveOccurred()) resp, _, errs := gorequest.New(). Get(f.IngressController.HTTPURL+"/something"). @@ -123,15 +114,12 @@ var _ = framework.IngressNginxDescribe("Annotations - Rewrite", func() { By("creating a regular ingress definition") ing := framework.NewSingleIngress("kube-lego", "/.well-known/acme/challenge", host, f.IngressController.Namespace, "http-svc", 80, &map[string]string{}) - _, err := f.EnsureIngress(ing) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) + f.EnsureIngress(ing) - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return strings.Contains(server, "/.well-known/acme/challenge") }) - Expect(err).NotTo(HaveOccurred()) By("making a request to the non-rewritten location") resp, body, errs := gorequest.New(). @@ -148,15 +136,13 @@ var _ = framework.IngressNginxDescribe("Annotations - Rewrite", func() { "nginx.ingress.kubernetes.io/rewrite-target": "/new/backend", } rewriteIng := framework.NewSingleIngress("rewrite-index", "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations) - _, err = f.EnsureIngress(rewriteIng) - Expect(err).NotTo(HaveOccurred()) - Expect(rewriteIng).NotTo(BeNil()) - err = f.WaitForNginxServer(host, + f.EnsureIngress(rewriteIng) + + f.WaitForNginxServer(host, func(server string) bool { return strings.Contains(server, "location ~* ^/ {") && strings.Contains(server, `location ~* "^/.well-known/acme/challenge" {`) }) - Expect(err).NotTo(HaveOccurred()) By("making a second request to the non-rewritten location") resp, body, errs = gorequest.New(). @@ -173,15 +159,12 @@ var _ = framework.IngressNginxDescribe("Annotations - Rewrite", func() { By("creating a regular ingress definition") ing := framework.NewSingleIngress("foo", "/foo", host, f.IngressController.Namespace, "http-svc", 80, &map[string]string{}) - _, err := f.EnsureIngress(ing) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) + f.EnsureIngress(ing) - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return strings.Contains(server, "location /foo {") }) - Expect(err).NotTo(HaveOccurred()) By(`creating an ingress definition with the use-regex amd rewrite-target annotation`) annotations := map[string]string{ @@ -189,15 +172,12 @@ var _ = framework.IngressNginxDescribe("Annotations - Rewrite", func() { "nginx.ingress.kubernetes.io/rewrite-target": "/new/backend", } ing = framework.NewSingleIngress("regex", "/foo.+", host, f.IngressController.Namespace, "http-svc", 80, &annotations) - _, err = f.EnsureIngress(ing) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) + f.EnsureIngress(ing) - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return strings.Contains(server, `location ~* "^/foo" {`) && strings.Contains(server, `location ~* "^/foo.+\/?(?.*)" {`) }) - Expect(err).NotTo(HaveOccurred()) By("ensuring '/foo' matches '~* ^/foo'") resp, body, errs := gorequest.New(). @@ -225,9 +205,7 @@ var _ = framework.IngressNginxDescribe("Annotations - Rewrite", func() { By("creating a regular ingress definition") ing := framework.NewSingleIngress("foo", "/foo/bar/bar", host, f.IngressController.Namespace, "http-svc", 80, &map[string]string{}) - _, err := f.EnsureIngress(ing) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) + f.EnsureIngress(ing) By(`creating an ingress definition with the use-regex annotation`) annotations := map[string]string{ @@ -235,15 +213,12 @@ var _ = framework.IngressNginxDescribe("Annotations - Rewrite", func() { "nginx.ingress.kubernetes.io/rewrite-target": "/new/backend", } ing = framework.NewSingleIngress("regex", "/foo/bar/[a-z]{3}", host, f.IngressController.Namespace, "http-svc", 80, &annotations) - _, err = f.EnsureIngress(ing) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) + f.EnsureIngress(ing) - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return strings.Contains(server, `location ~* "^/foo/bar/bar" {`) && strings.Contains(server, `location ~* "^/foo/bar/[a-z]{3}\/?(?.*)" {`) }) - Expect(err).NotTo(HaveOccurred()) By("check that '/foo/bar/bar' does not match the longest exact path") resp, body, errs := gorequest.New(). diff --git a/test/e2e/annotations/serversnippet.go b/test/e2e/annotations/serversnippet.go new file mode 100644 index 000000000..90bb36a25 --- /dev/null +++ b/test/e2e/annotations/serversnippet.go @@ -0,0 +1,53 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package annotations + +import ( + "strings" + + . "github.com/onsi/ginkgo" + + "k8s.io/ingress-nginx/test/e2e/framework" +) + +var _ = framework.IngressNginxDescribe("Annotations - ServerSnippet", func() { + f := framework.NewDefaultFramework("serversnippet") + + BeforeEach(func() { + f.NewEchoDeploymentWithReplicas(2) + }) + + AfterEach(func() { + }) + + It(`add valid directives to server via server snippet"`, func() { + host := "serversnippet.foo.com" + annotations := map[string]string{ + "nginx.ingress.kubernetes.io/server-snippet": ` + more_set_headers "Content-Length: $content_length"; + more_set_headers "Content-Type: $content_type";`, + } + + ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations) + f.EnsureIngress(ing) + + f.WaitForNginxServer(host, + func(server string) bool { + return strings.Contains(server, `more_set_headers "Content-Length: $content_length`) && strings.Contains(server, `more_set_headers "Content-Type: $content_type";`) + }) + }) +}) diff --git a/test/e2e/annotations/snippet.go b/test/e2e/annotations/snippet.go index 81ee9724e..1686fb37b 100644 --- a/test/e2e/annotations/snippet.go +++ b/test/e2e/annotations/snippet.go @@ -26,8 +26,7 @@ var _ = framework.IngressNginxDescribe("Annotations - Configurationsnippet", fun f := framework.NewDefaultFramework("configurationsnippet") BeforeEach(func() { - err := f.NewEchoDeploymentWithReplicas(2) - Expect(err).NotTo(HaveOccurred()) + f.NewEchoDeploymentWithReplicas(2) }) AfterEach(func() { @@ -41,15 +40,11 @@ var _ = framework.IngressNginxDescribe("Annotations - Configurationsnippet", fun } ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations) - _, err := f.EnsureIngress(ing) + f.EnsureIngress(ing) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) - - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return Expect(server).Should(ContainSubstring(`more_set_headers "Request-Id: $req_id";`)) }) - Expect(err).NotTo(HaveOccurred()) }) }) diff --git a/test/e2e/annotations/sslciphers.go b/test/e2e/annotations/sslciphers.go index b71a39cfe..859444022 100644 --- a/test/e2e/annotations/sslciphers.go +++ b/test/e2e/annotations/sslciphers.go @@ -27,8 +27,7 @@ var _ = framework.IngressNginxDescribe("Annotations - SSL CIPHERS", func() { f := framework.NewDefaultFramework("sslciphers") BeforeEach(func() { - err := f.NewEchoDeploymentWithReplicas(2) - Expect(err).NotTo(HaveOccurred()) + f.NewEchoDeploymentWithReplicas(2) }) AfterEach(func() { @@ -41,15 +40,11 @@ var _ = framework.IngressNginxDescribe("Annotations - SSL CIPHERS", func() { } ing := framework.NewSingleIngress(host, "/something", host, f.IngressController.Namespace, "http-svc", 80, &annotations) - _, err := f.EnsureIngress(ing) + f.EnsureIngress(ing) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) - - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return Expect(server).Should(ContainSubstring("ssl_ciphers ALL:!aNULL:!EXPORT56:RC4+RSA:+HIGH:+MEDIUM:+LOW:+SSLv2:+EXP;")) }) - Expect(err).NotTo(HaveOccurred()) }) }) diff --git a/test/e2e/annotations/upstreamvhost.go b/test/e2e/annotations/upstreamvhost.go index ff57396a5..ae5789425 100644 --- a/test/e2e/annotations/upstreamvhost.go +++ b/test/e2e/annotations/upstreamvhost.go @@ -26,8 +26,7 @@ var _ = framework.IngressNginxDescribe("Annotations - Upstreamvhost", func() { f := framework.NewDefaultFramework("upstreamvhost") BeforeEach(func() { - err := f.NewEchoDeploymentWithReplicas(2) - Expect(err).NotTo(HaveOccurred()) + f.NewEchoDeploymentWithReplicas(2) }) AfterEach(func() { @@ -40,15 +39,11 @@ var _ = framework.IngressNginxDescribe("Annotations - Upstreamvhost", func() { } ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations) - _, err := f.EnsureIngress(ing) + f.EnsureIngress(ing) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) - - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return Expect(server).Should(ContainSubstring(`proxy_set_header Host "upstreamvhost.bar.com";`)) }) - Expect(err).NotTo(HaveOccurred()) }) }) diff --git a/test/e2e/defaultbackend/custom_default_backend.go b/test/e2e/defaultbackend/custom_default_backend.go index ea9e9f5dd..c2104b415 100644 --- a/test/e2e/defaultbackend/custom_default_backend.go +++ b/test/e2e/defaultbackend/custom_default_backend.go @@ -23,6 +23,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/parnurzeal/gorequest" appsv1beta1 "k8s.io/api/apps/v1beta1" @@ -34,8 +35,7 @@ var _ = framework.IngressNginxDescribe("Dynamic Certificate", func() { f := framework.NewDefaultFramework("custom-default-backend") BeforeEach(func() { - err := f.NewEchoDeploymentWithReplicas(1) - Expect(err).NotTo(HaveOccurred()) + f.NewEchoDeploymentWithReplicas(1) framework.UpdateDeployment(f.KubeClientSet, f.IngressController.Namespace, "nginx-ingress-controller", 1, func(deployment *appsv1beta1.Deployment) error { @@ -47,11 +47,10 @@ var _ = framework.IngressNginxDescribe("Dynamic Certificate", func() { return err }) - err = f.WaitForNginxServer("_", + f.WaitForNginxServer("_", func(server string) bool { return strings.Contains(server, "set $proxy_upstream_name \"upstream-default-backend\"") }) - Expect(err).ToNot(HaveOccurred()) }) It("uses custom default backend", func() { diff --git a/test/e2e/dind-cluster-v1.11.sh b/test/e2e/dind-cluster-v1.11.sh deleted file mode 100755 index e642839cc..000000000 --- a/test/e2e/dind-cluster-v1.11.sh +++ /dev/null @@ -1,2347 +0,0 @@ -#!/bin/bash -# Copyright 2017 Mirantis -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -o errexit -set -o nounset -set -o pipefail -set -o errtrace - -if [ $(uname) = Darwin ]; then - readlinkf(){ perl -MCwd -e 'print Cwd::abs_path shift' "$1";} -else - readlinkf(){ readlink -f "$1"; } -fi -DIND_ROOT="$(cd $(dirname "$(readlinkf "${BASH_SOURCE}")"); pwd)" - -RUN_ON_BTRFS_ANYWAY="${RUN_ON_BTRFS_ANYWAY:-}" -if [[ ! ${RUN_ON_BTRFS_ANYWAY} ]] && docker info| grep -q '^Storage Driver: btrfs'; then - echo "ERROR: Docker is using btrfs storage driver which is unsupported by kubeadm-dind-cluster" >&2 - echo "Please refer to the documentation for more info." >&2 - echo "Set RUN_ON_BTRFS_ANYWAY to non-empty string to continue anyway." >&2 - exit 1 -fi - -# In case of linuxkit / moby linux, -v will not work so we can't -# mount /lib/modules and /boot. Also we'll be using localhost -# to access the apiserver. -using_linuxkit= -if ! docker info|grep -s '^Operating System: .*Docker for Windows' > /dev/null 2>&1 ; then - if docker info|grep -s '^Kernel Version: .*-moby$' >/dev/null 2>&1 || - docker info|grep -s '^Kernel Version: .*-linuxkit-' > /dev/null 2>&1 ; then - using_linuxkit=1 - fi -fi - -# Determine when using Linux and docker daemon running locally -using_local_linuxdocker= -if [[ $(uname) == Linux && -z ${DOCKER_HOST:-} ]]; then - using_local_linuxdocker=1 -fi - -EMBEDDED_CONFIG=y;DIND_IMAGE=mirantis/kubeadm-dind-cluster:v1.11 - -# dind::localhost provides the local host IP based on the address family used for service subnet. -function dind::localhost() { - if [[ ${SERVICE_NET_MODE} = "ipv6" ]]; then - echo '[::1]' - else - echo '127.0.0.1' - fi -} - -# dind::family-for indicates whether the CIDR or IP is for an IPv6 or IPv4 family. -function dind::family-for { - local addr=$1 - if [[ "$addr" = *":"* ]]; then - echo "ipv6" - else - echo "ipv4" - fi -} - -# dind::cluster-suffix builds a suffix used for resources, based on the DIND_LABEL. -function dind::cluster-suffix { - if [ "$DIND_LABEL" != "$DEFAULT_DIND_LABEL" ]; then - echo "-${DIND_LABEL}" - else - echo '' - fi -} - -function dind::net-name { - echo "kubeadm-dind-net$( dind::cluster-suffix )" -} - -# dind::add-cluster will inject the cluster ID to the IP address. For IPv4, it is -# assumed that the IP is a /24 with the third part of the address available for cluster ID. -# For IPv6, it is assumed that there is enough space for the cluster to be added, and the -# cluster ID will be added to the 16 bits before the double colon. For example: -# -# 10.192.0.0/24 => 10.192.5.0/24 -# fd00:77:20::/64 => fd00:77:20:5::/64 -# -# This function is intended to be used for management networks. -# -# TODO: Validate that there is enough space for cluster ID. -# TODO: For IPv6 could get fancy and handle case where cluster ID is placed in upper 8 bits of hextet -# TODO: Consider if want to do /16 for IPv4 management subnet. -# -function dind::add-cluster { - local cidr=$1 - local ip_mode=$2 - - if [[ ${ip_mode} = "ipv4" ]]; then - echo ${cidr} | sed "s/^\([0-9]*\.[0-9]*\.\).*\/24$/\1${CLUSTER_ID}.0\/24/" - else # IPv6 - echo ${cidr} | sed "s/^\(.*\)\(\:\:\/[0-9]*\)$/\1:${CLUSTER_ID}\2/" - fi -} - -# dind::get-and-validate-cidrs takes a list of CIDRs and validates them based on the ip -# mode, returning them. For IPv4 only and IPv6 only modes, only one CIDR is expected. For -# dual stack, two CIDRS are expected. It verifies that the CIDRs are the right family and -# will use the provided defaults, when CIDRs are missing. For dual-stack, the IPv4 address -# will be first. -# -# For the management network, the cluster ID will be injected into the CIDR. Also, if no -# MGMT_CIDRS value is specified, but the legacy DIND_SUBNET/DIND_SUBNET_SIZE is provided, -# that will be used for the (first) CIDR. -# -# NOTE: It is expected that the CIDR size is /24 for IPv4 management networks. -# -# For pod CIDRs, the size will be increased by 8, to leave room for the node ID to be -# injected into the address. -# -# NOTE: For IPv4, the pod size is expected to be /16 -> /24 in usage. -# -function dind::get-and-validate-cidrs { - IFS=', ' read -r -a cidrs <<< "$1" - IFS=', ' read -r -a defaults <<< "$2" - local is_mgmt=$3 - case ${IP_MODE} in - ipv4) - case ${#cidrs[@]} in - 0) - cidrs[0]="${defaults[0]}" - ;; - 1) - ;; - *) - echo "ERROR! More than one CIDR provided '$1'" - exit 1 - ;; - esac - if [[ $( dind::family-for "${cidrs[0]}" ) != "ipv4" ]]; then - echo "ERROR! CIDR must be IPv4 value" - exit 1 - fi - if [[ ${is_mgmt} = true ]]; then - cidrs[0]="$( dind::add-cluster "${cidrs[0]}" "${IP_MODE}" )" - fi - ;; - - ipv6) - case ${#cidrs[@]} in - 0) - cidrs[0]="${defaults[0]}" - ;; - 1) - ;; - *) - echo "ERROR! More than one CIDR provided '$1'" - exit 1 - ;; - esac - if [[ $( dind::family-for "${cidrs[0]}" ) != "ipv6" ]]; then - echo "ERROR! CIDR must be IPv6 value" - exit 1 - fi - if [[ ${is_mgmt} = true ]]; then - cidrs[0]="$( dind::add-cluster "${cidrs[0]}" "${IP_MODE}" )" - fi - ;; - - dual-stack) - case ${#cidrs[@]} in - 0) - cidrs[0]="${defaults[0]}" - cidrs[1]="${defaults[1]}" - ;; - 1) - if [[ $( dind::family-for "${cidrs[0]}" ) = "ipv6" ]]; then - cidrs[1]=${cidrs[0]} - cidrs[0]="${defaults[0]}" # Assuming first default is a V4 address - else - cidrs[1]="${defaults[1]}" - fi - ;; - 2) - # Force ordering to have V4 address first - if [[ $( dind::family-for "${cidrs[0]}" ) = "ipv6" ]]; then - local temp=${cidrs[0]} - cidrs[0]=${cidrs[1]} - cidrs[1]=${temp} - fi - ;; - *) - echo "ERROR! More than two CIDRs provided '$1'" - exit 1 - ;; - esac - local have_v4="" - local have_v6="" - for cidr in ${cidrs[@]}; do - if [[ $( dind::family-for "${cidr}" ) = "ipv6" ]]; then - have_v6=1 - else - have_v4=1 - fi - done - if [[ -z ${have_v4} ]]; then - echo "ERROR! Missing IPv4 CIDR in '$1'" - exit 1 - fi - if [[ -z ${have_v6} ]]; then - echo "ERROR! Missing IPv6 CIDR in '$1'" - exit 1 - fi - if [[ ${is_mgmt} = true ]]; then - cidrs[0]="$( dind::add-cluster "${cidrs[0]}" "${IP_MODE}" )" - cidrs[1]="$( dind::add-cluster "${cidrs[1]}" "${IP_MODE}" )" - fi - ;; - esac - echo "${cidrs[@]}" -} - -# dind::make-ip-from-cidr strips off the slash and size, and appends the -# interface part to the prefix to form an IP. For IPv4, it strips off the -# fourth part of the prefix, so that it can be replaced. It assumes that the -# resulting prefix will be of sufficient size. It also will use hex for the -# appended part for IPv6, and decimal for IPv4. -# -# fd00:20::/64 -> fd00:20::a -# 10.96.0.0/12 -> 10.96.0.10 -# -function dind::make-ip-from-cidr { - prefix="$(echo $1 | sed 's,/.*,,')" - if [[ $( dind::family-for ${prefix} ) == "ipv4" ]]; then - printf "%s%d" $( echo ${prefix} | sed 's/0$//' ) $2 - else - printf "%s%x" ${prefix} $2 - fi -} - -# dind::add-cluster-id-and-validate-nat64-prefix will modify the IPv4 mapping -# subnet prefix, by adding the cluster ID (default 0) to the second octet. -# It will produce an error, if the prefix is not in the 10.0.0.0/8 or -# 172.16.0.0/12 private networks. -function dind::add-cluster-id-and-validate-nat64-prefix { - local parts - IFS="." read -a parts <<<${NAT64_V4_SUBNET_PREFIX} - if [[ ${#parts[@]} -ne 2 ]]; then - echo "ERROR! NAT64_V4_SUBNET_PREFIX must be two octets (have '${NAT64_V4_SUBNET_PREFIX}')" - exit 1 - fi - (( parts[1]+=${CLUSTER_ID} )) - NAT64_V4_SUBNET_PREFIX="${parts[0]}.${parts[1]}" - echo "Added cluster ID offset (${CLUSTER_ID}) to NAT64_V4_SUBNET_PREFIX giving prefix '${NAT64_V4_SUBNET_PREFIX}'" - if [[ ${parts[0]} -eq 10 ]]; then - if [[ ${parts[1]} > 253 ]]; then - echo "ERROR! NAT64_V4_SUBNET_PREFIX is too large for 10.0.0.0/8 private net" - exit 1 - fi - elif [[ ${parts[0]} -eq 172 ]]; then - if [[ ${parts[1]} -lt 16 || ${parts[1]} -gt 31 ]]; then - echo "ERROR! NAT64_V4_SUBNET_PREFIX is outside of range for 172.16.0.0/12 private net" - exit 1 - fi - else - echo "ERROR! NAT64_V4_SUBNET_PREFIX is not in 10.0.0.0/8 or 172.16.0.0/12 private networks" - exit 1 - fi - echo "Using NAT64 V4 mapping network prefix: ${NAT64_V4_SUBNET_PREFIX}" -} - - -# START OF PROCESSING... - -IP_MODE="${IP_MODE:-ipv4}" # ipv4, ipv6, dual-stack -# FUTURE: Once dual-stack support is released, check K8s version, and reject for older versions. -if [[ ! ${EMBEDDED_CONFIG:-} ]]; then - source "${DIND_ROOT}/config.sh" -fi - -# Multicluster support -# Users can specify a cluster ID number from 1..254, represented as a string. -# This will be used to form resource names "cluster-#", and will be used in the -# management subnet to give unique networks for each cluster. If the cluster ID -# is not specified, or zero, it will be considered a single cluster or the first -# in the multi-cluster. This is the recommended usage. -# -# For legacy support, the user can specify DIND_LABEL, which will be used in the -# resource names. If a cluster ID is specified (a hybrid case, where people are -# using the new method, but want custom names), the resourse name will have the -# suffix "-#" with the cluster ID. If no cluster ID is specified (for backward -# compatibility), then the resource name will be just the DIND_LABEL, and a pseudo- -# random number from 1..13 will be generated for the cluster ID to be used in -# management network. The range is limited, because, in IPv6 mode, the cluster ID -# is used in the NAT64 V4 subnet prefix, which must be in a private network. -# The default is 172.18, so the cluster ID cannot be larger than 13 to guarantee -# a valid value. -# -# To get around that limitation, you can set the cluster ID, in addition to the -# DIND_LABEL, and optionally, change the NAT64_V4_SUBNET_PREFIX value. -# -DEFAULT_DIND_LABEL='mirantis.kubeadm_dind_cluster_runtime' -if [[ -z ${DIND_LABEL+x} ]]; then # No legacy DIND_LABEL set - if [[ -z ${CLUSTER_ID+x} ]]; then # No cluster ID set - DIND_LABEL=${DEFAULT_DIND_LABEL} # Single cluster mode - CLUSTER_ID="0" - else # Have cluster ID - if [[ ${CLUSTER_ID} = "0" ]]; then - DIND_LABEL=${DEFAULT_DIND_LABEL} # Single cluster mode or first cluster of multi-cluster - else - DIND_LABEL="cluster-${CLUSTER_ID}" # Multi-cluster - fi - fi -else # Legacy DIND_LABEL set for multi-cluster - if [[ -z ${CLUSTER_ID+x} ]]; then # No cluster ID set, make one from 1..13, but don't use in resource names - CLUSTER_ID="$(( ($RANDOM % 12) + 1 ))" - else - if [[ ${CLUSTER_ID} = "0" ]]; then - CLUSTER_ID="$(( ($RANDOM % 12) + 1 ))" # Force a pseudo-random cluster for additional legacy cluster - else - DIND_LABEL="${DIND_LABEL}-${CLUSTER_ID}" - fi - fi -fi - -CNI_PLUGIN="${CNI_PLUGIN:-bridge}" -GCE_HOSTED="${GCE_HOSTED:-}" -DIND_ALLOW_AAAA_USE="${DIND_ALLOW_AAAA_USE:-}" # Default is to use DNS64 always for IPv6 mode -KUBE_ROUTER_VERSION="${KUBE_ROUTER_VERSION:-v0.2.0}" - -# Use legacy DIND_SUBNET/DIND_SUBNET_SIZE, only if MGMT_CIDRS is not set. -legacy_mgmt_cidr="" -if [[ ${DIND_SUBNET:-} && ${DIND_SUBNET_SIZE:-} ]]; then - legacy_mgmt_cidr="${DIND_SUBNET}/${DIND_SUBNET_SIZE}" -fi - -if [[ ${IP_MODE} = "dual-stack" ]]; then - mgmt_net_defaults="10.192.0.0/24, fd00:20::/64" - - KUBE_RSYNC_ADDR="${KUBE_RSYNC_ADDR:-::1}" - SERVICE_CIDR="${SERVICE_CIDR:-fd00:30::/110}" # Will default to IPv6 service net family - - pod_net_defaults="10.244.0.0/16, fd00:40::/72" - - USE_HAIRPIN="${USE_HAIRPIN:-true}" # Default is to use hairpin for dual-stack - DIND_ALLOW_AAAA_USE=true # Forced, so can access external hosts via IPv6 - if [[ ${DIND_ALLOW_AAAA_USE} && ${GCE_HOSTED} ]]; then - echo "ERROR! GCE does not support use of IPv6 for external addresses - aborting." - exit 1 - fi -elif [[ ${IP_MODE} = "ipv6" ]]; then - mgmt_net_defaults="fd00:20::/64" - - KUBE_RSYNC_ADDR="${KUBE_RSYNC_ADDR:-::1}" - SERVICE_CIDR="${SERVICE_CIDR:-fd00:30::/110}" - - pod_net_defaults="fd00:40::/72" - - USE_HAIRPIN="${USE_HAIRPIN:-true}" # Default is to use hairpin for IPv6 - if [[ ${DIND_ALLOW_AAAA_USE} && ${GCE_HOSTED} ]]; then - echo "ERROR! GCE does not support use of IPv6 for external addresses - aborting." - exit 1 - fi -else # IPv4 mode - mgmt_net_defaults="10.192.0.0/24" - - KUBE_RSYNC_ADDR="${KUBE_RSYNC_ADDR:-127.0.0.1}" - SERVICE_CIDR="${SERVICE_CIDR:-10.96.0.0/12}" - - pod_net_defaults="10.244.0.0/16" - - USE_HAIRPIN="${USE_HAIRPIN:-false}" # Disabled for IPv4, as issue with Virtlet networking - if [[ ${DIND_ALLOW_AAAA_USE} ]]; then - echo "WARNING! The DIND_ALLOW_AAAA_USE option is for IPv6 mode - ignoring setting." - DIND_ALLOW_AAAA_USE= - fi - if [[ ${CNI_PLUGIN} = "calico" || ${CNI_PLUGIN} = "calico-kdd" ]]; then - pod_net_defaults="192.168.0.0/16" - fi -fi - -IFS=' ' read -r -a mgmt_net_cidrs <<<$( dind::get-and-validate-cidrs "${MGMT_CIDRS:-${legacy_mgmt_cidr}}" "${mgmt_net_defaults[@]}" true ) - -REMOTE_DNS64_V4SERVER="${REMOTE_DNS64_V4SERVER:-8.8.8.8}" -if [[ ${IP_MODE} == "ipv6" ]]; then - # Uses local DNS64 container - dns_server="$( dind::make-ip-from-cidr ${mgmt_net_cidrs[0]} 0x100 )" - DNS64_PREFIX="${DNS64_PREFIX:-fd00:10:64:ff9b::}" - DNS64_PREFIX_SIZE="${DNS64_PREFIX_SIZE:-96}" - DNS64_PREFIX_CIDR="${DNS64_PREFIX}/${DNS64_PREFIX_SIZE}" - - LOCAL_NAT64_SERVER="$( dind::make-ip-from-cidr ${mgmt_net_cidrs[0]} 0x200 )" - NAT64_V4_SUBNET_PREFIX="${NAT64_V4_SUBNET_PREFIX:-172.18}" - dind::add-cluster-id-and-validate-nat64-prefix -else - dns_server="${REMOTE_DNS64_V4SERVER}" -fi - -SERVICE_NET_MODE="$( dind::family-for ${SERVICE_CIDR} )" -DNS_SVC_IP="$( dind::make-ip-from-cidr ${SERVICE_CIDR} 10 )" - -ETCD_HOST="${ETCD_HOST:-$( dind::localhost )}" - -IFS=' ' read -r -a pod_net_cidrs <<<$( dind::get-and-validate-cidrs "${POD_NETWORK_CIDR:-}" "${pod_net_defaults[@]}" false ) - -declare -a pod_prefixes -declare -a pod_sizes -# Extract the prefix and size from the provided pod CIDR(s), based on the IP mode of each. The -# size will be increased by 8, to make room for the node ID to be added to the prefix later. -# Bridge and PTP plugins can process IPv4 and IPv6 pod CIDRs, other plugins must be IPv4 only. -for pod_cidr in "${pod_net_cidrs[@]}"; do - if [[ $( dind::family-for "${pod_cidr}" ) = "ipv4" ]]; then - actual_size=$( echo ${pod_cidr} | sed 's,.*/,,' ) - if [[ ${actual_size} -ne 16 ]]; then - echo "ERROR! For IPv4 CIDRs, the size must be /16. Have '${pod_cidr}'" - exit 1 - fi - pod_sizes+=( 24 ) - pod_prefixes+=( "$(echo ${pod_cidr} | sed 's/^\([0-9]*\.[0-9]*\.\).*/\1/')" ) - else # IPv6 - if [[ ${CNI_PLUGIN} != "bridge" && ${CNI_PLUGIN} != "ptp" ]]; then - echo "ERROR! IPv6 pod networks are only supported by bridge and PTP CNI plugins" - exit 1 - fi - # There are several cases to address. First, is normal split of prefix and size: - # fd00:10:20:30::/64 ---> fd00:10:20:30: /72 - # - # Second, is when the prefix needs to be padded, so that node ID can be added later: - # fd00:10::/64 ---> fd00:10:0:0: /72 - # - # Third, is when the low order part of the address, must be removed for the prefix, - # as the node ID will be placed in the lower byte: - # fd00:10:20:30:4000::/72 ---> fd00:10:20:30:40 /80 - # - # We will attempt to check for three error cases. One is when the address part is - # way too big for the size specified: - # fd00:10:20:30:40::/48 ---> fd00:10:20: /56 desired, but conflict with 30:40: - # - # Another is when the address part, once trimmed for the size, would loose info: - # fd00:10:20:1234::/56 ---> fd00:10:20:12 /64, but lost 34:, which conflicts - # - # Lastly, again, trimming would leave high byte in hextet, conflicting with - # the node ID: - # fd00:10:20:30:1200::/64 ---> fd00:10:20:30:12 /72, but 12 conflicts - # - # Note: later, the node ID will be appended to the prefix generated. - # - cluster_size="$(echo ${pod_cidr} | sed 's,.*::/,,')" - pod_sizes+=( $((${cluster_size}+8)) ) - - pod_prefix="$(echo ${pod_cidr} | sed 's,::/.*,:,')" - num_colons="$(grep -o ":" <<< "${pod_prefix}" | wc -l)" - need_zero_pads=$((${cluster_size}/16)) - - if [[ ${num_colons} -gt $((need_zero_pads + 1)) ]]; then - echo "ERROR! Address part of CIDR (${pod_prefix}) is too large for /${cluster_size}" - exit 1 - fi - if [[ ${num_colons} -gt ${need_zero_pads} ]]; then - # Will be replacing lowest byte with node ID, so pull off lower byte and colon - if [[ ${pod_prefix: -3} != "00:" ]]; then # last byte is not zero - echo "ERROR! Cannot trim address part of CIDR (${pod_prefix}) to fit in /${cluster_size}" - exit 1 - fi - pod_prefix=${pod_prefix::-3} - if [[ $(( ${cluster_size} % 16 )) -eq 0 && $( ${pod_prefix: -1} ) != ":" ]]; then # should not be upper byte for this size CIDR - echo "ERROR! Trimmed address part of CIDR (${pod_prefix}) is still too large for /${cluster_size}" - exit 1 - fi - fi - # Add in zeros to pad 16 bits at a time, up to the padding needed, which is - # need_zero_pads - num_colons. - while [ ${num_colons} -lt ${need_zero_pads} ]; do - pod_prefix+="0:" - ((num_colons++)) - done - pod_prefixes+=( "${pod_prefix}" ) - fi -done - -DIND_IMAGE="${DIND_IMAGE:-}" -BUILD_KUBEADM="${BUILD_KUBEADM:-}" -BUILD_HYPERKUBE="${BUILD_HYPERKUBE:-}" -KUBEADM_SOURCE="${KUBEADM_SOURCE-}" -HYPERKUBE_SOURCE="${HYPERKUBE_SOURCE-}" -NUM_NODES=${NUM_NODES:-2} -EXTRA_PORTS="${EXTRA_PORTS:-}" -LOCAL_KUBECTL_VERSION=${LOCAL_KUBECTL_VERSION:-} -KUBECTL_DIR="${KUBECTL_DIR:-${HOME}/.kubeadm-dind-cluster}" -DASHBOARD_URL="${DASHBOARD_URL:-https://rawgit.com/kubernetes/dashboard/bfab10151f012d1acc5dfb1979f3172e2400aa3c/src/deploy/kubernetes-dashboard.yaml}" -SKIP_SNAPSHOT="${SKIP_SNAPSHOT:-}" -E2E_REPORT_DIR="${E2E_REPORT_DIR:-}" -DIND_NO_PARALLEL_E2E="${DIND_NO_PARALLEL_E2E:-}" -DNS_SERVICE="${DNS_SERVICE:-kube-dns}" -DIND_STORAGE_DRIVER="${DIND_STORAGE_DRIVER:-overlay2}" - -DIND_CA_CERT_URL="${DIND_CA_CERT_URL:-}" -DIND_PROPAGATE_HTTP_PROXY="${DIND_PROPAGATE_HTTP_PROXY:-}" -DIND_HTTP_PROXY="${DIND_HTTP_PROXY:-}" -DIND_HTTPS_PROXY="${DIND_HTTPS_PROXY:-}" -DIND_NO_PROXY="${DIND_NO_PROXY:-}" - -DIND_DAEMON_JSON_FILE="${DIND_DAEMON_JSON_FILE:-/etc/docker/daemon.json}" # can be set to /dev/null -DIND_REGISTRY_MIRROR="${DIND_REGISTRY_MIRROR:-}" # plain string format -DIND_INSECURE_REGISTRIES="${DIND_INSECURE_REGISTRIES:-}" # json list format - -FEATURE_GATES="${FEATURE_GATES:-MountPropagation=true}" -# you can set special value 'none' not to set any kubelet's feature gates. -KUBELET_FEATURE_GATES="${KUBELET_FEATURE_GATES:-MountPropagation=true,DynamicKubeletConfig=true}" - -KUBELET_EXTRA_ARGS="--sync-frequency=30s" - -if [[ ! ${LOCAL_KUBECTL_VERSION:-} && ${DIND_IMAGE:-} =~ :(v[0-9]+\.[0-9]+)$ ]]; then - LOCAL_KUBECTL_VERSION="${BASH_REMATCH[1]}" -fi - -ENABLE_CEPH="${ENABLE_CEPH:-}" - -# TODO: Test multi-cluster for IPv6, before enabling -if [[ "${DIND_LABEL}" != "${DEFAULT_DIND_LABEL}" && "${IP_MODE}" == 'dual-stack' ]]; then - echo "Multiple parallel clusters currently not supported for dual-stack mode" >&2 - exit 1 -fi - -# not configurable for now, would need to setup context for kubectl _inside_ the cluster -readonly INTERNAL_APISERVER_PORT=8080 - -function dind::need-source { - if [[ ! -f cluster/kubectl.sh ]]; then - echo "$0 must be called from the Kubernetes repository root directory" 1>&2 - exit 1 - fi -} - -build_tools_dir="build" -use_k8s_source=y -if [[ ! ${BUILD_KUBEADM} && ! ${BUILD_HYPERKUBE} ]]; then - use_k8s_source= -fi -if [[ ${use_k8s_source} ]]; then - dind::need-source - kubectl=cluster/kubectl.sh - if [[ ! -f ${build_tools_dir}/common.sh ]]; then - build_tools_dir="build-tools" - fi -else - if [[ ! ${LOCAL_KUBECTL_VERSION:-} ]] && ! hash kubectl 2>/dev/null; then - echo "You need kubectl binary in your PATH to use prebuilt DIND image" 1>&2 - exit 1 - fi - kubectl=kubectl -fi - -function dind::retry { - # based on retry function in hack/jenkins/ scripts in k8s source - for i in {1..10}; do - "$@" && return 0 || sleep ${i} - done - "$@" -} - -busybox_image="busybox:1.26.2" -e2e_base_image="golang:1.9.2" -sys_volume_args=() -build_volume_args=() - -function dind::set-build-volume-args { - if [ ${#build_volume_args[@]} -gt 0 ]; then - return 0 - fi - build_container_name= - if [ -n "${KUBEADM_DIND_LOCAL:-}" ]; then - build_volume_args=(-v "$PWD:/go/src/k8s.io/kubernetes") - else - build_container_name="$(KUBE_ROOT=${PWD} ETCD_HOST=${ETCD_HOST} && - . ${build_tools_dir}/common.sh && - kube::build::verify_prereqs >&2 && - echo "${KUBE_DATA_CONTAINER_NAME:-${KUBE_BUILD_DATA_CONTAINER_NAME}}")" - build_volume_args=(--volumes-from "${build_container_name}") - fi -} - -function dind::volume-exists { - local name="$1" - if docker volume inspect "${name}" >& /dev/null; then - return 0 - fi - return 1 -} - -function dind::create-volume { - local name="$1" - docker volume create --label "${DIND_LABEL}" --name "${name}" >/dev/null -} - -# We mount /boot and /lib/modules into the container -# below to in case some of the workloads need them. -# This includes virtlet, for instance. Also this may be -# useful in future if we want DIND nodes to pass -# preflight checks. -# Unfortunately we can't do this when using Mac Docker -# (unless a remote docker daemon on Linux is used) -# NB: there's no /boot on recent Mac dockers -function dind::prepare-sys-mounts { - if [[ ! ${using_linuxkit} ]]; then - sys_volume_args=() - if [[ -d /boot ]]; then - sys_volume_args+=(-v /boot:/boot) - fi - if [[ -d /lib/modules ]]; then - sys_volume_args+=(-v /lib/modules:/lib/modules) - fi - return 0 - fi - local dind_sys_vol_name - dind_sys_vol_name="kubeadm-dind-sys$( dind::cluster-suffix )" - if ! dind::volume-exists "$dind_sys_vol_name"; then - dind::step "Saving a copy of docker host's /lib/modules" - dind::create-volume "$dind_sys_vol_name" - # Use a dirty nsenter trick to fool Docker on Mac and grab system - # /lib/modules into sys.tar file on kubeadm-dind-sys volume. - local nsenter="nsenter --mount=/proc/1/ns/mnt --" - docker run \ - --rm \ - --privileged \ - -v "$dind_sys_vol_name":/dest \ - --pid=host \ - "${busybox_image}" \ - /bin/sh -c \ - "if ${nsenter} test -d /lib/modules; then ${nsenter} tar -C / -c lib/modules >/dest/sys.tar; fi" - fi - sys_volume_args=(-v "$dind_sys_vol_name":/dind-sys) -} - -tmp_containers=() - -function dind::cleanup { - if [ ${#tmp_containers[@]} -gt 0 ]; then - for name in "${tmp_containers[@]}"; do - docker rm -vf "${name}" 2>/dev/null - done - fi -} - -trap dind::cleanup EXIT - -function dind::check-image { - local name="$1" - if docker inspect --format 'x' "${name}" >&/dev/null; then - return 0 - else - return 1 - fi -} - -function dind::filter-make-output { - # these messages make output too long and make Travis CI choke - egrep -v --line-buffered 'I[0-9][0-9][0-9][0-9] .*(parse|conversion|defaulter|deepcopy)\.go:[0-9]+\]' -} - -function dind::run-build-command { - # this is like build/run.sh, but it doesn't rsync back the binaries, - # only the generated files. - local cmd=("$@") - ( - # The following is taken from build/run.sh and build/common.sh - # of Kubernetes source tree. It differs in - # --filter='+ /_output/dockerized/bin/**' - # being removed from rsync - . ${build_tools_dir}/common.sh - kube::build::verify_prereqs - kube::build::build_image - kube::build::run_build_command "$@" - - kube::log::status "Syncing out of container" - - kube::build::start_rsyncd_container - - local rsync_extra="" - if (( ${KUBE_VERBOSE} >= 6 )); then - rsync_extra="-iv" - fi - - # The filter syntax for rsync is a little obscure. It filters on files and - # directories. If you don't go in to a directory you won't find any files - # there. Rules are evaluated in order. The last two rules are a little - # magic. '+ */' says to go in to every directory and '- /**' says to ignore - # any file or directory that isn't already specifically allowed. - # - # We are looking to copy out all of the built binaries along with various - # generated files. - kube::build::rsync \ - --filter='- /vendor/' \ - --filter='- /_temp/' \ - --filter='+ zz_generated.*' \ - --filter='+ generated.proto' \ - --filter='+ *.pb.go' \ - --filter='+ types.go' \ - --filter='+ */' \ - --filter='- /**' \ - "rsync://k8s@${KUBE_RSYNC_ADDR}/k8s/" "${KUBE_ROOT}" - - kube::build::stop_rsyncd_container - ) -} - -function dind::make-for-linux { - local copy="$1" - shift - dind::step "Building binaries:" "$*" - if [ -n "${KUBEADM_DIND_LOCAL:-}" ]; then - dind::step "+ make WHAT=\"$*\"" - make WHAT="$*" 2>&1 | dind::filter-make-output - elif [ "${copy}" = "y" ]; then - dind::step "+ ${build_tools_dir}/run.sh make WHAT=\"$*\"" - "${build_tools_dir}/run.sh" make WHAT="$*" 2>&1 | dind::filter-make-output - else - dind::step "+ [using the build container] make WHAT=\"$*\"" - dind::run-build-command make WHAT="$*" 2>&1 | dind::filter-make-output - fi -} - -function dind::check-binary { - local filename="$1" - local dockerized="_output/dockerized/bin/linux/amd64/${filename}" - local plain="_output/local/bin/linux/amd64/${filename}" - dind::set-build-volume-args - # FIXME: don't hardcode amd64 arch - if [ -n "${KUBEADM_DIND_LOCAL:-${force_local:-}}" ]; then - if [ -f "${dockerized}" -o -f "${plain}" ]; then - return 0 - fi - elif docker run --rm "${build_volume_args[@]}" \ - "${busybox_image}" \ - test -f "/go/src/k8s.io/kubernetes/${dockerized}" >&/dev/null; then - return 0 - fi - return 1 -} - -function dind::ensure-downloaded-kubectl { - local kubectl_url - local kubectl_sha1 - local kubectl_sha1_linux - local kubectl_sha1_darwin - local kubectl_link - local kubectl_os - local full_kubectl_version - - case "${LOCAL_KUBECTL_VERSION}" in - v1.8) - full_kubectl_version=v1.8.15 - kubectl_sha1_linux=52a1ee321e1e8c0ecfd6e83c38bf972c2c60adf2 - kubectl_sha1_darwin=ac3f823d7aa104237929a1e35ea400c6aa3cc356 - ;; - v1.9) - full_kubectl_version=v1.9.9 - kubectl_sha1_linux=c8163a6360119c56d163fbd8cef8727e9841e712 - kubectl_sha1_darwin=09585552eb7616954481789489ec382c633a0162 - ;; - v1.10) - full_kubectl_version=v1.10.5 - kubectl_sha1_linux=dbe431b2684f8ff4188335b3b3cea185d5a9ec44 - kubectl_sha1_darwin=08e58440949c71053b45bfadf80532ea3d752d12 - ;; - v1.11) - full_kubectl_version=v1.11.0 - kubectl_sha1_linux=e23f251ca0cb848802f3cb0f69a4ba297d07bfc6 - kubectl_sha1_darwin=6eff29a328c4bc00879fd6a0c8b33690c6f75908 - ;; - "") - return 0 - ;; - *) - echo "Invalid kubectl version" >&2 - exit 1 - esac - - export PATH="${KUBECTL_DIR}:$PATH" - - if [ $(uname) = Darwin ]; then - kubectl_sha1="${kubectl_sha1_darwin}" - kubectl_os=darwin - else - kubectl_sha1="${kubectl_sha1_linux}" - kubectl_os=linux - fi - local link_target="kubectl-${full_kubectl_version}" - local link_name="${KUBECTL_DIR}"/kubectl - if [[ -h "${link_name}" && "$(readlink "${link_name}")" = "${link_target}" ]]; then - return 0 - fi - - local path="${KUBECTL_DIR}/${link_target}" - if [[ ! -f "${path}" ]]; then - mkdir -p "${KUBECTL_DIR}" - curl -sSLo "${path}" "https://storage.googleapis.com/kubernetes-release/release/${full_kubectl_version}/bin/${kubectl_os}/amd64/kubectl" - echo "${kubectl_sha1} ${path}" | sha1sum -c - chmod +x "${path}" - fi - - ln -fs "${link_target}" "${KUBECTL_DIR}/kubectl" -} - -function dind::ensure-kubectl { - if [[ ! ${use_k8s_source} ]]; then - # already checked on startup - dind::ensure-downloaded-kubectl - return 0 - fi - if [ $(uname) = Darwin ]; then - if [ ! -f _output/local/bin/darwin/amd64/kubectl ]; then - dind::step "Building kubectl" - dind::step "+ make WHAT=cmd/kubectl" - make WHAT=cmd/kubectl 2>&1 | dind::filter-make-output - fi - elif ! force_local=y dind::check-binary kubectl; then - dind::make-for-linux y cmd/kubectl - fi -} - -function dind::ensure-binaries { - local -a to_build=() - for name in "$@"; do - if ! dind::check-binary "$(basename "${name}")"; then - to_build+=("${name}") - fi - done - if [ "${#to_build[@]}" -gt 0 ]; then - dind::make-for-linux n "${to_build[@]}" - fi - return 0 -} - -# dind::ensure-network creates the management network for the cluster. For IPv4 -# only it will have the management network CIDR. For IPv6 only, it will have -# the IPv6 management network CIDR and the NAT64 V4 mapping network CIDR. For -# dual stack, it will have the IPv4 and IPv6 management CIDRs. Each of the -# management networks (not the NAT64 network) will have a gateway specified. -# -function dind::ensure-network { - if ! docker network inspect $(dind::net-name) >&/dev/null; then - local -a args - for cidr in "${mgmt_net_cidrs[@]}"; do - if [[ $( dind::family-for ${cidr} ) = "ipv6" ]]; then - args+=(--ipv6) - fi - args+=(--subnet="${cidr}") - local gw=$( dind::make-ip-from-cidr ${cidr} 1 ) - args+=(--gateway="${gw}") - done - if [[ ${IP_MODE} = "ipv6" ]]; then - # Need second network for NAT64 V4 mapping network - args+=(--subnet=${NAT64_V4_SUBNET_PREFIX}.0.0/16) - fi - docker network create ${args[@]} $(dind::net-name) >/dev/null - fi -} - -function dind::ensure-volume { - local reuse_volume= - if [[ $1 = -r ]]; then - reuse_volume=1 - shift - fi - local name="$1" - if dind::volume-exists "${name}"; then - if [[ ! ${reuse_volume} ]]; then - docker volume rm "${name}" >/dev/null - fi - fi - dind::create-volume "${name}" -} - -function dind::ensure-dns { - if [[ ${IP_MODE} = "ipv6" ]]; then - local dns64_name="bind9$( dind::cluster-suffix )" - if ! docker inspect ${dns64_name} >&/dev/null; then - local force_dns64_for="" - if [[ ! ${DIND_ALLOW_AAAA_USE} ]]; then - # Normally, if have an AAAA record, it is used. This clause tells - # bind9 to do ignore AAAA records for the specified networks - # and/or addresses and lookup A records and synthesize new AAAA - # records. In this case, we select "any" networks that have AAAA - # records meaning we ALWAYS use A records and do NAT64. - force_dns64_for="exclude { any; };" - fi - read -r -d '' bind9_conf </named.conf && named -c /named.conf -g -u named' >/dev/null - ipv4_addr="$(docker exec ${dns64_name} ip addr list eth0 | grep "inet" | awk '$1 == "inet" {print $2}')" - docker exec ${dns64_name} ip addr del ${ipv4_addr} dev eth0 - docker exec ${dns64_name} ip -6 route add ${DNS64_PREFIX_CIDR} via ${LOCAL_NAT64_SERVER} - fi - fi -} - -function dind::ensure-nat { - if [[ ${IP_MODE} = "ipv6" ]]; then - local nat64_name="tayga$( dind::cluster-suffix )" - if ! docker ps | grep ${nat64_name} >&/dev/null; then - docker run -d --name ${nat64_name} --hostname ${nat64_name} --net "$(dind::net-name)" --label "dind-support$( dind::cluster-suffix )" \ - --sysctl net.ipv6.conf.all.disable_ipv6=0 --sysctl net.ipv6.conf.all.forwarding=1 \ - --privileged=true --ip ${NAT64_V4_SUBNET_PREFIX}.0.200 --ip6 ${LOCAL_NAT64_SERVER} --dns ${REMOTE_DNS64_V4SERVER} --dns ${dns_server} \ - -e TAYGA_CONF_PREFIX=${DNS64_PREFIX_CIDR} -e TAYGA_CONF_IPV4_ADDR=${NAT64_V4_SUBNET_PREFIX}.0.200 \ - -e TAYGA_CONF_DYNAMIC_POOL=${NAT64_V4_SUBNET_PREFIX}.0.128/25 danehans/tayga:latest >/dev/null - # Need to check/create, as "clean" may remove route - local route="$(ip route | egrep "^${NAT64_V4_SUBNET_PREFIX}.0.128/25")" - if [[ -z "${route}" ]]; then - docker run --net=host --rm --privileged ${busybox_image} ip route add ${NAT64_V4_SUBNET_PREFIX}.0.128/25 via ${NAT64_V4_SUBNET_PREFIX}.0.200 - fi - fi - fi -} - -function dind::run { - local reuse_volume= - if [[ $1 = -r ]]; then - reuse_volume="-r" - shift - fi - local container_name="${1:-}" - local node_id=${2:-0} - local portforward="${3:-}" - if [[ $# -gt 3 ]]; then - shift 3 - else - shift $# - fi - - local -a opts=("$@") - local ip_mode="--ip" - for cidr in "${mgmt_net_cidrs[@]}"; do - if [[ $( dind::family-for ${cidr} ) = "ipv6" ]]; then - ip_mode="--ip6" - fi - opts+=("${ip_mode}" "$( dind::make-ip-from-cidr ${cidr} $((${node_id}+1)) )") - done - opts+=("$@") - - local -a args=("systemd.setenv=CNI_PLUGIN=${CNI_PLUGIN}") - args+=("systemd.setenv=IP_MODE=${IP_MODE}") - args+=("systemd.setenv=DIND_STORAGE_DRIVER=${DIND_STORAGE_DRIVER}") - - if [[ ${IP_MODE} != "ipv4" ]]; then - opts+=(--sysctl net.ipv6.conf.all.disable_ipv6=0) - opts+=(--sysctl net.ipv6.conf.all.forwarding=1) - fi - - if [[ ${IP_MODE} = "ipv6" ]]; then - opts+=(--dns ${dns_server}) - args+=("systemd.setenv=DNS64_PREFIX_CIDR=${DNS64_PREFIX_CIDR}") - args+=("systemd.setenv=LOCAL_NAT64_SERVER=${LOCAL_NAT64_SERVER}") - fi - - declare -a pod_nets - local i=0 - if [[ ${IP_MODE} = "ipv4" || ${IP_MODE} = "dual-stack" ]]; then - pod_nets+=("${pod_prefixes[$i]}${node_id}") - i=$((i+1)) - fi - if [[ ${IP_MODE} = "ipv6" || ${IP_MODE} = "dual-stack" ]]; then - # For prefix, if node ID will be in the upper byte, push it over - if [[ $((${pod_sizes[$i]} % 16)) -ne 0 ]]; then - n_id=$(printf "%02x00\n" "${node_id}") - else - if [[ "${pod_prefixes[$i]: -1}" = ":" ]]; then - n_id=$(printf "%x\n" "${node_id}") - else - n_id=$(printf "%02x\n" "${node_id}") # In lower byte, so ensure two chars - fi - fi - pod_nets+=("${pod_prefixes[$i]}${n_id}") - fi - - args+=("systemd.setenv=POD_NET_PREFIX=\"${pod_nets[0]}\"") - args+=("systemd.setenv=POD_NET_SIZE=\"${pod_sizes[0]}\"") - args+=("systemd.setenv=POD_NET2_PREFIX=\"${pod_nets[1]:-}\"") - args+=("systemd.setenv=POD_NET2_SIZE=\"${pod_sizes[1]:-}\"") - args+=("systemd.setenv=SERVICE_NET_MODE=${SERVICE_NET_MODE}") - args+=("systemd.setenv=USE_HAIRPIN=${USE_HAIRPIN}") - args+=("systemd.setenv=DNS_SVC_IP=${DNS_SVC_IP}") - args+=("systemd.setenv=DNS_SERVICE=${DNS_SERVICE}") - if [[ ! "${container_name}" ]]; then - echo >&2 "Must specify container name" - exit 1 - fi - - # remove any previously created containers with the same name - docker rm -vf "${container_name}" >&/dev/null || true - - if [[ "${portforward}" ]]; then - IFS=';' read -ra array <<< "${portforward}" - for element in "${array[@]}"; do - opts+=(-p "${element}") - done - fi - - opts+=(${sys_volume_args[@]+"${sys_volume_args[@]}"}) - - dind::step "Starting DIND container:" "${container_name}" - - if [[ ! ${using_linuxkit} ]]; then - opts+=(-v /boot:/boot -v /lib/modules:/lib/modules) - fi - - if [[ ${ENABLE_CEPH} ]]; then - opts+=(-v /dev:/dev - -v /sys/bus:/sys/bus - -v /var/run/docker.sock:/opt/outer-docker.sock) - fi - - local volume_name="kubeadm-dind-${container_name}" - dind::ensure-network - dind::ensure-volume ${reuse_volume} "${volume_name}" - dind::ensure-nat - dind::ensure-dns - - # TODO: create named volume for binaries and mount it to /k8s - # in case of the source build - - # Start the new container. - docker run \ - -e IP_MODE="${IP_MODE}" \ - -e KUBEADM_SOURCE="${KUBEADM_SOURCE}" \ - -e HYPERKUBE_SOURCE="${HYPERKUBE_SOURCE}" \ - -d --privileged \ - --net "$(dind::net-name)" \ - --name "${container_name}" \ - --hostname "${container_name}" \ - -l "${DIND_LABEL}" \ - -v "${volume_name}:/dind" \ - ${opts[@]+"${opts[@]}"} \ - "${DIND_IMAGE}" \ - ${args[@]+"${args[@]}"} -} - -function dind::kubeadm { - local container_id="$1" - shift - dind::step "Running kubeadm:" "$*" - status=0 - # See image/bare/wrapkubeadm. - # Capturing output is necessary to grab flags for 'kubeadm join' - kubelet_feature_gates="-e KUBELET_FEATURE_GATES=${KUBELET_FEATURE_GATES}" - if ! docker exec ${kubelet_feature_gates} "${container_id}" /usr/local/bin/wrapkubeadm "$@" 2>&1 | tee /dev/fd/2; then - echo "*** kubeadm failed" >&2 - return 1 - fi - return ${status} -} - -# function dind::bare { -# local container_name="${1:-}" -# if [[ ! "${container_name}" ]]; then -# echo >&2 "Must specify container name" -# exit 1 -# fi -# shift -# run_opts=(${@+"$@"}) -# dind::run "${container_name}" -# } - -function dind::configure-kubectl { - dind::step "Setting cluster config" - local host="$(dind::localhost)" - if [[ -z "$using_local_linuxdocker" ]]; then - host="127.0.0.1" - fi - local context_name cluster_name - context_name="$(dind::context-name)" - cluster_name="$(dind::context-name)" - "${kubectl}" config set-cluster "$cluster_name" \ - --server="http://${host}:$(dind::apiserver-port)" \ - --insecure-skip-tls-verify=true - "${kubectl}" config set-context "$context_name" --cluster="$cluster_name" - if [[ ${DIND_LABEL} = ${DEFAULT_DIND_LABEL} ]]; then - # Single cluster mode - "${kubectl}" config use-context "$context_name" - fi -} - -force_make_binaries= -function dind::set-master-opts { - master_opts=() - if [[ ${BUILD_KUBEADM} || ${BUILD_HYPERKUBE} ]]; then - # share binaries pulled from the build container between nodes - local dind_k8s_bin_vol_name - dind_k8s_bin_vol_name="dind-k8s-binaries$(dind::cluster-suffix)" - dind::ensure-volume -r "${dind_k8s_bin_vol_name}" - dind::set-build-volume-args - master_opts+=("${build_volume_args[@]}" -v "${dind_k8s_bin_vol_name}:/k8s") - local -a bins - if [[ ${BUILD_KUBEADM} ]]; then - master_opts+=(-e KUBEADM_SOURCE=build://) - bins+=(cmd/kubeadm) - else - master_opts+=(-e ${KUBEADM_SOURCE}) - fi - if [[ ${BUILD_HYPERKUBE} ]]; then - master_opts+=(-e HYPERKUBE_SOURCE=build://) - bins+=(cmd/hyperkube) - fi - if [[ ${force_make_binaries} ]]; then - dind::make-for-linux n "${bins[@]}" - else - dind::ensure-binaries "${bins[@]}" - fi - fi - if [[ ${MASTER_EXTRA_OPTS:-} ]]; then - master_opts+=( ${MASTER_EXTRA_OPTS} ) - fi -} - -function dind::ensure-dashboard-clusterrolebinding { - local ctx - ctx="$(dind::context-name)" - # 'create' may cause etcd timeout, yet create the clusterrolebinding. - # So use 'apply' to actually create it - "${kubectl}" --context "$ctx" create clusterrolebinding add-on-cluster-admin \ - --clusterrole=cluster-admin \ - --serviceaccount=kube-system:default \ - -o json --dry-run | - docker exec -i "$(dind::master-name)" jq '.apiVersion="rbac.authorization.k8s.io/v1beta1"|.kind|="ClusterRoleBinding"' | - "${kubectl}" --context "$ctx" apply -f - -} - -function dind::deploy-dashboard { - dind::step "Deploying k8s dashboard" - dind::retry "${kubectl}" --context "$(dind::context-name)" apply -f "${DASHBOARD_URL}" - # https://kubernetes-io-vnext-staging.netlify.com/docs/admin/authorization/rbac/#service-account-permissions - # Thanks @liggitt for the hint - dind::retry dind::ensure-dashboard-clusterrolebinding -} - -function dind::kubeadm-version { - if [[ ${use_k8s_source} ]]; then - (cluster/kubectl.sh version --short 2>/dev/null || true) | - grep Client | - sed 's/^.*: v\([0-9.]*\).*/\1/' - else - docker exec "$(dind::master-name)" \ - /bin/bash -c 'kubeadm version -o json | jq -r .clientVersion.gitVersion' | - sed 's/^v\([0-9.]*\).*/\1/' - fi -} - -function dind::kubeadm-skip-checks-flag { - kubeadm_version="$(dind::kubeadm-version)" - if [[ ${kubeadm_version} =~ 1\.8\. ]]; then - echo -n "--skip-preflight-checks" - else - echo -n "--ignore-preflight-errors=all" - fi -} - -function dind::verify-image-compatibility { - # We can't tell in advance, if the image selected supports dual-stack, - # but will do the next best thing, and check as soon as start up kube-master - local master_name=$1 - if [[ ${IP_MODE} = "dual-stack" ]]; then - local dual_stack_support="$(docker exec ${master_name} cat /node-info 2>/dev/null | grep "dual-stack-support" | wc -l)" - if [[ ${dual_stack_support} -eq 0 ]]; then - echo "ERROR! DinD image (${DIND_IMAGE}) does not support dual-stack mode - aborting!" - dind::remove-images "${DIND_LABEL}" - exit 1 - fi - fi -} - -function dind::init { - local -a opts - dind::set-master-opts - local local_host master_name container_id - master_name="$(dind::master-name)" - local_host="$( dind::localhost )" - container_id=$(dind::run "${master_name}" 1 "${local_host}:$(dind::apiserver-port):${INTERNAL_APISERVER_PORT}" ${master_opts[@]+"${master_opts[@]}"}) - - dind::verify-image-compatibility ${master_name} - - # FIXME: I tried using custom tokens with 'kubeadm ex token create' but join failed with: - # 'failed to parse response as JWS object [square/go-jose: compact JWS format must have three parts]' - # So we just pick the line from 'kubeadm init' output - # Using a template file in the image to build a kubeadm.conf file and to customize - # it based on CNI plugin, IP mode, and environment settings. User can add additional - # customizations to template and then rebuild the image used (build/build-local.sh). - local pod_subnet_disable="# " - # TODO: May want to specify each of the plugins that require --pod-network-cidr - if [[ ${CNI_PLUGIN} != "bridge" && ${CNI_PLUGIN} != "ptp" ]]; then - pod_subnet_disable="" - fi - local bind_address="0.0.0.0" - if [[ ${SERVICE_NET_MODE} = "ipv6" ]]; then - bind_address="::" - fi - dind::proxy "$master_name" - dind::custom-docker-opts "$master_name" - - # HACK: Indicating mode, so that wrapkubeadm will not set a cluster CIDR for kube-proxy - # in IPv6 (only) mode. - if [[ ${SERVICE_NET_MODE} = "ipv6" ]]; then - docker exec --privileged -i "$master_name" touch /v6-mode - fi - - feature_gates="{}" - if [[ ${DNS_SERVICE} == "coredns" ]]; then - # can't just use 'CoreDNS: false' because - # it'll break k8s 1.8. FIXME: simplify - # after 1.8 support is removed - feature_gates="{CoreDNS: true}" - elif docker exec "$master_name" kubeadm init --help 2>&1 | grep -q CoreDNS; then - # FIXME: CoreDNS should be the default in 1.11 - feature_gates="{CoreDNS: false}" - fi - - component_feature_gates="" - if [ "${FEATURE_GATES}" != "none" ]; then - component_feature_gates="feature-gates: \\\"${FEATURE_GATES}\\\"" - fi - - apiserver_extra_args="" - for e in $(set -o posix ; set | grep -E "^APISERVER_[a-z_]+=" | cut -d'=' -f 1); do - opt_name=$(echo ${e#APISERVER_} | sed 's/_/-/g') - apiserver_extra_args+=" ${opt_name}: \\\"$(eval echo \$$e)\\\"\\n" - done - - controller_manager_extra_args="" - for e in $(set -o posix ; set | grep -E "^CONTROLLER_MANAGER_[a-z_]+=" | cut -d'=' -f 1); do - opt_name=$(echo ${e#CONTROLLER_MANAGER_} | sed 's/_/-/g') - controller_manager_extra_args+=" ${opt_name}: \\\"$(eval echo \$$e)\\\"\\n" - done - - scheduler_extra_args="" - for e in $(set -o posix ; set | grep -E "^SCHEDULER_[a-z_]+=" | cut -d'=' -f 1); do - opt_name=$(echo ${e#SCHEDULER_} | sed 's/_/-/g') - scheduler_extra_args+=" ${opt_name}: \\\"$(eval echo \$$e)\\\"\\n" - done - - kubeadm_version="$(dind::kubeadm-version)" - api_version="kubeadm.k8s.io/v1alpha3" - kind="ClusterConfiguration" - api_endpoint="apiEndpoint:" - if [[ ${kubeadm_version} =~ 1\.(8|9|10)\. ]]; then - api_version="kubeadm.k8s.io/v1alpha1" - kind="MasterConfiguration" - api_endpoint="api:" - elif [[ ${kubeadm_version} =~ 1\.(11|12)\. ]]; then - api_version="kubeadm.k8s.io/v1alpha2" - kind="MasterConfiguration" - api_endpoint="api:" - fi - local mgmt_cidr=${mgmt_net_cidrs[0]} - if [[ ${IP_MODE} = "dual-stack" && ${SERVICE_NET_MODE} = "ipv6" ]]; then - mgmt_cidr=${mgmt_net_cidrs[1]} - fi - local master_ip=$( dind::make-ip-from-cidr ${mgmt_cidr} 2 ) - docker exec -i "$master_name" bash < /etc/kubeadm.conf -EOF - init_args=(--config /etc/kubeadm.conf) - skip_preflight_arg="$(dind::kubeadm-skip-checks-flag)" - # required when building from source - if [[ ${BUILD_KUBEADM} || ${BUILD_HYPERKUBE} ]]; then - docker exec "$master_name" mount --make-shared /k8s - fi - kubeadm_join_flags="$(dind::kubeadm "${container_id}" init "${init_args[@]}" "${skip_preflight_arg}" "$@" | grep '^ *kubeadm join' | sed 's/^ *kubeadm join //')" - dind::configure-kubectl - dind::start-port-forwarder -} - -function dind::create-node-container { - local reuse_volume next_node_index node_name - reuse_volume='' - if [[ ${1:-} = -r ]]; then - reuse_volume="-r" - shift - fi - # if there's just one node currently, it's master, thus we need to use - # kube-node-1 hostname, if there are two nodes, we should pick - # kube-node-2 and so on - next_node_index=${1:-$(docker ps -q --filter=label="${DIND_LABEL}" | wc -l | sed 's/^ *//g')} - local -a opts - if [[ ${BUILD_KUBEADM} || ${BUILD_HYPERKUBE} ]]; then - opts+=(-v "dind-k8s-binaries$(dind::cluster-suffix)":/k8s) - if [[ ${BUILD_KUBEADM} ]]; then - opts+=(-e KUBEADM_SOURCE=build://) - fi - if [[ ${BUILD_HYPERKUBE} ]]; then - opts+=(-e HYPERKUBE_SOURCE=build://) - fi - fi - node_name="$(dind::node-name ${next_node_index})" - dind::run ${reuse_volume} "$node_name" $((next_node_index + 1)) "${EXTRA_PORTS}" ${opts[@]+"${opts[@]}"} -} - -function dind::join { - local container_id="$1" - shift - dind::proxy "${container_id}" - dind::custom-docker-opts "${container_id}" - skip_preflight_arg="$(dind::kubeadm-skip-checks-flag)" - dind::kubeadm "${container_id}" join "${skip_preflight_arg}" "$@" >/dev/null -} - -function dind::escape-e2e-name { - sed 's/[]\$*.^()[]/\\&/g; s/\s\+/\\s+/g' <<< "$1" | tr -d '\n' -} - -function dind::accelerate-kube-dns { - if [[ ${DNS_SERVICE} == "kube-dns" ]]; then - dind::step "Patching kube-dns deployment to make it start faster" - # Could do this on the host, too, but we don't want to require jq here - # TODO: do this in wrapkubeadm - docker exec "$(dind::master-name)" /bin/bash -c \ - "kubectl get deployment kube-dns -n kube-system -o json | jq '.spec.template.spec.containers[0].readinessProbe.initialDelaySeconds = 3|.spec.template.spec.containers[0].readinessProbe.periodSeconds = 3' | kubectl apply --force -f -" - fi -} - -function dind::component-ready { - local label="$1" - local out - if ! out="$("${kubectl}" --context "$(dind::context-name)" get pod -l "${label}" -n kube-system \ - -o jsonpath='{ .items[*].status.conditions[?(@.type == "Ready")].status }' 2>/dev/null)"; then - return 1 - fi - if ! grep -v False <<<"${out}" | grep -q True; then - return 1 - fi - return 0 -} - -function dind::kill-failed-pods { - local pods ctx - ctx="$(dind::context-name)" - # workaround for https://github.com/kubernetes/kubernetes/issues/36482 - if ! pods="$(kubectl --context "$ctx" get pod -n kube-system -o jsonpath='{ .items[?(@.status.phase == "Failed")].metadata.name }' 2>/dev/null)"; then - return - fi - for name in ${pods}; do - kubectl --context "$ctx" delete pod --now -n kube-system "${name}" >&/dev/null || true - done -} - -function dind::create-static-routes { - echo "Creating static routes for bridge/PTP plugin" - for ((i=0; i <= NUM_NODES; i++)); do - if [[ ${i} -eq 0 ]]; then - node="$(dind::master-name)" - else - node="$(dind::node-name $i)" - fi - for ((j=0; j <= NUM_NODES; j++)); do - if [[ ${i} -eq ${j} ]]; then - continue - fi - if [[ ${j} -eq 0 ]]; then - dest_node="$(dind::master-name)" - else - dest_node="$(dind::node-name $j)" - fi - id=$((${j}+1)) - if [[ ${IP_MODE} = "ipv4" || ${IP_MODE} = "dual-stack" ]]; then - # Assuming pod subnets will all be /24 - dest="${pod_prefixes[0]}${id}.0/24" - gw=`docker exec ${dest_node} ip addr show eth0 | grep -w inet | awk '{ print $2 }' | sed 's,/.*,,'` - docker exec "${node}" ip route add "${dest}" via "${gw}" - fi - if [[ ${IP_MODE} = "ipv6" || ${IP_MODE} = "dual-stack" ]]; then - local position=0 - if [[ ${IP_MODE} = "dual-stack" ]]; then - position=1 - fi - instance=$(printf "%02x" ${id}) - if [[ $((${pod_sizes[$position]} % 16)) -ne 0 ]]; then - instance+="00" # Move node ID to upper byte - fi - dest="${pod_prefixes[$position]}${instance}::/${pod_sizes[$position]}" - gw=`docker exec ${dest_node} ip addr show eth0 | grep -w inet6 | grep -i global | head -1 | awk '{ print $2 }' | sed 's,/.*,,'` - docker exec "${node}" ip route add "${dest}" via "${gw}" - fi - done - done -} - -# If we are allowing AAAA record use, then provide SNAT for IPv6 packets from -# node containers, and forward packets to bridge used for $(dind::net-name). -# This gives pods access to external IPv6 sites, when using IPv6 addresses. -function dind::setup_external_access_on_host { - if [[ ! ${DIND_ALLOW_AAAA_USE} ]]; then - return - fi - local main_if=`ip route | grep default | awk '{print $5}'` - dind::ip6tables-on-hostnet -t nat -A POSTROUTING -o $main_if -j MASQUERADE - if [[ ${IP_MODE} = "dual-stack" ]]; then - return - fi - local bridge_if=`ip route | grep ${NAT64_V4_SUBNET_PREFIX}.0.0 | awk '{print $3}'` - if [[ -n "$bridge_if" ]]; then - dind::ip6tables-on-hostnet -A FORWARD -i $bridge_if -j ACCEPT - else - echo "WARNING! No $(dind::net-name) bridge with NAT64 - unable to setup forwarding/SNAT" - fi -} - -# Remove ip6tables rules for SNAT and forwarding, if they exist. -function dind::remove_external_access_on_host { - if [[ ! ${DIND_ALLOW_AAAA_USE} ]]; then - return - fi - local have_rule - local main_if="$(ip route | grep default | awk '{print $5}')" - have_rule="$(dind::ip6tables-on-hostnet -S -t nat | grep "\-o $main_if" || true)" - if [[ -n "$have_rule" ]]; then - dind::ip6tables-on-hostnet -t nat -D POSTROUTING -o $main_if -j MASQUERADE - else - echo "Skipping delete of ip6tables rule for SNAT, as rule non-existent" - fi - - if [[ ${IP_MODE} = "dual-stack" ]]; then - return - fi - local bridge_if="$(ip route | grep ${NAT64_V4_SUBNET_PREFIX}.0.0 | awk '{print $3}')" - if [[ -n "$bridge_if" ]]; then - have_rule="$(dind::ip6tables-on-hostnet -S | grep "\-i $bridge_if" || true)" - if [[ -n "$have_rule" ]]; then - dind::ip6tables-on-hostnet -D FORWARD -i $bridge_if -j ACCEPT - else - echo "Skipping delete of ip6tables rule for forwarding, as rule non-existent" - fi - else - echo "Skipping delete of ip6tables rule for forwarding, as no bridge interface using NAT64" - fi -} - -function dind::ip6tables-on-hostnet { - local mod_path='/lib/modules' - docker run -v "${mod_path}:${mod_path}" --entrypoint /sbin/ip6tables --net=host --rm --privileged "${DIND_IMAGE}" "$@" -} - -function dind::wait-for-ready { - local app="kube-proxy" - if [[ ${CNI_PLUGIN} = "kube-router" ]]; then - app=kube-router - fi - dind::step "Waiting for ${app} and the nodes" - local app_ready - local nodes_ready - local n=3 - local ntries=200 - local ctx - ctx="$(dind::context-name)" - while true; do - dind::kill-failed-pods - if "${kubectl}" --context "$ctx" get nodes 2>/dev/null | grep -q NotReady; then - nodes_ready= - else - nodes_ready=y - fi - if dind::component-ready k8s-app=${app}; then - app_ready=y - else - app_ready= - fi - if [[ ${nodes_ready} && ${app_ready} ]]; then - if ((--n == 0)); then - echo "[done]" >&2 - break - fi - else - n=3 - fi - if ((--ntries == 0)); then - echo "Error waiting for ${app} and the nodes" >&2 - exit 1 - fi - echo -n "." >&2 - sleep 1 - done - - dind::step "Bringing up ${DNS_SERVICE} and kubernetes-dashboard" - # on Travis 'scale' sometimes fails with 'error: Scaling the resource failed with: etcdserver: request timed out; Current resource version 442' here - dind::retry "${kubectl}" --context "$ctx" scale deployment --replicas=1 -n kube-system ${DNS_SERVICE} - dind::retry "${kubectl}" --context "$ctx" scale deployment --replicas=1 -n kube-system kubernetes-dashboard - - ntries=200 - while ! dind::component-ready k8s-app=kube-dns || ! dind::component-ready app=kubernetes-dashboard; do - if ((--ntries == 0)); then - echo "Error bringing up ${DNS_SERVICE} and kubernetes-dashboard" >&2 - exit 1 - fi - echo -n "." >&2 - dind::kill-failed-pods - sleep 1 - done - echo "[done]" >&2 - - dind::retry "${kubectl}" --context "$ctx" get nodes >&2 - - local local_host - local_host="$( dind::localhost )" - dind::step "Access dashboard at:" "http://${local_host}:$(dind::apiserver-port)/api/v1/namespaces/kube-system/services/kubernetes-dashboard:/proxy" -} - -# dind::make-kube-router-yaml creates a temp file with contents of the configuration needed for the kube-router CNI -# plugin at a specific version, instead of using the publically available file, which uses the latest version. This -# allows us to control the version used. If/when updating, be sure to update the KUBE_ROUTER_VERSION env variable -# ensure the YAML contents below, reflect the configuration in: -# -# https://raw.githubusercontent.com/cloudnativelabs/kube-router/master/daemonset/kubeadm-kuberouter-all-feature.yaml -# -# FUTURE: May be able to remove this, if/when kube-router "latest" is stable, and use the public YAML file instead. -function dind::make-kube-router-yaml { - tmp_yaml=$(mktemp /tmp/kube-router-yaml.XXXXXX) - cat >${tmp_yaml} <&2 "*** Failed to start node container ${n}" - exit 1 - else - node_containers+=(${container_id}) - dind::step "Node container started:" ${n} - fi - done - dind::fix-mounts - status=0 - local -a pids - for ((n=1; n <= NUM_NODES; n++)); do - ( - dind::step "Joining node:" ${n} - container_id="${node_containers[${n}-1]}" - if ! dind::join ${container_id} ${kubeadm_join_flags}; then - echo >&2 "*** Failed to start node container ${n}" - exit 1 - else - dind::step "Node joined:" ${n} - fi - )& - pids[${n}]=$! - done - if ((NUM_NODES > 0)); then - for pid in ${pids[*]}; do - wait ${pid} - done - else - # FIXME: this may fail depending on k8s/kubeadm version - # FIXME: check for taint & retry if it's there - "${kubectl}" --context "$ctx" taint nodes $(dind::master-name) node-role.kubernetes.io/master- || true - fi - case "${CNI_PLUGIN}" in - bridge | ptp) - dind::create-static-routes - dind::setup_external_access_on_host - ;; - flannel) - # without --validate=false this will fail on older k8s versions - dind::retry "${kubectl}" --context "$ctx" apply --validate=false -f "https://github.com/coreos/flannel/blob/master/Documentation/kube-flannel.yml?raw=true" - ;; - calico) - dind::retry "${kubectl}" --context "$ctx" apply -f https://docs.projectcalico.org/v2.6/getting-started/kubernetes/installation/hosted/kubeadm/1.6/calico.yaml - ;; - calico-kdd) - dind::retry "${kubectl}" --context "$ctx" apply -f https://docs.projectcalico.org/v2.6/getting-started/kubernetes/installation/hosted/rbac-kdd.yaml - dind::retry "${kubectl}" --context "$ctx" apply -f https://docs.projectcalico.org/v2.6/getting-started/kubernetes/installation/hosted/kubernetes-datastore/calico-networking/1.7/calico.yaml - ;; - weave) - dind::retry "${kubectl}" --context "$ctx" apply -f "https://github.com/weaveworks/weave/blob/master/prog/weave-kube/weave-daemonset-k8s-1.6.yaml?raw=true" - ;; - kube-router) - kube_router_config="$( dind::make-kube-router-yaml )" - dind::retry "${kubectl}" --context "$ctx" apply -f ${kube_router_config} - rm "${kube_router_config}" - dind::retry "${kubectl}" --context "$ctx" -n kube-system delete ds kube-proxy - docker run --privileged --net=host k8s.gcr.io/kube-proxy-amd64:v1.10.2 kube-proxy --cleanup - ;; - *) - echo "Unsupported CNI plugin '${CNI_PLUGIN}'" >&2 - ;; - esac - dind::deploy-dashboard - dind::accelerate-kube-dns - if [[ (${CNI_PLUGIN} != "bridge" && ${CNI_PLUGIN} != "ptp") || ${SKIP_SNAPSHOT} ]]; then - # This is especially important in case of Calico - - # the cluster will not recover after snapshotting - # (at least not after restarting from the snapshot) - # if Calico installation is interrupted - dind::wait-for-ready - fi - dind::step "Cluster Info" - echo "Network Mode: ${IP_MODE}" - echo "Cluster context: $( dind::context-name )" - echo "Cluster ID: ${CLUSTER_ID}" - echo "Management CIDR(s): ${mgmt_net_cidrs[@]}" - echo "Service CIDR/mode: ${SERVICE_CIDR}/${SERVICE_NET_MODE}" - echo "Pod CIDR(s): ${pod_net_cidrs[@]}" -} - -function dind::fix-mounts { - local node_name - for ((n=0; n <= NUM_NODES; n++)); do - node_name="$(dind::master-name)" - if ((n > 0)); then - node_name="$(dind::node-name $n)" - fi - docker exec "${node_name}" mount --make-shared /run - if [[ ! ${using_linuxkit} ]]; then - docker exec "${node_name}" mount --make-shared /lib/modules/ - fi - # required when building from source - if [[ ${BUILD_KUBEADM} || ${BUILD_HYPERKUBE} ]]; then - docker exec "${node_name}" mount --make-shared /k8s - fi - # docker exec "${node_name}" mount --make-shared /sys/kernel/debug - done -} - -function dind::snapshot_container { - local container_name="$1" - docker exec -i ${container_name} /usr/local/bin/snapshot prepare - # remove the hidden *plnk directories - docker diff ${container_name} | grep -v plnk | docker exec -i ${container_name} /usr/local/bin/snapshot save -} - -function dind::snapshot { - dind::step "Taking snapshot of the cluster" - dind::snapshot_container "$(dind::master-name)" - for ((n=1; n <= NUM_NODES; n++)); do - dind::snapshot_container "$(dind::node-name $n)" - done - dind::wait-for-ready -} - -restore_cmd=restore -function dind::restore_container { - local container_id="$1" - docker exec ${container_id} /usr/local/bin/snapshot "${restore_cmd}" -} - -function dind::restore { - local apiserver_port local_host pid pids - dind::down - dind::step "Restoring containers" - dind::set-master-opts - local_host="$( dind::localhost )" - apiserver_port="$( dind::apiserver-port )" - for ((n=0; n <= NUM_NODES; n++)); do - ( - if [[ n -eq 0 ]]; then - dind::step "Restoring master container" - dind::restore_container "$(dind::run -r "$(dind::master-name)" 1 "${local_host}:${apiserver_port}:${INTERNAL_APISERVER_PORT}" ${master_opts[@]+"${master_opts[@]}"})" - dind::verify-image-compatibility "$(dind::master-name)" - dind::step "Master container restored" - else - dind::step "Restoring node container:" ${n} - if ! container_id="$(dind::create-node-container -r ${n})"; then - echo >&2 "*** Failed to start node container ${n}" - exit 1 - else - dind::restore_container "${container_id}" - dind::step "Node container restored:" ${n} - fi - fi - )& - pids[${n}]=$! - done - for pid in ${pids[*]}; do - wait ${pid} - done - if [[ ${CNI_PLUGIN} = "bridge" || ${CNI_PLUGIN} = "ptp" ]]; then - dind::create-static-routes - dind::setup_external_access_on_host - fi - dind::fix-mounts - # Recheck kubectl config. It's possible that the cluster was started - # on this docker from different host - dind::configure-kubectl - dind::start-port-forwarder - dind::wait-for-ready -} - -function dind::down { - dind::remove-images "${DIND_LABEL}" - if [[ ${CNI_PLUGIN} = "bridge" || ${CNI_PLUGIN} = "ptp" ]]; then - dind::remove_external_access_on_host - elif [[ "${CNI_PLUGIN}" = "kube-router" ]]; then - if [[ ${COMMAND} = "down" || ${COMMAND} = "clean" ]]; then - # FUTURE: Updated pinned version, after verifying operation - docker run --privileged --net=host cloudnativelabs/kube-router:${KUBE_ROUTER_VERSION} --cleanup-config - fi - fi -} - -function dind::apiserver-port { - # APISERVER_PORT is explicitely set - if [ -n "${APISERVER_PORT:-}" ] - then - echo "$APISERVER_PORT" - return - fi - - # Get the port from the master - local master port - master="$(dind::master-name)" - # 8080/tcp -> 127.0.0.1:8082 => 8082 - port="$( docker port "$master" 2>/dev/null | awk -F: "/^${INTERNAL_APISERVER_PORT}/{ print \$NF }" )" - if [ -n "$port" ] - then - APISERVER_PORT="$port" - echo "$APISERVER_PORT" - return - fi - - # get a random free port - APISERVER_PORT=0 - echo "$APISERVER_PORT" -} - -function dind::master-name { - echo "kube-master$( dind::cluster-suffix )" -} - -function dind::node-name { - local nr="$1" - echo "kube-node-${nr}$( dind::cluster-suffix )" -} - -function dind::context-name { - echo "dind$( dind::cluster-suffix )" -} - -function dind::remove-volumes { - # docker 1.13+: docker volume ls -q -f label="${DIND_LABEL}" - local nameRE - nameRE="^kubeadm-dind-(sys|kube-master|kube-node-[0-9]+)$(dind::cluster-suffix)$" - docker volume ls -q | (grep -E "$nameRE" || true) | while read -r volume_id; do - dind::step "Removing volume:" "${volume_id}" - docker volume rm "${volume_id}" - done -} - -function dind::remove-images { - local which=$1 - docker ps -a -q --filter=label="${which}" | while read container_id; do - dind::step "Removing container:" "${container_id}" - docker rm -fv "${container_id}" - done -} - -function dind::remove-cluster { - cluster_name="dind$(dind::cluster-suffix)" - if ${kubectl} config get-clusters | grep -qE "^${cluster_name}$"; then - dind::step "Removing cluster from config:" "${cluster_name}" - ${kubectl} config delete-cluster ${cluster_name} 2>/dev/null || true - fi -} - -function dind::remove-context { - context_name="$(dind::context-name)" - if ${kubectl} config get-contexts | grep -qE "${context_name}\\s"; then - dind::step "Removing context from config:" "${context_name}" - ${kubectl} config delete-context ${context_name} 2>/dev/null || true - fi -} - -function dind::start-port-forwarder { - local fwdr port - fwdr="${DIND_PORT_FORWARDER:-}" - - [ -n "$fwdr" ] || return 0 - - [ -x "$fwdr" ] || { - echo "'${fwdr}' is not executable." >&2 - return 1 - } - - port="$( dind::apiserver-port )" - dind::step "+ Setting up port-forwarding for :${port}" - "$fwdr" "$port" -} - -function dind::check-for-snapshot { - if ! dind::volume-exists "kubeadm-dind-$(dind::master-name)"; then - return 1 - fi - for ((n=1; n <= NUM_NODES; n++)); do - if ! dind::volume-exists "kubeadm-dind-$(dind::node-name ${n})"; then - return 1 - fi - done -} - -function dind::do-run-e2e { - local parallel="${1:-}" - local focus="${2:-}" - local skip="${3:-}" - local host="$(dind::localhost)" - if [[ -z "$using_local_linuxdocker" ]]; then - host="127.0.0.1" - fi - dind::need-source - local kubeapi test_args term= - local -a e2e_volume_opts=() - kubeapi="http://${host}:$(dind::apiserver-port)" - test_args="--host=${kubeapi}" - if [[ ${focus} ]]; then - test_args="--ginkgo.focus=${focus} ${test_args}" - fi - if [[ ${skip} ]]; then - test_args="--ginkgo.skip=${skip} ${test_args}" - fi - if [[ ${E2E_REPORT_DIR} ]]; then - test_args="--report-dir=/report ${test_args}" - e2e_volume_opts=(-v "${E2E_REPORT_DIR}:/report") - fi - dind::make-for-linux n "cmd/kubectl test/e2e/e2e.test vendor/github.com/onsi/ginkgo/ginkgo" - dind::step "Running e2e tests with args:" "${test_args}" - dind::set-build-volume-args - if [ -t 1 ] ; then - term="-it" - test_args="--ginkgo.noColor --num-nodes=2 ${test_args}" - fi - docker run \ - --rm ${term} \ - --net=host \ - "${build_volume_args[@]}" \ - -e KUBERNETES_PROVIDER=dind \ - -e KUBE_MASTER_IP="${kubeapi}" \ - -e KUBE_MASTER=local \ - -e KUBERNETES_CONFORMANCE_TEST=y \ - -e GINKGO_PARALLEL=${parallel} \ - ${e2e_volume_opts[@]+"${e2e_volume_opts[@]}"} \ - -w /go/src/k8s.io/kubernetes \ - "${e2e_base_image}" \ - bash -c "cluster/kubectl.sh config set-cluster dind --server='${kubeapi}' --insecure-skip-tls-verify=true && - cluster/kubectl.sh config set-context dind --cluster=dind && - cluster/kubectl.sh config use-context dind && - go run hack/e2e.go -- --v 6 --test --check-version-skew=false --test_args='${test_args}'" -} - -function dind::clean { - dind::down - dind::remove-images "dind-support$( dind::cluster-suffix )" - dind::remove-volumes - local net_name - net_name="$(dind::net-name)" - if docker network inspect "$net_name" >&/dev/null; then - docker network rm "$net_name" - fi - dind::remove-cluster - dind::remove-context -} - -function dind::copy-image { - local image="${2:-}" - local image_path="/tmp/save_${image//\//_}" - if [[ -f "${image_path}" ]]; then - rm -fr "${image_path}" - fi - docker save "${image}" -o "${image_path}" - docker ps -a -q --filter=label="${DIND_LABEL}" | while read container_id; do - cat "${image_path}" | docker exec -i "${container_id}" docker load - done - rm -fr "${image_path}" -} - -function dind::run-e2e { - local focus="${1:-}" - local skip="${2:-[Serial]}" - skip="$(dind::escape-e2e-name "${skip}")" - if [[ "$focus" ]]; then - focus="$(dind::escape-e2e-name "${focus}")" - else - focus="\[Conformance\]" - fi - local parallel=y - if [[ ${DIND_NO_PARALLEL_E2E} ]]; then - parallel= - fi - dind::do-run-e2e "${parallel}" "${focus}" "${skip}" -} - -function dind::run-e2e-serial { - local focus="${1:-}" - local skip="${2:-}" - skip="$(dind::escape-e2e-name "${skip}")" - dind::need-source - if [[ "$focus" ]]; then - focus="$(dind::escape-e2e-name "${focus}")" - else - focus="\[Serial\].*\[Conformance\]" - fi - dind::do-run-e2e n "${focus}" "${skip}" - # TBD: specify filter -} - -function dind::step { - local OPTS="" - if [ "$1" = "-n" ]; then - shift - OPTS+="-n" - fi - GREEN="$1" - shift - if [ -t 2 ] ; then - echo -e ${OPTS} "\x1B[97m* \x1B[92m${GREEN}\x1B[39m $*" 1>&2 - else - echo ${OPTS} "* ${GREEN} $*" 1>&2 - fi -} - -function dind::dump { - set +e - echo "*** Dumping cluster state ***" - for node in $(docker ps --format '{{.Names}}' --filter label="${DIND_LABEL}"); do - for service in kubelet.service dindnet.service criproxy.service dockershim.service; do - if docker exec "${node}" systemctl is-enabled "${service}" >&/dev/null; then - echo "@@@ service-${node}-${service}.log @@@" - docker exec "${node}" systemctl status "${service}" - docker exec "${node}" journalctl -xe -n all -u "${service}" - fi - done - echo "@@@ psaux-${node}.txt @@@" - docker exec "${node}" ps auxww - echo "@@@ dockerps-a-${node}.txt @@@" - docker exec "${node}" docker ps -a - echo "@@@ ip-a-${node}.txt @@@" - docker exec "${node}" ip a - echo "@@@ ip-r-${node}.txt @@@" - docker exec "${node}" ip r - done - local ctx master_name - master_name="$(dind::master-name)" - ctx="$(dind::context-name)" - docker exec "$master_name" kubectl get pods --all-namespaces \ - -o go-template='{{range $x := .items}}{{range $x.spec.containers}}{{$x.spec.nodeName}}{{" "}}{{$x.metadata.namespace}}{{" "}}{{$x.metadata.name}}{{" "}}{{.name}}{{"\n"}}{{end}}{{end}}' | - while read node ns pod container; do - echo "@@@ pod-${node}-${ns}-${pod}--${container}.log @@@" - docker exec "$master_name" kubectl logs -n "${ns}" -c "${container}" "${pod}" - done - echo "@@@ kubectl-all.txt @@@" - docker exec "$master_name" kubectl get all --all-namespaces -o wide - echo "@@@ describe-all.txt @@@" - docker exec "$master_name" kubectl describe all --all-namespaces - echo "@@@ nodes.txt @@@" - docker exec "$master_name" kubectl get nodes -o wide -} - -function dind::dump64 { - echo "%%% start-base64 %%%" - dind::dump | docker exec -i "$(dind::master-name)" /bin/sh -c "lzma | base64 -w 100" - echo "%%% end-base64 %%%" -} - -function dind::split-dump { - mkdir -p cluster-dump - cd cluster-dump - awk '!/^@@@ .* @@@$/{print >out}; /^@@@ .* @@@$/{out=$2}' out=/dev/null - ls -l -} - -function dind::split-dump64 { - decode_opt=-d - if base64 --help | grep -q '^ *-D'; then - # Mac OS X - decode_opt=-D - fi - sed -n '/^%%% start-base64 %%%$/,/^%%% end-base64 %%%$/p' | - sed '1d;$d' | - base64 "${decode_opt}" | - lzma -dc | - dind::split-dump -} - -function dind::proxy { - local container_id="$1" - if [[ ${DIND_CA_CERT_URL} ]] ; then - dind::step "+ Adding certificate on ${container_id}" - docker exec ${container_id} /bin/sh -c "cd /usr/local/share/ca-certificates; curl -sSO ${DIND_CA_CERT_URL}" - docker exec ${container_id} update-ca-certificates - fi - if [[ "${DIND_PROPAGATE_HTTP_PROXY}" || "${DIND_HTTP_PROXY}" || "${DIND_HTTPS_PROXY}" || "${DIND_NO_PROXY}" ]]; then - dind::step "+ Setting *_PROXY for docker service on ${container_id}" - local proxy_env="[Service]"$'\n'"Environment=" - if [[ "${DIND_PROPAGATE_HTTP_PROXY}" ]]; then - # take *_PROXY values from container environment - proxy_env+=$(docker exec ${container_id} env | grep -i _proxy | awk '{ print "\""$0"\""}' | xargs -d'\n') - else - if [[ "${DIND_HTTP_PROXY}" ]] ; then proxy_env+="\"HTTP_PROXY=${DIND_HTTP_PROXY}\" "; fi - if [[ "${DIND_HTTPS_PROXY}" ]] ; then proxy_env+="\"HTTPS_PROXY=${DIND_HTTPS_PROXY}\" "; fi - if [[ "${DIND_NO_PROXY}" ]] ; then proxy_env+="\"NO_PROXY=${DIND_NO_PROXY}\" "; fi - fi - docker exec -i ${container_id} /bin/sh -c "cat > /etc/systemd/system/docker.service.d/30-proxy.conf" <<< "${proxy_env}" - docker exec ${container_id} systemctl daemon-reload - docker exec ${container_id} systemctl restart docker - fi -} - -function dind::custom-docker-opts { - local container_id="$1" - local -a jq=() - if [[ ! -f ${DIND_DAEMON_JSON_FILE} ]] ; then - jq[0]="{}" - else - jq+=("$(cat ${DIND_DAEMON_JSON_FILE})") - fi - if [[ ${DIND_REGISTRY_MIRROR} ]] ; then - dind::step "+ Setting up registry mirror on ${container_id}" - jq+=("{\"registry-mirrors\": [\"${DIND_REGISTRY_MIRROR}\"]}") - fi - if [[ ${DIND_INSECURE_REGISTRIES} ]] ; then - dind::step "+ Setting up insecure-registries on ${container_id}" - jq+=("{\"insecure-registries\": ${DIND_INSECURE_REGISTRIES}}") - fi - if [[ ${jq} ]] ; then - local json=$(IFS="+"; echo "${jq[*]}") - docker exec -i ${container_id} /bin/sh -c "mkdir -p /etc/docker && jq -n '${json}' > /etc/docker/daemon.json" - docker exec ${container_id} systemctl daemon-reload - docker exec ${container_id} systemctl restart docker - fi -} - -COMMAND="${1:-}" - -case ${COMMAND} in - up) - if [[ ! ( ${DIND_IMAGE} =~ local ) && ! ${DIND_SKIP_PULL:-} ]]; then - dind::step "Making sure DIND image is up to date" - docker pull "${DIND_IMAGE}" >&2 - fi - - dind::prepare-sys-mounts - dind::ensure-kubectl - if [[ ${SKIP_SNAPSHOT} ]]; then - force_make_binaries=y dind::up - elif ! dind::check-for-snapshot; then - force_make_binaries=y dind::up - dind::snapshot - else - dind::restore - fi - ;; - reup) - dind::prepare-sys-mounts - dind::ensure-kubectl - if [[ ${SKIP_SNAPSHOT} ]]; then - force_make_binaries=y dind::up - elif ! dind::check-for-snapshot; then - force_make_binaries=y dind::up - dind::snapshot - else - force_make_binaries=y - restore_cmd=update_and_restore - dind::restore - fi - ;; - down) - dind::down - ;; - init) - shift - dind::prepare-sys-mounts - dind::ensure-kubectl - dind::init "$@" - ;; - join) - shift - dind::prepare-sys-mounts - dind::ensure-kubectl - dind::join "$(dind::create-node-container)" "$@" - ;; - # bare) - # shift - # dind::bare "$@" - # ;; - snapshot) - shift - dind::snapshot - ;; - restore) - shift - dind::restore - ;; - clean) - dind::clean - ;; - copy-image) - dind::copy-image "$@" - ;; - e2e) - shift - dind::run-e2e "$@" - ;; - e2e-serial) - shift - dind::run-e2e-serial "$@" - ;; - dump) - dind::dump - ;; - dump64) - dind::dump64 - ;; - split-dump) - dind::split-dump - ;; - split-dump64) - dind::split-dump64 - ;; - apiserver-port) - dind::apiserver-port - ;; - *) - echo "usage:" >&2 - echo " $0 up" >&2 - echo " $0 reup" >&2 - echo " $0 down" >&2 - echo " $0 init kubeadm-args..." >&2 - echo " $0 join kubeadm-args..." >&2 - # echo " $0 bare container_name [docker_options...]" - echo " $0 clean" - echo " $0 copy-image [image_name]" >&2 - echo " $0 e2e [test-name-substring]" >&2 - echo " $0 e2e-serial [test-name-substring]" >&2 - echo " $0 dump" >&2 - echo " $0 dump64" >&2 - echo " $0 split-dump" >&2 - echo " $0 split-dump64" >&2 - exit 1 - ;; -esac \ No newline at end of file diff --git a/test/e2e/e2e.go b/test/e2e/e2e.go index d42336254..773635cf5 100644 --- a/test/e2e/e2e.go +++ b/test/e2e/e2e.go @@ -17,13 +17,14 @@ limitations under the License. package e2e import ( + "os" "testing" - "github.com/golang/glog" "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/config" "github.com/onsi/gomega" "k8s.io/apiserver/pkg/util/logs" + // required _ "k8s.io/client-go/plugin/pkg/client/auth" @@ -36,6 +37,7 @@ import ( _ "k8s.io/ingress-nginx/test/e2e/servicebackend" _ "k8s.io/ingress-nginx/test/e2e/settings" _ "k8s.io/ingress-nginx/test/e2e/ssl" + _ "k8s.io/ingress-nginx/test/e2e/status" ) // RunE2ETests checks configuration parameters (specified through flags) and then runs @@ -50,7 +52,12 @@ func RunE2ETests(t *testing.T) { config.GinkgoConfig.SkipString = `\[Flaky\]|\[Feature:.+\]` } - glog.Infof("Starting e2e run %q on Ginkgo node %d", framework.RunID, config.GinkgoConfig.ParallelNode) + if os.Getenv("KUBECTL_PATH") != "" { + framework.KubectlPath = os.Getenv("KUBECTL_PATH") + framework.Logf("Using kubectl path '%s'", framework.KubectlPath) + } + + framework.Logf("Starting e2e run %q on Ginkgo node %d", framework.RunID, config.GinkgoConfig.ParallelNode) ginkgo.RunSpecs(t, "nginx-ingress-controller e2e suite") } diff --git a/test/e2e/framework/deployment.go b/test/e2e/framework/deployment.go index 122c20f1a..a01e3d333 100644 --- a/test/e2e/framework/deployment.go +++ b/test/e2e/framework/deployment.go @@ -17,10 +17,9 @@ limitations under the License. package framework import ( - "fmt" "time" - "github.com/pkg/errors" + . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" extensions "k8s.io/api/extensions/v1beta1" @@ -30,23 +29,23 @@ import ( ) // NewEchoDeployment creates a new single replica deployment of the echoserver image in a particular namespace -func (f *Framework) NewEchoDeployment() error { - return f.NewEchoDeploymentWithReplicas(1) +func (f *Framework) NewEchoDeployment() { + f.NewEchoDeploymentWithReplicas(1) } // NewEchoDeploymentWithReplicas creates a new deployment of the echoserver image in a particular namespace. Number of // replicas is configurable -func (f *Framework) NewEchoDeploymentWithReplicas(replicas int32) error { - return f.NewDeployment("http-svc", "gcr.io/kubernetes-e2e-test-images/echoserver:2.1", 8080, replicas) +func (f *Framework) NewEchoDeploymentWithReplicas(replicas int32) { + f.NewDeployment("http-svc", "gcr.io/kubernetes-e2e-test-images/echoserver:2.1", 8080, replicas) } // NewHttpbinDeployment creates a new single replica deployment of the httpbin image in a particular namespace. -func (f *Framework) NewHttpbinDeployment() error { - return f.NewDeployment("httpbin", "kennethreitz/httpbin", 80, 1) +func (f *Framework) NewHttpbinDeployment() { + f.NewDeployment("httpbin", "kennethreitz/httpbin", 80, 1) } // NewDeployment creates a new deployment in a particular namespace. -func (f *Framework) NewDeployment(name, image string, port int32, replicas int32) error { +func (f *Framework) NewDeployment(name, image string, port int32, replicas int32) { deployment := &extensions.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -86,20 +85,13 @@ func (f *Framework) NewDeployment(name, image string, port int32, replicas int32 } d, err := f.EnsureDeployment(deployment) - if err != nil { - return err - } - - if d == nil { - return fmt.Errorf("unexpected error creating deployement %s", name) - } + Expect(err).NotTo(HaveOccurred(), "failed to create a deployment") + Expect(d).NotTo(BeNil(), "expected a deployement but none returned") err = WaitForPodsReady(f.KubeClientSet, 5*time.Minute, int(replicas), f.IngressController.Namespace, metav1.ListOptions{ LabelSelector: fields.SelectorFromSet(fields.Set(d.Spec.Template.ObjectMeta.Labels)).String(), }) - if err != nil { - return errors.Wrap(err, "failed to wait for to become ready") - } + Expect(err).NotTo(HaveOccurred(), "failed to wait for to become ready") service := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ @@ -121,14 +113,6 @@ func (f *Framework) NewDeployment(name, image string, port int32, replicas int32 }, } - s, err := f.EnsureService(service) - if err != nil { - return err - } - - if s == nil { - return fmt.Errorf("unexpected error creating service %s", name) - } - - return nil + s := f.EnsureService(service) + Expect(s).NotTo(BeNil(), "expected a service but none returned") } diff --git a/test/e2e/framework/exec.go b/test/e2e/framework/exec.go index f6f82911f..893e343de 100644 --- a/test/e2e/framework/exec.go +++ b/test/e2e/framework/exec.go @@ -19,7 +19,11 @@ package framework import ( "bytes" "fmt" + "io" "os/exec" + "regexp" + "strconv" + "strings" "k8s.io/api/core/v1" ) @@ -31,14 +35,14 @@ func (f *Framework) ExecCommand(pod *v1.Pod, command string) (string, error) { execErr bytes.Buffer ) - args := fmt.Sprintf("kubectl exec --namespace %v %v --container nginx-ingress-controller -- %v", pod.Namespace, pod.Name, command) - cmd := exec.Command("/bin/bash", "-c", args) + cmd := exec.Command("/bin/bash", "-c", fmt.Sprintf("%v exec --namespace %s %s --container nginx-ingress-controller -- %s", KubectlPath, pod.Namespace, pod.Name, command)) cmd.Stdout = &execOut cmd.Stderr = &execErr err := cmd.Run() if err != nil { - return "", fmt.Errorf("could not execute: %v", err) + return "", fmt.Errorf("could not execute '%s %s': %v", cmd.Path, cmd.Args, err) + } if execErr.Len() > 0 { @@ -59,3 +63,51 @@ func (f *Framework) NewIngressController(namespace string) error { return nil } + +var ( + proxyRegexp = regexp.MustCompile("Starting to serve on .*:([0-9]+)") +) + +// KubectlProxy creates a proxy to kubernetes apiserver +func (f *Framework) KubectlProxy(port int) (int, *exec.Cmd, error) { + cmd := exec.Command("/bin/bash", "-c", fmt.Sprintf("%s proxy --accept-hosts=.* --address=0.0.0.0 --port=%d", KubectlPath, port)) + stdout, stderr, err := startCmdAndStreamOutput(cmd) + if err != nil { + return -1, nil, err + } + + defer stdout.Close() + defer stderr.Close() + + buf := make([]byte, 128) + var n int + if n, err = stdout.Read(buf); err != nil { + return -1, cmd, fmt.Errorf("Failed to read from kubectl proxy stdout: %v", err) + } + + output := string(buf[:n]) + match := proxyRegexp.FindStringSubmatch(output) + if len(match) == 2 { + if port, err := strconv.Atoi(match[1]); err == nil { + return port, cmd, nil + } + } + + return -1, cmd, fmt.Errorf("Failed to parse port from proxy stdout: %s", output) +} + +func startCmdAndStreamOutput(cmd *exec.Cmd) (stdout, stderr io.ReadCloser, err error) { + stdout, err = cmd.StdoutPipe() + if err != nil { + return + } + + stderr, err = cmd.StderrPipe() + if err != nil { + return + } + + Logf("Asynchronously running '%s'", strings.Join(cmd.Args, " ")) + err = cmd.Start() + return +} diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go index 10234a719..d2ec36035 100644 --- a/test/e2e/framework/framework.go +++ b/test/e2e/framework/framework.go @@ -46,6 +46,11 @@ const ( HTTPS RequestScheme = "https" ) +var ( + // KubectlPath defines the full path of the kubectl binary + KubectlPath = "/usr/local/bin/kubectl" +) + // Framework supports common operations used by e2e tests; it will keep a client & a namespace for you. type Framework struct { BaseName string @@ -112,14 +117,10 @@ func (f *Framework) BeforeEach() { }) Expect(err).NotTo(HaveOccurred()) - HTTPURL, err := f.GetNginxURL(HTTP) - Expect(err).NotTo(HaveOccurred()) - + HTTPURL := f.GetNginxURL(HTTP) f.IngressController.HTTPURL = HTTPURL - HTTPSURL, err := f.GetNginxURL(HTTPS) - Expect(err).NotTo(HaveOccurred()) - + HTTPSURL := f.GetNginxURL(HTTPS) f.IngressController.HTTPSURL = HTTPSURL // we wait for any change in the informers and SSL certificate generation @@ -149,8 +150,10 @@ func IngressNginxDescribe(text string, body func()) bool { // GetNginxIP returns the IP address of the minikube cluster // where the NGINX ingress controller is running -func (f *Framework) GetNginxIP() (string, error) { - return os.Getenv("NODE_IP"), nil +func (f *Framework) GetNginxIP() string { + nodeIP := os.Getenv("NODE_IP") + Expect(nodeIP).NotTo(BeEmpty(), "env variable NODE_IP is empty") + return nodeIP } // GetNginxPort returns the number of TCP port where NGINX is running @@ -173,33 +176,28 @@ func (f *Framework) GetNginxPort(name string) (int, error) { } // GetNginxURL returns the URL should be used to make a request to NGINX -func (f *Framework) GetNginxURL(scheme RequestScheme) (string, error) { - ip, err := f.GetNginxIP() - if err != nil { - return "", err - } - +func (f *Framework) GetNginxURL(scheme RequestScheme) string { + ip := f.GetNginxIP() port, err := f.GetNginxPort(fmt.Sprintf("%v", scheme)) - if err != nil { - return "", err - } + Expect(err).NotTo(HaveOccurred(), "unexpected error obtaning NGINX Port") - return fmt.Sprintf("%v://%v:%v", scheme, ip, port), nil + return fmt.Sprintf("%v://%v:%v", scheme, ip, port) } // WaitForNginxServer waits until the nginx configuration contains a particular server section -func (f *Framework) WaitForNginxServer(name string, matcher func(cfg string) bool) error { - return wait.Poll(Poll, time.Minute*5, f.matchNginxConditions(name, matcher)) +func (f *Framework) WaitForNginxServer(name string, matcher func(cfg string) bool) { + err := wait.Poll(Poll, time.Minute*5, f.matchNginxConditions(name, matcher)) + Expect(err).NotTo(HaveOccurred(), "unexpected error waiting for nginx server condition/s") } // WaitForNginxConfiguration waits until the nginx configuration contains a particular configuration -func (f *Framework) WaitForNginxConfiguration(matcher func(cfg string) bool) error { - return wait.Poll(Poll, time.Minute*5, f.matchNginxConditions("", matcher)) +func (f *Framework) WaitForNginxConfiguration(matcher func(cfg string) bool) { + err := wait.Poll(Poll, time.Minute*5, f.matchNginxConditions("", matcher)) + Expect(err).NotTo(HaveOccurred(), "unexpected error waiting for nginx server condition/s") } -// NginxLogs returns the logs of the nginx ingress controller pod running -func (f *Framework) NginxLogs() (string, error) { - l, err := f.KubeClientSet.CoreV1().Pods(f.IngressController.Namespace).List(metav1.ListOptions{ +func nginxLogs(client kubernetes.Interface, namespace string) (string, error) { + l, err := client.CoreV1().Pods(namespace).List(metav1.ListOptions{ LabelSelector: "app.kubernetes.io/name=ingress-nginx", }) if err != nil { @@ -209,7 +207,7 @@ func (f *Framework) NginxLogs() (string, error) { for _, pod := range l.Items { if strings.HasPrefix(pod.GetName(), "nginx-ingress-controller") { if isRunning, err := podRunningReady(&pod); err == nil && isRunning { - return f.Logs(&pod) + return Logs(&pod) } } } @@ -217,6 +215,11 @@ func (f *Framework) NginxLogs() (string, error) { return "", fmt.Errorf("no nginx ingress controller pod is running (logs)") } +// NginxLogs returns the logs of the nginx ingress controller pod running +func (f *Framework) NginxLogs() (string, error) { + return nginxLogs(f.KubeClientSet, f.IngressController.Namespace) +} + func (f *Framework) matchNginxConditions(name string, matcher func(cfg string) bool) wait.ConditionFunc { return func() (bool, error) { l, err := f.KubeClientSet.CoreV1().Pods(f.IngressController.Namespace).List(metav1.ListOptions{ @@ -311,16 +314,12 @@ func (f *Framework) GetNginxConfigMapData() (map[string]string, error) { } // SetNginxConfigMapData sets ingress-nginx's nginx-configuration configMap data -func (f *Framework) SetNginxConfigMapData(cmData map[string]string) error { +func (f *Framework) SetNginxConfigMapData(cmData map[string]string) { // Needs to do a Get and Set, Update will not take just the Data field // or a configMap that is not the very last revision config, err := f.getNginxConfigMap() - if err != nil { - return err - } - if config == nil { - return fmt.Errorf("Unable to get nginx-configuration configMap") - } + Expect(err).NotTo(HaveOccurred()) + Expect(config).NotTo(BeNil(), "expected a configmap but none returned") config.Data = cmData @@ -328,25 +327,19 @@ func (f *Framework) SetNginxConfigMapData(cmData map[string]string) error { CoreV1(). ConfigMaps(f.IngressController.Namespace). Update(config) - if err != nil { - return err - } + Expect(err).NotTo(HaveOccurred()) time.Sleep(5 * time.Second) - - return err } // UpdateNginxConfigMapData updates single field in ingress-nginx's nginx-configuration map data -func (f *Framework) UpdateNginxConfigMapData(key string, value string) error { +func (f *Framework) UpdateNginxConfigMapData(key string, value string) { config, err := f.GetNginxConfigMapData() - if err != nil { - return err - } + Expect(err).NotTo(HaveOccurred(), "unexpected error reading configmap") config[key] = value - return f.SetNginxConfigMapData(config) + f.SetNginxConfigMapData(config) } // UpdateDeployment runs the given updateFunc on the deployment and waits for it to be updated @@ -380,7 +373,7 @@ func UpdateDeployment(kubeClientSet kubernetes.Interface, namespace string, name LabelSelector: fields.SelectorFromSet(fields.Set(deployment.Spec.Template.ObjectMeta.Labels)).String(), }) if err != nil { - return errors.Wrapf(err, "failed to wait for nginx-ingress-controller replica count to be %v", replicas) + return errors.Wrapf(err, "waiting for nginx-ingress-controller replica count to be %v", replicas) } return nil @@ -428,6 +421,7 @@ func newSingleIngress(name, path, host, ns, service string, port int, annotation }, }, } + if withTLS { ing.Spec.TLS = []extensions.IngressTLS{ { diff --git a/test/e2e/framework/grpc_fortune_teller.go b/test/e2e/framework/grpc_fortune_teller.go index 7a1e2c77e..d2075e017 100644 --- a/test/e2e/framework/grpc_fortune_teller.go +++ b/test/e2e/framework/grpc_fortune_teller.go @@ -17,10 +17,9 @@ limitations under the License. package framework import ( - "fmt" "time" - "github.com/pkg/errors" + . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" extensions "k8s.io/api/extensions/v1beta1" @@ -31,13 +30,13 @@ import ( // NewGRPCFortuneTellerDeployment creates a new single replica // deployment of the fortune teller image in a particular namespace -func (f *Framework) NewGRPCFortuneTellerDeployment() error { - return f.NewNewGRPCFortuneTellerDeploymentWithReplicas(1) +func (f *Framework) NewGRPCFortuneTellerDeployment() { + f.NewNewGRPCFortuneTellerDeploymentWithReplicas(1) } // NewNewGRPCFortuneTellerDeploymentWithReplicas creates a new deployment of the // fortune teller image in a particular namespace. Number of replicas is configurable -func (f *Framework) NewNewGRPCFortuneTellerDeploymentWithReplicas(replicas int32) error { +func (f *Framework) NewNewGRPCFortuneTellerDeploymentWithReplicas(replicas int32) { deployment := &extensions.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: "fortune-teller", @@ -77,20 +76,13 @@ func (f *Framework) NewNewGRPCFortuneTellerDeploymentWithReplicas(replicas int32 } d, err := f.EnsureDeployment(deployment) - if err != nil { - return err - } - - if d == nil { - return fmt.Errorf("unexpected error creating deployement for fortune-teller") - } + Expect(err).NotTo(HaveOccurred()) + Expect(d).NotTo(BeNil(), "expected a fortune-teller deployment") err = WaitForPodsReady(f.KubeClientSet, 5*time.Minute, int(replicas), f.IngressController.Namespace, metav1.ListOptions{ LabelSelector: fields.SelectorFromSet(fields.Set(d.Spec.Template.ObjectMeta.Labels)).String(), }) - if err != nil { - return errors.Wrap(err, "failed to wait for to become ready") - } + Expect(err).NotTo(HaveOccurred(), "failed to wait for to become ready") service := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ @@ -112,14 +104,5 @@ func (f *Framework) NewNewGRPCFortuneTellerDeploymentWithReplicas(replicas int32 }, } - s, err := f.EnsureService(service) - if err != nil { - return err - } - - if s == nil { - return fmt.Errorf("unexpected error creating service for fortune-teller deployment") - } - - return nil + f.EnsureService(service) } diff --git a/test/e2e/framework/influxdb.go b/test/e2e/framework/influxdb.go index 82ef69b80..0ad895e72 100644 --- a/test/e2e/framework/influxdb.go +++ b/test/e2e/framework/influxdb.go @@ -17,10 +17,9 @@ limitations under the License. package framework import ( - "fmt" "time" - "github.com/pkg/errors" + . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" extensions "k8s.io/api/extensions/v1beta1" @@ -60,7 +59,7 @@ bind-address = "0.0.0.0:8088" // NewInfluxDBDeployment creates an InfluxDB server configured to reply // on 8086/tcp and 8089/udp -func (f *Framework) NewInfluxDBDeployment() error { +func (f *Framework) NewInfluxDBDeployment() { configuration := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: "influxdb-config", @@ -72,13 +71,9 @@ func (f *Framework) NewInfluxDBDeployment() error { } cm, err := f.EnsureConfigMap(configuration) - if err != nil { - return err - } + Expect(err).NotTo(HaveOccurred(), "failed to create an Influxdb deployment") - if cm == nil { - return fmt.Errorf("unexpected error creating configmap for influxdb") - } + Expect(cm).NotTo(BeNil(), "expected a configmap but none returned") deployment := &extensions.Deployment{ ObjectMeta: metav1.ObjectMeta{ @@ -143,20 +138,12 @@ func (f *Framework) NewInfluxDBDeployment() error { } d, err := f.EnsureDeployment(deployment) - if err != nil { - return err - } + Expect(err).NotTo(HaveOccurred(), "failed to create an Influxdb deployment") - if d == nil { - return fmt.Errorf("unexpected error creating deployement for influxdb") - } + Expect(d).NotTo(BeNil(), "unexpected error creating deployement for influxdb") err = WaitForPodsReady(f.KubeClientSet, 5*time.Minute, 1, f.IngressController.Namespace, metav1.ListOptions{ LabelSelector: fields.SelectorFromSet(fields.Set(d.Spec.Template.ObjectMeta.Labels)).String(), }) - if err != nil { - return errors.Wrap(err, "failed to wait for influxdb to become ready") - } - - return nil + Expect(err).NotTo(HaveOccurred(), "failed to wait for influxdb to become ready") } diff --git a/test/e2e/framework/k8s.go b/test/e2e/framework/k8s.go index e5594497f..5a40fe100 100644 --- a/test/e2e/framework/k8s.go +++ b/test/e2e/framework/k8s.go @@ -20,6 +20,8 @@ import ( "fmt" "time" + . "github.com/onsi/gomega" + api "k8s.io/api/core/v1" core "k8s.io/api/core/v1" extensions "k8s.io/api/extensions/v1beta1" @@ -31,15 +33,23 @@ import ( ) // EnsureSecret creates a Secret object or returns it if it already exists. -func (f *Framework) EnsureSecret(secret *api.Secret) (*api.Secret, error) { +func (f *Framework) EnsureSecret(secret *api.Secret) *api.Secret { s, err := f.KubeClientSet.CoreV1().Secrets(secret.Namespace).Create(secret) if err != nil { if k8sErrors.IsAlreadyExists(err) { - return f.KubeClientSet.CoreV1().Secrets(secret.Namespace).Update(secret) + s, err := f.KubeClientSet.CoreV1().Secrets(secret.Namespace).Update(secret) + Expect(err).NotTo(HaveOccurred(), "unexpected error updating secret") + + return s } - return nil, err + + Expect(err).NotTo(HaveOccurred(), "unexpected error creating secret") } - return s, nil + + Expect(s).NotTo(BeNil()) + Expect(s.ObjectMeta).NotTo(BeNil()) + + return s } // EnsureConfigMap creates a ConfigMap object or returns it if it already exists. @@ -51,40 +61,49 @@ func (f *Framework) EnsureConfigMap(configMap *api.ConfigMap) (*api.ConfigMap, e } return nil, err } + return cm, nil } // EnsureIngress creates an Ingress object or returns it if it already exists. -func (f *Framework) EnsureIngress(ingress *extensions.Ingress) (*extensions.Ingress, error) { - s, err := f.KubeClientSet.ExtensionsV1beta1().Ingresses(ingress.Namespace).Update(ingress) +func (f *Framework) EnsureIngress(ingress *extensions.Ingress) *extensions.Ingress { + ing, err := f.KubeClientSet.ExtensionsV1beta1().Ingresses(ingress.Namespace).Update(ingress) if err != nil { if k8sErrors.IsNotFound(err) { - s, err = f.KubeClientSet.ExtensionsV1beta1().Ingresses(ingress.Namespace).Create(ingress) - if err != nil { - return nil, err - } - } else { - return nil, err + ing, err = f.KubeClientSet.ExtensionsV1beta1().Ingresses(ingress.Namespace).Create(ingress) + Expect(err).NotTo(HaveOccurred(), "unexpected error creating ingress") + return ing } + + Expect(err).NotTo(HaveOccurred()) } - if s.Annotations == nil { - s.Annotations = make(map[string]string) + Expect(ing).NotTo(BeNil()) + + if ing.Annotations == nil { + ing.Annotations = make(map[string]string) } - return s, nil + return ing } // EnsureService creates a Service object or returns it if it already exists. -func (f *Framework) EnsureService(service *core.Service) (*core.Service, error) { +func (f *Framework) EnsureService(service *core.Service) *core.Service { s, err := f.KubeClientSet.CoreV1().Services(service.Namespace).Update(service) if err != nil { if k8sErrors.IsNotFound(err) { - return f.KubeClientSet.CoreV1().Services(service.Namespace).Create(service) + s, err := f.KubeClientSet.CoreV1().Services(service.Namespace).Create(service) + Expect(err).NotTo(HaveOccurred(), "unexpected error creating service") + return s + } - return nil, err + + Expect(err).NotTo(HaveOccurred()) } - return s, nil + + Expect(s).NotTo(BeNil(), "expected a service but none returned") + + return s } // EnsureDeployment creates a Deployment object or returns it if it already exists. diff --git a/test/e2e/framework/logs.go b/test/e2e/framework/logs.go index f1a08b82d..21a93e569 100644 --- a/test/e2e/framework/logs.go +++ b/test/e2e/framework/logs.go @@ -25,7 +25,7 @@ import ( ) // Logs returns the log entries of a given Pod. -func (f *Framework) Logs(pod *v1.Pod) (string, error) { +func Logs(pod *v1.Pod) (string, error) { var ( execOut bytes.Buffer execErr bytes.Buffer @@ -35,14 +35,13 @@ func (f *Framework) Logs(pod *v1.Pod) (string, error) { return "", fmt.Errorf("could not determine which container to use") } - args := fmt.Sprintf("kubectl logs -n %v %v", pod.Namespace, pod.Name) - cmd := exec.Command("/bin/bash", "-c", args) + cmd := exec.Command("/bin/bash", "-c", fmt.Sprintf("%v logs --namespace %s %s", KubectlPath, pod.Namespace, pod.Name)) cmd.Stdout = &execOut cmd.Stderr = &execErr err := cmd.Run() if err != nil { - return "", fmt.Errorf("could not execute: %v", err) + return "", fmt.Errorf("could not execute '%s %s': %v", cmd.Path, cmd.Args, err) } if execErr.Len() > 0 { diff --git a/test/e2e/framework/ssl.go b/test/e2e/framework/ssl.go index 9b6055e2c..803f5f034 100644 --- a/test/e2e/framework/ssl.go +++ b/test/e2e/framework/ssl.go @@ -32,6 +32,8 @@ import ( "strings" "time" + . "github.com/onsi/gomega" + "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" @@ -140,8 +142,9 @@ func CreateIngressMASecret(client kubernetes.Interface, host string, secretName, } // WaitForTLS waits until the TLS handshake with a given server completes successfully. -func WaitForTLS(url string, tlsConfig *tls.Config) error { - return wait.Poll(Poll, 30*time.Second, matchTLSServerName(url, tlsConfig)) +func WaitForTLS(url string, tlsConfig *tls.Config) { + err := wait.Poll(Poll, 30*time.Second, matchTLSServerName(url, tlsConfig)) + Expect(err).NotTo(HaveOccurred(), "timeout waiting for TLS configuration in URL %s", url) } // generateRSACert generates a basic self signed certificate using a key length diff --git a/test/e2e/lua/dynamic_certificates.go b/test/e2e/lua/dynamic_certificates.go index 143db355f..3541997df 100644 --- a/test/e2e/lua/dynamic_certificates.go +++ b/test/e2e/lua/dynamic_certificates.go @@ -27,7 +27,6 @@ import ( appsv1beta1 "k8s.io/api/apps/v1beta1" extensions "k8s.io/api/extensions/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" "k8s.io/ingress-nginx/test/e2e/framework" ) @@ -37,17 +36,24 @@ var _ = framework.IngressNginxDescribe("Dynamic Certificate", func() { host := "foo.com" BeforeEach(func() { - err := enableDynamicCertificates(f.IngressController.Namespace, f.KubeClientSet) - Expect(err).NotTo(HaveOccurred()) + err := framework.UpdateDeployment(f.KubeClientSet, f.IngressController.Namespace, "nginx-ingress-controller", 1, + func(deployment *appsv1beta1.Deployment) error { + args := deployment.Spec.Template.Spec.Containers[0].Args + args = append(args, "--enable-dynamic-certificates") + args = append(args, "--enable-ssl-chain-completion=false") + deployment.Spec.Template.Spec.Containers[0].Args = args + _, err := f.KubeClientSet.AppsV1beta1().Deployments(f.IngressController.Namespace).Update(deployment) - err = f.WaitForNginxConfiguration( - func(cfg string) bool { - return strings.Contains(cfg, "ok, res = pcall(require, \"certificate\")") + return err }) Expect(err).NotTo(HaveOccurred()) - err = f.NewEchoDeploymentWithReplicas(1) - Expect(err).NotTo(HaveOccurred()) + f.WaitForNginxConfiguration( + func(cfg string) bool { + return strings.Contains(cfg, "ok, res = pcall(require, \"certificate\")") + }) + + f.NewEchoDeploymentWithReplicas(1) }) It("picks up the certificate when we add TLS spec to existing ingress", func() { @@ -74,27 +80,27 @@ var _ = framework.IngressNginxDescribe("Dynamic Certificate", func() { }) It("picks up the previously missing secret for a given ingress without reloading", func() { - ing, err := f.EnsureIngress(framework.NewSingleIngressWithTLS(host, "/", host, f.IngressController.Namespace, "http-svc", 80, nil)) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) + ing := framework.NewSingleIngressWithTLS(host, "/", host, f.IngressController.Namespace, "http-svc", 80, nil) + f.EnsureIngress(ing) + time.Sleep(waitForLuaSync) + ensureHTTPSRequest(fmt.Sprintf("%s?id=dummy_log_splitter_foo_bar", f.IngressController.HTTPSURL), host, "ingress.local") - _, err = framework.CreateIngressTLSSecret(f.KubeClientSet, + _, err := framework.CreateIngressTLSSecret(f.KubeClientSet, ing.Spec.TLS[0].Hosts, ing.Spec.TLS[0].SecretName, ing.Namespace) Expect(err).ToNot(HaveOccurred()) By("configuring certificate_by_lua and skipping Nginx configuration of the new certificate") - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return strings.Contains(server, "ssl_certificate_by_lua_block") && !strings.Contains(server, fmt.Sprintf("ssl_certificate /etc/ingress-controller/ssl/%s-%s.pem;", ing.Namespace, host)) && !strings.Contains(server, fmt.Sprintf("ssl_certificate_key /etc/ingress-controller/ssl/%s-%s.pem;", ing.Namespace, host)) && strings.Contains(server, "listen 443") }) - Expect(err).ToNot(HaveOccurred()) time.Sleep(waitForLuaSync) @@ -114,14 +120,13 @@ var _ = framework.IngressNginxDescribe("Dynamic Certificate", func() { Context("given an ingress with TLS correctly configured", func() { BeforeEach(func() { - ing, err := f.EnsureIngress(framework.NewSingleIngressWithTLS(host, "/", host, f.IngressController.Namespace, "http-svc", 80, nil)) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) + ing := f.EnsureIngress(framework.NewSingleIngressWithTLS(host, "/", host, f.IngressController.Namespace, "http-svc", 80, nil)) + time.Sleep(waitForLuaSync) ensureHTTPSRequest(f.IngressController.HTTPSURL, host, "ingress.local") - _, err = framework.CreateIngressTLSSecret(f.KubeClientSet, + _, err := framework.CreateIngressTLSSecret(f.KubeClientSet, ing.Spec.TLS[0].Hosts, ing.Spec.TLS[0].SecretName, ing.Namespace) @@ -129,14 +134,13 @@ var _ = framework.IngressNginxDescribe("Dynamic Certificate", func() { time.Sleep(waitForLuaSync) By("configuring certificate_by_lua and skipping Nginx configuration of the new certificate") - err = f.WaitForNginxServer(ing.Spec.TLS[0].Hosts[0], + f.WaitForNginxServer(ing.Spec.TLS[0].Hosts[0], func(server string) bool { return strings.Contains(server, "ssl_certificate_by_lua_block") && !strings.Contains(server, fmt.Sprintf("ssl_certificate /etc/ingress-controller/ssl/%s-%s.pem;", ing.Namespace, host)) && !strings.Contains(server, fmt.Sprintf("ssl_certificate_key /etc/ingress-controller/ssl/%s-%s.pem;", ing.Namespace, host)) && strings.Contains(server, "listen 443") }) - Expect(err).ToNot(HaveOccurred()) time.Sleep(waitForLuaSync) @@ -146,6 +150,7 @@ var _ = framework.IngressNginxDescribe("Dynamic Certificate", func() { It("picks up the updated certificate without reloading", func() { ing, err := f.KubeClientSet.ExtensionsV1beta1().Ingresses(f.IngressController.Namespace).Get(host, metav1.GetOptions{}) + Expect(err).ToNot(HaveOccurred()) ensureHTTPSRequest(fmt.Sprintf("%s?id=dummy_log_splitter_foo_bar", f.IngressController.HTTPSURL), host, host) @@ -157,14 +162,13 @@ var _ = framework.IngressNginxDescribe("Dynamic Certificate", func() { time.Sleep(waitForLuaSync) By("configuring certificate_by_lua and skipping Nginx configuration of the new certificate") - err = f.WaitForNginxServer(ing.Spec.TLS[0].Hosts[0], + f.WaitForNginxServer(ing.Spec.TLS[0].Hosts[0], func(server string) bool { return strings.Contains(server, "ssl_certificate_by_lua_block") && !strings.Contains(server, fmt.Sprintf("ssl_certificate /etc/ingress-controller/ssl/%s-%s.pem;", ing.Namespace, host)) && !strings.Contains(server, fmt.Sprintf("ssl_certificate_key /etc/ingress-controller/ssl/%s-%s.pem;", ing.Namespace, host)) && strings.Contains(server, "listen 443") }) - Expect(err).ToNot(HaveOccurred()) time.Sleep(waitForLuaSync) @@ -192,14 +196,13 @@ var _ = framework.IngressNginxDescribe("Dynamic Certificate", func() { time.Sleep(waitForLuaSync) By("configuring certificate_by_lua and skipping Nginx configuration of the new certificate") - err = f.WaitForNginxServer(ing.Spec.TLS[0].Hosts[0], + f.WaitForNginxServer(ing.Spec.TLS[0].Hosts[0], func(server string) bool { return strings.Contains(server, "ssl_certificate_by_lua_block") && strings.Contains(server, "ssl_certificate /etc/ingress-controller/ssl/default-fake-certificate.pem;") && strings.Contains(server, "ssl_certificate_key /etc/ingress-controller/ssl/default-fake-certificate.pem;") && strings.Contains(server, "listen 443") }) - Expect(err).ToNot(HaveOccurred()) time.Sleep(waitForLuaSync) @@ -242,16 +245,3 @@ var _ = framework.IngressNginxDescribe("Dynamic Certificate", func() { }) }) }) - -func enableDynamicCertificates(namespace string, kubeClientSet kubernetes.Interface) error { - return framework.UpdateDeployment(kubeClientSet, namespace, "nginx-ingress-controller", 1, - func(deployment *appsv1beta1.Deployment) error { - args := deployment.Spec.Template.Spec.Containers[0].Args - args = append(args, "--enable-dynamic-certificates") - args = append(args, "--enable-ssl-chain-completion=false") - deployment.Spec.Template.Spec.Containers[0].Args = args - _, err := kubeClientSet.AppsV1beta1().Deployments(namespace).Update(deployment) - - return err - }) -} diff --git a/test/e2e/lua/dynamic_configuration.go b/test/e2e/lua/dynamic_configuration.go index 19c5e70a6..74ddf428b 100644 --- a/test/e2e/lua/dynamic_configuration.go +++ b/test/e2e/lua/dynamic_configuration.go @@ -47,60 +47,54 @@ var _ = framework.IngressNginxDescribe("Dynamic Configuration", func() { f := framework.NewDefaultFramework("dynamic-configuration") BeforeEach(func() { - err := f.NewEchoDeploymentWithReplicas(1) - Expect(err).NotTo(HaveOccurred()) - + f.NewEchoDeploymentWithReplicas(1) ensureIngress(f, "foo.com") }) It("configures balancer Lua middleware correctly", func() { - err := f.WaitForNginxConfiguration(func(cfg string) bool { + f.WaitForNginxConfiguration(func(cfg string) bool { return strings.Contains(cfg, "balancer.init_worker()") && strings.Contains(cfg, "balancer.balance()") }) - Expect(err).NotTo(HaveOccurred()) host := "foo.com" - err = f.WaitForNginxServer(host, func(server string) bool { + f.WaitForNginxServer(host, func(server string) bool { return strings.Contains(server, "balancer.rewrite()") && strings.Contains(server, "balancer.log()") }) - Expect(err).NotTo(HaveOccurred()) }) It("sets nameservers for Lua", func() { - err := f.WaitForNginxConfiguration(func(cfg string) bool { + f.WaitForNginxConfiguration(func(cfg string) bool { r := regexp.MustCompile(`configuration.nameservers = { [".,0-9a-zA-Z]+ }`) return r.MatchString(cfg) }) - Expect(err).NotTo(HaveOccurred()) }) Context("when only backends change", func() { It("handles endpoints only changes", func() { var nginxConfig string - err := f.WaitForNginxConfiguration(func(cfg string) bool { + f.WaitForNginxConfiguration(func(cfg string) bool { nginxConfig = cfg return true }) replicas := 2 - err = framework.UpdateDeployment(f.KubeClientSet, f.IngressController.Namespace, "http-svc", replicas, nil) + err := framework.UpdateDeployment(f.KubeClientSet, f.IngressController.Namespace, "http-svc", replicas, nil) Expect(err).NotTo(HaveOccurred()) time.Sleep(waitForLuaSync) ensureRequest(f, "foo.com") var newNginxConfig string - err = f.WaitForNginxConfiguration(func(cfg string) bool { + f.WaitForNginxConfiguration(func(cfg string) bool { newNginxConfig = cfg return true }) - Expect(nginxConfig).Should(Equal(newNginxConfig)) }) It("handles an annotation change", func() { var nginxConfig string - err := f.WaitForNginxConfiguration(func(cfg string) bool { + f.WaitForNginxConfiguration(func(cfg string) bool { nginxConfig = cfg return true }) @@ -116,7 +110,7 @@ var _ = framework.IngressNginxDescribe("Dynamic Configuration", func() { ensureRequest(f, "foo.com") var newNginxConfig string - err = f.WaitForNginxConfiguration(func(cfg string) bool { + f.WaitForNginxConfiguration(func(cfg string) bool { newNginxConfig = cfg return true }) @@ -127,7 +121,7 @@ var _ = framework.IngressNginxDescribe("Dynamic Configuration", func() { It("handles a non backend update", func() { var nginxConfig string - err := f.WaitForNginxConfiguration(func(cfg string) bool { + f.WaitForNginxConfiguration(func(cfg string) bool { nginxConfig = cfg return true }) @@ -149,11 +143,10 @@ var _ = framework.IngressNginxDescribe("Dynamic Configuration", func() { Expect(err).ToNot(HaveOccurred()) var newNginxConfig string - err = f.WaitForNginxConfiguration(func(cfg string) bool { + f.WaitForNginxConfiguration(func(cfg string) bool { newNginxConfig = cfg return true }) - Expect(nginxConfig).ShouldNot(Equal(newNginxConfig)) }) @@ -181,7 +174,7 @@ func ensureIngress(f *framework.Framework, host string) *extensions.Ingress { "nginx.ingress.kubernetes.io/session-cookie-max-age": "172800"})) Expect(err).NotTo(HaveOccurred()) Expect(ing).NotTo(BeNil()) - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return strings.Contains(server, fmt.Sprintf("server_name %s ;", host)) && strings.Contains(server, "proxy_pass http://upstream_balancer;") diff --git a/test/e2e/servicebackend/service_backend.go b/test/e2e/servicebackend/service_backend.go index e60319084..7e28d0315 100644 --- a/test/e2e/servicebackend/service_backend.go +++ b/test/e2e/servicebackend/service_backend.go @@ -45,15 +45,12 @@ var _ = framework.IngressNginxDescribe("Service backend - 503", func() { host := "nonexistent.svc.com" bi := buildIngressWithNonexistentService(host, f.IngressController.Namespace, "/") - ing, err := f.EnsureIngress(bi) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) + f.EnsureIngress(bi) - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { - return strings.Contains(server, "return 503;") + return strings.Contains(server, "proxy_pass http://upstream_balancer;") }) - Expect(err).NotTo(HaveOccurred()) resp, _, errs := gorequest.New(). Get(f.IngressController.HTTPURL). @@ -68,19 +65,15 @@ var _ = framework.IngressNginxDescribe("Service backend - 503", func() { bi, bs := buildIngressWithUnavailableServiceEndpoints(host, f.IngressController.Namespace, "/") - svc, err := f.EnsureService(bs) - Expect(err).NotTo(HaveOccurred()) + svc := f.EnsureService(bs) Expect(svc).NotTo(BeNil()) - ing, err := f.EnsureIngress(bi) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) + f.EnsureIngress(bi) - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { - return strings.Contains(server, "return 503;") + return strings.Contains(server, "proxy_pass http://upstream_balancer;") }) - Expect(err).NotTo(HaveOccurred()) resp, _, errs := gorequest.New(). Get(f.IngressController.HTTPURL). diff --git a/test/e2e/settings/configmap_change.go b/test/e2e/settings/configmap_change.go index ff88687f7..4e19ca90d 100644 --- a/test/e2e/settings/configmap_change.go +++ b/test/e2e/settings/configmap_change.go @@ -30,8 +30,7 @@ var _ = framework.IngressNginxDescribe("Configmap change", func() { f := framework.NewDefaultFramework("configmap-change") BeforeEach(func() { - err := f.NewEchoDeployment() - Expect(err).NotTo(HaveOccurred()) + f.NewEchoDeployment() }) AfterEach(func() { @@ -40,22 +39,20 @@ var _ = framework.IngressNginxDescribe("Configmap change", func() { It("should reload after an update in the configuration", func() { host := "configmap-change" - ing, err := f.EnsureIngress(framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, nil)) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) + ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, nil) + f.EnsureIngress(ing) wlKey := "whitelist-source-range" wlValue := "1.1.1.1" By("adding a whitelist-source-range") - err = f.UpdateNginxConfigMapData(wlKey, wlValue) - Expect(err).NotTo(HaveOccurred()) + f.UpdateNginxConfigMapData(wlKey, wlValue) checksumRegex := regexp.MustCompile("Configuration checksum:\\s+(\\d+)") checksum := "" - err = f.WaitForNginxConfiguration( + f.WaitForNginxConfiguration( func(cfg string) bool { // before returning, extract the current checksum match := checksumRegex.FindStringSubmatch(cfg) @@ -66,16 +63,14 @@ var _ = framework.IngressNginxDescribe("Configmap change", func() { return strings.Contains(cfg, "geo $the_real_ip $deny_") && strings.Contains(cfg, "1.1.1.1 0") }) - Expect(err).NotTo(HaveOccurred()) Expect(checksum).NotTo(BeEmpty()) By("changing error-log-level") - err = f.UpdateNginxConfigMapData("error-log-level", "debug") - Expect(err).NotTo(HaveOccurred()) + f.UpdateNginxConfigMapData("error-log-level", "debug") newChecksum := "" - err = f.WaitForNginxConfiguration( + f.WaitForNginxConfiguration( func(cfg string) bool { match := checksumRegex.FindStringSubmatch(cfg) if len(match) > 0 { @@ -84,8 +79,6 @@ var _ = framework.IngressNginxDescribe("Configmap change", func() { return strings.ContainsAny(cfg, "error_log /var/log/nginx/error.log debug;") }) - Expect(err).NotTo(HaveOccurred()) - Expect(checksum).NotTo(BeEquivalentTo(newChecksum)) }) }) diff --git a/test/e2e/settings/forwarded_headers.go b/test/e2e/settings/forwarded_headers.go index 35f9a5fff..fe451655e 100644 --- a/test/e2e/settings/forwarded_headers.go +++ b/test/e2e/settings/forwarded_headers.go @@ -34,11 +34,8 @@ var _ = framework.IngressNginxDescribe("X-Forwarded headers", func() { setting := "use-forwarded-headers" BeforeEach(func() { - err := f.NewEchoDeployment() - Expect(err).NotTo(HaveOccurred()) - - err = f.UpdateNginxConfigMapData(setting, "false") - Expect(err).NotTo(HaveOccurred()) + f.NewEchoDeployment() + f.UpdateNginxConfigMapData(setting, "false") }) AfterEach(func() { @@ -47,18 +44,15 @@ var _ = framework.IngressNginxDescribe("X-Forwarded headers", func() { It("should trust X-Forwarded headers when setting is true", func() { host := "forwarded-headers" - err := f.UpdateNginxConfigMapData(setting, "true") - Expect(err).NotTo(HaveOccurred()) + f.UpdateNginxConfigMapData(setting, "true") - ing, err := f.EnsureIngress(framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, nil)) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) + ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, nil) + f.EnsureIngress(ing) - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return strings.Contains(server, "server_name forwarded-headers") }) - Expect(err).NotTo(HaveOccurred()) resp, body, errs := gorequest.New(). Get(f.IngressController.HTTPURL). @@ -80,18 +74,14 @@ var _ = framework.IngressNginxDescribe("X-Forwarded headers", func() { It("should not trust X-Forwarded headers when setting is false", func() { host := "forwarded-headers" - err := f.UpdateNginxConfigMapData(setting, "false") - Expect(err).NotTo(HaveOccurred()) + f.UpdateNginxConfigMapData(setting, "false") - ing, err := f.EnsureIngress(framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, nil)) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) + f.EnsureIngress(framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, nil)) - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return strings.Contains(server, "server_name forwarded-headers") }) - Expect(err).NotTo(HaveOccurred()) resp, body, errs := gorequest.New(). Get(f.IngressController.HTTPURL). diff --git a/test/e2e/settings/geoip2.go b/test/e2e/settings/geoip2.go new file mode 100644 index 000000000..325691b51 --- /dev/null +++ b/test/e2e/settings/geoip2.go @@ -0,0 +1,90 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package settings + +import ( + "strings" + + "net/http" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/parnurzeal/gorequest" + "k8s.io/ingress-nginx/test/e2e/framework" +) + +var _ = framework.IngressNginxDescribe("Geoip2", func() { + f := framework.NewDefaultFramework("geoip2") + + host := "geoip2" + + BeforeEach(func() { + f.NewEchoDeployment() + }) + + It("should only allow requests from specific countries", func() { + f.UpdateNginxConfigMapData("use-geoip2", "true") + + httpSnippetAllowingOnlyAustralia := + `map $geoip2_city_country_code $blocked_country { + default 1; + AU 0; +}` + f.UpdateNginxConfigMapData("http-snippet", httpSnippetAllowingOnlyAustralia) + + f.WaitForNginxConfiguration( + func(cfg string) bool { + return strings.Contains(cfg, "map $geoip2_city_country_code $blocked_country") + }) + + configSnippet := + `if ($blocked_country) { + return 403; +}` + + annotations := map[string]string{ + "nginx.ingress.kubernetes.io/configuration-snippet": configSnippet, + } + + f.EnsureIngress(framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations)) + + f.WaitForNginxConfiguration( + func(cfg string) bool { + return strings.Contains(cfg, "if ($blocked_country)") + }) + + // Should be blocked + usIP := "8.8.8.8" + resp, _, errs := gorequest.New(). + Get(f.IngressController.HTTPURL). + Set("Host", host). + Set("X-Forwarded-For", usIP). + End() + Expect(errs).To(BeNil()) + Expect(resp.StatusCode).Should(Equal(http.StatusForbidden)) + + // Shouldn't be blocked + australianIP := "1.1.1.1" + resp, _, errs = gorequest.New(). + Get(f.IngressController.HTTPURL). + Set("Host", host). + Set("X-Forwarded-For", australianIP). + End() + Expect(errs).To(BeNil()) + Expect(resp.StatusCode).Should(Equal(http.StatusOK)) + }) +}) diff --git a/test/e2e/settings/global_access_block.go b/test/e2e/settings/global_access_block.go index 3caec7328..924283b18 100644 --- a/test/e2e/settings/global_access_block.go +++ b/test/e2e/settings/global_access_block.go @@ -33,28 +33,22 @@ var _ = framework.IngressNginxDescribe("Global access block", func() { host := "global-access-block" BeforeEach(func() { - err := f.NewEchoDeploymentWithReplicas(1) - Expect(err).NotTo(HaveOccurred()) - - ing, err := f.EnsureIngress(framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, nil)) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) + f.NewEchoDeploymentWithReplicas(1) + f.EnsureIngress(framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, nil)) }) AfterEach(func() { }) It("should block CIDRs defined in the ConfigMap", func() { - err := f.UpdateNginxConfigMapData("block-cidrs", "172.16.0.0/12,192.168.0.0/16,10.0.0.0/8") - Expect(err).NotTo(HaveOccurred()) + f.UpdateNginxConfigMapData("block-cidrs", "172.16.0.0/12,192.168.0.0/16,10.0.0.0/8") - err = f.WaitForNginxConfiguration( + f.WaitForNginxConfiguration( func(cfg string) bool { return strings.Contains(cfg, "deny 172.16.0.0/12;") && strings.Contains(cfg, "deny 192.168.0.0/16;") && strings.Contains(cfg, "deny 10.0.0.0/8;") }) - Expect(err).NotTo(HaveOccurred()) // This test works for minikube, but may have problems with real kubernetes clusters, // especially if connection is done via Internet. In this case, the test should be disabled/removed. @@ -67,15 +61,13 @@ var _ = framework.IngressNginxDescribe("Global access block", func() { }) It("should block User-Agents defined in the ConfigMap", func() { - err := f.UpdateNginxConfigMapData("block-user-agents", "~*chrome\\/68\\.0\\.3440\\.106\\ safari\\/537\\.36,AlphaBot") - Expect(err).NotTo(HaveOccurred()) + f.UpdateNginxConfigMapData("block-user-agents", "~*chrome\\/68\\.0\\.3440\\.106\\ safari\\/537\\.36,AlphaBot") - err = f.WaitForNginxConfiguration( + f.WaitForNginxConfiguration( func(cfg string) bool { return strings.Contains(cfg, "~*chrome\\/68\\.0\\.3440\\.106\\ safari\\/537\\.36 1;") && strings.Contains(cfg, "AlphaBot 1;") }) - Expect(err).NotTo(HaveOccurred()) // Should be blocked resp, _, errs := gorequest.New(). @@ -105,15 +97,13 @@ var _ = framework.IngressNginxDescribe("Global access block", func() { }) It("should block Referers defined in the ConfigMap", func() { - err := f.UpdateNginxConfigMapData("block-referers", "~*example\\.com,qwerty") - Expect(err).NotTo(HaveOccurred()) + f.UpdateNginxConfigMapData("block-referers", "~*example\\.com,qwerty") - err = f.WaitForNginxConfiguration( + f.WaitForNginxConfiguration( func(cfg string) bool { return strings.Contains(cfg, "~*example\\.com 1;") && strings.Contains(cfg, "qwerty 1;") }) - Expect(err).NotTo(HaveOccurred()) // Should be blocked resp, _, errs := gorequest.New(). diff --git a/test/e2e/settings/main_snippet.go b/test/e2e/settings/main_snippet.go index d71ec5c91..0a7d4e4cc 100644 --- a/test/e2e/settings/main_snippet.go +++ b/test/e2e/settings/main_snippet.go @@ -20,7 +20,6 @@ import ( "strings" . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" "k8s.io/ingress-nginx/test/e2e/framework" ) @@ -31,13 +30,11 @@ var _ = framework.IngressNginxDescribe("Main Snippet", func() { It("should add value of main-snippet setting to nginx config", func() { expectedComment := "# main snippet" - err := f.UpdateNginxConfigMapData(mainSnippet, expectedComment) - Expect(err).NotTo(HaveOccurred()) + f.UpdateNginxConfigMapData(mainSnippet, expectedComment) - err = f.WaitForNginxConfiguration( + f.WaitForNginxConfiguration( func(cfg string) bool { return strings.Contains(cfg, expectedComment) }) - Expect(err).NotTo(HaveOccurred()) }) }) diff --git a/test/e2e/settings/multi_accept.go b/test/e2e/settings/multi_accept.go index d067204e9..94d1349fe 100644 --- a/test/e2e/settings/multi_accept.go +++ b/test/e2e/settings/multi_accept.go @@ -20,7 +20,6 @@ import ( "strings" . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" "k8s.io/ingress-nginx/test/e2e/framework" ) @@ -31,34 +30,29 @@ var _ = framework.IngressNginxDescribe("Multi Accept", func() { It("should be enabled by default", func() { expectedDirective := "multi_accept on;" - err := f.WaitForNginxConfiguration( + f.WaitForNginxConfiguration( func(cfg string) bool { return strings.Contains(cfg, expectedDirective) }) - Expect(err).NotTo(HaveOccurred()) }) It("should be enabled when set to true", func() { expectedDirective := "multi_accept on;" - err := f.UpdateNginxConfigMapData(multiAccept, "true") - Expect(err).NotTo(HaveOccurred()) + f.UpdateNginxConfigMapData(multiAccept, "true") - err = f.WaitForNginxConfiguration( + f.WaitForNginxConfiguration( func(cfg string) bool { return strings.Contains(cfg, expectedDirective) }) - Expect(err).NotTo(HaveOccurred()) }) It("should be disabled when set to false", func() { expectedDirective := "multi_accept off;" - err := f.UpdateNginxConfigMapData(multiAccept, "false") - Expect(err).NotTo(HaveOccurred()) + f.UpdateNginxConfigMapData(multiAccept, "false") - err = f.WaitForNginxConfiguration( + f.WaitForNginxConfiguration( func(cfg string) bool { return strings.Contains(cfg, expectedDirective) }) - Expect(err).NotTo(HaveOccurred()) }) }) diff --git a/test/e2e/settings/no_auth_locations.go b/test/e2e/settings/no_auth_locations.go index a91c473f2..a1eb741fc 100644 --- a/test/e2e/settings/no_auth_locations.go +++ b/test/e2e/settings/no_auth_locations.go @@ -43,32 +43,24 @@ var _ = framework.IngressNginxDescribe("No Auth locations", func() { noAuthPath := "/noauth" BeforeEach(func() { - err := f.NewEchoDeployment() - Expect(err).NotTo(HaveOccurred()) + f.NewEchoDeployment() - s, err := f.EnsureSecret(buildSecret(username, password, secretName, f.IngressController.Namespace)) - Expect(err).NotTo(HaveOccurred()) - Expect(s).NotTo(BeNil()) - Expect(s.ObjectMeta).NotTo(BeNil()) + s := f.EnsureSecret(buildSecret(username, password, secretName, f.IngressController.Namespace)) - err = f.UpdateNginxConfigMapData(setting, noAuthPath) - Expect(err).NotTo(HaveOccurred()) + f.UpdateNginxConfigMapData(setting, noAuthPath) bi := buildBasicAuthIngressWithSecondPath(host, f.IngressController.Namespace, s.Name, noAuthPath) - ing, err := f.EnsureIngress(bi) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) + f.EnsureIngress(bi) }) AfterEach(func() { }) It("should return status code 401 when accessing '/' unauthentication", func() { - err := f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return Expect(server).Should(ContainSubstring("test auth")) }) - Expect(err).NotTo(HaveOccurred()) resp, body, errs := gorequest.New(). Get(f.IngressController.HTTPURL). @@ -81,11 +73,10 @@ var _ = framework.IngressNginxDescribe("No Auth locations", func() { }) It("should return status code 200 when accessing '/' authentication", func() { - err := f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return Expect(server).Should(ContainSubstring("test auth")) }) - Expect(err).NotTo(HaveOccurred()) resp, _, errs := gorequest.New(). Get(f.IngressController.HTTPURL). @@ -98,11 +89,10 @@ var _ = framework.IngressNginxDescribe("No Auth locations", func() { }) It("should return status code 200 when accessing '/noauth' unauthenticated", func() { - err := f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return Expect(server).Should(ContainSubstring("test auth")) }) - Expect(err).NotTo(HaveOccurred()) resp, _, errs := gorequest.New(). Get(fmt.Sprintf("%s/noauth", f.IngressController.HTTPURL)). @@ -156,8 +146,9 @@ func buildBasicAuthIngressWithSecondPath(host, namespace, secretName, pathName s func buildSecret(username, password, name, namespace string) *corev1.Secret { out, err := exec.Command("openssl", "passwd", "-crypt", password).CombinedOutput() + Expect(err).NotTo(HaveOccurred(), "creating password") + encpass := fmt.Sprintf("%v:%s\n", username, out) - Expect(err).NotTo(HaveOccurred()) return &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ diff --git a/test/e2e/settings/proxy_protocol.go b/test/e2e/settings/proxy_protocol.go index acf10d718..8a2956515 100644 --- a/test/e2e/settings/proxy_protocol.go +++ b/test/e2e/settings/proxy_protocol.go @@ -35,11 +35,8 @@ var _ = framework.IngressNginxDescribe("Proxy Protocol", func() { setting := "use-proxy-protocol" BeforeEach(func() { - err := f.NewEchoDeployment() - Expect(err).NotTo(HaveOccurred()) - - err = f.UpdateNginxConfigMapData(setting, "false") - Expect(err).NotTo(HaveOccurred()) + f.NewEchoDeployment() + f.UpdateNginxConfigMapData(setting, "false") }) AfterEach(func() { @@ -48,27 +45,22 @@ var _ = framework.IngressNginxDescribe("Proxy Protocol", func() { It("should respect port passed by the PROXY Protocol", func() { host := "proxy-protocol" - err := f.UpdateNginxConfigMapData(setting, "true") - Expect(err).NotTo(HaveOccurred()) + f.UpdateNginxConfigMapData(setting, "true") - ing, err := f.EnsureIngress(framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, nil)) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) + f.EnsureIngress(framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, nil)) - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return strings.Contains(server, "server_name proxy-protocol") && strings.Contains(server, "listen 80 proxy_protocol") }) - Expect(err).NotTo(HaveOccurred()) - ip, err := f.GetNginxIP() - Expect(err).NotTo(HaveOccurred()) + ip := f.GetNginxIP() port, err := f.GetNginxPort("http") - Expect(err).NotTo(HaveOccurred()) + Expect(err).NotTo(HaveOccurred(), "unexpected error obtaning NGINX Port") conn, err := net.Dial("tcp", net.JoinHostPort(ip, strconv.Itoa(port))) - Expect(err).NotTo(HaveOccurred()) + Expect(err).NotTo(HaveOccurred(), "unexpected error creating connection to %s:%d", ip, port) defer conn.Close() header := "PROXY TCP4 192.168.0.1 192.168.0.11 56324 1234\r\n" @@ -76,7 +68,7 @@ var _ = framework.IngressNginxDescribe("Proxy Protocol", func() { conn.Write([]byte("GET / HTTP/1.1\r\nHost: proxy-protocol\r\n\r\n")) data, err := ioutil.ReadAll(conn) - Expect(err).NotTo(HaveOccurred()) + Expect(err).NotTo(HaveOccurred(), "unexpected error reading connection data") body := string(data) Expect(body).Should(ContainSubstring(fmt.Sprintf("host=%v", "proxy-protocol"))) Expect(body).Should(ContainSubstring(fmt.Sprintf("x-forwarded-port=80"))) diff --git a/test/e2e/settings/server_tokens.go b/test/e2e/settings/server_tokens.go index a5aecd6b8..2189f2f79 100644 --- a/test/e2e/settings/server_tokens.go +++ b/test/e2e/settings/server_tokens.go @@ -20,7 +20,6 @@ import ( "strings" . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" "k8s.io/api/extensions/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -33,34 +32,28 @@ var _ = framework.IngressNginxDescribe("Server Tokens", func() { serverTokens := "server-tokens" BeforeEach(func() { - err := f.NewEchoDeployment() - Expect(err).NotTo(HaveOccurred()) + f.NewEchoDeployment() }) AfterEach(func() { }) It("should not exists Server header in the response", func() { - err := f.UpdateNginxConfigMapData(serverTokens, "false") - Expect(err).NotTo(HaveOccurred()) + f.UpdateNginxConfigMapData(serverTokens, "false") - ing, err := f.EnsureIngress(framework.NewSingleIngress(serverTokens, "/", serverTokens, f.IngressController.Namespace, "http-svc", 80, nil)) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) + f.EnsureIngress(framework.NewSingleIngress(serverTokens, "/", serverTokens, f.IngressController.Namespace, "http-svc", 80, nil)) - err = f.WaitForNginxConfiguration( + f.WaitForNginxConfiguration( func(cfg string) bool { return strings.Contains(cfg, "server_tokens off") && strings.Contains(cfg, "more_clear_headers Server;") }) - Expect(err).NotTo(HaveOccurred()) }) It("should exists Server header in the response when is enabled", func() { - err := f.UpdateNginxConfigMapData(serverTokens, "true") - Expect(err).NotTo(HaveOccurred()) + f.UpdateNginxConfigMapData(serverTokens, "true") - ing, err := f.EnsureIngress(&v1beta1.Ingress{ + f.EnsureIngress(&v1beta1.Ingress{ ObjectMeta: metav1.ObjectMeta{ Name: serverTokens, Namespace: f.IngressController.Namespace, @@ -88,13 +81,9 @@ var _ = framework.IngressNginxDescribe("Server Tokens", func() { }, }) - Expect(err).NotTo(HaveOccurred()) - Expect(ing).NotTo(BeNil()) - - err = f.WaitForNginxConfiguration( + f.WaitForNginxConfiguration( func(cfg string) bool { return strings.Contains(cfg, "server_tokens on") }) - Expect(err).NotTo(HaveOccurred()) }) }) diff --git a/test/e2e/settings/tls.go b/test/e2e/settings/tls.go index 0e7333ae9..2247ebff9 100644 --- a/test/e2e/settings/tls.go +++ b/test/e2e/settings/tls.go @@ -34,8 +34,7 @@ var _ = framework.IngressNginxDescribe("Settings - TLS)", func() { host := "settings-tls" BeforeEach(func() { - err := f.NewEchoDeployment() - Expect(err).NotTo(HaveOccurred()) + f.NewEchoDeployment() }) AfterEach(func() { @@ -52,19 +51,15 @@ var _ = framework.IngressNginxDescribe("Settings - TLS)", func() { tlsConfig, err := tlsEndpoint(f, host) Expect(err).NotTo(HaveOccurred()) - err = framework.WaitForTLS(f.IngressController.HTTPSURL, tlsConfig) - Expect(err).NotTo(HaveOccurred()) + framework.WaitForTLS(f.IngressController.HTTPSURL, tlsConfig) By("setting cipher suite") + f.UpdateNginxConfigMapData(sslCiphers, testCiphers) - err = f.UpdateNginxConfigMapData(sslCiphers, testCiphers) - Expect(err).NotTo(HaveOccurred()) - - err = f.WaitForNginxConfiguration( + f.WaitForNginxConfiguration( func(cfg string) bool { return strings.Contains(cfg, fmt.Sprintf("ssl_ciphers '%s';", testCiphers)) }) - Expect(err).NotTo(HaveOccurred()) resp, _, errs := gorequest.New(). Get(f.IngressController.HTTPSURL). @@ -78,15 +73,12 @@ var _ = framework.IngressNginxDescribe("Settings - TLS)", func() { Expect(resp.TLS.CipherSuite).Should(BeNumerically("==", tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384)) By("enforcing TLS v1.0") + f.UpdateNginxConfigMapData(sslProtocols, "TLSv1") - err = f.UpdateNginxConfigMapData(sslProtocols, "TLSv1") - Expect(err).NotTo(HaveOccurred()) - - err = f.WaitForNginxConfiguration( + f.WaitForNginxConfiguration( func(cfg string) bool { return strings.Contains(cfg, "ssl_protocols TLSv1;") }) - Expect(err).NotTo(HaveOccurred()) resp, _, errs = gorequest.New(). Get(f.IngressController.HTTPSURL). @@ -108,19 +100,15 @@ var _ = framework.IngressNginxDescribe("Settings - TLS)", func() { tlsConfig, err := tlsEndpoint(f, host) Expect(err).NotTo(HaveOccurred()) - err = framework.WaitForTLS(f.IngressController.HTTPSURL, tlsConfig) - Expect(err).NotTo(HaveOccurred()) + framework.WaitForTLS(f.IngressController.HTTPSURL, tlsConfig) By("setting max-age parameter") + f.UpdateNginxConfigMapData(hstsMaxAge, "86400") - err = f.UpdateNginxConfigMapData(hstsMaxAge, "86400") - Expect(err).NotTo(HaveOccurred()) - - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return strings.Contains(server, "Strict-Transport-Security: max-age=86400; includeSubDomains\"") }) - Expect(err).NotTo(HaveOccurred()) resp, _, errs := gorequest.New(). Get(f.IngressController.HTTPSURL). @@ -133,15 +121,12 @@ var _ = framework.IngressNginxDescribe("Settings - TLS)", func() { Expect(resp.Header.Get("Strict-Transport-Security")).Should(ContainSubstring("max-age=86400")) By("setting includeSubDomains parameter") + f.UpdateNginxConfigMapData(hstsIncludeSubdomains, "false") - err = f.UpdateNginxConfigMapData(hstsIncludeSubdomains, "false") - Expect(err).NotTo(HaveOccurred()) - - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return strings.Contains(server, "Strict-Transport-Security: max-age=86400\"") }) - Expect(err).NotTo(HaveOccurred()) resp, _, errs = gorequest.New(). Get(f.IngressController.HTTPSURL). @@ -154,15 +139,12 @@ var _ = framework.IngressNginxDescribe("Settings - TLS)", func() { Expect(resp.Header.Get("Strict-Transport-Security")).ShouldNot(ContainSubstring("includeSubDomains")) By("setting preload parameter") + f.UpdateNginxConfigMapData(hstsPreload, "true") - err = f.UpdateNginxConfigMapData(hstsPreload, "true") - Expect(err).NotTo(HaveOccurred()) - - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return strings.Contains(server, "Strict-Transport-Security: max-age=86400; preload\"") }) - Expect(err).NotTo(HaveOccurred()) resp, _, errs = gorequest.New(). Get(f.IngressController.HTTPSURL). @@ -177,11 +159,7 @@ var _ = framework.IngressNginxDescribe("Settings - TLS)", func() { }) func tlsEndpoint(f *framework.Framework, host string) (*tls.Config, error) { - ing, err := f.EnsureIngress(framework.NewSingleIngressWithTLS(host, "/", host, f.IngressController.Namespace, "http-svc", 80, nil)) - if err != nil { - return nil, err - } - + ing := f.EnsureIngress(framework.NewSingleIngressWithTLS(host, "/", host, f.IngressController.Namespace, "http-svc", 80, nil)) return framework.CreateIngressTLSSecret(f.KubeClientSet, ing.Spec.TLS[0].Hosts, ing.Spec.TLS[0].SecretName, diff --git a/test/e2e/ssl/secret_update.go b/test/e2e/ssl/secret_update.go index 9050d37ad..8dded8a45 100644 --- a/test/e2e/ssl/secret_update.go +++ b/test/e2e/ssl/secret_update.go @@ -33,8 +33,7 @@ var _ = framework.IngressNginxDescribe("SSL", func() { f := framework.NewDefaultFramework("ssl") BeforeEach(func() { - err := f.NewEchoDeployment() - Expect(err).NotTo(HaveOccurred()) + f.NewEchoDeployment() }) AfterEach(func() { @@ -43,7 +42,7 @@ var _ = framework.IngressNginxDescribe("SSL", func() { It("should not appear references to secret updates not used in ingress rules", func() { host := "ssl-update" - dummySecret, err := f.EnsureSecret(&v1.Secret{ + dummySecret := f.EnsureSecret(&v1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: "dummy", Namespace: f.IngressController.Namespace, @@ -52,24 +51,20 @@ var _ = framework.IngressNginxDescribe("SSL", func() { "key": []byte("value"), }, }) - Expect(err).NotTo(HaveOccurred()) - ing, err := f.EnsureIngress(framework.NewSingleIngressWithTLS(host, "/", host, f.IngressController.Namespace, "http-svc", 80, nil)) - Expect(err).ToNot(HaveOccurred()) - Expect(ing).ToNot(BeNil()) + ing := f.EnsureIngress(framework.NewSingleIngressWithTLS(host, "/", host, f.IngressController.Namespace, "http-svc", 80, nil)) - _, err = framework.CreateIngressTLSSecret(f.KubeClientSet, + _, err := framework.CreateIngressTLSSecret(f.KubeClientSet, ing.Spec.TLS[0].Hosts, ing.Spec.TLS[0].SecretName, ing.Namespace) Expect(err).ToNot(HaveOccurred()) - err = f.WaitForNginxServer(host, + f.WaitForNginxServer(host, func(server string) bool { return strings.Contains(server, "server_name ssl-update") && strings.Contains(server, "listen 443") }) - Expect(err).ToNot(HaveOccurred()) log, err := f.NginxLogs() Expect(err).ToNot(HaveOccurred()) diff --git a/test/e2e/status/update.go b/test/e2e/status/update.go new file mode 100644 index 000000000..728dbf2fa --- /dev/null +++ b/test/e2e/status/update.go @@ -0,0 +1,141 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package settings + +import ( + "fmt" + "log" + "net" + "strings" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + appsv1beta1 "k8s.io/api/apps/v1beta1" + apiv1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + + "k8s.io/ingress-nginx/test/e2e/framework" +) + +var _ = framework.IngressNginxDescribe("Status Update [Status]", func() { + f := framework.NewDefaultFramework("status-update") + host := "status-update" + address := getHostIP() + + BeforeEach(func() { + }) + + AfterEach(func() { + }) + + It("should update status field after client-go reconnection", func() { + port, cmd, err := f.KubectlProxy(0) + Expect(err).NotTo(HaveOccurred(), "unexpected error starting kubectl proxy") + + err = framework.UpdateDeployment(f.KubeClientSet, f.IngressController.Namespace, "nginx-ingress-controller", 1, + func(deployment *appsv1beta1.Deployment) error { + args := deployment.Spec.Template.Spec.Containers[0].Args + args = append(args, fmt.Sprintf("--apiserver-host=http://%s:%d", address.String(), port)) + args = append(args, "--publish-status-address=1.1.0.0") + // flags --publish-service and --publish-status-address are mutually exclusive + var index int + for k, v := range args { + if strings.Index(v, "--publish-service") != -1 { + index = k + break + } + } + if index > -1 { + args[index] = "" + } + + deployment.Spec.Template.Spec.Containers[0].Args = args + _, err := f.KubeClientSet.AppsV1beta1().Deployments(f.IngressController.Namespace).Update(deployment) + return err + }) + Expect(err).NotTo(HaveOccurred(), "unexpected error updating ingress controller deployment flags") + + f.NewEchoDeploymentWithReplicas(1) + + ing := f.EnsureIngress(framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, nil)) + + f.WaitForNginxConfiguration( + func(cfg string) bool { + return strings.Contains(cfg, fmt.Sprintf("server_name %s", host)) + }) + + framework.Logf("waiting for leader election and initial status update") + time.Sleep(30 * time.Second) + + err = cmd.Process.Kill() + Expect(err).NotTo(HaveOccurred(), "unexpected error terminating kubectl proxy") + + ing, err = f.KubeClientSet.Extensions().Ingresses(f.IngressController.Namespace).Get(host, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred(), "unexpected error getting %s/%v Ingress", f.IngressController.Namespace, host) + + ing.Status.LoadBalancer.Ingress = []apiv1.LoadBalancerIngress{} + _, err = f.KubeClientSet.Extensions().Ingresses(f.IngressController.Namespace).UpdateStatus(ing) + Expect(err).NotTo(HaveOccurred(), "unexpected error cleaning Ingress status") + time.Sleep(10 * time.Second) + + err = f.KubeClientSet.CoreV1(). + ConfigMaps(f.IngressController.Namespace). + Delete("ingress-controller-leader-nginx", &metav1.DeleteOptions{}) + Expect(err).NotTo(HaveOccurred(), "unexpected error deleting leader election configmap") + + _, cmd, err = f.KubectlProxy(port) + Expect(err).NotTo(HaveOccurred(), "unexpected error starting kubectl proxy") + defer func() { + if cmd != nil { + err := cmd.Process.Kill() + Expect(err).NotTo(HaveOccurred(), "unexpected error terminating kubectl proxy") + } + }() + + err = wait.Poll(10*time.Second, time.Minute*3, func() (done bool, err error) { + ing, err = f.KubeClientSet.Extensions().Ingresses(f.IngressController.Namespace).Get(host, metav1.GetOptions{}) + if err != nil { + return false, err + } + + if len(ing.Status.LoadBalancer.Ingress) != 1 { + return false, nil + } + + return true, nil + }) + Expect(err).NotTo(HaveOccurred(), "unexpected error waiting for ingress status") + Expect(ing.Status.LoadBalancer.Ingress).Should(Equal([]apiv1.LoadBalancerIngress{ + {IP: "1.1.0.0"}, + })) + }) +}) + +func getHostIP() net.IP { + conn, err := net.Dial("udp", "8.8.8.8:80") + if err != nil { + log.Fatal(err) + } + defer conn.Close() + + localAddr := conn.LocalAddr().(*net.UDPAddr) + + return localAddr.IP +} diff --git a/test/e2e/up.sh b/test/e2e/up.sh index 470dde0e9..a16b5c73c 100755 --- a/test/e2e/up.sh +++ b/test/e2e/up.sh @@ -22,7 +22,7 @@ if test -e kubectl; then echo "skipping download of kubectl" else echo "downloading kubectl..." - curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/v1.11.0/bin/linux/amd64/kubectl && \ + curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/v1.12.0/bin/linux/amd64/kubectl && \ chmod +x kubectl && sudo mv kubectl /usr/local/bin/ fi @@ -31,6 +31,9 @@ touch ${HOME}/.kube/config export KUBECONFIG=${HOME}/.kube/config echo "starting Kubernetes cluster..." +curl -Lo $DIR/dind-cluster-v1.11.sh https://raw.githubusercontent.com/kubernetes-sigs/kubeadm-dind-cluster/master/fixed/dind-cluster-v1.11.sh && \ + chmod +x $DIR/dind-cluster-v1.11.sh + $DIR/dind-cluster-v1.11.sh up kubectl config use-context dind