diff --git a/.github/workflows/perftest.yaml b/.github/workflows/perftest.yaml new file mode 100644 index 000000000..370b37d87 --- /dev/null +++ b/.github/workflows/perftest.yaml @@ -0,0 +1,67 @@ +name: Performance Test +on: + workflow_dispatch: + inputs: + logLevel: + description: 'Log level' + required: true + default: 'warning' + tags: + description: 'K6 Load Test' + +jobs: + k6_test_run: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v1 + + - name: Install K6 + run: | + wget https://github.com/grafana/k6/releases/download/v0.38.2/k6-v0.38.2-linux-amd64.tar.gz + tar -xvf k6-v0.38.2-linux-amd64.tar.gz k6-v0.38.2-linux-amd64/k6 + mv k6-v0.38.2-linux-amd64/k6 . + ./k6 + + - name: Make dev-env + run: | + mkdir $HOME/.kube + make dev-env + podName=`kubectl -n ingress-nginx get po | grep -i controller | awk '{print $1}'` + if [[ -z ${podName} ]] ; then + sleep 5 + fi + kubectl wait pod -n ingress-nginx --for condition=Ready $podName + kubectl get all -A + + - name: Deploy workload + run: | + kubectl create deploy k6 --image kennethreitz/httpbin --port 80 && \ + kubectl expose deploy k6 --port 80 && \ + kubectl create ing k6 --class nginx \ + --rule test.ingress-nginx-controller.ga/*=k6:80 + podName=`kubectl get po | grep -i k6 | awk '{print $1}'` + if [[ -z ${podName} ]] ; then + sleep 5 + fi + kubectl wait pod --for condition=Ready $podName + kubectl get all,secrets,ing + + - name: Tune OS + run : | + sudo sysctl -A 2>/dev/null | egrep -i "local_port_range|tw_reuse|tcp_timestamps" + sudo sh -c "ulimit" + sudo sysctl -w net.ipv4.ip_local_port_range="1024 65535" + sudo sysctl -w net.ipv4.tcp_tw_reuse=1 + sudo sysctl -w net.ipv4.tcp_timestamps=1 + sudo sh -c "ulimit " + + - name: Run smoke test + run: | + vmstat -at 5 | tee vmstat_report & + #./k6 login cloud -t $K6_TOKEN + #./k6 run -o cloud ./smoketest.js + ./k6 run test/k6/smoketest.js + pkill vmstat + cat vmstat_report diff --git a/build/dev-env.sh b/build/dev-env.sh index b2e504b15..c0c99526b 100755 --- a/build/dev-env.sh +++ b/build/dev-env.sh @@ -61,7 +61,7 @@ echo "[dev-env] building image" make build image docker tag "${REGISTRY}/controller:${TAG}" "${DEV_IMAGE}" -export K8S_VERSION=${K8S_VERSION:-v1.21.1@sha256:69860bda5563ac81e3c0057d654b5253219618a22ec3a346306239bba8cfa1a6} +export K8S_VERSION=${K8S_VERSION:-v1.21.12@sha256:f316b33dd88f8196379f38feb80545ef3ed44d9197dca1bfd48bcb1583210207} KIND_CLUSTER_NAME="ingress-nginx-dev" diff --git a/test/k6/README.md b/test/k6/README.md new file mode 100644 index 000000000..81169d3f6 --- /dev/null +++ b/test/k6/README.md @@ -0,0 +1,83 @@ +# Performance testing ingress-nginx-controller in GithubAction-CI +This README will evolve as the development of testing occurs. + +## INFORMATION +### 1. No CPU/Memory for stress +- Github-Actions job runner is a 2core 7Gig VM so that limits what/how we test https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners + + image + + - Need to eventually get our own beefy runner, with enough cpu/memory to handle stress level load + +### 2. Scale is work-in-progress +- We are grateful to have got a free account on K6.io, as part of their OSS Program. But it is limited to 600 tests per year. + +### 3. No Testplans +- Testplan discussion and coding is needed for more practical real-world testing reports + +## DESCRIPTION + +### What +- An issue was created for performance tests, for the ingress-nginx-controller builds, https://github.com/kubernetes/ingress-nginx/issues/8033 . + +### How +- A step by step guide to using https://k6.io with GithubActions is here https://k6.io/blog/load-testing-using-github-actions/ + + - The link above contains sample code + image + + + - Copy sample test code from website and edit to taste + image + + - The CI launches a ubuntu environment and uses `make dev-env` to create a kind cluster. The popular https://httpbin.org api docker image is used to create a workload + + image + + - We don't want the test to block CI so this syntax from Github-Actions creates a button to run the test + image + + - The button looks like this (the `Run Workflow` dropdown at bottom right of screenshot) + image + + image + + +### fqdn +- Obtained a freenom domain `ingress-nginx-controller.ga` + + - The test uses a fqdn `test.ingress-nginx-controller.ga` + + - The K6 api has configuration options for dns resolution of (above mentioned fqdn) to localhost/loopback/127.0.0.1 (`make dev-env` cluster) + image + + - Will need to discuss and decide on fqdn, as it relates to tls secret + +### tls +- Procured a letsencrypt wildcard certificate for `*.ingress-nginx-controller.ga` + + - base64 encoded hash of the cert + key is stored in the `Github Project Settings Secrets` as a variable + + - The `GithubActions secrets` variables are decoded in the CI to create the TLS secret + + image + + +### Visualization +- Plan is to run tests locally on a kind cluster, in the CI pipeline, but push results to K6-cloud + + - Pushing and visualization on K6 cloud is as simple as executing `k6 run -o cloud test.js` + + - Currently there is a personal account in trial period (50 tests or 1 year limit) bing used + + - Pushing test-results from K6 tests on laptop, to K6-cloud personal trial account on K6-Cloud, to see what the graphs look like + + image + + image + + - The cli result looks like this + image + +- Before merging the PR, the testing is being done on personal Github project with exact same code as this PR here https://github.com/longwuyuan/k6-loadtest-example/runs/6545706269?check_suite_focus=true + diff --git a/test/k6/loadtest.js b/test/k6/loadtest.js new file mode 100644 index 000000000..2396948fc --- /dev/null +++ b/test/k6/loadtest.js @@ -0,0 +1,49 @@ +// This is a loadtest under development +// Test here is spec'd to have 100virtual-users +// Other specs currently similar to smoktest +// But loadtest needs testplan that likely uses auth & data-transfer + +import http from 'k6/http'; +import { sleep } from 'k6'; + +export const options = { + hosts: { + 'test.ingress-nginx-controller.ga:80': '127.0.0.1:80', + 'test.ingress-nginx-controller.ga:443': '127.0.0.1:443', + }, + duration: '1m', + vus: 100, + thresholds: { + http_req_failed: ['rate<0.01'], // http errors should be less than 1% + http_req_duration: ['p(95)<500'], // 95 percent of response times must be below 500ms + http_req_duration: ['p(99)<1500'], // 99 percent of response times must be below 1500ms + }, +}; + +export default function () { + const params = { + headers: {'host': 'test.ingress-nginx-controller.ga'}, + }; + const req1 = { + method: 'GET', + url: 'http://test.ingress-nginx-controller.ga/ip', + }; + const req2 = { + method: 'GET', + url: 'http://test.ingress-nginx-controller.ga/image/svg', + }; + const req3 = { + params: { + headers: { + 'Content-Type': 'application/x-www-form-urlencoded' + }, + }, + method: 'POST', + url: 'https://test.ingress-nginx-controller.ga/post', + body: { + hello: 'world!', + }, + }; + const res = http.batch([req1, req2, req3], params); + sleep(1); +} diff --git a/test/k6/smoketest.js b/test/k6/smoketest.js new file mode 100644 index 000000000..b5ab577a9 --- /dev/null +++ b/test/k6/smoketest.js @@ -0,0 +1,64 @@ +// smotest.js edited after copy/pasting from https://k6.io docs +// Using this like loadtest because of limited cpu/memory/other + +import http from 'k6/http'; +import { sleep } from 'k6'; + +export const options = { + // testbed created with "make dev-env" requires this name resolution + // this does not set the host header + hosts: { + 'test.ingress-nginx-controller.ga:80': '127.0.0.1:80', + 'test.ingress-nginx-controller.ga:443': '127.0.0.1:443', + }, + // below 3 lines documented at https://k6.io + duration: '1m', + vus: 50, + thresholds: { + http_req_failed: ['rate<0.01'], // http errors should be less than 1% + http_req_duration: ['p(95)<500'], // 95 percent of response times must be below 500ms + http_req_duration: ['p(99)<1500'], // 99 percent of response times must be below 1500ms + }, +}; + +export default function () { + // docs of k6 say this is how to adds host header + // needed as ingress is created with this host value + const params = { + headers: {'host': 'test.ingress-nginx-controller.ga'}, + }; + // httpbin.org documents these requests + const req1 = { + method: 'GET', + url: 'http://test.ingress-nginx-controller.ga/ip', + }; + const req2 = { + method: 'GET', + url: 'http://test.ingress-nginx-controller.ga/image/svg', + }; + const req3 = { + params: { + headers: { + 'Content-Type': 'application/json' + }, + }, + method: 'POST', + url: 'https://test.ingress-nginx-controller.ga/post', + body: { + 'key1': 'Hello World!', + }, + }; + const req4 = { + method: 'GET', + url: 'https://test.ingress-nginx-controller.ga/basic-auth/admin/admin', + params: { + headers: { + 'accept': 'application/jsom', + } + } + } + for(let i=0; i<20; i++){ + const res = http.batch([req0, req1, req2, req3, req4], params); + sleep(1); + } +}