developing k6 perftest (#8581)

.github/workflows/perftest.yaml

@@ -0,0 +1,67 @@
+name: Performance Test
+on: 
+  workflow_dispatch:
+    inputs:
+      logLevel:
+        description: 'Log level'     
+        required: true
+        default: 'warning'
+      tags:
+        description: 'K6 Load Test'  
+
+jobs:
+  k6_test_run:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v1
+ 
+      - name: Install K6
+        run: |
+          wget https://github.com/grafana/k6/releases/download/v0.38.2/k6-v0.38.2-linux-amd64.tar.gz
+          tar -xvf k6-v0.38.2-linux-amd64.tar.gz k6-v0.38.2-linux-amd64/k6
+          mv k6-v0.38.2-linux-amd64/k6 .
+          ./k6
+
+      - name: Make dev-env
+        run: |
+          mkdir $HOME/.kube
+          make dev-env
+          podName=`kubectl -n ingress-nginx get po | grep -i controller | awk '{print $1}'`
+          if [[ -z ${podName} ]] ; then 
+            sleep 5
+          fi
+          kubectl wait pod -n ingress-nginx --for condition=Ready $podName
+          kubectl get all -A
+
+      - name: Deploy workload
+        run: |
+          kubectl create deploy k6 --image kennethreitz/httpbin --port 80 && \
+            kubectl expose deploy k6 --port 80 && \
+            kubectl create ing k6 --class nginx \
+            --rule test.ingress-nginx-controller.ga/*=k6:80
+          podName=`kubectl get po | grep -i k6 | awk '{print $1}'`
+          if [[ -z ${podName} ]] ; then 
+            sleep 5
+          fi
+          kubectl wait pod --for condition=Ready $podName
+          kubectl get all,secrets,ing
+
+      - name: Tune OS
+        run : |
+          sudo sysctl -A 2>/dev/null | egrep -i "local_port_range|tw_reuse|tcp_timestamps"
+          sudo sh -c "ulimit"
+          sudo sysctl -w net.ipv4.ip_local_port_range="1024 65535"
+          sudo sysctl -w net.ipv4.tcp_tw_reuse=1
+          sudo sysctl -w net.ipv4.tcp_timestamps=1
+          sudo sh -c "ulimit "
+
+      - name: Run smoke test
+        run: |
+          vmstat -at 5 | tee vmstat_report &
+          #./k6 login cloud -t $K6_TOKEN
+          #./k6 run -o cloud ./smoketest.js
+          ./k6 run test/k6/smoketest.js
+          pkill vmstat
+          cat vmstat_report

build/dev-env.sh

@@ -61,7 +61,7 @@ echo "[dev-env] building image"
 make build image
 docker tag "${REGISTRY}/controller:${TAG}" "${DEV_IMAGE}"
 
-export K8S_VERSION=${K8S_VERSION:-v1.21.1@sha256:69860bda5563ac81e3c0057d654b5253219618a22ec3a346306239bba8cfa1a6}
+export K8S_VERSION=${K8S_VERSION:-v1.21.12@sha256:f316b33dd88f8196379f38feb80545ef3ed44d9197dca1bfd48bcb1583210207}
 
 KIND_CLUSTER_NAME="ingress-nginx-dev"
 

test/k6/README.md

@@ -0,0 +1,83 @@
+# Performance testing ingress-nginx-controller in GithubAction-CI
+This README will evolve as the development of testing occurs.
+
+## INFORMATION
+### 1. No CPU/Memory for stress
+- Github-Actions job runner is a 2core 7Gig VM so that limits what/how we test https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners
+
+  <img width="863" alt="image" src="https://user-images.githubusercontent.com/5085914/169713673-7daa56ed-dffe-4c49-8600-4e9a01754d37.png">
+  
+  - Need to eventually get our own beefy runner, with enough cpu/memory to handle stress level load
+
+### 2. Scale is work-in-progress
+- We are grateful to have got a free account on K6.io, as part of their OSS Program. But it is limited to 600 tests per year.
+
+### 3. No Testplans
+- Testplan discussion and coding is needed for more practical real-world testing reports
+
+## DESCRIPTION
+ 
+### What
+- An issue was created for performance tests, for the ingress-nginx-controller builds, https://github.com/kubernetes/ingress-nginx/issues/8033 .
+
+### How
+- A step by step guide to using https://k6.io with GithubActions is here https://k6.io/blog/load-testing-using-github-actions/
+
+  - The link above contains sample code
+    <img width="901" alt="image" src="https://user-images.githubusercontent.com/5085914/169714798-6ff62542-a591-4379-8f50-7678b6024936.png">
+
+
+  - Copy sample test code from website and edit to taste
+   <img width="646" alt="image" src="https://user-images.githubusercontent.com/5085914/169714937-1f6f2a86-36c0-4826-8e28-5e450d461353.png">
+
+  - The CI launches a ubuntu environment and uses `make dev-env` to create a kind cluster. The popular https://httpbin.org api docker image is used to create a workload
+ 
+    <img width="646" alt="image" src="https://user-images.githubusercontent.com/5085914/169714872-613ffd6a-36b5-4317-81fe-c39765a76649.png">
+
+  - We don't want the test to block CI so this syntax from Github-Actions creates a button to run the test
+    <img width="257" alt="image" src="https://user-images.githubusercontent.com/5085914/169715159-e7fbeada-fcb4-4c25-a65f-8be8547b7c19.png">
+
+  - The button looks like this (the `Run Workflow` dropdown at bottom right of screenshot)
+    <img width="1385" alt="image" src="https://user-images.githubusercontent.com/5085914/169715301-0752c5ed-9f84-4560-9a5e-c872c32041de.png">
+
+    <img width="1380" alt="image" src="https://user-images.githubusercontent.com/5085914/169715321-8e76ee6b-2a85-4ed2-ba4e-9d8410518c42.png">
+
+
+### fqdn
+- Obtained a freenom domain `ingress-nginx-controller.ga` 
+
+  - The test uses a fqdn `test.ingress-nginx-controller.ga`
+
+  - The K6 api has configuration options for dns resolution of (above mentioned fqdn) to localhost/loopback/127.0.0.1 (`make dev-env` cluster)
+    <img width="445" alt="image" src="https://user-images.githubusercontent.com/5085914/169716036-213d43c1-4801-4b4f-aee6-1c11c7812e6d.png">
+
+  - Will need to discuss and decide on fqdn, as it relates to tls secret
+
+### tls
+- Procured a letsencrypt wildcard certificate for `*.ingress-nginx-controller.ga`
+
+  - base64 encoded hash of the cert + key is stored in the `Github Project Settings Secrets` as a variable
+
+  - The `GithubActions secrets` variables are decoded in the CI to create the TLS secret
+
+    <img width="1250" alt="image" src="https://user-images.githubusercontent.com/5085914/169716088-030b9f6f-cdb1-470b-b10c-ea4a0fb8199f.png">
+
+
+### Visualization
+- Plan is to run tests locally on a kind cluster, in the CI pipeline, but push results to K6-cloud
+
+  - Pushing and visualization on K6 cloud is as simple as executing `k6 run -o cloud test.js`
+
+  - Currently there is a personal account in trial period (50 tests or 1 year limit) bing used
+
+  - Pushing test-results from K6 tests on laptop, to K6-cloud personal trial account on K6-Cloud, to see what the graphs look like
+
+    <img width="954" alt="image" src="https://user-images.githubusercontent.com/5085914/169713896-2cc3b775-38d9-43c6-8792-2a43329a8cfb.png">
+
+    <img width="950" alt="image" src="https://user-images.githubusercontent.com/5085914/169713941-1671426d-9356-4c50-956b-b4003df4aa78.png">
+
+  - The cli result looks like this
+    <img width="835" alt="image" src="https://user-images.githubusercontent.com/5085914/169715209-68aa116a-020b-4f2d-8c8e-ec2c5f68b7b0.png">
+
+- Before merging the PR, the testing is being done on personal Github project with exact same code as this PR here https://github.com/longwuyuan/k6-loadtest-example/runs/6545706269?check_suite_focus=true
+

test/k6/loadtest.js

@@ -0,0 +1,49 @@
+// This is a loadtest under development
+// Test here is spec'd to have 100virtual-users
+// Other specs currently similar to smoktest
+// But loadtest needs testplan that likely uses auth & data-transfer
+
+import http from 'k6/http';
+import { sleep } from 'k6';
+
+export const options = {
+  hosts: {
+    'test.ingress-nginx-controller.ga:80': '127.0.0.1:80',
+    'test.ingress-nginx-controller.ga:443': '127.0.0.1:443',
+  },
+  duration: '1m',
+  vus: 100,
+  thresholds: {
+    http_req_failed: ['rate<0.01'], // http errors should be less than 1%
+    http_req_duration: ['p(95)<500'], // 95 percent of response times must be below 500ms
+    http_req_duration: ['p(99)<1500'], // 99 percent of response times must be below 1500ms
+  },
+};
+
+export default function () {
+  const params = {
+    headers: {'host': 'test.ingress-nginx-controller.ga'},
+  };
+  const req1 = {
+  	method: 'GET',
+  	url: 'http://test.ingress-nginx-controller.ga/ip',
+  };
+  const req2 = {
+  	method: 'GET',
+  	url: 'http://test.ingress-nginx-controller.ga/image/svg',
+  };
+  const req3 = {
+  	params: {
+  	  headers: {
+        'Content-Type': 'application/x-www-form-urlencoded' 
+      },
+  	},
+  	method: 'POST',
+  	url: 'https://test.ingress-nginx-controller.ga/post',
+  	body: {
+  	  hello: 'world!',
+  	},
+  };
+  const res = http.batch([req1, req2, req3], params);
+  sleep(1);
+}

test/k6/smoketest.js

@@ -0,0 +1,64 @@
+// smotest.js edited after copy/pasting from https://k6.io docs
+// Using this like loadtest because of limited cpu/memory/other
+
+import http from 'k6/http';
+import { sleep } from 'k6';
+
+export const options = {
+  // testbed created with "make dev-env" requires this name resolution
+  // this does not set the host header
+  hosts: {
+    'test.ingress-nginx-controller.ga:80': '127.0.0.1:80',
+    'test.ingress-nginx-controller.ga:443': '127.0.0.1:443',
+  },
+  // below 3 lines documented at https://k6.io
+  duration: '1m',
+  vus: 50,
+  thresholds: {
+    http_req_failed: ['rate<0.01'], // http errors should be less than 1%
+    http_req_duration: ['p(95)<500'], // 95 percent of response times must be below 500ms
+    http_req_duration: ['p(99)<1500'], // 99 percent of response times must be below 1500ms
+  },
+};
+
+export default function () {
+  // docs of k6 say this is how to adds host header
+  // needed as ingress is created with this host value
+  const params = {
+    headers: {'host': 'test.ingress-nginx-controller.ga'},
+  };
+  // httpbin.org documents these requests
+  const req1 = {
+  	method: 'GET',
+  	url: 'http://test.ingress-nginx-controller.ga/ip',
+  };
+  const req2 = {
+  	method: 'GET',
+  	url: 'http://test.ingress-nginx-controller.ga/image/svg',
+  };
+  const req3 = {
+	params: {
+	  headers: {
+	    'Content-Type': 'application/json' 
+      },
+	},
+  	method: 'POST',
+  	url: 'https://test.ingress-nginx-controller.ga/post',
+  	body: {
+  	  'key1': 'Hello World!',
+  	},
+  };
+  const req4 = {
+    method: 'GET',
+    url: 'https://test.ingress-nginx-controller.ga/basic-auth/admin/admin',
+    params: {
+      headers: {
+        'accept': 'application/jsom',
+      }
+    }
+  }
+  for(let i=0; i<20; i++){
+    const res = http.batch([req0, req1, req2, req3, req4], params);
+    sleep(1);
+  }
+}