developing k6 perftest (#8581)
This commit is contained in:
Long Wu Yuan
GitHub
parent
0ff500c23f
commit
bb1f9deafd
5 changed files with 264 additions and 1 deletions
67
.github/workflows/perftest.yaml
vendored
Normal file
67
.github/workflows/perftest.yaml
vendored
Normal file
|
|
@ -0,0 +1,67 @@
|
||||||
|
name: Performance Test
|
||||||
|
on:
|
||||||
|
workflow_dispatch:
|
||||||
|
inputs:
|
||||||
|
logLevel:
|
||||||
|
description: 'Log level'
|
||||||
|
required: true
|
||||||
|
default: 'warning'
|
||||||
|
tags:
|
||||||
|
description: 'K6 Load Test'
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
k6_test_run:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: Checkout
|
||||||
|
uses: actions/checkout@v1
|
||||||
|
|
||||||
|
- name: Install K6
|
||||||
|
run: |
|
||||||
|
wget https://github.com/grafana/k6/releases/download/v0.38.2/k6-v0.38.2-linux-amd64.tar.gz
|
||||||
|
tar -xvf k6-v0.38.2-linux-amd64.tar.gz k6-v0.38.2-linux-amd64/k6
|
||||||
|
mv k6-v0.38.2-linux-amd64/k6 .
|
||||||
|
./k6
|
||||||
|
|
||||||
|
- name: Make dev-env
|
||||||
|
run: |
|
||||||
|
mkdir $HOME/.kube
|
||||||
|
make dev-env
|
||||||
|
podName=`kubectl -n ingress-nginx get po | grep -i controller | awk '{print $1}'`
|
||||||
|
if [[ -z ${podName} ]] ; then
|
||||||
|
sleep 5
|
||||||
|
fi
|
||||||
|
kubectl wait pod -n ingress-nginx --for condition=Ready $podName
|
||||||
|
kubectl get all -A
|
||||||
|
|
||||||
|
- name: Deploy workload
|
||||||
|
run: |
|
||||||
|
kubectl create deploy k6 --image kennethreitz/httpbin --port 80 && \
|
||||||
|
kubectl expose deploy k6 --port 80 && \
|
||||||
|
kubectl create ing k6 --class nginx \
|
||||||
|
--rule test.ingress-nginx-controller.ga/*=k6:80
|
||||||
|
podName=`kubectl get po | grep -i k6 | awk '{print $1}'`
|
||||||
|
if [[ -z ${podName} ]] ; then
|
||||||
|
sleep 5
|
||||||
|
fi
|
||||||
|
kubectl wait pod --for condition=Ready $podName
|
||||||
|
kubectl get all,secrets,ing
|
||||||
|
|
||||||
|
- name: Tune OS
|
||||||
|
run : |
|
||||||
|
sudo sysctl -A 2>/dev/null | egrep -i "local_port_range|tw_reuse|tcp_timestamps"
|
||||||
|
sudo sh -c "ulimit"
|
||||||
|
sudo sysctl -w net.ipv4.ip_local_port_range="1024 65535"
|
||||||
|
sudo sysctl -w net.ipv4.tcp_tw_reuse=1
|
||||||
|
sudo sysctl -w net.ipv4.tcp_timestamps=1
|
||||||
|
sudo sh -c "ulimit "
|
||||||
|
|
||||||
|
- name: Run smoke test
|
||||||
|
run: |
|
||||||
|
vmstat -at 5 | tee vmstat_report &
|
||||||
|
#./k6 login cloud -t $K6_TOKEN
|
||||||
|
#./k6 run -o cloud ./smoketest.js
|
||||||
|
./k6 run test/k6/smoketest.js
|
||||||
|
pkill vmstat
|
||||||
|
cat vmstat_report
|
||||||
|
|
@ -61,7 +61,7 @@ echo "[dev-env] building image"
|
||||||
make build image
|
make build image
|
||||||
docker tag "${REGISTRY}/controller:${TAG}" "${DEV_IMAGE}"
|
docker tag "${REGISTRY}/controller:${TAG}" "${DEV_IMAGE}"
|
||||||
|
|
||||||
export K8S_VERSION=${K8S_VERSION:-v1.21.1@sha256:69860bda5563ac81e3c0057d654b5253219618a22ec3a346306239bba8cfa1a6}
|
export K8S_VERSION=${K8S_VERSION:-v1.21.12@sha256:f316b33dd88f8196379f38feb80545ef3ed44d9197dca1bfd48bcb1583210207}
|
||||||
|
|
||||||
KIND_CLUSTER_NAME="ingress-nginx-dev"
|
KIND_CLUSTER_NAME="ingress-nginx-dev"
|
||||||
|
|
||||||
|
|
|
||||||
83
test/k6/README.md
Normal file
83
test/k6/README.md
Normal file
|
|
@ -0,0 +1,83 @@
|
||||||
|
# Performance testing ingress-nginx-controller in GithubAction-CI
|
||||||
|
This README will evolve as the development of testing occurs.
|
||||||
|
|
||||||
|
## INFORMATION
|
||||||
|
### 1. No CPU/Memory for stress
|
||||||
|
- Github-Actions job runner is a 2core 7Gig VM so that limits what/how we test https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners
|
||||||
|
|
||||||
|
<img width="863" alt="image" src="https://user-images.githubusercontent.com/5085914/169713673-7daa56ed-dffe-4c49-8600-4e9a01754d37.png">
|
||||||
|
|
||||||
|
- Need to eventually get our own beefy runner, with enough cpu/memory to handle stress level load
|
||||||
|
|
||||||
|
### 2. Scale is work-in-progress
|
||||||
|
- We are grateful to have got a free account on K6.io, as part of their OSS Program. But it is limited to 600 tests per year.
|
||||||
|
|
||||||
|
### 3. No Testplans
|
||||||
|
- Testplan discussion and coding is needed for more practical real-world testing reports
|
||||||
|
|
||||||
|
## DESCRIPTION
|
||||||
|
|
||||||
|
### What
|
||||||
|
- An issue was created for performance tests, for the ingress-nginx-controller builds, https://github.com/kubernetes/ingress-nginx/issues/8033 .
|
||||||
|
|
||||||
|
### How
|
||||||
|
- A step by step guide to using https://k6.io with GithubActions is here https://k6.io/blog/load-testing-using-github-actions/
|
||||||
|
|
||||||
|
- The link above contains sample code
|
||||||
|
<img width="901" alt="image" src="https://user-images.githubusercontent.com/5085914/169714798-6ff62542-a591-4379-8f50-7678b6024936.png">
|
||||||
|
|
||||||
|
|
||||||
|
- Copy sample test code from website and edit to taste
|
||||||
|
<img width="646" alt="image" src="https://user-images.githubusercontent.com/5085914/169714937-1f6f2a86-36c0-4826-8e28-5e450d461353.png">
|
||||||
|
|
||||||
|
- The CI launches a ubuntu environment and uses `make dev-env` to create a kind cluster. The popular https://httpbin.org api docker image is used to create a workload
|
||||||
|
|
||||||
|
<img width="646" alt="image" src="https://user-images.githubusercontent.com/5085914/169714872-613ffd6a-36b5-4317-81fe-c39765a76649.png">
|
||||||
|
|
||||||
|
- We don't want the test to block CI so this syntax from Github-Actions creates a button to run the test
|
||||||
|
<img width="257" alt="image" src="https://user-images.githubusercontent.com/5085914/169715159-e7fbeada-fcb4-4c25-a65f-8be8547b7c19.png">
|
||||||
|
|
||||||
|
- The button looks like this (the `Run Workflow` dropdown at bottom right of screenshot)
|
||||||
|
<img width="1385" alt="image" src="https://user-images.githubusercontent.com/5085914/169715301-0752c5ed-9f84-4560-9a5e-c872c32041de.png">
|
||||||
|
|
||||||
|
<img width="1380" alt="image" src="https://user-images.githubusercontent.com/5085914/169715321-8e76ee6b-2a85-4ed2-ba4e-9d8410518c42.png">
|
||||||
|
|
||||||
|
|
||||||
|
### fqdn
|
||||||
|
- Obtained a freenom domain `ingress-nginx-controller.ga`
|
||||||
|
|
||||||
|
- The test uses a fqdn `test.ingress-nginx-controller.ga`
|
||||||
|
|
||||||
|
- The K6 api has configuration options for dns resolution of (above mentioned fqdn) to localhost/loopback/127.0.0.1 (`make dev-env` cluster)
|
||||||
|
<img width="445" alt="image" src="https://user-images.githubusercontent.com/5085914/169716036-213d43c1-4801-4b4f-aee6-1c11c7812e6d.png">
|
||||||
|
|
||||||
|
- Will need to discuss and decide on fqdn, as it relates to tls secret
|
||||||
|
|
||||||
|
### tls
|
||||||
|
- Procured a letsencrypt wildcard certificate for `*.ingress-nginx-controller.ga`
|
||||||
|
|
||||||
|
- base64 encoded hash of the cert + key is stored in the `Github Project Settings Secrets` as a variable
|
||||||
|
|
||||||
|
- The `GithubActions secrets` variables are decoded in the CI to create the TLS secret
|
||||||
|
|
||||||
|
<img width="1250" alt="image" src="https://user-images.githubusercontent.com/5085914/169716088-030b9f6f-cdb1-470b-b10c-ea4a0fb8199f.png">
|
||||||
|
|
||||||
|
|
||||||
|
### Visualization
|
||||||
|
- Plan is to run tests locally on a kind cluster, in the CI pipeline, but push results to K6-cloud
|
||||||
|
|
||||||
|
- Pushing and visualization on K6 cloud is as simple as executing `k6 run -o cloud test.js`
|
||||||
|
|
||||||
|
- Currently there is a personal account in trial period (50 tests or 1 year limit) bing used
|
||||||
|
|
||||||
|
- Pushing test-results from K6 tests on laptop, to K6-cloud personal trial account on K6-Cloud, to see what the graphs look like
|
||||||
|
|
||||||
|
<img width="954" alt="image" src="https://user-images.githubusercontent.com/5085914/169713896-2cc3b775-38d9-43c6-8792-2a43329a8cfb.png">
|
||||||
|
|
||||||
|
<img width="950" alt="image" src="https://user-images.githubusercontent.com/5085914/169713941-1671426d-9356-4c50-956b-b4003df4aa78.png">
|
||||||
|
|
||||||
|
- The cli result looks like this
|
||||||
|
<img width="835" alt="image" src="https://user-images.githubusercontent.com/5085914/169715209-68aa116a-020b-4f2d-8c8e-ec2c5f68b7b0.png">
|
||||||
|
|
||||||
|
- Before merging the PR, the testing is being done on personal Github project with exact same code as this PR here https://github.com/longwuyuan/k6-loadtest-example/runs/6545706269?check_suite_focus=true
|
||||||
|
|
||||||
49
test/k6/loadtest.js
Normal file
49
test/k6/loadtest.js
Normal file
|
|
@ -0,0 +1,49 @@
|
||||||
|
// This is a loadtest under development
|
||||||
|
// Test here is spec'd to have 100virtual-users
|
||||||
|
// Other specs currently similar to smoktest
|
||||||
|
// But loadtest needs testplan that likely uses auth & data-transfer
|
||||||
|
|
||||||
|
import http from 'k6/http';
|
||||||
|
import { sleep } from 'k6';
|
||||||
|
|
||||||
|
export const options = {
|
||||||
|
hosts: {
|
||||||
|
'test.ingress-nginx-controller.ga:80': '127.0.0.1:80',
|
||||||
|
'test.ingress-nginx-controller.ga:443': '127.0.0.1:443',
|
||||||
|
},
|
||||||
|
duration: '1m',
|
||||||
|
vus: 100,
|
||||||
|
thresholds: {
|
||||||
|
http_req_failed: ['rate<0.01'], // http errors should be less than 1%
|
||||||
|
http_req_duration: ['p(95)<500'], // 95 percent of response times must be below 500ms
|
||||||
|
http_req_duration: ['p(99)<1500'], // 99 percent of response times must be below 1500ms
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
export default function () {
|
||||||
|
const params = {
|
||||||
|
headers: {'host': 'test.ingress-nginx-controller.ga'},
|
||||||
|
};
|
||||||
|
const req1 = {
|
||||||
|
method: 'GET',
|
||||||
|
url: 'http://test.ingress-nginx-controller.ga/ip',
|
||||||
|
};
|
||||||
|
const req2 = {
|
||||||
|
method: 'GET',
|
||||||
|
url: 'http://test.ingress-nginx-controller.ga/image/svg',
|
||||||
|
};
|
||||||
|
const req3 = {
|
||||||
|
params: {
|
||||||
|
headers: {
|
||||||
|
'Content-Type': 'application/x-www-form-urlencoded'
|
||||||
|
},
|
||||||
|
},
|
||||||
|
method: 'POST',
|
||||||
|
url: 'https://test.ingress-nginx-controller.ga/post',
|
||||||
|
body: {
|
||||||
|
hello: 'world!',
|
||||||
|
},
|
||||||
|
};
|
||||||
|
const res = http.batch([req1, req2, req3], params);
|
||||||
|
sleep(1);
|
||||||
|
}
|
||||||
64
test/k6/smoketest.js
Normal file
64
test/k6/smoketest.js
Normal file
|
|
@ -0,0 +1,64 @@
|
||||||
|
// smotest.js edited after copy/pasting from https://k6.io docs
|
||||||
|
// Using this like loadtest because of limited cpu/memory/other
|
||||||
|
|
||||||
|
import http from 'k6/http';
|
||||||
|
import { sleep } from 'k6';
|
||||||
|
|
||||||
|
export const options = {
|
||||||
|
// testbed created with "make dev-env" requires this name resolution
|
||||||
|
// this does not set the host header
|
||||||
|
hosts: {
|
||||||
|
'test.ingress-nginx-controller.ga:80': '127.0.0.1:80',
|
||||||
|
'test.ingress-nginx-controller.ga:443': '127.0.0.1:443',
|
||||||
|
},
|
||||||
|
// below 3 lines documented at https://k6.io
|
||||||
|
duration: '1m',
|
||||||
|
vus: 50,
|
||||||
|
thresholds: {
|
||||||
|
http_req_failed: ['rate<0.01'], // http errors should be less than 1%
|
||||||
|
http_req_duration: ['p(95)<500'], // 95 percent of response times must be below 500ms
|
||||||
|
http_req_duration: ['p(99)<1500'], // 99 percent of response times must be below 1500ms
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
export default function () {
|
||||||
|
// docs of k6 say this is how to adds host header
|
||||||
|
// needed as ingress is created with this host value
|
||||||
|
const params = {
|
||||||
|
headers: {'host': 'test.ingress-nginx-controller.ga'},
|
||||||
|
};
|
||||||
|
// httpbin.org documents these requests
|
||||||
|
const req1 = {
|
||||||
|
method: 'GET',
|
||||||
|
url: 'http://test.ingress-nginx-controller.ga/ip',
|
||||||
|
};
|
||||||
|
const req2 = {
|
||||||
|
method: 'GET',
|
||||||
|
url: 'http://test.ingress-nginx-controller.ga/image/svg',
|
||||||
|
};
|
||||||
|
const req3 = {
|
||||||
|
params: {
|
||||||
|
headers: {
|
||||||
|
'Content-Type': 'application/json'
|
||||||
|
},
|
||||||
|
},
|
||||||
|
method: 'POST',
|
||||||
|
url: 'https://test.ingress-nginx-controller.ga/post',
|
||||||
|
body: {
|
||||||
|
'key1': 'Hello World!',
|
||||||
|
},
|
||||||
|
};
|
||||||
|
const req4 = {
|
||||||
|
method: 'GET',
|
||||||
|
url: 'https://test.ingress-nginx-controller.ga/basic-auth/admin/admin',
|
||||||
|
params: {
|
||||||
|
headers: {
|
||||||
|
'accept': 'application/jsom',
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for(let i=0; i<20; i++){
|
||||||
|
const res = http.batch([req0, req1, req2, req3, req4], params);
|
||||||
|
sleep(1);
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
Reference in a new issue