Merge pull request #2643 from aledbf/remove-vts
Remove VTS from the ingress controller
This commit is contained in:
k8s-ci-robot
GitHub
commit
3cbd2d66bf
18 changed files with 1 additions and 250 deletions
|
|
@ -42,6 +42,3 @@ $ kubectl exec nginx-ingress-controller-v1ppm cat /etc/nginx/nginx.conf
|
|||
}
|
||||
....
|
||||
```
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
Binary file not shown.
|
Before Width: | Height: | Size: 59 KiB |
|
|
@ -1,103 +0,0 @@
|
|||
# Custom VTS metrics with Prometheus
|
||||
|
||||
This example aims to demonstrate the deployment of an nginx ingress controller and use a ConfigMap to enable [nginx vts module](https://github.com/vozlt/nginx-module-vts
|
||||
) to export metrics in prometheus format.
|
||||
|
||||
## vts-metrics
|
||||
|
||||
Vts-metrics export NGINX metrics. To deploy all the files simply run `kubectl apply -f nginx`. A deployment and service will be
|
||||
created which already has a `prometheus.io/scrape: 'true'` annotation and if you added
|
||||
the recommended Prometheus service-endpoint scraping [configuration](https://raw.githubusercontent.com/prometheus/prometheus/master/documentation/examples/prometheus-kubernetes.yml),
|
||||
Prometheus will scrape it automatically and you start using the generated metrics right away.
|
||||
|
||||
## Custom configuration
|
||||
|
||||
```console
|
||||
apiVersion: v1
|
||||
data:
|
||||
enable-vts-status: "true"
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: nginx-configuration
|
||||
namespace: ingress-nginx
|
||||
labels:
|
||||
app: ingress-nginx
|
||||
```
|
||||
|
||||
```console
|
||||
$ kubectl apply -f nginx-vts-metrics-conf.yaml
|
||||
```
|
||||
|
||||
## Result
|
||||
|
||||
Check whether the ingress controller successfully generated the NGINX vts status:
|
||||
|
||||
```console
|
||||
$ kubectl exec nginx-ingress-controller-873061567-4n3k2 -n ingress-nginx cat /etc/nginx/nginx.conf|grep vhost_traffic_status_display
|
||||
vhost_traffic_status_display;
|
||||
vhost_traffic_status_display_format html;
|
||||
```
|
||||
|
||||
### NGINX vts dashboard
|
||||
|
||||
The vts dashboard provides real time metrics.
|
||||
|
||||
|
||||
|
||||
Because the vts port it's not yet exposed, you should forward the controller port to see it.
|
||||
|
||||
```console
|
||||
$ kubectl port-forward $(kubectl get pods --selector=k8s-app=nginx-ingress-controller -n ingress-nginx --output=jsonpath={.items..metadata.name}) -n ingress-nginx 18080
|
||||
```
|
||||
|
||||
Now open the url [http://localhost:18080/nginx_status](http://localhost:18080/nginx_status) in your browser.
|
||||
|
||||
### Prometheus metrics output
|
||||
|
||||
NGINX Ingress controller already has a parser to convert vts metrics to Prometheus format. It exports prometheus metrics to the address `:10254/metrics`.
|
||||
|
||||
```console
|
||||
$ kubectl exec -ti -n ingress-nginx $(kubectl get pods --selector=k8s-app=nginx-ingress-controller -n kube-system --output=jsonpath={.items..metadata.name}) curl localhost:10254/metrics
|
||||
ingress_controller_ssl_expire_time_seconds{host="foo.bar.com"} -6.21355968e+10
|
||||
# HELP ingress_controller_success Cumulative number of Ingress controller reload operations
|
||||
# TYPE ingress_controller_success counter
|
||||
ingress_controller_success{count="reloads"} 3
|
||||
# HELP nginx_bytes_total Nginx bytes count
|
||||
# TYPE nginx_bytes_total counter
|
||||
nginx_bytes_total{direction="in",ingress_class="nginx",namespace="",server_zone="*"} 3708
|
||||
nginx_bytes_total{direction="in",ingress_class="nginx",namespace="",server_zone="_"} 3708
|
||||
nginx_bytes_total{direction="out",ingress_class="nginx",namespace="",server_zone="*"} 5256
|
||||
nginx_bytes_total{direction="out",ingress_class="nginx",namespace="",server_zone="_"} 5256
|
||||
```
|
||||
|
||||
### Customize metrics
|
||||
|
||||
The default [vts vhost key](https://github.com/vozlt/nginx-module-vts#vhost_traffic_status_filter_by_set_key) is `$geoip_country_code country::*` that expose metrics grouped by server and country code. The example below show how to have metrics grouped by server and server path.
|
||||
|
||||
|
||||
|
||||
## NGINX custom configuration ( http level )
|
||||
|
||||
```
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
data:
|
||||
enable-vts-status: "true"
|
||||
vts-default-filter-key: "$server_name"
|
||||
...
|
||||
```
|
||||
|
||||
## Customize ingress
|
||||
|
||||
```
|
||||
apiVersion: extensions/v1beta1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
annotations:
|
||||
nginx.ingress.kubernetes.io/vts-filter-key: $uri $server_name
|
||||
name: ingress
|
||||
```
|
||||
|
||||
## Result
|
||||
|
||||
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 969 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 451 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 244 KiB |
|
|
@ -1,9 +0,0 @@
|
|||
apiVersion: v1
|
||||
data:
|
||||
enable-vts-status: "true"
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: nginx-configuration
|
||||
namespace: ingress-nginx
|
||||
labels:
|
||||
app: ingress-nginx
|
||||
|
|
@ -16,7 +16,6 @@ Customization | [Custom DH parameters for perfect forward secrecy](customization
|
|||
Customization | [Custom errors](customization/custom-errors/README.md) | serve custom error pages from the default backend | Intermediate
|
||||
Customization | [Custom headers](customization/custom-headers/README.md) | set custom headers before sending traffic to backends | Advanced
|
||||
Customization | [Custom upstream check](customization/custom-upstream-check/README.md) | TODO | TODO
|
||||
Customization | [Custom VTS metrics with Prometheus](customization/custom-vts-metrics-prometheus/README.md) | TODO | TODO
|
||||
Customization | [External authentication with response header propagation](customization/external-auth-headers/README.md) | TODO | TODO
|
||||
Customization | [Sysctl tuning](customization/sysctl/README.md) | TODO | TODO
|
||||
Features | [Rewrite](rewrite/README.md) | TODO | TODO
|
||||
|
|
|
|||
|
|
@ -44,10 +44,6 @@ The following table shows a configuration option's name, type, and the default v
|
|||
|[disable-ipv6-dns](#disable-ipv6-dns)|bool|false|
|
||||
|[enable-underscores-in-headers](#enable-underscores-in-headers)|bool|false|
|
||||
|[ignore-invalid-headers](#ignore-invalid-headers)|bool|true|
|
||||
|[enable-vts-status](#enable-vts-status)|bool|false|
|
||||
|[vts-status-zone-size](#vts-status-zone-size)|string|"10m"|
|
||||
|[vts-sum-key](#vts-sum-key)|string|"*"|
|
||||
|[vts-default-filter-key](#vts-default-filter-key)|string|"$geoip_country_code country::*"|
|
||||
|[retry-non-idempotent](#retry-non-idempotent)|bool|"false"|
|
||||
|[error-log-level](#error-log-level)|string|"notice"|
|
||||
|[http2-max-field-size](#http2-max-field-size)|string|"4k"|
|
||||
|
|
@ -240,32 +236,6 @@ Enables underscores in header names. _**default:**_ is disabled
|
|||
Set if header fields with invalid names should be ignored.
|
||||
_**default:**_ is enabled
|
||||
|
||||
## enable-vts-status
|
||||
|
||||
Allows the replacement of the default status page with a third party module named [nginx-module-vts](https://github.com/vozlt/nginx-module-vts).
|
||||
_**default:**_ is disabled
|
||||
|
||||
## vts-status-zone-size
|
||||
|
||||
Vts config on http level sets parameters for a shared memory zone that will keep states for various keys. The cache is shared between all worker processes. _**default:**_ 10m
|
||||
|
||||
_References:_
|
||||
[https://github.com/vozlt/nginx-module-vts#vhost_traffic_status_zone](https://github.com/vozlt/nginx-module-vts#vhost_traffic_status_zone)
|
||||
|
||||
## vts-default-filter-key
|
||||
|
||||
Vts config on http level enables the keys by user defined variable. The key is a key string to calculate traffic. The name is a group string to calculate traffic. The key and name can contain variables such as $host, $server_name. The name's group belongs to filterZones if specified. The key's group belongs to serverZones if not specified second argument name. _**default:**_ $geoip_country_code country::*
|
||||
|
||||
_References:_
|
||||
[https://github.com/vozlt/nginx-module-vts#vhost_traffic_status_filter_by_set_key](https://github.com/vozlt/nginx-module-vts#vhost_traffic_status_filter_by_set_key)
|
||||
|
||||
## vts-sum-key
|
||||
|
||||
For metrics keyed (or when using Prometheus, labeled) by server zone, this value is used to indicate metrics for all server zones combined. _**default:**_ *
|
||||
|
||||
_References:_
|
||||
[https://github.com/vozlt/nginx-module-vts#vhost_traffic_status_display_sum_key](https://github.com/vozlt/nginx-module-vts#vhost_traffic_status_display_sum_key)
|
||||
|
||||
## retry-non-idempotent
|
||||
|
||||
Since 1.9.13 NGINX will not retry non-idempotent requests (POST, LOCK, PATCH) in case of an error in the upstream server. The previous behavior can be restored using the value "true".
|
||||
|
|
|
|||
|
|
@ -2,10 +2,3 @@
|
|||
|
||||
The [ngx_http_stub_status_module](http://nginx.org/en/docs/http/ngx_http_stub_status_module.html) module provides access to basic status information.
|
||||
This is the default module active in the url `/nginx_status` in the status port (default is 18080).
|
||||
|
||||
This controller provides an alternative to this module using the [nginx-module-vts](https://github.com/vozlt/nginx-module-vts) module.
|
||||
To use this module just set in the configuration configmap `enable-vts-status: "true"`.
|
||||
|
||||
|
||||
|
||||
To extract the information in JSON format the module provides a custom URL: `/nginx_status/format/json`
|
||||
|
|
|
|||
|
|
@ -6,7 +6,6 @@ nginx [engine x] is an HTTP and reverse proxy server, a mail proxy server, and a
|
|||
This custom nginx image contains:
|
||||
|
||||
- [stream](http://nginx.org/en/docs/stream/ngx_stream_core_module.html) tcp support for upstreams
|
||||
- nginx stats [nginx-module-vts](https://github.com/vozlt/nginx-module-vts)
|
||||
- [Dynamic TLS record sizing](https://blog.cloudflare.com/optimizing-tls-over-tcp-to-reduce-latency/)
|
||||
- [ngx_devel_kit](https://github.com/simpl/ngx_devel_kit)
|
||||
- [set-misc-nginx-module](https://github.com/openresty/set-misc-nginx-module)
|
||||
|
|
|
|||
|
|
@ -54,7 +54,6 @@ import (
|
|||
"k8s.io/ingress-nginx/internal/ingress/annotations/sslpassthrough"
|
||||
"k8s.io/ingress-nginx/internal/ingress/annotations/upstreamhashby"
|
||||
"k8s.io/ingress-nginx/internal/ingress/annotations/upstreamvhost"
|
||||
"k8s.io/ingress-nginx/internal/ingress/annotations/vtsfilterkey"
|
||||
"k8s.io/ingress-nginx/internal/ingress/annotations/xforwardedprefix"
|
||||
"k8s.io/ingress-nginx/internal/ingress/errors"
|
||||
"k8s.io/ingress-nginx/internal/ingress/resolver"
|
||||
|
|
@ -90,7 +89,6 @@ type Ingress struct {
|
|||
UpstreamHashBy string
|
||||
LoadBalancing string
|
||||
UpstreamVhost string
|
||||
VtsFilterKey string
|
||||
Whitelist ipwhitelist.SourceRange
|
||||
XForwardedPrefix bool
|
||||
SSLCiphers string
|
||||
|
|
@ -132,7 +130,6 @@ func NewAnnotationExtractor(cfg resolver.Resolver) Extractor {
|
|||
"UpstreamHashBy": upstreamhashby.NewParser(cfg),
|
||||
"LoadBalancing": loadbalancing.NewParser(cfg),
|
||||
"UpstreamVhost": upstreamvhost.NewParser(cfg),
|
||||
"VtsFilterKey": vtsfilterkey.NewParser(cfg),
|
||||
"Whitelist": ipwhitelist.NewParser(cfg),
|
||||
"XForwardedPrefix": xforwardedprefix.NewParser(cfg),
|
||||
"SSLCiphers": sslcipher.NewParser(cfg),
|
||||
|
|
|
|||
|
|
@ -1,40 +0,0 @@
|
|||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package vtsfilterkey
|
||||
|
||||
import (
|
||||
extensions "k8s.io/api/extensions/v1beta1"
|
||||
|
||||
"k8s.io/ingress-nginx/internal/ingress/annotations/parser"
|
||||
"k8s.io/ingress-nginx/internal/ingress/resolver"
|
||||
)
|
||||
|
||||
type vtsFilterKey struct {
|
||||
r resolver.Resolver
|
||||
}
|
||||
|
||||
// NewParser creates a new vts filter key annotation parser
|
||||
func NewParser(r resolver.Resolver) parser.IngressAnnotation {
|
||||
return vtsFilterKey{r}
|
||||
}
|
||||
|
||||
// Parse parses the annotations contained in the ingress rule
|
||||
// used to indicate if the location/s contains a fragment of
|
||||
// configuration to be included inside the paths of the rules
|
||||
func (a vtsFilterKey) Parse(ing *extensions.Ingress) (interface{}, error) {
|
||||
return parser.GetStringAnnotation("vts-filter-key", ing)
|
||||
}
|
||||
|
|
@ -161,31 +161,6 @@ type Configuration struct {
|
|||
// By default this is enabled
|
||||
IgnoreInvalidHeaders bool `json:"ignore-invalid-headers"`
|
||||
|
||||
// EnableVtsStatus allows the replacement of the default status page with a third party module named
|
||||
// nginx-module-vts - https://github.com/vozlt/nginx-module-vts
|
||||
// By default this is disabled
|
||||
EnableVtsStatus bool `json:"enable-vts-status,omitempty"`
|
||||
|
||||
// Vts config on http level
|
||||
// Description: Sets parameters for a shared memory zone that will keep states for various keys. The cache is shared between all worker processe
|
||||
// https://github.com/vozlt/nginx-module-vts#vhost_traffic_status_zone
|
||||
// Default value is 10m
|
||||
VtsStatusZoneSize string `json:"vts-status-zone-size,omitempty"`
|
||||
|
||||
// Vts config on http level
|
||||
// Description: Enables the keys by user defined variable. The key is a key string to calculate traffic.
|
||||
// The name is a group string to calculate traffic. The key and name can contain variables such as $host,
|
||||
// $server_name. The name's group belongs to filterZones if specified. The key's group belongs to serverZones
|
||||
// if not specified second argument name. The example with geoip module is as follows:
|
||||
// https://github.com/vozlt/nginx-module-vts#vhost_traffic_status_filter_by_set_key
|
||||
// Default value is $geoip_country_code country::*
|
||||
VtsDefaultFilterKey string `json:"vts-default-filter-key,omitempty"`
|
||||
|
||||
// Description: Sets sum key used by vts json output, and the sum label in prometheus output.
|
||||
// These indicate metrics values for all server zones combined, rather than for a specific one.
|
||||
// Default value is *
|
||||
VtsSumKey string `json:"vts-sum-key,omitempty"`
|
||||
|
||||
// RetryNonIdempotent since 1.9.13 NGINX will not retry non-idempotent requests (POST, LOCK, PATCH)
|
||||
// in case of an error. The previous behavior can be restored using the value true
|
||||
RetryNonIdempotent bool `json:"retry-non-idempotent"`
|
||||
|
|
@ -603,9 +578,6 @@ func NewDefault() Configuration {
|
|||
WorkerProcesses: strconv.Itoa(runtime.NumCPU()),
|
||||
WorkerShutdownTimeout: "10s",
|
||||
LoadBalanceAlgorithm: defaultLoadBalancerAlgorithm,
|
||||
VtsStatusZoneSize: "10m",
|
||||
VtsDefaultFilterKey: "$geoip_country_code country::*",
|
||||
VtsSumKey: "*",
|
||||
VariablesHashBucketSize: 128,
|
||||
VariablesHashMaxSize: 2048,
|
||||
UseHTTP2: true,
|
||||
|
|
|
|||
|
|
@ -437,7 +437,6 @@ func (n *NGINXController) getBackendServers(ingresses []*extensions.Ingress) ([]
|
|||
loc.Redirect = anns.Redirect
|
||||
loc.Rewrite = anns.Rewrite
|
||||
loc.UpstreamVhost = anns.UpstreamVhost
|
||||
loc.VtsFilterKey = anns.VtsFilterKey
|
||||
loc.Whitelist = anns.Whitelist
|
||||
loc.Denied = anns.Denied
|
||||
loc.XForwardedPrefix = anns.XForwardedPrefix
|
||||
|
|
@ -476,7 +475,6 @@ func (n *NGINXController) getBackendServers(ingresses []*extensions.Ingress) ([]
|
|||
Redirect: anns.Redirect,
|
||||
Rewrite: anns.Rewrite,
|
||||
UpstreamVhost: anns.UpstreamVhost,
|
||||
VtsFilterKey: anns.VtsFilterKey,
|
||||
Whitelist: anns.Whitelist,
|
||||
Denied: anns.Denied,
|
||||
XForwardedPrefix: anns.XForwardedPrefix,
|
||||
|
|
@ -913,7 +911,6 @@ func (n *NGINXController) createServers(data []*extensions.Ingress,
|
|||
// defLoc.Redirect = anns.Redirect
|
||||
// defLoc.Rewrite = anns.Rewrite
|
||||
defLoc.UpstreamVhost = anns.UpstreamVhost
|
||||
defLoc.VtsFilterKey = anns.VtsFilterKey
|
||||
defLoc.Whitelist = anns.Whitelist
|
||||
defLoc.Denied = anns.Denied
|
||||
defLoc.GRPC = anns.GRPC
|
||||
|
|
|
|||
|
|
@ -242,10 +242,6 @@ type Location struct {
|
|||
// UsePortInRedirects indicates if redirects must specify the port
|
||||
// +optional
|
||||
UsePortInRedirects bool `json:"usePortInRedirects"`
|
||||
// VtsFilterKey contains the vts filter key on the location level
|
||||
// https://github.com/vozlt/nginx-module-vts#vhost_traffic_status_filter_by_set_key
|
||||
// +optional
|
||||
VtsFilterKey string `json:"vtsFilterKey,omitempty"`
|
||||
// ConfigurationSnippet contains additional configuration for the backend
|
||||
// to be considered in the configuration of the location
|
||||
ConfigurationSnippet string `json:"configurationSnippet"`
|
||||
|
|
|
|||
|
|
@ -106,11 +106,6 @@ http {
|
|||
geoip_proxy_recursive on;
|
||||
{{ end }}
|
||||
|
||||
{{ if $cfg.EnableVtsStatus }}
|
||||
vhost_traffic_status_zone shared:vhost_traffic_status:{{ $cfg.VtsStatusZoneSize }};
|
||||
vhost_traffic_status_filter_by_set_key {{ $cfg.VtsDefaultFilterKey }};
|
||||
{{ end }}
|
||||
|
||||
aio threads;
|
||||
aio_write on;
|
||||
|
||||
|
|
@ -543,14 +538,8 @@ http {
|
|||
opentracing off;
|
||||
{{ end }}
|
||||
|
||||
{{ if $cfg.EnableVtsStatus }}
|
||||
vhost_traffic_status_display;
|
||||
vhost_traffic_status_display_format html;
|
||||
vhost_traffic_status_display_sum_key {{ $cfg.VtsSumKey }};
|
||||
{{ else }}
|
||||
access_log off;
|
||||
stub_status on;
|
||||
{{ end }}
|
||||
}
|
||||
|
||||
{{ if $all.DynamicConfigurationEnabled }}
|
||||
|
|
@ -935,8 +924,6 @@ stream {
|
|||
|
||||
port_in_redirect {{ if $location.UsePortInRedirects }}on{{ else }}off{{ end }};
|
||||
|
||||
{{ if $all.Cfg.EnableVtsStatus }}{{ if $location.VtsFilterKey }} vhost_traffic_status_filter_by_set_key {{ $location.VtsFilterKey }};{{ end }}{{ end }}
|
||||
|
||||
set $proxy_upstream_name "{{ buildUpstreamName $server.Hostname $all.Backends $location $all.DynamicConfigurationEnabled }}";
|
||||
|
||||
{{/* redirect to HTTPS can be achieved forcing the redirect or having a SSL Certificate configured for the server */}}
|
||||
|
|
|
|||
|
|
@ -21,7 +21,6 @@
|
|||
"bodySize": "1m",
|
||||
"enableDynamicTlsRecords": true,
|
||||
"enableSpdy": false,
|
||||
"enableVtsStatus": true,
|
||||
"errorLogLevel": "notice",
|
||||
"gzipTypes": "application/atom+xml application/javascript application/x-javascript application/json application/rss+xml application/vnd.ms-fontobject application/x-font-ttf application/x-web-app-manifest+json application/xhtml+xml application/xml font/opentype image/svg+xml image/x-icon text/css text/plain text/x-component",
|
||||
"hsts": true,
|
||||
|
|
@ -46,7 +45,6 @@
|
|||
"useGzip": true,
|
||||
"useHttp2": true,
|
||||
"proxyStreamTimeout": "600s",
|
||||
"vtsStatusZoneSize": "10m",
|
||||
"workerProcesses": 1,
|
||||
"limitConnZoneVariable": "$the_real_ip"
|
||||
},
|
||||
|
|
@ -117,9 +115,7 @@
|
|||
"keyFilename": "",
|
||||
"caFilename": "",
|
||||
"pemSha": ""
|
||||
},
|
||||
"vtsDefaultFilterKey": "$uri $server_name"
|
||||
|
||||
}
|
||||
}, {
|
||||
"path": "/",
|
||||
"isDefBackend": true,
|
||||
|
|
|
|||
Loading…
Reference in a new issue