From 74b66beda9c4a9b8b274656c7040b7701526265f Mon Sep 17 00:00:00 2001 From: Manuel de Brito Fontes Date: Wed, 1 Jun 2016 14:47:37 -0400 Subject: [PATCH] Add support for services running ssl --- controllers/nginx/Changelog.md | 9 +- controllers/nginx/README.md | 240 +----------------- controllers/nginx/controller.go | 20 +- controllers/nginx/nginx/nginx.go | 14 +- .../nginx/nginx/secureupstream/main.go | 50 ++++ .../nginx/nginx/secureupstream/main_test.go | 80 ++++++ controllers/nginx/nginx/template.go | 14 +- controllers/nginx/utils.go | 4 +- 8 files changed, 172 insertions(+), 259 deletions(-) create mode 100644 controllers/nginx/nginx/secureupstream/main.go create mode 100644 controllers/nginx/nginx/secureupstream/main_test.go diff --git a/controllers/nginx/Changelog.md b/controllers/nginx/Changelog.md index 91037b8a3..3c6181adc 100644 --- a/controllers/nginx/Changelog.md +++ b/controllers/nginx/Changelog.md @@ -14,7 +14,12 @@ Changelog - [X] [#1079](https://github.com/kubernetes/contrib/pull/1079) path rewrite - [X] [#1093](https://github.com/kubernetes/contrib/pull/1093) rate limiting - [X] [#1102](https://github.com/kubernetes/contrib/pull/1102) geolocation of traffic in stats +- [X] [#884](https://github.com/kubernetes/contrib/issues/884) support services running ssl +- [X] [#930](https://github.com/kubernetes/contrib/issues/930) detect changes in configuration configmaps + + +TODO + - [ ] [#1063](https://github.com/kubernetes/contrib/pull/1063) watches referenced tls secrets - [ ] [#850](https://github.com/kubernetes/contrib/pull/850) adds configurable SSL redirect nginx controller -- [ ] review docs -- [ ] missing examples + diff --git a/controllers/nginx/README.md b/controllers/nginx/README.md index 663310e10..fef82cf2d 100644 --- a/controllers/nginx/README.md +++ b/controllers/nginx/README.md @@ -198,9 +198,7 @@ Use the [custom-template](examples/custom-template/README.md) example as a guide ### Custom NGINX upstream checks -NGINX exposes some flags in the [upstream configuration](http://nginx.org/en/docs/http/ngx_http_upstream_module.html#upstream) that enables the configuration of each server in the upstream. The ingress controller allows custom `max_fails` and `fail_timeout` parameters in a global context using `upstream-max-fails` or `upstream-fail-timeout` in the NGINX Configmap or in a particular Ingress rule. By default this values are 0. This means NGINX will respect the `readinessProbe`, if is defined. If there is no probe, NGINX will not mark a server inside an upstream down. - -**With the default values NGINX will not health check your backends, and whenever the endpoints controller notices a readiness probe failure that pod's ip will be removed from the list of endpoints, causing nginx to also remove it from the upstreams.** +NGINX exposes some flags in the [upstream configuration](http://nginx.org/en/docs/http/ngx_http_upstream_module.html#upstream) that enabled configuration of each server in the upstream. The ingress controller allows custom `max_fails` and `fail_timeout` parameters in a global context using `upstream-max-fails` or `upstream-fail-timeout` in the NGINX Configmap or in a particular Ingress rule. By default this values are 0. This means NGINX will respect the `livenessProbe`, if is defined. If there is no probe, NGINX will not mark a server inside an upstream down. To use custom values in an Ingress rule define this annotations: @@ -260,239 +258,6 @@ In case of an error in a request the body of the response is obtained from the ` Using this two headers is possible to use a custom backend service like [this one](https://github.com/aledbf/contrib/tree/nginx-debug-server/Ingress/images/nginx-error-server) that inspect each request and returns a custom error page with the format expected by the client. Please check the example [custom-errors](examples/custom-errors/README.md) -### Annotations - -|Annotation |Values|Description| -|---------------------------|------|-----------| -|ingress.kubernetes.io/rewrite-target|URI| | -|ingress.kubernetes.io/add-base-url|true\|false| | -|ingress.kubernetes.io/limit-connections| || -|ingress.kubernetes.io/limit-rps||| -|ingress.kubernetes.io/auth-type|basic or digest|Indicates the [HTTP Authentication Type: Basic or Digest Access Authentication](https://tools.ietf.org/html/rfc2617)|| -|ingress.kubernetes.io/auth-secret|string|Name of the secret that contains the usernames and passwords. -| | |The secret must be created in the same namespace than the Ingress rule|| -|ingress.kubernetes.io/auth-realm|string| | -|ingress.kubernetes.io/upstream-max-fails| | -|ingress.kubernetes.io/upstream-max-fails| | -|ingress.kubernetes.io/upstream-fail-timeout| | - - - -### Custom configuration options - -Running `/nginx-ingress-controller --dump-nginx-configuration` is possible to get the value of the options that can be changed. -The next table shows the options, the default value and a description - -|name |default| -|---------------------------|------| -|body-size|1m| -|custom-http-errors|" "| -|enable-sticky-sessions|"false"| -|enable-vts-status|"false"| -|error-log-level|notice| -|gzip-types|| -|hsts|"true"| -|hsts-include-subdomains|"true"| -|hsts-max-age|"15724800"| -|keep-alive|"75"| -|max-worker-connections|"16384"| -|proxy-connect-timeout|"5"| -|proxy-read-timeout|"60"| -|proxy-real-ip-cidr|0.0.0.0/0| -|proxy-send-timeout|"60"| -|retry-non-idempotent|"false"| -|server-name-hash-bucket-size|"64"| -|server-name-hash-max-size|"512"| -|ssl-buffer-size|4k| -|ssl-ciphers|| -|ssl-protocols|TLSv1 TLSv1.1 TLSv1.2| -|ssl-session-cache|"true"| -|ssl-session-cache-size|10m| -|ssl-session-tickets|"true"| -|ssl-session-timeout|10m| -|use-gzip|"true"| -|use-http2|"true"| -|vts-status-zone-size|10m| -|worker-processes|| - - -**Description:** - -*body-size:* - -Sets the maximum allowed size of the client request body. See NGINX [client_max_body_size](http://nginx.org/en/docs/http/ngx_http_core_module.html#client_max_body_size) - - -*custom-http-errors:* - -Enables which HTTP codes should be passed for processing with the [error_page directive](http://nginx.org/en/docs/http/ngx_http_core_module.html#error_page) -Setting at least one code this also enables [proxy_intercept_errors](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_intercept_errors) (required to process error_page) - - -*enable-sticky-sessions:* - -Enables sticky sessions using cookies. This is provided by [nginx-sticky-module-ng](https://bitbucket.org/nginx-goodies/nginx-sticky-module-ng) module - - -*enable-vts-status:* - -Allows the replacement of the default status page with a third party module named [nginx-module-vts](https://github.com/vozlt/nginx-module-vts) - - -*error-log-level:* - -Configures the logging level of errors. Log levels above are listed in the order of increasing severity -http://nginx.org/en/docs/ngx_core_module.html#error_log - - -*retry-non-idempotent:* - -Since 1.9.13 NGINX will not retry non-idempotent requests (POST, LOCK, PATCH) in case of an error in the upstream server. -The previous behavior can be restored using the value "true" - - -*hsts:* - -Enables or disables the header HSTS in servers running SSL. -HTTP Strict Transport Security (often abbreviated as HSTS) is a security feature (HTTP header) that tell browsers that it should only be communicated with using HTTPS, instead of using HTTP. -https://developer.mozilla.org/en-US/docs/Web/Security/HTTP_strict_transport_security -Why HSTS is important? - - -*hsts-include-subdomains:* - -Enables or disables the use of HSTS in all the subdomains of the servername - - -*hsts-max-age:* - -Sets the time, in seconds, that the browser should remember that this site is only to be accessed using HTTPS. - - -*keep-alive:* - -Sets the time during which a keep-alive client connection will stay open on the server side. -The zero value disables keep-alive client connections -http://nginx.org/en/docs/http/ngx_http_core_module.html#keepalive_timeout - - -*max-worker-connections:* - -Sets the maximum number of simultaneous connections that can be opened by each [worker process](http://nginx.org/en/docs/ngx_core_module.html#worker_connections) - - -*proxy-connect-timeout*: - -Sets the timeout for [establishing a connection with a proxied server](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_connect_timeout). It should be noted that this timeout cannot usually exceed 75 seconds. - - -*proxy-read-timeout:* - -Sets the timeout in seconds for [reading a response from the proxied server](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_read_timeout). The timeout is set only between two successive read operations, not for the transmission of the whole response - - -*proxy-send-timeout:* - -Sets the timeout in seconds for [transmitting a request to the proxied server](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_send_timeout). The timeout is set only between two successive write operations, not for the transmission of the whole request. - - -*resolver:* - -Configures name servers used to [resolve](http://nginx.org/en/docs/http/ngx_http_core_module.html#resolver) names of upstream servers into addresses - - -*server-name-hash-max-size:* - -Sets the maximum size of the [server names hash tables](http://nginx.org/en/docs/http/ngx_http_core_module.html#server_names_hash_max_size) used in server names, map directive’s values, MIME types, names of request header strings, etc. -http://nginx.org/en/docs/hash.html - - -*server-name-hash-bucket-size:* - -Sets the size of the bucker for the server names hash tables -http://nginx.org/en/docs/hash.html -http://nginx.org/en/docs/http/ngx_http_core_module.html#server_names_hash_bucket_size - -*ssl-buffer-size:* - -Sets the size of the [SSL buffer](http://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_buffer_size) used for sending data. -4k helps NGINX to improve TLS Time To First Byte (TTTFB) -https://www.igvita.com/2013/12/16/optimizing-nginx-tls-time-to-first-byte/ - -*ssl-ciphers:* - -Sets the [ciphers](http://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_ciphers) list to enable. The ciphers are specified in the format understood by the OpenSSL library - - -*ssl-dh-param:* - -Base64 string that contains Diffie-Hellman key to help with "Perfect Forward Secrecy" -https://www.openssl.org/docs/manmaster/apps/dhparam.html -https://wiki.mozilla.org/Security/Server_Side_TLS#DHE_handshake_and_dhparam -http://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_dhparam - -*ssl-protocols:* - -Sets the [SSL protocols](http://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_protocols) to use - - -*ssl-session-cache:* - -Enables or disables the use of shared [SSL cache](http://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_session_cache) among worker processes. - - -*ssl-session-cache-size:* - -Sets the size of the [SSL shared session cache](http://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_session_cache) between all worker processes. - - -*ssl-session-tickets:* - -Enables or disables session resumption through [TLS session tickets](http://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_session_tickets) - - -*ssl-session-timeout:* - -Sets the time during which a client may [reuse the session](http://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_session_timeout) parameters stored in a cache. - - -*upstream-max-fails:* - -Sets the number of unsuccessful attempts to communicate with the [server](http://nginx.org/en/docs/http/ngx_http_upstream_module.html#upstream) that should happen in the duration set by the fail_timeout parameter to consider the server unavailable - - -*upstream-fail-timeout:* - -Sets the time during which the specified number of unsuccessful attempts to communicate with the [server](http://nginx.org/en/docs/http/ngx_http_upstream_module.html#upstream) should happen to consider the server unavailable - - -*use-proxy-protocol:* - -Enables or disables the use of the [PROXY protocol](https://www.nginx.com/resources/admin-guide/proxy-protocol/) to receive client connection (real IP address) information passed through proxy servers and load balancers such as HAproxy and Amazon Elastic Load Balancer (ELB). - - -*use-gzip:* - -Enables or disables the use of the nginx module that compresses responses using the ["gzip" module](http://nginx.org/en/docs/http/ngx_http_gzip_module.html) - - -*use-http2:* - -Enables or disables the [HTTP/2](http://nginx.org/en/docs/http/ngx_http_v2_module.html) support in secure connections - - -*gzip-types:* - -MIME types in addition to "text/html" to compress. The special value "*"" matches any MIME type. -Responses with the "text/html" type are always compressed if `use-gzip` is enabled - - -*worker-processes:* - -Sets the number of [worker processes](http://nginx.org/en/docs/ngx_core_module.html#worker_processes). By default "auto" means number of available CPU cores - - - ## Troubleshooting Problems encountered during [1.2.0-alpha7 deployment](https://github.com/kubernetes/kubernetes/blob/master/docs/getting-started-guides/docker.md): @@ -534,5 +299,4 @@ The previous behavior can be restored using `retry-non-idempotent=true` in the c ## Limitations - Ingress rules for TLS require the definition of the field `host` - - +- The IP address in the status of loadBalancer could contain old values diff --git a/controllers/nginx/controller.go b/controllers/nginx/controller.go index 1338426b2..3b0eb562e 100644 --- a/controllers/nginx/controller.go +++ b/controllers/nginx/controller.go @@ -45,6 +45,7 @@ import ( "k8s.io/contrib/ingress/controllers/nginx/nginx/healthcheck" "k8s.io/contrib/ingress/controllers/nginx/nginx/ratelimit" "k8s.io/contrib/ingress/controllers/nginx/nginx/rewrite" + "k8s.io/contrib/ingress/controllers/nginx/nginx/secureupstream" ) const ( @@ -93,7 +94,7 @@ type loadBalancerController struct { ingLister StoreToIngressLister svcLister cache.StoreToServiceLister endpLister cache.StoreToEndpointsLister - mapLister StoreToMapLister + mapLister StoreToConfigmapLister nginx *nginx.Manager podInfo *podInfo defaultSvc string @@ -635,6 +636,11 @@ func (lbc *loadBalancerController) getUpstreamServers(ngxCfg nginx.Configuration glog.V(3).Infof("error reading rate limit annotation in Ingress %v/%v: %v", ing.GetNamespace(), ing.GetName(), err) } + secUpstream, err := secureupstream.ParseAnnotations(ing) + if err != nil { + glog.V(3).Infof("error reading secure upstream in Ingress %v/%v: %v", ing.GetNamespace(), ing.GetName(), err) + } + host := rule.Host if host == "" { host = defServerName @@ -664,6 +670,7 @@ func (lbc *loadBalancerController) getUpstreamServers(ngxCfg nginx.Configuration loc.Upstream = *ups loc.Auth = *nginxAuth loc.RateLimit = *rl + loc.SecureUpstream = secUpstream locRew, err := rewrite.ParseAnnotations(ing) if err != nil { @@ -690,11 +697,12 @@ func (lbc *loadBalancerController) getUpstreamServers(ngxCfg nginx.Configuration } server.Locations = append(server.Locations, &nginx.Location{ - Path: nginxPath, - Upstream: *ups, - Auth: *nginxAuth, - RateLimit: *rl, - Redirect: *locRew, + Path: nginxPath, + Upstream: *ups, + Auth: *nginxAuth, + RateLimit: *rl, + Redirect: *locRew, + SecureUpstream: secUpstream, }) } } diff --git a/controllers/nginx/nginx/nginx.go b/controllers/nginx/nginx/nginx.go index df8268977..b39a245bc 100644 --- a/controllers/nginx/nginx/nginx.go +++ b/controllers/nginx/nginx/nginx.go @@ -34,6 +34,7 @@ type IngressConfig struct { type Upstream struct { Name string Backends []UpstreamServer + Secure bool } // UpstreamByNameServers sorts upstreams by name @@ -91,12 +92,13 @@ func (c ServerByName) Less(i, j int) bool { // Location describes an NGINX location type Location struct { - Path string - IsDefBackend bool - Upstream Upstream - Auth auth.Nginx - RateLimit ratelimit.RateLimit - Redirect rewrite.Redirect + Path string + IsDefBackend bool + Upstream Upstream + Auth auth.Nginx + RateLimit ratelimit.RateLimit + Redirect rewrite.Redirect + SecureUpstream bool } // LocationByPath sorts location by path diff --git a/controllers/nginx/nginx/secureupstream/main.go b/controllers/nginx/nginx/secureupstream/main.go new file mode 100644 index 000000000..89f2674d5 --- /dev/null +++ b/controllers/nginx/nginx/secureupstream/main.go @@ -0,0 +1,50 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package secureupstream + +import ( + "errors" + "strconv" + + "k8s.io/kubernetes/pkg/apis/extensions" +) + +const ( + secureUpstream = "ingress.kubernetes.io/secure-upstream" +) + +type ingAnnotations map[string]string + +func (a ingAnnotations) secureUpstream() bool { + val, ok := a[secureUpstream] + if ok { + if b, err := strconv.ParseBool(val); err == nil { + return b + } + } + return false +} + +// ParseAnnotations parses the annotations contained in the ingress +// rule used to indicate if the upstream servers should use SSL +func ParseAnnotations(ing *extensions.Ingress) (bool, error) { + if ing.GetAnnotations() == nil { + return false, errors.New("no annotations present") + } + + return ingAnnotations(ing.GetAnnotations()).secureUpstream(), nil +} diff --git a/controllers/nginx/nginx/secureupstream/main_test.go b/controllers/nginx/nginx/secureupstream/main_test.go new file mode 100644 index 000000000..5da751b28 --- /dev/null +++ b/controllers/nginx/nginx/secureupstream/main_test.go @@ -0,0 +1,80 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package secureupstream + +import ( + "testing" + + "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/apis/extensions" + "k8s.io/kubernetes/pkg/util/intstr" +) + +func buildIngress() *extensions.Ingress { + defaultBackend := extensions.IngressBackend{ + ServiceName: "default-backend", + ServicePort: intstr.FromInt(80), + } + + return &extensions.Ingress{ + ObjectMeta: api.ObjectMeta{ + Name: "foo", + Namespace: api.NamespaceDefault, + }, + Spec: extensions.IngressSpec{ + Backend: &extensions.IngressBackend{ + ServiceName: "default-backend", + ServicePort: intstr.FromInt(80), + }, + Rules: []extensions.IngressRule{ + { + Host: "foo.bar.com", + IngressRuleValue: extensions.IngressRuleValue{ + HTTP: &extensions.HTTPIngressRuleValue{ + Paths: []extensions.HTTPIngressPath{ + { + Path: "/foo", + Backend: defaultBackend, + }, + }, + }, + }, + }, + }, + }, + } +} + +func TestAnnotations(t *testing.T) { + ing := buildIngress() + data := map[string]string{} + data[secureUpstream] = "true" + ing.SetAnnotations(data) + + su := ingAnnotations(ing.GetAnnotations()).secureUpstream() + if !su { + t.Errorf("Expected true in secure-upstgream but %v was returned", su) + } +} + +func TestWithoutAnnotations(t *testing.T) { + ing := buildIngress() + _, err := ParseAnnotations(ing) + if err == nil { + t.Error("Expected error with ingress without annotations") + } +} diff --git a/controllers/nginx/nginx/template.go b/controllers/nginx/nginx/template.go index 0e12f1d04..60fd1b5e3 100644 --- a/controllers/nginx/nginx/template.go +++ b/controllers/nginx/nginx/template.go @@ -139,8 +139,12 @@ func buildProxyPass(input interface{}) string { path := location.Path + proto := "http" + if location.SecureUpstream { + proto = "https" + } // defProxyPass returns the default proxy_pass, just the name of the upstream - defProxyPass := fmt.Sprintf("proxy_pass http://%s;", location.Upstream.Name) + defProxyPass := fmt.Sprintf("proxy_pass %s://%s;", proto, location.Upstream.Name) // if the path in the ingress rule is equals to the target: no special rewrite if path == location.Redirect.Target { return defProxyPass @@ -169,14 +173,14 @@ func buildProxyPass(input interface{}) string { return fmt.Sprintf(` rewrite %s / break; rewrite %s(.*) /$1 break; - proxy_pass http://%s; - %v`, location.Path, path, location.Upstream.Name, abu) + proxy_pass %s://%s; + %v`, location.Path, path, proto, location.Upstream.Name, abu) } return fmt.Sprintf(` rewrite %s(.*) %s/$1 break; - proxy_pass http://%s; - %v`, path, location.Redirect.Target, location.Upstream.Name, abu) + proxy_pass %s://%s; + %v`, path, location.Redirect.Target, proto, location.Upstream.Name, abu) } // default proxy_pass diff --git a/controllers/nginx/utils.go b/controllers/nginx/utils.go index ecf8ebea3..dcca0ea74 100644 --- a/controllers/nginx/utils.go +++ b/controllers/nginx/utils.go @@ -37,8 +37,8 @@ type StoreToIngressLister struct { cache.Store } -// StoreToMapLister makes a Store that lists Secrets. -type StoreToMapLister struct { +// StoreToConfigmapLister makes a Store that lists Configmap. +type StoreToConfigmapLister struct { cache.Store }