Add proxy_ssl_* directives

Add support for backends which require client certificate (eg. NiFi)
authentication. The `proxy-ssl-secret` k8s annotation references a
secret which is used to authenticate to the backend server. All other
directives fine tune the backend communication.

The following annotations are supported:
* proxy-ssl-secret
* proxy-ssl-ciphers
* proxy-ssl-protocol
* proxy-ssl-verify
* proxy-ssl-verify-depth
This commit is contained in:
Gabor Lekeny 2019-07-17 02:23:32 +02:00
parent 589c9a20f9
commit def13fc06c
8 changed files with 476 additions and 0 deletions

View file

@ -67,6 +67,11 @@ You can add these Kubernetes annotations to specific Ingress objects to customiz
|[nginx.ingress.kubernetes.io/proxy-redirect-from](#proxy-redirect)|string|
|[nginx.ingress.kubernetes.io/proxy-redirect-to](#proxy-redirect)|string|
|[nginx.ingress.kubernetes.io/proxy-http-version](#proxy-http-version)|"1.0" or "1.1"|
|[nginx.ingress.kubernetes.io/proxy-ssl-secret](#backend-certificate-authentication)|string|
|[nginx.ingress.kubernetes.io/proxy-ssl-ciphers](#backend-certificate-authentication)|string|
|[nginx.ingress.kubernetes.io/proxy-ssl-protocols](#backend-certificate-authentication)|string|
|[nginx.ingress.kubernetes.io/proxy-ssl-verify](#backend-certificate-authentication)|string|
|[nginx.ingress.kubernetes.io/proxy-ssl-verify-depth](#backend-certificate-authentication)|number|
|[nginx.ingress.kubernetes.io/enable-rewrite-log](#enable-rewrite-log)|"true" or "false"|
|[nginx.ingress.kubernetes.io/rewrite-target](#rewrite)|URI|
|[nginx.ingress.kubernetes.io/satisfy](#satisfy)|string|
@ -233,6 +238,21 @@ The annotations are:
Only Authenticated Origin Pulls are allowed and can be configured by following their tutorial: [https://support.cloudflare.com/hc/en-us/articles/204494148-Setting-up-NGINX-to-use-TLS-Authenticated-Origin-Pulls](https://support.cloudflare.com/hc/en-us/articles/204494148-Setting-up-NGINX-to-use-TLS-Authenticated-Origin-Pulls)
### Backend Certificate Authentication
It is possible to authenticate to a proxied HTTPS backend with certificate using additional annotations in Ingress Rule.
* `nginx.ingress.kubernetes.io/proxy-ssl-secret: secretName`:
Specifies a Secret with the certificate `tls.crt`, key `tls.key` in PEM format used for authentication to a proxied HTTPS server. It should also contain trusted CA certificates `ca.crt` in PEM format used to verify the certificate of the proxied HTTPS server.
This annotation also accepts the alternative form "namespace/secretName", in which case the Secret lookup is performed in the referenced namespace instead of the Ingress namespace.
* `nginx.ingress.kubernetes.io/proxy-ssl-verify`:
Enables or disables verification of the proxied HTTPS server certificate. (default: off)
* `nginx.ingress.kubernetes.io/proxy-ssl-verify-depth`:
Sets the verification depth in the proxied HTTPS server certificates chain. (default: 1)
* `nginx.ingress.kubernetes.io/proxy-ssl-ciphers`:
Specifies the enabled [ciphers](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_ssl_ciphers) for requests to a proxied HTTPS server. The ciphers are specified in the format understood by the OpenSSL library.
* `nginx.ingress.kubernetes.io/proxy-ssl-protocols`:
Enables the specified [protocols](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_ssl_protocols) for requests to a proxied HTTPS server.
### Configuration snippet

View file

@ -20,6 +20,7 @@ import (
"github.com/imdario/mergo"
"k8s.io/ingress-nginx/internal/ingress/annotations/canary"
"k8s.io/ingress-nginx/internal/ingress/annotations/modsecurity"
"k8s.io/ingress-nginx/internal/ingress/annotations/proxyssl"
"k8s.io/ingress-nginx/internal/ingress/annotations/sslcipher"
"k8s.io/klog"
@ -87,6 +88,7 @@ type Ingress struct {
EnableGlobalAuth bool
HTTP2PushPreload bool
Proxy proxy.Config
ProxySSL proxyssl.Config
RateLimit ratelimit.Config
Redirect redirect.Config
Rewrite rewrite.Config
@ -132,6 +134,7 @@ func NewAnnotationExtractor(cfg resolver.Resolver) Extractor {
"EnableGlobalAuth": authreqglobal.NewParser(cfg),
"HTTP2PushPreload": http2pushpreload.NewParser(cfg),
"Proxy": proxy.NewParser(cfg),
"ProxySSL": proxyssl.NewParser(cfg),
"RateLimit": ratelimit.NewParser(cfg),
"Redirect": redirect.NewParser(cfg),
"Rewrite": rewrite.NewParser(cfg),

View file

@ -0,0 +1,157 @@
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package proxyssl
import (
"regexp"
"sort"
"strings"
"github.com/pkg/errors"
networking "k8s.io/api/networking/v1beta1"
"k8s.io/ingress-nginx/internal/ingress/annotations/parser"
ing_errors "k8s.io/ingress-nginx/internal/ingress/errors"
"k8s.io/ingress-nginx/internal/ingress/resolver"
"k8s.io/ingress-nginx/internal/k8s"
)
const (
defaultProxySSLCiphers = "DEFAULT"
defaultProxySSLProtocols = "TLSv1 TLSv1.1 TLSv1.2"
defaultProxySSLVerify = "off"
defaultProxySSLVerifyDepth = 1
)
var (
proxySSLOnOffRegex = regexp.MustCompile(`^(on|off)$`)
proxySSLProtocolRegex = regexp.MustCompile(`^(SSLv2|SSLv3|TLSv1|TLSv1\.1|TLSv1\.2|TLSv1\.3)$`)
)
// Config contains the AuthSSLCert used for mutual authentication
// and the configured VerifyDepth
type Config struct {
resolver.AuthSSLCert
Ciphers string `json:"ciphers"`
Protocols string `json:"protocols"`
Verify string `json:"verify"`
VerifyDepth int `json:"verifyDepth"`
}
// Equal tests for equality between two Config types
func (pssl1 *Config) Equal(pssl2 *Config) bool {
if pssl1 == pssl2 {
return true
}
if pssl1 == nil || pssl2 == nil {
return false
}
if !(&pssl1.AuthSSLCert).Equal(&pssl2.AuthSSLCert) {
return false
}
if pssl1.Ciphers != pssl2.Ciphers {
return false
}
if pssl1.Protocols != pssl2.Protocols {
return false
}
if pssl1.Verify != pssl2.Verify {
return false
}
if pssl1.VerifyDepth != pssl2.VerifyDepth {
return false
}
return true
}
// NewParser creates a new TLS authentication annotation parser
func NewParser(resolver resolver.Resolver) parser.IngressAnnotation {
return proxySSL{resolver}
}
type proxySSL struct {
r resolver.Resolver
}
func sortProtocols(protocols string) string {
protolist := strings.Split(protocols, " ")
n := 0
for _, proto := range protolist {
proto = strings.TrimSpace(proto)
if proto == "" || !proxySSLProtocolRegex.MatchString(proto) {
continue
}
protolist[n] = proto
n++
}
if n == 0 {
return defaultProxySSLProtocols
}
protolist = protolist[:n]
sort.Strings(protolist)
return strings.Join(protolist, " ")
}
// Parse parses the annotations contained in the ingress
// rule used to use a Certificate as authentication method
func (p proxySSL) Parse(ing *networking.Ingress) (interface{}, error) {
var err error
config := &Config{}
proxysslsecret, err := parser.GetStringAnnotation("proxy-ssl-secret", ing)
if err != nil {
return &Config{}, err
}
_, _, err = k8s.ParseNameNS(proxysslsecret)
if err != nil {
return &Config{}, ing_errors.NewLocationDenied(err.Error())
}
proxyCert, err := p.r.GetAuthCertificate(proxysslsecret)
if err != nil {
e := errors.Wrap(err, "error obtaining certificate")
return &Config{}, ing_errors.LocationDenied{Reason: e}
}
config.AuthSSLCert = *proxyCert
config.Ciphers, err = parser.GetStringAnnotation("proxy-ssl-ciphers", ing)
if err != nil {
config.Ciphers = defaultProxySSLCiphers
}
config.Protocols, err = parser.GetStringAnnotation("proxy-ssl-protocols", ing)
if err != nil {
config.Protocols = defaultProxySSLProtocols
} else {
config.Protocols = sortProtocols(config.Protocols)
}
config.Verify, err = parser.GetStringAnnotation("proxy-ssl-verify", ing)
if err != nil || !proxySSLOnOffRegex.MatchString(config.Verify) {
config.Verify = defaultProxySSLVerify
}
config.VerifyDepth, err = parser.GetIntAnnotation("proxy-ssl-verify-depth", ing)
if err != nil || config.VerifyDepth == 0 {
config.VerifyDepth = defaultProxySSLVerifyDepth
}
return config, nil
}

View file

@ -0,0 +1,265 @@
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package proxyssl
import (
"testing"
api "k8s.io/api/core/v1"
networking "k8s.io/api/networking/v1beta1"
meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/ingress-nginx/internal/ingress/annotations/parser"
"k8s.io/ingress-nginx/internal/ingress/errors"
"k8s.io/ingress-nginx/internal/ingress/resolver"
)
func buildIngress() *networking.Ingress {
defaultBackend := networking.IngressBackend{
ServiceName: "default-backend",
ServicePort: intstr.FromInt(80),
}
return &networking.Ingress{
ObjectMeta: meta_v1.ObjectMeta{
Name: "foo",
Namespace: api.NamespaceDefault,
},
Spec: networking.IngressSpec{
Backend: &networking.IngressBackend{
ServiceName: "default-backend",
ServicePort: intstr.FromInt(80),
},
Rules: []networking.IngressRule{
{
Host: "foo.bar.com",
IngressRuleValue: networking.IngressRuleValue{
HTTP: &networking.HTTPIngressRuleValue{
Paths: []networking.HTTPIngressPath{
{
Path: "/foo",
Backend: defaultBackend,
},
},
},
},
},
},
},
}
}
// mocks the resolver for proxySSL
type mockSecret struct {
resolver.Mock
}
// GetAuthCertificate from mockSecret mocks the GetAuthCertificate for backend certificate authentication
func (m mockSecret) GetAuthCertificate(name string) (*resolver.AuthSSLCert, error) {
if name != "default/demo-secret" {
return nil, errors.Errorf("there is no secret with name %v", name)
}
return &resolver.AuthSSLCert{
Secret: "default/demo-secret",
CAFileName: "/ssl/ca.crt",
PemSHA: "abc",
}, nil
}
func TestAnnotations(t *testing.T) {
ing := buildIngress()
data := map[string]string{}
data[parser.GetAnnotationWithPrefix("proxy-ssl-secret")] = "default/demo-secret"
data[parser.GetAnnotationWithPrefix("proxy-ssl-ciphers")] = "HIGH:-SHA"
data[parser.GetAnnotationWithPrefix("proxy-ssl-name")] = "$host"
data[parser.GetAnnotationWithPrefix("proxy-ssl-protocols")] = "TLSv1.3 SSLv2 TLSv1 TLSv1.2"
data[parser.GetAnnotationWithPrefix("proxy-ssl-server-name")] = "off"
data[parser.GetAnnotationWithPrefix("proxy-ssl-session-reuse")] = "off"
data[parser.GetAnnotationWithPrefix("proxy-ssl-verify")] = "on"
data[parser.GetAnnotationWithPrefix("proxy-ssl-verify-depth")] = "3"
ing.SetAnnotations(data)
fakeSecret := &mockSecret{}
i, err := NewParser(fakeSecret).Parse(ing)
if err != nil {
t.Errorf("Uxpected error with ingress: %v", err)
}
u, ok := i.(*Config)
if !ok {
t.Errorf("expected *Config but got %v", u)
}
secret, err := fakeSecret.GetAuthCertificate("default/demo-secret")
if err != nil {
t.Errorf("unexpected error getting secret %v", err)
}
if u.AuthSSLCert.Secret != secret.Secret {
t.Errorf("expected %v but got %v", secret.Secret, u.AuthSSLCert.Secret)
}
if u.Ciphers != "HIGH:-SHA" {
t.Errorf("expected %v but got %v", "HIGH:-SHA", u.Ciphers)
}
if u.Protocols != "SSLv2 TLSv1 TLSv1.2 TLSv1.3" {
t.Errorf("expected %v but got %v", "SSLv2 TLSv1 TLSv1.2 TLSv1.3", u.Protocols)
}
if u.Verify != "on" {
t.Errorf("expected %v but got %v", "on", u.Verify)
}
if u.VerifyDepth != 3 {
t.Errorf("expected %v but got %v", 3, u.VerifyDepth)
}
}
func TestInvalidAnnotations(t *testing.T) {
ing := buildIngress()
fakeSecret := &mockSecret{}
data := map[string]string{}
// No annotation
_, err := NewParser(fakeSecret).Parse(ing)
if err == nil {
t.Errorf("Expected error with ingress but got nil")
}
// Invalid NameSpace
data[parser.GetAnnotationWithPrefix("proxy-ssl-secret")] = "demo-secret"
ing.SetAnnotations(data)
_, err = NewParser(fakeSecret).Parse(ing)
if err == nil {
t.Errorf("Expected error with ingress but got nil")
}
// Invalid Proxy Certificate
data[parser.GetAnnotationWithPrefix("proxy-ssl-secret")] = "default/invalid-demo-secret"
ing.SetAnnotations(data)
_, err = NewParser(fakeSecret).Parse(ing)
if err == nil {
t.Errorf("Expected error with ingress but got nil")
}
// Invalid optional Annotations
data[parser.GetAnnotationWithPrefix("proxy-ssl-secret")] = "default/demo-secret"
data[parser.GetAnnotationWithPrefix("proxy-ssl-protocols")] = "TLSv111 SSLv1"
data[parser.GetAnnotationWithPrefix("proxy-ssl-server-name")] = "w00t"
data[parser.GetAnnotationWithPrefix("proxy-ssl-session-reuse")] = "w00t"
data[parser.GetAnnotationWithPrefix("proxy-ssl-verify")] = "w00t"
data[parser.GetAnnotationWithPrefix("proxy-ssl-verify-depth")] = "abcd"
ing.SetAnnotations(data)
i, err := NewParser(fakeSecret).Parse(ing)
if err != nil {
t.Errorf("Uxpected error with ingress: %v", err)
}
u, ok := i.(*Config)
if !ok {
t.Errorf("expected *Config but got %v", u)
}
if u.Protocols != defaultProxySSLProtocols {
t.Errorf("expected %v but got %v", defaultProxySSLProtocols, u.Protocols)
}
if u.Verify != defaultProxySSLVerify {
t.Errorf("expected %v but got %v", defaultProxySSLVerify, u.Verify)
}
if u.VerifyDepth != defaultProxySSLVerifyDepth {
t.Errorf("expected %v but got %v", defaultProxySSLVerifyDepth, u.VerifyDepth)
}
}
func TestEquals(t *testing.T) {
cfg1 := &Config{}
cfg2 := &Config{}
// Same config
result := cfg1.Equal(cfg1)
if result != true {
t.Errorf("Expected true")
}
// compare nil
result = cfg1.Equal(nil)
if result != false {
t.Errorf("Expected false")
}
// Different Certs
sslCert1 := resolver.AuthSSLCert{
Secret: "default/demo-secret",
CAFileName: "/ssl/ca.crt",
PemSHA: "abc",
}
sslCert2 := resolver.AuthSSLCert{
Secret: "default/other-demo-secret",
CAFileName: "/ssl/ca.crt",
PemSHA: "abc",
}
cfg1.AuthSSLCert = sslCert1
cfg2.AuthSSLCert = sslCert2
result = cfg1.Equal(cfg2)
if result != false {
t.Errorf("Expected false")
}
cfg2.AuthSSLCert = sslCert1
// Different Ciphers
cfg1.Ciphers = "DEFAULT"
cfg2.Ciphers = "HIGH:-SHA"
result = cfg1.Equal(cfg2)
if result != false {
t.Errorf("Expected false")
}
cfg2.Ciphers = "DEFAULT"
// Different Protocols
cfg1.Protocols = "SSLv2 TLSv1 TLSv1.2 TLSv1.3"
cfg2.Protocols = "SSLv3 TLSv1 TLSv1.2 TLSv1.3"
result = cfg1.Equal(cfg2)
if result != false {
t.Errorf("Expected false")
}
cfg2.Protocols = "SSLv2 TLSv1 TLSv1.2 TLSv1.3"
// Different Verify
cfg1.Verify = "off"
cfg2.Verify = "on"
result = cfg1.Equal(cfg2)
if result != false {
t.Errorf("Expected false")
}
cfg2.Verify = "off"
// Different VerifyDepth
cfg1.VerifyDepth = 1
cfg2.VerifyDepth = 2
result = cfg1.Equal(cfg2)
if result != false {
t.Errorf("Expected false")
}
cfg2.VerifyDepth = 1
// Equal Configs
result = cfg1.Equal(cfg2)
if result != true {
t.Errorf("Expected true")
}
}

View file

@ -481,6 +481,17 @@ func (n *NGINXController) getBackendServers(ingresses []*ingress.Ingress) ([]*in
server.Hostname, ingKey)
}
if server.ProxySSL.CAFileName == "" {
server.ProxySSL = anns.ProxySSL
if server.ProxySSL.Secret != "" && server.ProxySSL.CAFileName == "" {
klog.V(3).Infof("Secret %q has no 'ca.crt' key, client cert authentication disabled for Ingress %q",
server.ProxySSL.Secret, ingKey)
}
} else {
klog.V(3).Infof("Server %q is already configured for client cert authentication (Ingress %q)",
server.Hostname, ingKey)
}
if rule.HTTP == nil {
klog.V(3).Infof("Ingress %q does not contain any HTTP rule, using default backend", ingKey)
continue

View file

@ -684,6 +684,7 @@ func (s *k8sStore) updateSecretIngressMap(ing *networkingv1beta1.Ingress) {
secretAnnotations := []string{
"auth-secret",
"auth-tls-secret",
"proxy-ssl-secret",
}
for _, ann := range secretAnnotations {
secrKey, err := objectRefAnnotationNsKey(ann, ing)

View file

@ -33,6 +33,7 @@ import (
"k8s.io/ingress-nginx/internal/ingress/annotations/luarestywaf"
"k8s.io/ingress-nginx/internal/ingress/annotations/modsecurity"
"k8s.io/ingress-nginx/internal/ingress/annotations/proxy"
"k8s.io/ingress-nginx/internal/ingress/annotations/proxyssl"
"k8s.io/ingress-nginx/internal/ingress/annotations/ratelimit"
"k8s.io/ingress-nginx/internal/ingress/annotations/redirect"
"k8s.io/ingress-nginx/internal/ingress/annotations/rewrite"
@ -188,6 +189,9 @@ type Server struct {
// CertificateAuth indicates the this server requires mutual authentication
// +optional
CertificateAuth authtls.Config `json:"certificateAuth"`
// ProxySSL indicates the this server uses client certificate to access backends
// +optional
ProxySSL proxyssl.Config `json:"proxySSL"`
// ServerSnippet returns the snippet of server
// +optional
ServerSnippet string `json:"serverSnippet"`
@ -273,6 +277,10 @@ type Location struct {
// to be used in connections against endpoints
// +optional
Proxy proxy.Config `json:"proxy,omitempty"`
// ProxySSL contains information about SSL configuration parameters
// to be used in connections against endpoints
// +optional
ProxySSL proxyssl.Config `json:"proxySSL,omitempty"`
// UsePortInRedirects indicates if redirects must specify the port
// +optional
UsePortInRedirects bool `json:"usePortInRedirects"`

View file

@ -864,6 +864,17 @@ stream {
{{ end }}
{{ end }}
{{ if not (empty $server.ProxySSL.CAFileName) }}
# PEM sha: {{ $server.ProxySSL.PemSHA }}
proxy_ssl_certificate {{ $server.ProxySSL.CAFileName }};
proxy_ssl_certificate_key {{ $server.ProxySSL.CAFileName }};
proxy_ssl_trusted_certificate {{ $server.ProxySSL.CAFileName }};
proxy_ssl_ciphers {{ $server.ProxySSL.Ciphers }};
proxy_ssl_protocols {{ $server.ProxySSL.Protocols }};
proxy_ssl_verify {{ $server.ProxySSL.Verify }};
proxy_ssl_verify_depth {{ $server.ProxySSL.VerifyDepth }};
{{ end }}
{{ if not (empty $server.SSLCiphers) }}
ssl_ciphers {{ $server.SSLCiphers }};
{{ end }}