Adding ipdenylist annotation (#8795)

* feat: Add support for IP Deny List

* fixed gomod

* Update package

* go mod tidy

* Revert "go mod tidy"

This reverts commit e6a837e1e7.

* update ginko version

* Updates e2e tests

* fix test typo
This commit is contained in:
Phil Nichol 2023-01-08 22:43:28 +00:00 committed by GitHub
parent bbf7c79f96
commit 8ed3a27e25
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
15 changed files with 3568 additions and 0 deletions

View file

@ -109,6 +109,7 @@ You can add these Kubernetes annotations to specific Ingress objects to customiz
|[nginx.ingress.kubernetes.io/x-forwarded-prefix](#x-forwarded-prefix-header)|string|
|[nginx.ingress.kubernetes.io/load-balance](#custom-nginx-load-balancing)|string|
|[nginx.ingress.kubernetes.io/upstream-vhost](#custom-nginx-upstream-vhost)|string|
|[nginx.ingress.kubernetes.io/denylist-source-range](#denylist-source-range)|CIDR|
|[nginx.ingress.kubernetes.io/whitelist-source-range](#whitelist-source-range)|CIDR|
|[nginx.ingress.kubernetes.io/proxy-buffering](#proxy-buffering)|string|
|[nginx.ingress.kubernetes.io/proxy-buffers-number](#proxy-buffers-number)|number|
@ -638,6 +639,17 @@ To enable this feature use the annotation `nginx.ingress.kubernetes.io/from-to-w
!!! attention
For HTTPS to HTTPS redirects is mandatory the SSL Certificate defined in the Secret, located in the TLS section of Ingress, contains both FQDN in the common name of the certificate.
### Denylist source range
You can specify blocked client IP source ranges through the `nginx.ingress.kubernetes.io/denylist-source-range` annotation.
The value is a comma separated list of [CIDRs](https://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing), e.g. `10.0.0.0/24,172.10.0.1`.
To configure this setting globally for all Ingress rules, the `denylist-source-range` value may be set in the [NGINX ConfigMap](./configmap.md#denylist-source-range).
!!! note
Adding an annotation to an Ingress rule overrides any global restriction.
### Whitelist source range
You can specify allowed client IP source ranges through the `nginx.ingress.kubernetes.io/whitelist-source-range` annotation.

View file

@ -176,6 +176,7 @@ The following table shows a configuration option's name, type, and the default v
|[proxy-request-buffering](#proxy-request-buffering)|string|"on"|
|[ssl-redirect](#ssl-redirect)|bool|"true"|
|[force-ssl-redirect](#force-ssl-redirect)|bool|"false"|
|[denylist-source-range](#denylist-source-range)|[]string|[]string{}|
|[whitelist-source-range](#whitelist-source-range)|[]string|[]string{}|
|[skip-access-log-urls](#skip-access-log-urls)|[]string|[]string{}|
|[limit-rate](#limit-rate)|int|0|
@ -1096,6 +1097,11 @@ _**default:**_ "true"
Sets the global value of redirects (308) to HTTPS if the server has a default TLS certificate (defined in extra-args).
_**default:**_ "false"
## denylist-source-range
Sets the default denylisted IPs for each `server` block. This can be overwritten by an annotation on an Ingress rule.
See [ngx_http_access_module](https://nginx.org/en/docs/http/ngx_http_access_module.html).
## whitelist-source-range
Sets the default whitelisted IPs for each `server` block. This can be overwritten by an annotation on an Ingress rule.

View file

@ -44,6 +44,7 @@ import (
"k8s.io/ingress-nginx/internal/ingress/annotations/globalratelimit"
"k8s.io/ingress-nginx/internal/ingress/annotations/http2pushpreload"
"k8s.io/ingress-nginx/internal/ingress/annotations/influxdb"
"k8s.io/ingress-nginx/internal/ingress/annotations/ipdenylist"
"k8s.io/ingress-nginx/internal/ingress/annotations/ipwhitelist"
"k8s.io/ingress-nginx/internal/ingress/annotations/loadbalancing"
"k8s.io/ingress-nginx/internal/ingress/annotations/log"
@ -110,6 +111,7 @@ type Ingress struct {
LoadBalancing string
UpstreamVhost string
Whitelist ipwhitelist.SourceRange
Denylist ipdenylist.SourceRange
XForwardedPrefix string
SSLCipher sslcipher.Config
Logs log.Config
@ -160,6 +162,7 @@ func NewAnnotationExtractor(cfg resolver.Resolver) Extractor {
"LoadBalancing": loadbalancing.NewParser(cfg),
"UpstreamVhost": upstreamvhost.NewParser(cfg),
"Whitelist": ipwhitelist.NewParser(cfg),
"Denylist": ipdenylist.NewParser(cfg),
"XForwardedPrefix": xforwardedprefix.NewParser(cfg),
"SSLCipher": sslcipher.NewParser(cfg),
"Logs": log.NewParser(cfg),

View file

@ -0,0 +1,95 @@
/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package ipdenylist
import (
"fmt"
"sort"
"strings"
networking "k8s.io/api/networking/v1"
"k8s.io/ingress-nginx/internal/net"
"k8s.io/ingress-nginx/internal/ingress/annotations/parser"
ing_errors "k8s.io/ingress-nginx/internal/ingress/errors"
"k8s.io/ingress-nginx/internal/ingress/resolver"
"k8s.io/ingress-nginx/pkg/util/sets"
)
// SourceRange returns the CIDR
type SourceRange struct {
CIDR []string `json:"cidr,omitempty"`
}
// Equal tests for equality between two SourceRange types
func (sr1 *SourceRange) Equal(sr2 *SourceRange) bool {
if sr1 == sr2 {
return true
}
if sr1 == nil || sr2 == nil {
return false
}
return sets.StringElementsMatch(sr1.CIDR, sr2.CIDR)
}
type ipdenylist struct {
r resolver.Resolver
}
// NewParser creates a new denylist annotation parser
func NewParser(r resolver.Resolver) parser.IngressAnnotation {
return ipdenylist{r}
}
// ParseAnnotations parses the annotations contained in the ingress
// rule used to limit access to certain client addresses or networks.
// Multiple ranges can specified using commas as separator
// e.g. `18.0.0.0/8,56.0.0.0/8`
func (a ipdenylist) Parse(ing *networking.Ingress) (interface{}, error) {
defBackend := a.r.GetDefaultBackend()
defaultDenylistSourceRange := make([]string, len(defBackend.DenylistSourceRange))
copy(defaultDenylistSourceRange, defBackend.DenylistSourceRange)
sort.Strings(defaultDenylistSourceRange)
val, err := parser.GetStringAnnotation("denylist-source-range", ing)
// A missing annotation is not a problem, just use the default
if err == ing_errors.ErrMissingAnnotations {
return &SourceRange{CIDR: defaultDenylistSourceRange}, nil
}
values := strings.Split(val, ",")
ipnets, ips, err := net.ParseIPNets(values...)
if err != nil && len(ips) == 0 {
return &SourceRange{CIDR: defaultDenylistSourceRange}, ing_errors.LocationDenied{
Reason: fmt.Errorf("the annotation does not contain a valid IP address or network: %w", err),
}
}
cidrs := []string{}
for k := range ipnets {
cidrs = append(cidrs, k)
}
for k := range ips {
cidrs = append(cidrs, k)
}
sort.Strings(cidrs)
return &SourceRange{cidrs}, nil
}

View file

@ -0,0 +1,216 @@
/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package ipdenylist
import (
"testing"
api "k8s.io/api/core/v1"
networking "k8s.io/api/networking/v1"
meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/ingress-nginx/internal/ingress/annotations/parser"
"k8s.io/ingress-nginx/internal/ingress/defaults"
"k8s.io/ingress-nginx/internal/ingress/resolver"
)
func buildIngress() *networking.Ingress {
defaultBackend := networking.IngressBackend{
Service: &networking.IngressServiceBackend{
Name: "default-backend",
Port: networking.ServiceBackendPort{
Number: 80,
},
},
}
return &networking.Ingress{
ObjectMeta: meta_v1.ObjectMeta{
Name: "foo",
Namespace: api.NamespaceDefault,
},
Spec: networking.IngressSpec{
DefaultBackend: &networking.IngressBackend{
Service: &networking.IngressServiceBackend{
Name: "default-backend",
Port: networking.ServiceBackendPort{
Number: 80,
},
},
},
Rules: []networking.IngressRule{
{
Host: "foo.bar.com",
IngressRuleValue: networking.IngressRuleValue{
HTTP: &networking.HTTPIngressRuleValue{
Paths: []networking.HTTPIngressPath{
{
Path: "/foo",
Backend: defaultBackend,
},
},
},
},
},
},
},
}
}
func TestParseAnnotations(t *testing.T) {
ing := buildIngress()
tests := map[string]struct {
net string
expectCidr []string
expectErr bool
errOut string
}{
"test parse a valid net": {
net: "10.0.0.0/24",
expectCidr: []string{"10.0.0.0/24"},
expectErr: false,
},
"test parse a invalid net": {
net: "ww",
expectErr: true,
errOut: "the annotation does not contain a valid IP address or network: invalid CIDR address: ww",
},
"test parse a empty net": {
net: "",
expectErr: true,
errOut: "the annotation does not contain a valid IP address or network: invalid CIDR address: ",
},
"test parse a malicious escaped string": {
net: `10.0.0.0/8"rm /tmp",11.0.0.0/8`,
expectErr: true,
errOut: `the annotation does not contain a valid IP address or network: invalid CIDR address: 10.0.0.0/8"rm /tmp"`,
},
"test parse multiple valid cidr": {
net: "2.2.2.2/32,1.1.1.1/32,3.3.3.0/24",
expectCidr: []string{"1.1.1.1/32", "2.2.2.2/32", "3.3.3.0/24"},
expectErr: false,
},
}
for testName, test := range tests {
data := map[string]string{}
data[parser.GetAnnotationWithPrefix("denylist-source-range")] = test.net
ing.SetAnnotations(data)
p := NewParser(&resolver.Mock{})
i, err := p.Parse(ing)
if err != nil && !test.expectErr {
t.Errorf("%v:unexpected error: %v", testName, err)
}
if test.expectErr {
if err.Error() != test.errOut {
t.Errorf("%v:expected error: %v but %v return", testName, test.errOut, err.Error())
}
}
if !test.expectErr {
sr, ok := i.(*SourceRange)
if !ok {
t.Errorf("%v:expected a SourceRange type", testName)
}
if !strsEquals(sr.CIDR, test.expectCidr) {
t.Errorf("%v:expected %v CIDR but %v returned", testName, test.expectCidr, sr.CIDR)
}
}
}
}
type mockBackend struct {
resolver.Mock
}
// GetDefaultBackend returns the backend that must be used as default
func (m mockBackend) GetDefaultBackend() defaults.Backend {
return defaults.Backend{
DenylistSourceRange: []string{"4.4.4.0/24", "1.2.3.4/32"},
}
}
// Test that when we have a denylist set on the Backend that is used when we
// don't have the annotation
func TestParseAnnotationsWithDefaultConfig(t *testing.T) {
ing := buildIngress()
mockBackend := mockBackend{}
tests := map[string]struct {
net string
expectCidr []string
expectErr bool
errOut string
}{
"test parse a valid net": {
net: "10.0.0.0/24",
expectCidr: []string{"10.0.0.0/24"},
expectErr: false,
},
"test parse a invalid net": {
net: "ww",
expectErr: true,
errOut: "the annotation does not contain a valid IP address or network: invalid CIDR address: ww",
},
"test parse a empty net": {
net: "",
expectErr: true,
errOut: "the annotation does not contain a valid IP address or network: invalid CIDR address: ",
},
"test parse multiple valid cidr": {
net: "2.2.2.2/32,1.1.1.1/32,3.3.3.0/24",
expectCidr: []string{"1.1.1.1/32", "2.2.2.2/32", "3.3.3.0/24"},
expectErr: false,
},
}
for testName, test := range tests {
data := map[string]string{}
data[parser.GetAnnotationWithPrefix("denylist-source-range")] = test.net
ing.SetAnnotations(data)
p := NewParser(mockBackend)
i, err := p.Parse(ing)
if err != nil && !test.expectErr {
t.Errorf("%v:unexpected error: %v", testName, err)
}
if test.expectErr {
if err.Error() != test.errOut {
t.Errorf("%v:expected error: %v but %v return", testName, test.errOut, err.Error())
}
}
if !test.expectErr {
sr, ok := i.(*SourceRange)
if !ok {
t.Errorf("%v:expected a SourceRange type", testName)
}
if !strsEquals(sr.CIDR, test.expectCidr) {
t.Errorf("%v:expected %v CIDR but %v returned", testName, test.expectCidr, sr.CIDR)
}
}
}
}
func strsEquals(a, b []string) bool {
if len(a) != len(b) {
return false
}
for i, v := range a {
if v != b[i] {
return false
}
}
return true
}

View file

@ -895,6 +895,7 @@ func NewDefault() Configuration {
PreserveTrailingSlash: false,
SSLRedirect: true,
CustomHTTPErrors: []int{},
DenylistSourceRange: []string{},
WhitelistSourceRange: []string{},
SkipAccessLogURLs: []string{},
LimitRate: 0,

View file

@ -1412,6 +1412,7 @@ func locationApplyAnnotations(loc *ingress.Location, anns *annotations.Ingress)
loc.Redirect = anns.Redirect
loc.Rewrite = anns.Rewrite
loc.UpstreamVhost = anns.UpstreamVhost
loc.Denylist = anns.Denylist
loc.Whitelist = anns.Whitelist
loc.Denied = anns.Denied
loc.XForwardedPrefix = anns.XForwardedPrefix

View file

@ -41,6 +41,7 @@ const (
customHTTPErrors = "custom-http-errors"
skipAccessLogUrls = "skip-access-log-urls"
whitelistSourceRange = "whitelist-source-range"
denylistSourceRange = "denylist-source-range"
proxyRealIPCIDR = "proxy-real-ip-cidr"
bindAddress = "bind-address"
httpRedirectCode = "http-redirect-code"
@ -100,6 +101,7 @@ func ReadConfig(src map[string]string) config.Configuration {
to := config.NewDefault()
errors := make([]int, 0)
skipUrls := make([]string, 0)
denyList := make([]string, 0)
whiteList := make([]string, 0)
proxyList := make([]string, 0)
hideHeadersList := make([]string, 0)
@ -169,6 +171,11 @@ func ReadConfig(src map[string]string) config.Configuration {
skipUrls = splitAndTrimSpace(val, ",")
}
if val, ok := conf[denylistSourceRange]; ok {
delete(conf, denylistSourceRange)
denyList = append(denyList, splitAndTrimSpace(val, ",")...)
}
if val, ok := conf[whitelistSourceRange]; ok {
delete(conf, whitelistSourceRange)
whiteList = append(whiteList, splitAndTrimSpace(val, ",")...)
@ -395,6 +402,7 @@ func ReadConfig(src map[string]string) config.Configuration {
to.CustomHTTPErrors = filterErrors(errors)
to.SkipAccessLogURLs = skipUrls
to.DenylistSourceRange = denyList
to.WhitelistSourceRange = whiteList
to.ProxyRealIPCIDR = proxyList
to.BindAddressIpv4 = bindAddressIpv4List

View file

@ -149,6 +149,7 @@ func TestMergeConfigMapToStruct(t *testing.T) {
def = config.NewDefault()
def.LuaSharedDicts = defaultLuaSharedDicts
def.DenylistSourceRange = []string{"2.2.2.2/32"}
def.WhitelistSourceRange = []string{"1.1.1.1/32"}
def.DisableIpv6DNS = true
@ -161,6 +162,7 @@ func TestMergeConfigMapToStruct(t *testing.T) {
def.Checksum = fmt.Sprintf("%v", hash)
to = ReadConfig(map[string]string{
"denylist-source-range": "2.2.2.2/32",
"whitelist-source-range": "1.1.1.1/32",
"disable-ipv6-dns": "true",
})

View file

@ -139,6 +139,10 @@ type Backend struct {
// http://nginx.org/en/docs/http/ngx_http_access_module.html
WhitelistSourceRange []string `json:"whitelist-source-range"`
// DenylistSourceRange allows limiting access to certain client addresses
// http://nginx.org/en/docs/http/ngx_http_access_module.html
DenylistSourceRange []string `json:"denylist-source-range"`
// Limits the rate of response transmission to a client.
// The rate is specified in bytes per second. The zero value disables rate limiting.
// The limit is set per a request, and so if a client simultaneously opens two connections,

View file

@ -30,6 +30,7 @@ import (
"k8s.io/ingress-nginx/internal/ingress/annotations/fastcgi"
"k8s.io/ingress-nginx/internal/ingress/annotations/globalratelimit"
"k8s.io/ingress-nginx/internal/ingress/annotations/influxdb"
"k8s.io/ingress-nginx/internal/ingress/annotations/ipdenylist"
"k8s.io/ingress-nginx/internal/ingress/annotations/ipwhitelist"
"k8s.io/ingress-nginx/internal/ingress/annotations/log"
"k8s.io/ingress-nginx/internal/ingress/annotations/mirror"
@ -222,6 +223,7 @@ type Server struct {
// In some cases when more than one annotations is defined a particular order in the execution
// is required.
// The chain in the execution order of annotations should be:
// - Denylist
// - Whitelist
// - RateLimit
// - BasicDigestAuth
@ -292,6 +294,10 @@ type Location struct {
// Rewrite describes the redirection this location.
// +optional
Rewrite rewrite.Config `json:"rewrite,omitempty"`
// Denylist indicates only connections from certain client
// addresses or networks are allowed.
// +optional
Denylist ipdenylist.SourceRange `json:"denylist,omitempty"`
// Whitelist indicates only connections from certain client
// addresses or networks are allowed.
// +optional

View file

@ -401,6 +401,9 @@ func (l1 *Location) Equal(l2 *Location) bool {
if !(&l1.Rewrite).Equal(&l2.Rewrite) {
return false
}
if !(&l1.Denylist).Equal(&l2.Denylist) {
return false
}
if !(&l1.Whitelist).Equal(&l2.Whitelist) {
return false
}

View file

@ -1262,6 +1262,10 @@ stream {
{{ buildModSecurityForLocation $all.Cfg $location }}
{{ if isLocationAllowed $location }}
{{ if gt (len $location.Denylist.CIDR) 0 }}
{{ range $ip := $location.Denylist.CIDR }}
deny {{ $ip }};{{ end }}
{{ end }}
{{ if gt (len $location.Whitelist.CIDR) 0 }}
{{ range $ip := $location.Whitelist.CIDR }}
allow {{ $ip }};{{ end }}

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,147 @@
/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package annotations
import (
"net/http"
"strings"
"github.com/onsi/ginkgo/v2"
"k8s.io/ingress-nginx/test/e2e/framework"
)
var _ = framework.DescribeAnnotation("denylist-source-range", func() {
f := framework.NewDefaultFramework("ipdenylist")
ginkgo.BeforeEach(func() {
f.NewEchoDeployment()
})
ginkgo.It("only deny explicitly denied IPs, allow all others", func() {
host := "ipdenylist.foo.com"
namespace := f.Namespace
annotations := map[string]string{
"nginx.ingress.kubernetes.io/denylist-source-range": "18.0.0.0/8, 56.0.0.1",
}
ing := framework.NewSingleIngress(host, "/", host, namespace, framework.EchoService, 80, annotations)
// Temporarily trust forwarded headers so we can test IP based access control
f.UpdateNginxConfigMapData("use-forwarded-headers", "true")
defer func() {
// Return to the original value
f.UpdateNginxConfigMapData("use-forwarded-headers", "false")
}()
f.EnsureIngress(ing)
f.WaitForNginxServer(host,
func(server string) bool {
return strings.Contains(server, "deny 18.0.0.0/8;") &&
strings.Contains(server, "deny 56.0.0.1;") &&
!strings.Contains(server, "deny all;")
})
ginkgo.By("sending request from an explicitly denied IP range")
f.HTTPTestClient().
GET("/").
WithHeader("Host", host).
WithHeader("X-Forwarded-For", "18.0.0.1").
Expect().
Status(http.StatusForbidden)
ginkgo.By("sending request from an explicitly denied IP address")
f.HTTPTestClient().
GET("/").
WithHeader("Host", host).
WithHeader("X-Forwarded-For", "56.0.0.1").
Expect().
Status(http.StatusForbidden)
ginkgo.By("sending request from an implicitly allowed IP range")
f.HTTPTestClient().
GET("/").
WithHeader("Host", host).
WithHeader("X-Forwarded-For", "56.0.0.2").
Expect().
Status(http.StatusOK)
})
ginkgo.It("only allow explicitly allowed IPs, deny all others", func() {
host := "ipdenylist.foo.com"
namespace := f.Namespace
annotations := map[string]string{
"nginx.ingress.kubernetes.io/denylist-source-range": "18.1.0.0/16, 56.0.0.0/8",
"nginx.ingress.kubernetes.io/whitelist-source-range": "18.0.0.0/8, 55.0.0.0/8",
}
ing := framework.NewSingleIngress(host, "/", host, namespace, framework.EchoService, 80, annotations)
// Temporarily trust forwarded headers so we can test IP based access control
f.UpdateNginxConfigMapData("use-forwarded-headers", "true")
defer func() {
// Return to the original value
f.UpdateNginxConfigMapData("use-forwarded-headers", "false")
}()
f.EnsureIngress(ing)
f.WaitForNginxServer(host,
func(server string) bool {
return strings.Contains(server, "deny 18.1.0.0/16;") &&
strings.Contains(server, "deny 56.0.0.0/8;") &&
strings.Contains(server, "allow 18.0.0.0/8;") &&
strings.Contains(server, "allow 55.0.0.0/8;") &&
strings.Contains(server, "deny all;")
})
ginkgo.By("sending request from an explicitly denied IP range")
f.HTTPTestClient().
GET("/").
WithHeader("Host", host).
WithHeader("X-Forwarded-For", "18.1.0.1").
Expect().
Status(http.StatusForbidden)
ginkgo.By("sending request from an implicitly denied IP")
f.HTTPTestClient().
GET("/").
WithHeader("Host", host).
WithHeader("X-Forwarded-For", "10.10.10.10").
Expect().
Status(http.StatusForbidden)
ginkgo.By("sending request from an explicitly allowed IP range")
f.HTTPTestClient().
GET("/").
WithHeader("Host", host).
WithHeader("X-Forwarded-For", "18.4.0.1").
Expect().
Status(http.StatusOK)
ginkgo.By("sending request from an explicitly allowed IP range")
f.HTTPTestClient().
GET("/").
WithHeader("Host", host).
WithHeader("X-Forwarded-For", "55.55.55.55").
Expect().
Status(http.StatusOK)
})
})