diff --git a/charts/ingress-nginx/templates/_params.tpl b/charts/ingress-nginx/templates/_params.tpl index c628ec6f7..d74798b98 100644 --- a/charts/ingress-nginx/templates/_params.tpl +++ b/charts/ingress-nginx/templates/_params.tpl @@ -18,6 +18,9 @@ {{- if .Values.controller.scope.enabled }} - --watch-namespace={{ default "$(POD_NAMESPACE)" .Values.controller.scope.namespace }} {{- end }} +{{- if and (not .Values.controller.scope.enabled) .Values.controller.scope.namespaceSelector }} +- --watch-namespace-selector={{ default "" .Values.controller.scope.namespaceSelector }} +{{- end }} {{- if and .Values.controller.reportNodeInternalIp .Values.controller.hostNetwork }} - --report-node-internal-ip-address={{ .Values.controller.reportNodeInternalIp }} {{- end }} diff --git a/charts/ingress-nginx/templates/clusterrole.yaml b/charts/ingress-nginx/templates/clusterrole.yaml index c1f901d50..efc7d2682 100644 --- a/charts/ingress-nginx/templates/clusterrole.yaml +++ b/charts/ingress-nginx/templates/clusterrole.yaml @@ -20,6 +20,9 @@ rules: - nodes - pods - secrets +{{- if not .Values.controller.scope.enabled }} + - namespaces +{{- end}} verbs: - list - watch diff --git a/charts/ingress-nginx/values.yaml b/charts/ingress-nginx/values.yaml index 86028fdd2..3f9771b31 100644 --- a/charts/ingress-nginx/values.yaml +++ b/charts/ingress-nginx/values.yaml @@ -137,6 +137,9 @@ controller: scope: enabled: false namespace: "" # defaults to $(POD_NAMESPACE) + # When scope.enabled == false, instead of watching all namespaces, we watching namespaces whose labels + # only match with namespaceSelector. Format like foo=bar. Defaults to empty, means watching all namespaces. + namespaceSelector: "" ## Allows customization of the configmap / nginx-configmap namespace ## diff --git a/cmd/nginx/flags.go b/cmd/nginx/flags.go index 42c14dd51..72a2bfb8d 100644 --- a/cmd/nginx/flags.go +++ b/cmd/nginx/flags.go @@ -24,6 +24,7 @@ import ( "github.com/spf13/pflag" apiv1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/ingress-nginx/internal/ingress/annotations/parser" "k8s.io/ingress-nginx/internal/ingress/controller" ngx_config "k8s.io/ingress-nginx/internal/ingress/controller/config" @@ -100,6 +101,9 @@ either be a port name or number.`) This includes Ingresses, Services and all configuration resources. All namespaces are watched if this parameter is left empty.`) + watchNamespaceSelector = flags.String("watch-namespace-selector", "", + `Selector selects namespaces the controller watches for updates to Kubernetes objects.`) + profiling = flags.Bool("profiling", true, `Enable profiling via web interface host:port/debug/pprof/`) @@ -266,6 +270,19 @@ https://blog.maxmind.com/2019/12/18/significant-changes-to-accessing-and-using-g nginx.HealthCheckTimeout = time.Duration(*defHealthCheckTimeout) * time.Second } + if len(*watchNamespace) != 0 && len(*watchNamespaceSelector) != 0 { + return false, nil, fmt.Errorf("flags --watch-namespace and --watch-namespace-selector are mutually exclusive") + } + + var namespaceSelector labels.Selector + if len(*watchNamespaceSelector) != 0 { + var err error + namespaceSelector, err = labels.Parse(*watchNamespaceSelector) + if err != nil { + return false, nil, fmt.Errorf("failed to parse --watch-namespace-selector=%s, error: %v", *watchNamespaceSelector, err) + } + } + ngx_config.EnableSSLChainCompletion = *enableSSLChainCompletion config := &controller.Configuration{ @@ -282,6 +299,7 @@ https://blog.maxmind.com/2019/12/18/significant-changes-to-accessing-and-using-g ResyncPeriod: *resyncPeriod, DefaultService: *defaultSvc, Namespace: *watchNamespace, + WatchNamespaceSelector: namespaceSelector, ConfigMapName: *configMap, TCPConfigMapName: *tcpConfigMapName, UDPConfigMapName: *udpConfigMapName, diff --git a/deploy/static/provider/aws/deploy.yaml b/deploy/static/provider/aws/deploy.yaml index 273d90dfd..3b3b45bc4 100644 --- a/deploy/static/provider/aws/deploy.yaml +++ b/deploy/static/provider/aws/deploy.yaml @@ -59,6 +59,7 @@ rules: - nodes - pods - secrets + - namespaces verbs: - list - watch diff --git a/deploy/static/provider/baremetal/deploy.yaml b/deploy/static/provider/baremetal/deploy.yaml index 10f08bb6e..106b85306 100644 --- a/deploy/static/provider/baremetal/deploy.yaml +++ b/deploy/static/provider/baremetal/deploy.yaml @@ -59,6 +59,7 @@ rules: - nodes - pods - secrets + - namespaces verbs: - list - watch diff --git a/deploy/static/provider/cloud/deploy.yaml b/deploy/static/provider/cloud/deploy.yaml index 38dc08bf7..bd9dd511e 100644 --- a/deploy/static/provider/cloud/deploy.yaml +++ b/deploy/static/provider/cloud/deploy.yaml @@ -59,6 +59,7 @@ rules: - nodes - pods - secrets + - namespaces verbs: - list - watch diff --git a/deploy/static/provider/do/deploy.yaml b/deploy/static/provider/do/deploy.yaml index 5d148249e..1e701f3e8 100644 --- a/deploy/static/provider/do/deploy.yaml +++ b/deploy/static/provider/do/deploy.yaml @@ -60,6 +60,7 @@ rules: - nodes - pods - secrets + - namespaces verbs: - list - watch diff --git a/deploy/static/provider/exoscale/deploy.yaml b/deploy/static/provider/exoscale/deploy.yaml index f86766fbc..b050d7e7b 100644 --- a/deploy/static/provider/exoscale/deploy.yaml +++ b/deploy/static/provider/exoscale/deploy.yaml @@ -59,6 +59,7 @@ rules: - nodes - pods - secrets + - namespaces verbs: - list - watch diff --git a/deploy/static/provider/kind/deploy.yaml b/deploy/static/provider/kind/deploy.yaml index 322d63e0e..a426cd1c2 100644 --- a/deploy/static/provider/kind/deploy.yaml +++ b/deploy/static/provider/kind/deploy.yaml @@ -59,6 +59,7 @@ rules: - nodes - pods - secrets + - namespaces verbs: - list - watch diff --git a/deploy/static/provider/scw/deploy.yaml b/deploy/static/provider/scw/deploy.yaml index b7bdefba5..4010e5fc6 100644 --- a/deploy/static/provider/scw/deploy.yaml +++ b/deploy/static/provider/scw/deploy.yaml @@ -60,6 +60,7 @@ rules: - nodes - pods - secrets + - namespaces verbs: - list - watch diff --git a/docs/user-guide/cli-arguments.md b/docs/user-guide/cli-arguments.md index ef1c0feb2..dc31830ef 100644 --- a/docs/user-guide/cli-arguments.md +++ b/docs/user-guide/cli-arguments.md @@ -65,3 +65,4 @@ They are set in the container spec of the `nginx-ingress-controller` Deployment | `--version` | Show release information about the NGINX Ingress controller and exit. | | `--vmodule` | comma-separated list of pattern=N settings for file-filtered logging | | `--watch-namespace` | Namespace the controller watches for updates to Kubernetes objects. This includes Ingresses, Services and all configuration resources. All namespaces are watched if this parameter is left empty. | +| `--watch-namespace-selector` | The controller will watch namespaces whose labels match the given selector. This flag only takes effective when `--watch-namespace` is empty. | diff --git a/go.mod b/go.mod index 49e380774..f7dc52167 100644 --- a/go.mod +++ b/go.mod @@ -63,6 +63,7 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver v3.5.1+incompatible // indirect github.com/cespare/xxhash/v2 v2.1.1 // indirect + github.com/client9/misspell v0.3.4 // indirect github.com/coreos/go-systemd/v22 v22.3.2 // indirect github.com/cyphar/filepath-securejoin v0.2.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect diff --git a/go.sum b/go.sum index 21a2bb5c4..02750db0d 100644 --- a/go.sum +++ b/go.sum @@ -120,6 +120,7 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= +github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= diff --git a/internal/ingress/controller/controller.go b/internal/ingress/controller/controller.go index b51071630..b1dbf9cd1 100644 --- a/internal/ingress/controller/controller.go +++ b/internal/ingress/controller/controller.go @@ -27,6 +27,7 @@ import ( apiv1 "k8s.io/api/core/v1" networking "k8s.io/api/networking/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" @@ -67,6 +68,8 @@ type Configuration struct { Namespace string + WatchNamespaceSelector labels.Selector + // +optional TCPConfigMapName string // +optional diff --git a/internal/ingress/controller/controller_test.go b/internal/ingress/controller/controller_test.go index f9d60974f..15367bc0d 100644 --- a/internal/ingress/controller/controller_test.go +++ b/internal/ingress/controller/controller_test.go @@ -36,6 +36,7 @@ import ( v1 "k8s.io/api/core/v1" networking "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/kubernetes/fake" "k8s.io/ingress-nginx/internal/file" @@ -2378,6 +2379,7 @@ func newNGINXController(t *testing.T) *NGINXController { storer := store.New( ns, + labels.Nothing(), fmt.Sprintf("%v/config", ns), fmt.Sprintf("%v/tcp", ns), fmt.Sprintf("%v/udp", ns), @@ -2441,6 +2443,7 @@ func newDynamicNginxController(t *testing.T, setConfigMap func(string) *v1.Confi storer := store.New( ns, + labels.Nothing(), fmt.Sprintf("%v/config", ns), fmt.Sprintf("%v/tcp", ns), fmt.Sprintf("%v/udp", ns), diff --git a/internal/ingress/controller/nginx.go b/internal/ingress/controller/nginx.go index b77d1d870..4d1aa3916 100644 --- a/internal/ingress/controller/nginx.go +++ b/internal/ingress/controller/nginx.go @@ -122,6 +122,7 @@ func NewNGINXController(config *Configuration, mc metric.Collector) *NGINXContro n.store = store.New( config.Namespace, + config.WatchNamespaceSelector, config.ConfigMapName, config.TCPConfigMapName, config.UDPConfigMapName, diff --git a/internal/ingress/controller/store/namespace.go b/internal/ingress/controller/store/namespace.go new file mode 100644 index 000000000..b29eb0326 --- /dev/null +++ b/internal/ingress/controller/store/namespace.go @@ -0,0 +1,39 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package store + +import ( + apiv1 "k8s.io/api/core/v1" + "k8s.io/client-go/tools/cache" +) + +// NamespaceLister makes a Store that lists Namespaces. +type NamespaceLister struct { + cache.Store +} + +// ByKey returns the Namespace matching key in the local Namespace Store. +func (cml *NamespaceLister) ByKey(key string) (*apiv1.Namespace, error) { + s, exists, err := cml.GetByKey(key) + if err != nil { + return nil, err + } + if !exists { + return nil, NotExistsError(key) + } + return s.(*apiv1.Namespace), nil +} diff --git a/internal/ingress/controller/store/store.go b/internal/ingress/controller/store/store.go index a91443549..fe0d1e0d7 100644 --- a/internal/ingress/controller/store/store.go +++ b/internal/ingress/controller/store/store.go @@ -32,6 +32,7 @@ import ( networkingv1 "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/labels" k8sruntime "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" @@ -127,6 +128,7 @@ type Informer struct { Service cache.SharedIndexInformer Secret cache.SharedIndexInformer ConfigMap cache.SharedIndexInformer + Namespace cache.SharedIndexInformer } // Lister contains object listers (stores). @@ -137,6 +139,7 @@ type Lister struct { Endpoint EndpointLister Secret SecretLister ConfigMap ConfigMapLister + Namespace NamespaceLister IngressWithAnnotation IngressWithAnnotationsLister } @@ -172,6 +175,15 @@ func (i *Informer) Run(stopCh chan struct{}) { runtime.HandleError(fmt.Errorf("timed out waiting for ingress classcaches to sync")) } + // when limit controller scope to one namespace, skip sync namespaces at cluster scope + if i.Namespace != nil { + go i.Namespace.Run(stopCh) + + if !cache.WaitForCacheSync(stopCh, i.Namespace.HasSynced) { + runtime.HandleError(fmt.Errorf("timed out waiting for caches to sync")) + } + } + // in big clusters, deltas can keep arriving even after HasSynced // functions have returned 'true' time.Sleep(1 * time.Second) @@ -225,7 +237,9 @@ type k8sStore struct { // New creates a new object store to be used in the ingress controller func New( - namespace, configmap, tcp, udp, defaultSSLCertificate string, + namespace string, + namespaceSelector labels.Selector, + configmap, tcp, udp, defaultSSLCertificate string, resyncPeriod time.Duration, client clientset.Interface, updateCh *channels.RingChannel, @@ -322,6 +336,35 @@ func New( store.informers.Service = infFactory.Core().V1().Services().Informer() store.listers.Service.Store = store.informers.Service.GetStore() + // avoid caching namespaces at cluster scope when watching single namespace + if namespaceSelector != nil && !namespaceSelector.Empty() { + // cache informers factory for namespaces + infFactoryNamespaces := informers.NewSharedInformerFactoryWithOptions(client, resyncPeriod, + informers.WithTweakListOptions(labelsTweakListOptionsFunc), + ) + + store.informers.Namespace = infFactoryNamespaces.Core().V1().Namespaces().Informer() + store.listers.Namespace.Store = store.informers.Namespace.GetStore() + } + + watchedNamespace := func(namespace string) bool { + if namespaceSelector == nil || namespaceSelector.Empty() { + return true + } + + item, ok, err := store.listers.Namespace.GetByKey(namespace) + if !ok { + klog.Errorf("Namespace %s not existed: %v.", namespace, err) + return false + } + ns, ok := item.(*corev1.Namespace) + if !ok { + return false + } + + return namespaceSelector.Matches(labels.Set(ns.Labels)) + } + ingDeleteHandler := func(obj interface{}) { ing, ok := toIngress(obj) if !ok { @@ -338,6 +381,10 @@ func New( } } + if !watchedNamespace(ing.Namespace) { + return + } + _, err := store.GetIngressClass(ing, icConfig) if err != nil { klog.InfoS("Ignoring ingress because of error while validating ingress class", "ingress", klog.KObj(ing), "error", err) @@ -363,6 +410,11 @@ func New( ingEventHandler := cache.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { ing, _ := toIngress(obj) + + if !watchedNamespace(ing.Namespace) { + return + } + ic, err := store.GetIngressClass(ing, icConfig) if err != nil { klog.InfoS("Ignoring ingress because of error while validating ingress class", "ingress", klog.KObj(ing), "error", err) @@ -392,6 +444,10 @@ func New( oldIng, _ := toIngress(old) curIng, _ := toIngress(cur) + if !watchedNamespace(oldIng.Namespace) { + return + } + var errOld, errCur error var classCur string if !icConfig.IgnoreIngressClass { @@ -528,6 +584,10 @@ func New( sec := cur.(*corev1.Secret) key := k8s.MetaNamespaceKey(sec) + if !watchedNamespace(sec.Namespace) { + return + } + if store.defaultSSLCertificate == key { store.syncSecret(store.defaultSSLCertificate) } @@ -566,6 +626,10 @@ func New( } } + if !watchedNamespace(sec.Namespace) { + return + } + store.sslStore.Delete(k8s.MetaNamespaceKey(sec)) key := k8s.MetaNamespaceKey(sec) diff --git a/internal/ingress/controller/store/store_test.go b/internal/ingress/controller/store/store_test.go index 9004094a3..735208001 100644 --- a/internal/ingress/controller/store/store_test.go +++ b/internal/ingress/controller/store/store_test.go @@ -31,6 +31,7 @@ import ( networking "k8s.io/api/networking/v1" k8sErrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" "sigs.k8s.io/controller-runtime/pkg/envtest" @@ -89,6 +90,8 @@ func TestStore(t *testing.T) { t.Fatalf("error: %v", err) } + emptySelector, _ := labels.Parse("") + defer te.Stop() clientSet, err := kubernetes.NewForConfig(cfg) @@ -112,6 +115,7 @@ func TestStore(t *testing.T) { storer := New( ns, + emptySelector, fmt.Sprintf("%v/config", ns), fmt.Sprintf("%v/tcp", ns), fmt.Sprintf("%v/udp", ns), @@ -191,6 +195,7 @@ func TestStore(t *testing.T) { storer := New( ns, + emptySelector, fmt.Sprintf("%v/config", ns), fmt.Sprintf("%v/tcp", ns), fmt.Sprintf("%v/udp", ns), @@ -293,6 +298,7 @@ func TestStore(t *testing.T) { storer := New( ns, + emptySelector, fmt.Sprintf("%v/config", ns), fmt.Sprintf("%v/tcp", ns), fmt.Sprintf("%v/udp", ns), @@ -407,6 +413,7 @@ func TestStore(t *testing.T) { storer := New( ns, + emptySelector, fmt.Sprintf("%v/config", ns), fmt.Sprintf("%v/tcp", ns), fmt.Sprintf("%v/udp", ns), @@ -535,6 +542,7 @@ func TestStore(t *testing.T) { storer := New( ns, + emptySelector, fmt.Sprintf("%v/config", ns), fmt.Sprintf("%v/tcp", ns), fmt.Sprintf("%v/udp", ns), @@ -633,6 +641,7 @@ func TestStore(t *testing.T) { storer := New( ns, + emptySelector, fmt.Sprintf("%v/config", ns), fmt.Sprintf("%v/tcp", ns), fmt.Sprintf("%v/udp", ns), @@ -725,6 +734,7 @@ func TestStore(t *testing.T) { storer := New( ns, + emptySelector, fmt.Sprintf("%v/config", ns), fmt.Sprintf("%v/tcp", ns), fmt.Sprintf("%v/udp", ns), @@ -809,6 +819,7 @@ func TestStore(t *testing.T) { storer := New( ns, + emptySelector, fmt.Sprintf("%v/config", ns), fmt.Sprintf("%v/tcp", ns), fmt.Sprintf("%v/udp", ns), @@ -903,6 +914,7 @@ func TestStore(t *testing.T) { storer := New( ns, + emptySelector, fmt.Sprintf("%v/config", ns), fmt.Sprintf("%v/tcp", ns), fmt.Sprintf("%v/udp", ns), @@ -1025,6 +1037,7 @@ func TestStore(t *testing.T) { storer := New( ns, + emptySelector, fmt.Sprintf("%v/config", ns), fmt.Sprintf("%v/tcp", ns), fmt.Sprintf("%v/udp", ns), @@ -1107,6 +1120,102 @@ func TestStore(t *testing.T) { } }) + t.Run("should not receive events whose namespace doesn't match watch namespace selector", func(t *testing.T) { + ns := createNamespace(clientSet, t) + defer deleteNamespace(ns, clientSet, t) + createConfigMap(clientSet, ns, t) + + stopCh := make(chan struct{}) + updateCh := channels.NewRingChannel(1024) + + var add uint64 + var upd uint64 + var del uint64 + + go func(ch *channels.RingChannel) { + for { + evt, ok := <-ch.Out() + if !ok { + return + } + + e := evt.(Event) + if e.Obj == nil { + continue + } + switch e.Type { + case CreateEvent: + atomic.AddUint64(&add, 1) + case UpdateEvent: + atomic.AddUint64(&upd, 1) + case DeleteEvent: + atomic.AddUint64(&del, 1) + } + } + }(updateCh) + + namesapceSelector, _ := labels.Parse("foo=bar") + storer := New( + ns, + namesapceSelector, + fmt.Sprintf("%v/config", ns), + fmt.Sprintf("%v/tcp", ns), + fmt.Sprintf("%v/udp", ns), + "", + 10*time.Minute, + clientSet, + updateCh, + false, + DefaultClassConfig) + + storer.Run(stopCh) + + ing := ensureIngress(&networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dummy", + Namespace: ns, + }, + Spec: networking.IngressSpec{ + Rules: []networking.IngressRule{ + { + Host: "dummy", + IngressRuleValue: networking.IngressRuleValue{ + HTTP: &networking.HTTPIngressRuleValue{ + Paths: []networking.HTTPIngressPath{ + { + Path: "/", + PathType: &pathPrefix, + Backend: networking.IngressBackend{ + Service: &networking.IngressServiceBackend{ + Name: "http-svc", + Port: networking.ServiceBackendPort{ + Number: 80, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, clientSet, t) + defer deleteIngress(ing, clientSet, t) + + time.Sleep(1 * time.Second) + + if atomic.LoadUint64(&add) != 0 { + t.Errorf("expected 0 events of type Create but %v occurred", add) + } + if atomic.LoadUint64(&upd) != 0 { + t.Errorf("expected 0 events of type Update but %v occurred", upd) + } + if atomic.LoadUint64(&del) != 0 { + t.Errorf("expected 0 events of type Delete but %v occurred", del) + } + + }) // test add ingress with secret it doesn't exists and then add secret // check secret is generated on fs // check ocsp diff --git a/test/e2e-image/namespace-overlays/namespace-selector/values.yaml b/test/e2e-image/namespace-overlays/namespace-selector/values.yaml new file mode 100644 index 000000000..e4c0e7a87 --- /dev/null +++ b/test/e2e-image/namespace-overlays/namespace-selector/values.yaml @@ -0,0 +1,36 @@ +# TODO: remove the need to use fullnameOverride +fullnameOverride: nginx-ingress +controller: + image: + repository: ingress-controller/controller + tag: 1.0.0-dev + digest: + containerPort: + http: "1080" + https: "1443" + + extraArgs: + http-port: "1080" + https-port: "1443" + # e2e tests do not require information about ingress status + update-status: "false" + ingressClassResource: + # We will create and remove each IC/ClusterRole/ClusterRoleBinding per test so there's no conflict + enabled: false + scope: + enabled: false + namespaceSelector: "foo=bar" + + config: + worker-processes: "1" + service: + type: NodePort + admissionWebhooks: + enabled: false + +defaultBackend: + enabled: false + +rbac: + create: true + scope: false diff --git a/test/e2e/framework/deployment.go b/test/e2e/framework/deployment.go index 444045036..c5fded856 100644 --- a/test/e2e/framework/deployment.go +++ b/test/e2e/framework/deployment.go @@ -55,7 +55,15 @@ func (f *Framework) NewEchoDeploymentWithReplicas(replicas int) { // replicas is configurable and // name is configurable func (f *Framework) NewEchoDeploymentWithNameAndReplicas(name string, replicas int) { - deployment := newDeployment(name, f.Namespace, "k8s.gcr.io/ingress-nginx/e2e-test-echo@sha256:131ece0637b29231470cfaa04690c2966a2e0b147d3c9df080a0857b78982410", 80, int32(replicas), + f.newEchoDeployment(f.Namespace, name, replicas) +} + +func (f *Framework) NewEchoDeploymentWithNamespaceAndReplicas(namespace string, replicas int) { + f.newEchoDeployment(namespace, EchoService, replicas) +} + +func (f *Framework) newEchoDeployment(namespace, name string, replicas int) { + deployment := newDeployment(name, namespace, "k8s.gcr.io/ingress-nginx/e2e-test-echo@sha256:131ece0637b29231470cfaa04690c2966a2e0b147d3c9df080a0857b78982410", 80, int32(replicas), nil, []corev1.VolumeMount{}, []corev1.Volume{}, @@ -66,7 +74,7 @@ func (f *Framework) NewEchoDeploymentWithNameAndReplicas(name string, replicas i service := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: name, - Namespace: f.Namespace, + Namespace: namespace, }, Spec: corev1.ServiceSpec{ Ports: []corev1.ServicePort{ @@ -85,7 +93,7 @@ func (f *Framework) NewEchoDeploymentWithNameAndReplicas(name string, replicas i f.EnsureService(service) - err := WaitForEndpoints(f.KubeClientSet, DefaultTimeout, name, f.Namespace, replicas) + err := WaitForEndpoints(f.KubeClientSet, DefaultTimeout, name, namespace, replicas) assert.Nil(ginkgo.GinkgoT(), err, "waiting for endpoints to become ready") } diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go index e31fd1e4e..11405c69f 100644 --- a/test/e2e/framework/framework.go +++ b/test/e2e/framework/framework.go @@ -126,7 +126,7 @@ func (f *Framework) AfterEach() { defer func(kubeClient kubernetes.Interface, ns string) { go func() { defer ginkgo.GinkgoRecover() - err := deleteKubeNamespace(kubeClient, ns) + err := DeleteKubeNamespace(kubeClient, ns) assert.Nil(ginkgo.GinkgoT(), err, "deleting namespace %v", f.Namespace) }() }(f.KubeClientSet, f.Namespace) @@ -588,6 +588,12 @@ func NewSingleIngress(name, path, host, ns, service string, port int, annotation return newSingleIngressWithRules(name, path, host, ns, service, port, annotations, nil) } +func NewSingleIngressWithIngressClass(name, path, host, ns, service, ingressClass string, port int, annotations map[string]string) *networking.Ingress { + ing := newSingleIngressWithRules(name, path, host, ns, service, port, annotations, nil) + ing.Spec.IngressClassName = &ingressClass + return ing +} + // NewSingleIngressWithMultiplePaths creates a simple ingress rule with multiple paths func NewSingleIngressWithMultiplePaths(name string, paths []string, host, ns, service string, port int, annotations map[string]string) *networking.Ingress { pathtype := networking.PathTypePrefix diff --git a/test/e2e/framework/k8s.go b/test/e2e/framework/k8s.go index ea34960b6..7f434beb8 100644 --- a/test/e2e/framework/k8s.go +++ b/test/e2e/framework/k8s.go @@ -38,7 +38,7 @@ import ( // EnsureSecret creates a Secret object or returns it if it already exists. func (f *Framework) EnsureSecret(secret *api.Secret) *api.Secret { - err := createSecretWithRetries(f.KubeClientSet, f.Namespace, secret) + err := createSecretWithRetries(f.KubeClientSet, secret.Namespace, secret) assert.Nil(ginkgo.GinkgoT(), err, "creating secret") s, err := f.KubeClientSet.CoreV1().Secrets(secret.Namespace).Get(context.TODO(), secret.Name, metav1.GetOptions{}) @@ -50,10 +50,10 @@ func (f *Framework) EnsureSecret(secret *api.Secret) *api.Secret { // EnsureConfigMap creates a ConfigMap object or returns it if it already exists. func (f *Framework) EnsureConfigMap(configMap *api.ConfigMap) (*api.ConfigMap, error) { - cm, err := f.KubeClientSet.CoreV1().ConfigMaps(f.Namespace).Create(context.TODO(), configMap, metav1.CreateOptions{}) + cm, err := f.KubeClientSet.CoreV1().ConfigMaps(configMap.Namespace).Create(context.TODO(), configMap, metav1.CreateOptions{}) if err != nil { if k8sErrors.IsAlreadyExists(err) { - return f.KubeClientSet.CoreV1().ConfigMaps(f.Namespace).Update(context.TODO(), configMap, metav1.UpdateOptions{}) + return f.KubeClientSet.CoreV1().ConfigMaps(configMap.Namespace).Update(context.TODO(), configMap, metav1.UpdateOptions{}) } return nil, err } @@ -72,13 +72,13 @@ func (f *Framework) GetIngress(namespace string, name string) *networking.Ingres // EnsureIngress creates an Ingress object and returns it, throws error if it already exists. func (f *Framework) EnsureIngress(ingress *networking.Ingress) *networking.Ingress { fn := func() { - err := createIngressWithRetries(f.KubeClientSet, f.Namespace, ingress) + err := createIngressWithRetries(f.KubeClientSet, ingress.Namespace, ingress) assert.Nil(ginkgo.GinkgoT(), err, "creating ingress") } f.WaitForReload(fn) - ing := f.GetIngress(f.Namespace, ingress.Name) + ing := f.GetIngress(ingress.Namespace, ingress.Name) if ing.Annotations == nil { ing.Annotations = make(map[string]string) } @@ -88,10 +88,10 @@ func (f *Framework) EnsureIngress(ingress *networking.Ingress) *networking.Ingre // UpdateIngress updates an Ingress object and returns the updated object. func (f *Framework) UpdateIngress(ingress *networking.Ingress) *networking.Ingress { - err := updateIngressWithRetries(f.KubeClientSet, f.Namespace, ingress) + err := updateIngressWithRetries(f.KubeClientSet, ingress.Namespace, ingress) assert.Nil(ginkgo.GinkgoT(), err, "updating ingress") - ing := f.GetIngress(f.Namespace, ingress.Name) + ing := f.GetIngress(ingress.Namespace, ingress.Name) if ing.Annotations == nil { ing.Annotations = make(map[string]string) } @@ -113,15 +113,15 @@ func (f *Framework) GetService(namespace string, name string) *core.Service { // EnsureService creates a Service object and returns it, throws error if it already exists. func (f *Framework) EnsureService(service *core.Service) *core.Service { - err := createServiceWithRetries(f.KubeClientSet, f.Namespace, service) + err := createServiceWithRetries(f.KubeClientSet, service.Namespace, service) assert.Nil(ginkgo.GinkgoT(), err, "creating service") - return f.GetService(f.Namespace, service.Name) + return f.GetService(service.Namespace, service.Name) } // EnsureDeployment creates a Deployment object and returns it, throws error if it already exists. func (f *Framework) EnsureDeployment(deployment *appsv1.Deployment) *appsv1.Deployment { - err := createDeploymentWithRetries(f.KubeClientSet, f.Namespace, deployment) + err := createDeploymentWithRetries(f.KubeClientSet, deployment.Namespace, deployment) assert.Nil(ginkgo.GinkgoT(), err, "creating deployment") d, err := f.KubeClientSet.AppsV1().Deployments(deployment.Namespace).Get(context.TODO(), deployment.Name, metav1.GetOptions{}) diff --git a/test/e2e/framework/util.go b/test/e2e/framework/util.go index 753e31bfc..3befb8369 100644 --- a/test/e2e/framework/util.go +++ b/test/e2e/framework/util.go @@ -85,14 +85,15 @@ func RestclientConfig(config, context string) (*api.Config, error) { // RunID unique identifier of the e2e run var RunID = uuid.NewUUID() -// CreateKubeNamespace creates a new namespace in the cluster -func CreateKubeNamespace(baseName string, c kubernetes.Interface) (string, error) { +func createNamespace(baseName string, labels map[string]string, c kubernetes.Interface) (string, error) { ts := time.Now().UnixNano() ns := &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ GenerateName: fmt.Sprintf("e2e-tests-%v-%v-", baseName, ts), + Labels: labels, }, } + // Be robust about making the namespace creation call. var got *corev1.Namespace var err error @@ -111,8 +112,20 @@ func CreateKubeNamespace(baseName string, c kubernetes.Interface) (string, error return got.Name, nil } -// deleteKubeNamespace deletes a namespace and all the objects inside -func deleteKubeNamespace(c kubernetes.Interface, namespace string) error { +// CreateKubeNamespace creates a new namespace in the cluster +func CreateKubeNamespace(baseName string, c kubernetes.Interface) (string, error) { + + return createNamespace(baseName, nil, c) +} + +// CreateKubeNamespaceWithLabel creates a new namespace with given labels in the cluster +func CreateKubeNamespaceWithLabel(baseName string, labels map[string]string, c kubernetes.Interface) (string, error) { + + return createNamespace(baseName, labels, c) +} + +// DeleteKubeNamespace deletes a namespace and all the objects inside +func DeleteKubeNamespace(c kubernetes.Interface, namespace string) error { grace := int64(0) pb := metav1.DeletePropagationBackground return c.CoreV1().Namespaces().Delete(context.TODO(), namespace, metav1.DeleteOptions{ diff --git a/test/e2e/settings/namespace_selector.go b/test/e2e/settings/namespace_selector.go new file mode 100644 index 000000000..4fa28826a --- /dev/null +++ b/test/e2e/settings/namespace_selector.go @@ -0,0 +1,123 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package settings + +import ( + "context" + "net/http" + "strings" + + "github.com/onsi/ginkgo" + "github.com/stretchr/testify/assert" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/ingress-nginx/test/e2e/framework" +) + +var _ = framework.IngressNginxDescribe("[Flag] watch namespace selector", func() { + f := framework.NewDefaultFramework("namespace-selector") + notMatchedHost, matchedHost := "bar", "foo" + var notMatchedNs string + var matchedNs string + + // create a test namespace, under which create an ingress and backend deployment + prepareTestIngress := func(baseName string, host string, labels map[string]string) string { + ns, err := framework.CreateKubeNamespaceWithLabel(f.BaseName, labels, f.KubeClientSet) + assert.Nil(ginkgo.GinkgoT(), err, "creating test namespace") + f.NewEchoDeploymentWithNamespaceAndReplicas(ns, 1) + ing := framework.NewSingleIngressWithIngressClass(host, "/", host, ns, framework.EchoService, f.IngressClass, 80, nil) + f.EnsureIngress(ing) + return ns + } + + cleanupNamespace := func(ns string) { + err := framework.DeleteKubeNamespace(f.KubeClientSet, ns) + assert.Nil(ginkgo.GinkgoT(), err, "deleting temporarily crated namespace") + } + + ginkgo.BeforeEach(func() { + notMatchedNs = prepareTestIngress(notMatchedHost, notMatchedHost, nil) // create namespace without label "foo=bar" + matchedNs = prepareTestIngress(matchedHost, matchedHost, map[string]string{"foo": "bar"}) + }) + + ginkgo.AfterEach(func() { + cleanupNamespace(notMatchedNs) + cleanupNamespace(matchedNs) + + // cleanup clusterrole/clusterrolebinding created by installing chart with controller.scope.enabled=false + err := f.KubeClientSet.RbacV1().ClusterRoles().Delete(context.TODO(), "nginx-ingress", metav1.DeleteOptions{}) + assert.Nil(ginkgo.GinkgoT(), err, "deleting clusterrole nginx-ingress") + + err = f.KubeClientSet.RbacV1().ClusterRoleBindings().Delete(context.TODO(), "nginx-ingress", metav1.DeleteOptions{}) + assert.Nil(ginkgo.GinkgoT(), err, "deleting clusterrolebinging nginx-ingress") + }) + + ginkgo.Context("With specific watch-namespace-selector flags", func() { + + ginkgo.It("should ingore Ingress of namespace without label foo=bar and accept those of namespace with label foo=bar", func() { + + f.WaitForNginxConfiguration(func(cfg string) bool { + return !strings.Contains(cfg, "server_name bar") && + strings.Contains(cfg, "server_name foo") + }) + + f.HTTPTestClient(). + GET("/"). + WithHeader("Host", matchedHost). + Expect(). + Status(http.StatusOK) + + f.HTTPTestClient(). + GET("/"). + WithHeader("Host", notMatchedHost). + Expect(). + Status(http.StatusNotFound) + + // should accept Ingress when namespace labeled with foo=bar + ns, err := f.KubeClientSet.CoreV1().Namespaces().Get(context.TODO(), notMatchedNs, metav1.GetOptions{}) + assert.Nil(ginkgo.GinkgoT(), err) + + if ns.Labels == nil { + ns.Labels = make(map[string]string) + } + ns.Labels["foo"] = "bar" + + _, err = f.KubeClientSet.CoreV1().Namespaces().Update(context.TODO(), ns, metav1.UpdateOptions{}) + assert.Nil(ginkgo.GinkgoT(), err, "labeling not matched namespace") + + // update ingress to trigger reconcilation + ing, err := f.KubeClientSet.NetworkingV1().Ingresses(notMatchedNs).Get(context.TODO(), notMatchedHost, metav1.GetOptions{}) + assert.Nil(ginkgo.GinkgoT(), err, "retrieve test ingress") + if ing.Labels == nil { + ing.Labels = make(map[string]string) + } + ing.Labels["foo"] = "bar" + + _, err = f.KubeClientSet.NetworkingV1().Ingresses(notMatchedNs).Update(context.TODO(), ing, metav1.UpdateOptions{}) + assert.Nil(ginkgo.GinkgoT(), err, "updating ingress") + + f.WaitForNginxConfiguration(func(cfg string) bool { + return strings.Contains(cfg, "server_name bar") + }) + + f.HTTPTestClient(). + GET("/"). + WithHeader("Host", notMatchedHost). + Expect(). + Status(http.StatusOK) + }) + }) +})