Add support for IngressClass and ingress.class annotation

This commit is contained in:
Manuel Alejandro de Brito Fontes 2020-04-20 17:38:50 -04:00
parent d4e0657991
commit efbb3f9fc8
17 changed files with 350 additions and 53 deletions

View file

@ -65,4 +65,12 @@ rules:
- ingresses/status
verbs:
- update
- apiGroups:
- "networking.k8s.io" # k8s 1.14+
resources:
- ingressclasses
verbs:
- get
- list
- watch
{{- end }}

View file

@ -49,6 +49,14 @@ rules:
- ingresses/status
verbs:
- update
- apiGroups:
- "networking.k8s.io" # k8s 1.14+
resources:
- ingressclasses
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:

View file

@ -29,7 +29,6 @@ import (
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"
@ -42,6 +41,7 @@ import (
"k8s.io/klog"
"k8s.io/ingress-nginx/internal/file"
"k8s.io/ingress-nginx/internal/ingress/annotations/class"
"k8s.io/ingress-nginx/internal/ingress/controller"
"k8s.io/ingress-nginx/internal/ingress/metric"
"k8s.io/ingress-nginx/internal/k8s"
@ -87,8 +87,10 @@ func main() {
if errors.IsUnauthorized(err) || errors.IsForbidden(err) {
klog.Fatal("✖ The cluster seems to be running with a restrictive Authorization mode and the Ingress controller does not have the required permissions to operate normally.")
}
klog.Fatalf("No service with name %v found: %v", conf.DefaultService, err)
}
klog.Infof("Validated %v as the default backend.", conf.DefaultService)
}
@ -107,8 +109,28 @@ func main() {
klog.Warningf("Using deprecated \"k8s.io/api/extensions/v1beta1\" package because Kubernetes version is < v1.14.0")
}
if !k8s.IsIngressV1Ready {
klog.Infof("Enabling new Ingress features availables since v1.18.0")
if k8s.IsIngressV1Ready {
klog.Infof("Enabling new Ingress features available since Kubernetes v1.18")
k8s.IngressClass, err = kubeClient.NetworkingV1beta1().IngressClasses().
Get(context.TODO(), class.IngressClass, metav1.GetOptions{})
if err != nil {
if !errors.IsNotFound(err) {
if !errors.IsUnauthorized(err) && !errors.IsForbidden(err) {
klog.Fatalf("Error searching IngressClass: %v", err)
}
klog.Errorf("Unexpected error searching IngressClass: %v", err)
}
klog.Warningf("No IngressClass resource with name %v found. Only annotation will be used.", class.IngressClass)
// TODO: remove once this is fixed in client-go
k8s.IngressClass = nil
}
if k8s.IngressClass != nil && k8s.IngressClass.Spec.Controller != k8s.IngressNGINXController {
klog.Fatalf("IngressClass with name %v is not valid for ingress-nginx (invalid Spec.Controller)", class.IngressClass)
}
}
conf.Client = kubeClient

View file

@ -106,6 +106,14 @@ rules:
- ingresses/status
verbs:
- update
- apiGroups:
- networking.k8s.io # k8s 1.14+
resources:
- ingressclasses
verbs:
- get
- list
- watch
---
# Source: ingress-nginx/templates/clusterrolebinding.yaml
apiVersion: rbac.authorization.k8s.io/v1
@ -184,6 +192,14 @@ rules:
- ingresses/status
verbs:
- update
- apiGroups:
- networking.k8s.io # k8s 1.14+
resources:
- ingressclasses
verbs:
- get
- list
- watch
- apiGroups:
- ''
resources:

View file

@ -99,6 +99,14 @@ rules:
- ingresses/status
verbs:
- update
- apiGroups:
- networking.k8s.io # k8s 1.14+
resources:
- ingressclasses
verbs:
- get
- list
- watch
---
# Source: ingress-nginx/templates/clusterrolebinding.yaml
apiVersion: rbac.authorization.k8s.io/v1
@ -177,6 +185,14 @@ rules:
- ingresses/status
verbs:
- update
- apiGroups:
- networking.k8s.io # k8s 1.14+
resources:
- ingressclasses
verbs:
- get
- list
- watch
- apiGroups:
- ''
resources:

View file

@ -99,6 +99,14 @@ rules:
- ingresses/status
verbs:
- update
- apiGroups:
- networking.k8s.io # k8s 1.14+
resources:
- ingressclasses
verbs:
- get
- list
- watch
---
# Source: ingress-nginx/templates/clusterrolebinding.yaml
apiVersion: rbac.authorization.k8s.io/v1
@ -177,6 +185,14 @@ rules:
- ingresses/status
verbs:
- update
- apiGroups:
- networking.k8s.io # k8s 1.14+
resources:
- ingressclasses
verbs:
- get
- list
- watch
- apiGroups:
- ''
resources:

View file

@ -99,6 +99,14 @@ rules:
- ingresses/status
verbs:
- update
- apiGroups:
- networking.k8s.io # k8s 1.14+
resources:
- ingressclasses
verbs:
- get
- list
- watch
---
# Source: ingress-nginx/templates/clusterrolebinding.yaml
apiVersion: rbac.authorization.k8s.io/v1
@ -177,6 +185,14 @@ rules:
- ingresses/status
verbs:
- update
- apiGroups:
- networking.k8s.io # k8s 1.14+
resources:
- ingressclasses
verbs:
- get
- list
- watch
- apiGroups:
- ''
resources:

View file

@ -99,6 +99,14 @@ rules:
- ingresses/status
verbs:
- update
- apiGroups:
- networking.k8s.io # k8s 1.14+
resources:
- ingressclasses
verbs:
- get
- list
- watch
---
# Source: ingress-nginx/templates/clusterrolebinding.yaml
apiVersion: rbac.authorization.k8s.io/v1
@ -177,6 +185,14 @@ rules:
- ingresses/status
verbs:
- update
- apiGroups:
- networking.k8s.io # k8s 1.14+
resources:
- ingressclasses
verbs:
- get
- list
- watch
- apiGroups:
- ''
resources:

View file

@ -18,6 +18,7 @@ package class
import (
networking "k8s.io/api/networking/v1beta1"
"k8s.io/ingress-nginx/internal/k8s"
)
const (
@ -37,30 +38,30 @@ var (
IngressClass = "nginx"
)
// IsValid returns true if the given Ingress either doesn't specify
// the ingress.class annotation, or it's set to the configured in the
// ingress controller.
// IsValid returns true if the given Ingress specify the ingress.class
// annotation or IngressClassName resource for Kubernetes >= v1.18
func IsValid(ing *networking.Ingress) bool {
className := ing.Spec.IngressClassName
// 1. with annotation
ingress, ok := ing.GetAnnotations()[IngressKey]
if ok {
// empty annotation and same annotation on ingress
if ingress == "" && IngressClass == DefaultClass {
return true
}
// we have 2 valid combinations
// 1 - ingress with default class | blank annotation on ingress
// 2 - ingress with specific class | same annotation on ingress
//
// and 2 invalid combinations
// 3 - ingress with default class | fixed annotation on ingress
// 4 - ingress with specific class | different annotation on ingress
if className != nil {
return *className == IngressClass
return ingress == IngressClass
}
if IngressClass == DefaultClass {
return true
// 2. k8s < v1.18. Check default annotation
if !k8s.IsIngressV1Ready {
return IngressClass == DefaultClass
}
if IngressClass == "" {
return true
// 3. without annotation and IngressClass. Check default annotation
if k8s.IngressClass == nil {
return IngressClass == DefaultClass
}
return false
// 4. with IngressClass
return k8s.IngressClass.Name == *ing.Spec.IngressClassName
}

View file

@ -54,10 +54,10 @@ func TestIsValidClass(t *testing.T) {
},
}
data := map[string]string{}
ing.SetAnnotations(data)
for _, test := range tests {
if test.ingress != "" {
ing.Spec.IngressClassName = &test.ingress
}
ing.Annotations[IngressKey] = test.ingress
IngressClass = test.controller
DefaultClass = test.defClass

View file

@ -190,8 +190,7 @@ func TestCheckIngress(t *testing.T) {
}
t.Run("When the ingress class differs from nginx", func(t *testing.T) {
class := "different"
ing.Spec.IngressClassName = &class
ing.ObjectMeta.Annotations["kubernetes.io/ingress.class"] = "different"
nginx.command = testNginxTestCommand{
t: t,
err: fmt.Errorf("test error"),
@ -202,8 +201,7 @@ func TestCheckIngress(t *testing.T) {
})
t.Run("when the class is the nginx one", func(t *testing.T) {
class := "nginx"
ing.Spec.IngressClassName = &class
ing.ObjectMeta.Annotations["kubernetes.io/ingress.class"] = "nginx"
nginx.command = testNginxTestCommand{
t: t,
err: nil,

View file

@ -949,30 +949,18 @@ func toIngress(obj interface{}) (*networkingv1beta1.Ingress, bool) {
return nil, false
}
ing.Spec.IngressClassName = extractClassName(ing)
setDefaultPathTypeIfEmpty(ing)
return ing, true
}
if ing, ok := obj.(*networkingv1beta1.Ingress); ok {
ing.Spec.IngressClassName = extractClassName(ing)
setDefaultPathTypeIfEmpty(ing)
return ing, true
}
return nil, false
}
func extractClassName(ing *networkingv1beta1.Ingress) *string {
if c, ok := ing.Annotations[class.IngressKey]; ok {
return &c
}
return nil
}
// Default path type is Prefix to not break existing definitions
var defaultPathType = networkingv1beta1.PathTypePrefix

View file

@ -25,6 +25,7 @@ import (
"k8s.io/klog"
apiv1 "k8s.io/api/core/v1"
networkingv1beta1 "k8s.io/api/networking/v1beta1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/version"
clientset "k8s.io/client-go/kubernetes"
@ -121,6 +122,13 @@ var IsNetworkingIngressAvailable bool
// IsIngressV1Ready indicates if the running Kubernetes version is at least v1.18.0
var IsIngressV1Ready bool
// IngressClass indicates the class of the Ingress to use as filter
var IngressClass *networkingv1beta1.IngressClass
// IngressNGINXController defines the valid value of IngressClass
// Controller field for ingress-nginx
const IngressNGINXController = "k8s.io/ingress-nginx"
// NetworkingIngressAvailable checks if the package "k8s.io/api/networking/v1beta1"
// is available or not and if Ingress V1 is supported (k8s >= v1.18.0)
func NetworkingIngressAvailable(client clientset.Interface) (bool, bool) {

View file

@ -55,7 +55,7 @@ func (f *Framework) GetLbAlgorithm(serviceName string, servicePort int) (string,
// ExecIngressPod executes a command inside the first container in ingress controller running pod
func (f *Framework) ExecIngressPod(command string) (string, error) {
pod, err := getIngressNGINXPod(f.Namespace, f.KubeClientSet)
pod, err := GetIngressNGINXPod(f.Namespace, f.KubeClientSet)
if err != nil {
return "", err
}

View file

@ -35,6 +35,7 @@ import (
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/kubernetes"
restclient "k8s.io/client-go/rest"
"k8s.io/ingress-nginx/internal/k8s"
"k8s.io/klog"
)
@ -56,6 +57,8 @@ var (
type Framework struct {
BaseName string
IsIngressV1Ready bool
// A Kubernetes and Service Catalog client
KubeClientSet kubernetes.Interface
KubeConfig *restclient.Config
@ -78,10 +81,13 @@ func NewDefaultFramework(baseName string) *Framework {
kubeClient, err := kubernetes.NewForConfig(kubeConfig)
assert.Nil(ginkgo.GinkgoT(), err, "creating Kubernetes API client")
_, isIngressV1Ready := k8s.NetworkingIngressAvailable(kubeClient)
f := &Framework{
BaseName: baseName,
KubeConfig: kubeConfig,
KubeClientSet: kubeClient,
BaseName: baseName,
KubeConfig: kubeConfig,
KubeClientSet: kubeClient,
IsIngressV1Ready: isIngressV1Ready,
}
ginkgo.BeforeEach(f.BeforeEach)
@ -109,7 +115,7 @@ func (f *Framework) BeforeEach() {
// AfterEach deletes the namespace, after reading its events.
func (f *Framework) AfterEach() {
if ginkgo.CurrentGinkgoTestDescription().Failed {
pod, err := getIngressNGINXPod(f.Namespace, f.KubeClientSet)
pod, err := GetIngressNGINXPod(f.Namespace, f.KubeClientSet)
if err != nil {
Logf("Unexpected error searching for ingress controller pod: %v", err)
return
@ -221,7 +227,7 @@ func (f *Framework) WaitForNginxCustomConfiguration(from string, to string, matc
}
func nginxLogs(client kubernetes.Interface, namespace string) (string, error) {
pod, err := getIngressNGINXPod(namespace, client)
pod, err := GetIngressNGINXPod(namespace, client)
if err != nil {
return "", err
}
@ -240,7 +246,7 @@ func (f *Framework) NginxLogs() (string, error) {
func (f *Framework) matchNginxConditions(name string, matcher func(cfg string) bool) wait.ConditionFunc {
return func() (bool, error) {
pod, err := getIngressNGINXPod(f.Namespace, f.KubeClientSet)
pod, err := GetIngressNGINXPod(f.Namespace, f.KubeClientSet)
if err != nil {
return false, nil
}
@ -272,7 +278,7 @@ func (f *Framework) matchNginxConditions(name string, matcher func(cfg string) b
func (f *Framework) matchNginxCustomConditions(from string, to string, matcher func(cfg string) bool) wait.ConditionFunc {
return func() (bool, error) {
pod, err := getIngressNGINXPod(f.Namespace, f.KubeClientSet)
pod, err := GetIngressNGINXPod(f.Namespace, f.KubeClientSet)
if err != nil {
return false, nil
}
@ -367,14 +373,14 @@ func (f *Framework) UpdateNginxConfigMapData(key string, value string) {
// Grace period to wait for pod shutdown is in seconds.
func (f *Framework) DeleteNGINXPod(grace int64) {
ns := f.Namespace
pod, err := getIngressNGINXPod(ns, f.KubeClientSet)
pod, err := GetIngressNGINXPod(ns, f.KubeClientSet)
assert.Nil(ginkgo.GinkgoT(), err, "expected ingress nginx pod to be running")
err = f.KubeClientSet.CoreV1().Pods(ns).Delete(context.TODO(), pod.GetName(), *metav1.NewDeleteOptions(grace))
assert.Nil(ginkgo.GinkgoT(), err, "deleting ingress nginx pod")
err = wait.Poll(Poll, DefaultTimeout, func() (bool, error) {
pod, err := getIngressNGINXPod(ns, f.KubeClientSet)
pod, err := GetIngressNGINXPod(ns, f.KubeClientSet)
if err != nil || pod == nil {
return false, nil
}

View file

@ -213,7 +213,8 @@ func podRunningReady(p *core.Pod) (bool, error) {
return true, nil
}
func getIngressNGINXPod(ns string, kubeClientSet kubernetes.Interface) (*core.Pod, error) {
// GetIngressNGINXPod returns the ingress controller running pod
func GetIngressNGINXPod(ns string, kubeClientSet kubernetes.Interface) (*core.Pod, error) {
l, err := kubeClientSet.CoreV1().Pods(ns).List(context.TODO(), metav1.ListOptions{
LabelSelector: "app.kubernetes.io/name=ingress-nginx",
})

View file

@ -18,27 +18,51 @@ package settings
import (
"context"
"fmt"
"net/http"
"strings"
"time"
"github.com/onsi/ginkgo"
"github.com/stretchr/testify/assert"
appsv1 "k8s.io/api/apps/v1"
networking "k8s.io/api/networking/v1beta1"
rbacv1 "k8s.io/api/rbac/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/ingress-nginx/internal/ingress/annotations/class"
"k8s.io/ingress-nginx/internal/k8s"
"k8s.io/ingress-nginx/test/e2e/framework"
)
var _ = framework.IngressNginxDescribe("[Flag] ingress-class", func() {
f := framework.NewDefaultFramework("ingress-class")
f.KubeClientSet.RbacV1().ClusterRoles().Create(context.TODO(), &rbacv1.ClusterRole{
ObjectMeta: metav1.ObjectMeta{Name: "ingress-nginx-class"},
Rules: []rbacv1.PolicyRule{{
APIGroups: []string{"networking.k8s.io"},
Resources: []string{"ingressclasses"},
Verbs: []string{"get", "list", "watch"},
}},
}, metav1.CreateOptions{})
f.KubeClientSet.RbacV1().ClusterRoleBindings().Create(context.TODO(), &rbacv1.ClusterRoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: "ingress-nginx-class",
},
RoleRef: rbacv1.RoleRef{
APIGroup: "rbac.authorization.k8s.io",
Kind: "ClusterRole",
Name: "ingress-nginx-class",
},
}, metav1.CreateOptions{})
ginkgo.BeforeEach(func() {
f.NewEchoDeploymentWithReplicas(1)
})
ginkgo.Context("Without a specific ingress-class", func() {
ginkgo.It("should ignore Ingress with class", func() {
invalidHost := "foo"
annotations := map[string]string{
@ -74,7 +98,15 @@ var _ = framework.IngressNginxDescribe("[Flag] ingress-class", func() {
ginkgo.BeforeEach(func() {
err := framework.UpdateDeployment(f.KubeClientSet, f.Namespace, "nginx-ingress-controller", 1,
func(deployment *appsv1.Deployment) error {
args := deployment.Spec.Template.Spec.Containers[0].Args
args := []string{}
for _, v := range deployment.Spec.Template.Spec.Containers[0].Args {
if strings.Contains(v, "--ingress-class") {
continue
}
args = append(args, v)
}
args = append(args, "--ingress-class=testclass")
deployment.Spec.Template.Spec.Containers[0].Args = args
_, err := f.KubeClientSet.AppsV1().Deployments(f.Namespace).Update(context.TODO(), deployment, metav1.UpdateOptions{})
@ -154,4 +186,149 @@ var _ = framework.IngressNginxDescribe("[Flag] ingress-class", func() {
Status(http.StatusNotFound)
})
})
ginkgo.It("check scenarios for IngressClass and ingress.class annotation", func() {
if !f.IsIngressV1Ready {
ginkgo.Skip("Test requires Kubernetes v1.18 or higher")
}
ingressClassName := "test-new-ingress-class"
ingressClass, err := f.KubeClientSet.NetworkingV1beta1().IngressClasses().
Create(context.TODO(), &networking.IngressClass{
ObjectMeta: metav1.ObjectMeta{
Name: ingressClassName,
},
Spec: networking.IngressClassSpec{
Controller: k8s.IngressNGINXController,
},
}, metav1.CreateOptions{})
if ingressClass == nil {
assert.Nil(ginkgo.GinkgoT(), err, "creating IngressClass")
}
pod, err := framework.GetIngressNGINXPod(f.Namespace, f.KubeClientSet)
assert.Nil(ginkgo.GinkgoT(), err, "searching ingress controller pod")
serviceAccount := pod.Spec.ServiceAccountName
crb, err := f.KubeClientSet.RbacV1().ClusterRoleBindings().Get(context.Background(), "ingress-nginx-class", metav1.GetOptions{})
assert.Nil(ginkgo.GinkgoT(), err, "searching cluster role binding")
// add service of current namespace
crb.Subjects = append(crb.Subjects, rbacv1.Subject{
APIGroup: "",
Kind: "ServiceAccount",
Name: serviceAccount,
Namespace: f.Namespace,
})
_, err = f.KubeClientSet.RbacV1().ClusterRoleBindings().Update(context.Background(), crb, metav1.UpdateOptions{})
assert.Nil(ginkgo.GinkgoT(), err, "searching cluster role binding")
err = framework.UpdateDeployment(f.KubeClientSet, f.Namespace, "nginx-ingress-controller", 1,
func(deployment *appsv1.Deployment) error {
args := []string{}
for _, v := range deployment.Spec.Template.Spec.Containers[0].Args {
if strings.Contains(v, "--ingress-class") {
continue
}
args = append(args, v)
}
args = append(args, fmt.Sprintf("--ingress-class=%v", ingressClassName))
deployment.Spec.Template.Spec.Containers[0].Args = args
_, err := f.KubeClientSet.AppsV1().Deployments(f.Namespace).Update(context.TODO(), deployment, metav1.UpdateOptions{})
return err
})
assert.Nil(ginkgo.GinkgoT(), err, "updating ingress controller deployment flags")
host := "ingress.class"
ginkgo.By("only having IngressClassName")
ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, nil)
ing.Spec.IngressClassName = &ingressClassName
f.EnsureIngress(ing)
f.WaitForNginxServer(host, func(cfg string) bool {
return strings.Contains(cfg, fmt.Sprintf("server_name %v", host))
})
f.HTTPTestClient().
GET("/").
WithHeader("Host", host).
Expect().
Status(http.StatusOK)
ginkgo.By("only having ingress.class annotation")
ing, err = f.KubeClientSet.NetworkingV1beta1().Ingresses(f.Namespace).Get(context.TODO(), host, metav1.GetOptions{})
assert.Nil(ginkgo.GinkgoT(), err)
ing.Annotations = map[string]string{
class.IngressKey: ingressClassName,
}
ing.Spec.IngressClassName = nil
_, err = f.KubeClientSet.NetworkingV1beta1().Ingresses(f.Namespace).Update(context.TODO(), ing, metav1.UpdateOptions{})
assert.Nil(ginkgo.GinkgoT(), err)
f.WaitForNginxConfiguration(func(cfg string) bool {
return strings.Contains(cfg, fmt.Sprintf("server_name %v", host))
})
time.Sleep(2 * time.Second)
f.HTTPTestClient().
GET("/").
WithHeader("Host", host).
Expect().
Status(http.StatusOK)
ginkgo.By("having an invalid ingress.class annotation and no IngressClassName")
ing, err = f.KubeClientSet.NetworkingV1beta1().Ingresses(f.Namespace).Get(context.TODO(), host, metav1.GetOptions{})
assert.Nil(ginkgo.GinkgoT(), err)
ing.Annotations = map[string]string{
class.IngressKey: "invalid",
}
ing.Spec.IngressClassName = nil
_, err = f.KubeClientSet.NetworkingV1beta1().Ingresses(f.Namespace).Update(context.TODO(), ing, metav1.UpdateOptions{})
assert.Nil(ginkgo.GinkgoT(), err)
time.Sleep(2 * time.Second)
f.WaitForNginxConfiguration(func(cfg string) bool {
return !strings.Contains(cfg, fmt.Sprintf("server_name %v", host))
})
f.HTTPTestClient().
GET("/").
WithHeader("Host", host).
Expect().
Status(http.StatusNotFound)
ginkgo.By("not having ingress.class annotation and invalid IngressClassName")
ing, err = f.KubeClientSet.NetworkingV1beta1().Ingresses(f.Namespace).Get(context.TODO(), host, metav1.GetOptions{})
assert.Nil(ginkgo.GinkgoT(), err)
ing.Annotations = map[string]string{}
invalidClassName := "invalidclass"
ing.Spec.IngressClassName = &invalidClassName
_, err = f.KubeClientSet.NetworkingV1beta1().Ingresses(f.Namespace).Update(context.TODO(), ing, metav1.UpdateOptions{})
assert.Nil(ginkgo.GinkgoT(), err)
time.Sleep(2 * time.Second)
f.WaitForNginxConfiguration(func(cfg string) bool {
return !strings.Contains(cfg, fmt.Sprintf("server_name %v", host))
})
f.HTTPTestClient().
GET("/").
WithHeader("Host", host).
Expect().
Status(http.StatusNotFound)
})
})