ingress-nginx-helm/test/e2e/framework/util.go

/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package framework

import (
	"context"
	"fmt"
	"os"
	"time"

	"github.com/onsi/ginkgo/v2"
	corev1 "k8s.io/api/core/v1"
	networkingv1 "k8s.io/api/networking/v1"
	rbacv1 "k8s.io/api/rbac/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/uuid"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/tools/clientcmd"
	"k8s.io/client-go/tools/clientcmd/api"

	"k8s.io/ingress-nginx/internal/k8s"
)

const (
	// Poll how often to poll for conditions
	Poll = 2 * time.Second

	// DefaultTimeout time to wait for operations to complete
	DefaultTimeout = 90 * time.Second
)

func nowStamp() string {
	return time.Now().Format(time.StampMilli)
}

func logf(level, format string, args ...interface{}) {
	fmt.Fprintf(ginkgo.GinkgoWriter, nowStamp()+": "+level+": "+format+"\n", args...)
}

// Logf logs to the INFO logs.
func Logf(format string, args ...interface{}) {
	logf("INFO", format, args...)
}

// Failf logs to the INFO logs and fails the test.
func Failf(format string, args ...interface{}) {
	msg := fmt.Sprintf(format, args...)
	logf("INFO", msg)
	ginkgo.Fail(nowStamp()+": "+msg, 1)
}

// RestclientConfig deserializes the contents of a kubeconfig file into a Config object.
func RestclientConfig(config, newContext string) (*api.Config, error) {
	Logf(">>> config: %s\n", config)
	if config == "" {
		return nil, fmt.Errorf("config file must be specified to load client config")
	}
	c, err := clientcmd.LoadFromFile(config)
	if err != nil {
		return nil, fmt.Errorf("error loading config: %v", err.Error())
	}
	if newContext != "" {
		Logf(">>> context: %s\n", newContext)
		c.CurrentContext = newContext
	}
	return c, nil
}

// RunID unique identifier of the e2e run
var RunID = uuid.NewUUID()

func createNamespace(baseName string, labels map[string]string, c kubernetes.Interface) (string, error) {
	ts := time.Now().UnixNano()
	ns := &corev1.Namespace{
		ObjectMeta: metav1.ObjectMeta{
			GenerateName: fmt.Sprintf("e2e-tests-%v-%v-", baseName, ts),
			Labels:       labels,
		},
	}

	// Be robust about making the namespace creation call.
	var got *corev1.Namespace
	var err error

	//nolint:staticcheck // TODO: will replace it since wait.Poll is deprecated
	err = wait.Poll(Poll, DefaultTimeout, func() (bool, error) {
		got, err = c.CoreV1().Namespaces().Create(context.TODO(), ns, metav1.CreateOptions{})
		if err != nil {
			Logf("Unexpected error while creating namespace: %v", err)
			return false, nil
		}
		return true, nil
	})
	if err != nil {
		return "", err
	}
	return got.Name, nil
}

// CreateKubeNamespace creates a new namespace in the cluster
func CreateKubeNamespace(baseName string, c kubernetes.Interface) (string, error) {
	return createNamespace(baseName, nil, c)
}

// CreateKubeNamespaceWithLabel creates a new namespace with given labels in the cluster
func CreateKubeNamespaceWithLabel(baseName string, labels map[string]string, c kubernetes.Interface) (string, error) {
	return createNamespace(baseName, labels, c)
}

// DeleteKubeNamespace deletes a namespace and all the objects inside
func DeleteKubeNamespace(c kubernetes.Interface, namespace string) error {
	grace := int64(0)
	pb := metav1.DeletePropagationBackground
	return c.CoreV1().Namespaces().Delete(context.TODO(), namespace, metav1.DeleteOptions{
		GracePeriodSeconds: &grace,
		PropagationPolicy:  &pb,
	})
}

// CreateIngressClass creates a new IngressClass related to a test/namespace and also
// the required ClusterRole/ClusterRoleBinding
func CreateIngressClass(namespace string, c kubernetes.Interface) (string, error) {
	icname := fmt.Sprintf("ic-%s", namespace)
	var err error

	ic, err := c.NetworkingV1().IngressClasses().
		Create(context.TODO(), &networkingv1.IngressClass{
			ObjectMeta: metav1.ObjectMeta{
				Name: icname,
			},
			Spec: networkingv1.IngressClassSpec{
				Controller: k8s.IngressNGINXController,
			},
		}, metav1.CreateOptions{})
	if err != nil {
		return "", fmt.Errorf("unexpected error creating IngressClass %s: %v", icname, err)
	}

	_, err = c.RbacV1().ClusterRoles().Create(context.TODO(), &rbacv1.ClusterRole{
		ObjectMeta: metav1.ObjectMeta{Name: icname},
		Rules: []rbacv1.PolicyRule{{
			APIGroups: []string{"networking.k8s.io"},
			Resources: []string{"ingressclasses"},
			Verbs:     []string{"get", "list", "watch"},
		}},
	}, metav1.CreateOptions{})
	if err != nil {
		return "", fmt.Errorf("unexpected error creating IngressClass ClusterRole %s: %v", icname, err)
	}

	_, err = c.RbacV1().ClusterRoleBindings().Create(context.TODO(), &rbacv1.ClusterRoleBinding{
		ObjectMeta: metav1.ObjectMeta{
			Name: icname,
		},
		RoleRef: rbacv1.RoleRef{
			APIGroup: "rbac.authorization.k8s.io",
			Kind:     "ClusterRole",
			Name:     icname,
		},
		Subjects: []rbacv1.Subject{
			{
				APIGroup:  "",
				Kind:      "ServiceAccount",
				Namespace: namespace,
				Name:      "nginx-ingress",
			},
		},
	}, metav1.CreateOptions{})
	if err != nil {
		return "", fmt.Errorf("unexpected error creating IngressClass ClusterRoleBinding %s: %v", icname, err)
	}
	return ic.Name, nil
}

// deleteIngressClass deletes an IngressClass and its related ClusterRole* objects
func deleteIngressClass(c kubernetes.Interface, ingressclass string) error {
	var err error
	grace := int64(0)
	pb := metav1.DeletePropagationBackground
	deleteOptions := metav1.DeleteOptions{
		GracePeriodSeconds: &grace,
		PropagationPolicy:  &pb,
	}
	err = c.NetworkingV1().IngressClasses().Delete(context.TODO(), ingressclass, deleteOptions)
	if err != nil {
		return fmt.Errorf("unexpected error deleting IngressClass %s: %v", ingressclass, err)
	}

	err = c.RbacV1().ClusterRoleBindings().Delete(context.TODO(), ingressclass, deleteOptions)
	if err != nil {
		return fmt.Errorf("unexpected error deleting IngressClass ClusterRoleBinding %s: %v", ingressclass, err)
	}
	err = c.RbacV1().ClusterRoles().Delete(context.TODO(), ingressclass, deleteOptions)
	if err != nil {
		return fmt.Errorf("unexpected error deleting IngressClass ClusterRole %s: %v", ingressclass, err)
	}

	return nil
}

// GetIngressClassName returns the default IngressClassName given a namespace
func GetIngressClassName(namespace string) *string {
	icname := fmt.Sprintf("ic-%s", namespace)
	return &icname
}

// WaitForKubeNamespaceNotExist waits until a namespaces is not present in the cluster
func WaitForKubeNamespaceNotExist(c kubernetes.Interface, namespace string) error {
	//nolint:staticcheck // TODO: will replace it since wait.Poll is deprecated
	return wait.Poll(Poll, DefaultTimeout, namespaceNotExist(c, namespace))
}

func namespaceNotExist(c kubernetes.Interface, namespace string) wait.ConditionFunc {
	return func() (bool, error) {
		_, err := c.CoreV1().Namespaces().Get(context.TODO(), namespace, metav1.GetOptions{})
		if apierrors.IsNotFound(err) {
			return true, nil
		}
		if err != nil {
			return false, err
		}
		return false, nil
	}
}

// WaitForNoPodsInNamespace waits until there are no pods running in a namespace
func WaitForNoPodsInNamespace(c kubernetes.Interface, namespace string) error {
	//nolint:staticcheck // TODO: will replace it since wait.Poll is deprecated
	return wait.Poll(Poll, DefaultTimeout, noPodsInNamespace(c, namespace))
}

func noPodsInNamespace(c kubernetes.Interface, namespace string) wait.ConditionFunc {
	return func() (bool, error) {
		items, err := c.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{})
		if apierrors.IsNotFound(err) {
			return true, nil
		}
		if err != nil {
			return false, err
		}

		if len(items.Items) == 0 {
			return true, nil
		}
		return false, nil
	}
}

// WaitForPodRunningInNamespace waits a default amount of time (PodStartTimeout) for the specified pod to become running.
// Returns an error if timeout occurs first, or pod goes in to failed state.
func WaitForPodRunningInNamespace(c kubernetes.Interface, pod *corev1.Pod) error {
	if pod.Status.Phase == corev1.PodRunning {
		return nil
	}
	return waitTimeoutForPodRunningInNamespace(c, pod.Name, pod.Namespace)
}

func waitTimeoutForPodRunningInNamespace(c kubernetes.Interface, podName, namespace string) error {
	//nolint:staticcheck // TODO: will replace it since wait.Poll is deprecated
	return wait.Poll(Poll, DefaultTimeout, podRunning(c, podName, namespace))
}

// WaitForSecretInNamespace waits a default amount of time for the specified secret is present in a particular namespace
func WaitForSecretInNamespace(c kubernetes.Interface, namespace, name string) error {
	//nolint:staticcheck // TODO: will replace it since wait.Poll is deprecated
	return wait.Poll(Poll, DefaultTimeout, secretInNamespace(c, namespace, name))
}

func secretInNamespace(c kubernetes.Interface, namespace, name string) wait.ConditionFunc {
	return func() (bool, error) {
		s, err := c.CoreV1().Secrets(namespace).Get(context.TODO(), name, metav1.GetOptions{})
		if apierrors.IsNotFound(err) {
			return false, nil
		}
		if err != nil {
			return false, err
		}

		if s != nil {
			return true, nil
		}
		return false, nil
	}
}

// WaitForFileInFS waits a default amount of time for the specified file is present in the filesystem
func WaitForFileInFS(file string) error {
	//nolint:staticcheck // TODO: will replace it since wait.Poll is deprecated
	return wait.Poll(Poll, DefaultTimeout, fileInFS(file))
}

func fileInFS(file string) wait.ConditionFunc {
	return func() (bool, error) {
		stat, err := os.Stat(file)
		if err != nil {
			return false, err
		}

		if stat == nil {
			return false, fmt.Errorf("file %v does not exist", file)
		}

		if stat.Size() > 0 {
			return true, nil
		}

		return false, fmt.Errorf("the file %v exists but it is empty", file)
	}
}

// WaitForNoIngressInNamespace waits until there is no ingress object in a particular namespace
func WaitForNoIngressInNamespace(c kubernetes.Interface, namespace, name string) error {
	//nolint:staticcheck // TODO: will replace it since wait.Poll is deprecated
	return wait.Poll(Poll, DefaultTimeout, noIngressInNamespace(c, namespace, name))
}

func noIngressInNamespace(c kubernetes.Interface, namespace, name string) wait.ConditionFunc {
	return func() (bool, error) {
		ing, err := c.NetworkingV1().Ingresses(namespace).Get(context.TODO(), name, metav1.GetOptions{})
		if apierrors.IsNotFound(err) {
			return true, nil
		}
		if err != nil {
			return false, err
		}

		if ing == nil {
			return true, nil
		}
		return false, nil
	}
}

// WaitForIngressInNamespace waits until a particular ingress object exists namespace
func WaitForIngressInNamespace(c kubernetes.Interface, namespace, name string) error {
	//nolint:staticcheck // TODO: will replace it since wait.Poll is deprecated
	return wait.Poll(Poll, DefaultTimeout, ingressInNamespace(c, namespace, name))
}

func ingressInNamespace(c kubernetes.Interface, namespace, name string) wait.ConditionFunc {
	return func() (bool, error) {
		ing, err := c.NetworkingV1().Ingresses(namespace).Get(context.TODO(), name, metav1.GetOptions{})
		if apierrors.IsNotFound(err) {
			return false, nil
		}
		if err != nil {
			return false, err
		}

		if ing != nil {
			return true, nil
		}
		return false, nil
	}
}

func podRunning(c kubernetes.Interface, podName, namespace string) wait.ConditionFunc {
	return func() (bool, error) {
		pod, err := c.CoreV1().Pods(namespace).Get(context.TODO(), podName, metav1.GetOptions{})
		if err != nil {
			return false, nil
		}
		switch pod.Status.Phase {
		case corev1.PodRunning:
			return true, nil
		case corev1.PodFailed, corev1.PodSucceeded:
			return false, fmt.Errorf("pod ran to completion")
		}
		return false, nil
	}
}

func labelSelectorToString(labels map[string]string) string {
	var str string
	for k, v := range labels {
		str += fmt.Sprintf("%s=%s,", k, v)
	}
	return str[:len(str)-1]
}

// NewInt32 converts int32 to a pointer
func NewInt32(val int32) *int32 {
	p := new(int32)
	*p = val
	return p
}

// NewInt64 converts int64 to a pointer
func NewInt64(val int64) *int64 {
	p := new(int64)
	*p = val
	return p
}