423 lines
13 KiB
Go
423 lines
13 KiB
Go
/*
|
|
Copyright 2019 The Kubernetes Authors.
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package envtest
|
|
|
|
import (
|
|
"context"
|
|
"encoding/base64"
|
|
"fmt"
|
|
"io/ioutil"
|
|
"net"
|
|
"os"
|
|
"path/filepath"
|
|
"time"
|
|
|
|
"k8s.io/apimachinery/pkg/api/errors"
|
|
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
|
"k8s.io/apimachinery/pkg/runtime"
|
|
"k8s.io/apimachinery/pkg/runtime/schema"
|
|
"k8s.io/apimachinery/pkg/util/sets"
|
|
"k8s.io/apimachinery/pkg/util/wait"
|
|
"k8s.io/client-go/rest"
|
|
"sigs.k8s.io/controller-runtime/pkg/client"
|
|
"sigs.k8s.io/controller-runtime/pkg/internal/testing/integration"
|
|
"sigs.k8s.io/controller-runtime/pkg/internal/testing/integration/addr"
|
|
"sigs.k8s.io/yaml"
|
|
)
|
|
|
|
// WebhookInstallOptions are the options for installing mutating or validating webhooks
|
|
type WebhookInstallOptions struct {
|
|
// Paths is a list of paths to the directories containing the mutating or validating webhooks yaml or json configs.
|
|
DirectoryPaths []string
|
|
|
|
// MutatingWebhooks is a list of MutatingWebhookConfigurations to install
|
|
MutatingWebhooks []runtime.Object
|
|
|
|
// ValidatingWebhooks is a list of ValidatingWebhookConfigurations to install
|
|
ValidatingWebhooks []runtime.Object
|
|
|
|
// IgnoreErrorIfPathMissing will ignore an error if a DirectoryPath does not exist when set to true
|
|
IgnoreErrorIfPathMissing bool
|
|
|
|
// LocalServingHost is the host for serving webhooks on.
|
|
// it will be automatically populated
|
|
LocalServingHost string
|
|
|
|
// LocalServingPort is the allocated port for serving webhooks on.
|
|
// it will be automatically populated by a random available local port
|
|
LocalServingPort int
|
|
|
|
// LocalServingCertDir is the allocated directory for serving certificates.
|
|
// it will be automatically populated by the local temp dir
|
|
LocalServingCertDir string
|
|
|
|
// MaxTime is the max time to wait
|
|
MaxTime time.Duration
|
|
|
|
// PollInterval is the interval to check
|
|
PollInterval time.Duration
|
|
}
|
|
|
|
// ModifyWebhookDefinitions modifies webhook definitions by:
|
|
// - applying CABundle based on the provided tinyca
|
|
// - if webhook client config uses service spec, it's removed and replaced with direct url
|
|
func (o *WebhookInstallOptions) ModifyWebhookDefinitions(caData []byte) error {
|
|
hostPort, err := o.generateHostPort()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
for i, unstructuredHook := range runtimeListToUnstructured(o.MutatingWebhooks) {
|
|
webhooks, found, err := unstructured.NestedSlice(unstructuredHook.Object, "webhooks")
|
|
if !found || err != nil {
|
|
return fmt.Errorf("unexpected object, %v", err)
|
|
}
|
|
for j := range webhooks {
|
|
webhook, err := modifyWebhook(webhooks[j].(map[string]interface{}), caData, hostPort)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
webhooks[j] = webhook
|
|
unstructuredHook.Object["webhooks"] = webhooks
|
|
o.MutatingWebhooks[i] = unstructuredHook
|
|
}
|
|
}
|
|
|
|
for i, unstructuredHook := range runtimeListToUnstructured(o.ValidatingWebhooks) {
|
|
webhooks, found, err := unstructured.NestedSlice(unstructuredHook.Object, "webhooks")
|
|
if !found || err != nil {
|
|
return fmt.Errorf("unexpected object, %v", err)
|
|
}
|
|
for j := range webhooks {
|
|
webhook, err := modifyWebhook(webhooks[j].(map[string]interface{}), caData, hostPort)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
webhooks[j] = webhook
|
|
unstructuredHook.Object["webhooks"] = webhooks
|
|
o.ValidatingWebhooks[i] = unstructuredHook
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func modifyWebhook(webhook map[string]interface{}, caData []byte, hostPort string) (map[string]interface{}, error) {
|
|
clientConfig, found, err := unstructured.NestedMap(webhook, "clientConfig")
|
|
if !found || err != nil {
|
|
return nil, fmt.Errorf("cannot find clientconfig: %v", err)
|
|
}
|
|
clientConfig["caBundle"] = base64.StdEncoding.EncodeToString(caData)
|
|
servicePath, found, err := unstructured.NestedString(clientConfig, "service", "path")
|
|
if found && err == nil {
|
|
// we cannot use service in integration tests since we're running controller outside cluster
|
|
// the intent here is that we swap out service for raw address because we don't have an actually standard kube service network.
|
|
// We want to users to be able to use your standard config though
|
|
url := fmt.Sprintf("https://%s/%s", hostPort, servicePath)
|
|
clientConfig["url"] = url
|
|
clientConfig["service"] = nil
|
|
}
|
|
webhook["clientConfig"] = clientConfig
|
|
return webhook, nil
|
|
}
|
|
|
|
func (o *WebhookInstallOptions) generateHostPort() (string, error) {
|
|
port, host, err := addr.Suggest()
|
|
if err != nil {
|
|
return "", fmt.Errorf("unable to grab random port for serving webhooks on: %v", err)
|
|
}
|
|
o.LocalServingPort = port
|
|
o.LocalServingHost = host
|
|
return net.JoinHostPort(host, fmt.Sprintf("%d", port)), nil
|
|
}
|
|
|
|
// Install installs specified webhooks to the API server
|
|
func (o *WebhookInstallOptions) Install(config *rest.Config) error {
|
|
hookCA, err := o.setupCA()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := parseWebhookDirs(o); err != nil {
|
|
return err
|
|
}
|
|
|
|
err = o.ModifyWebhookDefinitions(hookCA)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := createWebhooks(config, o.MutatingWebhooks, o.ValidatingWebhooks); err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := WaitForWebhooks(config, o.MutatingWebhooks, o.ValidatingWebhooks, *o); err != nil {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// Cleanup cleans up cert directories
|
|
func (o *WebhookInstallOptions) Cleanup() error {
|
|
if o.LocalServingCertDir != "" {
|
|
return os.RemoveAll(o.LocalServingCertDir)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// WaitForWebhooks waits for the Webhooks to be available through API server
|
|
func WaitForWebhooks(config *rest.Config,
|
|
mutatingWebhooks []runtime.Object,
|
|
validatingWebhooks []runtime.Object,
|
|
options WebhookInstallOptions) error {
|
|
|
|
waitingFor := map[schema.GroupVersionKind]*sets.String{}
|
|
|
|
for _, hook := range runtimeListToUnstructured(append(validatingWebhooks, mutatingWebhooks...)) {
|
|
if _, ok := waitingFor[hook.GroupVersionKind()]; !ok {
|
|
waitingFor[hook.GroupVersionKind()] = &sets.String{}
|
|
}
|
|
waitingFor[hook.GroupVersionKind()].Insert(hook.GetName())
|
|
}
|
|
|
|
// Poll until all resources are found in discovery
|
|
p := &webhookPoller{config: config, waitingFor: waitingFor}
|
|
return wait.PollImmediate(options.PollInterval, options.MaxTime, p.poll)
|
|
}
|
|
|
|
// poller checks if all the resources have been found in discovery, and returns false if not
|
|
type webhookPoller struct {
|
|
// config is used to get discovery
|
|
config *rest.Config
|
|
|
|
// waitingFor is the map of resources keyed by group version that have not yet been found in discovery
|
|
waitingFor map[schema.GroupVersionKind]*sets.String
|
|
}
|
|
|
|
// poll checks if all the resources have been found in discovery, and returns false if not
|
|
func (p *webhookPoller) poll() (done bool, err error) {
|
|
// Create a new clientset to avoid any client caching of discovery
|
|
c, err := client.New(p.config, client.Options{})
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
|
|
allFound := true
|
|
for gvk, names := range p.waitingFor {
|
|
if names.Len() == 0 {
|
|
delete(p.waitingFor, gvk)
|
|
continue
|
|
}
|
|
for _, name := range names.List() {
|
|
var obj = &unstructured.Unstructured{}
|
|
obj.SetGroupVersionKind(gvk)
|
|
err := c.Get(context.Background(), client.ObjectKey{
|
|
Namespace: "",
|
|
Name: name,
|
|
}, obj)
|
|
|
|
if err == nil {
|
|
names.Delete(name)
|
|
}
|
|
|
|
if errors.IsNotFound(err) {
|
|
allFound = false
|
|
}
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
}
|
|
}
|
|
return allFound, nil
|
|
}
|
|
|
|
// setupCA creates CA for testing and writes them to disk
|
|
func (o *WebhookInstallOptions) setupCA() ([]byte, error) {
|
|
hookCA, err := integration.NewTinyCA()
|
|
if err != nil {
|
|
return nil, fmt.Errorf("unable to set up webhook CA: %v", err)
|
|
}
|
|
|
|
hookCert, err := hookCA.NewServingCert()
|
|
if err != nil {
|
|
return nil, fmt.Errorf("unable to set up webhook serving certs: %v", err)
|
|
}
|
|
|
|
localServingCertsDir, err := ioutil.TempDir("", "envtest-serving-certs-")
|
|
o.LocalServingCertDir = localServingCertsDir
|
|
if err != nil {
|
|
return nil, fmt.Errorf("unable to create directory for webhook serving certs: %v", err)
|
|
}
|
|
|
|
certData, keyData, err := hookCert.AsBytes()
|
|
if err != nil {
|
|
return nil, fmt.Errorf("unable to marshal webhook serving certs: %v", err)
|
|
}
|
|
|
|
if err := ioutil.WriteFile(filepath.Join(localServingCertsDir, "tls.crt"), certData, 0640); err != nil {
|
|
return nil, fmt.Errorf("unable to write webhook serving cert to disk: %v", err)
|
|
}
|
|
if err := ioutil.WriteFile(filepath.Join(localServingCertsDir, "tls.key"), keyData, 0640); err != nil {
|
|
return nil, fmt.Errorf("unable to write webhook serving key to disk: %v", err)
|
|
}
|
|
|
|
return certData, nil
|
|
}
|
|
|
|
func createWebhooks(config *rest.Config, mutHooks []runtime.Object, valHooks []runtime.Object) error {
|
|
cs, err := client.New(config, client.Options{})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Create each webhook
|
|
for _, hook := range runtimeListToUnstructured(mutHooks) {
|
|
log.V(1).Info("installing mutating webhook", "webhook", hook.GetName())
|
|
if err := ensureCreated(cs, hook); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
for _, hook := range runtimeListToUnstructured(valHooks) {
|
|
log.V(1).Info("installing validating webhook", "webhook", hook.GetName())
|
|
if err := ensureCreated(cs, hook); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// ensureCreated creates or update object if already exists in the cluster
|
|
func ensureCreated(cs client.Client, obj *unstructured.Unstructured) error {
|
|
existing := obj.DeepCopy()
|
|
err := cs.Get(context.Background(), client.ObjectKey{Name: obj.GetName()}, existing)
|
|
switch {
|
|
case apierrors.IsNotFound(err):
|
|
if err := cs.Create(context.Background(), obj); err != nil {
|
|
return err
|
|
}
|
|
case err != nil:
|
|
return err
|
|
default:
|
|
log.V(1).Info("Webhook configuration already exists, updating", "webhook", obj.GetName())
|
|
obj.SetResourceVersion(existing.GetResourceVersion())
|
|
if err := cs.Update(context.Background(), obj); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// parseWebhookDirs reads the directories of Webhooks in options.DirectoryPaths and adds the Webhook structs to options
|
|
func parseWebhookDirs(options *WebhookInstallOptions) error {
|
|
if len(options.DirectoryPaths) > 0 {
|
|
for _, path := range options.DirectoryPaths {
|
|
_, err := os.Stat(path)
|
|
if options.IgnoreErrorIfPathMissing && os.IsNotExist(err) {
|
|
continue // skip this path
|
|
}
|
|
if !options.IgnoreErrorIfPathMissing && os.IsNotExist(err) {
|
|
return err // treat missing path as error
|
|
}
|
|
mutHooks, valHooks, err := readWebhooks(path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
options.MutatingWebhooks = append(options.MutatingWebhooks, mutHooks...)
|
|
options.ValidatingWebhooks = append(options.ValidatingWebhooks, valHooks...)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// readWebhooks reads the Webhooks from files and Unmarshals them into structs
|
|
// returns slice of mutating and validating webhook configurations
|
|
func readWebhooks(path string) ([]runtime.Object, []runtime.Object, error) {
|
|
// Get the webhook files
|
|
var files []os.FileInfo
|
|
var err error
|
|
log.V(1).Info("reading Webhooks from path", "path", path)
|
|
if files, err = ioutil.ReadDir(path); err != nil {
|
|
return nil, nil, err
|
|
}
|
|
|
|
// file extensions that may contain Webhooks
|
|
resourceExtensions := sets.NewString(".json", ".yaml", ".yml")
|
|
|
|
var mutHooks []runtime.Object
|
|
var valHooks []runtime.Object
|
|
for _, file := range files {
|
|
// Only parse whitelisted file types
|
|
if !resourceExtensions.Has(filepath.Ext(file.Name())) {
|
|
continue
|
|
}
|
|
|
|
// Unmarshal Webhooks from file into structs
|
|
docs, err := readDocuments(filepath.Join(path, file.Name()))
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
|
|
for _, doc := range docs {
|
|
var generic metav1.PartialObjectMetadata
|
|
if err = yaml.Unmarshal(doc, &generic); err != nil {
|
|
return nil, nil, err
|
|
}
|
|
|
|
const (
|
|
admissionregv1 = "admissionregistration.k8s.io/v1"
|
|
admissionregv1beta1 = "admissionregistration.k8s.io/v1beta1"
|
|
)
|
|
switch {
|
|
case generic.Kind == "MutatingWebhookConfiguration":
|
|
if generic.APIVersion != admissionregv1beta1 && generic.APIVersion != admissionregv1 {
|
|
return nil, nil, fmt.Errorf("only v1beta1 and v1 are supported right now for MutatingWebhookConfiguration (name: %s)", generic.Name)
|
|
}
|
|
hook := &unstructured.Unstructured{}
|
|
if err := yaml.Unmarshal(doc, &hook); err != nil {
|
|
return nil, nil, err
|
|
}
|
|
mutHooks = append(mutHooks, hook)
|
|
case generic.Kind == "ValidatingWebhookConfiguration":
|
|
if generic.APIVersion != admissionregv1beta1 && generic.APIVersion != admissionregv1 {
|
|
return nil, nil, fmt.Errorf("only v1beta1 and v1 are supported right now for ValidatingWebhookConfiguration (name: %s)", generic.Name)
|
|
}
|
|
hook := &unstructured.Unstructured{}
|
|
if err := yaml.Unmarshal(doc, &hook); err != nil {
|
|
return nil, nil, err
|
|
}
|
|
valHooks = append(valHooks, hook)
|
|
default:
|
|
continue
|
|
}
|
|
}
|
|
|
|
log.V(1).Info("read webhooks from file", "file", file.Name())
|
|
}
|
|
return mutHooks, valHooks, nil
|
|
}
|
|
|
|
func runtimeListToUnstructured(l []runtime.Object) []*unstructured.Unstructured {
|
|
res := []*unstructured.Unstructured{}
|
|
for _, obj := range l {
|
|
m, err := runtime.DefaultUnstructuredConverter.ToUnstructured(obj.DeepCopyObject())
|
|
if err != nil {
|
|
continue
|
|
}
|
|
res = append(res, &unstructured.Unstructured{
|
|
Object: m,
|
|
})
|
|
}
|
|
return res
|
|
}
|