From c3b896dfbc547379cd1936adb176291ecec91c6c Mon Sep 17 00:00:00 2001 From: Lorenzo Fontana Date: Sat, 19 May 2018 16:45:17 +0200 Subject: [PATCH] InfluxDB annotations e2e tests Signed-off-by: Lorenzo Fontana --- .../ingress/annotations/influxdb/main_test.go | 4 +- test/e2e/annotations/influxdb.go | 193 ++++++++++++++++++ test/e2e/framework/influxdb.go | 162 +++++++++++++++ test/e2e/framework/k8s.go | 12 ++ 4 files changed, 369 insertions(+), 2 deletions(-) create mode 100644 test/e2e/annotations/influxdb.go create mode 100644 test/e2e/framework/influxdb.go diff --git a/internal/ingress/annotations/influxdb/main_test.go b/internal/ingress/annotations/influxdb/main_test.go index f9ddef59b..b8c67ba59 100644 --- a/internal/ingress/annotations/influxdb/main_test.go +++ b/internal/ingress/annotations/influxdb/main_test.go @@ -69,7 +69,7 @@ func TestIngressInfluxDB(t *testing.T) { data[parser.GetAnnotationWithPrefix("enable-influxdb")] = "true" data[parser.GetAnnotationWithPrefix("influxdb-measurement")] = "nginxmeasures" data[parser.GetAnnotationWithPrefix("influxdb-port")] = "9091" - data[parser.GetAnnotationWithPrefix("influxdb-host")] = "mytelegrafserver.mycompany.mytld" + data[parser.GetAnnotationWithPrefix("influxdb-host")] = "10.99.0.13" data[parser.GetAnnotationWithPrefix("influxdb-server-name")] = "nginx-test-1" ing.SetAnnotations(data) @@ -91,7 +91,7 @@ func TestIngressInfluxDB(t *testing.T) { t.Errorf("expected port not found. Found %v", nginxInflux.InfluxDBPort) } - if nginxInflux.InfluxDBHost != "mytelegrafserver.mycompany.mytld" { + if nginxInflux.InfluxDBHost != "10.99.0.13" { t.Errorf("expected host not found. Found %v", nginxInflux.InfluxDBHost) } diff --git a/test/e2e/annotations/influxdb.go b/test/e2e/annotations/influxdb.go new file mode 100644 index 000000000..f72825ba5 --- /dev/null +++ b/test/e2e/annotations/influxdb.go @@ -0,0 +1,193 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package annotations + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "os/exec" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/parnurzeal/gorequest" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/ingress-nginx/test/e2e/framework" +) + +var _ = framework.IngressNginxDescribe("Annotations - influxdb", func() { + f := framework.NewDefaultFramework("influxdb") + + BeforeEach(func() { + err := f.NewInfluxDBDeployment() + Expect(err).NotTo(HaveOccurred()) + err = f.NewEchoDeployment() + Expect(err).NotTo(HaveOccurred()) + }) + + Context("when influxdb is enabled", func() { + It("should send the request metric to the influxdb server", func() { + ifs, err := createInfluxDBService(f) + + Expect(err).NotTo(HaveOccurred()) + + // Ingress configured with InfluxDB annotations + host := "influxdb.e2e.local" + createInfluxDBIngress( + f, + host, + "http-svc", + 8080, + map[string]string{ + "nginx.ingress.kubernetes.io/enable-influxdb": "true", + "nginx.ingress.kubernetes.io/influxdb-host": ifs.Spec.ClusterIP, + "nginx.ingress.kubernetes.io/influxdb-port": "8089", + "nginx.ingress.kubernetes.io/influxdb-measurement": "requests", + "nginx.ingress.kubernetes.io/influxdb-servername": "e2e-nginx-srv", + }, + ) + + // Do a request to the echo server ingress that sends metrics + // to the InfluxDB backend. + res, _, errs := gorequest.New(). + Get(f.IngressController.HTTPURL). + Set("Host", host). + End() + + Expect(len(errs)).Should(Equal(0)) + Expect(res.StatusCode).Should(Equal(http.StatusOK)) + + time.Sleep(5 * time.Second) + measurements, err := extractInfluxDBMeasurements(f) + + var results map[string][]map[string]interface{} + json.Unmarshal([]byte(measurements), &results) + + Expect(err).NotTo(HaveOccurred()) + Expect(len(measurements)).ShouldNot(Equal(0)) + for _, elem := range results["results"] { + Expect(len(elem)).ShouldNot(Equal(0)) + } + }) + }) +}) + +func createInfluxDBService(f *framework.Framework) (*corev1.Service, error) { + service := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "inflxudb-svc", + Namespace: f.IngressController.Namespace, + }, + Spec: corev1.ServiceSpec{Ports: []corev1.ServicePort{ + { + Name: "udp", + Port: 8089, + TargetPort: intstr.FromInt(8089), + Protocol: "UDP", + }, + }, + Selector: map[string]string{ + "app": "influxdb-svc", + }, + }, + } + + s, err := f.EnsureService(service) + if err != nil { + return nil, err + } + + if s == nil { + return nil, fmt.Errorf("unexpected error creating service for influxdb deployment") + } + + return s, nil +} + +func createInfluxDBIngress(f *framework.Framework, host, service string, port int, annotations map[string]string) { + ing, err := f.EnsureIngress(framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, service, port, &annotations)) + Expect(err).NotTo(HaveOccurred()) + Expect(ing).NotTo(BeNil()) + + err = f.WaitForNginxServer(host, + func(server string) bool { + return Expect(server).Should(ContainSubstring(fmt.Sprintf("server_name %v", host))) && + Expect(server).ShouldNot(ContainSubstring("return 503")) + }) + Expect(err).NotTo(HaveOccurred()) +} + +func extractInfluxDBMeasurements(f *framework.Framework) (string, error) { + l, err := f.KubeClientSet.CoreV1().Pods(f.IngressController.Namespace).List(metav1.ListOptions{ + LabelSelector: "app=influxdb-svc", + }) + if err != nil { + return "", err + } + + if len(l.Items) == 0 { + return "", err + } + + cmd := "influx -database 'nginx' -execute 'select * from requests' -format 'json' -pretty" + + var pod *corev1.Pod + for _, p := range l.Items { + pod = &p + break + } + + if pod == nil { + return "", fmt.Errorf("no influxdb pods found") + } + + o, err := execInfluxDBCommand(pod, cmd) + if err != nil { + return "", err + } + + return o, nil +} + +func execInfluxDBCommand(pod *corev1.Pod, command string) (string, error) { + var ( + execOut bytes.Buffer + execErr bytes.Buffer + ) + + args := fmt.Sprintf("kubectl exec --namespace %v %v -- %v", pod.Namespace, pod.Name, command) + cmd := exec.Command("/bin/bash", "-c", args) + cmd.Stdout = &execOut + cmd.Stderr = &execErr + + err := cmd.Run() + + if execErr.Len() > 0 { + return "", fmt.Errorf("stderr: %v", execErr.String()) + } + + if err != nil { + return "", fmt.Errorf("could not execute: %v", err) + } + + return execOut.String(), nil +} diff --git a/test/e2e/framework/influxdb.go b/test/e2e/framework/influxdb.go new file mode 100644 index 000000000..82ef69b80 --- /dev/null +++ b/test/e2e/framework/influxdb.go @@ -0,0 +1,162 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +import ( + "fmt" + "time" + + "github.com/pkg/errors" + + corev1 "k8s.io/api/core/v1" + extensions "k8s.io/api/extensions/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" +) + +const influxConfig = ` +reporting-disabled = true +bind-address = "0.0.0.0:8088" + +[meta] + dir = "/var/lib/influxdb/meta" + retention-autocreate = true + logging-enabled = true + +[data] + dir = "/var/lib/influxdb/data" + index-version = "inmem" + wal-dir = "/var/lib/influxdb/wal" + wal-fsync-delay = "0s" + query-log-enabled = true + cache-max-memory-size = 1073741824 + cache-snapshot-memory-size = 26214400 + cache-snapshot-write-cold-duration = "10m0s" + compact-full-write-cold-duration = "4h0m0s" + max-series-per-database = 1000000 + max-values-per-tag = 100000 + max-concurrent-compactions = 0 + trace-logging-enabled = false + +[[udp]] + enabled = true + bind-address = ":8089" + database = "nginx" +` + +// NewInfluxDBDeployment creates an InfluxDB server configured to reply +// on 8086/tcp and 8089/udp +func (f *Framework) NewInfluxDBDeployment() error { + configuration := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "influxdb-config", + Namespace: f.IngressController.Namespace, + }, + Data: map[string]string{ + "influxd.conf": influxConfig, + }, + } + + cm, err := f.EnsureConfigMap(configuration) + if err != nil { + return err + } + + if cm == nil { + return fmt.Errorf("unexpected error creating configmap for influxdb") + } + + deployment := &extensions.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "influxdb-svc", + Namespace: f.IngressController.Namespace, + }, + Spec: extensions.DeploymentSpec{ + Replicas: NewInt32(1), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "influxdb-svc", + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "influxdb-svc", + }, + }, + Spec: corev1.PodSpec{ + TerminationGracePeriodSeconds: NewInt64(0), + Volumes: []corev1.Volume{ + { + Name: "influxdb-config", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "influxdb-config", + }, + }, + }, + }, + }, + Containers: []corev1.Container{ + { + Name: "influxdb-svc", + Image: "docker.io/influxdb:1.5", + Env: []corev1.EnvVar{}, + Command: []string{"influxd", "-config", "/influxdb-config/influxd.conf"}, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "influxdb-config", + ReadOnly: true, + MountPath: "/influxdb-config", + }, + }, + Ports: []corev1.ContainerPort{ + { + Name: "http", + ContainerPort: 8086, + }, + { + Name: "udp", + ContainerPort: 8089, + }, + }, + }, + }, + }, + }, + }, + } + + d, err := f.EnsureDeployment(deployment) + if err != nil { + return err + } + + if d == nil { + return fmt.Errorf("unexpected error creating deployement for influxdb") + } + + err = WaitForPodsReady(f.KubeClientSet, 5*time.Minute, 1, f.IngressController.Namespace, metav1.ListOptions{ + LabelSelector: fields.SelectorFromSet(fields.Set(d.Spec.Template.ObjectMeta.Labels)).String(), + }) + if err != nil { + return errors.Wrap(err, "failed to wait for influxdb to become ready") + } + + return nil +} diff --git a/test/e2e/framework/k8s.go b/test/e2e/framework/k8s.go index db21765ed..e5594497f 100644 --- a/test/e2e/framework/k8s.go +++ b/test/e2e/framework/k8s.go @@ -42,6 +42,18 @@ func (f *Framework) EnsureSecret(secret *api.Secret) (*api.Secret, error) { return s, nil } +// EnsureConfigMap creates a ConfigMap object or returns it if it already exists. +func (f *Framework) EnsureConfigMap(configMap *api.ConfigMap) (*api.ConfigMap, error) { + cm, err := f.KubeClientSet.CoreV1().ConfigMaps(configMap.Namespace).Create(configMap) + if err != nil { + if k8sErrors.IsAlreadyExists(err) { + return f.KubeClientSet.CoreV1().ConfigMaps(configMap.Namespace).Update(configMap) + } + return nil, err + } + return cm, nil +} + // EnsureIngress creates an Ingress object or returns it if it already exists. func (f *Framework) EnsureIngress(ingress *extensions.Ingress) (*extensions.Ingress, error) { s, err := f.KubeClientSet.ExtensionsV1beta1().Ingresses(ingress.Namespace).Update(ingress)