From ce9deaa33280c6125d67a4da765a94114d048862 Mon Sep 17 00:00:00 2001 From: Tobias Salzmann <796084+Eun@users.noreply.github.com> Date: Thu, 23 Dec 2021 20:46:30 +0100 Subject: [PATCH] Add stream-snippet as a ConfigMap and Annotation option (#8029) * stream snippet * gofmt -s --- .../nginx-configuration/annotations.md | 18 +++ .../nginx-configuration/configmap.md | 5 + internal/ingress/annotations/annotations.go | 3 + .../ingress/annotations/streamsnippet/main.go | 40 +++++ .../annotations/streamsnippet/main_test.go | 64 ++++++++ internal/ingress/controller/config/config.go | 12 +- internal/ingress/controller/controller.go | 17 +++ internal/ingress/controller/nginx.go | 1 + internal/ingress/types.go | 2 + rootfs/etc/nginx/template/nginx.tmpl | 5 + test/e2e/annotations/streamsnippet.go | 138 ++++++++++++++++++ test/e2e/settings/stream_snippet.go | 85 +++++++++++ 12 files changed, 386 insertions(+), 4 deletions(-) create mode 100644 internal/ingress/annotations/streamsnippet/main.go create mode 100644 internal/ingress/annotations/streamsnippet/main_test.go create mode 100644 test/e2e/annotations/streamsnippet.go create mode 100644 test/e2e/settings/stream_snippet.go diff --git a/docs/user-guide/nginx-configuration/annotations.md b/docs/user-guide/nginx-configuration/annotations.md index 46f7c23d4..5a217c27d 100755 --- a/docs/user-guide/nginx-configuration/annotations.md +++ b/docs/user-guide/nginx-configuration/annotations.md @@ -99,6 +99,7 @@ You can add these Kubernetes annotations to specific Ingress objects to customiz |[nginx.ingress.kubernetes.io/session-cookie-conditional-samesite-none](#cookie-affinity)|"true" or "false"| |[nginx.ingress.kubernetes.io/ssl-redirect](#server-side-https-enforcement-through-redirect)|"true" or "false"| |[nginx.ingress.kubernetes.io/ssl-passthrough](#ssl-passthrough)|"true" or "false"| +|[nginx.ingress.kubernetes.io/stream-snippet](#stream-snippet)|string| |[nginx.ingress.kubernetes.io/upstream-hash-by](#custom-nginx-upstream-hashing)|string| |[nginx.ingress.kubernetes.io/x-forwarded-prefix](#x-forwarded-prefix-header)|string| |[nginx.ingress.kubernetes.io/load-balance](#custom-nginx-load-balancing)|string| @@ -927,3 +928,20 @@ nginx.ingress.kubernetes.io/mirror-request-body: "off" The request sent to the mirror is linked to the original request. If you have a slow mirror backend, then the original request will throttle. For more information on the mirror module see [ngx_http_mirror_module](https://nginx.org/en/docs/http/ngx_http_mirror_module.html) + + +### Stream snippet + +Using the annotation `nginx.ingress.kubernetes.io/stream-snippet` it is possible to add custom stream configuration. + +```yaml +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + annotations: + nginx.ingress.kubernetes.io/stream-snippet: | + server { + listen 8000; + proxy_pass 127.0.0.1:80; + } +``` \ No newline at end of file diff --git a/docs/user-guide/nginx-configuration/configmap.md b/docs/user-guide/nginx-configuration/configmap.md index b217eee49..d35a19b1a 100755 --- a/docs/user-guide/nginx-configuration/configmap.md +++ b/docs/user-guide/nginx-configuration/configmap.md @@ -156,6 +156,7 @@ The following table shows a configuration option's name, type, and the default v |[main-snippet](#main-snippet)|string|""| |[http-snippet](#http-snippet)|string|""| |[server-snippet](#server-snippet)|string|""| +|[stream-snippet](#stream-snippet)|string|""| |[location-snippet](#location-snippet)|string|""| |[custom-http-errors](#custom-http-errors)|[]int|[]int{}| |[proxy-body-size](#proxy-body-size)|string|"1m"| @@ -988,6 +989,10 @@ Adds custom configuration to the http section of the nginx configuration. Adds custom configuration to all the servers in the nginx configuration. +## stream-snippet + +Adds custom configuration to the stream section of the nginx configuration. + ## location-snippet Adds custom configuration to all the locations in the nginx configuration. diff --git a/internal/ingress/annotations/annotations.go b/internal/ingress/annotations/annotations.go index 9fb53dd1e..fe7400ac7 100644 --- a/internal/ingress/annotations/annotations.go +++ b/internal/ingress/annotations/annotations.go @@ -22,6 +22,7 @@ import ( "k8s.io/ingress-nginx/internal/ingress/annotations/modsecurity" "k8s.io/ingress-nginx/internal/ingress/annotations/proxyssl" "k8s.io/ingress-nginx/internal/ingress/annotations/sslcipher" + "k8s.io/ingress-nginx/internal/ingress/annotations/streamsnippet" "k8s.io/klog/v2" apiv1 "k8s.io/api/core/v1" @@ -115,6 +116,7 @@ type Ingress struct { InfluxDB influxdb.Config ModSecurity modsecurity.Config Mirror mirror.Config + StreamSnippet string } // Extractor defines the annotation parsers to be used in the extraction of annotations @@ -165,6 +167,7 @@ func NewAnnotationExtractor(cfg resolver.Resolver) Extractor { "BackendProtocol": backendprotocol.NewParser(cfg), "ModSecurity": modsecurity.NewParser(cfg), "Mirror": mirror.NewParser(cfg), + "StreamSnippet": streamsnippet.NewParser(cfg), }, } } diff --git a/internal/ingress/annotations/streamsnippet/main.go b/internal/ingress/annotations/streamsnippet/main.go new file mode 100644 index 000000000..fb22f754c --- /dev/null +++ b/internal/ingress/annotations/streamsnippet/main.go @@ -0,0 +1,40 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package streamsnippet + +import ( + networking "k8s.io/api/networking/v1" + + "k8s.io/ingress-nginx/internal/ingress/annotations/parser" + "k8s.io/ingress-nginx/internal/ingress/resolver" +) + +type streamSnippet struct { + r resolver.Resolver +} + +// NewParser creates a new server snippet annotation parser +func NewParser(r resolver.Resolver) parser.IngressAnnotation { + return streamSnippet{r} +} + +// Parse parses the annotations contained in the ingress rule +// used to indicate if the location/s contains a fragment of +// configuration to be included inside the paths of the rules +func (a streamSnippet) Parse(ing *networking.Ingress) (interface{}, error) { + return parser.GetStringAnnotation("stream-snippet", ing) +} diff --git a/internal/ingress/annotations/streamsnippet/main_test.go b/internal/ingress/annotations/streamsnippet/main_test.go new file mode 100644 index 000000000..0b8e3e3aa --- /dev/null +++ b/internal/ingress/annotations/streamsnippet/main_test.go @@ -0,0 +1,64 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package streamsnippet + +import ( + "testing" + + api "k8s.io/api/core/v1" + networking "k8s.io/api/networking/v1" + meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/ingress-nginx/internal/ingress/annotations/parser" + "k8s.io/ingress-nginx/internal/ingress/resolver" +) + +func TestParse(t *testing.T) { + annotation := parser.GetAnnotationWithPrefix("stream-snippet") + + ap := NewParser(&resolver.Mock{}) + if ap == nil { + t.Fatalf("expected a parser.IngressAnnotation but returned nil") + } + + testCases := []struct { + annotations map[string]string + expected string + }{ + {map[string]string{annotation: "server { listen: 8000; proxy_pass 127.0.0.1:80}"}, + "server { listen: 8000; proxy_pass 127.0.0.1:80}", + }, + {map[string]string{annotation: "false"}, "false"}, + {map[string]string{}, ""}, + {nil, ""}, + } + + ing := &networking.Ingress{ + ObjectMeta: meta_v1.ObjectMeta{ + Name: "foo", + Namespace: api.NamespaceDefault, + }, + Spec: networking.IngressSpec{}, + } + + for _, testCase := range testCases { + ing.SetAnnotations(testCase.annotations) + result, _ := ap.Parse(ing) + if result != testCase.expected { + t.Errorf("expected %v but returned %v, annotations: %s", testCase.expected, result, testCase.annotations) + } + } +} diff --git a/internal/ingress/controller/config/config.go b/internal/ingress/controller/config/config.go index d17135f1c..f37516e78 100644 --- a/internal/ingress/controller/config/config.go +++ b/internal/ingress/controller/config/config.go @@ -659,6 +659,9 @@ type Configuration struct { // ServerSnippet adds custom configuration to all the servers in the nginx configuration ServerSnippet string `json:"server-snippet"` + // StreamSnippet adds custom configuration to the stream section of the nginx configuration + StreamSnippet string `json:"stream-snippet"` + // LocationSnippet adds custom configuration to all the locations in the nginx configuration LocationSnippet string `json:"location-snippet"` @@ -956,10 +959,11 @@ type TemplateConfig struct { MaxmindEditionFiles *[]string MonitorMaxBatchSize int - PID string - StatusPath string - StatusPort int - StreamPort int + PID string + StatusPath string + StatusPort int + StreamPort int + StreamSnippets []string } // ListenPorts describe the ports required to run the diff --git a/internal/ingress/controller/controller.go b/internal/ingress/controller/controller.go index 99d1c8f35..7d958a626 100644 --- a/internal/ingress/controller/controller.go +++ b/internal/ingress/controller/controller.go @@ -538,6 +538,7 @@ func (n *NGINXController) getConfiguration(ingresses []*ingress.Ingress) (sets.S PassthroughBackends: passUpstreams, BackendConfigChecksum: n.store.GetBackendConfiguration().Checksum, DefaultSSLCertificate: n.getDefaultSSLCertificate(), + StreamSnippets: n.getStreamSnippets(ingresses), } } @@ -562,6 +563,11 @@ func dropSnippetDirectives(anns *annotations.Ingress, ingKey string) { anns.ExternalAuth.AuthSnippet = "" } + if anns.StreamSnippet != "" { + klog.V(3).Infof("Ingress %q tried to use stream-snippet and the annotation is disabled by the admin. Removing the annotation", ingKey) + anns.StreamSnippet = "" + } + } } @@ -1779,3 +1785,14 @@ func ingressForHostPath(hostname, path string, servers []*ingress.Server) []*net return ingresses } + +func (n *NGINXController) getStreamSnippets(ingresses []*ingress.Ingress) []string { + snippets := make([]string, 0, len(ingresses)) + for _, i := range ingresses { + if i.ParsedAnnotations.StreamSnippet == "" { + continue + } + snippets = append(snippets, i.ParsedAnnotations.StreamSnippet) + } + return snippets +} diff --git a/internal/ingress/controller/nginx.go b/internal/ingress/controller/nginx.go index 4d1aa3916..ed5590c3e 100644 --- a/internal/ingress/controller/nginx.go +++ b/internal/ingress/controller/nginx.go @@ -599,6 +599,7 @@ func (n NGINXController) generateTemplate(cfg ngx_config.Configuration, ingressC StatusPath: nginx.StatusPath, StatusPort: nginx.StatusPort, StreamPort: nginx.StreamPort, + StreamSnippets: append(ingressCfg.StreamSnippets, cfg.StreamSnippet), } tc.Cfg.Checksum = ingressCfg.ConfigurationChecksum diff --git a/internal/ingress/types.go b/internal/ingress/types.go index 78c2245ff..db4f37f99 100644 --- a/internal/ingress/types.go +++ b/internal/ingress/types.go @@ -76,6 +76,8 @@ type Configuration struct { ConfigurationChecksum string `json:"configurationChecksum,omitempty"` DefaultSSLCertificate *SSLCert `json:"-"` + + StreamSnippets []string } // Backend describes one or more remote server/s (endpoints) associated with a service diff --git a/rootfs/etc/nginx/template/nginx.tmpl b/rootfs/etc/nginx/template/nginx.tmpl index 0cc8d3cab..bf780fe80 100755 --- a/rootfs/etc/nginx/template/nginx.tmpl +++ b/rootfs/etc/nginx/template/nginx.tmpl @@ -827,6 +827,11 @@ stream { proxy_pass upstream_balancer; } {{ end }} + + # Stream Snippets + {{ range $snippet := .StreamSnippets }} + {{ $snippet }} + {{ end }} } {{/* definition of templates to avoid repetitions */}} diff --git a/test/e2e/annotations/streamsnippet.go b/test/e2e/annotations/streamsnippet.go new file mode 100644 index 000000000..cc9aca715 --- /dev/null +++ b/test/e2e/annotations/streamsnippet.go @@ -0,0 +1,138 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package annotations + +import ( + "context" + "fmt" + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "net/http" + "strings" + + "github.com/onsi/ginkgo" + + "k8s.io/ingress-nginx/test/e2e/framework" +) + +var _ = framework.DescribeSetting("stream-snippet", func() { + f := framework.NewDefaultFramework("stream-snippet") + + ginkgo.BeforeEach(func() { + f.NewEchoDeployment() + }) + + ginkgo.It("should add value of stream-snippet to nginx config", func() { + host := "foo.com" + + snippet := `server {listen 8000; proxy_pass 127.0.0.1:80;}` + + ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, map[string]string{ + "nginx.ingress.kubernetes.io/stream-snippet": snippet, + }) + f.EnsureIngress(ing) + + svc, err := f.KubeClientSet. + CoreV1(). + Services(f.Namespace). + Get(context.TODO(), "nginx-ingress-controller", metav1.GetOptions{}) + assert.Nil(ginkgo.GinkgoT(), err, "unexpected error obtaining ingress-nginx service") + assert.NotNil(ginkgo.GinkgoT(), svc, "expected a service but none returned") + + svc.Spec.Ports = append(svc.Spec.Ports, corev1.ServicePort{ + Name: framework.EchoService, + Port: 8000, + TargetPort: intstr.FromInt(8000), + }) + + _, err = f.KubeClientSet. + CoreV1(). + Services(f.Namespace). + Update(context.TODO(), svc, metav1.UpdateOptions{}) + assert.Nil(ginkgo.GinkgoT(), err, "unexpected error updating service") + + // Sleep a while just to guarantee that the configmap is applied + framework.Sleep() + + f.WaitForNginxConfiguration( + func(cfg string) bool { + return strings.Contains(cfg, snippet) + }) + + f.HTTPTestClient(). + GET("/healthz"). + WithURL(fmt.Sprintf("http://%v:8000/healthz", f.GetNginxIP())). + Expect(). + Status(http.StatusOK) + }) + + ginkgo.It("should add stream-snippet and drop annotations per admin config", func() { + host := "cm.foo.com" + hostAnnot := "annot.foo.com" + + cmSnippet := `server {listen 8000; proxy_pass 127.0.0.1:80;}` + annotSnippet := `server {listen 8001; proxy_pass 127.0.0.1:80;}` + + f.SetNginxConfigMapData(map[string]string{ + "allow-snippet-annotations": "false", + "stream-snippet": cmSnippet, + }) + + ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, nil) + f.EnsureIngress(ing) + + ing1 := framework.NewSingleIngress(hostAnnot, "/", hostAnnot, f.Namespace, framework.EchoService, 80, map[string]string{ + "nginx.ingress.kubernetes.io/stream-snippet": annotSnippet, + }) + f.EnsureIngress(ing1) + + svc, err := f.KubeClientSet. + CoreV1(). + Services(f.Namespace). + Get(context.TODO(), "nginx-ingress-controller", metav1.GetOptions{}) + assert.Nil(ginkgo.GinkgoT(), err, "unexpected error obtaining ingress-nginx service") + assert.NotNil(ginkgo.GinkgoT(), svc, "expected a service but none returned") + + svc.Spec.Ports = append(svc.Spec.Ports, corev1.ServicePort{ + Name: framework.EchoService, + Port: 8000, + TargetPort: intstr.FromInt(8000), + }) + + _, err = f.KubeClientSet. + CoreV1(). + Services(f.Namespace). + Update(context.TODO(), svc, metav1.UpdateOptions{}) + assert.Nil(ginkgo.GinkgoT(), err, "unexpected error updating service") + + // Sleep a while just to guarantee that the configmap is applied + framework.Sleep() + + f.WaitForNginxConfiguration( + func(cfg string) bool { + return strings.Contains(cfg, cmSnippet) && !strings.Contains(cfg, annotSnippet) + }) + + f.HTTPTestClient(). + GET("/healthz"). + WithURL(fmt.Sprintf("http://%v:8000/healthz", f.GetNginxIP())). + Expect(). + Status(http.StatusOK) + }) +}) diff --git a/test/e2e/settings/stream_snippet.go b/test/e2e/settings/stream_snippet.go new file mode 100644 index 000000000..90f928c23 --- /dev/null +++ b/test/e2e/settings/stream_snippet.go @@ -0,0 +1,85 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package settings + +import ( + "context" + "fmt" + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "net/http" + "strings" + + "github.com/onsi/ginkgo" + + "k8s.io/ingress-nginx/test/e2e/framework" +) + +var _ = framework.DescribeSetting("configmap stream-snippet", func() { + f := framework.NewDefaultFramework("cm-stream-snippet") + + ginkgo.BeforeEach(func() { + f.NewEchoDeployment() + }) + + ginkgo.It("should add value of stream-snippet via config map to nginx config", func() { + host := "foo.com" + snippet := `server {listen 8000; proxy_pass 127.0.0.1:80;}` + + f.SetNginxConfigMapData(map[string]string{ + "stream-snippet": snippet, + }) + + ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, nil) + f.EnsureIngress(ing) + + svc, err := f.KubeClientSet. + CoreV1(). + Services(f.Namespace). + Get(context.TODO(), "nginx-ingress-controller", metav1.GetOptions{}) + assert.Nil(ginkgo.GinkgoT(), err, "unexpected error obtaining ingress-nginx service") + assert.NotNil(ginkgo.GinkgoT(), svc, "expected a service but none returned") + + svc.Spec.Ports = append(svc.Spec.Ports, corev1.ServicePort{ + Name: framework.EchoService, + Port: 8000, + TargetPort: intstr.FromInt(8000), + }) + + _, err = f.KubeClientSet. + CoreV1(). + Services(f.Namespace). + Update(context.TODO(), svc, metav1.UpdateOptions{}) + assert.Nil(ginkgo.GinkgoT(), err, "unexpected error updating service") + + // Sleep a while just to guarantee that the configmap is applied + framework.Sleep() + + f.WaitForNginxConfiguration( + func(cfg string) bool { + return strings.Contains(cfg, snippet) + }) + + f.HTTPTestClient(). + GET("/healthz"). + WithURL(fmt.Sprintf("http://%v:8000/healthz", f.GetNginxIP())). + Expect(). + Status(http.StatusOK) + }) +})