Add stream-snippet as a ConfigMap and Annotation option (#8029)

* stream snippet

* gofmt -s

docs/user-guide/nginx-configuration/annotations.md

@@ -99,6 +99,7 @@ You can add these Kubernetes annotations to specific Ingress objects to customiz
 |[nginx.ingress.kubernetes.io/session-cookie-conditional-samesite-none](#cookie-affinity)|"true" or "false"|
 |[nginx.ingress.kubernetes.io/ssl-redirect](#server-side-https-enforcement-through-redirect)|"true" or "false"|
 |[nginx.ingress.kubernetes.io/ssl-passthrough](#ssl-passthrough)|"true" or "false"|
+|[nginx.ingress.kubernetes.io/stream-snippet](#stream-snippet)|string|
 |[nginx.ingress.kubernetes.io/upstream-hash-by](#custom-nginx-upstream-hashing)|string|
 |[nginx.ingress.kubernetes.io/x-forwarded-prefix](#x-forwarded-prefix-header)|string|
 |[nginx.ingress.kubernetes.io/load-balance](#custom-nginx-load-balancing)|string|
@@ -927,3 +928,20 @@ nginx.ingress.kubernetes.io/mirror-request-body: "off"
 The request sent to the mirror is linked to the original request. If you have a slow mirror backend, then the original request will throttle.
 
 For more information on the mirror module see [ngx_http_mirror_module](https://nginx.org/en/docs/http/ngx_http_mirror_module.html)
+
+
+### Stream snippet
+
+Using the annotation `nginx.ingress.kubernetes.io/stream-snippet` it is possible to add custom stream configuration.
+
+```yaml
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    nginx.ingress.kubernetes.io/stream-snippet: |
+      server {
+        listen 8000;
+        proxy_pass 127.0.0.1:80;
+      }
+```

docs/user-guide/nginx-configuration/configmap.md

@@ -156,6 +156,7 @@ The following table shows a configuration option's name, type, and the default v
 |[main-snippet](#main-snippet)|string|""|
 |[http-snippet](#http-snippet)|string|""|
 |[server-snippet](#server-snippet)|string|""|
+|[stream-snippet](#stream-snippet)|string|""|
 |[location-snippet](#location-snippet)|string|""|
 |[custom-http-errors](#custom-http-errors)|[]int|[]int{}|
 |[proxy-body-size](#proxy-body-size)|string|"1m"|
@@ -988,6 +989,10 @@ Adds custom configuration to the http section of the nginx configuration.
 
 Adds custom configuration to all the servers in the nginx configuration.
 
+## stream-snippet
+
+Adds custom configuration to the stream section of the nginx configuration.
+
 ## location-snippet
 
 Adds custom configuration to all the locations in the nginx configuration.

internal/ingress/annotations/annotations.go

@@ -22,6 +22,7 @@ import (
 	"k8s.io/ingress-nginx/internal/ingress/annotations/modsecurity"
 	"k8s.io/ingress-nginx/internal/ingress/annotations/proxyssl"
 	"k8s.io/ingress-nginx/internal/ingress/annotations/sslcipher"
+	"k8s.io/ingress-nginx/internal/ingress/annotations/streamsnippet"
 	"k8s.io/klog/v2"
 
 	apiv1 "k8s.io/api/core/v1"
@@ -115,6 +116,7 @@ type Ingress struct {
 	InfluxDB           influxdb.Config
 	ModSecurity        modsecurity.Config
 	Mirror             mirror.Config
+	StreamSnippet      string
 }
 
 // Extractor defines the annotation parsers to be used in the extraction of annotations
@@ -165,6 +167,7 @@ func NewAnnotationExtractor(cfg resolver.Resolver) Extractor {
 			"BackendProtocol":      backendprotocol.NewParser(cfg),
 			"ModSecurity":          modsecurity.NewParser(cfg),
 			"Mirror":               mirror.NewParser(cfg),
+			"StreamSnippet":        streamsnippet.NewParser(cfg),
 		},
 	}
 }

internal/ingress/annotations/streamsnippet/main.go

@@ -0,0 +1,40 @@
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package streamsnippet
+
+import (
+	networking "k8s.io/api/networking/v1"
+
+	"k8s.io/ingress-nginx/internal/ingress/annotations/parser"
+	"k8s.io/ingress-nginx/internal/ingress/resolver"
+)
+
+type streamSnippet struct {
+	r resolver.Resolver
+}
+
+// NewParser creates a new server snippet annotation parser
+func NewParser(r resolver.Resolver) parser.IngressAnnotation {
+	return streamSnippet{r}
+}
+
+// Parse parses the annotations contained in the ingress rule
+// used to indicate if the location/s contains a fragment of
+// configuration to be included inside the paths of the rules
+func (a streamSnippet) Parse(ing *networking.Ingress) (interface{}, error) {
+	return parser.GetStringAnnotation("stream-snippet", ing)
+}

internal/ingress/annotations/streamsnippet/main_test.go

@@ -0,0 +1,64 @@
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package streamsnippet
+
+import (
+	"testing"
+
+	api "k8s.io/api/core/v1"
+	networking "k8s.io/api/networking/v1"
+	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/ingress-nginx/internal/ingress/annotations/parser"
+	"k8s.io/ingress-nginx/internal/ingress/resolver"
+)
+
+func TestParse(t *testing.T) {
+	annotation := parser.GetAnnotationWithPrefix("stream-snippet")
+
+	ap := NewParser(&resolver.Mock{})
+	if ap == nil {
+		t.Fatalf("expected a parser.IngressAnnotation but returned nil")
+	}
+
+	testCases := []struct {
+		annotations map[string]string
+		expected    string
+	}{
+		{map[string]string{annotation: "server { listen: 8000; proxy_pass 127.0.0.1:80}"},
+			"server { listen: 8000; proxy_pass 127.0.0.1:80}",
+		},
+		{map[string]string{annotation: "false"}, "false"},
+		{map[string]string{}, ""},
+		{nil, ""},
+	}
+
+	ing := &networking.Ingress{
+		ObjectMeta: meta_v1.ObjectMeta{
+			Name:      "foo",
+			Namespace: api.NamespaceDefault,
+		},
+		Spec: networking.IngressSpec{},
+	}
+
+	for _, testCase := range testCases {
+		ing.SetAnnotations(testCase.annotations)
+		result, _ := ap.Parse(ing)
+		if result != testCase.expected {
+			t.Errorf("expected %v but returned %v, annotations: %s", testCase.expected, result, testCase.annotations)
+		}
+	}
+}

internal/ingress/controller/config/config.go

@@ -659,6 +659,9 @@ type Configuration struct {
 	// ServerSnippet adds custom configuration to all the servers in the nginx configuration
 	ServerSnippet string `json:"server-snippet"`
 
+	// StreamSnippet adds custom configuration to the stream section of the nginx configuration
+	StreamSnippet string `json:"stream-snippet"`
+
 	// LocationSnippet adds custom configuration to all the locations in the nginx configuration
 	LocationSnippet string `json:"location-snippet"`
 
@@ -956,10 +959,11 @@ type TemplateConfig struct {
 	MaxmindEditionFiles      *[]string
 	MonitorMaxBatchSize      int
 
-	PID        string
+	PID            string
-	StatusPath string
+	StatusPath     string
-	StatusPort int
+	StatusPort     int
-	StreamPort int
+	StreamPort     int
+	StreamSnippets []string
 }
 
 // ListenPorts describe the ports required to run the

internal/ingress/controller/controller.go

@@ -538,6 +538,7 @@ func (n *NGINXController) getConfiguration(ingresses []*ingress.Ingress) (sets.S
 		PassthroughBackends:   passUpstreams,
 		BackendConfigChecksum: n.store.GetBackendConfiguration().Checksum,
 		DefaultSSLCertificate: n.getDefaultSSLCertificate(),
+		StreamSnippets:        n.getStreamSnippets(ingresses),
 	}
 }
 
@@ -562,6 +563,11 @@ func dropSnippetDirectives(anns *annotations.Ingress, ingKey string) {
 			anns.ExternalAuth.AuthSnippet = ""
 		}
 
+		if anns.StreamSnippet != "" {
+			klog.V(3).Infof("Ingress %q tried to use stream-snippet and the annotation is disabled by the admin. Removing the annotation", ingKey)
+			anns.StreamSnippet = ""
+		}
+
 	}
 }
 
@@ -1779,3 +1785,14 @@ func ingressForHostPath(hostname, path string, servers []*ingress.Server) []*net
 
 	return ingresses
 }
+
+func (n *NGINXController) getStreamSnippets(ingresses []*ingress.Ingress) []string {
+	snippets := make([]string, 0, len(ingresses))
+	for _, i := range ingresses {
+		if i.ParsedAnnotations.StreamSnippet == "" {
+			continue
+		}
+		snippets = append(snippets, i.ParsedAnnotations.StreamSnippet)
+	}
+	return snippets
+}

internal/ingress/controller/nginx.go

@@ -599,6 +599,7 @@ func (n NGINXController) generateTemplate(cfg ngx_config.Configuration, ingressC
 		StatusPath:               nginx.StatusPath,
 		StatusPort:               nginx.StatusPort,
 		StreamPort:               nginx.StreamPort,
+		StreamSnippets:           append(ingressCfg.StreamSnippets, cfg.StreamSnippet),
 	}
 
 	tc.Cfg.Checksum = ingressCfg.ConfigurationChecksum

internal/ingress/types.go

@@ -76,6 +76,8 @@ type Configuration struct {
 	ConfigurationChecksum string `json:"configurationChecksum,omitempty"`
 
 	DefaultSSLCertificate *SSLCert `json:"-"`
+
+	StreamSnippets []string
 }
 
 // Backend describes one or more remote server/s (endpoints) associated with a service

rootfs/etc/nginx/template/nginx.tmpl

@@ -827,6 +827,11 @@ stream {
         proxy_pass              upstream_balancer;
     }
     {{ end }}
+
+    # Stream Snippets
+    {{ range $snippet := .StreamSnippets }}
+    {{ $snippet }}
+    {{ end }}
 }
 
 {{/* definition of templates to avoid repetitions */}}

test/e2e/annotations/streamsnippet.go

@@ -0,0 +1,138 @@
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package annotations
+
+import (
+	"context"
+	"fmt"
+	"github.com/stretchr/testify/assert"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	"net/http"
+	"strings"
+
+	"github.com/onsi/ginkgo"
+
+	"k8s.io/ingress-nginx/test/e2e/framework"
+)
+
+var _ = framework.DescribeSetting("stream-snippet", func() {
+	f := framework.NewDefaultFramework("stream-snippet")
+
+	ginkgo.BeforeEach(func() {
+		f.NewEchoDeployment()
+	})
+
+	ginkgo.It("should add value of stream-snippet to nginx config", func() {
+		host := "foo.com"
+
+		snippet := `server {listen 8000; proxy_pass 127.0.0.1:80;}`
+
+		ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, map[string]string{
+			"nginx.ingress.kubernetes.io/stream-snippet": snippet,
+		})
+		f.EnsureIngress(ing)
+
+		svc, err := f.KubeClientSet.
+			CoreV1().
+			Services(f.Namespace).
+			Get(context.TODO(), "nginx-ingress-controller", metav1.GetOptions{})
+		assert.Nil(ginkgo.GinkgoT(), err, "unexpected error obtaining ingress-nginx service")
+		assert.NotNil(ginkgo.GinkgoT(), svc, "expected a service but none returned")
+
+		svc.Spec.Ports = append(svc.Spec.Ports, corev1.ServicePort{
+			Name:       framework.EchoService,
+			Port:       8000,
+			TargetPort: intstr.FromInt(8000),
+		})
+
+		_, err = f.KubeClientSet.
+			CoreV1().
+			Services(f.Namespace).
+			Update(context.TODO(), svc, metav1.UpdateOptions{})
+		assert.Nil(ginkgo.GinkgoT(), err, "unexpected error updating service")
+
+		// Sleep a while just to guarantee that the configmap is applied
+		framework.Sleep()
+
+		f.WaitForNginxConfiguration(
+			func(cfg string) bool {
+				return strings.Contains(cfg, snippet)
+			})
+
+		f.HTTPTestClient().
+			GET("/healthz").
+			WithURL(fmt.Sprintf("http://%v:8000/healthz", f.GetNginxIP())).
+			Expect().
+			Status(http.StatusOK)
+	})
+
+	ginkgo.It("should add stream-snippet and drop annotations per admin config", func() {
+		host := "cm.foo.com"
+		hostAnnot := "annot.foo.com"
+
+		cmSnippet := `server {listen 8000; proxy_pass 127.0.0.1:80;}`
+		annotSnippet := `server {listen 8001; proxy_pass 127.0.0.1:80;}`
+
+		f.SetNginxConfigMapData(map[string]string{
+			"allow-snippet-annotations": "false",
+			"stream-snippet":            cmSnippet,
+		})
+
+		ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, nil)
+		f.EnsureIngress(ing)
+
+		ing1 := framework.NewSingleIngress(hostAnnot, "/", hostAnnot, f.Namespace, framework.EchoService, 80, map[string]string{
+			"nginx.ingress.kubernetes.io/stream-snippet": annotSnippet,
+		})
+		f.EnsureIngress(ing1)
+
+		svc, err := f.KubeClientSet.
+			CoreV1().
+			Services(f.Namespace).
+			Get(context.TODO(), "nginx-ingress-controller", metav1.GetOptions{})
+		assert.Nil(ginkgo.GinkgoT(), err, "unexpected error obtaining ingress-nginx service")
+		assert.NotNil(ginkgo.GinkgoT(), svc, "expected a service but none returned")
+
+		svc.Spec.Ports = append(svc.Spec.Ports, corev1.ServicePort{
+			Name:       framework.EchoService,
+			Port:       8000,
+			TargetPort: intstr.FromInt(8000),
+		})
+
+		_, err = f.KubeClientSet.
+			CoreV1().
+			Services(f.Namespace).
+			Update(context.TODO(), svc, metav1.UpdateOptions{})
+		assert.Nil(ginkgo.GinkgoT(), err, "unexpected error updating service")
+
+		// Sleep a while just to guarantee that the configmap is applied
+		framework.Sleep()
+
+		f.WaitForNginxConfiguration(
+			func(cfg string) bool {
+				return strings.Contains(cfg, cmSnippet) && !strings.Contains(cfg, annotSnippet)
+			})
+
+		f.HTTPTestClient().
+			GET("/healthz").
+			WithURL(fmt.Sprintf("http://%v:8000/healthz", f.GetNginxIP())).
+			Expect().
+			Status(http.StatusOK)
+	})
+})

test/e2e/settings/stream_snippet.go

@@ -0,0 +1,85 @@
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package settings
+
+import (
+	"context"
+	"fmt"
+	"github.com/stretchr/testify/assert"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	"net/http"
+	"strings"
+
+	"github.com/onsi/ginkgo"
+
+	"k8s.io/ingress-nginx/test/e2e/framework"
+)
+
+var _ = framework.DescribeSetting("configmap stream-snippet", func() {
+	f := framework.NewDefaultFramework("cm-stream-snippet")
+
+	ginkgo.BeforeEach(func() {
+		f.NewEchoDeployment()
+	})
+
+	ginkgo.It("should add value of stream-snippet via config map to nginx config", func() {
+		host := "foo.com"
+		snippet := `server {listen 8000; proxy_pass 127.0.0.1:80;}`
+
+		f.SetNginxConfigMapData(map[string]string{
+			"stream-snippet": snippet,
+		})
+
+		ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, nil)
+		f.EnsureIngress(ing)
+
+		svc, err := f.KubeClientSet.
+			CoreV1().
+			Services(f.Namespace).
+			Get(context.TODO(), "nginx-ingress-controller", metav1.GetOptions{})
+		assert.Nil(ginkgo.GinkgoT(), err, "unexpected error obtaining ingress-nginx service")
+		assert.NotNil(ginkgo.GinkgoT(), svc, "expected a service but none returned")
+
+		svc.Spec.Ports = append(svc.Spec.Ports, corev1.ServicePort{
+			Name:       framework.EchoService,
+			Port:       8000,
+			TargetPort: intstr.FromInt(8000),
+		})
+
+		_, err = f.KubeClientSet.
+			CoreV1().
+			Services(f.Namespace).
+			Update(context.TODO(), svc, metav1.UpdateOptions{})
+		assert.Nil(ginkgo.GinkgoT(), err, "unexpected error updating service")
+
+		// Sleep a while just to guarantee that the configmap is applied
+		framework.Sleep()
+
+		f.WaitForNginxConfiguration(
+			func(cfg string) bool {
+				return strings.Contains(cfg, snippet)
+			})
+
+		f.HTTPTestClient().
+			GET("/healthz").
+			WithURL(fmt.Sprintf("http://%v:8000/healthz", f.GetNginxIP())).
+			Expect().
+			Status(http.StatusOK)
+	})
+})