From f6aded2c5112623fee0865320cd39a94fd67ae04 Mon Sep 17 00:00:00 2001 From: Kevin Pullin Date: Sun, 17 Feb 2019 14:12:10 -0800 Subject: [PATCH 1/2] Fix DNS failures in L4 services --- .../etc/nginx/lua/tcp_udp_configuration.lua | 4 +- rootfs/etc/nginx/template/nginx.tmpl | 9 +- test/e2e/tcpudp/tcp.go | 96 ++++++++++++++++++- 3 files changed, 103 insertions(+), 6 deletions(-) diff --git a/rootfs/etc/nginx/lua/tcp_udp_configuration.lua b/rootfs/etc/nginx/lua/tcp_udp_configuration.lua index 8cc4111cf..902ac59b6 100644 --- a/rootfs/etc/nginx/lua/tcp_udp_configuration.lua +++ b/rootfs/etc/nginx/lua/tcp_udp_configuration.lua @@ -1,9 +1,7 @@ -- this is the Lua representation of TCP/UDP Configuration local tcp_udp_configuration_data = ngx.shared.tcp_udp_configuration_data -local _M = { - nameservers = {} -} +local _M = {} function _M.get_backends_data() return tcp_udp_configuration_data:get("backends") diff --git a/rootfs/etc/nginx/template/nginx.tmpl b/rootfs/etc/nginx/template/nginx.tmpl index ba5355c4d..cb008d2dc 100644 --- a/rootfs/etc/nginx/template/nginx.tmpl +++ b/rootfs/etc/nginx/template/nginx.tmpl @@ -689,12 +689,19 @@ stream { -- init modules local ok, res + ok, res = pcall(require, "configuration") + if not ok then + error("require failed: " .. tostring(res)) + else + configuration = res + configuration.nameservers = { {{ buildResolversForLua $cfg.Resolver $cfg.DisableIpv6DNS }} } + end + ok, res = pcall(require, "tcp_udp_configuration") if not ok then error("require failed: " .. tostring(res)) else tcp_udp_configuration = res - tcp_udp_configuration.nameservers = { {{ buildResolversForLua $cfg.Resolver $cfg.DisableIpv6DNS }} } end ok, res = pcall(require, "tcp_udp_balancer") diff --git a/test/e2e/tcpudp/tcp.go b/test/e2e/tcpudp/tcp.go index cebb90be2..23a75b947 100644 --- a/test/e2e/tcpudp/tcp.go +++ b/test/e2e/tcpudp/tcp.go @@ -17,10 +17,12 @@ limitations under the License. package settings import ( + "context" "fmt" + "github.com/parnurzeal/gorequest" + "net" "strings" - "github.com/parnurzeal/gorequest" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" @@ -55,6 +57,7 @@ var _ = framework.IngressNginxDescribe("TCP Feature", func() { } config.Data["8080"] = fmt.Sprintf("%v/http-svc:80", f.IngressController.Namespace) + _, err = f.KubeClientSet. CoreV1(). ConfigMaps(f.IngressController.Namespace). @@ -86,7 +89,7 @@ var _ = framework.IngressNginxDescribe("TCP Feature", func() { ip := f.GetNginxIP() port, err := f.GetNginxPort("http-svc") - Expect(err).NotTo(HaveOccurred(), "unexpected error obtaning service port") + Expect(err).NotTo(HaveOccurred(), "unexpected error obtaining service port") resp, _, errs := gorequest.New(). Get(fmt.Sprintf("http://%v:%v", ip, port)). @@ -94,4 +97,93 @@ var _ = framework.IngressNginxDescribe("TCP Feature", func() { Expect(errs).Should(BeEmpty()) Expect(resp.StatusCode).Should(Equal(200)) }) + + It("should expose an ExternalName TCP service", func() { + // Setup: + // - Create an external name service for DNS lookups on port 5353. Point it to google's DNS server + // - Expose port 5353 on the nginx ingress NodePort service to open a hole for this test + // - Update the `tcp-services` configmap to proxy traffic to the configured external name service + + // Create an external service for DNS + externalService := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dns-external-name-svc", + Namespace: f.IngressController.Namespace, + }, + + Spec: corev1.ServiceSpec{ + ExternalName: "google-public-dns-a.google.com", + Ports: []corev1.ServicePort{ + { + Name: "dns-external-name-svc", + Port: 5353, + TargetPort: intstr.FromInt(53), + }, + }, + Type: corev1.ServiceTypeExternalName, + }, + } + f.EnsureService(externalService) + + // Expose the `external name` port on the `ingress-nginx` service + svc, err := f.KubeClientSet. + CoreV1(). + Services(f.IngressController.Namespace). + Get("ingress-nginx", metav1.GetOptions{}) + Expect(err).To(BeNil(), "unexpected error obtaining ingress-nginx service") + Expect(svc).NotTo(BeNil(), "expected a service but none returned") + + svc.Spec.Ports = append(svc.Spec.Ports, corev1.ServicePort{ + Name: "dns-svc", + Port: 5353, + TargetPort: intstr.FromInt(5353), + }) + _, err = f.KubeClientSet. + CoreV1(). + Services(f.IngressController.Namespace). + Update(svc) + Expect(err).NotTo(HaveOccurred(), "unexpected error updating service") + + // Update the TCP configmap to link port 5353 to the DNS external name service + config, err := f.KubeClientSet. + CoreV1(). + ConfigMaps(f.IngressController.Namespace). + Get("tcp-services", metav1.GetOptions{}) + Expect(err).To(BeNil(), "unexpected error obtaining tcp-services configmap") + Expect(config).NotTo(BeNil(), "expected a configmap but none returned") + + if config.Data == nil { + config.Data = map[string]string{} + } + + config.Data["5353"] = fmt.Sprintf("%v/dns-external-name-svc:5353", f.IngressController.Namespace) + + _, err = f.KubeClientSet. + CoreV1(). + ConfigMaps(f.IngressController.Namespace). + Update(config) + Expect(err).NotTo(HaveOccurred(), "unexpected error updating configmap") + + // Validate that the generated nginx config contains the expected `proxy_upstream_name` value + f.WaitForNginxConfiguration( + func(cfg string) bool { + return strings.Contains(cfg, fmt.Sprintf(`ngx.var.proxy_upstream_name="tcp-%v-dns-external-name-svc-5353"`, f.IngressController.Namespace)) + }) + + // Execute the test. Use the `external name` service to resolve a domain name. + ip := f.GetNginxIP() + port, err := f.GetNginxPort("dns-svc") + Expect(err).NotTo(HaveOccurred(), "unexpected error obtaining service port") + + resolver := net.Resolver{ + PreferGo: true, + Dial: func(ctx context.Context, network, address string) (net.Conn, error) { + d := net.Dialer{} + return d.DialContext(ctx, "tcp", fmt.Sprintf("%v:%v", ip, port)) + }, + } + ips, err := resolver.LookupHost(context.Background(), "google-public-dns-b.google.com") + Expect(ips).Should(ContainElement("8.8.4.4")) + + }) }) From 3ac57097f61b1a1dcdae9146ca10323fb7ef6907 Mon Sep 17 00:00:00 2001 From: Kevin Pullin Date: Sun, 17 Feb 2019 20:32:27 -0800 Subject: [PATCH 2/2] Add 'waitForLuaSync' sleep in external name TCP test --- test/e2e/tcpudp/tcp.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/e2e/tcpudp/tcp.go b/test/e2e/tcpudp/tcp.go index 23a75b947..df93ca776 100644 --- a/test/e2e/tcpudp/tcp.go +++ b/test/e2e/tcpudp/tcp.go @@ -22,6 +22,7 @@ import ( "github.com/parnurzeal/gorequest" "net" "strings" + "time" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -33,6 +34,10 @@ import ( "k8s.io/ingress-nginx/test/e2e/framework" ) +const ( + waitForLuaSync = 5 * time.Second +) + var _ = framework.IngressNginxDescribe("TCP Feature", func() { f := framework.NewDefaultFramework("tcp") @@ -164,6 +169,8 @@ var _ = framework.IngressNginxDescribe("TCP Feature", func() { Update(config) Expect(err).NotTo(HaveOccurred(), "unexpected error updating configmap") + time.Sleep(waitForLuaSync) + // Validate that the generated nginx config contains the expected `proxy_upstream_name` value f.WaitForNginxConfiguration( func(cfg string) bool { @@ -183,6 +190,7 @@ var _ = framework.IngressNginxDescribe("TCP Feature", func() { }, } ips, err := resolver.LookupHost(context.Background(), "google-public-dns-b.google.com") + Expect(err).NotTo(HaveOccurred(), "unexpected error from DNS resolver") Expect(ips).Should(ContainElement("8.8.4.4")) })