From 0d9769c80d710eccd0ea110f78ce0bbabc060950 Mon Sep 17 00:00:00 2001 From: Manuel de Brito Fontes Date: Sun, 9 Apr 2017 20:51:38 -0300 Subject: [PATCH] Add golang server to handle TLS hello --- controllers/nginx/pkg/cmd/controller/nginx.go | 59 ++++++++++++ controllers/nginx/pkg/cmd/controller/tcp.go | 90 +++++++++++++++++++ controllers/nginx/pkg/config/config.go | 2 +- controllers/nginx/pkg/template/template.go | 57 +++--------- .../rootfs/etc/nginx/template/nginx.tmpl | 24 ++--- core/pkg/ingress/controller/controller.go | 6 ++ core/pkg/ingress/types.go | 10 ++- 7 files changed, 183 insertions(+), 65 deletions(-) create mode 100644 controllers/nginx/pkg/cmd/controller/tcp.go diff --git a/controllers/nginx/pkg/cmd/controller/nginx.go b/controllers/nginx/pkg/cmd/controller/nginx.go index 760070eb6..0334726ea 100644 --- a/controllers/nginx/pkg/cmd/controller/nginx.go +++ b/controllers/nginx/pkg/cmd/controller/nginx.go @@ -80,9 +80,31 @@ func newNGINXController() ingress.Controller { binary: ngx, configmap: &api_v1.ConfigMap{}, isIPV6Enabled: isIPv6Enabled(), +<<<<<<< fc67b1d5e2a51cc0037a434583af6530efa1a59c resolver: h, +======= + proxy: &proxy{}, +>>>>>>> } + listener, err := net.Listen("tcp", ":443") + if err != nil { + glog.Fatalf("%v", err) + } + + // start goroutine that accepts tcp connections in port 443 + go func() { + for { + conn, err := listener.Accept() + if err != nil { + glog.Warningf("unexpected error accepting tcp connection: %v", err) + continue + } + glog.V(3).Infof("%s -> %s", conn.RemoteAddr(), conn.LocalAddr()) + go n.proxy.Handle(conn) + } + }() + var onChange func() onChange = func() { template, err := ngx_template.NewTemplate(tmplPath, onChange) @@ -134,7 +156,11 @@ type NGINXController struct { // returns true if IPV6 is enabled in the pod isIPV6Enabled bool +<<<<<<< fc67b1d5e2a51cc0037a434583af6530efa1a59c resolver []net.IP +======= + proxy *proxy +>>>>>>> } // Start start a new NGINX master process running in foreground. @@ -446,6 +472,39 @@ func (n *NGINXController) OnUpdate(ingressCfg ingress.Configuration) ([]byte, er return nil, err } + servers := []*server{} + for _, pb := range ingressCfg.PassthroughBackends { + svc := pb.Service + if svc == nil { + continue + } + port, err := strconv.Atoi(pb.Port.String()) + if err != nil { + for _, sp := range svc.Spec.Ports { + if sp.Name == pb.Port.String() { + port = int(sp.Port) + break + } + } + } else { + for _, sp := range svc.Spec.Ports { + if sp.Port == int32(port) { + port = int(sp.Port) + break + } + } + } + + servers = append(servers, &server{ + Hostname: pb.Hostname, + IP: svc.Spec.ClusterIP, + Port: port, + }) + } + + glog.Infof("%v", servers) + n.proxy.ServerList = servers + return content, nil } diff --git a/controllers/nginx/pkg/cmd/controller/tcp.go b/controllers/nginx/pkg/cmd/controller/tcp.go new file mode 100644 index 000000000..754ee275a --- /dev/null +++ b/controllers/nginx/pkg/cmd/controller/tcp.go @@ -0,0 +1,90 @@ +package main + +import ( + "fmt" + "io" + "net" + + "github.com/golang/glog" + "github.com/paultag/sniff/parser" +) + +type server struct { + Hostname string + IP string + Port int +} + +type proxy struct { + ServerList []*server + Default *server +} + +func (p *proxy) Get(host string) *server { + for _, s := range p.ServerList { + if s.Hostname == host { + return s + } + } + + return &server{ + Hostname: "localhost", + IP: "127.0.0.1", + Port: 442, + } +} + +func (p *proxy) Handle(conn net.Conn) { + defer conn.Close() + data := make([]byte, 4096) + + length, err := conn.Read(data) + if err != nil { + glog.V(4).Infof("Error reading the first 4k of the connection: %s", err) + return + } + + var proxy *server + hostname, err := parser.GetHostname(data[:]) + if err == nil { + glog.V(2).Infof("Parsed hostname: %s", hostname) + proxy = p.Get(hostname) + if proxy == nil { + return + } + } else { + proxy = p.Default + if proxy == nil { + return + } + } + + clientConn, err := net.Dial("tcp", fmt.Sprintf("%s:%d", proxy.IP, proxy.Port)) + if err != nil { + return + } + defer clientConn.Close() + + _, err = clientConn.Write(data[:length]) + if err != nil { + clientConn.Close() + } + pipe(clientConn, conn) +} + +func pipe(client, server net.Conn) { + doCopy := func(s, c net.Conn, cancel chan<- bool) { + io.Copy(s, c) + cancel <- true + } + + cancel := make(chan bool, 2) + + go doCopy(server, client, cancel) + go doCopy(client, server, cancel) + + select { + case <-cancel: + return + } +} diff --git a/controllers/nginx/pkg/config/config.go b/controllers/nginx/pkg/config/config.go index 66a7bbb06..3b22c10ba 100644 --- a/controllers/nginx/pkg/config/config.go +++ b/controllers/nginx/pkg/config/config.go @@ -50,7 +50,7 @@ const ( logFormatUpstream = `%v - [$proxy_add_x_forwarded_for] - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" $request_length $request_time [$proxy_upstream_name] $upstream_addr $upstream_response_length $upstream_response_time $upstream_status` - logFormatStream = `[$time_local] $protocol [$ssl_preread_server_name] [$stream_upstream] $status $bytes_sent $bytes_received $session_time` + logFormatStream = `[$time_local] $protocol $status $bytes_sent $bytes_received $session_time` // http://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_buffer_size // Sets the size of the buffer used for sending data. diff --git a/controllers/nginx/pkg/template/template.go b/controllers/nginx/pkg/template/template.go index 7560d8190..dcfd91a8a 100644 --- a/controllers/nginx/pkg/template/template.go +++ b/controllers/nginx/pkg/template/template.go @@ -130,21 +130,20 @@ var ( } return true }, - "buildLocation": buildLocation, - "buildAuthLocation": buildAuthLocation, - "buildAuthResponseHeaders": buildAuthResponseHeaders, - "buildProxyPass": buildProxyPass, - "buildRateLimitZones": buildRateLimitZones, - "buildRateLimit": buildRateLimit, - "buildSSLPassthroughUpstreams": buildSSLPassthroughUpstreams, - "buildResolvers": buildResolvers, - "isLocationAllowed": isLocationAllowed, - "buildLogFormatUpstream": buildLogFormatUpstream, - "contains": strings.Contains, - "hasPrefix": strings.HasPrefix, - "hasSuffix": strings.HasSuffix, - "toUpper": strings.ToUpper, - "toLower": strings.ToLower, + "buildLocation": buildLocation, + "buildAuthLocation": buildAuthLocation, + "buildAuthResponseHeaders": buildAuthResponseHeaders, + "buildProxyPass": buildProxyPass, + "buildRateLimitZones": buildRateLimitZones, + "buildRateLimit": buildRateLimit, + "buildResolvers": buildResolvers, + "isLocationAllowed": isLocationAllowed, + "buildLogFormatUpstream": buildLogFormatUpstream, + "contains": strings.Contains, + "hasPrefix": strings.HasPrefix, + "hasSuffix": strings.HasSuffix, + "toUpper": strings.ToUpper, + "toLower": strings.ToLower, } ) @@ -169,34 +168,6 @@ func buildResolvers(a interface{}) string { return strings.Join(r, " ") } -func buildSSLPassthroughUpstreams(b interface{}, sslb interface{}) string { - backends := b.([]*ingress.Backend) - sslBackends := sslb.([]*ingress.SSLPassthroughBackend) - buf := bytes.NewBuffer(make([]byte, 0, 10)) - - // multiple services can use the same upstream. - // avoid duplications using a map[name]=true - u := make(map[string]bool) - for _, passthrough := range sslBackends { - if u[passthrough.Backend] { - continue - } - u[passthrough.Backend] = true - fmt.Fprintf(buf, "upstream %v {\n", passthrough.Backend) - for _, backend := range backends { - if backend.Name == passthrough.Backend { - for _, server := range backend.Endpoints { - fmt.Fprintf(buf, "\t\tserver %v:%v;\n", server.Address, server.Port) - } - break - } - } - fmt.Fprint(buf, "\t}\n\n") - } - - return buf.String() -} - // buildLocation produces the location string, if the ingress has redirects // (specified through the ingress.kubernetes.io/rewrite-to annotation) func buildLocation(input interface{}) string { diff --git a/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl b/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl index 920a487e6..9ae66274c 100644 --- a/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl +++ b/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl @@ -222,10 +222,10 @@ http { listen 80{{ if $cfg.UseProxyProtocol }} proxy_protocol{{ end }}{{ if eq $server.Hostname "_"}} default_server reuseport backlog={{ $backlogSize }}{{end}}; {{ if $IsIPV6Enabled }}listen [::]:80{{ if $cfg.UseProxyProtocol }} proxy_protocol{{ end }}{{ if eq $server.Hostname "_"}} default_server reuseport backlog={{ $backlogSize }}{{ end }};{{ end }} - {{/* Listen on 442 because port 443 is used in the stream section */}} + {{/* Listen on 442 because port 443 is used in the TLS sni server */}} {{/* This listen on port 442 cannot contains proxy_protocol directive because port 443 is in charge of decoding the protocol */}} - {{ if not (empty $server.SSLCertificate) }}listen {{ if gt (len $passthroughBackends) 0 }}442{{ else }}443 {{ if $cfg.UseProxyProtocol }} proxy_protocol {{ end }}{{ end }} {{ if eq $server.Hostname "_"}} default_server reuseport backlog={{ $backlogSize }}{{end}} ssl {{ if $cfg.UseHTTP2 }}http2{{ end }}; - {{ if $IsIPV6Enabled }}{{ if not (empty $server.SSLCertificate) }}listen {{ if gt (len $passthroughBackends) 0 }}[::]:442{{ else }}[::]:443 {{ end }}{{ if $cfg.UseProxyProtocol }} proxy_protocol {{ end }}{{ end }} {{ if eq $server.Hostname "_"}} default_server reuseport backlog={{ $backlogSize }}{{end}} ssl {{ if $cfg.UseHTTP2 }}http2{{ end }};{{ end }} + {{ if not (empty $server.SSLCertificate) }}listen 442 {{ if $cfg.UseProxyProtocol }} proxy_protocol {{ end }} {{ if eq $server.Hostname "_"}} default_server reuseport backlog={{ $backlogSize }}{{end}} ssl {{ if $cfg.UseHTTP2 }}http2{{ end }}; + {{ if $IsIPV6Enabled }}{{ if not (empty $server.SSLCertificate) }}listen [::]:442 {{ if $cfg.UseProxyProtocol }} proxy_protocol {{ end }}{{ end }} {{ if eq $server.Hostname "_"}} default_server reuseport backlog={{ $backlogSize }}{{end}} ssl {{ if $cfg.UseHTTP2 }}http2{{ end }};{{ end }} {{/* comment PEM sha is required to detect changes in the generated configuration and force a reload */}} # PEM sha: {{ $server.SSLPemChecksum }} ssl_certificate {{ $server.SSLCertificate }}; @@ -476,6 +476,7 @@ http { } stream { +<<<<<<< fc67b1d5e2a51cc0037a434583af6530efa1a59c {{ if gt (len $passthroughBackends) 0 }} # map FQDN that requires SSL passthrough map $ssl_preread_server_name $stream_upstream { @@ -486,6 +487,8 @@ stream { # send SSL traffic to this nginx in a different port default nginx-ssl-backend; } +======= +>>>>>>> log_format log_stream {{ $cfg.LogFormatStream }}; @@ -497,21 +500,6 @@ stream { error_log /var/log/nginx/error.log; - # configure default backend for SSL - upstream nginx-ssl-backend { - server 127.0.0.1:442; - } - - {{ buildSSLPassthroughUpstreams $backends .PassthroughBackends }} - - server { - listen 443 {{ if $cfg.UseProxyProtocol }} proxy_protocol{{ end }}; - {{ if $IsIPV6Enabled }}listen [::]:443 {{ if $cfg.UseProxyProtocol }} proxy_protocol{{ end }};{{ end }} - proxy_pass $stream_upstream; - ssl_preread on; - } - {{ end }} - # TCP services {{ range $i, $tcpServer := .TCPBackends }} upstream tcp-{{ $tcpServer.Backend.Namespace }}-{{ $tcpServer.Backend.Name }}-{{ $tcpServer.Backend.Port }} { diff --git a/core/pkg/ingress/controller/controller.go b/core/pkg/ingress/controller/controller.go index 6b21589c3..d4b350286 100644 --- a/core/pkg/ingress/controller/controller.go +++ b/core/pkg/ingress/controller/controller.go @@ -790,6 +790,12 @@ func (ic *GenericController) createUpstreams(data []interface{}) map[string]*ing continue } upstream.Endpoints = endp + + s, e, _ := ic.svcLister.Store.GetByKey(svcKey) + if e { + upstream.Service = s.(*api.Service) + } + upstream.Port = path.Backend.ServicePort } } } diff --git a/core/pkg/ingress/types.go b/core/pkg/ingress/types.go index 8b9626549..548b3b612 100644 --- a/core/pkg/ingress/types.go +++ b/core/pkg/ingress/types.go @@ -147,7 +147,9 @@ type Configuration struct { // Backend describes one or more remote server/s (endpoints) associated with a service type Backend struct { // Name represents an unique api.Service name formatted as -- - Name string `json:"name"` + Name string `json:"name"` + Service *api.Service + Port intstr.IntOrString // This indicates if the communication protocol between the backend and the endpoint is HTTP or HTTPS // Allowing the use of HTTPS // The endpoint/s must provide a TLS connection. @@ -158,8 +160,7 @@ type Backend struct { SSLPassthrough bool `json:"sslPassthrough"` // Endpoints contains the list of endpoints currently running Endpoints []Endpoint `json:"endpoints"` - // StickySession contains the StickyConfig object with stickness configuration - + // StickySessionAffinitySession contains the StickyConfig object with stickness configuration SessionAffinity SessionAffinityConfig } @@ -291,6 +292,9 @@ type Location struct { // The endpoints must provide the TLS termination exposing the required SSL certificate. // The ingress controller only pipes the underlying TCP connection type SSLPassthroughBackend struct { + Namespace string + Service *api.Service + Port intstr.IntOrString // Backend describes the endpoints to use. Backend string `json:"namespace,omitempty"` // Hostname returns the FQDN of the server