From e91c23ff2d6fa56bc477a7480258deb91867ffce Mon Sep 17 00:00:00 2001 From: Manuel de Brito Fontes Date: Sun, 7 Aug 2016 18:53:08 -0400 Subject: [PATCH] Refactoring of templates --- controllers/nginx/Dockerfile | 1 + controllers/nginx/configuration.md | 1 - controllers/nginx/controller.go | 93 ++++++++++--------- controllers/nginx/main.go | 44 ++++++--- controllers/nginx/nginx/command.go | 13 ++- .../nginx/nginx/healthcheck/main_test.go | 4 +- .../nginx/nginx/{ => ingress}/nginx.go | 28 +++--- controllers/nginx/nginx/main.go | 67 +++++++------ controllers/nginx/nginx/ssl.go | 30 ++---- .../nginx/nginx/template/file_watcher.go | 72 ++++++++++++++ .../nginx/nginx/{ => template}/template.go | 84 +++++++++++------ .../nginx/{ => template}/template_test.go | 9 +- controllers/nginx/nginx/utils.go | 23 +---- 13 files changed, 289 insertions(+), 180 deletions(-) rename controllers/nginx/nginx/{ => ingress}/nginx.go (84%) create mode 100644 controllers/nginx/nginx/template/file_watcher.go rename controllers/nginx/nginx/{ => template}/template.go (77%) rename controllers/nginx/nginx/{ => template}/template_test.go (94%) diff --git a/controllers/nginx/Dockerfile b/controllers/nginx/Dockerfile index 3a295fb07..ab4997e92 100644 --- a/controllers/nginx/Dockerfile +++ b/controllers/nginx/Dockerfile @@ -23,6 +23,7 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get update && apt-get install -y \ COPY nginx-ingress-controller / COPY nginx.tmpl /etc/nginx/template/nginx.tmpl +COPY nginx.tmpl /etc/nginx/nginx.tmpl COPY default.conf /etc/nginx/nginx.conf COPY lua /etc/nginx/lua/ diff --git a/controllers/nginx/configuration.md b/controllers/nginx/configuration.md index e8b160406..0716b27e5 100644 --- a/controllers/nginx/configuration.md +++ b/controllers/nginx/configuration.md @@ -282,7 +282,6 @@ Responses with the "text/html" type are always compressed if `use-gzip` is enabl ### Default configuration options -Running `/nginx-ingress-controller --dump-nginx-configuration` is possible to get the value of the options that can be changed. The next table shows the options, the default value and a description |name |default| diff --git a/controllers/nginx/controller.go b/controllers/nginx/controller.go index 727b66c36..29e71b374 100644 --- a/controllers/nginx/controller.go +++ b/controllers/nginx/controller.go @@ -44,6 +44,7 @@ import ( "k8s.io/contrib/ingress/controllers/nginx/nginx/auth" "k8s.io/contrib/ingress/controllers/nginx/nginx/config" "k8s.io/contrib/ingress/controllers/nginx/nginx/healthcheck" + "k8s.io/contrib/ingress/controllers/nginx/nginx/ingress" "k8s.io/contrib/ingress/controllers/nginx/nginx/ipwhitelist" "k8s.io/contrib/ingress/controllers/nginx/nginx/ratelimit" "k8s.io/contrib/ingress/controllers/nginx/nginx/rewrite" @@ -453,7 +454,7 @@ func (lbc *loadBalancerController) sync(key string) error { ings := lbc.ingLister.Store.List() upstreams, servers := lbc.getUpstreamServers(ngxConfig, ings) - return lbc.nginx.CheckAndReload(ngxConfig, nginx.IngressConfig{ + return lbc.nginx.CheckAndReload(ngxConfig, ingress.Configuration{ Upstreams: upstreams, Servers: servers, TCPUpstreams: lbc.getTCPServices(), @@ -513,48 +514,48 @@ func (lbc *loadBalancerController) isStatusIPDefined(lbings []api.LoadBalancerIn return false } -func (lbc *loadBalancerController) getTCPServices() []*nginx.Location { +func (lbc *loadBalancerController) getTCPServices() []*ingress.Location { if lbc.tcpConfigMap == "" { // no configmap for TCP services - return []*nginx.Location{} + return []*ingress.Location{} } ns, name, err := parseNsName(lbc.tcpConfigMap) if err != nil { glog.Warningf("%v", err) - return []*nginx.Location{} + return []*ingress.Location{} } tcpMap, err := lbc.getTCPConfigMap(ns, name) if err != nil { glog.V(3).Infof("no configured tcp services found: %v", err) - return []*nginx.Location{} + return []*ingress.Location{} } return lbc.getStreamServices(tcpMap.Data, api.ProtocolTCP) } -func (lbc *loadBalancerController) getUDPServices() []*nginx.Location { +func (lbc *loadBalancerController) getUDPServices() []*ingress.Location { if lbc.udpConfigMap == "" { // no configmap for TCP services - return []*nginx.Location{} + return []*ingress.Location{} } ns, name, err := parseNsName(lbc.udpConfigMap) if err != nil { glog.Warningf("%v", err) - return []*nginx.Location{} + return []*ingress.Location{} } tcpMap, err := lbc.getUDPConfigMap(ns, name) if err != nil { glog.V(3).Infof("no configured tcp services found: %v", err) - return []*nginx.Location{} + return []*ingress.Location{} } return lbc.getStreamServices(tcpMap.Data, api.ProtocolUDP) } -func (lbc *loadBalancerController) getStreamServices(data map[string]string, proto api.Protocol) []*nginx.Location { - var svcs []*nginx.Location +func (lbc *loadBalancerController) getStreamServices(data map[string]string, proto api.Protocol) []*ingress.Location { + var svcs []*ingress.Location // k -> port to expose in nginx // v -> /: for k, v := range data { @@ -598,7 +599,7 @@ func (lbc *loadBalancerController) getStreamServices(data map[string]string, pro svc := svcObj.(*api.Service) - var endps []nginx.UpstreamServer + var endps []ingress.UpstreamServer targetPort, err := strconv.Atoi(svcPort) if err != nil { for _, sp := range svc.Spec.Ports { @@ -624,9 +625,9 @@ func (lbc *loadBalancerController) getStreamServices(data map[string]string, pro continue } - svcs = append(svcs, &nginx.Location{ + svcs = append(svcs, &ingress.Location{ Path: k, - Upstream: nginx.Upstream{ + Upstream: ingress.Upstream{ Name: fmt.Sprintf("%v-%v-%v", svcNs, svcName, port), Backends: endps, }, @@ -636,8 +637,8 @@ func (lbc *loadBalancerController) getStreamServices(data map[string]string, pro return svcs } -func (lbc *loadBalancerController) getDefaultUpstream() *nginx.Upstream { - upstream := &nginx.Upstream{ +func (lbc *loadBalancerController) getDefaultUpstream() *ingress.Upstream { + upstream := &ingress.Upstream{ Name: defUpstreamName, } svcKey := lbc.defaultSvc @@ -667,7 +668,7 @@ func (lbc *loadBalancerController) getDefaultUpstream() *nginx.Upstream { return upstream } -func (lbc *loadBalancerController) getUpstreamServers(ngxCfg config.Configuration, data []interface{}) ([]*nginx.Upstream, []*nginx.Server) { +func (lbc *loadBalancerController) getUpstreamServers(ngxCfg config.Configuration, data []interface{}) ([]*ingress.Upstream, []*ingress.Server) { upstreams := lbc.createUpstreams(ngxCfg, data) servers := lbc.createServers(data) @@ -754,7 +755,7 @@ func (lbc *loadBalancerController) getUpstreamServers(ngxCfg config.Configuratio if addLoc { - server.Locations = append(server.Locations, &nginx.Location{ + server.Locations = append(server.Locations, &ingress.Location{ Path: nginxPath, Upstream: *ups, Auth: *nginxAuth, @@ -771,31 +772,31 @@ func (lbc *loadBalancerController) getUpstreamServers(ngxCfg config.Configuratio // TODO: find a way to make this more readable // The structs must be ordered to always generate the same file // if the content does not change. - aUpstreams := make([]*nginx.Upstream, 0, len(upstreams)) + aUpstreams := make([]*ingress.Upstream, 0, len(upstreams)) for _, value := range upstreams { if len(value.Backends) == 0 { glog.Warningf("upstream %v does not have any active endpoints. Using default backend", value.Name) value.Backends = append(value.Backends, nginx.NewDefaultServer()) } - sort.Sort(nginx.UpstreamServerByAddrPort(value.Backends)) + sort.Sort(ingress.UpstreamServerByAddrPort(value.Backends)) aUpstreams = append(aUpstreams, value) } - sort.Sort(nginx.UpstreamByNameServers(aUpstreams)) + sort.Sort(ingress.UpstreamByNameServers(aUpstreams)) - aServers := make([]*nginx.Server, 0, len(servers)) + aServers := make([]*ingress.Server, 0, len(servers)) for _, value := range servers { - sort.Sort(nginx.LocationByPath(value.Locations)) + sort.Sort(ingress.LocationByPath(value.Locations)) aServers = append(aServers, value) } - sort.Sort(nginx.ServerByName(aServers)) + sort.Sort(ingress.ServerByName(aServers)) return aUpstreams, aServers } // createUpstreams creates the NGINX upstreams for each service referenced in // Ingress rules. The servers inside the upstream are endpoints. -func (lbc *loadBalancerController) createUpstreams(ngxCfg config.Configuration, data []interface{}) map[string]*nginx.Upstream { - upstreams := make(map[string]*nginx.Upstream) +func (lbc *loadBalancerController) createUpstreams(ngxCfg config.Configuration, data []interface{}) map[string]*ingress.Upstream { + upstreams := make(map[string]*ingress.Upstream) upstreams[defUpstreamName] = lbc.getDefaultUpstream() for _, ingIf := range data { @@ -852,12 +853,12 @@ func (lbc *loadBalancerController) createUpstreams(ngxCfg config.Configuration, return upstreams } -func (lbc *loadBalancerController) createServers(data []interface{}) map[string]*nginx.Server { - servers := make(map[string]*nginx.Server) +func (lbc *loadBalancerController) createServers(data []interface{}) map[string]*ingress.Server { + servers := make(map[string]*ingress.Server) pems := lbc.getPemsFromIngress(data) - var ngxCert nginx.SSLCert + var ngxCert ingress.SSLCert var err error if lbc.defSSLCertificate == "" { @@ -868,13 +869,13 @@ func (lbc *loadBalancerController) createServers(data []interface{}) map[string] ngxCert, err = lbc.getPemCertificate(lbc.defSSLCertificate) } - locs := []*nginx.Location{} - locs = append(locs, &nginx.Location{ + locs := []*ingress.Location{} + locs = append(locs, &ingress.Location{ Path: rootLocation, IsDefBackend: true, Upstream: *lbc.getDefaultUpstream(), }) - servers[defServerName] = &nginx.Server{Name: defServerName, Locations: locs} + servers[defServerName] = &ingress.Server{Name: defServerName, Locations: locs} if err == nil { pems[defServerName] = ngxCert @@ -896,13 +897,13 @@ func (lbc *loadBalancerController) createServers(data []interface{}) map[string] } if _, ok := servers[host]; !ok { - locs := []*nginx.Location{} - locs = append(locs, &nginx.Location{ + locs := []*ingress.Location{} + locs = append(locs, &ingress.Location{ Path: rootLocation, IsDefBackend: true, Upstream: *lbc.getDefaultUpstream(), }) - servers[host] = &nginx.Server{Name: host, Locations: locs} + servers[host] = &ingress.Server{Name: host, Locations: locs} } if ngxCert, ok := pems[host]; ok { @@ -918,8 +919,8 @@ func (lbc *loadBalancerController) createServers(data []interface{}) map[string] return servers } -func (lbc *loadBalancerController) getPemsFromIngress(data []interface{}) map[string]nginx.SSLCert { - pems := make(map[string]nginx.SSLCert) +func (lbc *loadBalancerController) getPemsFromIngress(data []interface{}) map[string]ingress.SSLCert { + pems := make(map[string]ingress.SSLCert) for _, ingIf := range data { ing := ingIf.(*extensions.Ingress) @@ -946,23 +947,23 @@ func (lbc *loadBalancerController) getPemsFromIngress(data []interface{}) map[st return pems } -func (lbc *loadBalancerController) getPemCertificate(secretName string) (nginx.SSLCert, error) { +func (lbc *loadBalancerController) getPemCertificate(secretName string) (ingress.SSLCert, error) { secretInterface, exists, err := lbc.secrLister.Store.GetByKey(secretName) if err != nil { - return nginx.SSLCert{}, fmt.Errorf("Error retriveing secret %v: %v", secretName, err) + return ingress.SSLCert{}, fmt.Errorf("Error retriveing secret %v: %v", secretName, err) } if !exists { - return nginx.SSLCert{}, fmt.Errorf("Secret %v does not exists", secretName) + return ingress.SSLCert{}, fmt.Errorf("Secret %v does not exists", secretName) } secret := secretInterface.(*api.Secret) cert, ok := secret.Data[api.TLSCertKey] if !ok { - return nginx.SSLCert{}, fmt.Errorf("Secret %v has no private key", secretName) + return ingress.SSLCert{}, fmt.Errorf("Secret %v has no private key", secretName) } key, ok := secret.Data[api.TLSPrivateKeyKey] if !ok { - return nginx.SSLCert{}, fmt.Errorf("Secret %v has no cert", secretName) + return ingress.SSLCert{}, fmt.Errorf("Secret %v has no cert", secretName) } nsSecName := strings.Replace(secretName, "/", "-", -1) @@ -986,15 +987,15 @@ func (lbc *loadBalancerController) secrReferenced(namespace string, name string) } // getEndpoints returns a list of : for a given service/target port combination. -func (lbc *loadBalancerController) getEndpoints(s *api.Service, servicePort intstr.IntOrString, proto api.Protocol, hz *healthcheck.Upstream) []nginx.UpstreamServer { +func (lbc *loadBalancerController) getEndpoints(s *api.Service, servicePort intstr.IntOrString, proto api.Protocol, hz *healthcheck.Upstream) []ingress.UpstreamServer { glog.V(3).Infof("getting endpoints for service %v/%v and port %v", s.Namespace, s.Name, servicePort.String()) ep, err := lbc.endpLister.GetServiceEndpoints(s) if err != nil { glog.Warningf("unexpected error obtaining service endpoints: %v", err) - return []nginx.UpstreamServer{} + return []ingress.UpstreamServer{} } - upsServers := []nginx.UpstreamServer{} + upsServers := []ingress.UpstreamServer{} for _, ss := range ep.Subsets { for _, epPort := range ss.Ports { @@ -1044,7 +1045,7 @@ func (lbc *loadBalancerController) getEndpoints(s *api.Service, servicePort ints } for _, epAddress := range ss.Addresses { - ups := nginx.UpstreamServer{ + ups := ingress.UpstreamServer{ Address: epAddress.IP, Port: fmt.Sprintf("%v", targetPort), MaxFails: hz.MaxFails, diff --git a/controllers/nginx/main.go b/controllers/nginx/main.go index f561e3b30..737d69359 100644 --- a/controllers/nginx/main.go +++ b/controllers/nginx/main.go @@ -19,6 +19,7 @@ package main import ( "flag" "fmt" + "io/ioutil" "net/http" "net/http/pprof" "os" @@ -29,8 +30,6 @@ import ( "github.com/golang/glog" "github.com/spf13/pflag" - "k8s.io/contrib/ingress/controllers/nginx/nginx" - "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/client/unversioned" "k8s.io/kubernetes/pkg/healthz" @@ -77,9 +76,6 @@ var ( healthzPort = flags.Int("healthz-port", healthPort, "port for healthz endpoint.") - buildCfg = flags.Bool("dump-nginx-configuration", false, `Returns a ConfigMap with the default nginx conguration. - This can be used as a guide to create a custom configuration.`) - profiling = flags.Bool("profiling", true, `Enable profiling via web interface host:port/debug/pprof/`) defSSLCertificate = flags.String("default-ssl-certificate", "", `Name of the secret that contains a SSL @@ -93,11 +89,6 @@ func main() { glog.Infof("Using build: %v - %v", gitRepo, version) - if *buildCfg { - fmt.Printf("Example of ConfigMap to customize NGINX configuration:\n%v", nginx.ConfigMapAsString()) - os.Exit(0) - } - if *defaultSvc == "" { glog.Fatalf("Please specify --default-backend-service") } @@ -123,12 +114,14 @@ func main() { glog.Infof("Validated %v as the default backend", *defaultSvc) if *nxgConfigMap != "" { - _, _, err := parseNsName(*nxgConfigMap) + _, _, err = parseNsName(*nxgConfigMap) if err != nil { glog.Fatalf("configmap error: %v", err) } } + checkTemplate() + lbc, err := newLoadBalancerController(kubeClient, *resyncPeriod, *defaultSvc, *watchNamespace, *nxgConfigMap, *tcpConfigMapName, *udpConfigMapName, *defSSLCertificate, runtimePodInfo) @@ -158,12 +151,12 @@ func registerHandlers(lbc *loadBalancerController) { mux := http.NewServeMux() healthz.InstallHandler(mux, lbc.nginx) - http.HandleFunc("/build", func(w http.ResponseWriter, r *http.Request) { + mux.HandleFunc("/build", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) fmt.Fprintf(w, "build: %v - %v", gitRepo, version) }) - http.HandleFunc("/stop", func(w http.ResponseWriter, r *http.Request) { + mux.HandleFunc("/stop", func(w http.ResponseWriter, r *http.Request) { lbc.Stop() }) @@ -195,3 +188,28 @@ func handleSigterm(lbc *loadBalancerController) { glog.Infof("Exiting with %v", exitCode) os.Exit(exitCode) } + +const ( + defTmpl = "/etc/nginx/template/nginx.tmpl" + fallbackTmpl = "/etc/nginx/nginx.tmpl" +) + +// checkTemplate verifies the NGINX template exists (file /etc/nginx/template/nginx.tmpl) +// If the file does not exists it means: +// a. custom docker image +// b. standard image using watch-resource sidecar with emptyDir volume +// If the file /etc/nginx/nginx.tmpl exists copy the file to /etc/nginx/template/nginx.tmpl +// or terminate the execution (It is not possible to start NGINX without a template) +func checkTemplate() { + _, err := os.Stat(defTmpl) + if err != nil { + glog.Warningf("error checking template %v: %v", defTmpl, err) + data, err := ioutil.ReadFile(fallbackTmpl) + if err != nil { + glog.Fatalf("error reading template %v: %v", fallbackTmpl, err) + } + if err = ioutil.WriteFile(defTmpl, data, 0644); err != nil { + glog.Fatalf("error copying %v to %v: %v", fallbackTmpl, defTmpl, err) + } + } +} diff --git a/controllers/nginx/nginx/command.go b/controllers/nginx/nginx/command.go index 39512670a..ec69a12a8 100644 --- a/controllers/nginx/nginx/command.go +++ b/controllers/nginx/nginx/command.go @@ -27,6 +27,7 @@ import ( "k8s.io/kubernetes/pkg/healthz" "k8s.io/contrib/ingress/controllers/nginx/nginx/config" + "k8s.io/contrib/ingress/controllers/nginx/nginx/ingress" ) // Start starts a nginx (master process) and waits. If the process ends @@ -56,19 +57,23 @@ func (ngx *Manager) Start() { // shut down, stop accepting new connections and continue to service current requests // until all such requests are serviced. After that, the old worker processes exit. // http://nginx.org/en/docs/beginners_guide.html#control -func (ngx *Manager) CheckAndReload(cfg config.Configuration, ingressCfg IngressConfig) error { +func (ngx *Manager) CheckAndReload(cfg config.Configuration, ingressCfg ingress.Configuration) error { ngx.reloadRateLimiter.Accept() ngx.reloadLock.Lock() defer ngx.reloadLock.Unlock() - newCfg, err := ngx.writeCfg(cfg, ingressCfg) - + newCfg, err := ngx.template.Write(cfg, ingressCfg) if err != nil { return fmt.Errorf("failed to write new nginx configuration. Avoiding reload: %v", err) } - if newCfg { + changed, err := ngx.needsReload(newCfg) + if err != nil { + return err + } + + if changed { if err := ngx.shellOut("nginx -s reload"); err != nil { return fmt.Errorf("error reloading nginx: %v", err) } diff --git a/controllers/nginx/nginx/healthcheck/main_test.go b/controllers/nginx/nginx/healthcheck/main_test.go index 348df45e6..aa96795f6 100644 --- a/controllers/nginx/nginx/healthcheck/main_test.go +++ b/controllers/nginx/nginx/healthcheck/main_test.go @@ -84,7 +84,7 @@ func TestAnnotations(t *testing.T) { t.Errorf("Unexpected error: %v", err) } if mf != 1 { - t.Errorf("Expected 1 but returned %s", mf) + t.Errorf("Expected 1 but returned %v", mf) } ft, err := ingAnnotations(ing.GetAnnotations()).failTimeout() @@ -92,7 +92,7 @@ func TestAnnotations(t *testing.T) { t.Errorf("Unexpected error: %v", err) } if ft != 1 { - t.Errorf("Expected 1 but returned %s", ft) + t.Errorf("Expected 1 but returned %v", ft) } } diff --git a/controllers/nginx/nginx/nginx.go b/controllers/nginx/nginx/ingress/nginx.go similarity index 84% rename from controllers/nginx/nginx/nginx.go rename to controllers/nginx/nginx/ingress/nginx.go index 31986d9b9..6dcff198a 100644 --- a/controllers/nginx/nginx/nginx.go +++ b/controllers/nginx/nginx/ingress/nginx.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package nginx +package ingress import ( "k8s.io/contrib/ingress/controllers/nginx/nginx/auth" @@ -23,8 +23,8 @@ import ( "k8s.io/contrib/ingress/controllers/nginx/nginx/rewrite" ) -// IngressConfig describes an NGINX configuration -type IngressConfig struct { +// Configuration describes an NGINX configuration +type Configuration struct { Upstreams []*Upstream Servers []*Server TCPUpstreams []*Location @@ -113,15 +113,15 @@ func (c LocationByPath) Less(i, j int) bool { return c[i].Path > c[j].Path } -// NewDefaultServer return an UpstreamServer to be use as default server that returns 503. -func NewDefaultServer() UpstreamServer { - return UpstreamServer{Address: "127.0.0.1", Port: "8181"} -} - -// NewUpstream creates an upstream without servers. -func NewUpstream(name string) *Upstream { - return &Upstream{ - Name: name, - Backends: []UpstreamServer{}, - } +// SSLCert describes a SSL certificate to be used in NGINX +type SSLCert struct { + CertFileName string + KeyFileName string + // PemFileName contains the path to the file with the certificate and key concatenated + PemFileName string + // PemSHA contains the sha1 of the pem file. + // This is used to detect changes in the secret that contains the certificates + PemSHA string + // CN contains all the common names defined in the SSL certificate + CN []string } diff --git a/controllers/nginx/nginx/main.go b/controllers/nginx/nginx/main.go index f60ad2dad..9f0d68cb7 100644 --- a/controllers/nginx/nginx/main.go +++ b/controllers/nginx/nginx/main.go @@ -17,22 +17,22 @@ limitations under the License. package nginx import ( - "fmt" "os" "strings" "sync" - "text/template" "github.com/golang/glog" - "github.com/fatih/structs" - "github.com/ghodss/yaml" - - "k8s.io/kubernetes/pkg/api" client "k8s.io/kubernetes/pkg/client/unversioned" "k8s.io/kubernetes/pkg/util/flowcontrol" "k8s.io/contrib/ingress/controllers/nginx/nginx/config" + "k8s.io/contrib/ingress/controllers/nginx/nginx/ingress" + ngx_template "k8s.io/contrib/ingress/controllers/nginx/nginx/template" +) + +var ( + tmplPath = "/etc/nginx/template/nginx.tmpl" ) // Manager ... @@ -48,11 +48,24 @@ type Manager struct { reloadRateLimiter flowcontrol.RateLimiter // template loaded ready to be used to generate the nginx configuration file - template *template.Template + template *ngx_template.Template reloadLock *sync.Mutex } +// NewDefaultServer return an UpstreamServer to be use as default server that returns 503. +func NewDefaultServer() ingress.UpstreamServer { + return ingress.UpstreamServer{Address: "127.0.0.1", Port: "8181"} +} + +// NewUpstream creates an upstream without servers. +func NewUpstream(name string) *ingress.Upstream { + return &ingress.Upstream{ + Name: name, + Backends: []ingress.UpstreamServer{}, + } +} + // NewManager ... func NewManager(kubeClient *client.Client) *Manager { ngx := &Manager{ @@ -67,10 +80,26 @@ func NewManager(kubeClient *client.Client) *Manager { ngx.sslDHParam = ngx.SearchDHParamFile(config.SSLDirectory) - if err := ngx.loadTemplate(); err != nil { + var onChange func() + + onChange = func() { + template, err := ngx_template.NewTemplate(tmplPath, onChange) + if err != nil { + glog.Warningf("invalid NGINX template: %v", err) + return + } + + ngx.template.Close() + ngx.template = template + glog.Info("new NGINX template loaded") + } + + template, err := ngx_template.NewTemplate(tmplPath, onChange) + if err != nil { glog.Fatalf("invalid NGINX template: %v", err) } + ngx.template = template return ngx } @@ -83,25 +112,3 @@ func (nginx *Manager) createCertsDir(base string) { glog.Fatalf("Couldn't create directory %v: %v", base, err) } } - -// ConfigMapAsString returns a ConfigMap with the default NGINX -// configuration to be used a guide to provide a custom configuration -func ConfigMapAsString() string { - cfg := &api.ConfigMap{} - cfg.Name = "custom-name" - cfg.Namespace = "a-valid-namespace" - cfg.Data = make(map[string]string) - - data := structs.Map(config.NewDefault()) - for k, v := range data { - cfg.Data[k] = fmt.Sprintf("%v", v) - } - - out, err := yaml.Marshal(cfg) - if err != nil { - glog.Warningf("Unexpected error creating default configuration: %v", err) - return "" - } - - return string(out) -} diff --git a/controllers/nginx/nginx/ssl.go b/controllers/nginx/nginx/ssl.go index 33234756e..7f7d8bb28 100644 --- a/controllers/nginx/nginx/ssl.go +++ b/controllers/nginx/nginx/ssl.go @@ -28,54 +28,42 @@ import ( "github.com/golang/glog" "k8s.io/contrib/ingress/controllers/nginx/nginx/config" + "k8s.io/contrib/ingress/controllers/nginx/nginx/ingress" ) -// SSLCert describes a SSL certificate to be used in NGINX -type SSLCert struct { - CertFileName string - KeyFileName string - // PemFileName contains the path to the file with the certificate and key concatenated - PemFileName string - // PemSHA contains the sha1 of the pem file. - // This is used to detect changes in the secret that contains the certificates - PemSHA string - // CN contains all the common names defined in the SSL certificate - CN []string -} - // AddOrUpdateCertAndKey creates a .pem file wth the cert and the key with the specified name -func (nginx *Manager) AddOrUpdateCertAndKey(name string, cert string, key string) (SSLCert, error) { +func (nginx *Manager) AddOrUpdateCertAndKey(name string, cert string, key string) (ingress.SSLCert, error) { temporaryPemFileName := fmt.Sprintf("%v.pem", name) pemFileName := fmt.Sprintf("%v/%v.pem", config.SSLDirectory, name) temporaryPemFile, err := ioutil.TempFile("", temporaryPemFileName) if err != nil { - return SSLCert{}, fmt.Errorf("Couldn't create temp pem file %v: %v", temporaryPemFile.Name(), err) + return ingress.SSLCert{}, fmt.Errorf("Couldn't create temp pem file %v: %v", temporaryPemFile.Name(), err) } _, err = temporaryPemFile.WriteString(fmt.Sprintf("%v\n%v", cert, key)) if err != nil { - return SSLCert{}, fmt.Errorf("Couldn't write to pem file %v: %v", temporaryPemFile.Name(), err) + return ingress.SSLCert{}, fmt.Errorf("Couldn't write to pem file %v: %v", temporaryPemFile.Name(), err) } err = temporaryPemFile.Close() if err != nil { - return SSLCert{}, fmt.Errorf("Couldn't close temp pem file %v: %v", temporaryPemFile.Name(), err) + return ingress.SSLCert{}, fmt.Errorf("Couldn't close temp pem file %v: %v", temporaryPemFile.Name(), err) } cn, err := nginx.commonNames(temporaryPemFile.Name()) if err != nil { os.Remove(temporaryPemFile.Name()) - return SSLCert{}, err + return ingress.SSLCert{}, err } err = os.Rename(temporaryPemFile.Name(), pemFileName) if err != nil { os.Remove(temporaryPemFile.Name()) - return SSLCert{}, fmt.Errorf("Couldn't move temp pem file %v to destination %v: %v", temporaryPemFile.Name(), pemFileName, err) + return ingress.SSLCert{}, fmt.Errorf("Couldn't move temp pem file %v to destination %v: %v", temporaryPemFile.Name(), pemFileName, err) } - return SSLCert{ + return ingress.SSLCert{ CertFileName: cert, KeyFileName: key, PemFileName: pemFileName, @@ -133,6 +121,8 @@ func (nginx *Manager) SearchDHParamFile(baseDir string) string { return "" } +// pemSHA1 returns the SHA1 of a pem file. This is used to +// reload NGINX in case a secret with a SSL certificate changed. func (nginx *Manager) pemSHA1(filename string) string { hasher := sha1.New() s, err := ioutil.ReadFile(filename) diff --git a/controllers/nginx/nginx/template/file_watcher.go b/controllers/nginx/nginx/template/file_watcher.go new file mode 100644 index 000000000..e35afba11 --- /dev/null +++ b/controllers/nginx/nginx/template/file_watcher.go @@ -0,0 +1,72 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package template + +import ( + "log" + "path" + + "gopkg.in/fsnotify.v1" +) + +type fileWatcher struct { + file string + watcher *fsnotify.Watcher + onEvent func() +} + +func newFileWatcher(file string, onEvent func()) (fileWatcher, error) { + fw := fileWatcher{ + file: file, + onEvent: onEvent, + } + + err := fw.watch() + return fw, err +} + +func (f fileWatcher) close() error { + return f.watcher.Close() +} + +// watch creates a fsnotify watcher for a file and create of write events +func (f *fileWatcher) watch() error { + watcher, err := fsnotify.NewWatcher() + if err != nil { + return err + } + f.watcher = watcher + + dir, file := path.Split(f.file) + go func(file string) { + for { + select { + case event := <-watcher.Events: + if event.Op&fsnotify.Write == fsnotify.Write || + event.Op&fsnotify.Create == fsnotify.Create && + event.Name == file { + f.onEvent() + } + case err := <-watcher.Errors: + if err != nil { + log.Printf("error watching file: %v\n", err) + } + } + } + }(file) + return watcher.Add(dir) +} diff --git a/controllers/nginx/nginx/template.go b/controllers/nginx/nginx/template/template.go similarity index 77% rename from controllers/nginx/nginx/template.go rename to controllers/nginx/nginx/template/template.go index f5cd24f5c..dd1d09dd4 100644 --- a/controllers/nginx/nginx/template.go +++ b/controllers/nginx/nginx/template/template.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package nginx +package template import ( "bytes" @@ -22,12 +22,14 @@ import ( "fmt" "regexp" "strings" - "text/template" + text_template "text/template" "github.com/fatih/structs" "github.com/golang/glog" "k8s.io/contrib/ingress/controllers/nginx/nginx/config" + "k8s.io/contrib/ingress/controllers/nginx/nginx/ingress" + "k8s.io/kubernetes/pkg/util/sysctl" ) const ( @@ -36,9 +38,8 @@ const ( var ( camelRegexp = regexp.MustCompile("[0-9A-Za-z]+") - tmplPath = "/etc/nginx/template/nginx.tmpl" - funcMap = template.FuncMap{ + funcMap = text_template.FuncMap{ "empty": func(input interface{}) bool { check, ok := input.(string) if ok { @@ -54,48 +55,64 @@ var ( } ) -func (ngx *Manager) loadTemplate() error { - tmpl, err := template.New("nginx.tmpl").Funcs(funcMap).ParseFiles(tmplPath) - if err != nil { - return err - } - ngx.template = tmpl - return nil +// Template ... +type Template struct { + tmpl *text_template.Template + fw fileWatcher } -func (ngx *Manager) writeCfg(cfg config.Configuration, ingressCfg IngressConfig) (bool, error) { +//NewTemplate returns a new Template instance or an +//error if the specified template file contains errors +func NewTemplate(file string, onChange func()) (*Template, error) { + tmpl, err := text_template.New("nginx.tmpl").Funcs(funcMap).ParseFiles(file) + if err != nil { + return nil, err + } + fw, err := newFileWatcher(file, onChange) + if err != nil { + return nil, err + } + + return &Template{ + tmpl: tmpl, + fw: fw, + }, nil +} + +// Close removes the file watcher +func (t *Template) Close() { + t.fw.close() +} + +// Write populates a buffer using a template with NGINX configuration +// and the servers and upstreams created by Ingress rules +func (t *Template) Write(cfg config.Configuration, ingressCfg ingress.Configuration) ([]byte, error) { conf := make(map[string]interface{}) conf["backlogSize"] = sysctlSomaxconn() conf["upstreams"] = ingressCfg.Upstreams conf["servers"] = ingressCfg.Servers conf["tcpUpstreams"] = ingressCfg.TCPUpstreams conf["udpUpstreams"] = ingressCfg.UDPUpstreams - conf["defResolver"] = ngx.defResolver - conf["sslDHParam"] = ngx.sslDHParam + conf["defResolver"] = cfg.Resolver + conf["sslDHParam"] = cfg.SSLDHParam conf["customErrors"] = len(cfg.CustomHTTPErrors) > 0 conf["cfg"] = fixKeyNames(structs.Map(cfg)) if glog.V(3) { b, err := json.Marshal(conf) if err != nil { - glog.Errorf("unexpected error:", err) + glog.Errorf("unexpected error: %v", err) } glog.Infof("NGINX configuration: %v", string(b)) } buffer := new(bytes.Buffer) - err := ngx.template.Execute(buffer, conf) + err := t.tmpl.Execute(buffer, conf) if err != nil { glog.V(3).Infof("%v", string(buffer.Bytes())) - return false, err } - changed, err := ngx.needsReload(buffer) - if err != nil { - return false, err - } - - return changed, nil + return buffer.Bytes(), err } func fixKeyNames(data map[string]interface{}) map[string]interface{} { @@ -121,7 +138,7 @@ func toCamelCase(src string) string { // buildLocation produces the location string, if the ingress has redirects // (specified through the ingress.kubernetes.io/rewrite-to annotation) func buildLocation(input interface{}) string { - location, ok := input.(*Location) + location, ok := input.(*ingress.Location) if !ok { return slash } @@ -139,7 +156,7 @@ func buildLocation(input interface{}) string { // If the annotation ingress.kubernetes.io/add-base-url:"true" is specified it will // add a base tag in the head of the response from the service func buildProxyPass(input interface{}) string { - location, ok := input.(*Location) + location, ok := input.(*ingress.Location) if !ok { return "" } @@ -200,7 +217,7 @@ func buildProxyPass(input interface{}) string { func buildRateLimitZones(input interface{}) []string { zones := []string{} - servers, ok := input.([]*Server) + servers, ok := input.([]*ingress.Server) if !ok { return zones } @@ -230,7 +247,7 @@ func buildRateLimitZones(input interface{}) []string { func buildRateLimit(input interface{}) []string { limits := []string{} - loc, ok := input.(*Location) + loc, ok := input.(*ingress.Location) if !ok { return limits } @@ -249,3 +266,16 @@ func buildRateLimit(input interface{}) []string { return limits } + +// sysctlSomaxconn returns the value of net.core.somaxconn, i.e. +// maximum number of connections that can be queued for acceptance +// http://nginx.org/en/docs/http/ngx_http_core_module.html#listen +func sysctlSomaxconn() int { + maxConns, err := sysctl.GetSysctl("net/core/somaxconn") + if err != nil || maxConns < 512 { + glog.Warningf("system net.core.somaxconn=%v. Using NGINX default (511)", maxConns) + return 511 + } + + return maxConns +} diff --git a/controllers/nginx/nginx/template_test.go b/controllers/nginx/nginx/template/template_test.go similarity index 94% rename from controllers/nginx/nginx/template_test.go rename to controllers/nginx/nginx/template/template_test.go index f13aaf339..36650561c 100644 --- a/controllers/nginx/nginx/template_test.go +++ b/controllers/nginx/nginx/template/template_test.go @@ -14,12 +14,13 @@ See the License for the specific language governing permissions and limitations under the License. */ -package nginx +package template import ( "strings" "testing" + "k8s.io/contrib/ingress/controllers/nginx/nginx/ingress" "k8s.io/contrib/ingress/controllers/nginx/nginx/rewrite" ) @@ -70,7 +71,7 @@ var ( func TestBuildLocation(t *testing.T) { for k, tc := range tmplFuncTestcases { - loc := &Location{ + loc := &ingress.Location{ Path: tc.Path, Redirect: rewrite.Redirect{Target: tc.Target, AddBaseURL: tc.AddBaseURL}, } @@ -84,10 +85,10 @@ func TestBuildLocation(t *testing.T) { func TestBuildProxyPass(t *testing.T) { for k, tc := range tmplFuncTestcases { - loc := &Location{ + loc := &ingress.Location{ Path: tc.Path, Redirect: rewrite.Redirect{Target: tc.Target, AddBaseURL: tc.AddBaseURL}, - Upstream: Upstream{Name: "upstream-name"}, + Upstream: ingress.Upstream{Name: "upstream-name"}, } pp := buildProxyPass(loc) diff --git a/controllers/nginx/nginx/utils.go b/controllers/nginx/nginx/utils.go index fa18be392..0c86933b8 100644 --- a/controllers/nginx/nginx/utils.go +++ b/controllers/nginx/nginx/utils.go @@ -28,7 +28,6 @@ import ( "github.com/golang/glog" "github.com/mitchellh/mapstructure" "k8s.io/kubernetes/pkg/api" - "k8s.io/kubernetes/pkg/util/sysctl" "k8s.io/contrib/ingress/controllers/nginx/nginx/config" ) @@ -160,7 +159,7 @@ func (ngx *Manager) filterErrors(errCodes []int) []int { return fa } -func (ngx *Manager) needsReload(data *bytes.Buffer) (bool, error) { +func (ngx *Manager) needsReload(data []byte) (bool, error) { filename := ngx.ConfigFile in, err := os.Open(filename) if err != nil { @@ -173,14 +172,13 @@ func (ngx *Manager) needsReload(data *bytes.Buffer) (bool, error) { return false, err } - res := data.Bytes() - if !bytes.Equal(src, res) { - err = ioutil.WriteFile(filename, res, 0644) + if !bytes.Equal(src, data) { + err = ioutil.WriteFile(filename, data, 0644) if err != nil { return false, err } - dData, err := diff(src, res) + dData, err := diff(src, data) if err != nil { glog.Errorf("error computing diff: %s", err) return true, nil @@ -221,16 +219,3 @@ func diff(b1, b2 []byte) (data []byte, err error) { } return } - -// sysctlSomaxconn returns the value of net.core.somaxconn, i.e. -// maximum number of connections that can be queued for acceptance -// http://nginx.org/en/docs/http/ngx_http_core_module.html#listen -func sysctlSomaxconn() int { - maxConns, err := sysctl.GetSysctl("net/core/somaxconn") - if err != nil || maxConns < 512 { - glog.V(3).Infof("system net.core.somaxconn=%v. Using NGINX default (511)", maxConns) - return 511 - } - - return maxConns -}