DevFW-CICD/ingress-nginx-helm
16
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Refactoring of templates

Browse source
This commit is contained in:
Manuel de Brito Fontes 2016-08-07 18:53:08 -04:00
parent e4236ad0f2
commit e91c23ff2d
13 changed files with 289 additions and 180 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
controllers/nginx/Dockerfile
View file

@ -23,6 +23,7 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get update && apt-get install -y \
COPY nginx-ingress-controller / COPY nginx-ingress-controller /
COPY nginx.tmpl /etc/nginx/template/nginx.tmpl COPY nginx.tmpl /etc/nginx/template/nginx.tmpl
COPY nginx.tmpl /etc/nginx/nginx.tmpl
COPY default.conf /etc/nginx/nginx.conf COPY default.conf /etc/nginx/nginx.conf
COPY lua /etc/nginx/lua/ COPY lua /etc/nginx/lua/

1
controllers/nginx/configuration.md
View file

@ -282,7 +282,6 @@ Responses with the "text/html" type are always compressed if `use-gzip` is enabl
### Default configuration options ### Default configuration options
Running `/nginx-ingress-controller --dump-nginx-configuration` is possible to get the value of the options that can be changed.
The next table shows the options, the default value and a description The next table shows the options, the default value and a description
|name |default| |name |default|

93
controllers/nginx/controller.go
View file

@ -44,6 +44,7 @@ import (
"k8s.io/contrib/ingress/controllers/nginx/nginx/auth" "k8s.io/contrib/ingress/controllers/nginx/nginx/auth"
"k8s.io/contrib/ingress/controllers/nginx/nginx/config" "k8s.io/contrib/ingress/controllers/nginx/nginx/config"
"k8s.io/contrib/ingress/controllers/nginx/nginx/healthcheck" "k8s.io/contrib/ingress/controllers/nginx/nginx/healthcheck"
"k8s.io/contrib/ingress/controllers/nginx/nginx/ingress"
"k8s.io/contrib/ingress/controllers/nginx/nginx/ipwhitelist" "k8s.io/contrib/ingress/controllers/nginx/nginx/ipwhitelist"
"k8s.io/contrib/ingress/controllers/nginx/nginx/ratelimit" "k8s.io/contrib/ingress/controllers/nginx/nginx/ratelimit"
"k8s.io/contrib/ingress/controllers/nginx/nginx/rewrite" "k8s.io/contrib/ingress/controllers/nginx/nginx/rewrite"
@ -453,7 +454,7 @@ func (lbc *loadBalancerController) sync(key string) error {
ings := lbc.ingLister.Store.List() ings := lbc.ingLister.Store.List()
upstreams, servers := lbc.getUpstreamServers(ngxConfig, ings) upstreams, servers := lbc.getUpstreamServers(ngxConfig, ings)
return lbc.nginx.CheckAndReload(ngxConfig, nginx.IngressConfig{ return lbc.nginx.CheckAndReload(ngxConfig, ingress.Configuration{
Upstreams: upstreams, Upstreams: upstreams,
Servers: servers, Servers: servers,
TCPUpstreams: lbc.getTCPServices(), TCPUpstreams: lbc.getTCPServices(),
@ -513,48 +514,48 @@ func (lbc *loadBalancerController) isStatusIPDefined(lbings []api.LoadBalancerIn
return false return false
} }
func (lbc *loadBalancerController) getTCPServices() []*nginx.Location { func (lbc *loadBalancerController) getTCPServices() []*ingress.Location {
if lbc.tcpConfigMap == "" { if lbc.tcpConfigMap == "" {
// no configmap for TCP services // no configmap for TCP services
return []*nginx.Location{} return []*ingress.Location{}
} }
ns, name, err := parseNsName(lbc.tcpConfigMap) ns, name, err := parseNsName(lbc.tcpConfigMap)
if err != nil { if err != nil {
glog.Warningf("%v", err) glog.Warningf("%v", err)
return []*nginx.Location{} return []*ingress.Location{}
} }
tcpMap, err := lbc.getTCPConfigMap(ns, name) tcpMap, err := lbc.getTCPConfigMap(ns, name)
if err != nil { if err != nil {
glog.V(3).Infof("no configured tcp services found: %v", err) glog.V(3).Infof("no configured tcp services found: %v", err)
return []*nginx.Location{} return []*ingress.Location{}
} }
return lbc.getStreamServices(tcpMap.Data, api.ProtocolTCP) return lbc.getStreamServices(tcpMap.Data, api.ProtocolTCP)
} }
func (lbc *loadBalancerController) getUDPServices() []*nginx.Location { func (lbc *loadBalancerController) getUDPServices() []*ingress.Location {
if lbc.udpConfigMap == "" { if lbc.udpConfigMap == "" {
// no configmap for TCP services // no configmap for TCP services
return []*nginx.Location{} return []*ingress.Location{}
} }
ns, name, err := parseNsName(lbc.udpConfigMap) ns, name, err := parseNsName(lbc.udpConfigMap)
if err != nil { if err != nil {
glog.Warningf("%v", err) glog.Warningf("%v", err)
return []*nginx.Location{} return []*ingress.Location{}
} }
tcpMap, err := lbc.getUDPConfigMap(ns, name) tcpMap, err := lbc.getUDPConfigMap(ns, name)
if err != nil { if err != nil {
glog.V(3).Infof("no configured tcp services found: %v", err) glog.V(3).Infof("no configured tcp services found: %v", err)
return []*nginx.Location{} return []*ingress.Location{}
} }
return lbc.getStreamServices(tcpMap.Data, api.ProtocolUDP) return lbc.getStreamServices(tcpMap.Data, api.ProtocolUDP)
} }
func (lbc *loadBalancerController) getStreamServices(data map[string]string, proto api.Protocol) []*nginx.Location { func (lbc *loadBalancerController) getStreamServices(data map[string]string, proto api.Protocol) []*ingress.Location {
var svcs []*nginx.Location var svcs []*ingress.Location
// k -> port to expose in nginx // k -> port to expose in nginx
// v -> <namespace>/<service name>:<port from service to be used> // v -> <namespace>/<service name>:<port from service to be used>
for k, v := range data { for k, v := range data {
@ -598,7 +599,7 @@ func (lbc *loadBalancerController) getStreamServices(data map[string]string, pro
svc := svcObj.(*api.Service) svc := svcObj.(*api.Service)
var endps []nginx.UpstreamServer var endps []ingress.UpstreamServer
targetPort, err := strconv.Atoi(svcPort) targetPort, err := strconv.Atoi(svcPort)
if err != nil { if err != nil {
for _, sp := range svc.Spec.Ports { for _, sp := range svc.Spec.Ports {
@ -624,9 +625,9 @@ func (lbc *loadBalancerController) getStreamServices(data map[string]string, pro
continue continue
} }
svcs = append(svcs, &nginx.Location{ svcs = append(svcs, &ingress.Location{
Path: k, Path: k,
Upstream: nginx.Upstream{ Upstream: ingress.Upstream{
Name: fmt.Sprintf("%v-%v-%v", svcNs, svcName, port), Name: fmt.Sprintf("%v-%v-%v", svcNs, svcName, port),
Backends: endps, Backends: endps,
}, },
@ -636,8 +637,8 @@ func (lbc *loadBalancerController) getStreamServices(data map[string]string, pro
return svcs return svcs
} }
func (lbc *loadBalancerController) getDefaultUpstream() *nginx.Upstream { func (lbc *loadBalancerController) getDefaultUpstream() *ingress.Upstream {
upstream := &nginx.Upstream{ upstream := &ingress.Upstream{
Name: defUpstreamName, Name: defUpstreamName,
} }
svcKey := lbc.defaultSvc svcKey := lbc.defaultSvc
@ -667,7 +668,7 @@ func (lbc *loadBalancerController) getDefaultUpstream() *nginx.Upstream {
return upstream return upstream
} }
func (lbc *loadBalancerController) getUpstreamServers(ngxCfg config.Configuration, data []interface{}) ([]*nginx.Upstream, []*nginx.Server) { func (lbc *loadBalancerController) getUpstreamServers(ngxCfg config.Configuration, data []interface{}) ([]*ingress.Upstream, []*ingress.Server) {
upstreams := lbc.createUpstreams(ngxCfg, data) upstreams := lbc.createUpstreams(ngxCfg, data)
servers := lbc.createServers(data) servers := lbc.createServers(data)
@ -754,7 +755,7 @@ func (lbc *loadBalancerController) getUpstreamServers(ngxCfg config.Configuratio
if addLoc { if addLoc {
server.Locations = append(server.Locations, &nginx.Location{ server.Locations = append(server.Locations, &ingress.Location{
Path: nginxPath, Path: nginxPath,
Upstream: *ups, Upstream: *ups,
Auth: *nginxAuth, Auth: *nginxAuth,
@ -771,31 +772,31 @@ func (lbc *loadBalancerController) getUpstreamServers(ngxCfg config.Configuratio
// TODO: find a way to make this more readable // TODO: find a way to make this more readable
// The structs must be ordered to always generate the same file // The structs must be ordered to always generate the same file
// if the content does not change. // if the content does not change.
aUpstreams := make([]*nginx.Upstream, 0, len(upstreams)) aUpstreams := make([]*ingress.Upstream, 0, len(upstreams))
for _, value := range upstreams { for _, value := range upstreams {
if len(value.Backends) == 0 { if len(value.Backends) == 0 {
glog.Warningf("upstream %v does not have any active endpoints. Using default backend", value.Name) glog.Warningf("upstream %v does not have any active endpoints. Using default backend", value.Name)
value.Backends = append(value.Backends, nginx.NewDefaultServer()) value.Backends = append(value.Backends, nginx.NewDefaultServer())
} }
sort.Sort(nginx.UpstreamServerByAddrPort(value.Backends)) sort.Sort(ingress.UpstreamServerByAddrPort(value.Backends))
aUpstreams = append(aUpstreams, value) aUpstreams = append(aUpstreams, value)
} }
sort.Sort(nginx.UpstreamByNameServers(aUpstreams)) sort.Sort(ingress.UpstreamByNameServers(aUpstreams))
aServers := make([]*nginx.Server, 0, len(servers)) aServers := make([]*ingress.Server, 0, len(servers))
for _, value := range servers { for _, value := range servers {
sort.Sort(nginx.LocationByPath(value.Locations)) sort.Sort(ingress.LocationByPath(value.Locations))
aServers = append(aServers, value) aServers = append(aServers, value)
} }
sort.Sort(nginx.ServerByName(aServers)) sort.Sort(ingress.ServerByName(aServers))
return aUpstreams, aServers return aUpstreams, aServers
} }
// createUpstreams creates the NGINX upstreams for each service referenced in // createUpstreams creates the NGINX upstreams for each service referenced in
// Ingress rules. The servers inside the upstream are endpoints. // Ingress rules. The servers inside the upstream are endpoints.
func (lbc *loadBalancerController) createUpstreams(ngxCfg config.Configuration, data []interface{}) map[string]*nginx.Upstream { func (lbc *loadBalancerController) createUpstreams(ngxCfg config.Configuration, data []interface{}) map[string]*ingress.Upstream {
upstreams := make(map[string]*nginx.Upstream) upstreams := make(map[string]*ingress.Upstream)
upstreams[defUpstreamName] = lbc.getDefaultUpstream() upstreams[defUpstreamName] = lbc.getDefaultUpstream()
for _, ingIf := range data { for _, ingIf := range data {
@ -852,12 +853,12 @@ func (lbc *loadBalancerController) createUpstreams(ngxCfg config.Configuration,
return upstreams return upstreams
} }
func (lbc *loadBalancerController) createServers(data []interface{}) map[string]*nginx.Server { func (lbc *loadBalancerController) createServers(data []interface{}) map[string]*ingress.Server {
servers := make(map[string]*nginx.Server) servers := make(map[string]*ingress.Server)
pems := lbc.getPemsFromIngress(data) pems := lbc.getPemsFromIngress(data)
var ngxCert nginx.SSLCert var ngxCert ingress.SSLCert
var err error var err error
if lbc.defSSLCertificate == "" { if lbc.defSSLCertificate == "" {
@ -868,13 +869,13 @@ func (lbc *loadBalancerController) createServers(data []interface{}) map[string]
ngxCert, err = lbc.getPemCertificate(lbc.defSSLCertificate) ngxCert, err = lbc.getPemCertificate(lbc.defSSLCertificate)
} }
locs := []*nginx.Location{} locs := []*ingress.Location{}
locs = append(locs, &nginx.Location{ locs = append(locs, &ingress.Location{
Path: rootLocation, Path: rootLocation,
IsDefBackend: true, IsDefBackend: true,
Upstream: *lbc.getDefaultUpstream(), Upstream: *lbc.getDefaultUpstream(),
}) })
servers[defServerName] = &nginx.Server{Name: defServerName, Locations: locs} servers[defServerName] = &ingress.Server{Name: defServerName, Locations: locs}
if err == nil { if err == nil {
pems[defServerName] = ngxCert pems[defServerName] = ngxCert
@ -896,13 +897,13 @@ func (lbc *loadBalancerController) createServers(data []interface{}) map[string]
} }
if _, ok := servers[host]; !ok { if _, ok := servers[host]; !ok {
locs := []*nginx.Location{} locs := []*ingress.Location{}
locs = append(locs, &nginx.Location{ locs = append(locs, &ingress.Location{
Path: rootLocation, Path: rootLocation,
IsDefBackend: true, IsDefBackend: true,
Upstream: *lbc.getDefaultUpstream(), Upstream: *lbc.getDefaultUpstream(),
}) })
servers[host] = &nginx.Server{Name: host, Locations: locs} servers[host] = &ingress.Server{Name: host, Locations: locs}
} }
if ngxCert, ok := pems[host]; ok { if ngxCert, ok := pems[host]; ok {
@ -918,8 +919,8 @@ func (lbc *loadBalancerController) createServers(data []interface{}) map[string]
return servers return servers
} }
func (lbc *loadBalancerController) getPemsFromIngress(data []interface{}) map[string]nginx.SSLCert { func (lbc *loadBalancerController) getPemsFromIngress(data []interface{}) map[string]ingress.SSLCert {
pems := make(map[string]nginx.SSLCert) pems := make(map[string]ingress.SSLCert)
for _, ingIf := range data { for _, ingIf := range data {
ing := ingIf.(*extensions.Ingress) ing := ingIf.(*extensions.Ingress)
@ -946,23 +947,23 @@ func (lbc *loadBalancerController) getPemsFromIngress(data []interface{}) map[st
return pems return pems
} }
func (lbc *loadBalancerController) getPemCertificate(secretName string) (nginx.SSLCert, error) { func (lbc *loadBalancerController) getPemCertificate(secretName string) (ingress.SSLCert, error) {
secretInterface, exists, err := lbc.secrLister.Store.GetByKey(secretName) secretInterface, exists, err := lbc.secrLister.Store.GetByKey(secretName)
if err != nil { if err != nil {
return nginx.SSLCert{}, fmt.Errorf("Error retriveing secret %v: %v", secretName, err) return ingress.SSLCert{}, fmt.Errorf("Error retriveing secret %v: %v", secretName, err)
} }
if !exists { if !exists {
return nginx.SSLCert{}, fmt.Errorf("Secret %v does not exists", secretName) return ingress.SSLCert{}, fmt.Errorf("Secret %v does not exists", secretName)
} }
secret := secretInterface.(*api.Secret) secret := secretInterface.(*api.Secret)
cert, ok := secret.Data[api.TLSCertKey] cert, ok := secret.Data[api.TLSCertKey]
if !ok { if !ok {
return nginx.SSLCert{}, fmt.Errorf("Secret %v has no private key", secretName) return ingress.SSLCert{}, fmt.Errorf("Secret %v has no private key", secretName)
} }
key, ok := secret.Data[api.TLSPrivateKeyKey] key, ok := secret.Data[api.TLSPrivateKeyKey]
if !ok { if !ok {
return nginx.SSLCert{}, fmt.Errorf("Secret %v has no cert", secretName) return ingress.SSLCert{}, fmt.Errorf("Secret %v has no cert", secretName)
} }
nsSecName := strings.Replace(secretName, "/", "-", -1) nsSecName := strings.Replace(secretName, "/", "-", -1)
@ -986,15 +987,15 @@ func (lbc *loadBalancerController) secrReferenced(namespace string, name string)
} }
// getEndpoints returns a list of <endpoint ip>:<port> for a given service/target port combination. // getEndpoints returns a list of <endpoint ip>:<port> for a given service/target port combination.
func (lbc *loadBalancerController) getEndpoints(s *api.Service, servicePort intstr.IntOrString, proto api.Protocol, hz *healthcheck.Upstream) []nginx.UpstreamServer { func (lbc *loadBalancerController) getEndpoints(s *api.Service, servicePort intstr.IntOrString, proto api.Protocol, hz *healthcheck.Upstream) []ingress.UpstreamServer {
glog.V(3).Infof("getting endpoints for service %v/%v and port %v", s.Namespace, s.Name, servicePort.String()) glog.V(3).Infof("getting endpoints for service %v/%v and port %v", s.Namespace, s.Name, servicePort.String())
ep, err := lbc.endpLister.GetServiceEndpoints(s) ep, err := lbc.endpLister.GetServiceEndpoints(s)
if err != nil { if err != nil {
glog.Warningf("unexpected error obtaining service endpoints: %v", err) glog.Warningf("unexpected error obtaining service endpoints: %v", err)
return []nginx.UpstreamServer{} return []ingress.UpstreamServer{}
} }
upsServers := []nginx.UpstreamServer{} upsServers := []ingress.UpstreamServer{}
for _, ss := range ep.Subsets { for _, ss := range ep.Subsets {
for _, epPort := range ss.Ports { for _, epPort := range ss.Ports {
@ -1044,7 +1045,7 @@ func (lbc *loadBalancerController) getEndpoints(s *api.Service, servicePort ints
} }
for _, epAddress := range ss.Addresses { for _, epAddress := range ss.Addresses {
ups := nginx.UpstreamServer{ ups := ingress.UpstreamServer{
Address: epAddress.IP, Address: epAddress.IP,
Port: fmt.Sprintf("%v", targetPort), Port: fmt.Sprintf("%v", targetPort),
MaxFails: hz.MaxFails, MaxFails: hz.MaxFails,

44
controllers/nginx/main.go
View file

@ -19,6 +19,7 @@ package main
import ( import (
"flag" "flag"
"fmt" "fmt"
"io/ioutil"
"net/http" "net/http"
"net/http/pprof" "net/http/pprof"
"os" "os"
@ -29,8 +30,6 @@ import (
"github.com/golang/glog" "github.com/golang/glog"
"github.com/spf13/pflag" "github.com/spf13/pflag"
"k8s.io/contrib/ingress/controllers/nginx/nginx"
"k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/client/unversioned" "k8s.io/kubernetes/pkg/client/unversioned"
"k8s.io/kubernetes/pkg/healthz" "k8s.io/kubernetes/pkg/healthz"
@ -77,9 +76,6 @@ var (
healthzPort = flags.Int("healthz-port", healthPort, "port for healthz endpoint.") healthzPort = flags.Int("healthz-port", healthPort, "port for healthz endpoint.")
buildCfg = flags.Bool("dump-nginx-configuration", false, `Returns a ConfigMap with the default nginx conguration.
This can be used as a guide to create a custom configuration.`)
profiling = flags.Bool("profiling", true, `Enable profiling via web interface host:port/debug/pprof/`) profiling = flags.Bool("profiling", true, `Enable profiling via web interface host:port/debug/pprof/`)
defSSLCertificate = flags.String("default-ssl-certificate", "", `Name of the secret that contains a SSL defSSLCertificate = flags.String("default-ssl-certificate", "", `Name of the secret that contains a SSL
@ -93,11 +89,6 @@ func main() {
glog.Infof("Using build: %v - %v", gitRepo, version) glog.Infof("Using build: %v - %v", gitRepo, version)
if *buildCfg {
fmt.Printf("Example of ConfigMap to customize NGINX configuration:\n%v", nginx.ConfigMapAsString())
os.Exit(0)
}
if *defaultSvc == "" { if *defaultSvc == "" {
glog.Fatalf("Please specify --default-backend-service") glog.Fatalf("Please specify --default-backend-service")
} }
@ -123,12 +114,14 @@ func main() {
glog.Infof("Validated %v as the default backend", *defaultSvc) glog.Infof("Validated %v as the default backend", *defaultSvc)
if *nxgConfigMap != "" { if *nxgConfigMap != "" {
_, _, err := parseNsName(*nxgConfigMap) _, _, err = parseNsName(*nxgConfigMap)
if err != nil { if err != nil {
glog.Fatalf("configmap error: %v", err) glog.Fatalf("configmap error: %v", err)
} }
} }
checkTemplate()
lbc, err := newLoadBalancerController(kubeClient, *resyncPeriod, lbc, err := newLoadBalancerController(kubeClient, *resyncPeriod,
*defaultSvc, *watchNamespace, *nxgConfigMap, *tcpConfigMapName, *defaultSvc, *watchNamespace, *nxgConfigMap, *tcpConfigMapName,
*udpConfigMapName, *defSSLCertificate, runtimePodInfo) *udpConfigMapName, *defSSLCertificate, runtimePodInfo)
@ -158,12 +151,12 @@ func registerHandlers(lbc *loadBalancerController) {
mux := http.NewServeMux() mux := http.NewServeMux()
healthz.InstallHandler(mux, lbc.nginx) healthz.InstallHandler(mux, lbc.nginx)
http.HandleFunc("/build", func(w http.ResponseWriter, r *http.Request) { mux.HandleFunc("/build", func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK) w.WriteHeader(http.StatusOK)
fmt.Fprintf(w, "build: %v - %v", gitRepo, version) fmt.Fprintf(w, "build: %v - %v", gitRepo, version)
}) })
http.HandleFunc("/stop", func(w http.ResponseWriter, r *http.Request) { mux.HandleFunc("/stop", func(w http.ResponseWriter, r *http.Request) {
lbc.Stop() lbc.Stop()
}) })
@ -195,3 +188,28 @@ func handleSigterm(lbc *loadBalancerController) {
glog.Infof("Exiting with %v", exitCode) glog.Infof("Exiting with %v", exitCode)
os.Exit(exitCode) os.Exit(exitCode)
} }
const (
defTmpl = "/etc/nginx/template/nginx.tmpl"
fallbackTmpl = "/etc/nginx/nginx.tmpl"
)
// checkTemplate verifies the NGINX template exists (file /etc/nginx/template/nginx.tmpl)
// If the file does not exists it means:
// a. custom docker image
// b. standard image using watch-resource sidecar with emptyDir volume
// If the file /etc/nginx/nginx.tmpl exists copy the file to /etc/nginx/template/nginx.tmpl
// or terminate the execution (It is not possible to start NGINX without a template)
func checkTemplate() {
_, err := os.Stat(defTmpl)
if err != nil {
glog.Warningf("error checking template %v: %v", defTmpl, err)
data, err := ioutil.ReadFile(fallbackTmpl)
if err != nil {
glog.Fatalf("error reading template %v: %v", fallbackTmpl, err)
}
if err = ioutil.WriteFile(defTmpl, data, 0644); err != nil {
glog.Fatalf("error copying %v to %v: %v", fallbackTmpl, defTmpl, err)
}
}
}

13
controllers/nginx/nginx/command.go
View file

@ -27,6 +27,7 @@ import (
"k8s.io/kubernetes/pkg/healthz" "k8s.io/kubernetes/pkg/healthz"
"k8s.io/contrib/ingress/controllers/nginx/nginx/config" "k8s.io/contrib/ingress/controllers/nginx/nginx/config"
"k8s.io/contrib/ingress/controllers/nginx/nginx/ingress"
) )
// Start starts a nginx (master process) and waits. If the process ends // Start starts a nginx (master process) and waits. If the process ends
@ -56,19 +57,23 @@ func (ngx *Manager) Start() {
// shut down, stop accepting new connections and continue to service current requests // shut down, stop accepting new connections and continue to service current requests
// until all such requests are serviced. After that, the old worker processes exit. // until all such requests are serviced. After that, the old worker processes exit.
// http://nginx.org/en/docs/beginners_guide.html#control // http://nginx.org/en/docs/beginners_guide.html#control
func (ngx *Manager) CheckAndReload(cfg config.Configuration, ingressCfg IngressConfig) error { func (ngx *Manager) CheckAndReload(cfg config.Configuration, ingressCfg ingress.Configuration) error {
ngx.reloadRateLimiter.Accept() ngx.reloadRateLimiter.Accept()
ngx.reloadLock.Lock() ngx.reloadLock.Lock()
defer ngx.reloadLock.Unlock() defer ngx.reloadLock.Unlock()
newCfg, err := ngx.writeCfg(cfg, ingressCfg) newCfg, err := ngx.template.Write(cfg, ingressCfg)
if err != nil { if err != nil {
return fmt.Errorf("failed to write new nginx configuration. Avoiding reload: %v", err) return fmt.Errorf("failed to write new nginx configuration. Avoiding reload: %v", err)
} }
if newCfg { changed, err := ngx.needsReload(newCfg)
if err != nil {
return err
}
if changed {
if err := ngx.shellOut("nginx -s reload"); err != nil { if err := ngx.shellOut("nginx -s reload"); err != nil {
return fmt.Errorf("error reloading nginx: %v", err) return fmt.Errorf("error reloading nginx: %v", err)
} }

4
controllers/nginx/nginx/healthcheck/main_test.go
View file

@ -84,7 +84,7 @@ func TestAnnotations(t *testing.T) {
t.Errorf("Unexpected error: %v", err) t.Errorf("Unexpected error: %v", err)
} }
if mf != 1 { if mf != 1 {
t.Errorf("Expected 1 but returned %s", mf) t.Errorf("Expected 1 but returned %v", mf)
} }
ft, err := ingAnnotations(ing.GetAnnotations()).failTimeout() ft, err := ingAnnotations(ing.GetAnnotations()).failTimeout()
@ -92,7 +92,7 @@ func TestAnnotations(t *testing.T) {
t.Errorf("Unexpected error: %v", err) t.Errorf("Unexpected error: %v", err)
} }
if ft != 1 { if ft != 1 {
t.Errorf("Expected 1 but returned %s", ft) t.Errorf("Expected 1 but returned %v", ft)
} }
} }

28
controllers/nginx/nginx/nginx.go → controllers/nginx/nginx/ingress/nginx.go
View file

@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License. limitations under the License.
*/ */
package nginx package ingress
import ( import (
"k8s.io/contrib/ingress/controllers/nginx/nginx/auth" "k8s.io/contrib/ingress/controllers/nginx/nginx/auth"
@ -23,8 +23,8 @@ import (
"k8s.io/contrib/ingress/controllers/nginx/nginx/rewrite" "k8s.io/contrib/ingress/controllers/nginx/nginx/rewrite"
) )
// IngressConfig describes an NGINX configuration // Configuration describes an NGINX configuration
type IngressConfig struct { type Configuration struct {
Upstreams []*Upstream Upstreams []*Upstream
Servers []*Server Servers []*Server
TCPUpstreams []*Location TCPUpstreams []*Location
@ -113,15 +113,15 @@ func (c LocationByPath) Less(i, j int) bool {
return c[i].Path > c[j].Path return c[i].Path > c[j].Path
} }
// NewDefaultServer return an UpstreamServer to be use as default server that returns 503. // SSLCert describes a SSL certificate to be used in NGINX
func NewDefaultServer() UpstreamServer { type SSLCert struct {
return UpstreamServer{Address: "127.0.0.1", Port: "8181"} CertFileName string
} KeyFileName string
// PemFileName contains the path to the file with the certificate and key concatenated
// NewUpstream creates an upstream without servers. PemFileName string
func NewUpstream(name string) *Upstream { // PemSHA contains the sha1 of the pem file.
return &Upstream{ // This is used to detect changes in the secret that contains the certificates
Name: name, PemSHA string
Backends: []UpstreamServer{}, // CN contains all the common names defined in the SSL certificate
} CN []string
} }

67
controllers/nginx/nginx/main.go
View file

@ -17,22 +17,22 @@ limitations under the License.
package nginx package nginx
import ( import (
"fmt"
"os" "os"
"strings" "strings"
"sync" "sync"
"text/template"
"github.com/golang/glog" "github.com/golang/glog"
"github.com/fatih/structs"
"github.com/ghodss/yaml"
"k8s.io/kubernetes/pkg/api"
client "k8s.io/kubernetes/pkg/client/unversioned" client "k8s.io/kubernetes/pkg/client/unversioned"
"k8s.io/kubernetes/pkg/util/flowcontrol" "k8s.io/kubernetes/pkg/util/flowcontrol"
"k8s.io/contrib/ingress/controllers/nginx/nginx/config" "k8s.io/contrib/ingress/controllers/nginx/nginx/config"
"k8s.io/contrib/ingress/controllers/nginx/nginx/ingress"
ngx_template "k8s.io/contrib/ingress/controllers/nginx/nginx/template"
)
var (
tmplPath = "/etc/nginx/template/nginx.tmpl"
) )
// Manager ... // Manager ...
@ -48,11 +48,24 @@ type Manager struct {
reloadRateLimiter flowcontrol.RateLimiter reloadRateLimiter flowcontrol.RateLimiter
// template loaded ready to be used to generate the nginx configuration file // template loaded ready to be used to generate the nginx configuration file
template *template.Template template *ngx_template.Template
reloadLock *sync.Mutex reloadLock *sync.Mutex
} }
// NewDefaultServer return an UpstreamServer to be use as default server that returns 503.
func NewDefaultServer() ingress.UpstreamServer {
return ingress.UpstreamServer{Address: "127.0.0.1", Port: "8181"}
}
// NewUpstream creates an upstream without servers.
func NewUpstream(name string) *ingress.Upstream {
return &ingress.Upstream{
Name: name,
Backends: []ingress.UpstreamServer{},
}
}
// NewManager ... // NewManager ...
func NewManager(kubeClient *client.Client) *Manager { func NewManager(kubeClient *client.Client) *Manager {
ngx := &Manager{ ngx := &Manager{
@ -67,10 +80,26 @@ func NewManager(kubeClient *client.Client) *Manager {
ngx.sslDHParam = ngx.SearchDHParamFile(config.SSLDirectory) ngx.sslDHParam = ngx.SearchDHParamFile(config.SSLDirectory)
if err := ngx.loadTemplate(); err != nil { var onChange func()
onChange = func() {
template, err := ngx_template.NewTemplate(tmplPath, onChange)
if err != nil {
glog.Warningf("invalid NGINX template: %v", err)
return
}
ngx.template.Close()
ngx.template = template
glog.Info("new NGINX template loaded")
}
template, err := ngx_template.NewTemplate(tmplPath, onChange)
if err != nil {
glog.Fatalf("invalid NGINX template: %v", err) glog.Fatalf("invalid NGINX template: %v", err)
} }
ngx.template = template
return ngx return ngx
} }
@ -83,25 +112,3 @@ func (nginx *Manager) createCertsDir(base string) {
glog.Fatalf("Couldn't create directory %v: %v", base, err) glog.Fatalf("Couldn't create directory %v: %v", base, err)
} }
} }
// ConfigMapAsString returns a ConfigMap with the default NGINX
// configuration to be used a guide to provide a custom configuration
func ConfigMapAsString() string {
cfg := &api.ConfigMap{}
cfg.Name = "custom-name"
cfg.Namespace = "a-valid-namespace"
cfg.Data = make(map[string]string)
data := structs.Map(config.NewDefault())
for k, v := range data {
cfg.Data[k] = fmt.Sprintf("%v", v)
}
out, err := yaml.Marshal(cfg)
if err != nil {
glog.Warningf("Unexpected error creating default configuration: %v", err)
return ""
}
return string(out)
}

30
controllers/nginx/nginx/ssl.go
View file

@ -28,54 +28,42 @@ import (
"github.com/golang/glog" "github.com/golang/glog"
"k8s.io/contrib/ingress/controllers/nginx/nginx/config" "k8s.io/contrib/ingress/controllers/nginx/nginx/config"
"k8s.io/contrib/ingress/controllers/nginx/nginx/ingress"
) )
// SSLCert describes a SSL certificate to be used in NGINX
type SSLCert struct {
CertFileName string
KeyFileName string
// PemFileName contains the path to the file with the certificate and key concatenated
PemFileName string
// PemSHA contains the sha1 of the pem file.
// This is used to detect changes in the secret that contains the certificates
PemSHA string
// CN contains all the common names defined in the SSL certificate
CN []string
}
// AddOrUpdateCertAndKey creates a .pem file wth the cert and the key with the specified name // AddOrUpdateCertAndKey creates a .pem file wth the cert and the key with the specified name
func (nginx *Manager) AddOrUpdateCertAndKey(name string, cert string, key string) (SSLCert, error) { func (nginx *Manager) AddOrUpdateCertAndKey(name string, cert string, key string) (ingress.SSLCert, error) {
temporaryPemFileName := fmt.Sprintf("%v.pem", name) temporaryPemFileName := fmt.Sprintf("%v.pem", name)
pemFileName := fmt.Sprintf("%v/%v.pem", config.SSLDirectory, name) pemFileName := fmt.Sprintf("%v/%v.pem", config.SSLDirectory, name)
temporaryPemFile, err := ioutil.TempFile("", temporaryPemFileName) temporaryPemFile, err := ioutil.TempFile("", temporaryPemFileName)
if err != nil { if err != nil {
return SSLCert{}, fmt.Errorf("Couldn't create temp pem file %v: %v", temporaryPemFile.Name(), err) return ingress.SSLCert{}, fmt.Errorf("Couldn't create temp pem file %v: %v", temporaryPemFile.Name(), err)
} }
_, err = temporaryPemFile.WriteString(fmt.Sprintf("%v\n%v", cert, key)) _, err = temporaryPemFile.WriteString(fmt.Sprintf("%v\n%v", cert, key))
if err != nil { if err != nil {
return SSLCert{}, fmt.Errorf("Couldn't write to pem file %v: %v", temporaryPemFile.Name(), err) return ingress.SSLCert{}, fmt.Errorf("Couldn't write to pem file %v: %v", temporaryPemFile.Name(), err)
} }
err = temporaryPemFile.Close() err = temporaryPemFile.Close()
if err != nil { if err != nil {
return SSLCert{}, fmt.Errorf("Couldn't close temp pem file %v: %v", temporaryPemFile.Name(), err) return ingress.SSLCert{}, fmt.Errorf("Couldn't close temp pem file %v: %v", temporaryPemFile.Name(), err)
} }
cn, err := nginx.commonNames(temporaryPemFile.Name()) cn, err := nginx.commonNames(temporaryPemFile.Name())
if err != nil { if err != nil {
os.Remove(temporaryPemFile.Name()) os.Remove(temporaryPemFile.Name())
return SSLCert{}, err return ingress.SSLCert{}, err
} }
err = os.Rename(temporaryPemFile.Name(), pemFileName) err = os.Rename(temporaryPemFile.Name(), pemFileName)
if err != nil { if err != nil {
os.Remove(temporaryPemFile.Name()) os.Remove(temporaryPemFile.Name())
return SSLCert{}, fmt.Errorf("Couldn't move temp pem file %v to destination %v: %v", temporaryPemFile.Name(), pemFileName, err) return ingress.SSLCert{}, fmt.Errorf("Couldn't move temp pem file %v to destination %v: %v", temporaryPemFile.Name(), pemFileName, err)
} }
return SSLCert{ return ingress.SSLCert{
CertFileName: cert, CertFileName: cert,
KeyFileName: key, KeyFileName: key,
PemFileName: pemFileName, PemFileName: pemFileName,
@ -133,6 +121,8 @@ func (nginx *Manager) SearchDHParamFile(baseDir string) string {
return "" return ""
} }
// pemSHA1 returns the SHA1 of a pem file. This is used to
// reload NGINX in case a secret with a SSL certificate changed.
func (nginx *Manager) pemSHA1(filename string) string { func (nginx *Manager) pemSHA1(filename string) string {
hasher := sha1.New() hasher := sha1.New()
s, err := ioutil.ReadFile(filename) s, err := ioutil.ReadFile(filename)

72
controllers/nginx/nginx/template/file_watcher.go Normal file
View file

@ -0,0 +1,72 @@
/*
Copyright 2016 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package template
import (
"log"
"path"
"gopkg.in/fsnotify.v1"
)
type fileWatcher struct {
file string
watcher *fsnotify.Watcher
onEvent func()
}
func newFileWatcher(file string, onEvent func()) (fileWatcher, error) {
fw := fileWatcher{
file: file,
onEvent: onEvent,
}
err := fw.watch()
return fw, err
}
func (f fileWatcher) close() error {
return f.watcher.Close()
}
// watch creates a fsnotify watcher for a file and create of write events
func (f *fileWatcher) watch() error {
watcher, err := fsnotify.NewWatcher()
if err != nil {
return err
}
f.watcher = watcher
dir, file := path.Split(f.file)
go func(file string) {
for {
select {
case event := <-watcher.Events:
if event.Op&fsnotify.Write == fsnotify.Write ||
event.Op&fsnotify.Create == fsnotify.Create &&
event.Name == file {
f.onEvent()
}
case err := <-watcher.Errors:
if err != nil {
log.Printf("error watching file: %v\n", err)
}
}
}
}(file)
return watcher.Add(dir)
}

84
controllers/nginx/nginx/template.go → controllers/nginx/nginx/template/template.go
View file

@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License. limitations under the License.
*/ */
package nginx package template
import ( import (
"bytes" "bytes"
@ -22,12 +22,14 @@ import (
"fmt" "fmt"
"regexp" "regexp"
"strings" "strings"
"text/template" text_template "text/template"
"github.com/fatih/structs" "github.com/fatih/structs"
"github.com/golang/glog" "github.com/golang/glog"
"k8s.io/contrib/ingress/controllers/nginx/nginx/config" "k8s.io/contrib/ingress/controllers/nginx/nginx/config"
"k8s.io/contrib/ingress/controllers/nginx/nginx/ingress"
"k8s.io/kubernetes/pkg/util/sysctl"
) )
const ( const (
@ -36,9 +38,8 @@ const (
var ( var (
camelRegexp = regexp.MustCompile("[0-9A-Za-z]+") camelRegexp = regexp.MustCompile("[0-9A-Za-z]+")
tmplPath = "/etc/nginx/template/nginx.tmpl"
funcMap = template.FuncMap{ funcMap = text_template.FuncMap{
"empty": func(input interface{}) bool { "empty": func(input interface{}) bool {
check, ok := input.(string) check, ok := input.(string)
if ok { if ok {
@ -54,48 +55,64 @@ var (
} }
) )
func (ngx *Manager) loadTemplate() error { // Template ...
tmpl, err := template.New("nginx.tmpl").Funcs(funcMap).ParseFiles(tmplPath) type Template struct {
if err != nil { tmpl *text_template.Template
return err fw fileWatcher
}
ngx.template = tmpl
return nil
} }
func (ngx *Manager) writeCfg(cfg config.Configuration, ingressCfg IngressConfig) (bool, error) { //NewTemplate returns a new Template instance or an
//error if the specified template file contains errors
func NewTemplate(file string, onChange func()) (*Template, error) {
tmpl, err := text_template.New("nginx.tmpl").Funcs(funcMap).ParseFiles(file)
if err != nil {
return nil, err
}
fw, err := newFileWatcher(file, onChange)
if err != nil {
return nil, err
}
return &Template{
tmpl: tmpl,
fw: fw,
}, nil
}
// Close removes the file watcher
func (t *Template) Close() {
t.fw.close()
}
// Write populates a buffer using a template with NGINX configuration
// and the servers and upstreams created by Ingress rules
func (t *Template) Write(cfg config.Configuration, ingressCfg ingress.Configuration) ([]byte, error) {
conf := make(map[string]interface{}) conf := make(map[string]interface{})
conf["backlogSize"] = sysctlSomaxconn() conf["backlogSize"] = sysctlSomaxconn()
conf["upstreams"] = ingressCfg.Upstreams conf["upstreams"] = ingressCfg.Upstreams
conf["servers"] = ingressCfg.Servers conf["servers"] = ingressCfg.Servers
conf["tcpUpstreams"] = ingressCfg.TCPUpstreams conf["tcpUpstreams"] = ingressCfg.TCPUpstreams
conf["udpUpstreams"] = ingressCfg.UDPUpstreams conf["udpUpstreams"] = ingressCfg.UDPUpstreams
conf["defResolver"] = ngx.defResolver conf["defResolver"] = cfg.Resolver
conf["sslDHParam"] = ngx.sslDHParam conf["sslDHParam"] = cfg.SSLDHParam
conf["customErrors"] = len(cfg.CustomHTTPErrors) > 0 conf["customErrors"] = len(cfg.CustomHTTPErrors) > 0
conf["cfg"] = fixKeyNames(structs.Map(cfg)) conf["cfg"] = fixKeyNames(structs.Map(cfg))
if glog.V(3) { if glog.V(3) {
b, err := json.Marshal(conf) b, err := json.Marshal(conf)
if err != nil { if err != nil {
glog.Errorf("unexpected error:", err) glog.Errorf("unexpected error: %v", err)
} }
glog.Infof("NGINX configuration: %v", string(b)) glog.Infof("NGINX configuration: %v", string(b))
} }
buffer := new(bytes.Buffer) buffer := new(bytes.Buffer)
err := ngx.template.Execute(buffer, conf) err := t.tmpl.Execute(buffer, conf)
if err != nil { if err != nil {
glog.V(3).Infof("%v", string(buffer.Bytes())) glog.V(3).Infof("%v", string(buffer.Bytes()))
return false, err
} }
changed, err := ngx.needsReload(buffer) return buffer.Bytes(), err
if err != nil {
return false, err
}
return changed, nil
} }
func fixKeyNames(data map[string]interface{}) map[string]interface{} { func fixKeyNames(data map[string]interface{}) map[string]interface{} {
@ -121,7 +138,7 @@ func toCamelCase(src string) string {
// buildLocation produces the location string, if the ingress has redirects // buildLocation produces the location string, if the ingress has redirects
// (specified through the ingress.kubernetes.io/rewrite-to annotation) // (specified through the ingress.kubernetes.io/rewrite-to annotation)
func buildLocation(input interface{}) string { func buildLocation(input interface{}) string {
location, ok := input.(*Location) location, ok := input.(*ingress.Location)
if !ok { if !ok {
return slash return slash
} }
@ -139,7 +156,7 @@ func buildLocation(input interface{}) string {
// If the annotation ingress.kubernetes.io/add-base-url:"true" is specified it will // If the annotation ingress.kubernetes.io/add-base-url:"true" is specified it will
// add a base tag in the head of the response from the service // add a base tag in the head of the response from the service
func buildProxyPass(input interface{}) string { func buildProxyPass(input interface{}) string {
location, ok := input.(*Location) location, ok := input.(*ingress.Location)
if !ok { if !ok {
return "" return ""
} }
@ -200,7 +217,7 @@ func buildProxyPass(input interface{}) string {
func buildRateLimitZones(input interface{}) []string { func buildRateLimitZones(input interface{}) []string {
zones := []string{} zones := []string{}
servers, ok := input.([]*Server) servers, ok := input.([]*ingress.Server)
if !ok { if !ok {
return zones return zones
} }
@ -230,7 +247,7 @@ func buildRateLimitZones(input interface{}) []string {
func buildRateLimit(input interface{}) []string { func buildRateLimit(input interface{}) []string {
limits := []string{} limits := []string{}
loc, ok := input.(*Location) loc, ok := input.(*ingress.Location)
if !ok { if !ok {
return limits return limits
} }
@ -249,3 +266,16 @@ func buildRateLimit(input interface{}) []string {
return limits return limits
} }
// sysctlSomaxconn returns the value of net.core.somaxconn, i.e.
// maximum number of connections that can be queued for acceptance
// http://nginx.org/en/docs/http/ngx_http_core_module.html#listen
func sysctlSomaxconn() int {
maxConns, err := sysctl.GetSysctl("net/core/somaxconn")
if err != nil || maxConns < 512 {
glog.Warningf("system net.core.somaxconn=%v. Using NGINX default (511)", maxConns)
return 511
}
return maxConns
}

9
controllers/nginx/nginx/template_test.go → controllers/nginx/nginx/template/template_test.go
View file

@ -14,12 +14,13 @@ See the License for the specific language governing permissions and
limitations under the License. limitations under the License.
*/ */
package nginx package template
import ( import (
"strings" "strings"
"testing" "testing"
"k8s.io/contrib/ingress/controllers/nginx/nginx/ingress"
"k8s.io/contrib/ingress/controllers/nginx/nginx/rewrite" "k8s.io/contrib/ingress/controllers/nginx/nginx/rewrite"
) )
@ -70,7 +71,7 @@ var (
func TestBuildLocation(t *testing.T) { func TestBuildLocation(t *testing.T) {
for k, tc := range tmplFuncTestcases { for k, tc := range tmplFuncTestcases {
loc := &Location{ loc := &ingress.Location{
Path: tc.Path, Path: tc.Path,
Redirect: rewrite.Redirect{Target: tc.Target, AddBaseURL: tc.AddBaseURL}, Redirect: rewrite.Redirect{Target: tc.Target, AddBaseURL: tc.AddBaseURL},
} }
@ -84,10 +85,10 @@ func TestBuildLocation(t *testing.T) {
func TestBuildProxyPass(t *testing.T) { func TestBuildProxyPass(t *testing.T) {
for k, tc := range tmplFuncTestcases { for k, tc := range tmplFuncTestcases {
loc := &Location{ loc := &ingress.Location{
Path: tc.Path, Path: tc.Path,
Redirect: rewrite.Redirect{Target: tc.Target, AddBaseURL: tc.AddBaseURL}, Redirect: rewrite.Redirect{Target: tc.Target, AddBaseURL: tc.AddBaseURL},
Upstream: Upstream{Name: "upstream-name"}, Upstream: ingress.Upstream{Name: "upstream-name"},
} }
pp := buildProxyPass(loc) pp := buildProxyPass(loc)

23
controllers/nginx/nginx/utils.go
View file

@ -28,7 +28,6 @@ import (
"github.com/golang/glog" "github.com/golang/glog"
"github.com/mitchellh/mapstructure" "github.com/mitchellh/mapstructure"
"k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/util/sysctl"
"k8s.io/contrib/ingress/controllers/nginx/nginx/config" "k8s.io/contrib/ingress/controllers/nginx/nginx/config"
) )
@ -160,7 +159,7 @@ func (ngx *Manager) filterErrors(errCodes []int) []int {
return fa return fa
} }
func (ngx *Manager) needsReload(data *bytes.Buffer) (bool, error) { func (ngx *Manager) needsReload(data []byte) (bool, error) {
filename := ngx.ConfigFile filename := ngx.ConfigFile
in, err := os.Open(filename) in, err := os.Open(filename)
if err != nil { if err != nil {
@ -173,14 +172,13 @@ func (ngx *Manager) needsReload(data *bytes.Buffer) (bool, error) {
return false, err return false, err
} }
res := data.Bytes() if !bytes.Equal(src, data) {
if !bytes.Equal(src, res) { err = ioutil.WriteFile(filename, data, 0644)
err = ioutil.WriteFile(filename, res, 0644)
if err != nil { if err != nil {
return false, err return false, err
} }
dData, err := diff(src, res) dData, err := diff(src, data)
if err != nil { if err != nil {
glog.Errorf("error computing diff: %s", err) glog.Errorf("error computing diff: %s", err)
return true, nil return true, nil
@ -221,16 +219,3 @@ func diff(b1, b2 []byte) (data []byte, err error) {
} }
return return
} }
// sysctlSomaxconn returns the value of net.core.somaxconn, i.e.
// maximum number of connections that can be queued for acceptance
// http://nginx.org/en/docs/http/ngx_http_core_module.html#listen
func sysctlSomaxconn() int {
maxConns, err := sysctl.GetSysctl("net/core/somaxconn")
if err != nil || maxConns < 512 {
glog.V(3).Infof("system net.core.somaxconn=%v. Using NGINX default (511)", maxConns)
return 511
}
return maxConns
}