Update opentracing configuration (#2676)

Makefile

@@ -59,7 +59,7 @@ IMAGE = $(REGISTRY)/$(IMGNAME)
 MULTI_ARCH_IMG = $(IMAGE)-$(ARCH)
 
 # Set default base image dynamically for each arch
-BASEIMAGE?=quay.io/kubernetes-ingress-controller/nginx-$(ARCH):0.52
+BASEIMAGE?=quay.io/kubernetes-ingress-controller/nginx-$(ARCH):0.53
 
 ifeq ($(ARCH),arm)
 	QEMUARCH=arm

internal/ingress/controller/nginx.go

@@ -31,6 +31,7 @@ import (
 	"strings"
 	"sync"
 	"syscall"
+	"text/template"
 	"time"
 
 	"github.com/golang/glog"
@@ -593,11 +594,17 @@ func (n *NGINXController) OnUpdate(ingressCfg ingress.Configuration) error {
 	}
 
 	content, err := n.t.Write(tc)
-
 	if err != nil {
 		return err
 	}
 
+	if cfg.EnableOpentracing {
+		err := createOpentracingCfg(cfg)
+		if err != nil {
+			return err
+		}
+	}
+
 	err = n.testTemplate(content)
 	if err != nil {
 		return err
@@ -779,3 +786,47 @@ func configureDynamically(pcfg *ingress.Configuration, port int) error {
 
 	return nil
 }
+
+const zipkinTmpl = `{
+  "service_name": "{{ .ZipkinServiceName }}",
+  "collector_host": "{{ .ZipkinCollectorHost }}",
+  "collector_port": {{ .ZipkinCollectorPort }}
+}`
+
+const jaegerTmpl = `{
+  "service_name": "{{ .JaegerServiceName }}",
+  "sampler": {
+	"type": "{{ .JaegerSamplerType }}",
+	"param": {{ .JaegerSamplerParam }}
+  },
+  "reporter": {
+	"localAgentHostPort": "{{ .JaegerCollectorHost }}:{{ .JaegerCollectorPort }}"
+  }
+}`
+
+func createOpentracingCfg(cfg ngx_config.Configuration) error {
+	var tmpl *template.Template
+	var err error
+
+	if cfg.ZipkinCollectorHost != "" {
+		tmpl, err = template.New("zipkin").Parse(zipkinTmpl)
+		if err != nil {
+			return err
+		}
+	} else if cfg.JaegerCollectorHost != "" {
+		tmpl, err = template.New("jarger").Parse(jaegerTmpl)
+		if err != nil {
+			return err
+		}
+	} else {
+		tmpl, _ = template.New("empty").Parse("{}")
+	}
+
+	tmplBuf := bytes.NewBuffer(make([]byte, 0))
+	err = tmpl.Execute(tmplBuf, cfg)
+	if err != nil {
+		return err
+	}
+
+	return ioutil.WriteFile("/etc/nginx/opentracing.json", tmplBuf.Bytes(), file.ReadWriteByUser)
+}

internal/ingress/controller/template/template.go

@@ -151,7 +151,6 @@ var (
 		"isValidClientBodyBufferSize": isValidClientBodyBufferSize,
 		"buildForwardedFor":           buildForwardedFor,
 		"buildAuthSignURL":            buildAuthSignURL,
-		"buildOpentracingLoad":        buildOpentracingLoad,
 		"buildOpentracing":            buildOpentracing,
 		"proxySetHeader":              proxySetHeader,
 		"buildInfluxDB":               buildInfluxDB,
@@ -841,31 +840,6 @@ func randomString() string {
 	return string(b)
 }
 
-func buildOpentracingLoad(input interface{}) string {
-	cfg, ok := input.(config.Configuration)
-	if !ok {
-		glog.Errorf("expected a 'config.Configuration' type but %T was returned", input)
-		return ""
-	}
-
-	if !cfg.EnableOpentracing {
-		return ""
-	}
-
-	buf := bytes.NewBufferString("load_module /etc/nginx/modules/ngx_http_opentracing_module.so;")
-	buf.WriteString("\r\n")
-
-	if cfg.ZipkinCollectorHost != "" {
-		buf.WriteString("load_module /etc/nginx/modules/ngx_http_zipkin_module.so;")
-	} else if cfg.JaegerCollectorHost != "" {
-		buf.WriteString("load_module /etc/nginx/modules/ngx_http_jaeger_module.so;")
-	}
-
-	buf.WriteString("\r\n")
-
-	return buf.String()
-}
-
 func buildOpentracing(input interface{}) string {
 	cfg, ok := input.(config.Configuration)
 	if !ok {
@@ -878,24 +852,14 @@ func buildOpentracing(input interface{}) string {
 	}
 
 	buf := bytes.NewBufferString("")
-
 	if cfg.ZipkinCollectorHost != "" {
-		buf.WriteString(fmt.Sprintf("zipkin_collector_host                   %v;", cfg.ZipkinCollectorHost))
+		buf.WriteString("opentracing_load_tracer /usr/local/lib/libzipkin_opentracing.so /etc/nginx/opentracing.json;")
-		buf.WriteString("\r\n")
-		buf.WriteString(fmt.Sprintf("zipkin_collector_port                   %v;", cfg.ZipkinCollectorPort))
-		buf.WriteString("\r\n")
-		buf.WriteString(fmt.Sprintf("zipkin_service_name                     %v;", cfg.ZipkinServiceName))
 	} else if cfg.JaegerCollectorHost != "" {
-		buf.WriteString(fmt.Sprintf("jaeger_reporter_local_agent_host_port   %v:%v;", cfg.JaegerCollectorHost, cfg.JaegerCollectorPort))
+		buf.WriteString("opentracing_load_tracer /usr/local/lib/libjaegertracing.so 	 /etc/nginx/opentracing.json;")
-		buf.WriteString("\r\n")
-		buf.WriteString(fmt.Sprintf("jaeger_service_name                     %v;", cfg.JaegerServiceName))
-		buf.WriteString("\r\n")
-		buf.WriteString(fmt.Sprintf("jaeger_sampler_type                     %v;", cfg.JaegerSamplerType))
-		buf.WriteString("\r\n")
-		buf.WriteString(fmt.Sprintf("jaeger_sampler_param                    %v;", cfg.JaegerSamplerParam))
 	}
 
 	buf.WriteString("\r\n")
+
 	return buf.String()
 }
 

rootfs/Dockerfile

@@ -20,35 +20,35 @@ WORKDIR  /etc/nginx
 
 RUN clean-install \
   diffutils \
-  dumb-init \
+  libcap2-bin \
-  libcap2-bin 
+  dumb-init
 
 COPY . /
 
-RUN setcap cap_net_bind_service=+ep /usr/sbin/nginx \
+RUN setcap cap_net_bind_service=+ep /nginx-ingress-controller
-  && setcap cap_net_bind_service=+ep /nginx-ingress-controller
 
-RUN bash -eux -c ' \
+# Create symlinks to redirect nginx logs to stdout and stderr docker log collector
+# This only works if nginx is started with CMD or ENTRYPOINT
+# Required because clean-install removes /var/log content
+# We cannot chown /etc/nginx recursively because that adds 100MB to the image
+RUN  mkdir -p /var/log/nginx \
+  && ln -sf /dev/stdout /var/log/nginx/access.log \
+  && ln -sf /dev/stderr /var/log/nginx/error.log \
+  && bash -eux -c ' \
     writeDirs=( \
-        /etc/nginx \
+      /etc/nginx/template \
       /etc/ingress-controller/ssl \
       /etc/ingress-controller/auth \
       /var/log \
       /var/log/nginx \
-        /opt/modsecurity/var/log \
-        /opt/modsecurity/var/upload \
-        /opt/modsecurity/var/audit \
     ); \
     for dir in "${writeDirs[@]}"; do \
       mkdir -p ${dir}; \
       chown -R www-data.www-data ${dir}; \
     done \
-    '
+    ' \
-
+  && chown www-data.www-data /etc/nginx/nginx.conf \
-# Create symlinks to redirect nginx logs to stdout and stderr docker log collector
+  && chown www-data.www-data /etc/nginx/opentracing.json
-# This only works if nginx is started with CMD or ENTRYPOINT
-RUN  ln -sf /dev/stdout /var/log/nginx/access.log \
-  && ln -sf /dev/stderr /var/log/nginx/error.log
 
 USER www-data
 

rootfs/etc/nginx/opentracing.json

@@ -0,0 +1 @@
+{}

rootfs/etc/nginx/template/nginx.tmpl

@@ -16,7 +16,9 @@ pid /tmp/nginx.pid;
 load_module /etc/nginx/modules/ngx_http_modsecurity_module.so;
 {{ end }}
 
-{{ buildOpentracingLoad $cfg }}
+{{ if $cfg.EnableOpentracing }}
+load_module /etc/nginx/modules/ngx_http_opentracing_module.so;
+{{ end }}
 
 daemon off;
 
@@ -846,6 +848,10 @@ stream {
             set $service_port   "{{ $location.Port }}";
             set $location_path  "{{ $location.Path }}";
 
+            {{ if $all.Cfg.EnableOpentracing }}
+            opentracing_propagate_context;
+            {{ end }}
+
             {{ if not $all.DisableLua }}
             rewrite_by_lua_block {
                 {{ if $all.DynamicConfigurationEnabled}}