From df76d4b481b934d2f8c7b6a8d3923d24fd005eb4 Mon Sep 17 00:00:00 2001 From: Manuel Alejandro de Brito Fontes Date: Thu, 21 Jun 2018 18:15:18 -0400 Subject: [PATCH] Update opentracing configuration (#2676) --- Makefile | 4 +- internal/ingress/controller/nginx.go | 53 ++++++++++++++++++- .../ingress/controller/template/template.go | 42 ++------------- rootfs/Dockerfile | 38 ++++++------- rootfs/etc/nginx/opentracing.json | 1 + rootfs/etc/nginx/template/nginx.tmpl | 8 ++- 6 files changed, 84 insertions(+), 62 deletions(-) create mode 100644 rootfs/etc/nginx/opentracing.json diff --git a/Makefile b/Makefile index ee9ccba23..b5771458f 100644 --- a/Makefile +++ b/Makefile @@ -59,7 +59,7 @@ IMAGE = $(REGISTRY)/$(IMGNAME) MULTI_ARCH_IMG = $(IMAGE)-$(ARCH) # Set default base image dynamically for each arch -BASEIMAGE?=quay.io/kubernetes-ingress-controller/nginx-$(ARCH):0.52 +BASEIMAGE?=quay.io/kubernetes-ingress-controller/nginx-$(ARCH):0.53 ifeq ($(ARCH),arm) QEMUARCH=arm @@ -149,7 +149,7 @@ clean: build: clean CGO_ENABLED=0 GOOS=${GOOS} GOARCH=${GOARCH} go build -a -installsuffix cgo \ -ldflags "-s -w -X ${PKG}/version.RELEASE=${TAG} -X ${PKG}/version.COMMIT=${COMMIT} -X ${PKG}/version.REPO=${REPO_INFO}" \ - -o ${TEMP_DIR}/rootfs/nginx-ingress-controller ${PKG}/cmd/nginx + -o ${TEMP_DIR}/rootfs/nginx-ingress-controller ${PKG}/cmd/nginx .PHONY: verify-all verify-all: diff --git a/internal/ingress/controller/nginx.go b/internal/ingress/controller/nginx.go index 1739c1caf..28148f67f 100644 --- a/internal/ingress/controller/nginx.go +++ b/internal/ingress/controller/nginx.go @@ -31,6 +31,7 @@ import ( "strings" "sync" "syscall" + "text/template" "time" "github.com/golang/glog" @@ -593,11 +594,17 @@ func (n *NGINXController) OnUpdate(ingressCfg ingress.Configuration) error { } content, err := n.t.Write(tc) - if err != nil { return err } + if cfg.EnableOpentracing { + err := createOpentracingCfg(cfg) + if err != nil { + return err + } + } + err = n.testTemplate(content) if err != nil { return err @@ -779,3 +786,47 @@ func configureDynamically(pcfg *ingress.Configuration, port int) error { return nil } + +const zipkinTmpl = `{ + "service_name": "{{ .ZipkinServiceName }}", + "collector_host": "{{ .ZipkinCollectorHost }}", + "collector_port": {{ .ZipkinCollectorPort }} +}` + +const jaegerTmpl = `{ + "service_name": "{{ .JaegerServiceName }}", + "sampler": { + "type": "{{ .JaegerSamplerType }}", + "param": {{ .JaegerSamplerParam }} + }, + "reporter": { + "localAgentHostPort": "{{ .JaegerCollectorHost }}:{{ .JaegerCollectorPort }}" + } +}` + +func createOpentracingCfg(cfg ngx_config.Configuration) error { + var tmpl *template.Template + var err error + + if cfg.ZipkinCollectorHost != "" { + tmpl, err = template.New("zipkin").Parse(zipkinTmpl) + if err != nil { + return err + } + } else if cfg.JaegerCollectorHost != "" { + tmpl, err = template.New("jarger").Parse(jaegerTmpl) + if err != nil { + return err + } + } else { + tmpl, _ = template.New("empty").Parse("{}") + } + + tmplBuf := bytes.NewBuffer(make([]byte, 0)) + err = tmpl.Execute(tmplBuf, cfg) + if err != nil { + return err + } + + return ioutil.WriteFile("/etc/nginx/opentracing.json", tmplBuf.Bytes(), file.ReadWriteByUser) +} diff --git a/internal/ingress/controller/template/template.go b/internal/ingress/controller/template/template.go index 320c31553..b4babf9e3 100644 --- a/internal/ingress/controller/template/template.go +++ b/internal/ingress/controller/template/template.go @@ -151,7 +151,6 @@ var ( "isValidClientBodyBufferSize": isValidClientBodyBufferSize, "buildForwardedFor": buildForwardedFor, "buildAuthSignURL": buildAuthSignURL, - "buildOpentracingLoad": buildOpentracingLoad, "buildOpentracing": buildOpentracing, "proxySetHeader": proxySetHeader, "buildInfluxDB": buildInfluxDB, @@ -841,31 +840,6 @@ func randomString() string { return string(b) } -func buildOpentracingLoad(input interface{}) string { - cfg, ok := input.(config.Configuration) - if !ok { - glog.Errorf("expected a 'config.Configuration' type but %T was returned", input) - return "" - } - - if !cfg.EnableOpentracing { - return "" - } - - buf := bytes.NewBufferString("load_module /etc/nginx/modules/ngx_http_opentracing_module.so;") - buf.WriteString("\r\n") - - if cfg.ZipkinCollectorHost != "" { - buf.WriteString("load_module /etc/nginx/modules/ngx_http_zipkin_module.so;") - } else if cfg.JaegerCollectorHost != "" { - buf.WriteString("load_module /etc/nginx/modules/ngx_http_jaeger_module.so;") - } - - buf.WriteString("\r\n") - - return buf.String() -} - func buildOpentracing(input interface{}) string { cfg, ok := input.(config.Configuration) if !ok { @@ -878,24 +852,14 @@ func buildOpentracing(input interface{}) string { } buf := bytes.NewBufferString("") - if cfg.ZipkinCollectorHost != "" { - buf.WriteString(fmt.Sprintf("zipkin_collector_host %v;", cfg.ZipkinCollectorHost)) - buf.WriteString("\r\n") - buf.WriteString(fmt.Sprintf("zipkin_collector_port %v;", cfg.ZipkinCollectorPort)) - buf.WriteString("\r\n") - buf.WriteString(fmt.Sprintf("zipkin_service_name %v;", cfg.ZipkinServiceName)) + buf.WriteString("opentracing_load_tracer /usr/local/lib/libzipkin_opentracing.so /etc/nginx/opentracing.json;") } else if cfg.JaegerCollectorHost != "" { - buf.WriteString(fmt.Sprintf("jaeger_reporter_local_agent_host_port %v:%v;", cfg.JaegerCollectorHost, cfg.JaegerCollectorPort)) - buf.WriteString("\r\n") - buf.WriteString(fmt.Sprintf("jaeger_service_name %v;", cfg.JaegerServiceName)) - buf.WriteString("\r\n") - buf.WriteString(fmt.Sprintf("jaeger_sampler_type %v;", cfg.JaegerSamplerType)) - buf.WriteString("\r\n") - buf.WriteString(fmt.Sprintf("jaeger_sampler_param %v;", cfg.JaegerSamplerParam)) + buf.WriteString("opentracing_load_tracer /usr/local/lib/libjaegertracing.so /etc/nginx/opentracing.json;") } buf.WriteString("\r\n") + return buf.String() } diff --git a/rootfs/Dockerfile b/rootfs/Dockerfile index a6c702021..812215278 100644 --- a/rootfs/Dockerfile +++ b/rootfs/Dockerfile @@ -20,35 +20,35 @@ WORKDIR /etc/nginx RUN clean-install \ diffutils \ - dumb-init \ - libcap2-bin + libcap2-bin \ + dumb-init COPY . / -RUN setcap cap_net_bind_service=+ep /usr/sbin/nginx \ - && setcap cap_net_bind_service=+ep /nginx-ingress-controller +RUN setcap cap_net_bind_service=+ep /nginx-ingress-controller -RUN bash -eux -c ' \ +# Create symlinks to redirect nginx logs to stdout and stderr docker log collector +# This only works if nginx is started with CMD or ENTRYPOINT +# Required because clean-install removes /var/log content +# We cannot chown /etc/nginx recursively because that adds 100MB to the image +RUN mkdir -p /var/log/nginx \ + && ln -sf /dev/stdout /var/log/nginx/access.log \ + && ln -sf /dev/stderr /var/log/nginx/error.log \ + && bash -eux -c ' \ writeDirs=( \ - /etc/nginx \ - /etc/ingress-controller/ssl \ - /etc/ingress-controller/auth \ - /var/log \ - /var/log/nginx \ - /opt/modsecurity/var/log \ - /opt/modsecurity/var/upload \ - /opt/modsecurity/var/audit \ + /etc/nginx/template \ + /etc/ingress-controller/ssl \ + /etc/ingress-controller/auth \ + /var/log \ + /var/log/nginx \ ); \ for dir in "${writeDirs[@]}"; do \ mkdir -p ${dir}; \ chown -R www-data.www-data ${dir}; \ done \ - ' - -# Create symlinks to redirect nginx logs to stdout and stderr docker log collector -# This only works if nginx is started with CMD or ENTRYPOINT -RUN ln -sf /dev/stdout /var/log/nginx/access.log \ - && ln -sf /dev/stderr /var/log/nginx/error.log + ' \ + && chown www-data.www-data /etc/nginx/nginx.conf \ + && chown www-data.www-data /etc/nginx/opentracing.json USER www-data diff --git a/rootfs/etc/nginx/opentracing.json b/rootfs/etc/nginx/opentracing.json new file mode 100644 index 000000000..9e26dfeeb --- /dev/null +++ b/rootfs/etc/nginx/opentracing.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/rootfs/etc/nginx/template/nginx.tmpl b/rootfs/etc/nginx/template/nginx.tmpl index 1f30b0eea..451d6c489 100644 --- a/rootfs/etc/nginx/template/nginx.tmpl +++ b/rootfs/etc/nginx/template/nginx.tmpl @@ -16,7 +16,9 @@ pid /tmp/nginx.pid; load_module /etc/nginx/modules/ngx_http_modsecurity_module.so; {{ end }} -{{ buildOpentracingLoad $cfg }} +{{ if $cfg.EnableOpentracing }} +load_module /etc/nginx/modules/ngx_http_opentracing_module.so; +{{ end }} daemon off; @@ -846,6 +848,10 @@ stream { set $service_port "{{ $location.Port }}"; set $location_path "{{ $location.Path }}"; + {{ if $all.Cfg.EnableOpentracing }} + opentracing_propagate_context; + {{ end }} + {{ if not $all.DisableLua }} rewrite_by_lua_block { {{ if $all.DynamicConfigurationEnabled}}