diff --git a/controllers/nginx/pkg/config/config.go b/controllers/nginx/pkg/config/config.go index bfe4cc79e..c23c15867 100644 --- a/controllers/nginx/pkg/config/config.go +++ b/controllers/nginx/pkg/config/config.go @@ -21,6 +21,7 @@ import ( "github.com/golang/glog" + "fmt" "k8s.io/ingress/core/pkg/ingress" "k8s.io/ingress/core/pkg/ingress/defaults" ) @@ -46,6 +47,10 @@ const ( gzipTypes = "application/atom+xml application/javascript application/x-javascript application/json application/rss+xml application/vnd.ms-fontobject application/x-font-ttf application/x-web-app-manifest+json application/xhtml+xml application/xml font/opentype image/svg+xml image/x-icon text/css text/plain text/x-component" + logFormatUpstream = "'%v - [$proxy_add_x_forwarded_for] - $remote_user [$time_local] \"$request\" $status $body_bytes_sent \"$http_referer\" \"$http_user_agent\" $request_length $request_time [$proxy_upstream_name] $upstream_addr $upstream_response_length $upstream_response_time $upstream_status'" + + logFormatStream = "'$remote_addr [$time_local] $protocol [$ssl_preread_server_name] [$stream_upstream] $status $bytes_sent $bytes_received $session_time'" + // http://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_buffer_size // Sets the size of the buffer used for sending data. // 4k helps NGINX to improve TLS Time To First Byte (TTTFB) @@ -143,6 +148,14 @@ type Configuration struct { // Default: 4 8k LargeClientHeaderBuffers string `json:"large-client-header-buffers"` + // Customize upstream log_format + // http://nginx.org/en/docs/http/ngx_http_log_module.html#log_format + LogFormatUpstream string `json:"log-format-upstream,omitempty"` + + // Customize stream log_format + // http://nginx.org/en/docs/http/ngx_http_log_module.html#log_format + LogFormatStream string `json:"log-format-stream,omitempty"` + // Maximum number of simultaneous connections that can be opened by each worker process // http://nginx.org/en/docs/ngx_core_module.html#worker_connections MaxWorkerConnections int `json:"max-worker-connections,omitempty"` @@ -250,6 +263,8 @@ func NewDefault() Configuration { GzipTypes: gzipTypes, KeepAlive: 75, LargeClientHeaderBuffers: "4 8k", + LogFormatStream: logFormatStream, + LogFormatUpstream: BuildLogFormatUpstream(false), MaxWorkerConnections: 16384, MapHashBucketSize: 64, ProxyRealIPCIDR: defIPCIDR, @@ -291,6 +306,15 @@ func NewDefault() Configuration { return cfg } +// BuildLogFormatUpstream format the log_format upstream based on proxy_protocol +func BuildLogFormatUpstream(useProxyProtocol bool) string { + + if useProxyProtocol { + return fmt.Sprintf(logFormatUpstream, "$proxy_protocol_addr") + } + return fmt.Sprintf(logFormatUpstream, "$remote_addr") +} + // TemplateConfig contains the nginx configuration to render the file nginx.conf type TemplateConfig struct { ProxySetHeaders map[string]string diff --git a/controllers/nginx/pkg/config/config_test.go b/controllers/nginx/pkg/config/config_test.go new file mode 100644 index 000000000..d0fccc69b --- /dev/null +++ b/controllers/nginx/pkg/config/config_test.go @@ -0,0 +1,27 @@ +package config + +import ( + "fmt" + "testing" +) + +func TestBuildLogFormatUpstream(t *testing.T) { + + testCases := []struct { + useProxyProtocol bool // use proxy protocol + expected string + }{ + {true, fmt.Sprintf(logFormatUpstream, "$proxy_protocol_addr")}, + {false, fmt.Sprintf(logFormatUpstream, "$remote_addr")}, + } + + for _, testCase := range testCases { + + result := BuildLogFormatUpstream(testCase.useProxyProtocol) + + if result != testCase.expected { + t.Errorf(" expected %v but return %v", testCase.expected, result) + } + + } +} diff --git a/controllers/nginx/pkg/template/template.go b/controllers/nginx/pkg/template/template.go index 62975cfe5..cd3e43afa 100644 --- a/controllers/nginx/pkg/template/template.go +++ b/controllers/nginx/pkg/template/template.go @@ -31,6 +31,7 @@ import ( "github.com/golang/glog" "k8s.io/ingress/controllers/nginx/pkg/config" + nginxconfig "k8s.io/ingress/controllers/nginx/pkg/config" "k8s.io/ingress/core/pkg/ingress" ing_net "k8s.io/ingress/core/pkg/net" "k8s.io/ingress/core/pkg/watch" @@ -135,12 +136,12 @@ var ( "buildSSLPassthroughUpstreams": buildSSLPassthroughUpstreams, "buildResolvers": buildResolvers, "isLocationAllowed": isLocationAllowed, - - "contains": strings.Contains, - "hasPrefix": strings.HasPrefix, - "hasSuffix": strings.HasSuffix, - "toUpper": strings.ToUpper, - "toLower": strings.ToLower, + "buildLogFormatUpstream": buildLogFormatUpstream, + "contains": strings.Contains, + "hasPrefix": strings.HasPrefix, + "hasSuffix": strings.HasSuffix, + "toUpper": strings.ToUpper, + "toLower": strings.ToLower, } ) @@ -248,6 +249,17 @@ func buildAuthResponseHeaders(input interface{}) []string { return res } +func buildLogFormatUpstream(input interface{}) string { + config, ok := input.(config.Configuration) + + if !ok { + glog.Errorf("error an ingress.buildLogFormatUpstream type but %T was returned", input) + } + + return nginxconfig.BuildLogFormatUpstream(config.UseProxyProtocol) + +} + // buildProxyPass produces the proxy pass string, if the ingress has redirects // (specified through the ingress.kubernetes.io/rewrite-to annotation) // If the annotation ingress.kubernetes.io/add-base-url:"true" is specified it will diff --git a/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl b/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl index f79439d0d..44ee1c25f 100644 --- a/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl +++ b/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl @@ -77,11 +77,9 @@ http { gzip_proxied any; {{ end }} - server_tokens {{ if $cfg.ShowServerTokens }}on{{ else }}off{{ end }}; + server_tokens {{ if $cfg.ShowServerTokens }}on{{ else }}off{{ end }}; - log_format upstreaminfo '{{ if $cfg.UseProxyProtocol }}$proxy_protocol_addr{{ else }}$remote_addr{{ end }} - ' - '[$proxy_add_x_forwarded_for] - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" ' - '$request_length $request_time [$proxy_upstream_name] $upstream_addr $upstream_response_length $upstream_response_time $upstream_status'; + log_format upstreaminfo {{ buildLogFormatUpstream $cfg }}; {{/* map urls that should not appear in access.log */}} {{/* http://nginx.org/en/docs/http/ngx_http_log_module.html#access_log */}} @@ -451,7 +449,7 @@ stream { default nginx-ssl-backend; } - log_format log_stream '$remote_addr [$time_local] $protocol [$ssl_preread_server_name] [$stream_upstream] $status $bytes_sent $bytes_received $session_time'; + log_format log_stream {{ $cfg.LogFormatStream }}; {{ if $cfg.DisableAccessLog }} access_log off; diff --git a/core/pkg/net/ssl/ssl.go b/core/pkg/net/ssl/ssl.go index 62d2f6b7e..ea492940f 100644 --- a/core/pkg/net/ssl/ssl.go +++ b/core/pkg/net/ssl/ssl.go @@ -71,6 +71,11 @@ func AddOrUpdateCertAndKey(name string, cert, key, ca []byte) (*ingress.SSLCert, return nil, fmt.Errorf("No valid PEM formatted block found") } + // If the file does not start with 'BEGIN CERTIFICATE' it's invalid and must not be used. + if pemBlock.Type != "CERTIFICATE" { + return nil, fmt.Errorf("Certificate %v contains invalid data, and must be created with 'kubectl create secret tls'", name) + } + pemCert, err := x509.ParseCertificate(pemBlock.Bytes) if err != nil { return nil, err @@ -138,6 +143,10 @@ func AddCertAuth(name string, ca []byte) (*ingress.SSLCert, error) { if pemCABlock == nil { return nil, fmt.Errorf("No valid PEM formatted block found") } + // If the first certificate does not start with 'BEGIN CERTIFICATE' it's invalid and must not be used. + if pemCABlock.Type != "CERTIFICATE" { + return nil, fmt.Errorf("CA File %v contains invalid data, and must be created only with PEM formated certificates", name) + } _, err := x509.ParseCertificate(pemCABlock.Bytes) if err != nil { diff --git a/examples/daemonset/haproxy/README.md b/examples/daemonset/haproxy/README.md new file mode 100644 index 000000000..75fe32eb5 --- /dev/null +++ b/examples/daemonset/haproxy/README.md @@ -0,0 +1,62 @@ +# Haproxy Ingress DaemonSet + +In some cases, the Ingress controller will be required to be run at all the nodes in cluster. Using [DaemonSet](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/daemon.md) can achieve this requirement. + +## Prerequisites + +This ingress controller doesn't yet have support for +[ingress classes](/examples/PREREQUISITES.md#ingress-class). You MUST turn +down any existing ingress controllers before running HAProxy Ingress controller or +they will fight for Ingresses. This includes any cloudprovider controller. + +This document has also the following prerequisites: + +* Create a [TLS secret](/examples/PREREQUISITES.md#tls-certificates) named `tls-secret` to be used as default TLS certificate + +Creating the TLS secret: + +```console +$ openssl req \ + -x509 -newkey rsa:2048 -nodes -days 365 \ + -keyout tls.key -out tls.crt -subj '/CN=localhost' +$ kubectl create secret tls tls-secret --cert=tls.crt --key=tls.key +$ rm -v tls.crt tls.key +``` + +## Default Backend + +The default backend is a service of handling all url paths and hosts the haproxy controller doesn't understand. Deploy the default-http-backend as follow: + +```console +$ kubectl apply -f ../../deployment/nginx/default-backend.yaml +deployment "default-http-backend" configured +service "default-http-backend" configured + +$ kubectl -n kube-system get svc +NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE +default-http-backend 192.168.3.4 80/TCP 30m + +$ kubectl -n kube-system get pods +NAME READY STATUS RESTARTS AGE +default-http-backend-q5sb6 1/1 Running 0 30m +``` + +## Ingress DaemonSet + +Deploy the daemonset as follows: + +```console +$ kubectl apply -f haproxy-ingress-daemonset.yaml +``` + +Check if the controller was successfully deployed: +```console +$ kubectl -n kube-system get ds +NAME DESIRED CURRENT READY NODE-SELECTOR AGE +haproxy-ingress 2 2 2 45s + +$ kubectl -n kube-system get pods +NAME READY STATUS RESTARTS AGE +default-http-backend-q5sb6 1/1 Running 0 45m +haproxy-ingress-km32x 1/1 Running 0 1m +``` diff --git a/examples/daemonset/haproxy/haproxy-ingress-daemonset.yaml b/examples/daemonset/haproxy/haproxy-ingress-daemonset.yaml new file mode 100644 index 000000000..6d6d689ad --- /dev/null +++ b/examples/daemonset/haproxy/haproxy-ingress-daemonset.yaml @@ -0,0 +1,35 @@ +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + labels: + run: haproxy-ingress + name: haproxy-ingress +spec: + template: + metadata: + labels: + run: haproxy-ingress + spec: + containers: + - name: haproxy-ingress + image: quay.io/jcmoraisjr/haproxy-ingress + imagePullPolicy: IfNotPresent + args: + - --default-backend-service=default/default-http-backend + - --default-ssl-certificate=default/tls-secret + ports: + - name: http + containerPort: 80 + - name: https + containerPort: 443 + - name: stat + containerPort: 1936 + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace diff --git a/examples/scaling-deployment/nginx/README.md b/examples/scaling-deployment/nginx/README.md new file mode 100644 index 000000000..e38174015 --- /dev/null +++ b/examples/scaling-deployment/nginx/README.md @@ -0,0 +1,41 @@ +# Deploying multi Nginx Ingress Controllers + +This example aims to demonstrate the Deployment of multi nginx ingress controllers. + +## Default Backend + +The default backend is a service of handling all url paths and hosts the nginx controller doesn't understand. Deploy the default-http-backend as follow: + +```console +$ kubectl apply -f ../../deployment/nginx/default-backend.yaml +deployment "default-http-backend" configured +service "default-http-backend" configured + +$ kubectl -n kube-system get svc +NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE +default-http-backend 192.168.3.52 80/TCP 6m + +$ kubectl -n kube-system get po +NAME READY STATUS RESTARTS AGE +default-http-backend-2657704409-wz6o3 1/1 Running 0 6m +``` + +## Ingress Deployment + +Deploy the Deployment of multi controllers as follows: + +```console +$ kubectl apply -f nginx-ingress-deployment.yaml +deployment "nginx-ingress-controller" created + +$ kubectl -n kube-system get deployment +NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE +default-http-backend 1 1 1 1 16m +nginx-ingress-controller 2 2 2 2 24s + +$ kubectl -n kube-system get po +NAME READY STATUS RESTARTS AGE +default-http-backend-2657704409-wz6o3 1/1 Running 0 16m +nginx-ingress-controller-3752011415-0qbi6 1/1 Running 0 39s +nginx-ingress-controller-3752011415-vi8fq 1/1 Running 0 39s +``` diff --git a/examples/scaling-deployment/nginx/nginx-ingress-deployment.yaml b/examples/scaling-deployment/nginx/nginx-ingress-deployment.yaml new file mode 100644 index 000000000..f0ee65b31 --- /dev/null +++ b/examples/scaling-deployment/nginx/nginx-ingress-deployment.yaml @@ -0,0 +1,47 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: nginx-ingress-controller + labels: + k8s-app: nginx-ingress-controller + namespace: kube-system +spec: + replicas: 2 + template: + metadata: + labels: + k8s-app: nginx-ingress-controller + spec: + terminationGracePeriodSeconds: 60 + containers: + - image: gcr.io/google_containers/nginx-ingress-controller:0.9.0-beta.2 + name: nginx-ingress-controller + readinessProbe: + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + livenessProbe: + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + initialDelaySeconds: 10 + timeoutSeconds: 1 + ports: + - containerPort: 80 + hostPort: 80 + - containerPort: 443 + hostPort: 443 + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + args: + - /nginx-ingress-controller + - --default-backend-service=$(POD_NAMESPACE)/default-http-backend