From 5f73b611f4da2c770b23169908b9ec3621d7f263 Mon Sep 17 00:00:00 2001 From: Joao Morais Date: Thu, 2 Feb 2017 22:56:44 -0200 Subject: [PATCH] Initial version of HAProxy controller --- controllers/haproxy/.gitignore | 1 + controllers/haproxy/Makefile | 16 +++ controllers/haproxy/pkg/controller/config.go | 56 ++++++++ controllers/haproxy/pkg/controller/haproxy.go | 124 ++++++++++++++++++ controllers/haproxy/pkg/controller/main.go | 46 +++++++ .../haproxy/pkg/controller/template.go | 59 +++++++++ controllers/haproxy/pkg/version/version.go | 26 ++++ controllers/haproxy/rootfs/Dockerfile | 28 ++++ controllers/haproxy/rootfs/haproxy-wrapper | 31 +++++ controllers/haproxy/rootfs/haproxy.tmpl | 113 ++++++++++++++++ 10 files changed, 500 insertions(+) create mode 100644 controllers/haproxy/.gitignore create mode 100644 controllers/haproxy/Makefile create mode 100644 controllers/haproxy/pkg/controller/config.go create mode 100644 controllers/haproxy/pkg/controller/haproxy.go create mode 100644 controllers/haproxy/pkg/controller/main.go create mode 100644 controllers/haproxy/pkg/controller/template.go create mode 100644 controllers/haproxy/pkg/version/version.go create mode 100644 controllers/haproxy/rootfs/Dockerfile create mode 100755 controllers/haproxy/rootfs/haproxy-wrapper create mode 100644 controllers/haproxy/rootfs/haproxy.tmpl diff --git a/controllers/haproxy/.gitignore b/controllers/haproxy/.gitignore new file mode 100644 index 000000000..4eb2366a1 --- /dev/null +++ b/controllers/haproxy/.gitignore @@ -0,0 +1 @@ +rootfs/haproxy-ingress-controller diff --git a/controllers/haproxy/Makefile b/controllers/haproxy/Makefile new file mode 100644 index 000000000..a1e89277c --- /dev/null +++ b/controllers/haproxy/Makefile @@ -0,0 +1,16 @@ +RELEASE=0.9.0-beta.1 +PREFIX=localhost/haproxy-ingress-controller + +PKG=k8s.io/ingress/controllers/haproxy/pkg +REPO_INFO=$(shell git config --get remote.origin.url) +COMMIT=git-$(shell git rev-parse --short HEAD) + +GOOS=linux + +build: + CGO_ENABLED=0 GOOS=$(GOOS) go build \ + -v -installsuffix cgo \ + -ldflags "-s -w -X $(PKG)/version.RELEASE=$(RELEASE) -X $(PKG)/version.COMMIT=$(COMMIT) -X $(PKG)/version.REPO=$(REPO_INFO)" \ + -o rootfs/haproxy-ingress-controller $(PKG)/controller +container: build + docker build -t $(PREFIX):$(RELEASE) rootfs diff --git a/controllers/haproxy/pkg/controller/config.go b/controllers/haproxy/pkg/controller/config.go new file mode 100644 index 000000000..0479825d3 --- /dev/null +++ b/controllers/haproxy/pkg/controller/config.go @@ -0,0 +1,56 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "github.com/golang/glog" + "github.com/mitchellh/mapstructure" + "k8s.io/ingress/core/pkg/ingress" +) + +type ( + configuration struct { + Backends []*ingress.Backend + Servers []*ingress.Server + TCPEndpoints []*ingress.Location + UDPEndpoints []*ingress.Location + PassthroughBackends []*ingress.SSLPassthroughBackend + Syslog string `json:"syslog-endpoint"` + } +) + +func newConfig(cfg *ingress.Configuration, data map[string]string) *configuration { + conf := configuration{ + Backends: cfg.Backends, + Servers: cfg.Servers, + TCPEndpoints: cfg.TCPEndpoints, + UDPEndpoints: cfg.UPDEndpoints, + PassthroughBackends: cfg.PassthroughBackends, + } + decoder, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ + WeaklyTypedInput: true, + Result: &conf, + TagName: "json", + }) + if err != nil { + glog.Warningf("error configuring decoder: %v", err) + } + if err = decoder.Decode(data); err != nil { + glog.Warningf("error decoding config: %v", err) + } + return &conf +} diff --git a/controllers/haproxy/pkg/controller/haproxy.go b/controllers/haproxy/pkg/controller/haproxy.go new file mode 100644 index 000000000..00f4d5d39 --- /dev/null +++ b/controllers/haproxy/pkg/controller/haproxy.go @@ -0,0 +1,124 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "bytes" + "github.com/golang/glog" + "io/ioutil" + "k8s.io/ingress/controllers/haproxy/pkg/version" + "k8s.io/ingress/core/pkg/ingress" + "k8s.io/ingress/core/pkg/ingress/controller" + "k8s.io/ingress/core/pkg/ingress/defaults" + "k8s.io/kubernetes/pkg/api" + "net/http" + "os" + "os/exec" +) + +type haproxyController struct { + controller *controller.GenericController + configMap *api.ConfigMap + command string + configFile string + template *template +} + +func newHAProxyController() *haproxyController { + return &haproxyController{ + command: "/haproxy-wrapper", + configFile: "/usr/local/etc/haproxy/haproxy.cfg", + template: newTemplate("haproxy.tmpl", "/usr/local/etc/haproxy/haproxy.tmpl"), + } +} + +func (haproxy *haproxyController) Info() *ingress.BackendInfo { + return &ingress.BackendInfo{ + Name: "HAProxy", + Release: version.RELEASE, + Build: version.COMMIT, + Repository: version.REPO, + } +} + +func (haproxy *haproxyController) Start() { + controller := controller.NewIngressController(haproxy) + haproxy.controller = controller + haproxy.controller.Start() +} + +func (haproxy *haproxyController) Stop() error { + err := haproxy.controller.Stop() + return err +} + +func (haproxy *haproxyController) Name() string { + return "HAProxy Ingress Controller" +} + +func (haproxy *haproxyController) Check(_ *http.Request) error { + return nil +} + +func (haproxy *haproxyController) SetConfig(configMap *api.ConfigMap) { + haproxy.configMap = configMap +} + +func (haproxy *haproxyController) BackendDefaults() defaults.Backend { + return defaults.Backend{} +} + +func (haproxy *haproxyController) OnUpdate(cfg ingress.Configuration) ([]byte, error) { + conf := newConfig(&cfg, haproxy.configMap.Data) + data, err := haproxy.template.execute(conf) + if err != nil { + return nil, err + } + return data, nil +} + +func (haproxy *haproxyController) Reload(data []byte) ([]byte, bool, error) { + if !haproxy.configChanged(data) { + return nil, false, nil + } + // TODO missing HAProxy validation before overwrite and try to reload + err := ioutil.WriteFile(haproxy.configFile, data, 0644) + if err != nil { + return nil, false, err + } + out, err := haproxy.reloadHaproxy() + if len(out) > 0 { + glog.Infof("HAProxy output:\n%v", string(out)) + } + return out, true, err +} + +func (haproxy *haproxyController) configChanged(data []byte) bool { + if _, err := os.Stat(haproxy.configFile); os.IsNotExist(err) { + return true + } + cfg, err := ioutil.ReadFile(haproxy.configFile) + if err != nil { + return false + } + return !bytes.Equal(cfg, data) +} + +func (haproxy *haproxyController) reloadHaproxy() ([]byte, error) { + out, err := exec.Command(haproxy.command, haproxy.configFile).CombinedOutput() + return out, err +} diff --git a/controllers/haproxy/pkg/controller/main.go b/controllers/haproxy/pkg/controller/main.go new file mode 100644 index 000000000..c91672e81 --- /dev/null +++ b/controllers/haproxy/pkg/controller/main.go @@ -0,0 +1,46 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "github.com/golang/glog" + "os" + "os/signal" + "syscall" +) + +func main() { + hc := newHAProxyController() + errCh := make(chan error) + go handleSignal(hc, errCh) + hc.Start() + code := 0 + err := <-errCh + if err != nil { + glog.Warningf("Error stopping Ingress: %v", err) + code++ + } + glog.Infof("Exiting (%v)", code) + os.Exit(code) +} + +func handleSignal(hc *haproxyController, err chan error) { + sig := make(chan os.Signal, 1) + signal.Notify(sig, syscall.SIGTERM, syscall.SIGINT) + glog.Infof("Shutting down with signal %v", <-sig) + err <- hc.Stop() +} diff --git a/controllers/haproxy/pkg/controller/template.go b/controllers/haproxy/pkg/controller/template.go new file mode 100644 index 000000000..8e0f08835 --- /dev/null +++ b/controllers/haproxy/pkg/controller/template.go @@ -0,0 +1,59 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "bytes" + "github.com/golang/glog" + "os/exec" + gotemplate "text/template" +) + +type template struct { + tmpl *gotemplate.Template + rawConfig *bytes.Buffer + fmtConfig *bytes.Buffer +} + +func newTemplate(name string, file string) *template { + tmpl, err := gotemplate.New(name).ParseFiles(file) + if err != nil { + glog.Fatalf("Cannot read template file: %v", err) + } + return &template{ + tmpl: tmpl, + rawConfig: bytes.NewBuffer(make([]byte, 0, 16384)), + fmtConfig: bytes.NewBuffer(make([]byte, 0, 16384)), + } +} + +func (t *template) execute(conf *configuration) ([]byte, error) { + t.rawConfig.Reset() + t.fmtConfig.Reset() + if err := t.tmpl.Execute(t.rawConfig, conf); err != nil { + return nil, err + } + cmd := exec.Command("sed", "/^ *$/d") + cmd.Stdin = t.rawConfig + cmd.Stdout = t.fmtConfig + if err := cmd.Run(); err != nil { + glog.Errorf("Template cleaning has failed: %v", err) + // TODO recover and return raw buffer + return nil, err + } + return t.fmtConfig.Bytes(), nil +} diff --git a/controllers/haproxy/pkg/version/version.go b/controllers/haproxy/pkg/version/version.go new file mode 100644 index 000000000..fbba0be53 --- /dev/null +++ b/controllers/haproxy/pkg/version/version.go @@ -0,0 +1,26 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package version + +var ( + // RELEASE Release version + RELEASE = "UNKNOWN" + // REPO Git repository URL + REPO = "UNKNOWN" + // COMMIT Short sha from git commit + COMMIT = "UNKNOWN" +) diff --git a/controllers/haproxy/rootfs/Dockerfile b/controllers/haproxy/rootfs/Dockerfile new file mode 100644 index 000000000..e71885f96 --- /dev/null +++ b/controllers/haproxy/rootfs/Dockerfile @@ -0,0 +1,28 @@ +# Copyright 2017 The Kubernetes Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM haproxy:1.7-alpine +RUN apk --no-cache add openssl + +# dumb-init kindly manages SIGCHLD from forked HAProxy processes +ARG DUMB_INIT_SHA256=81231da1cd074fdc81af62789fead8641ef3f24b6b07366a1c34e5b059faf363 +RUN wget -O/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64\ + && echo "$DUMB_INIT_SHA256 /dumb-init" | sha256sum -c -\ + && chmod +x /dumb-init + +COPY haproxy-ingress-controller / +COPY haproxy-wrapper / +COPY haproxy.tmpl /usr/local/etc/haproxy/ + +ENTRYPOINT ["/dumb-init", "--", "/haproxy-ingress-controller"] diff --git a/controllers/haproxy/rootfs/haproxy-wrapper b/controllers/haproxy/rootfs/haproxy-wrapper new file mode 100755 index 000000000..62a21d74c --- /dev/null +++ b/controllers/haproxy/rootfs/haproxy-wrapper @@ -0,0 +1,31 @@ +#!/bin/sh +# +# Copyright 2017 The Kubernetes Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# A script to help with haproxy reloads. Needs sudo for :80. Running it for the +# first time starts haproxy, each subsequent invocation will perform a +# soft-reload. +# Receives /path/to/haproxy.cfg as the first parameter +# HAProxy options: +# -f config file +# -p pid file +# -D run as daemon +# -sf soft reload, wait for pids to finish handling requests +# send pids a resume signal if reload of new config fails + +set -e + +pidFile="/var/run/haproxy.pid" +haproxy -f "$1" -p "$pidFile" -D -sf $(cat "$pidFile" 2>/dev/null || :) diff --git a/controllers/haproxy/rootfs/haproxy.tmpl b/controllers/haproxy/rootfs/haproxy.tmpl new file mode 100644 index 000000000..aa18c9bac --- /dev/null +++ b/controllers/haproxy/rootfs/haproxy.tmpl @@ -0,0 +1,113 @@ +{{ $cfg := . }} +global + daemon + stats socket /tmp/haproxy + #server-state-file global + #server-state-base /var/state/haproxy/ +{{ if ne $cfg.Syslog "" }} + log {{ $cfg.Syslog }} format rfc5424 local0 + log-tag ingress +{{ end }} + tune.ssl.default-dh-param 1024 + ssl-default-bind-ciphers ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!3DES:!MD5:!PSK + ssl-default-bind-options no-tls-tickets + +defaults + log global + #load-server-state-from-file global + option redispatch + option dontlognull + option http-server-close + option http-keep-alive + timeout http-request 5s + timeout connect 5s + timeout client 50s + timeout client-fin 50s + timeout server 50s + timeout tunnel 1h + timeout http-keep-alive 60s + #default_backend #default-backend + +###### +###### Backends +###### +{{ range $backend := $cfg.Backends }} +backend {{ $backend.Name }} + mode http + balance roundrobin +{{ range $endpoint := $backend.Endpoints }} +{{ $target := (print $endpoint.Address ":" $endpoint.Port) }} + server {{ $target }} {{ $target }} check port {{ $endpoint.Port }} inter 2s +{{ end }} +{{ end }} + +###### +###### HTTP frontend +###### +frontend httpfront + bind *:80 + mode http +{{ if ne $cfg.Syslog "" }} + option httplog +{{ end }} + option forwardfor +{{ range $server := $cfg.Servers }} +{{ if and (ne $server.Hostname "_") (ne $server.SSLCertificate "") }} + redirect scheme https if { hdr(host) {{ $server.Hostname }} } +{{ end }} +{{ end }} +{{ range $server := $cfg.Servers }} +{{ if and (ne $server.Hostname "_") (eq $server.SSLCertificate "") }} +{{ range $location := $server.Locations }} + use_backend {{ $location.Backend }} if { hdr(host) {{ $server.Hostname }} } { path_beg {{ $location.Path }} } +{{ end }} +{{ end }} +{{ end }} + +###### +###### HTTPS frontends (tcp mode) +###### +frontend httpsfront + bind :443 + mode tcp + tcp-request inspect-delay 5s + tcp-request content accept if { req.ssl_hello_type 1 } +{{ range $server := $cfg.Servers }} +{{ if and (ne $server.Hostname "_") (ne $server.SSLCertificate "") }} + use_backend httpsback-{{ $server.Hostname }} if { req.ssl_sni -i {{ $server.Hostname }} } +{{ end }} +{{ end }} + +{{ range $server := $cfg.Servers }} +{{ if and (ne $server.Hostname "_") (ne $server.SSLCertificate "") }} +## +## {{ $server.Hostname }} +backend httpsback-{{ $server.Hostname }} + mode tcp + server {{ $server.Hostname }} unix@/var/run/haproxy-{{ $server.Hostname }}.sock send-proxy-v2 + +frontend httpsfront-{{ $server.Hostname }} + # CRT PEM checksum: {{ $server.SSLPemChecksum }} + bind unix@/var/run/haproxy-{{ $server.Hostname }}.sock ssl crt {{ $server.SSLCertificate }} no-sslv3 accept-proxy + mode http +{{ if ne $cfg.Syslog "" }} + option httplog +{{ end }} + option forwardfor + rspadd Strict-Transport-Security:\ max-age=15768000 +{{ range $location := $server.Locations }} + use_backend {{ $location.Backend }} if { path_beg {{ $location.Path }} } +{{ end }} +{{ end }} +{{ end }} + +###### +###### Status page +###### +listen stats + bind *:1936 + mode http + stats enable + stats realm Haproxy\ Statistics + stats uri / + no log