Merge pull request #1239 from aledbf/pc

Add flags to customize listen ports and detect port collisions
This commit is contained in:
Manuel Alejandro de Brito Fontes 2017-08-24 10:23:59 -04:00 committed by GitHub
commit e7d2ff6fac
12 changed files with 127 additions and 47 deletions

View file

@ -47,13 +47,19 @@ Anytime we reference a tls secret, we mean (x509, pem encoded, RSA 2048, etc). Y
Usage of :
--alsologtostderr log to standard error as well as files
--apiserver-host string The address of the Kubernetes Apiserver to connect to in the format of protocol://address:port, e.g., http://localhost:8080. If not specified, the assumption is that the binary runs inside a Kubernetes cluster and local discovery is attempted.
--configmap string Name of the ConfigMap that contains the custom configuration use
--default-backend-service string Service used to serve a 404 page for the default backend. Takes the form namespace/name. The controller uses the first node port of this Service for the default backend.
--configmap string Name of the ConfigMap that contains the custom configuration to use
--default-backend-service string Service used to serve a 404 page for the default backend. Takes the form
namespace/name. The controller uses the first node port of this Service for
the default backend.
--default-server-port int Default port to use for exposing the default server (catch all) (default 8181)
--default-ssl-certificate string Name of the secret that contains a SSL certificate to be used as default for a HTTPS catch-all server
--election-id string Election id to use for status update. (default "ingress-controller-leader")
--enable-ssl-passthrough Enable SSL passthrough feature. Default is disabled
--force-namespace-isolation Force namespace isolation. This flag is required to avoid the reference of secrets or configmaps located in a different namespace than the specified in the flag --watch-namespace.
--health-check-path string Defines the URL to be used as health check inside in the default server in NGINX. (default "/healthz")
--healthz-port int port for healthz endpoint. (default 10254)
--http-port int Indicates the port to use for HTTP traffic (default 80)
--https-port int Indicates the port to use for HTTPS traffic (default 443)
--ingress-class string Name of the ingress class to route through this controller.
--kubeconfig string Path to kubeconfig file with authorization and master location information.
--log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0)
@ -61,15 +67,16 @@ Usage of :
--logtostderr log to standard error instead of files
--profiling Enable profiling via web interface host:port/debug/pprof/ (default true)
--publish-service string Service fronting the ingress controllers. Takes the form namespace/name. The controller will set the endpoint records on the ingress objects to reflect those on the service.
--sort-backends Defines if backends and it's endpoints should be sorted
--ssl-passtrough-proxy-port int Default port to use internally for SSL when SSL Passthgough is enabled (default 442)
--status-port int Indicates the TCP port to use for exposing the nginx status page (default 18080)
--stderrthreshold severity logs at or above this threshold go to stderr (default 2)
--sync-period duration Relist and confirm cloud resources this often. (default 1m0s)
--tcp-services-configmap string Name of the ConfigMap that contains the definition of the TCP services to expose.
The key in the map indicates the external port to be used. The value is the name of the service with the format namespace/serviceName and the port of the service could be a number of the name of the port.
The ports 80 and 443 are not allowed as external ports. This ports are reserved for the backend
--udp-services-configmap string Name of the ConfigMap that contains the definition of the UDP services to expose.
The key in the map indicates the external port to be used. The value is the name of the service with the format namespace/serviceName and the port of the service could be a number of the name of the port.
--sync-period duration Relist and confirm cloud resources this often. Default is 10 minutes (default 10m0s)
--tcp-services-configmap string Name of the ConfigMap that contains the definition of the TCP services to expose. The key in the map indicates the external port to be used. The value is the name of theservice with the format namespace/serviceName and the port of the service could be a number of the name of the port. The ports 80 and 443 are not allowed as external ports. This ports are reserved for the backend
--udp-services-configmap string Name of the ConfigMap that contains the definition of the UDP services to expose. The key in the map indicates the external port to be used. The value is the name of theservice with the format namespace/serviceName and the port of the service could be a number of the name of the port.
--update-status Indicates if the ingress controller should update the Ingress status IP/hostname. Default is true (default true)
-v, --v Level log level for V logs
--update-status-on-shutdown Indicates if the ingress controller should update the Ingress status IP/hostname when the controller is being stopped. Default is true (default true)
-v, --v Level log level for V logs
--vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging
--watch-namespace string Namespace to watch for Ingress. Default is to watch all namespaces
```

View file

@ -45,6 +45,8 @@ type statsCollector struct {
namespace string
watchClass string
healthPort int
}
func (s *statsCollector) stop(sm statusModule) {
@ -61,17 +63,17 @@ func (s *statsCollector) stop(sm statusModule) {
func (s *statsCollector) start(sm statusModule) {
switch sm {
case defaultStatusModule:
s.basic = collector.NewNginxStatus(s.namespace, s.watchClass, ngxHealthPort, ngxStatusPath)
s.basic = collector.NewNginxStatus(s.namespace, s.watchClass, s.healthPort, ngxStatusPath)
prometheus.Register(s.basic)
break
case vtsStatusModule:
s.vts = collector.NewNGINXVTSCollector(s.namespace, s.watchClass, ngxHealthPort, ngxVtsPath)
s.vts = collector.NewNGINXVTSCollector(s.namespace, s.watchClass, s.healthPort, ngxVtsPath)
prometheus.Register(s.vts)
break
}
}
func newStatsCollector(ns, class, binary string) *statsCollector {
func newStatsCollector(ns, class, binary string, hz int) *statsCollector {
glog.Infof("starting new nginx stats collector for Ingress controller running in namespace %v (class %v)", ns, class)
pc, err := collector.NewNamedProcess(true, collector.BinaryNameMatcher{
Name: "nginx",
@ -89,5 +91,6 @@ func newStatsCollector(ns, class, binary string) *statsCollector {
namespace: ns,
watchClass: class,
process: pc,
healthPort: hz,
}
}

View file

@ -34,6 +34,7 @@ import (
"github.com/spf13/pflag"
proxyproto "github.com/armon/go-proxyproto"
api "k8s.io/api/core/v1"
api_v1 "k8s.io/api/core/v1"
extensions "k8s.io/api/extensions/v1beta1"
@ -50,7 +51,6 @@ import (
type statusModule string
const (
ngxHealthPort = 18080
ngxHealthPath = "/healthz"
defaultStatusModule statusModule = "default"
@ -87,6 +87,7 @@ func newNGINXController() ingress.Controller {
configmap: &api_v1.ConfigMap{},
isIPV6Enabled: isIPv6Enabled(),
resolver: h,
ports: &config.ListenPorts{},
}
fcgiListener, err := net.Listen("unix", fastCGISocket)
@ -161,6 +162,8 @@ type NGINXController struct {
isSSLPassthroughEnabled bool
proxy *proxy
ports *config.ListenPorts
}
// Start start a new NGINX master process running in foreground.
@ -280,14 +283,42 @@ func (n NGINXController) Info() *ingress.BackendInfo {
}
}
// DefaultEndpoint returns the default endpoint to be use as default server that returns 404.
func (n NGINXController) DefaultEndpoint() ingress.Endpoint {
return ingress.Endpoint{
Address: "127.0.0.1",
Port: fmt.Sprintf("%v", n.ports.Default),
Target: &api.ObjectReference{},
}
}
// ConfigureFlags allow to configure more flags before the parsing of
// command line arguments
func (n *NGINXController) ConfigureFlags(flags *pflag.FlagSet) {
flags.BoolVar(&n.isSSLPassthroughEnabled, "enable-ssl-passthrough", false, `Enable SSL passthrough feature. Default is disabled`)
flags.IntVar(&n.ports.HTTP, "http-port", 80, `Indicates the port to use for HTTP traffic`)
flags.IntVar(&n.ports.HTTPS, "https-port", 443, `Indicates the port to use for HTTPS traffic`)
flags.IntVar(&n.ports.Status, "status-port", 18080, `Indicates the TCP port to use for exposing the nginx status page`)
flags.IntVar(&n.ports.SSLProxy, "ssl-passtrough-proxy-port", 442, `Default port to use internally for SSL when SSL Passthgough is enabled`)
flags.IntVar(&n.ports.Default, "default-server-port", 8181, `Default port to use for exposing the default server (catch all)`)
}
// OverrideFlags customize NGINX controller flags
func (n *NGINXController) OverrideFlags(flags *pflag.FlagSet) {
// we check port collisions
if !isPortAvailable(n.ports.HTTP) {
glog.Fatalf("Port %v is already in use. Please check the flag --http-port", n.ports.HTTP)
}
if !isPortAvailable(n.ports.HTTPS) {
glog.Fatalf("Port %v is already in use. Please check the flag --https-port", n.ports.HTTPS)
}
if !isPortAvailable(n.ports.Status) {
glog.Fatalf("Port %v is already in use. Please check the flag --status-port", n.ports.Status)
}
if !isPortAvailable(n.ports.Default) {
glog.Fatalf("Port %v is already in use. Please check the flag --default-server-port", n.ports.Default)
}
ic, _ := flags.GetString("ingress-class")
wc, _ := flags.GetString("watch-namespace")
@ -300,20 +331,24 @@ func (n *NGINXController) OverrideFlags(flags *pflag.FlagSet) {
}
flags.Set("ingress-class", ic)
n.stats = newStatsCollector(wc, ic, n.binary)
n.stats = newStatsCollector(wc, ic, n.binary, n.ports.Health)
if n.isSSLPassthroughEnabled {
if !isPortAvailable(n.ports.SSLProxy) {
glog.Fatalf("Port %v is already in use. Please check the flag --ssl-passtrough-proxy-port", n.ports.SSLProxy)
}
glog.Info("starting TLS proxy for SSL passthrough")
n.proxy = &proxy{
Default: &server{
Hostname: "localhost",
IP: "127.0.0.1",
Port: 442,
Port: n.ports.SSLProxy,
ProxyProtocol: true,
},
}
listener, err := net.Listen("tcp", ":443")
listener, err := net.Listen("tcp", fmt.Sprintf(":%v", n.ports.HTTPS))
if err != nil {
glog.Fatalf("%v", err)
}
@ -594,6 +629,7 @@ func (n *NGINXController) OnUpdate(ingressCfg ingress.Configuration) error {
IsIPV6Enabled: n.isIPV6Enabled && !cfg.DisableIpv6,
RedirectServers: redirectServers,
IsSSLPassthroughEnabled: n.isSSLPassthroughEnabled,
ListenPorts: n.ports,
}
// We need to extract the endpoints to be used in the fastcgi error handler
@ -651,7 +687,7 @@ func (n NGINXController) Name() string {
// Check returns if the nginx healthz endpoint is returning ok (status code 200)
func (n NGINXController) Check(_ *http.Request) error {
res, err := http.Get(fmt.Sprintf("http://localhost:%v%v", ngxHealthPort, ngxHealthPath))
res, err := http.Get(fmt.Sprintf("http://localhost:%v%v", n.ports.Status, ngxHealthPath))
if err != nil {
return err
}

View file

@ -17,7 +17,9 @@ limitations under the License.
package main
import (
"fmt"
"io/ioutil"
"net"
"os"
"os/exec"
"syscall"
@ -74,3 +76,12 @@ func diff(b1, b2 []byte) ([]byte, error) {
out, _ := exec.Command("diff", "-u", f1.Name(), f2.Name()).CombinedOutput()
return out, nil
}
func isPortAvailable(p int) bool {
ln, err := net.Listen("tcp", fmt.Sprintf(":%v", p))
if err != nil {
return false
}
ln.Close()
return true
}

View file

@ -440,4 +440,16 @@ type TemplateConfig struct {
IsIPV6Enabled bool
IsSSLPassthroughEnabled bool
RedirectServers map[string]string
ListenPorts *ListenPorts
}
// ListenPorts describe the ports required to run the
// NGINX Ingress controller
type ListenPorts struct {
HTTP int
HTTPS int
Status int
Health int
Default int
SSLProxy int
}

View file

@ -175,7 +175,9 @@ func TestTemplateWithData(t *testing.T) {
if err := json.Unmarshal(data, &dat); err != nil {
t.Errorf("unexpected error unmarshalling json: %v", err)
}
if dat.ListenPorts == nil {
dat.ListenPorts = &config.ListenPorts{}
}
tf, err := os.Open(path.Join(pwd, "../../rootfs/etc/nginx/template/nginx.tmpl"))
if err != nil {
t.Errorf("unexpected error reading json file: %v", err)

View file

@ -160,9 +160,9 @@ http {
{{ end }}
{{ if $all.IsSSLPassthroughEnabled }}
# map port 442 to 443 for header X-Forwarded-Port
# map port {{ $all.ListenPorts.SSLProxy }} to 443 for header X-Forwarded-Port
map $pass_server_port $pass_port {
442 443;
{{ $all.ListenPorts.SSLProxy }} 443;
default $pass_server_port;
}
{{ else }}
@ -319,11 +319,11 @@ http {
{{/* Build server redirects (from/to www) */}}
{{ range $hostname, $to := .RedirectServers }}
server {
listen 80{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }};
listen {{ if $all.IsSSLPassthroughEnabled }}442 proxy_protocol{{ else }}443{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}{{ end }} ssl;
listen {{ $all.ListenPorts.HTTP }}{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }};
listen {{ if $all.IsSSLPassthroughEnabled }}{{ $all.ListenPorts.SSLProxy }} proxy_protocol{{ else }}{{ $all.ListenPorts.HTTPS }}{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}{{ end }} ssl;
{{ if $IsIPV6Enabled }}
listen [::]:80{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }};
listen {{ if $all.IsSSLPassthroughEnabled }}[::]:442 proxy_protocol{{ else }}[::]:443{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}{{ end }};
listen [::]:{{ $all.ListenPorts.HTTP }}{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }};
listen [::]:{{ if $all.IsSSLPassthroughEnabled }}{{ $all.ListenPorts.SSLProxy }} proxy_protocol{{ else }}{{ $all.ListenPorts.HTTPS }}{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}{{ end }};
{{ end }}
server_name {{ $hostname }};
return 301 $scheme://{{ $to }}$request_uri;
@ -347,11 +347,11 @@ http {
# default server, used for NGINX healthcheck and access to nginx stats
server {
# Use the port 18080 (random value just to avoid known ports) as default port for nginx.
# Use the port {{ $all.ListenPorts.Status }} (random value just to avoid known ports) as default port for nginx.
# Changing this value requires a change in:
# https://github.com/kubernetes/ingress/blob/master/controllers/nginx/pkg/cmd/controller/nginx.go
listen 18080 default_server reuseport backlog={{ .BacklogSize }};
{{ if $IsIPV6Enabled }}listen [::]:18080 default_server reuseport backlog={{ .BacklogSize }};{{ end }}
listen {{ $all.ListenPorts.Status }} default_server reuseport backlog={{ .BacklogSize }};
{{ if $IsIPV6Enabled }}listen [::]:{{ $all.ListenPorts.Status }} default_server reuseport backlog={{ .BacklogSize }};{{ end }}
set $proxy_upstream_name "-";
location {{ $healthzURI }} {
@ -394,7 +394,7 @@ http {
# default server for services without endpoints
server {
listen 8181;
listen {{ $all.ListenPorts.Default }};
set $proxy_upstream_name "-";
location / {
@ -518,14 +518,15 @@ stream {
{{ define "SERVER" }}
{{ $all := .First }}
{{ $server := .Second }}
listen 80{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}{{ if eq $server.Hostname "_"}} default_server reuseport backlog={{ $all.BacklogSize }}{{end}};
{{ if $all.IsIPV6Enabled }}listen [::]:80{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}{{ if eq $server.Hostname "_"}} default_server reuseport backlog={{ $all.BacklogSize }}{{ end }};{{ end }}
listen {{ $all.ListenPorts.HTTP }}{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}{{ if eq $server.Hostname "_"}} default_server reuseport backlog={{ $all.BacklogSize }}{{end}};
{{ if $all.IsIPV6Enabled }}listen [::]:{{ $all.ListenPorts.HTTP }}{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}{{ if eq $server.Hostname "_"}} default_server reuseport backlog={{ $all.BacklogSize }}{{ end }};{{ end }}
set $proxy_upstream_name "-";
{{/* Listen on 442 because port 443 is used in the TLS sni server */}}
{{/* Listen on {{ $all.ListenPorts.SSLProxy }} because port {{ $all.ListenPorts.HTTPS }} is used in the TLS sni server */}}
{{/* This listener must always have proxy_protocol enabled, because the SNI listener forwards on source IP info in it. */}}
{{ if not (empty $server.SSLCertificate) }}listen {{ if $all.IsSSLPassthroughEnabled }}442 proxy_protocol {{ else }}443{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}{{ end }} {{ if eq $server.Hostname "_"}} default_server reuseport backlog={{ $all.BacklogSize }}{{end}} ssl {{ if $all.Cfg.UseHTTP2 }}http2{{ end }};
{{ if $all.IsIPV6Enabled }}{{ if not (empty $server.SSLCertificate) }}listen {{ if $all.IsSSLPassthroughEnabled }}[::]:442 proxy_protocol{{ else }}[::]:443{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}{{ end }}{{ end }} {{ if eq $server.Hostname "_"}} default_server reuseport backlog={{ $all.BacklogSize }}{{end}} ssl {{ if $all.Cfg.UseHTTP2 }}http2{{ end }};{{ end }} {{/* comment PEM sha is required to detect changes in the generated configuration and force a reload */}}
{{ if not (empty $server.SSLCertificate) }}listen {{ if $all.IsSSLPassthroughEnabled }}{{ $all.ListenPorts.SSLProxy }} proxy_protocol {{ else }}{{ $all.ListenPorts.HTTPS }}{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}{{ end }} {{ if eq $server.Hostname "_"}} default_server reuseport backlog={{ $all.BacklogSize }}{{end}} ssl {{ if $all.Cfg.UseHTTP2 }}http2{{ end }};
{{ if $all.IsIPV6Enabled }}{{ if not (empty $server.SSLCertificate) }}listen [::]:{{ if $all.IsSSLPassthroughEnabled }}{{ $all.ListenPorts.SSLProxy }} proxy_protocol{{ else }}{{ $all.ListenPorts.HTTPS }}{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}{{ end }}{{ end }} {{ if eq $server.Hostname "_"}} default_server reuseport backlog={{ $all.BacklogSize }}{{end}} ssl {{ if $all.Cfg.UseHTTP2 }}http2{{ end }};{{ end }}
{{/* comment PEM sha is required to detect changes in the generated configuration and force a reload */}}
# PEM sha: {{ $server.SSLPemChecksum }}
ssl_certificate {{ $server.SSLCertificate }};
ssl_certificate_key {{ $server.SSLCertificate }};
@ -708,7 +709,7 @@ stream {
{{ end }}
{{ if eq $server.Hostname "_" }}
# health checks in cloud providers require the use of port 80
# health checks in cloud providers require the use of port {{ $all.ListenPorts.HTTP }}
location {{ $all.HealthzURI }} {
access_log off;
return 200;

View file

@ -569,13 +569,13 @@ func (ic *GenericController) getDefaultUpstream() *ingress.Backend {
svcObj, svcExists, err := ic.svcLister.Store.GetByKey(svcKey)
if err != nil {
glog.Warningf("unexpected error searching the default backend %v: %v", ic.cfg.DefaultService, err)
upstream.Endpoints = append(upstream.Endpoints, newDefaultServer())
upstream.Endpoints = append(upstream.Endpoints, ic.cfg.Backend.DefaultEndpoint())
return upstream
}
if !svcExists {
glog.Warningf("service %v does not exist", svcKey)
upstream.Endpoints = append(upstream.Endpoints, newDefaultServer())
upstream.Endpoints = append(upstream.Endpoints, ic.cfg.Backend.DefaultEndpoint())
return upstream
}
@ -583,7 +583,7 @@ func (ic *GenericController) getDefaultUpstream() *ingress.Backend {
endps := ic.getEndpoints(svc, &svc.Spec.Ports[0], api.ProtocolTCP, &healthcheck.Upstream{})
if len(endps) == 0 {
glog.Warningf("service %v does not have any active endpoints", svcKey)
endps = []ingress.Endpoint{newDefaultServer()}
endps = []ingress.Endpoint{ic.cfg.Backend.DefaultEndpoint()}
}
upstream.Service = svc
@ -760,7 +760,7 @@ func (ic *GenericController) getBackendServers() ([]*ingress.Backend, []*ingress
for _, value := range upstreams {
if len(value.Endpoints) == 0 {
glog.V(3).Infof("upstream %v does not have any active endpoints. Using default backend", value.Name)
value.Endpoints = append(value.Endpoints, newDefaultServer())
value.Endpoints = append(value.Endpoints, ic.cfg.Backend.DefaultEndpoint())
}
aUpstreams = append(aUpstreams, value)
}

View file

@ -90,11 +90,11 @@ func NewIngressController(backend ingress.Controller) *GenericController {
`Force namespace isolation. This flag is required to avoid the reference of secrets or
configmaps located in a different namespace than the specified in the flag --watch-namespace.`)
UpdateStatusOnShutdown = flags.Bool("update-status-on-shutdown", true, `Indicates if the
updateStatusOnShutdown = flags.Bool("update-status-on-shutdown", true, `Indicates if the
ingress controller should update the Ingress status IP/hostname when the controller
is being stopped. Default is true`)
SortBackends = flags.Bool("sort-backends", false,
sortBackends = flags.Bool("sort-backends", false,
`Defines if backends and it's endpoints should be sorted`)
)
@ -175,8 +175,8 @@ func NewIngressController(backend ingress.Controller) *GenericController {
PublishService: *publishSvc,
Backend: backend,
ForceNamespaceIsolation: *forceIsolation,
UpdateStatusOnShutdown: *UpdateStatusOnShutdown,
SortBackends: *SortBackends,
UpdateStatusOnShutdown: *updateStatusOnShutdown,
SortBackends: *sortBackends,
}
ic := newIngressController(config)

View file

@ -28,11 +28,6 @@ import (
// DeniedKeyName name of the key that contains the reason to deny a location
const DeniedKeyName = "Denied"
// newDefaultServer return an BackendServer to be use as default server that returns 503.
func newDefaultServer() ingress.Endpoint {
return ingress.Endpoint{Address: "127.0.0.1", Port: "8181", Target: &api.ObjectReference{}}
}
// newUpstream creates an upstream without servers.
func newUpstream(name string) *ingress.Backend {
return &ingress.Backend{

View file

@ -98,6 +98,10 @@ type Controller interface {
// This allows custom implementations
// If the function returns nil the standard functions will be executed.
UpdateIngressStatus(*extensions.Ingress) []api.LoadBalancerIngress
// DefaultEndpoint returns the Endpoint to use as default when the
// referenced service does not exists. This should return the content
// of to the default backend
DefaultEndpoint() Endpoint
}
// StoreLister returns the configured stores for ingresses, services,

View file

@ -110,3 +110,12 @@ func (n DummyController) DefaultIngressClass() string {
func (n DummyController) UpdateIngressStatus(*extensions.Ingress) []api.LoadBalancerIngress {
return nil
}
// DefaultEndpoint returns the default endpoint to be use as default server that returns 404.
func (n DummyController) DefaultEndpoint() ingress.Endpoint {
return ingress.Endpoint{
Address: "127.0.0.1",
Port: "8181",
Target: &api.ObjectReference{},
}
}