2018-07-07 17:46:18 +00:00
|
|
|
/*
|
|
|
|
Copyright 2017 The Kubernetes Authors.
|
|
|
|
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
you may not use this file except in compliance with the License.
|
|
|
|
You may obtain a copy of the License at
|
|
|
|
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
See the License for the specific language governing permissions and
|
|
|
|
limitations under the License.
|
|
|
|
*/
|
|
|
|
|
|
|
|
package metric
|
|
|
|
|
|
|
|
import (
|
|
|
|
"os"
|
2019-06-28 22:29:58 +00:00
|
|
|
"sync/atomic"
|
2018-07-30 13:56:09 +00:00
|
|
|
"time"
|
2018-07-07 17:46:18 +00:00
|
|
|
|
|
|
|
"github.com/prometheus/client_golang/prometheus"
|
2020-08-08 23:31:02 +00:00
|
|
|
"k8s.io/klog/v2"
|
2019-06-28 22:29:58 +00:00
|
|
|
|
2018-09-22 17:25:01 +00:00
|
|
|
"k8s.io/apimachinery/pkg/util/sets"
|
2018-07-07 17:46:18 +00:00
|
|
|
"k8s.io/ingress-nginx/internal/ingress/metric/collectors"
|
2022-07-22 00:32:48 +00:00
|
|
|
"k8s.io/ingress-nginx/pkg/apis/ingress"
|
2018-07-07 17:46:18 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
// Collector defines the interface for a metric collector
|
|
|
|
type Collector interface {
|
|
|
|
ConfigSuccess(uint64, bool)
|
|
|
|
|
|
|
|
IncReloadCount()
|
|
|
|
IncReloadErrorCount()
|
|
|
|
|
2021-11-02 17:54:34 +00:00
|
|
|
SetAdmissionMetrics(float64, float64, float64, float64, float64, float64)
|
|
|
|
|
2019-03-10 22:12:33 +00:00
|
|
|
OnStartedLeading(string)
|
|
|
|
OnStoppedLeading(string)
|
|
|
|
|
2019-02-21 19:45:21 +00:00
|
|
|
IncCheckCount(string, string)
|
|
|
|
IncCheckErrorCount(string, string)
|
2023-01-16 12:22:51 +00:00
|
|
|
IncOrphanIngress(string, string, string)
|
|
|
|
DecOrphanIngress(string, string, string)
|
2019-02-21 19:45:21 +00:00
|
|
|
|
2024-04-08 19:32:21 +00:00
|
|
|
RemoveMetrics(ingresses, certificates []string)
|
2018-07-07 17:46:18 +00:00
|
|
|
|
|
|
|
SetSSLExpireTime([]*ingress.Server)
|
Add a certificate info metric (#8253)
When the ingress controller loads certificates (new ones or following a
secret update), it performs a series of check to ensure its validity.
In our systems, we detected a case where, when the secret object is
compromised, for example when the certificate does not match the secret
key, different pods of the ingress controller are serving a different
version of the certificate.
This behaviour is due to the cache mechanism of the ingress controller,
keeping the last known certificate in case of corruption. When this
happens, old ingress-controller pods will keep serving the old one,
while new pods, by failing to load the corrupted certificates, would
use the default certificate, causing invalid certificates for its
clients.
This generates a random error on the client side, depending on the
actual pod instance it reaches.
In order to allow detecting occurences of those situations, add a metric
to expose, for all ingress controlller pods, detailed informations of
the currently loaded certificate.
This will, for example, allow setting an alert when there is a
certificate discrepency across all ingress controller pods using a query
similar to `sum(nginx_ingress_controller_ssl_certificate_info{host="name.tld"})by(serial_number)`
This also allows to catch other exceptions loading certificates (failing
to load the certificate from the k8s API, ...
Co-authored-by: Daniel Ricart <danielricart@users.noreply.github.com>
Co-authored-by: Daniel Ricart <danielricart@users.noreply.github.com>
2022-02-24 15:08:32 +00:00
|
|
|
SetSSLInfo(servers []*ingress.Server)
|
2018-07-07 17:46:18 +00:00
|
|
|
|
2018-09-22 17:25:01 +00:00
|
|
|
// SetHosts sets the hostnames that are being served by the ingress controller
|
2023-02-16 14:05:48 +00:00
|
|
|
SetHosts(set sets.Set[string])
|
2018-09-22 17:25:01 +00:00
|
|
|
|
2021-11-02 17:54:34 +00:00
|
|
|
Start(string)
|
|
|
|
Stop(string)
|
2018-07-07 17:46:18 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
type collector struct {
|
|
|
|
nginxStatus collectors.NGINXStatusCollector
|
|
|
|
nginxProcess collectors.NGINXProcessCollector
|
|
|
|
|
2021-11-02 17:54:34 +00:00
|
|
|
ingressController *collectors.Controller
|
|
|
|
admissionController *collectors.AdmissionCollector
|
2018-07-07 17:46:18 +00:00
|
|
|
|
|
|
|
socket *collectors.SocketCollector
|
|
|
|
|
|
|
|
registry *prometheus.Registry
|
|
|
|
}
|
|
|
|
|
|
|
|
// NewCollector creates a new metric collector the for ingress controller
|
2023-04-11 08:01:18 +00:00
|
|
|
func NewCollector(metricsPerHost, reportStatusClasses bool, registry *prometheus.Registry, ingressclass string, buckets collectors.HistogramBuckets, excludedSocketMetrics []string) (Collector, error) {
|
2018-07-07 17:46:18 +00:00
|
|
|
podNamespace := os.Getenv("POD_NAMESPACE")
|
|
|
|
if podNamespace == "" {
|
|
|
|
podNamespace = "default"
|
|
|
|
}
|
|
|
|
|
|
|
|
podName := os.Getenv("POD_NAME")
|
|
|
|
|
2021-08-21 20:42:00 +00:00
|
|
|
nc, err := collectors.NewNGINXStatus(podName, podNamespace, ingressclass)
|
2018-07-07 17:46:18 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
2021-08-21 20:42:00 +00:00
|
|
|
pc, err := collectors.NewNGINXProcess(podName, podNamespace, ingressclass)
|
2018-07-07 17:46:18 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
2023-04-11 08:01:18 +00:00
|
|
|
s, err := collectors.NewSocketCollector(podName, podNamespace, ingressclass, metricsPerHost, reportStatusClasses, buckets, excludedSocketMetrics)
|
2018-07-07 17:46:18 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
2021-08-21 20:42:00 +00:00
|
|
|
ic := collectors.NewController(podName, podNamespace, ingressclass)
|
2018-07-07 17:46:18 +00:00
|
|
|
|
2021-11-02 17:54:34 +00:00
|
|
|
am := collectors.NewAdmissionCollector(podName, podNamespace, ingressclass)
|
|
|
|
|
2018-07-07 17:46:18 +00:00
|
|
|
return Collector(&collector{
|
|
|
|
nginxStatus: nc,
|
|
|
|
nginxProcess: pc,
|
|
|
|
|
2021-11-02 17:54:34 +00:00
|
|
|
admissionController: am,
|
|
|
|
ingressController: ic,
|
2018-07-07 17:46:18 +00:00
|
|
|
|
|
|
|
socket: s,
|
|
|
|
|
|
|
|
registry: registry,
|
|
|
|
}), nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *collector) ConfigSuccess(hash uint64, success bool) {
|
|
|
|
c.ingressController.ConfigSuccess(hash, success)
|
|
|
|
}
|
|
|
|
|
2023-08-31 07:36:48 +00:00
|
|
|
func (c *collector) IncCheckCount(namespace, name string) {
|
2019-02-21 19:45:21 +00:00
|
|
|
c.ingressController.IncCheckCount(namespace, name)
|
|
|
|
}
|
|
|
|
|
2023-08-31 07:36:48 +00:00
|
|
|
func (c *collector) IncCheckErrorCount(namespace, name string) {
|
2019-02-21 19:45:21 +00:00
|
|
|
c.ingressController.IncCheckErrorCount(namespace, name)
|
|
|
|
}
|
|
|
|
|
2018-07-07 17:46:18 +00:00
|
|
|
func (c *collector) IncReloadCount() {
|
|
|
|
c.ingressController.IncReloadCount()
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *collector) IncReloadErrorCount() {
|
|
|
|
c.ingressController.IncReloadErrorCount()
|
|
|
|
}
|
|
|
|
|
2024-04-08 19:32:21 +00:00
|
|
|
func (c *collector) RemoveMetrics(ingresses, certificates []string) {
|
2018-07-07 17:46:18 +00:00
|
|
|
c.socket.RemoveMetrics(ingresses, c.registry)
|
2024-04-08 19:32:21 +00:00
|
|
|
c.ingressController.RemoveMetrics(certificates, c.registry)
|
2018-07-07 17:46:18 +00:00
|
|
|
}
|
|
|
|
|
2021-11-02 17:54:34 +00:00
|
|
|
func (c *collector) Start(admissionStatus string) {
|
2018-07-07 17:46:18 +00:00
|
|
|
c.registry.MustRegister(c.nginxStatus)
|
|
|
|
c.registry.MustRegister(c.nginxProcess)
|
2021-11-02 17:54:34 +00:00
|
|
|
if admissionStatus != "" {
|
|
|
|
c.registry.MustRegister(c.admissionController)
|
|
|
|
}
|
2018-07-07 17:46:18 +00:00
|
|
|
c.registry.MustRegister(c.ingressController)
|
|
|
|
c.registry.MustRegister(c.socket)
|
|
|
|
|
2018-07-30 13:56:09 +00:00
|
|
|
// the default nginx.conf does not contains
|
|
|
|
// a server section with the status port
|
|
|
|
go func() {
|
|
|
|
time.Sleep(5 * time.Second)
|
|
|
|
c.nginxStatus.Start()
|
|
|
|
}()
|
2018-07-07 17:46:18 +00:00
|
|
|
go c.nginxProcess.Start()
|
|
|
|
go c.socket.Start()
|
|
|
|
}
|
|
|
|
|
2021-11-02 17:54:34 +00:00
|
|
|
func (c *collector) Stop(admissionStatus string) {
|
2018-07-07 17:46:18 +00:00
|
|
|
c.registry.Unregister(c.nginxStatus)
|
|
|
|
c.registry.Unregister(c.nginxProcess)
|
2021-11-02 17:54:34 +00:00
|
|
|
if admissionStatus != "" {
|
|
|
|
c.registry.Unregister(c.admissionController)
|
|
|
|
}
|
2018-07-07 17:46:18 +00:00
|
|
|
c.registry.Unregister(c.ingressController)
|
|
|
|
c.registry.Unregister(c.socket)
|
|
|
|
|
|
|
|
c.nginxStatus.Stop()
|
|
|
|
c.nginxProcess.Stop()
|
|
|
|
c.socket.Stop()
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *collector) SetSSLExpireTime(servers []*ingress.Server) {
|
2019-06-28 22:29:58 +00:00
|
|
|
if !isLeader() {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2020-09-27 20:32:40 +00:00
|
|
|
klog.V(2).InfoS("Updating ssl expiration metrics")
|
2018-07-07 17:46:18 +00:00
|
|
|
c.ingressController.SetSSLExpireTime(servers)
|
|
|
|
}
|
2018-09-22 17:25:01 +00:00
|
|
|
|
Add a certificate info metric (#8253)
When the ingress controller loads certificates (new ones or following a
secret update), it performs a series of check to ensure its validity.
In our systems, we detected a case where, when the secret object is
compromised, for example when the certificate does not match the secret
key, different pods of the ingress controller are serving a different
version of the certificate.
This behaviour is due to the cache mechanism of the ingress controller,
keeping the last known certificate in case of corruption. When this
happens, old ingress-controller pods will keep serving the old one,
while new pods, by failing to load the corrupted certificates, would
use the default certificate, causing invalid certificates for its
clients.
This generates a random error on the client side, depending on the
actual pod instance it reaches.
In order to allow detecting occurences of those situations, add a metric
to expose, for all ingress controlller pods, detailed informations of
the currently loaded certificate.
This will, for example, allow setting an alert when there is a
certificate discrepency across all ingress controller pods using a query
similar to `sum(nginx_ingress_controller_ssl_certificate_info{host="name.tld"})by(serial_number)`
This also allows to catch other exceptions loading certificates (failing
to load the certificate from the k8s API, ...
Co-authored-by: Daniel Ricart <danielricart@users.noreply.github.com>
Co-authored-by: Daniel Ricart <danielricart@users.noreply.github.com>
2022-02-24 15:08:32 +00:00
|
|
|
func (c *collector) SetSSLInfo(servers []*ingress.Server) {
|
|
|
|
klog.V(2).Infof("Updating ssl certificate info metrics")
|
|
|
|
c.ingressController.SetSSLInfo(servers)
|
|
|
|
}
|
|
|
|
|
2023-08-31 07:36:48 +00:00
|
|
|
func (c *collector) IncOrphanIngress(namespace, name, orphanityType string) {
|
2023-01-16 12:22:51 +00:00
|
|
|
c.ingressController.IncOrphanIngress(namespace, name, orphanityType)
|
|
|
|
}
|
|
|
|
|
2023-08-31 07:36:48 +00:00
|
|
|
func (c *collector) DecOrphanIngress(namespace, name, orphanityType string) {
|
2023-01-16 12:22:51 +00:00
|
|
|
c.ingressController.DecOrphanIngress(namespace, name, orphanityType)
|
|
|
|
}
|
|
|
|
|
2023-02-16 14:05:48 +00:00
|
|
|
func (c *collector) SetHosts(hosts sets.Set[string]) {
|
2018-09-22 17:25:01 +00:00
|
|
|
c.socket.SetHosts(hosts)
|
|
|
|
}
|
2019-03-10 22:12:33 +00:00
|
|
|
|
2023-08-31 07:36:48 +00:00
|
|
|
func (c *collector) SetAdmissionMetrics(testedIngressLength, testedIngressTime, renderingIngressLength, renderingIngressTime, testedConfigurationSize, admissionTime float64) {
|
2021-11-02 17:54:34 +00:00
|
|
|
c.admissionController.SetAdmissionMetrics(
|
|
|
|
testedIngressLength,
|
|
|
|
testedIngressTime,
|
|
|
|
renderingIngressLength,
|
|
|
|
renderingIngressTime,
|
|
|
|
testedConfigurationSize,
|
|
|
|
admissionTime,
|
|
|
|
)
|
|
|
|
}
|
|
|
|
|
2019-03-11 16:20:41 +00:00
|
|
|
// OnStartedLeading indicates the pod was elected as the leader
|
2019-03-10 22:12:33 +00:00
|
|
|
func (c *collector) OnStartedLeading(electionID string) {
|
2019-06-28 22:29:58 +00:00
|
|
|
setLeader(true)
|
2019-03-10 22:12:33 +00:00
|
|
|
c.ingressController.OnStartedLeading(electionID)
|
|
|
|
}
|
|
|
|
|
2019-03-11 16:20:41 +00:00
|
|
|
// OnStoppedLeading indicates the pod stopped being the leader
|
2019-03-10 22:12:33 +00:00
|
|
|
func (c *collector) OnStoppedLeading(electionID string) {
|
2019-06-28 22:29:58 +00:00
|
|
|
setLeader(false)
|
2019-03-10 22:12:33 +00:00
|
|
|
c.ingressController.OnStoppedLeading(electionID)
|
Add a certificate info metric (#8253)
When the ingress controller loads certificates (new ones or following a
secret update), it performs a series of check to ensure its validity.
In our systems, we detected a case where, when the secret object is
compromised, for example when the certificate does not match the secret
key, different pods of the ingress controller are serving a different
version of the certificate.
This behaviour is due to the cache mechanism of the ingress controller,
keeping the last known certificate in case of corruption. When this
happens, old ingress-controller pods will keep serving the old one,
while new pods, by failing to load the corrupted certificates, would
use the default certificate, causing invalid certificates for its
clients.
This generates a random error on the client side, depending on the
actual pod instance it reaches.
In order to allow detecting occurences of those situations, add a metric
to expose, for all ingress controlller pods, detailed informations of
the currently loaded certificate.
This will, for example, allow setting an alert when there is a
certificate discrepency across all ingress controller pods using a query
similar to `sum(nginx_ingress_controller_ssl_certificate_info{host="name.tld"})by(serial_number)`
This also allows to catch other exceptions loading certificates (failing
to load the certificate from the k8s API, ...
Co-authored-by: Daniel Ricart <danielricart@users.noreply.github.com>
Co-authored-by: Daniel Ricart <danielricart@users.noreply.github.com>
2022-02-24 15:08:32 +00:00
|
|
|
c.ingressController.RemoveAllSSLMetrics(c.registry)
|
2019-03-10 22:12:33 +00:00
|
|
|
}
|
2019-06-28 22:29:58 +00:00
|
|
|
|
2023-08-31 07:36:48 +00:00
|
|
|
var currentLeader uint32
|
2019-06-28 22:29:58 +00:00
|
|
|
|
|
|
|
func setLeader(leader bool) {
|
|
|
|
var i uint32
|
|
|
|
if leader {
|
|
|
|
i = 1
|
|
|
|
}
|
|
|
|
|
|
|
|
atomic.StoreUint32(¤tLeader, i)
|
|
|
|
}
|
|
|
|
|
|
|
|
func isLeader() bool {
|
|
|
|
return atomic.LoadUint32(¤tLeader) != 0
|
|
|
|
}
|