mirror of
https://github.com/prometheus-operator/prometheus-operator.git
synced 2025-04-21 03:38:43 +00:00
Merge pull request #6127 from simonpasquier/drop-krp-reload-cert
chore: use apiserver controller to reload certs
This commit is contained in:
Simon Pasquier
GitHub
commit
0d5dd20f00
3 changed files with 98 additions and 52 deletions
3
go.mod
3
go.mod
|
|
@ -6,7 +6,6 @@ require (
|
|||
github.com/alecthomas/kingpin/v2 v2.4.0
|
||||
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2
|
||||
github.com/blang/semver/v4 v4.0.0
|
||||
github.com/brancz/kube-rbac-proxy v0.15.0
|
||||
github.com/cespare/xxhash/v2 v2.2.0
|
||||
github.com/distribution/reference v0.5.0
|
||||
github.com/evanphx/json-patch/v5 v5.7.0
|
||||
|
|
@ -36,6 +35,7 @@ require (
|
|||
k8s.io/api v0.28.4
|
||||
k8s.io/apiextensions-apiserver v0.28.4
|
||||
k8s.io/apimachinery v0.28.4
|
||||
k8s.io/apiserver v0.28.4
|
||||
k8s.io/client-go v0.28.4
|
||||
k8s.io/component-base v0.28.4
|
||||
k8s.io/klog/v2 v2.110.1
|
||||
|
|
@ -49,6 +49,7 @@ require (
|
|||
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
|
||||
github.com/docker/distribution v2.8.3+incompatible // indirect
|
||||
github.com/golang-jwt/jwt/v5 v5.0.0 // indirect
|
||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
|
||||
github.com/google/gnostic-models v0.6.8 // indirect
|
||||
github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
|
||||
github.com/mitchellh/go-ps v1.0.0 // indirect
|
||||
|
|
|
|||
6
go.sum
6
go.sum
|
|
@ -76,8 +76,6 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
|||
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
||||
github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
|
||||
github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
|
||||
github.com/brancz/kube-rbac-proxy v0.15.0 h1:ZBNVyiiV1vmyp0/A1jxkqz0U3FHJLdLL93eLk/o2EPI=
|
||||
github.com/brancz/kube-rbac-proxy v0.15.0/go.mod h1:ne9y1bsoAwZP4welF93ZcI3j2m/qX7gZRCYkh3fd0VY=
|
||||
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
||||
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
|
||||
|
|
@ -237,6 +235,8 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfU
|
|||
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
|
||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
|
||||
|
|
@ -947,6 +947,8 @@ k8s.io/apiextensions-apiserver v0.28.4 h1:AZpKY/7wQ8n+ZYDtNHbAJBb+N4AXXJvyZx6ww6
|
|||
k8s.io/apiextensions-apiserver v0.28.4/go.mod h1:pgQIZ1U8eJSMQcENew/0ShUTlePcSGFq6dxSxf2mwPM=
|
||||
k8s.io/apimachinery v0.28.4 h1:zOSJe1mc+GxuMnFzD4Z/U1wst50X28ZNsn5bhgIIao8=
|
||||
k8s.io/apimachinery v0.28.4/go.mod h1:wI37ncBvfAoswfq626yPTe6Bz1c22L7uaJ8dho83mgg=
|
||||
k8s.io/apiserver v0.28.4 h1:BJXlaQbAU/RXYX2lRz+E1oPe3G3TKlozMMCZWu5GMgg=
|
||||
k8s.io/apiserver v0.28.4/go.mod h1:Idq71oXugKZoVGUUL2wgBCTHbUR+FYTWa4rq9j4n23w=
|
||||
k8s.io/client-go v0.28.4 h1:Np5ocjlZcTrkyRJ3+T3PkXDpe4UpatQxj85+xjaD2wY=
|
||||
k8s.io/client-go v0.28.4/go.mod h1:0VDZFpgoZfelyP5Wqu0/r/TRYcLYuJ2U1KEeoaPa1N4=
|
||||
k8s.io/component-base v0.28.4 h1:c/iQLWPdUgI90O+T9TeECg8o7N3YJTiuz2sKxILYcYo=
|
||||
|
|
|
|||
|
|
@ -17,7 +17,6 @@ package server
|
|||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"flag"
|
||||
"fmt"
|
||||
stdlog "log"
|
||||
|
|
@ -27,9 +26,9 @@ import (
|
|||
"path/filepath"
|
||||
"time"
|
||||
|
||||
rbacproxytls "github.com/brancz/kube-rbac-proxy/pkg/tls"
|
||||
"github.com/go-kit/log"
|
||||
"github.com/go-kit/log/level"
|
||||
"k8s.io/apiserver/pkg/server/dynamiccertificates"
|
||||
kflag "k8s.io/component-base/cli/flag"
|
||||
|
||||
"github.com/prometheus-operator/prometheus-operator/pkg/operator"
|
||||
|
|
@ -43,6 +42,7 @@ const (
|
|||
func DefaultConfig(listenAddress string, enableTLS bool) Config {
|
||||
return Config{
|
||||
ListenAddress: listenAddress,
|
||||
|
||||
// Mitigate CVE-2023-44487 by disabling HTTP2 by default until the Go
|
||||
// standard library and golang.org/x/net are fully fixed.
|
||||
// Right now, it is possible for authenticated and unauthenticated users to
|
||||
|
|
@ -52,6 +52,7 @@ func DefaultConfig(listenAddress string, enableTLS bool) Config {
|
|||
// * https://github.com/kubernetes/kubernetes/issues/121197
|
||||
// * https://github.com/golang/go/issues/63417#issuecomment-1758858612
|
||||
EnableHTTP2: false,
|
||||
|
||||
TLSConfig: TLSConfig{
|
||||
Enabled: enableTLS,
|
||||
CertFile: filepath.Join(defaultTLSDir, "tls.crt"),
|
||||
|
|
@ -103,6 +104,7 @@ type TLSConfig struct {
|
|||
}
|
||||
|
||||
// Convert returns a *tls.Config from the given TLSConfig.
|
||||
// It returns nil when TLS isn't enabled/configured.
|
||||
func (tc *TLSConfig) Convert(logger log.Logger) (*tls.Config, error) {
|
||||
if logger == nil {
|
||||
logger = log.NewNopLogger()
|
||||
|
|
@ -155,23 +157,13 @@ func (tc *TLSConfig) Convert(logger log.Logger) (*tls.Config, error) {
|
|||
info, err := os.Stat(tc.ClientCAFile)
|
||||
switch {
|
||||
case err != nil:
|
||||
level.Warn(logger).Log("msg", "server TLS client verification disabled", "err", err, "client_ca_file", tc.ClientCAFile)
|
||||
level.Warn(logger).Log("msg", "server TLS client verification disabled", "client_ca_file", tc.ClientCAFile, "err", err)
|
||||
|
||||
case !info.Mode().IsRegular():
|
||||
level.Warn(logger).Log("msg", "server TLS client verification disabled", "client_ca_file", tc.ClientCAFile, "file_mode", info.Mode().String())
|
||||
|
||||
default:
|
||||
caPEM, err := os.ReadFile(tc.ClientCAFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading client CA %q: %w", tc.ClientCAFile, err)
|
||||
}
|
||||
|
||||
certPool := x509.NewCertPool()
|
||||
if !certPool.AppendCertsFromPEM(caPEM) {
|
||||
return nil, fmt.Errorf("client CA %q: failed to parse certificate", tc.ClientCAFile)
|
||||
}
|
||||
|
||||
tlsCfg.ClientCAs = certPool
|
||||
// The client CA content will be checked by the cert controller.
|
||||
tlsCfg.ClientAuth = tls.RequireAndVerifyClientCert
|
||||
level.Info(logger).Log("msg", "server TLS client verification enabled", "client_ca_file", tc.ClientCAFile)
|
||||
}
|
||||
|
|
@ -181,22 +173,94 @@ func (tc *TLSConfig) Convert(logger log.Logger) (*tls.Config, error) {
|
|||
|
||||
// Server is a web server.
|
||||
type Server struct {
|
||||
logger log.Logger
|
||||
srv *http.Server
|
||||
logger log.Logger
|
||||
|
||||
listener net.Listener
|
||||
cfg *Config
|
||||
srv *http.Server
|
||||
runners []func(context.Context)
|
||||
|
||||
cfg *Config
|
||||
}
|
||||
|
||||
// NewServer initializes a web server with the given handler (typically an http.MuxServe).
|
||||
func NewServer(logger log.Logger, c *Config, handler http.Handler) (*Server, error) {
|
||||
listener, err := net.Listen("tcp", c.ListenAddress)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
tlsConfig, err := c.TLSConfig.Convert(logger)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create TLS configuration: %w", err)
|
||||
}
|
||||
|
||||
listener, err := net.Listen("tcp", c.ListenAddress)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
var runners []func(context.Context)
|
||||
if tlsConfig != nil {
|
||||
var (
|
||||
certController *dynamiccertificates.DynamicServingCertificateController
|
||||
clientCA dynamiccertificates.CAContentProvider
|
||||
servingCert dynamiccertificates.CertKeyContentProvider
|
||||
)
|
||||
|
||||
if tlsConfig.ClientAuth == tls.RequireAndVerifyClientCert {
|
||||
clientCA, err = dynamiccertificates.NewDynamicCAContentFromFile("clientCA", c.TLSConfig.ClientCAFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to load client CA certificate: %w", err)
|
||||
}
|
||||
|
||||
if err = clientCA.(*dynamiccertificates.DynamicFileCAContent).RunOnce(context.Background()); err != nil {
|
||||
return nil, fmt.Errorf("failed to sync client CA certificate: %w", err)
|
||||
}
|
||||
|
||||
runners = append(runners, func(ctx context.Context) {
|
||||
clientCA.(*dynamiccertificates.DynamicFileCAContent).Run(ctx, 1)
|
||||
})
|
||||
}
|
||||
|
||||
if c.TLSConfig.CertFile != "" && c.TLSConfig.KeyFile != "" {
|
||||
servingCert, err = dynamiccertificates.NewDynamicServingContentFromFiles("servingCert", c.TLSConfig.CertFile, c.TLSConfig.KeyFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to load serving certificate and key: %w", err)
|
||||
}
|
||||
|
||||
if err = servingCert.(*dynamiccertificates.DynamicCertKeyPairContent).RunOnce(context.Background()); err != nil {
|
||||
return nil, fmt.Errorf("failed to sync serving certificate: %w", err)
|
||||
}
|
||||
|
||||
runners = append(runners, func(ctx context.Context) {
|
||||
servingCert.(*dynamiccertificates.DynamicCertKeyPairContent).Run(ctx, 1)
|
||||
})
|
||||
}
|
||||
|
||||
certController = dynamiccertificates.NewDynamicServingCertificateController(
|
||||
tlsConfig,
|
||||
clientCA,
|
||||
servingCert,
|
||||
nil,
|
||||
nil,
|
||||
)
|
||||
|
||||
if clientCA != nil {
|
||||
clientCA.AddListener(certController)
|
||||
}
|
||||
|
||||
if servingCert != nil {
|
||||
servingCert.AddListener(certController)
|
||||
}
|
||||
|
||||
// Ensure that the configuration is valid.
|
||||
err = certController.RunOnce()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to sync certificates: %w", err)
|
||||
}
|
||||
|
||||
runners = append(runners, func(ctx context.Context) {
|
||||
certController.Run(1, ctx.Done())
|
||||
})
|
||||
|
||||
tlsConfig.GetConfigForClient = certController.GetConfigForClient
|
||||
|
||||
listener = tls.NewListener(listener, tlsConfig)
|
||||
}
|
||||
|
||||
srv := &http.Server{
|
||||
|
|
@ -218,6 +282,7 @@ func NewServer(logger log.Logger, c *Config, handler http.Handler) (*Server, err
|
|||
logger: logger,
|
||||
srv: srv,
|
||||
listener: listener,
|
||||
runners: runners,
|
||||
cfg: c,
|
||||
}, nil
|
||||
}
|
||||
|
|
@ -225,39 +290,17 @@ func NewServer(logger log.Logger, c *Config, handler http.Handler) (*Server, err
|
|||
// Serve starts the web server. It will block until the server is shutted down
|
||||
// or an error occurs.
|
||||
func (s *Server) Serve(ctx context.Context) error {
|
||||
for _, r := range s.runners {
|
||||
go r(ctx)
|
||||
}
|
||||
|
||||
if s.srv.TLSConfig == nil {
|
||||
level.Info(s.logger).Log("msg", "starting insecure server", "address", s.listener.Addr().String())
|
||||
if err := s.srv.Serve(s.listener); err != http.ErrServerClosed {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
} else {
|
||||
level.Info(s.logger).Log("msg", "starting secure server", "address", s.listener.Addr().String(), "http2", s.cfg.EnableHTTP2)
|
||||
}
|
||||
|
||||
r, err := rbacproxytls.NewCertReloader(
|
||||
s.cfg.TLSConfig.CertFile,
|
||||
s.cfg.TLSConfig.KeyFile,
|
||||
s.cfg.TLSConfig.ReloadInterval,
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to initialize certificate reloader: %w", err)
|
||||
}
|
||||
|
||||
s.srv.TLSConfig.GetCertificate = r.GetCertificate
|
||||
go func() {
|
||||
for {
|
||||
// r.Watch will wait ReloadInterval, so this is not
|
||||
// a hot loop
|
||||
if err := r.Watch(ctx); err != nil {
|
||||
level.Warn(s.logger).Log(
|
||||
"msg", "error watching certificate reloader",
|
||||
"err", err)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
level.Info(s.logger).Log("msg", "starting secure server", "address", s.listener.Addr().String(), "http2", s.cfg.EnableHTTP2)
|
||||
if err := s.srv.ServeTLS(s.listener, "", ""); err != http.ErrServerClosed {
|
||||
if err := s.srv.Serve(s.listener); err != http.ErrServerClosed {
|
||||
return err
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue