mirrors/kyverno
1
0
Fork 0
mirror of https://github.com/kyverno/kyverno.git synced 2024-12-15 17:51:20 +00:00
Code Activity

feat: gracefull certificates rotation support (#3890)

Browse source 
* refactor: remove deployment hash on certs secrets

Signed-off-by: Charles-Edouard Brétéché <charled.breteche@gmail.com>

* feat: add label on kyverno webhooks

Signed-off-by: Charles-Edouard Brétéché <charled.breteche@gmail.com>

* feat: implement update ca bundle

Signed-off-by: Charles-Edouard Brétéché <charled.breteche@gmail.com>

* test: set very low validity and expiration intervals

Signed-off-by: Charles-Edouard Brétéché <charled.breteche@gmail.com>

* fix: writing secret

Signed-off-by: Charles-Edouard Brétéché <charled.breteche@gmail.com>

* add renew ca

Signed-off-by: Charles-Edouard Brétéché <charled.breteche@gmail.com>

* decouple ca and tls validity duration

Signed-off-by: Charles-Edouard Brétéché <charled.breteche@gmail.com>

* refactored code, everything is in place to finalize implementation

Signed-off-by: Charles-Edouard Brétéché <charled.breteche@gmail.com>

* use real validity periods

Signed-off-by: Charles-Edouard Brétéché <charled.breteche@gmail.com>

Co-authored-by: Vyankatesh Kudtarkar <vyankateshkd@gmail.com>
This commit is contained in:
Charles-Edouard Brétéché 2022-05-12 16:07:25 +02:00 committed by GitHub
parent c15ad0c520
commit 97cf1b3e95
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 352 additions and 279 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
cmd/kyverno/main.go
View file

@ -330,12 +330,20 @@ func main() {
promConfig,
)
certRenewer, err := tls.NewCertRenewer(kubeClient, clientConfig, tls.CertRenewalInterval, tls.CertValidityDuration, serverIP, log.Log.WithName("CertRenewer"))
certRenewer, err := tls.NewCertRenewer(
kubeClient,
clientConfig,
tls.CertRenewalInterval,
tls.CAValidityDuration,
tls.TLSValidityDuration,
serverIP,
log.Log.WithName("CertRenewer"),
)
if err != nil {
setupLog.Error(err, "failed to initialize CertRenewer")
os.Exit(1)
}
certManager, err := certmanager.NewController(kubeKyvernoInformer.Core().V1().Secrets(), certRenewer)
certManager, err := certmanager.NewController(kubeKyvernoInformer.Core().V1().Secrets(), certRenewer, webhookCfg.UpdateWebhooksCaBundle)
if err != nil {
setupLog.Error(err, "failed to initialize CertManager")
os.Exit(1)

58
pkg/controllers/certmanager/controller.go
View file

@ -5,10 +5,10 @@ import (
"reflect"
"time"
"github.com/kyverno/kyverno/pkg/common"
"github.com/kyverno/kyverno/pkg/config"
"github.com/kyverno/kyverno/pkg/tls"
v1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
informerv1 "k8s.io/client-go/informers/core/v1"
listersv1 "k8s.io/client-go/listers/core/v1"
"k8s.io/client-go/tools/cache"
@ -23,16 +23,18 @@ type Controller interface {
}
type controller struct {
renewer *tls.CertRenewer
secretLister listersv1.SecretLister
secretQueue chan bool
renewer *tls.CertRenewer
secretLister listersv1.SecretLister
secretQueue chan bool
onSecretChanged func() error
}
func NewController(secretInformer informerv1.SecretInformer, certRenewer *tls.CertRenewer) (Controller, error) {
func NewController(secretInformer informerv1.SecretInformer, certRenewer *tls.CertRenewer, onSecretChanged func() error) (Controller, error) {
manager := &controller{
renewer: certRenewer,
secretLister: secretInformer.Lister(),
secretQueue: make(chan bool, 1),
renewer: certRenewer,
secretLister: secretInformer.Lister(),
secretQueue: make(chan bool, 1),
onSecretChanged: onSecretChanged,
}
secretInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
AddFunc: manager.addSecretFunc,
@ -67,21 +69,33 @@ func (m *controller) GetTLSPemPair() ([]byte, []byte, error) {
return secret.Data[v1.TLSCertKey], secret.Data[v1.TLSPrivateKeyKey], nil
}
func (m *controller) validateCerts() error {
valid, err := m.renewer.ValidCert()
if err != nil {
if apierrors.IsNotFound(err) {
return nil
}
logger.Error(err, "failed to validate cert")
func (m *controller) renewCertificates() error {
if err := common.RetryFunc(time.Second, 5*time.Second, m.renewer.RenewCA, "failed to renew CA", logger)(); err != nil {
return err
}
if !valid {
logger.Info("rootCA has changed or is about to expire, trigger a rolling update to renew the cert")
return m.renewer.RollingUpdate()
if m.onSecretChanged != nil {
if err := common.RetryFunc(time.Second, 5*time.Second, m.onSecretChanged, "failed to renew CA", logger)(); err != nil {
return err
}
}
if err := common.RetryFunc(time.Second, 5*time.Second, m.renewer.RenewTLS, "failed to renew TLS", logger)(); err != nil {
return err
}
return nil
}
func (m *controller) GetCAPem() ([]byte, error) {
secret, err := m.secretLister.Secrets(config.KyvernoNamespace()).Get(tls.GenerateRootCASecretName())
if err != nil {
return nil, err
}
result := secret.Data[v1.TLSCertKey]
if len(result) == 0 {
result = secret.Data[tls.RootCAKey]
}
return result, nil
}
func (m *controller) Run(stopCh <-chan struct{}) {
logger.Info("start managing certificate")
certsRenewalTicker := time.NewTicker(tls.CertRenewalInterval)
@ -89,13 +103,13 @@ func (m *controller) Run(stopCh <-chan struct{}) {
for {
select {
case <-certsRenewalTicker.C:
if err := m.validateCerts(); err != nil {
logger.Error(err, "unable to trigger a rolling update, force restarting")
if err := m.renewCertificates(); err != nil {
logger.Error(err, "unable to renew certificates, force restarting")
os.Exit(1)
}
case <-m.secretQueue:
if err := m.validateCerts(); err != nil {
logger.Error(err, "unable to trigger a rolling update, force restarting")
if err := m.renewCertificates(); err != nil {
logger.Error(err, "unable to renew certificates, force restarting")
os.Exit(1)
}
case <-stopCh:

45
pkg/tls/keypair.go
View file

@ -14,20 +14,17 @@ import (
"github.com/kyverno/kyverno/pkg/config"
)
// keyPair ...
type keyPair struct {
cert *x509.Certificate
key *rsa.PrivateKey
}
// generateCA creates the self-signed CA cert and private key
// it will be used to sign the webhook server certificate
func generateCA(certValidityDuration time.Duration) (*keyPair, error) {
func generateCA(key *rsa.PrivateKey, certValidityDuration time.Duration) (*rsa.PrivateKey, *x509.Certificate, error) {
now := time.Now()
begin, end := now.Add(-1*time.Hour), now.Add(certValidityDuration)
key, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
return nil, fmt.Errorf("error generating key: %v", err)
if key == nil {
newKey, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
return nil, nil, err
}
key = newKey
}
templ := &x509.Certificate{
SerialNumber: big.NewInt(0),
@ -42,21 +39,18 @@ func generateCA(certValidityDuration time.Duration) (*keyPair, error) {
}
der, err := x509.CreateCertificate(rand.Reader, templ, templ, key.Public(), key)
if err != nil {
return nil, fmt.Errorf("error creating certificate: %v", err)
return nil, nil, err
}
cert, err := x509.ParseCertificate(der)
if err != nil {
return nil, fmt.Errorf("error parsing certificate %v", err)
return nil, nil, err
}
return &keyPair{
cert: cert,
key: key,
}, nil
return key, cert, nil
}
// generateCert takes the results of GenerateCACert and uses it to create the
// generateTLS takes the results of GenerateCACert and uses it to create the
// PEM-encoded public certificate and private key, respectively
func generateCert(caCert *keyPair, props *certificateProps, serverIP string, certValidityDuration time.Duration) (*keyPair, error) {
func generateTLS(props *certificateProps, serverIP string, caCert *x509.Certificate, caKey *rsa.PrivateKey, certValidityDuration time.Duration) (*rsa.PrivateKey, *x509.Certificate, error) {
now := time.Now()
begin, end := now.Add(-1*time.Hour), now.Add(certValidityDuration)
dnsNames := []string{
@ -88,18 +82,17 @@ func generateCert(caCert *keyPair, props *certificateProps, serverIP string, cer
}
key, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
return nil, fmt.Errorf("error generating key for webhook %v", err)
return nil, nil, err
}
der, err := x509.CreateCertificate(rand.Reader, templ, caCert.cert, key.Public(), caCert.key)
der, err := x509.CreateCertificate(rand.Reader, templ, caCert, key.Public(), caKey)
if err != nil {
return nil, fmt.Errorf("error creating certificate for webhook %v", err)
logger.Error(err, "create certificate failed")
return nil, nil, err
}
cert, err := x509.ParseCertificate(der)
if err != nil {
return nil, fmt.Errorf("error parsing webhook certificate %v", err)
logger.Error(err, "parse certificate failed")
return nil, nil, err
}
return &keyPair{
cert: cert,
key: key,
}, nil
return key, cert, nil
}

5
pkg/tls/log.go Normal file
View file

@ -0,0 +1,5 @@
package tls
import "sigs.k8s.io/controller-runtime/pkg/log"
var logger = log.Log.WithName("tls")

2
pkg/tls/reader.go
View file

@ -23,7 +23,7 @@ func ReadRootCASecret(client kubernetes.Interface) ([]byte, error) {
result := stlsca.Data[v1.TLSCertKey]
// if not there, try old "rootCA.crt"
if len(result) == 0 {
result = stlsca.Data[rootCAKey]
result = stlsca.Data[RootCAKey]
}
if len(result) == 0 {
return nil, errors.Errorf("%s in secret %s/%s", ErrorsNotFound, config.KyvernoNamespace(), stlsca.Name)

367
pkg/tls/renewer.go
View file

@ -2,16 +2,13 @@ package tls
import (
"context"
"crypto/tls"
"crypto/rsa"
"crypto/x509"
"encoding/pem"
"fmt"
"time"
"github.com/cenkalti/backoff"
"github.com/go-logr/logr"
"github.com/kyverno/kyverno/pkg/config"
"github.com/pkg/errors"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@ -22,92 +19,137 @@ import (
const (
// CertRenewalInterval is the renewal interval for rootCA
CertRenewalInterval time.Duration = 12 * time.Hour
// CertValidityDuration is the valid duration for a new cert
CertValidityDuration time.Duration = 365 * 24 * time.Hour
// CAValidityDuration is the valid duration for CA certificates
CAValidityDuration time.Duration = 365 * 24 * time.Hour
// TLSValidityDuration is the valid duration for TLS certificates
TLSValidityDuration time.Duration = 150 * 24 * time.Hour
// ManagedByLabel is added to Kyverno managed secrets
ManagedByLabel string = "cert.kyverno.io/managed-by"
rootCAKey string = "rootCA.crt"
rollingUpdateAnnotation string = "update.kyverno.io/force-rolling-update"
ManagedByLabel string = "cert.kyverno.io/managed-by"
RootCAKey string = "rootCA.crt"
)
// CertRenewer creates rootCA and pem pair to register
// webhook configurations and webhook server
// renews RootCA at the given interval
type CertRenewer struct {
client kubernetes.Interface
certRenewalInterval time.Duration
certValidityDuration time.Duration
certProps *certificateProps
client kubernetes.Interface
certRenewalInterval time.Duration
caValidityDuration time.Duration
tlsValidityDuration time.Duration
certProps *certificateProps
// IP address where Kyverno controller runs. Only required if out-of-cluster.
serverIP string
log logr.Logger
}
// NewCertRenewer returns an instance of CertRenewer
func NewCertRenewer(client kubernetes.Interface, clientConfig *rest.Config, certRenewalInterval, certValidityDuration time.Duration, serverIP string, log logr.Logger) (*CertRenewer, error) {
func NewCertRenewer(client kubernetes.Interface, clientConfig *rest.Config, certRenewalInterval, caValidityDuration, tlsValidityDuration time.Duration, serverIP string, log logr.Logger) (*CertRenewer, error) {
certProps, err := newCertificateProps(clientConfig)
if err != nil {
return nil, err
}
return &CertRenewer{
client: client,
certRenewalInterval: certRenewalInterval,
certValidityDuration: certValidityDuration,
certProps: certProps,
serverIP: serverIP,
log: log,
client: client,
certRenewalInterval: certRenewalInterval,
caValidityDuration: caValidityDuration,
tlsValidityDuration: tlsValidityDuration,
certProps: certProps,
serverIP: serverIP,
}, nil
}
// InitTLSPemPair Loads or creates PEM private key and TLS certificate for webhook server.
// Created pair is stored in cluster's secret.
func (c *CertRenewer) InitTLSPemPair() error {
logger := c.log.WithName("InitTLSPemPair")
ca, err := c.getCASecret()
if err != nil && !apierrors.IsNotFound(err) {
if err := c.RenewCA(); err != nil {
return err
}
tls, err := c.getTLSSecret()
if err != nil && !apierrors.IsNotFound(err) {
if err := c.RenewTLS(); err != nil {
return err
}
// check they are valid
if ca != nil && tls != nil {
if validCert(ca, tls, logger) {
return nil
}
}
// if not valid, check we can renew them
if !IsSecretManagedByKyverno(ca) || !IsSecretManagedByKyverno(tls) {
return fmt.Errorf("tls is not valid but certificates are not managed by kyverno, we can't renew them")
}
// renew them
logger.Info("building key/certificate pair for TLS")
return c.buildTLSPemPairAndWriteToSecrets(c.serverIP)
}
// buildTLSPemPairAndWriteToSecrets Issues TLS certificate for webhook server using self-signed CA cert
// Returns signed and approved TLS certificate in PEM format
func (c *CertRenewer) buildTLSPemPairAndWriteToSecrets(serverIP string) error {
caCert, err := generateCA(c.certValidityDuration)
if err != nil {
return err
}
tlsPair, err := generateCert(caCert, c.certProps, serverIP, c.certValidityDuration)
if err != nil {
return err
}
if err := c.writeCASecret(caCert); err != nil {
return fmt.Errorf("failed to write CA cert to secret: %v", err)
}
if err = c.writeTLSSecret(tlsPair); err != nil {
return fmt.Errorf("unable to save TLS pair to the cluster: %v", err)
}
return nil
}
// RenewTLS renews the CA certificate if needed
func (c *CertRenewer) RenewCA() error {
secret, key, certs, err := c.decodeCASecret()
if err != nil && !apierrors.IsNotFound(err) {
logger.Error(err, "failed to read CA")
return err
}
now := time.Now()
certs = removeExpiredCertificates(now, certs...)
if !allCertificatesExpired(now.Add(5*c.certRenewalInterval), certs...) {
logger.V(4).Info("CA certificate does not need to be renewed")
return nil
}
if !IsSecretManagedByKyverno(secret) {
err := fmt.Errorf("tls is not valid but certificates are not managed by kyverno, we can't renew them")
logger.Error(err, "tls is not valid but certificates are not managed by kyverno, we can't renew them")
return err
}
caKey, caCert, err := generateCA(key, c.caValidityDuration)
if err != nil {
logger.Error(err, "failed to generate CA")
return err
}
certs = append(certs, caCert)
if err := c.writeCASecret(caKey, certs...); err != nil {
logger.Error(err, "failed to write CA")
return err
}
logger.Info("CA was renewed")
return nil
}
// RenewTLS renews the TLS certificate if needed
func (c *CertRenewer) RenewTLS() error {
_, caKey, caCerts, err := c.decodeCASecret()
if err != nil {
logger.Error(err, "failed to read CA")
return err
}
secret, _, cert, err := c.decodeTLSSecret()
if err != nil && !apierrors.IsNotFound(err) {
logger.Error(err, "failed to read TLS")
return err
}
now := time.Now()
if cert != nil && !allCertificatesExpired(now.Add(5*c.certRenewalInterval), cert) {
logger.V(4).Info("TLS certificate does not need to be renewed")
return nil
}
if !IsSecretManagedByKyverno(secret) {
err := fmt.Errorf("tls is not valid but certificates are not managed by kyverno, we can't renew them")
logger.Error(err, "tls is not valid but certificates are not managed by kyverno, we can't renew them")
return err
}
tlsKey, tlsCert, err := generateTLS(c.certProps, c.serverIP, caCerts[len(caCerts)-1], caKey, c.tlsValidityDuration)
if err != nil {
logger.Error(err, "failed to generate TLS")
return err
}
if err := c.writeTLSSecret(tlsKey, tlsCert); err != nil {
logger.Error(err, "failed to write TLS")
return err
}
logger.Info("TLS was renewed")
return nil
}
// ValidateCert validates the CA Cert
func (c *CertRenewer) ValidateCert() (bool, error) {
_, _, caCerts, err := c.decodeCASecret()
if err != nil {
return false, err
}
_, _, cert, err := c.decodeTLSSecret()
if err != nil {
return false, err
}
return validateCert(time.Now(), cert, caCerts...), nil
}
func (c *CertRenewer) getSecret(name string) (*corev1.Secret, error) {
if s, err := c.client.CoreV1().Secrets(config.KyvernoNamespace()).Get(context.TODO(), name, metav1.GetOptions{}); err != nil {
return nil, err
@ -116,16 +158,72 @@ func (c *CertRenewer) getSecret(name string) (*corev1.Secret, error) {
}
}
func (c *CertRenewer) getCASecret() (*corev1.Secret, error) {
return c.getSecret(GenerateRootCASecretName())
func (c *CertRenewer) decodeSecret(name string) (*corev1.Secret, *rsa.PrivateKey, []*x509.Certificate, error) {
secret, err := c.getSecret(name)
if err != nil {
return nil, nil, nil, err
}
var certBytes, keyBytes []byte
if secret != nil {
keyBytes = secret.Data[corev1.TLSPrivateKeyKey]
certBytes = secret.Data[corev1.TLSCertKey]
if len(certBytes) == 0 {
certBytes = secret.Data[RootCAKey]
}
}
var key *rsa.PrivateKey
if keyBytes != nil {
usedkey, err := pemToPrivateKey(keyBytes)
if err != nil {
return nil, nil, nil, err
}
key = usedkey
}
return secret, key, pemToCertificates(certBytes), nil
}
func (c *CertRenewer) getTLSSecret() (*corev1.Secret, error) {
return c.getSecret(GenerateTLSPairSecretName())
func (c *CertRenewer) decodeCASecret() (*corev1.Secret, *rsa.PrivateKey, []*x509.Certificate, error) {
return c.decodeSecret(GenerateRootCASecretName())
}
func (c *CertRenewer) writeSecret(secret *corev1.Secret, logger logr.Logger) error {
logger = logger.WithValues("name", secret.GetName(), "namespace", secret.GetNamespace())
func (c *CertRenewer) decodeTLSSecret() (*corev1.Secret, *rsa.PrivateKey, *x509.Certificate, error) {
secret, key, certs, err := c.decodeSecret(GenerateTLSPairSecretName())
if err != nil {
return nil, nil, nil, err
}
if len(certs) == 0 {
return secret, key, nil, nil
} else if len(certs) == 1 {
return secret, key, certs[0], nil
} else {
return nil, nil, nil, err
}
}
func (c *CertRenewer) writeSecret(name string, key *rsa.PrivateKey, certs ...*x509.Certificate) error {
logger := logger.WithValues("name", name, "namespace", config.KyvernoNamespace())
secret, err := c.getSecret(name)
if err != nil && !apierrors.IsNotFound(err) {
logger.Error(err, "failed to get CA secret")
return err
}
if secret == nil {
secret = &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: config.KyvernoNamespace(),
Labels: map[string]string{
ManagedByLabel: "kyverno",
},
},
Type: corev1.SecretTypeTLS,
}
}
secret.Type = corev1.SecretTypeTLS
secret.Data = map[string][]byte{
corev1.TLSCertKey: certificateToPem(certs...),
corev1.TLSPrivateKeyKey: privateKeyToPem(key),
}
if secret.ResourceVersion == "" {
if _, err := c.client.CoreV1().Secrets(config.KyvernoNamespace()).Create(context.TODO(), secret, metav1.CreateOptions{}); err != nil {
logger.Error(err, "failed to update secret")
@ -145,142 +243,11 @@ func (c *CertRenewer) writeSecret(secret *corev1.Secret, logger logr.Logger) err
}
// writeCASecret stores the CA cert in secret
func (c *CertRenewer) writeCASecret(ca *keyPair) error {
logger := c.log.WithName("writeCASecret")
secret, err := c.getCASecret()
if err != nil && !apierrors.IsNotFound(err) {
logger.Error(err, "failed to get CA secret")
return err
}
if secret == nil {
secret = &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: GenerateRootCASecretName(),
Namespace: config.KyvernoNamespace(),
Labels: map[string]string{
ManagedByLabel: "kyverno",
},
},
Type: corev1.SecretTypeTLS,
}
}
secret.Type = corev1.SecretTypeTLS
secret.Data = map[string][]byte{
corev1.TLSCertKey: CertificateToPem(ca.cert),
corev1.TLSPrivateKeyKey: PrivateKeyToPem(ca.key),
}
return c.writeSecret(secret, logger)
func (c *CertRenewer) writeCASecret(key *rsa.PrivateKey, certs ...*x509.Certificate) error {
return c.writeSecret(GenerateRootCASecretName(), key, certs...)
}
// writeTLSSecret Writes the pair of TLS certificate and key to the specified secret.
func (c *CertRenewer) writeTLSSecret(tls *keyPair) error {
logger := c.log.WithName("writeTLSSecret")
secret, err := c.getTLSSecret()
if err != nil && !apierrors.IsNotFound(err) {
logger.Error(err, "failed to get TLS secret")
return err
}
if secret == nil {
secret = &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: GenerateTLSPairSecretName(),
Namespace: config.KyvernoNamespace(),
Labels: map[string]string{
ManagedByLabel: "kyverno",
},
},
Type: corev1.SecretTypeTLS,
}
}
secret.Data = map[string][]byte{
corev1.TLSCertKey: CertificateToPem(tls.cert),
corev1.TLSPrivateKeyKey: PrivateKeyToPem(tls.key),
}
return c.writeSecret(secret, logger)
}
// ValidCert validates the CA Cert
func (c *CertRenewer) ValidCert() (bool, error) {
logger := c.log.WithName("validCert")
ca, err := c.getCASecret()
if err != nil {
logger.Error(err, "unable to read CA secret")
return false, err
}
tls, err := c.getTLSSecret()
if err != nil {
logger.Error(err, "unable to read TLS secret")
return false, err
}
return validCert(ca, tls, logger), nil
}
func validCert(caSecret *corev1.Secret, tlsSecret *corev1.Secret, logger logr.Logger) bool {
caPem := caSecret.Data[corev1.TLSCertKey]
if len(caPem) == 0 {
caPem = caSecret.Data[rootCAKey]
}
// build cert pool
pool := x509.NewCertPool()
if !pool.AppendCertsFromPEM(caPem) {
logger.Info("bad certificate")
return false
}
// valid PEM pair
_, err := tls.X509KeyPair(tlsSecret.Data[corev1.TLSCertKey], tlsSecret.Data[corev1.TLSPrivateKeyKey])
if err != nil {
logger.Error(err, "invalid PEM pair")
return false
}
certPemBlock, _ := pem.Decode(tlsSecret.Data[corev1.TLSCertKey])
if certPemBlock == nil {
logger.Error(err, "bad private key")
return false
}
cert, err := x509.ParseCertificate(certPemBlock.Bytes)
if err != nil {
logger.Error(err, "failed to parse cert")
return false
}
if _, err = cert.Verify(x509.VerifyOptions{
Roots: pool,
CurrentTime: time.Now(),
}); err != nil {
logger.Error(err, "invalid cert")
return false
}
return true
}
// RollingUpdate triggers a rolling update of Kyverno pod.
// It is used when the rootCA is renewed, the restart of
// Kyverno pod will register webhook server with new cert
func (c *CertRenewer) RollingUpdate() error {
update := func() error {
deploy, err := c.client.AppsV1().Deployments(config.KyvernoNamespace()).Get(context.TODO(), config.KyvernoDeploymentName(), metav1.GetOptions{})
if err != nil {
return errors.Wrap(err, "failed to find Kyverno")
}
if IsKyvernoInRollingUpdate(deploy, c.log) {
return nil
}
if deploy.Spec.Template.Annotations == nil {
deploy.Spec.Template.Annotations = map[string]string{}
}
deploy.Spec.Template.Annotations[rollingUpdateAnnotation] = time.Now().String()
if _, err = c.client.AppsV1().Deployments(config.KyvernoNamespace()).Update(context.TODO(), deploy, metav1.UpdateOptions{}); err != nil {
return errors.Wrap(err, "update Kyverno deployment")
}
return nil
}
exbackoff := &backoff.ExponentialBackOff{
InitialInterval: 500 * time.Millisecond,
RandomizationFactor: 0.5,
Multiplier: 1.5,
MaxInterval: time.Second,
MaxElapsedTime: 3 * time.Second,
Clock: backoff.SystemClock,
}
exbackoff.Reset()
return backoff.Retry(update, exbackoff)
func (c *CertRenewer) writeTLSSecret(key *rsa.PrivateKey, cert *x509.Certificate) error {
return c.writeSecret(GenerateTLSPairSecretName(), key, cert)
}

74
pkg/tls/utils.go
View file

@ -4,15 +4,14 @@ import (
"crypto/rsa"
"crypto/x509"
"encoding/pem"
"time"
"github.com/go-logr/logr"
"github.com/kyverno/kyverno/pkg/config"
appsv1 "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
)
// PrivateKeyToPem Creates PEM block from private key object
func PrivateKeyToPem(rsaKey *rsa.PrivateKey) []byte {
func privateKeyToPem(rsaKey *rsa.PrivateKey) []byte {
privateKey := &pem.Block{
Type: "PRIVATE KEY",
Bytes: x509.MarshalPKCS1PrivateKey(rsaKey),
@ -20,17 +19,72 @@ func PrivateKeyToPem(rsaKey *rsa.PrivateKey) []byte {
return pem.EncodeToMemory(privateKey)
}
// CertificateToPem Creates PEM block from certificate object
func CertificateToPem(cert *x509.Certificate) []byte {
certificate := &pem.Block{
Type: "CERTIFICATE",
Bytes: cert.Raw,
func certificateToPem(certs ...*x509.Certificate) []byte {
var raw []byte
for _, cert := range certs {
certificate := &pem.Block{
Type: "CERTIFICATE",
Bytes: cert.Raw,
}
raw = append(raw, pem.EncodeToMemory(certificate)...)
}
return pem.EncodeToMemory(certificate)
return raw
}
func pemToPrivateKey(raw []byte) (*rsa.PrivateKey, error) {
block, _ := pem.Decode(raw)
return x509.ParsePKCS1PrivateKey(block.Bytes)
}
func pemToCertificates(raw []byte) []*x509.Certificate {
var certs []*x509.Certificate
for {
certPemBlock, next := pem.Decode(raw)
if certPemBlock == nil {
return certs
}
raw = next
cert, err := x509.ParseCertificate(certPemBlock.Bytes)
if err == nil {
certs = append(certs, cert)
} else {
logger.Error(err, "failed to parse cert")
}
}
}
func removeExpiredCertificates(now time.Time, certs ...*x509.Certificate) []*x509.Certificate {
var result []*x509.Certificate
for _, cert := range certs {
if !now.After(cert.NotAfter) {
result = append(result, cert)
}
}
return result
}
func allCertificatesExpired(now time.Time, certs ...*x509.Certificate) bool {
for _, cert := range certs {
if !now.After(cert.NotAfter) {
return false
}
}
return true
}
func validateCert(now time.Time, cert *x509.Certificate, caCerts ...*x509.Certificate) bool {
pool := x509.NewCertPool()
for _, cert := range caCerts {
pool.AddCert(cert)
}
if _, err := cert.Verify(x509.VerifyOptions{Roots: pool, CurrentTime: now}); err != nil {
return false
}
return true
}
// IsKyvernoInRollingUpdate returns true if Kyverno is in rolling update
func IsKyvernoInRollingUpdate(deploy *appsv1.Deployment, logger logr.Logger) bool {
func IsKyvernoInRollingUpdate(deploy *appsv1.Deployment) bool {
var replicas int32 = 1
if deploy.Spec.Replicas != nil {
replicas = *deploy.Spec.Replicas

14
pkg/webhookconfig/common.go
View file

@ -16,6 +16,11 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
const (
managedByLabel string = "webhook.kyverno.io/managed-by"
kyvernoValue string = "kyverno"
)
var (
noneOnDryRun = admregapi.SideEffectClassNoneOnDryRun
never = admregapi.NeverReinvocationPolicy
@ -32,7 +37,7 @@ var (
}
vertifyObjectSelector = &metav1.LabelSelector{
MatchLabels: map[string]string{
"app.kubernetes.io/name": "kyverno",
"app.kubernetes.io/name": kyvernoValue,
},
}
update = []admregapi.OperationType{admregapi.Update}
@ -69,7 +74,7 @@ func getHealthyPodsIP(pods []corev1.Pod) []string {
func (wrc *Register) GetKubePolicyClusterRoleName() (*rbacv1.ClusterRole, error) {
selector := &metav1.LabelSelector{
MatchLabels: map[string]string{
"app.kubernetes.io/name": "kyverno",
"app.kubernetes.io/name": kyvernoValue,
},
}
clusterRoles, err := wrc.kubeClient.RbacV1().ClusterRoles().List(context.TODO(), metav1.ListOptions{LabelSelector: metav1.FormatLabelSelector(selector)})
@ -190,7 +195,10 @@ func generateValidatingWebhook(name, servicePath string, caData []byte, timeoutS
func generateObjectMeta(name string, owner ...metav1.OwnerReference) metav1.ObjectMeta {
return metav1.ObjectMeta{
Name: name,
Name: name,
Labels: map[string]string{
managedByLabel: kyvernoValue,
},
OwnerReferences: owner,
}
}

3
pkg/webhookconfig/monitor.go
View file

@ -229,6 +229,5 @@ func skipWebhookCheck(register *Register, logger logr.Logger) bool {
logger.Info("unable to get Kyverno deployment", "reason", err.Error())
return false
}
return tls.IsKyvernoInRollingUpdate(deploy, logger)
return tls.IsKyvernoInRollingUpdate(deploy)
}

51
pkg/webhookconfig/registration.go
View file

@ -14,7 +14,6 @@ import (
kyvernoinformer "github.com/kyverno/kyverno/pkg/client/informers/externalversions/kyverno/v1"
"github.com/kyverno/kyverno/pkg/config"
client "github.com/kyverno/kyverno/pkg/dclient"
"github.com/kyverno/kyverno/pkg/tls"
"github.com/kyverno/kyverno/pkg/utils"
"github.com/pkg/errors"
admregapi "k8s.io/api/admissionregistration/v1"
@ -169,7 +168,6 @@ func (wrc *Register) Remove(cleanUp chan<- struct{}) {
}
if wrc.shouldCleanupKyvernoResource() {
wrc.removeWebhookConfigurations()
wrc.removeSecrets()
}
}
@ -178,6 +176,44 @@ func (wrc *Register) GetWebhookTimeOut() time.Duration {
return time.Duration(wrc.timeoutSeconds)
}
func (wrc *Register) UpdateWebhooksCaBundle() error {
selector := &metav1.LabelSelector{
MatchLabels: map[string]string{
managedByLabel: kyvernoValue,
},
}
caData := wrc.readCaData()
m := wrc.kubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations()
v := wrc.kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations()
if list, err := m.List(context.TODO(), metav1.ListOptions{LabelSelector: metav1.FormatLabelSelector(selector)}); err != nil {
return err
} else {
for _, item := range list.Items {
copy := item
for r := range copy.Webhooks {
copy.Webhooks[r].ClientConfig.CABundle = caData
}
if _, err := m.Update(context.TODO(), &copy, metav1.UpdateOptions{}); err != nil {
return err
}
}
}
if list, err := v.List(context.TODO(), metav1.ListOptions{LabelSelector: metav1.FormatLabelSelector(selector)}); err != nil {
return err
} else {
for _, item := range list.Items {
copy := item
for r := range copy.Webhooks {
copy.Webhooks[r].ClientConfig.CABundle = caData
}
if _, err := v.Update(context.TODO(), &copy, metav1.UpdateOptions{}); err != nil {
return err
}
}
}
return nil
}
// UpdateWebhookConfigurations updates resource webhook configurations dynamically
// based on the UPDATEs of Kyverno ConfigMap defined in INIT_CONFIG env
//
@ -501,17 +537,6 @@ func (wrc *Register) shouldCleanupKyvernoResource() bool {
return false
}
func (wrc *Register) removeSecrets() {
selector := &metav1.LabelSelector{
MatchLabels: map[string]string{
tls.ManagedByLabel: "kyverno",
},
}
if err := wrc.kubeClient.CoreV1().Secrets(config.KyvernoNamespace()).DeleteCollection(context.TODO(), metav1.DeleteOptions{}, metav1.ListOptions{LabelSelector: metav1.FormatLabelSelector(selector)}); err != nil {
wrc.log.Error(err, "failed to clean up Kyverno managed secrets")
}
}
func (wrc *Register) removeWebhookConfigurations() {
startTime := time.Now()
wrc.log.V(3).Info("deleting all webhook configurations")