From 8f825bb0402a51ff189e4c94c14f8f3de2143b8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Charles-Edouard=20Br=C3=A9t=C3=A9ch=C3=A9?= Date: Wed, 11 May 2022 16:58:14 +0200 Subject: [PATCH] refactor: remove deployment hash on certs secrets (#3886) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Charles-Edouard Brétéché --- cmd/initContainer/main.go | 39 +-- pkg/controllers/certmanager/controller.go | 12 +- pkg/tls/certRenewer.go | 357 ---------------------- pkg/tls/keypair.go | 40 +-- pkg/tls/pempair.go | 25 -- pkg/tls/props.go | 33 +- pkg/tls/reader.go | 83 +---- pkg/tls/renewer.go | 287 +++++++++++++++++ pkg/tls/utils.go | 55 +++- pkg/webhookconfig/common.go | 2 +- 10 files changed, 376 insertions(+), 557 deletions(-) delete mode 100644 pkg/tls/certRenewer.go delete mode 100644 pkg/tls/pempair.go create mode 100644 pkg/tls/renewer.go diff --git a/cmd/initContainer/main.go b/cmd/initContainer/main.go index 4c35c29082..ff11be944a 100644 --- a/cmd/initContainer/main.go +++ b/cmd/initContainer/main.go @@ -6,7 +6,6 @@ package main import ( "context" "flag" - "fmt" "os" "sync" "time" @@ -131,52 +130,22 @@ func main() { failure := false run := func() { - certProps, err := tls.NewCertificateProps(clientConfig) - if err != nil { - log.Log.Info("failed to get cert properties: %v", err.Error()) - os.Exit(1) - } - - depl, err := kubeClient.AppsV1().Deployments(config.KyvernoNamespace()).Get(context.TODO(), config.KyvernoDeploymentName(), metav1.GetOptions{}) - deplHash := "" - if err != nil { - log.Log.Info("failed to fetch deployment '%v': %v", config.KyvernoDeploymentName(), err.Error()) - os.Exit(1) - } - deplHash = fmt.Sprintf("%v", depl.GetUID()) - - name := certProps.GenerateRootCASecretName() - secret, err := kubeClient.CoreV1().Secrets(config.KyvernoNamespace()).Get(context.TODO(), name, metav1.GetOptions{}) + name := tls.GenerateRootCASecretName() + _, err = kubeClient.CoreV1().Secrets(config.KyvernoNamespace()).Get(context.TODO(), name, metav1.GetOptions{}) if err != nil { log.Log.Info("failed to fetch root CA secret", "name", name, "error", err.Error()) - if !errors.IsNotFound(err) { os.Exit(1) } - } else if tls.CanAddAnnotationToSecret(deplHash, secret) { - secret.SetAnnotations(map[string]string{tls.MasterDeploymentUID: deplHash}) - _, err = kubeClient.CoreV1().Secrets(config.KyvernoNamespace()).Update(context.TODO(), secret, metav1.UpdateOptions{}) - if err != nil { - log.Log.Info("failed to update cert: %v", err.Error()) - os.Exit(1) - } } - name = certProps.GenerateTLSPairSecretName() - secret, err = kubeClient.CoreV1().Secrets(config.KyvernoNamespace()).Get(context.TODO(), name, metav1.GetOptions{}) + name = tls.GenerateTLSPairSecretName() + _, err = kubeClient.CoreV1().Secrets(config.KyvernoNamespace()).Get(context.TODO(), name, metav1.GetOptions{}) if err != nil { log.Log.Info("failed to fetch TLS Pair secret", "name", name, "error", err.Error()) - if !errors.IsNotFound(err) { os.Exit(1) } - } else if tls.CanAddAnnotationToSecret(deplHash, secret) { - secret.SetAnnotations(map[string]string{tls.MasterDeploymentUID: deplHash}) - _, err = kubeClient.CoreV1().Secrets(certProps.Namespace).Update(context.TODO(), secret, metav1.UpdateOptions{}) - if err != nil { - log.Log.Info("failed to update cert: %v", err.Error()) - os.Exit(1) - } } if err = acquireLeader(ctx, kubeClient); err != nil { diff --git a/pkg/controllers/certmanager/controller.go b/pkg/controllers/certmanager/controller.go index e12bc87fd8..a4f69d98e9 100644 --- a/pkg/controllers/certmanager/controller.go +++ b/pkg/controllers/certmanager/controller.go @@ -3,12 +3,12 @@ package certmanager import ( "os" "reflect" - "strings" "time" "github.com/kyverno/kyverno/pkg/config" "github.com/kyverno/kyverno/pkg/tls" v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" informerv1 "k8s.io/client-go/informers/core/v1" listersv1 "k8s.io/client-go/listers/core/v1" "k8s.io/client-go/tools/cache" @@ -43,7 +43,7 @@ func NewController(secretInformer informerv1.SecretInformer, certRenewer *tls.Ce func (m *controller) addSecretFunc(obj interface{}) { secret := obj.(*v1.Secret) - if secret.GetNamespace() == config.KyvernoNamespace() && secret.GetName() == m.renewer.GenerateTLSPairSecretName() { + if secret.GetNamespace() == config.KyvernoNamespace() && secret.GetName() == tls.GenerateTLSPairSecretName() { m.secretQueue <- true } } @@ -51,7 +51,7 @@ func (m *controller) addSecretFunc(obj interface{}) { func (m *controller) updateSecretFunc(oldObj interface{}, newObj interface{}) { old := oldObj.(*v1.Secret) new := newObj.(*v1.Secret) - if new.GetNamespace() == config.KyvernoNamespace() && new.GetName() == m.renewer.GenerateTLSPairSecretName() { + if new.GetNamespace() == config.KyvernoNamespace() && new.GetName() == tls.GenerateTLSPairSecretName() { if !reflect.DeepEqual(old.DeepCopy().Data, new.DeepCopy().Data) { m.secretQueue <- true logger.V(4).Info("secret updated, reconciling webhook configurations") @@ -60,7 +60,7 @@ func (m *controller) updateSecretFunc(oldObj interface{}, newObj interface{}) { } func (m *controller) GetTLSPemPair() ([]byte, []byte, error) { - secret, err := m.secretLister.Secrets(config.KyvernoNamespace()).Get(m.renewer.GenerateTLSPairSecretName()) + secret, err := m.secretLister.Secrets(config.KyvernoNamespace()).Get(tls.GenerateTLSPairSecretName()) if err != nil { return nil, nil, err } @@ -70,10 +70,10 @@ func (m *controller) GetTLSPemPair() ([]byte, []byte, error) { func (m *controller) validateCerts() error { valid, err := m.renewer.ValidCert() if err != nil { - logger.Error(err, "failed to validate cert") - if !strings.Contains(err.Error(), tls.ErrorsNotFound) { + if apierrors.IsNotFound(err) { return nil } + logger.Error(err, "failed to validate cert") } if !valid { logger.Info("rootCA has changed or is about to expire, trigger a rolling update to renew the cert") diff --git a/pkg/tls/certRenewer.go b/pkg/tls/certRenewer.go deleted file mode 100644 index 6135d318da..0000000000 --- a/pkg/tls/certRenewer.go +++ /dev/null @@ -1,357 +0,0 @@ -package tls - -import ( - "context" - "crypto/tls" - "crypto/x509" - "encoding/pem" - "fmt" - "time" - - "github.com/cenkalti/backoff" - "github.com/go-logr/logr" - "github.com/kyverno/kyverno/pkg/config" - "github.com/pkg/errors" - v1 "k8s.io/api/core/v1" - k8errors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" -) - -const ( - // CertRenewalInterval is the renewal interval for rootCA - CertRenewalInterval time.Duration = 12 * time.Hour - // CertValidityDuration is the valid duration for a new cert - CertValidityDuration time.Duration = 365 * 24 * time.Hour - // ManagedByLabel is added to Kyverno managed secrets - ManagedByLabel string = "cert.kyverno.io/managed-by" - MasterDeploymentUID string = "cert.kyverno.io/master-deployment-uid" - rootCAKey string = "rootCA.crt" - rollingUpdateAnnotation string = "update.kyverno.io/force-rolling-update" -) - -// CertRenewer creates rootCA and pem pair to register -// webhook configurations and webhook server -// renews RootCA at the given interval -type CertRenewer struct { - client kubernetes.Interface - clientConfig *rest.Config - certRenewalInterval time.Duration - certValidityDuration time.Duration - certProps *CertificateProps - - // IP address where Kyverno controller runs. Only required if out-of-cluster. - serverIP string - - log logr.Logger -} - -// NewCertRenewer returns an instance of CertRenewer -func NewCertRenewer(client kubernetes.Interface, clientConfig *rest.Config, certRenewalInterval, certValidityDuration time.Duration, serverIP string, log logr.Logger) (*CertRenewer, error) { - certProps, err := NewCertificateProps(clientConfig) - if err != nil { - return nil, err - } - return &CertRenewer{ - client: client, - clientConfig: clientConfig, - certRenewalInterval: certRenewalInterval, - certValidityDuration: certValidityDuration, - certProps: certProps, - serverIP: serverIP, - log: log, - }, nil -} - -func (c *CertRenewer) Client() kubernetes.Interface { - return c.client -} - -func (c *CertRenewer) ClientConfig() *rest.Config { - return c.clientConfig -} - -func (c *CertRenewer) GenerateTLSPairSecretName() string { - return c.certProps.GenerateTLSPairSecretName() -} - -func (c *CertRenewer) GenerateRootCASecretName() string { - return c.certProps.GenerateRootCASecretName() -} - -// InitTLSPemPair Loads or creates PEM private key and TLS certificate for webhook server. -// Created pair is stored in cluster's secret. -// Returns struct with key/certificate pair. -func (c *CertRenewer) InitTLSPemPair() error { - logger := c.log.WithName("InitTLSPemPair") - if valid, err := c.ValidCert(); err == nil && valid { - if _, _, err := ReadTLSPair(c.clientConfig, c.client); err == nil { - logger.Info("using existing TLS key/certificate pair") - return nil - } - } else if err != nil { - logger.V(3).Info("unable to find TLS pair", "reason", err.Error()) - } - - logger.Info("building key/certificate pair for TLS") - return c.buildTLSPemPairAndWriteToSecrets(c.serverIP) -} - -// buildTLSPemPairAndWriteToSecrets Issues TLS certificate for webhook server using self-signed CA cert -// Returns signed and approved TLS certificate in PEM format -func (c *CertRenewer) buildTLSPemPairAndWriteToSecrets(serverIP string) error { - caCert, err := GenerateCA(c.certValidityDuration) - if err != nil { - return err - } - tlsPair, err := GenerateCert(caCert, c.certProps, serverIP, c.certValidityDuration) - if err != nil { - return err - } - if err := c.WriteCACertToSecret(caCert); err != nil { - return fmt.Errorf("failed to write CA cert to secret: %v", err) - } - if err = c.WriteTLSPairToSecret(tlsPair); err != nil { - return fmt.Errorf("unable to save TLS pair to the cluster: %v", err) - } - return nil -} - -// WriteCACertToSecret stores the CA cert in secret -func (c *CertRenewer) WriteCACertToSecret(ca *KeyPair) error { - logger := c.log.WithName("CAcert") - name := c.certProps.GenerateRootCASecretName() - caBytes := CertificateToPem(ca.Cert) - keyBytes := PrivateKeyToPem(ca.Key) - depl, err := c.client.AppsV1().Deployments(c.certProps.Namespace).Get(context.TODO(), config.KyvernoDeploymentName(), metav1.GetOptions{}) - deplHash := "" - if err == nil { - deplHash = fmt.Sprintf("%v", depl.GetUID()) - } - secret, err := c.client.CoreV1().Secrets(c.certProps.Namespace).Get(context.TODO(), name, metav1.GetOptions{}) - if err != nil { - if k8errors.IsNotFound(err) { - secret = &v1.Secret{ - TypeMeta: metav1.TypeMeta{ - Kind: "Secret", - APIVersion: "v1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: c.certProps.Namespace, - Annotations: map[string]string{ - MasterDeploymentUID: deplHash, - }, - Labels: map[string]string{ - ManagedByLabel: "kyverno", - }, - }, - Data: map[string][]byte{ - v1.TLSCertKey: caBytes, - v1.TLSPrivateKeyKey: keyBytes, - }, - Type: v1.SecretTypeTLS, - } - _, err = c.client.CoreV1().Secrets(c.certProps.Namespace).Create(context.TODO(), secret, metav1.CreateOptions{}) - if err == nil { - logger.Info("secret created", "name", name, "namespace", c.certProps.Namespace) - } - } - return err - } else if CanAddAnnotationToSecret(deplHash, secret) { - _, err = c.client.CoreV1().Secrets(c.certProps.Namespace).Update(context.TODO(), secret, metav1.UpdateOptions{}) - if err == nil { - logger.Info("secret updated", "name", name, "namespace", c.certProps.Namespace) - } - return err - } - secret.Type = v1.SecretTypeTLS - secret.Data = map[string][]byte{ - v1.TLSCertKey: caBytes, - v1.TLSPrivateKeyKey: keyBytes, - } - _, err = c.client.CoreV1().Secrets(c.certProps.Namespace).Update(context.TODO(), secret, metav1.UpdateOptions{}) - if err != nil { - return err - } - logger.Info("secret updated", "name", name, "namespace", c.certProps.Namespace) - return nil -} - -// WriteTLSPairToSecret Writes the pair of TLS certificate and key to the specified secret. -// Updates existing secret or creates new one. -func (c *CertRenewer) WriteTLSPairToSecret(tls *KeyPair) error { - logger := c.log.WithName("WriteTLSPair") - name := c.certProps.GenerateTLSPairSecretName() - certBytes := CertificateToPem(tls.Cert) - keyBytes := PrivateKeyToPem(tls.Key) - depl, err := c.client.AppsV1().Deployments(c.certProps.Namespace).Get(context.TODO(), config.KyvernoDeploymentName(), metav1.GetOptions{}) - deplHash := "" - if err == nil { - deplHash = fmt.Sprintf("%v", depl.GetUID()) - } - secret, err := c.client.CoreV1().Secrets(c.certProps.Namespace).Get(context.TODO(), name, metav1.GetOptions{}) - if err != nil { - if k8errors.IsNotFound(err) { - secret = &v1.Secret{ - TypeMeta: metav1.TypeMeta{ - Kind: "Secret", - APIVersion: "v1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: c.certProps.Namespace, - Annotations: map[string]string{ - MasterDeploymentUID: deplHash, - }, - Labels: map[string]string{ - ManagedByLabel: "kyverno", - }, - }, - Data: map[string][]byte{ - v1.TLSCertKey: certBytes, - v1.TLSPrivateKeyKey: keyBytes, - }, - Type: v1.SecretTypeTLS, - } - _, err = c.client.CoreV1().Secrets(c.certProps.Namespace).Create(context.TODO(), secret, metav1.CreateOptions{}) - if err == nil { - logger.Info("secret created", "name", name, "namespace", c.certProps.Namespace) - } - } - return err - } else if CanAddAnnotationToSecret(deplHash, secret) { - _, err = c.client.CoreV1().Secrets(c.certProps.Namespace).Update(context.TODO(), secret, metav1.UpdateOptions{}) - if err == nil { - logger.Info("secret updated", "name", name, "namespace", c.certProps.Namespace) - } - return err - } - secret.Data = map[string][]byte{ - v1.TLSCertKey: certBytes, - v1.TLSPrivateKeyKey: keyBytes, - } - _, err = c.client.CoreV1().Secrets(c.certProps.Namespace).Update(context.TODO(), secret, metav1.UpdateOptions{}) - if err != nil { - return err - } - logger.Info("secret updated", "name", name, "namespace", c.certProps.Namespace) - return nil -} - -// ValidCert validates the CA Cert -func (c *CertRenewer) ValidCert() (bool, error) { - logger := c.log.WithName("ValidCert") - var managedByKyverno bool - snameTLS := c.certProps.GenerateTLSPairSecretName() - snameCA := c.certProps.GenerateRootCASecretName() - secret, err := c.client.CoreV1().Secrets(c.certProps.Namespace).Get(context.TODO(), snameTLS, metav1.GetOptions{}) - if err != nil { - return false, nil - } - - if label, ok := secret.GetLabels()[ManagedByLabel]; ok { - managedByKyverno = label == "kyverno" - } - - _, ok := secret.GetAnnotations()[MasterDeploymentUID] - if managedByKyverno && !ok { - return false, nil - } - - secret, err = c.client.CoreV1().Secrets(c.certProps.Namespace).Get(context.TODO(), snameCA, metav1.GetOptions{}) - if err != nil { - return false, nil - } - - if label, ok := secret.GetLabels()[ManagedByLabel]; ok { - managedByKyverno = label == "kyverno" - } - - _, ok = secret.GetAnnotations()[MasterDeploymentUID] - if managedByKyverno && !ok { - return false, nil - } - - rootCA, err := ReadRootCASecret(c.clientConfig, c.client) - if err != nil { - return false, errors.Wrap(err, "unable to read CA from secret") - } - - certPem, keyPem, err := ReadTLSPair(c.clientConfig, c.client) - if err != nil { - // wait till next reconcile - logger.Info("unable to read TLS PEM Pair from secret", "reason", err.Error()) - return false, errors.Wrap(err, "unable to read TLS PEM Pair from secret") - } - - // build cert pool - pool := x509.NewCertPool() - if !pool.AppendCertsFromPEM(rootCA) { - logger.Error(err, "bad certificate") - return false, nil - } - - // valid PEM pair - _, err = tls.X509KeyPair(certPem, keyPem) - if err != nil { - logger.Error(err, "invalid PEM pair") - return false, nil - } - - certPemBlock, _ := pem.Decode(certPem) - if certPem == nil { - logger.Error(err, "bad private key") - return false, nil - } - - cert, err := x509.ParseCertificate(certPemBlock.Bytes) - if err != nil { - logger.Error(err, "failed to parse cert") - return false, nil - } - - if _, err = cert.Verify(x509.VerifyOptions{ - Roots: pool, - CurrentTime: time.Now().Add(c.certRenewalInterval), - }); err != nil { - logger.Error(err, "invalid cert") - return false, nil - } - - return true, nil -} - -// RollingUpdate triggers a rolling update of Kyverno pod. -// It is used when the rootCA is renewed, the restart of -// Kyverno pod will register webhook server with new cert -func (c *CertRenewer) RollingUpdate() error { - update := func() error { - deploy, err := c.client.AppsV1().Deployments(config.KyvernoNamespace()).Get(context.TODO(), config.KyvernoDeploymentName(), metav1.GetOptions{}) - if err != nil { - return errors.Wrap(err, "failed to find Kyverno") - } - if IsKyvernoInRollingUpdate(deploy, c.log) { - return nil - } - if deploy.Spec.Template.Annotations == nil { - deploy.Spec.Template.Annotations = map[string]string{} - } - deploy.Spec.Template.Annotations[rollingUpdateAnnotation] = time.Now().String() - if _, err = c.client.AppsV1().Deployments(config.KyvernoNamespace()).Update(context.TODO(), deploy, metav1.UpdateOptions{}); err != nil { - return errors.Wrap(err, "update Kyverno deployment") - } - return nil - } - exbackoff := &backoff.ExponentialBackOff{ - InitialInterval: 500 * time.Millisecond, - RandomizationFactor: 0.5, - Multiplier: 1.5, - MaxInterval: time.Second, - MaxElapsedTime: 3 * time.Second, - Clock: backoff.SystemClock, - } - exbackoff.Reset() - return backoff.Retry(update, exbackoff) -} diff --git a/pkg/tls/keypair.go b/pkg/tls/keypair.go index c1181b799b..a7c43ec467 100644 --- a/pkg/tls/keypair.go +++ b/pkg/tls/keypair.go @@ -10,17 +10,19 @@ import ( "net" "strings" "time" + + "github.com/kyverno/kyverno/pkg/config" ) -// KeyPair ... -type KeyPair struct { - Cert *x509.Certificate - Key *rsa.PrivateKey +// keyPair ... +type keyPair struct { + cert *x509.Certificate + key *rsa.PrivateKey } -// GenerateCA creates the self-signed CA cert and private key +// generateCA creates the self-signed CA cert and private key // it will be used to sign the webhook server certificate -func GenerateCA(certValidityDuration time.Duration) (*KeyPair, error) { +func generateCA(certValidityDuration time.Duration) (*keyPair, error) { now := time.Now() begin, end := now.Add(-1*time.Hour), now.Add(certValidityDuration) key, err := rsa.GenerateKey(rand.Reader, 2048) @@ -46,21 +48,21 @@ func GenerateCA(certValidityDuration time.Duration) (*KeyPair, error) { if err != nil { return nil, fmt.Errorf("error parsing certificate %v", err) } - return &KeyPair{ - Cert: cert, - Key: key, + return &keyPair{ + cert: cert, + key: key, }, nil } -// GenerateCert takes the results of GenerateCACert and uses it to create the +// generateCert takes the results of GenerateCACert and uses it to create the // PEM-encoded public certificate and private key, respectively -func GenerateCert(caCert *KeyPair, props *CertificateProps, serverIP string, certValidityDuration time.Duration) (*KeyPair, error) { +func generateCert(caCert *keyPair, props *certificateProps, serverIP string, certValidityDuration time.Duration) (*keyPair, error) { now := time.Now() begin, end := now.Add(-1*time.Hour), now.Add(certValidityDuration) dnsNames := []string{ - props.Service, - fmt.Sprintf("%s.%s", props.Service, props.Namespace), - props.inClusterServiceName(), + config.KyvernoServiceName(), + fmt.Sprintf("%s.%s", config.KyvernoServiceName(), config.KyvernoNamespace()), + InClusterServiceName(), } var ips []net.IP if serverIP != "" { @@ -74,7 +76,7 @@ func GenerateCert(caCert *KeyPair, props *CertificateProps, serverIP string, cer templ := &x509.Certificate{ SerialNumber: big.NewInt(1), Subject: pkix.Name{ - CommonName: props.Service, + CommonName: config.KyvernoServiceName(), }, DNSNames: dnsNames, IPAddresses: ips, @@ -88,7 +90,7 @@ func GenerateCert(caCert *KeyPair, props *CertificateProps, serverIP string, cer if err != nil { return nil, fmt.Errorf("error generating key for webhook %v", err) } - der, err := x509.CreateCertificate(rand.Reader, templ, caCert.Cert, key.Public(), caCert.Key) + der, err := x509.CreateCertificate(rand.Reader, templ, caCert.cert, key.Public(), caCert.key) if err != nil { return nil, fmt.Errorf("error creating certificate for webhook %v", err) } @@ -96,8 +98,8 @@ func GenerateCert(caCert *KeyPair, props *CertificateProps, serverIP string, cer if err != nil { return nil, fmt.Errorf("error parsing webhook certificate %v", err) } - return &KeyPair{ - Cert: cert, - Key: key, + return &keyPair{ + cert: cert, + key: key, }, nil } diff --git a/pkg/tls/pempair.go b/pkg/tls/pempair.go deleted file mode 100644 index 6ab70b714a..0000000000 --- a/pkg/tls/pempair.go +++ /dev/null @@ -1,25 +0,0 @@ -package tls - -import ( - "crypto/rsa" - "crypto/x509" - "encoding/pem" -) - -// PrivateKeyToPem Creates PEM block from private key object -func PrivateKeyToPem(rsaKey *rsa.PrivateKey) []byte { - privateKey := &pem.Block{ - Type: "PRIVATE KEY", - Bytes: x509.MarshalPKCS1PrivateKey(rsaKey), - } - return pem.EncodeToMemory(privateKey) -} - -// CertificateToPem Creates PEM block from certificate object -func CertificateToPem(cert *x509.Certificate) []byte { - certificate := &pem.Block{ - Type: "CERTIFICATE", - Bytes: cert.Raw, - } - return pem.EncodeToMemory(certificate) -} diff --git a/pkg/tls/props.go b/pkg/tls/props.go index e3c16d80ad..0c677b9774 100644 --- a/pkg/tls/props.go +++ b/pkg/tls/props.go @@ -3,40 +3,21 @@ package tls import ( "net/url" - "github.com/kyverno/kyverno/pkg/config" "k8s.io/client-go/rest" ) -// CertificateProps Properties of TLS certificate which should be issued for webhook server -type CertificateProps struct { - Service string - Namespace string - APIServerHost string +// certificateProps Properties of TLS certificate which should be issued for webhook server +type certificateProps struct { + apiServerHost string } -// NewCertificateProps creates CertificateProps from a *rest.Config -func NewCertificateProps(configuration *rest.Config) (*CertificateProps, error) { +// newCertificateProps creates CertificateProps from a *rest.Config +func newCertificateProps(configuration *rest.Config) (*certificateProps, error) { apiServerURL, err := url.Parse(configuration.Host) if err != nil { return nil, err } - return &CertificateProps{ - Service: config.KyvernoServiceName(), - Namespace: config.KyvernoNamespace(), - APIServerHost: apiServerURL.Hostname(), + return &certificateProps{ + apiServerHost: apiServerURL.Hostname(), }, nil } - -// inClusterServiceName The generated service name should be the common name for TLS certificate -// TODO: could be static -func (props *CertificateProps) inClusterServiceName() string { - return props.Service + "." + props.Namespace + ".svc" -} - -func (props *CertificateProps) GenerateTLSPairSecretName() string { - return props.inClusterServiceName() + ".kyverno-tls-pair" -} - -func (props *CertificateProps) GenerateRootCASecretName() string { - return props.inClusterServiceName() + ".kyverno-tls-ca" -} diff --git a/pkg/tls/reader.go b/pkg/tls/reader.go index bdeb0a1268..3225cf4576 100644 --- a/pkg/tls/reader.go +++ b/pkg/tls/reader.go @@ -2,48 +2,23 @@ package tls import ( "context" - "fmt" "github.com/kyverno/kyverno/pkg/config" "github.com/pkg/errors" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" ) var ErrorsNotFound = "root CA certificate not found" // ReadRootCASecret returns the RootCA from the pre-defined secret -func ReadRootCASecret(restConfig *rest.Config, client kubernetes.Interface) ([]byte, error) { - certProps, err := NewCertificateProps(restConfig) - if err != nil { - return nil, errors.Wrap(err, "failed to get TLS Cert Properties") - } - - depl, err := client.AppsV1().Deployments(certProps.Namespace).Get(context.TODO(), config.KyvernoDeploymentName(), metav1.GetOptions{}) - - deplHash := "" - if err == nil { - deplHash = fmt.Sprintf("%v", depl.GetUID()) - } - - var deplHashSec string - var ok, managedByKyverno bool - - sname := certProps.GenerateRootCASecretName() - stlsca, err := client.CoreV1().Secrets(certProps.Namespace).Get(context.TODO(), sname, metav1.GetOptions{}) +func ReadRootCASecret(client kubernetes.Interface) ([]byte, error) { + sname := GenerateRootCASecretName() + stlsca, err := client.CoreV1().Secrets(config.KyvernoNamespace()).Get(context.TODO(), sname, metav1.GetOptions{}) if err != nil { return nil, err } - - if label, ok := stlsca.GetLabels()[ManagedByLabel]; ok { - managedByKyverno = label == "kyverno" - } - deplHashSec, ok = stlsca.GetAnnotations()[MasterDeploymentUID] - if managedByKyverno && (ok && deplHashSec != deplHash) { - return nil, fmt.Errorf("outdated secret") - } // try "tls.crt" result := stlsca.Data[v1.TLSCertKey] // if not there, try old "rootCA.crt" @@ -51,57 +26,7 @@ func ReadRootCASecret(restConfig *rest.Config, client kubernetes.Interface) ([]b result = stlsca.Data[rootCAKey] } if len(result) == 0 { - return nil, errors.Errorf("%s in secret %s/%s", ErrorsNotFound, certProps.Namespace, stlsca.Name) + return nil, errors.Errorf("%s in secret %s/%s", ErrorsNotFound, config.KyvernoNamespace(), stlsca.Name) } return result, nil } - -// ReadTLSPair returns the pem pair from the pre-defined secret -func ReadTLSPair(restConfig *rest.Config, client kubernetes.Interface) ([]byte, []byte, error) { - certProps, err := NewCertificateProps(restConfig) - if err != nil { - return nil, nil, errors.Wrap(err, "failed to get TLS Cert Properties") - } - - depl, err := client.AppsV1().Deployments(certProps.Namespace).Get(context.TODO(), config.KyvernoDeploymentName(), metav1.GetOptions{}) - - deplHash := "" - if err == nil { - deplHash = fmt.Sprintf("%v", depl.GetUID()) - } - - var deplHashSec string - var ok, managedByKyverno bool - - sname := certProps.GenerateTLSPairSecretName() - secret, err := client.CoreV1().Secrets(certProps.Namespace).Get(context.TODO(), sname, metav1.GetOptions{}) - if err != nil { - return nil, nil, fmt.Errorf("failed to get secret %s/%s: %v", certProps.Namespace, sname, err) - } - if label, ok := secret.GetLabels()[ManagedByLabel]; ok { - managedByKyverno = label == "kyverno" - } - deplHashSec, ok = secret.GetAnnotations()[MasterDeploymentUID] - if managedByKyverno && (ok && deplHashSec != deplHash) { - return nil, nil, fmt.Errorf("outdated secret") - } - - // If secret contains annotation 'self-signed-cert', then it's created using helper scripts to setup self-signed certificates. - // As the root CA used to sign the certificate is required for webhook configuration, check if the corresponding secret is created - { - sname := certProps.GenerateRootCASecretName() - _, err := client.CoreV1().Secrets(certProps.Namespace).Get(context.TODO(), sname, metav1.GetOptions{}) - if err != nil { - return nil, nil, fmt.Errorf("rootCA secret is required while using self-signed certificate TLS pair, defaulting to generating new TLS pair %s/%s", certProps.Namespace, sname) - } - } - - if len(secret.Data[v1.TLSCertKey]) == 0 { - return nil, nil, fmt.Errorf("TLS Certificate not found in secret %s/%s", certProps.Namespace, sname) - } - if len(secret.Data[v1.TLSPrivateKeyKey]) == 0 { - return nil, nil, fmt.Errorf("TLS PrivateKey not found in secret %s/%s", certProps.Namespace, sname) - } - - return secret.Data[v1.TLSCertKey], secret.Data[v1.TLSPrivateKeyKey], nil -} diff --git a/pkg/tls/renewer.go b/pkg/tls/renewer.go new file mode 100644 index 0000000000..f516afb3c6 --- /dev/null +++ b/pkg/tls/renewer.go @@ -0,0 +1,287 @@ +package tls + +import ( + "context" + "crypto/tls" + "crypto/x509" + "encoding/pem" + "fmt" + "time" + + "github.com/cenkalti/backoff" + "github.com/go-logr/logr" + "github.com/kyverno/kyverno/pkg/config" + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" +) + +const ( + // CertRenewalInterval is the renewal interval for rootCA + CertRenewalInterval time.Duration = 12 * time.Hour + // CertValidityDuration is the valid duration for a new cert + CertValidityDuration time.Duration = 365 * 24 * time.Hour + // ManagedByLabel is added to Kyverno managed secrets + ManagedByLabel string = "cert.kyverno.io/managed-by" + rootCAKey string = "rootCA.crt" + rollingUpdateAnnotation string = "update.kyverno.io/force-rolling-update" +) + +// CertRenewer creates rootCA and pem pair to register +// webhook configurations and webhook server +// renews RootCA at the given interval +type CertRenewer struct { + client kubernetes.Interface + certRenewalInterval time.Duration + certValidityDuration time.Duration + certProps *certificateProps + + // IP address where Kyverno controller runs. Only required if out-of-cluster. + serverIP string + + log logr.Logger +} + +// NewCertRenewer returns an instance of CertRenewer +func NewCertRenewer(client kubernetes.Interface, clientConfig *rest.Config, certRenewalInterval, certValidityDuration time.Duration, serverIP string, log logr.Logger) (*CertRenewer, error) { + certProps, err := newCertificateProps(clientConfig) + if err != nil { + return nil, err + } + return &CertRenewer{ + client: client, + certRenewalInterval: certRenewalInterval, + certValidityDuration: certValidityDuration, + certProps: certProps, + serverIP: serverIP, + log: log, + }, nil +} + +// InitTLSPemPair Loads or creates PEM private key and TLS certificate for webhook server. +// Created pair is stored in cluster's secret. +func (c *CertRenewer) InitTLSPemPair() error { + logger := c.log.WithName("InitTLSPemPair") + ca, err := c.getCASecret() + if err != nil && !apierrors.IsNotFound(err) { + return err + } + tls, err := c.getTLSSecret() + if err != nil && !apierrors.IsNotFound(err) { + return err + } + // check they are valid + if ca != nil && tls != nil { + if validCert(ca, tls, logger) { + return nil + } + } + // if not valid, check we can renew them + if !IsSecretManagedByKyverno(ca) || !IsSecretManagedByKyverno(tls) { + return fmt.Errorf("tls is not valid but certificates are not managed by kyverno, we can't renew them") + } + // renew them + logger.Info("building key/certificate pair for TLS") + return c.buildTLSPemPairAndWriteToSecrets(c.serverIP) +} + +// buildTLSPemPairAndWriteToSecrets Issues TLS certificate for webhook server using self-signed CA cert +// Returns signed and approved TLS certificate in PEM format +func (c *CertRenewer) buildTLSPemPairAndWriteToSecrets(serverIP string) error { + caCert, err := generateCA(c.certValidityDuration) + if err != nil { + return err + } + tlsPair, err := generateCert(caCert, c.certProps, serverIP, c.certValidityDuration) + if err != nil { + return err + } + if err := c.writeCASecret(caCert); err != nil { + return fmt.Errorf("failed to write CA cert to secret: %v", err) + } + if err = c.writeTLSSecret(tlsPair); err != nil { + return fmt.Errorf("unable to save TLS pair to the cluster: %v", err) + } + return nil +} + +func (c *CertRenewer) getSecret(name string) (*corev1.Secret, error) { + if s, err := c.client.CoreV1().Secrets(config.KyvernoNamespace()).Get(context.TODO(), name, metav1.GetOptions{}); err != nil { + return nil, err + } else { + return s, nil + } +} + +func (c *CertRenewer) getCASecret() (*corev1.Secret, error) { + return c.getSecret(GenerateRootCASecretName()) +} + +func (c *CertRenewer) getTLSSecret() (*corev1.Secret, error) { + return c.getSecret(GenerateTLSPairSecretName()) +} + +func (c *CertRenewer) writeSecret(secret *corev1.Secret, logger logr.Logger) error { + logger = logger.WithValues("name", secret.GetName(), "namespace", secret.GetNamespace()) + if _, err := c.client.CoreV1().Secrets(config.KyvernoNamespace()).Create(context.TODO(), secret, metav1.CreateOptions{}); err != nil { + if apierrors.IsAlreadyExists(err) { + if _, err := c.client.CoreV1().Secrets(config.KyvernoNamespace()).Update(context.TODO(), secret, metav1.UpdateOptions{}); err != nil { + logger.Error(err, "failed to update secret") + return err + } else { + logger.Info("secret updated") + return nil + } + } else { + logger.Error(err, "failed to create secret") + return err + } + } else { + logger.Info("secret created") + return nil + } +} + +// writeCASecret stores the CA cert in secret +func (c *CertRenewer) writeCASecret(ca *keyPair) error { + logger := c.log.WithName("writeCASecret") + secret, err := c.getCASecret() + if err != nil && !apierrors.IsNotFound(err) { + logger.Error(err, "failed to get CA secret") + return err + } + if secret == nil { + secret = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: GenerateRootCASecretName(), + Namespace: config.KyvernoNamespace(), + Labels: map[string]string{ + ManagedByLabel: "kyverno", + }, + }, + Type: corev1.SecretTypeTLS, + } + } + secret.Type = corev1.SecretTypeTLS + secret.Data = map[string][]byte{ + corev1.TLSCertKey: CertificateToPem(ca.cert), + corev1.TLSPrivateKeyKey: PrivateKeyToPem(ca.key), + } + return c.writeSecret(secret, logger) +} + +// writeTLSSecret Writes the pair of TLS certificate and key to the specified secret. +func (c *CertRenewer) writeTLSSecret(tls *keyPair) error { + logger := c.log.WithName("writeTLSSecret") + secret, err := c.getTLSSecret() + if err != nil && !apierrors.IsNotFound(err) { + logger.Error(err, "failed to get TLS secret") + return err + } + if secret == nil { + secret = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: GenerateTLSPairSecretName(), + Namespace: config.KyvernoNamespace(), + Labels: map[string]string{ + ManagedByLabel: "kyverno", + }, + }, + Type: corev1.SecretTypeTLS, + } + } + secret.Data = map[string][]byte{ + corev1.TLSCertKey: CertificateToPem(tls.cert), + corev1.TLSPrivateKeyKey: PrivateKeyToPem(tls.key), + } + return c.writeSecret(secret, logger) +} + +// ValidCert validates the CA Cert +func (c *CertRenewer) ValidCert() (bool, error) { + logger := c.log.WithName("validCert") + ca, err := c.getCASecret() + if err != nil { + logger.Error(err, "unable to read CA secret") + return false, err + } + tls, err := c.getTLSSecret() + if err != nil { + logger.Error(err, "unable to read TLS secret") + return false, err + } + return validCert(ca, tls, logger), nil +} + +func validCert(caSecret *corev1.Secret, tlsSecret *corev1.Secret, logger logr.Logger) bool { + caPem := caSecret.Data[corev1.TLSCertKey] + if len(caPem) == 0 { + caPem = caSecret.Data[rootCAKey] + } + // build cert pool + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(caPem) { + logger.Info("bad certificate") + return false + } + // valid PEM pair + _, err := tls.X509KeyPair(tlsSecret.Data[corev1.TLSCertKey], tlsSecret.Data[corev1.TLSPrivateKeyKey]) + if err != nil { + logger.Error(err, "invalid PEM pair") + return false + } + certPemBlock, _ := pem.Decode(tlsSecret.Data[corev1.TLSCertKey]) + if certPemBlock == nil { + logger.Error(err, "bad private key") + return false + } + cert, err := x509.ParseCertificate(certPemBlock.Bytes) + if err != nil { + logger.Error(err, "failed to parse cert") + return false + } + if _, err = cert.Verify(x509.VerifyOptions{ + Roots: pool, + CurrentTime: time.Now(), + }); err != nil { + logger.Error(err, "invalid cert") + return false + } + return true +} + +// RollingUpdate triggers a rolling update of Kyverno pod. +// It is used when the rootCA is renewed, the restart of +// Kyverno pod will register webhook server with new cert +func (c *CertRenewer) RollingUpdate() error { + update := func() error { + deploy, err := c.client.AppsV1().Deployments(config.KyvernoNamespace()).Get(context.TODO(), config.KyvernoDeploymentName(), metav1.GetOptions{}) + if err != nil { + return errors.Wrap(err, "failed to find Kyverno") + } + if IsKyvernoInRollingUpdate(deploy, c.log) { + return nil + } + if deploy.Spec.Template.Annotations == nil { + deploy.Spec.Template.Annotations = map[string]string{} + } + deploy.Spec.Template.Annotations[rollingUpdateAnnotation] = time.Now().String() + if _, err = c.client.AppsV1().Deployments(config.KyvernoNamespace()).Update(context.TODO(), deploy, metav1.UpdateOptions{}); err != nil { + return errors.Wrap(err, "update Kyverno deployment") + } + return nil + } + exbackoff := &backoff.ExponentialBackOff{ + InitialInterval: 500 * time.Millisecond, + RandomizationFactor: 0.5, + Multiplier: 1.5, + MaxInterval: time.Second, + MaxElapsedTime: 3 * time.Second, + Clock: backoff.SystemClock, + } + exbackoff.Reset() + return backoff.Retry(update, exbackoff) +} diff --git a/pkg/tls/utils.go b/pkg/tls/utils.go index 17c899b9a0..5952308684 100644 --- a/pkg/tls/utils.go +++ b/pkg/tls/utils.go @@ -1,11 +1,34 @@ package tls import ( + "crypto/rsa" + "crypto/x509" + "encoding/pem" + "github.com/go-logr/logr" + "github.com/kyverno/kyverno/pkg/config" appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" ) +// PrivateKeyToPem Creates PEM block from private key object +func PrivateKeyToPem(rsaKey *rsa.PrivateKey) []byte { + privateKey := &pem.Block{ + Type: "PRIVATE KEY", + Bytes: x509.MarshalPKCS1PrivateKey(rsaKey), + } + return pem.EncodeToMemory(privateKey) +} + +// CertificateToPem Creates PEM block from certificate object +func CertificateToPem(cert *x509.Certificate) []byte { + certificate := &pem.Block{ + Type: "CERTIFICATE", + Bytes: cert.Raw, + } + return pem.EncodeToMemory(certificate) +} + // IsKyvernoInRollingUpdate returns true if Kyverno is in rolling update func IsKyvernoInRollingUpdate(deploy *appsv1.Deployment, logger logr.Logger) bool { var replicas int32 = 1 @@ -20,14 +43,28 @@ func IsKyvernoInRollingUpdate(deploy *appsv1.Deployment, logger logr.Logger) boo return false } -func CanAddAnnotationToSecret(deplHash string, secret *v1.Secret) bool { - var deplHashSec string - var ok, managedByKyverno bool - - if label, ok := secret.GetLabels()[ManagedByLabel]; ok { - managedByKyverno = label == "kyverno" +func IsSecretManagedByKyverno(secret *v1.Secret) bool { + if secret != nil { + labels := secret.GetLabels() + if labels == nil { + return false + } + if labels[ManagedByLabel] != "kyverno" { + return false + } } - deplHashSec, ok = secret.GetAnnotations()[MasterDeploymentUID] - - return managedByKyverno && (!ok || deplHashSec != deplHash) + return true +} + +// InClusterServiceName The generated service name should be the common name for TLS certificate +func InClusterServiceName() string { + return config.KyvernoServiceName() + "." + config.KyvernoNamespace() + ".svc" +} + +func GenerateTLSPairSecretName() string { + return InClusterServiceName() + ".kyverno-tls-pair" +} + +func GenerateRootCASecretName() string { + return InClusterServiceName() + ".kyverno-tls-ca" } diff --git a/pkg/webhookconfig/common.go b/pkg/webhookconfig/common.go index 913b84dc2b..5a351af9f0 100644 --- a/pkg/webhookconfig/common.go +++ b/pkg/webhookconfig/common.go @@ -47,7 +47,7 @@ func (wrc *Register) readCaData() []byte { // Check if ca is defined in the secret tls-ca // assume the key and signed cert have been defined in secret tls.kyverno - if caData, err = tls.ReadRootCASecret(wrc.clientConfig, wrc.kubeClient); err == nil { + if caData, err = tls.ReadRootCASecret(wrc.kubeClient); err == nil { logger.V(4).Info("read CA from secret") return caData }