mirrors/kyverno
1
0
Fork 0
mirror of https://github.com/kyverno/kyverno.git synced 2024-12-14 11:57:48 +00:00
Code Activity

refactor: remove deployment hash on certs secrets (#3886)

Browse source 
Signed-off-by: Charles-Edouard Brétéché <charled.breteche@gmail.com>
This commit is contained in:
Charles-Edouard Brétéché 2022-05-11 16:58:14 +02:00 committed by GitHub
parent 747f4128ef
commit 8f825bb040
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 376 additions and 557 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

39
cmd/initContainer/main.go
View file

@ -6,7 +6,6 @@ package main
import (
"context"
"flag"
"fmt"
"os"
"sync"
"time"
@ -131,52 +130,22 @@ func main() {
failure := false
run := func() {
certProps, err := tls.NewCertificateProps(clientConfig)
if err != nil {
log.Log.Info("failed to get cert properties: %v", err.Error())
os.Exit(1)
}
depl, err := kubeClient.AppsV1().Deployments(config.KyvernoNamespace()).Get(context.TODO(), config.KyvernoDeploymentName(), metav1.GetOptions{})
deplHash := ""
if err != nil {
log.Log.Info("failed to fetch deployment '%v': %v", config.KyvernoDeploymentName(), err.Error())
os.Exit(1)
}
deplHash = fmt.Sprintf("%v", depl.GetUID())
name := certProps.GenerateRootCASecretName()
secret, err := kubeClient.CoreV1().Secrets(config.KyvernoNamespace()).Get(context.TODO(), name, metav1.GetOptions{})
name := tls.GenerateRootCASecretName()
_, err = kubeClient.CoreV1().Secrets(config.KyvernoNamespace()).Get(context.TODO(), name, metav1.GetOptions{})
if err != nil {
log.Log.Info("failed to fetch root CA secret", "name", name, "error", err.Error())
if !errors.IsNotFound(err) {
os.Exit(1)
}
} else if tls.CanAddAnnotationToSecret(deplHash, secret) {
secret.SetAnnotations(map[string]string{tls.MasterDeploymentUID: deplHash})
_, err = kubeClient.CoreV1().Secrets(config.KyvernoNamespace()).Update(context.TODO(), secret, metav1.UpdateOptions{})
if err != nil {
log.Log.Info("failed to update cert: %v", err.Error())
os.Exit(1)
}
}
name = certProps.GenerateTLSPairSecretName()
secret, err = kubeClient.CoreV1().Secrets(config.KyvernoNamespace()).Get(context.TODO(), name, metav1.GetOptions{})
name = tls.GenerateTLSPairSecretName()
_, err = kubeClient.CoreV1().Secrets(config.KyvernoNamespace()).Get(context.TODO(), name, metav1.GetOptions{})
if err != nil {
log.Log.Info("failed to fetch TLS Pair secret", "name", name, "error", err.Error())
if !errors.IsNotFound(err) {
os.Exit(1)
}
} else if tls.CanAddAnnotationToSecret(deplHash, secret) {
secret.SetAnnotations(map[string]string{tls.MasterDeploymentUID: deplHash})
_, err = kubeClient.CoreV1().Secrets(certProps.Namespace).Update(context.TODO(), secret, metav1.UpdateOptions{})
if err != nil {
log.Log.Info("failed to update cert: %v", err.Error())
os.Exit(1)
}
}
if err = acquireLeader(ctx, kubeClient); err != nil {

12
pkg/controllers/certmanager/controller.go
View file

@ -3,12 +3,12 @@ package certmanager
import (
"os"
"reflect"
"strings"
"time"
"github.com/kyverno/kyverno/pkg/config"
"github.com/kyverno/kyverno/pkg/tls"
v1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
informerv1 "k8s.io/client-go/informers/core/v1"
listersv1 "k8s.io/client-go/listers/core/v1"
"k8s.io/client-go/tools/cache"
@ -43,7 +43,7 @@ func NewController(secretInformer informerv1.SecretInformer, certRenewer *tls.Ce
func (m *controller) addSecretFunc(obj interface{}) {
secret := obj.(*v1.Secret)
if secret.GetNamespace() == config.KyvernoNamespace() && secret.GetName() == m.renewer.GenerateTLSPairSecretName() {
if secret.GetNamespace() == config.KyvernoNamespace() && secret.GetName() == tls.GenerateTLSPairSecretName() {
m.secretQueue <- true
}
}
@ -51,7 +51,7 @@ func (m *controller) addSecretFunc(obj interface{}) {
func (m *controller) updateSecretFunc(oldObj interface{}, newObj interface{}) {
old := oldObj.(*v1.Secret)
new := newObj.(*v1.Secret)
if new.GetNamespace() == config.KyvernoNamespace() && new.GetName() == m.renewer.GenerateTLSPairSecretName() {
if new.GetNamespace() == config.KyvernoNamespace() && new.GetName() == tls.GenerateTLSPairSecretName() {
if !reflect.DeepEqual(old.DeepCopy().Data, new.DeepCopy().Data) {
m.secretQueue <- true
logger.V(4).Info("secret updated, reconciling webhook configurations")
@ -60,7 +60,7 @@ func (m *controller) updateSecretFunc(oldObj interface{}, newObj interface{}) {
}
func (m *controller) GetTLSPemPair() ([]byte, []byte, error) {
secret, err := m.secretLister.Secrets(config.KyvernoNamespace()).Get(m.renewer.GenerateTLSPairSecretName())
secret, err := m.secretLister.Secrets(config.KyvernoNamespace()).Get(tls.GenerateTLSPairSecretName())
if err != nil {
return nil, nil, err
}
@ -70,10 +70,10 @@ func (m *controller) GetTLSPemPair() ([]byte, []byte, error) {
func (m *controller) validateCerts() error {
valid, err := m.renewer.ValidCert()
if err != nil {
logger.Error(err, "failed to validate cert")
if !strings.Contains(err.Error(), tls.ErrorsNotFound) {
if apierrors.IsNotFound(err) {
return nil
}
logger.Error(err, "failed to validate cert")
}
if !valid {
logger.Info("rootCA has changed or is about to expire, trigger a rolling update to renew the cert")

357
pkg/tls/certRenewer.go
View file

@ -1,357 +0,0 @@
package tls
import (
"context"
"crypto/tls"
"crypto/x509"
"encoding/pem"
"fmt"
"time"
"github.com/cenkalti/backoff"
"github.com/go-logr/logr"
"github.com/kyverno/kyverno/pkg/config"
"github.com/pkg/errors"
v1 "k8s.io/api/core/v1"
k8errors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
)
const (
// CertRenewalInterval is the renewal interval for rootCA
CertRenewalInterval time.Duration = 12 * time.Hour
// CertValidityDuration is the valid duration for a new cert
CertValidityDuration time.Duration = 365 * 24 * time.Hour
// ManagedByLabel is added to Kyverno managed secrets
ManagedByLabel string = "cert.kyverno.io/managed-by"
MasterDeploymentUID string = "cert.kyverno.io/master-deployment-uid"
rootCAKey string = "rootCA.crt"
rollingUpdateAnnotation string = "update.kyverno.io/force-rolling-update"
)
// CertRenewer creates rootCA and pem pair to register
// webhook configurations and webhook server
// renews RootCA at the given interval
type CertRenewer struct {
client kubernetes.Interface
clientConfig *rest.Config
certRenewalInterval time.Duration
certValidityDuration time.Duration
certProps *CertificateProps
// IP address where Kyverno controller runs. Only required if out-of-cluster.
serverIP string
log logr.Logger
}
// NewCertRenewer returns an instance of CertRenewer
func NewCertRenewer(client kubernetes.Interface, clientConfig *rest.Config, certRenewalInterval, certValidityDuration time.Duration, serverIP string, log logr.Logger) (*CertRenewer, error) {
certProps, err := NewCertificateProps(clientConfig)
if err != nil {
return nil, err
}
return &CertRenewer{
client: client,
clientConfig: clientConfig,
certRenewalInterval: certRenewalInterval,
certValidityDuration: certValidityDuration,
certProps: certProps,
serverIP: serverIP,
log: log,
}, nil
}
func (c *CertRenewer) Client() kubernetes.Interface {
return c.client
}
func (c *CertRenewer) ClientConfig() *rest.Config {
return c.clientConfig
}
func (c *CertRenewer) GenerateTLSPairSecretName() string {
return c.certProps.GenerateTLSPairSecretName()
}
func (c *CertRenewer) GenerateRootCASecretName() string {
return c.certProps.GenerateRootCASecretName()
}
// InitTLSPemPair Loads or creates PEM private key and TLS certificate for webhook server.
// Created pair is stored in cluster's secret.
// Returns struct with key/certificate pair.
func (c *CertRenewer) InitTLSPemPair() error {
logger := c.log.WithName("InitTLSPemPair")
if valid, err := c.ValidCert(); err == nil && valid {
if _, _, err := ReadTLSPair(c.clientConfig, c.client); err == nil {
logger.Info("using existing TLS key/certificate pair")
return nil
}
} else if err != nil {
logger.V(3).Info("unable to find TLS pair", "reason", err.Error())
}
logger.Info("building key/certificate pair for TLS")
return c.buildTLSPemPairAndWriteToSecrets(c.serverIP)
}
// buildTLSPemPairAndWriteToSecrets Issues TLS certificate for webhook server using self-signed CA cert
// Returns signed and approved TLS certificate in PEM format
func (c *CertRenewer) buildTLSPemPairAndWriteToSecrets(serverIP string) error {
caCert, err := GenerateCA(c.certValidityDuration)
if err != nil {
return err
}
tlsPair, err := GenerateCert(caCert, c.certProps, serverIP, c.certValidityDuration)
if err != nil {
return err
}
if err := c.WriteCACertToSecret(caCert); err != nil {
return fmt.Errorf("failed to write CA cert to secret: %v", err)
}
if err = c.WriteTLSPairToSecret(tlsPair); err != nil {
return fmt.Errorf("unable to save TLS pair to the cluster: %v", err)
}
return nil
}
// WriteCACertToSecret stores the CA cert in secret
func (c *CertRenewer) WriteCACertToSecret(ca *KeyPair) error {
logger := c.log.WithName("CAcert")
name := c.certProps.GenerateRootCASecretName()
caBytes := CertificateToPem(ca.Cert)
keyBytes := PrivateKeyToPem(ca.Key)
depl, err := c.client.AppsV1().Deployments(c.certProps.Namespace).Get(context.TODO(), config.KyvernoDeploymentName(), metav1.GetOptions{})
deplHash := ""
if err == nil {
deplHash = fmt.Sprintf("%v", depl.GetUID())
}
secret, err := c.client.CoreV1().Secrets(c.certProps.Namespace).Get(context.TODO(), name, metav1.GetOptions{})
if err != nil {
if k8errors.IsNotFound(err) {
secret = &v1.Secret{
TypeMeta: metav1.TypeMeta{
Kind: "Secret",
APIVersion: "v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: c.certProps.Namespace,
Annotations: map[string]string{
MasterDeploymentUID: deplHash,
},
Labels: map[string]string{
ManagedByLabel: "kyverno",
},
},
Data: map[string][]byte{
v1.TLSCertKey: caBytes,
v1.TLSPrivateKeyKey: keyBytes,
},
Type: v1.SecretTypeTLS,
}
_, err = c.client.CoreV1().Secrets(c.certProps.Namespace).Create(context.TODO(), secret, metav1.CreateOptions{})
if err == nil {
logger.Info("secret created", "name", name, "namespace", c.certProps.Namespace)
}
}
return err
} else if CanAddAnnotationToSecret(deplHash, secret) {
_, err = c.client.CoreV1().Secrets(c.certProps.Namespace).Update(context.TODO(), secret, metav1.UpdateOptions{})
if err == nil {
logger.Info("secret updated", "name", name, "namespace", c.certProps.Namespace)
}
return err
}
secret.Type = v1.SecretTypeTLS
secret.Data = map[string][]byte{
v1.TLSCertKey: caBytes,
v1.TLSPrivateKeyKey: keyBytes,
}
_, err = c.client.CoreV1().Secrets(c.certProps.Namespace).Update(context.TODO(), secret, metav1.UpdateOptions{})
if err != nil {
return err
}
logger.Info("secret updated", "name", name, "namespace", c.certProps.Namespace)
return nil
}
// WriteTLSPairToSecret Writes the pair of TLS certificate and key to the specified secret.
// Updates existing secret or creates new one.
func (c *CertRenewer) WriteTLSPairToSecret(tls *KeyPair) error {
logger := c.log.WithName("WriteTLSPair")
name := c.certProps.GenerateTLSPairSecretName()
certBytes := CertificateToPem(tls.Cert)
keyBytes := PrivateKeyToPem(tls.Key)
depl, err := c.client.AppsV1().Deployments(c.certProps.Namespace).Get(context.TODO(), config.KyvernoDeploymentName(), metav1.GetOptions{})
deplHash := ""
if err == nil {
deplHash = fmt.Sprintf("%v", depl.GetUID())
}
secret, err := c.client.CoreV1().Secrets(c.certProps.Namespace).Get(context.TODO(), name, metav1.GetOptions{})
if err != nil {
if k8errors.IsNotFound(err) {
secret = &v1.Secret{
TypeMeta: metav1.TypeMeta{
Kind: "Secret",
APIVersion: "v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: c.certProps.Namespace,
Annotations: map[string]string{
MasterDeploymentUID: deplHash,
},
Labels: map[string]string{
ManagedByLabel: "kyverno",
},
},
Data: map[string][]byte{
v1.TLSCertKey: certBytes,
v1.TLSPrivateKeyKey: keyBytes,
},
Type: v1.SecretTypeTLS,
}
_, err = c.client.CoreV1().Secrets(c.certProps.Namespace).Create(context.TODO(), secret, metav1.CreateOptions{})
if err == nil {
logger.Info("secret created", "name", name, "namespace", c.certProps.Namespace)
}
}
return err
} else if CanAddAnnotationToSecret(deplHash, secret) {
_, err = c.client.CoreV1().Secrets(c.certProps.Namespace).Update(context.TODO(), secret, metav1.UpdateOptions{})
if err == nil {
logger.Info("secret updated", "name", name, "namespace", c.certProps.Namespace)
}
return err
}
secret.Data = map[string][]byte{
v1.TLSCertKey: certBytes,
v1.TLSPrivateKeyKey: keyBytes,
}
_, err = c.client.CoreV1().Secrets(c.certProps.Namespace).Update(context.TODO(), secret, metav1.UpdateOptions{})
if err != nil {
return err
}
logger.Info("secret updated", "name", name, "namespace", c.certProps.Namespace)
return nil
}
// ValidCert validates the CA Cert
func (c *CertRenewer) ValidCert() (bool, error) {
logger := c.log.WithName("ValidCert")
var managedByKyverno bool
snameTLS := c.certProps.GenerateTLSPairSecretName()
snameCA := c.certProps.GenerateRootCASecretName()
secret, err := c.client.CoreV1().Secrets(c.certProps.Namespace).Get(context.TODO(), snameTLS, metav1.GetOptions{})
if err != nil {
return false, nil
}
if label, ok := secret.GetLabels()[ManagedByLabel]; ok {
managedByKyverno = label == "kyverno"
}
_, ok := secret.GetAnnotations()[MasterDeploymentUID]
if managedByKyverno && !ok {
return false, nil
}
secret, err = c.client.CoreV1().Secrets(c.certProps.Namespace).Get(context.TODO(), snameCA, metav1.GetOptions{})
if err != nil {
return false, nil
}
if label, ok := secret.GetLabels()[ManagedByLabel]; ok {
managedByKyverno = label == "kyverno"
}
_, ok = secret.GetAnnotations()[MasterDeploymentUID]
if managedByKyverno && !ok {
return false, nil
}
rootCA, err := ReadRootCASecret(c.clientConfig, c.client)
if err != nil {
return false, errors.Wrap(err, "unable to read CA from secret")
}
certPem, keyPem, err := ReadTLSPair(c.clientConfig, c.client)
if err != nil {
// wait till next reconcile
logger.Info("unable to read TLS PEM Pair from secret", "reason", err.Error())
return false, errors.Wrap(err, "unable to read TLS PEM Pair from secret")
}
// build cert pool
pool := x509.NewCertPool()
if !pool.AppendCertsFromPEM(rootCA) {
logger.Error(err, "bad certificate")
return false, nil
}
// valid PEM pair
_, err = tls.X509KeyPair(certPem, keyPem)
if err != nil {
logger.Error(err, "invalid PEM pair")
return false, nil
}
certPemBlock, _ := pem.Decode(certPem)
if certPem == nil {
logger.Error(err, "bad private key")
return false, nil
}
cert, err := x509.ParseCertificate(certPemBlock.Bytes)
if err != nil {
logger.Error(err, "failed to parse cert")
return false, nil
}
if _, err = cert.Verify(x509.VerifyOptions{
Roots: pool,
CurrentTime: time.Now().Add(c.certRenewalInterval),
}); err != nil {
logger.Error(err, "invalid cert")
return false, nil
}
return true, nil
}
// RollingUpdate triggers a rolling update of Kyverno pod.
// It is used when the rootCA is renewed, the restart of
// Kyverno pod will register webhook server with new cert
func (c *CertRenewer) RollingUpdate() error {
update := func() error {
deploy, err := c.client.AppsV1().Deployments(config.KyvernoNamespace()).Get(context.TODO(), config.KyvernoDeploymentName(), metav1.GetOptions{})
if err != nil {
return errors.Wrap(err, "failed to find Kyverno")
}
if IsKyvernoInRollingUpdate(deploy, c.log) {
return nil
}
if deploy.Spec.Template.Annotations == nil {
deploy.Spec.Template.Annotations = map[string]string{}
}
deploy.Spec.Template.Annotations[rollingUpdateAnnotation] = time.Now().String()
if _, err = c.client.AppsV1().Deployments(config.KyvernoNamespace()).Update(context.TODO(), deploy, metav1.UpdateOptions{}); err != nil {
return errors.Wrap(err, "update Kyverno deployment")
}
return nil
}
exbackoff := &backoff.ExponentialBackOff{
InitialInterval: 500 * time.Millisecond,
RandomizationFactor: 0.5,
Multiplier: 1.5,
MaxInterval: time.Second,
MaxElapsedTime: 3 * time.Second,
Clock: backoff.SystemClock,
}
exbackoff.Reset()
return backoff.Retry(update, exbackoff)
}

40
pkg/tls/keypair.go
View file

@ -10,17 +10,19 @@ import (
"net"
"strings"
"time"
"github.com/kyverno/kyverno/pkg/config"
)
// KeyPair ...
type KeyPair struct {
Cert *x509.Certificate
Key *rsa.PrivateKey
// keyPair ...
type keyPair struct {
cert *x509.Certificate
key *rsa.PrivateKey
}
// GenerateCA creates the self-signed CA cert and private key
// generateCA creates the self-signed CA cert and private key
// it will be used to sign the webhook server certificate
func GenerateCA(certValidityDuration time.Duration) (*KeyPair, error) {
func generateCA(certValidityDuration time.Duration) (*keyPair, error) {
now := time.Now()
begin, end := now.Add(-1*time.Hour), now.Add(certValidityDuration)
key, err := rsa.GenerateKey(rand.Reader, 2048)
@ -46,21 +48,21 @@ func GenerateCA(certValidityDuration time.Duration) (*KeyPair, error) {
if err != nil {
return nil, fmt.Errorf("error parsing certificate %v", err)
}
return &KeyPair{
Cert: cert,
Key: key,
return &keyPair{
cert: cert,
key: key,
}, nil
}
// GenerateCert takes the results of GenerateCACert and uses it to create the
// generateCert takes the results of GenerateCACert and uses it to create the
// PEM-encoded public certificate and private key, respectively
func GenerateCert(caCert *KeyPair, props *CertificateProps, serverIP string, certValidityDuration time.Duration) (*KeyPair, error) {
func generateCert(caCert *keyPair, props *certificateProps, serverIP string, certValidityDuration time.Duration) (*keyPair, error) {
now := time.Now()
begin, end := now.Add(-1*time.Hour), now.Add(certValidityDuration)
dnsNames := []string{
props.Service,
fmt.Sprintf("%s.%s", props.Service, props.Namespace),
props.inClusterServiceName(),
config.KyvernoServiceName(),
fmt.Sprintf("%s.%s", config.KyvernoServiceName(), config.KyvernoNamespace()),
InClusterServiceName(),
}
var ips []net.IP
if serverIP != "" {
@ -74,7 +76,7 @@ func GenerateCert(caCert *KeyPair, props *CertificateProps, serverIP string, cer
templ := &x509.Certificate{
SerialNumber: big.NewInt(1),
Subject: pkix.Name{
CommonName: props.Service,
CommonName: config.KyvernoServiceName(),
},
DNSNames: dnsNames,
IPAddresses: ips,
@ -88,7 +90,7 @@ func GenerateCert(caCert *KeyPair, props *CertificateProps, serverIP string, cer
if err != nil {
return nil, fmt.Errorf("error generating key for webhook %v", err)
}
der, err := x509.CreateCertificate(rand.Reader, templ, caCert.Cert, key.Public(), caCert.Key)
der, err := x509.CreateCertificate(rand.Reader, templ, caCert.cert, key.Public(), caCert.key)
if err != nil {
return nil, fmt.Errorf("error creating certificate for webhook %v", err)
}
@ -96,8 +98,8 @@ func GenerateCert(caCert *KeyPair, props *CertificateProps, serverIP string, cer
if err != nil {
return nil, fmt.Errorf("error parsing webhook certificate %v", err)
}
return &KeyPair{
Cert: cert,
Key: key,
return &keyPair{
cert: cert,
key: key,
}, nil
}

25
pkg/tls/pempair.go
View file

@ -1,25 +0,0 @@
package tls
import (
"crypto/rsa"
"crypto/x509"
"encoding/pem"
)
// PrivateKeyToPem Creates PEM block from private key object
func PrivateKeyToPem(rsaKey *rsa.PrivateKey) []byte {
privateKey := &pem.Block{
Type: "PRIVATE KEY",
Bytes: x509.MarshalPKCS1PrivateKey(rsaKey),
}
return pem.EncodeToMemory(privateKey)
}
// CertificateToPem Creates PEM block from certificate object
func CertificateToPem(cert *x509.Certificate) []byte {
certificate := &pem.Block{
Type: "CERTIFICATE",
Bytes: cert.Raw,
}
return pem.EncodeToMemory(certificate)
}

33
pkg/tls/props.go
View file

@ -3,40 +3,21 @@ package tls
import (
"net/url"
"github.com/kyverno/kyverno/pkg/config"
"k8s.io/client-go/rest"
)
// CertificateProps Properties of TLS certificate which should be issued for webhook server
type CertificateProps struct {
Service string
Namespace string
APIServerHost string
// certificateProps Properties of TLS certificate which should be issued for webhook server
type certificateProps struct {
apiServerHost string
}
// NewCertificateProps creates CertificateProps from a *rest.Config
func NewCertificateProps(configuration *rest.Config) (*CertificateProps, error) {
// newCertificateProps creates CertificateProps from a *rest.Config
func newCertificateProps(configuration *rest.Config) (*certificateProps, error) {
apiServerURL, err := url.Parse(configuration.Host)
if err != nil {
return nil, err
}
return &CertificateProps{
Service: config.KyvernoServiceName(),
Namespace: config.KyvernoNamespace(),
APIServerHost: apiServerURL.Hostname(),
return &certificateProps{
apiServerHost: apiServerURL.Hostname(),
}, nil
}
// inClusterServiceName The generated service name should be the common name for TLS certificate
// TODO: could be static
func (props *CertificateProps) inClusterServiceName() string {
return props.Service + "." + props.Namespace + ".svc"
}
func (props *CertificateProps) GenerateTLSPairSecretName() string {
return props.inClusterServiceName() + ".kyverno-tls-pair"
}
func (props *CertificateProps) GenerateRootCASecretName() string {
return props.inClusterServiceName() + ".kyverno-tls-ca"
}

83
pkg/tls/reader.go
View file

@ -2,48 +2,23 @@ package tls
import (
"context"
"fmt"
"github.com/kyverno/kyverno/pkg/config"
"github.com/pkg/errors"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
)
var ErrorsNotFound = "root CA certificate not found"
// ReadRootCASecret returns the RootCA from the pre-defined secret
func ReadRootCASecret(restConfig *rest.Config, client kubernetes.Interface) ([]byte, error) {
certProps, err := NewCertificateProps(restConfig)
if err != nil {
return nil, errors.Wrap(err, "failed to get TLS Cert Properties")
}
depl, err := client.AppsV1().Deployments(certProps.Namespace).Get(context.TODO(), config.KyvernoDeploymentName(), metav1.GetOptions{})
deplHash := ""
if err == nil {
deplHash = fmt.Sprintf("%v", depl.GetUID())
}
var deplHashSec string
var ok, managedByKyverno bool
sname := certProps.GenerateRootCASecretName()
stlsca, err := client.CoreV1().Secrets(certProps.Namespace).Get(context.TODO(), sname, metav1.GetOptions{})
func ReadRootCASecret(client kubernetes.Interface) ([]byte, error) {
sname := GenerateRootCASecretName()
stlsca, err := client.CoreV1().Secrets(config.KyvernoNamespace()).Get(context.TODO(), sname, metav1.GetOptions{})
if err != nil {
return nil, err
}
if label, ok := stlsca.GetLabels()[ManagedByLabel]; ok {
managedByKyverno = label == "kyverno"
}
deplHashSec, ok = stlsca.GetAnnotations()[MasterDeploymentUID]
if managedByKyverno && (ok && deplHashSec != deplHash) {
return nil, fmt.Errorf("outdated secret")
}
// try "tls.crt"
result := stlsca.Data[v1.TLSCertKey]
// if not there, try old "rootCA.crt"
@ -51,57 +26,7 @@ func ReadRootCASecret(restConfig *rest.Config, client kubernetes.Interface) ([]b
result = stlsca.Data[rootCAKey]
}
if len(result) == 0 {
return nil, errors.Errorf("%s in secret %s/%s", ErrorsNotFound, certProps.Namespace, stlsca.Name)
return nil, errors.Errorf("%s in secret %s/%s", ErrorsNotFound, config.KyvernoNamespace(), stlsca.Name)
}
return result, nil
}
// ReadTLSPair returns the pem pair from the pre-defined secret
func ReadTLSPair(restConfig *rest.Config, client kubernetes.Interface) ([]byte, []byte, error) {
certProps, err := NewCertificateProps(restConfig)
if err != nil {
return nil, nil, errors.Wrap(err, "failed to get TLS Cert Properties")
}
depl, err := client.AppsV1().Deployments(certProps.Namespace).Get(context.TODO(), config.KyvernoDeploymentName(), metav1.GetOptions{})
deplHash := ""
if err == nil {
deplHash = fmt.Sprintf("%v", depl.GetUID())
}
var deplHashSec string
var ok, managedByKyverno bool
sname := certProps.GenerateTLSPairSecretName()
secret, err := client.CoreV1().Secrets(certProps.Namespace).Get(context.TODO(), sname, metav1.GetOptions{})
if err != nil {
return nil, nil, fmt.Errorf("failed to get secret %s/%s: %v", certProps.Namespace, sname, err)
}
if label, ok := secret.GetLabels()[ManagedByLabel]; ok {
managedByKyverno = label == "kyverno"
}
deplHashSec, ok = secret.GetAnnotations()[MasterDeploymentUID]
if managedByKyverno && (ok && deplHashSec != deplHash) {
return nil, nil, fmt.Errorf("outdated secret")
}
// If secret contains annotation 'self-signed-cert', then it's created using helper scripts to setup self-signed certificates.
// As the root CA used to sign the certificate is required for webhook configuration, check if the corresponding secret is created
{
sname := certProps.GenerateRootCASecretName()
_, err := client.CoreV1().Secrets(certProps.Namespace).Get(context.TODO(), sname, metav1.GetOptions{})
if err != nil {
return nil, nil, fmt.Errorf("rootCA secret is required while using self-signed certificate TLS pair, defaulting to generating new TLS pair %s/%s", certProps.Namespace, sname)
}
}
if len(secret.Data[v1.TLSCertKey]) == 0 {
return nil, nil, fmt.Errorf("TLS Certificate not found in secret %s/%s", certProps.Namespace, sname)
}
if len(secret.Data[v1.TLSPrivateKeyKey]) == 0 {
return nil, nil, fmt.Errorf("TLS PrivateKey not found in secret %s/%s", certProps.Namespace, sname)
}
return secret.Data[v1.TLSCertKey], secret.Data[v1.TLSPrivateKeyKey], nil
}

287
pkg/tls/renewer.go Normal file
View file

@ -0,0 +1,287 @@
package tls
import (
"context"
"crypto/tls"
"crypto/x509"
"encoding/pem"
"fmt"
"time"
"github.com/cenkalti/backoff"
"github.com/go-logr/logr"
"github.com/kyverno/kyverno/pkg/config"
"github.com/pkg/errors"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
)
const (
// CertRenewalInterval is the renewal interval for rootCA
CertRenewalInterval time.Duration = 12 * time.Hour
// CertValidityDuration is the valid duration for a new cert
CertValidityDuration time.Duration = 365 * 24 * time.Hour
// ManagedByLabel is added to Kyverno managed secrets
ManagedByLabel string = "cert.kyverno.io/managed-by"
rootCAKey string = "rootCA.crt"
rollingUpdateAnnotation string = "update.kyverno.io/force-rolling-update"
)
// CertRenewer creates rootCA and pem pair to register
// webhook configurations and webhook server
// renews RootCA at the given interval
type CertRenewer struct {
client kubernetes.Interface
certRenewalInterval time.Duration
certValidityDuration time.Duration
certProps *certificateProps
// IP address where Kyverno controller runs. Only required if out-of-cluster.
serverIP string
log logr.Logger
}
// NewCertRenewer returns an instance of CertRenewer
func NewCertRenewer(client kubernetes.Interface, clientConfig *rest.Config, certRenewalInterval, certValidityDuration time.Duration, serverIP string, log logr.Logger) (*CertRenewer, error) {
certProps, err := newCertificateProps(clientConfig)
if err != nil {
return nil, err
}
return &CertRenewer{
client: client,
certRenewalInterval: certRenewalInterval,
certValidityDuration: certValidityDuration,
certProps: certProps,
serverIP: serverIP,
log: log,
}, nil
}
// InitTLSPemPair Loads or creates PEM private key and TLS certificate for webhook server.
// Created pair is stored in cluster's secret.
func (c *CertRenewer) InitTLSPemPair() error {
logger := c.log.WithName("InitTLSPemPair")
ca, err := c.getCASecret()
if err != nil && !apierrors.IsNotFound(err) {
return err
}
tls, err := c.getTLSSecret()
if err != nil && !apierrors.IsNotFound(err) {
return err
}
// check they are valid
if ca != nil && tls != nil {
if validCert(ca, tls, logger) {
return nil
}
}
// if not valid, check we can renew them
if !IsSecretManagedByKyverno(ca) || !IsSecretManagedByKyverno(tls) {
return fmt.Errorf("tls is not valid but certificates are not managed by kyverno, we can't renew them")
}
// renew them
logger.Info("building key/certificate pair for TLS")
return c.buildTLSPemPairAndWriteToSecrets(c.serverIP)
}
// buildTLSPemPairAndWriteToSecrets Issues TLS certificate for webhook server using self-signed CA cert
// Returns signed and approved TLS certificate in PEM format
func (c *CertRenewer) buildTLSPemPairAndWriteToSecrets(serverIP string) error {
caCert, err := generateCA(c.certValidityDuration)
if err != nil {
return err
}
tlsPair, err := generateCert(caCert, c.certProps, serverIP, c.certValidityDuration)
if err != nil {
return err
}
if err := c.writeCASecret(caCert); err != nil {
return fmt.Errorf("failed to write CA cert to secret: %v", err)
}
if err = c.writeTLSSecret(tlsPair); err != nil {
return fmt.Errorf("unable to save TLS pair to the cluster: %v", err)
}
return nil
}
func (c *CertRenewer) getSecret(name string) (*corev1.Secret, error) {
if s, err := c.client.CoreV1().Secrets(config.KyvernoNamespace()).Get(context.TODO(), name, metav1.GetOptions{}); err != nil {
return nil, err
} else {
return s, nil
}
}
func (c *CertRenewer) getCASecret() (*corev1.Secret, error) {
return c.getSecret(GenerateRootCASecretName())
}
func (c *CertRenewer) getTLSSecret() (*corev1.Secret, error) {
return c.getSecret(GenerateTLSPairSecretName())
}
func (c *CertRenewer) writeSecret(secret *corev1.Secret, logger logr.Logger) error {
logger = logger.WithValues("name", secret.GetName(), "namespace", secret.GetNamespace())
if _, err := c.client.CoreV1().Secrets(config.KyvernoNamespace()).Create(context.TODO(), secret, metav1.CreateOptions{}); err != nil {
if apierrors.IsAlreadyExists(err) {
if _, err := c.client.CoreV1().Secrets(config.KyvernoNamespace()).Update(context.TODO(), secret, metav1.UpdateOptions{}); err != nil {
logger.Error(err, "failed to update secret")
return err
} else {
logger.Info("secret updated")
return nil
}
} else {
logger.Error(err, "failed to create secret")
return err
}
} else {
logger.Info("secret created")
return nil
}
}
// writeCASecret stores the CA cert in secret
func (c *CertRenewer) writeCASecret(ca *keyPair) error {
logger := c.log.WithName("writeCASecret")
secret, err := c.getCASecret()
if err != nil && !apierrors.IsNotFound(err) {
logger.Error(err, "failed to get CA secret")
return err
}
if secret == nil {
secret = &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: GenerateRootCASecretName(),
Namespace: config.KyvernoNamespace(),
Labels: map[string]string{
ManagedByLabel: "kyverno",
},
},
Type: corev1.SecretTypeTLS,
}
}
secret.Type = corev1.SecretTypeTLS
secret.Data = map[string][]byte{
corev1.TLSCertKey: CertificateToPem(ca.cert),
corev1.TLSPrivateKeyKey: PrivateKeyToPem(ca.key),
}
return c.writeSecret(secret, logger)
}
// writeTLSSecret Writes the pair of TLS certificate and key to the specified secret.
func (c *CertRenewer) writeTLSSecret(tls *keyPair) error {
logger := c.log.WithName("writeTLSSecret")
secret, err := c.getTLSSecret()
if err != nil && !apierrors.IsNotFound(err) {
logger.Error(err, "failed to get TLS secret")
return err
}
if secret == nil {
secret = &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: GenerateTLSPairSecretName(),
Namespace: config.KyvernoNamespace(),
Labels: map[string]string{
ManagedByLabel: "kyverno",
},
},
Type: corev1.SecretTypeTLS,
}
}
secret.Data = map[string][]byte{
corev1.TLSCertKey: CertificateToPem(tls.cert),
corev1.TLSPrivateKeyKey: PrivateKeyToPem(tls.key),
}
return c.writeSecret(secret, logger)
}
// ValidCert validates the CA Cert
func (c *CertRenewer) ValidCert() (bool, error) {
logger := c.log.WithName("validCert")
ca, err := c.getCASecret()
if err != nil {
logger.Error(err, "unable to read CA secret")
return false, err
}
tls, err := c.getTLSSecret()
if err != nil {
logger.Error(err, "unable to read TLS secret")
return false, err
}
return validCert(ca, tls, logger), nil
}
func validCert(caSecret *corev1.Secret, tlsSecret *corev1.Secret, logger logr.Logger) bool {
caPem := caSecret.Data[corev1.TLSCertKey]
if len(caPem) == 0 {
caPem = caSecret.Data[rootCAKey]
}
// build cert pool
pool := x509.NewCertPool()
if !pool.AppendCertsFromPEM(caPem) {
logger.Info("bad certificate")
return false
}
// valid PEM pair
_, err := tls.X509KeyPair(tlsSecret.Data[corev1.TLSCertKey], tlsSecret.Data[corev1.TLSPrivateKeyKey])
if err != nil {
logger.Error(err, "invalid PEM pair")
return false
}
certPemBlock, _ := pem.Decode(tlsSecret.Data[corev1.TLSCertKey])
if certPemBlock == nil {
logger.Error(err, "bad private key")
return false
}
cert, err := x509.ParseCertificate(certPemBlock.Bytes)
if err != nil {
logger.Error(err, "failed to parse cert")
return false
}
if _, err = cert.Verify(x509.VerifyOptions{
Roots: pool,
CurrentTime: time.Now(),
}); err != nil {
logger.Error(err, "invalid cert")
return false
}
return true
}
// RollingUpdate triggers a rolling update of Kyverno pod.
// It is used when the rootCA is renewed, the restart of
// Kyverno pod will register webhook server with new cert
func (c *CertRenewer) RollingUpdate() error {
update := func() error {
deploy, err := c.client.AppsV1().Deployments(config.KyvernoNamespace()).Get(context.TODO(), config.KyvernoDeploymentName(), metav1.GetOptions{})
if err != nil {
return errors.Wrap(err, "failed to find Kyverno")
}
if IsKyvernoInRollingUpdate(deploy, c.log) {
return nil
}
if deploy.Spec.Template.Annotations == nil {
deploy.Spec.Template.Annotations = map[string]string{}
}
deploy.Spec.Template.Annotations[rollingUpdateAnnotation] = time.Now().String()
if _, err = c.client.AppsV1().Deployments(config.KyvernoNamespace()).Update(context.TODO(), deploy, metav1.UpdateOptions{}); err != nil {
return errors.Wrap(err, "update Kyverno deployment")
}
return nil
}
exbackoff := &backoff.ExponentialBackOff{
InitialInterval: 500 * time.Millisecond,
RandomizationFactor: 0.5,
Multiplier: 1.5,
MaxInterval: time.Second,
MaxElapsedTime: 3 * time.Second,
Clock: backoff.SystemClock,
}
exbackoff.Reset()
return backoff.Retry(update, exbackoff)
}

55
pkg/tls/utils.go
View file

@ -1,11 +1,34 @@
package tls
import (
"crypto/rsa"
"crypto/x509"
"encoding/pem"
"github.com/go-logr/logr"
"github.com/kyverno/kyverno/pkg/config"
appsv1 "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
)
// PrivateKeyToPem Creates PEM block from private key object
func PrivateKeyToPem(rsaKey *rsa.PrivateKey) []byte {
privateKey := &pem.Block{
Type: "PRIVATE KEY",
Bytes: x509.MarshalPKCS1PrivateKey(rsaKey),
}
return pem.EncodeToMemory(privateKey)
}
// CertificateToPem Creates PEM block from certificate object
func CertificateToPem(cert *x509.Certificate) []byte {
certificate := &pem.Block{
Type: "CERTIFICATE",
Bytes: cert.Raw,
}
return pem.EncodeToMemory(certificate)
}
// IsKyvernoInRollingUpdate returns true if Kyverno is in rolling update
func IsKyvernoInRollingUpdate(deploy *appsv1.Deployment, logger logr.Logger) bool {
var replicas int32 = 1
@ -20,14 +43,28 @@ func IsKyvernoInRollingUpdate(deploy *appsv1.Deployment, logger logr.Logger) boo
return false
}
func CanAddAnnotationToSecret(deplHash string, secret *v1.Secret) bool {
var deplHashSec string
var ok, managedByKyverno bool
if label, ok := secret.GetLabels()[ManagedByLabel]; ok {
managedByKyverno = label == "kyverno"
func IsSecretManagedByKyverno(secret *v1.Secret) bool {
if secret != nil {
labels := secret.GetLabels()
if labels == nil {
return false
}
deplHashSec, ok = secret.GetAnnotations()[MasterDeploymentUID]
return managedByKyverno && (!ok || deplHashSec != deplHash)
if labels[ManagedByLabel] != "kyverno" {
return false
}
}
return true
}
// InClusterServiceName The generated service name should be the common name for TLS certificate
func InClusterServiceName() string {
return config.KyvernoServiceName() + "." + config.KyvernoNamespace() + ".svc"
}
func GenerateTLSPairSecretName() string {
return InClusterServiceName() + ".kyverno-tls-pair"
}
func GenerateRootCASecretName() string {
return InClusterServiceName() + ".kyverno-tls-ca"
}

2
pkg/webhookconfig/common.go
View file

@ -47,7 +47,7 @@ func (wrc *Register) readCaData() []byte {
// Check if ca is defined in the secret tls-ca
// assume the key and signed cert have been defined in secret tls.kyverno
if caData, err = tls.ReadRootCASecret(wrc.clientConfig, wrc.kubeClient); err == nil {
if caData, err = tls.ReadRootCASecret(wrc.kubeClient); err == nil {
logger.V(4).Info("read CA from secret")
return caData
}