mirrors/kyverno
mirrors/kyverno
1
0
Fork 0
mirror of https://github.com/kyverno/kyverno.git synced 2025-03-13 19:28:55 +00:00
Code Activity

refactor: cleanup tls package (#3854)

Browse source 
* refactor: init certs with certs renewer directly

Signed-off-by: Charles-Edouard Brétéché <charled.breteche@gmail.com>

* refactor: tls package

Signed-off-by: Charles-Edouard Brétéché <charled.breteche@gmail.com>

* refactor: cleanup tls package

Signed-off-by: Charles-Edouard Brétéché <charled.breteche@gmail.com>
This commit is contained in:
Charles-Edouard Brétéché 2022-05-11 10:05:13 +02:00 • committed by GitHub
parent d8a4c709f8
commit c2602d8181
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 333 additions and 429 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
cmd/initContainer/main.go
View file

@ -131,7 +131,7 @@ func main() {
failure := false
run := func() {
certProps, err := tls.GetTLSCertProps(clientConfig)
certProps, err := tls.NewCertificateProps(clientConfig)
if err != nil {
log.Log.Info("failed to get cert properties: %v", err.Error())
os.Exit(1)
@ -145,7 +145,7 @@ func main() {
}
deplHash = fmt.Sprintf("%v", depl.GetUID())
name := tls.GenerateRootCASecretName(certProps)
name := certProps.GenerateRootCASecretName()
secret, err := kubeClient.CoreV1().Secrets(config.KyvernoNamespace()).Get(context.TODO(), name, metav1.GetOptions{})
if err != nil {
log.Log.Info("failed to fetch root CA secret", "name", name, "error", err.Error())
@ -162,7 +162,7 @@ func main() {
}
}
name = tls.GenerateTLSPairSecretName(certProps)
name = certProps.GenerateTLSPairSecretName()
secret, err = kubeClient.CoreV1().Secrets(config.KyvernoNamespace()).Get(context.TODO(), name, metav1.GetOptions{})
if err != nil {
log.Log.Info("failed to fetch TLS Pair secret", "name", name, "error", err.Error())

2
cmd/kyverno/main.go
View file

@ -343,7 +343,7 @@ func main() {
registerWrapperRetry := common.RetryFunc(time.Second, webhookRegistrationTimeout, webhookCfg.Register, "failed to register webhook", setupLog)
registerWebhookConfigurations := func() {
if _, err := certRenewer.InitTLSPemPair(); err != nil {
if err := certRenewer.InitTLSPemPair(); err != nil {
setupLog.Error(err, "tls initialization error")
os.Exit(1)
}

56
pkg/controllers/certmanager/controller.go
View file

@ -19,7 +19,7 @@ type Controller interface {
Run(stopCh <-chan struct{})
// GetTLSPemPair gets the existing TLSPemPair from the secret
GetTLSPemPair() (*tls.PemPair, error)
GetTLSPemPair() ([]byte, []byte, error)
}
type controller struct {
@ -59,15 +59,27 @@ func (m *controller) updateSecretFunc(oldObj interface{}, newObj interface{}) {
}
}
func (m *controller) GetTLSPemPair() (*tls.PemPair, error) {
func (m *controller) GetTLSPemPair() ([]byte, []byte, error) {
secret, err := m.secretLister.Secrets(config.KyvernoNamespace()).Get(m.renewer.GenerateTLSPairSecretName())
if err != nil {
return nil, err
return nil, nil, err
}
return &tls.PemPair{
Certificate: secret.Data[v1.TLSCertKey],
PrivateKey: secret.Data[v1.TLSPrivateKeyKey],
}, nil
return secret.Data[v1.TLSCertKey], secret.Data[v1.TLSPrivateKeyKey], nil
}
func (m *controller) validateCerts() error {
valid, err := m.renewer.ValidCert()
if err != nil {
logger.Error(err, "failed to validate cert")
if !strings.Contains(err.Error(), tls.ErrorsNotFound) {
return nil
}
}
if !valid {
logger.Info("rootCA has changed or is about to expire, trigger a rolling update to renew the cert")
return m.renewer.RollingUpdate()
}
return nil
}
func (m *controller) Run(stopCh <-chan struct{}) {
@ -77,35 +89,13 @@ func (m *controller) Run(stopCh <-chan struct{}) {
for {
select {
case <-certsRenewalTicker.C:
valid, err := m.renewer.ValidCert()
if err != nil {
logger.Error(err, "failed to validate cert")
if !strings.Contains(err.Error(), tls.ErrorsNotFound) {
continue
}
}
if valid {
continue
}
logger.Info("rootCA is about to expire, trigger a rolling update to renew the cert")
if err := m.renewer.RollingUpdate(); err != nil {
logger.Error(err, "unable to trigger a rolling update to renew rootCA, force restarting")
if err := m.validateCerts(); err != nil {
logger.Error(err, "unable to trigger a rolling update, force restarting")
os.Exit(1)
}
case <-m.secretQueue:
valid, err := m.renewer.ValidCert()
if err != nil {
logger.Error(err, "failed to validate cert")
if !strings.Contains(err.Error(), tls.ErrorsNotFound) {
continue
}
}
if valid {
continue
}
logger.Info("rootCA has changed, updating webhook configurations")
if err := m.renewer.RollingUpdate(); err != nil {
logger.Error(err, "unable to trigger a rolling update to re-register webhook server, force restarting")
if err := m.validateCerts(); err != nil {
logger.Error(err, "unable to trigger a rolling update, force restarting")
os.Exit(1)
}
case <-stopCh:

227
pkg/tls/certRenewer.go
View file

@ -12,7 +12,6 @@ import (
"github.com/go-logr/logr"
"github.com/kyverno/kyverno/pkg/config"
"github.com/pkg/errors"
appsv1 "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
k8errors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@ -21,6 +20,10 @@ import (
)
const (
// CertRenewalInterval is the renewal interval for rootCA
CertRenewalInterval time.Duration = 12 * time.Hour
// CertValidityDuration is the valid duration for a new cert
CertValidityDuration time.Duration = 365 * 24 * time.Hour
// ManagedByLabel is added to Kyverno managed secrets
ManagedByLabel string = "cert.kyverno.io/managed-by"
MasterDeploymentUID string = "cert.kyverno.io/master-deployment-uid"
@ -46,7 +49,7 @@ type CertRenewer struct {
// NewCertRenewer returns an instance of CertRenewer
func NewCertRenewer(client kubernetes.Interface, clientConfig *rest.Config, certRenewalInterval, certValidityDuration time.Duration, serverIP string, log logr.Logger) (*CertRenewer, error) {
certProps, err := GetTLSCertProps(clientConfig)
certProps, err := NewCertificateProps(clientConfig)
if err != nil {
return nil, err
}
@ -69,15 +72,23 @@ func (c *CertRenewer) ClientConfig() *rest.Config {
return c.clientConfig
}
func (c *CertRenewer) GenerateTLSPairSecretName() string {
return c.certProps.GenerateTLSPairSecretName()
}
func (c *CertRenewer) GenerateRootCASecretName() string {
return c.certProps.GenerateRootCASecretName()
}
// InitTLSPemPair Loads or creates PEM private key and TLS certificate for webhook server.
// Created pair is stored in cluster's secret.
// Returns struct with key/certificate pair.
func (c *CertRenewer) InitTLSPemPair() (*PemPair, error) {
func (c *CertRenewer) InitTLSPemPair() error {
logger := c.log.WithName("InitTLSPemPair")
if valid, err := c.ValidCert(); err == nil && valid {
if tlsPair, err := ReadTLSPair(c.clientConfig, c.client); err == nil {
if _, _, err := ReadTLSPair(c.clientConfig, c.client); err == nil {
logger.Info("using existing TLS key/certificate pair")
return tlsPair, nil
return nil
}
} else if err != nil {
logger.V(3).Info("unable to find TLS pair", "reason", err.Error())
@ -89,44 +100,36 @@ func (c *CertRenewer) InitTLSPemPair() (*PemPair, error) {
// buildTLSPemPairAndWriteToSecrets Issues TLS certificate for webhook server using self-signed CA cert
// Returns signed and approved TLS certificate in PEM format
func (c *CertRenewer) buildTLSPemPairAndWriteToSecrets(serverIP string) (*PemPair, error) {
caCert, caPEM, err := GenerateCACert(c.certValidityDuration)
func (c *CertRenewer) buildTLSPemPairAndWriteToSecrets(serverIP string) error {
caCert, err := GenerateCA(c.certValidityDuration)
if err != nil {
return nil, err
return err
}
if err := c.WriteCACertToSecret(caPEM); err != nil {
return nil, fmt.Errorf("failed to write CA cert to secret: %v", err)
}
tlsPair, err := GenerateCertPem(caCert, c.certProps, serverIP, c.certValidityDuration)
tlsPair, err := GenerateCert(caCert, c.certProps, serverIP, c.certValidityDuration)
if err != nil {
return nil, err
return err
}
if err := c.WriteCACertToSecret(caCert); err != nil {
return fmt.Errorf("failed to write CA cert to secret: %v", err)
}
if err = c.WriteTLSPairToSecret(tlsPair); err != nil {
return nil, fmt.Errorf("unable to save TLS pair to the cluster: %v", err)
return fmt.Errorf("unable to save TLS pair to the cluster: %v", err)
}
return tlsPair, nil
return nil
}
// ReadTLSPair Reads the pair of TLS certificate and key from the specified secret.
// WriteCACertToSecret stores the CA cert in secret
func (c *CertRenewer) WriteCACertToSecret(caPEM *PemPair) error {
func (c *CertRenewer) WriteCACertToSecret(ca *KeyPair) error {
logger := c.log.WithName("CAcert")
name := c.GenerateRootCASecretName()
name := c.certProps.GenerateRootCASecretName()
caBytes := CertificateToPem(ca.Cert)
keyBytes := PrivateKeyToPem(ca.Key)
depl, err := c.client.AppsV1().Deployments(c.certProps.Namespace).Get(context.TODO(), config.KyvernoDeploymentName(), metav1.GetOptions{})
deplHash := ""
if err == nil {
deplHash = fmt.Sprintf("%v", depl.GetUID())
}
secret, err := c.client.CoreV1().Secrets(c.certProps.Namespace).Get(context.TODO(), name, metav1.GetOptions{})
if err != nil {
if k8errors.IsNotFound(err) {
secret = &v1.Secret{
@ -145,8 +148,8 @@ func (c *CertRenewer) WriteCACertToSecret(caPEM *PemPair) error {
},
},
Data: map[string][]byte{
v1.TLSCertKey: caPEM.Certificate,
v1.TLSPrivateKeyKey: caPEM.PrivateKey,
v1.TLSCertKey: caBytes,
v1.TLSPrivateKeyKey: keyBytes,
},
Type: v1.SecretTypeTLS,
}
@ -163,14 +166,11 @@ func (c *CertRenewer) WriteCACertToSecret(caPEM *PemPair) error {
}
return err
}
dataMap := map[string][]byte{
v1.TLSCertKey: caPEM.Certificate,
v1.TLSPrivateKeyKey: caPEM.PrivateKey,
}
secret.Type = v1.SecretTypeTLS
secret.Data = dataMap
secret.Data = map[string][]byte{
v1.TLSCertKey: caBytes,
v1.TLSPrivateKeyKey: keyBytes,
}
_, err = c.client.CoreV1().Secrets(c.certProps.Namespace).Update(context.TODO(), secret, metav1.UpdateOptions{})
if err != nil {
return err
@ -181,20 +181,17 @@ func (c *CertRenewer) WriteCACertToSecret(caPEM *PemPair) error {
// WriteTLSPairToSecret Writes the pair of TLS certificate and key to the specified secret.
// Updates existing secret or creates new one.
func (c *CertRenewer) WriteTLSPairToSecret(pemPair *PemPair) error {
func (c *CertRenewer) WriteTLSPairToSecret(tls *KeyPair) error {
logger := c.log.WithName("WriteTLSPair")
name := c.GenerateTLSPairSecretName()
name := c.certProps.GenerateTLSPairSecretName()
certBytes := CertificateToPem(tls.Cert)
keyBytes := PrivateKeyToPem(tls.Key)
depl, err := c.client.AppsV1().Deployments(c.certProps.Namespace).Get(context.TODO(), config.KyvernoDeploymentName(), metav1.GetOptions{})
deplHash := ""
if err == nil {
deplHash = fmt.Sprintf("%v", depl.GetUID())
}
secret, err := c.client.CoreV1().Secrets(c.certProps.Namespace).Get(context.TODO(), name, metav1.GetOptions{})
if err != nil {
if k8errors.IsNotFound(err) {
secret = &v1.Secret{
@ -213,8 +210,8 @@ func (c *CertRenewer) WriteTLSPairToSecret(pemPair *PemPair) error {
},
},
Data: map[string][]byte{
v1.TLSCertKey: pemPair.Certificate,
v1.TLSPrivateKeyKey: pemPair.PrivateKey,
v1.TLSCertKey: certBytes,
v1.TLSPrivateKeyKey: keyBytes,
},
Type: v1.SecretTypeTLS,
}
@ -231,74 +228,25 @@ func (c *CertRenewer) WriteTLSPairToSecret(pemPair *PemPair) error {
}
return err
}
dataMap := map[string][]byte{
v1.TLSCertKey: pemPair.Certificate,
v1.TLSPrivateKeyKey: pemPair.PrivateKey,
secret.Data = map[string][]byte{
v1.TLSCertKey: certBytes,
v1.TLSPrivateKeyKey: keyBytes,
}
secret.Data = dataMap
_, err = c.client.CoreV1().Secrets(c.certProps.Namespace).Update(context.TODO(), secret, metav1.UpdateOptions{})
if err != nil {
return err
}
logger.Info("secret updated", "name", name, "namespace", c.certProps.Namespace)
return nil
}
// RollingUpdate triggers a rolling update of Kyverno pod.
// It is used when the rootCA is renewed, the restart of
// Kyverno pod will register webhook server with new cert
func (c *CertRenewer) RollingUpdate() error {
update := func() error {
deploy, err := c.client.AppsV1().Deployments(config.KyvernoNamespace()).Get(context.TODO(), config.KyvernoDeploymentName(), metav1.GetOptions{})
if err != nil {
return errors.Wrap(err, "failed to find Kyverno")
}
if IsKyvernoInRollingUpdate(deploy, c.log) {
return nil
}
if deploy.Spec.Template.Annotations == nil {
deploy.Spec.Template.Annotations = map[string]string{}
}
deploy.Spec.Template.Annotations[rollingUpdateAnnotation] = time.Now().String()
if _, err = c.client.AppsV1().Deployments(config.KyvernoNamespace()).Update(context.TODO(), deploy, metav1.UpdateOptions{}); err != nil {
return errors.Wrap(err, "update Kyverno deployment")
}
return nil
}
exbackoff := &backoff.ExponentialBackOff{
InitialInterval: 500 * time.Millisecond,
RandomizationFactor: 0.5,
Multiplier: 1.5,
MaxInterval: time.Second,
MaxElapsedTime: 3 * time.Second,
Clock: backoff.SystemClock,
}
exbackoff.Reset()
return backoff.Retry(update, exbackoff)
}
// ValidCert validates the CA Cert
func (c *CertRenewer) ValidCert() (bool, error) {
logger := c.log.WithName("ValidCert")
certProps, err := GetTLSCertProps(c.clientConfig)
if err != nil {
return false, nil
}
var managedByKyverno bool
snameTLS := GenerateTLSPairSecretName(certProps)
snameCA := GenerateRootCASecretName(certProps)
secret, err := c.client.CoreV1().Secrets(certProps.Namespace).Get(context.TODO(), snameTLS, metav1.GetOptions{})
snameTLS := c.certProps.GenerateTLSPairSecretName()
snameCA := c.certProps.GenerateRootCASecretName()
secret, err := c.client.CoreV1().Secrets(c.certProps.Namespace).Get(context.TODO(), snameTLS, metav1.GetOptions{})
if err != nil {
return false, nil
}
@ -312,7 +260,7 @@ func (c *CertRenewer) ValidCert() (bool, error) {
return false, nil
}
secret, err = c.client.CoreV1().Secrets(certProps.Namespace).Get(context.TODO(), snameCA, metav1.GetOptions{})
secret, err = c.client.CoreV1().Secrets(c.certProps.Namespace).Get(context.TODO(), snameCA, metav1.GetOptions{})
if err != nil {
return false, nil
}
@ -331,7 +279,7 @@ func (c *CertRenewer) ValidCert() (bool, error) {
return false, errors.Wrap(err, "unable to read CA from secret")
}
tlsPair, err := ReadTLSPair(c.clientConfig, c.client)
certPem, keyPem, err := ReadTLSPair(c.clientConfig, c.client)
if err != nil {
// wait till next reconcile
logger.Info("unable to read TLS PEM Pair from secret", "reason", err.Error())
@ -346,19 +294,19 @@ func (c *CertRenewer) ValidCert() (bool, error) {
}
// valid PEM pair
_, err = tls.X509KeyPair(tlsPair.Certificate, tlsPair.PrivateKey)
_, err = tls.X509KeyPair(certPem, keyPem)
if err != nil {
logger.Error(err, "invalid PEM pair")
return false, nil
}
certPem, _ := pem.Decode(tlsPair.Certificate)
certPemBlock, _ := pem.Decode(certPem)
if certPem == nil {
logger.Error(err, "bad private key")
return false, nil
}
cert, err := x509.ParseCertificate(certPem.Bytes)
cert, err := x509.ParseCertificate(certPemBlock.Bytes)
if err != nil {
logger.Error(err, "failed to parse cert")
return false, nil
@ -375,46 +323,35 @@ func (c *CertRenewer) ValidCert() (bool, error) {
return true, nil
}
// IsKyvernoInRollingUpdate returns true if Kyverno is in rolling update
func IsKyvernoInRollingUpdate(deploy *appsv1.Deployment, logger logr.Logger) bool {
var replicas int32 = 1
if deploy.Spec.Replicas != nil {
replicas = *deploy.Spec.Replicas
// RollingUpdate triggers a rolling update of Kyverno pod.
// It is used when the rootCA is renewed, the restart of
// Kyverno pod will register webhook server with new cert
func (c *CertRenewer) RollingUpdate() error {
update := func() error {
deploy, err := c.client.AppsV1().Deployments(config.KyvernoNamespace()).Get(context.TODO(), config.KyvernoDeploymentName(), metav1.GetOptions{})
if err != nil {
return errors.Wrap(err, "failed to find Kyverno")
}
if IsKyvernoInRollingUpdate(deploy, c.log) {
return nil
}
if deploy.Spec.Template.Annotations == nil {
deploy.Spec.Template.Annotations = map[string]string{}
}
deploy.Spec.Template.Annotations[rollingUpdateAnnotation] = time.Now().String()
if _, err = c.client.AppsV1().Deployments(config.KyvernoNamespace()).Update(context.TODO(), deploy, metav1.UpdateOptions{}); err != nil {
return errors.Wrap(err, "update Kyverno deployment")
}
return nil
}
nonTerminatedReplicas := deploy.Status.Replicas
if nonTerminatedReplicas > replicas {
logger.Info("detect Kyverno is in rolling update, won't trigger the update again")
return true
exbackoff := &backoff.ExponentialBackOff{
InitialInterval: 500 * time.Millisecond,
RandomizationFactor: 0.5,
Multiplier: 1.5,
MaxInterval: time.Second,
MaxElapsedTime: 3 * time.Second,
Clock: backoff.SystemClock,
}
return false
}
func (c *CertRenewer) GenerateTLSPairSecretName() string {
return GenerateTLSPairSecretName(c.certProps)
}
func (c *CertRenewer) GenerateRootCASecretName() string {
return GenerateRootCASecretName(c.certProps)
}
func GenerateTLSPairSecretName(props *CertificateProps) string {
return generateInClusterServiceName(props) + ".kyverno-tls-pair"
}
func GenerateRootCASecretName(props *CertificateProps) string {
return generateInClusterServiceName(props) + ".kyverno-tls-ca"
}
func CanAddAnnotationToSecret(deplHash string, secret *v1.Secret) bool {
var deplHashSec string
var ok, managedByKyverno bool
if label, ok := secret.GetLabels()[ManagedByLabel]; ok {
managedByKyverno = label == "kyverno"
}
deplHashSec, ok = secret.GetAnnotations()[MasterDeploymentUID]
return managedByKyverno && (!ok || deplHashSec != deplHash)
exbackoff.Reset()
return backoff.Retry(update, exbackoff)
}

103
pkg/tls/keypair.go Normal file
View file

@ -0,0 +1,103 @@
package tls
import (
"crypto/rand"
"crypto/rsa"
"crypto/x509"
"crypto/x509/pkix"
"fmt"
"math/big"
"net"
"strings"
"time"
)
// KeyPair ...
type KeyPair struct {
Cert *x509.Certificate
Key *rsa.PrivateKey
}
// GenerateCA creates the self-signed CA cert and private key
// it will be used to sign the webhook server certificate
func GenerateCA(certValidityDuration time.Duration) (*KeyPair, error) {
now := time.Now()
begin, end := now.Add(-1*time.Hour), now.Add(certValidityDuration)
key, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
return nil, fmt.Errorf("error generating key: %v", err)
}
templ := &x509.Certificate{
SerialNumber: big.NewInt(0),
Subject: pkix.Name{
CommonName: "*.kyverno.svc",
},
NotBefore: begin,
NotAfter: end,
KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment | x509.KeyUsageCertSign,
BasicConstraintsValid: true,
IsCA: true,
}
der, err := x509.CreateCertificate(rand.Reader, templ, templ, key.Public(), key)
if err != nil {
return nil, fmt.Errorf("error creating certificate: %v", err)
}
cert, err := x509.ParseCertificate(der)
if err != nil {
return nil, fmt.Errorf("error parsing certificate %v", err)
}
return &KeyPair{
Cert: cert,
Key: key,
}, nil
}
// GenerateCert takes the results of GenerateCACert and uses it to create the
// PEM-encoded public certificate and private key, respectively
func GenerateCert(caCert *KeyPair, props *CertificateProps, serverIP string, certValidityDuration time.Duration) (*KeyPair, error) {
now := time.Now()
begin, end := now.Add(-1*time.Hour), now.Add(certValidityDuration)
dnsNames := []string{
props.Service,
fmt.Sprintf("%s.%s", props.Service, props.Namespace),
props.inClusterServiceName(),
}
var ips []net.IP
if serverIP != "" {
if strings.Contains(serverIP, ":") {
host, _, _ := net.SplitHostPort(serverIP)
serverIP = host
}
ip := net.ParseIP(serverIP)
ips = append(ips, ip)
}
templ := &x509.Certificate{
SerialNumber: big.NewInt(1),
Subject: pkix.Name{
CommonName: props.Service,
},
DNSNames: dnsNames,
IPAddresses: ips,
NotBefore: begin,
NotAfter: end,
KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment,
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
BasicConstraintsValid: true,
}
key, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
return nil, fmt.Errorf("error generating key for webhook %v", err)
}
der, err := x509.CreateCertificate(rand.Reader, templ, caCert.Cert, key.Public(), caCert.Key)
if err != nil {
return nil, fmt.Errorf("error creating certificate for webhook %v", err)
}
cert, err := x509.ParseCertificate(der)
if err != nil {
return nil, fmt.Errorf("error parsing webhook certificate %v", err)
}
return &KeyPair{
Cert: cert,
Key: key,
}, nil
}

25
pkg/tls/pempair.go Normal file
View file

@ -0,0 +1,25 @@
package tls
import (
"crypto/rsa"
"crypto/x509"
"encoding/pem"
)
// PrivateKeyToPem Creates PEM block from private key object
func PrivateKeyToPem(rsaKey *rsa.PrivateKey) []byte {
privateKey := &pem.Block{
Type: "PRIVATE KEY",
Bytes: x509.MarshalPKCS1PrivateKey(rsaKey),
}
return pem.EncodeToMemory(privateKey)
}
// CertificateToPem Creates PEM block from certificate object
func CertificateToPem(cert *x509.Certificate) []byte {
certificate := &pem.Block{
Type: "CERTIFICATE",
Bytes: cert.Raw,
}
return pem.EncodeToMemory(certificate)
}

42
pkg/tls/props.go Normal file
View file

@ -0,0 +1,42 @@
package tls
import (
"net/url"
"github.com/kyverno/kyverno/pkg/config"
"k8s.io/client-go/rest"
)
// CertificateProps Properties of TLS certificate which should be issued for webhook server
type CertificateProps struct {
Service string
Namespace string
APIServerHost string
}
// NewCertificateProps creates CertificateProps from a *rest.Config
func NewCertificateProps(configuration *rest.Config) (*CertificateProps, error) {
apiServerURL, err := url.Parse(configuration.Host)
if err != nil {
return nil, err
}
return &CertificateProps{
Service: config.KyvernoServiceName(),
Namespace: config.KyvernoNamespace(),
APIServerHost: apiServerURL.Hostname(),
}, nil
}
// inClusterServiceName The generated service name should be the common name for TLS certificate
// TODO: could be static
func (props *CertificateProps) inClusterServiceName() string {
return props.Service + "." + props.Namespace + ".svc"
}
func (props *CertificateProps) GenerateTLSPairSecretName() string {
return props.inClusterServiceName() + ".kyverno-tls-pair"
}
func (props *CertificateProps) GenerateRootCASecretName() string {
return props.inClusterServiceName() + ".kyverno-tls-ca"
}

55
pkg/tls/reader.go
View file

@ -3,7 +3,6 @@ package tls
import (
"context"
"fmt"
"net/url"
"github.com/kyverno/kyverno/pkg/config"
"github.com/pkg/errors"
@ -16,8 +15,8 @@ import (
var ErrorsNotFound = "root CA certificate not found"
// ReadRootCASecret returns the RootCA from the pre-defined secret
func ReadRootCASecret(restConfig *rest.Config, client kubernetes.Interface) (result []byte, err error) {
certProps, err := GetTLSCertProps(restConfig)
func ReadRootCASecret(restConfig *rest.Config, client kubernetes.Interface) ([]byte, error) {
certProps, err := NewCertificateProps(restConfig)
if err != nil {
return nil, errors.Wrap(err, "failed to get TLS Cert Properties")
}
@ -32,7 +31,7 @@ func ReadRootCASecret(restConfig *rest.Config, client kubernetes.Interface) (res
var deplHashSec string
var ok, managedByKyverno bool
sname := GenerateRootCASecretName(certProps)
sname := certProps.GenerateRootCASecretName()
stlsca, err := client.CoreV1().Secrets(certProps.Namespace).Get(context.TODO(), sname, metav1.GetOptions{})
if err != nil {
return nil, err
@ -46,7 +45,7 @@ func ReadRootCASecret(restConfig *rest.Config, client kubernetes.Interface) (res
return nil, fmt.Errorf("outdated secret")
}
// try "tls.crt"
result = stlsca.Data[v1.TLSCertKey]
result := stlsca.Data[v1.TLSCertKey]
// if not there, try old "rootCA.crt"
if len(result) == 0 {
result = stlsca.Data[rootCAKey]
@ -58,10 +57,10 @@ func ReadRootCASecret(restConfig *rest.Config, client kubernetes.Interface) (res
}
// ReadTLSPair returns the pem pair from the pre-defined secret
func ReadTLSPair(restConfig *rest.Config, client kubernetes.Interface) (*PemPair, error) {
certProps, err := GetTLSCertProps(restConfig)
func ReadTLSPair(restConfig *rest.Config, client kubernetes.Interface) ([]byte, []byte, error) {
certProps, err := NewCertificateProps(restConfig)
if err != nil {
return nil, errors.Wrap(err, "failed to get TLS Cert Properties")
return nil, nil, errors.Wrap(err, "failed to get TLS Cert Properties")
}
depl, err := client.AppsV1().Deployments(certProps.Namespace).Get(context.TODO(), config.KyvernoDeploymentName(), metav1.GetOptions{})
@ -74,53 +73,35 @@ func ReadTLSPair(restConfig *rest.Config, client kubernetes.Interface) (*PemPair
var deplHashSec string
var ok, managedByKyverno bool
sname := GenerateTLSPairSecretName(certProps)
sname := certProps.GenerateTLSPairSecretName()
secret, err := client.CoreV1().Secrets(certProps.Namespace).Get(context.TODO(), sname, metav1.GetOptions{})
if err != nil {
return nil, fmt.Errorf("failed to get secret %s/%s: %v", certProps.Namespace, sname, err)
return nil, nil, fmt.Errorf("failed to get secret %s/%s: %v", certProps.Namespace, sname, err)
}
if label, ok := secret.GetLabels()[ManagedByLabel]; ok {
managedByKyverno = label == "kyverno"
}
deplHashSec, ok = secret.GetAnnotations()[MasterDeploymentUID]
if managedByKyverno && (ok && deplHashSec != deplHash) {
return nil, fmt.Errorf("outdated secret")
return nil, nil, fmt.Errorf("outdated secret")
}
// If secret contains annotation 'self-signed-cert', then it's created using helper scripts to setup self-signed certificates.
// As the root CA used to sign the certificate is required for webhook configuration, check if the corresponding secret is created
{
sname := GenerateRootCASecretName(certProps)
sname := certProps.GenerateRootCASecretName()
_, err := client.CoreV1().Secrets(certProps.Namespace).Get(context.TODO(), sname, metav1.GetOptions{})
if err != nil {
return nil, fmt.Errorf("rootCA secret is required while using self-signed certificate TLS pair, defaulting to generating new TLS pair %s/%s", certProps.Namespace, sname)
return nil, nil, fmt.Errorf("rootCA secret is required while using self-signed certificate TLS pair, defaulting to generating new TLS pair %s/%s", certProps.Namespace, sname)
}
}
pemPair := PemPair{
Certificate: secret.Data[v1.TLSCertKey],
PrivateKey: secret.Data[v1.TLSPrivateKeyKey],
if len(secret.Data[v1.TLSCertKey]) == 0 {
return nil, nil, fmt.Errorf("TLS Certificate not found in secret %s/%s", certProps.Namespace, sname)
}
if len(secret.Data[v1.TLSPrivateKeyKey]) == 0 {
return nil, nil, fmt.Errorf("TLS PrivateKey not found in secret %s/%s", certProps.Namespace, sname)
}
if len(pemPair.Certificate) == 0 {
return nil, fmt.Errorf("TLS Certificate not found in secret %s/%s", certProps.Namespace, sname)
}
if len(pemPair.PrivateKey) == 0 {
return nil, fmt.Errorf("TLS PrivateKey not found in secret %s/%s", certProps.Namespace, sname)
}
return &pemPair, nil
}
//GetTLSCertProps provides the TLS Certificate Properties
func GetTLSCertProps(configuration *rest.Config) (*CertificateProps, error) {
apiServerURL, err := url.Parse(configuration.Host)
if err != nil {
return nil, err
}
return &CertificateProps{
Service: config.KyvernoServiceName(),
Namespace: config.KyvernoNamespace(),
APIServerHost: apiServerURL.Hostname(),
}, nil
return secret.Data[v1.TLSCertKey], secret.Data[v1.TLSPrivateKeyKey], nil
}

206
pkg/tls/tls.go
View file

@ -1,206 +0,0 @@
package tls
import (
"crypto/rand"
"crypto/rsa"
"crypto/x509"
"crypto/x509/pkix"
"encoding/pem"
"errors"
"fmt"
"math/big"
"net"
"strings"
"time"
)
// CertRenewalInterval is the renewal interval for rootCA
const CertRenewalInterval time.Duration = 12 * time.Hour
// CertValidityDuration is the valid duration for a new cert
const CertValidityDuration time.Duration = 365 * 24 * time.Hour
// CertificateProps Properties of TLS certificate which should be issued for webhook server
type CertificateProps struct {
Service string
Namespace string
APIServerHost string
ServerIP string
}
// PemPair The pair of TLS certificate corresponding private key, both in PEM format
type PemPair struct {
Certificate []byte
PrivateKey []byte
}
// KeyPair ...
type KeyPair struct {
Cert *x509.Certificate
Key *rsa.PrivateKey
}
// GeneratePrivateKey Generates RSA private key
func GeneratePrivateKey() (*rsa.PrivateKey, error) {
return rsa.GenerateKey(rand.Reader, 2048)
}
// PrivateKeyToPem Creates PEM block from private key object
func PrivateKeyToPem(rsaKey *rsa.PrivateKey) []byte {
privateKey := &pem.Block{
Type: "PRIVATE KEY",
Bytes: x509.MarshalPKCS1PrivateKey(rsaKey),
}
return pem.EncodeToMemory(privateKey)
}
// CertificateToPem ...
func CertificateToPem(certificateDER []byte) []byte {
certificate := &pem.Block{
Type: "CERTIFICATE",
Bytes: certificateDER,
}
return pem.EncodeToMemory(certificate)
}
// GenerateCACert creates the self-signed CA cert and private key
// it will be used to sign the webhook server certificate
func GenerateCACert(certValidityDuration time.Duration) (*KeyPair, *PemPair, error) {
now := time.Now()
begin := now.Add(-1 * time.Hour)
end := now.Add(certValidityDuration)
templ := &x509.Certificate{
SerialNumber: big.NewInt(0),
Subject: pkix.Name{
CommonName: "*.kyverno.svc",
},
NotBefore: begin,
NotAfter: end,
KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment | x509.KeyUsageCertSign,
BasicConstraintsValid: true,
IsCA: true,
}
key, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
return nil, nil, fmt.Errorf("error generating key: %v", err)
}
der, err := x509.CreateCertificate(rand.Reader, templ, templ, key.Public(), key)
if err != nil {
return nil, nil, fmt.Errorf("error creating certificate: %v", err)
}
pemPair := &PemPair{
Certificate: CertificateToPem(der),
PrivateKey: PrivateKeyToPem(key),
}
cert, err := x509.ParseCertificate(der)
if err != nil {
return nil, nil, fmt.Errorf("error parsing certificate %v", err)
}
caCert := &KeyPair{
Cert: cert,
Key: key,
}
return caCert, pemPair, nil
}
// GenerateCertPem takes the results of GenerateCACert and uses it to create the
// PEM-encoded public certificate and private key, respectively
func GenerateCertPem(caCert *KeyPair, props *CertificateProps, serverIP string, certValidityDuration time.Duration) (*PemPair, error) {
now := time.Now()
begin := now.Add(-1 * time.Hour)
end := now.Add(certValidityDuration)
dnsNames := make([]string, 3)
dnsNames[0] = props.Service
csCommonName := dnsNames[0]
dnsNames[1] = fmt.Sprintf("%s.%s", props.Service, props.Namespace)
// The full service name is the CommonName for the certificate
commonName := generateInClusterServiceName(props)
dnsNames[2] = commonName
var ips []net.IP
if serverIP != "" {
if strings.Contains(serverIP, ":") {
host, _, _ := net.SplitHostPort(serverIP)
serverIP = host
}
ip := net.ParseIP(serverIP)
ips = append(ips, ip)
}
templ := &x509.Certificate{
SerialNumber: big.NewInt(1),
Subject: pkix.Name{
CommonName: csCommonName,
},
DNSNames: dnsNames,
IPAddresses: ips,
NotBefore: begin,
NotAfter: end,
KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment,
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
BasicConstraintsValid: true,
}
key, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
return nil, fmt.Errorf("error generating key for webhook %v", err)
}
der, err := x509.CreateCertificate(rand.Reader, templ, caCert.Cert, key.Public(), caCert.Key)
if err != nil {
return nil, fmt.Errorf("error creating certificate for webhook %v", err)
}
pemPair := &PemPair{
Certificate: CertificateToPem(der),
PrivateKey: PrivateKeyToPem(key),
}
return pemPair, nil
}
//GenerateInClusterServiceName The generated service name should be the common name for TLS certificate
func generateInClusterServiceName(props *CertificateProps) string {
return props.Service + "." + props.Namespace + ".svc"
}
//TlsCertificateGetExpirationDate Gets NotAfter property from raw certificate
func tlsCertificateGetExpirationDate(certData []byte) (*time.Time, error) {
block, _ := pem.Decode(certData)
if block == nil {
return nil, errors.New("failed to decode PEM")
}
cert, err := x509.ParseCertificate(block.Bytes)
if err != nil {
return nil, errors.New("failed to parse certificate: %v" + err.Error())
}
return &cert.NotAfter, nil
}
// The certificate is valid for a year, but we update it earlier to avoid using
// an expired certificate in a controller that has been running for a long time
const timeReserveBeforeCertificateExpiration time.Duration = time.Hour * 24 * 30 * 6 // About half a year
//IsTLSPairShouldBeUpdated checks if TLS pair has expited and needs to be updated
func IsTLSPairShouldBeUpdated(tlsPair *PemPair) bool {
if tlsPair == nil {
return true
}
expirationDate, err := tlsCertificateGetExpirationDate(tlsPair.Certificate)
if err != nil {
return true
}
return time.Until(*expirationDate) < timeReserveBeforeCertificateExpiration
}

33
pkg/tls/utils.go Normal file
View file

@ -0,0 +1,33 @@
package tls
import (
"github.com/go-logr/logr"
appsv1 "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
)
// IsKyvernoInRollingUpdate returns true if Kyverno is in rolling update
func IsKyvernoInRollingUpdate(deploy *appsv1.Deployment, logger logr.Logger) bool {
var replicas int32 = 1
if deploy.Spec.Replicas != nil {
replicas = *deploy.Spec.Replicas
}
nonTerminatedReplicas := deploy.Status.Replicas
if nonTerminatedReplicas > replicas {
logger.Info("detect Kyverno is in rolling update, won't trigger the update again")
return true
}
return false
}
func CanAddAnnotationToSecret(deplHash string, secret *v1.Secret) bool {
var deplHashSec string
var ok, managedByKyverno bool
if label, ok := secret.GetLabels()[ManagedByLabel]; ok {
managedByKyverno = label == "kyverno"
}
deplHashSec, ok = secret.GetAnnotations()[MasterDeploymentUID]
return managedByKyverno && (!ok || deplHashSec != deplHash)
}

7
pkg/webhooks/server.go
View file

@ -25,7 +25,6 @@ import (
"github.com/kyverno/kyverno/pkg/openapi"
"github.com/kyverno/kyverno/pkg/policycache"
"github.com/kyverno/kyverno/pkg/policyreport"
tlsutils "github.com/kyverno/kyverno/pkg/tls"
"github.com/kyverno/kyverno/pkg/userinfo"
"github.com/kyverno/kyverno/pkg/utils"
"github.com/kyverno/kyverno/pkg/webhookconfig"
@ -97,7 +96,7 @@ type WebhookServer struct {
func NewWebhookServer(
kyvernoClient kyvernoclient.Interface,
client client.Interface,
tlsPair func() (*tlsutils.PemPair, error),
tlsPair func() ([]byte, []byte, error),
urInformer urinformer.UpdateRequestInformer,
pInformer kyvernoinformer.ClusterPolicyInformer,
rbInformer rbacinformer.RoleBindingInformer,
@ -157,11 +156,11 @@ func NewWebhookServer(
Addr: ":9443", // Listen on port for HTTPS requests
TLSConfig: &tls.Config{
GetCertificate: func(*tls.ClientHelloInfo) (*tls.Certificate, error) {
tlsPair, err := tlsPair()
certPem, keyPem, err := tlsPair()
if err != nil {
return nil, err
}
pair, err := tls.X509KeyPair(tlsPair.Certificate, tlsPair.PrivateKey)
pair, err := tls.X509KeyPair(certPem, keyPem)
if err != nil {
return nil, err
}