diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 874b6e55ed..6ee9864815 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -41,6 +41,9 @@ them, don't hesitate to ask. We're here to help! This is simply a reminder of wh - [] I have read the [contributing guidelines](https://github.com/kyverno/kyverno/blob/main/CONTRIBUTING.md). - [] I have added tests that prove my fix is effective or that my feature works. - [] I have added or changed [the documentation](https://github.com/kyverno/website). + - If not, I have raised an issue in [kyverno/website](https://github.com/kyverno/website) to track the doc update: + + ## Further comments diff --git a/cmd/initContainer/main.go b/cmd/initContainer/main.go index 7e09ecff8a..4f0dc8986b 100644 --- a/cmd/initContainer/main.go +++ b/cmd/initContainer/main.go @@ -10,7 +10,6 @@ import ( "sync" "time" - "github.com/kyverno/kyverno/pkg/config" client "github.com/kyverno/kyverno/pkg/dclient" "github.com/kyverno/kyverno/pkg/signal" "github.com/kyverno/kyverno/pkg/utils" @@ -28,8 +27,6 @@ var ( ) const ( - mutatingWebhookConfigKind string = "MutatingWebhookConfiguration" - validatingWebhookConfigKind string = "ValidatingWebhookConfiguration" policyReportKind string = "PolicyReport" clusterPolicyReportKind string = "ClusterPolicyReport" reportChangeRequestKind string = "ReportChangeRequest" @@ -72,16 +69,6 @@ func main() { } requests := []request{ - {validatingWebhookConfigKind, config.ValidatingWebhookConfigurationName}, - {validatingWebhookConfigKind, config.ValidatingWebhookConfigurationDebugName}, - {mutatingWebhookConfigKind, config.MutatingWebhookConfigurationName}, - {mutatingWebhookConfigKind, config.MutatingWebhookConfigurationDebugName}, - - {validatingWebhookConfigKind, config.PolicyValidatingWebhookConfigurationName}, - {validatingWebhookConfigKind, config.PolicyValidatingWebhookConfigurationDebugName}, - {mutatingWebhookConfigKind, config.PolicyMutatingWebhookConfigurationName}, - {mutatingWebhookConfigKind, config.PolicyMutatingWebhookConfigurationDebugName}, - {policyReportKind, ""}, {clusterPolicyReportKind, ""}, @@ -120,8 +107,6 @@ func main() { func executeRequest(client *client.Client, req request) error { switch req.kind { - case mutatingWebhookConfigKind, validatingWebhookConfigKind: - return removeWebhookIfExists(client, req.kind, req.name) case policyReportKind: return removePolicyReport(client, req.kind) case clusterPolicyReportKind: @@ -236,29 +221,6 @@ func merge(done <-chan struct{}, stopCh <-chan struct{}, processes ...<-chan err return out } -func removeWebhookIfExists(client *client.Client, kind string, name string) error { - logger := log.Log.WithName("removeExistingWebhook").WithValues("kind", kind, "name", name) - var err error - // Get resource - _, err = client.GetResource("", kind, "", name) - if errors.IsNotFound(err) { - logger.V(4).Info("resource not found") - return nil - } - if err != nil { - logger.Error(err, "failed to get resource") - return err - } - // Delete resource - err = client.DeleteResource("", kind, "", name, false) - if err != nil { - logger.Error(err, "failed to delete resource") - return err - } - logger.Info("removed the resource") - return nil -} - func removeClusterPolicyReport(client *client.Client, kind string) error { logger := log.Log.WithName("removeClusterPolicyReport") diff --git a/cmd/kyverno/main.go b/cmd/kyverno/main.go index 61e8548ac9..01ab102708 100755 --- a/cmd/kyverno/main.go +++ b/cmd/kyverno/main.go @@ -24,6 +24,7 @@ import ( "github.com/kyverno/kyverno/pkg/policystatus" "github.com/kyverno/kyverno/pkg/resourcecache" "github.com/kyverno/kyverno/pkg/signal" + ktls "github.com/kyverno/kyverno/pkg/tls" "github.com/kyverno/kyverno/pkg/utils" "github.com/kyverno/kyverno/pkg/version" "github.com/kyverno/kyverno/pkg/webhookconfig" @@ -144,7 +145,7 @@ func main() { log.Log) // Resource Mutating Webhook Watcher - webhookMonitor := webhookconfig.NewMonitor(log.Log.WithName("WebhookMonitor")) + webhookMonitor := webhookconfig.NewMonitor(rCache, log.Log.WithName("WebhookMonitor")) // KYVERNO CRD INFORMER // watches CRD resources: @@ -283,8 +284,9 @@ func main() { client, ) + certRenewer := ktls.NewCertRenewer(client, clientConfig, ktls.CertRenewalInterval, ktls.CertValidityDuration, log.Log.WithName("CertRenewer")) // Configure certificates - tlsPair, err := client.InitTLSPemPair(clientConfig, serverIP) + tlsPair, err := certRenewer.InitTLSPemPair(serverIP) if err != nil { setupLog.Error(err, "Failed to initialize TLS key/certificate pair") os.Exit(1) @@ -329,6 +331,7 @@ func main() { pCacheController.Cache, webhookCfg, webhookMonitor, + certRenewer, statusSync.Listener, configData, reportReqGen, diff --git a/pkg/dclient/certificates.go b/pkg/dclient/certificates.go deleted file mode 100644 index 784fa5c947..0000000000 --- a/pkg/dclient/certificates.go +++ /dev/null @@ -1,243 +0,0 @@ -package client - -import ( - "encoding/base64" - "fmt" - "net/url" - - "github.com/kyverno/kyverno/pkg/config" - tls "github.com/kyverno/kyverno/pkg/tls" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/client-go/rest" -) - -// InitTLSPemPair Loads or creates PEM private key and TLS certificate for webhook server. -// Created pair is stored in cluster's secret. -// Returns struct with key/certificate pair. -func (c *Client) InitTLSPemPair(configuration *rest.Config, serverIP string) (*tls.PemPair, error) { - logger := c.log - certProps, err := c.GetTLSCertProps(configuration) - if err != nil { - return nil, err - } - - logger.Info("Building key/certificate pair for TLS") - tlsPair, err := c.buildTLSPemPair(certProps, serverIP) - if err != nil { - return nil, err - } - if err = c.WriteTLSPairToSecret(certProps, tlsPair); err != nil { - return nil, fmt.Errorf("Unable to save TLS pair to the cluster: %v", err) - } - - return tlsPair, nil -} - -// buildTLSPemPair Issues TLS certificate for webhook server using self-signed CA cert -// Returns signed and approved TLS certificate in PEM format -func (c *Client) buildTLSPemPair(props tls.CertificateProps, serverIP string) (*tls.PemPair, error) { - caCert, caPEM, err := tls.GenerateCACert() - if err != nil { - return nil, err - } - - if err := c.WriteCACertToSecret(caPEM, props); err != nil { - return nil, fmt.Errorf("failed to write CA cert to secret: %v", err) - } - - return tls.GenerateCertPem(caCert, props, serverIP) -} - -//ReadRootCASecret returns the RootCA from the pre-defined secret -func (c *Client) ReadRootCASecret() (result []byte) { - logger := c.log.WithName("ReadRootCASecret") - certProps, err := c.GetTLSCertProps(c.clientConfig) - if err != nil { - logger.Error(err, "failed to get TLS Cert Properties") - return result - } - sname := generateRootCASecretName(certProps) - stlsca, err := c.GetResource("", Secrets, certProps.Namespace, sname) - if err != nil { - return result - } - tlsca, err := convertToSecret(stlsca) - if err != nil { - logger.Error(err, "failed to convert secret", "name", sname, "namespace", certProps.Namespace) - return result - } - - result = tlsca.Data[rootCAKey] - if len(result) == 0 { - logger.Info("root CA certificate not found in secret", "name", tlsca.Name, "namespace", certProps.Namespace) - return result - } - logger.V(4).Info("using CA bundle defined in secret to validate the webhook's server certificate", "name", tlsca.Name, "namespace", certProps.Namespace) - return result -} - -const selfSignedAnnotation string = "self-signed-cert" -const rootCAKey string = "rootCA.crt" - -// ReadTLSPair Reads the pair of TLS certificate and key from the specified secret. -func (c *Client) ReadTLSPair(props tls.CertificateProps) *tls.PemPair { - logger := c.log.WithName("ReadTLSPair") - sname := generateTLSPairSecretName(props) - unstrSecret, err := c.GetResource("", Secrets, props.Namespace, sname) - if err != nil { - logger.Error(err, "Failed to get secret", "name", sname, "namespace", props.Namespace) - return nil - } - - // If secret contains annotation 'self-signed-cert', then it's created using helper scripts to setup self-signed certificates. - // As the root CA used to sign the certificate is required for webhook cnofiguration, check if the corresponding secret is created - annotations := unstrSecret.GetAnnotations() - if _, ok := annotations[selfSignedAnnotation]; ok { - sname := generateRootCASecretName(props) - _, err := c.GetResource("", Secrets, props.Namespace, sname) - if err != nil { - logger.Error(err, "Root CA secret is required while using self-signed certificates TLS pair, defaulting to generating new TLS pair", "name", sname, "namespace", props.Namespace) - return nil - } - } - secret, err := convertToSecret(unstrSecret) - if err != nil { - return nil - } - pemPair := tls.PemPair{ - Certificate: secret.Data[v1.TLSCertKey], - PrivateKey: secret.Data[v1.TLSPrivateKeyKey], - } - if len(pemPair.Certificate) == 0 { - logger.Info("TLS Certificate not found in secret", "name", sname, "namespace", props.Namespace) - return nil - } - if len(pemPair.PrivateKey) == 0 { - logger.Info("TLS PrivateKey not found in secret", "name", sname, "namespace", props.Namespace) - return nil - } - return &pemPair -} - -// WriteCACertToSecret stores the CA cert in secret -func (c *Client) WriteCACertToSecret(caPEM *tls.PemPair, props tls.CertificateProps) error { - logger := c.log.WithName("CAcert") - name := generateRootCASecretName(props) - - secretUnstr, err := c.GetResource("", Secrets, props.Namespace, name) - if err != nil { - secret := &v1.Secret{ - TypeMeta: metav1.TypeMeta{ - Kind: "Secret", - APIVersion: "v1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: props.Namespace, - Annotations: map[string]string{ - selfSignedAnnotation: "true", - }, - }, - Data: map[string][]byte{ - rootCAKey: caPEM.Certificate, - }, - Type: v1.SecretTypeOpaque, - } - - _, err := c.CreateResource("", Secrets, props.Namespace, secret, false) - if err == nil { - logger.Info("secret created", "name", name, "namespace", props.Namespace) - } - return err - } - - if _, ok := secretUnstr.GetAnnotations()[selfSignedAnnotation]; !ok { - secretUnstr.SetAnnotations(map[string]string{selfSignedAnnotation: "true"}) - } - - dataMap := map[string]interface{}{ - rootCAKey: base64.StdEncoding.EncodeToString(caPEM.Certificate)} - - if err := unstructured.SetNestedMap(secretUnstr.Object, dataMap, "data"); err != nil { - return err - } - - _, err = c.UpdateResource("", Secrets, props.Namespace, secretUnstr, false) - if err != nil { - return err - } - logger.Info("secret updated", "name", name, "namespace", props.Namespace) - return nil -} - -// WriteTLSPairToSecret Writes the pair of TLS certificate and key to the specified secret. -// Updates existing secret or creates new one. -func (c *Client) WriteTLSPairToSecret(props tls.CertificateProps, pemPair *tls.PemPair) error { - logger := c.log.WithName("WriteTLSPair") - name := generateTLSPairSecretName(props) - secretUnstr, err := c.GetResource("", Secrets, props.Namespace, name) - if err != nil { - secret := &v1.Secret{ - TypeMeta: metav1.TypeMeta{ - Kind: "Secret", - APIVersion: "v1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: props.Namespace, - }, - Data: map[string][]byte{ - v1.TLSCertKey: pemPair.Certificate, - v1.TLSPrivateKeyKey: pemPair.PrivateKey, - }, - Type: v1.SecretTypeTLS, - } - - _, err := c.CreateResource("", Secrets, props.Namespace, secret, false) - if err == nil { - logger.Info("secret created", "name", name, "namespace", props.Namespace) - } - return err - } - - dataMap := map[string]interface{}{ - v1.TLSCertKey: base64.StdEncoding.EncodeToString(pemPair.Certificate), - v1.TLSPrivateKeyKey: base64.StdEncoding.EncodeToString(pemPair.PrivateKey), - } - - if err := unstructured.SetNestedMap(secretUnstr.Object, dataMap, "data"); err != nil { - return err - } - - _, err = c.UpdateResource("", Secrets, props.Namespace, secretUnstr, false) - if err != nil { - return err - } - - logger.Info("secret updated", "name", name, "namespace", props.Namespace) - return nil -} - -func generateTLSPairSecretName(props tls.CertificateProps) string { - return tls.GenerateInClusterServiceName(props) + ".kyverno-tls-pair" -} - -func generateRootCASecretName(props tls.CertificateProps) string { - return tls.GenerateInClusterServiceName(props) + ".kyverno-tls-ca" -} - -//GetTLSCertProps provides the TLS Certificate Properties -func (c *Client) GetTLSCertProps(configuration *rest.Config) (certProps tls.CertificateProps, err error) { - apiServerURL, err := url.Parse(configuration.Host) - if err != nil { - return certProps, err - } - certProps = tls.CertificateProps{ - Service: config.KyvernoServiceName, - Namespace: config.KyvernoNamespace, - APIServerHost: apiServerURL.Hostname(), - } - return certProps, nil -} diff --git a/pkg/dclient/client.go b/pkg/dclient/client.go index 3fa1cf66b1..cbd31c42e1 100644 --- a/pkg/dclient/client.go +++ b/pkg/dclient/client.go @@ -9,7 +9,6 @@ import ( "github.com/go-logr/logr" openapiv2 "github.com/googleapis/gnostic/openapiv2" certificates "k8s.io/api/certificates/v1beta1" - v1 "k8s.io/api/core/v1" helperv1 "k8s.io/apimachinery/pkg/apis/meta/v1" meta "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -198,15 +197,6 @@ func convertToUnstructured(obj interface{}) *unstructured.Unstructured { return &unstructured.Unstructured{Object: unstructuredObj} } -//To-Do remove this to use unstructured type -func convertToSecret(obj *unstructured.Unstructured) (v1.Secret, error) { - secret := v1.Secret{} - if err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), &secret); err != nil { - return secret, err - } - return secret, nil -} - //To-Do remove this to use unstructured type func convertToCSR(obj *unstructured.Unstructured) (*certificates.CertificateSigningRequest, error) { csr := certificates.CertificateSigningRequest{} diff --git a/pkg/dclient/utils.go b/pkg/dclient/utils.go index d4cf657eb3..6b6dc19b08 100644 --- a/pkg/dclient/utils.go +++ b/pkg/dclient/utils.go @@ -15,17 +15,6 @@ import ( kubernetesfake "k8s.io/client-go/kubernetes/fake" ) -const ( - // CSRs CertificateSigningRequest - CSRs string = "CertificateSigningRequest" - // Secrets Secret - Secrets string = "Secret" - // ConfigMaps ConfigMap - ConfigMaps string = "ConfigMap" - // Namespaces Namespace - Namespaces string = "Namespace" -) - //NewMockClient ---testing utilities func NewMockClient(scheme *runtime.Scheme, gvrToListKind map[schema.GroupVersionResource]string, objects ...runtime.Object) (*Client, error) { client := fake.NewSimpleDynamicClientWithCustomListKinds(scheme, gvrToListKind, objects...) diff --git a/pkg/resourcecache/gvrcache.go b/pkg/resourcecache/gvrcache.go index 8d8d76d0ce..befc315a8f 100644 --- a/pkg/resourcecache/gvrcache.go +++ b/pkg/resourcecache/gvrcache.go @@ -4,6 +4,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/dynamic/dynamiclister" "k8s.io/client-go/informers" + "k8s.io/client-go/tools/cache" ) // GenericCache - allows operation on a single resource @@ -13,6 +14,7 @@ type GenericCache interface { Lister() dynamiclister.Lister NamespacedLister(namespace string) dynamiclister.NamespaceLister GVR() schema.GroupVersionResource + GetInformer() cache.SharedIndexInformer } type genericCache struct { @@ -56,3 +58,8 @@ func (gc *genericCache) Lister() dynamiclister.Lister { func (gc *genericCache) NamespacedLister(namespace string) dynamiclister.NamespaceLister { return dynamiclister.New(gc.genericInformer.Informer().GetIndexer(), gc.GVR()).Namespace(namespace) } + +// GetInformer gets SharedIndexInformer +func (gc *genericCache) GetInformer() cache.SharedIndexInformer { + return gc.genericInformer.Informer() +} diff --git a/pkg/resourcecache/main.go b/pkg/resourcecache/main.go index b902c2f945..f8cc3c5f1b 100644 --- a/pkg/resourcecache/main.go +++ b/pkg/resourcecache/main.go @@ -10,11 +10,15 @@ import ( ) // ResourceCache - allows the creation, deletion and saving the resource informers as a cache +// the resource cache can be registered by gvks as follows: +// - group/version/kind +// - group/kind +// - kind type ResourceCache interface { - CreateInformers(resources ...string) []error - CreateGVKInformer(kind string) (GenericCache, error) - StopResourceInformer(resource string) - GetGVRCache(resource string) (GenericCache, bool) + CreateInformers(gvks ...string) []error + CreateGVKInformer(gvk string) (GenericCache, error) + StopResourceInformer(gvk string) + GetGVRCache(gvk string) (GenericCache, bool) } type resourceCache struct { diff --git a/pkg/tls/certRenewer.go b/pkg/tls/certRenewer.go new file mode 100644 index 0000000000..f4cb20581c --- /dev/null +++ b/pkg/tls/certRenewer.go @@ -0,0 +1,348 @@ +package tls + +import ( + "crypto/tls" + "crypto/x509" + "encoding/base64" + "encoding/pem" + "fmt" + "time" + + "github.com/cenkalti/backoff" + "github.com/go-logr/logr" + "github.com/kyverno/kyverno/pkg/config" + client "github.com/kyverno/kyverno/pkg/dclient" + "github.com/pkg/errors" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/client-go/rest" +) + +const ( + // ManagedByLabel is added to Kyverno managed secrets + ManagedByLabel string = "cert.kyverno.io/managed-by" + + SelfSignedAnnotation string = "self-signed-cert" + RootCAKey string = "rootCA.crt" + rollingUpdateAnnotation string = "update.kyverno.io/force-rolling-update" +) + +// CertRenewer creates rootCA and pem pair to register +// webhook configurations and webhook server +// renews RootCA at the given interval +type CertRenewer struct { + client *client.Client + clientConfig *rest.Config + certRenewalInterval time.Duration + certValidityDuration time.Duration + log logr.Logger +} + +// NewCertRenewer returns an instance of CertRenewer +func NewCertRenewer(client *client.Client, clientConfig *rest.Config, certRenewalInterval, certValidityDuration time.Duration, log logr.Logger) *CertRenewer { + return &CertRenewer{ + client: client, + clientConfig: clientConfig, + certRenewalInterval: certRenewalInterval, + certValidityDuration: certValidityDuration, + log: log, + } +} + +// InitTLSPemPair Loads or creates PEM private key and TLS certificate for webhook server. +// Created pair is stored in cluster's secret. +// Returns struct with key/certificate pair. +func (c *CertRenewer) InitTLSPemPair(serverIP string) (*PemPair, error) { + logger := c.log.WithName("InitTLSPemPair") + certProps, err := GetTLSCertProps(c.clientConfig) + if err != nil { + return nil, err + } + + if valid, err := c.ValidCert(); err == nil && valid { + if tlsPair, err := ReadTLSPair(c.clientConfig, c.client); err == nil { + logger.Info("using existing TLS key/certificate pair") + return tlsPair, nil + } + } else { + logger.V(3).Info("unable to find TLS pair", "reason", err.Error()) + } + + logger.Info("building key/certificate pair for TLS") + return c.buildTLSPemPairAndWriteToSecrets(certProps, serverIP) +} + +// buildTLSPemPairAndWriteToSecrets Issues TLS certificate for webhook server using self-signed CA cert +// Returns signed and approved TLS certificate in PEM format +func (c *CertRenewer) buildTLSPemPairAndWriteToSecrets(props CertificateProps, serverIP string) (*PemPair, error) { + caCert, caPEM, err := GenerateCACert(c.certValidityDuration) + if err != nil { + return nil, err + } + + if err := c.WriteCACertToSecret(caPEM, props); err != nil { + return nil, fmt.Errorf("failed to write CA cert to secret: %v", err) + } + + tlsPair, err := GenerateCertPem(caCert, props, serverIP, c.certValidityDuration) + if err != nil { + return nil, err + } + + if err = c.WriteTLSPairToSecret(props, tlsPair); err != nil { + return nil, fmt.Errorf("unable to save TLS pair to the cluster: %v", err) + } + + return tlsPair, nil +} + +// ReadTLSPair Reads the pair of TLS certificate and key from the specified secret. + +// WriteCACertToSecret stores the CA cert in secret +func (c *CertRenewer) WriteCACertToSecret(caPEM *PemPair, props CertificateProps) error { + logger := c.log.WithName("CAcert") + name := generateRootCASecretName(props) + + secretUnstr, err := c.client.GetResource("", "Secret", props.Namespace, name) + if err != nil { + secret := &v1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: props.Namespace, + Annotations: map[string]string{ + SelfSignedAnnotation: "true", + }, + Labels: map[string]string{ + ManagedByLabel: "kyverno", + }, + }, + Data: map[string][]byte{ + RootCAKey: caPEM.Certificate, + }, + Type: v1.SecretTypeOpaque, + } + + _, err := c.client.CreateResource("", "Secret", props.Namespace, secret, false) + if err == nil { + logger.Info("secret created", "name", name, "namespace", props.Namespace) + } + return err + } + + if _, ok := secretUnstr.GetAnnotations()[SelfSignedAnnotation]; !ok { + secretUnstr.SetAnnotations(map[string]string{SelfSignedAnnotation: "true"}) + } + + dataMap := map[string]interface{}{ + RootCAKey: base64.StdEncoding.EncodeToString(caPEM.Certificate)} + + if err := unstructured.SetNestedMap(secretUnstr.Object, dataMap, "data"); err != nil { + return err + } + + _, err = c.client.UpdateResource("", "Secret", props.Namespace, secretUnstr, false) + if err != nil { + return err + } + logger.Info("secret updated", "name", name, "namespace", props.Namespace) + return nil +} + +// WriteTLSPairToSecret Writes the pair of TLS certificate and key to the specified secret. +// Updates existing secret or creates new one. +func (c *CertRenewer) WriteTLSPairToSecret(props CertificateProps, pemPair *PemPair) error { + logger := c.log.WithName("WriteTLSPair") + + name := generateTLSPairSecretName(props) + secretUnstr, err := c.client.GetResource("", "Secret", props.Namespace, name) + if err != nil { + secret := &v1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: props.Namespace, + Labels: map[string]string{ + ManagedByLabel: "kyverno", + }, + }, + Data: map[string][]byte{ + v1.TLSCertKey: pemPair.Certificate, + v1.TLSPrivateKeyKey: pemPair.PrivateKey, + }, + Type: v1.SecretTypeTLS, + } + + _, err := c.client.CreateResource("", "Secret", props.Namespace, secret, false) + if err == nil { + logger.Info("secret created", "name", name, "namespace", props.Namespace) + } + return err + } + + dataMap := map[string][]byte{ + v1.TLSCertKey: pemPair.Certificate, + v1.TLSPrivateKeyKey: pemPair.PrivateKey, + } + + secret, err := convertToSecret(secretUnstr) + if err != nil { + return err + } + + secret.Data = dataMap + _, err = c.client.UpdateResource("", "Secret", props.Namespace, secret, false) + if err != nil { + return err + } + + logger.Info("secret updated", "name", name, "namespace", props.Namespace) + return nil +} + +// RollingUpdate triggers a rolling update of Kyverno pod. +// It is used when the rootCA is renewed, the restart of +// Kyverno pod will register webhook server with new cert +func (c *CertRenewer) RollingUpdate() error { + + update := func() error { + deploy, err := c.client.GetResource("", "Deployment", config.KyvernoNamespace, config.KyvernoDeploymentName) + if err != nil { + return errors.Wrap(err, "failed to find Kyverno") + } + + if IsKyvernoIsInRollingUpdate(deploy.UnstructuredContent(), c.log) { + return nil + } + + annotations, ok, err := unstructured.NestedStringMap(deploy.UnstructuredContent(), "spec", "template", "metadata", "annotations") + if err != nil { + return errors.Wrap(err, "bad annotations") + } + + if !ok { + annotations = map[string]string{} + } + + annotations[rollingUpdateAnnotation] = time.Now().String() + if err = unstructured.SetNestedStringMap(deploy.UnstructuredContent(), + annotations, + "spec", "template", "metadata", "annotations", + ); err != nil { + return errors.Wrapf(err, "set annotation %s", rollingUpdateAnnotation) + } + + if _, err = c.client.UpdateResource("", "Deployment", config.KyvernoNamespace, deploy, false); err != nil { + return errors.Wrap(err, "update Kyverno deployment") + } + return nil + } + + exbackoff := &backoff.ExponentialBackOff{ + InitialInterval: 500 * time.Millisecond, + RandomizationFactor: 0.5, + Multiplier: 1.5, + MaxInterval: time.Second, + MaxElapsedTime: 3 * time.Second, + Clock: backoff.SystemClock, + } + + exbackoff.Reset() + return backoff.Retry(update, exbackoff) +} + +// ValidCert validates the CA Cert +func (c *CertRenewer) ValidCert() (bool, error) { + logger := c.log.WithName("ValidCert") + + rootCA, err := ReadRootCASecret(c.clientConfig, c.client) + if err != nil { + return false, errors.Wrap(err, "unable to read CA from secret") + } + + tlsPair, err := ReadTLSPair(c.clientConfig, c.client) + if err != nil { + // wait till next reconcile + logger.Info("unable to read TLS PEM Pair from secret", "reason", err.Error()) + return false, errors.Wrap(err, "unable to read TLS PEM Pair from secret") + } + + // build cert pool + pool := x509.NewCertPool() + caPem, _ := pem.Decode(rootCA) + if caPem == nil { + logger.Error(err, "bad certificate") + return false, nil + } + + cac, err := x509.ParseCertificate(caPem.Bytes) + if err != nil { + logger.Error(err, "failed to parse CA cert") + return false, nil + } + pool.AddCert(cac) + + // valid PEM pair + _, err = tls.X509KeyPair(tlsPair.Certificate, tlsPair.PrivateKey) + if err != nil { + logger.Error(err, "invalid PEM pair") + return false, nil + } + + certPem, _ := pem.Decode(tlsPair.Certificate) + if certPem == nil { + logger.Error(err, "bad private key") + return false, nil + } + + cert, err := x509.ParseCertificate(certPem.Bytes) + if err != nil { + logger.Error(err, "failed to parse cert") + return false, nil + } + + if _, err = cert.Verify(x509.VerifyOptions{ + Roots: pool, + CurrentTime: time.Now().Add(c.certRenewalInterval), + }); err != nil { + logger.Error(err, "invalid cert") + return false, nil + } + + return true, nil +} + +// IsKyvernoIsInRollingUpdate returns true if Kyverno is in rolling update +func IsKyvernoIsInRollingUpdate(deploy map[string]interface{}, logger logr.Logger) bool { + replicas, _, err := unstructured.NestedInt64(deploy, "spec", "replicas") + if err != nil { + logger.Error(err, "unable to fetch spec.replicas") + } + + nonTerminatedReplicas, _, err := unstructured.NestedInt64(deploy, "status", "replicas") + if err != nil { + logger.Error(err, "unable to fetch status.replicas") + } + + if nonTerminatedReplicas > replicas { + logger.Info("detect Kyverno is in rolling update, won't trigger the update again") + return true + } + + return false +} + +func generateTLSPairSecretName(props CertificateProps) string { + return generateInClusterServiceName(props) + ".kyverno-tls-pair" +} + +func generateRootCASecretName(props CertificateProps) string { + return generateInClusterServiceName(props) + ".kyverno-tls-ca" +} diff --git a/pkg/tls/reader.go b/pkg/tls/reader.go new file mode 100644 index 0000000000..4ddebc9a29 --- /dev/null +++ b/pkg/tls/reader.go @@ -0,0 +1,106 @@ +package tls + +import ( + "fmt" + "net/url" + + "github.com/kyverno/kyverno/pkg/config" + client "github.com/kyverno/kyverno/pkg/dclient" + "github.com/pkg/errors" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/rest" +) + +// ReadRootCASecret returns the RootCA from the pre-defined secret +func ReadRootCASecret(restConfig *rest.Config, client *client.Client) (result []byte, err error) { + certProps, err := GetTLSCertProps(restConfig) + if err != nil { + return nil, errors.Wrap(err, "failed to get TLS Cert Properties") + } + + sname := generateRootCASecretName(certProps) + stlsca, err := client.GetResource("", "Secret", certProps.Namespace, sname) + if err != nil { + return nil, err + } + + tlsca, err := convertToSecret(stlsca) + if err != nil { + return nil, errors.Wrapf(err, "failed to convert secret %s/%s", certProps.Namespace, sname) + } + + result = tlsca.Data[RootCAKey] + if len(result) == 0 { + return nil, errors.Errorf("root CA certificate not found in secret %s/%s", certProps.Namespace, tlsca.Name) + } + + return result, nil +} + +// ReadTLSPair returns the pem pair from the pre-defined secret +func ReadTLSPair(restConfig *rest.Config, client *client.Client) (*PemPair, error) { + certProps, err := GetTLSCertProps(restConfig) + if err != nil { + return nil, errors.Wrap(err, "failed to get TLS Cert Properties") + } + + sname := generateTLSPairSecretName(certProps) + unstrSecret, err := client.GetResource("", "Secret", certProps.Namespace, sname) + if err != nil { + return nil, fmt.Errorf("failed to get secret %s/%s: %v", certProps.Namespace, sname, err) + } + + // If secret contains annotation 'self-signed-cert', then it's created using helper scripts to setup self-signed certificates. + // As the root CA used to sign the certificate is required for webhook configuration, check if the corresponding secret is created + annotations := unstrSecret.GetAnnotations() + if _, ok := annotations[SelfSignedAnnotation]; ok { + sname := generateRootCASecretName(certProps) + _, err := client.GetResource("", "Secret", certProps.Namespace, sname) + if err != nil { + return nil, fmt.Errorf("rootCA secret is required while using self-signed certificate TLS pair, defaulting to generating new TLS pair %s/%s", certProps.Namespace, sname) + } + } + secret, err := convertToSecret(unstrSecret) + if err != nil { + return nil, err + } + + pemPair := PemPair{ + Certificate: secret.Data[v1.TLSCertKey], + PrivateKey: secret.Data[v1.TLSPrivateKeyKey], + } + + if len(pemPair.Certificate) == 0 { + return nil, fmt.Errorf("TLS Certificate not found in secret %s/%s", certProps.Namespace, sname) + } + if len(pemPair.PrivateKey) == 0 { + return nil, fmt.Errorf("TLS PrivateKey not found in secret %s/%s", certProps.Namespace, sname) + } + + return &pemPair, nil +} + +//GetTLSCertProps provides the TLS Certificate Properties +func GetTLSCertProps(configuration *rest.Config) (certProps CertificateProps, err error) { + apiServerURL, err := url.Parse(configuration.Host) + if err != nil { + return certProps, err + } + + certProps = CertificateProps{ + Service: config.KyvernoServiceName, + Namespace: config.KyvernoNamespace, + APIServerHost: apiServerURL.Hostname(), + } + return certProps, nil +} + +func convertToSecret(obj *unstructured.Unstructured) (v1.Secret, error) { + secret := v1.Secret{} + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), &secret); err != nil { + return secret, err + } + return secret, nil +} diff --git a/pkg/tls/tls.go b/pkg/tls/tls.go index 023db9e699..880dc9f6ea 100644 --- a/pkg/tls/tls.go +++ b/pkg/tls/tls.go @@ -14,7 +14,11 @@ import ( "time" ) -const certValidityDuration = 10 * 365 * 24 * time.Hour +// CertRenewalInterval is the renewal interval for rootCA +const CertRenewalInterval time.Duration = 12 * time.Hour + +// CertValidityDuration is the valid duration for a new cert +const CertValidityDuration time.Duration = 365 * 24 * time.Hour // CertificateProps Properties of TLS certificate which should be issued for webhook server type CertificateProps struct { @@ -63,7 +67,7 @@ func CertificateToPem(certificateDER []byte) []byte { // GenerateCACert creates the self-signed CA cert and private key // it will be used to sign the webhook server certificate -func GenerateCACert() (*KeyPair, *PemPair, error) { +func GenerateCACert(certValidityDuration time.Duration) (*KeyPair, *PemPair, error) { now := time.Now() begin := now.Add(-1 * time.Hour) end := now.Add(certValidityDuration) @@ -110,7 +114,7 @@ func GenerateCACert() (*KeyPair, *PemPair, error) { // GenerateCertPem takes the results of GenerateCACert and uses it to create the // PEM-encoded public certificate and private key, respectively -func GenerateCertPem(caCert *KeyPair, props CertificateProps, serverIP string) (*PemPair, error) { +func GenerateCertPem(caCert *KeyPair, props CertificateProps, serverIP string, certValidityDuration time.Duration) (*PemPair, error) { now := time.Now() begin := now.Add(-1 * time.Hour) end := now.Add(certValidityDuration) @@ -121,7 +125,7 @@ func GenerateCertPem(caCert *KeyPair, props CertificateProps, serverIP string) ( dnsNames[1] = fmt.Sprintf("%s.%s", props.Service, props.Namespace) // The full service name is the CommonName for the certificate - commonName := GenerateInClusterServiceName(props) + commonName := generateInClusterServiceName(props) dnsNames[2] = fmt.Sprintf("%s", commonName) var ips []net.IP @@ -174,7 +178,7 @@ func GenerateCertPem(caCert *KeyPair, props CertificateProps, serverIP string) ( } //GenerateInClusterServiceName The generated service name should be the common name for TLS certificate -func GenerateInClusterServiceName(props CertificateProps) string { +func generateInClusterServiceName(props CertificateProps) string { return props.Service + "." + props.Namespace + ".svc" } diff --git a/pkg/webhookconfig/common.go b/pkg/webhookconfig/common.go index 0ec76c8958..58a3a2b0d8 100644 --- a/pkg/webhookconfig/common.go +++ b/pkg/webhookconfig/common.go @@ -4,28 +4,34 @@ import ( "io/ioutil" "github.com/kyverno/kyverno/pkg/config" + "github.com/kyverno/kyverno/pkg/tls" admregapi "k8s.io/api/admissionregistration/v1beta1" apps "k8s.io/api/apps/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" rest "k8s.io/client-go/rest" ) func (wrc *Register) readCaData() []byte { - logger := wrc.log + logger := wrc.log.WithName("readCaData") var caData []byte + var err error + // Check if ca is defined in the secret tls-ca // assume the key and signed cert have been defined in secret tls.kyverno - if caData = wrc.client.ReadRootCASecret(); len(caData) != 0 { + if caData, err = tls.ReadRootCASecret(wrc.clientConfig, wrc.client); err == nil { logger.V(4).Info("read CA from secret") return caData } - logger.V(4).Info("failed to read CA from secret, reading from kubeconfig") + + logger.V(4).Info("failed to read CA from secret, reading from kubeconfig", "reason", err.Error()) // load the CA from kubeconfig if caData = extractCA(wrc.clientConfig); len(caData) != 0 { logger.V(4).Info("read CA from kubeconfig") return caData } + logger.V(4).Info("failed to read CA from kubeconfig") return nil } @@ -49,8 +55,8 @@ func extractCA(config *rest.Config) (result []byte) { func (wrc *Register) constructOwner() v1.OwnerReference { logger := wrc.log - kubePolicyDeployment, err := wrc.getKubePolicyDeployment() + kubePolicyDeployment, _, err := wrc.GetKubePolicyDeployment() if err != nil { logger.Error(err, "failed to construct OwnerReference") return v1.OwnerReference{} @@ -64,17 +70,18 @@ func (wrc *Register) constructOwner() v1.OwnerReference { } } -func (wrc *Register) getKubePolicyDeployment() (*apps.Deployment, error) { +// GetKubePolicyDeployment gets Kyverno deployment +func (wrc *Register) GetKubePolicyDeployment() (*apps.Deployment, *unstructured.Unstructured, error) { lister, _ := wrc.resCache.GetGVRCache("Deployment") kubePolicyDeployment, err := lister.NamespacedLister(config.KyvernoNamespace).Get(config.KyvernoDeploymentName) if err != nil { - return nil, err + return nil, nil, err } deploy := apps.Deployment{} if err = runtime.DefaultUnstructuredConverter.FromUnstructured(kubePolicyDeployment.UnstructuredContent(), &deploy); err != nil { - return nil, err + return nil, kubePolicyDeployment, err } - return &deploy, nil + return &deploy, kubePolicyDeployment, nil } // debug mutating webhook diff --git a/pkg/webhookconfig/monitor.go b/pkg/webhookconfig/monitor.go index af59210c98..b39ecbe963 100644 --- a/pkg/webhookconfig/monitor.go +++ b/pkg/webhookconfig/monitor.go @@ -2,12 +2,18 @@ package webhookconfig import ( "fmt" + "os" + "reflect" "sync" "time" "github.com/go-logr/logr" - dclient "github.com/kyverno/kyverno/pkg/dclient" + "github.com/kyverno/kyverno/pkg/config" "github.com/kyverno/kyverno/pkg/event" + "github.com/kyverno/kyverno/pkg/resourcecache" + "github.com/kyverno/kyverno/pkg/tls" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/client-go/tools/cache" ) //maxRetryCount defines the max deadline count @@ -28,17 +34,34 @@ const ( // like the webhook settings. // type Monitor struct { - t time.Time - mu sync.RWMutex - log logr.Logger + t time.Time + mu sync.RWMutex + secretQueue chan bool + log logr.Logger } -//NewMonitor returns a new instance of LastRequestTime store -func NewMonitor(log logr.Logger) *Monitor { - return &Monitor{ - t: time.Now(), - log: log, +//NewMonitor returns a new instance of webhook monitor +func NewMonitor(resCache resourcecache.ResourceCache, log logr.Logger) *Monitor { + monitor := &Monitor{ + t: time.Now(), + secretQueue: make(chan bool, 1), + log: log, } + + var err error + secretCache, ok := resCache.GetGVRCache("Secret") + if !ok { + if secretCache, err = resCache.CreateGVKInformer("Secret"); err != nil { + log.Error(err, "unable to start Secret's informer") + } + } + + secretCache.GetInformer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: monitor.addSecretFunc, + UpdateFunc: monitor.updateSecretFunc, + }) + + return monitor } //Time returns the last request time @@ -56,18 +79,59 @@ func (t *Monitor) SetTime(tm time.Time) { t.t = tm } +func (t *Monitor) addSecretFunc(obj interface{}) { + secret := obj.(*unstructured.Unstructured) + if secret.GetNamespace() != config.KyvernoNamespace { + return + } + + val, ok := secret.GetAnnotations()[tls.SelfSignedAnnotation] + if !ok || val != "true" { + return + } + + t.secretQueue <- true +} + +func (t *Monitor) updateSecretFunc(oldObj interface{}, newObj interface{}) { + old := oldObj.(*unstructured.Unstructured) + new := newObj.(*unstructured.Unstructured) + if new.GetNamespace() != config.KyvernoNamespace { + return + } + + val, ok := new.GetAnnotations()[tls.SelfSignedAnnotation] + if !ok || val != "true" { + return + } + + if reflect.DeepEqual(old.UnstructuredContent()["data"], new.UnstructuredContent()["data"]) { + return + } + + t.secretQueue <- true + t.log.V(4).Info("secret updated, reconciling webhook configurations") +} + //Run runs the checker and verify the resource update -func (t *Monitor) Run(register *Register, eventGen event.Interface, client *dclient.Client, stopCh <-chan struct{}) { +func (t *Monitor) Run(register *Register, certRenewer *tls.CertRenewer, eventGen event.Interface, stopCh <-chan struct{}) { logger := t.log logger.V(4).Info("starting webhook monitor", "interval", idleCheckInterval) - status := newStatusControl(client, eventGen, logger.WithName("WebhookStatusControl")) + status := newStatusControl(register.client, eventGen, logger.WithName("WebhookStatusControl")) ticker := time.NewTicker(tickerInterval) defer ticker.Stop() + certsRenewalTicker := time.NewTicker(tls.CertRenewalInterval) + defer certsRenewalTicker.Stop() + for { select { case <-ticker.C: + if skipWebhookCheck(register, logger.WithName("statusCheck/skipWebhookCheck")) { + logger.Info("skip validating webhook status, Kyverno is in rolling update") + continue + } if err := register.Check(); err != nil { t.log.Error(err, "missing webhooks") @@ -95,6 +159,10 @@ func (t *Monitor) Run(register *Register, eventGen event.Interface, client *dcli if timeDiff > idleCheckInterval { logger.V(1).Info("webhook idle time exceeded", "deadline", idleCheckInterval) + if skipWebhookCheck(register, logger.WithName("skipWebhookCheck")) { + logger.Info("skip validating webhook status, Kyverno is in rolling update") + continue + } // send request to update the Kyverno deployment if err := status.IncrementAnnotation(); err != nil { @@ -110,6 +178,40 @@ func (t *Monitor) Run(register *Register, eventGen event.Interface, client *dcli logger.Error(err, "failed to annotate deployment webhook status to success") } + case <-certsRenewalTicker.C: + valid, err := certRenewer.ValidCert() + if err != nil { + logger.Error(err, "failed to validate cert") + continue + } + + if valid { + continue + } + + logger.Info("rootCA is about to expire, trigger a rolling update to renew the cert") + if err := certRenewer.RollingUpdate(); err != nil { + logger.Error(err, "unable to trigger a rolling update to renew rootCA, force restarting") + os.Exit(1) + } + + case <-t.secretQueue: + valid, err := certRenewer.ValidCert() + if err != nil { + logger.Error(err, "failed to validate cert") + continue + } + + if valid { + continue + } + + logger.Info("rootCA has changed, updating webhook configurations") + if err := certRenewer.RollingUpdate(); err != nil { + logger.Error(err, "unable to trigger a rolling update to re-register webhook server, force restarting") + os.Exit(1) + } + case <-stopCh: // handler termination signal logger.V(2).Info("stopping webhook monitor") @@ -117,3 +219,14 @@ func (t *Monitor) Run(register *Register, eventGen event.Interface, client *dcli } } } + +// skipWebhookCheck returns true if Kyverno is in rolling update +func skipWebhookCheck(register *Register, logger logr.Logger) bool { + _, deploy, err := register.GetKubePolicyDeployment() + if err != nil { + logger.Info("unable to get Kyverno deployment", "reason", err.Error()) + return false + } + + return tls.IsKyvernoIsInRollingUpdate(deploy.UnstructuredContent(), logger) +} diff --git a/pkg/webhookconfig/registration.go b/pkg/webhookconfig/registration.go index 64ace56bf5..aef5331a49 100644 --- a/pkg/webhookconfig/registration.go +++ b/pkg/webhookconfig/registration.go @@ -11,9 +11,11 @@ import ( "github.com/kyverno/kyverno/pkg/config" client "github.com/kyverno/kyverno/pkg/dclient" "github.com/kyverno/kyverno/pkg/resourcecache" + "github.com/kyverno/kyverno/pkg/tls" admregapi "k8s.io/api/admissionregistration/v1beta1" errorsapi "k8s.io/apimachinery/pkg/api/errors" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" rest "k8s.io/client-go/rest" ) @@ -64,24 +66,29 @@ func (wrc *Register) Register() error { wrc.removeWebhookConfigurations() + caData := wrc.readCaData() + if caData == nil { + return errors.New("Unable to extract CA data from configuration") + } + errors := make([]string, 0) - if err := wrc.createVerifyMutatingWebhookConfiguration(); err != nil { + if err := wrc.createVerifyMutatingWebhookConfiguration(caData); err != nil { errors = append(errors, err.Error()) } - if err := wrc.createPolicyValidatingWebhookConfiguration(); err != nil { + if err := wrc.createPolicyValidatingWebhookConfiguration(caData); err != nil { errors = append(errors, err.Error()) } - if err := wrc.createPolicyMutatingWebhookConfiguration(); err != nil { + if err := wrc.createPolicyMutatingWebhookConfiguration(caData); err != nil { errors = append(errors, err.Error()) } - if err := wrc.createResourceValidatingWebhookConfiguration(); err != nil { + if err := wrc.createResourceValidatingWebhookConfiguration(caData); err != nil { errors = append(errors, err.Error()) } - if err := wrc.createResourceMutatingWebhookConfiguration(); err != nil { + if err := wrc.createResourceMutatingWebhookConfiguration(caData); err != nil { errors = append(errors, err.Error()) } @@ -122,19 +129,46 @@ func (wrc *Register) Check() error { // Remove removes all webhook configurations func (wrc *Register) Remove(cleanUp chan<- struct{}) { + defer close(cleanUp) + if !wrc.cleanupKyvernoResource() { + return + } + wrc.removeWebhookConfigurations() - close(cleanUp) + wrc.removeSecrets() } -func (wrc *Register) createResourceMutatingWebhookConfiguration() error { - - var caData []byte - var config *admregapi.MutatingWebhookConfiguration - - if caData = wrc.readCaData(); caData == nil { - return errors.New("Unable to extract CA data from configuration") +// cleanupKyvernoResource returns true if Kyverno deployment is terminating +func (wrc *Register) cleanupKyvernoResource() bool { + logger := wrc.log.WithName("cleanupKyvernoResource") + deploy, err := wrc.client.GetResource("", "Deployment", deployNamespace, deployName) + if err != nil { + logger.Error(err, "failed to get deployment") + return false } + if deploy.GetDeletionTimestamp() != nil { + logger.Info("Kyverno is terminating, clean up Kyverno resources") + return true + } + + replicas, _, err := unstructured.NestedInt64(deploy.UnstructuredContent(), "spec", "replicas") + if err != nil { + logger.Error(err, "unable to fetch spec.replicas of Kyverno deployment") + } + + if replicas == 0 { + logger.Info("Kyverno is scaled to zero, clean up Kyverno resources") + return true + } + + logger.Info("updating Kyverno Pod, won't clean up Kyverno resources") + return false +} + +func (wrc *Register) createResourceMutatingWebhookConfiguration(caData []byte) error { + var config *admregapi.MutatingWebhookConfiguration + if wrc.serverIP != "" { config = wrc.constructDebugMutatingWebhookConfig(caData) } else { @@ -158,13 +192,9 @@ func (wrc *Register) createResourceMutatingWebhookConfiguration() error { return nil } -func (wrc *Register) createResourceValidatingWebhookConfiguration() error { - var caData []byte +func (wrc *Register) createResourceValidatingWebhookConfiguration(caData []byte) error { var config *admregapi.ValidatingWebhookConfiguration - if caData = wrc.readCaData(); caData == nil { - return errors.New("Unable to extract CA data from configuration") - } if wrc.serverIP != "" { config = wrc.constructDebugValidatingWebhookConfig(caData) } else { @@ -189,15 +219,9 @@ func (wrc *Register) createResourceValidatingWebhookConfiguration() error { } //registerPolicyValidatingWebhookConfiguration create a Validating webhook configuration for Policy CRD -func (wrc *Register) createPolicyValidatingWebhookConfiguration() error { - var caData []byte +func (wrc *Register) createPolicyValidatingWebhookConfiguration(caData []byte) error { var config *admregapi.ValidatingWebhookConfiguration - // read certificate data - if caData = wrc.readCaData(); caData == nil { - return errors.New("Unable to extract CA data from configuration") - } - if wrc.serverIP != "" { config = wrc.contructDebugPolicyValidatingWebhookConfig(caData) } else { @@ -217,14 +241,9 @@ func (wrc *Register) createPolicyValidatingWebhookConfiguration() error { return nil } -func (wrc *Register) createPolicyMutatingWebhookConfiguration() error { - var caData []byte +func (wrc *Register) createPolicyMutatingWebhookConfiguration(caData []byte) error { var config *admregapi.MutatingWebhookConfiguration - if caData = wrc.readCaData(); caData == nil { - return errors.New("Unable to extract CA data from configuration") - } - if wrc.serverIP != "" { config = wrc.contructDebugPolicyMutatingWebhookConfig(caData) } else { @@ -245,14 +264,9 @@ func (wrc *Register) createPolicyMutatingWebhookConfiguration() error { return nil } -func (wrc *Register) createVerifyMutatingWebhookConfiguration() error { - var caData []byte +func (wrc *Register) createVerifyMutatingWebhookConfiguration(caData []byte) error { var config *admregapi.MutatingWebhookConfiguration - if caData = wrc.readCaData(); caData == nil { - return errors.New("Unable to extract CA data from configuration") - } - if wrc.serverIP != "" { config = wrc.constructDebugVerifyMutatingWebhookConfig(caData) } else { @@ -435,3 +449,24 @@ func (wrc *Register) getVerifyWebhookMutatingWebhookName() string { func (wrc *Register) GetWebhookTimeOut() time.Duration { return time.Duration(wrc.timeoutSeconds) } + +// removeSecrets removes Kyverno managed secrets +func (wrc *Register) removeSecrets() { + selector := &v1.LabelSelector{ + MatchLabels: map[string]string{ + tls.ManagedByLabel: "kyverno", + }, + } + + secretList, err := wrc.client.ListResource("", "Secret", config.KyvernoNamespace, selector) + if err != nil && errorsapi.IsNotFound(err) { + wrc.log.Error(err, "failed to clean up Kyverno managed secrets") + return + } + + for _, secret := range secretList.Items { + if err := wrc.client.DeleteResource("", "Secret", secret.GetNamespace(), secret.GetName(), false); err != nil { + wrc.log.Error(err, "failed to delete secret", "ns", secret.GetNamespace(), "name", secret.GetName()) + } + } +} diff --git a/pkg/webhooks/server.go b/pkg/webhooks/server.go index 14d279135a..7a48730675 100644 --- a/pkg/webhooks/server.go +++ b/pkg/webhooks/server.go @@ -27,6 +27,7 @@ import ( "github.com/kyverno/kyverno/pkg/policyreport" "github.com/kyverno/kyverno/pkg/policystatus" "github.com/kyverno/kyverno/pkg/resourcecache" + ktls "github.com/kyverno/kyverno/pkg/tls" tlsutils "github.com/kyverno/kyverno/pkg/tls" userinfo "github.com/kyverno/kyverno/pkg/userinfo" "github.com/kyverno/kyverno/pkg/utils" @@ -104,6 +105,8 @@ type WebhookServer struct { // last request time webhookMonitor *webhookconfig.Monitor + certRenewer *ktls.CertRenewer + // policy report generator prGenerator policyreport.GeneratorInterface @@ -148,6 +151,7 @@ func NewWebhookServer( pCache policycache.Interface, webhookRegistrationClient *webhookconfig.Register, webhookMonitor *webhookconfig.Monitor, + certRenewer *ktls.CertRenewer, statusSync policystatus.Listener, configHandler config.Interface, prGenerator policyreport.GeneratorInterface, @@ -198,6 +202,7 @@ func NewWebhookServer( configHandler: configHandler, cleanUp: cleanUp, webhookMonitor: webhookMonitor, + certRenewer: certRenewer, prGenerator: prGenerator, grGenerator: grGenerator, grController: grc, @@ -512,7 +517,7 @@ func (ws *WebhookServer) RunAsync(stopCh <-chan struct{}) { logger.Info("starting service") if !ws.debug { - go ws.webhookMonitor.Run(ws.webhookRegister, ws.eventGen, ws.client, stopCh) + go ws.webhookMonitor.Run(ws.webhookRegister, ws.certRenewer, ws.eventGen, stopCh) } }