From 972be16ad38460daded1a3cc3b3b97487785c513 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Charles-Edouard=20Br=C3=A9t=C3=A9ch=C3=A9?= Date: Mon, 2 May 2022 12:58:04 +0200 Subject: [PATCH] refactor: remove unstructured usage from webhookconfig (#3737) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * refactor: use typed informers and add tombstone support to webhookconfig Signed-off-by: Charles-Edouard Brétéché * refactor: remove unstructured usage from webhookconfig Signed-off-by: Charles-Edouard Brétéché --- cmd/kyverno/main.go | 8 - pkg/config/config.go | 13 +- pkg/webhookconfig/certmanager.go | 26 +-- pkg/webhookconfig/common.go | 44 ++--- pkg/webhookconfig/configmanager.go | 5 - pkg/webhookconfig/monitor.go | 2 +- pkg/webhookconfig/policy.go | 3 - pkg/webhookconfig/registration.go | 286 ++++++----------------------- pkg/webhookconfig/resource.go | 20 +- pkg/webhookconfig/status.go | 8 +- 10 files changed, 100 insertions(+), 315 deletions(-) diff --git a/cmd/kyverno/main.go b/cmd/kyverno/main.go index d89b64bd5d..08523ca4ef 100755 --- a/cmd/kyverno/main.go +++ b/cmd/kyverno/main.go @@ -34,7 +34,6 @@ import ( "github.com/kyverno/kyverno/pkg/policycache" "github.com/kyverno/kyverno/pkg/policyreport" "github.com/kyverno/kyverno/pkg/registryclient" - "github.com/kyverno/kyverno/pkg/resourcecache" "github.com/kyverno/kyverno/pkg/signal" ktls "github.com/kyverno/kyverno/pkg/tls" "github.com/kyverno/kyverno/pkg/toggle" @@ -156,12 +155,6 @@ func main() { kubeKyvernoInformer := kubeinformers.NewSharedInformerFactoryWithOptions(kubeClient, resyncPeriod, kubeinformers.WithNamespace(config.KyvernoNamespace)) kubedynamicInformer := client.NewDynamicSharedInformerFactory(resyncPeriod) - rCache, err := resourcecache.NewResourceCache(client, kubedynamicInformer, log.Log.WithName("resourcecache")) - if err != nil { - setupLog.Error(err, "ConfigMap lookup disabled: failed to create resource cache") - os.Exit(1) - } - // load image registry secrets secrets := strings.Split(imagePullSecrets, ",") if imagePullSecrets != "" && len(secrets) > 0 { @@ -221,7 +214,6 @@ func main() { pclient, kubeInformer.Admissionregistration().V1().MutatingWebhookConfigurations(), kubeInformer.Admissionregistration().V1().ValidatingWebhookConfigurations(), - rCache, kubeKyvernoInformer.Apps().V1().Deployments(), pInformer.Kyverno().V1().ClusterPolicies(), pInformer.Kyverno().V1().Policies(), diff --git a/pkg/config/config.go b/pkg/config/config.go index c49529b43e..7b971f5a5e 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -47,23 +47,12 @@ const ( // Issue: https://github.com/kubernetes/kubernetes/pull/63972 // When the issue is closed, we should use TypeMeta struct instead of this constants - // DeploymentKind define the default deployment resource kind - DeploymentKind = "Deployment" - - // DeploymentAPIVersion define the default deployment resource apiVersion - DeploymentAPIVersion = "apps/v1" - - // NamespaceKind define the default namespace resource kind - NamespaceKind = "Namespace" - - // NamespaceAPIVersion define the default namespace resource apiVersion - NamespaceAPIVersion = "v1" - // ClusterRoleAPIVersion define the default clusterrole resource apiVersion ClusterRoleAPIVersion = "rbac.authorization.k8s.io/v1" // ClusterRoleKind define the default clusterrole resource kind ClusterRoleKind = "ClusterRole" + //MutatingWebhookServicePath is the path for mutation webhook MutatingWebhookServicePath = "/mutate" diff --git a/pkg/webhookconfig/certmanager.go b/pkg/webhookconfig/certmanager.go index 4de6303469..9df0ffc24f 100644 --- a/pkg/webhookconfig/certmanager.go +++ b/pkg/webhookconfig/certmanager.go @@ -9,7 +9,7 @@ import ( "github.com/go-logr/logr" "github.com/kyverno/kyverno/pkg/common" "github.com/kyverno/kyverno/pkg/config" - ktls "github.com/kyverno/kyverno/pkg/tls" + "github.com/kyverno/kyverno/pkg/tls" v1 "k8s.io/api/core/v1" informerv1 "k8s.io/client-go/informers/core/v1" "k8s.io/client-go/kubernetes" @@ -25,17 +25,17 @@ type Interface interface { InitTLSPemPair() // GetTLSPemPair gets the existing TLSPemPair from the secret - GetTLSPemPair() (*ktls.PemPair, error) + GetTLSPemPair() (*tls.PemPair, error) } type certManager struct { - renewer *ktls.CertRenewer + renewer *tls.CertRenewer secretInformer informerv1.SecretInformer secretQueue chan bool stopCh <-chan struct{} log logr.Logger } -func NewCertManager(secretInformer informerv1.SecretInformer, kubeClient kubernetes.Interface, certRenewer *ktls.CertRenewer, log logr.Logger, stopCh <-chan struct{}) (Interface, error) { +func NewCertManager(secretInformer informerv1.SecretInformer, kubeClient kubernetes.Interface, certRenewer *tls.CertRenewer, log logr.Logger, stopCh <-chan struct{}) (Interface, error) { manager := &certManager{ renewer: certRenewer, secretInformer: secretInformer, @@ -58,7 +58,7 @@ func (m *certManager) addSecretFunc(obj interface{}) { return } - val, ok := secret.GetAnnotations()[ktls.SelfSignedAnnotation] + val, ok := secret.GetAnnotations()[tls.SelfSignedAnnotation] if !ok || val != "true" { return } @@ -73,7 +73,7 @@ func (m *certManager) updateSecretFunc(oldObj interface{}, newObj interface{}) { return } - val, ok := new.GetAnnotations()[ktls.SelfSignedAnnotation] + val, ok := new.GetAnnotations()[tls.SelfSignedAnnotation] if !ok || val != "true" { return } @@ -94,12 +94,12 @@ func (m *certManager) InitTLSPemPair() { } } -func (m *certManager) GetTLSPemPair() (*ktls.PemPair, error) { - var tls *ktls.PemPair +func (m *certManager) GetTLSPemPair() (*tls.PemPair, error) { + var keyPair *tls.PemPair var err error retryReadTLS := func() error { - tls, err = ktls.ReadTLSPair(m.renewer.ClientConfig(), m.renewer.Client()) + keyPair, err = tls.ReadTLSPair(m.renewer.ClientConfig(), m.renewer.Client()) if err != nil { return err } @@ -112,7 +112,7 @@ func (m *certManager) GetTLSPemPair() (*ktls.PemPair, error) { f := common.RetryFunc(time.Second, time.Minute, retryReadTLS, msg, m.log.WithName("GetTLSPemPair/Retry")) err = f() - return tls, err + return keyPair, err } func (m *certManager) Run(stopCh <-chan struct{}) { @@ -127,7 +127,7 @@ func (m *certManager) Run(stopCh <-chan struct{}) { }) m.log.Info("start managing certificate") - certsRenewalTicker := time.NewTicker(ktls.CertRenewalInterval) + certsRenewalTicker := time.NewTicker(tls.CertRenewalInterval) defer certsRenewalTicker.Stop() for { @@ -137,7 +137,7 @@ func (m *certManager) Run(stopCh <-chan struct{}) { if err != nil { m.log.Error(err, "failed to validate cert") - if !strings.Contains(err.Error(), ktls.ErrorsNotFound) { + if !strings.Contains(err.Error(), tls.ErrorsNotFound) { continue } } @@ -157,7 +157,7 @@ func (m *certManager) Run(stopCh <-chan struct{}) { if err != nil { m.log.Error(err, "failed to validate cert") - if !strings.Contains(err.Error(), ktls.ErrorsNotFound) { + if !strings.Contains(err.Error(), tls.ErrorsNotFound) { continue } } diff --git a/pkg/webhookconfig/common.go b/pkg/webhookconfig/common.go index 3528ede022..fe8b8699fd 100644 --- a/pkg/webhookconfig/common.go +++ b/pkg/webhookconfig/common.go @@ -1,7 +1,7 @@ package webhookconfig import ( - "encoding/json" + "context" "errors" "io/ioutil" "path/filepath" @@ -11,11 +11,10 @@ import ( "github.com/kyverno/kyverno/pkg/config" "github.com/kyverno/kyverno/pkg/tls" admregapi "k8s.io/api/admissionregistration/v1" - apps "k8s.io/api/apps/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime/schema" - rest "k8s.io/client-go/rest" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/rest" ) func (wrc *Register) readCaData() []byte { @@ -60,16 +59,14 @@ func extractCA(config *rest.Config) (result []byte) { return config.TLSClientConfig.CAData } -func (wrc *Register) constructOwner() v1.OwnerReference { +func (wrc *Register) constructOwner() metav1.OwnerReference { logger := wrc.log - kubeClusterRoleName, err := wrc.GetKubePolicyClusterRoleName() if err != nil { logger.Error(err, "failed to get cluster role") - return v1.OwnerReference{} + return metav1.OwnerReference{} } - - return v1.OwnerReference{ + return metav1.OwnerReference{ APIVersion: config.ClusterRoleAPIVersion, Kind: config.ClusterRoleKind, Name: kubeClusterRoleName.GetName(), @@ -77,8 +74,13 @@ func (wrc *Register) constructOwner() v1.OwnerReference { } } -func (wrc *Register) GetKubePolicyClusterRoleName() (*unstructured.Unstructured, error) { - clusterRoles, err := wrc.client.ListResource(config.ClusterRoleAPIVersion, config.ClusterRoleKind, "", &v1.LabelSelector{MatchLabels: map[string]string{"app.kubernetes.io/name": "kyverno"}}) +func (wrc *Register) GetKubePolicyClusterRoleName() (*corev1.ClusterRole, error) { + selector := &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app.kubernetes.io/name": "kyverno", + }, + } + clusterRoles, err := wrc.kubeClient.RbacV1().ClusterRoles().List(context.TODO(), metav1.ListOptions{LabelSelector: metav1.FormatLabelSelector(selector)}) if err != nil { return nil, err } @@ -92,22 +94,12 @@ func (wrc *Register) GetKubePolicyClusterRoleName() (*unstructured.Unstructured, // GetKubePolicyDeployment gets Kyverno deployment using the resource cache // it does not initialize any client call -func (wrc *Register) GetKubePolicyDeployment() (*apps.Deployment, *unstructured.Unstructured, error) { +func (wrc *Register) GetKubePolicyDeployment() (*appsv1.Deployment, error) { deploy, err := wrc.kDeplLister.Deployments(config.KyvernoNamespace).Get(config.KyvernoDeploymentName) if err != nil { - return nil, nil, err + return nil, err } - deploy.SetGroupVersionKind(schema.GroupVersionKind{Group: "", Version: "apps/v1", Kind: "Deployment"}) - kubePolicyDeployment := unstructured.Unstructured{} - rawDepl, err := json.Marshal(deploy) - if err != nil { - return nil, nil, err - } - err = json.Unmarshal(rawDepl, &kubePolicyDeployment.Object) - if err != nil { - return deploy, nil, err - } - return deploy, &kubePolicyDeployment, nil + return deploy, nil } // debug mutating webhook diff --git a/pkg/webhookconfig/configmanager.go b/pkg/webhookconfig/configmanager.go index e619c57481..e1a4dd44a5 100644 --- a/pkg/webhookconfig/configmanager.go +++ b/pkg/webhookconfig/configmanager.go @@ -18,7 +18,6 @@ import ( "github.com/kyverno/kyverno/pkg/common" "github.com/kyverno/kyverno/pkg/config" client "github.com/kyverno/kyverno/pkg/dclient" - "github.com/kyverno/kyverno/pkg/resourcecache" "github.com/kyverno/kyverno/pkg/utils" kubeutils "github.com/kyverno/kyverno/pkg/utils/kube" "github.com/pkg/errors" @@ -58,8 +57,6 @@ type webhookConfigManager struct { // npListerSynced returns true if the namespace policy store has been synced at least once npListerSynced cache.InformerSynced - resCache resourcecache.ResourceCache - mutateInformer adminformers.MutatingWebhookConfigurationInformer validateInformer adminformers.ValidatingWebhookConfigurationInformer mutateLister admlisters.MutatingWebhookConfigurationLister @@ -95,7 +92,6 @@ func newWebhookConfigManager( npInformer kyvernoinformer.PolicyInformer, mwcInformer adminformers.MutatingWebhookConfigurationInformer, vwcInformer adminformers.ValidatingWebhookConfigurationInformer, - resCache resourcecache.ResourceCache, serverIP string, autoUpdateWebhooks bool, createDefaultWebhook chan<- string, @@ -107,7 +103,6 @@ func newWebhookConfigManager( kyvernoClient: kyvernoClient, pInformer: pInformer, npInformer: npInformer, - resCache: resCache, queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "configmanager"), wildcardPolicy: 0, serverIP: serverIP, diff --git a/pkg/webhookconfig/monitor.go b/pkg/webhookconfig/monitor.go index bcbe636936..0ec98b8906 100644 --- a/pkg/webhookconfig/monitor.go +++ b/pkg/webhookconfig/monitor.go @@ -225,7 +225,7 @@ func lastRequestTimeFromAnnotation(leaseClient coordinationv1.LeaseInterface, lo // skipWebhookCheck returns true if Kyverno is in rolling update func skipWebhookCheck(register *Register, logger logr.Logger) bool { - deploy, _, err := register.GetKubePolicyDeployment() + deploy, err := register.GetKubePolicyDeployment() if err != nil { logger.Info("unable to get Kyverno deployment", "reason", err.Error()) return false diff --git a/pkg/webhookconfig/policy.go b/pkg/webhookconfig/policy.go index 4d36b3ebb2..57f7cd47d0 100644 --- a/pkg/webhookconfig/policy.go +++ b/pkg/webhookconfig/policy.go @@ -9,7 +9,6 @@ import ( ) func (wrc *Register) constructPolicyValidatingWebhookConfig(caData []byte) *admregapi.ValidatingWebhookConfiguration { - return &admregapi.ValidatingWebhookConfiguration{ ObjectMeta: v1.ObjectMeta{ Name: config.PolicyValidatingWebhookConfigurationName, @@ -40,7 +39,6 @@ func (wrc *Register) constructDebugPolicyValidatingWebhookConfig(caData []byte) logger := wrc.log url := fmt.Sprintf("https://%s%s", wrc.serverIP, config.PolicyValidatingWebhookServicePath) logger.V(4).Info("Debug PolicyValidatingWebhookConfig is registered with url ", "url", url) - return &admregapi.ValidatingWebhookConfiguration{ ObjectMeta: v1.ObjectMeta{ Name: config.PolicyValidatingWebhookConfigurationDebugName, @@ -95,7 +93,6 @@ func (wrc *Register) constructDebugPolicyMutatingWebhookConfig(caData []byte) *a logger := wrc.log url := fmt.Sprintf("https://%s%s", wrc.serverIP, config.PolicyMutatingWebhookServicePath) logger.V(4).Info("Debug PolicyMutatingWebhookConfig is registered with url ", "url", url) - return &admregapi.MutatingWebhookConfiguration{ ObjectMeta: v1.ObjectMeta{ Name: config.PolicyMutatingWebhookConfigurationDebugName, diff --git a/pkg/webhookconfig/registration.go b/pkg/webhookconfig/registration.go index 1c04f4557b..fa08a66e50 100644 --- a/pkg/webhookconfig/registration.go +++ b/pkg/webhookconfig/registration.go @@ -14,7 +14,6 @@ import ( kyvernoinformer "github.com/kyverno/kyverno/pkg/client/informers/externalversions/kyverno/v1" "github.com/kyverno/kyverno/pkg/config" client "github.com/kyverno/kyverno/pkg/dclient" - "github.com/kyverno/kyverno/pkg/resourcecache" "github.com/kyverno/kyverno/pkg/tls" "github.com/kyverno/kyverno/pkg/utils" "github.com/pkg/errors" @@ -22,9 +21,6 @@ import ( corev1 "k8s.io/api/core/v1" errorsapi "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" adminformers "k8s.io/client-go/informers/admissionregistration/v1" informers "k8s.io/client-go/informers/apps/v1" "k8s.io/client-go/kubernetes" @@ -46,16 +42,19 @@ const ( // 4. Resource Mutation // 5. Webhook Status Mutation type Register struct { - client *client.Client + // clients kubeClient kubernetes.Interface clientConfig *rest.Config - resCache resourcecache.ResourceCache - mwcLister admlisters.MutatingWebhookConfigurationLister - vwcLister admlisters.ValidatingWebhookConfigurationLister + // listers + mwcLister admlisters.MutatingWebhookConfigurationLister + vwcLister admlisters.ValidatingWebhookConfigurationLister + kDeplLister listers.DeploymentLister - mwcListerSynced func() bool - vwcListerSynced func() bool + // sync + mwcListerSynced cache.InformerSynced + vwcListerSynced cache.InformerSynced + kDeplListerSynced cache.InformerSynced serverIP string // when running outside a cluster timeoutSeconds int32 @@ -67,9 +66,6 @@ type Register struct { UpdateWebhookChan chan bool createDefaultWebhook chan string - kDeplLister listers.DeploymentLister - kDeplListerSynced func() bool - // manage implements methods to manage webhook configurations manage } @@ -82,7 +78,6 @@ func NewRegister( kyvernoClient *kyvernoclient.Clientset, mwcInformer adminformers.MutatingWebhookConfigurationInformer, vwcInformer adminformers.ValidatingWebhookConfigurationInformer, - resCache resourcecache.ResourceCache, kDeplInformer informers.DeploymentInformer, pInformer kyvernoinformer.ClusterPolicyInformer, npInformer kyvernoinformer.PolicyInformer, @@ -94,9 +89,13 @@ func NewRegister( log logr.Logger) *Register { register := &Register{ clientConfig: clientConfig, - client: client, kubeClient: kubeClient, - resCache: resCache, + mwcLister: mwcInformer.Lister(), + vwcLister: vwcInformer.Lister(), + kDeplLister: kDeplInformer.Lister(), + mwcListerSynced: mwcInformer.Informer().HasSynced, + vwcListerSynced: vwcInformer.Informer().HasSynced, + kDeplListerSynced: kDeplInformer.Informer().HasSynced, serverIP: serverIP, timeoutSeconds: webhookTimeout, log: log.WithName("Register"), @@ -104,16 +103,10 @@ func NewRegister( autoUpdateWebhooks: autoUpdateWebhooks, UpdateWebhookChan: make(chan bool), createDefaultWebhook: make(chan string), - kDeplLister: kDeplInformer.Lister(), - kDeplListerSynced: kDeplInformer.Informer().HasSynced, - mwcLister: mwcInformer.Lister(), - vwcLister: vwcInformer.Lister(), - mwcListerSynced: mwcInformer.Informer().HasSynced, - vwcListerSynced: vwcInformer.Informer().HasSynced, stopCh: stopCh, } - register.manage = newWebhookConfigManager(client, kyvernoClient, pInformer, npInformer, mwcInformer, vwcInformer, resCache, serverIP, register.autoUpdateWebhooks, register.createDefaultWebhook, stopCh, log.WithName("WebhookConfigManager")) + register.manage = newWebhookConfigManager(client, kyvernoClient, pInformer, npInformer, mwcInformer, vwcInformer, serverIP, register.autoUpdateWebhooks, register.createDefaultWebhook, stopCh, log.WithName("WebhookConfigManager")) return register } @@ -252,24 +245,16 @@ func (wrc *Register) UpdateWebhookConfigurations(configHandler config.Interface) func (wrc *Register) ValidateWebhookConfigurations(namespace, name string) error { logger := wrc.log.WithName("ValidateWebhookConfigurations") - - cm, err := wrc.client.GetResource("", "ConfigMap", namespace, name) + cm, err := wrc.kubeClient.CoreV1().ConfigMaps(namespace).Get(context.TODO(), name, metav1.GetOptions{}) if err != nil { logger.Error(err, "unable to fetch ConfigMap", "namespace", namespace, "name", name) return nil } - - webhooks, ok, err := unstructured.NestedString(cm.UnstructuredContent(), "data", "webhooks") - if err != nil { - logger.Error(err, "failed to fetch tag 'webhooks' from the ConfigMap") - return nil - } - + webhooks, ok := cm.Data["webhooks"] if !ok { logger.V(4).Info("webhook configurations not defined") return nil } - webhookCfgs := make([]config.WebhookConfig, 0, 10) return json.Unmarshal([]byte(webhooks), &webhookCfgs) } @@ -277,13 +262,12 @@ func (wrc *Register) ValidateWebhookConfigurations(namespace, name string) error // cleanupKyvernoResource returns true if Kyverno is terminating func (wrc *Register) cleanupKyvernoResource() bool { logger := wrc.log.WithName("cleanupKyvernoResource") - deploy, err := wrc.client.GetResource("", "Deployment", config.KyvernoNamespace, config.KyvernoDeploymentName) + deploy, err := wrc.kubeClient.AppsV1().Deployments(config.KyvernoNamespace).Get(context.TODO(), config.KyvernoDeploymentName, metav1.GetOptions{}) if err != nil { if errorsapi.IsNotFound(err) { logger.Info("Kyverno deployment not found, cleanup Kyverno resources") return true } - logger.Error(err, "failed to get deployment, not cleaning up kyverno resources") return false } @@ -291,17 +275,10 @@ func (wrc *Register) cleanupKyvernoResource() bool { logger.Info("Kyverno is terminating, cleanup Kyverno resources") return true } - - replicas, _, err := unstructured.NestedInt64(deploy.UnstructuredContent(), "spec", "replicas") - if err != nil { - logger.Error(err, "unable to fetch spec.replicas of Kyverno deployment") - } - - if replicas == 0 { + if deploy.Spec.Replicas == nil && *deploy.Spec.Replicas == 0 { logger.Info("Kyverno is scaled to zero, cleanup Kyverno resources") return true } - logger.Info("updating Kyverno Pod, won't clean up Kyverno resources") return false } @@ -317,7 +294,7 @@ func (wrc *Register) createResourceMutatingWebhookConfiguration(caData []byte) e logger := wrc.log.WithValues("kind", kindMutating, "name", config.Name) - _, err := wrc.client.CreateResource("", kindMutating, "", *config, false) + _, err := wrc.kubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Create(context.TODO(), config, metav1.CreateOptions{}) if errorsapi.IsAlreadyExists(err) { logger.V(6).Info("resource mutating webhook configuration already exists", "name", config.Name) err = wrc.updateMutatingWebhookConfiguration(config) @@ -346,7 +323,7 @@ func (wrc *Register) createResourceValidatingWebhookConfiguration(caData []byte) logger := wrc.log.WithValues("kind", kindValidating, "name", config.Name) - _, err := wrc.client.CreateResource("", kindValidating, "", *config, false) + _, err := wrc.kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Create(context.TODO(), config, metav1.CreateOptions{}) if errorsapi.IsAlreadyExists(err) { logger.V(6).Info("resource validating webhook configuration already exists", "name", config.Name) err = wrc.updateValidatingWebhookConfiguration(config) @@ -375,7 +352,7 @@ func (wrc *Register) createPolicyValidatingWebhookConfiguration(caData []byte) e config = wrc.constructPolicyValidatingWebhookConfig(caData) } - if _, err := wrc.client.CreateResource("", kindValidating, "", *config, false); err != nil { + if _, err := wrc.kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Create(context.TODO(), config, metav1.CreateOptions{}); err != nil { if errorsapi.IsAlreadyExists(err) { wrc.log.V(6).Info("webhook already exists", "kind", kindValidating, "name", config.Name) err = wrc.updateValidatingWebhookConfiguration(config) @@ -401,8 +378,7 @@ func (wrc *Register) createPolicyMutatingWebhookConfiguration(caData []byte) err config = wrc.constructPolicyMutatingWebhookConfig(caData) } - // create mutating webhook configuration resource - if _, err := wrc.client.CreateResource("", kindMutating, "", *config, false); err != nil { + if _, err := wrc.kubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Create(context.TODO(), config, metav1.CreateOptions{}); err != nil { if errorsapi.IsAlreadyExists(err) { wrc.log.V(6).Info("webhook already exists", "kind", kindMutating, "name", config.Name) err = wrc.updateMutatingWebhookConfiguration(config) @@ -428,7 +404,7 @@ func (wrc *Register) createVerifyMutatingWebhookConfiguration(caData []byte) err config = wrc.constructVerifyMutatingWebhookConfig(caData) } - if _, err := wrc.client.CreateResource("", kindMutating, "", *config, false); err != nil { + if _, err := wrc.kubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Create(context.TODO(), config, metav1.CreateOptions{}); err != nil { if errorsapi.IsAlreadyExists(err) { wrc.log.V(6).Info("webhook already exists", "kind", kindMutating, "name", config.Name) err = wrc.updateMutatingWebhookConfiguration(config) @@ -476,7 +452,7 @@ func (wrc *Register) removePolicyMutatingWebhookConfiguration(wg *sync.WaitGroup return } - err := wrc.client.DeleteResource("", kindMutating, "", mutatingConfig, false) + err := wrc.kubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete(context.TODO(), mutatingConfig, metav1.DeleteOptions{}) if errorsapi.IsNotFound(err) { logger.V(5).Info("policy mutating webhook configuration not found") return @@ -512,7 +488,7 @@ func (wrc *Register) removePolicyValidatingWebhookConfiguration(wg *sync.WaitGro } logger.V(4).Info("removing validating webhook configuration") - err := wrc.client.DeleteResource("", kindValidating, "", validatingConfig, false) + err := wrc.kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Delete(context.TODO(), validatingConfig, metav1.DeleteOptions{}) if errorsapi.IsNotFound(err) { logger.V(5).Info("policy validating webhook configuration not found") return @@ -615,7 +591,7 @@ func (wrc *Register) removeVerifyWebhookMutatingWebhookConfig(wg *sync.WaitGroup return } - err = wrc.client.DeleteResource("", kindMutating, "", mutatingConfig, false) + err = wrc.kubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete(context.TODO(), mutatingConfig, metav1.DeleteOptions{}) if errorsapi.IsNotFound(err) { logger.V(5).Info("verify webhook configuration not found") return @@ -651,52 +627,37 @@ func (wrc *Register) removeSecrets() { tls.ManagedByLabel: "kyverno", }, } - - secretList, err := wrc.client.ListResource("", "Secret", config.KyvernoNamespace, selector) - if err != nil { + if err := wrc.kubeClient.CoreV1().Secrets(config.KyvernoNamespace).DeleteCollection(context.TODO(), metav1.DeleteOptions{}, metav1.ListOptions{LabelSelector: metav1.FormatLabelSelector(selector)}); err != nil { wrc.log.Error(err, "failed to clean up Kyverno managed secrets") return } - - for _, secret := range secretList.Items { - if err := wrc.client.DeleteResource("", "Secret", secret.GetNamespace(), secret.GetName(), false); err != nil { - if !errorsapi.IsNotFound(err) { - wrc.log.Error(err, "failed to delete secret", "ns", secret.GetNamespace(), "name", secret.GetName()) - } - } - } } func (wrc *Register) checkEndpoint() error { - obj, err := wrc.client.GetResource("", "Endpoints", config.KyvernoNamespace, config.KyvernoServiceName) + endpoint, err := wrc.kubeClient.CoreV1().Endpoints(config.KyvernoNamespace).Get(context.TODO(), config.KyvernoServiceName, metav1.GetOptions{}) if err != nil { return fmt.Errorf("failed to get endpoint %s/%s: %v", config.KyvernoNamespace, config.KyvernoServiceName, err) } - var endpoint corev1.Endpoints - err = runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), &endpoint) - if err != nil { - return fmt.Errorf("failed to convert endpoint %s/%s from unstructured: %v", config.KyvernoNamespace, config.KyvernoServiceName, err) + selector := &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app.kubernetes.io/name": "kyverno", + }, } - - pods, err := wrc.client.ListResource("", "Pod", config.KyvernoNamespace, &metav1.LabelSelector{MatchLabels: map[string]string{"app.kubernetes.io/name": "kyverno"}}) + pods, err := wrc.kubeClient.CoreV1().Pods(config.KyvernoNamespace).List(context.TODO(), metav1.ListOptions{LabelSelector: metav1.FormatLabelSelector(selector)}) if err != nil { return fmt.Errorf("failed to list Kyverno Pod: %v", err) } - ips, errs := getHealthyPodsIP(pods.Items) if len(errs) != 0 { return fmt.Errorf("error getting pod's IP: %v", errs) } - if len(ips) == 0 { return fmt.Errorf("pod is not assigned to any node yet") } - for _, subset := range endpoint.Subsets { if len(subset.Addresses) == 0 { continue } - for _, addr := range subset.Addresses { if utils.ContainsString(ips, addr.IP) { wrc.log.Info("Endpoint ready", "ns", config.KyvernoNamespace, "name", config.KyvernoServiceName) @@ -704,130 +665,36 @@ func (wrc *Register) checkEndpoint() error { } } } - err = fmt.Errorf("endpoint not ready") wrc.log.V(3).Info(err.Error(), "ns", config.KyvernoNamespace, "name", config.KyvernoServiceName) return err } -func getHealthyPodsIP(pods []unstructured.Unstructured) (ips []string, errs []error) { +func getHealthyPodsIP(pods []corev1.Pod) (ips []string, errs []error) { for _, pod := range pods { - phase, _, err := unstructured.NestedString(pod.UnstructuredContent(), "status", "phase") - if err != nil { - errs = append(errs, fmt.Errorf("failed to get pod %s status: %v", pod.GetName(), err)) + if pod.Status.Phase != "Running" { continue } - - if phase != "Running" { - continue - } - - ip, _, err := unstructured.NestedString(pod.UnstructuredContent(), "status", "podIP") - if err != nil { - errs = append(errs, fmt.Errorf("failed to extract pod %s IP: %v", pod.GetName(), err)) - continue - } - - ips = append(ips, ip) + ips = append(ips, pod.Status.PodIP) } - return } -func convertLabelSelector(selector *metav1.LabelSelector, logger logr.Logger) (map[string]interface{}, error) { - if selector == nil { - return nil, nil - } - if selectorBytes, err := json.Marshal(*selector); err != nil { - logger.Error(err, "failed to serialize selector") - return nil, err - } else { - var nsSelector map[string]interface{} - if err := json.Unmarshal(selectorBytes, &nsSelector); err != nil { - logger.Error(err, "failed to convert namespaceSelector to the map") - return nil, err - } - return nsSelector, nil - } -} - -func configureSelector(webhook map[string]interface{}, selector *metav1.LabelSelector, key string, logger logr.Logger) (bool, error) { - currentSelector, _, err := unstructured.NestedMap(webhook, key) - if err != nil { - return false, err - } - if expectedSelector, err := convertLabelSelector(selector, logger); err != nil { - return false, err - } else { - if reflect.DeepEqual(expectedSelector, currentSelector) { - return false, nil - } else { - if err = unstructured.SetNestedMap(webhook, expectedSelector, key); err != nil { - return false, err - } - return true, nil - } - } -} - func (wrc *Register) updateResourceValidatingWebhookConfiguration(webhookCfg config.WebhookConfig) error { - resourceValidatingTyped, err := wrc.vwcLister.Get(getResourceValidatingWebhookConfigName(wrc.serverIP)) + resource, err := wrc.vwcLister.Get(getResourceValidatingWebhookConfigName(wrc.serverIP)) if err != nil { return errors.Wrapf(err, "unable to get validatingWebhookConfigurations") } - resourceValidatingTyped.SetGroupVersionKind(schema.GroupVersionKind{Group: "", Version: "admissionregistration.k8s.io/v1", Kind: kindValidating}) - - resourceValidating := &unstructured.Unstructured{} - rawResc, err := json.Marshal(resourceValidatingTyped) - if err != nil { - return err + copy := resource.DeepCopy() + for i := range copy.Webhooks { + copy.Webhooks[i].ObjectSelector = webhookCfg.ObjectSelector + copy.Webhooks[i].NamespaceSelector = webhookCfg.NamespaceSelector } - err = json.Unmarshal(rawResc, &resourceValidating.Object) - if err != nil { - return err - } - - webhooksUntyped, _, err := unstructured.NestedSlice(resourceValidating.UnstructuredContent(), "webhooks") - if err != nil { - return errors.Wrapf(err, "unable to load validatingWebhookConfigurations.webhooks") - } - - var ( - webhook map[string]interface{} - webhooks []interface{} - ok bool - ) - - webhookChanged := false - for i, whu := range webhooksUntyped { - webhook, ok = whu.(map[string]interface{}) - if !ok { - return errors.Wrapf(err, "type mismatched, expected map[string]interface{}, got %T", webhooksUntyped[i]) - } - if changed, err := configureSelector(webhook, webhookCfg.ObjectSelector, "objectSelector", wrc.log); err != nil { - return errors.Wrapf(err, "unable to get validatingWebhookConfigurations.webhooks["+fmt.Sprint(i)+"].objectSelector") - } else { - if changed { - webhookChanged = true - } - } - if changed, err := configureSelector(webhook, webhookCfg.NamespaceSelector, "namespaceSelector", wrc.log); err != nil { - return errors.Wrapf(err, "unable to get validatingWebhookConfigurations.webhooks["+fmt.Sprint(i)+"].namespaceSelector") - } else { - if changed { - webhookChanged = true - } - } - webhooks = append(webhooks, webhook) - } - if !webhookChanged { + if reflect.DeepEqual(resource.Webhooks, copy.Webhooks) { wrc.log.V(4).Info("namespaceSelector unchanged, skip updating validatingWebhookConfigurations") return nil } - if err = unstructured.SetNestedSlice(resourceValidating.UnstructuredContent(), webhooks, "webhooks"); err != nil { - return errors.Wrapf(err, "unable to set validatingWebhookConfigurations.webhooks") - } - if _, err := wrc.client.UpdateResource(resourceValidating.GetAPIVersion(), resourceValidating.GetKind(), "", resourceValidating, false); err != nil { + if _, err := wrc.kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Update(context.TODO(), copy, metav1.UpdateOptions{}); err != nil { return err } wrc.log.V(3).Info("successfully updated validatingWebhookConfigurations", "name", getResourceMutatingWebhookConfigName(wrc.serverIP)) @@ -835,63 +702,20 @@ func (wrc *Register) updateResourceValidatingWebhookConfiguration(webhookCfg con } func (wrc *Register) updateResourceMutatingWebhookConfiguration(webhookCfg config.WebhookConfig) error { - resourceMutatingTyped, err := wrc.mwcLister.Get(getResourceMutatingWebhookConfigName(wrc.serverIP)) + resource, err := wrc.mwcLister.Get(getResourceMutatingWebhookConfigName(wrc.serverIP)) if err != nil { return errors.Wrapf(err, "unable to get mutatingWebhookConfigurations") } - resourceMutatingTyped.SetGroupVersionKind(schema.GroupVersionKind{Group: "", Version: "admissionregistration.k8s.io/v1", Kind: kindMutating}) - - resourceMutating := &unstructured.Unstructured{} - rawResc, err := json.Marshal(resourceMutatingTyped) - if err != nil { - return err + copy := resource.DeepCopy() + for i := range copy.Webhooks { + copy.Webhooks[i].ObjectSelector = webhookCfg.ObjectSelector + copy.Webhooks[i].NamespaceSelector = webhookCfg.NamespaceSelector } - err = json.Unmarshal(rawResc, &resourceMutating.Object) - if err != nil { - return err - } - - webhooksUntyped, _, err := unstructured.NestedSlice(resourceMutating.UnstructuredContent(), "webhooks") - if err != nil { - return errors.Wrapf(err, "unable to load mutatingWebhookConfigurations.webhooks") - } - - var ( - webhook map[string]interface{} - webhooks []interface{} - ok bool - ) - - webhookChanged := false - for i, whu := range webhooksUntyped { - webhook, ok = whu.(map[string]interface{}) - if !ok { - return errors.Wrapf(err, "type mismatched, expected map[string]interface{}, got %T", webhooksUntyped[i]) - } - if changed, err := configureSelector(webhook, webhookCfg.ObjectSelector, "objectSelector", wrc.log); err != nil { - return errors.Wrapf(err, "unable to get mutatingWebhookConfigurations.webhooks["+fmt.Sprint(i)+"].objectSelector") - } else { - if changed { - webhookChanged = true - } - } - if changed, err := configureSelector(webhook, webhookCfg.NamespaceSelector, "namespaceSelector", wrc.log); err != nil { - return errors.Wrapf(err, "unable to get mutatingWebhookConfigurations.webhooks["+fmt.Sprint(i)+"].namespaceSelector") - } else { - if changed { - webhookChanged = true - } - } - webhooks = append(webhooks, webhook) - } - if !webhookChanged { + if reflect.DeepEqual(resource.Webhooks, copy.Webhooks) { wrc.log.V(4).Info("namespaceSelector unchanged, skip updating mutatingWebhookConfigurations") return nil } - if err = unstructured.SetNestedSlice(resourceMutating.UnstructuredContent(), webhooks, "webhooks"); err != nil { - return errors.Wrapf(err, "unable to set mutatingWebhookConfigurations.webhooks") - } - if _, err := wrc.client.UpdateResource(resourceMutating.GetAPIVersion(), resourceMutating.GetKind(), "", resourceMutating, false); err != nil { + if _, err := wrc.kubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Update(context.TODO(), copy, metav1.UpdateOptions{}); err != nil { return err } wrc.log.V(3).Info("successfully updated mutatingWebhookConfigurations", "name", getResourceMutatingWebhookConfigName(wrc.serverIP)) @@ -935,8 +759,7 @@ func (wrc *Register) updateMutatingWebhookConfiguration(targetConfig *admregapi. } // Update the current configuration. currentConfiguration.Webhooks = newWebhooks - _, err = wrc.client.UpdateResource("", kindMutating, "", currentConfiguration, false) - if err != nil { + if _, err := wrc.kubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Update(context.TODO(), currentConfiguration, metav1.UpdateOptions{}); err != nil { return err } wrc.log.V(3).Info("successfully updated mutatingWebhookConfigurations", "name", targetConfig.Name) @@ -980,8 +803,7 @@ func (wrc *Register) updateValidatingWebhookConfiguration(targetConfig *admregap } // Update the current configuration. currentConfiguration.Webhooks = newWebhooks - _, err = wrc.client.UpdateResource("", kindValidating, "", currentConfiguration, false) - if err != nil { + if _, err := wrc.kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Update(context.TODO(), currentConfiguration, metav1.UpdateOptions{}); err != nil { return err } wrc.log.V(3).Info("successfully updated validatingWebhookConfigurations", "name", targetConfig.Name) diff --git a/pkg/webhookconfig/resource.go b/pkg/webhookconfig/resource.go index aec4975c87..fb072e91e5 100644 --- a/pkg/webhookconfig/resource.go +++ b/pkg/webhookconfig/resource.go @@ -1,13 +1,14 @@ package webhookconfig import ( + "context" "fmt" "sync" "github.com/kyverno/kyverno/pkg/config" admregapi "k8s.io/api/admissionregistration/v1" errorsapi "k8s.io/apimachinery/pkg/api/errors" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func (wrc *Register) defaultResourceWebhookRule() admregapi.Rule { @@ -27,7 +28,7 @@ func (wrc *Register) constructDefaultDebugMutatingWebhookConfig(caData []byte) * url := fmt.Sprintf("https://%s%s", wrc.serverIP, config.MutatingWebhookServicePath) logger.V(4).Info("Debug MutatingWebhookConfig registered", "url", url) webhook := &admregapi.MutatingWebhookConfiguration{ - ObjectMeta: v1.ObjectMeta{ + ObjectMeta: metav1.ObjectMeta{ Name: config.MutatingWebhookConfigurationDebugName, }, Webhooks: []admregapi.MutatingWebhook{ @@ -61,9 +62,9 @@ func (wrc *Register) constructDefaultDebugMutatingWebhookConfig(caData []byte) * func (wrc *Register) constructDefaultMutatingWebhookConfig(caData []byte) *admregapi.MutatingWebhookConfiguration { webhook := &admregapi.MutatingWebhookConfiguration{ - ObjectMeta: v1.ObjectMeta{ + ObjectMeta: metav1.ObjectMeta{ Name: config.MutatingWebhookConfigurationName, - OwnerReferences: []v1.OwnerReference{ + OwnerReferences: []metav1.OwnerReference{ wrc.constructOwner(), }, }, @@ -115,8 +116,7 @@ func (wrc *Register) removeResourceMutatingWebhookConfiguration(wg *sync.WaitGro return } - // delete webhook configuration - err := wrc.client.DeleteResource("", kindMutating, "", configName, false) + err := wrc.kubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete(context.TODO(), configName, metav1.DeleteOptions{}) if errorsapi.IsNotFound(err) { logger.V(4).Info("webhook configuration not found") return @@ -134,7 +134,7 @@ func (wrc *Register) constructDefaultDebugValidatingWebhookConfig(caData []byte) url := fmt.Sprintf("https://%s%s", wrc.serverIP, config.ValidatingWebhookServicePath) webhook := &admregapi.ValidatingWebhookConfiguration{ - ObjectMeta: v1.ObjectMeta{ + ObjectMeta: metav1.ObjectMeta{ Name: config.ValidatingWebhookConfigurationDebugName, }, Webhooks: []admregapi.ValidatingWebhook{ @@ -168,9 +168,9 @@ func (wrc *Register) constructDefaultDebugValidatingWebhookConfig(caData []byte) func (wrc *Register) constructDefaultValidatingWebhookConfig(caData []byte) *admregapi.ValidatingWebhookConfiguration { webhook := &admregapi.ValidatingWebhookConfiguration{ - ObjectMeta: v1.ObjectMeta{ + ObjectMeta: metav1.ObjectMeta{ Name: config.ValidatingWebhookConfigurationName, - OwnerReferences: []v1.OwnerReference{ + OwnerReferences: []metav1.OwnerReference{ wrc.constructOwner(), }, }, @@ -223,7 +223,7 @@ func (wrc *Register) removeResourceValidatingWebhookConfiguration(wg *sync.WaitG return } - err := wrc.client.DeleteResource("", kindValidating, "", configName, false) + err := wrc.kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Delete(context.TODO(), configName, metav1.DeleteOptions{}) if errorsapi.IsNotFound(err) { logger.V(5).Info("webhook configuration not found") return diff --git a/pkg/webhookconfig/status.go b/pkg/webhookconfig/status.go index 5744fda4ea..e2e7ff0959 100644 --- a/pkg/webhookconfig/status.go +++ b/pkg/webhookconfig/status.go @@ -13,10 +13,8 @@ import ( coordinationv1 "k8s.io/client-go/kubernetes/typed/coordination/v1" ) -var leaseName string = "kyverno" -var leaseNamespace string = config.KyvernoNamespace - const ( + leaseName string = "kyverno" annWebhookStatus string = "kyverno.io/webhookActive" annLastRequestTime string = "kyverno.io/last-request-time" ) @@ -48,7 +46,7 @@ func newStatusControl(leaseClient coordinationv1.LeaseInterface, eventGen event. } func (vc statusControl) setStatus(status string) error { - logger := vc.log.WithValues("name", leaseName, "namespace", leaseNamespace) + logger := vc.log.WithValues("name", leaseName, "namespace", config.KyvernoNamespace) var ann map[string]string var err error @@ -90,7 +88,7 @@ func (vc statusControl) setStatus(status string) error { func createStatusUpdateEvent(status string, eventGen event.Interface) { e := event.Info{} e.Kind = "Lease" - e.Namespace = leaseNamespace + e.Namespace = config.KyvernoNamespace e.Name = leaseName e.Reason = "Update" e.Message = fmt.Sprintf("admission control webhook active status changed to %s", status)