refactor: remove unstructured usage from webhookconfig (#3737)

* refactor: use typed informers and add tombstone support to webhookconfig

Signed-off-by: Charles-Edouard Brétéché <charled.breteche@gmail.com>

* refactor: remove unstructured usage from webhookconfig

Signed-off-by: Charles-Edouard Brétéché <charled.breteche@gmail.com>

cmd/kyverno/main.go

@@ -34,7 +34,6 @@ import (
 	"github.com/kyverno/kyverno/pkg/policycache"
 	"github.com/kyverno/kyverno/pkg/policyreport"
 	"github.com/kyverno/kyverno/pkg/registryclient"
-	"github.com/kyverno/kyverno/pkg/resourcecache"
 	"github.com/kyverno/kyverno/pkg/signal"
 	ktls "github.com/kyverno/kyverno/pkg/tls"
 	"github.com/kyverno/kyverno/pkg/toggle"
@@ -156,12 +155,6 @@ func main() {
 	kubeKyvernoInformer := kubeinformers.NewSharedInformerFactoryWithOptions(kubeClient, resyncPeriod, kubeinformers.WithNamespace(config.KyvernoNamespace))
 	kubedynamicInformer := client.NewDynamicSharedInformerFactory(resyncPeriod)
 
-	rCache, err := resourcecache.NewResourceCache(client, kubedynamicInformer, log.Log.WithName("resourcecache"))
-	if err != nil {
-		setupLog.Error(err, "ConfigMap lookup disabled: failed to create resource cache")
-		os.Exit(1)
-	}
-
 	// load image registry secrets
 	secrets := strings.Split(imagePullSecrets, ",")
 	if imagePullSecrets != "" && len(secrets) > 0 {
@@ -221,7 +214,6 @@ func main() {
 		pclient,
 		kubeInformer.Admissionregistration().V1().MutatingWebhookConfigurations(),
 		kubeInformer.Admissionregistration().V1().ValidatingWebhookConfigurations(),
-		rCache,
 		kubeKyvernoInformer.Apps().V1().Deployments(),
 		pInformer.Kyverno().V1().ClusterPolicies(),
 		pInformer.Kyverno().V1().Policies(),

pkg/config/config.go

@@ -47,23 +47,12 @@ const (
 	// Issue: https://github.com/kubernetes/kubernetes/pull/63972
 	// When the issue is closed, we should use TypeMeta struct instead of this constants
 
-	// DeploymentKind define the default deployment resource kind
-	DeploymentKind = "Deployment"
-
-	// DeploymentAPIVersion define the default deployment resource apiVersion
-	DeploymentAPIVersion = "apps/v1"
-
-	// NamespaceKind define the default namespace resource kind
-	NamespaceKind = "Namespace"
-
-	// NamespaceAPIVersion define the default namespace resource apiVersion
-	NamespaceAPIVersion = "v1"
-
 	// ClusterRoleAPIVersion define the default clusterrole resource apiVersion
 	ClusterRoleAPIVersion = "rbac.authorization.k8s.io/v1"
 
 	// ClusterRoleKind define the default clusterrole resource kind
 	ClusterRoleKind = "ClusterRole"
+
 	//MutatingWebhookServicePath is the path for mutation webhook
 	MutatingWebhookServicePath = "/mutate"
 

pkg/webhookconfig/certmanager.go

@@ -9,7 +9,7 @@ import (
 	"github.com/go-logr/logr"
 	"github.com/kyverno/kyverno/pkg/common"
 	"github.com/kyverno/kyverno/pkg/config"
-	ktls "github.com/kyverno/kyverno/pkg/tls"
+	"github.com/kyverno/kyverno/pkg/tls"
 	v1 "k8s.io/api/core/v1"
 	informerv1 "k8s.io/client-go/informers/core/v1"
 	"k8s.io/client-go/kubernetes"
@@ -25,17 +25,17 @@ type Interface interface {
 	InitTLSPemPair()
 
 	// GetTLSPemPair gets the existing TLSPemPair from the secret
-	GetTLSPemPair() (*ktls.PemPair, error)
+	GetTLSPemPair() (*tls.PemPair, error)
 }
 type certManager struct {
-	renewer        *ktls.CertRenewer
+	renewer        *tls.CertRenewer
 	secretInformer informerv1.SecretInformer
 	secretQueue    chan bool
 	stopCh         <-chan struct{}
 	log            logr.Logger
 }
 
-func NewCertManager(secretInformer informerv1.SecretInformer, kubeClient kubernetes.Interface, certRenewer *ktls.CertRenewer, log logr.Logger, stopCh <-chan struct{}) (Interface, error) {
+func NewCertManager(secretInformer informerv1.SecretInformer, kubeClient kubernetes.Interface, certRenewer *tls.CertRenewer, log logr.Logger, stopCh <-chan struct{}) (Interface, error) {
 	manager := &certManager{
 		renewer:        certRenewer,
 		secretInformer: secretInformer,
@@ -58,7 +58,7 @@ func (m *certManager) addSecretFunc(obj interface{}) {
 		return
 	}
 
-	val, ok := secret.GetAnnotations()[ktls.SelfSignedAnnotation]
+	val, ok := secret.GetAnnotations()[tls.SelfSignedAnnotation]
 	if !ok || val != "true" {
 		return
 	}
@@ -73,7 +73,7 @@ func (m *certManager) updateSecretFunc(oldObj interface{}, newObj interface{}) {
 		return
 	}
 
-	val, ok := new.GetAnnotations()[ktls.SelfSignedAnnotation]
+	val, ok := new.GetAnnotations()[tls.SelfSignedAnnotation]
 	if !ok || val != "true" {
 		return
 	}
@@ -94,12 +94,12 @@ func (m *certManager) InitTLSPemPair() {
 	}
 }
 
-func (m *certManager) GetTLSPemPair() (*ktls.PemPair, error) {
+func (m *certManager) GetTLSPemPair() (*tls.PemPair, error) {
-	var tls *ktls.PemPair
+	var keyPair *tls.PemPair
 	var err error
 
 	retryReadTLS := func() error {
-		tls, err = ktls.ReadTLSPair(m.renewer.ClientConfig(), m.renewer.Client())
+		keyPair, err = tls.ReadTLSPair(m.renewer.ClientConfig(), m.renewer.Client())
 		if err != nil {
 			return err
 		}
@@ -112,7 +112,7 @@ func (m *certManager) GetTLSPemPair() (*ktls.PemPair, error) {
 	f := common.RetryFunc(time.Second, time.Minute, retryReadTLS, msg, m.log.WithName("GetTLSPemPair/Retry"))
 	err = f()
 
-	return tls, err
+	return keyPair, err
 }
 
 func (m *certManager) Run(stopCh <-chan struct{}) {
@@ -127,7 +127,7 @@ func (m *certManager) Run(stopCh <-chan struct{}) {
 	})
 
 	m.log.Info("start managing certificate")
-	certsRenewalTicker := time.NewTicker(ktls.CertRenewalInterval)
+	certsRenewalTicker := time.NewTicker(tls.CertRenewalInterval)
 	defer certsRenewalTicker.Stop()
 
 	for {
@@ -137,7 +137,7 @@ func (m *certManager) Run(stopCh <-chan struct{}) {
 			if err != nil {
 				m.log.Error(err, "failed to validate cert")
 
-				if !strings.Contains(err.Error(), ktls.ErrorsNotFound) {
+				if !strings.Contains(err.Error(), tls.ErrorsNotFound) {
 					continue
 				}
 			}
@@ -157,7 +157,7 @@ func (m *certManager) Run(stopCh <-chan struct{}) {
 			if err != nil {
 				m.log.Error(err, "failed to validate cert")
 
-				if !strings.Contains(err.Error(), ktls.ErrorsNotFound) {
+				if !strings.Contains(err.Error(), tls.ErrorsNotFound) {
 					continue
 				}
 			}

pkg/webhookconfig/common.go

@@ -1,7 +1,7 @@
 package webhookconfig
 
 import (
-	"encoding/json"
+	"context"
 	"errors"
 	"io/ioutil"
 	"path/filepath"
@@ -11,11 +11,10 @@ import (
 	"github.com/kyverno/kyverno/pkg/config"
 	"github.com/kyverno/kyverno/pkg/tls"
 	admregapi "k8s.io/api/admissionregistration/v1"
-	apps "k8s.io/api/apps/v1"
+	appsv1 "k8s.io/api/apps/v1"
-	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	corev1 "k8s.io/api/rbac/v1"
-	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/rest"
-	rest "k8s.io/client-go/rest"
 )
 
 func (wrc *Register) readCaData() []byte {
@@ -60,16 +59,14 @@ func extractCA(config *rest.Config) (result []byte) {
 	return config.TLSClientConfig.CAData
 }
 
-func (wrc *Register) constructOwner() v1.OwnerReference {
+func (wrc *Register) constructOwner() metav1.OwnerReference {
 	logger := wrc.log
-
 	kubeClusterRoleName, err := wrc.GetKubePolicyClusterRoleName()
 	if err != nil {
 		logger.Error(err, "failed to get cluster role")
-		return v1.OwnerReference{}
+		return metav1.OwnerReference{}
 	}
-
+	return metav1.OwnerReference{
-	return v1.OwnerReference{
 		APIVersion: config.ClusterRoleAPIVersion,
 		Kind:       config.ClusterRoleKind,
 		Name:       kubeClusterRoleName.GetName(),
@@ -77,8 +74,13 @@ func (wrc *Register) constructOwner() v1.OwnerReference {
 	}
 }
 
-func (wrc *Register) GetKubePolicyClusterRoleName() (*unstructured.Unstructured, error) {
+func (wrc *Register) GetKubePolicyClusterRoleName() (*corev1.ClusterRole, error) {
-	clusterRoles, err := wrc.client.ListResource(config.ClusterRoleAPIVersion, config.ClusterRoleKind, "", &v1.LabelSelector{MatchLabels: map[string]string{"app.kubernetes.io/name": "kyverno"}})
+	selector := &metav1.LabelSelector{
+		MatchLabels: map[string]string{
+			"app.kubernetes.io/name": "kyverno",
+		},
+	}
+	clusterRoles, err := wrc.kubeClient.RbacV1().ClusterRoles().List(context.TODO(), metav1.ListOptions{LabelSelector: metav1.FormatLabelSelector(selector)})
 	if err != nil {
 		return nil, err
 	}
@@ -92,22 +94,12 @@ func (wrc *Register) GetKubePolicyClusterRoleName() (*unstructured.Unstructured,
 
 // GetKubePolicyDeployment gets Kyverno deployment using the resource cache
 // it does not initialize any client call
-func (wrc *Register) GetKubePolicyDeployment() (*apps.Deployment, *unstructured.Unstructured, error) {
+func (wrc *Register) GetKubePolicyDeployment() (*appsv1.Deployment, error) {
 	deploy, err := wrc.kDeplLister.Deployments(config.KyvernoNamespace).Get(config.KyvernoDeploymentName)
 	if err != nil {
-		return nil, nil, err
+		return nil, err
 	}
-	deploy.SetGroupVersionKind(schema.GroupVersionKind{Group: "", Version: "apps/v1", Kind: "Deployment"})
+	return deploy, nil
-	kubePolicyDeployment := unstructured.Unstructured{}
-	rawDepl, err := json.Marshal(deploy)
-	if err != nil {
-		return nil, nil, err
-	}
-	err = json.Unmarshal(rawDepl, &kubePolicyDeployment.Object)
-	if err != nil {
-		return deploy, nil, err
-	}
-	return deploy, &kubePolicyDeployment, nil
 }
 
 // debug mutating webhook

pkg/webhookconfig/configmanager.go

@@ -18,7 +18,6 @@ import (
 	"github.com/kyverno/kyverno/pkg/common"
 	"github.com/kyverno/kyverno/pkg/config"
 	client "github.com/kyverno/kyverno/pkg/dclient"
-	"github.com/kyverno/kyverno/pkg/resourcecache"
 	"github.com/kyverno/kyverno/pkg/utils"
 	kubeutils "github.com/kyverno/kyverno/pkg/utils/kube"
 	"github.com/pkg/errors"
@@ -58,8 +57,6 @@ type webhookConfigManager struct {
 	// npListerSynced returns true if the namespace policy store has been synced at least once
 	npListerSynced cache.InformerSynced
 
-	resCache resourcecache.ResourceCache
-
 	mutateInformer         adminformers.MutatingWebhookConfigurationInformer
 	validateInformer       adminformers.ValidatingWebhookConfigurationInformer
 	mutateLister           admlisters.MutatingWebhookConfigurationLister
@@ -95,7 +92,6 @@ func newWebhookConfigManager(
 	npInformer kyvernoinformer.PolicyInformer,
 	mwcInformer adminformers.MutatingWebhookConfigurationInformer,
 	vwcInformer adminformers.ValidatingWebhookConfigurationInformer,
-	resCache resourcecache.ResourceCache,
 	serverIP string,
 	autoUpdateWebhooks bool,
 	createDefaultWebhook chan<- string,
@@ -107,7 +103,6 @@ func newWebhookConfigManager(
 		kyvernoClient:        kyvernoClient,
 		pInformer:            pInformer,
 		npInformer:           npInformer,
-		resCache:             resCache,
 		queue:                workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "configmanager"),
 		wildcardPolicy:       0,
 		serverIP:             serverIP,

pkg/webhookconfig/monitor.go

@@ -225,7 +225,7 @@ func lastRequestTimeFromAnnotation(leaseClient coordinationv1.LeaseInterface, lo
 
 // skipWebhookCheck returns true if Kyverno is in rolling update
 func skipWebhookCheck(register *Register, logger logr.Logger) bool {
-	deploy, _, err := register.GetKubePolicyDeployment()
+	deploy, err := register.GetKubePolicyDeployment()
 	if err != nil {
 		logger.Info("unable to get Kyverno deployment", "reason", err.Error())
 		return false

pkg/webhookconfig/policy.go

@@ -9,7 +9,6 @@ import (
 )
 
 func (wrc *Register) constructPolicyValidatingWebhookConfig(caData []byte) *admregapi.ValidatingWebhookConfiguration {
-
 	return &admregapi.ValidatingWebhookConfiguration{
 		ObjectMeta: v1.ObjectMeta{
 			Name: config.PolicyValidatingWebhookConfigurationName,
@@ -40,7 +39,6 @@ func (wrc *Register) constructDebugPolicyValidatingWebhookConfig(caData []byte)
 	logger := wrc.log
 	url := fmt.Sprintf("https://%s%s", wrc.serverIP, config.PolicyValidatingWebhookServicePath)
 	logger.V(4).Info("Debug PolicyValidatingWebhookConfig is registered with url ", "url", url)
-
 	return &admregapi.ValidatingWebhookConfiguration{
 		ObjectMeta: v1.ObjectMeta{
 			Name: config.PolicyValidatingWebhookConfigurationDebugName,
@@ -95,7 +93,6 @@ func (wrc *Register) constructDebugPolicyMutatingWebhookConfig(caData []byte) *a
 	logger := wrc.log
 	url := fmt.Sprintf("https://%s%s", wrc.serverIP, config.PolicyMutatingWebhookServicePath)
 	logger.V(4).Info("Debug PolicyMutatingWebhookConfig is registered with url ", "url", url)
-
 	return &admregapi.MutatingWebhookConfiguration{
 		ObjectMeta: v1.ObjectMeta{
 			Name: config.PolicyMutatingWebhookConfigurationDebugName,

pkg/webhookconfig/registration.go

@@ -14,7 +14,6 @@ import (
 	kyvernoinformer "github.com/kyverno/kyverno/pkg/client/informers/externalversions/kyverno/v1"
 	"github.com/kyverno/kyverno/pkg/config"
 	client "github.com/kyverno/kyverno/pkg/dclient"
-	"github.com/kyverno/kyverno/pkg/resourcecache"
 	"github.com/kyverno/kyverno/pkg/tls"
 	"github.com/kyverno/kyverno/pkg/utils"
 	"github.com/pkg/errors"
@@ -22,9 +21,6 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	errorsapi "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
-	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/apimachinery/pkg/runtime/schema"
 	adminformers "k8s.io/client-go/informers/admissionregistration/v1"
 	informers "k8s.io/client-go/informers/apps/v1"
 	"k8s.io/client-go/kubernetes"
@@ -46,16 +42,19 @@ const (
 // 4. Resource Mutation
 // 5. Webhook Status Mutation
 type Register struct {
-	client       *client.Client
+	// clients
 	kubeClient   kubernetes.Interface
 	clientConfig *rest.Config
-	resCache     resourcecache.ResourceCache
 
-	mwcLister admlisters.MutatingWebhookConfigurationLister
+	// listers
-	vwcLister admlisters.ValidatingWebhookConfigurationLister
+	mwcLister   admlisters.MutatingWebhookConfigurationLister
+	vwcLister   admlisters.ValidatingWebhookConfigurationLister
+	kDeplLister listers.DeploymentLister
 
-	mwcListerSynced func() bool
+	// sync
-	vwcListerSynced func() bool
+	mwcListerSynced   cache.InformerSynced
+	vwcListerSynced   cache.InformerSynced
+	kDeplListerSynced cache.InformerSynced
 
 	serverIP           string // when running outside a cluster
 	timeoutSeconds     int32
@@ -67,9 +66,6 @@ type Register struct {
 	UpdateWebhookChan    chan bool
 	createDefaultWebhook chan string
 
-	kDeplLister       listers.DeploymentLister
-	kDeplListerSynced func() bool
-
 	// manage implements methods to manage webhook configurations
 	manage
 }
@@ -82,7 +78,6 @@ func NewRegister(
 	kyvernoClient *kyvernoclient.Clientset,
 	mwcInformer adminformers.MutatingWebhookConfigurationInformer,
 	vwcInformer adminformers.ValidatingWebhookConfigurationInformer,
-	resCache resourcecache.ResourceCache,
 	kDeplInformer informers.DeploymentInformer,
 	pInformer kyvernoinformer.ClusterPolicyInformer,
 	npInformer kyvernoinformer.PolicyInformer,
@@ -94,9 +89,13 @@ func NewRegister(
 	log logr.Logger) *Register {
 	register := &Register{
 		clientConfig:         clientConfig,
-		client:               client,
 		kubeClient:           kubeClient,
-		resCache:             resCache,
+		mwcLister:            mwcInformer.Lister(),
+		vwcLister:            vwcInformer.Lister(),
+		kDeplLister:          kDeplInformer.Lister(),
+		mwcListerSynced:      mwcInformer.Informer().HasSynced,
+		vwcListerSynced:      vwcInformer.Informer().HasSynced,
+		kDeplListerSynced:    kDeplInformer.Informer().HasSynced,
 		serverIP:             serverIP,
 		timeoutSeconds:       webhookTimeout,
 		log:                  log.WithName("Register"),
@@ -104,16 +103,10 @@ func NewRegister(
 		autoUpdateWebhooks:   autoUpdateWebhooks,
 		UpdateWebhookChan:    make(chan bool),
 		createDefaultWebhook: make(chan string),
-		kDeplLister:          kDeplInformer.Lister(),
-		kDeplListerSynced:    kDeplInformer.Informer().HasSynced,
-		mwcLister:            mwcInformer.Lister(),
-		vwcLister:            vwcInformer.Lister(),
-		mwcListerSynced:      mwcInformer.Informer().HasSynced,
-		vwcListerSynced:      vwcInformer.Informer().HasSynced,
 		stopCh:               stopCh,
 	}
 
-	register.manage = newWebhookConfigManager(client, kyvernoClient, pInformer, npInformer, mwcInformer, vwcInformer, resCache, serverIP, register.autoUpdateWebhooks, register.createDefaultWebhook, stopCh, log.WithName("WebhookConfigManager"))
+	register.manage = newWebhookConfigManager(client, kyvernoClient, pInformer, npInformer, mwcInformer, vwcInformer, serverIP, register.autoUpdateWebhooks, register.createDefaultWebhook, stopCh, log.WithName("WebhookConfigManager"))
 
 	return register
 }
@@ -252,24 +245,16 @@ func (wrc *Register) UpdateWebhookConfigurations(configHandler config.Interface)
 
 func (wrc *Register) ValidateWebhookConfigurations(namespace, name string) error {
 	logger := wrc.log.WithName("ValidateWebhookConfigurations")
-
+	cm, err := wrc.kubeClient.CoreV1().ConfigMaps(namespace).Get(context.TODO(), name, metav1.GetOptions{})
-	cm, err := wrc.client.GetResource("", "ConfigMap", namespace, name)
 	if err != nil {
 		logger.Error(err, "unable to fetch ConfigMap", "namespace", namespace, "name", name)
 		return nil
 	}
-
+	webhooks, ok := cm.Data["webhooks"]
-	webhooks, ok, err := unstructured.NestedString(cm.UnstructuredContent(), "data", "webhooks")
-	if err != nil {
-		logger.Error(err, "failed to fetch tag 'webhooks' from the ConfigMap")
-		return nil
-	}
-
 	if !ok {
 		logger.V(4).Info("webhook configurations not defined")
 		return nil
 	}
-
 	webhookCfgs := make([]config.WebhookConfig, 0, 10)
 	return json.Unmarshal([]byte(webhooks), &webhookCfgs)
 }
@@ -277,13 +262,12 @@ func (wrc *Register) ValidateWebhookConfigurations(namespace, name string) error
 // cleanupKyvernoResource returns true if Kyverno is terminating
 func (wrc *Register) cleanupKyvernoResource() bool {
 	logger := wrc.log.WithName("cleanupKyvernoResource")
-	deploy, err := wrc.client.GetResource("", "Deployment", config.KyvernoNamespace, config.KyvernoDeploymentName)
+	deploy, err := wrc.kubeClient.AppsV1().Deployments(config.KyvernoNamespace).Get(context.TODO(), config.KyvernoDeploymentName, metav1.GetOptions{})
 	if err != nil {
 		if errorsapi.IsNotFound(err) {
 			logger.Info("Kyverno deployment not found, cleanup Kyverno resources")
 			return true
 		}
-
 		logger.Error(err, "failed to get deployment, not cleaning up kyverno resources")
 		return false
 	}
@@ -291,17 +275,10 @@ func (wrc *Register) cleanupKyvernoResource() bool {
 		logger.Info("Kyverno is terminating, cleanup Kyverno resources")
 		return true
 	}
-
+	if deploy.Spec.Replicas == nil && *deploy.Spec.Replicas == 0 {
-	replicas, _, err := unstructured.NestedInt64(deploy.UnstructuredContent(), "spec", "replicas")
-	if err != nil {
-		logger.Error(err, "unable to fetch spec.replicas of Kyverno deployment")
-	}
-
-	if replicas == 0 {
 		logger.Info("Kyverno is scaled to zero, cleanup Kyverno resources")
 		return true
 	}
-
 	logger.Info("updating Kyverno Pod, won't clean up Kyverno resources")
 	return false
 }
@@ -317,7 +294,7 @@ func (wrc *Register) createResourceMutatingWebhookConfiguration(caData []byte) e
 
 	logger := wrc.log.WithValues("kind", kindMutating, "name", config.Name)
 
-	_, err := wrc.client.CreateResource("", kindMutating, "", *config, false)
+	_, err := wrc.kubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Create(context.TODO(), config, metav1.CreateOptions{})
 	if errorsapi.IsAlreadyExists(err) {
 		logger.V(6).Info("resource mutating webhook configuration already exists", "name", config.Name)
 		err = wrc.updateMutatingWebhookConfiguration(config)
@@ -346,7 +323,7 @@ func (wrc *Register) createResourceValidatingWebhookConfiguration(caData []byte)
 
 	logger := wrc.log.WithValues("kind", kindValidating, "name", config.Name)
 
-	_, err := wrc.client.CreateResource("", kindValidating, "", *config, false)
+	_, err := wrc.kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Create(context.TODO(), config, metav1.CreateOptions{})
 	if errorsapi.IsAlreadyExists(err) {
 		logger.V(6).Info("resource validating webhook configuration already exists", "name", config.Name)
 		err = wrc.updateValidatingWebhookConfiguration(config)
@@ -375,7 +352,7 @@ func (wrc *Register) createPolicyValidatingWebhookConfiguration(caData []byte) e
 		config = wrc.constructPolicyValidatingWebhookConfig(caData)
 	}
 
-	if _, err := wrc.client.CreateResource("", kindValidating, "", *config, false); err != nil {
+	if _, err := wrc.kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Create(context.TODO(), config, metav1.CreateOptions{}); err != nil {
 		if errorsapi.IsAlreadyExists(err) {
 			wrc.log.V(6).Info("webhook already exists", "kind", kindValidating, "name", config.Name)
 			err = wrc.updateValidatingWebhookConfiguration(config)
@@ -401,8 +378,7 @@ func (wrc *Register) createPolicyMutatingWebhookConfiguration(caData []byte) err
 		config = wrc.constructPolicyMutatingWebhookConfig(caData)
 	}
 
-	// create mutating webhook configuration resource
+	if _, err := wrc.kubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Create(context.TODO(), config, metav1.CreateOptions{}); err != nil {
-	if _, err := wrc.client.CreateResource("", kindMutating, "", *config, false); err != nil {
 		if errorsapi.IsAlreadyExists(err) {
 			wrc.log.V(6).Info("webhook already exists", "kind", kindMutating, "name", config.Name)
 			err = wrc.updateMutatingWebhookConfiguration(config)
@@ -428,7 +404,7 @@ func (wrc *Register) createVerifyMutatingWebhookConfiguration(caData []byte) err
 		config = wrc.constructVerifyMutatingWebhookConfig(caData)
 	}
 
-	if _, err := wrc.client.CreateResource("", kindMutating, "", *config, false); err != nil {
+	if _, err := wrc.kubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Create(context.TODO(), config, metav1.CreateOptions{}); err != nil {
 		if errorsapi.IsAlreadyExists(err) {
 			wrc.log.V(6).Info("webhook already exists", "kind", kindMutating, "name", config.Name)
 			err = wrc.updateMutatingWebhookConfiguration(config)
@@ -476,7 +452,7 @@ func (wrc *Register) removePolicyMutatingWebhookConfiguration(wg *sync.WaitGroup
 		return
 	}
 
-	err := wrc.client.DeleteResource("", kindMutating, "", mutatingConfig, false)
+	err := wrc.kubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete(context.TODO(), mutatingConfig, metav1.DeleteOptions{})
 	if errorsapi.IsNotFound(err) {
 		logger.V(5).Info("policy mutating webhook configuration not found")
 		return
@@ -512,7 +488,7 @@ func (wrc *Register) removePolicyValidatingWebhookConfiguration(wg *sync.WaitGro
 	}
 
 	logger.V(4).Info("removing validating webhook configuration")
-	err := wrc.client.DeleteResource("", kindValidating, "", validatingConfig, false)
+	err := wrc.kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Delete(context.TODO(), validatingConfig, metav1.DeleteOptions{})
 	if errorsapi.IsNotFound(err) {
 		logger.V(5).Info("policy validating webhook configuration not found")
 		return
@@ -615,7 +591,7 @@ func (wrc *Register) removeVerifyWebhookMutatingWebhookConfig(wg *sync.WaitGroup
 		return
 	}
 
-	err = wrc.client.DeleteResource("", kindMutating, "", mutatingConfig, false)
+	err = wrc.kubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete(context.TODO(), mutatingConfig, metav1.DeleteOptions{})
 	if errorsapi.IsNotFound(err) {
 		logger.V(5).Info("verify webhook configuration not found")
 		return
@@ -651,52 +627,37 @@ func (wrc *Register) removeSecrets() {
 			tls.ManagedByLabel: "kyverno",
 		},
 	}
-
+	if err := wrc.kubeClient.CoreV1().Secrets(config.KyvernoNamespace).DeleteCollection(context.TODO(), metav1.DeleteOptions{}, metav1.ListOptions{LabelSelector: metav1.FormatLabelSelector(selector)}); err != nil {
-	secretList, err := wrc.client.ListResource("", "Secret", config.KyvernoNamespace, selector)
-	if err != nil {
 		wrc.log.Error(err, "failed to clean up Kyverno managed secrets")
 		return
 	}
-
-	for _, secret := range secretList.Items {
-		if err := wrc.client.DeleteResource("", "Secret", secret.GetNamespace(), secret.GetName(), false); err != nil {
-			if !errorsapi.IsNotFound(err) {
-				wrc.log.Error(err, "failed to delete secret", "ns", secret.GetNamespace(), "name", secret.GetName())
-			}
-		}
-	}
 }
 
 func (wrc *Register) checkEndpoint() error {
-	obj, err := wrc.client.GetResource("", "Endpoints", config.KyvernoNamespace, config.KyvernoServiceName)
+	endpoint, err := wrc.kubeClient.CoreV1().Endpoints(config.KyvernoNamespace).Get(context.TODO(), config.KyvernoServiceName, metav1.GetOptions{})
 	if err != nil {
 		return fmt.Errorf("failed to get endpoint %s/%s: %v", config.KyvernoNamespace, config.KyvernoServiceName, err)
 	}
-	var endpoint corev1.Endpoints
+	selector := &metav1.LabelSelector{
-	err = runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), &endpoint)
+		MatchLabels: map[string]string{
-	if err != nil {
+			"app.kubernetes.io/name": "kyverno",
-		return fmt.Errorf("failed to convert endpoint %s/%s from unstructured: %v", config.KyvernoNamespace, config.KyvernoServiceName, err)
+		},
 	}
-
+	pods, err := wrc.kubeClient.CoreV1().Pods(config.KyvernoNamespace).List(context.TODO(), metav1.ListOptions{LabelSelector: metav1.FormatLabelSelector(selector)})
-	pods, err := wrc.client.ListResource("", "Pod", config.KyvernoNamespace, &metav1.LabelSelector{MatchLabels: map[string]string{"app.kubernetes.io/name": "kyverno"}})
 	if err != nil {
 		return fmt.Errorf("failed to list Kyverno Pod: %v", err)
 	}
-
 	ips, errs := getHealthyPodsIP(pods.Items)
 	if len(errs) != 0 {
 		return fmt.Errorf("error getting pod's IP: %v", errs)
 	}
-
 	if len(ips) == 0 {
 		return fmt.Errorf("pod is not assigned to any node yet")
 	}
-
 	for _, subset := range endpoint.Subsets {
 		if len(subset.Addresses) == 0 {
 			continue
 		}
-
 		for _, addr := range subset.Addresses {
 			if utils.ContainsString(ips, addr.IP) {
 				wrc.log.Info("Endpoint ready", "ns", config.KyvernoNamespace, "name", config.KyvernoServiceName)
@@ -704,130 +665,36 @@ func (wrc *Register) checkEndpoint() error {
 			}
 		}
 	}
-
 	err = fmt.Errorf("endpoint not ready")
 	wrc.log.V(3).Info(err.Error(), "ns", config.KyvernoNamespace, "name", config.KyvernoServiceName)
 	return err
 }
 
-func getHealthyPodsIP(pods []unstructured.Unstructured) (ips []string, errs []error) {
+func getHealthyPodsIP(pods []corev1.Pod) (ips []string, errs []error) {
 	for _, pod := range pods {
-		phase, _, err := unstructured.NestedString(pod.UnstructuredContent(), "status", "phase")
+		if pod.Status.Phase != "Running" {
-		if err != nil {
-			errs = append(errs, fmt.Errorf("failed to get pod %s status: %v", pod.GetName(), err))
 			continue
 		}
-
+		ips = append(ips, pod.Status.PodIP)
-		if phase != "Running" {
-			continue
-		}
-
-		ip, _, err := unstructured.NestedString(pod.UnstructuredContent(), "status", "podIP")
-		if err != nil {
-			errs = append(errs, fmt.Errorf("failed to extract pod %s IP: %v", pod.GetName(), err))
-			continue
-		}
-
-		ips = append(ips, ip)
 	}
-
 	return
 }
 
-func convertLabelSelector(selector *metav1.LabelSelector, logger logr.Logger) (map[string]interface{}, error) {
-	if selector == nil {
-		return nil, nil
-	}
-	if selectorBytes, err := json.Marshal(*selector); err != nil {
-		logger.Error(err, "failed to serialize selector")
-		return nil, err
-	} else {
-		var nsSelector map[string]interface{}
-		if err := json.Unmarshal(selectorBytes, &nsSelector); err != nil {
-			logger.Error(err, "failed to convert namespaceSelector to the map")
-			return nil, err
-		}
-		return nsSelector, nil
-	}
-}
-
-func configureSelector(webhook map[string]interface{}, selector *metav1.LabelSelector, key string, logger logr.Logger) (bool, error) {
-	currentSelector, _, err := unstructured.NestedMap(webhook, key)
-	if err != nil {
-		return false, err
-	}
-	if expectedSelector, err := convertLabelSelector(selector, logger); err != nil {
-		return false, err
-	} else {
-		if reflect.DeepEqual(expectedSelector, currentSelector) {
-			return false, nil
-		} else {
-			if err = unstructured.SetNestedMap(webhook, expectedSelector, key); err != nil {
-				return false, err
-			}
-			return true, nil
-		}
-	}
-}
-
 func (wrc *Register) updateResourceValidatingWebhookConfiguration(webhookCfg config.WebhookConfig) error {
-	resourceValidatingTyped, err := wrc.vwcLister.Get(getResourceValidatingWebhookConfigName(wrc.serverIP))
+	resource, err := wrc.vwcLister.Get(getResourceValidatingWebhookConfigName(wrc.serverIP))
 	if err != nil {
 		return errors.Wrapf(err, "unable to get validatingWebhookConfigurations")
 	}
-	resourceValidatingTyped.SetGroupVersionKind(schema.GroupVersionKind{Group: "", Version: "admissionregistration.k8s.io/v1", Kind: kindValidating})
+	copy := resource.DeepCopy()
-
+	for i := range copy.Webhooks {
-	resourceValidating := &unstructured.Unstructured{}
+		copy.Webhooks[i].ObjectSelector = webhookCfg.ObjectSelector
-	rawResc, err := json.Marshal(resourceValidatingTyped)
+		copy.Webhooks[i].NamespaceSelector = webhookCfg.NamespaceSelector
-	if err != nil {
-		return err
 	}
-	err = json.Unmarshal(rawResc, &resourceValidating.Object)
+	if reflect.DeepEqual(resource.Webhooks, copy.Webhooks) {
-	if err != nil {
-		return err
-	}
-
-	webhooksUntyped, _, err := unstructured.NestedSlice(resourceValidating.UnstructuredContent(), "webhooks")
-	if err != nil {
-		return errors.Wrapf(err, "unable to load validatingWebhookConfigurations.webhooks")
-	}
-
-	var (
-		webhook  map[string]interface{}
-		webhooks []interface{}
-		ok       bool
-	)
-
-	webhookChanged := false
-	for i, whu := range webhooksUntyped {
-		webhook, ok = whu.(map[string]interface{})
-		if !ok {
-			return errors.Wrapf(err, "type mismatched, expected map[string]interface{}, got %T", webhooksUntyped[i])
-		}
-		if changed, err := configureSelector(webhook, webhookCfg.ObjectSelector, "objectSelector", wrc.log); err != nil {
-			return errors.Wrapf(err, "unable to get validatingWebhookConfigurations.webhooks["+fmt.Sprint(i)+"].objectSelector")
-		} else {
-			if changed {
-				webhookChanged = true
-			}
-		}
-		if changed, err := configureSelector(webhook, webhookCfg.NamespaceSelector, "namespaceSelector", wrc.log); err != nil {
-			return errors.Wrapf(err, "unable to get validatingWebhookConfigurations.webhooks["+fmt.Sprint(i)+"].namespaceSelector")
-		} else {
-			if changed {
-				webhookChanged = true
-			}
-		}
-		webhooks = append(webhooks, webhook)
-	}
-	if !webhookChanged {
 		wrc.log.V(4).Info("namespaceSelector unchanged, skip updating validatingWebhookConfigurations")
 		return nil
 	}
-	if err = unstructured.SetNestedSlice(resourceValidating.UnstructuredContent(), webhooks, "webhooks"); err != nil {
+	if _, err := wrc.kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Update(context.TODO(), copy, metav1.UpdateOptions{}); err != nil {
-		return errors.Wrapf(err, "unable to set validatingWebhookConfigurations.webhooks")
-	}
-	if _, err := wrc.client.UpdateResource(resourceValidating.GetAPIVersion(), resourceValidating.GetKind(), "", resourceValidating, false); err != nil {
 		return err
 	}
 	wrc.log.V(3).Info("successfully updated validatingWebhookConfigurations", "name", getResourceMutatingWebhookConfigName(wrc.serverIP))
@@ -835,63 +702,20 @@ func (wrc *Register) updateResourceValidatingWebhookConfiguration(webhookCfg con
 }
 
 func (wrc *Register) updateResourceMutatingWebhookConfiguration(webhookCfg config.WebhookConfig) error {
-	resourceMutatingTyped, err := wrc.mwcLister.Get(getResourceMutatingWebhookConfigName(wrc.serverIP))
+	resource, err := wrc.mwcLister.Get(getResourceMutatingWebhookConfigName(wrc.serverIP))
 	if err != nil {
 		return errors.Wrapf(err, "unable to get mutatingWebhookConfigurations")
 	}
-	resourceMutatingTyped.SetGroupVersionKind(schema.GroupVersionKind{Group: "", Version: "admissionregistration.k8s.io/v1", Kind: kindMutating})
+	copy := resource.DeepCopy()
-
+	for i := range copy.Webhooks {
-	resourceMutating := &unstructured.Unstructured{}
+		copy.Webhooks[i].ObjectSelector = webhookCfg.ObjectSelector
-	rawResc, err := json.Marshal(resourceMutatingTyped)
+		copy.Webhooks[i].NamespaceSelector = webhookCfg.NamespaceSelector
-	if err != nil {
-		return err
 	}
-	err = json.Unmarshal(rawResc, &resourceMutating.Object)
+	if reflect.DeepEqual(resource.Webhooks, copy.Webhooks) {
-	if err != nil {
-		return err
-	}
-
-	webhooksUntyped, _, err := unstructured.NestedSlice(resourceMutating.UnstructuredContent(), "webhooks")
-	if err != nil {
-		return errors.Wrapf(err, "unable to load mutatingWebhookConfigurations.webhooks")
-	}
-
-	var (
-		webhook  map[string]interface{}
-		webhooks []interface{}
-		ok       bool
-	)
-
-	webhookChanged := false
-	for i, whu := range webhooksUntyped {
-		webhook, ok = whu.(map[string]interface{})
-		if !ok {
-			return errors.Wrapf(err, "type mismatched, expected map[string]interface{}, got %T", webhooksUntyped[i])
-		}
-		if changed, err := configureSelector(webhook, webhookCfg.ObjectSelector, "objectSelector", wrc.log); err != nil {
-			return errors.Wrapf(err, "unable to get mutatingWebhookConfigurations.webhooks["+fmt.Sprint(i)+"].objectSelector")
-		} else {
-			if changed {
-				webhookChanged = true
-			}
-		}
-		if changed, err := configureSelector(webhook, webhookCfg.NamespaceSelector, "namespaceSelector", wrc.log); err != nil {
-			return errors.Wrapf(err, "unable to get mutatingWebhookConfigurations.webhooks["+fmt.Sprint(i)+"].namespaceSelector")
-		} else {
-			if changed {
-				webhookChanged = true
-			}
-		}
-		webhooks = append(webhooks, webhook)
-	}
-	if !webhookChanged {
 		wrc.log.V(4).Info("namespaceSelector unchanged, skip updating mutatingWebhookConfigurations")
 		return nil
 	}
-	if err = unstructured.SetNestedSlice(resourceMutating.UnstructuredContent(), webhooks, "webhooks"); err != nil {
+	if _, err := wrc.kubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Update(context.TODO(), copy, metav1.UpdateOptions{}); err != nil {
-		return errors.Wrapf(err, "unable to set mutatingWebhookConfigurations.webhooks")
-	}
-	if _, err := wrc.client.UpdateResource(resourceMutating.GetAPIVersion(), resourceMutating.GetKind(), "", resourceMutating, false); err != nil {
 		return err
 	}
 	wrc.log.V(3).Info("successfully updated mutatingWebhookConfigurations", "name", getResourceMutatingWebhookConfigName(wrc.serverIP))
@@ -935,8 +759,7 @@ func (wrc *Register) updateMutatingWebhookConfiguration(targetConfig *admregapi.
 	}
 	// Update the current configuration.
 	currentConfiguration.Webhooks = newWebhooks
-	_, err = wrc.client.UpdateResource("", kindMutating, "", currentConfiguration, false)
+	if _, err := wrc.kubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Update(context.TODO(), currentConfiguration, metav1.UpdateOptions{}); err != nil {
-	if err != nil {
 		return err
 	}
 	wrc.log.V(3).Info("successfully updated mutatingWebhookConfigurations", "name", targetConfig.Name)
@@ -980,8 +803,7 @@ func (wrc *Register) updateValidatingWebhookConfiguration(targetConfig *admregap
 	}
 	// Update the current configuration.
 	currentConfiguration.Webhooks = newWebhooks
-	_, err = wrc.client.UpdateResource("", kindValidating, "", currentConfiguration, false)
+	if _, err := wrc.kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Update(context.TODO(), currentConfiguration, metav1.UpdateOptions{}); err != nil {
-	if err != nil {
 		return err
 	}
 	wrc.log.V(3).Info("successfully updated validatingWebhookConfigurations", "name", targetConfig.Name)

pkg/webhookconfig/resource.go

@@ -1,13 +1,14 @@
 package webhookconfig
 
 import (
+	"context"
 	"fmt"
 	"sync"
 
 	"github.com/kyverno/kyverno/pkg/config"
 	admregapi "k8s.io/api/admissionregistration/v1"
 	errorsapi "k8s.io/apimachinery/pkg/api/errors"
-	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 func (wrc *Register) defaultResourceWebhookRule() admregapi.Rule {
@@ -27,7 +28,7 @@ func (wrc *Register) constructDefaultDebugMutatingWebhookConfig(caData []byte) *
 	url := fmt.Sprintf("https://%s%s", wrc.serverIP, config.MutatingWebhookServicePath)
 	logger.V(4).Info("Debug MutatingWebhookConfig registered", "url", url)
 	webhook := &admregapi.MutatingWebhookConfiguration{
-		ObjectMeta: v1.ObjectMeta{
+		ObjectMeta: metav1.ObjectMeta{
 			Name: config.MutatingWebhookConfigurationDebugName,
 		},
 		Webhooks: []admregapi.MutatingWebhook{
@@ -61,9 +62,9 @@ func (wrc *Register) constructDefaultDebugMutatingWebhookConfig(caData []byte) *
 
 func (wrc *Register) constructDefaultMutatingWebhookConfig(caData []byte) *admregapi.MutatingWebhookConfiguration {
 	webhook := &admregapi.MutatingWebhookConfiguration{
-		ObjectMeta: v1.ObjectMeta{
+		ObjectMeta: metav1.ObjectMeta{
 			Name: config.MutatingWebhookConfigurationName,
-			OwnerReferences: []v1.OwnerReference{
+			OwnerReferences: []metav1.OwnerReference{
 				wrc.constructOwner(),
 			},
 		},
@@ -115,8 +116,7 @@ func (wrc *Register) removeResourceMutatingWebhookConfiguration(wg *sync.WaitGro
 		return
 	}
 
-	// delete webhook configuration
+	err := wrc.kubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete(context.TODO(), configName, metav1.DeleteOptions{})
-	err := wrc.client.DeleteResource("", kindMutating, "", configName, false)
 	if errorsapi.IsNotFound(err) {
 		logger.V(4).Info("webhook configuration not found")
 		return
@@ -134,7 +134,7 @@ func (wrc *Register) constructDefaultDebugValidatingWebhookConfig(caData []byte)
 	url := fmt.Sprintf("https://%s%s", wrc.serverIP, config.ValidatingWebhookServicePath)
 
 	webhook := &admregapi.ValidatingWebhookConfiguration{
-		ObjectMeta: v1.ObjectMeta{
+		ObjectMeta: metav1.ObjectMeta{
 			Name: config.ValidatingWebhookConfigurationDebugName,
 		},
 		Webhooks: []admregapi.ValidatingWebhook{
@@ -168,9 +168,9 @@ func (wrc *Register) constructDefaultDebugValidatingWebhookConfig(caData []byte)
 
 func (wrc *Register) constructDefaultValidatingWebhookConfig(caData []byte) *admregapi.ValidatingWebhookConfiguration {
 	webhook := &admregapi.ValidatingWebhookConfiguration{
-		ObjectMeta: v1.ObjectMeta{
+		ObjectMeta: metav1.ObjectMeta{
 			Name: config.ValidatingWebhookConfigurationName,
-			OwnerReferences: []v1.OwnerReference{
+			OwnerReferences: []metav1.OwnerReference{
 				wrc.constructOwner(),
 			},
 		},
@@ -223,7 +223,7 @@ func (wrc *Register) removeResourceValidatingWebhookConfiguration(wg *sync.WaitG
 		return
 	}
 
-	err := wrc.client.DeleteResource("", kindValidating, "", configName, false)
+	err := wrc.kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Delete(context.TODO(), configName, metav1.DeleteOptions{})
 	if errorsapi.IsNotFound(err) {
 		logger.V(5).Info("webhook configuration not found")
 		return

pkg/webhookconfig/status.go

@@ -13,10 +13,8 @@ import (
 	coordinationv1 "k8s.io/client-go/kubernetes/typed/coordination/v1"
 )
 
-var leaseName string = "kyverno"
-var leaseNamespace string = config.KyvernoNamespace
-
 const (
+	leaseName          string = "kyverno"
 	annWebhookStatus   string = "kyverno.io/webhookActive"
 	annLastRequestTime string = "kyverno.io/last-request-time"
 )
@@ -48,7 +46,7 @@ func newStatusControl(leaseClient coordinationv1.LeaseInterface, eventGen event.
 }
 
 func (vc statusControl) setStatus(status string) error {
-	logger := vc.log.WithValues("name", leaseName, "namespace", leaseNamespace)
+	logger := vc.log.WithValues("name", leaseName, "namespace", config.KyvernoNamespace)
 	var ann map[string]string
 	var err error
 
@@ -90,7 +88,7 @@ func (vc statusControl) setStatus(status string) error {
 func createStatusUpdateEvent(status string, eventGen event.Interface) {
 	e := event.Info{}
 	e.Kind = "Lease"
-	e.Namespace = leaseNamespace
+	e.Namespace = config.KyvernoNamespace
 	e.Name = leaseName
 	e.Reason = "Update"
 	e.Message = fmt.Sprintf("admission control webhook active status changed to %s", status)