1
0
Fork 0
mirror of https://github.com/kyverno/kyverno.git synced 2025-03-29 10:55:05 +00:00

refactor: remove unstructured usage from webhookconfig (#3737)

* refactor: use typed informers and add tombstone support to webhookconfig

Signed-off-by: Charles-Edouard Brétéché <charled.breteche@gmail.com>

* refactor: remove unstructured usage from webhookconfig

Signed-off-by: Charles-Edouard Brétéché <charled.breteche@gmail.com>
This commit is contained in:
Charles-Edouard Brétéché 2022-05-02 12:58:04 +02:00 committed by GitHub
parent 87880ad6f1
commit 972be16ad3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 100 additions and 315 deletions

View file

@ -34,7 +34,6 @@ import (
"github.com/kyverno/kyverno/pkg/policycache"
"github.com/kyverno/kyverno/pkg/policyreport"
"github.com/kyverno/kyverno/pkg/registryclient"
"github.com/kyverno/kyverno/pkg/resourcecache"
"github.com/kyverno/kyverno/pkg/signal"
ktls "github.com/kyverno/kyverno/pkg/tls"
"github.com/kyverno/kyverno/pkg/toggle"
@ -156,12 +155,6 @@ func main() {
kubeKyvernoInformer := kubeinformers.NewSharedInformerFactoryWithOptions(kubeClient, resyncPeriod, kubeinformers.WithNamespace(config.KyvernoNamespace))
kubedynamicInformer := client.NewDynamicSharedInformerFactory(resyncPeriod)
rCache, err := resourcecache.NewResourceCache(client, kubedynamicInformer, log.Log.WithName("resourcecache"))
if err != nil {
setupLog.Error(err, "ConfigMap lookup disabled: failed to create resource cache")
os.Exit(1)
}
// load image registry secrets
secrets := strings.Split(imagePullSecrets, ",")
if imagePullSecrets != "" && len(secrets) > 0 {
@ -221,7 +214,6 @@ func main() {
pclient,
kubeInformer.Admissionregistration().V1().MutatingWebhookConfigurations(),
kubeInformer.Admissionregistration().V1().ValidatingWebhookConfigurations(),
rCache,
kubeKyvernoInformer.Apps().V1().Deployments(),
pInformer.Kyverno().V1().ClusterPolicies(),
pInformer.Kyverno().V1().Policies(),

View file

@ -47,23 +47,12 @@ const (
// Issue: https://github.com/kubernetes/kubernetes/pull/63972
// When the issue is closed, we should use TypeMeta struct instead of this constants
// DeploymentKind define the default deployment resource kind
DeploymentKind = "Deployment"
// DeploymentAPIVersion define the default deployment resource apiVersion
DeploymentAPIVersion = "apps/v1"
// NamespaceKind define the default namespace resource kind
NamespaceKind = "Namespace"
// NamespaceAPIVersion define the default namespace resource apiVersion
NamespaceAPIVersion = "v1"
// ClusterRoleAPIVersion define the default clusterrole resource apiVersion
ClusterRoleAPIVersion = "rbac.authorization.k8s.io/v1"
// ClusterRoleKind define the default clusterrole resource kind
ClusterRoleKind = "ClusterRole"
//MutatingWebhookServicePath is the path for mutation webhook
MutatingWebhookServicePath = "/mutate"

View file

@ -9,7 +9,7 @@ import (
"github.com/go-logr/logr"
"github.com/kyverno/kyverno/pkg/common"
"github.com/kyverno/kyverno/pkg/config"
ktls "github.com/kyverno/kyverno/pkg/tls"
"github.com/kyverno/kyverno/pkg/tls"
v1 "k8s.io/api/core/v1"
informerv1 "k8s.io/client-go/informers/core/v1"
"k8s.io/client-go/kubernetes"
@ -25,17 +25,17 @@ type Interface interface {
InitTLSPemPair()
// GetTLSPemPair gets the existing TLSPemPair from the secret
GetTLSPemPair() (*ktls.PemPair, error)
GetTLSPemPair() (*tls.PemPair, error)
}
type certManager struct {
renewer *ktls.CertRenewer
renewer *tls.CertRenewer
secretInformer informerv1.SecretInformer
secretQueue chan bool
stopCh <-chan struct{}
log logr.Logger
}
func NewCertManager(secretInformer informerv1.SecretInformer, kubeClient kubernetes.Interface, certRenewer *ktls.CertRenewer, log logr.Logger, stopCh <-chan struct{}) (Interface, error) {
func NewCertManager(secretInformer informerv1.SecretInformer, kubeClient kubernetes.Interface, certRenewer *tls.CertRenewer, log logr.Logger, stopCh <-chan struct{}) (Interface, error) {
manager := &certManager{
renewer: certRenewer,
secretInformer: secretInformer,
@ -58,7 +58,7 @@ func (m *certManager) addSecretFunc(obj interface{}) {
return
}
val, ok := secret.GetAnnotations()[ktls.SelfSignedAnnotation]
val, ok := secret.GetAnnotations()[tls.SelfSignedAnnotation]
if !ok || val != "true" {
return
}
@ -73,7 +73,7 @@ func (m *certManager) updateSecretFunc(oldObj interface{}, newObj interface{}) {
return
}
val, ok := new.GetAnnotations()[ktls.SelfSignedAnnotation]
val, ok := new.GetAnnotations()[tls.SelfSignedAnnotation]
if !ok || val != "true" {
return
}
@ -94,12 +94,12 @@ func (m *certManager) InitTLSPemPair() {
}
}
func (m *certManager) GetTLSPemPair() (*ktls.PemPair, error) {
var tls *ktls.PemPair
func (m *certManager) GetTLSPemPair() (*tls.PemPair, error) {
var keyPair *tls.PemPair
var err error
retryReadTLS := func() error {
tls, err = ktls.ReadTLSPair(m.renewer.ClientConfig(), m.renewer.Client())
keyPair, err = tls.ReadTLSPair(m.renewer.ClientConfig(), m.renewer.Client())
if err != nil {
return err
}
@ -112,7 +112,7 @@ func (m *certManager) GetTLSPemPair() (*ktls.PemPair, error) {
f := common.RetryFunc(time.Second, time.Minute, retryReadTLS, msg, m.log.WithName("GetTLSPemPair/Retry"))
err = f()
return tls, err
return keyPair, err
}
func (m *certManager) Run(stopCh <-chan struct{}) {
@ -127,7 +127,7 @@ func (m *certManager) Run(stopCh <-chan struct{}) {
})
m.log.Info("start managing certificate")
certsRenewalTicker := time.NewTicker(ktls.CertRenewalInterval)
certsRenewalTicker := time.NewTicker(tls.CertRenewalInterval)
defer certsRenewalTicker.Stop()
for {
@ -137,7 +137,7 @@ func (m *certManager) Run(stopCh <-chan struct{}) {
if err != nil {
m.log.Error(err, "failed to validate cert")
if !strings.Contains(err.Error(), ktls.ErrorsNotFound) {
if !strings.Contains(err.Error(), tls.ErrorsNotFound) {
continue
}
}
@ -157,7 +157,7 @@ func (m *certManager) Run(stopCh <-chan struct{}) {
if err != nil {
m.log.Error(err, "failed to validate cert")
if !strings.Contains(err.Error(), ktls.ErrorsNotFound) {
if !strings.Contains(err.Error(), tls.ErrorsNotFound) {
continue
}
}

View file

@ -1,7 +1,7 @@
package webhookconfig
import (
"encoding/json"
"context"
"errors"
"io/ioutil"
"path/filepath"
@ -11,11 +11,10 @@ import (
"github.com/kyverno/kyverno/pkg/config"
"github.com/kyverno/kyverno/pkg/tls"
admregapi "k8s.io/api/admissionregistration/v1"
apps "k8s.io/api/apps/v1"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime/schema"
rest "k8s.io/client-go/rest"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/rbac/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/rest"
)
func (wrc *Register) readCaData() []byte {
@ -60,16 +59,14 @@ func extractCA(config *rest.Config) (result []byte) {
return config.TLSClientConfig.CAData
}
func (wrc *Register) constructOwner() v1.OwnerReference {
func (wrc *Register) constructOwner() metav1.OwnerReference {
logger := wrc.log
kubeClusterRoleName, err := wrc.GetKubePolicyClusterRoleName()
if err != nil {
logger.Error(err, "failed to get cluster role")
return v1.OwnerReference{}
return metav1.OwnerReference{}
}
return v1.OwnerReference{
return metav1.OwnerReference{
APIVersion: config.ClusterRoleAPIVersion,
Kind: config.ClusterRoleKind,
Name: kubeClusterRoleName.GetName(),
@ -77,8 +74,13 @@ func (wrc *Register) constructOwner() v1.OwnerReference {
}
}
func (wrc *Register) GetKubePolicyClusterRoleName() (*unstructured.Unstructured, error) {
clusterRoles, err := wrc.client.ListResource(config.ClusterRoleAPIVersion, config.ClusterRoleKind, "", &v1.LabelSelector{MatchLabels: map[string]string{"app.kubernetes.io/name": "kyverno"}})
func (wrc *Register) GetKubePolicyClusterRoleName() (*corev1.ClusterRole, error) {
selector := &metav1.LabelSelector{
MatchLabels: map[string]string{
"app.kubernetes.io/name": "kyverno",
},
}
clusterRoles, err := wrc.kubeClient.RbacV1().ClusterRoles().List(context.TODO(), metav1.ListOptions{LabelSelector: metav1.FormatLabelSelector(selector)})
if err != nil {
return nil, err
}
@ -92,22 +94,12 @@ func (wrc *Register) GetKubePolicyClusterRoleName() (*unstructured.Unstructured,
// GetKubePolicyDeployment gets Kyverno deployment using the resource cache
// it does not initialize any client call
func (wrc *Register) GetKubePolicyDeployment() (*apps.Deployment, *unstructured.Unstructured, error) {
func (wrc *Register) GetKubePolicyDeployment() (*appsv1.Deployment, error) {
deploy, err := wrc.kDeplLister.Deployments(config.KyvernoNamespace).Get(config.KyvernoDeploymentName)
if err != nil {
return nil, nil, err
return nil, err
}
deploy.SetGroupVersionKind(schema.GroupVersionKind{Group: "", Version: "apps/v1", Kind: "Deployment"})
kubePolicyDeployment := unstructured.Unstructured{}
rawDepl, err := json.Marshal(deploy)
if err != nil {
return nil, nil, err
}
err = json.Unmarshal(rawDepl, &kubePolicyDeployment.Object)
if err != nil {
return deploy, nil, err
}
return deploy, &kubePolicyDeployment, nil
return deploy, nil
}
// debug mutating webhook

View file

@ -18,7 +18,6 @@ import (
"github.com/kyverno/kyverno/pkg/common"
"github.com/kyverno/kyverno/pkg/config"
client "github.com/kyverno/kyverno/pkg/dclient"
"github.com/kyverno/kyverno/pkg/resourcecache"
"github.com/kyverno/kyverno/pkg/utils"
kubeutils "github.com/kyverno/kyverno/pkg/utils/kube"
"github.com/pkg/errors"
@ -58,8 +57,6 @@ type webhookConfigManager struct {
// npListerSynced returns true if the namespace policy store has been synced at least once
npListerSynced cache.InformerSynced
resCache resourcecache.ResourceCache
mutateInformer adminformers.MutatingWebhookConfigurationInformer
validateInformer adminformers.ValidatingWebhookConfigurationInformer
mutateLister admlisters.MutatingWebhookConfigurationLister
@ -95,7 +92,6 @@ func newWebhookConfigManager(
npInformer kyvernoinformer.PolicyInformer,
mwcInformer adminformers.MutatingWebhookConfigurationInformer,
vwcInformer adminformers.ValidatingWebhookConfigurationInformer,
resCache resourcecache.ResourceCache,
serverIP string,
autoUpdateWebhooks bool,
createDefaultWebhook chan<- string,
@ -107,7 +103,6 @@ func newWebhookConfigManager(
kyvernoClient: kyvernoClient,
pInformer: pInformer,
npInformer: npInformer,
resCache: resCache,
queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "configmanager"),
wildcardPolicy: 0,
serverIP: serverIP,

View file

@ -225,7 +225,7 @@ func lastRequestTimeFromAnnotation(leaseClient coordinationv1.LeaseInterface, lo
// skipWebhookCheck returns true if Kyverno is in rolling update
func skipWebhookCheck(register *Register, logger logr.Logger) bool {
deploy, _, err := register.GetKubePolicyDeployment()
deploy, err := register.GetKubePolicyDeployment()
if err != nil {
logger.Info("unable to get Kyverno deployment", "reason", err.Error())
return false

View file

@ -9,7 +9,6 @@ import (
)
func (wrc *Register) constructPolicyValidatingWebhookConfig(caData []byte) *admregapi.ValidatingWebhookConfiguration {
return &admregapi.ValidatingWebhookConfiguration{
ObjectMeta: v1.ObjectMeta{
Name: config.PolicyValidatingWebhookConfigurationName,
@ -40,7 +39,6 @@ func (wrc *Register) constructDebugPolicyValidatingWebhookConfig(caData []byte)
logger := wrc.log
url := fmt.Sprintf("https://%s%s", wrc.serverIP, config.PolicyValidatingWebhookServicePath)
logger.V(4).Info("Debug PolicyValidatingWebhookConfig is registered with url ", "url", url)
return &admregapi.ValidatingWebhookConfiguration{
ObjectMeta: v1.ObjectMeta{
Name: config.PolicyValidatingWebhookConfigurationDebugName,
@ -95,7 +93,6 @@ func (wrc *Register) constructDebugPolicyMutatingWebhookConfig(caData []byte) *a
logger := wrc.log
url := fmt.Sprintf("https://%s%s", wrc.serverIP, config.PolicyMutatingWebhookServicePath)
logger.V(4).Info("Debug PolicyMutatingWebhookConfig is registered with url ", "url", url)
return &admregapi.MutatingWebhookConfiguration{
ObjectMeta: v1.ObjectMeta{
Name: config.PolicyMutatingWebhookConfigurationDebugName,

View file

@ -14,7 +14,6 @@ import (
kyvernoinformer "github.com/kyverno/kyverno/pkg/client/informers/externalversions/kyverno/v1"
"github.com/kyverno/kyverno/pkg/config"
client "github.com/kyverno/kyverno/pkg/dclient"
"github.com/kyverno/kyverno/pkg/resourcecache"
"github.com/kyverno/kyverno/pkg/tls"
"github.com/kyverno/kyverno/pkg/utils"
"github.com/pkg/errors"
@ -22,9 +21,6 @@ import (
corev1 "k8s.io/api/core/v1"
errorsapi "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
adminformers "k8s.io/client-go/informers/admissionregistration/v1"
informers "k8s.io/client-go/informers/apps/v1"
"k8s.io/client-go/kubernetes"
@ -46,16 +42,19 @@ const (
// 4. Resource Mutation
// 5. Webhook Status Mutation
type Register struct {
client *client.Client
// clients
kubeClient kubernetes.Interface
clientConfig *rest.Config
resCache resourcecache.ResourceCache
mwcLister admlisters.MutatingWebhookConfigurationLister
vwcLister admlisters.ValidatingWebhookConfigurationLister
// listers
mwcLister admlisters.MutatingWebhookConfigurationLister
vwcLister admlisters.ValidatingWebhookConfigurationLister
kDeplLister listers.DeploymentLister
mwcListerSynced func() bool
vwcListerSynced func() bool
// sync
mwcListerSynced cache.InformerSynced
vwcListerSynced cache.InformerSynced
kDeplListerSynced cache.InformerSynced
serverIP string // when running outside a cluster
timeoutSeconds int32
@ -67,9 +66,6 @@ type Register struct {
UpdateWebhookChan chan bool
createDefaultWebhook chan string
kDeplLister listers.DeploymentLister
kDeplListerSynced func() bool
// manage implements methods to manage webhook configurations
manage
}
@ -82,7 +78,6 @@ func NewRegister(
kyvernoClient *kyvernoclient.Clientset,
mwcInformer adminformers.MutatingWebhookConfigurationInformer,
vwcInformer adminformers.ValidatingWebhookConfigurationInformer,
resCache resourcecache.ResourceCache,
kDeplInformer informers.DeploymentInformer,
pInformer kyvernoinformer.ClusterPolicyInformer,
npInformer kyvernoinformer.PolicyInformer,
@ -94,9 +89,13 @@ func NewRegister(
log logr.Logger) *Register {
register := &Register{
clientConfig: clientConfig,
client: client,
kubeClient: kubeClient,
resCache: resCache,
mwcLister: mwcInformer.Lister(),
vwcLister: vwcInformer.Lister(),
kDeplLister: kDeplInformer.Lister(),
mwcListerSynced: mwcInformer.Informer().HasSynced,
vwcListerSynced: vwcInformer.Informer().HasSynced,
kDeplListerSynced: kDeplInformer.Informer().HasSynced,
serverIP: serverIP,
timeoutSeconds: webhookTimeout,
log: log.WithName("Register"),
@ -104,16 +103,10 @@ func NewRegister(
autoUpdateWebhooks: autoUpdateWebhooks,
UpdateWebhookChan: make(chan bool),
createDefaultWebhook: make(chan string),
kDeplLister: kDeplInformer.Lister(),
kDeplListerSynced: kDeplInformer.Informer().HasSynced,
mwcLister: mwcInformer.Lister(),
vwcLister: vwcInformer.Lister(),
mwcListerSynced: mwcInformer.Informer().HasSynced,
vwcListerSynced: vwcInformer.Informer().HasSynced,
stopCh: stopCh,
}
register.manage = newWebhookConfigManager(client, kyvernoClient, pInformer, npInformer, mwcInformer, vwcInformer, resCache, serverIP, register.autoUpdateWebhooks, register.createDefaultWebhook, stopCh, log.WithName("WebhookConfigManager"))
register.manage = newWebhookConfigManager(client, kyvernoClient, pInformer, npInformer, mwcInformer, vwcInformer, serverIP, register.autoUpdateWebhooks, register.createDefaultWebhook, stopCh, log.WithName("WebhookConfigManager"))
return register
}
@ -252,24 +245,16 @@ func (wrc *Register) UpdateWebhookConfigurations(configHandler config.Interface)
func (wrc *Register) ValidateWebhookConfigurations(namespace, name string) error {
logger := wrc.log.WithName("ValidateWebhookConfigurations")
cm, err := wrc.client.GetResource("", "ConfigMap", namespace, name)
cm, err := wrc.kubeClient.CoreV1().ConfigMaps(namespace).Get(context.TODO(), name, metav1.GetOptions{})
if err != nil {
logger.Error(err, "unable to fetch ConfigMap", "namespace", namespace, "name", name)
return nil
}
webhooks, ok, err := unstructured.NestedString(cm.UnstructuredContent(), "data", "webhooks")
if err != nil {
logger.Error(err, "failed to fetch tag 'webhooks' from the ConfigMap")
return nil
}
webhooks, ok := cm.Data["webhooks"]
if !ok {
logger.V(4).Info("webhook configurations not defined")
return nil
}
webhookCfgs := make([]config.WebhookConfig, 0, 10)
return json.Unmarshal([]byte(webhooks), &webhookCfgs)
}
@ -277,13 +262,12 @@ func (wrc *Register) ValidateWebhookConfigurations(namespace, name string) error
// cleanupKyvernoResource returns true if Kyverno is terminating
func (wrc *Register) cleanupKyvernoResource() bool {
logger := wrc.log.WithName("cleanupKyvernoResource")
deploy, err := wrc.client.GetResource("", "Deployment", config.KyvernoNamespace, config.KyvernoDeploymentName)
deploy, err := wrc.kubeClient.AppsV1().Deployments(config.KyvernoNamespace).Get(context.TODO(), config.KyvernoDeploymentName, metav1.GetOptions{})
if err != nil {
if errorsapi.IsNotFound(err) {
logger.Info("Kyverno deployment not found, cleanup Kyverno resources")
return true
}
logger.Error(err, "failed to get deployment, not cleaning up kyverno resources")
return false
}
@ -291,17 +275,10 @@ func (wrc *Register) cleanupKyvernoResource() bool {
logger.Info("Kyverno is terminating, cleanup Kyverno resources")
return true
}
replicas, _, err := unstructured.NestedInt64(deploy.UnstructuredContent(), "spec", "replicas")
if err != nil {
logger.Error(err, "unable to fetch spec.replicas of Kyverno deployment")
}
if replicas == 0 {
if deploy.Spec.Replicas == nil && *deploy.Spec.Replicas == 0 {
logger.Info("Kyverno is scaled to zero, cleanup Kyverno resources")
return true
}
logger.Info("updating Kyverno Pod, won't clean up Kyverno resources")
return false
}
@ -317,7 +294,7 @@ func (wrc *Register) createResourceMutatingWebhookConfiguration(caData []byte) e
logger := wrc.log.WithValues("kind", kindMutating, "name", config.Name)
_, err := wrc.client.CreateResource("", kindMutating, "", *config, false)
_, err := wrc.kubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Create(context.TODO(), config, metav1.CreateOptions{})
if errorsapi.IsAlreadyExists(err) {
logger.V(6).Info("resource mutating webhook configuration already exists", "name", config.Name)
err = wrc.updateMutatingWebhookConfiguration(config)
@ -346,7 +323,7 @@ func (wrc *Register) createResourceValidatingWebhookConfiguration(caData []byte)
logger := wrc.log.WithValues("kind", kindValidating, "name", config.Name)
_, err := wrc.client.CreateResource("", kindValidating, "", *config, false)
_, err := wrc.kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Create(context.TODO(), config, metav1.CreateOptions{})
if errorsapi.IsAlreadyExists(err) {
logger.V(6).Info("resource validating webhook configuration already exists", "name", config.Name)
err = wrc.updateValidatingWebhookConfiguration(config)
@ -375,7 +352,7 @@ func (wrc *Register) createPolicyValidatingWebhookConfiguration(caData []byte) e
config = wrc.constructPolicyValidatingWebhookConfig(caData)
}
if _, err := wrc.client.CreateResource("", kindValidating, "", *config, false); err != nil {
if _, err := wrc.kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Create(context.TODO(), config, metav1.CreateOptions{}); err != nil {
if errorsapi.IsAlreadyExists(err) {
wrc.log.V(6).Info("webhook already exists", "kind", kindValidating, "name", config.Name)
err = wrc.updateValidatingWebhookConfiguration(config)
@ -401,8 +378,7 @@ func (wrc *Register) createPolicyMutatingWebhookConfiguration(caData []byte) err
config = wrc.constructPolicyMutatingWebhookConfig(caData)
}
// create mutating webhook configuration resource
if _, err := wrc.client.CreateResource("", kindMutating, "", *config, false); err != nil {
if _, err := wrc.kubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Create(context.TODO(), config, metav1.CreateOptions{}); err != nil {
if errorsapi.IsAlreadyExists(err) {
wrc.log.V(6).Info("webhook already exists", "kind", kindMutating, "name", config.Name)
err = wrc.updateMutatingWebhookConfiguration(config)
@ -428,7 +404,7 @@ func (wrc *Register) createVerifyMutatingWebhookConfiguration(caData []byte) err
config = wrc.constructVerifyMutatingWebhookConfig(caData)
}
if _, err := wrc.client.CreateResource("", kindMutating, "", *config, false); err != nil {
if _, err := wrc.kubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Create(context.TODO(), config, metav1.CreateOptions{}); err != nil {
if errorsapi.IsAlreadyExists(err) {
wrc.log.V(6).Info("webhook already exists", "kind", kindMutating, "name", config.Name)
err = wrc.updateMutatingWebhookConfiguration(config)
@ -476,7 +452,7 @@ func (wrc *Register) removePolicyMutatingWebhookConfiguration(wg *sync.WaitGroup
return
}
err := wrc.client.DeleteResource("", kindMutating, "", mutatingConfig, false)
err := wrc.kubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete(context.TODO(), mutatingConfig, metav1.DeleteOptions{})
if errorsapi.IsNotFound(err) {
logger.V(5).Info("policy mutating webhook configuration not found")
return
@ -512,7 +488,7 @@ func (wrc *Register) removePolicyValidatingWebhookConfiguration(wg *sync.WaitGro
}
logger.V(4).Info("removing validating webhook configuration")
err := wrc.client.DeleteResource("", kindValidating, "", validatingConfig, false)
err := wrc.kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Delete(context.TODO(), validatingConfig, metav1.DeleteOptions{})
if errorsapi.IsNotFound(err) {
logger.V(5).Info("policy validating webhook configuration not found")
return
@ -615,7 +591,7 @@ func (wrc *Register) removeVerifyWebhookMutatingWebhookConfig(wg *sync.WaitGroup
return
}
err = wrc.client.DeleteResource("", kindMutating, "", mutatingConfig, false)
err = wrc.kubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete(context.TODO(), mutatingConfig, metav1.DeleteOptions{})
if errorsapi.IsNotFound(err) {
logger.V(5).Info("verify webhook configuration not found")
return
@ -651,52 +627,37 @@ func (wrc *Register) removeSecrets() {
tls.ManagedByLabel: "kyverno",
},
}
secretList, err := wrc.client.ListResource("", "Secret", config.KyvernoNamespace, selector)
if err != nil {
if err := wrc.kubeClient.CoreV1().Secrets(config.KyvernoNamespace).DeleteCollection(context.TODO(), metav1.DeleteOptions{}, metav1.ListOptions{LabelSelector: metav1.FormatLabelSelector(selector)}); err != nil {
wrc.log.Error(err, "failed to clean up Kyverno managed secrets")
return
}
for _, secret := range secretList.Items {
if err := wrc.client.DeleteResource("", "Secret", secret.GetNamespace(), secret.GetName(), false); err != nil {
if !errorsapi.IsNotFound(err) {
wrc.log.Error(err, "failed to delete secret", "ns", secret.GetNamespace(), "name", secret.GetName())
}
}
}
}
func (wrc *Register) checkEndpoint() error {
obj, err := wrc.client.GetResource("", "Endpoints", config.KyvernoNamespace, config.KyvernoServiceName)
endpoint, err := wrc.kubeClient.CoreV1().Endpoints(config.KyvernoNamespace).Get(context.TODO(), config.KyvernoServiceName, metav1.GetOptions{})
if err != nil {
return fmt.Errorf("failed to get endpoint %s/%s: %v", config.KyvernoNamespace, config.KyvernoServiceName, err)
}
var endpoint corev1.Endpoints
err = runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), &endpoint)
if err != nil {
return fmt.Errorf("failed to convert endpoint %s/%s from unstructured: %v", config.KyvernoNamespace, config.KyvernoServiceName, err)
selector := &metav1.LabelSelector{
MatchLabels: map[string]string{
"app.kubernetes.io/name": "kyverno",
},
}
pods, err := wrc.client.ListResource("", "Pod", config.KyvernoNamespace, &metav1.LabelSelector{MatchLabels: map[string]string{"app.kubernetes.io/name": "kyverno"}})
pods, err := wrc.kubeClient.CoreV1().Pods(config.KyvernoNamespace).List(context.TODO(), metav1.ListOptions{LabelSelector: metav1.FormatLabelSelector(selector)})
if err != nil {
return fmt.Errorf("failed to list Kyverno Pod: %v", err)
}
ips, errs := getHealthyPodsIP(pods.Items)
if len(errs) != 0 {
return fmt.Errorf("error getting pod's IP: %v", errs)
}
if len(ips) == 0 {
return fmt.Errorf("pod is not assigned to any node yet")
}
for _, subset := range endpoint.Subsets {
if len(subset.Addresses) == 0 {
continue
}
for _, addr := range subset.Addresses {
if utils.ContainsString(ips, addr.IP) {
wrc.log.Info("Endpoint ready", "ns", config.KyvernoNamespace, "name", config.KyvernoServiceName)
@ -704,130 +665,36 @@ func (wrc *Register) checkEndpoint() error {
}
}
}
err = fmt.Errorf("endpoint not ready")
wrc.log.V(3).Info(err.Error(), "ns", config.KyvernoNamespace, "name", config.KyvernoServiceName)
return err
}
func getHealthyPodsIP(pods []unstructured.Unstructured) (ips []string, errs []error) {
func getHealthyPodsIP(pods []corev1.Pod) (ips []string, errs []error) {
for _, pod := range pods {
phase, _, err := unstructured.NestedString(pod.UnstructuredContent(), "status", "phase")
if err != nil {
errs = append(errs, fmt.Errorf("failed to get pod %s status: %v", pod.GetName(), err))
if pod.Status.Phase != "Running" {
continue
}
if phase != "Running" {
continue
}
ip, _, err := unstructured.NestedString(pod.UnstructuredContent(), "status", "podIP")
if err != nil {
errs = append(errs, fmt.Errorf("failed to extract pod %s IP: %v", pod.GetName(), err))
continue
}
ips = append(ips, ip)
ips = append(ips, pod.Status.PodIP)
}
return
}
func convertLabelSelector(selector *metav1.LabelSelector, logger logr.Logger) (map[string]interface{}, error) {
if selector == nil {
return nil, nil
}
if selectorBytes, err := json.Marshal(*selector); err != nil {
logger.Error(err, "failed to serialize selector")
return nil, err
} else {
var nsSelector map[string]interface{}
if err := json.Unmarshal(selectorBytes, &nsSelector); err != nil {
logger.Error(err, "failed to convert namespaceSelector to the map")
return nil, err
}
return nsSelector, nil
}
}
func configureSelector(webhook map[string]interface{}, selector *metav1.LabelSelector, key string, logger logr.Logger) (bool, error) {
currentSelector, _, err := unstructured.NestedMap(webhook, key)
if err != nil {
return false, err
}
if expectedSelector, err := convertLabelSelector(selector, logger); err != nil {
return false, err
} else {
if reflect.DeepEqual(expectedSelector, currentSelector) {
return false, nil
} else {
if err = unstructured.SetNestedMap(webhook, expectedSelector, key); err != nil {
return false, err
}
return true, nil
}
}
}
func (wrc *Register) updateResourceValidatingWebhookConfiguration(webhookCfg config.WebhookConfig) error {
resourceValidatingTyped, err := wrc.vwcLister.Get(getResourceValidatingWebhookConfigName(wrc.serverIP))
resource, err := wrc.vwcLister.Get(getResourceValidatingWebhookConfigName(wrc.serverIP))
if err != nil {
return errors.Wrapf(err, "unable to get validatingWebhookConfigurations")
}
resourceValidatingTyped.SetGroupVersionKind(schema.GroupVersionKind{Group: "", Version: "admissionregistration.k8s.io/v1", Kind: kindValidating})
resourceValidating := &unstructured.Unstructured{}
rawResc, err := json.Marshal(resourceValidatingTyped)
if err != nil {
return err
copy := resource.DeepCopy()
for i := range copy.Webhooks {
copy.Webhooks[i].ObjectSelector = webhookCfg.ObjectSelector
copy.Webhooks[i].NamespaceSelector = webhookCfg.NamespaceSelector
}
err = json.Unmarshal(rawResc, &resourceValidating.Object)
if err != nil {
return err
}
webhooksUntyped, _, err := unstructured.NestedSlice(resourceValidating.UnstructuredContent(), "webhooks")
if err != nil {
return errors.Wrapf(err, "unable to load validatingWebhookConfigurations.webhooks")
}
var (
webhook map[string]interface{}
webhooks []interface{}
ok bool
)
webhookChanged := false
for i, whu := range webhooksUntyped {
webhook, ok = whu.(map[string]interface{})
if !ok {
return errors.Wrapf(err, "type mismatched, expected map[string]interface{}, got %T", webhooksUntyped[i])
}
if changed, err := configureSelector(webhook, webhookCfg.ObjectSelector, "objectSelector", wrc.log); err != nil {
return errors.Wrapf(err, "unable to get validatingWebhookConfigurations.webhooks["+fmt.Sprint(i)+"].objectSelector")
} else {
if changed {
webhookChanged = true
}
}
if changed, err := configureSelector(webhook, webhookCfg.NamespaceSelector, "namespaceSelector", wrc.log); err != nil {
return errors.Wrapf(err, "unable to get validatingWebhookConfigurations.webhooks["+fmt.Sprint(i)+"].namespaceSelector")
} else {
if changed {
webhookChanged = true
}
}
webhooks = append(webhooks, webhook)
}
if !webhookChanged {
if reflect.DeepEqual(resource.Webhooks, copy.Webhooks) {
wrc.log.V(4).Info("namespaceSelector unchanged, skip updating validatingWebhookConfigurations")
return nil
}
if err = unstructured.SetNestedSlice(resourceValidating.UnstructuredContent(), webhooks, "webhooks"); err != nil {
return errors.Wrapf(err, "unable to set validatingWebhookConfigurations.webhooks")
}
if _, err := wrc.client.UpdateResource(resourceValidating.GetAPIVersion(), resourceValidating.GetKind(), "", resourceValidating, false); err != nil {
if _, err := wrc.kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Update(context.TODO(), copy, metav1.UpdateOptions{}); err != nil {
return err
}
wrc.log.V(3).Info("successfully updated validatingWebhookConfigurations", "name", getResourceMutatingWebhookConfigName(wrc.serverIP))
@ -835,63 +702,20 @@ func (wrc *Register) updateResourceValidatingWebhookConfiguration(webhookCfg con
}
func (wrc *Register) updateResourceMutatingWebhookConfiguration(webhookCfg config.WebhookConfig) error {
resourceMutatingTyped, err := wrc.mwcLister.Get(getResourceMutatingWebhookConfigName(wrc.serverIP))
resource, err := wrc.mwcLister.Get(getResourceMutatingWebhookConfigName(wrc.serverIP))
if err != nil {
return errors.Wrapf(err, "unable to get mutatingWebhookConfigurations")
}
resourceMutatingTyped.SetGroupVersionKind(schema.GroupVersionKind{Group: "", Version: "admissionregistration.k8s.io/v1", Kind: kindMutating})
resourceMutating := &unstructured.Unstructured{}
rawResc, err := json.Marshal(resourceMutatingTyped)
if err != nil {
return err
copy := resource.DeepCopy()
for i := range copy.Webhooks {
copy.Webhooks[i].ObjectSelector = webhookCfg.ObjectSelector
copy.Webhooks[i].NamespaceSelector = webhookCfg.NamespaceSelector
}
err = json.Unmarshal(rawResc, &resourceMutating.Object)
if err != nil {
return err
}
webhooksUntyped, _, err := unstructured.NestedSlice(resourceMutating.UnstructuredContent(), "webhooks")
if err != nil {
return errors.Wrapf(err, "unable to load mutatingWebhookConfigurations.webhooks")
}
var (
webhook map[string]interface{}
webhooks []interface{}
ok bool
)
webhookChanged := false
for i, whu := range webhooksUntyped {
webhook, ok = whu.(map[string]interface{})
if !ok {
return errors.Wrapf(err, "type mismatched, expected map[string]interface{}, got %T", webhooksUntyped[i])
}
if changed, err := configureSelector(webhook, webhookCfg.ObjectSelector, "objectSelector", wrc.log); err != nil {
return errors.Wrapf(err, "unable to get mutatingWebhookConfigurations.webhooks["+fmt.Sprint(i)+"].objectSelector")
} else {
if changed {
webhookChanged = true
}
}
if changed, err := configureSelector(webhook, webhookCfg.NamespaceSelector, "namespaceSelector", wrc.log); err != nil {
return errors.Wrapf(err, "unable to get mutatingWebhookConfigurations.webhooks["+fmt.Sprint(i)+"].namespaceSelector")
} else {
if changed {
webhookChanged = true
}
}
webhooks = append(webhooks, webhook)
}
if !webhookChanged {
if reflect.DeepEqual(resource.Webhooks, copy.Webhooks) {
wrc.log.V(4).Info("namespaceSelector unchanged, skip updating mutatingWebhookConfigurations")
return nil
}
if err = unstructured.SetNestedSlice(resourceMutating.UnstructuredContent(), webhooks, "webhooks"); err != nil {
return errors.Wrapf(err, "unable to set mutatingWebhookConfigurations.webhooks")
}
if _, err := wrc.client.UpdateResource(resourceMutating.GetAPIVersion(), resourceMutating.GetKind(), "", resourceMutating, false); err != nil {
if _, err := wrc.kubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Update(context.TODO(), copy, metav1.UpdateOptions{}); err != nil {
return err
}
wrc.log.V(3).Info("successfully updated mutatingWebhookConfigurations", "name", getResourceMutatingWebhookConfigName(wrc.serverIP))
@ -935,8 +759,7 @@ func (wrc *Register) updateMutatingWebhookConfiguration(targetConfig *admregapi.
}
// Update the current configuration.
currentConfiguration.Webhooks = newWebhooks
_, err = wrc.client.UpdateResource("", kindMutating, "", currentConfiguration, false)
if err != nil {
if _, err := wrc.kubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Update(context.TODO(), currentConfiguration, metav1.UpdateOptions{}); err != nil {
return err
}
wrc.log.V(3).Info("successfully updated mutatingWebhookConfigurations", "name", targetConfig.Name)
@ -980,8 +803,7 @@ func (wrc *Register) updateValidatingWebhookConfiguration(targetConfig *admregap
}
// Update the current configuration.
currentConfiguration.Webhooks = newWebhooks
_, err = wrc.client.UpdateResource("", kindValidating, "", currentConfiguration, false)
if err != nil {
if _, err := wrc.kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Update(context.TODO(), currentConfiguration, metav1.UpdateOptions{}); err != nil {
return err
}
wrc.log.V(3).Info("successfully updated validatingWebhookConfigurations", "name", targetConfig.Name)

View file

@ -1,13 +1,14 @@
package webhookconfig
import (
"context"
"fmt"
"sync"
"github.com/kyverno/kyverno/pkg/config"
admregapi "k8s.io/api/admissionregistration/v1"
errorsapi "k8s.io/apimachinery/pkg/api/errors"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func (wrc *Register) defaultResourceWebhookRule() admregapi.Rule {
@ -27,7 +28,7 @@ func (wrc *Register) constructDefaultDebugMutatingWebhookConfig(caData []byte) *
url := fmt.Sprintf("https://%s%s", wrc.serverIP, config.MutatingWebhookServicePath)
logger.V(4).Info("Debug MutatingWebhookConfig registered", "url", url)
webhook := &admregapi.MutatingWebhookConfiguration{
ObjectMeta: v1.ObjectMeta{
ObjectMeta: metav1.ObjectMeta{
Name: config.MutatingWebhookConfigurationDebugName,
},
Webhooks: []admregapi.MutatingWebhook{
@ -61,9 +62,9 @@ func (wrc *Register) constructDefaultDebugMutatingWebhookConfig(caData []byte) *
func (wrc *Register) constructDefaultMutatingWebhookConfig(caData []byte) *admregapi.MutatingWebhookConfiguration {
webhook := &admregapi.MutatingWebhookConfiguration{
ObjectMeta: v1.ObjectMeta{
ObjectMeta: metav1.ObjectMeta{
Name: config.MutatingWebhookConfigurationName,
OwnerReferences: []v1.OwnerReference{
OwnerReferences: []metav1.OwnerReference{
wrc.constructOwner(),
},
},
@ -115,8 +116,7 @@ func (wrc *Register) removeResourceMutatingWebhookConfiguration(wg *sync.WaitGro
return
}
// delete webhook configuration
err := wrc.client.DeleteResource("", kindMutating, "", configName, false)
err := wrc.kubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete(context.TODO(), configName, metav1.DeleteOptions{})
if errorsapi.IsNotFound(err) {
logger.V(4).Info("webhook configuration not found")
return
@ -134,7 +134,7 @@ func (wrc *Register) constructDefaultDebugValidatingWebhookConfig(caData []byte)
url := fmt.Sprintf("https://%s%s", wrc.serverIP, config.ValidatingWebhookServicePath)
webhook := &admregapi.ValidatingWebhookConfiguration{
ObjectMeta: v1.ObjectMeta{
ObjectMeta: metav1.ObjectMeta{
Name: config.ValidatingWebhookConfigurationDebugName,
},
Webhooks: []admregapi.ValidatingWebhook{
@ -168,9 +168,9 @@ func (wrc *Register) constructDefaultDebugValidatingWebhookConfig(caData []byte)
func (wrc *Register) constructDefaultValidatingWebhookConfig(caData []byte) *admregapi.ValidatingWebhookConfiguration {
webhook := &admregapi.ValidatingWebhookConfiguration{
ObjectMeta: v1.ObjectMeta{
ObjectMeta: metav1.ObjectMeta{
Name: config.ValidatingWebhookConfigurationName,
OwnerReferences: []v1.OwnerReference{
OwnerReferences: []metav1.OwnerReference{
wrc.constructOwner(),
},
},
@ -223,7 +223,7 @@ func (wrc *Register) removeResourceValidatingWebhookConfiguration(wg *sync.WaitG
return
}
err := wrc.client.DeleteResource("", kindValidating, "", configName, false)
err := wrc.kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Delete(context.TODO(), configName, metav1.DeleteOptions{})
if errorsapi.IsNotFound(err) {
logger.V(5).Info("webhook configuration not found")
return

View file

@ -13,10 +13,8 @@ import (
coordinationv1 "k8s.io/client-go/kubernetes/typed/coordination/v1"
)
var leaseName string = "kyverno"
var leaseNamespace string = config.KyvernoNamespace
const (
leaseName string = "kyverno"
annWebhookStatus string = "kyverno.io/webhookActive"
annLastRequestTime string = "kyverno.io/last-request-time"
)
@ -48,7 +46,7 @@ func newStatusControl(leaseClient coordinationv1.LeaseInterface, eventGen event.
}
func (vc statusControl) setStatus(status string) error {
logger := vc.log.WithValues("name", leaseName, "namespace", leaseNamespace)
logger := vc.log.WithValues("name", leaseName, "namespace", config.KyvernoNamespace)
var ann map[string]string
var err error
@ -90,7 +88,7 @@ func (vc statusControl) setStatus(status string) error {
func createStatusUpdateEvent(status string, eventGen event.Interface) {
e := event.Info{}
e.Kind = "Lease"
e.Namespace = leaseNamespace
e.Namespace = config.KyvernoNamespace
e.Name = leaseName
e.Reason = "Update"
e.Message = fmt.Sprintf("admission control webhook active status changed to %s", status)