fix conflicts (#7109)

Signed-off-by: ShutingZhao <shuting@nirmata.com>
Co-authored-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

CHANGELOG.md

@@ -1,3 +1,9 @@
+## v1.9.3
+
+### Note
+
+- Added support for configuring webhook annotations in the config map through `webhookAnnotations` stanza.
+
 ## v1.9.0-rc.1
 
 ### Note

charts/kyverno/Chart.yaml

@@ -26,7 +26,5 @@ annotations:
       url: https://kyverno.io/docs
   # valid kinds are: added, changed, deprecated, removed, fixed and security
   artifacthub.io/changes: |
-    - kind: changed
+    - kind: added
-      description: Syntax change for webhooksCleanup switch to match with the rest of the file
+      description: support for webhook annotations in config map
-    - kind: fixed
-      description: Handle multiple extraArgs in init container

charts/kyverno/README.md

@@ -186,6 +186,7 @@ The command removes all the Kubernetes components associated with the chart and
 | config.generateSuccessEvents | bool | `false` | Generate success events. |
 | config.metricsConfig | object | `{"annotations":{},"namespaces":{"exclude":[],"include":[]}}` | Metrics config. |
 | config.metricsConfig.annotations | object | `{}` | Additional annotations to add to the metricsconfigmap |
+| config.webhookAnnotations | object | `{}` | Defines annotations to set on webhook configurations. |
 | updateStrategy | object | See [values.yaml](values.yaml) | Deployment update strategy. Ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy |
 | service.port | int | `443` | Service port. |
 | service.type | string | `"ClusterIP"` | Service type. |

charts/kyverno/templates/configmap.yaml

@@ -31,4 +31,7 @@ data:
   {{- if .Values.config.generateSuccessEvents }}
   generateSuccessEvents: {{ .Values.config.generateSuccessEvents | quote }}
   {{- end -}}
+  {{- with .Values.config.webhookAnnotations }}
+  webhookAnnotations: {{ toJson . | quote }}
+  {{- end }}
 {{- end -}}

charts/kyverno/values.yaml

@@ -392,6 +392,12 @@ config:
 
   # Or provide an existing metrics config-map by uncommenting the below line
   # existingMetricsConfig: sample-metrics-configmap. Refer to the ./templates/metricsconfigmap.yaml for the structure of metrics configmap.
+
+  # -- Defines annotations to set on webhook configurations.
+  webhookAnnotations: {}
+    # Example to disable admission enforcer on AKS:
+    # 'admissions.enforcer/disabled': 'true'
+
 # -- Deployment update strategy.
 # Ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy
 # @default -- See [values.yaml](values.yaml)

cmd/cleanup-controller/main.go

@@ -121,6 +121,7 @@ func main() {
 					kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations(),
 					kubeInformer.Admissionregistration().V1().ValidatingWebhookConfigurations(),
 					kubeKyvernoInformer.Core().V1().Secrets(),
+					kubeKyvernoInformer.Core().V1().ConfigMaps(),
 					config.CleanupValidatingWebhookConfigurationName,
 					config.CleanupValidatingWebhookServicePath,
 					serverIP,

cmd/kyverno/main.go

@@ -321,6 +321,7 @@ func createrLeaderControllers(
 		kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations(),
 		kubeInformer.Admissionregistration().V1().ValidatingWebhookConfigurations(),
 		kubeKyvernoInformer.Core().V1().Secrets(),
+		kubeKyvernoInformer.Core().V1().ConfigMaps(),
 		config.ExceptionValidatingWebhookConfigurationName,
 		config.ExceptionValidatingWebhookServicePath,
 		serverIP,

pkg/config/config.go

@@ -6,7 +6,7 @@ import (
 	"sync"
 
 	osutils "github.com/kyverno/kyverno/pkg/utils/os"
-	wildcard "github.com/kyverno/kyverno/pkg/utils/wildcard"
+	"github.com/kyverno/kyverno/pkg/utils/wildcard"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -142,18 +142,21 @@ type Configuration interface {
 	FilterNamespaces(namespaces []string) []string
 	// GetWebhooks returns the webhook configs
 	GetWebhooks() []WebhookConfig
+	// GetWebhookAnnotations returns annotations to set on webhook configs
+	GetWebhookAnnotations() map[string]string
 	// Load loads configuration from a configmap
 	Load(cm *corev1.ConfigMap)
 }
 
 // configuration stores the configuration
 type configuration struct {
-	mux                   sync.RWMutex
 	filters               []filter
 	excludeGroupRole      []string
 	excludeUsername       []string
 	webhooks              []WebhookConfig
 	generateSuccessEvents bool
+	webhookAnnotations    map[string]string
+	mux                   sync.RWMutex
 }
 
 // NewDefaultConfiguration ...
@@ -227,6 +230,12 @@ func (cd *configuration) GetWebhooks() []WebhookConfig {
 	return cd.webhooks
 }
 
+func (cd *configuration) GetWebhookAnnotations() map[string]string {
+	cd.mux.RLock()
+	defer cd.mux.RUnlock()
+	return cd.webhookAnnotations
+}
+
 func (cd *configuration) Load(cm *corev1.ConfigMap) {
 	if cm != nil {
 		cd.load(cm)
@@ -275,6 +284,16 @@ func (cd *configuration) load(cm *corev1.ConfigMap) {
 			cd.webhooks = webhooks
 		}
 	}
+	// load webhook annotations
+	webhookAnnotations, ok := cm.Data["webhookAnnotations"]
+	if ok {
+		webhookAnnotations, err := parseWebhookAnnotations(webhookAnnotations)
+		if err != nil {
+			logger.Error(err, "failed to parse webhook annotations")
+		} else {
+			cd.webhookAnnotations = webhookAnnotations
+		}
+	}
 }
 
 func (cd *configuration) unload() {
@@ -286,4 +305,5 @@ func (cd *configuration) unload() {
 	cd.generateSuccessEvents = false
 	cd.webhooks = nil
 	cd.excludeGroupRole = append(cd.excludeGroupRole, defaultExcludeGroupRole...)
+	cd.webhookAnnotations = nil
 }

pkg/config/types.go

@@ -21,6 +21,14 @@ func parseWebhooks(webhooks string) ([]WebhookConfig, error) {
 	return webhookCfgs, nil
 }
 
+func parseWebhookAnnotations(in string) (map[string]string, error) {
+	var out map[string]string
+	if err := json.Unmarshal([]byte(in), &out); err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
 func parseRbac(list string) []string {
 	return strings.Split(list, ",")
 }

pkg/controllers/generic/webhook/controller.go

@@ -40,8 +40,9 @@ type controller struct {
 	vwcClient controllerutils.ObjectClient[*admissionregistrationv1.ValidatingWebhookConfiguration]
 
 	// listers
-	vwcLister    admissionregistrationv1listers.ValidatingWebhookConfigurationLister
+	vwcLister       admissionregistrationv1listers.ValidatingWebhookConfigurationLister
-	secretLister corev1listers.SecretNamespaceLister
+	secretLister    corev1listers.SecretNamespaceLister
+	configMapLister corev1listers.ConfigMapLister
 
 	// queue
 	queue workqueue.RateLimitingInterface
@@ -62,6 +63,7 @@ func NewController(
 	vwcClient controllerutils.ObjectClient[*admissionregistrationv1.ValidatingWebhookConfiguration],
 	vwcInformer admissionregistrationv1informers.ValidatingWebhookConfigurationInformer,
 	secretInformer corev1informers.SecretInformer,
+	configMapInformer corev1informers.ConfigMapInformer,
 	webhookName string,
 	path string,
 	server string,
@@ -71,18 +73,19 @@ func NewController(
 ) controllers.Controller {
 	queue := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), controllerName)
 	c := controller{
-		vwcClient:      vwcClient,
+		vwcClient:       vwcClient,
-		vwcLister:      vwcInformer.Lister(),
+		vwcLister:       vwcInformer.Lister(),
-		secretLister:   secretInformer.Lister().Secrets(config.KyvernoNamespace()),
+		secretLister:    secretInformer.Lister().Secrets(config.KyvernoNamespace()),
-		queue:          queue,
+		configMapLister: configMapInformer.Lister(),
-		controllerName: controllerName,
+		queue:           queue,
-		logger:         logging.ControllerLogger(controllerName),
+		controllerName:  controllerName,
-		webhookName:    webhookName,
+		logger:          logging.ControllerLogger(controllerName),
-		path:           path,
+		webhookName:     webhookName,
-		server:         server,
+		path:            path,
-		rules:          rules,
+		server:          server,
-		failurePolicy:  failurePolicy,
+		rules:           rules,
-		sideEffects:    sideEffects,
+		failurePolicy:   failurePolicy,
+		sideEffects:     sideEffects,
 	}
 	controllerutils.AddDefaultEventHandlers(c.logger, vwcInformer.Informer(), queue)
 	controllerutils.AddEventHandlersT(
@@ -103,6 +106,24 @@ func NewController(
 			}
 		},
 	)
+	controllerutils.AddEventHandlersT(
+		configMapInformer.Informer(),
+		func(obj *corev1.ConfigMap) {
+			if obj.GetNamespace() == config.KyvernoNamespace() && obj.GetName() == config.KyvernoConfigMapName() {
+				c.enqueue()
+			}
+		},
+		func(_, obj *corev1.ConfigMap) {
+			if obj.GetNamespace() == config.KyvernoNamespace() && obj.GetName() == config.KyvernoConfigMapName() {
+				c.enqueue()
+			}
+		},
+		func(obj *corev1.ConfigMap) {
+			if obj.GetNamespace() == config.KyvernoNamespace() && obj.GetName() == config.KyvernoConfigMapName() {
+				c.enqueue()
+			}
+		},
+	)
 	return &c
 }
 
@@ -115,6 +136,15 @@ func (c *controller) enqueue() {
 	c.queue.Add(c.webhookName)
 }
 
+func (c *controller) loadConfig() config.Configuration {
+	cfg := config.NewDefaultConfiguration()
+	cm, err := c.configMapLister.ConfigMaps(config.KyvernoNamespace()).Get(config.KyvernoConfigMapName())
+	if err == nil {
+		cfg.Load(cm)
+	}
+	return cfg
+}
+
 func (c *controller) reconcile(ctx context.Context, logger logr.Logger, key, _, _ string) error {
 	if key != c.webhookName {
 		return nil
@@ -123,7 +153,7 @@ func (c *controller) reconcile(ctx context.Context, logger logr.Logger, key, _,
 	if err != nil {
 		return err
 	}
-	desired, err := c.build(caData)
+	desired, err := c.build(c.loadConfig(), caData)
 	if err != nil {
 		return err
 	}
@@ -137,6 +167,7 @@ func (c *controller) reconcile(ctx context.Context, logger logr.Logger, key, _,
 	}
 	_, err = controllerutils.Update(ctx, observed, c.vwcClient, func(w *admissionregistrationv1.ValidatingWebhookConfiguration) error {
 		w.Labels = desired.Labels
+		w.Annotations = desired.Annotations
 		w.OwnerReferences = desired.OwnerReferences
 		w.Webhooks = desired.Webhooks
 		return nil
@@ -144,19 +175,20 @@ func (c *controller) reconcile(ctx context.Context, logger logr.Logger, key, _,
 	return err
 }
 
-func objectMeta(name string, owner ...metav1.OwnerReference) metav1.ObjectMeta {
+func objectMeta(name string, annotations map[string]string, owner ...metav1.OwnerReference) metav1.ObjectMeta {
 	return metav1.ObjectMeta{
 		Name: name,
 		Labels: map[string]string{
 			utils.ManagedByLabel: kyvernov1.ValueKyvernoApp,
 		},
+		Annotations:     annotations,
 		OwnerReferences: owner,
 	}
 }
 
-func (c *controller) build(caBundle []byte) (*admissionregistrationv1.ValidatingWebhookConfiguration, error) {
+func (c *controller) build(cfg config.Configuration, caBundle []byte) (*admissionregistrationv1.ValidatingWebhookConfiguration, error) {
 	return &admissionregistrationv1.ValidatingWebhookConfiguration{
-			ObjectMeta: objectMeta(c.webhookName),
+			ObjectMeta: objectMeta(c.webhookName, cfg.GetWebhookAnnotations()),
 			Webhooks: []admissionregistrationv1.ValidatingWebhook{{
 				Name:                    fmt.Sprintf("%s.%s.svc", config.KyvernoServiceName(), config.KyvernoNamespace()),
 				ClientConfig:            c.clientConfig(caBundle),

pkg/controllers/webhook/controller.go

@@ -361,12 +361,12 @@ func (c *controller) reconcileVerifyMutatingWebhookConfiguration(ctx context.Con
 	return c.reconcileMutatingWebhookConfiguration(ctx, true, c.buildVerifyMutatingWebhookConfiguration)
 }
 
-func (c *controller) reconcileValidatingWebhookConfiguration(ctx context.Context, autoUpdateWebhooks bool, build func([]byte) (*admissionregistrationv1.ValidatingWebhookConfiguration, error)) error {
+func (c *controller) reconcileValidatingWebhookConfiguration(ctx context.Context, autoUpdateWebhooks bool, build func(config.Configuration, []byte) (*admissionregistrationv1.ValidatingWebhookConfiguration, error)) error {
 	caData, err := tls.ReadRootCASecret(c.secretLister.Secrets(config.KyvernoNamespace()))
 	if err != nil {
 		return err
 	}
-	desired, err := build(caData)
+	desired, err := build(c.loadConfig(), caData)
 	if err != nil {
 		return err
 	}
@@ -383,6 +383,7 @@ func (c *controller) reconcileValidatingWebhookConfiguration(ctx context.Context
 	}
 	_, err = controllerutils.Update(ctx, observed, c.vwcClient, func(w *admissionregistrationv1.ValidatingWebhookConfiguration) error {
 		w.Labels = desired.Labels
+		w.Annotations = desired.Annotations
 		w.OwnerReferences = desired.OwnerReferences
 		w.Webhooks = desired.Webhooks
 		return nil
@@ -390,12 +391,12 @@ func (c *controller) reconcileValidatingWebhookConfiguration(ctx context.Context
 	return err
 }
 
-func (c *controller) reconcileMutatingWebhookConfiguration(ctx context.Context, autoUpdateWebhooks bool, build func([]byte) (*admissionregistrationv1.MutatingWebhookConfiguration, error)) error {
+func (c *controller) reconcileMutatingWebhookConfiguration(ctx context.Context, autoUpdateWebhooks bool, build func(config.Configuration, []byte) (*admissionregistrationv1.MutatingWebhookConfiguration, error)) error {
 	caData, err := tls.ReadRootCASecret(c.secretLister.Secrets(config.KyvernoNamespace()))
 	if err != nil {
 		return err
 	}
-	desired, err := build(caData)
+	desired, err := build(c.loadConfig(), caData)
 	if err != nil {
 		return err
 	}
@@ -412,6 +413,7 @@ func (c *controller) reconcileMutatingWebhookConfiguration(ctx context.Context,
 	}
 	_, err = controllerutils.Update(ctx, observed, c.mwcClient, func(w *admissionregistrationv1.MutatingWebhookConfiguration) error {
 		w.Labels = desired.Labels
+		w.Annotations = desired.Annotations
 		w.OwnerReferences = desired.OwnerReferences
 		w.Webhooks = desired.Webhooks
 		return nil
@@ -516,9 +518,9 @@ func (c *controller) reconcile(ctx context.Context, logger logr.Logger, key, nam
 	return nil
 }
 
-func (c *controller) buildVerifyMutatingWebhookConfiguration(caBundle []byte) (*admissionregistrationv1.MutatingWebhookConfiguration, error) {
+func (c *controller) buildVerifyMutatingWebhookConfiguration(cfg config.Configuration, caBundle []byte) (*admissionregistrationv1.MutatingWebhookConfiguration, error) {
 	return &admissionregistrationv1.MutatingWebhookConfiguration{
-			ObjectMeta: objectMeta(config.VerifyMutatingWebhookConfigurationName, c.buildOwner()...),
+			ObjectMeta: objectMeta(config.VerifyMutatingWebhookConfigurationName, cfg.GetWebhookAnnotations(), c.buildOwner()...),
 			Webhooks: []admissionregistrationv1.MutatingWebhook{{
 				Name:         config.VerifyMutatingWebhookName,
 				ClientConfig: c.clientConfig(caBundle, config.VerifyMutatingWebhookServicePath),
@@ -542,9 +544,9 @@ func (c *controller) buildVerifyMutatingWebhookConfiguration(caBundle []byte) (*
 		nil
 }
 
-func (c *controller) buildPolicyMutatingWebhookConfiguration(caBundle []byte) (*admissionregistrationv1.MutatingWebhookConfiguration, error) {
+func (c *controller) buildPolicyMutatingWebhookConfiguration(cfg config.Configuration, caBundle []byte) (*admissionregistrationv1.MutatingWebhookConfiguration, error) {
 	return &admissionregistrationv1.MutatingWebhookConfiguration{
-			ObjectMeta: objectMeta(config.PolicyMutatingWebhookConfigurationName, c.buildOwner()...),
+			ObjectMeta: objectMeta(config.PolicyMutatingWebhookConfigurationName, cfg.GetWebhookAnnotations(), c.buildOwner()...),
 			Webhooks: []admissionregistrationv1.MutatingWebhook{{
 				Name:         config.PolicyMutatingWebhookName,
 				ClientConfig: c.clientConfig(caBundle, config.PolicyMutatingWebhookServicePath),
@@ -564,9 +566,9 @@ func (c *controller) buildPolicyMutatingWebhookConfiguration(caBundle []byte) (*
 		nil
 }
 
-func (c *controller) buildPolicyValidatingWebhookConfiguration(caBundle []byte) (*admissionregistrationv1.ValidatingWebhookConfiguration, error) {
+func (c *controller) buildPolicyValidatingWebhookConfiguration(cfg config.Configuration, caBundle []byte) (*admissionregistrationv1.ValidatingWebhookConfiguration, error) {
 	return &admissionregistrationv1.ValidatingWebhookConfiguration{
-			ObjectMeta: objectMeta(config.PolicyValidatingWebhookConfigurationName, c.buildOwner()...),
+			ObjectMeta: objectMeta(config.PolicyValidatingWebhookConfigurationName, cfg.GetWebhookAnnotations(), c.buildOwner()...),
 			Webhooks: []admissionregistrationv1.ValidatingWebhook{{
 				Name:         config.PolicyValidatingWebhookName,
 				ClientConfig: c.clientConfig(caBundle, config.PolicyValidatingWebhookServicePath),
@@ -585,9 +587,9 @@ func (c *controller) buildPolicyValidatingWebhookConfiguration(caBundle []byte)
 		nil
 }
 
-func (c *controller) buildDefaultResourceMutatingWebhookConfiguration(caBundle []byte) (*admissionregistrationv1.MutatingWebhookConfiguration, error) {
+func (c *controller) buildDefaultResourceMutatingWebhookConfiguration(cfg config.Configuration, caBundle []byte) (*admissionregistrationv1.MutatingWebhookConfiguration, error) {
 	return &admissionregistrationv1.MutatingWebhookConfiguration{
-			ObjectMeta: objectMeta(config.MutatingWebhookConfigurationName, c.buildOwner()...),
+			ObjectMeta: objectMeta(config.MutatingWebhookConfigurationName, cfg.GetWebhookAnnotations(), c.buildOwner()...),
 			Webhooks: []admissionregistrationv1.MutatingWebhook{{
 				Name:         config.MutatingWebhookName + "-ignore",
 				ClientConfig: c.clientConfig(caBundle, config.MutatingWebhookServicePath+"/ignore"),
@@ -612,9 +614,9 @@ func (c *controller) buildDefaultResourceMutatingWebhookConfiguration(caBundle [
 		nil
 }
 
-func (c *controller) buildResourceMutatingWebhookConfiguration(caBundle []byte) (*admissionregistrationv1.MutatingWebhookConfiguration, error) {
+func (c *controller) buildResourceMutatingWebhookConfiguration(cfg config.Configuration, caBundle []byte) (*admissionregistrationv1.MutatingWebhookConfiguration, error) {
 	result := admissionregistrationv1.MutatingWebhookConfiguration{
-		ObjectMeta: objectMeta(config.MutatingWebhookConfigurationName, c.buildOwner()...),
+		ObjectMeta: objectMeta(config.MutatingWebhookConfigurationName, cfg.GetWebhookAnnotations(), c.buildOwner()...),
 		Webhooks:   []admissionregistrationv1.MutatingWebhook{},
 	}
 	if c.watchdogCheck() {
@@ -641,7 +643,6 @@ func (c *controller) buildResourceMutatingWebhookConfiguration(caBundle []byte)
 				}
 			}
 		}
-		cfg := c.loadConfig()
 		webhookCfg := config.WebhookConfig{}
 		webhookCfgs := cfg.GetWebhooks()
 		if len(webhookCfgs) > 0 {
@@ -687,13 +688,13 @@ func (c *controller) buildResourceMutatingWebhookConfiguration(caBundle []byte)
 	return &result, nil
 }
 
-func (c *controller) buildDefaultResourceValidatingWebhookConfiguration(caBundle []byte) (*admissionregistrationv1.ValidatingWebhookConfiguration, error) {
+func (c *controller) buildDefaultResourceValidatingWebhookConfiguration(cfg config.Configuration, caBundle []byte) (*admissionregistrationv1.ValidatingWebhookConfiguration, error) {
 	sideEffects := &none
 	if c.admissionReports {
 		sideEffects = &noneOnDryRun
 	}
 	return &admissionregistrationv1.ValidatingWebhookConfiguration{
-			ObjectMeta: objectMeta(config.ValidatingWebhookConfigurationName, c.buildOwner()...),
+			ObjectMeta: objectMeta(config.ValidatingWebhookConfigurationName, cfg.GetWebhookAnnotations(), c.buildOwner()...),
 			Webhooks: []admissionregistrationv1.ValidatingWebhook{{
 				Name:         config.ValidatingWebhookName + "-ignore",
 				ClientConfig: c.clientConfig(caBundle, config.ValidatingWebhookServicePath+"/ignore"),
@@ -719,9 +720,9 @@ func (c *controller) buildDefaultResourceValidatingWebhookConfiguration(caBundle
 		nil
 }
 
-func (c *controller) buildResourceValidatingWebhookConfiguration(caBundle []byte) (*admissionregistrationv1.ValidatingWebhookConfiguration, error) {
+func (c *controller) buildResourceValidatingWebhookConfiguration(cfg config.Configuration, caBundle []byte) (*admissionregistrationv1.ValidatingWebhookConfiguration, error) {
 	result := admissionregistrationv1.ValidatingWebhookConfiguration{
-		ObjectMeta: objectMeta(config.ValidatingWebhookConfigurationName, c.buildOwner()...),
+		ObjectMeta: objectMeta(config.ValidatingWebhookConfigurationName, cfg.GetWebhookAnnotations(), c.buildOwner()...),
 		Webhooks:   []admissionregistrationv1.ValidatingWebhook{},
 	}
 	if c.watchdogCheck() {
@@ -748,7 +749,6 @@ func (c *controller) buildResourceValidatingWebhookConfiguration(caBundle []byte
 				}
 			}
 		}
-		cfg := c.loadConfig()
 		webhookCfg := config.WebhookConfig{}
 		webhookCfgs := cfg.GetWebhooks()
 		if len(webhookCfgs) > 0 {

pkg/controllers/webhook/utils.go

@@ -98,12 +98,13 @@ func hasWildcard(policies ...kyvernov1.PolicyInterface) bool {
 	return false
 }
 
-func objectMeta(name string, owner ...metav1.OwnerReference) metav1.ObjectMeta {
+func objectMeta(name string, annotations map[string]string, owner ...metav1.OwnerReference) metav1.ObjectMeta {
 	return metav1.ObjectMeta{
 		Name: name,
 		Labels: map[string]string{
 			utils.ManagedByLabel: kyvernov1.ValueKyvernoApp,
 		},
+		Annotations:     annotations,
 		OwnerReferences: owner,
 	}
 }