diff --git a/charts/kyverno/README.md b/charts/kyverno/README.md index aaafcb8a19..adf86c00d0 100644 --- a/charts/kyverno/README.md +++ b/charts/kyverno/README.md @@ -702,6 +702,7 @@ The chart values are organised per component. | Key | Type | Default | Description | |-----|------|---------|-------------| | webhooksCleanup.enabled | bool | `true` | Create a helm pre-delete hook to cleanup webhooks. | +| webhooksCleanup.autoDeleteWebhooks.enabled | bool | `false` | Allow webhooks controller to delete webhooks using finalizers | | webhooksCleanup.image.registry | string | `nil` | Image registry | | webhooksCleanup.image.repository | string | `"bitnami/kubectl"` | Image repository | | webhooksCleanup.image.tag | string | `"1.30.2"` | Image tag Defaults to `latest` if omitted | diff --git a/charts/kyverno/templates/admission-controller/clusterrole.yaml b/charts/kyverno/templates/admission-controller/clusterrole.yaml index c6e8e7e0fd..faf876ab56 100644 --- a/charts/kyverno/templates/admission-controller/clusterrole.yaml +++ b/charts/kyverno/templates/admission-controller/clusterrole.yaml @@ -16,6 +16,14 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: {{ template "kyverno.admission-controller.roleName" . }}:core + {{- if .Values.webhooksCleanup.autoDeleteWebhooks.enabled }} + {{- if not .Values.templating.enabled }} + finalizers: + - kyverno.io/webhooks + - kyverno.io/exceptionwebhooks + - kyverno.io/globalcontextwebhooks + {{- end }} + {{- end }} labels: {{- include "kyverno.admission-controller.labels" . | nindent 4 }} rules: @@ -139,6 +147,31 @@ rules: - get - list - watch + {{- if .Values.webhooksCleanup.autoDeleteWebhooks.enabled }} + {{- if not .Values.templating.enabled }} + - apiGroups: + - rbac.authorization.k8s.io + resources: + - clusterroles + - clusterrolebindings + resourceNames: + - {{ template "kyverno.admission-controller.roleName" . }} + - {{ template "kyverno.admission-controller.roleName" . }}:core + - {{ template "kyverno.admission-controller.roleName" . }}:temporary + verbs: + - get + - patch + - update + - apiGroups: + - rbac.authorization.k8s.io + resources: + - clusterroles + - clusterrolebindings + verbs: + - create + - list + {{- end }} + {{- end }} {{- with .Values.admissionController.rbac.coreClusterRole.extraResources }} {{- toYaml . | nindent 2 }} {{- end }} @@ -153,4 +186,4 @@ metadata: rules: {{- toYaml . | nindent 2 }} {{- end }} -{{- end }} \ No newline at end of file +{{- end }} diff --git a/charts/kyverno/templates/admission-controller/deployment.yaml b/charts/kyverno/templates/admission-controller/deployment.yaml index 8ca7daeb36..610e039eca 100644 --- a/charts/kyverno/templates/admission-controller/deployment.yaml +++ b/charts/kyverno/templates/admission-controller/deployment.yaml @@ -4,6 +4,14 @@ kind: Deployment metadata: name: {{ template "kyverno.admission-controller.name" . }} namespace: {{ template "kyverno.namespace" . }} + {{- if .Values.webhooksCleanup.autoDeleteWebhooks.enabled }} + {{- if not .Values.templating.enabled }} + finalizers: + - kyverno.io/webhooks + - kyverno.io/exceptionwebhooks + - kyverno.io/globalcontextwebhooks + {{- end }} + {{- end }} labels: {{- include "kyverno.admission-controller.labels" . | nindent 4 }} {{- with .Values.admissionController.annotations }} @@ -105,6 +113,8 @@ spec: env: - name: KYVERNO_SERVICEACCOUNT_NAME value: {{ template "kyverno.admission-controller.serviceAccountName" . }} + - name: KYVERNO_ROLE_NAME + value: {{ template "kyverno.admission-controller.roleName" . }} - name: INIT_CONFIG value: {{ template "kyverno.config.configMapName" . }} - name: METRICS_CONFIG @@ -138,6 +148,9 @@ spec: - --reportsServiceAccountName=system:serviceaccount:{{ include "kyverno.namespace" . }}:{{ include "kyverno.reports-controller.serviceAccountName" . }} - --servicePort={{ .Values.admissionController.service.port }} - --webhookServerPort={{ .Values.admissionController.webhookServer.port }} + {{- if .Values.webhooksCleanup.autoDeleteWebhooks.enabled }} + - --autoDeleteWebhooks + {{- end }} {{- if .Values.admissionController.tracing.enabled }} - --enableTracing - --tracingAddress={{ .Values.admissionController.tracing.address }} @@ -220,6 +233,8 @@ spec: fieldPath: metadata.name - name: KYVERNO_SERVICEACCOUNT_NAME value: {{ template "kyverno.admission-controller.serviceAccountName" . }} + - name: KYVERNO_ROLE_NAME + value: {{ template "kyverno.admission-controller.roleName" . }} - name: KYVERNO_SVC value: {{ template "kyverno.admission-controller.serviceName" . }} - name: TUF_ROOT diff --git a/charts/kyverno/templates/admission-controller/role.yaml b/charts/kyverno/templates/admission-controller/role.yaml index b5d621786c..c24a6ea544 100644 --- a/charts/kyverno/templates/admission-controller/role.yaml +++ b/charts/kyverno/templates/admission-controller/role.yaml @@ -4,6 +4,14 @@ kind: Role metadata: name: {{ template "kyverno.admission-controller.roleName" . }} namespace: {{ template "kyverno.namespace" . }} + {{- if .Values.webhooksCleanup.autoDeleteWebhooks.enabled }} + {{- if not .Values.templating.enabled }} + finalizers: + - kyverno.io/webhooks + - kyverno.io/exceptionwebhooks + - kyverno.io/globalcontextwebhooks + {{- end }} + {{- end }} labels: {{- include "kyverno.admission-controller.labels" . | nindent 4 }} rules: @@ -11,10 +19,12 @@ rules: - '' resources: - secrets + - serviceaccounts verbs: - get - list - watch + - patch - create - update - delete @@ -39,6 +49,29 @@ rules: - get - patch - update + {{- if .Values.webhooksCleanup.autoDeleteWebhooks.enabled }} + {{- if not .Values.templating.enabled }} + - apiGroups: + - rbac.authorization.k8s.io + resources: + - roles + - rolebindings + resourceNames: + - {{ template "kyverno.admission-controller.roleName" . }} + - {{ template "kyverno.admission-controller.roleName" . }}:temporary + verbs: + - get + - patch + - update + - apiGroups: + - rbac.authorization.k8s.io + resources: + - roles + - rolebindings + verbs: + - create + {{- end }} + {{- end }} # Allow update of Kyverno deployment annotations - apiGroups: - apps diff --git a/charts/kyverno/templates/admission-controller/rolebinding.yaml b/charts/kyverno/templates/admission-controller/rolebinding.yaml index b2045b17b6..e54046c78d 100644 --- a/charts/kyverno/templates/admission-controller/rolebinding.yaml +++ b/charts/kyverno/templates/admission-controller/rolebinding.yaml @@ -4,6 +4,14 @@ apiVersion: rbac.authorization.k8s.io/v1 metadata: name: {{ template "kyverno.admission-controller.roleName" . }} namespace: {{ template "kyverno.namespace" . }} + {{- if .Values.webhooksCleanup.autoDeleteWebhooks.enabled }} + {{- if not .Values.templating.enabled }} + finalizers: + - kyverno.io/webhooks + - kyverno.io/exceptionwebhooks + - kyverno.io/globalcontextwebhooks + {{- end }} + {{- end }} labels: {{- include "kyverno.admission-controller.labels" . | nindent 4 }} roleRef: diff --git a/charts/kyverno/templates/admission-controller/serviceaccount.yaml b/charts/kyverno/templates/admission-controller/serviceaccount.yaml index e78f6bff4a..e6fe87ce7f 100644 --- a/charts/kyverno/templates/admission-controller/serviceaccount.yaml +++ b/charts/kyverno/templates/admission-controller/serviceaccount.yaml @@ -4,6 +4,14 @@ kind: ServiceAccount metadata: name: {{ template "kyverno.admission-controller.serviceAccountName" . }} namespace: {{ template "kyverno.namespace" . }} + {{- if .Values.webhooksCleanup.autoDeleteWebhooks.enabled }} + {{- if not .Values.templating.enabled }} + finalizers: + - kyverno.io/webhooks + - kyverno.io/exceptionwebhooks + - kyverno.io/globalcontextwebhooks + {{- end }} + {{- end }} labels: {{- include "kyverno.admission-controller.labels" . | nindent 4 }} {{- with .Values.admissionController.rbac.serviceAccount.annotations }} diff --git a/charts/kyverno/templates/cleanup-controller/clusterrole.yaml b/charts/kyverno/templates/cleanup-controller/clusterrole.yaml index 68a9d96a62..8f1dd0ce17 100644 --- a/charts/kyverno/templates/cleanup-controller/clusterrole.yaml +++ b/charts/kyverno/templates/cleanup-controller/clusterrole.yaml @@ -17,6 +17,13 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: {{ template "kyverno.cleanup-controller.roleName" . }}:core + {{- if .Values.webhooksCleanup.autoDeleteWebhooks.enabled }} + {{- if not .Values.templating.enabled }} + finalizers: + - kyverno.io/policywebhooks + - kyverno.io/ttlwebhooks + {{- end }} + {{- end }} labels: {{- include "kyverno.cleanup-controller.labels" . | nindent 4 }} rules: @@ -97,6 +104,31 @@ rules: - subjectaccessreviews verbs: - create + {{- if .Values.webhooksCleanup.autoDeleteWebhooks.enabled }} + {{- if not .Values.templating.enabled }} + - apiGroups: + - rbac.authorization.k8s.io + resources: + - clusterroles + - clusterrolebindings + resourceNames: + - {{ template "kyverno.cleanup-controller.roleName" . }} + - {{ template "kyverno.cleanup-controller.roleName" . }}:core + - {{ template "kyverno.cleanup-controller.roleName" . }}:temporary + verbs: + - get + - patch + - update + - apiGroups: + - rbac.authorization.k8s.io + resources: + - clusterroles + - clusterrolebindings + verbs: + - create + - list + {{- end }} + {{- end }} {{- with .Values.cleanupController.rbac.clusterRole.extraResources }} --- apiVersion: rbac.authorization.k8s.io/v1 @@ -109,4 +141,4 @@ rules: {{- toYaml . | nindent 2 }} {{- end }} {{- end }} -{{- end }} \ No newline at end of file +{{- end }} diff --git a/charts/kyverno/templates/cleanup-controller/deployment.yaml b/charts/kyverno/templates/cleanup-controller/deployment.yaml index a957fe602a..5894a7f25e 100644 --- a/charts/kyverno/templates/cleanup-controller/deployment.yaml +++ b/charts/kyverno/templates/cleanup-controller/deployment.yaml @@ -5,6 +5,13 @@ kind: Deployment metadata: name: {{ template "kyverno.cleanup-controller.name" . }} namespace: {{ template "kyverno.namespace" . }} + {{- if .Values.webhooksCleanup.autoDeleteWebhooks.enabled }} + {{- if not .Values.templating.enabled }} + finalizers: + - kyverno.io/policywebhooks + - kyverno.io/ttlwebhooks + {{- end }} + {{- end }} labels: {{- include "kyverno.cleanup-controller.labels" . | nindent 4 }} {{- with .Values.cleanupController.annotations }} @@ -101,6 +108,9 @@ spec: - --servicePort={{ .Values.cleanupController.service.port }} - --cleanupServerPort={{ .Values.cleanupController.server.port }} - --webhookServerPort={{ .Values.cleanupController.webhookServer.port }} + {{- if .Values.webhooksCleanup.autoDeleteWebhooks.enabled }} + - --autoDeleteWebhooks + {{- end }} {{- if .Values.cleanupController.tracing.enabled }} - --enableTracing - --tracingAddress={{ .Values.cleanupController.tracing.address }} @@ -150,6 +160,8 @@ spec: fieldPath: metadata.name - name: KYVERNO_SERVICEACCOUNT_NAME value: {{ template "kyverno.cleanup-controller.serviceAccountName" . }} + - name: KYVERNO_ROLE_NAME + value: {{ template "kyverno.cleanup-controller.roleName" . }} - name: KYVERNO_NAMESPACE valueFrom: fieldRef: diff --git a/charts/kyverno/templates/cleanup-controller/role.yaml b/charts/kyverno/templates/cleanup-controller/role.yaml index 82db80d7dd..7aebf849ac 100644 --- a/charts/kyverno/templates/cleanup-controller/role.yaml +++ b/charts/kyverno/templates/cleanup-controller/role.yaml @@ -4,6 +4,13 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: {{ template "kyverno.cleanup-controller.roleName" . }} + {{- if .Values.webhooksCleanup.autoDeleteWebhooks.enabled }} + {{- if not .Values.templating.enabled }} + finalizers: + - kyverno.io/policywebhooks + - kyverno.io/ttlwebhooks + {{- end }} + {{- end }} labels: {{- include "kyverno.cleanup-controller.labels" . | nindent 4 }} namespace: {{ template "kyverno.namespace" . }} @@ -27,6 +34,22 @@ rules: resourceNames: - {{ template "kyverno.cleanup-controller.name" . }}.{{ template "kyverno.namespace" . }}.svc.kyverno-tls-ca - {{ template "kyverno.cleanup-controller.name" . }}.{{ template "kyverno.namespace" . }}.svc.kyverno-tls-pair + {{- if .Values.webhooksCleanup.autoDeleteWebhooks.enabled }} + {{- if not .Values.templating.enabled }} + - apiGroups: + - '' + resources: + - serviceaccounts + verbs: + - delete + - get + - list + - update + - watch + resourceNames: + - {{ template "kyverno.cleanup-controller.serviceAccountName" . }} + {{- end }} + {{- end }} - apiGroups: - '' resources: @@ -55,5 +78,42 @@ rules: - update resourceNames: - kyverno-cleanup-controller + {{- if .Values.webhooksCleanup.autoDeleteWebhooks.enabled }} + {{- if not .Values.templating.enabled }} + - apiGroups: + - rbac.authorization.k8s.io + resources: + - roles + - rolebindings + resourceNames: + - {{ template "kyverno.cleanup-controller.roleName" . }} + - {{ template "kyverno.cleanup-controller.roleName" . }}:temporary + verbs: + - get + - patch + - update + - apiGroups: + - rbac.authorization.k8s.io + resources: + - roles + - rolebindings + verbs: + - create + {{- end }} + {{- end }} + - apiGroups: + - apps + resources: + - deployments + verbs: + - get + - list + - watch + {{- if .Values.webhooksCleanup.autoDeleteWebhooks.enabled }} + {{- if not .Values.templating.enabled }} + - patch + - update + {{- end }} + {{- end }} {{- end -}} {{- end -}} diff --git a/charts/kyverno/templates/cleanup-controller/rolebinding.yaml b/charts/kyverno/templates/cleanup-controller/rolebinding.yaml index 2096f16238..8ccec904bd 100644 --- a/charts/kyverno/templates/cleanup-controller/rolebinding.yaml +++ b/charts/kyverno/templates/cleanup-controller/rolebinding.yaml @@ -4,6 +4,13 @@ kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: {{ template "kyverno.cleanup-controller.roleName" . }} + {{- if .Values.webhooksCleanup.autoDeleteWebhooks.enabled }} + {{- if not .Values.templating.enabled }} + finalizers: + - kyverno.io/policywebhooks + - kyverno.io/ttlwebhooks + {{- end }} + {{- end }} labels: {{- include "kyverno.cleanup-controller.labels" . | nindent 4 }} namespace: {{ template "kyverno.namespace" . }} diff --git a/charts/kyverno/templates/cleanup-controller/serviceaccount.yaml b/charts/kyverno/templates/cleanup-controller/serviceaccount.yaml index a20a186446..d06987029f 100644 --- a/charts/kyverno/templates/cleanup-controller/serviceaccount.yaml +++ b/charts/kyverno/templates/cleanup-controller/serviceaccount.yaml @@ -5,6 +5,13 @@ kind: ServiceAccount metadata: name: {{ template "kyverno.cleanup-controller.serviceAccountName" . }} namespace: {{ template "kyverno.namespace" . }} + {{- if .Values.webhooksCleanup.autoDeleteWebhooks.enabled }} + {{- if not .Values.templating.enabled }} + finalizers: + - kyverno.io/policywebhooks + - kyverno.io/ttlwebhooks + {{- end }} + {{- end }} labels: {{- include "kyverno.cleanup-controller.labels" . | nindent 4 }} {{- with .Values.cleanupController.rbac.serviceAccount.annotations }} diff --git a/charts/kyverno/values.yaml b/charts/kyverno/values.yaml index 2632a51d05..3f299d1148 100644 --- a/charts/kyverno/values.yaml +++ b/charts/kyverno/values.yaml @@ -469,6 +469,10 @@ webhooksCleanup: # -- Create a helm pre-delete hook to cleanup webhooks. enabled: true + autoDeleteWebhooks: + # -- Allow webhooks controller to delete webhooks using finalizers + enabled: false + image: # -- (string) Image registry registry: ~ diff --git a/cmd/cleanup-controller/main.go b/cmd/cleanup-controller/main.go index 661e28ba99..209a397410 100644 --- a/cmd/cleanup-controller/main.go +++ b/cmd/cleanup-controller/main.go @@ -21,6 +21,7 @@ import ( genericwebhookcontroller "github.com/kyverno/kyverno/pkg/controllers/generic/webhook" globalcontextcontroller "github.com/kyverno/kyverno/pkg/controllers/globalcontext" ttlcontroller "github.com/kyverno/kyverno/pkg/controllers/ttl" + webhookcontroller "github.com/kyverno/kyverno/pkg/controllers/webhook" "github.com/kyverno/kyverno/pkg/event" "github.com/kyverno/kyverno/pkg/globalcontext/store" "github.com/kyverno/kyverno/pkg/informers" @@ -29,6 +30,7 @@ import ( "github.com/kyverno/kyverno/pkg/tls" "github.com/kyverno/kyverno/pkg/toggle" kubeutils "github.com/kyverno/kyverno/pkg/utils/kube" + runtimeutils "github.com/kyverno/kyverno/pkg/utils/runtime" "github.com/kyverno/kyverno/pkg/webhooks" admissionregistrationv1 "k8s.io/api/admissionregistration/v1" corev1 "k8s.io/api/core/v1" @@ -38,10 +40,12 @@ import ( ) const ( - resyncPeriod = 15 * time.Minute - webhookWorkers = 2 - policyWebhookControllerName = "policy-webhook-controller" - ttlWebhookControllerName = "ttl-webhook-controller" + resyncPeriod = 15 * time.Minute + webhookWorkers = 2 + policyWebhookControllerName = "policy-webhook-controller" + ttlWebhookControllerName = "ttl-webhook-controller" + policyWebhookControllerFinalizerName = "kyverno.io/policywebhooks" + ttlWebhookControllerFinalizerName = "kyverno.io/ttlwebhooks" ) var ( @@ -78,6 +82,7 @@ func main() { interval time.Duration renewBefore time.Duration maxAPICallResponseLength int64 + autoDeleteWebhooks bool ) flagset := flag.NewFlagSet("cleanup-controller", flag.ExitOnError) flagset.BoolVar(&dumpPayload, "dumpPayload", false, "Set this flag to activate/deactivate debug mode.") @@ -91,6 +96,7 @@ func main() { flagset.StringVar(&tlsSecretName, "tlsSecretName", "", "Name of the secret containing TLS pair.") flagset.DurationVar(&renewBefore, "renewBefore", 15*24*time.Hour, "The certificate renewal time before expiration") flagset.Int64Var(&maxAPICallResponseLength, "maxAPICallResponseLength", 2*1000*1000, "Maximum allowed response size from API Calls. A value of 0 bypasses checks (not recommended).") + flagset.BoolVar(&autoDeleteWebhooks, "autoDeleteWebhooks", false, "Set this flag to 'true' to enable autodeletion of webhook configurations using finalizers (requires extra permissions).") // config appConfig := internal.NewConfiguration( internal.WithProfiling(), @@ -129,7 +135,8 @@ func main() { // certificates informers caSecret := informers.NewSecretInformer(setup.KubeClient, config.KyvernoNamespace(), caSecretName, resyncPeriod) tlsSecret := informers.NewSecretInformer(setup.KubeClient, config.KyvernoNamespace(), tlsSecretName, resyncPeriod) - if !informers.StartInformersAndWaitForCacheSync(ctx, setup.Logger, caSecret, tlsSecret) { + kyvernoDeployment := informers.NewDeploymentInformer(setup.KubeClient, config.KyvernoNamespace(), config.KyvernoDeploymentName(), resyncPeriod) + if !informers.StartInformersAndWaitForCacheSync(ctx, setup.Logger, caSecret, tlsSecret, kyvernoDeployment) { setup.Logger.Error(errors.New("failed to wait for cache sync"), "failed to wait for cache sync") os.Exit(1) } @@ -180,6 +187,12 @@ func main() { if !internal.StartInformersAndWaitForCacheSync(ctx, setup.Logger, kubeInformer, kyvernoInformer) { os.Exit(1) } + runtime := runtimeutils.NewRuntime( + setup.Logger.WithName("runtime-checks"), + serverIP, + kyvernoDeployment, + nil, + ) // setup leader election le, err := leaderelection.New( setup.Logger.WithName("leader-election"), @@ -229,6 +242,7 @@ func main() { setup.KubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations(), kubeInformer.Admissionregistration().V1().ValidatingWebhookConfigurations(), caSecret, + kyvernoDeployment, config.CleanupValidatingWebhookConfigurationName, config.CleanupValidatingWebhookServicePath, serverIP, @@ -255,6 +269,10 @@ func main() { genericwebhookcontroller.None, setup.Configuration, caSecretName, + runtime, + autoDeleteWebhooks, + webhookcontroller.WebhookCleanupSetup(setup.KubeClient, policyWebhookControllerFinalizerName), + webhookcontroller.WebhookCleanupHandler(setup.KubeClient, policyWebhookControllerFinalizerName), ), webhookWorkers, ) @@ -265,6 +283,7 @@ func main() { setup.KubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations(), kubeInformer.Admissionregistration().V1().ValidatingWebhookConfigurations(), caSecret, + kyvernoDeployment, config.TtlValidatingWebhookConfigurationName, config.TtlValidatingWebhookServicePath, serverIP, @@ -295,6 +314,10 @@ func main() { genericwebhookcontroller.None, setup.Configuration, caSecretName, + runtime, + autoDeleteWebhooks, + webhookcontroller.WebhookCleanupSetup(setup.KubeClient, ttlWebhookControllerFinalizerName), + webhookcontroller.WebhookCleanupHandler(setup.KubeClient, ttlWebhookControllerFinalizerName), ), webhookWorkers, ) diff --git a/cmd/kyverno/main.go b/cmd/kyverno/main.go index bb9ca85dae..ed2a9ac657 100644 --- a/cmd/kyverno/main.go +++ b/cmd/kyverno/main.go @@ -51,15 +51,19 @@ import ( corev1 "k8s.io/api/core/v1" apiserver "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" kubeinformers "k8s.io/client-go/informers" + appsv1informers "k8s.io/client-go/informers/apps/v1" corev1informers "k8s.io/client-go/informers/core/v1" "k8s.io/client-go/kubernetes" kyamlopenapi "sigs.k8s.io/kustomize/kyaml/openapi" ) const ( - resyncPeriod = 15 * time.Minute - exceptionWebhookControllerName = "exception-webhook-controller" - gctxWebhookControllerName = "global-context-webhook-controller" + resyncPeriod = 15 * time.Minute + exceptionWebhookControllerName = "exception-webhook-controller" + gctxWebhookControllerName = "global-context-webhook-controller" + webhookControllerFinalizerName = "kyverno.io/webhooks" + exceptionControllerFinalizerName = "kyverno.io/exceptionwebhooks" + gctxControllerFinalizerName = "kyverno.io/globalcontextwebhooks" ) var ( @@ -107,11 +111,13 @@ func createrLeaderControllers( serverIP string, webhookTimeout int, autoUpdateWebhooks bool, + autoDeleteWebhooks bool, kubeInformer kubeinformers.SharedInformerFactory, kubeKyvernoInformer kubeinformers.SharedInformerFactory, kyvernoInformer kyvernoinformer.SharedInformerFactory, caInformer corev1informers.SecretInformer, tlsInformer corev1informers.SecretInformer, + deploymentInformer appsv1informers.DeploymentInformer, kubeClient kubernetes.Interface, kyvernoClient versioned.Interface, dynamicClient dclient.Interface, @@ -141,6 +147,7 @@ func createrLeaderControllers( kubeInformer.Admissionregistration().V1().ValidatingWebhookConfigurations(), kyvernoInformer.Kyverno().V1().ClusterPolicies(), kyvernoInformer.Kyverno().V1().Policies(), + deploymentInformer, caInformer, kubeKyvernoInformer.Coordination().V1().Leases(), kubeInformer.Rbac().V1().ClusterRoles(), @@ -150,16 +157,20 @@ func createrLeaderControllers( servicePort, webhookServerPort, autoUpdateWebhooks, + autoDeleteWebhooks, admissionReports, runtime, configuration, caSecretName, + webhookcontroller.WebhookCleanupSetup(kubeClient, webhookControllerFinalizerName), + webhookcontroller.WebhookCleanupHandler(kubeClient, webhookControllerFinalizerName), ) exceptionWebhookController := genericwebhookcontroller.NewController( exceptionWebhookControllerName, kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations(), kubeInformer.Admissionregistration().V1().ValidatingWebhookConfigurations(), caInformer, + deploymentInformer, config.ExceptionValidatingWebhookConfigurationName, config.ExceptionValidatingWebhookServicePath, serverIP, @@ -181,12 +192,17 @@ func createrLeaderControllers( genericwebhookcontroller.None, configuration, caSecretName, + runtime, + autoDeleteWebhooks, + webhookcontroller.WebhookCleanupSetup(kubeClient, exceptionControllerFinalizerName), + webhookcontroller.WebhookCleanupHandler(kubeClient, exceptionControllerFinalizerName), ) gctxWebhookController := genericwebhookcontroller.NewController( gctxWebhookControllerName, kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations(), kubeInformer.Admissionregistration().V1().ValidatingWebhookConfigurations(), caInformer, + deploymentInformer, config.GlobalContextValidatingWebhookConfigurationName, config.GlobalContextValidatingWebhookServicePath, serverIP, @@ -208,6 +224,10 @@ func createrLeaderControllers( genericwebhookcontroller.None, configuration, caSecretName, + runtime, + autoDeleteWebhooks, + webhookcontroller.WebhookCleanupSetup(kubeClient, gctxControllerFinalizerName), + webhookcontroller.WebhookCleanupHandler(kubeClient, gctxControllerFinalizerName), ) leaderControllers = append(leaderControllers, internal.NewController(certmanager.ControllerName, certManager, certmanager.Workers)) leaderControllers = append(leaderControllers, internal.NewController(webhookcontroller.ControllerName, webhookController, webhookcontroller.Workers)) @@ -241,6 +261,7 @@ func main() { maxQueuedEvents int omitEvents string autoUpdateWebhooks bool + autoDeleteWebhooks bool webhookRegistrationTimeout time.Duration admissionReports bool dumpPayload bool @@ -261,6 +282,7 @@ func main() { flagset.StringVar(&omitEvents, "omitEvents", "", "Set this flag to a comma sperated list of PolicyViolation, PolicyApplied, PolicyError, PolicySkipped to disable events, e.g. --omitEvents=PolicyApplied,PolicyViolation") flagset.StringVar(&serverIP, "serverIP", "", "IP address where Kyverno controller runs. Only required if out-of-cluster.") flagset.BoolVar(&autoUpdateWebhooks, "autoUpdateWebhooks", true, "Set this flag to 'false' to disable auto-configuration of the webhook.") + flagset.BoolVar(&autoDeleteWebhooks, "autoDeleteWebhooks", false, "Set this flag to 'true' to enable autodeletion of webhook configurations using finalizers (requires extra permissions).") flagset.DurationVar(&webhookRegistrationTimeout, "webhookRegistrationTimeout", 120*time.Second, "Timeout for webhook registration, e.g., 30s, 1m, 5m.") flagset.Func(toggle.ProtectManagedResourcesFlagName, toggle.ProtectManagedResourcesDescription, toggle.ProtectManagedResources.Parse) flagset.Func(toggle.ForceFailurePolicyIgnoreFlagName, toggle.ForceFailurePolicyIgnoreDescription, toggle.ForceFailurePolicyIgnore.Parse) @@ -324,7 +346,8 @@ func main() { } caSecret := informers.NewSecretInformer(setup.KubeClient, config.KyvernoNamespace(), caSecretName, resyncPeriod) tlsSecret := informers.NewSecretInformer(setup.KubeClient, config.KyvernoNamespace(), tlsSecretName, resyncPeriod) - if !informers.StartInformersAndWaitForCacheSync(signalCtx, setup.Logger, caSecret, tlsSecret) { + kyvernoDeployment := informers.NewDeploymentInformer(setup.KubeClient, config.KyvernoNamespace(), config.KyvernoDeploymentName(), resyncPeriod) + if !informers.StartInformersAndWaitForCacheSync(signalCtx, setup.Logger, caSecret, tlsSecret, kyvernoDeployment) { setup.Logger.Error(errors.New("failed to wait for cache sync"), "failed to wait for cache sync") os.Exit(1) } @@ -342,6 +365,7 @@ func main() { kubeInformer := kubeinformers.NewSharedInformerFactory(setup.KubeClient, resyncPeriod) kubeKyvernoInformer := kubeinformers.NewSharedInformerFactoryWithOptions(setup.KubeClient, resyncPeriod, kubeinformers.WithNamespace(config.KyvernoNamespace())) kyvernoInformer := kyvernoinformer.NewSharedInformerFactory(setup.KyvernoClient, resyncPeriod) + certRenewer := tls.NewCertRenewer( setup.KubeClient.CoreV1().Secrets(config.KyvernoNamespace()), tls.CertRenewalInterval, @@ -463,11 +487,13 @@ func main() { serverIP, webhookTimeout, autoUpdateWebhooks, + autoDeleteWebhooks, kubeInformer, kubeKyvernoInformer, kyvernoInformer, caSecret, tlsSecret, + kyvernoDeployment, setup.KubeClient, setup.KyvernoClient, setup.KyvernoDynamicClient, diff --git a/config/install-latest-testing.yaml b/config/install-latest-testing.yaml index f59074d99d..4318cea955 100644 --- a/config/install-latest-testing.yaml +++ b/config/install-latest-testing.yaml @@ -48730,10 +48730,12 @@ rules: - '' resources: - secrets + - serviceaccounts verbs: - get - list - watch + - patch - create - update - delete @@ -48874,6 +48876,14 @@ rules: - update resourceNames: - kyverno-cleanup-controller + - apiGroups: + - apps + resources: + - deployments + verbs: + - get + - list + - watch --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role @@ -49205,6 +49215,8 @@ spec: env: - name: KYVERNO_SERVICEACCOUNT_NAME value: kyverno-admission-controller + - name: KYVERNO_ROLE_NAME + value: kyverno:admission-controller - name: INIT_CONFIG value: kyverno - name: METRICS_CONFIG @@ -49291,6 +49303,8 @@ spec: fieldPath: metadata.name - name: KYVERNO_SERVICEACCOUNT_NAME value: kyverno-admission-controller + - name: KYVERNO_ROLE_NAME + value: kyverno:admission-controller - name: KYVERNO_SVC value: kyverno-svc - name: TUF_ROOT @@ -49522,6 +49536,8 @@ spec: fieldPath: metadata.name - name: KYVERNO_SERVICEACCOUNT_NAME value: kyverno-cleanup-controller + - name: KYVERNO_ROLE_NAME + value: kyverno:cleanup-controller - name: KYVERNO_NAMESPACE valueFrom: fieldRef: diff --git a/pkg/config/config.go b/pkg/config/config.go index c30bd39096..5df2258b25 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -106,6 +106,8 @@ var ( kyvernoNamespace = osutils.GetEnvWithFallback("KYVERNO_NAMESPACE", "kyverno") // kyvernoServiceAccountName is the Kyverno service account name kyvernoServiceAccountName = osutils.GetEnvWithFallback("KYVERNO_SERVICEACCOUNT_NAME", "kyverno") + // kyvernoRoleName is the Kyverno rbac name + kyvernoRoleName = osutils.GetEnvWithFallback("KYVERNO_ROLE_NAME", "kyverno") // kyvernoDeploymentName is the Kyverno deployment name kyvernoDeploymentName = osutils.GetEnvWithFallback("KYVERNO_DEPLOYMENT", "kyverno") // kyvernoServiceName is the Kyverno service name @@ -132,6 +134,10 @@ func KyvernoServiceAccountName() string { return kyvernoServiceAccountName } +func KyvernoRoleName() string { + return kyvernoRoleName +} + func KyvernoDeploymentName() string { return kyvernoDeploymentName } diff --git a/pkg/controllers/generic/webhook/controller.go b/pkg/controllers/generic/webhook/controller.go index 40d3223997..f0b5d43807 100644 --- a/pkg/controllers/generic/webhook/controller.go +++ b/pkg/controllers/generic/webhook/controller.go @@ -12,14 +12,18 @@ import ( "github.com/kyverno/kyverno/pkg/logging" "github.com/kyverno/kyverno/pkg/tls" controllerutils "github.com/kyverno/kyverno/pkg/utils/controller" + runtimeutils "github.com/kyverno/kyverno/pkg/utils/runtime" "golang.org/x/exp/maps" admissionregistrationv1 "k8s.io/api/admissionregistration/v1" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" admissionregistrationv1informers "k8s.io/client-go/informers/admissionregistration/v1" + appsv1informers "k8s.io/client-go/informers/apps/v1" corev1informers "k8s.io/client-go/informers/core/v1" admissionregistrationv1listers "k8s.io/client-go/listers/admissionregistration/v1" + appsv1listers "k8s.io/client-go/listers/apps/v1" corev1listers "k8s.io/client-go/listers/core/v1" "k8s.io/client-go/util/workqueue" ) @@ -42,25 +46,31 @@ type controller struct { vwcClient controllerutils.ObjectClient[*admissionregistrationv1.ValidatingWebhookConfiguration] // listers - vwcLister admissionregistrationv1listers.ValidatingWebhookConfigurationLister - secretLister corev1listers.SecretNamespaceLister + vwcLister admissionregistrationv1listers.ValidatingWebhookConfigurationLister + secretLister corev1listers.SecretNamespaceLister + deploymentLister appsv1listers.DeploymentNamespaceLister // queue queue workqueue.TypedRateLimitingInterface[any] // config - controllerName string - logger logr.Logger - webhookName string - path string - server string - servicePort int32 - rules []admissionregistrationv1.RuleWithOperations - failurePolicy *admissionregistrationv1.FailurePolicyType - sideEffects *admissionregistrationv1.SideEffectClass - configuration config.Configuration - labelSelector *metav1.LabelSelector - caSecretName string + controllerName string + logger logr.Logger + webhookName string + path string + server string + servicePort int32 + rules []admissionregistrationv1.RuleWithOperations + failurePolicy *admissionregistrationv1.FailurePolicyType + sideEffects *admissionregistrationv1.SideEffectClass + runtime runtimeutils.Runtime + configuration config.Configuration + labelSelector *metav1.LabelSelector + caSecretName string + webhooksDeleted bool + autoDeleteWebhooks bool + webhookCleanupSetup func(context.Context, logr.Logger) error + postWebhookCleanup func(context.Context, logr.Logger) error } func NewController( @@ -68,6 +78,7 @@ func NewController( vwcClient controllerutils.ObjectClient[*admissionregistrationv1.ValidatingWebhookConfiguration], vwcInformer admissionregistrationv1informers.ValidatingWebhookConfigurationInformer, secretInformer corev1informers.SecretInformer, + deploymentInformer appsv1informers.DeploymentInformer, webhookName string, path string, server string, @@ -79,25 +90,34 @@ func NewController( sideEffects *admissionregistrationv1.SideEffectClass, configuration config.Configuration, caSecretName string, + runtime runtimeutils.Runtime, + autoDeleteWebhooks bool, + webhookCleanupSetup func(context.Context, logr.Logger) error, + postWebhookCleanup func(context.Context, logr.Logger) error, ) controllers.Controller { queue := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultTypedControllerRateLimiter[any](), controllerName) c := controller{ - vwcClient: vwcClient, - vwcLister: vwcInformer.Lister(), - secretLister: secretInformer.Lister().Secrets(config.KyvernoNamespace()), - queue: queue, - controllerName: controllerName, - logger: logging.ControllerLogger(controllerName), - webhookName: webhookName, - path: path, - server: server, - servicePort: servicePort, - rules: rules, - failurePolicy: failurePolicy, - sideEffects: sideEffects, - configuration: configuration, - labelSelector: labelSelector, - caSecretName: caSecretName, + vwcClient: vwcClient, + vwcLister: vwcInformer.Lister(), + secretLister: secretInformer.Lister().Secrets(config.KyvernoNamespace()), + deploymentLister: deploymentInformer.Lister().Deployments(config.KyvernoNamespace()), + queue: queue, + controllerName: controllerName, + logger: logging.ControllerLogger(controllerName), + webhookName: webhookName, + path: path, + server: server, + servicePort: servicePort, + rules: rules, + failurePolicy: failurePolicy, + sideEffects: sideEffects, + configuration: configuration, + labelSelector: labelSelector, + caSecretName: caSecretName, + runtime: runtime, + autoDeleteWebhooks: autoDeleteWebhooks, + webhookCleanupSetup: webhookCleanupSetup, + postWebhookCleanup: postWebhookCleanup, } if _, _, err := controllerutils.AddDefaultEventHandlers(c.logger, vwcInformer.Informer(), queue); err != nil { c.logger.Error(err, "failed to register event handlers") @@ -122,11 +142,34 @@ func NewController( ); err != nil { c.logger.Error(err, "failed to register event handlers") } + if autoDeleteWebhooks { + if _, err := controllerutils.AddEventHandlersT( + deploymentInformer.Informer(), + func(obj *appsv1.Deployment) { + }, + func(_, obj *appsv1.Deployment) { + if obj.GetNamespace() == config.KyvernoNamespace() && obj.GetName() == config.KyvernoDeploymentName() { + c.enqueueCleanupAfter(1 * time.Second) + } + }, + func(obj *appsv1.Deployment) { + if obj.GetNamespace() == config.KyvernoNamespace() && obj.GetName() == config.KyvernoDeploymentName() { + c.enqueueCleanup() + } + }, + ); err != nil { + c.logger.Error(err, "failed to register event handlers") + } + } + configuration.OnChanged(c.enqueue) return &c } func (c *controller) Run(ctx context.Context, workers int) { + if err := c.webhookCleanupSetup(ctx, c.logger); err != nil { + c.logger.Error(err, "failed to setup webhook cleanup") + } c.enqueue() controllerutils.Run(ctx, c.logger, c.controllerName, time.Second, c.queue, workers, maxRetries, c.reconcile) } @@ -135,7 +178,22 @@ func (c *controller) enqueue() { c.queue.Add(c.webhookName) } +func (c *controller) enqueueCleanup() { + c.queue.Add(config.KyvernoDeploymentName()) +} + +func (c *controller) enqueueCleanupAfter(duration time.Duration) { + c.queue.AddAfter(config.KyvernoDeploymentName(), duration) +} + func (c *controller) reconcile(ctx context.Context, logger logr.Logger, key, _, _ string) error { + if c.autoDeleteWebhooks && c.runtime.IsGoingDown() { + return c.reconcileWebhookDeletion(ctx) + } + + if c.autoDeleteWebhooks && key == config.KyvernoDeploymentName() { + return c.reconcileWebhookDeletion(ctx) + } if key != c.webhookName { return nil } @@ -165,6 +223,37 @@ func (c *controller) reconcile(ctx context.Context, logger logr.Logger, key, _, return err } +func (c *controller) reconcileWebhookDeletion(ctx context.Context) error { + if c.autoDeleteWebhooks { + if c.runtime.IsGoingDown() { + if c.webhooksDeleted { + return nil + } + c.webhooksDeleted = true + if err := c.vwcClient.Delete(ctx, c.webhookName, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { + c.logger.Error(err, "failed to clean up validating webhook configuration", "label", kyverno.LabelWebhookManagedBy) + return err + } else if err == nil { + c.logger.Info("successfully deleted validating webhook configurations", "label", kyverno.LabelWebhookManagedBy) + } + + if err := c.postWebhookCleanup(ctx, c.logger); err != nil { + c.logger.Error(err, "failed to clean up temporary rbac") + return err + } else { + c.logger.Info("successfully deleted temporary rbac") + } + } else { + if err := c.webhookCleanupSetup(ctx, c.logger); err != nil { + c.logger.Error(err, "failed to reconcile webhook cleanup setup") + return err + } + c.logger.Info("reconciled webhook cleanup setup") + } + } + return nil +} + func objectMeta(name string, annotations map[string]string, labels map[string]string, owner ...metav1.OwnerReference) metav1.ObjectMeta { desiredLabels := make(map[string]string) defaultLabels := map[string]string{ diff --git a/pkg/controllers/webhook/cleanup.go b/pkg/controllers/webhook/cleanup.go new file mode 100644 index 0000000000..44cab92c76 --- /dev/null +++ b/pkg/controllers/webhook/cleanup.go @@ -0,0 +1,282 @@ +package webhook + +import ( + "context" + + "github.com/go-logr/logr" + "github.com/kyverno/kyverno/pkg/config" + controllerutils "github.com/kyverno/kyverno/pkg/utils/controller" + rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/util/retry" +) + +// WebhookCleanupSetup creates temporary rbac owned by kyverno resources, these roles and cluster roles get automatically deleted when kyverno is uninstalled +// It creates the following resources: +// 1. Creates a temporary cluster role binding to give permission to delete kyverno's cluster role and set its owner ref to aggregated cluster role itself. +// 2. Creates a temporary role and role binding with permissions to delete a service account, roles and role bindings with owner ref set to the service account. +func WebhookCleanupSetup( + kubeClient kubernetes.Interface, + finalizer string, +) func(context.Context, logr.Logger) error { + return func(ctx context.Context, logger logr.Logger) error { + name := config.KyvernoRoleName() + coreName := name + ":core" + tempRbacName := name + ":temporary" + + // create temporary rbac + cr, err := kubeClient.RbacV1().ClusterRoles().Get(ctx, coreName, metav1.GetOptions{}) + if err != nil { + logger.Error(err, "failed to get cluster role binding") + return err + } + + coreClusterRoleBinding := &rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: coreName, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "rbac.authorization.k8s.io/v1", + Kind: "ClusterRole", + Name: cr.Name, + UID: cr.UID, + }, + }, + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: config.KyvernoServiceAccountName(), + Namespace: config.KyvernoNamespace(), + APIGroup: "", + }, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "ClusterRole", + Name: coreName, + }, + } + + if crb, err := kubeClient.RbacV1().ClusterRoleBindings().Create(ctx, coreClusterRoleBinding, metav1.CreateOptions{}); err != nil && !apierrors.IsAlreadyExists(err) { + logger.Error(err, "failed to create temporary clusterrolebinding", "name", crb.Name) + return err + } else if !apierrors.IsAlreadyExists(err) { + logger.V(4).Info("temporary clusterrolebinding created", "clusterrolebinding", crb.Name) + } + + // create temporary rbac + sa, err := kubeClient.CoreV1().ServiceAccounts(config.KyvernoNamespace()).Get(ctx, config.KyvernoServiceAccountName(), metav1.GetOptions{}) + if err != nil { + logger.Error(err, "failed to get service account") + return err + } + + role := &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{ + Name: tempRbacName, + Namespace: config.KyvernoNamespace(), + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "v1", + Kind: "ServiceAccount", + Name: sa.Name, + UID: sa.UID, + }, + }, + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{"serviceaccounts"}, + ResourceNames: []string{config.KyvernoServiceAccountName()}, + Verbs: []string{"get", "update", "delete"}, + }, + { + APIGroups: []string{"rbac.authorization.k8s.io"}, + Resources: []string{"rolebindings", "roles"}, + ResourceNames: []string{name}, + Verbs: []string{"get", "update"}, + }, + { + APIGroups: []string{"apps"}, + Resources: []string{"deployments"}, + ResourceNames: []string{config.KyvernoDeploymentName()}, + Verbs: []string{"get", "update"}, + }, + }, + } + + if r, err := kubeClient.RbacV1().Roles(config.KyvernoNamespace()).Create(ctx, role, metav1.CreateOptions{}); err != nil && !apierrors.IsAlreadyExists(err) { + logger.Error(err, "failed to create temporary role", "name", r.Name) + return err + } else if !apierrors.IsAlreadyExists(err) { + logger.V(4).Info("temporary role created in kyverno namespace", "role", r.Name, "namespace", r.Namespace) + } + + roleBinding := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: tempRbacName, + Namespace: config.KyvernoNamespace(), + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "v1", + Kind: "ServiceAccount", + Name: sa.Name, + UID: sa.UID, + }, + }, + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: config.KyvernoServiceAccountName(), + Namespace: config.KyvernoNamespace(), + APIGroup: "", + }, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "Role", + Name: tempRbacName, + }, + } + + if rb, err := kubeClient.RbacV1().RoleBindings(config.KyvernoNamespace()).Create(ctx, roleBinding, metav1.CreateOptions{}); err != nil && !apierrors.IsAlreadyExists(err) { + logger.Error(err, "failed to create temporary rolebinding", "name", rb.Name) + return err + } else if !apierrors.IsAlreadyExists(err) { + logger.V(4).Info("temporary rolebinding created in kyverno namespace", "rolebinding", rb.Name, "namespace", rb.Namespace) + } + + // Add finalizers + if err := AddFinalizers(ctx, kubeClient.RbacV1().ClusterRoles(), coreName, finalizer); err != nil && !apierrors.IsNotFound(err) { + logger.Error(err, "failed to add finalizer to clusterrole", "name", coreName) + return err + } + + if err := AddFinalizers(ctx, kubeClient.RbacV1().RoleBindings(config.KyvernoNamespace()), name, finalizer); err != nil { + logger.Error(err, "failed to add finalizer to rolebindings", "name", name, "namespace", config.KyvernoNamespace()) + return err + } + + if err := AddFinalizers(ctx, kubeClient.RbacV1().Roles(config.KyvernoNamespace()), name, finalizer); err != nil { + logger.Error(err, "failed to add finalizer to role", "name", name, "namespace", config.KyvernoNamespace()) + return err + } + + if err := AddFinalizers(ctx, kubeClient.AppsV1().Deployments(config.KyvernoNamespace()), config.KyvernoDeploymentName(), finalizer); err != nil { + logger.Error(err, "failed to add finalizer to deployment", "name", config.KyvernoDeploymentName(), "namespace", config.KyvernoNamespace()) + return err + } + + if err := AddFinalizers(ctx, kubeClient.CoreV1().ServiceAccounts(config.KyvernoNamespace()), config.KyvernoServiceAccountName(), finalizer); err != nil { + logger.Error(err, "failed to add finalizer to serviceaccount", "name", config.KyvernoServiceAccountName(), "namespace", config.KyvernoNamespace()) + return err + } + return nil + } +} + +// WebhookCleanupHandler is run after webhook configuration cleanup is performed to delete roles and service account. +// Admission controller cluster and namespaced roles and role bindings have finalizers to block their deletion until admission controller terminates. +// This handler removes the finalizers on roles and service account after they are used to cleanup webhook cfg. +// It does the following: +// +// Deletes the cluster scoped rbac in order: +// a. Removes finalizers from controller cluster role binding +// b. Removes finalizers from controller core cluster role +// c. Removes finalizers from controller aggregated cluster role +// d. Temporary cluster role and cluster role binding created by WebhookCleanupSetup gets garbage collected after (c) automatically +// +// Deletes the namespace scoped rbac in order: +// a. Removes finalizers from controller role binding. +// b. Removes finalizers from controller role. +// c. Removes finalizers from controller service account +// d. Temporary role and role binding created by WebhookCleanupSetup gets garbage collected after (c) automatically +func WebhookCleanupHandler( + kubeClient kubernetes.Interface, + finalizer string, +) func(context.Context, logr.Logger) error { + return func(ctx context.Context, logger logr.Logger) error { + name := config.KyvernoRoleName() + coreName := name + ":core" + + // cleanup cluster scoped rbac + if err := DeleteFinalizers(ctx, kubeClient.RbacV1().ClusterRoles(), coreName, finalizer); err != nil { + logger.Error(err, "failed to delete finalizer from clusterrole", "name", coreName) + return err + } + + // cleanup namespace scoped rbac + if err := DeleteFinalizers(ctx, kubeClient.RbacV1().RoleBindings(config.KyvernoNamespace()), name, finalizer); err != nil { + logger.Error(err, "failed to delete finalizer from rolebindings", "name", name, "namespace", config.KyvernoNamespace()) + return err + } + + if err := DeleteFinalizers(ctx, kubeClient.RbacV1().Roles(config.KyvernoNamespace()), name, finalizer); err != nil { + logger.Error(err, "failed to delete finalizer from role", "name", name, "namespace", config.KyvernoNamespace()) + return err + } + + if err := DeleteFinalizers(ctx, kubeClient.AppsV1().Deployments(config.KyvernoNamespace()), config.KyvernoDeploymentName(), finalizer); err != nil { + logger.Error(err, "failed to delete finalizer from deployment", "name", config.KyvernoDeploymentName(), "namespace", config.KyvernoNamespace()) + return err + } + + if err := DeleteFinalizers(ctx, kubeClient.CoreV1().ServiceAccounts(config.KyvernoNamespace()), config.KyvernoServiceAccountName(), finalizer); err != nil { + logger.Error(err, "failed to delete finalizer from serviceaccount", "name", config.KyvernoServiceAccountName(), "namespace", config.KyvernoNamespace()) + return err + } + + return nil + } +} + +func DeleteFinalizers[T metav1.Object](ctx context.Context, client controllerutils.ObjectClient[T], name, finalizer string) error { + return retry.RetryOnConflict(retry.DefaultRetry, func() error { + obj, err := client.Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return err + } + finalizers := make([]string, 0) + for _, f := range obj.GetFinalizers() { + if f != finalizer { + finalizers = append(finalizers, f) + } + } + + obj.SetFinalizers(finalizers) + _, err = client.Update(ctx, obj, metav1.UpdateOptions{}) + if err != nil { + return err + } + return nil + }) +} + +func AddFinalizers[T metav1.Object](ctx context.Context, client controllerutils.ObjectClient[T], name, finalizer string) error { + return retry.RetryOnConflict(retry.DefaultRetry, func() error { + obj, err := client.Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return err + } + + finalizers := obj.GetFinalizers() + for _, f := range finalizers { + if f == finalizer { + return nil + } + } + finalizers = append(finalizers, finalizer) + obj.SetFinalizers(finalizers) + + _, err = client.Update(ctx, obj, metav1.UpdateOptions{}) + if err != nil { + return err + } + return nil + }) +} diff --git a/pkg/controllers/webhook/controller.go b/pkg/controllers/webhook/controller.go index 40ee5db079..58e9242540 100644 --- a/pkg/controllers/webhook/controller.go +++ b/pkg/controllers/webhook/controller.go @@ -26,6 +26,7 @@ import ( kubeutils "github.com/kyverno/kyverno/pkg/utils/kube" runtimeutils "github.com/kyverno/kyverno/pkg/utils/runtime" admissionregistrationv1 "k8s.io/api/admissionregistration/v1" + appsv1 "k8s.io/api/apps/v1" coordinationv1 "k8s.io/api/coordination/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -34,10 +35,12 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/sets" admissionregistrationv1informers "k8s.io/client-go/informers/admissionregistration/v1" + appsv1informers "k8s.io/client-go/informers/apps/v1" coordinationv1informers "k8s.io/client-go/informers/coordination/v1" corev1informers "k8s.io/client-go/informers/core/v1" rbacv1informers "k8s.io/client-go/informers/rbac/v1" admissionregistrationv1listers "k8s.io/client-go/listers/admissionregistration/v1" + appsv1listers "k8s.io/client-go/listers/apps/v1" coordinationv1listers "k8s.io/client-go/listers/coordination/v1" corev1listers "k8s.io/client-go/listers/core/v1" rbacv1listers "k8s.io/client-go/listers/rbac/v1" @@ -91,6 +94,7 @@ type controller struct { vwcLister admissionregistrationv1listers.ValidatingWebhookConfigurationLister cpolLister kyvernov1listers.ClusterPolicyLister polLister kyvernov1listers.PolicyLister + deploymentLister appsv1listers.DeploymentLister secretLister corev1listers.SecretLister leaseLister coordinationv1listers.LeaseLister clusterroleLister rbacv1listers.ClusterRoleLister @@ -100,14 +104,18 @@ type controller struct { queue workqueue.TypedRateLimitingInterface[any] // config - server string - defaultTimeout int32 - servicePort int32 - autoUpdateWebhooks bool - admissionReports bool - runtime runtimeutils.Runtime - configuration config.Configuration - caSecretName string + server string + defaultTimeout int32 + servicePort int32 + autoUpdateWebhooks bool + autoDeleteWebhooks bool + admissionReports bool + runtime runtimeutils.Runtime + configuration config.Configuration + caSecretName string + webhooksDeleted bool + webhookCleanupSetup func(context.Context, logr.Logger) error + postWebhookCleanup func(context.Context, logr.Logger) error // state lock sync.Mutex @@ -124,6 +132,7 @@ func NewController( vwcInformer admissionregistrationv1informers.ValidatingWebhookConfigurationInformer, cpolInformer kyvernov1informers.ClusterPolicyInformer, polInformer kyvernov1informers.PolicyInformer, + deploymentInformer appsv1informers.DeploymentInformer, secretInformer corev1informers.SecretInformer, leaseInformer coordinationv1informers.LeaseInformer, clusterroleInformer rbacv1informers.ClusterRoleInformer, @@ -133,35 +142,42 @@ func NewController( servicePort int32, webhookServerPort int32, autoUpdateWebhooks bool, + autoDeleteWebhooks bool, admissionReports bool, runtime runtimeutils.Runtime, configuration config.Configuration, caSecretName string, + webhookCleanupSetup func(context.Context, logr.Logger) error, + postWebhookCleanup func(context.Context, logr.Logger) error, ) controllers.Controller { queue := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultTypedControllerRateLimiter[any](), ControllerName) c := controller{ - discoveryClient: discoveryClient, - mwcClient: mwcClient, - vwcClient: vwcClient, - leaseClient: leaseClient, - kyvernoClient: kyvernoClient, - mwcLister: mwcInformer.Lister(), - vwcLister: vwcInformer.Lister(), - cpolLister: cpolInformer.Lister(), - polLister: polInformer.Lister(), - secretLister: secretInformer.Lister(), - leaseLister: leaseInformer.Lister(), - clusterroleLister: clusterroleInformer.Lister(), - gctxentryLister: gctxentryInformer.Lister(), - queue: queue, - server: server, - defaultTimeout: defaultTimeout, - servicePort: servicePort, - autoUpdateWebhooks: autoUpdateWebhooks, - admissionReports: admissionReports, - runtime: runtime, - configuration: configuration, - caSecretName: caSecretName, + discoveryClient: discoveryClient, + mwcClient: mwcClient, + vwcClient: vwcClient, + leaseClient: leaseClient, + kyvernoClient: kyvernoClient, + mwcLister: mwcInformer.Lister(), + vwcLister: vwcInformer.Lister(), + cpolLister: cpolInformer.Lister(), + polLister: polInformer.Lister(), + deploymentLister: deploymentInformer.Lister(), + secretLister: secretInformer.Lister(), + leaseLister: leaseInformer.Lister(), + clusterroleLister: clusterroleInformer.Lister(), + gctxentryLister: gctxentryInformer.Lister(), + queue: queue, + server: server, + defaultTimeout: defaultTimeout, + servicePort: servicePort, + autoUpdateWebhooks: autoUpdateWebhooks, + autoDeleteWebhooks: autoDeleteWebhooks, + admissionReports: admissionReports, + runtime: runtime, + configuration: configuration, + caSecretName: caSecretName, + webhookCleanupSetup: webhookCleanupSetup, + postWebhookCleanup: postWebhookCleanup, policyState: map[string]sets.Set[string]{ config.MutatingWebhookConfigurationName: sets.New[string](), config.ValidatingWebhookConfigurationName: sets.New[string](), @@ -193,6 +209,25 @@ func NewController( ); err != nil { logger.Error(err, "failed to register event handlers") } + if autoDeleteWebhooks { + if _, err := controllerutils.AddEventHandlersT( + deploymentInformer.Informer(), + func(obj *appsv1.Deployment) { + }, + func(_, obj *appsv1.Deployment) { + if obj.GetNamespace() == config.KyvernoNamespace() && obj.GetName() == config.KyvernoDeploymentName() { + c.enqueueCleanupAfter(1 * time.Second) + } + }, + func(obj *appsv1.Deployment) { + if obj.GetNamespace() == config.KyvernoNamespace() && obj.GetName() == config.KyvernoDeploymentName() { + c.enqueueCleanup() + } + }, + ); err != nil { + logger.Error(err, "failed to register event handlers") + } + } if _, err := controllerutils.AddEventHandlers( cpolInformer.Informer(), func(interface{}) { c.enqueueResourceWebhooks(0) }, @@ -214,6 +249,9 @@ func NewController( } func (c *controller) Run(ctx context.Context, workers int) { + if err := c.webhookCleanupSetup(ctx, logger); err != nil { + logger.Error(err, "failed to setup webhook cleanup") + } // add our known webhooks to the queue c.enqueueAll() controllerutils.Run(ctx, logger, ControllerName, time.Second, c.queue, workers, maxRetries, c.reconcile, c.watchdog) @@ -286,6 +324,14 @@ func (c *controller) enqueueAll() { c.enqueueVerifyWebhook() } +func (c *controller) enqueueCleanup() { + c.queue.Add(config.KyvernoDeploymentName()) +} + +func (c *controller) enqueueCleanupAfter(duration time.Duration) { + c.queue.AddAfter(config.KyvernoDeploymentName(), duration) +} + func (c *controller) enqueuePolicyWebhooks() { c.queue.Add(config.PolicyValidatingWebhookConfigurationName) c.queue.Add(config.PolicyMutatingWebhookConfigurationName) @@ -363,6 +409,47 @@ func (c *controller) reconcileVerifyMutatingWebhookConfiguration(ctx context.Con return c.reconcileMutatingWebhookConfiguration(ctx, true, c.buildVerifyMutatingWebhookConfiguration) } +func (c *controller) reconcileWebhookDeletion(ctx context.Context) error { + if c.autoUpdateWebhooks { + if c.runtime.IsGoingDown() { + if c.webhooksDeleted { + return nil + } + c.webhooksDeleted = true + if err := c.vwcClient.DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{ + LabelSelector: kyverno.LabelWebhookManagedBy, + }); err != nil && !apierrors.IsNotFound(err) { + logger.Error(err, "failed to clean up validating webhook configuration", "label", kyverno.LabelWebhookManagedBy) + return err + } else if err == nil { + logger.Info("successfully deleted validating webhook configurations", "label", kyverno.LabelWebhookManagedBy) + } + if err := c.mwcClient.DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{ + LabelSelector: kyverno.LabelWebhookManagedBy, + }); err != nil && !apierrors.IsNotFound(err) { + logger.Error(err, "failed to clean up mutating webhook configuration", "label", kyverno.LabelWebhookManagedBy) + return err + } else if err == nil { + logger.Info("successfully deleted mutating webhook configurations", "label", kyverno.LabelWebhookManagedBy) + } + + if err := c.postWebhookCleanup(ctx, logger); err != nil { + logger.Error(err, "failed to clean up temporary rbac") + return err + } else { + logger.Info("successfully deleted temporary rbac") + } + } else { + if err := c.webhookCleanupSetup(ctx, logger); err != nil { + logger.Error(err, "failed to reconcile webhook cleanup setup") + return err + } + logger.Info("reconciled webhook cleanup setup") + } + } + return nil +} + func (c *controller) reconcileValidatingWebhookConfiguration(ctx context.Context, autoUpdateWebhooks bool, build func(context.Context, config.Configuration, []byte) (*admissionregistrationv1.ValidatingWebhookConfiguration, error)) error { caData, err := tls.ReadRootCASecret(c.caSecretName, config.KyvernoNamespace(), c.secretLister.Secrets(config.KyvernoNamespace())) if err != nil { @@ -526,6 +613,10 @@ func (c *controller) updatePolicyStatuses(ctx context.Context) error { } func (c *controller) reconcile(ctx context.Context, logger logr.Logger, key, namespace, name string) error { + if c.autoDeleteWebhooks && c.runtime.IsGoingDown() { + return c.reconcileWebhookDeletion(ctx) + } + switch name { case config.MutatingWebhookConfigurationName: if c.runtime.IsRollingUpdate() { @@ -555,6 +646,8 @@ func (c *controller) reconcile(ctx context.Context, logger logr.Logger, key, nam return c.reconcilePolicyMutatingWebhookConfiguration(ctx) case config.VerifyMutatingWebhookConfigurationName: return c.reconcileVerifyMutatingWebhookConfiguration(ctx) + case config.KyvernoDeploymentName(): + return c.reconcileWebhookDeletion(ctx) } return nil } diff --git a/pkg/informers/deployment.go b/pkg/informers/deployment.go new file mode 100644 index 0000000000..176a79c92e --- /dev/null +++ b/pkg/informers/deployment.go @@ -0,0 +1,46 @@ +package informers + +import ( + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + appsv1informers "k8s.io/client-go/informers/apps/v1" + "k8s.io/client-go/kubernetes" + appsv1listers "k8s.io/client-go/listers/apps/v1" + "k8s.io/client-go/tools/cache" +) + +type deploymentInformer struct { + informer cache.SharedIndexInformer + lister appsv1listers.DeploymentLister +} + +func NewDeploymentInformer( + client kubernetes.Interface, + namespace string, + name string, + resyncPeriod time.Duration, +) appsv1informers.DeploymentInformer { + indexers := cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc} + options := func(lo *metav1.ListOptions) { + lo.FieldSelector = fields.OneTermEqualSelector(metav1.ObjectNameField, name).String() + } + informer := appsv1informers.NewFilteredDeploymentInformer( + client, + namespace, + resyncPeriod, + indexers, + options, + ) + lister := appsv1listers.NewDeploymentLister(informer.GetIndexer()) + return &deploymentInformer{informer, lister} +} + +func (i *deploymentInformer) Informer() cache.SharedIndexInformer { + return i.informer +} + +func (i *deploymentInformer) Lister() appsv1listers.DeploymentLister { + return i.lister +} diff --git a/pkg/utils/controller/utils.go b/pkg/utils/controller/utils.go index 389b7126cb..5c4226f931 100644 --- a/pkg/utils/controller/utils.go +++ b/pkg/utils/controller/utils.go @@ -35,8 +35,9 @@ type ObjectClient[T metav1.Object] interface { CreateClient[T] GetClient[T] UpdateClient[T] - DeleteClient PatchClient[T] + DeleteClient + DeleteCollectionClient } type DeleteCollectionClient interface { diff --git a/pkg/webhooks/server.go b/pkg/webhooks/server.go index 7429ffea0a..fe90978b94 100644 --- a/pkg/webhooks/server.go +++ b/pkg/webhooks/server.go @@ -245,6 +245,8 @@ func (s *server) cleanup(ctx context.Context) { deleteLease := func(name string) { if err := s.leaseClient.Delete(ctx, name, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { logger.Error(err, "failed to clean up lease", "name", name) + } else if err == nil { + logger.Info("successfully deleted leases", "label", kyverno.LabelWebhookManagedBy) } } deleteVwc := func() { @@ -252,6 +254,8 @@ func (s *server) cleanup(ctx context.Context) { LabelSelector: kyverno.LabelWebhookManagedBy, }); err != nil && !apierrors.IsNotFound(err) { logger.Error(err, "failed to clean up validating webhook configuration", "label", kyverno.LabelWebhookManagedBy) + } else if err == nil { + logger.Info("successfully deleted validating webhook configurations", "label", kyverno.LabelWebhookManagedBy) } } deleteMwc := func() { @@ -259,6 +263,8 @@ func (s *server) cleanup(ctx context.Context) { LabelSelector: kyverno.LabelWebhookManagedBy, }); err != nil && !apierrors.IsNotFound(err) { logger.Error(err, "failed to clean up mutating webhook configuration", "label", kyverno.LabelWebhookManagedBy) + } else if err == nil { + logger.Info("successfully deleted mutating webhook configurations", "label", kyverno.LabelWebhookManagedBy) } } deleteLease("kyvernopre-lock")