1
0
Fork 0
mirror of https://github.com/kyverno/kyverno.git synced 2025-03-15 20:20:22 +00:00

feat: delete webhook configurations after kyverno is uninstalled (#10782)

* feat: delete webhook configurations after kyverno is uninstalled

Signed-off-by: Vishal Choudhary <vishal.choudhary@nirmata.com>

* feat: optionally add permissions

Signed-off-by: Vishal Choudhary <vishal.choudhary@nirmata.com>

* fix: linter

Signed-off-by: Vishal Choudhary <vishal.choudhary@nirmata.com>

* fix: disable finalizers in latest manifest

Signed-off-by: Vishal Choudhary <vishal.choudhary@nirmata.com>

* fix: move webhook cleanup to webhooks controller

Signed-off-by: Vishal Choudhary <vishal.choudhary@nirmata.com>

* feat: add finalizers on deployment

Signed-off-by: Vishal Choudhary <vishal.choudhary@nirmata.com>

* fix: refactor

Signed-off-by: Vishal Choudhary <vishal.choudhary@nirmata.com>

* feat: add roles to cleanupcontroller

Signed-off-by: Vishal Choudhary <vishal.choudhary@nirmata.com>

* feat: add cleanup to generic controllers

Signed-off-by: Vishal Choudhary <vishal.choudhary@nirmata.com>

* feat: add webhook cleanup in generic controllers

Signed-off-by: Vishal Choudhary <vishal.choudhary@nirmata.com>

* feat: remove unnecessary clusterrole and clusterrole bindings

Signed-off-by: Vishal Choudhary <vishal.choudhary@nirmata.com>

* feat: make this behaviour opt-in

Signed-off-by: Vishal Choudhary <vishal.choudhary@nirmata.com>

* feat: reconcile webhook setup on deployment change

Signed-off-by: Vishal Choudhary <vishal.choudhary@nirmata.com>

* fix: update codegen and remove unused vars

Signed-off-by: Vishal Choudhary <vishal.choudhary@nirmata.com>

* feat: add finalizers to chart

Signed-off-by: Vishal Choudhary <vishal.choudhary@nirmata.com>

---------

Signed-off-by: Vishal Choudhary <vishal.choudhary@nirmata.com>
Co-authored-by: Jim Bugwadia <jim@nirmata.com>
Co-authored-by: shuting <shuting@nirmata.com>
This commit is contained in:
Vishal Choudhary 2024-09-04 16:29:59 +05:30 committed by GitHub
parent 416b7d2f8b
commit c0d6eaddb3
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
22 changed files with 880 additions and 72 deletions

View file

@ -702,6 +702,7 @@ The chart values are organised per component.
| Key | Type | Default | Description | | Key | Type | Default | Description |
|-----|------|---------|-------------| |-----|------|---------|-------------|
| webhooksCleanup.enabled | bool | `true` | Create a helm pre-delete hook to cleanup webhooks. | | webhooksCleanup.enabled | bool | `true` | Create a helm pre-delete hook to cleanup webhooks. |
| webhooksCleanup.autoDeleteWebhooks.enabled | bool | `false` | Allow webhooks controller to delete webhooks using finalizers |
| webhooksCleanup.image.registry | string | `nil` | Image registry | | webhooksCleanup.image.registry | string | `nil` | Image registry |
| webhooksCleanup.image.repository | string | `"bitnami/kubectl"` | Image repository | | webhooksCleanup.image.repository | string | `"bitnami/kubectl"` | Image repository |
| webhooksCleanup.image.tag | string | `"1.30.2"` | Image tag Defaults to `latest` if omitted | | webhooksCleanup.image.tag | string | `"1.30.2"` | Image tag Defaults to `latest` if omitted |

View file

@ -16,6 +16,14 @@ apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole kind: ClusterRole
metadata: metadata:
name: {{ template "kyverno.admission-controller.roleName" . }}:core name: {{ template "kyverno.admission-controller.roleName" . }}:core
{{- if .Values.webhooksCleanup.autoDeleteWebhooks.enabled }}
{{- if not .Values.templating.enabled }}
finalizers:
- kyverno.io/webhooks
- kyverno.io/exceptionwebhooks
- kyverno.io/globalcontextwebhooks
{{- end }}
{{- end }}
labels: labels:
{{- include "kyverno.admission-controller.labels" . | nindent 4 }} {{- include "kyverno.admission-controller.labels" . | nindent 4 }}
rules: rules:
@ -139,6 +147,31 @@ rules:
- get - get
- list - list
- watch - watch
{{- if .Values.webhooksCleanup.autoDeleteWebhooks.enabled }}
{{- if not .Values.templating.enabled }}
- apiGroups:
- rbac.authorization.k8s.io
resources:
- clusterroles
- clusterrolebindings
resourceNames:
- {{ template "kyverno.admission-controller.roleName" . }}
- {{ template "kyverno.admission-controller.roleName" . }}:core
- {{ template "kyverno.admission-controller.roleName" . }}:temporary
verbs:
- get
- patch
- update
- apiGroups:
- rbac.authorization.k8s.io
resources:
- clusterroles
- clusterrolebindings
verbs:
- create
- list
{{- end }}
{{- end }}
{{- with .Values.admissionController.rbac.coreClusterRole.extraResources }} {{- with .Values.admissionController.rbac.coreClusterRole.extraResources }}
{{- toYaml . | nindent 2 }} {{- toYaml . | nindent 2 }}
{{- end }} {{- end }}
@ -153,4 +186,4 @@ metadata:
rules: rules:
{{- toYaml . | nindent 2 }} {{- toYaml . | nindent 2 }}
{{- end }} {{- end }}
{{- end }} {{- end }}

View file

@ -4,6 +4,14 @@ kind: Deployment
metadata: metadata:
name: {{ template "kyverno.admission-controller.name" . }} name: {{ template "kyverno.admission-controller.name" . }}
namespace: {{ template "kyverno.namespace" . }} namespace: {{ template "kyverno.namespace" . }}
{{- if .Values.webhooksCleanup.autoDeleteWebhooks.enabled }}
{{- if not .Values.templating.enabled }}
finalizers:
- kyverno.io/webhooks
- kyverno.io/exceptionwebhooks
- kyverno.io/globalcontextwebhooks
{{- end }}
{{- end }}
labels: labels:
{{- include "kyverno.admission-controller.labels" . | nindent 4 }} {{- include "kyverno.admission-controller.labels" . | nindent 4 }}
{{- with .Values.admissionController.annotations }} {{- with .Values.admissionController.annotations }}
@ -105,6 +113,8 @@ spec:
env: env:
- name: KYVERNO_SERVICEACCOUNT_NAME - name: KYVERNO_SERVICEACCOUNT_NAME
value: {{ template "kyverno.admission-controller.serviceAccountName" . }} value: {{ template "kyverno.admission-controller.serviceAccountName" . }}
- name: KYVERNO_ROLE_NAME
value: {{ template "kyverno.admission-controller.roleName" . }}
- name: INIT_CONFIG - name: INIT_CONFIG
value: {{ template "kyverno.config.configMapName" . }} value: {{ template "kyverno.config.configMapName" . }}
- name: METRICS_CONFIG - name: METRICS_CONFIG
@ -138,6 +148,9 @@ spec:
- --reportsServiceAccountName=system:serviceaccount:{{ include "kyverno.namespace" . }}:{{ include "kyverno.reports-controller.serviceAccountName" . }} - --reportsServiceAccountName=system:serviceaccount:{{ include "kyverno.namespace" . }}:{{ include "kyverno.reports-controller.serviceAccountName" . }}
- --servicePort={{ .Values.admissionController.service.port }} - --servicePort={{ .Values.admissionController.service.port }}
- --webhookServerPort={{ .Values.admissionController.webhookServer.port }} - --webhookServerPort={{ .Values.admissionController.webhookServer.port }}
{{- if .Values.webhooksCleanup.autoDeleteWebhooks.enabled }}
- --autoDeleteWebhooks
{{- end }}
{{- if .Values.admissionController.tracing.enabled }} {{- if .Values.admissionController.tracing.enabled }}
- --enableTracing - --enableTracing
- --tracingAddress={{ .Values.admissionController.tracing.address }} - --tracingAddress={{ .Values.admissionController.tracing.address }}
@ -220,6 +233,8 @@ spec:
fieldPath: metadata.name fieldPath: metadata.name
- name: KYVERNO_SERVICEACCOUNT_NAME - name: KYVERNO_SERVICEACCOUNT_NAME
value: {{ template "kyverno.admission-controller.serviceAccountName" . }} value: {{ template "kyverno.admission-controller.serviceAccountName" . }}
- name: KYVERNO_ROLE_NAME
value: {{ template "kyverno.admission-controller.roleName" . }}
- name: KYVERNO_SVC - name: KYVERNO_SVC
value: {{ template "kyverno.admission-controller.serviceName" . }} value: {{ template "kyverno.admission-controller.serviceName" . }}
- name: TUF_ROOT - name: TUF_ROOT

View file

@ -4,6 +4,14 @@ kind: Role
metadata: metadata:
name: {{ template "kyverno.admission-controller.roleName" . }} name: {{ template "kyverno.admission-controller.roleName" . }}
namespace: {{ template "kyverno.namespace" . }} namespace: {{ template "kyverno.namespace" . }}
{{- if .Values.webhooksCleanup.autoDeleteWebhooks.enabled }}
{{- if not .Values.templating.enabled }}
finalizers:
- kyverno.io/webhooks
- kyverno.io/exceptionwebhooks
- kyverno.io/globalcontextwebhooks
{{- end }}
{{- end }}
labels: labels:
{{- include "kyverno.admission-controller.labels" . | nindent 4 }} {{- include "kyverno.admission-controller.labels" . | nindent 4 }}
rules: rules:
@ -11,10 +19,12 @@ rules:
- '' - ''
resources: resources:
- secrets - secrets
- serviceaccounts
verbs: verbs:
- get - get
- list - list
- watch - watch
- patch
- create - create
- update - update
- delete - delete
@ -39,6 +49,29 @@ rules:
- get - get
- patch - patch
- update - update
{{- if .Values.webhooksCleanup.autoDeleteWebhooks.enabled }}
{{- if not .Values.templating.enabled }}
- apiGroups:
- rbac.authorization.k8s.io
resources:
- roles
- rolebindings
resourceNames:
- {{ template "kyverno.admission-controller.roleName" . }}
- {{ template "kyverno.admission-controller.roleName" . }}:temporary
verbs:
- get
- patch
- update
- apiGroups:
- rbac.authorization.k8s.io
resources:
- roles
- rolebindings
verbs:
- create
{{- end }}
{{- end }}
# Allow update of Kyverno deployment annotations # Allow update of Kyverno deployment annotations
- apiGroups: - apiGroups:
- apps - apps

View file

@ -4,6 +4,14 @@ apiVersion: rbac.authorization.k8s.io/v1
metadata: metadata:
name: {{ template "kyverno.admission-controller.roleName" . }} name: {{ template "kyverno.admission-controller.roleName" . }}
namespace: {{ template "kyverno.namespace" . }} namespace: {{ template "kyverno.namespace" . }}
{{- if .Values.webhooksCleanup.autoDeleteWebhooks.enabled }}
{{- if not .Values.templating.enabled }}
finalizers:
- kyverno.io/webhooks
- kyverno.io/exceptionwebhooks
- kyverno.io/globalcontextwebhooks
{{- end }}
{{- end }}
labels: labels:
{{- include "kyverno.admission-controller.labels" . | nindent 4 }} {{- include "kyverno.admission-controller.labels" . | nindent 4 }}
roleRef: roleRef:

View file

@ -4,6 +4,14 @@ kind: ServiceAccount
metadata: metadata:
name: {{ template "kyverno.admission-controller.serviceAccountName" . }} name: {{ template "kyverno.admission-controller.serviceAccountName" . }}
namespace: {{ template "kyverno.namespace" . }} namespace: {{ template "kyverno.namespace" . }}
{{- if .Values.webhooksCleanup.autoDeleteWebhooks.enabled }}
{{- if not .Values.templating.enabled }}
finalizers:
- kyverno.io/webhooks
- kyverno.io/exceptionwebhooks
- kyverno.io/globalcontextwebhooks
{{- end }}
{{- end }}
labels: labels:
{{- include "kyverno.admission-controller.labels" . | nindent 4 }} {{- include "kyverno.admission-controller.labels" . | nindent 4 }}
{{- with .Values.admissionController.rbac.serviceAccount.annotations }} {{- with .Values.admissionController.rbac.serviceAccount.annotations }}

View file

@ -17,6 +17,13 @@ apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole kind: ClusterRole
metadata: metadata:
name: {{ template "kyverno.cleanup-controller.roleName" . }}:core name: {{ template "kyverno.cleanup-controller.roleName" . }}:core
{{- if .Values.webhooksCleanup.autoDeleteWebhooks.enabled }}
{{- if not .Values.templating.enabled }}
finalizers:
- kyverno.io/policywebhooks
- kyverno.io/ttlwebhooks
{{- end }}
{{- end }}
labels: labels:
{{- include "kyverno.cleanup-controller.labels" . | nindent 4 }} {{- include "kyverno.cleanup-controller.labels" . | nindent 4 }}
rules: rules:
@ -97,6 +104,31 @@ rules:
- subjectaccessreviews - subjectaccessreviews
verbs: verbs:
- create - create
{{- if .Values.webhooksCleanup.autoDeleteWebhooks.enabled }}
{{- if not .Values.templating.enabled }}
- apiGroups:
- rbac.authorization.k8s.io
resources:
- clusterroles
- clusterrolebindings
resourceNames:
- {{ template "kyverno.cleanup-controller.roleName" . }}
- {{ template "kyverno.cleanup-controller.roleName" . }}:core
- {{ template "kyverno.cleanup-controller.roleName" . }}:temporary
verbs:
- get
- patch
- update
- apiGroups:
- rbac.authorization.k8s.io
resources:
- clusterroles
- clusterrolebindings
verbs:
- create
- list
{{- end }}
{{- end }}
{{- with .Values.cleanupController.rbac.clusterRole.extraResources }} {{- with .Values.cleanupController.rbac.clusterRole.extraResources }}
--- ---
apiVersion: rbac.authorization.k8s.io/v1 apiVersion: rbac.authorization.k8s.io/v1
@ -109,4 +141,4 @@ rules:
{{- toYaml . | nindent 2 }} {{- toYaml . | nindent 2 }}
{{- end }} {{- end }}
{{- end }} {{- end }}
{{- end }} {{- end }}

View file

@ -5,6 +5,13 @@ kind: Deployment
metadata: metadata:
name: {{ template "kyverno.cleanup-controller.name" . }} name: {{ template "kyverno.cleanup-controller.name" . }}
namespace: {{ template "kyverno.namespace" . }} namespace: {{ template "kyverno.namespace" . }}
{{- if .Values.webhooksCleanup.autoDeleteWebhooks.enabled }}
{{- if not .Values.templating.enabled }}
finalizers:
- kyverno.io/policywebhooks
- kyverno.io/ttlwebhooks
{{- end }}
{{- end }}
labels: labels:
{{- include "kyverno.cleanup-controller.labels" . | nindent 4 }} {{- include "kyverno.cleanup-controller.labels" . | nindent 4 }}
{{- with .Values.cleanupController.annotations }} {{- with .Values.cleanupController.annotations }}
@ -101,6 +108,9 @@ spec:
- --servicePort={{ .Values.cleanupController.service.port }} - --servicePort={{ .Values.cleanupController.service.port }}
- --cleanupServerPort={{ .Values.cleanupController.server.port }} - --cleanupServerPort={{ .Values.cleanupController.server.port }}
- --webhookServerPort={{ .Values.cleanupController.webhookServer.port }} - --webhookServerPort={{ .Values.cleanupController.webhookServer.port }}
{{- if .Values.webhooksCleanup.autoDeleteWebhooks.enabled }}
- --autoDeleteWebhooks
{{- end }}
{{- if .Values.cleanupController.tracing.enabled }} {{- if .Values.cleanupController.tracing.enabled }}
- --enableTracing - --enableTracing
- --tracingAddress={{ .Values.cleanupController.tracing.address }} - --tracingAddress={{ .Values.cleanupController.tracing.address }}
@ -150,6 +160,8 @@ spec:
fieldPath: metadata.name fieldPath: metadata.name
- name: KYVERNO_SERVICEACCOUNT_NAME - name: KYVERNO_SERVICEACCOUNT_NAME
value: {{ template "kyverno.cleanup-controller.serviceAccountName" . }} value: {{ template "kyverno.cleanup-controller.serviceAccountName" . }}
- name: KYVERNO_ROLE_NAME
value: {{ template "kyverno.cleanup-controller.roleName" . }}
- name: KYVERNO_NAMESPACE - name: KYVERNO_NAMESPACE
valueFrom: valueFrom:
fieldRef: fieldRef:

View file

@ -4,6 +4,13 @@ apiVersion: rbac.authorization.k8s.io/v1
kind: Role kind: Role
metadata: metadata:
name: {{ template "kyverno.cleanup-controller.roleName" . }} name: {{ template "kyverno.cleanup-controller.roleName" . }}
{{- if .Values.webhooksCleanup.autoDeleteWebhooks.enabled }}
{{- if not .Values.templating.enabled }}
finalizers:
- kyverno.io/policywebhooks
- kyverno.io/ttlwebhooks
{{- end }}
{{- end }}
labels: labels:
{{- include "kyverno.cleanup-controller.labels" . | nindent 4 }} {{- include "kyverno.cleanup-controller.labels" . | nindent 4 }}
namespace: {{ template "kyverno.namespace" . }} namespace: {{ template "kyverno.namespace" . }}
@ -27,6 +34,22 @@ rules:
resourceNames: resourceNames:
- {{ template "kyverno.cleanup-controller.name" . }}.{{ template "kyverno.namespace" . }}.svc.kyverno-tls-ca - {{ template "kyverno.cleanup-controller.name" . }}.{{ template "kyverno.namespace" . }}.svc.kyverno-tls-ca
- {{ template "kyverno.cleanup-controller.name" . }}.{{ template "kyverno.namespace" . }}.svc.kyverno-tls-pair - {{ template "kyverno.cleanup-controller.name" . }}.{{ template "kyverno.namespace" . }}.svc.kyverno-tls-pair
{{- if .Values.webhooksCleanup.autoDeleteWebhooks.enabled }}
{{- if not .Values.templating.enabled }}
- apiGroups:
- ''
resources:
- serviceaccounts
verbs:
- delete
- get
- list
- update
- watch
resourceNames:
- {{ template "kyverno.cleanup-controller.serviceAccountName" . }}
{{- end }}
{{- end }}
- apiGroups: - apiGroups:
- '' - ''
resources: resources:
@ -55,5 +78,42 @@ rules:
- update - update
resourceNames: resourceNames:
- kyverno-cleanup-controller - kyverno-cleanup-controller
{{- if .Values.webhooksCleanup.autoDeleteWebhooks.enabled }}
{{- if not .Values.templating.enabled }}
- apiGroups:
- rbac.authorization.k8s.io
resources:
- roles
- rolebindings
resourceNames:
- {{ template "kyverno.cleanup-controller.roleName" . }}
- {{ template "kyverno.cleanup-controller.roleName" . }}:temporary
verbs:
- get
- patch
- update
- apiGroups:
- rbac.authorization.k8s.io
resources:
- roles
- rolebindings
verbs:
- create
{{- end }}
{{- end }}
- apiGroups:
- apps
resources:
- deployments
verbs:
- get
- list
- watch
{{- if .Values.webhooksCleanup.autoDeleteWebhooks.enabled }}
{{- if not .Values.templating.enabled }}
- patch
- update
{{- end }}
{{- end }}
{{- end -}} {{- end -}}
{{- end -}} {{- end -}}

View file

@ -4,6 +4,13 @@ kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1 apiVersion: rbac.authorization.k8s.io/v1
metadata: metadata:
name: {{ template "kyverno.cleanup-controller.roleName" . }} name: {{ template "kyverno.cleanup-controller.roleName" . }}
{{- if .Values.webhooksCleanup.autoDeleteWebhooks.enabled }}
{{- if not .Values.templating.enabled }}
finalizers:
- kyverno.io/policywebhooks
- kyverno.io/ttlwebhooks
{{- end }}
{{- end }}
labels: labels:
{{- include "kyverno.cleanup-controller.labels" . | nindent 4 }} {{- include "kyverno.cleanup-controller.labels" . | nindent 4 }}
namespace: {{ template "kyverno.namespace" . }} namespace: {{ template "kyverno.namespace" . }}

View file

@ -5,6 +5,13 @@ kind: ServiceAccount
metadata: metadata:
name: {{ template "kyverno.cleanup-controller.serviceAccountName" . }} name: {{ template "kyverno.cleanup-controller.serviceAccountName" . }}
namespace: {{ template "kyverno.namespace" . }} namespace: {{ template "kyverno.namespace" . }}
{{- if .Values.webhooksCleanup.autoDeleteWebhooks.enabled }}
{{- if not .Values.templating.enabled }}
finalizers:
- kyverno.io/policywebhooks
- kyverno.io/ttlwebhooks
{{- end }}
{{- end }}
labels: labels:
{{- include "kyverno.cleanup-controller.labels" . | nindent 4 }} {{- include "kyverno.cleanup-controller.labels" . | nindent 4 }}
{{- with .Values.cleanupController.rbac.serviceAccount.annotations }} {{- with .Values.cleanupController.rbac.serviceAccount.annotations }}

View file

@ -469,6 +469,10 @@ webhooksCleanup:
# -- Create a helm pre-delete hook to cleanup webhooks. # -- Create a helm pre-delete hook to cleanup webhooks.
enabled: true enabled: true
autoDeleteWebhooks:
# -- Allow webhooks controller to delete webhooks using finalizers
enabled: false
image: image:
# -- (string) Image registry # -- (string) Image registry
registry: ~ registry: ~

View file

@ -21,6 +21,7 @@ import (
genericwebhookcontroller "github.com/kyverno/kyverno/pkg/controllers/generic/webhook" genericwebhookcontroller "github.com/kyverno/kyverno/pkg/controllers/generic/webhook"
globalcontextcontroller "github.com/kyverno/kyverno/pkg/controllers/globalcontext" globalcontextcontroller "github.com/kyverno/kyverno/pkg/controllers/globalcontext"
ttlcontroller "github.com/kyverno/kyverno/pkg/controllers/ttl" ttlcontroller "github.com/kyverno/kyverno/pkg/controllers/ttl"
webhookcontroller "github.com/kyverno/kyverno/pkg/controllers/webhook"
"github.com/kyverno/kyverno/pkg/event" "github.com/kyverno/kyverno/pkg/event"
"github.com/kyverno/kyverno/pkg/globalcontext/store" "github.com/kyverno/kyverno/pkg/globalcontext/store"
"github.com/kyverno/kyverno/pkg/informers" "github.com/kyverno/kyverno/pkg/informers"
@ -29,6 +30,7 @@ import (
"github.com/kyverno/kyverno/pkg/tls" "github.com/kyverno/kyverno/pkg/tls"
"github.com/kyverno/kyverno/pkg/toggle" "github.com/kyverno/kyverno/pkg/toggle"
kubeutils "github.com/kyverno/kyverno/pkg/utils/kube" kubeutils "github.com/kyverno/kyverno/pkg/utils/kube"
runtimeutils "github.com/kyverno/kyverno/pkg/utils/runtime"
"github.com/kyverno/kyverno/pkg/webhooks" "github.com/kyverno/kyverno/pkg/webhooks"
admissionregistrationv1 "k8s.io/api/admissionregistration/v1" admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
corev1 "k8s.io/api/core/v1" corev1 "k8s.io/api/core/v1"
@ -38,10 +40,12 @@ import (
) )
const ( const (
resyncPeriod = 15 * time.Minute resyncPeriod = 15 * time.Minute
webhookWorkers = 2 webhookWorkers = 2
policyWebhookControllerName = "policy-webhook-controller" policyWebhookControllerName = "policy-webhook-controller"
ttlWebhookControllerName = "ttl-webhook-controller" ttlWebhookControllerName = "ttl-webhook-controller"
policyWebhookControllerFinalizerName = "kyverno.io/policywebhooks"
ttlWebhookControllerFinalizerName = "kyverno.io/ttlwebhooks"
) )
var ( var (
@ -78,6 +82,7 @@ func main() {
interval time.Duration interval time.Duration
renewBefore time.Duration renewBefore time.Duration
maxAPICallResponseLength int64 maxAPICallResponseLength int64
autoDeleteWebhooks bool
) )
flagset := flag.NewFlagSet("cleanup-controller", flag.ExitOnError) flagset := flag.NewFlagSet("cleanup-controller", flag.ExitOnError)
flagset.BoolVar(&dumpPayload, "dumpPayload", false, "Set this flag to activate/deactivate debug mode.") flagset.BoolVar(&dumpPayload, "dumpPayload", false, "Set this flag to activate/deactivate debug mode.")
@ -91,6 +96,7 @@ func main() {
flagset.StringVar(&tlsSecretName, "tlsSecretName", "", "Name of the secret containing TLS pair.") flagset.StringVar(&tlsSecretName, "tlsSecretName", "", "Name of the secret containing TLS pair.")
flagset.DurationVar(&renewBefore, "renewBefore", 15*24*time.Hour, "The certificate renewal time before expiration") flagset.DurationVar(&renewBefore, "renewBefore", 15*24*time.Hour, "The certificate renewal time before expiration")
flagset.Int64Var(&maxAPICallResponseLength, "maxAPICallResponseLength", 2*1000*1000, "Maximum allowed response size from API Calls. A value of 0 bypasses checks (not recommended).") flagset.Int64Var(&maxAPICallResponseLength, "maxAPICallResponseLength", 2*1000*1000, "Maximum allowed response size from API Calls. A value of 0 bypasses checks (not recommended).")
flagset.BoolVar(&autoDeleteWebhooks, "autoDeleteWebhooks", false, "Set this flag to 'true' to enable autodeletion of webhook configurations using finalizers (requires extra permissions).")
// config // config
appConfig := internal.NewConfiguration( appConfig := internal.NewConfiguration(
internal.WithProfiling(), internal.WithProfiling(),
@ -129,7 +135,8 @@ func main() {
// certificates informers // certificates informers
caSecret := informers.NewSecretInformer(setup.KubeClient, config.KyvernoNamespace(), caSecretName, resyncPeriod) caSecret := informers.NewSecretInformer(setup.KubeClient, config.KyvernoNamespace(), caSecretName, resyncPeriod)
tlsSecret := informers.NewSecretInformer(setup.KubeClient, config.KyvernoNamespace(), tlsSecretName, resyncPeriod) tlsSecret := informers.NewSecretInformer(setup.KubeClient, config.KyvernoNamespace(), tlsSecretName, resyncPeriod)
if !informers.StartInformersAndWaitForCacheSync(ctx, setup.Logger, caSecret, tlsSecret) { kyvernoDeployment := informers.NewDeploymentInformer(setup.KubeClient, config.KyvernoNamespace(), config.KyvernoDeploymentName(), resyncPeriod)
if !informers.StartInformersAndWaitForCacheSync(ctx, setup.Logger, caSecret, tlsSecret, kyvernoDeployment) {
setup.Logger.Error(errors.New("failed to wait for cache sync"), "failed to wait for cache sync") setup.Logger.Error(errors.New("failed to wait for cache sync"), "failed to wait for cache sync")
os.Exit(1) os.Exit(1)
} }
@ -180,6 +187,12 @@ func main() {
if !internal.StartInformersAndWaitForCacheSync(ctx, setup.Logger, kubeInformer, kyvernoInformer) { if !internal.StartInformersAndWaitForCacheSync(ctx, setup.Logger, kubeInformer, kyvernoInformer) {
os.Exit(1) os.Exit(1)
} }
runtime := runtimeutils.NewRuntime(
setup.Logger.WithName("runtime-checks"),
serverIP,
kyvernoDeployment,
nil,
)
// setup leader election // setup leader election
le, err := leaderelection.New( le, err := leaderelection.New(
setup.Logger.WithName("leader-election"), setup.Logger.WithName("leader-election"),
@ -229,6 +242,7 @@ func main() {
setup.KubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations(), setup.KubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations(),
kubeInformer.Admissionregistration().V1().ValidatingWebhookConfigurations(), kubeInformer.Admissionregistration().V1().ValidatingWebhookConfigurations(),
caSecret, caSecret,
kyvernoDeployment,
config.CleanupValidatingWebhookConfigurationName, config.CleanupValidatingWebhookConfigurationName,
config.CleanupValidatingWebhookServicePath, config.CleanupValidatingWebhookServicePath,
serverIP, serverIP,
@ -255,6 +269,10 @@ func main() {
genericwebhookcontroller.None, genericwebhookcontroller.None,
setup.Configuration, setup.Configuration,
caSecretName, caSecretName,
runtime,
autoDeleteWebhooks,
webhookcontroller.WebhookCleanupSetup(setup.KubeClient, policyWebhookControllerFinalizerName),
webhookcontroller.WebhookCleanupHandler(setup.KubeClient, policyWebhookControllerFinalizerName),
), ),
webhookWorkers, webhookWorkers,
) )
@ -265,6 +283,7 @@ func main() {
setup.KubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations(), setup.KubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations(),
kubeInformer.Admissionregistration().V1().ValidatingWebhookConfigurations(), kubeInformer.Admissionregistration().V1().ValidatingWebhookConfigurations(),
caSecret, caSecret,
kyvernoDeployment,
config.TtlValidatingWebhookConfigurationName, config.TtlValidatingWebhookConfigurationName,
config.TtlValidatingWebhookServicePath, config.TtlValidatingWebhookServicePath,
serverIP, serverIP,
@ -295,6 +314,10 @@ func main() {
genericwebhookcontroller.None, genericwebhookcontroller.None,
setup.Configuration, setup.Configuration,
caSecretName, caSecretName,
runtime,
autoDeleteWebhooks,
webhookcontroller.WebhookCleanupSetup(setup.KubeClient, ttlWebhookControllerFinalizerName),
webhookcontroller.WebhookCleanupHandler(setup.KubeClient, ttlWebhookControllerFinalizerName),
), ),
webhookWorkers, webhookWorkers,
) )

View file

@ -51,15 +51,19 @@ import (
corev1 "k8s.io/api/core/v1" corev1 "k8s.io/api/core/v1"
apiserver "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" apiserver "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
kubeinformers "k8s.io/client-go/informers" kubeinformers "k8s.io/client-go/informers"
appsv1informers "k8s.io/client-go/informers/apps/v1"
corev1informers "k8s.io/client-go/informers/core/v1" corev1informers "k8s.io/client-go/informers/core/v1"
"k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes"
kyamlopenapi "sigs.k8s.io/kustomize/kyaml/openapi" kyamlopenapi "sigs.k8s.io/kustomize/kyaml/openapi"
) )
const ( const (
resyncPeriod = 15 * time.Minute resyncPeriod = 15 * time.Minute
exceptionWebhookControllerName = "exception-webhook-controller" exceptionWebhookControllerName = "exception-webhook-controller"
gctxWebhookControllerName = "global-context-webhook-controller" gctxWebhookControllerName = "global-context-webhook-controller"
webhookControllerFinalizerName = "kyverno.io/webhooks"
exceptionControllerFinalizerName = "kyverno.io/exceptionwebhooks"
gctxControllerFinalizerName = "kyverno.io/globalcontextwebhooks"
) )
var ( var (
@ -107,11 +111,13 @@ func createrLeaderControllers(
serverIP string, serverIP string,
webhookTimeout int, webhookTimeout int,
autoUpdateWebhooks bool, autoUpdateWebhooks bool,
autoDeleteWebhooks bool,
kubeInformer kubeinformers.SharedInformerFactory, kubeInformer kubeinformers.SharedInformerFactory,
kubeKyvernoInformer kubeinformers.SharedInformerFactory, kubeKyvernoInformer kubeinformers.SharedInformerFactory,
kyvernoInformer kyvernoinformer.SharedInformerFactory, kyvernoInformer kyvernoinformer.SharedInformerFactory,
caInformer corev1informers.SecretInformer, caInformer corev1informers.SecretInformer,
tlsInformer corev1informers.SecretInformer, tlsInformer corev1informers.SecretInformer,
deploymentInformer appsv1informers.DeploymentInformer,
kubeClient kubernetes.Interface, kubeClient kubernetes.Interface,
kyvernoClient versioned.Interface, kyvernoClient versioned.Interface,
dynamicClient dclient.Interface, dynamicClient dclient.Interface,
@ -141,6 +147,7 @@ func createrLeaderControllers(
kubeInformer.Admissionregistration().V1().ValidatingWebhookConfigurations(), kubeInformer.Admissionregistration().V1().ValidatingWebhookConfigurations(),
kyvernoInformer.Kyverno().V1().ClusterPolicies(), kyvernoInformer.Kyverno().V1().ClusterPolicies(),
kyvernoInformer.Kyverno().V1().Policies(), kyvernoInformer.Kyverno().V1().Policies(),
deploymentInformer,
caInformer, caInformer,
kubeKyvernoInformer.Coordination().V1().Leases(), kubeKyvernoInformer.Coordination().V1().Leases(),
kubeInformer.Rbac().V1().ClusterRoles(), kubeInformer.Rbac().V1().ClusterRoles(),
@ -150,16 +157,20 @@ func createrLeaderControllers(
servicePort, servicePort,
webhookServerPort, webhookServerPort,
autoUpdateWebhooks, autoUpdateWebhooks,
autoDeleteWebhooks,
admissionReports, admissionReports,
runtime, runtime,
configuration, configuration,
caSecretName, caSecretName,
webhookcontroller.WebhookCleanupSetup(kubeClient, webhookControllerFinalizerName),
webhookcontroller.WebhookCleanupHandler(kubeClient, webhookControllerFinalizerName),
) )
exceptionWebhookController := genericwebhookcontroller.NewController( exceptionWebhookController := genericwebhookcontroller.NewController(
exceptionWebhookControllerName, exceptionWebhookControllerName,
kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations(), kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations(),
kubeInformer.Admissionregistration().V1().ValidatingWebhookConfigurations(), kubeInformer.Admissionregistration().V1().ValidatingWebhookConfigurations(),
caInformer, caInformer,
deploymentInformer,
config.ExceptionValidatingWebhookConfigurationName, config.ExceptionValidatingWebhookConfigurationName,
config.ExceptionValidatingWebhookServicePath, config.ExceptionValidatingWebhookServicePath,
serverIP, serverIP,
@ -181,12 +192,17 @@ func createrLeaderControllers(
genericwebhookcontroller.None, genericwebhookcontroller.None,
configuration, configuration,
caSecretName, caSecretName,
runtime,
autoDeleteWebhooks,
webhookcontroller.WebhookCleanupSetup(kubeClient, exceptionControllerFinalizerName),
webhookcontroller.WebhookCleanupHandler(kubeClient, exceptionControllerFinalizerName),
) )
gctxWebhookController := genericwebhookcontroller.NewController( gctxWebhookController := genericwebhookcontroller.NewController(
gctxWebhookControllerName, gctxWebhookControllerName,
kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations(), kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations(),
kubeInformer.Admissionregistration().V1().ValidatingWebhookConfigurations(), kubeInformer.Admissionregistration().V1().ValidatingWebhookConfigurations(),
caInformer, caInformer,
deploymentInformer,
config.GlobalContextValidatingWebhookConfigurationName, config.GlobalContextValidatingWebhookConfigurationName,
config.GlobalContextValidatingWebhookServicePath, config.GlobalContextValidatingWebhookServicePath,
serverIP, serverIP,
@ -208,6 +224,10 @@ func createrLeaderControllers(
genericwebhookcontroller.None, genericwebhookcontroller.None,
configuration, configuration,
caSecretName, caSecretName,
runtime,
autoDeleteWebhooks,
webhookcontroller.WebhookCleanupSetup(kubeClient, gctxControllerFinalizerName),
webhookcontroller.WebhookCleanupHandler(kubeClient, gctxControllerFinalizerName),
) )
leaderControllers = append(leaderControllers, internal.NewController(certmanager.ControllerName, certManager, certmanager.Workers)) leaderControllers = append(leaderControllers, internal.NewController(certmanager.ControllerName, certManager, certmanager.Workers))
leaderControllers = append(leaderControllers, internal.NewController(webhookcontroller.ControllerName, webhookController, webhookcontroller.Workers)) leaderControllers = append(leaderControllers, internal.NewController(webhookcontroller.ControllerName, webhookController, webhookcontroller.Workers))
@ -241,6 +261,7 @@ func main() {
maxQueuedEvents int maxQueuedEvents int
omitEvents string omitEvents string
autoUpdateWebhooks bool autoUpdateWebhooks bool
autoDeleteWebhooks bool
webhookRegistrationTimeout time.Duration webhookRegistrationTimeout time.Duration
admissionReports bool admissionReports bool
dumpPayload bool dumpPayload bool
@ -261,6 +282,7 @@ func main() {
flagset.StringVar(&omitEvents, "omitEvents", "", "Set this flag to a comma sperated list of PolicyViolation, PolicyApplied, PolicyError, PolicySkipped to disable events, e.g. --omitEvents=PolicyApplied,PolicyViolation") flagset.StringVar(&omitEvents, "omitEvents", "", "Set this flag to a comma sperated list of PolicyViolation, PolicyApplied, PolicyError, PolicySkipped to disable events, e.g. --omitEvents=PolicyApplied,PolicyViolation")
flagset.StringVar(&serverIP, "serverIP", "", "IP address where Kyverno controller runs. Only required if out-of-cluster.") flagset.StringVar(&serverIP, "serverIP", "", "IP address where Kyverno controller runs. Only required if out-of-cluster.")
flagset.BoolVar(&autoUpdateWebhooks, "autoUpdateWebhooks", true, "Set this flag to 'false' to disable auto-configuration of the webhook.") flagset.BoolVar(&autoUpdateWebhooks, "autoUpdateWebhooks", true, "Set this flag to 'false' to disable auto-configuration of the webhook.")
flagset.BoolVar(&autoDeleteWebhooks, "autoDeleteWebhooks", false, "Set this flag to 'true' to enable autodeletion of webhook configurations using finalizers (requires extra permissions).")
flagset.DurationVar(&webhookRegistrationTimeout, "webhookRegistrationTimeout", 120*time.Second, "Timeout for webhook registration, e.g., 30s, 1m, 5m.") flagset.DurationVar(&webhookRegistrationTimeout, "webhookRegistrationTimeout", 120*time.Second, "Timeout for webhook registration, e.g., 30s, 1m, 5m.")
flagset.Func(toggle.ProtectManagedResourcesFlagName, toggle.ProtectManagedResourcesDescription, toggle.ProtectManagedResources.Parse) flagset.Func(toggle.ProtectManagedResourcesFlagName, toggle.ProtectManagedResourcesDescription, toggle.ProtectManagedResources.Parse)
flagset.Func(toggle.ForceFailurePolicyIgnoreFlagName, toggle.ForceFailurePolicyIgnoreDescription, toggle.ForceFailurePolicyIgnore.Parse) flagset.Func(toggle.ForceFailurePolicyIgnoreFlagName, toggle.ForceFailurePolicyIgnoreDescription, toggle.ForceFailurePolicyIgnore.Parse)
@ -324,7 +346,8 @@ func main() {
} }
caSecret := informers.NewSecretInformer(setup.KubeClient, config.KyvernoNamespace(), caSecretName, resyncPeriod) caSecret := informers.NewSecretInformer(setup.KubeClient, config.KyvernoNamespace(), caSecretName, resyncPeriod)
tlsSecret := informers.NewSecretInformer(setup.KubeClient, config.KyvernoNamespace(), tlsSecretName, resyncPeriod) tlsSecret := informers.NewSecretInformer(setup.KubeClient, config.KyvernoNamespace(), tlsSecretName, resyncPeriod)
if !informers.StartInformersAndWaitForCacheSync(signalCtx, setup.Logger, caSecret, tlsSecret) { kyvernoDeployment := informers.NewDeploymentInformer(setup.KubeClient, config.KyvernoNamespace(), config.KyvernoDeploymentName(), resyncPeriod)
if !informers.StartInformersAndWaitForCacheSync(signalCtx, setup.Logger, caSecret, tlsSecret, kyvernoDeployment) {
setup.Logger.Error(errors.New("failed to wait for cache sync"), "failed to wait for cache sync") setup.Logger.Error(errors.New("failed to wait for cache sync"), "failed to wait for cache sync")
os.Exit(1) os.Exit(1)
} }
@ -342,6 +365,7 @@ func main() {
kubeInformer := kubeinformers.NewSharedInformerFactory(setup.KubeClient, resyncPeriod) kubeInformer := kubeinformers.NewSharedInformerFactory(setup.KubeClient, resyncPeriod)
kubeKyvernoInformer := kubeinformers.NewSharedInformerFactoryWithOptions(setup.KubeClient, resyncPeriod, kubeinformers.WithNamespace(config.KyvernoNamespace())) kubeKyvernoInformer := kubeinformers.NewSharedInformerFactoryWithOptions(setup.KubeClient, resyncPeriod, kubeinformers.WithNamespace(config.KyvernoNamespace()))
kyvernoInformer := kyvernoinformer.NewSharedInformerFactory(setup.KyvernoClient, resyncPeriod) kyvernoInformer := kyvernoinformer.NewSharedInformerFactory(setup.KyvernoClient, resyncPeriod)
certRenewer := tls.NewCertRenewer( certRenewer := tls.NewCertRenewer(
setup.KubeClient.CoreV1().Secrets(config.KyvernoNamespace()), setup.KubeClient.CoreV1().Secrets(config.KyvernoNamespace()),
tls.CertRenewalInterval, tls.CertRenewalInterval,
@ -463,11 +487,13 @@ func main() {
serverIP, serverIP,
webhookTimeout, webhookTimeout,
autoUpdateWebhooks, autoUpdateWebhooks,
autoDeleteWebhooks,
kubeInformer, kubeInformer,
kubeKyvernoInformer, kubeKyvernoInformer,
kyvernoInformer, kyvernoInformer,
caSecret, caSecret,
tlsSecret, tlsSecret,
kyvernoDeployment,
setup.KubeClient, setup.KubeClient,
setup.KyvernoClient, setup.KyvernoClient,
setup.KyvernoDynamicClient, setup.KyvernoDynamicClient,

View file

@ -48730,10 +48730,12 @@ rules:
- '' - ''
resources: resources:
- secrets - secrets
- serviceaccounts
verbs: verbs:
- get - get
- list - list
- watch - watch
- patch
- create - create
- update - update
- delete - delete
@ -48874,6 +48876,14 @@ rules:
- update - update
resourceNames: resourceNames:
- kyverno-cleanup-controller - kyverno-cleanup-controller
- apiGroups:
- apps
resources:
- deployments
verbs:
- get
- list
- watch
--- ---
apiVersion: rbac.authorization.k8s.io/v1 apiVersion: rbac.authorization.k8s.io/v1
kind: Role kind: Role
@ -49205,6 +49215,8 @@ spec:
env: env:
- name: KYVERNO_SERVICEACCOUNT_NAME - name: KYVERNO_SERVICEACCOUNT_NAME
value: kyverno-admission-controller value: kyverno-admission-controller
- name: KYVERNO_ROLE_NAME
value: kyverno:admission-controller
- name: INIT_CONFIG - name: INIT_CONFIG
value: kyverno value: kyverno
- name: METRICS_CONFIG - name: METRICS_CONFIG
@ -49291,6 +49303,8 @@ spec:
fieldPath: metadata.name fieldPath: metadata.name
- name: KYVERNO_SERVICEACCOUNT_NAME - name: KYVERNO_SERVICEACCOUNT_NAME
value: kyverno-admission-controller value: kyverno-admission-controller
- name: KYVERNO_ROLE_NAME
value: kyverno:admission-controller
- name: KYVERNO_SVC - name: KYVERNO_SVC
value: kyverno-svc value: kyverno-svc
- name: TUF_ROOT - name: TUF_ROOT
@ -49522,6 +49536,8 @@ spec:
fieldPath: metadata.name fieldPath: metadata.name
- name: KYVERNO_SERVICEACCOUNT_NAME - name: KYVERNO_SERVICEACCOUNT_NAME
value: kyverno-cleanup-controller value: kyverno-cleanup-controller
- name: KYVERNO_ROLE_NAME
value: kyverno:cleanup-controller
- name: KYVERNO_NAMESPACE - name: KYVERNO_NAMESPACE
valueFrom: valueFrom:
fieldRef: fieldRef:

View file

@ -106,6 +106,8 @@ var (
kyvernoNamespace = osutils.GetEnvWithFallback("KYVERNO_NAMESPACE", "kyverno") kyvernoNamespace = osutils.GetEnvWithFallback("KYVERNO_NAMESPACE", "kyverno")
// kyvernoServiceAccountName is the Kyverno service account name // kyvernoServiceAccountName is the Kyverno service account name
kyvernoServiceAccountName = osutils.GetEnvWithFallback("KYVERNO_SERVICEACCOUNT_NAME", "kyverno") kyvernoServiceAccountName = osutils.GetEnvWithFallback("KYVERNO_SERVICEACCOUNT_NAME", "kyverno")
// kyvernoRoleName is the Kyverno rbac name
kyvernoRoleName = osutils.GetEnvWithFallback("KYVERNO_ROLE_NAME", "kyverno")
// kyvernoDeploymentName is the Kyverno deployment name // kyvernoDeploymentName is the Kyverno deployment name
kyvernoDeploymentName = osutils.GetEnvWithFallback("KYVERNO_DEPLOYMENT", "kyverno") kyvernoDeploymentName = osutils.GetEnvWithFallback("KYVERNO_DEPLOYMENT", "kyverno")
// kyvernoServiceName is the Kyverno service name // kyvernoServiceName is the Kyverno service name
@ -132,6 +134,10 @@ func KyvernoServiceAccountName() string {
return kyvernoServiceAccountName return kyvernoServiceAccountName
} }
func KyvernoRoleName() string {
return kyvernoRoleName
}
func KyvernoDeploymentName() string { func KyvernoDeploymentName() string {
return kyvernoDeploymentName return kyvernoDeploymentName
} }

View file

@ -12,14 +12,18 @@ import (
"github.com/kyverno/kyverno/pkg/logging" "github.com/kyverno/kyverno/pkg/logging"
"github.com/kyverno/kyverno/pkg/tls" "github.com/kyverno/kyverno/pkg/tls"
controllerutils "github.com/kyverno/kyverno/pkg/utils/controller" controllerutils "github.com/kyverno/kyverno/pkg/utils/controller"
runtimeutils "github.com/kyverno/kyverno/pkg/utils/runtime"
"golang.org/x/exp/maps" "golang.org/x/exp/maps"
admissionregistrationv1 "k8s.io/api/admissionregistration/v1" admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1" corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors" apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
admissionregistrationv1informers "k8s.io/client-go/informers/admissionregistration/v1" admissionregistrationv1informers "k8s.io/client-go/informers/admissionregistration/v1"
appsv1informers "k8s.io/client-go/informers/apps/v1"
corev1informers "k8s.io/client-go/informers/core/v1" corev1informers "k8s.io/client-go/informers/core/v1"
admissionregistrationv1listers "k8s.io/client-go/listers/admissionregistration/v1" admissionregistrationv1listers "k8s.io/client-go/listers/admissionregistration/v1"
appsv1listers "k8s.io/client-go/listers/apps/v1"
corev1listers "k8s.io/client-go/listers/core/v1" corev1listers "k8s.io/client-go/listers/core/v1"
"k8s.io/client-go/util/workqueue" "k8s.io/client-go/util/workqueue"
) )
@ -42,25 +46,31 @@ type controller struct {
vwcClient controllerutils.ObjectClient[*admissionregistrationv1.ValidatingWebhookConfiguration] vwcClient controllerutils.ObjectClient[*admissionregistrationv1.ValidatingWebhookConfiguration]
// listers // listers
vwcLister admissionregistrationv1listers.ValidatingWebhookConfigurationLister vwcLister admissionregistrationv1listers.ValidatingWebhookConfigurationLister
secretLister corev1listers.SecretNamespaceLister secretLister corev1listers.SecretNamespaceLister
deploymentLister appsv1listers.DeploymentNamespaceLister
// queue // queue
queue workqueue.TypedRateLimitingInterface[any] queue workqueue.TypedRateLimitingInterface[any]
// config // config
controllerName string controllerName string
logger logr.Logger logger logr.Logger
webhookName string webhookName string
path string path string
server string server string
servicePort int32 servicePort int32
rules []admissionregistrationv1.RuleWithOperations rules []admissionregistrationv1.RuleWithOperations
failurePolicy *admissionregistrationv1.FailurePolicyType failurePolicy *admissionregistrationv1.FailurePolicyType
sideEffects *admissionregistrationv1.SideEffectClass sideEffects *admissionregistrationv1.SideEffectClass
configuration config.Configuration runtime runtimeutils.Runtime
labelSelector *metav1.LabelSelector configuration config.Configuration
caSecretName string labelSelector *metav1.LabelSelector
caSecretName string
webhooksDeleted bool
autoDeleteWebhooks bool
webhookCleanupSetup func(context.Context, logr.Logger) error
postWebhookCleanup func(context.Context, logr.Logger) error
} }
func NewController( func NewController(
@ -68,6 +78,7 @@ func NewController(
vwcClient controllerutils.ObjectClient[*admissionregistrationv1.ValidatingWebhookConfiguration], vwcClient controllerutils.ObjectClient[*admissionregistrationv1.ValidatingWebhookConfiguration],
vwcInformer admissionregistrationv1informers.ValidatingWebhookConfigurationInformer, vwcInformer admissionregistrationv1informers.ValidatingWebhookConfigurationInformer,
secretInformer corev1informers.SecretInformer, secretInformer corev1informers.SecretInformer,
deploymentInformer appsv1informers.DeploymentInformer,
webhookName string, webhookName string,
path string, path string,
server string, server string,
@ -79,25 +90,34 @@ func NewController(
sideEffects *admissionregistrationv1.SideEffectClass, sideEffects *admissionregistrationv1.SideEffectClass,
configuration config.Configuration, configuration config.Configuration,
caSecretName string, caSecretName string,
runtime runtimeutils.Runtime,
autoDeleteWebhooks bool,
webhookCleanupSetup func(context.Context, logr.Logger) error,
postWebhookCleanup func(context.Context, logr.Logger) error,
) controllers.Controller { ) controllers.Controller {
queue := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultTypedControllerRateLimiter[any](), controllerName) queue := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultTypedControllerRateLimiter[any](), controllerName)
c := controller{ c := controller{
vwcClient: vwcClient, vwcClient: vwcClient,
vwcLister: vwcInformer.Lister(), vwcLister: vwcInformer.Lister(),
secretLister: secretInformer.Lister().Secrets(config.KyvernoNamespace()), secretLister: secretInformer.Lister().Secrets(config.KyvernoNamespace()),
queue: queue, deploymentLister: deploymentInformer.Lister().Deployments(config.KyvernoNamespace()),
controllerName: controllerName, queue: queue,
logger: logging.ControllerLogger(controllerName), controllerName: controllerName,
webhookName: webhookName, logger: logging.ControllerLogger(controllerName),
path: path, webhookName: webhookName,
server: server, path: path,
servicePort: servicePort, server: server,
rules: rules, servicePort: servicePort,
failurePolicy: failurePolicy, rules: rules,
sideEffects: sideEffects, failurePolicy: failurePolicy,
configuration: configuration, sideEffects: sideEffects,
labelSelector: labelSelector, configuration: configuration,
caSecretName: caSecretName, labelSelector: labelSelector,
caSecretName: caSecretName,
runtime: runtime,
autoDeleteWebhooks: autoDeleteWebhooks,
webhookCleanupSetup: webhookCleanupSetup,
postWebhookCleanup: postWebhookCleanup,
} }
if _, _, err := controllerutils.AddDefaultEventHandlers(c.logger, vwcInformer.Informer(), queue); err != nil { if _, _, err := controllerutils.AddDefaultEventHandlers(c.logger, vwcInformer.Informer(), queue); err != nil {
c.logger.Error(err, "failed to register event handlers") c.logger.Error(err, "failed to register event handlers")
@ -122,11 +142,34 @@ func NewController(
); err != nil { ); err != nil {
c.logger.Error(err, "failed to register event handlers") c.logger.Error(err, "failed to register event handlers")
} }
if autoDeleteWebhooks {
if _, err := controllerutils.AddEventHandlersT(
deploymentInformer.Informer(),
func(obj *appsv1.Deployment) {
},
func(_, obj *appsv1.Deployment) {
if obj.GetNamespace() == config.KyvernoNamespace() && obj.GetName() == config.KyvernoDeploymentName() {
c.enqueueCleanupAfter(1 * time.Second)
}
},
func(obj *appsv1.Deployment) {
if obj.GetNamespace() == config.KyvernoNamespace() && obj.GetName() == config.KyvernoDeploymentName() {
c.enqueueCleanup()
}
},
); err != nil {
c.logger.Error(err, "failed to register event handlers")
}
}
configuration.OnChanged(c.enqueue) configuration.OnChanged(c.enqueue)
return &c return &c
} }
func (c *controller) Run(ctx context.Context, workers int) { func (c *controller) Run(ctx context.Context, workers int) {
if err := c.webhookCleanupSetup(ctx, c.logger); err != nil {
c.logger.Error(err, "failed to setup webhook cleanup")
}
c.enqueue() c.enqueue()
controllerutils.Run(ctx, c.logger, c.controllerName, time.Second, c.queue, workers, maxRetries, c.reconcile) controllerutils.Run(ctx, c.logger, c.controllerName, time.Second, c.queue, workers, maxRetries, c.reconcile)
} }
@ -135,7 +178,22 @@ func (c *controller) enqueue() {
c.queue.Add(c.webhookName) c.queue.Add(c.webhookName)
} }
func (c *controller) enqueueCleanup() {
c.queue.Add(config.KyvernoDeploymentName())
}
func (c *controller) enqueueCleanupAfter(duration time.Duration) {
c.queue.AddAfter(config.KyvernoDeploymentName(), duration)
}
func (c *controller) reconcile(ctx context.Context, logger logr.Logger, key, _, _ string) error { func (c *controller) reconcile(ctx context.Context, logger logr.Logger, key, _, _ string) error {
if c.autoDeleteWebhooks && c.runtime.IsGoingDown() {
return c.reconcileWebhookDeletion(ctx)
}
if c.autoDeleteWebhooks && key == config.KyvernoDeploymentName() {
return c.reconcileWebhookDeletion(ctx)
}
if key != c.webhookName { if key != c.webhookName {
return nil return nil
} }
@ -165,6 +223,37 @@ func (c *controller) reconcile(ctx context.Context, logger logr.Logger, key, _,
return err return err
} }
func (c *controller) reconcileWebhookDeletion(ctx context.Context) error {
if c.autoDeleteWebhooks {
if c.runtime.IsGoingDown() {
if c.webhooksDeleted {
return nil
}
c.webhooksDeleted = true
if err := c.vwcClient.Delete(ctx, c.webhookName, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) {
c.logger.Error(err, "failed to clean up validating webhook configuration", "label", kyverno.LabelWebhookManagedBy)
return err
} else if err == nil {
c.logger.Info("successfully deleted validating webhook configurations", "label", kyverno.LabelWebhookManagedBy)
}
if err := c.postWebhookCleanup(ctx, c.logger); err != nil {
c.logger.Error(err, "failed to clean up temporary rbac")
return err
} else {
c.logger.Info("successfully deleted temporary rbac")
}
} else {
if err := c.webhookCleanupSetup(ctx, c.logger); err != nil {
c.logger.Error(err, "failed to reconcile webhook cleanup setup")
return err
}
c.logger.Info("reconciled webhook cleanup setup")
}
}
return nil
}
func objectMeta(name string, annotations map[string]string, labels map[string]string, owner ...metav1.OwnerReference) metav1.ObjectMeta { func objectMeta(name string, annotations map[string]string, labels map[string]string, owner ...metav1.OwnerReference) metav1.ObjectMeta {
desiredLabels := make(map[string]string) desiredLabels := make(map[string]string)
defaultLabels := map[string]string{ defaultLabels := map[string]string{

View file

@ -0,0 +1,282 @@
package webhook
import (
"context"
"github.com/go-logr/logr"
"github.com/kyverno/kyverno/pkg/config"
controllerutils "github.com/kyverno/kyverno/pkg/utils/controller"
rbacv1 "k8s.io/api/rbac/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/util/retry"
)
// WebhookCleanupSetup creates temporary rbac owned by kyverno resources, these roles and cluster roles get automatically deleted when kyverno is uninstalled
// It creates the following resources:
// 1. Creates a temporary cluster role binding to give permission to delete kyverno's cluster role and set its owner ref to aggregated cluster role itself.
// 2. Creates a temporary role and role binding with permissions to delete a service account, roles and role bindings with owner ref set to the service account.
func WebhookCleanupSetup(
kubeClient kubernetes.Interface,
finalizer string,
) func(context.Context, logr.Logger) error {
return func(ctx context.Context, logger logr.Logger) error {
name := config.KyvernoRoleName()
coreName := name + ":core"
tempRbacName := name + ":temporary"
// create temporary rbac
cr, err := kubeClient.RbacV1().ClusterRoles().Get(ctx, coreName, metav1.GetOptions{})
if err != nil {
logger.Error(err, "failed to get cluster role binding")
return err
}
coreClusterRoleBinding := &rbacv1.ClusterRoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: coreName,
OwnerReferences: []metav1.OwnerReference{
{
APIVersion: "rbac.authorization.k8s.io/v1",
Kind: "ClusterRole",
Name: cr.Name,
UID: cr.UID,
},
},
},
Subjects: []rbacv1.Subject{
{
Kind: "ServiceAccount",
Name: config.KyvernoServiceAccountName(),
Namespace: config.KyvernoNamespace(),
APIGroup: "",
},
},
RoleRef: rbacv1.RoleRef{
APIGroup: "rbac.authorization.k8s.io",
Kind: "ClusterRole",
Name: coreName,
},
}
if crb, err := kubeClient.RbacV1().ClusterRoleBindings().Create(ctx, coreClusterRoleBinding, metav1.CreateOptions{}); err != nil && !apierrors.IsAlreadyExists(err) {
logger.Error(err, "failed to create temporary clusterrolebinding", "name", crb.Name)
return err
} else if !apierrors.IsAlreadyExists(err) {
logger.V(4).Info("temporary clusterrolebinding created", "clusterrolebinding", crb.Name)
}
// create temporary rbac
sa, err := kubeClient.CoreV1().ServiceAccounts(config.KyvernoNamespace()).Get(ctx, config.KyvernoServiceAccountName(), metav1.GetOptions{})
if err != nil {
logger.Error(err, "failed to get service account")
return err
}
role := &rbacv1.Role{
ObjectMeta: metav1.ObjectMeta{
Name: tempRbacName,
Namespace: config.KyvernoNamespace(),
OwnerReferences: []metav1.OwnerReference{
{
APIVersion: "v1",
Kind: "ServiceAccount",
Name: sa.Name,
UID: sa.UID,
},
},
},
Rules: []rbacv1.PolicyRule{
{
APIGroups: []string{""},
Resources: []string{"serviceaccounts"},
ResourceNames: []string{config.KyvernoServiceAccountName()},
Verbs: []string{"get", "update", "delete"},
},
{
APIGroups: []string{"rbac.authorization.k8s.io"},
Resources: []string{"rolebindings", "roles"},
ResourceNames: []string{name},
Verbs: []string{"get", "update"},
},
{
APIGroups: []string{"apps"},
Resources: []string{"deployments"},
ResourceNames: []string{config.KyvernoDeploymentName()},
Verbs: []string{"get", "update"},
},
},
}
if r, err := kubeClient.RbacV1().Roles(config.KyvernoNamespace()).Create(ctx, role, metav1.CreateOptions{}); err != nil && !apierrors.IsAlreadyExists(err) {
logger.Error(err, "failed to create temporary role", "name", r.Name)
return err
} else if !apierrors.IsAlreadyExists(err) {
logger.V(4).Info("temporary role created in kyverno namespace", "role", r.Name, "namespace", r.Namespace)
}
roleBinding := &rbacv1.RoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: tempRbacName,
Namespace: config.KyvernoNamespace(),
OwnerReferences: []metav1.OwnerReference{
{
APIVersion: "v1",
Kind: "ServiceAccount",
Name: sa.Name,
UID: sa.UID,
},
},
},
Subjects: []rbacv1.Subject{
{
Kind: "ServiceAccount",
Name: config.KyvernoServiceAccountName(),
Namespace: config.KyvernoNamespace(),
APIGroup: "",
},
},
RoleRef: rbacv1.RoleRef{
APIGroup: "rbac.authorization.k8s.io",
Kind: "Role",
Name: tempRbacName,
},
}
if rb, err := kubeClient.RbacV1().RoleBindings(config.KyvernoNamespace()).Create(ctx, roleBinding, metav1.CreateOptions{}); err != nil && !apierrors.IsAlreadyExists(err) {
logger.Error(err, "failed to create temporary rolebinding", "name", rb.Name)
return err
} else if !apierrors.IsAlreadyExists(err) {
logger.V(4).Info("temporary rolebinding created in kyverno namespace", "rolebinding", rb.Name, "namespace", rb.Namespace)
}
// Add finalizers
if err := AddFinalizers(ctx, kubeClient.RbacV1().ClusterRoles(), coreName, finalizer); err != nil && !apierrors.IsNotFound(err) {
logger.Error(err, "failed to add finalizer to clusterrole", "name", coreName)
return err
}
if err := AddFinalizers(ctx, kubeClient.RbacV1().RoleBindings(config.KyvernoNamespace()), name, finalizer); err != nil {
logger.Error(err, "failed to add finalizer to rolebindings", "name", name, "namespace", config.KyvernoNamespace())
return err
}
if err := AddFinalizers(ctx, kubeClient.RbacV1().Roles(config.KyvernoNamespace()), name, finalizer); err != nil {
logger.Error(err, "failed to add finalizer to role", "name", name, "namespace", config.KyvernoNamespace())
return err
}
if err := AddFinalizers(ctx, kubeClient.AppsV1().Deployments(config.KyvernoNamespace()), config.KyvernoDeploymentName(), finalizer); err != nil {
logger.Error(err, "failed to add finalizer to deployment", "name", config.KyvernoDeploymentName(), "namespace", config.KyvernoNamespace())
return err
}
if err := AddFinalizers(ctx, kubeClient.CoreV1().ServiceAccounts(config.KyvernoNamespace()), config.KyvernoServiceAccountName(), finalizer); err != nil {
logger.Error(err, "failed to add finalizer to serviceaccount", "name", config.KyvernoServiceAccountName(), "namespace", config.KyvernoNamespace())
return err
}
return nil
}
}
// WebhookCleanupHandler is run after webhook configuration cleanup is performed to delete roles and service account.
// Admission controller cluster and namespaced roles and role bindings have finalizers to block their deletion until admission controller terminates.
// This handler removes the finalizers on roles and service account after they are used to cleanup webhook cfg.
// It does the following:
//
// Deletes the cluster scoped rbac in order:
// a. Removes finalizers from controller cluster role binding
// b. Removes finalizers from controller core cluster role
// c. Removes finalizers from controller aggregated cluster role
// d. Temporary cluster role and cluster role binding created by WebhookCleanupSetup gets garbage collected after (c) automatically
//
// Deletes the namespace scoped rbac in order:
// a. Removes finalizers from controller role binding.
// b. Removes finalizers from controller role.
// c. Removes finalizers from controller service account
// d. Temporary role and role binding created by WebhookCleanupSetup gets garbage collected after (c) automatically
func WebhookCleanupHandler(
kubeClient kubernetes.Interface,
finalizer string,
) func(context.Context, logr.Logger) error {
return func(ctx context.Context, logger logr.Logger) error {
name := config.KyvernoRoleName()
coreName := name + ":core"
// cleanup cluster scoped rbac
if err := DeleteFinalizers(ctx, kubeClient.RbacV1().ClusterRoles(), coreName, finalizer); err != nil {
logger.Error(err, "failed to delete finalizer from clusterrole", "name", coreName)
return err
}
// cleanup namespace scoped rbac
if err := DeleteFinalizers(ctx, kubeClient.RbacV1().RoleBindings(config.KyvernoNamespace()), name, finalizer); err != nil {
logger.Error(err, "failed to delete finalizer from rolebindings", "name", name, "namespace", config.KyvernoNamespace())
return err
}
if err := DeleteFinalizers(ctx, kubeClient.RbacV1().Roles(config.KyvernoNamespace()), name, finalizer); err != nil {
logger.Error(err, "failed to delete finalizer from role", "name", name, "namespace", config.KyvernoNamespace())
return err
}
if err := DeleteFinalizers(ctx, kubeClient.AppsV1().Deployments(config.KyvernoNamespace()), config.KyvernoDeploymentName(), finalizer); err != nil {
logger.Error(err, "failed to delete finalizer from deployment", "name", config.KyvernoDeploymentName(), "namespace", config.KyvernoNamespace())
return err
}
if err := DeleteFinalizers(ctx, kubeClient.CoreV1().ServiceAccounts(config.KyvernoNamespace()), config.KyvernoServiceAccountName(), finalizer); err != nil {
logger.Error(err, "failed to delete finalizer from serviceaccount", "name", config.KyvernoServiceAccountName(), "namespace", config.KyvernoNamespace())
return err
}
return nil
}
}
func DeleteFinalizers[T metav1.Object](ctx context.Context, client controllerutils.ObjectClient[T], name, finalizer string) error {
return retry.RetryOnConflict(retry.DefaultRetry, func() error {
obj, err := client.Get(ctx, name, metav1.GetOptions{})
if err != nil {
return err
}
finalizers := make([]string, 0)
for _, f := range obj.GetFinalizers() {
if f != finalizer {
finalizers = append(finalizers, f)
}
}
obj.SetFinalizers(finalizers)
_, err = client.Update(ctx, obj, metav1.UpdateOptions{})
if err != nil {
return err
}
return nil
})
}
func AddFinalizers[T metav1.Object](ctx context.Context, client controllerutils.ObjectClient[T], name, finalizer string) error {
return retry.RetryOnConflict(retry.DefaultRetry, func() error {
obj, err := client.Get(ctx, name, metav1.GetOptions{})
if err != nil {
return err
}
finalizers := obj.GetFinalizers()
for _, f := range finalizers {
if f == finalizer {
return nil
}
}
finalizers = append(finalizers, finalizer)
obj.SetFinalizers(finalizers)
_, err = client.Update(ctx, obj, metav1.UpdateOptions{})
if err != nil {
return err
}
return nil
})
}

View file

@ -26,6 +26,7 @@ import (
kubeutils "github.com/kyverno/kyverno/pkg/utils/kube" kubeutils "github.com/kyverno/kyverno/pkg/utils/kube"
runtimeutils "github.com/kyverno/kyverno/pkg/utils/runtime" runtimeutils "github.com/kyverno/kyverno/pkg/utils/runtime"
admissionregistrationv1 "k8s.io/api/admissionregistration/v1" admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
appsv1 "k8s.io/api/apps/v1"
coordinationv1 "k8s.io/api/coordination/v1" coordinationv1 "k8s.io/api/coordination/v1"
corev1 "k8s.io/api/core/v1" corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors" apierrors "k8s.io/apimachinery/pkg/api/errors"
@ -34,10 +35,12 @@ import (
"k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/sets"
admissionregistrationv1informers "k8s.io/client-go/informers/admissionregistration/v1" admissionregistrationv1informers "k8s.io/client-go/informers/admissionregistration/v1"
appsv1informers "k8s.io/client-go/informers/apps/v1"
coordinationv1informers "k8s.io/client-go/informers/coordination/v1" coordinationv1informers "k8s.io/client-go/informers/coordination/v1"
corev1informers "k8s.io/client-go/informers/core/v1" corev1informers "k8s.io/client-go/informers/core/v1"
rbacv1informers "k8s.io/client-go/informers/rbac/v1" rbacv1informers "k8s.io/client-go/informers/rbac/v1"
admissionregistrationv1listers "k8s.io/client-go/listers/admissionregistration/v1" admissionregistrationv1listers "k8s.io/client-go/listers/admissionregistration/v1"
appsv1listers "k8s.io/client-go/listers/apps/v1"
coordinationv1listers "k8s.io/client-go/listers/coordination/v1" coordinationv1listers "k8s.io/client-go/listers/coordination/v1"
corev1listers "k8s.io/client-go/listers/core/v1" corev1listers "k8s.io/client-go/listers/core/v1"
rbacv1listers "k8s.io/client-go/listers/rbac/v1" rbacv1listers "k8s.io/client-go/listers/rbac/v1"
@ -91,6 +94,7 @@ type controller struct {
vwcLister admissionregistrationv1listers.ValidatingWebhookConfigurationLister vwcLister admissionregistrationv1listers.ValidatingWebhookConfigurationLister
cpolLister kyvernov1listers.ClusterPolicyLister cpolLister kyvernov1listers.ClusterPolicyLister
polLister kyvernov1listers.PolicyLister polLister kyvernov1listers.PolicyLister
deploymentLister appsv1listers.DeploymentLister
secretLister corev1listers.SecretLister secretLister corev1listers.SecretLister
leaseLister coordinationv1listers.LeaseLister leaseLister coordinationv1listers.LeaseLister
clusterroleLister rbacv1listers.ClusterRoleLister clusterroleLister rbacv1listers.ClusterRoleLister
@ -100,14 +104,18 @@ type controller struct {
queue workqueue.TypedRateLimitingInterface[any] queue workqueue.TypedRateLimitingInterface[any]
// config // config
server string server string
defaultTimeout int32 defaultTimeout int32
servicePort int32 servicePort int32
autoUpdateWebhooks bool autoUpdateWebhooks bool
admissionReports bool autoDeleteWebhooks bool
runtime runtimeutils.Runtime admissionReports bool
configuration config.Configuration runtime runtimeutils.Runtime
caSecretName string configuration config.Configuration
caSecretName string
webhooksDeleted bool
webhookCleanupSetup func(context.Context, logr.Logger) error
postWebhookCleanup func(context.Context, logr.Logger) error
// state // state
lock sync.Mutex lock sync.Mutex
@ -124,6 +132,7 @@ func NewController(
vwcInformer admissionregistrationv1informers.ValidatingWebhookConfigurationInformer, vwcInformer admissionregistrationv1informers.ValidatingWebhookConfigurationInformer,
cpolInformer kyvernov1informers.ClusterPolicyInformer, cpolInformer kyvernov1informers.ClusterPolicyInformer,
polInformer kyvernov1informers.PolicyInformer, polInformer kyvernov1informers.PolicyInformer,
deploymentInformer appsv1informers.DeploymentInformer,
secretInformer corev1informers.SecretInformer, secretInformer corev1informers.SecretInformer,
leaseInformer coordinationv1informers.LeaseInformer, leaseInformer coordinationv1informers.LeaseInformer,
clusterroleInformer rbacv1informers.ClusterRoleInformer, clusterroleInformer rbacv1informers.ClusterRoleInformer,
@ -133,35 +142,42 @@ func NewController(
servicePort int32, servicePort int32,
webhookServerPort int32, webhookServerPort int32,
autoUpdateWebhooks bool, autoUpdateWebhooks bool,
autoDeleteWebhooks bool,
admissionReports bool, admissionReports bool,
runtime runtimeutils.Runtime, runtime runtimeutils.Runtime,
configuration config.Configuration, configuration config.Configuration,
caSecretName string, caSecretName string,
webhookCleanupSetup func(context.Context, logr.Logger) error,
postWebhookCleanup func(context.Context, logr.Logger) error,
) controllers.Controller { ) controllers.Controller {
queue := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultTypedControllerRateLimiter[any](), ControllerName) queue := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultTypedControllerRateLimiter[any](), ControllerName)
c := controller{ c := controller{
discoveryClient: discoveryClient, discoveryClient: discoveryClient,
mwcClient: mwcClient, mwcClient: mwcClient,
vwcClient: vwcClient, vwcClient: vwcClient,
leaseClient: leaseClient, leaseClient: leaseClient,
kyvernoClient: kyvernoClient, kyvernoClient: kyvernoClient,
mwcLister: mwcInformer.Lister(), mwcLister: mwcInformer.Lister(),
vwcLister: vwcInformer.Lister(), vwcLister: vwcInformer.Lister(),
cpolLister: cpolInformer.Lister(), cpolLister: cpolInformer.Lister(),
polLister: polInformer.Lister(), polLister: polInformer.Lister(),
secretLister: secretInformer.Lister(), deploymentLister: deploymentInformer.Lister(),
leaseLister: leaseInformer.Lister(), secretLister: secretInformer.Lister(),
clusterroleLister: clusterroleInformer.Lister(), leaseLister: leaseInformer.Lister(),
gctxentryLister: gctxentryInformer.Lister(), clusterroleLister: clusterroleInformer.Lister(),
queue: queue, gctxentryLister: gctxentryInformer.Lister(),
server: server, queue: queue,
defaultTimeout: defaultTimeout, server: server,
servicePort: servicePort, defaultTimeout: defaultTimeout,
autoUpdateWebhooks: autoUpdateWebhooks, servicePort: servicePort,
admissionReports: admissionReports, autoUpdateWebhooks: autoUpdateWebhooks,
runtime: runtime, autoDeleteWebhooks: autoDeleteWebhooks,
configuration: configuration, admissionReports: admissionReports,
caSecretName: caSecretName, runtime: runtime,
configuration: configuration,
caSecretName: caSecretName,
webhookCleanupSetup: webhookCleanupSetup,
postWebhookCleanup: postWebhookCleanup,
policyState: map[string]sets.Set[string]{ policyState: map[string]sets.Set[string]{
config.MutatingWebhookConfigurationName: sets.New[string](), config.MutatingWebhookConfigurationName: sets.New[string](),
config.ValidatingWebhookConfigurationName: sets.New[string](), config.ValidatingWebhookConfigurationName: sets.New[string](),
@ -193,6 +209,25 @@ func NewController(
); err != nil { ); err != nil {
logger.Error(err, "failed to register event handlers") logger.Error(err, "failed to register event handlers")
} }
if autoDeleteWebhooks {
if _, err := controllerutils.AddEventHandlersT(
deploymentInformer.Informer(),
func(obj *appsv1.Deployment) {
},
func(_, obj *appsv1.Deployment) {
if obj.GetNamespace() == config.KyvernoNamespace() && obj.GetName() == config.KyvernoDeploymentName() {
c.enqueueCleanupAfter(1 * time.Second)
}
},
func(obj *appsv1.Deployment) {
if obj.GetNamespace() == config.KyvernoNamespace() && obj.GetName() == config.KyvernoDeploymentName() {
c.enqueueCleanup()
}
},
); err != nil {
logger.Error(err, "failed to register event handlers")
}
}
if _, err := controllerutils.AddEventHandlers( if _, err := controllerutils.AddEventHandlers(
cpolInformer.Informer(), cpolInformer.Informer(),
func(interface{}) { c.enqueueResourceWebhooks(0) }, func(interface{}) { c.enqueueResourceWebhooks(0) },
@ -214,6 +249,9 @@ func NewController(
} }
func (c *controller) Run(ctx context.Context, workers int) { func (c *controller) Run(ctx context.Context, workers int) {
if err := c.webhookCleanupSetup(ctx, logger); err != nil {
logger.Error(err, "failed to setup webhook cleanup")
}
// add our known webhooks to the queue // add our known webhooks to the queue
c.enqueueAll() c.enqueueAll()
controllerutils.Run(ctx, logger, ControllerName, time.Second, c.queue, workers, maxRetries, c.reconcile, c.watchdog) controllerutils.Run(ctx, logger, ControllerName, time.Second, c.queue, workers, maxRetries, c.reconcile, c.watchdog)
@ -286,6 +324,14 @@ func (c *controller) enqueueAll() {
c.enqueueVerifyWebhook() c.enqueueVerifyWebhook()
} }
func (c *controller) enqueueCleanup() {
c.queue.Add(config.KyvernoDeploymentName())
}
func (c *controller) enqueueCleanupAfter(duration time.Duration) {
c.queue.AddAfter(config.KyvernoDeploymentName(), duration)
}
func (c *controller) enqueuePolicyWebhooks() { func (c *controller) enqueuePolicyWebhooks() {
c.queue.Add(config.PolicyValidatingWebhookConfigurationName) c.queue.Add(config.PolicyValidatingWebhookConfigurationName)
c.queue.Add(config.PolicyMutatingWebhookConfigurationName) c.queue.Add(config.PolicyMutatingWebhookConfigurationName)
@ -363,6 +409,47 @@ func (c *controller) reconcileVerifyMutatingWebhookConfiguration(ctx context.Con
return c.reconcileMutatingWebhookConfiguration(ctx, true, c.buildVerifyMutatingWebhookConfiguration) return c.reconcileMutatingWebhookConfiguration(ctx, true, c.buildVerifyMutatingWebhookConfiguration)
} }
func (c *controller) reconcileWebhookDeletion(ctx context.Context) error {
if c.autoUpdateWebhooks {
if c.runtime.IsGoingDown() {
if c.webhooksDeleted {
return nil
}
c.webhooksDeleted = true
if err := c.vwcClient.DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{
LabelSelector: kyverno.LabelWebhookManagedBy,
}); err != nil && !apierrors.IsNotFound(err) {
logger.Error(err, "failed to clean up validating webhook configuration", "label", kyverno.LabelWebhookManagedBy)
return err
} else if err == nil {
logger.Info("successfully deleted validating webhook configurations", "label", kyverno.LabelWebhookManagedBy)
}
if err := c.mwcClient.DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{
LabelSelector: kyverno.LabelWebhookManagedBy,
}); err != nil && !apierrors.IsNotFound(err) {
logger.Error(err, "failed to clean up mutating webhook configuration", "label", kyverno.LabelWebhookManagedBy)
return err
} else if err == nil {
logger.Info("successfully deleted mutating webhook configurations", "label", kyverno.LabelWebhookManagedBy)
}
if err := c.postWebhookCleanup(ctx, logger); err != nil {
logger.Error(err, "failed to clean up temporary rbac")
return err
} else {
logger.Info("successfully deleted temporary rbac")
}
} else {
if err := c.webhookCleanupSetup(ctx, logger); err != nil {
logger.Error(err, "failed to reconcile webhook cleanup setup")
return err
}
logger.Info("reconciled webhook cleanup setup")
}
}
return nil
}
func (c *controller) reconcileValidatingWebhookConfiguration(ctx context.Context, autoUpdateWebhooks bool, build func(context.Context, config.Configuration, []byte) (*admissionregistrationv1.ValidatingWebhookConfiguration, error)) error { func (c *controller) reconcileValidatingWebhookConfiguration(ctx context.Context, autoUpdateWebhooks bool, build func(context.Context, config.Configuration, []byte) (*admissionregistrationv1.ValidatingWebhookConfiguration, error)) error {
caData, err := tls.ReadRootCASecret(c.caSecretName, config.KyvernoNamespace(), c.secretLister.Secrets(config.KyvernoNamespace())) caData, err := tls.ReadRootCASecret(c.caSecretName, config.KyvernoNamespace(), c.secretLister.Secrets(config.KyvernoNamespace()))
if err != nil { if err != nil {
@ -526,6 +613,10 @@ func (c *controller) updatePolicyStatuses(ctx context.Context) error {
} }
func (c *controller) reconcile(ctx context.Context, logger logr.Logger, key, namespace, name string) error { func (c *controller) reconcile(ctx context.Context, logger logr.Logger, key, namespace, name string) error {
if c.autoDeleteWebhooks && c.runtime.IsGoingDown() {
return c.reconcileWebhookDeletion(ctx)
}
switch name { switch name {
case config.MutatingWebhookConfigurationName: case config.MutatingWebhookConfigurationName:
if c.runtime.IsRollingUpdate() { if c.runtime.IsRollingUpdate() {
@ -555,6 +646,8 @@ func (c *controller) reconcile(ctx context.Context, logger logr.Logger, key, nam
return c.reconcilePolicyMutatingWebhookConfiguration(ctx) return c.reconcilePolicyMutatingWebhookConfiguration(ctx)
case config.VerifyMutatingWebhookConfigurationName: case config.VerifyMutatingWebhookConfigurationName:
return c.reconcileVerifyMutatingWebhookConfiguration(ctx) return c.reconcileVerifyMutatingWebhookConfiguration(ctx)
case config.KyvernoDeploymentName():
return c.reconcileWebhookDeletion(ctx)
} }
return nil return nil
} }

View file

@ -0,0 +1,46 @@
package informers
import (
"time"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
appsv1informers "k8s.io/client-go/informers/apps/v1"
"k8s.io/client-go/kubernetes"
appsv1listers "k8s.io/client-go/listers/apps/v1"
"k8s.io/client-go/tools/cache"
)
type deploymentInformer struct {
informer cache.SharedIndexInformer
lister appsv1listers.DeploymentLister
}
func NewDeploymentInformer(
client kubernetes.Interface,
namespace string,
name string,
resyncPeriod time.Duration,
) appsv1informers.DeploymentInformer {
indexers := cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}
options := func(lo *metav1.ListOptions) {
lo.FieldSelector = fields.OneTermEqualSelector(metav1.ObjectNameField, name).String()
}
informer := appsv1informers.NewFilteredDeploymentInformer(
client,
namespace,
resyncPeriod,
indexers,
options,
)
lister := appsv1listers.NewDeploymentLister(informer.GetIndexer())
return &deploymentInformer{informer, lister}
}
func (i *deploymentInformer) Informer() cache.SharedIndexInformer {
return i.informer
}
func (i *deploymentInformer) Lister() appsv1listers.DeploymentLister {
return i.lister
}

View file

@ -35,8 +35,9 @@ type ObjectClient[T metav1.Object] interface {
CreateClient[T] CreateClient[T]
GetClient[T] GetClient[T]
UpdateClient[T] UpdateClient[T]
DeleteClient
PatchClient[T] PatchClient[T]
DeleteClient
DeleteCollectionClient
} }
type DeleteCollectionClient interface { type DeleteCollectionClient interface {

View file

@ -245,6 +245,8 @@ func (s *server) cleanup(ctx context.Context) {
deleteLease := func(name string) { deleteLease := func(name string) {
if err := s.leaseClient.Delete(ctx, name, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { if err := s.leaseClient.Delete(ctx, name, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) {
logger.Error(err, "failed to clean up lease", "name", name) logger.Error(err, "failed to clean up lease", "name", name)
} else if err == nil {
logger.Info("successfully deleted leases", "label", kyverno.LabelWebhookManagedBy)
} }
} }
deleteVwc := func() { deleteVwc := func() {
@ -252,6 +254,8 @@ func (s *server) cleanup(ctx context.Context) {
LabelSelector: kyverno.LabelWebhookManagedBy, LabelSelector: kyverno.LabelWebhookManagedBy,
}); err != nil && !apierrors.IsNotFound(err) { }); err != nil && !apierrors.IsNotFound(err) {
logger.Error(err, "failed to clean up validating webhook configuration", "label", kyverno.LabelWebhookManagedBy) logger.Error(err, "failed to clean up validating webhook configuration", "label", kyverno.LabelWebhookManagedBy)
} else if err == nil {
logger.Info("successfully deleted validating webhook configurations", "label", kyverno.LabelWebhookManagedBy)
} }
} }
deleteMwc := func() { deleteMwc := func() {
@ -259,6 +263,8 @@ func (s *server) cleanup(ctx context.Context) {
LabelSelector: kyverno.LabelWebhookManagedBy, LabelSelector: kyverno.LabelWebhookManagedBy,
}); err != nil && !apierrors.IsNotFound(err) { }); err != nil && !apierrors.IsNotFound(err) {
logger.Error(err, "failed to clean up mutating webhook configuration", "label", kyverno.LabelWebhookManagedBy) logger.Error(err, "failed to clean up mutating webhook configuration", "label", kyverno.LabelWebhookManagedBy)
} else if err == nil {
logger.Info("successfully deleted mutating webhook configurations", "label", kyverno.LabelWebhookManagedBy)
} }
} }
deleteLease("kyvernopre-lock") deleteLease("kyvernopre-lock")