fix: make webhook cleanup setup optional and add cleanup ci test (#11077)

* fix: make webhook cleanup setup optional and add cleanup ci test

Signed-off-by: Vishal Choudhary <vishal.choudhary@nirmata.com>

* fix: add logging

Signed-off-by: Vishal Choudhary <vishal.choudhary@nirmata.com>

* fix: add validating webhook count check

Signed-off-by: Vishal Choudhary <vishal.choudhary@nirmata.com>

---------

Signed-off-by: Vishal Choudhary <vishal.choudhary@nirmata.com>
Co-authored-by: shuting <shuting@nirmata.com>

.github/workflows/conformance.yaml

@@ -866,6 +866,81 @@ jobs:
           KYVERNO_EXPERIMENTAL=true kyverno fix test ./test/cli --save --compress
           make verify-cli-tests
 
+  cleanup-test:
+    runs-on: ubuntu-latest
+    permissions:
+      packages: read
+    strategy:
+      fail-fast: false
+      matrix:
+        k8s-version:
+          - name: v1.31
+            version: v1.31.0
+        kyverno-config:
+          - name: cleanup
+            values:
+              - kyverno-cleanup
+    needs:
+      - prepare-images
+    name: ${{ matrix.k8s-version.name }} - kyverno uninstall
+    steps:
+      - name: Checkout kyverno/kyverno
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      - name: Install helm
+        id: helm
+        uses: azure/setup-helm@fe7b79cd5ee1e45176fcad797de68ecaf3ca4814 # v4.2.0
+      - name: Install Kubectl
+        run: |
+          set -e
+          curl -LO "https://dl.k8s.io/release/${{ matrix.k8s-version.version }}/bin/linux/amd64/kubectl"
+          sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
+      - name: Create kind cluster
+        uses: helm/kind-action@0025e74a8c7512023d06dc019c617aa3cf561fde # v1.10.0
+        with:
+          node_image: kindest/node:${{ matrix.k8s-version.version }}
+          cluster_name: kind
+          config: ./scripts/config/kind/default.yaml
+      - name: Download kyverno images archive
+        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
+        with:
+          name: kyverno.tar
+      - name: Load kyverno images archive in kind cluster
+        shell: bash
+        run: |
+          set -e
+          kind load image-archive kyverno.tar --name kind
+      - name: Install kyverno
+        shell: bash
+        run: |
+          set -e
+          export HELM=${{ steps.helm.outputs.helm-path }}
+          export USE_CONFIG=${{ join(matrix.kyverno-config.values, ',') }}
+          make kind-install-kyverno
+      - name: Wait for kyverno ready
+        uses: ./.github/actions/kyverno-wait-ready
+      - name: Log finalizers from deployments
+        shell: bash
+        run: |
+          set -e
+          kubectl get deploy kyverno-admission-controller -n kyverno --template='{{.metadata.finalizers}}'
+          kubectl get deploy kyverno-cleanup-controller -n kyverno --template='{{.metadata.finalizers}}'
+      - name: Uninstall kyverno
+        shell: bash
+        run: |
+          set -e
+          helm uninstall kyverno -n kyverno --wait --no-hooks
+      - name: Check validating webhook count
+        shell: bash
+        run: |
+          set -e
+          if [ `kubectl get validatingwebhookconfigurations -l webhook.kyverno.io/managed-by=kyverno --no-headers | wc -l` -gt 0 ] 
+          then
+          exit 1
+          fi
+      - name: Debug failure
+        if: failure()
+        uses: ./.github/actions/kyverno-logs
+
   conformance-required-success:
     name: conformance-required
     needs:

pkg/controllers/generic/webhook/controller.go

@@ -167,8 +167,10 @@ func NewController(
 }
 
 func (c *controller) Run(ctx context.Context, workers int) {
-	if err := c.webhookCleanupSetup(ctx, c.logger); err != nil {
-		c.logger.Error(err, "failed to setup webhook cleanup")
+	if c.autoDeleteWebhooks {
+		if err := c.webhookCleanupSetup(ctx, c.logger); err != nil {
+			c.logger.Error(err, "failed to setup webhook cleanup")
+		}
 	}
 	c.enqueue()
 	controllerutils.Run(ctx, c.logger, c.controllerName, time.Second, c.queue, workers, maxRetries, c.reconcile)

pkg/controllers/webhook/controller.go

@@ -249,8 +249,10 @@ func NewController(
 }
 
 func (c *controller) Run(ctx context.Context, workers int) {
-	if err := c.webhookCleanupSetup(ctx, logger); err != nil {
-		logger.Error(err, "failed to setup webhook cleanup")
+	if c.autoDeleteWebhooks {
+		if err := c.webhookCleanupSetup(ctx, logger); err != nil {
+			logger.Error(err, "failed to setup webhook cleanup")
+		}
 	}
 	// add our known webhooks to the queue
 	c.enqueueAll()

scripts/config/kyverno-cleanup/kyverno.yaml

@@ -0,0 +1,4 @@
+webhooksCleanup:
+  enabled: true
+  autoDeleteWebhooks:
+    enabled: true