feat(audit): use a worker pool for Audit policies (#10048)

* enhancement: split validation logic for enforce and audit policies to return admission response earlier

Signed-off-by: ShutingZhao <shuting@nirmata.com>

* chore: add missing file

Signed-off-by: ShutingZhao <shuting@nirmata.com>

* fix: unit tests

Signed-off-by: ShutingZhao <shuting@nirmata.com>

* fix: linter issues

Signed-off-by: ShutingZhao <shuting@nirmata.com>

* fix: unit tests

Signed-off-by: ShutingZhao <shuting@nirmata.com>

* fix: get latest policy object before updating status

Signed-off-by: ShutingZhao <shuting@nirmata.com>

* chore: remove debug code

Signed-off-by: ShutingZhao <shuting@nirmata.com>

* fix: compare before updates

Signed-off-by: ShutingZhao <shuting@nirmata.com>

* fix: initial reconcile

Signed-off-by: ShutingZhao <shuting@nirmata.com>

* fix: updates

Signed-off-by: ShutingZhao <shuting@nirmata.com>

* feat(audit): use a worker pool for Audit policies

Signed-off-by: Khaled Emara <khaled.emara@nirmata.com>

* fix: unit test

Signed-off-by: ShutingZhao <shuting@nirmata.com>

* fix(attempt): spin up go routine

Signed-off-by: ShutingZhao <shuting@nirmata.com>

* feat: add flags maxAuditWorkers, maxAuditCapacity

Signed-off-by: ShutingZhao <shuting@nirmata.com>

* fix: enable debug log on failure

Signed-off-by: ShutingZhao <shuting@nirmata.com>

* fix: wait group panic

Signed-off-by: ShutingZhao <shuting@nirmata.com>

* load-tests: add stess tests configurations

Signed-off-by: ShutingZhao <shuting@nirmata.com>

* load-tests: disable admissionreports

Signed-off-by: ShutingZhao <shuting@nirmata.com>

* fix: build policy contexts syncronously

Signed-off-by: Vishal Choudhary <vishal.choudhary@nirmata.com>

* fix: only run generate and mutate existing go routines when policies are present

Signed-off-by: Vishal Choudhary <vishal.choudhary@nirmata.com>

* fix: mutate and verify tests

Signed-off-by: Vishal Choudhary <vishal.choudhary@nirmata.com>

* fix: return early if no audit policy

Signed-off-by: ShutingZhao <shuting@nirmata.com>

* fix: run handlegenerate and mutate existing in all cases

Signed-off-by: Vishal Choudhary <vishal.choudhary@nirmata.com>

* fix: only test bgapplies in generate test

Signed-off-by: Vishal Choudhary <vishal.choudhary@nirmata.com>

* fix: defer wait in tests

Signed-off-by: Vishal Choudhary <vishal.choudhary@nirmata.com>

* enhancement: process validate enforce in a go routine

Signed-off-by: ShutingZhao <shuting@nirmata.com>

---------

Signed-off-by: ShutingZhao <shuting@nirmata.com>
Signed-off-by: Khaled Emara <khaled.emara@nirmata.com>
Signed-off-by: Vishal Choudhary <vishal.choudhary@nirmata.com>
Co-authored-by: ShutingZhao <shuting@nirmata.com>
Co-authored-by: Vishal Choudhary <vishal.choudhary@nirmata.com>

.github/workflows/load-testing.yml

@@ -56,9 +56,9 @@ jobs:
           - name: default
             values:
               - default-with-profiling
-          - name: standard
+          - name: stress
             values:
-              - standard-with-profiling
+              - stress-with-profiling
         test:
           - kyverno-pss
           - kyverno-mutate
@@ -139,3 +139,6 @@ jobs:
         with:
           name: pprof-heap-profiles
           path: heap.pprof
+      - name: Debug failure
+        if: failure()
+        uses: ./.github/actions/kyverno-logs

cmd/kyverno/main.go

@@ -254,6 +254,8 @@ func main() {
 		backgroundServiceAccountName string
 		maxAPICallResponseLength     int64
 		renewBefore                  time.Duration
+		maxAuditWorkers              int
+		maxAuditCapacity             int
 	)
 	flagset := flag.NewFlagSet("kyverno", flag.ExitOnError)
 	flagset.BoolVar(&dumpPayload, "dumpPayload", false, "Set this flag to activate/deactivate debug mode.")
@@ -274,6 +276,8 @@ func main() {
 	flagset.StringVar(&tlsSecretName, "tlsSecretName", "", "Name of the secret containing TLS pair.")
 	flagset.Int64Var(&maxAPICallResponseLength, "maxAPICallResponseLength", 10*1000*1000, "Configure the value of maximum allowed GET response size from API Calls")
 	flagset.DurationVar(&renewBefore, "renewBefore", 15*24*time.Hour, "The certificate renewal time before expiration")
+	flagset.IntVar(&maxAuditWorkers, "maxAuditWorkers", 8, "Maximum number of workers for audit policy processing")
+	flagset.IntVar(&maxAuditCapacity, "maxAuditCapacity", 1000, "Maximum capacity of the audit policy task queue")
 	// config
 	appConfig := internal.NewConfiguration(
 		internal.WithProfiling(),
@@ -533,6 +537,8 @@ func main() {
 		admissionReports,
 		backgroundServiceAccountName,
 		setup.Jp,
+		maxAuditWorkers,
+		maxAuditCapacity,
 	)
 	exceptionHandlers := webhooksexception.NewHandlers(exception.ValidationOptions{
 		Enabled:   internal.PolicyExceptionEnabled(),

go.mod

@@ -92,6 +92,8 @@ require (
 	sigs.k8s.io/yaml v1.4.0
 )
 
+require github.com/alitto/pond v1.8.3 // indirect
+
 require (
 	cloud.google.com/go/compute v1.25.0 // indirect
 	cloud.google.com/go/compute/metadata v0.2.3 // indirect

go.sum

@@ -132,6 +132,8 @@ github.com/alibabacloud-go/tea-utils v1.4.5/go.mod h1:KNcT0oXlZZxOXINnZBs6YvgOd5
 github.com/alibabacloud-go/tea-xml v1.1.2/go.mod h1:Rq08vgCcCAjHyRi/M7xlHKUykZCEtyBy9+DPF6GgEu8=
 github.com/alibabacloud-go/tea-xml v1.1.3 h1:7LYnm+JbOq2B+T/B0fHC4Ies4/FofC4zHzYtqw7dgt0=
 github.com/alibabacloud-go/tea-xml v1.1.3/go.mod h1:Rq08vgCcCAjHyRi/M7xlHKUykZCEtyBy9+DPF6GgEu8=
+github.com/alitto/pond v1.8.3 h1:ydIqygCLVPqIX/USe5EaV/aSRXTRXDEI9JwuDdu+/xs=
+github.com/alitto/pond v1.8.3/go.mod h1:CmvIIGd5jKLasGI3D87qDkQxjzChdKMmnXMg3fG6M6Q=
 github.com/aliyun/credentials-go v1.1.2/go.mod h1:ozcZaMR5kLM7pwtCMEpVmQ242suV6qTJya2bDq4X1Tw=
 github.com/aliyun/credentials-go v1.3.2 h1:L4WppI9rctC8PdlMgyTkF8bBsy9pyKQEzBD1bHMRl+g=
 github.com/aliyun/credentials-go v1.3.2/go.mod h1:tlpz4uys4Rn7Ik4/piGRrTbXy2uLKvePgQJJduE+Y5c=

pkg/webhooks/resource/fake.go

@@ -3,6 +3,7 @@ package resource
 import (
 	"context"
 
+	"github.com/alitto/pond"
 	fakekyvernov1 "github.com/kyverno/kyverno/pkg/client/clientset/versioned/fake"
 	kyvernoinformers "github.com/kyverno/kyverno/pkg/client/informers/externalversions"
 	"github.com/kyverno/kyverno/pkg/clients/dclient"
@@ -53,6 +54,7 @@ func NewFakeHandlers(ctx context.Context, policyCache policycache.Cache) *resour
 		urGenerator:   updaterequest.NewFake(),
 		eventGen:      event.NewFake(),
 		pcBuilder:     webhookutils.NewPolicyContextBuilder(configuration, jp),
+		auditPool:     pond.New(8, 1000),
 		engine: engine.NewEngine(
 			configuration,
 			config.NewDefaultMetricsConfiguration(),

pkg/webhooks/resource/handlers.go

@@ -8,6 +8,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/alitto/pond"
 	"github.com/go-logr/logr"
 	kyvernov1 "github.com/kyverno/kyverno/api/kyverno/v1"
 	"github.com/kyverno/kyverno/pkg/client/clientset/versioned"
@@ -37,8 +38,6 @@ import (
 )
 
 type resourceHandlers struct {
-	wg sync.WaitGroup
-
 	// clients
 	client        dclient.Interface
 	kyvernoClient versioned.Interface
@@ -63,6 +62,7 @@ type resourceHandlers struct {
 
 	admissionReports             bool
 	backgroundServiceAccountName string
+	auditPool                    *pond.WorkerPool
 }
 
 func NewHandlers(
@@ -81,6 +81,8 @@ func NewHandlers(
 	admissionReports bool,
 	backgroundServiceAccountName string,
 	jp jmespath.Interface,
+	maxAuditWorkers int,
+	maxAuditCapacity int,
 ) webhooks.ResourceHandlers {
 	return &resourceHandlers{
 		engine:                       engine,
@@ -98,6 +100,7 @@ func NewHandlers(
 		pcBuilder:                    webhookutils.NewPolicyContextBuilder(configuration, jp),
 		admissionReports:             admissionReports,
 		backgroundServiceAccountName: backgroundServiceAccountName,
+		auditPool:                    pond.New(maxAuditWorkers, maxAuditCapacity, pond.Strategy(pond.Lazy())),
 	}
 }
 
@@ -117,23 +120,33 @@ func (h *resourceHandlers) Validate(ctx context.Context, logger logr.Logger, req
 
 	logger.V(4).Info("processing policies for validate admission request", "validate", len(policies), "mutate", len(mutatePolicies), "generate", len(generatePolicies))
 
-	policyContext, err := h.buildPolicyContextFromAdmissionRequest(logger, request)
+	vh := validation.NewValidationHandler(logger, h.kyvernoClient, h.engine, h.pCache, h.pcBuilder, h.eventGen, h.admissionReports, h.metricsConfig, h.configuration, h.nsLister)
-	if err != nil {
+	var wg sync.WaitGroup
-		return errorResponse(logger, request.UID, err, "failed create policy context")
+	var ok bool
+	var msg string
+	var warnings []string
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		ok, msg, warnings = vh.HandleValidationEnforce(ctx, request, policies, startTime)
+	}()
+
+	go h.auditPool.Submit(func() {
+		vh.HandleValidationAudit(ctx, request)
+	})
+	if !admissionutils.IsDryRun(request.AdmissionRequest) {
+		h.handleBackgroundApplies(ctx, logger, request, generatePolicies, mutatePolicies, startTime, nil)
+	}
+	if len(policies) == 0 {
+		return admissionutils.ResponseSuccess(request.UID)
 	}
 
-	vh := validation.NewValidationHandler(logger, h.kyvernoClient, h.engine, h.pCache, h.pcBuilder, h.eventGen, h.admissionReports, h.metricsConfig, h.configuration)
+	wg.Wait()
-
-	ok, msg, warnings := vh.HandleValidation(ctx, request, policies, policyContext, startTime)
 	if !ok {
 		logger.Info("admission request denied")
 		return admissionutils.Response(request.UID, errors.New(msg), warnings...)
 	}
-	if !admissionutils.IsDryRun(request.AdmissionRequest) {
+
-		h.wg.Add(1)
-		go h.handleBackgroundApplies(ctx, logger, request, generatePolicies, mutatePolicies, startTime)
-		h.wg.Wait()
-	}
 	return admissionutils.ResponseSuccess(request.UID, warnings...)
 }
 

pkg/webhooks/resource/handlers_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"sync"
 	"testing"
 	"time"
 
@@ -212,8 +213,15 @@ var policyMutateAndVerify = `
                                 "entries": [
                                     {
                                         "keys": {
-                                            "publicKeys": "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE8nXRh950IZbRj8Ra/N9sbqOPZrfM\n5/KAQN0/KjHcorm/J5yctVd7iEcnessRQjU917hmKO6JWVGHpDguIyakZA==\n-----END PUBLIC KEY-----"
+                                            "publicKeys": "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE8nXRh950IZbRj8Ra/N9sbqOPZrfM\n5/KAQN0/KjHcorm/J5yctVd7iEcnessRQjU917hmKO6JWVGHpDguIyakZA==\n-----END PUBLIC KEY-----",
-                                        }
+																						"rekor": {
+																							"url": "https://rekor.sigstore.dev",
+																							"ignoreTlog": true
+																						},
+																						"ctlog": {
+																							"ignoreSCT": true
+																						}
+																				}
                                     }
                                 ]
                             }
@@ -533,7 +541,7 @@ func Test_MutateAndVerify(t *testing.T) {
 		AdmissionRequest: v1.AdmissionRequest{
 			Operation: v1.Create,
 			Kind:      metav1.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"},
-			Resource:  metav1.GroupVersionResource{Group: "", Version: "v1", Resource: "Pod"},
+			Resource:  metav1.GroupVersionResource{Group: "", Version: "v1", Resource: "pods"},
 			Object: apiruntime.RawExtension{
 				Raw: []byte(resourceMutateAndVerify),
 			},
@@ -578,7 +586,7 @@ func Test_MutateAndGenerate(t *testing.T) {
 		AdmissionRequest: v1.AdmissionRequest{
 			Operation: v1.Create,
 			Kind:      metav1.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"},
-			Resource:  metav1.GroupVersionResource{Group: "", Version: "v1", Resource: "Pod"},
+			Resource:  metav1.GroupVersionResource{Group: "", Version: "v1", Resource: "pods"},
 			Object: apiruntime.RawExtension{
 				Raw: []byte(resourceMutateandGenerate),
 			},
@@ -587,38 +595,29 @@ func Test_MutateAndGenerate(t *testing.T) {
 		},
 	}
 
-	response := resourceHandlers.Validate(ctx, logger, request, "", time.Now())
+	_, mutatePolicies, generatePolicies, _, err := resourceHandlers.retrieveAndCategorizePolicies(ctx, logger, request, "", false)
-
-	assert.Assert(t, len(mockPcBuilder.contexts) >= 3, fmt.Sprint("expected no of context ", 3, " received ", len(mockPcBuilder.contexts)))
-
-	validateJSONContext := mockPcBuilder.contexts[0].JSONContext()
-	mutateJSONContext := mockPcBuilder.contexts[1].JSONContext()
-	generateJSONContext := mockPcBuilder.contexts[2].JSONContext()
-
-	_, err = enginecontext.AddMockDeferredLoader(validateJSONContext, "key1", "value1")
-	assert.NilError(t, err)
-	_, err = enginecontext.AddMockDeferredLoader(mutateJSONContext, "key2", "value2")
-	assert.NilError(t, err)
-	_, err = enginecontext.AddMockDeferredLoader(generateJSONContext, "key3", "value3")
 	assert.NilError(t, err)
 
-	_, err = mutateJSONContext.Query("key1")
+	var wg sync.WaitGroup
-	assert.ErrorContains(t, err, `Unknown key "key1" in path`)
+	wg.Add(2)
+	resourceHandlers.handleBackgroundApplies(ctx, logger, request, generatePolicies, mutatePolicies, time.Now(), &wg)
+	wg.Wait()
+
+	assert.Assert(t, len(mockPcBuilder.contexts) >= 2, fmt.Sprint("expected no of context ", 2, " received ", len(mockPcBuilder.contexts)))
+
+	mutateJSONContext := mockPcBuilder.contexts[0].JSONContext()
+	generateJSONContext := mockPcBuilder.contexts[1].JSONContext()
+
+	_, err = enginecontext.AddMockDeferredLoader(mutateJSONContext, "key1", "value1")
+	assert.NilError(t, err)
+	_, err = enginecontext.AddMockDeferredLoader(generateJSONContext, "key2", "value2")
+	assert.NilError(t, err)
+
+	_, err = mutateJSONContext.Query("key2")
+	assert.ErrorContains(t, err, `Unknown key "key2" in path`)
+
 	_, err = generateJSONContext.Query("key1")
 	assert.ErrorContains(t, err, `Unknown key "key1" in path`)
-
-	_, err = validateJSONContext.Query("key2")
-	assert.ErrorContains(t, err, `Unknown key "key2" in path`)
-	_, err = generateJSONContext.Query("key2")
-	assert.ErrorContains(t, err, `Unknown key "key2" in path`)
-
-	_, err = validateJSONContext.Query("key3")
-	assert.ErrorContains(t, err, `Unknown key "key3" in path`)
-	_, err = mutateJSONContext.Query("key3")
-	assert.ErrorContains(t, err, `Unknown key "key3" in path`)
-
-	assert.Equal(t, response.Allowed, true)
-	assert.Equal(t, len(response.Warnings), 0)
 }
 
 func makeKey(policy kyverno.PolicyInterface) string {

pkg/webhooks/resource/updaterequest.go

@@ -3,6 +3,7 @@ package resource
 import (
 	"context"
 	"fmt"
+	"sync"
 	"time"
 
 	"github.com/go-logr/logr"
@@ -19,19 +20,20 @@ import (
 )
 
 // handleBackgroundApplies applies generate and mutateExisting policies, and creates update requests for background reconcile
-func (h *resourceHandlers) handleBackgroundApplies(ctx context.Context, logger logr.Logger, request handlers.AdmissionRequest, generatePolicies, mutatePolicies []kyvernov1.PolicyInterface, ts time.Time) {
+func (h *resourceHandlers) handleBackgroundApplies(ctx context.Context, logger logr.Logger, request handlers.AdmissionRequest, generatePolicies, mutatePolicies []kyvernov1.PolicyInterface, ts time.Time, wg *sync.WaitGroup) {
-	h.wg.Add(1)
+	go h.handleMutateExisting(ctx, logger, request, mutatePolicies, ts, wg)
-	go h.handleMutateExisting(ctx, logger, request, mutatePolicies, ts)
+	go h.handleGenerate(ctx, logger, request, generatePolicies, ts, wg)
-	h.handleGenerate(ctx, logger, request, generatePolicies, ts)
 }
 
-func (h *resourceHandlers) handleMutateExisting(ctx context.Context, logger logr.Logger, request handlers.AdmissionRequest, policies []kyvernov1.PolicyInterface, admissionRequestTimestamp time.Time) {
+func (h *resourceHandlers) handleMutateExisting(ctx context.Context, logger logr.Logger, request handlers.AdmissionRequest, policies []kyvernov1.PolicyInterface, admissionRequestTimestamp time.Time, wg *sync.WaitGroup) {
 	policyContext, err := h.buildPolicyContextFromAdmissionRequest(logger, request)
 	if err != nil {
 		logger.Error(err, "failed to create policy context")
 		return
 	}
-	h.wg.Done()
+	if wg != nil { // for unit testing purposes
+		defer wg.Done()
+	}
 
 	if request.AdmissionRequest.Operation == admissionv1.Delete {
 		policyContext = policyContext.WithNewResource(policyContext.OldResource())
@@ -92,13 +94,15 @@ func (h *resourceHandlers) handleMutateExisting(ctx context.Context, logger logr
 	}
 }
 
-func (h *resourceHandlers) handleGenerate(ctx context.Context, logger logr.Logger, request handlers.AdmissionRequest, generatePolicies []kyvernov1.PolicyInterface, ts time.Time) {
+func (h *resourceHandlers) handleGenerate(ctx context.Context, logger logr.Logger, request handlers.AdmissionRequest, generatePolicies []kyvernov1.PolicyInterface, ts time.Time, wg *sync.WaitGroup) {
 	policyContext, err := h.buildPolicyContextFromAdmissionRequest(logger, request)
 	if err != nil {
 		logger.Error(err, "failed to create policy context")
 		return
 	}
-	h.wg.Done()
+	if wg != nil { // for unit testing purposes
+		defer wg.Done()
+	}
 
 	gh := generation.NewGenerationHandler(logger, h.engine, h.client, h.kyvernoClient, h.nsLister, h.urLister, h.cpolLister, h.polLister, h.urGenerator, h.eventGen, h.metricsConfig, h.backgroundServiceAccountName)
 	var policies []kyvernov1.PolicyInterface
@@ -108,5 +112,5 @@ func (h *resourceHandlers) handleGenerate(ctx context.Context, logger logr.Logge
 			policies = append(policies, new)
 		}
 	}
-	go gh.Handle(ctx, request.AdmissionRequest, policies, policyContext)
+	gh.Handle(ctx, request.AdmissionRequest, policies, policyContext)
 }

pkg/webhooks/resource/validation/utils.go

@@ -0,0 +1,30 @@
+package validation
+
+import (
+	admissionutils "github.com/kyverno/kyverno/pkg/utils/admission"
+	reportutils "github.com/kyverno/kyverno/pkg/utils/report"
+	"github.com/kyverno/kyverno/pkg/webhooks/handlers"
+	admissionv1 "k8s.io/api/admission/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+)
+
+func needsReports(request handlers.AdmissionRequest, resource unstructured.Unstructured, admissionReport bool) bool {
+	createReport := admissionReport
+	if admissionutils.IsDryRun(request.AdmissionRequest) {
+		createReport = false
+	}
+	// we don't need reports for deletions
+	if request.Operation == admissionv1.Delete {
+		createReport = false
+	}
+	// check if the resource supports reporting
+	if !reportutils.IsGvkSupported(schema.GroupVersionKind(request.Kind)) {
+		createReport = false
+	}
+	// if the underlying resource has no UID don't create a report
+	if resource.GetUID() == "" {
+		createReport = false
+	}
+	return createReport
+}

pkg/webhooks/resource/validation/validation.go

@@ -9,27 +9,29 @@ import (
 	kyvernov1 "github.com/kyverno/kyverno/api/kyverno/v1"
 	"github.com/kyverno/kyverno/pkg/client/clientset/versioned"
 	"github.com/kyverno/kyverno/pkg/config"
-	"github.com/kyverno/kyverno/pkg/engine"
 	engineapi "github.com/kyverno/kyverno/pkg/engine/api"
+	"github.com/kyverno/kyverno/pkg/engine/policycontext"
 	"github.com/kyverno/kyverno/pkg/event"
 	"github.com/kyverno/kyverno/pkg/metrics"
 	"github.com/kyverno/kyverno/pkg/policycache"
 	"github.com/kyverno/kyverno/pkg/tracing"
 	admissionutils "github.com/kyverno/kyverno/pkg/utils/admission"
+	engineutils "github.com/kyverno/kyverno/pkg/utils/engine"
 	reportutils "github.com/kyverno/kyverno/pkg/utils/report"
 	"github.com/kyverno/kyverno/pkg/webhooks/handlers"
 	webhookutils "github.com/kyverno/kyverno/pkg/webhooks/utils"
 	"go.opentelemetry.io/otel/trace"
-	admissionv1 "k8s.io/api/admission/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	corev1listers "k8s.io/client-go/listers/core/v1"
 )
 
 type ValidationHandler interface {
 	// HandleValidation handles validating webhook admission request
 	// If there are no errors in validating rule we apply generation rules
 	// patchedResource is the (resource + patches) after applying mutation rules
-	HandleValidation(context.Context, handlers.AdmissionRequest, []kyvernov1.PolicyInterface, *engine.PolicyContext, time.Time) (bool, string, []string)
+	HandleValidationEnforce(context.Context, handlers.AdmissionRequest, []kyvernov1.PolicyInterface, time.Time) (bool, string, []string)
+	HandleValidationAudit(context.Context, handlers.AdmissionRequest)
 }
 
 func NewValidationHandler(
@@ -42,6 +44,7 @@ func NewValidationHandler(
 	admissionReports bool,
 	metrics metrics.MetricsConfigManager,
 	cfg config.Configuration,
+	nsLister corev1listers.NamespaceLister,
 ) ValidationHandler {
 	return &validationHandler{
 		log:              log,
@@ -53,6 +56,7 @@ func NewValidationHandler(
 		admissionReports: admissionReports,
 		metrics:          metrics,
 		cfg:              cfg,
+		nsLister:         nsLister,
 	}
 }
 
@@ -66,18 +70,27 @@ type validationHandler struct {
 	admissionReports bool
 	metrics          metrics.MetricsConfigManager
 	cfg              config.Configuration
+	nsLister         corev1listers.NamespaceLister
 }
 
-func (v *validationHandler) HandleValidation(
+func (v *validationHandler) HandleValidationEnforce(
 	ctx context.Context,
 	request handlers.AdmissionRequest,
 	policies []kyvernov1.PolicyInterface,
-	policyContext *engine.PolicyContext,
 	admissionRequestTimestamp time.Time,
 ) (bool, string, []string) {
 	resourceName := admissionutils.GetResourceName(request.AdmissionRequest)
 	logger := v.log.WithValues("action", "validate", "resource", resourceName, "operation", request.Operation, "gvk", request.Kind)
 
+	if len(policies) == 0 {
+		return true, "", nil
+	}
+
+	policyContext, err := v.buildPolicyContextFromAdmissionRequest(logger, request)
+	if err != nil {
+		return false, "failed create policy context", nil
+	}
+
 	var engineResponses []engineapi.EngineResponse
 	failurePolicy := kyvernov1.Ignore
 	for _, policy := range policies {
@@ -94,7 +107,7 @@ func (v *validationHandler) HandleValidation(
 				engineResponse := v.engine.Validate(ctx, policyContext)
 				if engineResponse.IsNil() {
 					// we get an empty response if old and new resources created the same response
-					// allow updates if resource update doesnt change the policy evaluation
+					// allow updates if resource update doesn't change the policy evaluation
 					return
 				}
 
@@ -120,24 +133,61 @@ func (v *validationHandler) HandleValidation(
 		return false, webhookutils.GetBlockedMessages(engineResponses), nil
 	}
 
-	go v.handleAudit(ctx, policyContext.NewResource(), request, policyContext.NamespaceLabels(), engineResponses...)
+	go func() {
+		if needsReports(request, policyContext.NewResource(), v.admissionReports) {
+			if err := v.createReports(ctx, policyContext.NewResource(), request, engineResponses...); err != nil {
+				v.log.Error(err, "failed to create report")
+			}
+		}
+	}()
 
 	warnings := webhookutils.GetWarningMessages(engineResponses)
 	return true, "", warnings
 }
 
-func (v *validationHandler) buildAuditResponses(
+func (v *validationHandler) HandleValidationAudit(
 	ctx context.Context,
-	resource unstructured.Unstructured,
 	request handlers.AdmissionRequest,
-	namespaceLabels map[string]string,
+) {
-) ([]engineapi.EngineResponse, error) {
 	gvr := schema.GroupVersionResource(request.Resource)
 	policies := v.pCache.GetPolicies(policycache.ValidateAudit, gvr, request.SubResource, request.Namespace)
-	policyContext, err := v.pcBuilder.Build(request.AdmissionRequest, request.Roles, request.ClusterRoles, request.GroupVersionKind)
+	if len(policies) == 0 {
-	if err != nil {
+		return
-		return nil, err
 	}
+
+	policyContext, err := v.buildPolicyContextFromAdmissionRequest(v.log, request)
+	if err != nil {
+		v.log.Error(err, "failed to build policy context")
+		return
+	}
+
+	needsReport := needsReports(request, policyContext.NewResource(), v.admissionReports)
+	tracing.Span(
+		context.Background(),
+		"",
+		fmt.Sprintf("AUDIT %s %s", request.Operation, request.Kind),
+		func(ctx context.Context, span trace.Span) {
+			responses, err := v.buildAuditResponses(ctx, policyContext, policies)
+			if err != nil {
+				v.log.Error(err, "failed to build audit responses")
+			}
+			events := webhookutils.GenerateEvents(responses, false)
+			v.eventGen.Add(events...)
+			if needsReport {
+				if err := v.createReports(ctx, policyContext.NewResource(), request, responses...); err != nil {
+					v.log.Error(err, "failed to create report")
+				}
+			}
+		},
+		trace.WithLinks(trace.LinkFromContext(ctx)),
+	)
+}
+
+func (v *validationHandler) buildAuditResponses(
+	ctx context.Context,
+	policyContext *policycontext.PolicyContext,
+	policies []kyvernov1.PolicyInterface,
+) ([]engineapi.EngineResponse, error) {
 	var responses []engineapi.EngineResponse
 	for _, policy := range policies {
 		tracing.ChildSpan(
@@ -145,7 +195,7 @@ func (v *validationHandler) buildAuditResponses(
 			"pkg/webhooks/resource/validate",
 			fmt.Sprintf("POLICY %s/%s", policy.GetNamespace(), policy.GetName()),
 			func(ctx context.Context, span trace.Span) {
-				policyContext := policyContext.WithPolicy(policy).WithNamespaceLabels(namespaceLabels)
+				policyContext := policyContext.WithPolicy(policy)
 				response := v.engine.Validate(ctx, policyContext)
 				responses = append(responses, response)
 			},
@@ -154,51 +204,31 @@ func (v *validationHandler) buildAuditResponses(
 	return responses, nil
 }
 
-func (v *validationHandler) handleAudit(
+func (v *validationHandler) buildPolicyContextFromAdmissionRequest(logger logr.Logger, request handlers.AdmissionRequest) (*policycontext.PolicyContext, error) {
+	policyContext, err := v.pcBuilder.Build(request.AdmissionRequest, request.Roles, request.ClusterRoles, request.GroupVersionKind)
+	if err != nil {
+		return nil, err
+	}
+	namespaceLabels := make(map[string]string)
+	if request.Kind.Kind != "Namespace" && request.Namespace != "" {
+		namespaceLabels = engineutils.GetNamespaceSelectorsFromNamespaceLister(request.Kind.Kind, request.Namespace, v.nsLister, logger)
+	}
+	policyContext = policyContext.WithNamespaceLabels(namespaceLabels)
+	return policyContext, nil
+}
+
+func (v *validationHandler) createReports(
 	ctx context.Context,
 	resource unstructured.Unstructured,
 	request handlers.AdmissionRequest,
-	namespaceLabels map[string]string,
 	engineResponses ...engineapi.EngineResponse,
-) {
+) error {
-	createReport := v.admissionReports
+	report := reportutils.BuildAdmissionReport(resource, request.AdmissionRequest, engineResponses...)
-	if admissionutils.IsDryRun(request.AdmissionRequest) {
+	if len(report.GetResults()) > 0 {
-		createReport = false
+		_, err := reportutils.CreateReport(ctx, report, v.kyvernoClient)
+		if err != nil {
+			return err
+		}
 	}
-	// we don't need reports for deletions
+	return nil
-	if request.Operation == admissionv1.Delete {
-		createReport = false
-	}
-	// check if the resource supports reporting
-	if !reportutils.IsGvkSupported(schema.GroupVersionKind(request.Kind)) {
-		createReport = false
-	}
-	// if the underlying resource has no UID don't create a report
-	if resource.GetUID() == "" {
-		createReport = false
-	}
-	tracing.Span(
-		context.Background(),
-		"",
-		fmt.Sprintf("AUDIT %s %s", request.Operation, request.Kind),
-		func(ctx context.Context, span trace.Span) {
-			responses, err := v.buildAuditResponses(ctx, resource, request, namespaceLabels)
-			if err != nil {
-				v.log.Error(err, "failed to build audit responses")
-			}
-			events := webhookutils.GenerateEvents(responses, false)
-			v.eventGen.Add(events...)
-			if createReport {
-				responses = append(responses, engineResponses...)
-				report := reportutils.BuildAdmissionReport(resource, request.AdmissionRequest, responses...)
-				if len(report.GetResults()) > 0 {
-					_, err = reportutils.CreateReport(ctx, report, v.kyvernoClient)
-					if err != nil {
-						v.log.Error(err, "failed to create report")
-					}
-				}
-			}
-		},
-		trace.WithLinks(trace.LinkFromContext(ctx)),
-	)
 }

scripts/config/standard-with-profiling/kyverno.yaml

@@ -1,47 +0,0 @@
-features:
-  policyExceptions:
-    enabled: true
-
-admissionController:
-  profiling:
-    enabled: true
-    serviceType: NodePort
-    nodePort: 30950
-
-backgroundController:
-  rbac:
-    clusterRole:
-      extraResources:
-        - apiGroups:
-            - "*"
-          resources:
-            - configmaps
-            - networkpolicies
-            - resourcequotas
-            - secrets
-            - roles
-            - rolebindings
-            - limitranges
-            - namespaces
-            - nodes
-            - nodes/status
-            - pods
-          verbs:
-            - create
-            - update
-            - patch
-            - delete
-            - get
-            - list
-
-cleanupController:
-  rbac:
-    clusterRole:
-      extraResources:
-        - apiGroups:
-            - ""
-          resources:
-            - pods
-          verbs:
-            - list
-            - delete

scripts/config/stress-with-profiling/kyverno.yaml

@@ -0,0 +1,19 @@
+# stress-with-profiling sets the most restricted configurations for the admission controller. It disables
+# any additional feature that could affect the performance of the admission controller. 
+features:
+  policyExceptions:
+    enabled: true
+  omitEvents:
+    eventTypes:
+      - PolicyApplied
+      - PolicySkipped
+      - PolicyViolation
+      - PolicyError
+  admissionReports:
+    enabled: false
+
+admissionController:
+  profiling:
+    enabled: true
+    serviceType: NodePort
+    nodePort: 30950