diff --git a/.github/workflows/load-testing.yml b/.github/workflows/load-testing.yml index 8a28c1d623..bcc503022a 100644 --- a/.github/workflows/load-testing.yml +++ b/.github/workflows/load-testing.yml @@ -56,9 +56,9 @@ jobs: - name: default values: - default-with-profiling - - name: standard + - name: stress values: - - standard-with-profiling + - stress-with-profiling test: - kyverno-pss - kyverno-mutate @@ -139,3 +139,6 @@ jobs: with: name: pprof-heap-profiles path: heap.pprof + - name: Debug failure + if: failure() + uses: ./.github/actions/kyverno-logs diff --git a/cmd/kyverno/main.go b/cmd/kyverno/main.go index 46499da43a..55674d5fa8 100644 --- a/cmd/kyverno/main.go +++ b/cmd/kyverno/main.go @@ -254,6 +254,8 @@ func main() { backgroundServiceAccountName string maxAPICallResponseLength int64 renewBefore time.Duration + maxAuditWorkers int + maxAuditCapacity int ) flagset := flag.NewFlagSet("kyverno", flag.ExitOnError) flagset.BoolVar(&dumpPayload, "dumpPayload", false, "Set this flag to activate/deactivate debug mode.") @@ -274,6 +276,8 @@ func main() { flagset.StringVar(&tlsSecretName, "tlsSecretName", "", "Name of the secret containing TLS pair.") flagset.Int64Var(&maxAPICallResponseLength, "maxAPICallResponseLength", 10*1000*1000, "Configure the value of maximum allowed GET response size from API Calls") flagset.DurationVar(&renewBefore, "renewBefore", 15*24*time.Hour, "The certificate renewal time before expiration") + flagset.IntVar(&maxAuditWorkers, "maxAuditWorkers", 8, "Maximum number of workers for audit policy processing") + flagset.IntVar(&maxAuditCapacity, "maxAuditCapacity", 1000, "Maximum capacity of the audit policy task queue") // config appConfig := internal.NewConfiguration( internal.WithProfiling(), @@ -533,6 +537,8 @@ func main() { admissionReports, backgroundServiceAccountName, setup.Jp, + maxAuditWorkers, + maxAuditCapacity, ) exceptionHandlers := webhooksexception.NewHandlers(exception.ValidationOptions{ Enabled: internal.PolicyExceptionEnabled(), diff --git a/go.mod b/go.mod index d11bfd0b87..d9c31b82ed 100644 --- a/go.mod +++ b/go.mod @@ -92,6 +92,8 @@ require ( sigs.k8s.io/yaml v1.4.0 ) +require github.com/alitto/pond v1.8.3 // indirect + require ( cloud.google.com/go/compute v1.25.0 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect diff --git a/go.sum b/go.sum index ce4f48daf8..0ce6c943d4 100644 --- a/go.sum +++ b/go.sum @@ -132,6 +132,8 @@ github.com/alibabacloud-go/tea-utils v1.4.5/go.mod h1:KNcT0oXlZZxOXINnZBs6YvgOd5 github.com/alibabacloud-go/tea-xml v1.1.2/go.mod h1:Rq08vgCcCAjHyRi/M7xlHKUykZCEtyBy9+DPF6GgEu8= github.com/alibabacloud-go/tea-xml v1.1.3 h1:7LYnm+JbOq2B+T/B0fHC4Ies4/FofC4zHzYtqw7dgt0= github.com/alibabacloud-go/tea-xml v1.1.3/go.mod h1:Rq08vgCcCAjHyRi/M7xlHKUykZCEtyBy9+DPF6GgEu8= +github.com/alitto/pond v1.8.3 h1:ydIqygCLVPqIX/USe5EaV/aSRXTRXDEI9JwuDdu+/xs= +github.com/alitto/pond v1.8.3/go.mod h1:CmvIIGd5jKLasGI3D87qDkQxjzChdKMmnXMg3fG6M6Q= github.com/aliyun/credentials-go v1.1.2/go.mod h1:ozcZaMR5kLM7pwtCMEpVmQ242suV6qTJya2bDq4X1Tw= github.com/aliyun/credentials-go v1.3.2 h1:L4WppI9rctC8PdlMgyTkF8bBsy9pyKQEzBD1bHMRl+g= github.com/aliyun/credentials-go v1.3.2/go.mod h1:tlpz4uys4Rn7Ik4/piGRrTbXy2uLKvePgQJJduE+Y5c= diff --git a/pkg/webhooks/resource/fake.go b/pkg/webhooks/resource/fake.go index a3c0deb84b..ee5d853d86 100644 --- a/pkg/webhooks/resource/fake.go +++ b/pkg/webhooks/resource/fake.go @@ -3,6 +3,7 @@ package resource import ( "context" + "github.com/alitto/pond" fakekyvernov1 "github.com/kyverno/kyverno/pkg/client/clientset/versioned/fake" kyvernoinformers "github.com/kyverno/kyverno/pkg/client/informers/externalversions" "github.com/kyverno/kyverno/pkg/clients/dclient" @@ -53,6 +54,7 @@ func NewFakeHandlers(ctx context.Context, policyCache policycache.Cache) *resour urGenerator: updaterequest.NewFake(), eventGen: event.NewFake(), pcBuilder: webhookutils.NewPolicyContextBuilder(configuration, jp), + auditPool: pond.New(8, 1000), engine: engine.NewEngine( configuration, config.NewDefaultMetricsConfiguration(), diff --git a/pkg/webhooks/resource/handlers.go b/pkg/webhooks/resource/handlers.go index ab0d6eae7c..a0328caec9 100644 --- a/pkg/webhooks/resource/handlers.go +++ b/pkg/webhooks/resource/handlers.go @@ -8,6 +8,7 @@ import ( "sync" "time" + "github.com/alitto/pond" "github.com/go-logr/logr" kyvernov1 "github.com/kyverno/kyverno/api/kyverno/v1" "github.com/kyverno/kyverno/pkg/client/clientset/versioned" @@ -37,8 +38,6 @@ import ( ) type resourceHandlers struct { - wg sync.WaitGroup - // clients client dclient.Interface kyvernoClient versioned.Interface @@ -63,6 +62,7 @@ type resourceHandlers struct { admissionReports bool backgroundServiceAccountName string + auditPool *pond.WorkerPool } func NewHandlers( @@ -81,6 +81,8 @@ func NewHandlers( admissionReports bool, backgroundServiceAccountName string, jp jmespath.Interface, + maxAuditWorkers int, + maxAuditCapacity int, ) webhooks.ResourceHandlers { return &resourceHandlers{ engine: engine, @@ -98,6 +100,7 @@ func NewHandlers( pcBuilder: webhookutils.NewPolicyContextBuilder(configuration, jp), admissionReports: admissionReports, backgroundServiceAccountName: backgroundServiceAccountName, + auditPool: pond.New(maxAuditWorkers, maxAuditCapacity, pond.Strategy(pond.Lazy())), } } @@ -117,23 +120,33 @@ func (h *resourceHandlers) Validate(ctx context.Context, logger logr.Logger, req logger.V(4).Info("processing policies for validate admission request", "validate", len(policies), "mutate", len(mutatePolicies), "generate", len(generatePolicies)) - policyContext, err := h.buildPolicyContextFromAdmissionRequest(logger, request) - if err != nil { - return errorResponse(logger, request.UID, err, "failed create policy context") + vh := validation.NewValidationHandler(logger, h.kyvernoClient, h.engine, h.pCache, h.pcBuilder, h.eventGen, h.admissionReports, h.metricsConfig, h.configuration, h.nsLister) + var wg sync.WaitGroup + var ok bool + var msg string + var warnings []string + wg.Add(1) + go func() { + defer wg.Done() + ok, msg, warnings = vh.HandleValidationEnforce(ctx, request, policies, startTime) + }() + + go h.auditPool.Submit(func() { + vh.HandleValidationAudit(ctx, request) + }) + if !admissionutils.IsDryRun(request.AdmissionRequest) { + h.handleBackgroundApplies(ctx, logger, request, generatePolicies, mutatePolicies, startTime, nil) + } + if len(policies) == 0 { + return admissionutils.ResponseSuccess(request.UID) } - vh := validation.NewValidationHandler(logger, h.kyvernoClient, h.engine, h.pCache, h.pcBuilder, h.eventGen, h.admissionReports, h.metricsConfig, h.configuration) - - ok, msg, warnings := vh.HandleValidation(ctx, request, policies, policyContext, startTime) + wg.Wait() if !ok { logger.Info("admission request denied") return admissionutils.Response(request.UID, errors.New(msg), warnings...) } - if !admissionutils.IsDryRun(request.AdmissionRequest) { - h.wg.Add(1) - go h.handleBackgroundApplies(ctx, logger, request, generatePolicies, mutatePolicies, startTime) - h.wg.Wait() - } + return admissionutils.ResponseSuccess(request.UID, warnings...) } diff --git a/pkg/webhooks/resource/handlers_test.go b/pkg/webhooks/resource/handlers_test.go index f419350dc1..c30d82b13c 100644 --- a/pkg/webhooks/resource/handlers_test.go +++ b/pkg/webhooks/resource/handlers_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "sync" "testing" "time" @@ -212,8 +213,15 @@ var policyMutateAndVerify = ` "entries": [ { "keys": { - "publicKeys": "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE8nXRh950IZbRj8Ra/N9sbqOPZrfM\n5/KAQN0/KjHcorm/J5yctVd7iEcnessRQjU917hmKO6JWVGHpDguIyakZA==\n-----END PUBLIC KEY-----" - } + "publicKeys": "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE8nXRh950IZbRj8Ra/N9sbqOPZrfM\n5/KAQN0/KjHcorm/J5yctVd7iEcnessRQjU917hmKO6JWVGHpDguIyakZA==\n-----END PUBLIC KEY-----", + "rekor": { + "url": "https://rekor.sigstore.dev", + "ignoreTlog": true + }, + "ctlog": { + "ignoreSCT": true + } + } } ] } @@ -533,7 +541,7 @@ func Test_MutateAndVerify(t *testing.T) { AdmissionRequest: v1.AdmissionRequest{ Operation: v1.Create, Kind: metav1.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"}, - Resource: metav1.GroupVersionResource{Group: "", Version: "v1", Resource: "Pod"}, + Resource: metav1.GroupVersionResource{Group: "", Version: "v1", Resource: "pods"}, Object: apiruntime.RawExtension{ Raw: []byte(resourceMutateAndVerify), }, @@ -578,7 +586,7 @@ func Test_MutateAndGenerate(t *testing.T) { AdmissionRequest: v1.AdmissionRequest{ Operation: v1.Create, Kind: metav1.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"}, - Resource: metav1.GroupVersionResource{Group: "", Version: "v1", Resource: "Pod"}, + Resource: metav1.GroupVersionResource{Group: "", Version: "v1", Resource: "pods"}, Object: apiruntime.RawExtension{ Raw: []byte(resourceMutateandGenerate), }, @@ -587,38 +595,29 @@ func Test_MutateAndGenerate(t *testing.T) { }, } - response := resourceHandlers.Validate(ctx, logger, request, "", time.Now()) - - assert.Assert(t, len(mockPcBuilder.contexts) >= 3, fmt.Sprint("expected no of context ", 3, " received ", len(mockPcBuilder.contexts))) - - validateJSONContext := mockPcBuilder.contexts[0].JSONContext() - mutateJSONContext := mockPcBuilder.contexts[1].JSONContext() - generateJSONContext := mockPcBuilder.contexts[2].JSONContext() - - _, err = enginecontext.AddMockDeferredLoader(validateJSONContext, "key1", "value1") - assert.NilError(t, err) - _, err = enginecontext.AddMockDeferredLoader(mutateJSONContext, "key2", "value2") - assert.NilError(t, err) - _, err = enginecontext.AddMockDeferredLoader(generateJSONContext, "key3", "value3") + _, mutatePolicies, generatePolicies, _, err := resourceHandlers.retrieveAndCategorizePolicies(ctx, logger, request, "", false) assert.NilError(t, err) - _, err = mutateJSONContext.Query("key1") - assert.ErrorContains(t, err, `Unknown key "key1" in path`) + var wg sync.WaitGroup + wg.Add(2) + resourceHandlers.handleBackgroundApplies(ctx, logger, request, generatePolicies, mutatePolicies, time.Now(), &wg) + wg.Wait() + + assert.Assert(t, len(mockPcBuilder.contexts) >= 2, fmt.Sprint("expected no of context ", 2, " received ", len(mockPcBuilder.contexts))) + + mutateJSONContext := mockPcBuilder.contexts[0].JSONContext() + generateJSONContext := mockPcBuilder.contexts[1].JSONContext() + + _, err = enginecontext.AddMockDeferredLoader(mutateJSONContext, "key1", "value1") + assert.NilError(t, err) + _, err = enginecontext.AddMockDeferredLoader(generateJSONContext, "key2", "value2") + assert.NilError(t, err) + + _, err = mutateJSONContext.Query("key2") + assert.ErrorContains(t, err, `Unknown key "key2" in path`) + _, err = generateJSONContext.Query("key1") assert.ErrorContains(t, err, `Unknown key "key1" in path`) - - _, err = validateJSONContext.Query("key2") - assert.ErrorContains(t, err, `Unknown key "key2" in path`) - _, err = generateJSONContext.Query("key2") - assert.ErrorContains(t, err, `Unknown key "key2" in path`) - - _, err = validateJSONContext.Query("key3") - assert.ErrorContains(t, err, `Unknown key "key3" in path`) - _, err = mutateJSONContext.Query("key3") - assert.ErrorContains(t, err, `Unknown key "key3" in path`) - - assert.Equal(t, response.Allowed, true) - assert.Equal(t, len(response.Warnings), 0) } func makeKey(policy kyverno.PolicyInterface) string { diff --git a/pkg/webhooks/resource/updaterequest.go b/pkg/webhooks/resource/updaterequest.go index 97d5d35295..85d38b318c 100644 --- a/pkg/webhooks/resource/updaterequest.go +++ b/pkg/webhooks/resource/updaterequest.go @@ -3,6 +3,7 @@ package resource import ( "context" "fmt" + "sync" "time" "github.com/go-logr/logr" @@ -19,19 +20,20 @@ import ( ) // handleBackgroundApplies applies generate and mutateExisting policies, and creates update requests for background reconcile -func (h *resourceHandlers) handleBackgroundApplies(ctx context.Context, logger logr.Logger, request handlers.AdmissionRequest, generatePolicies, mutatePolicies []kyvernov1.PolicyInterface, ts time.Time) { - h.wg.Add(1) - go h.handleMutateExisting(ctx, logger, request, mutatePolicies, ts) - h.handleGenerate(ctx, logger, request, generatePolicies, ts) +func (h *resourceHandlers) handleBackgroundApplies(ctx context.Context, logger logr.Logger, request handlers.AdmissionRequest, generatePolicies, mutatePolicies []kyvernov1.PolicyInterface, ts time.Time, wg *sync.WaitGroup) { + go h.handleMutateExisting(ctx, logger, request, mutatePolicies, ts, wg) + go h.handleGenerate(ctx, logger, request, generatePolicies, ts, wg) } -func (h *resourceHandlers) handleMutateExisting(ctx context.Context, logger logr.Logger, request handlers.AdmissionRequest, policies []kyvernov1.PolicyInterface, admissionRequestTimestamp time.Time) { +func (h *resourceHandlers) handleMutateExisting(ctx context.Context, logger logr.Logger, request handlers.AdmissionRequest, policies []kyvernov1.PolicyInterface, admissionRequestTimestamp time.Time, wg *sync.WaitGroup) { policyContext, err := h.buildPolicyContextFromAdmissionRequest(logger, request) if err != nil { logger.Error(err, "failed to create policy context") return } - h.wg.Done() + if wg != nil { // for unit testing purposes + defer wg.Done() + } if request.AdmissionRequest.Operation == admissionv1.Delete { policyContext = policyContext.WithNewResource(policyContext.OldResource()) @@ -92,13 +94,15 @@ func (h *resourceHandlers) handleMutateExisting(ctx context.Context, logger logr } } -func (h *resourceHandlers) handleGenerate(ctx context.Context, logger logr.Logger, request handlers.AdmissionRequest, generatePolicies []kyvernov1.PolicyInterface, ts time.Time) { +func (h *resourceHandlers) handleGenerate(ctx context.Context, logger logr.Logger, request handlers.AdmissionRequest, generatePolicies []kyvernov1.PolicyInterface, ts time.Time, wg *sync.WaitGroup) { policyContext, err := h.buildPolicyContextFromAdmissionRequest(logger, request) if err != nil { logger.Error(err, "failed to create policy context") return } - h.wg.Done() + if wg != nil { // for unit testing purposes + defer wg.Done() + } gh := generation.NewGenerationHandler(logger, h.engine, h.client, h.kyvernoClient, h.nsLister, h.urLister, h.cpolLister, h.polLister, h.urGenerator, h.eventGen, h.metricsConfig, h.backgroundServiceAccountName) var policies []kyvernov1.PolicyInterface @@ -108,5 +112,5 @@ func (h *resourceHandlers) handleGenerate(ctx context.Context, logger logr.Logge policies = append(policies, new) } } - go gh.Handle(ctx, request.AdmissionRequest, policies, policyContext) + gh.Handle(ctx, request.AdmissionRequest, policies, policyContext) } diff --git a/pkg/webhooks/resource/validation/utils.go b/pkg/webhooks/resource/validation/utils.go new file mode 100644 index 0000000000..cdf5e9d606 --- /dev/null +++ b/pkg/webhooks/resource/validation/utils.go @@ -0,0 +1,30 @@ +package validation + +import ( + admissionutils "github.com/kyverno/kyverno/pkg/utils/admission" + reportutils "github.com/kyverno/kyverno/pkg/utils/report" + "github.com/kyverno/kyverno/pkg/webhooks/handlers" + admissionv1 "k8s.io/api/admission/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +func needsReports(request handlers.AdmissionRequest, resource unstructured.Unstructured, admissionReport bool) bool { + createReport := admissionReport + if admissionutils.IsDryRun(request.AdmissionRequest) { + createReport = false + } + // we don't need reports for deletions + if request.Operation == admissionv1.Delete { + createReport = false + } + // check if the resource supports reporting + if !reportutils.IsGvkSupported(schema.GroupVersionKind(request.Kind)) { + createReport = false + } + // if the underlying resource has no UID don't create a report + if resource.GetUID() == "" { + createReport = false + } + return createReport +} diff --git a/pkg/webhooks/resource/validation/validation.go b/pkg/webhooks/resource/validation/validation.go index 981a516761..c424c6f1ac 100644 --- a/pkg/webhooks/resource/validation/validation.go +++ b/pkg/webhooks/resource/validation/validation.go @@ -9,27 +9,29 @@ import ( kyvernov1 "github.com/kyverno/kyverno/api/kyverno/v1" "github.com/kyverno/kyverno/pkg/client/clientset/versioned" "github.com/kyverno/kyverno/pkg/config" - "github.com/kyverno/kyverno/pkg/engine" engineapi "github.com/kyverno/kyverno/pkg/engine/api" + "github.com/kyverno/kyverno/pkg/engine/policycontext" "github.com/kyverno/kyverno/pkg/event" "github.com/kyverno/kyverno/pkg/metrics" "github.com/kyverno/kyverno/pkg/policycache" "github.com/kyverno/kyverno/pkg/tracing" admissionutils "github.com/kyverno/kyverno/pkg/utils/admission" + engineutils "github.com/kyverno/kyverno/pkg/utils/engine" reportutils "github.com/kyverno/kyverno/pkg/utils/report" "github.com/kyverno/kyverno/pkg/webhooks/handlers" webhookutils "github.com/kyverno/kyverno/pkg/webhooks/utils" "go.opentelemetry.io/otel/trace" - admissionv1 "k8s.io/api/admission/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" + corev1listers "k8s.io/client-go/listers/core/v1" ) type ValidationHandler interface { // HandleValidation handles validating webhook admission request // If there are no errors in validating rule we apply generation rules // patchedResource is the (resource + patches) after applying mutation rules - HandleValidation(context.Context, handlers.AdmissionRequest, []kyvernov1.PolicyInterface, *engine.PolicyContext, time.Time) (bool, string, []string) + HandleValidationEnforce(context.Context, handlers.AdmissionRequest, []kyvernov1.PolicyInterface, time.Time) (bool, string, []string) + HandleValidationAudit(context.Context, handlers.AdmissionRequest) } func NewValidationHandler( @@ -42,6 +44,7 @@ func NewValidationHandler( admissionReports bool, metrics metrics.MetricsConfigManager, cfg config.Configuration, + nsLister corev1listers.NamespaceLister, ) ValidationHandler { return &validationHandler{ log: log, @@ -53,6 +56,7 @@ func NewValidationHandler( admissionReports: admissionReports, metrics: metrics, cfg: cfg, + nsLister: nsLister, } } @@ -66,18 +70,27 @@ type validationHandler struct { admissionReports bool metrics metrics.MetricsConfigManager cfg config.Configuration + nsLister corev1listers.NamespaceLister } -func (v *validationHandler) HandleValidation( +func (v *validationHandler) HandleValidationEnforce( ctx context.Context, request handlers.AdmissionRequest, policies []kyvernov1.PolicyInterface, - policyContext *engine.PolicyContext, admissionRequestTimestamp time.Time, ) (bool, string, []string) { resourceName := admissionutils.GetResourceName(request.AdmissionRequest) logger := v.log.WithValues("action", "validate", "resource", resourceName, "operation", request.Operation, "gvk", request.Kind) + if len(policies) == 0 { + return true, "", nil + } + + policyContext, err := v.buildPolicyContextFromAdmissionRequest(logger, request) + if err != nil { + return false, "failed create policy context", nil + } + var engineResponses []engineapi.EngineResponse failurePolicy := kyvernov1.Ignore for _, policy := range policies { @@ -94,7 +107,7 @@ func (v *validationHandler) HandleValidation( engineResponse := v.engine.Validate(ctx, policyContext) if engineResponse.IsNil() { // we get an empty response if old and new resources created the same response - // allow updates if resource update doesnt change the policy evaluation + // allow updates if resource update doesn't change the policy evaluation return } @@ -120,24 +133,61 @@ func (v *validationHandler) HandleValidation( return false, webhookutils.GetBlockedMessages(engineResponses), nil } - go v.handleAudit(ctx, policyContext.NewResource(), request, policyContext.NamespaceLabels(), engineResponses...) + go func() { + if needsReports(request, policyContext.NewResource(), v.admissionReports) { + if err := v.createReports(ctx, policyContext.NewResource(), request, engineResponses...); err != nil { + v.log.Error(err, "failed to create report") + } + } + }() warnings := webhookutils.GetWarningMessages(engineResponses) return true, "", warnings } -func (v *validationHandler) buildAuditResponses( +func (v *validationHandler) HandleValidationAudit( ctx context.Context, - resource unstructured.Unstructured, request handlers.AdmissionRequest, - namespaceLabels map[string]string, -) ([]engineapi.EngineResponse, error) { +) { gvr := schema.GroupVersionResource(request.Resource) policies := v.pCache.GetPolicies(policycache.ValidateAudit, gvr, request.SubResource, request.Namespace) - policyContext, err := v.pcBuilder.Build(request.AdmissionRequest, request.Roles, request.ClusterRoles, request.GroupVersionKind) - if err != nil { - return nil, err + if len(policies) == 0 { + return } + + policyContext, err := v.buildPolicyContextFromAdmissionRequest(v.log, request) + if err != nil { + v.log.Error(err, "failed to build policy context") + return + } + + needsReport := needsReports(request, policyContext.NewResource(), v.admissionReports) + tracing.Span( + context.Background(), + "", + fmt.Sprintf("AUDIT %s %s", request.Operation, request.Kind), + func(ctx context.Context, span trace.Span) { + responses, err := v.buildAuditResponses(ctx, policyContext, policies) + if err != nil { + v.log.Error(err, "failed to build audit responses") + } + events := webhookutils.GenerateEvents(responses, false) + v.eventGen.Add(events...) + if needsReport { + if err := v.createReports(ctx, policyContext.NewResource(), request, responses...); err != nil { + v.log.Error(err, "failed to create report") + } + } + }, + trace.WithLinks(trace.LinkFromContext(ctx)), + ) +} + +func (v *validationHandler) buildAuditResponses( + ctx context.Context, + policyContext *policycontext.PolicyContext, + policies []kyvernov1.PolicyInterface, +) ([]engineapi.EngineResponse, error) { var responses []engineapi.EngineResponse for _, policy := range policies { tracing.ChildSpan( @@ -145,7 +195,7 @@ func (v *validationHandler) buildAuditResponses( "pkg/webhooks/resource/validate", fmt.Sprintf("POLICY %s/%s", policy.GetNamespace(), policy.GetName()), func(ctx context.Context, span trace.Span) { - policyContext := policyContext.WithPolicy(policy).WithNamespaceLabels(namespaceLabels) + policyContext := policyContext.WithPolicy(policy) response := v.engine.Validate(ctx, policyContext) responses = append(responses, response) }, @@ -154,51 +204,31 @@ func (v *validationHandler) buildAuditResponses( return responses, nil } -func (v *validationHandler) handleAudit( +func (v *validationHandler) buildPolicyContextFromAdmissionRequest(logger logr.Logger, request handlers.AdmissionRequest) (*policycontext.PolicyContext, error) { + policyContext, err := v.pcBuilder.Build(request.AdmissionRequest, request.Roles, request.ClusterRoles, request.GroupVersionKind) + if err != nil { + return nil, err + } + namespaceLabels := make(map[string]string) + if request.Kind.Kind != "Namespace" && request.Namespace != "" { + namespaceLabels = engineutils.GetNamespaceSelectorsFromNamespaceLister(request.Kind.Kind, request.Namespace, v.nsLister, logger) + } + policyContext = policyContext.WithNamespaceLabels(namespaceLabels) + return policyContext, nil +} + +func (v *validationHandler) createReports( ctx context.Context, resource unstructured.Unstructured, request handlers.AdmissionRequest, - namespaceLabels map[string]string, engineResponses ...engineapi.EngineResponse, -) { - createReport := v.admissionReports - if admissionutils.IsDryRun(request.AdmissionRequest) { - createReport = false +) error { + report := reportutils.BuildAdmissionReport(resource, request.AdmissionRequest, engineResponses...) + if len(report.GetResults()) > 0 { + _, err := reportutils.CreateReport(ctx, report, v.kyvernoClient) + if err != nil { + return err + } } - // we don't need reports for deletions - if request.Operation == admissionv1.Delete { - createReport = false - } - // check if the resource supports reporting - if !reportutils.IsGvkSupported(schema.GroupVersionKind(request.Kind)) { - createReport = false - } - // if the underlying resource has no UID don't create a report - if resource.GetUID() == "" { - createReport = false - } - tracing.Span( - context.Background(), - "", - fmt.Sprintf("AUDIT %s %s", request.Operation, request.Kind), - func(ctx context.Context, span trace.Span) { - responses, err := v.buildAuditResponses(ctx, resource, request, namespaceLabels) - if err != nil { - v.log.Error(err, "failed to build audit responses") - } - events := webhookutils.GenerateEvents(responses, false) - v.eventGen.Add(events...) - if createReport { - responses = append(responses, engineResponses...) - report := reportutils.BuildAdmissionReport(resource, request.AdmissionRequest, responses...) - if len(report.GetResults()) > 0 { - _, err = reportutils.CreateReport(ctx, report, v.kyvernoClient) - if err != nil { - v.log.Error(err, "failed to create report") - } - } - } - }, - trace.WithLinks(trace.LinkFromContext(ctx)), - ) + return nil } diff --git a/scripts/config/standard-with-profiling/kyverno.yaml b/scripts/config/standard-with-profiling/kyverno.yaml deleted file mode 100644 index 7f21fad15f..0000000000 --- a/scripts/config/standard-with-profiling/kyverno.yaml +++ /dev/null @@ -1,47 +0,0 @@ -features: - policyExceptions: - enabled: true - -admissionController: - profiling: - enabled: true - serviceType: NodePort - nodePort: 30950 - -backgroundController: - rbac: - clusterRole: - extraResources: - - apiGroups: - - "*" - resources: - - configmaps - - networkpolicies - - resourcequotas - - secrets - - roles - - rolebindings - - limitranges - - namespaces - - nodes - - nodes/status - - pods - verbs: - - create - - update - - patch - - delete - - get - - list - -cleanupController: - rbac: - clusterRole: - extraResources: - - apiGroups: - - "" - resources: - - pods - verbs: - - list - - delete diff --git a/scripts/config/stress-with-profiling/kyverno.yaml b/scripts/config/stress-with-profiling/kyverno.yaml new file mode 100644 index 0000000000..dfeb1358c7 --- /dev/null +++ b/scripts/config/stress-with-profiling/kyverno.yaml @@ -0,0 +1,19 @@ +# stress-with-profiling sets the most restricted configurations for the admission controller. It disables +# any additional feature that could affect the performance of the admission controller. +features: + policyExceptions: + enabled: true + omitEvents: + eventTypes: + - PolicyApplied + - PolicySkipped + - PolicyViolation + - PolicyError + admissionReports: + enabled: false + +admissionController: + profiling: + enabled: true + serviceType: NodePort + nodePort: 30950 \ No newline at end of file