From fe49e97fba369c273b1020824d27db844067bd08 Mon Sep 17 00:00:00 2001 From: Vishal Choudhary Date: Wed, 2 Oct 2024 17:35:05 +0530 Subject: [PATCH] feat: add reporting to mutate and generate rules (#11265) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add reports to standard mutatation Signed-off-by: Vishal Choudhary * feat: add warnings for permissions Signed-off-by: Vishal Choudhary * fix: remove unnecessary fields Signed-off-by: Vishal Choudhary * feat: add reporting to generate and mutate existing Signed-off-by: Vishal Choudhary * feat: add reporting to generate and mutate existing Signed-off-by: Vishal Choudhary * fix: codegen and add generate tests Signed-off-by: Vishal Choudhary * fix: linter Signed-off-by: Vishal Choudhary * fix: e2e matrix Signed-off-by: Vishal Choudhary * fix: cleanup Signed-off-by: Vishal Choudhary --------- Signed-off-by: Vishal Choudhary Co-authored-by: Charles-Edouard Brétéché --- .../background-controller/clusterrole.yaml | 14 ++++ cmd/background-controller/main.go | 20 +++++- config/install-latest-testing.yaml | 14 ++++ pkg/background/generate/controller.go | 63 +++++++++++++---- pkg/background/mutate/mutate.go | 64 +++++++++++++++--- pkg/background/update_request_controller.go | 36 +++++----- pkg/breaker/resource_counter.go | 17 +++++ .../report/aggregate/controller.go | 22 +++--- pkg/policy/generate/fake.go | 3 +- pkg/policy/generate/validate.go | 61 ++++++++++++----- pkg/policy/mutate/validate.go | 66 +++++++++++++----- pkg/utils/report/new.go | 18 +++++ pkg/utils/report/results.go | 65 ++++++++++++++++++ pkg/validation/policy/actions.go | 6 +- pkg/webhooks/resource/handlers.go | 4 +- pkg/webhooks/resource/mutation/mutation.go | 67 +++++++++++++++---- pkg/webhooks/resource/mutation/utils.go | 27 ++++++++ test/conformance/chainsaw/e2e-matrix.json | 3 +- .../reports/admission/mutation/README.md | 3 + .../mutation/chainsaw-step-01-apply-1-1.yaml | 19 ++++++ .../mutation/chainsaw-step-01-assert-1-1.yaml | 9 +++ .../mutation/chainsaw-step-02-apply-1-1.yaml | 13 ++++ .../mutation/chainsaw-step-02-assert-1-1.yaml | 7 ++ .../mutation/chainsaw-step-03-assert-1-1.yaml | 24 +++++++ .../admission/mutation/chainsaw-test.yaml | 22 ++++++ .../reports/background/generate/README.md | 3 + .../generate/chainsaw-step-01-apply-1-1.yaml | 21 ++++++ .../generate/chainsaw-step-01-apply-1-2.yaml | 8 +++ .../generate/chainsaw-step-01-assert-1-1.yaml | 9 +++ .../generate/chainsaw-step-02-apply-1-1.yaml | 4 ++ .../generate/chainsaw-step-03-assert-1-1.yaml | 25 +++++++ .../background/generate/chainsaw-test.yaml | 26 +++++++ .../background/generate/permissions.yaml | 20 ++++++ .../background/mutate-existing/README.md | 7 ++ .../chainsaw-step-01-apply-1-1.yaml | 8 +++ .../chainsaw-step-01-apply-1-2.yaml | 8 +++ .../chainsaw-step-01-apply-1-3.yaml | 25 +++++++ .../chainsaw-step-01-assert-1-1.yaml | 9 +++ .../chainsaw-step-02-apply-1-1.yaml | 11 +++ .../chainsaw-step-03-assert-1-1.yaml | 22 ++++++ .../mutate-existing/chainsaw-test.yaml | 24 +++++++ 41 files changed, 793 insertions(+), 104 deletions(-) create mode 100644 pkg/webhooks/resource/mutation/utils.go create mode 100644 test/conformance/chainsaw/reports/admission/mutation/README.md create mode 100755 test/conformance/chainsaw/reports/admission/mutation/chainsaw-step-01-apply-1-1.yaml create mode 100755 test/conformance/chainsaw/reports/admission/mutation/chainsaw-step-01-assert-1-1.yaml create mode 100755 test/conformance/chainsaw/reports/admission/mutation/chainsaw-step-02-apply-1-1.yaml create mode 100755 test/conformance/chainsaw/reports/admission/mutation/chainsaw-step-02-assert-1-1.yaml create mode 100755 test/conformance/chainsaw/reports/admission/mutation/chainsaw-step-03-assert-1-1.yaml create mode 100755 test/conformance/chainsaw/reports/admission/mutation/chainsaw-test.yaml create mode 100644 test/conformance/chainsaw/reports/background/generate/README.md create mode 100755 test/conformance/chainsaw/reports/background/generate/chainsaw-step-01-apply-1-1.yaml create mode 100755 test/conformance/chainsaw/reports/background/generate/chainsaw-step-01-apply-1-2.yaml create mode 100755 test/conformance/chainsaw/reports/background/generate/chainsaw-step-01-assert-1-1.yaml create mode 100755 test/conformance/chainsaw/reports/background/generate/chainsaw-step-02-apply-1-1.yaml create mode 100755 test/conformance/chainsaw/reports/background/generate/chainsaw-step-03-assert-1-1.yaml create mode 100755 test/conformance/chainsaw/reports/background/generate/chainsaw-test.yaml create mode 100644 test/conformance/chainsaw/reports/background/generate/permissions.yaml create mode 100644 test/conformance/chainsaw/reports/background/mutate-existing/README.md create mode 100755 test/conformance/chainsaw/reports/background/mutate-existing/chainsaw-step-01-apply-1-1.yaml create mode 100755 test/conformance/chainsaw/reports/background/mutate-existing/chainsaw-step-01-apply-1-2.yaml create mode 100755 test/conformance/chainsaw/reports/background/mutate-existing/chainsaw-step-01-apply-1-3.yaml create mode 100755 test/conformance/chainsaw/reports/background/mutate-existing/chainsaw-step-01-assert-1-1.yaml create mode 100755 test/conformance/chainsaw/reports/background/mutate-existing/chainsaw-step-02-apply-1-1.yaml create mode 100755 test/conformance/chainsaw/reports/background/mutate-existing/chainsaw-step-03-assert-1-1.yaml create mode 100755 test/conformance/chainsaw/reports/background/mutate-existing/chainsaw-test.yaml diff --git a/charts/kyverno/templates/background-controller/clusterrole.yaml b/charts/kyverno/templates/background-controller/clusterrole.yaml index c680ef262e..2b2ade2834 100644 --- a/charts/kyverno/templates/background-controller/clusterrole.yaml +++ b/charts/kyverno/templates/background-controller/clusterrole.yaml @@ -68,6 +68,20 @@ rules: - patch - update - watch + - apiGroups: + - reports.kyverno.io + resources: + - ephemeralreports + - clusterephemeralreports + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - deletecollection {{- with .Values.backgroundController.rbac.coreClusterRole.extraResources }} {{- toYaml . | nindent 2 }} {{- end }} diff --git a/cmd/background-controller/main.go b/cmd/background-controller/main.go index 92b85bfcb0..6e27e16e83 100644 --- a/cmd/background-controller/main.go +++ b/cmd/background-controller/main.go @@ -11,6 +11,7 @@ import ( "github.com/kyverno/kyverno/cmd/internal" "github.com/kyverno/kyverno/pkg/background" + "github.com/kyverno/kyverno/pkg/breaker" "github.com/kyverno/kyverno/pkg/client/clientset/versioned" kyvernoinformer "github.com/kyverno/kyverno/pkg/client/informers/externalversions" "github.com/kyverno/kyverno/pkg/clients/dclient" @@ -54,6 +55,7 @@ func createrLeaderControllers( jp jmespath.Interface, backgroundScanInterval time.Duration, urGenerator generator.UpdateRequestGenerator, + reportsBreaker breaker.Breaker, ) ([]internal.Controller, error) { policyCtrl, err := policy.NewPolicyController( kyvernoClient, @@ -85,6 +87,7 @@ func createrLeaderControllers( eventGenerator, configuration, jp, + reportsBreaker, ) return []internal.Controller{ internal.NewController("policy-controller", policyCtrl, 2), @@ -98,12 +101,14 @@ func main() { maxQueuedEvents int omitEvents string maxAPICallResponseLength int64 + maxBackgroundReports int ) flagset := flag.NewFlagSet("updaterequest-controller", flag.ExitOnError) flagset.IntVar(&genWorkers, "genWorkers", 10, "Workers for the background controller.") flagset.IntVar(&maxQueuedEvents, "maxQueuedEvents", 1000, "Maximum events to be queued.") flagset.StringVar(&omitEvents, "omitEvents", "", "Set this flag to a comma sperated list of PolicyViolation, PolicyApplied, PolicyError, PolicySkipped to disable events, e.g. --omitEvents=PolicyApplied,PolicyViolation") flagset.Int64Var(&maxAPICallResponseLength, "maxAPICallResponseLength", 2*1000*1000, "Maximum allowed response size from API Calls. A value of 0 bypasses checks (not recommended).") + flagset.IntVar(&maxBackgroundReports, "maxBackgroundReports", 10000, "Maximum number of background reports before we stop creating new ones") // config appConfig := internal.NewConfiguration( internal.WithProfiling(), @@ -131,7 +136,7 @@ func main() { signalCtx, setup, sdown := internal.Setup(appConfig, "kyverno-background-controller", false) defer sdown() var err error - bgscanInterval := time.Hour + bgscanInterval := 30 * time.Second val := os.Getenv("BACKGROUND_SCAN_INTERVAL") if val != "" { if bgscanInterval, err = time.ParseDuration(val); err != nil { @@ -198,6 +203,18 @@ func main() { polexCache, gcstore, ) + ephrs, err := breaker.StartBackgroundReportsCounter(signalCtx, setup.MetadataClient) + if err != nil { + setup.Logger.Error(err, "failed to start background-scan reports watcher") + os.Exit(1) + } + reportsBreaker := breaker.NewBreaker("background scan reports", func(context.Context) bool { + count, isRunning := ephrs.Count() + if !isRunning { + return true + } + return count > maxBackgroundReports + }) // start informers and wait for cache sync if !internal.StartInformersAndWaitForCacheSync(signalCtx, setup.Logger, kyvernoInformer) { setup.Logger.Error(errors.New("failed to wait for cache sync"), "failed to wait for cache sync") @@ -230,6 +247,7 @@ func main() { setup.Jp, bgscanInterval, urGenerator, + reportsBreaker, ) if err != nil { logger.Error(err, "failed to create leader controllers") diff --git a/config/install-latest-testing.yaml b/config/install-latest-testing.yaml index a4c20f13cc..bc217768d2 100644 --- a/config/install-latest-testing.yaml +++ b/config/install-latest-testing.yaml @@ -49622,6 +49622,20 @@ rules: - patch - update - watch + - apiGroups: + - reports.kyverno.io + resources: + - ephemeralreports + - clusterephemeralreports + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - deletecollection - apiGroups: - networking.k8s.io resources: diff --git a/pkg/background/generate/controller.go b/pkg/background/generate/controller.go index be9726e75b..b767130e40 100644 --- a/pkg/background/generate/controller.go +++ b/pkg/background/generate/controller.go @@ -14,6 +14,7 @@ import ( kyvernov1 "github.com/kyverno/kyverno/api/kyverno/v1" kyvernov2 "github.com/kyverno/kyverno/api/kyverno/v2" "github.com/kyverno/kyverno/pkg/background/common" + "github.com/kyverno/kyverno/pkg/breaker" "github.com/kyverno/kyverno/pkg/client/clientset/versioned" kyvernov1listers "github.com/kyverno/kyverno/pkg/client/listers/kyverno/v1" kyvernov2listers "github.com/kyverno/kyverno/pkg/client/listers/kyverno/v2" @@ -27,6 +28,7 @@ import ( admissionutils "github.com/kyverno/kyverno/pkg/utils/admission" engineutils "github.com/kyverno/kyverno/pkg/utils/engine" kubeutils "github.com/kyverno/kyverno/pkg/utils/kube" + reportutils "github.com/kyverno/kyverno/pkg/utils/report" validationpolicy "github.com/kyverno/kyverno/pkg/validation/policy" "go.uber.org/multierr" admissionv1 "k8s.io/api/admission/v1" @@ -55,6 +57,8 @@ type GenerateController struct { log logr.Logger jp jmespath.Interface + + reportsBreaker breaker.Breaker } // NewGenerateController returns an instance of the Generate-Request Controller @@ -71,20 +75,22 @@ func NewGenerateController( eventGen event.Interface, log logr.Logger, jp jmespath.Interface, + reportsBreaker breaker.Breaker, ) *GenerateController { c := GenerateController{ - client: client, - kyvernoClient: kyvernoClient, - statusControl: statusControl, - engine: engine, - policyLister: policyLister, - npolicyLister: npolicyLister, - urLister: urLister, - nsLister: nsLister, - configuration: dynamicConfig, - eventGen: eventGen, - log: log, - jp: jp, + client: client, + kyvernoClient: kyvernoClient, + statusControl: statusControl, + engine: engine, + policyLister: policyLister, + npolicyLister: npolicyLister, + urLister: urLister, + nsLister: nsLister, + configuration: dynamicConfig, + eventGen: eventGen, + log: log, + jp: jp, + reportsBreaker: reportsBreaker, } return &c } @@ -227,6 +233,11 @@ func (c *GenerateController) applyGenerate(trigger unstructured.Unstructured, ur logger.V(4).Info(doesNotApply) return nil, errors.New(doesNotApply) } + if c.needsReports(trigger) { + if err := c.createReports(context.TODO(), policyContext.NewResource(), engineResponse); err != nil { + c.log.Error(err, "failed to create report") + } + } var applicableRules []string for _, r := range engineResponse.PolicyResponse.Rules { @@ -359,6 +370,34 @@ func (c *GenerateController) GetUnstrResource(genResourceSpec kyvernov1.Resource return resource, nil } +func (c *GenerateController) needsReports(trigger unstructured.Unstructured) bool { + createReport := true + // check if the resource supports reporting + if !reportutils.IsGvkSupported(trigger.GroupVersionKind()) { + createReport = false + } + + return createReport +} + +func (c *GenerateController) createReports( + ctx context.Context, + resource unstructured.Unstructured, + engineResponses ...engineapi.EngineResponse, +) error { + report := reportutils.BuildGenerateReport(resource.GetNamespace(), resource.GetName(), resource.GroupVersionKind(), resource.GetName(), resource.GetUID(), engineResponses...) + if len(report.GetResults()) > 0 { + err := c.reportsBreaker.Do(ctx, func(ctx context.Context) error { + _, err := reportutils.CreateReport(ctx, report, c.kyvernoClient) + return err + }) + if err != nil { + return err + } + } + return nil +} + func updateStatus(statusControl common.StatusControlInterface, ur kyvernov2.UpdateRequest, err error, genResources []kyvernov1.ResourceSpec) error { if err != nil { if _, err := statusControl.Failed(ur.GetName(), err.Error(), genResources); err != nil { diff --git a/pkg/background/mutate/mutate.go b/pkg/background/mutate/mutate.go index 0d2e6130e2..fd7c9b2099 100644 --- a/pkg/background/mutate/mutate.go +++ b/pkg/background/mutate/mutate.go @@ -8,6 +8,7 @@ import ( kyvernov1 "github.com/kyverno/kyverno/api/kyverno/v1" kyvernov2 "github.com/kyverno/kyverno/api/kyverno/v2" "github.com/kyverno/kyverno/pkg/background/common" + "github.com/kyverno/kyverno/pkg/breaker" "github.com/kyverno/kyverno/pkg/client/clientset/versioned" kyvernov1listers "github.com/kyverno/kyverno/pkg/client/listers/kyverno/v1" "github.com/kyverno/kyverno/pkg/clients/dclient" @@ -17,6 +18,7 @@ import ( "github.com/kyverno/kyverno/pkg/event" admissionutils "github.com/kyverno/kyverno/pkg/utils/admission" engineutils "github.com/kyverno/kyverno/pkg/utils/engine" + reportutils "github.com/kyverno/kyverno/pkg/utils/report" "go.uber.org/multierr" admissionv1 "k8s.io/api/admission/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -44,6 +46,8 @@ type mutateExistingController struct { log logr.Logger jp jmespath.Interface + + reportsBreaker breaker.Breaker } // NewMutateExistingController returns an instance of the MutateExistingController @@ -59,19 +63,21 @@ func NewMutateExistingController( eventGen event.Interface, log logr.Logger, jp jmespath.Interface, + reportsBreaker breaker.Breaker, ) *mutateExistingController { c := mutateExistingController{ - client: client, - kyvernoClient: kyvernoClient, - statusControl: statusControl, - engine: engine, - policyLister: policyLister, - npolicyLister: npolicyLister, - nsLister: nsLister, - configuration: dynamicConfig, - eventGen: eventGen, - log: log, - jp: jp, + client: client, + kyvernoClient: kyvernoClient, + statusControl: statusControl, + engine: engine, + policyLister: policyLister, + npolicyLister: npolicyLister, + nsLister: nsLister, + configuration: dynamicConfig, + eventGen: eventGen, + log: log, + jp: jp, + reportsBreaker: reportsBreaker, } return &c } @@ -80,6 +86,7 @@ func (c *mutateExistingController) ProcessUR(ur *kyvernov2.UpdateRequest) error logger := c.log.WithValues("name", ur.GetName(), "policy", ur.Spec.GetPolicyKey(), "resource", ur.Spec.GetResource().String()) var errs []error + logger.Info("processing mutate existing") policy, err := c.getPolicy(ur) if err != nil { logger.Error(err, "failed to get policy") @@ -157,6 +164,11 @@ func (c *mutateExistingController) ProcessUR(ur *kyvernov2.UpdateRequest) error } er := c.engine.Mutate(context.TODO(), policyContext) + if c.needsReports(trigger) { + if err := c.createReports(context.TODO(), policyContext.NewResource(), er); err != nil { + c.log.Error(err, "failed to create report") + } + } for _, r := range er.PolicyResponse.Rules { patched, parentGVR, patchedSubresource := r.PatchedTarget() switch r.Status() { @@ -243,6 +255,36 @@ func (c *mutateExistingController) report(err error, policy kyvernov1.PolicyInte c.eventGen.Add(events...) } +func (c *mutateExistingController) needsReports(trigger *unstructured.Unstructured) bool { + createReport := true + if trigger == nil { + return createReport + } + if !reportutils.IsGvkSupported(trigger.GroupVersionKind()) { + createReport = false + } + + return createReport +} + +func (c *mutateExistingController) createReports( + ctx context.Context, + resource unstructured.Unstructured, + engineResponses ...engineapi.EngineResponse, +) error { + report := reportutils.BuildMutateExistingReport(resource.GetNamespace(), resource.GetName(), resource.GroupVersionKind(), resource.GetName(), resource.GetUID(), engineResponses...) + if len(report.GetResults()) > 0 { + err := c.reportsBreaker.Do(ctx, func(ctx context.Context) error { + _, err := reportutils.CreateReport(ctx, report, c.kyvernoClient) + return err + }) + if err != nil { + return err + } + } + return nil +} + func updateURStatus(statusControl common.StatusControlInterface, ur kyvernov2.UpdateRequest, err error) error { if err != nil { if _, err := statusControl.Failed(ur.GetName(), err.Error(), nil); err != nil { diff --git a/pkg/background/update_request_controller.go b/pkg/background/update_request_controller.go index 072ed46a0e..8c4d2880f9 100644 --- a/pkg/background/update_request_controller.go +++ b/pkg/background/update_request_controller.go @@ -10,6 +10,7 @@ import ( common "github.com/kyverno/kyverno/pkg/background/common" "github.com/kyverno/kyverno/pkg/background/generate" "github.com/kyverno/kyverno/pkg/background/mutate" + "github.com/kyverno/kyverno/pkg/breaker" "github.com/kyverno/kyverno/pkg/client/clientset/versioned" kyvernov1informers "github.com/kyverno/kyverno/pkg/client/informers/externalversions/kyverno/v1" kyvernov2informers "github.com/kyverno/kyverno/pkg/client/informers/externalversions/kyverno/v2" @@ -57,9 +58,10 @@ type controller struct { // queue queue workqueue.TypedRateLimitingInterface[any] - eventGen event.Interface - configuration config.Configuration - jp jmespath.Interface + eventGen event.Interface + configuration config.Configuration + jp jmespath.Interface + reportsBreaker breaker.Breaker } // NewController returns an instance of the Generate-Request Controller @@ -74,20 +76,22 @@ func NewController( eventGen event.Interface, configuration config.Configuration, jp jmespath.Interface, + reportsBreaker breaker.Breaker, ) Controller { urLister := urInformer.Lister().UpdateRequests(config.KyvernoNamespace()) c := controller{ - client: client, - kyvernoClient: kyvernoClient, - engine: engine, - cpolLister: cpolInformer.Lister(), - polLister: polInformer.Lister(), - urLister: urLister, - nsLister: namespaceInformer.Lister(), - queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultTypedControllerRateLimiter[any](), "background"), - eventGen: eventGen, - configuration: configuration, - jp: jp, + client: client, + kyvernoClient: kyvernoClient, + engine: engine, + cpolLister: cpolInformer.Lister(), + polLister: polInformer.Lister(), + urLister: urLister, + nsLister: namespaceInformer.Lister(), + queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultTypedControllerRateLimiter[any](), "background"), + eventGen: eventGen, + configuration: configuration, + jp: jp, + reportsBreaker: reportsBreaker, } _, _ = urInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: c.addUR, @@ -220,10 +224,10 @@ func (c *controller) processUR(ur *kyvernov2.UpdateRequest) error { statusControl := common.NewStatusControl(c.kyvernoClient, c.urLister) switch ur.Spec.GetRequestType() { case kyvernov2.Mutate: - ctrl := mutate.NewMutateExistingController(c.client, c.kyvernoClient, statusControl, c.engine, c.cpolLister, c.polLister, c.nsLister, c.configuration, c.eventGen, logger, c.jp) + ctrl := mutate.NewMutateExistingController(c.client, c.kyvernoClient, statusControl, c.engine, c.cpolLister, c.polLister, c.nsLister, c.configuration, c.eventGen, logger, c.jp, c.reportsBreaker) return ctrl.ProcessUR(ur) case kyvernov2.Generate: - ctrl := generate.NewGenerateController(c.client, c.kyvernoClient, statusControl, c.engine, c.cpolLister, c.polLister, c.urLister, c.nsLister, c.configuration, c.eventGen, logger, c.jp) + ctrl := generate.NewGenerateController(c.client, c.kyvernoClient, statusControl, c.engine, c.cpolLister, c.polLister, c.urLister, c.nsLister, c.configuration, c.eventGen, logger, c.jp, c.reportsBreaker) return ctrl.ProcessUR(ur) } return nil diff --git a/pkg/breaker/resource_counter.go b/pkg/breaker/resource_counter.go index 057da56cc0..ec097e8842 100644 --- a/pkg/breaker/resource_counter.go +++ b/pkg/breaker/resource_counter.go @@ -108,6 +108,23 @@ func StartAdmissionReportsCounter(ctx context.Context, client metadataclient.Int }, nil } +func StartBackgroundReportsCounter(ctx context.Context, client metadataclient.Interface) (Counter, error) { + tweakListOptions := func(lo *metav1.ListOptions) { + lo.LabelSelector = "audit.kyverno.io/source==background-scan" + } + ephrs, err := StartResourceCounter(ctx, client, reportsv1.SchemeGroupVersion.WithResource("ephemeralreports"), tweakListOptions) + if err != nil { + return nil, err + } + cephrs, err := StartResourceCounter(ctx, client, reportsv1.SchemeGroupVersion.WithResource("clusterephemeralreports"), tweakListOptions) + if err != nil { + return nil, err + } + return composite{ + inner: []Counter{ephrs, cephrs}, + }, nil +} + type composite struct { inner []Counter } diff --git a/pkg/controllers/report/aggregate/controller.go b/pkg/controllers/report/aggregate/controller.go index d0def4a5b1..170533eec6 100644 --- a/pkg/controllers/report/aggregate/controller.go +++ b/pkg/controllers/report/aggregate/controller.go @@ -330,27 +330,30 @@ func (c *controller) findResource(ctx context.Context, reportMeta *metav1.Partia return resource, nil } -func (c *controller) adopt(ctx context.Context, reportMeta *metav1.PartialObjectMetadata) bool { +func (c *controller) adopt(ctx context.Context, reportMeta *metav1.PartialObjectMetadata) (bool, bool) { resource, err := c.findResource(ctx, reportMeta) if err != nil { - return false + if apierrors.IsForbidden(err) { + return false, true + } + return false, false } if resource == nil { - return false + return false, false } report, err := c.getEphemeralReport(ctx, reportMeta.GetNamespace(), reportMeta.GetName()) if err != nil { - return false + return false, false } if report == nil { - return false + return false, false } controllerutils.SetOwner(report, resource.GetAPIVersion(), resource.GetKind(), resource.GetName(), resource.GetUID()) reportutils.SetResourceUid(report, resource.GetUID()) if _, err := updateReport(ctx, report, c.client); err != nil { - return false + return false, false } - return true + return true, false } func (c *controller) frontReconcile(ctx context.Context, logger logr.Logger, _, namespace, name string) error { @@ -371,8 +374,11 @@ func (c *controller) frontReconcile(ctx context.Context, logger logr.Logger, _, return nil } // try to find the owner - if c.adopt(ctx, reportMeta) { + if adopted, forbidden := c.adopt(ctx, reportMeta); adopted { return nil + } else if forbidden { + logger.Info("deleting because insufficient permission to fetch resource") + return c.deleteEphemeralReport(ctx, reportMeta.GetNamespace(), reportMeta.GetName()) } // if not found and too old, forget about it if isTooOld(reportMeta) { diff --git a/pkg/policy/generate/fake.go b/pkg/policy/generate/fake.go index bd30293fe8..be0ab5c963 100644 --- a/pkg/policy/generate/fake.go +++ b/pkg/policy/generate/fake.go @@ -16,8 +16,9 @@ type FakeGenerate struct { // fake/mock implementation for operation access(always returns true) func NewFakeGenerate(rule kyvernov1.Generation) *FakeGenerate { g := FakeGenerate{} - g.rule = rule + g.rule = &kyvernov1.Rule{Generation: &rule} g.authChecker = fake.NewFakeAuth() + g.authCheckerReports = fake.NewFakeAuth() g.log = logging.GlobalLogger() return &g } diff --git a/pkg/policy/generate/validate.go b/pkg/policy/generate/validate.go index b476e66dde..0e9b355dbb 100644 --- a/pkg/policy/generate/validate.go +++ b/pkg/policy/generate/validate.go @@ -17,19 +17,21 @@ import ( // Generate provides implementation to validate 'generate' rule type Generate struct { - user string - rule kyvernov1.Generation - authChecker auth.AuthChecks - log logr.Logger + user string + rule *kyvernov1.Rule + authChecker auth.AuthChecks + authCheckerReports auth.AuthChecks + log logr.Logger } // NewGenerateFactory returns a new instance of Generate validation checker -func NewGenerateFactory(client dclient.Interface, rule kyvernov1.Generation, user string, log logr.Logger) *Generate { +func NewGenerateFactory(client dclient.Interface, rule *kyvernov1.Rule, user, reportsSA string, log logr.Logger) *Generate { g := Generate{ - user: user, - rule: rule, - authChecker: auth.NewAuth(client, user, log), - log: log, + user: user, + rule: rule, + authChecker: auth.NewAuth(client, user, log), + authCheckerReports: auth.NewAuth(client, reportsSA, log), + log: log, } return &g @@ -38,13 +40,13 @@ func NewGenerateFactory(client dclient.Interface, rule kyvernov1.Generation, use // Validate validates the 'generate' rule func (g *Generate) Validate(ctx context.Context, verbs []string) (warnings []string, path string, err error) { rule := g.rule - if rule.CloneList.Selector != nil { - if wildcard.ContainsWildcard(rule.CloneList.Selector.String()) { + if rule.Generation.CloneList.Selector != nil { + if wildcard.ContainsWildcard(rule.Generation.CloneList.Selector.String()) { return nil, "selector", fmt.Errorf("wildcard characters `*/?` not supported") } } - if target := rule.GetData(); target != nil { + if target := rule.Generation.GetData(); target != nil { // TODO: is this required ?? as anchors can only be on pattern and not resource // we can add this check by not sure if its needed here if path, err := common.ValidatePattern(target, "/", nil); err != nil { @@ -57,18 +59,23 @@ func (g *Generate) Validate(ctx context.Context, verbs []string) (warnings []str // instructions to modify the RBAC for kyverno are mentioned at https://github.com/kyverno/kyverno/blob/master/documentation/installation.md // - operations required: create/update/delete/get // If kind and namespace contain variables, then we cannot resolve then so we skip the processing - if rule.ForEachGeneration != nil { - for _, forEach := range rule.ForEachGeneration { + if rule.Generation.ForEachGeneration != nil { + for _, forEach := range rule.Generation.ForEachGeneration { if err := g.validateAuth(ctx, verbs, forEach.GeneratePattern); err != nil { return nil, "foreach", err } } } else { - if err := g.validateAuth(ctx, verbs, rule.GeneratePattern); err != nil { + if err := g.validateAuth(ctx, verbs, rule.Generation.GeneratePattern); err != nil { return nil, "", err } } - return nil, "", nil + if w, err := g.validateAuthReports(ctx); err != nil { + return nil, "", err + } else if len(w) > 0 { + warnings = append(warnings, w...) + } + return warnings, "", nil } func (g *Generate) validateAuth(ctx context.Context, verbs []string, generate kyvernov1.GeneratePattern) error { @@ -92,7 +99,7 @@ func (g *Generate) canIGenerate(ctx context.Context, verbs []string, gvk, namesp if verbs == nil { verbs = []string{"get", "create"} - if g.rule.Synchronize { + if g.rule.Generation.Synchronize { verbs = []string{"get", "create", "update", "delete"} } } @@ -117,3 +124,23 @@ func parseCloneKind(gvks string) (gvk, sub string) { } return k, sub } + +func (g *Generate) validateAuthReports(ctx context.Context) (warnings []string, err error) { + kinds := g.rule.MatchResources.GetKinds() + for _, k := range kinds { + if wildcard.ContainsWildcard(k) { + return nil, nil + } + + verbs := []string{"get", "list", "watch"} + ok, msg, err := g.authCheckerReports.CanI(ctx, verbs, k, "", "", "") + if err != nil { + return nil, err + } + if !ok { + warnings = append(warnings, msg) + } + } + + return warnings, nil +} diff --git a/pkg/policy/mutate/validate.go b/pkg/policy/mutate/validate.go index fedc0dd187..e2251abc42 100644 --- a/pkg/policy/mutate/validate.go +++ b/pkg/policy/mutate/validate.go @@ -6,6 +6,7 @@ import ( "strings" kyvernov1 "github.com/kyverno/kyverno/api/kyverno/v1" + "github.com/kyverno/kyverno/ext/wildcard" "github.com/kyverno/kyverno/pkg/clients/dclient" "github.com/kyverno/kyverno/pkg/engine/variables/regex" "github.com/kyverno/kyverno/pkg/logging" @@ -17,22 +18,26 @@ import ( // Mutate provides implementation to validate 'mutate' rule type Mutate struct { - mutation kyvernov1.Mutation - authChecker auth.AuthChecks + rule *kyvernov1.Rule + authCheckerBackground auth.AuthChecks + authCheckerReports auth.AuthChecks } // NewMutateFactory returns a new instance of Mutate validation checker -func NewMutateFactory(m kyvernov1.Mutation, client dclient.Interface, mock bool, backgroundSA string) *Mutate { - var authCheck auth.AuthChecks +func NewMutateFactory(rule *kyvernov1.Rule, client dclient.Interface, mock bool, backgroundSA, reportsSA string) *Mutate { + var authCheckBackground, authCheckerReports auth.AuthChecks if mock { - authCheck = fake.NewFakeAuth() + authCheckBackground = fake.NewFakeAuth() + authCheckerReports = fake.NewFakeAuth() } else { - authCheck = auth.NewAuth(client, backgroundSA, logging.GlobalLogger()) + authCheckBackground = auth.NewAuth(client, backgroundSA, logging.GlobalLogger()) + authCheckerReports = auth.NewAuth(client, reportsSA, logging.GlobalLogger()) } return &Mutate{ - mutation: m, - authChecker: authCheck, + rule: rule, + authCheckerBackground: authCheckBackground, + authCheckerReports: authCheckerReports, } } @@ -43,19 +48,24 @@ func (m *Mutate) Validate(ctx context.Context, _ []string) (warnings []string, p return nil, "foreach", fmt.Errorf("only one of `foreach`, `patchStrategicMerge`, or `patchesJson6902` is allowed") } - return m.validateForEach("", m.mutation.ForEachMutation) + return m.validateForEach("", m.rule.Mutation.ForEachMutation) } if m.hasPatchesJSON6902() && m.hasPatchStrategicMerge() { return nil, "foreach", fmt.Errorf("only one of `patchStrategicMerge` or `patchesJson6902` is allowed") } - if m.mutation.Targets != nil { - if err := m.validateAuth(ctx, m.mutation.Targets); err != nil { - return nil, "targets", fmt.Errorf("auth check fails, additional privileges are required for the service account '%s': %v", m.authChecker.User(), err) + if m.rule.Mutation.Targets != nil { + if err := m.validateAuth(ctx, m.rule.Mutation.Targets); err != nil { + return nil, "targets", fmt.Errorf("auth check fails, additional privileges are required for the service account '%s': %v", m.authCheckerBackground.User(), err) } } - return nil, "", nil + if w, err := m.validateAuthReports(ctx); err != nil { + return nil, "", err + } else if len(w) > 0 { + warnings = append(warnings, w...) + } + return warnings, "", nil } func (m *Mutate) validateForEach(tag string, foreach []kyvernov1.ForEachMutation) (warnings []string, path string, err error) { @@ -88,18 +98,18 @@ func (m *Mutate) validateNestedForEach(tag string, j []kyvernov1.ForEachMutation } func (m *Mutate) hasForEach() bool { - return len(m.mutation.ForEachMutation) > 0 + return len(m.rule.Mutation.ForEachMutation) > 0 } func (m *Mutate) hasPatchStrategicMerge() bool { - return m.mutation.GetPatchStrategicMerge() != nil + return m.rule.Mutation.GetPatchStrategicMerge() != nil } func (m *Mutate) hasPatchesJSON6902() bool { - return m.mutation.PatchesJSON6902 != "" + return m.rule.Mutation.PatchesJSON6902 != "" } -func (m *Mutate) validateAuth(ctx context.Context, targets []kyvernov1.TargetResourceSpec) error { +func (m *Mutate) validateAuth(ctx context.Context, targets []kyvernov1.TargetResourceSpec) (err error) { var errs []error for _, target := range targets { if regex.IsVariable(target.Kind) { @@ -108,7 +118,7 @@ func (m *Mutate) validateAuth(ctx context.Context, targets []kyvernov1.TargetRes _, _, k, sub := kubeutils.ParseKindSelector(target.Kind) gvk := strings.Join([]string{target.APIVersion, k}, "/") verbs := []string{"get", "update"} - ok, msg, err := m.authChecker.CanI(ctx, verbs, gvk, target.Namespace, target.Name, sub) + ok, msg, err := m.authCheckerBackground.CanI(ctx, verbs, gvk, target.Namespace, target.Name, sub) if err != nil { return err } @@ -119,3 +129,23 @@ func (m *Mutate) validateAuth(ctx context.Context, targets []kyvernov1.TargetRes return multierr.Combine(errs...) } + +func (m *Mutate) validateAuthReports(ctx context.Context) (warnings []string, err error) { + kinds := m.rule.MatchResources.GetKinds() + for _, k := range kinds { + if wildcard.ContainsWildcard(k) { + return nil, nil + } + + verbs := []string{"get", "list", "watch"} + ok, msg, err := m.authCheckerReports.CanI(ctx, verbs, k, "", "", "") + if err != nil { + return nil, err + } + if !ok { + warnings = append(warnings, msg) + } + } + + return warnings, nil +} diff --git a/pkg/utils/report/new.go b/pkg/utils/report/new.go index e6d54ef1ca..e8888aa1cb 100644 --- a/pkg/utils/report/new.go +++ b/pkg/utils/report/new.go @@ -54,6 +54,24 @@ func NewBackgroundScanReport(namespace, name string, gvk schema.GroupVersionKind return report } +func BuildMutationReport(resource unstructured.Unstructured, request admissionv1.AdmissionRequest, responses ...engineapi.EngineResponse) reportsv1.ReportInterface { + report := NewAdmissionReport(resource.GetNamespace(), string(request.UID), schema.GroupVersionResource(request.Resource), schema.GroupVersionKind(request.Kind), resource) + SetMutationResponses(report, responses...) + return report +} + +func BuildMutateExistingReport(namespace, name string, gvk schema.GroupVersionKind, owner string, uid types.UID, responses ...engineapi.EngineResponse) reportsv1.ReportInterface { + report := NewBackgroundScanReport(namespace, name, gvk, owner, uid) + SetMutationResponses(report, responses...) + return report +} + +func BuildGenerateReport(namespace, name string, gvk schema.GroupVersionKind, owner string, uid types.UID, responses ...engineapi.EngineResponse) reportsv1.ReportInterface { + report := NewBackgroundScanReport(namespace, name, gvk, owner, uid) + SetGenerationResponses(report, responses...) + return report +} + func NewPolicyReport(namespace, name string, scope *corev1.ObjectReference, results ...policyreportv1alpha2.PolicyReportResult) reportsv1.ReportInterface { var report reportsv1.ReportInterface if namespace == "" { diff --git a/pkg/utils/report/results.go b/pkg/utils/report/results.go index f3ee14e884..0715a34747 100644 --- a/pkg/utils/report/results.go +++ b/pkg/utils/report/results.go @@ -15,6 +15,7 @@ import ( "github.com/kyverno/kyverno/pkg/pss/utils" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/tools/cache" ) @@ -191,6 +192,42 @@ func EngineResponseToReportResults(response engineapi.EngineResponse) []policyre return results } +func MutationEngineResponseToReportResults(response engineapi.EngineResponse) []policyreportv1alpha2.PolicyReportResult { + pol := response.Policy() + policyName, _ := cache.MetaNamespaceKeyFunc(pol.AsKyvernoPolicy()) + policyType := pol.GetType() + annotations := pol.GetAnnotations() + + results := make([]policyreportv1alpha2.PolicyReportResult, 0, len(response.PolicyResponse.Rules)) + for _, ruleResult := range response.PolicyResponse.Rules { + result := ToPolicyReportResult(policyType, policyName, ruleResult, annotations, nil) + if target, _, _ := ruleResult.PatchedTarget(); target != nil { + addProperty("patched-target", getResourceInfo(target.GroupVersionKind(), target.GetName(), target.GetNamespace()), &result) + } + results = append(results, result) + } + + return results +} + +func GenerationEngineResponseToReportResults(response engineapi.EngineResponse) []policyreportv1alpha2.PolicyReportResult { + pol := response.Policy() + policyName, _ := cache.MetaNamespaceKeyFunc(pol.AsKyvernoPolicy()) + policyType := pol.GetType() + annotations := pol.GetAnnotations() + + results := make([]policyreportv1alpha2.PolicyReportResult, 0, len(response.PolicyResponse.Rules)) + for _, ruleResult := range response.PolicyResponse.Rules { + result := ToPolicyReportResult(policyType, policyName, ruleResult, annotations, nil) + if generatedResource := ruleResult.GeneratedResource(); generatedResource.GetName() != "" { + addProperty("generated-resource", getResourceInfo(generatedResource.GroupVersionKind(), generatedResource.GetName(), generatedResource.GetNamespace()), &result) + } + results = append(results, result) + } + + return results +} + func SplitResultsByPolicy(logger logr.Logger, results []policyreportv1alpha2.PolicyReportResult) map[string][]policyreportv1alpha2.PolicyReportResult { resultsMap := map[string][]policyreportv1alpha2.PolicyReportResult{} keysMap := map[string]string{} @@ -230,3 +267,31 @@ func SetResponses(report reportsv1.ReportInterface, engineResponses ...engineapi } SetResults(report, ruleResults...) } + +func SetMutationResponses(report reportsv1.ReportInterface, engineResponses ...engineapi.EngineResponse) { + var ruleResults []policyreportv1alpha2.PolicyReportResult + for _, result := range engineResponses { + pol := result.Policy() + SetPolicyLabel(report, pol) + ruleResults = append(ruleResults, MutationEngineResponseToReportResults(result)...) + } + SetResults(report, ruleResults...) +} + +func SetGenerationResponses(report reportsv1.ReportInterface, engineResponses ...engineapi.EngineResponse) { + var ruleResults []policyreportv1alpha2.PolicyReportResult + for _, result := range engineResponses { + pol := result.Policy() + SetPolicyLabel(report, pol) + ruleResults = append(ruleResults, GenerationEngineResponseToReportResults(result)...) + } + SetResults(report, ruleResults...) +} + +func getResourceInfo(gvk schema.GroupVersionKind, name, namespace string) string { + info := gvk.String() + " Name=" + name + if len(namespace) != 0 { + info = info + " Namespace=" + namespace + } + return info +} diff --git a/pkg/validation/policy/actions.go b/pkg/validation/policy/actions.go index 53bf0d6368..a15a823f0c 100644 --- a/pkg/validation/policy/actions.go +++ b/pkg/validation/policy/actions.go @@ -35,7 +35,7 @@ func validateActions(idx int, rule *kyvernov1.Rule, client dclient.Interface, mo // Mutate if rule.HasMutate() { - checker = mutate.NewMutateFactory(*rule.Mutation, client, mock, backgroundSA) + checker = mutate.NewMutateFactory(rule, client, mock, backgroundSA, reportsSA) if w, path, err := checker.Validate(context.TODO(), nil); err != nil { return nil, fmt.Errorf("path: spec.rules[%d].mutate.%s.: %v", idx, path, err) } else if w != nil { @@ -79,14 +79,14 @@ func validateActions(idx int, rule *kyvernov1.Rule, client dclient.Interface, mo } else { if rule.Generation.Synchronize { admissionSA := fmt.Sprintf("system:serviceaccount:%s:%s", config.KyvernoNamespace(), config.KyvernoServiceAccountName()) - checker = generate.NewGenerateFactory(client, *rule.Generation, admissionSA, logging.GlobalLogger()) + checker = generate.NewGenerateFactory(client, rule, admissionSA, reportsSA, logging.GlobalLogger()) if w, path, err := checker.Validate(context.TODO(), []string{"list", "get"}); err != nil { return nil, fmt.Errorf("path: spec.rules[%d].generate.%s.: %v", idx, path, err) } else if warnings != nil { warnings = append(warnings, w...) } } - checker = generate.NewGenerateFactory(client, *rule.Generation, backgroundSA, logging.GlobalLogger()) + checker = generate.NewGenerateFactory(client, rule, backgroundSA, reportsSA, logging.GlobalLogger()) if w, path, err := checker.Validate(context.TODO(), nil); err != nil { return nil, fmt.Errorf("path: spec.rules[%d].generate.%s.: %v", idx, path, err) } else if warnings != nil { diff --git a/pkg/webhooks/resource/handlers.go b/pkg/webhooks/resource/handlers.go index e7f299b8da..fe67c6b28b 100644 --- a/pkg/webhooks/resource/handlers.go +++ b/pkg/webhooks/resource/handlers.go @@ -200,8 +200,8 @@ func (h *resourceHandlers) Mutate(ctx context.Context, logger logr.Logger, reque logger.Error(err, "failed to build policy context") return admissionutils.Response(request.UID, err) } - mh := mutation.NewMutationHandler(logger, h.engine, h.eventGen, h.nsLister, h.metricsConfig) - patches, warnings, err := mh.HandleMutation(ctx, request.AdmissionRequest, mutatePolicies, policyContext, startTime, h.configuration) + mh := mutation.NewMutationHandler(logger, h.kyvernoClient, h.engine, h.eventGen, h.nsLister, h.metricsConfig, h.admissionReports, h.reportsBreaker) + patches, warnings, err := mh.HandleMutation(ctx, request, mutatePolicies, policyContext, startTime, h.configuration) if err != nil { logger.Error(err, "mutation failed") return admissionutils.Response(request.UID, err) diff --git a/pkg/webhooks/resource/mutation/mutation.go b/pkg/webhooks/resource/mutation/mutation.go index 16c8a4967e..5ff1c09cb5 100644 --- a/pkg/webhooks/resource/mutation/mutation.go +++ b/pkg/webhooks/resource/mutation/mutation.go @@ -7,6 +7,8 @@ import ( "github.com/go-logr/logr" kyvernov1 "github.com/kyverno/kyverno/api/kyverno/v1" + "github.com/kyverno/kyverno/pkg/breaker" + "github.com/kyverno/kyverno/pkg/client/clientset/versioned" "github.com/kyverno/kyverno/pkg/config" "github.com/kyverno/kyverno/pkg/engine" engineapi "github.com/kyverno/kyverno/pkg/engine/api" @@ -17,10 +19,13 @@ import ( "github.com/kyverno/kyverno/pkg/tracing" engineutils "github.com/kyverno/kyverno/pkg/utils/engine" jsonutils "github.com/kyverno/kyverno/pkg/utils/json" + reportutils "github.com/kyverno/kyverno/pkg/utils/report" + "github.com/kyverno/kyverno/pkg/webhooks/handlers" webhookutils "github.com/kyverno/kyverno/pkg/webhooks/utils" "go.opentelemetry.io/otel/trace" "gomodules.xyz/jsonpatch/v2" admissionv1 "k8s.io/api/admission/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" corev1listers "k8s.io/client-go/listers/core/v1" ) @@ -28,36 +33,45 @@ type MutationHandler interface { // HandleMutation handles validating webhook admission request // If there are no errors in validating rule we apply generation rules // patchedResource is the (resource + patches) after applying mutation rules - HandleMutation(context.Context, admissionv1.AdmissionRequest, []kyvernov1.PolicyInterface, *engine.PolicyContext, time.Time, config.Configuration) ([]byte, []string, error) + HandleMutation(context.Context, handlers.AdmissionRequest, []kyvernov1.PolicyInterface, *engine.PolicyContext, time.Time, config.Configuration) ([]byte, []string, error) } func NewMutationHandler( log logr.Logger, + kyvernoClient versioned.Interface, engine engineapi.Engine, eventGen event.Interface, nsLister corev1listers.NamespaceLister, metrics metrics.MetricsConfigManager, + admissionReports bool, + reportsBreaker breaker.Breaker, ) MutationHandler { return &mutationHandler{ - log: log, - engine: engine, - eventGen: eventGen, - nsLister: nsLister, - metrics: metrics, + log: log, + kyvernoClient: kyvernoClient, + engine: engine, + eventGen: eventGen, + nsLister: nsLister, + metrics: metrics, + admissionReports: admissionReports, + reportsBreaker: reportsBreaker, } } type mutationHandler struct { - log logr.Logger - engine engineapi.Engine - eventGen event.Interface - nsLister corev1listers.NamespaceLister - metrics metrics.MetricsConfigManager + log logr.Logger + kyvernoClient versioned.Interface + engine engineapi.Engine + eventGen event.Interface + nsLister corev1listers.NamespaceLister + metrics metrics.MetricsConfigManager + admissionReports bool + reportsBreaker breaker.Breaker } func (h *mutationHandler) HandleMutation( ctx context.Context, - request admissionv1.AdmissionRequest, + request handlers.AdmissionRequest, policies []kyvernov1.PolicyInterface, policyContext *engine.PolicyContext, admissionRequestTimestamp time.Time, @@ -77,7 +91,7 @@ func (h *mutationHandler) HandleMutation( // return value: generated patches, triggered policies, engine responses correspdonding to the triggered policies func (v *mutationHandler) applyMutations( ctx context.Context, - request admissionv1.AdmissionRequest, + request handlers.AdmissionRequest, policies []kyvernov1.PolicyInterface, policyContext *engine.PolicyContext, cfg config.Configuration, @@ -107,7 +121,7 @@ func (v *mutationHandler) applyMutations( failurePolicy = kyvernov1.Fail } - engineResponse, policyPatches, err := v.applyMutation(ctx, request, currentContext, failurePolicy) + engineResponse, policyPatches, err := v.applyMutation(ctx, request.AdmissionRequest, currentContext, failurePolicy) if err != nil { return fmt.Errorf("mutation policy %s error: %v", policy.GetName(), err) } @@ -141,6 +155,12 @@ func (v *mutationHandler) applyMutations( events := webhookutils.GenerateEvents(engineResponses, false, cfg) v.eventGen.Add(events...) + if v.needsReports(request, policyContext.NewResource(), v.admissionReports) { + if err := v.createReports(ctx, policyContext.NewResource(), request, engineResponses...); err != nil { + v.log.Error(err, "failed to create report") + } + } + logMutationResponse(patches, engineResponses, v.log) // patches holds all the successful patches, if no patch is created, it returns nil @@ -168,6 +188,25 @@ func (h *mutationHandler) applyMutation(ctx context.Context, request admissionv1 return &engineResponse, policyPatches, nil } +func (h *mutationHandler) createReports( + ctx context.Context, + resource unstructured.Unstructured, + request handlers.AdmissionRequest, + engineResponses ...engineapi.EngineResponse, +) error { + report := reportutils.BuildMutationReport(resource, request.AdmissionRequest, engineResponses...) + if len(report.GetResults()) > 0 { + err := h.reportsBreaker.Do(ctx, func(ctx context.Context) error { + _, err := reportutils.CreateReport(ctx, report, h.kyvernoClient) + return err + }) + if err != nil { + return err + } + } + return nil +} + func logMutationResponse(patches []jsonpatch.JsonPatchOperation, engineResponses []engineapi.EngineResponse, logger logr.Logger) { if len(patches) != 0 { logger.V(4).Info("created patches", "count", len(patches)) diff --git a/pkg/webhooks/resource/mutation/utils.go b/pkg/webhooks/resource/mutation/utils.go new file mode 100644 index 0000000000..4562239eee --- /dev/null +++ b/pkg/webhooks/resource/mutation/utils.go @@ -0,0 +1,27 @@ +package mutation + +import ( + admissionutils "github.com/kyverno/kyverno/pkg/utils/admission" + reportutils "github.com/kyverno/kyverno/pkg/utils/report" + "github.com/kyverno/kyverno/pkg/webhooks/handlers" + admissionv1 "k8s.io/api/admission/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +func (v *mutationHandler) needsReports(request handlers.AdmissionRequest, resource unstructured.Unstructured, admissionReport bool) bool { + createReport := admissionReport + if admissionutils.IsDryRun(request.AdmissionRequest) { + createReport = false + } + // we don't need reports for deletions + if request.Operation == admissionv1.Delete { + createReport = false + } + // check if the resource supports reporting + if !reportutils.IsGvkSupported(schema.GroupVersionKind(request.Kind)) { + createReport = false + } + + return createReport +} diff --git a/test/conformance/chainsaw/e2e-matrix.json b/test/conformance/chainsaw/e2e-matrix.json index ccb56378e5..09341d1b2c 100644 --- a/test/conformance/chainsaw/e2e-matrix.json +++ b/test/conformance/chainsaw/e2e-matrix.json @@ -104,7 +104,8 @@ "^rbac$" ], "reports": [ - "^reports$" + "^reports$/^admission$/^(exception|mutation|namespaceselector|namespaceselector-assert|test-report-admission-mode|test-report-audit-warn|test-report-properties|two-rules-with-different-modes|update|update-deployment)\\[.*\\]$", + "^reports$/^background$/^(exception|exception-assert|exception-with-conditions|exception-with-podsecurity|generate|multiple-exceptions-with-pod-security|mutate-existing|report-deletion|test-report-background-mode|two-rules-with-different-modes|verify-image-fail|verify-image-pass)\\[.*\\]$" ], "sigstore-custom-tuf": [ "^sigstore-custom-tuf$" diff --git a/test/conformance/chainsaw/reports/admission/mutation/README.md b/test/conformance/chainsaw/reports/admission/mutation/README.md new file mode 100644 index 0000000000..7ca7b77a9e --- /dev/null +++ b/test/conformance/chainsaw/reports/admission/mutation/README.md @@ -0,0 +1,3 @@ +# Title + +This is a basic mutation test. diff --git a/test/conformance/chainsaw/reports/admission/mutation/chainsaw-step-01-apply-1-1.yaml b/test/conformance/chainsaw/reports/admission/mutation/chainsaw-step-01-apply-1-1.yaml new file mode 100755 index 0000000000..9efe7e7f6f --- /dev/null +++ b/test/conformance/chainsaw/reports/admission/mutation/chainsaw-step-01-apply-1-1.yaml @@ -0,0 +1,19 @@ +apiVersion: kyverno.io/v1 +kind: ClusterPolicy +metadata: + name: add-labels +spec: + admission: true + background: true + rules: + - match: + any: + - resources: + kinds: + - Pod + mutate: + patchStrategicMerge: + metadata: + labels: + foo: bar + name: add-labels diff --git a/test/conformance/chainsaw/reports/admission/mutation/chainsaw-step-01-assert-1-1.yaml b/test/conformance/chainsaw/reports/admission/mutation/chainsaw-step-01-assert-1-1.yaml new file mode 100755 index 0000000000..7e9f14965b --- /dev/null +++ b/test/conformance/chainsaw/reports/admission/mutation/chainsaw-step-01-assert-1-1.yaml @@ -0,0 +1,9 @@ +apiVersion: kyverno.io/v1 +kind: ClusterPolicy +metadata: + name: add-labels +status: + conditions: + - reason: Succeeded + status: "True" + type: Ready diff --git a/test/conformance/chainsaw/reports/admission/mutation/chainsaw-step-02-apply-1-1.yaml b/test/conformance/chainsaw/reports/admission/mutation/chainsaw-step-02-apply-1-1.yaml new file mode 100755 index 0000000000..3da6f315fb --- /dev/null +++ b/test/conformance/chainsaw/reports/admission/mutation/chainsaw-step-02-apply-1-1.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Pod +metadata: + name: good-pod + namespace: default +spec: + hostNetwork: false + containers: + - name: nginx1 + image: nginx + args: + - sleep + - 1d diff --git a/test/conformance/chainsaw/reports/admission/mutation/chainsaw-step-02-assert-1-1.yaml b/test/conformance/chainsaw/reports/admission/mutation/chainsaw-step-02-assert-1-1.yaml new file mode 100755 index 0000000000..1079c9cc90 --- /dev/null +++ b/test/conformance/chainsaw/reports/admission/mutation/chainsaw-step-02-assert-1-1.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Pod +metadata: + name: good-pod + namespace: default + labels: + foo: bar diff --git a/test/conformance/chainsaw/reports/admission/mutation/chainsaw-step-03-assert-1-1.yaml b/test/conformance/chainsaw/reports/admission/mutation/chainsaw-step-03-assert-1-1.yaml new file mode 100755 index 0000000000..9c41120da2 --- /dev/null +++ b/test/conformance/chainsaw/reports/admission/mutation/chainsaw-step-03-assert-1-1.yaml @@ -0,0 +1,24 @@ +apiVersion: wgpolicyk8s.io/v1alpha2 +kind: PolicyReport +metadata: + labels: + app.kubernetes.io/managed-by: kyverno + namespace: default +results: +- message: mutated Pod/good-pod in namespace default + policy: add-labels + result: pass + rule: add-labels + scored: true + source: kyverno +scope: + apiVersion: v1 + kind: Pod + name: good-pod + namespace: default +summary: + error: 0 + fail: 0 + pass: 1 + skip: 0 + warn: 0 \ No newline at end of file diff --git a/test/conformance/chainsaw/reports/admission/mutation/chainsaw-test.yaml b/test/conformance/chainsaw/reports/admission/mutation/chainsaw-test.yaml new file mode 100755 index 0000000000..66251ae75f --- /dev/null +++ b/test/conformance/chainsaw/reports/admission/mutation/chainsaw-test.yaml @@ -0,0 +1,22 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: basic-check-output +spec: + steps: + - name: step-01 + try: + - apply: + file: chainsaw-step-01-apply-1-1.yaml + - assert: + file: chainsaw-step-01-assert-1-1.yaml + - name: step-02 + try: + - apply: + file: chainsaw-step-02-apply-1-1.yaml + - assert: + file: chainsaw-step-02-assert-1-1.yaml + - name: step-03 + try: + - assert: + file: chainsaw-step-03-assert-1-1.yaml diff --git a/test/conformance/chainsaw/reports/background/generate/README.md b/test/conformance/chainsaw/reports/background/generate/README.md new file mode 100644 index 0000000000..329fde09c5 --- /dev/null +++ b/test/conformance/chainsaw/reports/background/generate/README.md @@ -0,0 +1,3 @@ +# Title + +This is a generate test to ensure cluster policy report is generated for generate rules. diff --git a/test/conformance/chainsaw/reports/background/generate/chainsaw-step-01-apply-1-1.yaml b/test/conformance/chainsaw/reports/background/generate/chainsaw-step-01-apply-1-1.yaml new file mode 100755 index 0000000000..a3832c8a5d --- /dev/null +++ b/test/conformance/chainsaw/reports/background/generate/chainsaw-step-01-apply-1-1.yaml @@ -0,0 +1,21 @@ +apiVersion: kyverno.io/v2beta1 +kind: ClusterPolicy +metadata: + name: cpol-nosync-clone +spec: + rules: + - generate: + apiVersion: v1 + clone: + name: regcred + namespace: default + kind: Secret + name: regcred + namespace: '{{request.object.metadata.name}}' + synchronize: false + match: + any: + - resources: + kinds: + - Namespace + name: clone-secret diff --git a/test/conformance/chainsaw/reports/background/generate/chainsaw-step-01-apply-1-2.yaml b/test/conformance/chainsaw/reports/background/generate/chainsaw-step-01-apply-1-2.yaml new file mode 100755 index 0000000000..8e534a8890 --- /dev/null +++ b/test/conformance/chainsaw/reports/background/generate/chainsaw-step-01-apply-1-2.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +data: + foo: YmFy +kind: Secret +metadata: + name: regcred + namespace: default +type: Opaque diff --git a/test/conformance/chainsaw/reports/background/generate/chainsaw-step-01-assert-1-1.yaml b/test/conformance/chainsaw/reports/background/generate/chainsaw-step-01-assert-1-1.yaml new file mode 100755 index 0000000000..bb748c5b1f --- /dev/null +++ b/test/conformance/chainsaw/reports/background/generate/chainsaw-step-01-assert-1-1.yaml @@ -0,0 +1,9 @@ +apiVersion: kyverno.io/v2beta1 +kind: ClusterPolicy +metadata: + name: cpol-nosync-clone +status: + conditions: + - reason: Succeeded + status: "True" + type: Ready diff --git a/test/conformance/chainsaw/reports/background/generate/chainsaw-step-02-apply-1-1.yaml b/test/conformance/chainsaw/reports/background/generate/chainsaw-step-02-apply-1-1.yaml new file mode 100755 index 0000000000..e0972e1cba --- /dev/null +++ b/test/conformance/chainsaw/reports/background/generate/chainsaw-step-02-apply-1-1.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: cpol-clone-nosync-create-ns diff --git a/test/conformance/chainsaw/reports/background/generate/chainsaw-step-03-assert-1-1.yaml b/test/conformance/chainsaw/reports/background/generate/chainsaw-step-03-assert-1-1.yaml new file mode 100755 index 0000000000..f6abe75a88 --- /dev/null +++ b/test/conformance/chainsaw/reports/background/generate/chainsaw-step-03-assert-1-1.yaml @@ -0,0 +1,25 @@ +apiVersion: wgpolicyk8s.io/v1alpha2 +kind: ClusterPolicyReport +metadata: + labels: + app.kubernetes.io/managed-by: kyverno + ownerReferences: + - apiVersion: v1 + kind: Namespace + name: cpol-clone-nosync-create-ns +results: +- policy: cpol-nosync-clone + result: pass + rule: clone-secret + scored: true + source: kyverno +scope: + apiVersion: v1 + kind: Namespace + name: cpol-clone-nosync-create-ns +summary: + error: 0 + fail: 0 + pass: 1 + skip: 0 + warn: 0 \ No newline at end of file diff --git a/test/conformance/chainsaw/reports/background/generate/chainsaw-test.yaml b/test/conformance/chainsaw/reports/background/generate/chainsaw-test.yaml new file mode 100755 index 0000000000..d540ed0b25 --- /dev/null +++ b/test/conformance/chainsaw/reports/background/generate/chainsaw-test.yaml @@ -0,0 +1,26 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: cpol-clone-nosync-create +spec: + steps: + - name: step-01 + try: + - apply: + file: permissions.yaml + - apply: + file: chainsaw-step-01-apply-1-1.yaml + - apply: + file: chainsaw-step-01-apply-1-2.yaml + - assert: + file: chainsaw-step-01-assert-1-1.yaml + - name: step-02 + try: + - apply: + file: chainsaw-step-02-apply-1-1.yaml + - sleep: + duration: 3s + - name: step-03 + try: + - assert: + file: chainsaw-step-03-assert-1-1.yaml diff --git a/test/conformance/chainsaw/reports/background/generate/permissions.yaml b/test/conformance/chainsaw/reports/background/generate/permissions.yaml new file mode 100644 index 0000000000..18e48f48c9 --- /dev/null +++ b/test/conformance/chainsaw/reports/background/generate/permissions.yaml @@ -0,0 +1,20 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kyverno:secrets-kqo1 + labels: + rbac.kyverno.io/aggregate-to-background-controller: "true" + rbac.kyverno.io/aggregate-to-reports-controller: "true" + rbac.kyverno.io/aggregate-to-admission-controller: "true" +rules: +- apiGroups: + - '' + resources: + - secrets + verbs: + - get + - list + - watch + - create + - update + - delete \ No newline at end of file diff --git a/test/conformance/chainsaw/reports/background/mutate-existing/README.md b/test/conformance/chainsaw/reports/background/mutate-existing/README.md new file mode 100644 index 0000000000..80b273231b --- /dev/null +++ b/test/conformance/chainsaw/reports/background/mutate-existing/README.md @@ -0,0 +1,7 @@ +## Description + +This is a basic test for the mutate existing capability which ensures that creating a triggering resource results in the correct mutation of a different resource and correct report being generated + +## Reference Issue(s) + +N/A \ No newline at end of file diff --git a/test/conformance/chainsaw/reports/background/mutate-existing/chainsaw-step-01-apply-1-1.yaml b/test/conformance/chainsaw/reports/background/mutate-existing/chainsaw-step-01-apply-1-1.yaml new file mode 100755 index 0000000000..446e1ac0e2 --- /dev/null +++ b/test/conformance/chainsaw/reports/background/mutate-existing/chainsaw-step-01-apply-1-1.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Namespace +metadata: + annotations: + cloud.platformzero.com/serviceClass: xl2 + labels: + app-type: corp + name: staging diff --git a/test/conformance/chainsaw/reports/background/mutate-existing/chainsaw-step-01-apply-1-2.yaml b/test/conformance/chainsaw/reports/background/mutate-existing/chainsaw-step-01-apply-1-2.yaml new file mode 100755 index 0000000000..a7d73395d3 --- /dev/null +++ b/test/conformance/chainsaw/reports/background/mutate-existing/chainsaw-step-01-apply-1-2.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +data: + foo: YmFy +kind: Secret +metadata: + name: secret-1 + namespace: staging +type: Opaque diff --git a/test/conformance/chainsaw/reports/background/mutate-existing/chainsaw-step-01-apply-1-3.yaml b/test/conformance/chainsaw/reports/background/mutate-existing/chainsaw-step-01-apply-1-3.yaml new file mode 100755 index 0000000000..468bbf3ee1 --- /dev/null +++ b/test/conformance/chainsaw/reports/background/mutate-existing/chainsaw-step-01-apply-1-3.yaml @@ -0,0 +1,25 @@ +apiVersion: kyverno.io/v1 +kind: ClusterPolicy +metadata: + name: mutate-existing-secret +spec: + rules: + - match: + any: + - resources: + kinds: + - Pod + namespaces: + - staging + mutate: + mutateExistingOnPolicyUpdate: false + patchStrategicMerge: + metadata: + labels: + foo: bar + targets: + - apiVersion: v1 + kind: Secret + name: secret-1 + namespace: '{{ request.object.metadata.namespace }}' + name: mutate-secret-on-configmap-create diff --git a/test/conformance/chainsaw/reports/background/mutate-existing/chainsaw-step-01-assert-1-1.yaml b/test/conformance/chainsaw/reports/background/mutate-existing/chainsaw-step-01-assert-1-1.yaml new file mode 100755 index 0000000000..450edc769b --- /dev/null +++ b/test/conformance/chainsaw/reports/background/mutate-existing/chainsaw-step-01-assert-1-1.yaml @@ -0,0 +1,9 @@ +apiVersion: kyverno.io/v1 +kind: ClusterPolicy +metadata: + name: mutate-existing-secret +status: + conditions: + - reason: Succeeded + status: "True" + type: Ready diff --git a/test/conformance/chainsaw/reports/background/mutate-existing/chainsaw-step-02-apply-1-1.yaml b/test/conformance/chainsaw/reports/background/mutate-existing/chainsaw-step-02-apply-1-1.yaml new file mode 100755 index 0000000000..7035d4c71b --- /dev/null +++ b/test/conformance/chainsaw/reports/background/mutate-existing/chainsaw-step-02-apply-1-1.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Pod +metadata: + name: myapp-pod + namespace: staging + labels: + app: myapp +spec: + containers: + - name: nginx + image: nginx:latest \ No newline at end of file diff --git a/test/conformance/chainsaw/reports/background/mutate-existing/chainsaw-step-03-assert-1-1.yaml b/test/conformance/chainsaw/reports/background/mutate-existing/chainsaw-step-03-assert-1-1.yaml new file mode 100755 index 0000000000..e9e2cba645 --- /dev/null +++ b/test/conformance/chainsaw/reports/background/mutate-existing/chainsaw-step-03-assert-1-1.yaml @@ -0,0 +1,22 @@ +apiVersion: wgpolicyk8s.io/v1alpha2 +kind: PolicyReport +metadata: + generation: 1 + labels: + app.kubernetes.io/managed-by: kyverno + namespace: staging +results: +- message: mutated Secret/secret-1 in namespace staging + policy: mutate-existing-secret + properties: + patched-target: /v1, Kind=Secret Name=secret-1 Namespace=staging + result: pass + rule: mutate-secret-on-configmap-create + scored: true + source: kyverno +summary: + error: 0 + fail: 0 + pass: 1 + skip: 0 + warn: 0 \ No newline at end of file diff --git a/test/conformance/chainsaw/reports/background/mutate-existing/chainsaw-test.yaml b/test/conformance/chainsaw/reports/background/mutate-existing/chainsaw-test.yaml new file mode 100755 index 0000000000..80b571cf07 --- /dev/null +++ b/test/conformance/chainsaw/reports/background/mutate-existing/chainsaw-test.yaml @@ -0,0 +1,24 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: basic-create +spec: + steps: + - name: step-01 + try: + - apply: + file: chainsaw-step-01-apply-1-1.yaml + - apply: + file: chainsaw-step-01-apply-1-2.yaml + - apply: + file: chainsaw-step-01-apply-1-3.yaml + - assert: + file: chainsaw-step-01-assert-1-1.yaml + - name: step-02 + try: + - apply: + file: chainsaw-step-02-apply-1-1.yaml + - name: step-03 + try: + - assert: + file: chainsaw-step-03-assert-1-1.yaml