diff --git a/charts/kyverno/README.md b/charts/kyverno/README.md index bef798d782..53dd2222d5 100644 --- a/charts/kyverno/README.md +++ b/charts/kyverno/README.md @@ -294,6 +294,7 @@ The chart values are organised per component. | features.admissionReports.enabled | bool | `true` | Enables the feature | | features.aggregateReports.enabled | bool | `true` | Enables the feature | | features.policyReports.enabled | bool | `true` | Enables the feature | +| features.validatingAdmissionPolicyReports.enabled | bool | `false` | Enables the feature | | features.autoUpdateWebhooks.enabled | bool | `true` | Enables the feature | | features.backgroundScan.enabled | bool | `true` | Enables the feature | | features.backgroundScan.backgroundScanWorkers | int | `2` | Number of background scan workers | diff --git a/charts/kyverno/templates/NOTES.txt b/charts/kyverno/templates/NOTES.txt index f45e54d83d..e4ead22711 100644 --- a/charts/kyverno/templates/NOTES.txt +++ b/charts/kyverno/templates/NOTES.txt @@ -39,4 +39,8 @@ The following components have been installed in your cluster: {{- with .Values.features.generateValidatingAdmissionPolicy.enabled }} ⚠️ WARNING: Generating validating admission policy requires a Kubernetes 1.26+ cluster with `ValidatingAdmissionPolicy` feature gate and `admissionregistration.k8s.io` API group enabled. -{{- end }} \ No newline at end of file +{{- end }} + +{{- with .Values.features.validatingAdmissionPolicyReports.enabled }} +⚠️ WARNING: Validating admission policy reports require a Kubernetes 1.26+ cluster with `ValidatingAdmissionPolicy` feature gate and `admissionregistration.k8s.io` API group enabled. +{{- end }} diff --git a/charts/kyverno/templates/_helpers.tpl b/charts/kyverno/templates/_helpers.tpl index 7d154efa17..72dc424bda 100644 --- a/charts/kyverno/templates/_helpers.tpl +++ b/charts/kyverno/templates/_helpers.tpl @@ -19,6 +19,9 @@ {{- with .policyReports -}} {{- $flags = append $flags (print "--policyReports=" .enabled) -}} {{- end -}} +{{- with .validatingAdmissionPolicyReports -}} + {{- $flags = append $flags (print "--validatingAdmissionPolicyReports=" .enabled) -}} +{{- end -}} {{- with .autoUpdateWebhooks -}} {{- $flags = append $flags (print "--autoUpdateWebhooks=" .enabled) -}} {{- end -}} diff --git a/charts/kyverno/templates/reports-controller/deployment.yaml b/charts/kyverno/templates/reports-controller/deployment.yaml index ca202410cc..ae71c0d602 100644 --- a/charts/kyverno/templates/reports-controller/deployment.yaml +++ b/charts/kyverno/templates/reports-controller/deployment.yaml @@ -112,6 +112,7 @@ spec: "admissionReports" "aggregateReports" "policyReports" + "validatingAdmissionPolicyReports" "backgroundScan" "configMapCaching" "deferredLoading" diff --git a/charts/kyverno/values.yaml b/charts/kyverno/values.yaml index 55aa8c00e7..f8ce06e127 100644 --- a/charts/kyverno/values.yaml +++ b/charts/kyverno/values.yaml @@ -381,6 +381,9 @@ features: policyReports: # -- Enables the feature enabled: true + validatingAdmissionPolicyReports: + # -- Enables the feature + enabled: false autoUpdateWebhooks: # -- Enables the feature enabled: true diff --git a/cmd/reports-controller/main.go b/cmd/reports-controller/main.go index 30ebfc6952..35e388e3f7 100644 --- a/cmd/reports-controller/main.go +++ b/cmd/reports-controller/main.go @@ -23,7 +23,9 @@ import ( "github.com/kyverno/kyverno/pkg/event" "github.com/kyverno/kyverno/pkg/leaderelection" "github.com/kyverno/kyverno/pkg/logging" + "k8s.io/apimachinery/pkg/runtime/schema" kubeinformers "k8s.io/client-go/informers" + admissionregistrationv1alpha1informers "k8s.io/client-go/informers/admissionregistration/v1alpha1" metadatainformers "k8s.io/client-go/metadata/metadatainformer" kyamlopenapi "sigs.k8s.io/kustomize/kyaml/openapi" ) @@ -38,6 +40,7 @@ func createReportControllers( admissionReports bool, aggregateReports bool, policyReports bool, + validatingAdmissionPolicyReports bool, reportsChunkSize int, backgroundScanWorkers int, client dclient.Interface, @@ -52,12 +55,19 @@ func createReportControllers( ) ([]internal.Controller, func(context.Context) error) { var ctrls []internal.Controller var warmups []func(context.Context) error + var vapInformer admissionregistrationv1alpha1informers.ValidatingAdmissionPolicyInformer + // check if validating admission policies are registered in the API server + if validatingAdmissionPolicyReports { + vapInformer = kubeInformer.Admissionregistration().V1alpha1().ValidatingAdmissionPolicies() + } + kyvernoV1 := kyvernoInformer.Kyverno().V1() if backgroundScan || admissionReports { resourceReportController := resourcereportcontroller.NewController( client, kyvernoV1.Policies(), kyvernoV1.ClusterPolicies(), + vapInformer, ) warmups = append(warmups, func(ctx context.Context) error { return resourceReportController.Warmup(ctx) @@ -93,25 +103,27 @@ func createReportControllers( )) } if backgroundScan { + backgroundScanController := backgroundscancontroller.NewController( + client, + kyvernoClient, + eng, + metadataFactory, + kyvernoV1.Policies(), + kyvernoV1.ClusterPolicies(), + vapInformer, + kubeInformer.Core().V1().Namespaces(), + resourceReportController, + backgroundScanInterval, + configuration, + jp, + eventGenerator, + policyReports, + ) ctrls = append(ctrls, internal.NewController( backgroundscancontroller.ControllerName, - backgroundscancontroller.NewController( - client, - kyvernoClient, - eng, - metadataFactory, - kyvernoV1.Policies(), - kyvernoV1.ClusterPolicies(), - kubeInformer.Core().V1().Namespaces(), - resourceReportController, - backgroundScanInterval, - configuration, - jp, - eventGenerator, - policyReports, - ), - backgroundScanWorkers, - )) + backgroundScanController, + backgroundScanWorkers), + ) } } return ctrls, func(ctx context.Context) error { @@ -130,6 +142,7 @@ func createrLeaderControllers( admissionReports bool, aggregateReports bool, policyReports bool, + validatingAdmissionPolicyReports bool, reportsChunkSize int, backgroundScanWorkers int, kubeInformer kubeinformers.SharedInformerFactory, @@ -148,6 +161,7 @@ func createrLeaderControllers( admissionReports, aggregateReports, policyReports, + validatingAdmissionPolicyReports, reportsChunkSize, backgroundScanWorkers, dynamicClient, @@ -165,22 +179,24 @@ func createrLeaderControllers( func main() { var ( - backgroundScan bool - admissionReports bool - aggregateReports bool - policyReports bool - reportsChunkSize int - backgroundScanWorkers int - backgroundScanInterval time.Duration - maxQueuedEvents int - omitEvents string - skipResourceFilters bool + backgroundScan bool + admissionReports bool + aggregateReports bool + policyReports bool + validatingAdmissionPolicyReports bool + reportsChunkSize int + backgroundScanWorkers int + backgroundScanInterval time.Duration + maxQueuedEvents int + omitEvents string + skipResourceFilters bool ) flagset := flag.NewFlagSet("reports-controller", flag.ExitOnError) flagset.BoolVar(&backgroundScan, "backgroundScan", true, "Enable or disable background scan.") flagset.BoolVar(&admissionReports, "admissionReports", true, "Enable or disable admission reports.") flagset.BoolVar(&aggregateReports, "aggregateReports", true, "Enable or disable aggregated policy reports.") flagset.BoolVar(&policyReports, "policyReports", true, "Enable or disable policy reports.") + flagset.BoolVar(&validatingAdmissionPolicyReports, "validatingAdmissionPolicyReports", false, "Enable or disable validating admission policy reports.") flagset.IntVar(&reportsChunkSize, "reportsChunkSize", 1000, "Max number of results in generated reports, reports will be split accordingly if there are more results to be stored.") flagset.IntVar(&backgroundScanWorkers, "backgroundScanWorkers", backgroundscancontroller.Workers, "Configure the number of background scan workers.") flagset.DurationVar(&backgroundScanInterval, "backgroundScanInterval", time.Hour, "Configure background scan interval.") @@ -219,6 +235,14 @@ func main() { // ELSE KYAML IS NOT THREAD SAFE kyamlopenapi.Schema() setup.Logger.Info("background scan interval", "duration", backgroundScanInterval.String()) + // check if validating admission policies are registered in the API server + if validatingAdmissionPolicyReports { + groupVersion := schema.GroupVersion{Group: "admissionregistration.k8s.io", Version: "v1alpha1"} + if _, err := setup.KyvernoDynamicClient.GetKubeClient().Discovery().ServerResourcesForGroupVersion(groupVersion.String()); err != nil { + setup.Logger.Error(err, "validating admission policies aren't supported.") + os.Exit(1) + } + } // informer factories kyvernoInformer := kyvernoinformer.NewSharedInformerFactory(setup.KyvernoClient, resyncPeriod) omitEventsValues := strings.Split(omitEvents, ",") @@ -277,6 +301,7 @@ func main() { admissionReports, aggregateReports, policyReports, + validatingAdmissionPolicyReports, reportsChunkSize, backgroundScanWorkers, kubeInformer, diff --git a/config/install-latest-testing.yaml b/config/install-latest-testing.yaml index b6816038c5..8b49c407ab 100644 --- a/config/install-latest-testing.yaml +++ b/config/install-latest-testing.yaml @@ -42208,6 +42208,7 @@ spec: - --admissionReports=true - --aggregateReports=true - --policyReports=true + - --validatingAdmissionPolicyReports=false - --backgroundScan=true - --backgroundScanWorkers=2 - --backgroundScanInterval=1h diff --git a/pkg/controllers/report/aggregate/controller.go b/pkg/controllers/report/aggregate/controller.go index 408ac2d71a..19f92e4a32 100644 --- a/pkg/controllers/report/aggregate/controller.go +++ b/pkg/controllers/report/aggregate/controller.go @@ -15,6 +15,7 @@ import ( kyvernov1listers "github.com/kyverno/kyverno/pkg/client/listers/kyverno/v1" "github.com/kyverno/kyverno/pkg/controllers" "github.com/kyverno/kyverno/pkg/controllers/report/resource" + engineapi "github.com/kyverno/kyverno/pkg/engine/api" controllerutils "github.com/kyverno/kyverno/pkg/utils/controller" datautils "github.com/kyverno/kyverno/pkg/utils/data" reportutils "github.com/kyverno/kyverno/pkg/utils/report" @@ -226,7 +227,7 @@ func (c *controller) reconcileReport(ctx context.Context, policyMap map[string]p for _, result := range results { policy := policyMap[result.Policy] if policy.policy != nil { - reportutils.SetPolicyLabel(report, policy.policy) + reportutils.SetPolicyLabel(report, engineapi.NewKyvernoPolicy(policy.policy)) } } return reportutils.CreateReport(ctx, report, c.client) @@ -238,7 +239,7 @@ func (c *controller) reconcileReport(ctx context.Context, policyMap map[string]p for _, result := range results { policy := policyMap[result.Policy] if policy.policy != nil { - reportutils.SetPolicyLabel(after, policy.policy) + reportutils.SetPolicyLabel(after, engineapi.NewKyvernoPolicy(policy.policy)) } } reportutils.SetResults(after, results...) diff --git a/pkg/controllers/report/background/controller.go b/pkg/controllers/report/background/controller.go index 5c35466a97..a829186778 100644 --- a/pkg/controllers/report/background/controller.go +++ b/pkg/controllers/report/background/controller.go @@ -22,11 +22,14 @@ import ( controllerutils "github.com/kyverno/kyverno/pkg/utils/controller" datautils "github.com/kyverno/kyverno/pkg/utils/data" reportutils "github.com/kyverno/kyverno/pkg/utils/report" + admissionregistrationv1alpha1 "k8s.io/api/admissionregistration/v1alpha1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" + admissionregistrationv1alpha1informers "k8s.io/client-go/informers/admissionregistration/v1alpha1" corev1informers "k8s.io/client-go/informers/core/v1" + admissionregistrationv1alpha1listers "k8s.io/client-go/listers/admissionregistration/v1alpha1" corev1listers "k8s.io/client-go/listers/core/v1" metadatainformers "k8s.io/client-go/metadata/metadatainformer" "k8s.io/client-go/tools/cache" @@ -51,6 +54,7 @@ type controller struct { // listers polLister kyvernov1listers.PolicyLister cpolLister kyvernov1listers.ClusterPolicyLister + vapLister admissionregistrationv1alpha1listers.ValidatingAdmissionPolicyLister bgscanrLister cache.GenericLister cbgscanrLister cache.GenericLister nsLister corev1listers.NamespaceLister @@ -76,6 +80,7 @@ func NewController( metadataFactory metadatainformers.SharedInformerFactory, polInformer kyvernov1informers.PolicyInformer, cpolInformer kyvernov1informers.ClusterPolicyInformer, + vapInformer admissionregistrationv1alpha1informers.ValidatingAdmissionPolicyInformer, nsInformer corev1informers.NamespaceInformer, metadataCache resource.MetadataCache, forceDelay time.Duration, @@ -104,6 +109,14 @@ func NewController( eventGen: eventGen, policyReports: policyReports, } + + if vapInformer != nil { + c.vapLister = vapInformer.Lister() + if _, err := controllerutils.AddEventHandlersT(vapInformer.Informer(), c.addVAP, c.updateVAP, c.deleteVAP); err != nil { + logger.Error(err, "failed to register even handlers") + } + } + if _, _, err := controllerutils.AddDefaultEventHandlers(logger, bgscanr.Informer(), queue); err != nil { logger.Error(err, "failed to register even handlers") } @@ -149,6 +162,20 @@ func (c *controller) deletePolicy(obj kyvernov1.PolicyInterface) { c.enqueueResources() } +func (c *controller) addVAP(obj *admissionregistrationv1alpha1.ValidatingAdmissionPolicy) { + c.enqueueResources() +} + +func (c *controller) updateVAP(old, obj *admissionregistrationv1alpha1.ValidatingAdmissionPolicy) { + if old.GetResourceVersion() != obj.GetResourceVersion() { + c.enqueueResources() + } +} + +func (c *controller) deleteVAP(obj *admissionregistrationv1alpha1.ValidatingAdmissionPolicy) { + c.enqueueResources() +} + func (c *controller) enqueueResources() { for _, key := range c.metadataCache.GetAllResourceKeys() { c.queue.Add(key) @@ -179,7 +206,7 @@ func (c *controller) getMeta(namespace, name string) (metav1.Object, error) { } } -func (c *controller) needsReconcile(namespace, name, hash string, backgroundPolicies ...kyvernov1.PolicyInterface) (bool, bool, error) { +func (c *controller) needsReconcile(namespace, name, hash string, policies ...engineapi.GenericPolicy) (bool, bool, error) { // if the reportMetadata does not exist, we need a full reconcile reportMetadata, err := c.getMeta(namespace, name) if err != nil { @@ -208,7 +235,7 @@ func (c *controller) needsReconcile(namespace, name, hash string, backgroundPoli } // if a policy changed, we need a partial reconcile expected := map[string]string{} - for _, policy := range backgroundPolicies { + for _, policy := range policies { expected[reportutils.PolicyLabel(policy)] = policy.GetResourceVersion() } actual := map[string]string{} @@ -232,7 +259,7 @@ func (c *controller) reconcileReport( uid types.UID, gvk schema.GroupVersionKind, resource resource.Resource, - backgroundPolicies ...kyvernov1.PolicyInterface, + policies ...engineapi.GenericPolicy, ) error { // namespace labels to be used by the scanner var nsLabels map[string]string @@ -258,7 +285,7 @@ func (c *controller) reconcileReport( } // build desired report expected := map[string]string{} - for _, policy := range backgroundPolicies { + for _, policy := range policies { expected[reportutils.PolicyLabel(policy)] = policy.GetResourceVersion() } actual := map[string]string{} @@ -270,8 +297,14 @@ func (c *controller) reconcileReport( var ruleResults []policyreportv1alpha2.PolicyReportResult if !full { policyNameToLabel := map[string]string{} - for _, policy := range backgroundPolicies { - key, err := cache.MetaNamespaceKeyFunc(policy) + for _, policy := range policies { + var key string + var err error + if policy.GetType() == engineapi.KyvernoPolicyType { + key, err = cache.MetaNamespaceKeyFunc(policy.GetPolicy().(kyvernov1.PolicyInterface)) + } else { + key, err = cache.MetaNamespaceKeyFunc(policy.GetPolicy().(admissionregistrationv1alpha1.ValidatingAdmissionPolicy)) + } if err != nil { return err } @@ -287,7 +320,7 @@ func (c *controller) reconcileReport( } } // calculate necessary results - for _, policy := range backgroundPolicies { + for _, policy := range policies { if full || actual[reportutils.PolicyLabel(policy)] != policy.GetResourceVersion() { scanner := utils.NewScanner(logger, c.engine, c.config, c.jp) for _, result := range scanner.ScanResource(ctx, *target, nsLabels, policy) { @@ -306,7 +339,7 @@ func (c *controller) reconcileReport( delete(desired.GetLabels(), key) } } - for _, policy := range backgroundPolicies { + for _, policy := range policies { reportutils.SetPolicyLabel(desired, policy) } reportutils.SetResourceVersionLabels(desired, target) @@ -365,8 +398,8 @@ func (c *controller) reconcile(ctx context.Context, log logr.Logger, key, namesp } } } - // load all policies - policies, err := utils.FetchClusterPolicies(c.cpolLister) + // load all kyverno policies + kyvernoPolicies, err := utils.FetchClusterPolicies(c.cpolLister) if err != nil { return err } @@ -375,22 +408,33 @@ func (c *controller) reconcile(ctx context.Context, log logr.Logger, key, namesp if err != nil { return err } - policies = append(policies, pols...) + kyvernoPolicies = append(kyvernoPolicies, pols...) } // load background policies - backgroundPolicies := utils.RemoveNonBackgroundPolicies(policies...) - if err != nil { - return err + kyvernoPolicies = utils.RemoveNonBackgroundPolicies(kyvernoPolicies...) + var policies []engineapi.GenericPolicy + for _, pol := range kyvernoPolicies { + policies = append(policies, engineapi.NewKyvernoPolicy(pol)) + } + if c.vapLister != nil { + // load validating admission policies + vapPolicies, err := utils.FetchValidatingAdmissionPolicies(c.vapLister) + if err != nil { + return err + } + for _, pol := range vapPolicies { + policies = append(policies, engineapi.NewValidatingAdmissionPolicy(pol)) + } } // we have the resource, check if we need to reconcile - if needsReconcile, full, err := c.needsReconcile(namespace, name, resource.Hash, backgroundPolicies...); err != nil { + if needsReconcile, full, err := c.needsReconcile(namespace, name, resource.Hash, policies...); err != nil { return err } else { defer func() { c.queue.AddAfter(key, c.forceDelay) }() if needsReconcile { - return c.reconcileReport(ctx, namespace, name, full, uid, gvk, resource, backgroundPolicies...) + return c.reconcileReport(ctx, namespace, name, full, uid, gvk, resource, policies...) } } return nil diff --git a/pkg/controllers/report/resource/controller.go b/pkg/controllers/report/resource/controller.go index 0d0fde1b78..1db7bb72b7 100644 --- a/pkg/controllers/report/resource/controller.go +++ b/pkg/controllers/report/resource/controller.go @@ -15,6 +15,7 @@ import ( controllerutils "github.com/kyverno/kyverno/pkg/utils/controller" kubeutils "github.com/kyverno/kyverno/pkg/utils/kube" reportutils "github.com/kyverno/kyverno/pkg/utils/report" + "github.com/kyverno/kyverno/pkg/validatingadmissionpolicy" "golang.org/x/exp/slices" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -22,6 +23,8 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/watch" + admissionregistrationv1alpha1informers "k8s.io/client-go/informers/admissionregistration/v1alpha1" + admissionregistrationv1alpha1listers "k8s.io/client-go/listers/admissionregistration/v1alpha1" "k8s.io/client-go/tools/cache" watchTools "k8s.io/client-go/tools/watch" "k8s.io/client-go/util/workqueue" @@ -76,6 +79,7 @@ type controller struct { // listers polLister kyvernov1listers.PolicyLister cpolLister kyvernov1listers.ClusterPolicyLister + vapLister admissionregistrationv1alpha1listers.ValidatingAdmissionPolicyLister // queue queue workqueue.RateLimitingInterface @@ -89,6 +93,7 @@ func NewController( client dclient.Interface, polInformer kyvernov1informers.PolicyInformer, cpolInformer kyvernov1informers.ClusterPolicyInformer, + vapInformer admissionregistrationv1alpha1informers.ValidatingAdmissionPolicyInformer, ) Controller { c := controller{ client: client, @@ -97,6 +102,14 @@ func NewController( queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), ControllerName), dynamicWatchers: map[schema.GroupVersionResource]*watcher{}, } + + if vapInformer != nil { + c.vapLister = vapInformer.Lister() + if _, _, err := controllerutils.AddDefaultEventHandlers(logger, vapInformer.Informer(), c.queue); err != nil { + logger.Error(err, "failed to register even handlers") + } + } + if _, _, err := controllerutils.AddDefaultEventHandlers(logger, polInformer.Informer(), c.queue); err != nil { logger.Error(err, "failed to register even handlers") } @@ -226,23 +239,19 @@ func (c *controller) updateDynamicWatchers(ctx context.Context) error { gvkToGvr := map[schema.GroupVersionKind]schema.GroupVersionResource{} for _, policyKind := range sets.List(kinds) { group, version, kind, subresource := kubeutils.ParseKindSelector(policyKind) - gvrss, err := c.client.Discovery().FindResources(group, version, kind, subresource) + c.addGVKToGVRMapping(group, version, kind, subresource, gvkToGvr) + } + if c.vapLister != nil { + vapPolicies, err := utils.FetchValidatingAdmissionPolicies(c.vapLister) if err != nil { - logger.Error(err, "failed to get gvr from kind", "kind", kind) - } else { - for gvrs, api := range gvrss { - if gvrs.SubResource == "" { - gvk := schema.GroupVersionKind{Group: gvrs.Group, Version: gvrs.Version, Kind: policyKind} - if !reportutils.IsGvkSupported(gvk) { - logger.Info("kind is not supported", "gvk", gvk) - } else { - if slices.Contains(api.Verbs, "list") && slices.Contains(api.Verbs, "watch") { - gvkToGvr[gvk] = gvrs.GroupVersionResource() - } else { - logger.Info("list/watch not supported for kind", "kind", kind) - } - } - } + return err + } + // fetch kinds from validating admission policies + for _, policy := range vapPolicies { + kinds := validatingadmissionpolicy.GetKinds(policy) + for _, kind := range kinds { + group, version, kind, subresource := kubeutils.ParseKindSelector(kind) + c.addGVKToGVRMapping(group, version, kind, subresource, gvkToGvr) } } } @@ -274,6 +283,28 @@ func (c *controller) updateDynamicWatchers(ctx context.Context) error { return nil } +func (c *controller) addGVKToGVRMapping(group, version, kind, subresource string, gvrMap map[schema.GroupVersionKind]schema.GroupVersionResource) { + gvrss, err := c.client.Discovery().FindResources(group, version, kind, subresource) + if err != nil { + logger.Error(err, "failed to get gvr from kind", "kind", kind) + } else { + for gvrs, api := range gvrss { + if gvrs.SubResource == "" { + gvk := schema.GroupVersionKind{Group: gvrs.Group, Version: gvrs.Version, Kind: kind} + if !reportutils.IsGvkSupported(gvk) { + logger.Info("kind is not supported", "gvk", gvk) + } else { + if slices.Contains(api.Verbs, "list") && slices.Contains(api.Verbs, "watch") { + gvrMap[gvk] = gvrs.GroupVersionResource() + } else { + logger.Info("list/watch not supported for kind", "kind", kind) + } + } + } + } + } +} + func (c *controller) stopDynamicWatchers() { c.lock.Lock() defer c.lock.Unlock() diff --git a/pkg/controllers/report/utils/scanner.go b/pkg/controllers/report/utils/scanner.go index 9c2c8cacb0..b2cf1ac862 100644 --- a/pkg/controllers/report/utils/scanner.go +++ b/pkg/controllers/report/utils/scanner.go @@ -10,7 +10,9 @@ import ( "github.com/kyverno/kyverno/pkg/engine" engineapi "github.com/kyverno/kyverno/pkg/engine/api" "github.com/kyverno/kyverno/pkg/engine/jmespath" + "github.com/kyverno/kyverno/pkg/validatingadmissionpolicy" "go.uber.org/multierr" + admissionregistrationv1alpha1 "k8s.io/api/admissionregistration/v1alpha1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" ) @@ -27,7 +29,7 @@ type ScanResult struct { } type Scanner interface { - ScanResource(context.Context, unstructured.Unstructured, map[string]string, ...kyvernov1.PolicyInterface) map[kyvernov1.PolicyInterface]ScanResult + ScanResource(context.Context, unstructured.Unstructured, map[string]string, ...engineapi.GenericPolicy) map[*engineapi.GenericPolicy]ScanResult } func NewScanner( @@ -44,30 +46,39 @@ func NewScanner( } } -func (s *scanner) ScanResource(ctx context.Context, resource unstructured.Unstructured, nsLabels map[string]string, policies ...kyvernov1.PolicyInterface) map[kyvernov1.PolicyInterface]ScanResult { - results := map[kyvernov1.PolicyInterface]ScanResult{} - for _, policy := range policies { +func (s *scanner) ScanResource(ctx context.Context, resource unstructured.Unstructured, nsLabels map[string]string, policies ...engineapi.GenericPolicy) map[*engineapi.GenericPolicy]ScanResult { + results := map[*engineapi.GenericPolicy]ScanResult{} + for i, policy := range policies { var errors []error logger := s.logger.WithValues("kind", resource.GetKind(), "namespace", resource.GetNamespace(), "name", resource.GetName()) - response, err := s.validateResource(ctx, resource, nsLabels, policy) - if err != nil { - logger.Error(err, "failed to scan resource") - errors = append(errors, err) - } - spec := policy.GetSpec() - if spec.HasVerifyImages() { - ivResponse, err := s.validateImages(ctx, resource, nsLabels, policy) + var response *engineapi.EngineResponse + if policy.GetType() == engineapi.KyvernoPolicyType { + var err error + pol := policy.GetPolicy().(kyvernov1.PolicyInterface) + response, err = s.validateResource(ctx, resource, nsLabels, pol) if err != nil { - logger.Error(err, "failed to scan images") + logger.Error(err, "failed to scan resource") errors = append(errors, err) } - if response == nil { - response = ivResponse - } else if ivResponse != nil { - response.PolicyResponse.Rules = append(response.PolicyResponse.Rules, ivResponse.PolicyResponse.Rules...) + spec := pol.GetSpec() + if spec.HasVerifyImages() && len(errors) == 0 { + ivResponse, err := s.validateImages(ctx, resource, nsLabels, pol) + if err != nil { + logger.Error(err, "failed to scan images") + errors = append(errors, err) + } + if response == nil { + response = ivResponse + } else if ivResponse != nil { + response.PolicyResponse.Rules = append(response.PolicyResponse.Rules, ivResponse.PolicyResponse.Rules...) + } } + } else { + pol := policy.GetPolicy().(admissionregistrationv1alpha1.ValidatingAdmissionPolicy) + res := validatingadmissionpolicy.Validate(pol, resource) + response = &res } - results[policy] = ScanResult{response, multierr.Combine(errors...)} + results[&policies[i]] = ScanResult{response, multierr.Combine(errors...)} } return results } diff --git a/pkg/controllers/report/utils/utils.go b/pkg/controllers/report/utils/utils.go index 1034029f8a..5db2e02cdc 100644 --- a/pkg/controllers/report/utils/utils.go +++ b/pkg/controllers/report/utils/utils.go @@ -8,9 +8,11 @@ import ( kyvernov1listers "github.com/kyverno/kyverno/pkg/client/listers/kyverno/v1" datautils "github.com/kyverno/kyverno/pkg/utils/data" policyvalidation "github.com/kyverno/kyverno/pkg/validation/policy" + admissionregistrationv1alpha1 "k8s.io/api/admissionregistration/v1alpha1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/sets" + admissionregistrationv1alpha1listers "k8s.io/client-go/listers/admissionregistration/v1alpha1" ) func CanBackgroundProcess(p kyvernov1.PolicyInterface) bool { @@ -103,3 +105,15 @@ func FetchPolicies(polLister kyvernov1listers.PolicyLister, namespace string) ([ } return policies, nil } + +func FetchValidatingAdmissionPolicies(vapLister admissionregistrationv1alpha1listers.ValidatingAdmissionPolicyLister) ([]admissionregistrationv1alpha1.ValidatingAdmissionPolicy, error) { + var policies []admissionregistrationv1alpha1.ValidatingAdmissionPolicy + if pols, err := vapLister.List(labels.Everything()); err != nil { + return nil, err + } else { + for _, pol := range pols { + policies = append(policies, *pol) + } + } + return policies, nil +} diff --git a/pkg/engine/api/policy.go b/pkg/engine/api/policy.go index d435624b64..b283333798 100644 --- a/pkg/engine/api/policy.go +++ b/pkg/engine/api/policy.go @@ -26,8 +26,14 @@ type GenericPolicy interface { GetName() string // GetNamespace returns policy namespace GetNamespace() string + // GetKind returns policy kind + GetKind() string + // GetResourceVersion returns policy resource version + GetResourceVersion() string // GetAnnotations returns policy annotations GetAnnotations() map[string]string + // IsNamespaced indicates if the policy is namespace scoped + IsNamespaced() bool } type KyvernoPolicy struct { @@ -50,10 +56,22 @@ func (p *KyvernoPolicy) GetNamespace() string { return p.policy.GetNamespace() } +func (p *KyvernoPolicy) GetKind() string { + return p.policy.GetKind() +} + +func (p *KyvernoPolicy) GetResourceVersion() string { + return p.policy.GetResourceVersion() +} + func (p *KyvernoPolicy) GetAnnotations() map[string]string { return p.policy.GetAnnotations() } +func (p *KyvernoPolicy) IsNamespaced() bool { + return p.policy.IsNamespaced() +} + func NewKyvernoPolicy(pol kyvernov1.PolicyInterface) GenericPolicy { return &KyvernoPolicy{ policy: pol, @@ -80,10 +98,22 @@ func (p *ValidatingAdmissionPolicy) GetNamespace() string { return p.policy.GetNamespace() } +func (p *ValidatingAdmissionPolicy) GetKind() string { + return p.policy.Kind +} + +func (p *ValidatingAdmissionPolicy) GetResourceVersion() string { + return p.policy.GetResourceVersion() +} + func (p *ValidatingAdmissionPolicy) GetAnnotations() map[string]string { return p.policy.GetAnnotations() } +func (p *ValidatingAdmissionPolicy) IsNamespaced() bool { + return false +} + func NewValidatingAdmissionPolicy(pol v1alpha1.ValidatingAdmissionPolicy) GenericPolicy { return &ValidatingAdmissionPolicy{ policy: pol, diff --git a/pkg/event/events.go b/pkg/event/events.go index 0ed484327c..3606eb46f3 100644 --- a/pkg/event/events.go +++ b/pkg/event/events.go @@ -16,10 +16,10 @@ func NewPolicyFailEvent(source Source, reason Reason, engineResponse engineapi.E action = ResourceBlocked } - pol := engineResponse.Policy().GetPolicy().(kyvernov1.PolicyInterface) + pol := engineResponse.Policy() return Info{ - Kind: getPolicyKind(pol), + Kind: pol.GetKind(), Name: pol.GetName(), Namespace: pol.GetNamespace(), RelatedAPIVersion: engineResponse.GetResourceSpec().APIVersion, @@ -53,13 +53,6 @@ func buildPolicyEventMessage(resp engineapi.RuleResponse, resource engineapi.Res return b.String() } -func getPolicyKind(policy kyvernov1.PolicyInterface) string { - if policy.IsNamespaced() { - return "Policy" - } - return "ClusterPolicy" -} - func getCleanupPolicyKind(policy kyvernov2alpha1.CleanupPolicyInterface) string { if policy.IsNamespaced() { return "CleanupPolicy" @@ -94,7 +87,7 @@ func NewPolicyAppliedEvent(source Source, engineResponse engineapi.EngineRespons } return Info{ - Kind: getPolicyKind(pol), + Kind: pol.GetKind(), Name: pol.GetName(), Namespace: pol.GetNamespace(), RelatedAPIVersion: resource.GetAPIVersion(), @@ -112,7 +105,7 @@ func NewResourceViolationEvent(source Source, reason Reason, engineResponse engi var bldr strings.Builder defer bldr.Reset() - pol := engineResponse.Policy().GetPolicy().(kyvernov1.PolicyInterface) + pol := engineResponse.Policy() fmt.Fprintf(&bldr, "policy %s/%s %s: %s", pol.GetName(), ruleResp.Name(), ruleResp.Status(), ruleResp.Message()) resource := engineResponse.GetResourceSpec() @@ -202,7 +195,7 @@ func NewPolicyExceptionEvents(engineResponse engineapi.EngineResponse, ruleResp exceptionMessage = fmt.Sprintf("resource %s was skipped from policy rule %s/%s/%s", resourceKey(engineResponse.PatchedResource), pol.GetNamespace(), pol.GetName(), ruleResp.Name()) } policyEvent := Info{ - Kind: getPolicyKind(pol), + Kind: pol.GetKind(), Name: pol.GetName(), Namespace: pol.GetNamespace(), RelatedAPIVersion: engineResponse.PatchedResource.GetAPIVersion(), diff --git a/pkg/utils/report/labels.go b/pkg/utils/report/labels.go index 0f64c59bae..72b9b672b0 100644 --- a/pkg/utils/report/labels.go +++ b/pkg/utils/report/labels.go @@ -10,6 +10,7 @@ import ( "github.com/kyverno/kyverno/api/kyverno" kyvernov1 "github.com/kyverno/kyverno/api/kyverno/v1" kyvernov1alpha2 "github.com/kyverno/kyverno/api/kyverno/v1alpha2" + engineapi "github.com/kyverno/kyverno/pkg/engine/api" controllerutils "github.com/kyverno/kyverno/pkg/utils/controller" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -26,10 +27,11 @@ const ( LabelResourceNamespace = "audit.kyverno.io/resource.namespace" LabelResourceName = "audit.kyverno.io/resource.name" // policy labels - LabelDomainClusterPolicy = "cpol.kyverno.io" - LabelDomainPolicy = "pol.kyverno.io" - LabelPrefixClusterPolicy = LabelDomainClusterPolicy + "/" - LabelPrefixPolicy = LabelDomainPolicy + "/" + LabelDomainClusterPolicy = "cpol.kyverno.io" + LabelDomainPolicy = "pol.kyverno.io" + LabelPrefixClusterPolicy = LabelDomainClusterPolicy + "/" + LabelPrefixPolicy = LabelDomainPolicy + "/" + LabelPrefixValidatingAdmissionPolicy = "validatingadmissionpolicy.apiserver.io/" // aggregated admission report label LabelAggregatedReport = "audit.kyverno.io/report.aggregate" ) @@ -50,11 +52,14 @@ func PolicyNameFromLabel(namespace, label string) (string, error) { return "", fmt.Errorf("cannot get policy name from label, incorrect format: %s", label) } -func PolicyLabelPrefix(policy kyvernov1.PolicyInterface) string { +func PolicyLabelPrefix(policy engineapi.GenericPolicy) string { if policy.IsNamespaced() { return LabelPrefixPolicy } - return LabelPrefixClusterPolicy + if policy.GetType() == engineapi.KyvernoPolicyType { + return LabelPrefixClusterPolicy + } + return LabelPrefixValidatingAdmissionPolicy } func PolicyLabelDomain(policy kyvernov1.PolicyInterface) string { @@ -64,7 +69,7 @@ func PolicyLabelDomain(policy kyvernov1.PolicyInterface) string { return LabelDomainClusterPolicy } -func PolicyLabel(policy kyvernov1.PolicyInterface) string { +func PolicyLabel(policy engineapi.GenericPolicy) string { return PolicyLabelPrefix(policy) + policy.GetName() } @@ -125,7 +130,7 @@ func SetResourceVersionLabels(report kyvernov1alpha2.ReportInterface, resource * } } -func SetPolicyLabel(report kyvernov1alpha2.ReportInterface, policy kyvernov1.PolicyInterface) { +func SetPolicyLabel(report kyvernov1alpha2.ReportInterface, policy engineapi.GenericPolicy) { controllerutils.SetLabel(report, PolicyLabel(policy), policy.GetResourceVersion()) } diff --git a/pkg/utils/report/results.go b/pkg/utils/report/results.go index fd6eeab7cf..f15c089643 100644 --- a/pkg/utils/report/results.go +++ b/pkg/utils/report/results.go @@ -87,45 +87,60 @@ func SeverityFromString(severity string) policyreportv1alpha2.PolicySeverity { } func EngineResponseToReportResults(response engineapi.EngineResponse) []policyreportv1alpha2.PolicyReportResult { - pol := response.Policy().GetPolicy().(kyvernov1.PolicyInterface) - key, _ := cache.MetaNamespaceKeyFunc(pol) + pol := response.Policy() var results []policyreportv1alpha2.PolicyReportResult - for _, ruleResult := range response.PolicyResponse.Rules { - annotations := pol.GetAnnotations() - result := policyreportv1alpha2.PolicyReportResult{ - Source: kyverno.ValueKyvernoApp, - Policy: key, - Rule: ruleResult.Name(), - Message: ruleResult.Message(), - Result: toPolicyResult(ruleResult.Status()), - Scored: annotations[kyverno.AnnotationPolicyScored] != "false", - Timestamp: metav1.Timestamp{ - Seconds: time.Now().Unix(), - }, - Category: annotations[kyverno.AnnotationPolicyCategory], - Severity: SeverityFromString(annotations[kyverno.AnnotationPolicySeverity]), - } - pss := ruleResult.PodSecurityChecks() - if pss != nil { - var controls []string - for _, check := range pss.Checks { - if !check.CheckResult.Allowed { - controls = append(controls, check.ID) + if pol.GetType() == engineapi.KyvernoPolicyType { + key, _ := cache.MetaNamespaceKeyFunc(pol.GetPolicy().(kyvernov1.PolicyInterface)) + for _, ruleResult := range response.PolicyResponse.Rules { + annotations := pol.GetAnnotations() + result := policyreportv1alpha2.PolicyReportResult{ + Source: kyverno.ValueKyvernoApp, + Policy: key, + Rule: ruleResult.Name(), + Message: ruleResult.Message(), + Result: toPolicyResult(ruleResult.Status()), + Scored: annotations[kyverno.AnnotationPolicyScored] != "false", + Timestamp: metav1.Timestamp{ + Seconds: time.Now().Unix(), + }, + Category: annotations[kyverno.AnnotationPolicyCategory], + Severity: SeverityFromString(annotations[kyverno.AnnotationPolicySeverity]), + } + pss := ruleResult.PodSecurityChecks() + if pss != nil { + var controls []string + for _, check := range pss.Checks { + if !check.CheckResult.Allowed { + controls = append(controls, check.ID) + } + } + if len(controls) > 0 { + sort.Strings(controls) + result.Properties = map[string]string{ + "standard": string(pss.Level), + "version": pss.Version, + "controls": strings.Join(controls, ","), + } } } - if len(controls) > 0 { - sort.Strings(controls) - result.Properties = map[string]string{ - "standard": string(pss.Level), - "version": pss.Version, - "controls": strings.Join(controls, ","), - } + if result.Result == "fail" && !result.Scored { + result.Result = "warn" } + results = append(results, result) } - if result.Result == "fail" && !result.Scored { - result.Result = "warn" + } else { + for _, ruleResult := range response.PolicyResponse.Rules { + result := policyreportv1alpha2.PolicyReportResult{ + Source: "ValidatingAdmissionPolicy", + Policy: ruleResult.Name(), + Message: ruleResult.Message(), + Result: toPolicyResult(ruleResult.Status()), + Timestamp: metav1.Timestamp{ + Seconds: time.Now().Unix(), + }, + } + results = append(results, result) } - results = append(results, result) } return results } @@ -163,7 +178,7 @@ func SetResults(report kyvernov1alpha2.ReportInterface, results ...policyreportv func SetResponses(report kyvernov1alpha2.ReportInterface, engineResponses ...engineapi.EngineResponse) { var ruleResults []policyreportv1alpha2.PolicyReportResult for _, result := range engineResponses { - pol := result.Policy().GetPolicy().(kyvernov1.PolicyInterface) + pol := result.Policy() SetPolicyLabel(report, pol) ruleResults = append(ruleResults, EngineResponseToReportResults(result)...) } diff --git a/pkg/utils/report/selector.go b/pkg/utils/report/selector.go index 0cbb2f1ccc..fd8a4a8736 100644 --- a/pkg/utils/report/selector.go +++ b/pkg/utils/report/selector.go @@ -1,7 +1,7 @@ package report import ( - kyvernov1 "github.com/kyverno/kyverno/api/kyverno/v1" + engineapi "github.com/kyverno/kyverno/pkg/engine/api" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/selection" "k8s.io/apimachinery/pkg/types" @@ -16,7 +16,7 @@ func SelectorResourceUidEquals(uid types.UID) (labels.Selector, error) { return selector, err } -func SelectorPolicyDoesNotExist(policy kyvernov1.PolicyInterface) (labels.Selector, error) { +func SelectorPolicyDoesNotExist(policy engineapi.GenericPolicy) (labels.Selector, error) { selector := labels.Everything() requirement, err := labels.NewRequirement(PolicyLabel(policy), selection.DoesNotExist, nil) if err == nil { @@ -25,7 +25,7 @@ func SelectorPolicyDoesNotExist(policy kyvernov1.PolicyInterface) (labels.Select return selector, err } -func SelectorPolicyExists(policy kyvernov1.PolicyInterface) (labels.Selector, error) { +func SelectorPolicyExists(policy engineapi.GenericPolicy) (labels.Selector, error) { selector := labels.Everything() requirement, err := labels.NewRequirement(PolicyLabel(policy), selection.Exists, nil) if err == nil { @@ -34,7 +34,7 @@ func SelectorPolicyExists(policy kyvernov1.PolicyInterface) (labels.Selector, er return selector, err } -func SelectorPolicyNotEquals(policy kyvernov1.PolicyInterface) (labels.Selector, error) { +func SelectorPolicyNotEquals(policy engineapi.GenericPolicy) (labels.Selector, error) { selector := labels.Everything() requirement, err := labels.NewRequirement(PolicyLabel(policy), selection.NotEquals, []string{policy.GetResourceVersion()}) if err == nil {