1
0
Fork 0
mirror of https://github.com/kyverno/kyverno.git synced 2025-03-06 07:57:07 +00:00

feat: implement background scan (#12101)

* feat: implement background scan

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* scanner

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* refactor request

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

---------

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>
This commit is contained in:
Charles-Edouard Brétéché 2025-02-06 04:49:41 +01:00 committed by GitHub
parent 208314b04a
commit 02fceb64f7
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
15 changed files with 351 additions and 136 deletions

View file

@ -36,6 +36,7 @@ import (
gitutils "github.com/kyverno/kyverno/pkg/utils/git"
policyvalidation "github.com/kyverno/kyverno/pkg/validation/policy"
"github.com/spf13/cobra"
admissionv1 "k8s.io/api/admission/v1"
admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@ -339,10 +340,21 @@ func (c *ApplyCommandConfig) applyValidatingPolicies(
}
responses := make([]engineapi.EngineResponse, 0)
for _, resource := range resources {
request := engine.EngineRequest{
Context: contextProvider,
Resource: resource,
}
request := engine.Request(
contextProvider,
resource.GroupVersionKind(),
// TODO
schema.GroupVersionResource{},
// TODO
"",
resource.GetName(),
resource.GetNamespace(),
admissionv1.Create,
resource,
nil,
false,
nil,
)
response, err := eng.Handle(ctx, request)
if err != nil {
if c.ContinueOnFail {

View file

@ -254,7 +254,7 @@ func getKindsFromValidatingAdmissionPolicy(policy admissionregistrationv1.Valida
resourceTypesMap := make(map[schema.GroupVersionKind]bool)
subresourceMap := make(map[schema.GroupVersionKind]v1alpha1.Subresource)
kinds := admissionpolicy.GetKinds(policy)
kinds := admissionpolicy.GetKinds(policy.Spec.MatchConstraints)
for _, kind := range kinds {
addGVKToResourceTypesMap(kind, resourceTypesMap, subresourceMap, client)
}

View file

@ -82,6 +82,7 @@ func createReportControllers(
client,
kyvernoV1.Policies(),
kyvernoV1.ClusterPolicies(),
kyvernoV2alpha1.ValidatingPolicies(),
vapInformer,
)
warmups = append(warmups, func(ctx context.Context) error {
@ -101,8 +102,8 @@ func createReportControllers(
metadataFactory,
kyvernoV1.Policies(),
kyvernoV1.ClusterPolicies(),
vapInformer,
kyvernoV2alpha1.ValidatingPolicies(),
vapInformer,
),
aggregationWorkers,
))
@ -115,6 +116,7 @@ func createReportControllers(
metadataFactory,
kyvernoV1.Policies(),
kyvernoV1.ClusterPolicies(),
kyvernoV2alpha1.ValidatingPolicies(),
kyvernoV2.PolicyExceptions(),
vapInformer,
vapBindingInformer,
@ -131,8 +133,8 @@ func createReportControllers(
ctrls = append(ctrls, internal.NewController(
backgroundscancontroller.ControllerName,
backgroundScanController,
backgroundScanWorkers),
)
backgroundScanWorkers,
))
}
}
return ctrls, func(ctx context.Context) error {

View file

@ -27,10 +27,8 @@ import (
celconfig "k8s.io/apiserver/pkg/apis/cel"
)
func GetKinds(policy admissionregistrationv1.ValidatingAdmissionPolicy) []string {
func GetKinds(matchResources *admissionregistrationv1.MatchResources) []string {
var kindList []string
matchResources := policy.Spec.MatchConstraints
for _, rule := range matchResources.ResourceRules {
group := rule.APIGroups[0]
version := rule.APIVersions[0]

View file

@ -128,7 +128,7 @@ spec:
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
_, policy, _, _, _ := yamlutils.GetPolicy(tt.policy)
kinds := GetKinds(policy[0])
kinds := GetKinds(policy[0].Spec.MatchConstraints)
if !reflect.DeepEqual(kinds, tt.wantKinds) {
t.Errorf("Expected %v, got %v", tt.wantKinds, kinds)
}

View file

@ -14,17 +14,62 @@ import (
admissionv1 "k8s.io/api/admission/v1"
admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apiserver/pkg/admission"
"k8s.io/utils/ptr"
)
type EngineRequest struct {
Request *admissionv1.AdmissionRequest
Resource *unstructured.Unstructured
Context contextlib.ContextInterface
request admissionv1.AdmissionRequest
context contextlib.ContextInterface
}
func RequestFromAdmission(context contextlib.ContextInterface, request admissionv1.AdmissionRequest) EngineRequest {
return EngineRequest{
request: request,
context: context,
}
}
func Request(
context contextlib.ContextInterface,
gvk schema.GroupVersionKind,
gvr schema.GroupVersionResource,
subResource string,
name string,
namespace string,
operation admissionv1.Operation,
// userInfo authenticationv1.UserInfo,
object runtime.Object,
oldObject runtime.Object,
dryRun bool,
options runtime.Object,
) EngineRequest {
request := admissionv1.AdmissionRequest{
Kind: metav1.GroupVersionKind(gvk),
Resource: metav1.GroupVersionResource(gvr),
SubResource: subResource,
RequestKind: ptr.To(metav1.GroupVersionKind(gvk)),
RequestResource: ptr.To(metav1.GroupVersionResource(gvr)),
RequestSubResource: subResource,
Name: name,
Namespace: namespace,
Operation: operation,
// UserInfo: userInfo,
Object: runtime.RawExtension{Object: object},
OldObject: runtime.RawExtension{Object: oldObject},
DryRun: &dryRun,
Options: runtime.RawExtension{Object: options},
}
return RequestFromAdmission(context, request)
}
func (r *EngineRequest) AdmissionRequest() admissionv1.AdmissionRequest {
return r.request
}
type EngineResponse struct {
@ -59,62 +104,49 @@ func NewEngine(provider Provider, nsResolver NamespaceResolver, matcher matching
}
func (e *engine) Handle(ctx context.Context, request EngineRequest) (EngineResponse, error) {
response := EngineResponse{
Resource: request.Resource,
}
var response EngineResponse
// fetch compiled policies
policies, err := e.provider.CompiledPolicies(ctx)
if err != nil {
return response, err
}
// load objects
object, oldObject, err := admissionutils.ExtractResources(nil, request.request)
if err != nil {
return response, err
}
response.Resource = &object
if response.Resource.Object == nil {
response.Resource = &oldObject
}
// default dry run
dryRun := false
if request.request.DryRun != nil {
dryRun = *request.request.DryRun
}
// create admission attributes
attr := admission.NewAttributesRecord(
&object,
&oldObject,
schema.GroupVersionKind(request.request.Kind),
request.request.Namespace,
request.request.Name,
schema.GroupVersionResource(request.request.Resource),
request.request.SubResource,
admission.Operation(request.request.Operation),
nil,
dryRun,
// TODO
nil,
)
// resolve namespace
var namespace runtime.Object
var attr admission.Attributes
if request.Request != nil {
object, oldObject, err := admissionutils.ExtractResources(nil, *request.Request)
if err != nil {
return response, err
}
dryRun := false
if request.Request.DryRun != nil {
dryRun = *request.Request.DryRun
}
attr = admission.NewAttributesRecord(
&object,
&oldObject,
schema.GroupVersionKind(request.Request.Kind),
request.Request.Namespace,
request.Request.Name,
schema.GroupVersionResource(request.Request.Resource),
request.Request.SubResource,
admission.Operation(request.Request.Operation),
nil,
dryRun,
// TODO
nil,
)
if ns := request.Request.Namespace; ns != "" {
namespace = e.nsResolver(ns)
}
} else {
attr = admission.NewAttributesRecord(
request.Resource,
nil,
request.Resource.GroupVersionKind(),
request.Resource.GetNamespace(),
request.Resource.GetName(),
schema.GroupVersionResource{},
"",
admission.Create,
nil,
false,
nil,
)
if ns := request.Resource.GetNamespace(); ns != "" {
namespace = e.nsResolver(ns)
}
if ns := request.request.Namespace; ns != "" {
namespace = e.nsResolver(ns)
}
// evaluate policies
for _, policy := range policies {
response.Policies = append(response.Policies, e.handlePolicy(ctx, policy, attr, request.Request, namespace, request.Context))
response.Policies = append(response.Policies, e.handlePolicy(ctx, policy, attr, &request.request, namespace, request.context))
}
return response, nil
}

View file

@ -52,8 +52,8 @@ type controller struct {
// listers
polLister kyvernov1listers.PolicyLister
cpolLister kyvernov1listers.ClusterPolicyLister
vapLister admissionregistrationv1listers.ValidatingAdmissionPolicyLister
vpolLister kyvernov2alpha1listers.ValidatingPolicyLister
vapLister admissionregistrationv1listers.ValidatingAdmissionPolicyLister
ephrLister cache.GenericLister
cephrLister cache.GenericLister
@ -73,8 +73,8 @@ func NewController(
metadataFactory metadatainformers.SharedInformerFactory,
polInformer kyvernov1informers.PolicyInformer,
cpolInformer kyvernov1informers.ClusterPolicyInformer,
vapInformer admissionregistrationv1informers.ValidatingAdmissionPolicyInformer,
vpolInformer kyvernov2alpha1informers.ValidatingPolicyInformer,
vapInformer admissionregistrationv1informers.ValidatingAdmissionPolicyInformer,
) controllers.Controller {
ephrInformer := metadataFactory.ForResource(reportsv1.SchemeGroupVersion.WithResource("ephemeralreports"))
cephrInformer := metadataFactory.ForResource(reportsv1.SchemeGroupVersion.WithResource("clusterephemeralreports"))
@ -133,10 +133,10 @@ func NewController(
); err != nil {
logger.Error(err, "failed to register event handlers")
}
if vapInformer != nil {
c.vapLister = vapInformer.Lister()
if vpolInformer != nil {
c.vpolLister = vpolInformer.Lister()
if _, err := controllerutils.AddEventHandlersT(
vapInformer.Informer(),
vpolInformer.Informer(),
func(_ metav1.Object) { enqueueAll() },
func(_, _ metav1.Object) { enqueueAll() },
func(_ metav1.Object) { enqueueAll() },
@ -144,10 +144,10 @@ func NewController(
logger.Error(err, "failed to register event handlers")
}
}
if vpolInformer != nil {
c.vpolLister = vpolInformer.Lister()
if vapInformer != nil {
c.vapLister = vapInformer.Lister()
if _, err := controllerutils.AddEventHandlersT(
vpolInformer.Informer(),
vapInformer.Informer(),
func(_ metav1.Object) { enqueueAll() },
func(_, _ metav1.Object) { enqueueAll() },
func(_ metav1.Object) { enqueueAll() },

View file

@ -8,14 +8,17 @@ import (
"github.com/go-logr/logr"
kyvernov1 "github.com/kyverno/kyverno/api/kyverno/v1"
kyvernov2 "github.com/kyverno/kyverno/api/kyverno/v2"
kyvernov2alpha1 "github.com/kyverno/kyverno/api/kyverno/v2alpha1"
policyreportv1alpha2 "github.com/kyverno/kyverno/api/policyreport/v1alpha2"
reportsv1 "github.com/kyverno/kyverno/api/reports/v1"
"github.com/kyverno/kyverno/pkg/breaker"
"github.com/kyverno/kyverno/pkg/client/clientset/versioned"
kyvernov1informers "github.com/kyverno/kyverno/pkg/client/informers/externalversions/kyverno/v1"
kyvernov2informers "github.com/kyverno/kyverno/pkg/client/informers/externalversions/kyverno/v2"
kyvernov2alpha1informers "github.com/kyverno/kyverno/pkg/client/informers/externalversions/kyverno/v2alpha1"
kyvernov1listers "github.com/kyverno/kyverno/pkg/client/listers/kyverno/v1"
kyvernov2listers "github.com/kyverno/kyverno/pkg/client/listers/kyverno/v2"
kyvernov2alpha1listers "github.com/kyverno/kyverno/pkg/client/listers/kyverno/v2alpha1"
"github.com/kyverno/kyverno/pkg/clients/dclient"
"github.com/kyverno/kyverno/pkg/config"
"github.com/kyverno/kyverno/pkg/controllers"
@ -60,6 +63,7 @@ type controller struct {
// listers
polLister kyvernov1listers.PolicyLister
cpolLister kyvernov1listers.ClusterPolicyLister
vpolLister kyvernov2alpha1listers.ValidatingPolicyLister
polexLister kyvernov2listers.PolicyExceptionLister
vapLister admissionregistrationv1listers.ValidatingAdmissionPolicyLister
vapBindingLister admissionregistrationv1listers.ValidatingAdmissionPolicyBindingLister
@ -68,7 +72,7 @@ type controller struct {
nsLister corev1listers.NamespaceLister
// queue
queue workqueue.TypedRateLimitingInterface[any]
queue workqueue.TypedRateLimitingInterface[string]
// cache
metadataCache resource.MetadataCache
@ -90,6 +94,7 @@ func NewController(
metadataFactory metadatainformers.SharedInformerFactory,
polInformer kyvernov1informers.PolicyInformer,
cpolInformer kyvernov1informers.ClusterPolicyInformer,
vpolInformer kyvernov2alpha1informers.ValidatingPolicyInformer,
polexInformer kyvernov2informers.PolicyExceptionInformer,
vapInformer admissionregistrationv1informers.ValidatingAdmissionPolicyInformer,
vapBindingInformer admissionregistrationv1informers.ValidatingAdmissionPolicyBindingInformer,
@ -106,8 +111,8 @@ func NewController(
ephrInformer := metadataFactory.ForResource(reportsv1.SchemeGroupVersion.WithResource("ephemeralreports"))
cephrInformer := metadataFactory.ForResource(reportsv1.SchemeGroupVersion.WithResource("clusterephemeralreports"))
queue := workqueue.NewTypedRateLimitingQueueWithConfig(
workqueue.DefaultTypedControllerRateLimiter[any](),
workqueue.TypedRateLimitingQueueConfig[any]{Name: ControllerName},
workqueue.DefaultTypedControllerRateLimiter[string](),
workqueue.TypedRateLimitingQueueConfig[string]{Name: ControllerName},
)
c := controller{
client: client,
@ -129,6 +134,12 @@ func NewController(
reportsConfig: reportsConfig,
breaker: breaker,
}
if vpolInformer != nil {
c.vpolLister = vpolInformer.Lister()
if _, err := controllerutils.AddEventHandlersT(vpolInformer.Informer(), c.addVP, c.updateVP, c.deleteVP); err != nil {
logger.Error(err, "failed to register event handlers")
}
}
if vapInformer != nil {
c.vapLister = vapInformer.Lister()
if _, err := controllerutils.AddEventHandlersT(vapInformer.Informer(), c.addVAP, c.updateVAP, c.deleteVAP); err != nil {
@ -197,6 +208,20 @@ func (c *controller) deleteException(obj *kyvernov2.PolicyException) {
c.enqueueResources()
}
func (c *controller) addVP(obj *kyvernov2alpha1.ValidatingPolicy) {
c.enqueueResources()
}
func (c *controller) updateVP(old, obj *kyvernov2alpha1.ValidatingPolicy) {
if old.GetResourceVersion() != obj.GetResourceVersion() {
c.enqueueResources()
}
}
func (c *controller) deleteVP(obj *kyvernov2alpha1.ValidatingPolicy) {
c.enqueueResources()
}
func (c *controller) addVAP(obj *admissionregistrationv1.ValidatingAdmissionPolicy) {
c.enqueueResources()
}
@ -313,6 +338,7 @@ func (c *controller) reconcileReport(
full bool,
uid types.UID,
gvk schema.GroupVersionKind,
gvr schema.GroupVersionResource,
resource resource.Resource,
exceptions []kyvernov2.PolicyException,
bindings []admissionregistrationv1.ValidatingAdmissionPolicyBinding,
@ -362,29 +388,21 @@ func (c *controller) reconcileReport(
policyNameToLabel := map[string]string{}
for _, policy := range policies {
var key string
var err error
if policy.AsKyvernoPolicy() != nil {
key, err = cache.MetaNamespaceKeyFunc(policy.AsKyvernoPolicy())
key = cache.MetaObjectToName(policy.AsKyvernoPolicy()).String()
} else if policy.AsValidatingAdmissionPolicy() != nil {
key, err = cache.MetaNamespaceKeyFunc(policy.AsValidatingAdmissionPolicy())
}
if err != nil {
return err
key = cache.MetaObjectToName(policy.AsValidatingAdmissionPolicy()).String()
} else if policy.AsValidatingPolicy() != nil {
key = cache.MetaObjectToName(policy.AsValidatingPolicy()).String()
}
policyNameToLabel[key] = reportutils.PolicyLabel(policy)
}
for i, exception := range exceptions {
key, err := cache.MetaNamespaceKeyFunc(&exceptions[i])
if err != nil {
return err
}
key := cache.MetaObjectToName(&exceptions[i]).String()
policyNameToLabel[key] = reportutils.PolicyExceptionLabel(exception)
}
for _, binding := range bindings {
key, err := cache.MetaNamespaceKeyFunc(binding)
if err != nil {
return err
}
key := cache.MetaObjectToName(&binding).String()
policyNameToLabel[key] = reportutils.ValidatingAdmissionPolicyBindingLabel(binding)
}
for _, result := range observed.GetResults() {
@ -401,7 +419,6 @@ func (c *controller) reconcileReport(
break
}
}
label := policyNameToLabel[result.Policy]
vapBindingLabel := policyNameToLabel[result.Properties["binding"]]
if (label != "" && expected[label] == actual[label]) ||
@ -430,7 +447,7 @@ func (c *controller) reconcileReport(
}
if full || reevaluate || actual[reportutils.PolicyLabel(policy)] != policy.GetResourceVersion() {
scanner := utils.NewScanner(logger, c.engine, c.config, c.jp, c.client, c.reportsConfig)
for _, result := range scanner.ScanResource(ctx, *target, ns, bindings, policy) {
for _, result := range scanner.ScanResource(ctx, *target, gvr, "", ns, bindings, policy) {
if result.Error != nil {
return result.Error
} else if result.EngineResponse != nil {
@ -499,7 +516,7 @@ func (c *controller) storeReport(ctx context.Context, observed, desired reportsv
func (c *controller) reconcile(ctx context.Context, log logr.Logger, key, namespace, name string) error {
// try to find resource from the cache
uid := types.UID(name)
resource, gvk, exists := c.metadataCache.GetResourceHash(uid)
resource, gvk, gvr, exists := c.metadataCache.GetResourceHash(uid)
// if the resource is not present it means we shouldn't have a report for it
// we can delete the report, we will recreate one if the resource comes back
if !exists {
@ -535,6 +552,16 @@ func (c *controller) reconcile(ctx context.Context, log logr.Logger, key, namesp
for _, pol := range kyvernoPolicies {
policies = append(policies, engineapi.NewKyvernoPolicy(pol))
}
if c.vpolLister != nil {
// load validating policies
vpols, err := utils.FetchValidatingPolicies(c.vpolLister)
if err != nil {
return err
}
for _, vpol := range vpols {
policies = append(policies, engineapi.NewValidatingPolicy(&vpol))
}
}
if c.vapLister != nil {
// load validating admission policies
vapPolicies, err := utils.FetchValidatingAdmissionPolicies(c.vapLister)
@ -566,7 +593,7 @@ func (c *controller) reconcile(ctx context.Context, log logr.Logger, key, namesp
c.queue.AddAfter(key, c.forceDelay)
}()
if needsReconcile {
return c.reconcileReport(ctx, namespace, name, full, uid, gvk, resource, exceptions, vapBindings, policies...)
return c.reconcileReport(ctx, namespace, name, full, uid, gvk, gvr, resource, exceptions, vapBindings, policies...)
}
}
return nil

View file

@ -10,7 +10,9 @@ import (
"github.com/go-logr/logr"
"github.com/kyverno/kyverno/pkg/admissionpolicy"
kyvernov1informers "github.com/kyverno/kyverno/pkg/client/informers/externalversions/kyverno/v1"
kyvernov2alpha1informers "github.com/kyverno/kyverno/pkg/client/informers/externalversions/kyverno/v2alpha1"
kyvernov1listers "github.com/kyverno/kyverno/pkg/client/listers/kyverno/v1"
kyvernov2alpha1listers "github.com/kyverno/kyverno/pkg/client/listers/kyverno/v2alpha1"
"github.com/kyverno/kyverno/pkg/clients/dclient"
"github.com/kyverno/kyverno/pkg/controllers"
"github.com/kyverno/kyverno/pkg/controllers/report/utils"
@ -55,7 +57,7 @@ const (
type EventHandler func(EventType, types.UID, schema.GroupVersionKind, Resource)
type MetadataCache interface {
GetResourceHash(uid types.UID) (Resource, schema.GroupVersionKind, bool)
GetResourceHash(uid types.UID) (Resource, schema.GroupVersionKind, schema.GroupVersionResource, bool)
GetAllResourceKeys() []string
AddEventHandler(EventHandler)
Warmup(ctx context.Context) error
@ -79,6 +81,7 @@ type controller struct {
// listers
polLister kyvernov1listers.PolicyLister
cpolLister kyvernov1listers.ClusterPolicyLister
vpolLister kyvernov2alpha1listers.ValidatingPolicyLister
vapLister admissionregistrationv1listers.ValidatingAdmissionPolicyLister
// queue
@ -93,6 +96,7 @@ func NewController(
client dclient.Interface,
polInformer kyvernov1informers.PolicyInformer,
cpolInformer kyvernov1informers.ClusterPolicyInformer,
vpolInformer kyvernov2alpha1informers.ValidatingPolicyInformer,
vapInformer admissionregistrationv1informers.ValidatingAdmissionPolicyInformer,
) Controller {
c := controller{
@ -105,6 +109,12 @@ func NewController(
),
dynamicWatchers: map[schema.GroupVersionResource]*watcher{},
}
if vpolInformer != nil {
c.vpolLister = vpolInformer.Lister()
if _, _, err := controllerutils.AddDefaultEventHandlers(logger, vpolInformer.Informer(), c.queue); err != nil {
logger.Error(err, "failed to register event handlers")
}
}
if vapInformer != nil {
c.vapLister = vapInformer.Lister()
if _, _, err := controllerutils.AddDefaultEventHandlers(logger, vapInformer.Informer(), c.queue); err != nil {
@ -129,15 +139,15 @@ func (c *controller) Run(ctx context.Context, workers int) {
c.stopDynamicWatchers()
}
func (c *controller) GetResourceHash(uid types.UID) (Resource, schema.GroupVersionKind, bool) {
func (c *controller) GetResourceHash(uid types.UID) (Resource, schema.GroupVersionKind, schema.GroupVersionResource, bool) {
c.lock.RLock()
defer c.lock.RUnlock()
for _, watcher := range c.dynamicWatchers {
for gvr, watcher := range c.dynamicWatchers {
if resource, exists := watcher.hashes[uid]; exists {
return resource, watcher.gvk, true
return resource, watcher.gvk, gvr, true
}
}
return Resource{}, schema.GroupVersionKind{}, false
return Resource{}, schema.GroupVersionKind{}, schema.GroupVersionResource{}, false
}
func (c *controller) GetAllResourceKeys() []string {
@ -249,7 +259,21 @@ func (c *controller) updateDynamicWatchers(ctx context.Context) error {
}
// fetch kinds from validating admission policies
for _, policy := range vapPolicies {
kinds := admissionpolicy.GetKinds(policy)
kinds := admissionpolicy.GetKinds(policy.Spec.MatchConstraints)
for _, kind := range kinds {
group, version, kind, subresource := kubeutils.ParseKindSelector(kind)
c.addGVKToGVRMapping(group, version, kind, subresource, gvkToGvr)
}
}
}
if c.vpolLister != nil {
vpols, err := utils.FetchValidatingPolicies(c.vpolLister)
if err != nil {
return err
}
// fetch kinds from validating admission policies
for _, policy := range vpols {
kinds := admissionpolicy.GetKinds(policy.Spec.MatchConstraints)
for _, kind := range kinds {
group, version, kind, subresource := kubeutils.ParseKindSelector(kind)
c.addGVKToGVRMapping(group, version, kind, subresource, gvkToGvr)

View file

@ -7,6 +7,9 @@ import (
"github.com/kyverno/kyverno/api/kyverno"
kyvernov1 "github.com/kyverno/kyverno/api/kyverno/v1"
"github.com/kyverno/kyverno/pkg/admissionpolicy"
celengine "github.com/kyverno/kyverno/pkg/cel/engine"
"github.com/kyverno/kyverno/pkg/cel/matching"
celpolicy "github.com/kyverno/kyverno/pkg/cel/policy"
"github.com/kyverno/kyverno/pkg/clients/dclient"
"github.com/kyverno/kyverno/pkg/config"
"github.com/kyverno/kyverno/pkg/engine"
@ -14,9 +17,11 @@ import (
"github.com/kyverno/kyverno/pkg/engine/jmespath"
reportutils "github.com/kyverno/kyverno/pkg/utils/report"
"go.uber.org/multierr"
admissionv1 "k8s.io/api/admission/v1"
admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime/schema"
)
type scanner struct {
@ -34,7 +39,15 @@ type ScanResult struct {
}
type Scanner interface {
ScanResource(context.Context, unstructured.Unstructured, *corev1.Namespace, []admissionregistrationv1.ValidatingAdmissionPolicyBinding, ...engineapi.GenericPolicy) map[*engineapi.GenericPolicy]ScanResult
ScanResource(
context.Context,
unstructured.Unstructured,
schema.GroupVersionResource,
string,
*corev1.Namespace,
[]admissionregistrationv1.ValidatingAdmissionPolicyBinding,
...engineapi.GenericPolicy,
) map[*engineapi.GenericPolicy]ScanResult
}
func NewScanner(
@ -55,18 +68,38 @@ func NewScanner(
}
}
func (s *scanner) ScanResource(ctx context.Context, resource unstructured.Unstructured, ns *corev1.Namespace, bindings []admissionregistrationv1.ValidatingAdmissionPolicyBinding, policies ...engineapi.GenericPolicy) map[*engineapi.GenericPolicy]ScanResult {
results := map[*engineapi.GenericPolicy]ScanResult{}
for i, policy := range policies {
var errors []error
logger := s.logger.WithValues("kind", resource.GetKind(), "namespace", resource.GetNamespace(), "name", resource.GetName())
var response *engineapi.EngineResponse
func (s *scanner) ScanResource(
ctx context.Context,
resource unstructured.Unstructured,
gvr schema.GroupVersionResource,
subResource string,
ns *corev1.Namespace,
bindings []admissionregistrationv1.ValidatingAdmissionPolicyBinding,
policies ...engineapi.GenericPolicy,
) map[*engineapi.GenericPolicy]ScanResult {
var kpols, vpols, vaps []engineapi.GenericPolicy
// split policies per nature
for _, policy := range policies {
if pol := policy.AsKyvernoPolicy(); pol != nil {
kpols = append(kpols, policy)
} else if pol := policy.AsValidatingPolicy(); pol != nil {
vpols = append(vpols, policy)
} else if pol := policy.AsValidatingAdmissionPolicy(); pol != nil {
vaps = append(vaps, policy)
}
}
logger := s.logger.WithValues("kind", resource.GetKind(), "namespace", resource.GetNamespace(), "name", resource.GetName())
results := map[*engineapi.GenericPolicy]ScanResult{}
// evaluate kyverno policies
var nsLabels map[string]string
if ns != nil {
nsLabels = ns.Labels
}
for i, policy := range kpols {
if pol := policy.AsKyvernoPolicy(); pol != nil {
var errors []error
var response *engineapi.EngineResponse
var err error
var nsLabels map[string]string
if ns != nil {
nsLabels = ns.Labels
}
if s.reportingConfig.ValidateReportsEnabled() {
response, err = s.validateResource(ctx, resource, nsLabels, pol)
if err != nil {
@ -86,7 +119,6 @@ func (s *scanner) ScanResource(ctx context.Context, resource unstructured.Unstru
}
response.PolicyResponse.Rules = ruleResponses
}
ivResponse, err := s.validateImages(ctx, resource, nsLabels, pol)
if err != nil {
logger.Error(err, "failed to scan images")
@ -98,7 +130,66 @@ func (s *scanner) ScanResource(ctx context.Context, resource unstructured.Unstru
response.PolicyResponse.Rules = append(response.PolicyResponse.Rules, ivResponse.PolicyResponse.Rules...)
}
}
} else if pol := policy.AsValidatingAdmissionPolicy(); pol != nil {
results[&kpols[i]] = ScanResult{response, multierr.Combine(errors...)}
}
}
// evaluate validating policies
for i, policy := range vpols {
if pol := policy.AsValidatingPolicy(); pol != nil {
// create compiler
compiler := celpolicy.NewCompiler()
// create provider
provider, err := celengine.NewProvider(compiler, *pol)
if err != nil {
logger.Error(err, "failed to create policy provider")
results[&vpols[i]] = ScanResult{nil, err}
continue
}
// create engine
engine := celengine.NewEngine(
provider,
func(name string) *corev1.Namespace { return ns },
matching.NewMatcher(),
)
// create context provider
context, err := celpolicy.NewContextProvider(
s.client.GetKubeClient(),
nil,
// TODO
// []imagedataloader.Option{imagedataloader.WithLocalCredentials(c.RegistryAccess)},
)
if err != nil {
logger.Error(err, "failed to create cel context provider")
results[&vpols[i]] = ScanResult{nil, err}
continue
}
request := celengine.Request(
context,
resource.GroupVersionKind(),
gvr,
subResource,
resource.GetName(),
resource.GetNamespace(),
admissionv1.Create,
&resource,
nil,
false,
nil,
)
engineResponse, err := engine.Handle(ctx, request)
response := engineapi.EngineResponse{
Resource: resource,
PolicyResponse: engineapi.PolicyResponse{
// TODO: policies at index 0
Rules: engineResponse.Policies[0].Rules,
},
}.WithPolicy(vpols[i])
results[&vpols[i]] = ScanResult{&response, err}
}
}
// evaluate validating admission policies
for i, policy := range vaps {
if pol := policy.AsValidatingAdmissionPolicy(); pol != nil {
policyData := admissionpolicy.NewPolicyData(*pol)
for _, binding := range bindings {
if binding.Spec.PolicyName == pol.Name {
@ -106,12 +197,8 @@ func (s *scanner) ScanResource(ctx context.Context, resource unstructured.Unstru
}
}
res, err := admissionpolicy.Validate(policyData, resource, map[string]map[string]string{}, s.client)
if err != nil {
errors = append(errors, err)
}
response = &res
results[&vaps[i]] = ScanResult{&res, err}
}
results[&policies[i]] = ScanResult{response, multierr.Combine(errors...)}
}
return results
}

View file

@ -4,10 +4,12 @@ import (
"github.com/go-logr/logr"
kyvernov1 "github.com/kyverno/kyverno/api/kyverno/v1"
kyvernov2 "github.com/kyverno/kyverno/api/kyverno/v2"
kyvernov2alpha1 "github.com/kyverno/kyverno/api/kyverno/v2alpha1"
reportsv1 "github.com/kyverno/kyverno/api/reports/v1"
"github.com/kyverno/kyverno/pkg/autogen"
kyvernov1listers "github.com/kyverno/kyverno/pkg/client/listers/kyverno/v1"
kyvernov2listers "github.com/kyverno/kyverno/pkg/client/listers/kyverno/v2"
kyvernov2alpha1listers "github.com/kyverno/kyverno/pkg/client/listers/kyverno/v2alpha1"
datautils "github.com/kyverno/kyverno/pkg/utils/data"
policyvalidation "github.com/kyverno/kyverno/pkg/validation/policy"
admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
@ -148,3 +150,15 @@ func FetchValidatingAdmissionPolicyBindings(vapBindingLister admissionregistrati
}
return bindings, nil
}
func FetchValidatingPolicies(vpolLister kyvernov2alpha1listers.ValidatingPolicyLister) ([]kyvernov2alpha1.ValidatingPolicy, error) {
var policies []kyvernov2alpha1.ValidatingPolicy
if pols, err := vpolLister.List(labels.Everything()); err != nil {
return nil, err
} else {
for _, pol := range pols {
policies = append(policies, *pol)
}
}
return policies, nil
}

View file

@ -6,6 +6,7 @@ import (
kubeutils "github.com/kyverno/kyverno/pkg/utils/kube"
admissionv1 "k8s.io/api/admission/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
)
@ -34,6 +35,12 @@ func ExtractResources(newRaw []byte, request admissionv1.AdmissionRequest) (unst
if err != nil {
return emptyResource, emptyResource, fmt.Errorf("failed to convert new raw to unstructured: %v", err)
}
} else if request.Object.Object != nil {
ret, err := runtime.DefaultUnstructuredConverter.ToUnstructured(request.Object.Object)
if err != nil {
return emptyResource, emptyResource, fmt.Errorf("failed to convert new raw to unstructured: %v", err)
}
newResource = unstructured.Unstructured{Object: ret}
}
// Old Resource
@ -43,6 +50,12 @@ func ExtractResources(newRaw []byte, request admissionv1.AdmissionRequest) (unst
if err != nil {
return emptyResource, emptyResource, fmt.Errorf("failed to convert old raw to unstructured: %v", err)
}
} else if request.OldObject.Object != nil {
ret, err := runtime.DefaultUnstructuredConverter.ToUnstructured(request.OldObject.Object)
if err != nil {
return emptyResource, emptyResource, fmt.Errorf("failed to convert old raw to unstructured: %v", err)
}
oldResource = unstructured.Unstructured{Object: ret}
}
return newResource, oldResource, err

View file

@ -54,7 +54,7 @@ func newControllerMetrics(logger logr.Logger, controllerName string) *controller
}
}
func Run(ctx context.Context, logger logr.Logger, controllerName string, period time.Duration, queue workqueue.TypedRateLimitingInterface[any], n, maxRetries int, r reconcileFunc, routines ...func(context.Context, logr.Logger)) {
func Run[T comparable](ctx context.Context, logger logr.Logger, controllerName string, period time.Duration, queue workqueue.TypedRateLimitingInterface[T], n, maxRetries int, r reconcileFunc, routines ...func(context.Context, logr.Logger)) {
logger.Info("starting ...")
defer logger.Info("stopped")
var wg sync.WaitGroup
@ -88,12 +88,12 @@ func Run(ctx context.Context, logger logr.Logger, controllerName string, period
logger.Info("waiting for workers to terminate ...")
}
func worker(ctx context.Context, logger logr.Logger, metric *controllerMetrics, queue workqueue.TypedRateLimitingInterface[any], maxRetries int, r reconcileFunc) {
func worker[T comparable](ctx context.Context, logger logr.Logger, metric *controllerMetrics, queue workqueue.TypedRateLimitingInterface[T], maxRetries int, r reconcileFunc) {
for processNextWorkItem(ctx, logger, metric, queue, maxRetries, r) {
}
}
func processNextWorkItem(ctx context.Context, logger logr.Logger, metric *controllerMetrics, queue workqueue.TypedRateLimitingInterface[any], maxRetries int, r reconcileFunc) bool {
func processNextWorkItem[T comparable](ctx context.Context, logger logr.Logger, metric *controllerMetrics, queue workqueue.TypedRateLimitingInterface[T], maxRetries int, r reconcileFunc) bool {
if obj, quit := queue.Get(); !quit {
defer queue.Done(obj)
handleErr(ctx, logger, metric, queue, maxRetries, reconcile(ctx, logger, obj, r), obj)
@ -102,7 +102,7 @@ func processNextWorkItem(ctx context.Context, logger logr.Logger, metric *contro
return false
}
func handleErr(ctx context.Context, logger logr.Logger, metric *controllerMetrics, queue workqueue.TypedRateLimitingInterface[any], maxRetries int, err error, obj interface{}) {
func handleErr[T comparable](ctx context.Context, logger logr.Logger, metric *controllerMetrics, queue workqueue.TypedRateLimitingInterface[T], maxRetries int, err error, obj T) {
if metric.reconcileTotal != nil {
metric.reconcileTotal.Add(ctx, 1, sdkmetric.WithAttributes(attribute.String("controller_name", metric.controllerName)))
}

View file

@ -35,8 +35,10 @@ const (
// policy labels
LabelDomainClusterPolicy = "cpol.kyverno.io"
LabelDomainPolicy = "pol.kyverno.io"
LabelDomainValidatingPolicy = "vpol.kyverno.io"
LabelPrefixClusterPolicy = LabelDomainClusterPolicy + "/"
LabelPrefixPolicy = LabelDomainPolicy + "/"
LabelPrefixValidatingPolicy = LabelDomainValidatingPolicy + "/"
LabelPrefixPolicyException = "polex.kyverno.io/"
LabelPrefixValidatingAdmissionPolicy = "validatingadmissionpolicy.apiserver.io/"
LabelPrefixValidatingAdmissionPolicyBinding = "validatingadmissionpolicybinding.apiserver.io/"
@ -47,6 +49,7 @@ const (
func IsPolicyLabel(label string) bool {
return strings.HasPrefix(label, LabelPrefixPolicy) ||
strings.HasPrefix(label, LabelPrefixClusterPolicy) ||
strings.HasPrefix(label, LabelPrefixValidatingPolicy) ||
strings.HasPrefix(label, LabelPrefixPolicyException) ||
strings.HasPrefix(label, LabelPrefixValidatingAdmissionPolicy) ||
strings.HasPrefix(label, LabelPrefixValidatingAdmissionPolicyBinding)
@ -65,12 +68,16 @@ func PolicyNameFromLabel(namespace, label string) (string, error) {
}
func PolicyLabelPrefix(policy engineapi.GenericPolicy) string {
if policy.IsNamespaced() {
return LabelPrefixPolicy
}
if policy.AsKyvernoPolicy() != nil {
if policy.IsNamespaced() {
return LabelPrefixPolicy
}
return LabelPrefixClusterPolicy
}
if policy.AsValidatingPolicy() != nil {
return LabelPrefixValidatingPolicy
}
// TODO: detect potential type not detected
return LabelPrefixValidatingAdmissionPolicy
}

View file

@ -40,26 +40,24 @@ func New(
}
}
func (h *handler) Validate(ctx context.Context, logger logr.Logger, request handlers.AdmissionRequest, failurePolicy string, startTime time.Time) handlers.AdmissionResponse {
response, err := h.engine.Handle(ctx, celengine.EngineRequest{
Request: &request.AdmissionRequest,
Context: h.context,
})
func (h *handler) Validate(ctx context.Context, logger logr.Logger, admissionRequest handlers.AdmissionRequest, failurePolicy string, startTime time.Time) handlers.AdmissionResponse {
request := celengine.RequestFromAdmission(h.context, admissionRequest.AdmissionRequest)
response, err := h.engine.Handle(ctx, request)
if err != nil {
return admissionutils.Response(request.UID, err)
return admissionutils.Response(admissionRequest.UID, err)
}
var group wait.Group
defer group.Wait()
group.Start(func() {
err := h.admissionReport(ctx, response, request)
err := h.admissionReport(ctx, request, response)
if err != nil {
logger.Error(err, "failed to create report")
}
})
return h.admissionResponse(response, request)
return h.admissionResponse(request, response)
}
func (h *handler) admissionResponse(response celengine.EngineResponse, request handlers.AdmissionRequest) handlers.AdmissionResponse {
func (h *handler) admissionResponse(request celengine.EngineRequest, response celengine.EngineResponse) handlers.AdmissionResponse {
var errs []error
var warnings []string
for _, policy := range response.Policies {
@ -84,11 +82,12 @@ func (h *handler) admissionResponse(response celengine.EngineResponse, request h
}
}
}
return admissionutils.Response(request.UID, multierr.Combine(errs...), warnings...)
return admissionutils.Response(request.AdmissionRequest().UID, multierr.Combine(errs...), warnings...)
}
func (h *handler) admissionReport(ctx context.Context, response celengine.EngineResponse, request handlers.AdmissionRequest) error {
object, oldObject, err := admissionutils.ExtractResources(nil, request.AdmissionRequest)
func (h *handler) admissionReport(ctx context.Context, request celengine.EngineRequest, response celengine.EngineResponse) error {
admissionRequest := request.AdmissionRequest()
object, oldObject, err := admissionutils.ExtractResources(nil, admissionRequest)
if err != nil {
return err
}
@ -106,7 +105,7 @@ func (h *handler) admissionReport(ctx context.Context, response celengine.Engine
engineResponse = engineResponse.WithPolicy(engineapi.NewValidatingPolicy(&r.Policy))
responses = append(responses, engineResponse)
}
report := reportutils.BuildAdmissionReport(object, request.AdmissionRequest, responses...)
report := reportutils.BuildAdmissionReport(object, admissionRequest, responses...)
if len(report.GetResults()) > 0 {
err := h.reportsBreaker.Do(ctx, func(ctx context.Context) error {
_, err := reportutils.CreateReport(ctx, report, h.kyvernoClient)