1
0
Fork 0
mirror of https://github.com/kyverno/kyverno.git synced 2024-12-14 11:57:48 +00:00

feat: enhance global context (#9710)

* feat(globalcontext): add event handling

Signed-off-by: Khaled Emara <khaled.emara@nirmata.com>

* feat(globalcontext): handle cache sync error

Signed-off-by: Khaled Emara <khaled.emara@nirmata.com>

* feat(globalcontext): ensure api is called during init

Signed-off-by: Khaled Emara <khaled.emara@nirmata.com>

* design(events): decouple events from policies a bit

Signed-off-by: Khaled Emara <khaled.emara@nirmata.com>

* feat(globalcontext): use status

Signed-off-by: Khaled Emara <khaled.emara@nirmata.com>

* fix(globalcontext): make status optional

Signed-off-by: Khaled Emara <khaled.emara@nirmata.com>

* fix(globalcontext): status update

Signed-off-by: Khaled Emara <khaled.emara@nirmata.com>

* fix(globalcontext): codegen

Signed-off-by: Khaled Emara <khaled.emara@nirmata.com>

* chore(globalcontext): delete yaml annotations

Signed-off-by: Khaled Emara <khaled.emara@nirmata.com>

* fix(globalcontext): fix status in tests

Signed-off-by: Khaled Emara <khaled.emara@nirmata.com>

* fix(globalcotext): update enqueue func

Signed-off-by: Khaled Emara <khaled.emara@nirmata.com>

* fix(globalcontext): error

Signed-off-by: Khaled Emara <khaled.emara@nirmata.com>

* chore(globalcontext): rbac

Signed-off-by: Khaled Emara <khaled.emara@nirmata.com>

* chore(globalcontext): retry logic

Signed-off-by: Khaled Emara <khaled.emara@nirmata.com>

* fix(globalcontext): unknown api call in test

Signed-off-by: Khaled Emara <khaled.emara@nirmata.com>

* bump

Signed-off-by: Khaled Emara <khaled.emara@nirmata.com>

* fix: set unique name for each testing resource

Signed-off-by: ShutingZhao <shuting@nirmata.com>

* fix: update readme

Signed-off-by: ShutingZhao <shuting@nirmata.com>

* fix: log msg

Signed-off-by: ShutingZhao <shuting@nirmata.com>

* fix: add delays

Signed-off-by: ShutingZhao <shuting@nirmata.com>

* fix: delay gctce creation

Signed-off-by: ShutingZhao <shuting@nirmata.com>

* debug: check Kyverno status

Signed-off-by: ShutingZhao <shuting@nirmata.com>

* debug: update chainsaw config

Signed-off-by: ShutingZhao <shuting@nirmata.com>

* debug: revert chainsaw config

Signed-off-by: ShutingZhao <shuting@nirmata.com>

* test(globalcontext): print actual status

Signed-off-by: Khaled Emara <khaled.emara@nirmata.com>

* fix(globalcontext): add necessary delays and check status before applying

Signed-off-by: Khaled Emara <khaled.emara@nirmata.com>

* test(globalcontext): long refreshInterval

Signed-off-by: Khaled Emara <khaled.emara@nirmata.com>

* debug: log success

Signed-off-by: Khaled Emara <khaled.emara@nirmata.com>

* debug: print informer data

Signed-off-by: Khaled Emara <khaled.emara@nirmata.com>

* fix(globalcontext): use client instead of informer

Signed-off-by: Khaled Emara <khaled.emara@nirmata.com>

* debug: print status after update

Signed-off-by: Khaled Emara <khaled.emara@nirmata.com>

* debug: print ResourceVersion

Signed-off-by: Khaled Emara <khaled.emara@nirmata.com>

* debug: remove gcecontroller from other controllers

Signed-off-by: Khaled Emara <khaled.emara@nirmata.com>

* fix(globalcontext): update status only once

Signed-off-by: Khaled Emara <khaled.emara@nirmata.com>

* chore: remove excess logs

Signed-off-by: Khaled Emara <khaled.emara@nirmata.com>

* fix(globalcontext): add store to cleanup controller

Signed-off-by: Khaled Emara <khaled.emara@nirmata.com>

---------

Signed-off-by: Khaled Emara <khaled.emara@nirmata.com>
Signed-off-by: ShutingZhao <shuting@nirmata.com>
Co-authored-by: shuting <shuting@nirmata.com>
Co-authored-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>
This commit is contained in:
Khaled Emara 2024-02-23 12:34:04 +02:00 committed by GitHub
parent 7a93dcdbc9
commit 2b2587469d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
62 changed files with 417 additions and 155 deletions

View file

@ -28,6 +28,8 @@ import (
// +kubebuilder:object:root=true
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
// +kubebuilder:resource:shortName=gctxentry,categories=kyverno,scope="Cluster"
// +kubebuilder:subresource:status
// +kubebuilder:printcolumn:name="READY",type=string,JSONPath=`.status.conditions[?(@.type == "Ready")].status`
// GlobalContextEntry declares resources to be cached.
type GlobalContextEntry struct {

View file

@ -24,7 +24,11 @@ spec:
singular: globalcontextentry
scope: Cluster
versions:
- name: v2alpha1
- additionalPrinterColumns:
- jsonPath: .status.conditions[?(@.type == "Ready")].status
name: READY
type: string
name: v2alpha1
schema:
openAPIV3Schema:
description: GlobalContextEntry declares resources to be cached.
@ -199,4 +203,6 @@ spec:
type: object
served: true
storage: true
subresources:
status: {}
{{- end }}

View file

@ -60,6 +60,8 @@ rules:
- clusterpolicies/status
- updaterequests
- updaterequests/status
- globalcontextentries
- globalcontextentries/status
- admissionreports
- clusteradmissionreports
- backgroundscanreports
@ -74,13 +76,6 @@ rules:
- update
- watch
- deletecollection
- apiGroups:
- kyverno.io
resources:
- globalcontextentries
verbs:
- list
- watch
- apiGroups:
- reports.kyverno.io
resources:

View file

@ -50,6 +50,8 @@ spec:
resources:
- clusterpolicies
- clusterpolicies/status
- globalcontextentries
- globalcontextentries/status
- clusteradmissionreports
- clusterbackgroundscanreports
verbs:

View file

@ -32,6 +32,8 @@ rules:
- policyexceptions
- updaterequests
- updaterequests/status
- globalcontextentries
- globalcontextentries/status
verbs:
- create
- delete
@ -41,13 +43,6 @@ rules:
- update
- watch
- deletecollection
- apiGroups:
- kyverno.io
resources:
- globalcontextentries
verbs:
- list
- watch
- apiGroups:
- ''
resources:

View file

@ -55,9 +55,16 @@ rules:
- kyverno.io
resources:
- globalcontextentries
- globalcontextentries/status
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- deletecollection
- apiGroups:
- kyverno.io
resources:

View file

@ -38,12 +38,7 @@ rules:
- kyverno.io
resources:
- globalcontextentries
verbs:
- list
- watch
- apiGroups:
- kyverno.io
resources:
- globalcontextentries/status
- admissionreports
- clusteradmissionreports
- backgroundscanreports

View file

@ -160,8 +160,11 @@ func main() {
globalcontextcontroller.NewController(
kyvernoInformer.Kyverno().V2alpha1().GlobalContextEntries(),
setup.KyvernoDynamicClient,
setup.KyvernoClient,
gcstore,
eventGenerator,
maxAPICallResponseLength,
false,
),
globalcontextcontroller.Workers,
) // this controller only subscribe to events, nothing is returned...

View file

@ -166,8 +166,11 @@ func main() {
globalcontextcontroller.NewController(
kyvernoInformer.Kyverno().V2alpha1().GlobalContextEntries(),
setup.KyvernoDynamicClient,
setup.KyvernoClient,
gcstore,
eventGenerator,
maxAPICallResponseLength,
false,
),
globalcontextcontroller.Workers,
)
@ -305,6 +308,7 @@ func main() {
cmResolver,
setup.Jp,
eventGenerator,
gcstore,
),
cleanup.Workers,
)

View file

@ -362,8 +362,11 @@ func main() {
globalcontextcontroller.NewController(
kyvernoInformer.Kyverno().V2alpha1().GlobalContextEntries(),
setup.KyvernoDynamicClient,
setup.KyvernoClient,
gcstore,
eventGenerator,
maxAPICallResponseLength,
true,
),
globalcontextcontroller.Workers,
)
@ -508,7 +511,7 @@ func main() {
)
policyHandlers := webhookspolicy.NewHandlers(
setup.KyvernoDynamicClient,
kyvernoInformer.Kyverno().V2alpha1().GlobalContextEntries(),
setup.KyvernoClient,
backgroundServiceAccountName,
)
resourceHandlers := webhooksresource.NewHandlers(

View file

@ -278,8 +278,11 @@ func main() {
globalcontextcontroller.NewController(
kyvernoInformer.Kyverno().V2alpha1().GlobalContextEntries(),
setup.KyvernoDynamicClient,
setup.KyvernoClient,
gcstore,
eventGenerator,
maxAPICallResponseLength,
false,
),
globalcontextcontroller.Workers,
)

View file

@ -18,7 +18,11 @@ spec:
singular: globalcontextentry
scope: Cluster
versions:
- name: v2alpha1
- additionalPrinterColumns:
- jsonPath: .status.conditions[?(@.type == "Ready")].status
name: READY
type: string
name: v2alpha1
schema:
openAPIV3Schema:
description: GlobalContextEntry declares resources to be cached.
@ -193,3 +197,5 @@ spec:
type: object
served: true
storage: true
subresources:
status: {}

View file

@ -28569,7 +28569,11 @@ spec:
singular: globalcontextentry
scope: Cluster
versions:
- name: v2alpha1
- additionalPrinterColumns:
- jsonPath: .status.conditions[?(@.type == "Ready")].status
name: READY
type: string
name: v2alpha1
schema:
openAPIV3Schema:
description: GlobalContextEntry declares resources to be cached.
@ -28744,6 +28748,8 @@ spec:
type: object
served: true
storage: true
subresources:
status: {}
---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
@ -51279,6 +51285,8 @@ rules:
- clusterpolicies/status
- updaterequests
- updaterequests/status
- globalcontextentries
- globalcontextentries/status
- admissionreports
- clusteradmissionreports
- backgroundscanreports
@ -51293,13 +51301,6 @@ rules:
- update
- watch
- deletecollection
- apiGroups:
- kyverno.io
resources:
- globalcontextentries
verbs:
- list
- watch
- apiGroups:
- reports.kyverno.io
resources:
@ -51414,6 +51415,8 @@ rules:
- policyexceptions
- updaterequests
- updaterequests/status
- globalcontextentries
- globalcontextentries/status
verbs:
- create
- delete
@ -51423,13 +51426,6 @@ rules:
- update
- watch
- deletecollection
- apiGroups:
- kyverno.io
resources:
- globalcontextentries
verbs:
- list
- watch
- apiGroups:
- ''
resources:
@ -51556,9 +51552,16 @@ rules:
- kyverno.io
resources:
- globalcontextentries
- globalcontextentries/status
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- deletecollection
- apiGroups:
- kyverno.io
resources:
@ -51867,12 +51870,7 @@ rules:
- kyverno.io
resources:
- globalcontextentries
verbs:
- list
- watch
- apiGroups:
- kyverno.io
resources:
- globalcontextentries/status
- admissionreports
- clusteradmissionreports
- backgroundscanreports

View file

@ -17,6 +17,7 @@ import (
"github.com/kyverno/kyverno/pkg/controllers"
engineapi "github.com/kyverno/kyverno/pkg/engine/api"
enginecontext "github.com/kyverno/kyverno/pkg/engine/context"
"github.com/kyverno/kyverno/pkg/engine/context/loaders"
"github.com/kyverno/kyverno/pkg/engine/factories"
"github.com/kyverno/kyverno/pkg/engine/jmespath"
"github.com/kyverno/kyverno/pkg/event"
@ -57,6 +58,7 @@ type controller struct {
eventGen event.Interface
jp jmespath.Interface
metrics cleanupMetrics
gctxStore loaders.Store
}
type cleanupMetrics struct {
@ -80,6 +82,7 @@ func NewController(
cmResolver engineapi.ConfigmapResolver,
jp jmespath.Interface,
eventGen event.Interface,
gctxStore loaders.Store,
) controllers.Controller {
queue := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), ControllerName)
keyFunc := controllerutils.MetaNamespaceKeyT[kyvernov2alpha1.CleanupPolicyInterface]
@ -112,6 +115,7 @@ func NewController(
eventGen: eventGen,
metrics: newCleanupMetrics(logger),
jp: jp,
gctxStore: gctxStore,
}
if _, err := controllerutils.AddEventHandlersT(
cpolInformer.Informer(),
@ -181,7 +185,7 @@ func (c *controller) cleanup(ctx context.Context, logger logr.Logger, policy kyv
var errs []error
enginectx := enginecontext.NewContext(c.jp)
ctxFactory := factories.DefaultContextLoaderFactory(c.cmResolver)
ctxFactory := factories.DefaultContextLoaderFactory(c.cmResolver, factories.WithGlobalContextStore(c.gctxStore))
loader := ctxFactory(nil, kyvernov1.Rule{})
if err := loader.Load(

View file

@ -6,17 +6,21 @@ import (
"github.com/go-logr/logr"
kyvernov2alpha1 "github.com/kyverno/kyverno/api/kyverno/v2alpha1"
"github.com/kyverno/kyverno/pkg/client/clientset/versioned"
kyvernov2alpha1informers "github.com/kyverno/kyverno/pkg/client/informers/externalversions/kyverno/v2alpha1"
kyvernov2alpha1listers "github.com/kyverno/kyverno/pkg/client/listers/kyverno/v2alpha1"
"github.com/kyverno/kyverno/pkg/clients/dclient"
"github.com/kyverno/kyverno/pkg/controllers"
"github.com/kyverno/kyverno/pkg/engine/adapters"
"github.com/kyverno/kyverno/pkg/event"
"github.com/kyverno/kyverno/pkg/globalcontext/externalapi"
"github.com/kyverno/kyverno/pkg/globalcontext/k8sresource"
"github.com/kyverno/kyverno/pkg/globalcontext/store"
controllerutils "github.com/kyverno/kyverno/pkg/utils/controller"
datautils "github.com/kyverno/kyverno/pkg/utils/data"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/tools/cache"
"k8s.io/client-go/util/workqueue"
)
@ -35,29 +39,66 @@ type controller struct {
queue workqueue.RateLimitingInterface
// state
dclient dclient.Interface
store store.Store
maxResponseLength int64
dclient dclient.Interface
kyvernoClient versioned.Interface
store store.Store
eventGen event.Interface
maxResponseLength int64
shouldUpdateStatus bool
}
func NewController(
gceInformer kyvernov2alpha1informers.GlobalContextEntryInformer,
dclient dclient.Interface,
kyvernoClient versioned.Interface,
storage store.Store,
eventGen event.Interface,
maxResponseLength int64,
shouldUpdateStatus bool,
) controllers.Controller {
queue := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), ControllerName)
_, _, err := controllerutils.AddDefaultEventHandlers(logger, gceInformer.Informer(), queue)
if err != nil {
c := &controller{
gceLister: gceInformer.Lister(),
queue: queue,
dclient: dclient,
kyvernoClient: kyvernoClient,
store: storage,
eventGen: eventGen,
maxResponseLength: maxResponseLength,
shouldUpdateStatus: shouldUpdateStatus,
}
if _, err := controllerutils.AddEventHandlersT(gceInformer.Informer(), c.addGTXEntry, c.updateGTXEntry, c.deleteGTXEntry); err != nil {
logger.Error(err, "failed to register event handlers")
}
return &controller{
gceLister: gceInformer.Lister(),
queue: queue,
dclient: dclient,
store: storage,
maxResponseLength: maxResponseLength,
return c
}
func (c *controller) addGTXEntry(obj *kyvernov2alpha1.GlobalContextEntry) {
logger.Info("globalcontextentry created", "uid", obj.GetUID(), "kind", obj.Kind, "name", obj.GetName())
c.enqueueGCTXEntry(obj)
}
func (c *controller) updateGTXEntry(old, obj *kyvernov2alpha1.GlobalContextEntry) {
if datautils.DeepEqual(old.Spec, obj.Spec) {
return
}
logger.Info("globalcontextentry updated", "uid", obj.GetUID(), "kind", obj.Kind, "name", obj.GetName())
c.enqueueGCTXEntry(obj)
}
func (c *controller) deleteGTXEntry(obj *kyvernov2alpha1.GlobalContextEntry) {
c.enqueueGCTXEntry(obj)
}
func (c *controller) enqueueGCTXEntry(gctxentry *kyvernov2alpha1.GlobalContextEntry) {
key, err := cache.MetaNamespaceKeyFunc(gctxentry)
if err != nil {
logger.Error(err, "failed to enqueue global context entry")
return
}
c.queue.Add(key)
}
func (c *controller) Run(ctx context.Context, workers int) {
@ -95,14 +136,29 @@ func (c *controller) makeStoreEntry(ctx context.Context, gce *kyvernov2alpha1.Gl
Version: gce.Spec.KubernetesResource.Version,
Resource: gce.Spec.KubernetesResource.Resource,
}
return k8sresource.New(ctx, c.dclient.GetDynamicInterface(), gvr, gce.Spec.KubernetesResource.Namespace)
return k8sresource.New(
ctx,
gce,
c.eventGen,
c.dclient.GetDynamicInterface(),
c.kyvernoClient,
logger,
gvr,
gce.Spec.KubernetesResource.Namespace,
c.shouldUpdateStatus,
)
}
return externalapi.New(
ctx,
gce,
c.eventGen,
c.kyvernoClient,
c.gceLister,
logger,
adapters.Client(c.dclient),
gce.Spec.APICall.APICall,
gce.Spec.APICall.RefreshInterval.Duration,
c.maxResponseLength,
c.shouldUpdateStatus,
)
}

View file

@ -426,7 +426,7 @@ func (c *controller) reconcileMutatingWebhookConfiguration(ctx context.Context,
func (c *controller) isGlobalContextEntryReady(name string, gctxentries []*kyvernov2alpha1.GlobalContextEntry) bool {
for _, gctxentry := range gctxentries {
if gctxentry.Name == name {
return true
return gctxentry.Status.IsReady()
}
}
return false

View file

@ -135,7 +135,9 @@ func (gen *controller) processNextWorkItem(ctx context.Context) bool {
func (gen *controller) emitEvent(key Info) {
logger := gen.logger
eventType := corev1.EventTypeWarning
if key.Reason == PolicyApplied || key.Reason == PolicySkipped {
if key.Type != "" {
eventType = key.Type
} else if key.Reason == PolicyApplied || key.Reason == PolicySkipped {
eventType = corev1.EventTypeNormal
}

View file

@ -14,6 +14,7 @@ type Info struct {
Message string
Action Action
Source Source
Type string
}
func (i *Info) Resource() string {

View file

@ -0,0 +1,22 @@
package event
import (
"github.com/kyverno/kyverno/pkg/event"
corev1 "k8s.io/api/core/v1"
)
const (
source = "globalcontext-controller"
action = "Retrying"
)
func NewErrorEvent(regarding corev1.ObjectReference, reason event.Reason, err error) event.Info {
return event.Info{
Regarding: regarding,
Source: source,
Reason: reason,
Message: err.Error(),
Action: action,
Type: corev1.EventTypeWarning,
}
}

View file

@ -0,0 +1,7 @@
package event
const (
ReasonResourceListFailure = "FailedToList"
ReasonAPICallFailure = "FailedToCallAPI"
ReasonCacheSyncFailure = "FailedToWaitForCacheSync"
)

View file

@ -8,8 +8,18 @@ import (
"github.com/go-logr/logr"
kyvernov1 "github.com/kyverno/kyverno/api/kyverno/v1"
kyvernov2alpha1 "github.com/kyverno/kyverno/api/kyverno/v2alpha1"
"github.com/kyverno/kyverno/pkg/client/clientset/versioned"
kyvernov2alpha1listers "github.com/kyverno/kyverno/pkg/client/listers/kyverno/v2alpha1"
"github.com/kyverno/kyverno/pkg/engine/apicall"
"github.com/kyverno/kyverno/pkg/event"
entryevent "github.com/kyverno/kyverno/pkg/globalcontext/event"
"github.com/kyverno/kyverno/pkg/globalcontext/store"
controllerutils "github.com/kyverno/kyverno/pkg/utils/controller"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/util/retry"
)
type entry struct {
@ -21,12 +31,17 @@ type entry struct {
func New(
ctx context.Context,
gce *kyvernov2alpha1.GlobalContextEntry,
eventGen event.Interface,
kyvernoClient versioned.Interface,
gceLister kyvernov2alpha1listers.GlobalContextEntryLister,
logger logr.Logger,
client apicall.ClientInterface,
call kyvernov1.APICall,
period time.Duration,
maxResponseLength int64,
) (*entry, error) {
shouldUpdateStatus bool,
) (store.Entry, error) {
var group wait.Group
ctx, cancel := context.WithCancel(ctx)
stop := func() {
@ -38,19 +53,42 @@ func New(
e := &entry{
stop: stop,
}
group.StartWithContext(ctx, func(ctx context.Context) {
// TODO: make sure we have called it at least once before returning
config := apicall.NewAPICallConfiguration(maxResponseLength)
caller := apicall.NewCaller(logger, "globalcontext", client, config)
wait.UntilWithContext(ctx, func(ctx context.Context) {
if data, err := doCall(ctx, caller, call); err != nil {
logger.Error(err, "failed to get data from api caller")
e.setData(nil, err)
logger.Error(err, "failed to get data from api caller")
eventGen.Add(entryevent.NewErrorEvent(corev1.ObjectReference{
APIVersion: gce.APIVersion,
Kind: gce.Kind,
Name: gce.Name,
Namespace: gce.Namespace,
UID: gce.UID,
}, entryevent.ReasonAPICallFailure, err))
if shouldUpdateStatus {
if updateErr := updateStatus(ctx, gce.Name, kyvernoClient, false, entryevent.ReasonAPICallFailure); updateErr != nil {
logger.Error(updateErr, "failed to update status")
}
}
} else {
e.setData(data, nil)
if shouldUpdateStatus {
if updateErr := updateStatus(ctx, gce.Name, kyvernoClient, true, "APICallSuccess"); updateErr != nil {
logger.Error(updateErr, "failed to update status")
}
}
}
}, period)
})
return e, nil
}
@ -89,3 +127,24 @@ func (e *entry) setData(data any, err error) {
func doCall(ctx context.Context, caller apicall.Caller, call kyvernov1.APICall) (any, error) {
return caller.Execute(ctx, &call)
}
func updateStatus(ctx context.Context, gceName string, kyvernoClient versioned.Interface, ready bool, reason string) error {
retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error {
latestGCE, getErr := kyvernoClient.KyvernoV2alpha1().GlobalContextEntries().Get(ctx, gceName, metav1.GetOptions{})
if getErr != nil {
return getErr
}
_, updateErr := controllerutils.UpdateStatus(ctx, latestGCE, kyvernoClient.KyvernoV2alpha1().GlobalContextEntries(), func(latest *kyvernov2alpha1.GlobalContextEntry) error {
if latest == nil {
return fmt.Errorf("failed to update status: %s", latestGCE.Name)
}
latest.Status.SetReady(ready, reason)
return nil
})
return updateErr
})
return retryErr
}

View file

@ -4,6 +4,15 @@ import (
"context"
"fmt"
"github.com/go-logr/logr"
kyvernov2alpha1 "github.com/kyverno/kyverno/api/kyverno/v2alpha1"
"github.com/kyverno/kyverno/pkg/client/clientset/versioned"
"github.com/kyverno/kyverno/pkg/event"
entryevent "github.com/kyverno/kyverno/pkg/globalcontext/event"
"github.com/kyverno/kyverno/pkg/globalcontext/invalid"
"github.com/kyverno/kyverno/pkg/globalcontext/store"
controllerutils "github.com/kyverno/kyverno/pkg/utils/controller"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime/schema"
@ -14,11 +23,24 @@ import (
)
type entry struct {
lister cache.GenericLister
stop func()
lister cache.GenericLister
stop func()
gce *kyvernov2alpha1.GlobalContextEntry
eventGen event.Interface
}
func New(ctx context.Context, client dynamic.Interface, gvr schema.GroupVersionResource, namespace string) (*entry, error) {
// TODO: Handle Kyverno Pod Ready State
func New(
ctx context.Context,
gce *kyvernov2alpha1.GlobalContextEntry,
eventGen event.Interface,
client dynamic.Interface,
kyvernoClient versioned.Interface,
logger logr.Logger,
gvr schema.GroupVersionResource,
namespace string,
shouldUpdateStatus bool,
) (store.Entry, error) {
indexers := cache.Indexers{
cache.NamespaceIndex: cache.MetaNamespaceIndexFunc,
}
@ -39,17 +61,48 @@ func New(ctx context.Context, client dynamic.Interface, gvr schema.GroupVersionR
})
if !cache.WaitForCacheSync(ctx.Done(), informer.Informer().HasSynced) {
stop()
return nil, fmt.Errorf("failed to wait for cache sync: %s", gvr.Resource)
if shouldUpdateStatus {
if err := updateStatus(ctx, gce, kyvernoClient, false, "CacheSyncFailure"); err != nil {
logger.Error(err, "failed to update status")
}
}
err := fmt.Errorf("failed to sync cache for %s", gvr)
eventGen.Add(entryevent.NewErrorEvent(corev1.ObjectReference{
APIVersion: gce.APIVersion,
Kind: gce.Kind,
Name: gce.Name,
Namespace: gce.Namespace,
UID: gce.UID,
}, entryevent.ReasonCacheSyncFailure, err))
return invalid.New(err), nil
}
if shouldUpdateStatus {
if err := updateStatus(ctx, gce, kyvernoClient, true, "CacheSyncSuccess"); err != nil {
logger.Error(err, "failed to update status")
}
}
return &entry{
lister: informer.Lister(),
stop: stop,
lister: informer.Lister(),
stop: stop,
eventGen: eventGen,
}, nil
}
func (e *entry) Get() (any, error) {
obj, err := e.lister.List(labels.Everything())
if err != nil {
e.eventGen.Add(entryevent.NewErrorEvent(corev1.ObjectReference{
APIVersion: e.gce.APIVersion,
Kind: e.gce.Kind,
Name: e.gce.Name,
Namespace: e.gce.Namespace,
UID: e.gce.UID,
}, entryevent.ReasonResourceListFailure, err))
return nil, err
}
return obj, nil
@ -58,3 +111,14 @@ func (e *entry) Get() (any, error) {
func (e *entry) Stop() {
e.stop()
}
func updateStatus(ctx context.Context, gce *kyvernov2alpha1.GlobalContextEntry, kyvernoClient versioned.Interface, ready bool, reason string) error {
_, err := controllerutils.UpdateStatus(ctx, gce, kyvernoClient.KyvernoV2alpha1().GlobalContextEntries(), func(latest *kyvernov2alpha1.GlobalContextEntry) error {
if latest == nil {
return fmt.Errorf("failed to update status: %s", gce.Name)
}
latest.Status.SetReady(ready, reason)
return nil
})
return err
}

View file

@ -1,6 +1,7 @@
package policy
import (
"context"
"encoding/json"
"errors"
"fmt"
@ -19,7 +20,7 @@ import (
kyvernov2alpha1 "github.com/kyverno/kyverno/api/kyverno/v2alpha1"
"github.com/kyverno/kyverno/ext/wildcard"
"github.com/kyverno/kyverno/pkg/autogen"
kyvernov2alpha1listers "github.com/kyverno/kyverno/pkg/client/listers/kyverno/v2alpha1"
"github.com/kyverno/kyverno/pkg/client/clientset/versioned"
"github.com/kyverno/kyverno/pkg/clients/dclient"
enginecontext "github.com/kyverno/kyverno/pkg/engine/context"
"github.com/kyverno/kyverno/pkg/engine/variables"
@ -33,7 +34,6 @@ import (
admissionregistrationv1alpha1 "k8s.io/api/admissionregistration/v1alpha1"
"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/validation/field"
"k8s.io/apimachinery/pkg/util/yaml"
@ -128,7 +128,7 @@ func checkValidationFailureAction(spec *kyvernov1.Spec) []string {
}
// Validate checks the policy and rules declarations for required configurations
func Validate(policy, oldPolicy kyvernov1.PolicyInterface, client dclient.Interface, gctxentryLister kyvernov2alpha1listers.GlobalContextEntryLister, mock bool, username string) ([]string, error) {
func Validate(policy, oldPolicy kyvernov1.PolicyInterface, client dclient.Interface, kyvernoClient versioned.Interface, mock bool, username string) ([]string, error) {
var warnings []string
spec := policy.GetSpec()
background := spec.BackgroundProcessingEnabled()
@ -404,8 +404,8 @@ func Validate(policy, oldPolicy kyvernov1.PolicyInterface, client dclient.Interf
}
// global context entry validation
if gctxentryLister != nil {
gctxentries, err := gctxentryLister.List(labels.Everything())
if kyvernoClient != nil {
gctxentries, err := kyvernoClient.KyvernoV2alpha1().GlobalContextEntries().List(context.Background(), metav1.ListOptions{})
if err != nil {
return nil, err
}
@ -416,7 +416,7 @@ func Validate(policy, oldPolicy kyvernov1.PolicyInterface, client dclient.Interf
for _, ctxEntry := range rule.Context {
if ctxEntry.GlobalReference != nil {
if !isGlobalContextEntryReady(ctxEntry.GlobalReference.Name, gctxentries) {
return nil, fmt.Errorf("global context entry %s is not ready", ctxEntry.Name)
return nil, fmt.Errorf("global context entry %s is not ready", ctxEntry.GlobalReference.Name)
}
}
}
@ -473,10 +473,10 @@ func Validate(policy, oldPolicy kyvernov1.PolicyInterface, client dclient.Interf
return warnings, nil
}
func isGlobalContextEntryReady(name string, gctxentries []*kyvernov2alpha1.GlobalContextEntry) bool {
for _, gctxentry := range gctxentries {
func isGlobalContextEntryReady(name string, gctxentries *kyvernov2alpha1.GlobalContextEntryList) bool {
for _, gctxentry := range gctxentries.Items {
if gctxentry.Name == name {
return true
return gctxentry.Status.IsReady()
}
}
return false

View file

@ -5,8 +5,7 @@ import (
"time"
"github.com/go-logr/logr"
kyvernov2alpha1informers "github.com/kyverno/kyverno/pkg/client/informers/externalversions/kyverno/v2alpha1"
kyvernov2alpha1listers "github.com/kyverno/kyverno/pkg/client/listers/kyverno/v2alpha1"
"github.com/kyverno/kyverno/pkg/client/clientset/versioned"
"github.com/kyverno/kyverno/pkg/clients/dclient"
admissionutils "github.com/kyverno/kyverno/pkg/utils/admission"
policyvalidate "github.com/kyverno/kyverno/pkg/validation/policy"
@ -16,14 +15,14 @@ import (
type policyHandlers struct {
client dclient.Interface
gctxentryLister kyvernov2alpha1listers.GlobalContextEntryLister
kyvernoClient versioned.Interface
backgroundServiceAccountName string
}
func NewHandlers(client dclient.Interface, gctxentryInformer kyvernov2alpha1informers.GlobalContextEntryInformer, serviceaccount string) webhooks.PolicyHandlers {
func NewHandlers(client dclient.Interface, kyvernoClient versioned.Interface, serviceaccount string) webhooks.PolicyHandlers {
return &policyHandlers{
client: client,
gctxentryLister: gctxentryInformer.Lister(),
kyvernoClient: kyvernoClient,
backgroundServiceAccountName: serviceaccount,
}
}
@ -34,7 +33,7 @@ func (h *policyHandlers) Validate(ctx context.Context, logger logr.Logger, reque
logger.Error(err, "failed to unmarshal policies from admission request")
return admissionutils.Response(request.UID, err)
}
warnings, err := policyvalidate.Validate(policy, oldPolicy, h.client, h.gctxentryLister, false, h.backgroundServiceAccountName)
warnings, err := policyvalidate.Validate(policy, oldPolicy, h.client, h.kyvernoClient, false, h.backgroundServiceAccountName)
if err != nil {
logger.Error(err, "policy validation errors")
}

View file

@ -2,7 +2,7 @@ apiVersion: chainsaw.kyverno.io/v1alpha1
kind: Test
metadata:
creationTimestamp: null
name: resource-correct
name: apicall-correct
spec:
steps:
- name: scenario
@ -13,11 +13,13 @@ spec:
file: main-deployment.yaml
- apply:
file: gctxentry.yaml
- sleep:
duration: 15s
- apply:
file: clusterpolicy.yaml
- assert:
file: clusterpolicy-ready.yaml
- apply:
file: new-deployment.yaml
- assert:
file: clusterpolicy-succeeded.yaml
- assert:
file: new-deployment-exists.yaml

View file

@ -1,7 +1,7 @@
apiVersion: kyverno.io/v1
kind: ClusterPolicy
metadata:
name: namespace-has-coordinator
name: cpol-apicall-correct
status:
conditions:
- reason: Succeeded

View file

@ -1,7 +1,7 @@
apiVersion: kyverno.io/v1
kind: ClusterPolicy
metadata:
name: namespace-has-coordinator
name: cpol-apicall-correct
spec:
validationFailureAction: Enforce
failurePolicy: Fail
@ -10,7 +10,7 @@ spec:
context:
- name: deploymentCount
globalReference:
name: deployments
name: gctxentry-apicall-correct
jmesPath: "items | length(@)"
match:
all:

View file

@ -1,8 +1,8 @@
apiVersion: kyverno.io/v2alpha1
kind: GlobalContextEntry
metadata:
name: deployments
name: gctxentry-apicall-correct
spec:
apiCall:
urlPath: "/apis/apps/v1/namespaces/test-globalcontext/deployments"
refreshInterval: 10s
urlPath: "/apis/apps/v1/namespaces/test-globalcontext-apicall-correct/deployments"
refreshInterval: 1h

View file

@ -2,7 +2,7 @@ apiVersion: apps/v1
kind: Deployment
metadata:
name: main-deployment
namespace: test-globalcontext
namespace: test-globalcontext-apicall-correct
labels:
app: main-deployment
spec:

View file

@ -1,4 +1,4 @@
apiVersion: v1
kind: Namespace
metadata:
name: test-globalcontext
name: test-globalcontext-apicall-correct

View file

@ -2,6 +2,6 @@ apiVersion: apps/v1
kind: Deployment
metadata:
name: new-deployment
namespace: test-globalcontext
namespace: test-globalcontext-apicall-correct
labels:
app: new-deployment

View file

@ -2,7 +2,7 @@ apiVersion: apps/v1
kind: Deployment
metadata:
name: new-deployment
namespace: test-globalcontext
namespace: test-globalcontext-apicall-correct
labels:
app: new-deployment
spec:

View file

@ -0,0 +1,11 @@
## Description
This test verifies that Global Context Entries are evaluated correctly.
## Expected Behavior
`new-deployment` should be created.
## Reference Issues

View file

@ -0,0 +1,13 @@
apiVersion: chainsaw.kyverno.io/v1alpha1
kind: Test
metadata:
creationTimestamp: null
name: apicall-failed
spec:
steps:
- name: scenario
try:
- apply:
file: gctxentry.yaml
- assert:
file: gctxentry-not-ready.yaml

View file

@ -0,0 +1,8 @@
apiVersion: kyverno.io/v2alpha1
kind: GlobalContextEntry
metadata:
name: gctxentry-apicall-failed
status:
conditions:
- status: "False"
type: Ready

View file

@ -0,0 +1,8 @@
apiVersion: kyverno.io/v2alpha1
kind: GlobalContextEntry
metadata:
name: gctxentry-apicall-failed
spec:
apiCall:
urlPath: "/apis/apps/v1/namespaces/default/unknown"
refreshInterval: 10s

View file

@ -1,10 +1,10 @@
## Description
This test verifies that policies are not ready if referenced Global Context Entries don't exist.
This test verifies that policies cannot be created if referenced Global Context Entries don't exist.
## Expected Behavior
`new-deployment` should not be created because the policy is not ready.
The clusterpolicy ` cpol-gctxentry-not-exist` should not be created because the globalcontextentry does not exist.
## Reference Issues

View file

@ -11,8 +11,6 @@ spec:
file: namespace.yaml
- apply:
file: main-deployment.yaml
- apply:
file: gctxentry.yaml
- name: negative
try:
- apply:

View file

@ -1,7 +1,7 @@
apiVersion: kyverno.io/v1
kind: ClusterPolicy
metadata:
name: namespace-has-coordinator
name: cpol-gctxentry-not-exist
spec:
validationFailureAction: Enforce
failurePolicy: Fail

View file

@ -1,10 +1,10 @@
apiVersion: kyverno.io/v2alpha1
kind: GlobalContextEntry
metadata:
name: deployments
name: non-existent-gctx
spec:
kubernetesResource:
group: apps
version: v1
resource: deployments
namespace: test-globalcontext
namespace: test-globalcontext-gctxentry-not-exist

View file

@ -2,7 +2,7 @@ apiVersion: apps/v1
kind: Deployment
metadata:
name: main-deployment
namespace: test-globalcontext
namespace: test-globalcontext-gctxentry-not-exist
labels:
app: main-deployment
spec:

View file

@ -1,4 +1,4 @@
apiVersion: v1
kind: Namespace
metadata:
name: test-globalcontext
name: test-globalcontext-gctxentry-not-exist

View file

@ -1,22 +0,0 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: new-deployment
namespace: test-globalcontext
labels:
app: new-deployment
spec:
replicas: 1
selector:
matchLabels:
app: new-deployment
template:
metadata:
labels:
app: new-deployment
spec:
containers:
- name: nginx
image: nginx:1.14.2
ports:
- containerPort: 80

View file

@ -1,6 +1,6 @@
## Description
This test verifies that policies are not ready if referenced Global Context Entries don't exist.
This test verifies that the policy becomes not ready if the referenced Global Context Entries don't exist anymore.
## Expected Behavior

View file

@ -13,15 +13,19 @@ spec:
file: main-deployment.yaml
- apply:
file: gctxentry.yaml
- assert:
file: gctxentry-exists.yaml
- sleep:
duration: 15s
- apply:
file: clusterpolicy.yaml
- assert:
file: clusterpolicy-ready.yaml
- delete:
ref:
apiVersion: kyverno.io/v2alpha1
kind: GlobalContextEntry
name: deployments
name: gctx-not-ready
- sleep:
duration: 5s
- assert:
file: clusterpolicy-failed.yaml
- apply:

View file

@ -1,7 +1,7 @@
apiVersion: kyverno.io/v1
kind: ClusterPolicy
metadata:
name: namespace-has-coordinator
name: cpol-not-ready
status:
conditions:
- reason: Failed

View file

@ -1,7 +1,7 @@
apiVersion: kyverno.io/v1
kind: ClusterPolicy
metadata:
name: namespace-has-coordinator
name: cpol-not-ready
status:
conditions:
- reason: Succeeded

View file

@ -1,7 +1,7 @@
apiVersion: kyverno.io/v1
kind: ClusterPolicy
metadata:
name: namespace-has-coordinator
name: cpol-not-ready
spec:
validationFailureAction: Enforce
failurePolicy: Fail
@ -10,7 +10,7 @@ spec:
context:
- name: deploymentCount
globalReference:
name: deployments
name: gctx-not-ready
jmesPath: "items | length(@)"
match:
all:

View file

@ -1,4 +0,0 @@
apiVersion: kyverno.io/v2alpha1
kind: GlobalContextEntry
metadata:
name: deployments

View file

@ -1,8 +1,8 @@
apiVersion: kyverno.io/v2alpha1
kind: GlobalContextEntry
metadata:
name: deployments
name: gctx-not-ready
spec:
apiCall:
urlPath: "/apis/apps/v1/namespaces/test-globalcontext/deployments"
urlPath: "/apis/apps/v1/namespaces/test-globalcontext-not-ready/deployments"
refreshInterval: 10s

View file

@ -2,7 +2,7 @@ apiVersion: apps/v1
kind: Deployment
metadata:
name: main-deployment
namespace: test-globalcontext
namespace: test-globalcontext-not-ready
labels:
app: main-deployment
spec:

View file

@ -1,4 +1,4 @@
apiVersion: v1
kind: Namespace
metadata:
name: test-globalcontext
name: test-globalcontext-not-ready

View file

@ -2,6 +2,6 @@ apiVersion: apps/v1
kind: Deployment
metadata:
name: new-deployment
namespace: test-globalcontext
namespace: test-globalcontext-not-ready
labels:
app: new-deployment

View file

@ -2,7 +2,7 @@ apiVersion: apps/v1
kind: Deployment
metadata:
name: new-deployment
namespace: test-globalcontext
namespace: test-globalcontext-not-ready
labels:
app: new-deployment
spec:

View file

@ -13,11 +13,13 @@ spec:
file: main-deployment.yaml
- apply:
file: gctxentry.yaml
- sleep:
duration: 5s
- apply:
file: clusterpolicy.yaml
- assert:
file: clusterpolicy-ready.yaml
- apply:
file: new-deployment.yaml
- assert:
file: clusterpolicy-succeeded.yaml
- assert:
file: new-deployment-exists.yaml

View file

@ -0,0 +1,9 @@
apiVersion: kyverno.io/v1
kind: ClusterPolicy
metadata:
name: cpol-resource-correct
status:
conditions:
- reason: Succeeded
status: "True"
type: Ready

View file

@ -1,7 +1,7 @@
apiVersion: kyverno.io/v1
kind: ClusterPolicy
metadata:
name: namespace-has-coordinator
name: cpol-resource-correct
spec:
validationFailureAction: Enforce
failurePolicy: Fail
@ -10,7 +10,7 @@ spec:
context:
- name: deploymentCount
globalReference:
name: deployments
name: gctxentry-resource-correct
jmesPath: "length(@)"
match:
all:

View file

@ -1,10 +1,10 @@
apiVersion: kyverno.io/v2alpha1
kind: GlobalContextEntry
metadata:
name: deployments
name: gctxentry-resource-correct
spec:
kubernetesResource:
group: apps
version: v1
resource: deployments
namespace: test-globalcontext
namespace: test-globalcontext-resource-correct

View file

@ -2,7 +2,7 @@ apiVersion: apps/v1
kind: Deployment
metadata:
name: main-deployment
namespace: test-globalcontext
namespace: test-globalcontext-resource-correct
labels:
app: main-deployment
spec:

View file

@ -1,4 +1,4 @@
apiVersion: v1
kind: Namespace
metadata:
name: test-globalcontext
name: test-globalcontext-resource-correct

View file

@ -2,6 +2,6 @@ apiVersion: apps/v1
kind: Deployment
metadata:
name: new-deployment
namespace: test-globalcontext
namespace: test-globalcontext-resource-correct
labels:
app: new-deployment

View file

@ -2,7 +2,7 @@ apiVersion: apps/v1
kind: Deployment
metadata:
name: new-deployment
namespace: test-globalcontext
namespace: test-globalcontext-resource-correct
labels:
app: new-deployment
spec: