From 9dc001e7584b1bd06a6a03ade781240b28f3ab3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Charles-Edouard=20Br=C3=A9t=C3=A9ch=C3=A9?= Date: Fri, 9 Dec 2022 11:24:04 +0100 Subject: [PATCH] feat: add conditions matching to cleanup controller (#5626) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Charles-Edouard Brétéché Signed-off-by: Charles-Edouard Brétéché Co-authored-by: shuting --- api/kyverno/v2beta1/common_types.go | 17 ++++++ .../cleanup-controller/clusterrole.yaml | 8 +++ .../handlers/cleanup/condition.go | 45 +++++++++++++++ .../handlers/cleanup/condition_test.go | 57 +++++++++++++++++++ .../handlers/cleanup/handlers.go | 56 ++++++++++++++---- pkg/engine/variables/vars.go | 13 ++--- 6 files changed, 178 insertions(+), 18 deletions(-) create mode 100644 cmd/cleanup-controller/handlers/cleanup/condition.go create mode 100644 cmd/cleanup-controller/handlers/cleanup/condition_test.go diff --git a/api/kyverno/v2beta1/common_types.go b/api/kyverno/v2beta1/common_types.go index 1b48eea3f2..ea2a5e9fb8 100644 --- a/api/kyverno/v2beta1/common_types.go +++ b/api/kyverno/v2beta1/common_types.go @@ -2,6 +2,7 @@ package v2beta1 import ( kyvernov1 "github.com/kyverno/kyverno/api/kyverno/v1" + "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" ) @@ -98,6 +99,22 @@ type Condition struct { RawValue *apiextv1.JSON `json:"value,omitempty" yaml:"value,omitempty"` } +func (c *Condition) GetKey() apiextensions.JSON { + return kyvernov1.FromJSON(c.RawKey) +} + +func (c *Condition) SetKey(in apiextensions.JSON) { + c.RawKey = kyvernov1.ToJSON(in) +} + +func (c *Condition) GetValue() apiextensions.JSON { + return kyvernov1.FromJSON(c.RawValue) +} + +func (c *Condition) SetValue(in apiextensions.JSON) { + c.RawValue = kyvernov1.ToJSON(in) +} + type AnyAllConditions struct { // AnyConditions enable variable-based conditional rule execution. This is useful for // finer control of when an rule is applied. A condition can reference object data diff --git a/charts/kyverno/templates/cleanup-controller/clusterrole.yaml b/charts/kyverno/templates/cleanup-controller/clusterrole.yaml index e36bf93c61..4e4d8a1dfe 100644 --- a/charts/kyverno/templates/cleanup-controller/clusterrole.yaml +++ b/charts/kyverno/templates/cleanup-controller/clusterrole.yaml @@ -7,6 +7,14 @@ metadata: labels: {{- include "kyverno.cleanup-controller.labels" . | nindent 4 }} rules: + - apiGroups: + - '' + resources: + - namespaces + verbs: + - get + - list + - watch - apiGroups: - kyverno.io resources: diff --git a/cmd/cleanup-controller/handlers/cleanup/condition.go b/cmd/cleanup-controller/handlers/cleanup/condition.go new file mode 100644 index 0000000000..64bf20cae8 --- /dev/null +++ b/cmd/cleanup-controller/handlers/cleanup/condition.go @@ -0,0 +1,45 @@ +package cleanup + +import ( + "github.com/go-logr/logr" + kyvernov1 "github.com/kyverno/kyverno/api/kyverno/v1" + kyvernov2beta1 "github.com/kyverno/kyverno/api/kyverno/v2beta1" + enginecontext "github.com/kyverno/kyverno/pkg/engine/context" + "github.com/kyverno/kyverno/pkg/engine/variables" + "github.com/kyverno/kyverno/pkg/engine/variables/operator" + "github.com/pkg/errors" +) + +func checkAnyAllConditions(logger logr.Logger, ctx enginecontext.Interface, condition kyvernov2beta1.AnyAllConditions) (bool, error) { + for _, condition := range condition.AllConditions { + if passed, err := checkCondition(logger, ctx, condition); err != nil { + return false, err + } else if !passed { + return false, nil + } + } + for _, condition := range condition.AnyConditions { + if passed, err := checkCondition(logger, ctx, condition); err != nil { + return false, err + } else if passed { + return true, nil + } + } + return true, nil +} + +func checkCondition(logger logr.Logger, ctx enginecontext.Interface, condition kyvernov2beta1.Condition) (bool, error) { + key, err := variables.SubstituteAllInPreconditions(logger, ctx, condition.GetKey()) + if err != nil { + return false, errors.Wrapf(err, "failed to substitute variables in condition key") + } + value, err := variables.SubstituteAllInPreconditions(logger, ctx, condition.GetValue()) + if err != nil { + return false, errors.Wrapf(err, "failed to substitute variables in condition value") + } + handler := operator.CreateOperatorHandler(logger, ctx, kyvernov1.ConditionOperator(condition.Operator)) + if handler == nil { + return false, errors.Wrapf(err, "failed to create handler for condition operator") + } + return handler.Evaluate(key, value), nil +} diff --git a/cmd/cleanup-controller/handlers/cleanup/condition_test.go b/cmd/cleanup-controller/handlers/cleanup/condition_test.go new file mode 100644 index 0000000000..aab19e7b27 --- /dev/null +++ b/cmd/cleanup-controller/handlers/cleanup/condition_test.go @@ -0,0 +1,57 @@ +package cleanup + +import ( + "testing" + + "github.com/go-logr/logr" + kyvernov2beta1 "github.com/kyverno/kyverno/api/kyverno/v2beta1" + enginecontext "github.com/kyverno/kyverno/pkg/engine/context" + "github.com/kyverno/kyverno/pkg/logging" + v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" +) + +func Test_checkCondition(t *testing.T) { + ctx := enginecontext.NewContext() + ctx.AddResource(map[string]interface{}{ + "name": "dummy", + }) + type args struct { + logger logr.Logger + ctx enginecontext.Interface + condition kyvernov2beta1.Condition + } + tests := []struct { + name string + args args + want bool + wantErr bool + }{{ + name: "basic", + args: args{ + logger: logging.GlobalLogger(), + ctx: ctx, + condition: kyvernov2beta1.Condition{ + RawKey: &v1.JSON{ + Raw: []byte(`"{{ request.object.name }}"`), + }, + Operator: kyvernov2beta1.ConditionOperators["Equals"], + RawValue: &v1.JSON{ + Raw: []byte(`"dummy"`), + }, + }, + }, + want: true, + }} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := checkCondition(tt.args.logger, tt.args.ctx, tt.args.condition) + if (err != nil) != tt.wantErr { + t.Errorf("checkCondition() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("checkCondition() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/cmd/cleanup-controller/handlers/cleanup/handlers.go b/cmd/cleanup-controller/handlers/cleanup/handlers.go index 10cc0d25e4..c6330e2330 100644 --- a/cmd/cleanup-controller/handlers/cleanup/handlers.go +++ b/cmd/cleanup-controller/handlers/cleanup/handlers.go @@ -8,6 +8,7 @@ import ( kyvernov2alpha1 "github.com/kyverno/kyverno/api/kyverno/v2alpha1" kyvernov2alpha1listers "github.com/kyverno/kyverno/pkg/client/listers/kyverno/v2alpha1" "github.com/kyverno/kyverno/pkg/clients/dclient" + enginecontext "github.com/kyverno/kyverno/pkg/engine/context" controllerutils "github.com/kyverno/kyverno/pkg/utils/controller" "go.uber.org/multierr" "k8s.io/apimachinery/pkg/util/sets" @@ -38,6 +39,7 @@ func New( func (h *handlers) Cleanup(ctx context.Context, logger logr.Logger, name string, _ time.Time) error { logger.Info("cleaning up...") + defer logger.Info("done") namespace, name, err := cache.SplitMetaNamespaceKey(name) if err != nil { return err @@ -60,53 +62,85 @@ func (h *handlers) lookupPolicy(namespace, name string) (kyvernov2alpha1.Cleanup func (h *handlers) executePolicy(ctx context.Context, logger logr.Logger, policy kyvernov2alpha1.CleanupPolicyInterface) error { spec := policy.GetSpec() kinds := sets.NewString(spec.MatchResources.GetKinds()...) + debug := logger.V(5) var errs []error for kind := range kinds { - logger := logger.WithValues("kind", kind) - logger.V(5).Info("processing...") + debug := debug.WithValues("kind", kind) + debug.Info("processing...") list, err := h.client.ListResource(ctx, "", kind, policy.GetNamespace(), nil) if err != nil { - logger.Error(err, "failed to list resources") + debug.Error(err, "failed to list resources") errs = append(errs, err) } else { for i := range list.Items { resource := list.Items[i] namespace := resource.GetNamespace() name := resource.GetName() - logger := logger.WithValues("name", name, "namespace", namespace) + debug := debug.WithValues("name", name, "namespace", namespace) if !controllerutils.IsManagedByKyverno(&resource) { var nsLabels map[string]string if namespace != "" { ns, err := h.nsLister.Get(namespace) if err != nil { - logger.Error(err, "failed to get namespace labels") + debug.Error(err, "failed to get namespace labels") errs = append(errs, err) } nsLabels = ns.GetLabels() } // match namespaces if err := checkNamespace(policy.GetNamespace(), resource); err != nil { - logger.V(5).Info("resource namespace didn't match policy namespace", "result", err) + debug.Info("resource namespace didn't match policy namespace", "result", err) } // match resource with match/exclude clause matched := checkMatchesResources(resource, spec.MatchResources, nsLabels) if matched != nil { - logger.V(5).Info("resource/match didn't match", "result", matched) + debug.Info("resource/match didn't match", "result", matched) continue } if spec.ExcludeResources != nil { excluded := checkMatchesResources(resource, *spec.ExcludeResources, nsLabels) if excluded == nil { - logger.V(5).Info("resource/exclude matched") + debug.Info("resource/exclude matched") continue } else { - logger.V(5).Info("resource/exclude didn't match", "result", excluded) + debug.Info("resource/exclude didn't match", "result", excluded) } } - logger.V(5).Info("resource matched, it will be deleted...") + // check conditions + if spec.Conditions != nil { + enginectx := enginecontext.NewContext() + if err := enginectx.AddResource(resource.Object); err != nil { + debug.Error(err, "failed to add resource in context") + errs = append(errs, err) + continue + } + if err := enginectx.AddNamespace(resource.GetNamespace()); err != nil { + debug.Error(err, "failed to add namespace in context") + errs = append(errs, err) + continue + } + if err := enginectx.AddImageInfos(&resource); err != nil { + debug.Error(err, "failed to add image infos in context") + errs = append(errs, err) + continue + } + passed, err := checkAnyAllConditions(logger, enginectx, *spec.Conditions) + if err != nil { + debug.Error(err, "failed to check condition") + errs = append(errs, err) + continue + } + if !passed { + debug.Info("conditions did not pass") + continue + } + } + debug.Info("resource matched, it will be deleted...") if err := h.client.DeleteResource(ctx, resource.GetAPIVersion(), resource.GetKind(), namespace, name, false); err != nil { - logger.Error(err, "failed to delete resource") + debug.Error(err, "failed to delete resource") errs = append(errs, err) + } else { + debug.Info("deleted") } } } diff --git a/pkg/engine/variables/vars.go b/pkg/engine/variables/vars.go index ecc35d4541..aa0a8e6491 100644 --- a/pkg/engine/variables/vars.go +++ b/pkg/engine/variables/vars.go @@ -78,17 +78,17 @@ func newPreconditionsVariableResolver(log logr.Logger) VariableResolver { // SubstituteAll substitutes variables and references in the document. The document must be JSON data // i.e. string, []interface{}, map[string]interface{} -func SubstituteAll(log logr.Logger, ctx context.EvalInterface, document interface{}) (_ interface{}, err error) { +func SubstituteAll(log logr.Logger, ctx context.EvalInterface, document interface{}) (interface{}, error) { return substituteAll(log, ctx, document, DefaultVariableResolver) } -func SubstituteAllInPreconditions(log logr.Logger, ctx context.EvalInterface, document interface{}) (_ interface{}, err error) { +func SubstituteAllInPreconditions(log logr.Logger, ctx context.EvalInterface, document interface{}) (interface{}, error) { // We must convert all incoming conditions to JSON data i.e. // string, []interface{}, map[string]interface{} // we cannot use structs otherwise json traverse doesn't work untypedDoc, err := DocumentToUntyped(document) if err != nil { - return document, err + return nil, err } return substituteAll(log, ctx, untypedDoc, newPreconditionsVariableResolver(log)) } @@ -180,12 +180,11 @@ func JSONObjectToConditions(data interface{}) ([]kyvernov1.AnyAllConditions, err return c, nil } -func substituteAll(log logr.Logger, ctx context.EvalInterface, document interface{}, resolver VariableResolver) (_ interface{}, err error) { - document, err = substituteReferences(log, document) +func substituteAll(log logr.Logger, ctx context.EvalInterface, document interface{}, resolver VariableResolver) (interface{}, error) { + document, err := substituteReferences(log, document) if err != nil { - return document, err + return nil, err } - return substituteVars(log, ctx, document, resolver) }