feat: support vps in cli test command (#12384)

* feat: support vps in cli test command

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* context in test

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

---------

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

cmd/cli/kubectl-kyverno/apis/v1alpha1/test.go

@@ -42,8 +42,11 @@ type Test struct {
 	// Values are the values to be used in the test
 	Values *ValuesSpec `json:"values,omitempty"`
 
-	// Policy Exceptions are the policy exceptions to be used in the test
+	// PolicyExceptions are the policy exceptions to be used in the test
 	PolicyExceptions []string `json:"exceptions,omitempty"`
+
+	// Context file containing context data for CEL policies
+	Context string `json:"context,omitempty"`
 }
 
 type CheckResult struct {

cmd/cli/kubectl-kyverno/apis/v1alpha1/test_result.go

@@ -15,10 +15,15 @@ type TestResultBase struct {
 	Rule string `json:"rule,omitempty"`
 
 	// IsValidatingAdmissionPolicy indicates if the policy is a validating admission policy.
-	// It's required in case policy is a validating admission policy.
+	// It's required in case the policy is a validating admission policy.
 	// +optional
 	IsValidatingAdmissionPolicy bool `json:"isValidatingAdmissionPolicy,omitempty"`
 
+	// IsValidatingPolicy indicates if the policy is a validating policy.
+	// It's required in case the policy is a validating policy.
+	// +optional
+	IsValidatingPolicy bool `json:"isValidatingPolicy,omitempty"`
+
 	// Result mentions the result that the user is expecting.
 	// Possible values are pass, fail and skip.
 	Result policyreportv1alpha2.PolicyResult `json:"result"`

cmd/cli/kubectl-kyverno/commands/apply/command.go

@@ -304,6 +304,80 @@ func (c *ApplyCommandConfig) getMutateLogPathIsDir() (bool, error) {
 	return mutateLogPathIsDir, nil
 }
 
+func (c *ApplyCommandConfig) applyPolicies(
+	out io.Writer,
+	store *store.Store,
+	vars *variables.Variables,
+	policies []kyvernov1.PolicyInterface,
+	resources []*unstructured.Unstructured,
+	exceptions []*kyvernov2.PolicyException,
+	skipInvalidPolicies *SkippedInvalidPolicies,
+	dClient dclient.Interface,
+	userInfo *kyvernov2.RequestInfo,
+	mutateLogPathIsDir bool,
+) (*processor.ResultCounts, []*unstructured.Unstructured, []engineapi.EngineResponse, error) {
+	if vars != nil {
+		vars.SetInStore(store)
+	}
+	var rc processor.ResultCounts
+	// validate policies
+	validPolicies := make([]kyvernov1.PolicyInterface, 0, len(policies))
+	for _, pol := range policies {
+		// TODO we should return this info to the caller
+		sa := config.KyvernoUserName(config.KyvernoServiceAccountName())
+		_, err := policyvalidation.Validate(pol, nil, nil, true, sa, sa)
+		if err != nil {
+			log.Log.Error(err, "policy validation error")
+			rc.IncrementError(1)
+			if strings.HasPrefix(err.Error(), "variable 'element.name'") {
+				skipInvalidPolicies.invalid = append(skipInvalidPolicies.invalid, pol.GetName())
+			} else {
+				skipInvalidPolicies.skipped = append(skipInvalidPolicies.skipped, pol.GetName())
+			}
+			continue
+		}
+		validPolicies = append(validPolicies, pol)
+	}
+	var responses []engineapi.EngineResponse
+	for _, resource := range resources {
+		processor := processor.PolicyProcessor{
+			Store:                store,
+			Policies:             validPolicies,
+			Resource:             *resource,
+			PolicyExceptions:     exceptions,
+			MutateLogPath:        c.MutateLogPath,
+			MutateLogPathIsDir:   mutateLogPathIsDir,
+			Variables:            vars,
+			UserInfo:             userInfo,
+			PolicyReport:         c.PolicyReport,
+			NamespaceSelectorMap: vars.NamespaceSelectors(),
+			Stdin:                c.Stdin,
+			Rc:                   &rc,
+			PrintPatchResource:   true,
+			Cluster:              c.Cluster,
+			Client:               dClient,
+			AuditWarn:            c.AuditWarn,
+			Subresources:         vars.Subresources(),
+			Out:                  out,
+		}
+		ers, err := processor.ApplyPoliciesOnResource()
+		if err != nil {
+			if c.ContinueOnFail {
+				log.Log.V(2).Info(fmt.Sprintf("failed to apply policies on resource %s (%s)\n", resource.GetName(), err.Error()))
+				continue
+			}
+			return &rc, resources, responses, fmt.Errorf("failed to apply policies on resource %s (%w)", resource.GetName(), err)
+		}
+		responses = append(responses, ers...)
+	}
+	for _, policy := range validPolicies {
+		if policy.GetNamespace() == "" && policy.GetKind() == "Policy" {
+			log.Log.V(3).Info(fmt.Sprintf("Policy %s has no namespace detected. Ensure that namespaced policies are correctly loaded.", policy.GetNamespace()))
+		}
+	}
+	return &rc, resources, responses, nil
+}
+
 func (c *ApplyCommandConfig) applyValidatingAdmissionPolicies(
 	vaps []admissionregistrationv1.ValidatingAdmissionPolicy,
 	vapBindings []admissionregistrationv1.ValidatingAdmissionPolicyBinding,
@@ -450,7 +524,6 @@ func (c *ApplyCommandConfig) applyValidatingPolicies(
 		}
 		responsesTemp = append(responsesTemp, reps)
 	}
-
 	// transform response into legacy engine responses
 	for _, response := range responsesTemp {
 		for _, r := range response.Policies {
@@ -468,80 +541,6 @@ func (c *ApplyCommandConfig) applyValidatingPolicies(
 	return responses, nil
 }
 
-func (c *ApplyCommandConfig) applyPolicies(
-	out io.Writer,
-	store *store.Store,
-	vars *variables.Variables,
-	policies []kyvernov1.PolicyInterface,
-	resources []*unstructured.Unstructured,
-	exceptions []*kyvernov2.PolicyException,
-	skipInvalidPolicies *SkippedInvalidPolicies,
-	dClient dclient.Interface,
-	userInfo *kyvernov2.RequestInfo,
-	mutateLogPathIsDir bool,
-) (*processor.ResultCounts, []*unstructured.Unstructured, []engineapi.EngineResponse, error) {
-	if vars != nil {
-		vars.SetInStore(store)
-	}
-	var rc processor.ResultCounts
-	// validate policies
-	validPolicies := make([]kyvernov1.PolicyInterface, 0, len(policies))
-	for _, pol := range policies {
-		// TODO we should return this info to the caller
-		sa := config.KyvernoUserName(config.KyvernoServiceAccountName())
-		_, err := policyvalidation.Validate(pol, nil, nil, true, sa, sa)
-		if err != nil {
-			log.Log.Error(err, "policy validation error")
-			rc.IncrementError(1)
-			if strings.HasPrefix(err.Error(), "variable 'element.name'") {
-				skipInvalidPolicies.invalid = append(skipInvalidPolicies.invalid, pol.GetName())
-			} else {
-				skipInvalidPolicies.skipped = append(skipInvalidPolicies.skipped, pol.GetName())
-			}
-			continue
-		}
-		validPolicies = append(validPolicies, pol)
-	}
-	var responses []engineapi.EngineResponse
-	for _, resource := range resources {
-		processor := processor.PolicyProcessor{
-			Store:                store,
-			Policies:             validPolicies,
-			Resource:             *resource,
-			PolicyExceptions:     exceptions,
-			MutateLogPath:        c.MutateLogPath,
-			MutateLogPathIsDir:   mutateLogPathIsDir,
-			Variables:            vars,
-			UserInfo:             userInfo,
-			PolicyReport:         c.PolicyReport,
-			NamespaceSelectorMap: vars.NamespaceSelectors(),
-			Stdin:                c.Stdin,
-			Rc:                   &rc,
-			PrintPatchResource:   true,
-			Cluster:              c.Cluster,
-			Client:               dClient,
-			AuditWarn:            c.AuditWarn,
-			Subresources:         vars.Subresources(),
-			Out:                  out,
-		}
-		ers, err := processor.ApplyPoliciesOnResource()
-		if err != nil {
-			if c.ContinueOnFail {
-				log.Log.V(2).Info(fmt.Sprintf("failed to apply policies on resource %s (%s)\n", resource.GetName(), err.Error()))
-				continue
-			}
-			return &rc, resources, responses, fmt.Errorf("failed to apply policies on resource %s (%w)", resource.GetName(), err)
-		}
-		responses = append(responses, ers...)
-	}
-	for _, policy := range validPolicies {
-		if policy.GetNamespace() == "" && policy.GetKind() == "Policy" {
-			log.Log.V(3).Info(fmt.Sprintf("Policy %s has no namespace detected. Ensure that namespaced policies are correctly loaded.", policy.GetNamespace()))
-		}
-	}
-	return &rc, resources, responses, nil
-}
-
 func (c *ApplyCommandConfig) loadResources(out io.Writer, paths []string, policies []kyvernov1.PolicyInterface, vap []admissionregistrationv1.ValidatingAdmissionPolicy, dClient dclient.Interface) ([]*unstructured.Unstructured, []*unstructured.Unstructured, error) {
 	resources, err := common.GetResourceAccordingToResourcePath(out, nil, paths, c.Cluster, policies, vap, dClient, c.Namespace, c.PolicyReport, "")
 	if err != nil {

cmd/cli/kubectl-kyverno/commands/test/command.go

@@ -190,7 +190,7 @@ func checkResult(test v1alpha1.TestResult, fs billy.Filesystem, resoucePath stri
 func lookupRuleResponses(test v1alpha1.TestResult, responses ...engineapi.RuleResponse) []engineapi.RuleResponse {
 	var matches []engineapi.RuleResponse
 	// Since there are no rules in case of validating admission policies, responses are returned without checking rule names.
-	if test.IsValidatingAdmissionPolicy {
+	if test.IsValidatingAdmissionPolicy || test.IsValidatingPolicy {
 		matches = responses
 	} else {
 		for _, response := range responses {

cmd/cli/kubectl-kyverno/commands/test/output.go

@@ -246,7 +246,7 @@ func printTestResult(
 					for _, rule := range lookupRuleResponses(test, response.PolicyResponse.Rules...) {
 						r := response.Resource
 
-						if test.IsValidatingAdmissionPolicy {
+						if test.IsValidatingAdmissionPolicy || test.IsValidatingPolicy {
 							ok, message, reason := checkResult(test, fs, resoucePath, response, rule, r)
 							if strings.Contains(message, "not found in manifest") {
 								resourceSkipped = true

cmd/cli/kubectl-kyverno/commands/test/test.go

@@ -1,11 +1,14 @@
 package test
 
 import (
+	"context"
 	"fmt"
 	"io"
 
 	kyvernov1 "github.com/kyverno/kyverno/api/kyverno/v1"
 	kyvernov2 "github.com/kyverno/kyverno/api/kyverno/v2"
+	clicontext "github.com/kyverno/kyverno/cmd/cli/kubectl-kyverno/context"
+	"github.com/kyverno/kyverno/cmd/cli/kubectl-kyverno/data"
 	"github.com/kyverno/kyverno/cmd/cli/kubectl-kyverno/deprecations"
 	"github.com/kyverno/kyverno/cmd/cli/kubectl-kyverno/exception"
 	"github.com/kyverno/kyverno/cmd/cli/kubectl-kyverno/log"
@@ -21,13 +24,19 @@ import (
 	"github.com/kyverno/kyverno/ext/output/pluralize"
 	"github.com/kyverno/kyverno/pkg/autogen"
 	"github.com/kyverno/kyverno/pkg/background/generate"
+	"github.com/kyverno/kyverno/pkg/cel/engine"
+	"github.com/kyverno/kyverno/pkg/cel/matching"
+	celpolicy "github.com/kyverno/kyverno/pkg/cel/policy"
 	"github.com/kyverno/kyverno/pkg/clients/dclient"
 	"github.com/kyverno/kyverno/pkg/config"
 	engineapi "github.com/kyverno/kyverno/pkg/engine/api"
 	policyvalidation "github.com/kyverno/kyverno/pkg/validation/policy"
+	admissionv1 "k8s.io/api/admission/v1"
+	"k8s.io/apimachinery/pkg/api/meta"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/restmapper"
 )
 
 type TestResponse struct {
@@ -138,7 +147,7 @@ func runTest(out io.Writer, testCase test.TestCase, registryAccess bool) (*TestR
 	for _, policy := range results.Policies {
 		for _, rule := range autogen.Default.ComputeRules(policy, "") {
 			for _, res := range testCase.Test.Results {
-				if res.IsValidatingAdmissionPolicy {
+				if res.IsValidatingAdmissionPolicy || res.IsValidatingPolicy {
 					continue
 				}
 				// TODO: what if two policies have a rule with the same name ?
@@ -247,6 +256,81 @@ func runTest(out io.Writer, testCase test.TestCase, registryAccess bool) (*TestR
 		resourceKey := generateResourceKey(resource)
 		testResponse.Trigger[resourceKey] = append(testResponse.Trigger[resourceKey], ers...)
 	}
+	if len(results.ValidatingPolicies) != 0 {
+		ctx := context.TODO()
+		compiler := celpolicy.NewCompiler()
+		provider, err := engine.NewProvider(compiler, results.ValidatingPolicies, polexLoader.CELExceptions)
+		if err != nil {
+			return nil, err
+		}
+		eng := engine.NewEngine(provider, vars.Namespace, matching.NewMatcher())
+		var restMapper meta.RESTMapper
+		var contextProvider celpolicy.Context
+		apiGroupResources, err := data.APIGroupResources()
+		if err != nil {
+			return nil, err
+		}
+		restMapper = restmapper.NewDiscoveryRESTMapper(apiGroupResources)
+		fakeContextProvider := celpolicy.NewFakeContextProvider()
+		if testCase.Test.Context != "" {
+			ctx, err := clicontext.Load(nil, testCase.Test.Context)
+			if err != nil {
+				return nil, err
+			}
+			for _, resource := range ctx.ContextSpec.Resources {
+				gvk := resource.GroupVersionKind()
+				mapping, err := restMapper.RESTMapping(gvk.GroupKind(), gvk.Version)
+				if err != nil {
+					return nil, err
+				}
+				if err := fakeContextProvider.AddResource(mapping.Resource, &resource); err != nil {
+					return nil, err
+				}
+			}
+		}
+		contextProvider = fakeContextProvider
+		for _, resource := range uniques {
+			// get gvk from resource
+			gvk := resource.GroupVersionKind()
+			// map gvk to gvr
+			mapping, err := restMapper.RESTMapping(gvk.GroupKind(), gvk.Version)
+			if err != nil {
+				return nil, fmt.Errorf("failed to map gvk to gvr %s (%v)\n", gvk, err)
+			}
+			gvr := mapping.Resource
+			// create engine request
+			request := engine.Request(
+				contextProvider,
+				gvk,
+				gvr,
+				// TODO: how to manage subresource ?
+				"",
+				resource.GetName(),
+				resource.GetNamespace(),
+				// TODO: how to manage other operations ?
+				admissionv1.Create,
+				resource,
+				nil,
+				false,
+				nil,
+			)
+			reps, err := eng.Handle(ctx, request)
+			if err != nil {
+				return nil, fmt.Errorf("failed to apply validating policies on resource %s (%w)", resource.GetName(), err)
+			}
+			resourceKey := generateResourceKey(resource)
+			for _, r := range reps.Policies {
+				engineResponse := engineapi.EngineResponse{
+					Resource: *reps.Resource,
+					PolicyResponse: engineapi.PolicyResponse{
+						Rules: r.Rules,
+					},
+				}
+				engineResponse = engineResponse.WithPolicy(engineapi.NewValidatingPolicy(&r.Policy))
+				testResponse.Trigger[resourceKey] = append(testResponse.Trigger[resourceKey], engineResponse)
+			}
+		}
+	}
 	// this is an array of responses of all policies, generated by all of their rules
 	return &testResponse, nil
 }

cmd/cli/kubectl-kyverno/config/crds/cli.kyverno.io_tests.yaml

@@ -61,8 +61,11 @@ spec:
               - error
               type: object
             type: array
+          context:
+            description: Context file containing context data for CEL policies
+            type: string
           exceptions:
-            description: Policy Exceptions are the policy exceptions to be used in
+            description: PolicyExceptions are the policy exceptions to be used in
               the test
             items:
               type: string
@@ -110,7 +113,12 @@ spec:
                 isValidatingAdmissionPolicy:
                   description: |-
                     IsValidatingAdmissionPolicy indicates if the policy is a validating admission policy.
-                    It's required in case policy is a validating admission policy.
+                    It's required in case the policy is a validating admission policy.
+                  type: boolean
+                isValidatingPolicy:
+                  description: |-
+                    IsValidatingPolicy indicates if the policy is a validating policy.
+                    It's required in case the policy is a validating policy.
                   type: boolean
                 kind:
                   description: Kind mentions the kind of the resource on which the

cmd/cli/kubectl-kyverno/data/crds/cli.kyverno.io_tests.yaml

@@ -61,8 +61,11 @@ spec:
               - error
               type: object
             type: array
+          context:
+            description: Context file containing context data for CEL policies
+            type: string
           exceptions:
-            description: Policy Exceptions are the policy exceptions to be used in
+            description: PolicyExceptions are the policy exceptions to be used in
               the test
             items:
               type: string
@@ -110,7 +113,12 @@ spec:
                 isValidatingAdmissionPolicy:
                   description: |-
                     IsValidatingAdmissionPolicy indicates if the policy is a validating admission policy.
-                    It's required in case policy is a validating admission policy.
+                    It's required in case the policy is a validating admission policy.
+                  type: boolean
+                isValidatingPolicy:
+                  description: |-
+                    IsValidatingPolicy indicates if the policy is a validating policy.
+                    It's required in case the policy is a validating policy.
                   type: boolean
                 kind:
                   description: Kind mentions the kind of the resource on which the

docs/user/cli/crd/index.html

@@ -279,7 +279,18 @@ ValuesSpec
 </em>
 </td>
 <td>
-<p>Policy Exceptions are the policy exceptions to be used in the test</p>
+<p>PolicyExceptions are the policy exceptions to be used in the test</p>
+</td>
+</tr>
+<tr>
+<td>
+<code>context</code><br/>
+<em>
+string
+</em>
+</td>
+<td>
+<p>Context file containing context data for CEL policies</p>
 </td>
 </tr>
 </tbody>
@@ -1102,7 +1113,20 @@ bool
 <td>
 <em>(Optional)</em>
 <p>IsValidatingAdmissionPolicy indicates if the policy is a validating admission policy.
-It&rsquo;s required in case policy is a validating admission policy.</p>
+It&rsquo;s required in case the policy is a validating admission policy.</p>
+</td>
+</tr>
+<tr>
+<td>
+<code>isValidatingPolicy</code><br/>
+<em>
+bool
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>IsValidatingPolicy indicates if the policy is a validating policy.
+It&rsquo;s required in case the policy is a validating policy.</p>
 </td>
 </tr>
 <tr>

docs/user/cli/crd/kyverno_kubectl.v1alpha1.html

@@ -565,7 +565,36 @@ This field is deprecated, use <code>metadata.name</code> instead</p>
         <td>
           
 
-          <p>Policy Exceptions are the policy exceptions to be used in the test</p>
+          <p>PolicyExceptions are the policy exceptions to be used in the test</p>
+
+
+          
+
+          
+        </td>
+      </tr>
+    
+  
+    
+    
+      <tr>
+        <td><code>context</code>
+          
+          <span style="color:blue;"> *</span>
+          
+          </br>
+
+          
+          
+            
+              <span style="font-family: monospace">string</span>
+            
+          
+        </td>
+        <td>
+          
+
+          <p>Context file containing context data for CEL policies</p>
 
 
           
@@ -2393,7 +2422,35 @@ It's required in case policy is a kyverno policy.</p>
           
 
           <p>IsValidatingAdmissionPolicy indicates if the policy is a validating admission policy.
-It's required in case policy is a validating admission policy.</p>
+It's required in case the policy is a validating admission policy.</p>
+
+
+          
+
+          
+        </td>
+      </tr>
+    
+  
+    
+    
+      <tr>
+        <td><code>isValidatingPolicy</code>
+          
+          </br>
+
+          
+          
+            
+              <span style="font-family: monospace">bool</span>
+            
+          
+        </td>
+        <td>
+          
+
+          <p>IsValidatingPolicy indicates if the policy is a validating policy.
+It's required in case the policy is a validating policy.</p>
 
 
           

pkg/imageverification/imagedataloader/options.go

@@ -33,7 +33,6 @@ var (
 		TLSHandshakeTimeout:   10 * time.Second,
 		ExpectContinueTimeout: 1 * time.Second,
 	}
-
 	UserAgent = fmt.Sprintf("Kyverno/%s (%s; %s)", version.GetVersionInfo().GitVersion, runtime.GOOS, runtime.GOARCH)
 )