feat: Support CEL expression warnings (#9566)

* feat: support CEL expression warnings

Signed-off-by: Mariam Fahmy <mariam.fahmy@nirmata.com>

* fix

Signed-off-by: Mariam Fahmy <mariam.fahmy@nirmata.com>

* fix: allow the policy creation but return warnings to the API server

Signed-off-by: Mariam Fahmy <mariam.fahmy@nirmata.com>

* fix tests

Signed-off-by: Mariam Fahmy <mariam.fahmy@nirmata.com>

---------

Signed-off-by: Mariam Fahmy <mariam.fahmy@nirmata.com>
Signed-off-by: ShutingZhao <shuting@nirmata.com>
Co-authored-by: ShutingZhao <shuting@nirmata.com>

pkg/controllers/validatingadmissionpolicy-generate/controller.go

@@ -21,10 +21,11 @@ import (
 	controllerutils "github.com/kyverno/kyverno/pkg/utils/controller"
 	datautils "github.com/kyverno/kyverno/pkg/utils/data"
 	kubeutils "github.com/kyverno/kyverno/pkg/utils/kube"
-	"github.com/kyverno/kyverno/pkg/utils/validatingadmissionpolicy"
+	"github.com/kyverno/kyverno/pkg/validatingadmissionpolicy"
 	admissionregistrationv1alpha1 "k8s.io/api/admissionregistration/v1alpha1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
 	admissionregistrationv1alpha1informers "k8s.io/client-go/informers/admissionregistration/v1alpha1"
 	"k8s.io/client-go/kubernetes"
 	admissionregistrationv1alpha1listers "k8s.io/client-go/listers/admissionregistration/v1alpha1"
@@ -265,87 +266,18 @@ func (c *controller) getValidatingAdmissionPolicyBinding(name string) (*admissio
 	return vapbinding, nil
 }
 
-func (c *controller) buildValidatingAdmissionPolicy(vap *admissionregistrationv1alpha1.ValidatingAdmissionPolicy, cpol kyvernov1.PolicyInterface) error {
-	// set owner reference
-	vap.OwnerReferences = []metav1.OwnerReference{
-		{
-			APIVersion: "kyverno.io/v1",
-			Kind:       cpol.GetKind(),
-			Name:       cpol.GetName(),
-			UID:        cpol.GetUID(),
-		},
+// hasExceptions checks if there is an exception that match both the policy and the rule.
+func (c *controller) hasExceptions(policyName, rule string) (bool, error) {
+	polexs, err := c.polexLister.List(labels.Everything())
+	if err != nil {
+		return false, err
 	}
-
-	// construct validating admission policy resource rules
-	var matchResources admissionregistrationv1alpha1.MatchResources
-	var matchRules []admissionregistrationv1alpha1.NamedRuleWithOperations
-
-	rule := cpol.GetSpec().Rules[0]
-	match := rule.MatchResources
-	if !match.ResourceDescription.IsEmpty() {
-		if err := c.translateResource(&matchResources, &matchRules, match.ResourceDescription); err != nil {
-			return err
+	for _, polex := range polexs {
+		if polex.Contains(policyName, rule) {
+			return true, nil
 		}
 	}
-
-	if match.Any != nil {
-		if err := c.translateResourceFilters(&matchResources, &matchRules, match.Any); err != nil {
-			return err
-		}
-	}
-	if match.All != nil {
-		if err := c.translateResourceFilters(&matchResources, &matchRules, match.All); err != nil {
-			return err
-		}
-	}
-
-	// set validating admission policy spec
-	vap.Spec = admissionregistrationv1alpha1.ValidatingAdmissionPolicySpec{
-		MatchConstraints: &matchResources,
-		ParamKind:        rule.Validation.CEL.ParamKind,
-		Variables:        rule.Validation.CEL.Variables,
-		Validations:      rule.Validation.CEL.Expressions,
-		AuditAnnotations: rule.Validation.CEL.AuditAnnotations,
-		MatchConditions:  rule.CELPreconditions,
-	}
-
-	// set labels
-	controllerutils.SetManagedByKyvernoLabel(vap)
-	return nil
-}
-
-func (c *controller) buildValidatingAdmissionPolicyBinding(vapbinding *admissionregistrationv1alpha1.ValidatingAdmissionPolicyBinding, cpol kyvernov1.PolicyInterface) error {
-	// set owner reference
-	vapbinding.OwnerReferences = []metav1.OwnerReference{
-		{
-			APIVersion: "kyverno.io/v1",
-			Kind:       cpol.GetKind(),
-			Name:       cpol.GetName(),
-			UID:        cpol.GetUID(),
-		},
-	}
-
-	// set validation action for vap binding
-	var validationActions []admissionregistrationv1alpha1.ValidationAction
-	action := cpol.GetSpec().ValidationFailureAction
-	if action.Enforce() {
-		validationActions = append(validationActions, admissionregistrationv1alpha1.Deny)
-	} else if action.Audit() {
-		validationActions = append(validationActions, admissionregistrationv1alpha1.Audit)
-		validationActions = append(validationActions, admissionregistrationv1alpha1.Warn)
-	}
-
-	// set validating admission policy binding spec
-	rule := cpol.GetSpec().Rules[0]
-	vapbinding.Spec = admissionregistrationv1alpha1.ValidatingAdmissionPolicyBindingSpec{
-		PolicyName:        cpol.GetName(),
-		ParamRef:          rule.Validation.CEL.ParamRef,
-		ValidationActions: validationActions,
-	}
-
-	// set labels
-	controllerutils.SetManagedByKyvernoLabel(vapbinding)
-	return nil
+	return false, nil
 }
 
 func constructVapBindingName(vapName string) string {
@@ -391,7 +323,7 @@ func (c *controller) reconcile(ctx context.Context, logger logr.Logger, key, nam
 	if err != nil {
 		return err
 	}
-	if ok, msg := canGenerateVAP(spec); !ok || hasExceptions {
+	if ok, msg := validatingadmissionpolicy.CanGenerateVAP(spec); !ok || hasExceptions {
 		// delete the ValidatingAdmissionPolicy if exist
 		if vapErr == nil {
 			err = c.client.AdmissionregistrationV1alpha1().ValidatingAdmissionPolicies().Delete(ctx, vapName, metav1.DeleteOptions{})
@@ -439,7 +371,7 @@ func (c *controller) reconcile(ctx context.Context, logger logr.Logger, key, nam
 	}
 
 	if observedVAP.ResourceVersion == "" {
-		err := c.buildValidatingAdmissionPolicy(observedVAP, policy)
+		err := validatingadmissionpolicy.BuildValidatingAdmissionPolicy(c.discoveryClient, observedVAP, policy)
 		if err != nil {
 			c.updateClusterPolicyStatus(ctx, *policy, false, err.Error())
 			return err
@@ -455,7 +387,7 @@ func (c *controller) reconcile(ctx context.Context, logger logr.Logger, key, nam
 			observedVAP,
 			c.client.AdmissionregistrationV1alpha1().ValidatingAdmissionPolicies(),
 			func(observed *admissionregistrationv1alpha1.ValidatingAdmissionPolicy) error {
-				return c.buildValidatingAdmissionPolicy(observed, policy)
+				return validatingadmissionpolicy.BuildValidatingAdmissionPolicy(c.discoveryClient, observed, policy)
 			})
 		if err != nil {
 			c.updateClusterPolicyStatus(ctx, *policy, false, err.Error())
@@ -464,7 +396,7 @@ func (c *controller) reconcile(ctx context.Context, logger logr.Logger, key, nam
 	}
 
 	if observedVAPbinding.ResourceVersion == "" {
-		err := c.buildValidatingAdmissionPolicyBinding(observedVAPbinding, policy)
+		err := validatingadmissionpolicy.BuildValidatingAdmissionPolicyBinding(observedVAPbinding, policy)
 		if err != nil {
 			c.updateClusterPolicyStatus(ctx, *policy, false, err.Error())
 			return err
@@ -480,7 +412,7 @@ func (c *controller) reconcile(ctx context.Context, logger logr.Logger, key, nam
 			observedVAPbinding,
 			c.client.AdmissionregistrationV1alpha1().ValidatingAdmissionPolicyBindings(),
 			func(observed *admissionregistrationv1alpha1.ValidatingAdmissionPolicyBinding) error {
-				return c.buildValidatingAdmissionPolicyBinding(observed, policy)
+				return validatingadmissionpolicy.BuildValidatingAdmissionPolicyBinding(observed, policy)
 			})
 		if err != nil {
 			c.updateClusterPolicyStatus(ctx, *policy, false, err.Error())

pkg/controllers/validatingadmissionpolicy-generate/translation.go

@@ -1,108 +0,0 @@
-package validatingadmissionpolicygenerate
-
-import (
-	"fmt"
-	"slices"
-
-	kyvernov1 "github.com/kyverno/kyverno/api/kyverno/v1"
-	kubeutils "github.com/kyverno/kyverno/pkg/utils/kube"
-	admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
-	"k8s.io/api/admissionregistration/v1alpha1"
-)
-
-func (c *controller) translateResourceFilters(matchResources *v1alpha1.MatchResources, rules *[]v1alpha1.NamedRuleWithOperations, resFilters kyvernov1.ResourceFilters) error {
-	for _, filter := range resFilters {
-		err := c.translateResource(matchResources, rules, filter.ResourceDescription)
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func (c *controller) translateResource(matchResources *v1alpha1.MatchResources, rules *[]v1alpha1.NamedRuleWithOperations, res kyvernov1.ResourceDescription) error {
-	err := c.constructValidatingAdmissionPolicyRules(rules, res.Kinds, res.GetOperations())
-	if err != nil {
-		return err
-	}
-
-	matchResources.ResourceRules = *rules
-	matchResources.NamespaceSelector = res.NamespaceSelector
-	matchResources.ObjectSelector = res.Selector
-	return nil
-}
-
-func (c *controller) constructValidatingAdmissionPolicyRules(rules *[]v1alpha1.NamedRuleWithOperations, kinds []string, operations []string) error {
-	// translate operations to their corresponding values in validating admission policy.
-	ops := c.translateOperations(operations)
-
-	// get kinds from kyverno policies and translate them to rules in validating admission policies.
-	// matched resources in kyverno policies are written in the following format:
-	// group/version/kind/subresource
-	// whereas matched resources in validating admission policies are written in the following format:
-	// apiGroups:   ["group"]
-	// apiVersions: ["version"]
-	// resources:   ["resource"]
-	for _, kind := range kinds {
-		group, version, kind, subresource := kubeutils.ParseKindSelector(kind)
-		gvrss, err := c.discoveryClient.FindResources(group, version, kind, subresource)
-		if err != nil {
-			return err
-		}
-		if len(gvrss) != 1 {
-			return fmt.Errorf("no unique match for kind %s", kind)
-		}
-
-		for topLevelApi, apiResource := range gvrss {
-			isNewRule := true
-			// If there's a rule that contains both group and version, then the resource is appended to the existing rule instead of creating a new one.
-			// Example:  apiGroups:   ["apps"]
-			//           apiVersions: ["v1"]
-			//           resources:   ["deployments", "statefulsets"]
-			// Otherwise, a new rule is created.
-			for i := range *rules {
-				if slices.Contains((*rules)[i].APIGroups, topLevelApi.Group) && slices.Contains((*rules)[i].APIVersions, topLevelApi.Version) {
-					(*rules)[i].Resources = append((*rules)[i].Resources, apiResource.Name)
-					isNewRule = false
-					break
-				}
-			}
-			if isNewRule {
-				r := v1alpha1.NamedRuleWithOperations{
-					RuleWithOperations: admissionregistrationv1.RuleWithOperations{
-						Rule: admissionregistrationv1.Rule{
-							Resources:   []string{apiResource.Name},
-							APIGroups:   []string{topLevelApi.Group},
-							APIVersions: []string{topLevelApi.Version},
-						},
-						Operations: ops,
-					},
-				}
-				*rules = append(*rules, r)
-			}
-		}
-	}
-	return nil
-}
-
-func (c *controller) translateOperations(operations []string) []admissionregistrationv1.OperationType {
-	var vapOperations []admissionregistrationv1.OperationType
-	for _, op := range operations {
-		if op == string(kyvernov1.Create) {
-			vapOperations = append(vapOperations, admissionregistrationv1.Create)
-		} else if op == string(kyvernov1.Update) {
-			vapOperations = append(vapOperations, admissionregistrationv1.Update)
-		} else if op == string(kyvernov1.Connect) {
-			vapOperations = append(vapOperations, admissionregistrationv1.Connect)
-		} else if op == string(kyvernov1.Delete) {
-			vapOperations = append(vapOperations, admissionregistrationv1.Delete)
-		}
-	}
-
-	// set default values for operations since it's a required field in validating admission policies
-	if len(vapOperations) == 0 {
-		vapOperations = append(vapOperations, admissionregistrationv1.Create)
-		vapOperations = append(vapOperations, admissionregistrationv1.Update)
-	}
-	return vapOperations
-}

pkg/validatingadmissionpolicy/builder.go

@@ -0,0 +1,196 @@
+package validatingadmissionpolicy
+
+import (
+	"fmt"
+	"slices"
+
+	kyvernov1 "github.com/kyverno/kyverno/api/kyverno/v1"
+	"github.com/kyverno/kyverno/pkg/clients/dclient"
+	controllerutils "github.com/kyverno/kyverno/pkg/utils/controller"
+	kubeutils "github.com/kyverno/kyverno/pkg/utils/kube"
+	admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
+	admissionregistrationv1alpha1 "k8s.io/api/admissionregistration/v1alpha1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// BuildValidatingAdmissionPolicy is used to build a Kubernetes ValidatingAdmissionPolicy from a Kyverno policy
+func BuildValidatingAdmissionPolicy(discoveryClient dclient.IDiscovery, vap *admissionregistrationv1alpha1.ValidatingAdmissionPolicy, cpol kyvernov1.PolicyInterface) error {
+	// set owner reference
+	vap.OwnerReferences = []metav1.OwnerReference{
+		{
+			APIVersion: "kyverno.io/v1",
+			Kind:       cpol.GetKind(),
+			Name:       cpol.GetName(),
+			UID:        cpol.GetUID(),
+		},
+	}
+
+	// construct validating admission policy resource rules
+	var matchResources admissionregistrationv1alpha1.MatchResources
+	var matchRules []admissionregistrationv1alpha1.NamedRuleWithOperations
+
+	rule := cpol.GetSpec().Rules[0]
+	match := rule.MatchResources
+	if !match.ResourceDescription.IsEmpty() {
+		if err := translateResource(discoveryClient, &matchResources, &matchRules, match.ResourceDescription); err != nil {
+			return err
+		}
+	}
+
+	if match.Any != nil {
+		if err := translateResourceFilters(discoveryClient, &matchResources, &matchRules, match.Any); err != nil {
+			return err
+		}
+	}
+	if match.All != nil {
+		if err := translateResourceFilters(discoveryClient, &matchResources, &matchRules, match.All); err != nil {
+			return err
+		}
+	}
+
+	// set validating admission policy spec
+	vap.Spec = admissionregistrationv1alpha1.ValidatingAdmissionPolicySpec{
+		MatchConstraints: &matchResources,
+		ParamKind:        rule.Validation.CEL.ParamKind,
+		Variables:        rule.Validation.CEL.Variables,
+		Validations:      rule.Validation.CEL.Expressions,
+		AuditAnnotations: rule.Validation.CEL.AuditAnnotations,
+		MatchConditions:  rule.CELPreconditions,
+	}
+
+	// set labels
+	controllerutils.SetManagedByKyvernoLabel(vap)
+	return nil
+}
+
+// BuildValidatingAdmissionPolicyBinding is used to build a Kubernetes ValidatingAdmissionPolicyBinding from a Kyverno policy
+func BuildValidatingAdmissionPolicyBinding(vapbinding *admissionregistrationv1alpha1.ValidatingAdmissionPolicyBinding, cpol kyvernov1.PolicyInterface) error {
+	// set owner reference
+	vapbinding.OwnerReferences = []metav1.OwnerReference{
+		{
+			APIVersion: "kyverno.io/v1",
+			Kind:       cpol.GetKind(),
+			Name:       cpol.GetName(),
+			UID:        cpol.GetUID(),
+		},
+	}
+
+	// set validation action for vap binding
+	var validationActions []admissionregistrationv1alpha1.ValidationAction
+	action := cpol.GetSpec().ValidationFailureAction
+	if action.Enforce() {
+		validationActions = append(validationActions, admissionregistrationv1alpha1.Deny)
+	} else if action.Audit() {
+		validationActions = append(validationActions, admissionregistrationv1alpha1.Audit)
+		validationActions = append(validationActions, admissionregistrationv1alpha1.Warn)
+	}
+
+	// set validating admission policy binding spec
+	rule := cpol.GetSpec().Rules[0]
+	vapbinding.Spec = admissionregistrationv1alpha1.ValidatingAdmissionPolicyBindingSpec{
+		PolicyName:        cpol.GetName(),
+		ParamRef:          rule.Validation.CEL.ParamRef,
+		ValidationActions: validationActions,
+	}
+
+	// set labels
+	controllerutils.SetManagedByKyvernoLabel(vapbinding)
+	return nil
+}
+
+func translateResourceFilters(discoveryClient dclient.IDiscovery, matchResources *admissionregistrationv1alpha1.MatchResources, rules *[]admissionregistrationv1alpha1.NamedRuleWithOperations, resFilters kyvernov1.ResourceFilters) error {
+	for _, filter := range resFilters {
+		err := translateResource(discoveryClient, matchResources, rules, filter.ResourceDescription)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func translateResource(discoveryClient dclient.IDiscovery, matchResources *admissionregistrationv1alpha1.MatchResources, rules *[]admissionregistrationv1alpha1.NamedRuleWithOperations, res kyvernov1.ResourceDescription) error {
+	err := constructValidatingAdmissionPolicyRules(discoveryClient, rules, res.Kinds, res.GetOperations())
+	if err != nil {
+		return err
+	}
+
+	matchResources.ResourceRules = *rules
+	matchResources.NamespaceSelector = res.NamespaceSelector
+	matchResources.ObjectSelector = res.Selector
+	return nil
+}
+
+func constructValidatingAdmissionPolicyRules(discoveryClient dclient.IDiscovery, rules *[]admissionregistrationv1alpha1.NamedRuleWithOperations, kinds []string, operations []string) error {
+	// translate operations to their corresponding values in validating admission policy.
+	ops := translateOperations(operations)
+
+	// get kinds from kyverno policies and translate them to rules in validating admission policies.
+	// matched resources in kyverno policies are written in the following format:
+	// group/version/kind/subresource
+	// whereas matched resources in validating admission policies are written in the following format:
+	// apiGroups:   ["group"]
+	// apiVersions: ["version"]
+	// resources:   ["resource"]
+	for _, kind := range kinds {
+		group, version, kind, subresource := kubeutils.ParseKindSelector(kind)
+		gvrss, err := discoveryClient.FindResources(group, version, kind, subresource)
+		if err != nil {
+			return err
+		}
+		if len(gvrss) != 1 {
+			return fmt.Errorf("no unique match for kind %s", kind)
+		}
+
+		for topLevelApi, apiResource := range gvrss {
+			isNewRule := true
+			// If there's a rule that contains both group and version, then the resource is appended to the existing rule instead of creating a new one.
+			// Example:  apiGroups:   ["apps"]
+			//           apiVersions: ["v1"]
+			//           resources:   ["deployments", "statefulsets"]
+			// Otherwise, a new rule is created.
+			for i := range *rules {
+				if slices.Contains((*rules)[i].APIGroups, topLevelApi.Group) && slices.Contains((*rules)[i].APIVersions, topLevelApi.Version) {
+					(*rules)[i].Resources = append((*rules)[i].Resources, apiResource.Name)
+					isNewRule = false
+					break
+				}
+			}
+			if isNewRule {
+				r := admissionregistrationv1alpha1.NamedRuleWithOperations{
+					RuleWithOperations: admissionregistrationv1.RuleWithOperations{
+						Rule: admissionregistrationv1.Rule{
+							Resources:   []string{apiResource.Name},
+							APIGroups:   []string{topLevelApi.Group},
+							APIVersions: []string{topLevelApi.Version},
+						},
+						Operations: ops,
+					},
+				}
+				*rules = append(*rules, r)
+			}
+		}
+	}
+	return nil
+}
+
+func translateOperations(operations []string) []admissionregistrationv1.OperationType {
+	var vapOperations []admissionregistrationv1.OperationType
+	for _, op := range operations {
+		if op == string(kyvernov1.Create) {
+			vapOperations = append(vapOperations, admissionregistrationv1.Create)
+		} else if op == string(kyvernov1.Update) {
+			vapOperations = append(vapOperations, admissionregistrationv1.Update)
+		} else if op == string(kyvernov1.Connect) {
+			vapOperations = append(vapOperations, admissionregistrationv1.Connect)
+		} else if op == string(kyvernov1.Delete) {
+			vapOperations = append(vapOperations, admissionregistrationv1.Delete)
+		}
+	}
+
+	// set default values for operations since it's a required field in validating admission policies
+	if len(vapOperations) == 0 {
+		vapOperations = append(vapOperations, admissionregistrationv1.Create)
+		vapOperations = append(vapOperations, admissionregistrationv1.Update)
+	}
+	return vapOperations
+}

pkg/validatingadmissionpolicy/kyvernopolicy_checker.go

@@ -1,43 +1,11 @@
-package validatingadmissionpolicygenerate
+package validatingadmissionpolicy
 
 import (
 	kyvernov1 "github.com/kyverno/kyverno/api/kyverno/v1"
-	"k8s.io/apimachinery/pkg/labels"
 )
 
-// hasExceptions checks if there is an exception that match both the policy and the rule.
-func (c *controller) hasExceptions(policyName, rule string) (bool, error) {
-	polexs, err := c.polexLister.List(labels.Everything())
-	if err != nil {
-		return false, err
-	}
-	for _, polex := range polexs {
-		if polex.Contains(policyName, rule) {
-			return true, nil
-		}
-	}
-	return false, nil
-}
-
-func checkResources(resource kyvernov1.ResourceDescription) (bool, string) {
-	var msg string
-	if len(resource.Namespaces) != 0 || len(resource.Annotations) != 0 {
-		msg = "skip generating ValidatingAdmissionPolicy: Namespaces / Annotations in resource description isn't applicable."
-		return false, msg
-	}
-	return true, msg
-}
-
-func checkUserInfo(info kyvernov1.UserInfo) (bool, string) {
-	var msg string
-	if !info.IsEmpty() {
-		msg = "skip generating ValidatingAdmissionPolicy: Roles / ClusterRoles / Subjects in `any/all` isn't applicable."
-		return false, msg
-	}
-	return true, msg
-}
-
-func canGenerateVAP(spec *kyvernov1.Spec) (bool, string) {
+// CanGenerateVAP check if Kyverno policy can be translated to a Kubernetes ValidatingAdmissionPolicy
+func CanGenerateVAP(spec *kyvernov1.Spec) (bool, string) {
 	var msg string
 	if len(spec.Rules) > 1 {
 		msg = "skip generating ValidatingAdmissionPolicy: multiple rules aren't applicable."
@@ -121,3 +89,21 @@ func canGenerateVAP(spec *kyvernov1.Spec) (bool, string) {
 
 	return true, msg
 }
+
+func checkResources(resource kyvernov1.ResourceDescription) (bool, string) {
+	var msg string
+	if len(resource.Namespaces) != 0 || len(resource.Annotations) != 0 {
+		msg = "skip generating ValidatingAdmissionPolicy: Namespaces / Annotations in resource description isn't applicable."
+		return false, msg
+	}
+	return true, msg
+}
+
+func checkUserInfo(info kyvernov1.UserInfo) (bool, string) {
+	var msg string
+	if !info.IsEmpty() {
+		msg = "skip generating ValidatingAdmissionPolicy: Roles / ClusterRoles / Subjects in `any/all` isn't applicable."
+		return false, msg
+	}
+	return true, msg
+}

pkg/validatingadmissionpolicy/kyvernopolicy_checker_test.go

@@ -1,4 +1,4 @@
-package validatingadmissionpolicygenerate
+package validatingadmissionpolicy
 
 import (
 	"encoding/json"
@@ -480,7 +480,7 @@ func Test_Can_Generate_ValidatingAdmissionPolicy(t *testing.T) {
 			policies, _, _, err := yamlutils.GetPolicy([]byte(test.policy))
 			assert.NilError(t, err)
 			assert.Equal(t, 1, len(policies))
-			out, _ := canGenerateVAP(policies[0].GetSpec())
+			out, _ := CanGenerateVAP(policies[0].GetSpec())
 			assert.Equal(t, out, test.expected)
 		})
 	}

pkg/validatingadmissionpolicy/utils.go

@@ -1,63 +0,0 @@
-package validatingadmissionpolicy
-
-import (
-	admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
-	"k8s.io/api/admissionregistration/v1alpha1"
-	"k8s.io/api/admissionregistration/v1beta1"
-)
-
-func convertRules(v1alpha1rules []v1alpha1.NamedRuleWithOperations) []v1beta1.NamedRuleWithOperations {
-	var v1beta1rules []v1beta1.NamedRuleWithOperations
-	for _, r := range v1alpha1rules {
-		v1beta1rules = append(v1beta1rules, v1beta1.NamedRuleWithOperations(r))
-	}
-	return v1beta1rules
-}
-
-func convertValidations(v1alpha1validations []v1alpha1.Validation) []v1beta1.Validation {
-	var v1beta1validations []v1beta1.Validation
-	for _, v := range v1alpha1validations {
-		v1beta1validations = append(v1beta1validations, v1beta1.Validation(v))
-	}
-	return v1beta1validations
-}
-
-func convertAuditAnnotations(v1alpha1auditanns []v1alpha1.AuditAnnotation) []v1beta1.AuditAnnotation {
-	var v1beta1auditanns []v1beta1.AuditAnnotation
-	for _, a := range v1alpha1auditanns {
-		v1beta1auditanns = append(v1beta1auditanns, v1beta1.AuditAnnotation(a))
-	}
-	return v1beta1auditanns
-}
-
-func convertMatchConditions(v1alpha1conditions []v1alpha1.MatchCondition) []v1beta1.MatchCondition {
-	var v1beta1conditions []v1beta1.MatchCondition
-	for _, m := range v1alpha1conditions {
-		v1beta1conditions = append(v1beta1conditions, v1beta1.MatchCondition(m))
-	}
-	return v1beta1conditions
-}
-
-func ConvertMatchConditionsV1(v1alpha1conditions []v1alpha1.MatchCondition) []admissionregistrationv1.MatchCondition {
-	var v1conditions []admissionregistrationv1.MatchCondition
-	for _, m := range v1alpha1conditions {
-		v1conditions = append(v1conditions, admissionregistrationv1.MatchCondition(m))
-	}
-	return v1conditions
-}
-
-func convertVariables(v1alpha1variables []v1alpha1.Variable) []v1beta1.Variable {
-	var v1beta1variables []v1beta1.Variable
-	for _, v := range v1alpha1variables {
-		v1beta1variables = append(v1beta1variables, v1beta1.Variable(v))
-	}
-	return v1beta1variables
-}
-
-func convertValidationActions(v1alpha1actions []v1alpha1.ValidationAction) []v1beta1.ValidationAction {
-	var v1beta1actions []v1beta1.ValidationAction
-	for _, a := range v1alpha1actions {
-		v1beta1actions = append(v1beta1actions, v1beta1.ValidationAction(a))
-	}
-	return v1beta1actions
-}

pkg/validatingadmissionpolicy/validate.go

@@ -14,8 +14,6 @@ import (
 	"golang.org/x/text/language"
 	admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
 	"k8s.io/api/admissionregistration/v1alpha1"
-	"k8s.io/api/admissionregistration/v1beta1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
@@ -80,30 +78,7 @@ func Validate(policyData PolicyData, resource unstructured.Unstructured, client
 		matcher := validatingadmissionpolicy.NewMatcher(matching.NewMatcher(nsLister, client.GetKubeClient()))
 
 		// convert policy from v1alpha1 to v1beta1
-		var namespaceSelector, objectSelector metav1.LabelSelector
-		if policy.Spec.MatchConstraints.NamespaceSelector != nil {
-			namespaceSelector = *policy.Spec.MatchConstraints.NamespaceSelector
-		}
-		if policy.Spec.MatchConstraints.ObjectSelector != nil {
-			objectSelector = *policy.Spec.MatchConstraints.ObjectSelector
-		}
-		v1beta1policy := &v1beta1.ValidatingAdmissionPolicy{
-			Spec: v1beta1.ValidatingAdmissionPolicySpec{
-				FailurePolicy: (*v1beta1.FailurePolicyType)(policy.Spec.FailurePolicy),
-				ParamKind:     (*v1beta1.ParamKind)(policy.Spec.ParamKind),
-				MatchConstraints: &v1beta1.MatchResources{
-					NamespaceSelector:    &namespaceSelector,
-					ObjectSelector:       &objectSelector,
-					ResourceRules:        convertRules(policy.Spec.MatchConstraints.ResourceRules),
-					ExcludeResourceRules: convertRules(policy.Spec.MatchConstraints.ExcludeResourceRules),
-					MatchPolicy:          (*v1beta1.MatchPolicyType)(policy.Spec.MatchConstraints.MatchPolicy),
-				},
-				Validations:      convertValidations(policy.Spec.Validations),
-				AuditAnnotations: convertAuditAnnotations(policy.Spec.AuditAnnotations),
-				MatchConditions:  convertMatchConditions(policy.Spec.MatchConditions),
-				Variables:        convertVariables(policy.Spec.Variables),
-			},
-		}
+		v1beta1policy := ConvertValidatingAdmissionPolicy(policy)
 
 		// construct admission attributes
 		gvr, err = client.Discovery().GetGVRFromGVK(resource.GroupVersionKind())
@@ -114,7 +89,7 @@ func Validate(policyData PolicyData, resource unstructured.Unstructured, client
 
 		// check if policy matches the incoming resource
 		o := admission.NewObjectInterfacesFromScheme(runtime.NewScheme())
-		isMatch, _, _, err := matcher.DefinitionMatches(a, o, v1beta1policy)
+		isMatch, _, _, err := matcher.DefinitionMatches(a, o, &v1beta1policy)
 		if err != nil {
 			return engineResponse, err
 		}
@@ -130,48 +105,8 @@ func Validate(policyData PolicyData, resource unstructured.Unstructured, client
 		} else {
 			for i, binding := range bindings {
 				// convert policy binding from v1alpha1 to v1beta1
-				var namespaceSelector, objectSelector, paramSelector metav1.LabelSelector
-				var resourceRules, excludeResourceRules []v1alpha1.NamedRuleWithOperations
-				var matchPolicy *v1alpha1.MatchPolicyType
-				if binding.Spec.MatchResources != nil {
-					if binding.Spec.MatchResources.NamespaceSelector != nil {
-						namespaceSelector = *binding.Spec.MatchResources.NamespaceSelector
-					}
-					if binding.Spec.MatchResources.ObjectSelector != nil {
-						objectSelector = *binding.Spec.MatchResources.ObjectSelector
-					}
-					resourceRules = binding.Spec.MatchResources.ResourceRules
-					excludeResourceRules = binding.Spec.MatchResources.ExcludeResourceRules
-					matchPolicy = binding.Spec.MatchResources.MatchPolicy
-				}
-
-				var paramRef v1beta1.ParamRef
-				if binding.Spec.ParamRef != nil {
-					paramRef.Name = binding.Spec.ParamRef.Name
-					paramRef.Namespace = binding.Spec.ParamRef.Namespace
-					if binding.Spec.ParamRef.Selector != nil {
-						paramRef.Selector = binding.Spec.ParamRef.Selector
-					} else {
-						paramRef.Selector = &paramSelector
-					}
-					paramRef.ParameterNotFoundAction = (*v1beta1.ParameterNotFoundActionType)(binding.Spec.ParamRef.ParameterNotFoundAction)
-				}
-
-				v1beta1binding := &v1beta1.ValidatingAdmissionPolicyBinding{
-					Spec: v1beta1.ValidatingAdmissionPolicyBindingSpec{
-						PolicyName: binding.Spec.PolicyName,
-						ParamRef:   &paramRef,
-						MatchResources: &v1beta1.MatchResources{
-							NamespaceSelector:    &namespaceSelector,
-							ObjectSelector:       &objectSelector,
-							ResourceRules:        convertRules(resourceRules),
-							ExcludeResourceRules: convertRules(excludeResourceRules),
-							MatchPolicy:          (*v1beta1.MatchPolicyType)(matchPolicy),
-						},
-						ValidationActions: convertValidationActions(binding.Spec.ValidationActions),
-					},
-				}
-				isMatch, err := matcher.BindingMatches(a, o, v1beta1binding)
+				v1beta1binding := ConvertValidatingAdmissionPolicyBinding(binding)
+				isMatch, err := matcher.BindingMatches(a, o, &v1beta1binding)
 				if err != nil {
 					return engineResponse, err
 				}

pkg/validatingadmissionpolicy/version_converter.go

@@ -0,0 +1,139 @@
+package validatingadmissionpolicy
+
+import (
+	admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
+	"k8s.io/api/admissionregistration/v1alpha1"
+	"k8s.io/api/admissionregistration/v1beta1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// ConvertValidatingAdmissionPolicy is used to convert v1alpha1 of ValidatingAdmissionPolicy to v1beta1
+func ConvertValidatingAdmissionPolicy(v1alpha1policy v1alpha1.ValidatingAdmissionPolicy) v1beta1.ValidatingAdmissionPolicy {
+	var namespaceSelector, objectSelector metav1.LabelSelector
+	if v1alpha1policy.Spec.MatchConstraints.NamespaceSelector != nil {
+		namespaceSelector = *v1alpha1policy.Spec.MatchConstraints.NamespaceSelector
+	}
+	if v1alpha1policy.Spec.MatchConstraints.ObjectSelector != nil {
+		objectSelector = *v1alpha1policy.Spec.MatchConstraints.ObjectSelector
+	}
+	v1beta1policy := v1beta1.ValidatingAdmissionPolicy{
+		Spec: v1beta1.ValidatingAdmissionPolicySpec{
+			FailurePolicy: (*v1beta1.FailurePolicyType)(v1alpha1policy.Spec.FailurePolicy),
+			ParamKind:     (*v1beta1.ParamKind)(v1alpha1policy.Spec.ParamKind),
+			MatchConstraints: &v1beta1.MatchResources{
+				NamespaceSelector:    &namespaceSelector,
+				ObjectSelector:       &objectSelector,
+				ResourceRules:        convertRules(v1alpha1policy.Spec.MatchConstraints.ResourceRules),
+				ExcludeResourceRules: convertRules(v1alpha1policy.Spec.MatchConstraints.ExcludeResourceRules),
+				MatchPolicy:          (*v1beta1.MatchPolicyType)(v1alpha1policy.Spec.MatchConstraints.MatchPolicy),
+			},
+			Validations:      convertValidations(v1alpha1policy.Spec.Validations),
+			AuditAnnotations: convertAuditAnnotations(v1alpha1policy.Spec.AuditAnnotations),
+			MatchConditions:  convertMatchConditions(v1alpha1policy.Spec.MatchConditions),
+			Variables:        convertVariables(v1alpha1policy.Spec.Variables),
+		},
+	}
+	return v1beta1policy
+}
+
+// ConvertValidatingAdmissionPolicyBinding is used to convert v1alpha1 of ValidatingAdmissionPolicyBinding to v1beta1
+func ConvertValidatingAdmissionPolicyBinding(v1alpha1binding v1alpha1.ValidatingAdmissionPolicyBinding) v1beta1.ValidatingAdmissionPolicyBinding {
+	var namespaceSelector, objectSelector, paramSelector metav1.LabelSelector
+	var resourceRules, excludeResourceRules []v1alpha1.NamedRuleWithOperations
+	var matchPolicy *v1alpha1.MatchPolicyType
+	if v1alpha1binding.Spec.MatchResources != nil {
+		if v1alpha1binding.Spec.MatchResources.NamespaceSelector != nil {
+			namespaceSelector = *v1alpha1binding.Spec.MatchResources.NamespaceSelector
+		}
+		if v1alpha1binding.Spec.MatchResources.ObjectSelector != nil {
+			objectSelector = *v1alpha1binding.Spec.MatchResources.ObjectSelector
+		}
+		resourceRules = v1alpha1binding.Spec.MatchResources.ResourceRules
+		excludeResourceRules = v1alpha1binding.Spec.MatchResources.ExcludeResourceRules
+		matchPolicy = v1alpha1binding.Spec.MatchResources.MatchPolicy
+	}
+
+	var paramRef v1beta1.ParamRef
+	if v1alpha1binding.Spec.ParamRef != nil {
+		paramRef.Name = v1alpha1binding.Spec.ParamRef.Name
+		paramRef.Namespace = v1alpha1binding.Spec.ParamRef.Namespace
+		if v1alpha1binding.Spec.ParamRef.Selector != nil {
+			paramRef.Selector = v1alpha1binding.Spec.ParamRef.Selector
+		} else {
+			paramRef.Selector = &paramSelector
+		}
+		paramRef.ParameterNotFoundAction = (*v1beta1.ParameterNotFoundActionType)(v1alpha1binding.Spec.ParamRef.ParameterNotFoundAction)
+	}
+
+	v1beta1binding := v1beta1.ValidatingAdmissionPolicyBinding{
+		Spec: v1beta1.ValidatingAdmissionPolicyBindingSpec{
+			PolicyName: v1alpha1binding.Spec.PolicyName,
+			ParamRef:   &paramRef,
+			MatchResources: &v1beta1.MatchResources{
+				NamespaceSelector:    &namespaceSelector,
+				ObjectSelector:       &objectSelector,
+				ResourceRules:        convertRules(resourceRules),
+				ExcludeResourceRules: convertRules(excludeResourceRules),
+				MatchPolicy:          (*v1beta1.MatchPolicyType)(matchPolicy),
+			},
+			ValidationActions: convertValidationActions(v1alpha1binding.Spec.ValidationActions),
+		},
+	}
+	return v1beta1binding
+}
+
+func convertRules(v1alpha1rules []v1alpha1.NamedRuleWithOperations) []v1beta1.NamedRuleWithOperations {
+	var v1beta1rules []v1beta1.NamedRuleWithOperations
+	for _, r := range v1alpha1rules {
+		v1beta1rules = append(v1beta1rules, v1beta1.NamedRuleWithOperations(r))
+	}
+	return v1beta1rules
+}
+
+func convertValidations(v1alpha1validations []v1alpha1.Validation) []v1beta1.Validation {
+	var v1beta1validations []v1beta1.Validation
+	for _, v := range v1alpha1validations {
+		v1beta1validations = append(v1beta1validations, v1beta1.Validation(v))
+	}
+	return v1beta1validations
+}
+
+func convertAuditAnnotations(v1alpha1auditanns []v1alpha1.AuditAnnotation) []v1beta1.AuditAnnotation {
+	var v1beta1auditanns []v1beta1.AuditAnnotation
+	for _, a := range v1alpha1auditanns {
+		v1beta1auditanns = append(v1beta1auditanns, v1beta1.AuditAnnotation(a))
+	}
+	return v1beta1auditanns
+}
+
+func convertMatchConditions(v1alpha1conditions []v1alpha1.MatchCondition) []v1beta1.MatchCondition {
+	var v1beta1conditions []v1beta1.MatchCondition
+	for _, m := range v1alpha1conditions {
+		v1beta1conditions = append(v1beta1conditions, v1beta1.MatchCondition(m))
+	}
+	return v1beta1conditions
+}
+
+func convertVariables(v1alpha1variables []v1alpha1.Variable) []v1beta1.Variable {
+	var v1beta1variables []v1beta1.Variable
+	for _, v := range v1alpha1variables {
+		v1beta1variables = append(v1beta1variables, v1beta1.Variable(v))
+	}
+	return v1beta1variables
+}
+
+func convertValidationActions(v1alpha1actions []v1alpha1.ValidationAction) []v1beta1.ValidationAction {
+	var v1beta1actions []v1beta1.ValidationAction
+	for _, a := range v1alpha1actions {
+		v1beta1actions = append(v1beta1actions, v1beta1.ValidationAction(a))
+	}
+	return v1beta1actions
+}
+
+func ConvertMatchConditionsV1(v1alpha1conditions []v1alpha1.MatchCondition) []admissionregistrationv1.MatchCondition {
+	var v1conditions []admissionregistrationv1.MatchCondition
+	for _, m := range v1alpha1conditions {
+		v1conditions = append(v1conditions, admissionregistrationv1.MatchCondition(m))
+	}
+	return v1conditions
+}

pkg/validation/policy/actions.go

@@ -15,7 +15,7 @@ import (
 	"github.com/kyverno/kyverno/pkg/policy/mutate"
 	"github.com/kyverno/kyverno/pkg/policy/validate"
 	"github.com/kyverno/kyverno/pkg/toggle"
-	"github.com/kyverno/kyverno/pkg/utils/validatingadmissionpolicy"
+	"github.com/kyverno/kyverno/pkg/validatingadmissionpolicy"
 )
 
 // Validation provides methods to validate a rule

pkg/validation/policy/validate.go

@@ -24,15 +24,20 @@ import (
 	"github.com/kyverno/kyverno/pkg/engine/variables/operator"
 	"github.com/kyverno/kyverno/pkg/engine/variables/regex"
 	"github.com/kyverno/kyverno/pkg/logging"
-	"github.com/kyverno/kyverno/pkg/utils/api"
 	apiutils "github.com/kyverno/kyverno/pkg/utils/api"
 	datautils "github.com/kyverno/kyverno/pkg/utils/data"
 	kubeutils "github.com/kyverno/kyverno/pkg/utils/kube"
+	vaputils "github.com/kyverno/kyverno/pkg/validatingadmissionpolicy"
+	admissionregistrationv1alpha1 "k8s.io/api/admissionregistration/v1alpha1"
 	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/validation/field"
 	"k8s.io/apimachinery/pkg/util/yaml"
+	"k8s.io/apiserver/pkg/admission/plugin/validatingadmissionpolicy"
+	"k8s.io/apiserver/pkg/cel/openapi/resolver"
 	"k8s.io/client-go/discovery"
+	"k8s.io/client-go/restmapper"
 )
 
 var (
@@ -394,6 +399,54 @@ func Validate(policy, oldPolicy kyvernov1.PolicyInterface, client dclient.Interf
 
 		checkForDeprecatedOperatorsInRule(rule, &warnings)
 	}
+
+	// check for CEL expression warnings in case of CEL subrules
+	if ok, _ := vaputils.CanGenerateVAP(spec); ok && client != nil {
+		resolver := &resolver.ClientDiscoveryResolver{
+			Discovery: client.GetKubeClient().Discovery(),
+		}
+		groupResources, err := restmapper.GetAPIGroupResources(client.GetKubeClient().Discovery())
+		if err != nil {
+			return nil, err
+		}
+		mapper := restmapper.NewDiscoveryRESTMapper(groupResources)
+		checker := &validatingadmissionpolicy.TypeChecker{
+			SchemaResolver: resolver,
+			RestMapper:     mapper,
+		}
+
+		// build Kubernetes ValidatingAdmissionPolicy
+		vap := &admissionregistrationv1alpha1.ValidatingAdmissionPolicy{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: policy.GetName(),
+			},
+		}
+		err = vaputils.BuildValidatingAdmissionPolicy(client.Discovery(), vap, policy)
+		if err != nil {
+			return nil, err
+		}
+		v1beta1vap := vaputils.ConvertValidatingAdmissionPolicy(*vap)
+
+		// check cel expression warnings
+		ctx := checker.CreateContext(&v1beta1vap)
+		fieldRef := field.NewPath("spec", "rules[0]", "validate", "cel", "expressions")
+		for i, e := range spec.Rules[0].Validation.CEL.Expressions {
+			results := checker.CheckExpression(ctx, e.Expression)
+			if len(results) != 0 {
+				msg := fmt.Sprintf("%s:%s", fieldRef.Index(i).Child("expression").String(), strings.ReplaceAll(results.String(), "\n", ";"))
+				warnings = append(warnings, msg)
+			}
+
+			if e.MessageExpression == "" {
+				continue
+			}
+			results = checker.CheckExpression(ctx, e.MessageExpression)
+			if len(results) != 0 {
+				msg := fmt.Sprintf("%s:%s", fieldRef.Index(i).Child("messageExpression").String(), strings.ReplaceAll(results.String(), "\n", ";"))
+				warnings = append(warnings, msg)
+			}
+		}
+	}
 	return warnings, nil
 }
 
@@ -913,7 +966,7 @@ func validateValidationForEach(foreach []kyvernov1.ForEachValidation, schemaKey
 			}
 		}
 		if fe.ForEachValidation != nil {
-			nestedForEach, err := api.DeserializeJSONArray[kyvernov1.ForEachValidation](fe.ForEachValidation)
+			nestedForEach, err := apiutils.DeserializeJSONArray[kyvernov1.ForEachValidation](fe.ForEachValidation)
 			if err != nil {
 				return schemaKey, err
 			}
@@ -933,7 +986,7 @@ func validateMutationForEach(foreach []kyvernov1.ForEachMutation, schemaKey stri
 			}
 		}
 		if fe.ForEachMutation != nil {
-			nestedForEach, err := api.DeserializeJSONArray[kyvernov1.ForEachMutation](fe.ForEachMutation)
+			nestedForEach, err := apiutils.DeserializeJSONArray[kyvernov1.ForEachMutation](fe.ForEachMutation)
 			if err != nil {
 				return schemaKey, err
 			}

test/conformance/chainsaw/policy-validation/cluster-policy/cel-expressions/README.md

@@ -0,0 +1,10 @@
+## Description
+
+This test tries to create a policy with invalid CEL expression
+## Expected Behavior
+
+Policy should be rejected.
+
+## Related Issue
+
+https://github.com/kyverno/kyverno/issues/9351

test/conformance/chainsaw/policy-validation/cluster-policy/cel-expressions/chainsaw-test.yaml

@@ -0,0 +1,23 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  creationTimestamp: null
+  name: cel-expressions
+spec:
+  steps:
+  - name: Apply the first policy
+    try:
+    - script:
+        content: kubectl apply -f policy-1.yaml
+        check:
+          # This check ensures that the string "undefined field 'automountServiceAccountToken';" is found
+          # in stderr or else fails
+          (contains($stderr, 'undefined field \'automountServiceAccountToken\';')): true
+  - name: Apply the second policy
+    try:
+    - script:
+        content: kubectl apply -f policy-2.yaml
+        check:
+          # This check ensures that the string "undefined field 'replicas';" is found
+          # in stderr or else fails
+          (contains($stderr, 'undefined field \'replicas\';')): true

test/conformance/chainsaw/policy-validation/cluster-policy/cel-expressions/policy-1.yaml

@@ -0,0 +1,23 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: deny-secret-service-account-token
+spec:
+  validationFailureAction: Enforce
+  background: true
+  rules:
+    - name: check-service-account-token
+      match:
+        any:
+        - resources:
+            kinds:
+              - Secret
+      validate:
+        cel:
+          expressions:
+            - message: "long lived API tokens are not allowed"
+              expression: >-
+                !has(object.type) || object.type != "kubernetes.io/service-account-token"
+            - message: "automounting of tokens is not allowed"
+              expression: >-
+                !has(object.automountServiceAccountToken)

test/conformance/chainsaw/policy-validation/cluster-policy/cel-expressions/policy-2.yaml

@@ -0,0 +1,19 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: check-deployment-replicas
+spec:
+  validationFailureAction: Enforce
+  background: true
+  rules:
+    - name: check-deployment-replicas
+      match:
+        any:
+        - resources:
+            kinds:
+              - Deployment
+      validate:
+        cel:
+          expressions:
+            - expression: "object.replicas > 1" # should be "object.spec.replicas > 1"
+              message: "must be replicated"