feat: allow generate pattern changes (#11202)

* chore: remove duplicate test steps

Signed-off-by: ShutingZhao <shuting@nirmata.com>

* feat: remove validation checks upon generate policy changes

Signed-off-by: ShutingZhao <shuting@nirmata.com>

* fix: return nil

Signed-off-by: ShutingZhao <shuting@nirmata.com>

* fix: chainsaw tests

Signed-off-by: ShutingZhao <shuting@nirmata.com>

* fix: restrict rule spec changes except for generate pattern

Signed-off-by: ShutingZhao <shuting@nirmata.com>

* chore: rename tests

Signed-off-by: ShutingZhao <shuting@nirmata.com>

* fix: unit tests

Signed-off-by: ShutingZhao <shuting@nirmata.com>

* chore: upadte e2e matrix

Signed-off-by: ShutingZhao <shuting@nirmata.com>

---------

Signed-off-by: ShutingZhao <shuting@nirmata.com>

pkg/policy/generate.go

@@ -9,12 +9,15 @@ import (
 	kyvernov2 "github.com/kyverno/kyverno/api/kyverno/v2"
 	"github.com/kyverno/kyverno/pkg/autogen"
 	"github.com/kyverno/kyverno/pkg/background/common"
+	backgroundcommon "github.com/kyverno/kyverno/pkg/background/common"
 	generateutils "github.com/kyverno/kyverno/pkg/background/generate"
 	"github.com/kyverno/kyverno/pkg/config"
+	datautils "github.com/kyverno/kyverno/pkg/utils/data"
 	engineutils "github.com/kyverno/kyverno/pkg/utils/engine"
 	"go.uber.org/multierr"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 )
 
 func (pc *policyController) handleGenerate(policyKey string, policy kyvernov1.PolicyInterface) error {
@@ -225,28 +228,99 @@ func (pc *policyController) buildUrForDataRuleChanges(policy kyvernov1.PolicyInt
 	return ur, nil
 }
 
+func (pc *policyController) unlabelDownstream(selector updatedResource) {
+	for _, ruleSelector := range selector.ruleResources {
+		for _, kind := range ruleSelector.kinds {
+			updated, err := pc.client.ListResource(context.TODO(), "", kind, "", &metav1.LabelSelector{
+				MatchLabels: map[string]string{
+					backgroundcommon.GeneratePolicyLabel:          selector.policy,
+					backgroundcommon.GeneratePolicyNamespaceLabel: selector.policyNamespace,
+					backgroundcommon.GenerateRuleLabel:            ruleSelector.rule,
+				},
+			},
+			)
+			if err != nil {
+				utilruntime.HandleError(fmt.Errorf("failed to list old targets: %v", err))
+				continue
+			}
+
+			for _, obj := range updated.Items {
+				labels := obj.GetLabels()
+				delete(labels, backgroundcommon.GeneratePolicyLabel)
+				delete(labels, backgroundcommon.GeneratePolicyNamespaceLabel)
+				delete(labels, backgroundcommon.GenerateRuleLabel)
+				obj.SetLabels(labels)
+				_, err = pc.client.UpdateResource(context.TODO(), obj.GetAPIVersion(), obj.GetKind(), obj.GetNamespace(), &obj, false)
+				if err != nil {
+					utilruntime.HandleError(fmt.Errorf("failed to un-label old targets %s/%s/%s/%s: %v", obj.GetAPIVersion(), obj.GetKind(), obj.GetNamespace(), obj.GetName(), err))
+					continue
+				}
+			}
+		}
+	}
+}
+
+type updatedResource struct {
+	policy          string
+	policyNamespace string
+	ruleResources   []ruleResource
+}
+
+type ruleResource struct {
+	rule  string
+	kinds []string
+}
+
 // ruleDeletion returns true if any rule is deleted, along with deleted rules
-func ruleDeletion(old, new kyvernov1.PolicyInterface) (_ kyvernov1.PolicyInterface, ruleDeleted bool) {
+func ruleChange(old, new kyvernov1.PolicyInterface) (_ kyvernov1.PolicyInterface, ruleDeleted bool, _ updatedResource) {
 	if !new.GetDeletionTimestamp().IsZero() {
-		return nil, false
+		return nil, false, updatedResource{}
 	}
 
 	newRules := new.GetSpec().Rules
 	oldRules := old.GetSpec().Rules
-	newRulesMap := make(map[string]bool, len(newRules))
+	newRulesMap := make(map[string]kyvernov1.Rule, len(newRules))
 	var deletedRules []kyvernov1.Rule
+	updatedResources := updatedResource{
+		policy:          new.GetName(),
+		policyNamespace: new.GetNamespace(),
+	}
 
 	for _, r := range newRules {
-		newRulesMap[r.Name] = true
+		newRulesMap[r.Name] = r
 	}
-	for _, r := range oldRules {
-		if exist := newRulesMap[r.Name]; !exist {
-			deletedRules = append(deletedRules, r)
+	for _, oldRule := range oldRules {
+		if newRule, exist := newRulesMap[oldRule.Name]; !exist {
+			deletedRules = append(deletedRules, oldRule)
 			ruleDeleted = true
+		} else {
+			ruleRsrc := ruleResource{rule: oldRule.Name}
+			old, new := oldRule.Generation, newRule.Generation
+			if old.ResourceSpec != new.ResourceSpec || old.Clone != new.Clone {
+				ruleRsrc.kinds = append(ruleRsrc.kinds, old.ResourceSpec.GetKind())
+			}
+			if !datautils.DeepEqual(old.CloneList, new.CloneList) {
+				ruleRsrc.kinds = append(ruleRsrc.kinds, old.CloneList.Kinds...)
+			}
+
+			for _, oldForeach := range old.ForEachGeneration {
+				for _, newForeach := range new.ForEachGeneration {
+					if oldForeach.List == newForeach.List {
+						if oldForeach.ResourceSpec != newForeach.ResourceSpec || oldForeach.Clone != newForeach.Clone {
+							ruleRsrc.kinds = append(ruleRsrc.kinds, old.ResourceSpec.GetKind())
+						}
+
+						if !datautils.DeepEqual(oldForeach.CloneList, newForeach.CloneList) {
+							ruleRsrc.kinds = append(ruleRsrc.kinds, old.CloneList.Kinds...)
+						}
+					}
+				}
+			}
+			updatedResources.ruleResources = append(updatedResources.ruleResources, ruleRsrc)
 		}
 	}
 
-	return buildPolicyWithDeletedRules(old, deletedRules), ruleDeleted
+	return buildPolicyWithDeletedRules(old, deletedRules), ruleDeleted, updatedResources
 }
 
 func buildPolicyWithDeletedRules(policy kyvernov1.PolicyInterface, deletedRules []kyvernov1.Rule) kyvernov1.PolicyInterface {

pkg/policy/policy_controller.go

@@ -201,11 +201,13 @@ func (pc *policyController) updatePolicy(old, cur interface{}) {
 	}
 
 	logger.V(2).Info("updating policy", "name", oldP.GetName())
-	if deleted, ok := ruleDeletion(oldP, curP); ok {
+	if deleted, ok, selector := ruleChange(oldP, curP); ok {
 		err := pc.createURForDownstreamDeletion(deleted)
 		if err != nil {
 			utilruntime.HandleError(fmt.Errorf("failed to create UR on rule deletion, clean up downstream resource may be failed: %v", err))
 		}
+	} else {
+		pc.unlabelDownstream(selector)
 	}
 
 	pc.enqueuePolicy(curP)

pkg/validation/policy/generate.go

@@ -11,61 +11,66 @@ import (
 	"k8s.io/apimachinery/pkg/util/sets"
 )
 
-func immutableGenerateFields(new, old kyvernov1.PolicyInterface) error {
+func immutableGenerateFields(new, old kyvernov1.PolicyInterface) (string, error) {
 	if new == nil || old == nil {
-		return nil
+		return "", nil
 	}
 
-	if !new.GetSpec().HasGenerate() {
-		return nil
-	}
-
-	oldRuleHashes, err := buildHashes(old.GetSpec().Rules)
+	oldRuleHashes, oldGenerationHashes, err := buildHashes(old.GetSpec().Rules)
 	if err != nil {
-		return err
+		return "", err
 	}
-	newRuleHashes, err := buildHashes(new.GetSpec().Rules)
+	newRuleHashes, newGenerationHashes, err := buildHashes(new.GetSpec().Rules)
 	if err != nil {
-		return err
+		return "", err
 	}
 
-	switch len(old.GetSpec().Rules) <= len(new.GetSpec().Rules) {
-	case true:
-		if newRuleHashes.IsSuperset(oldRuleHashes) {
-			return nil
-		} else {
-			return errors.New("change of immutable fields for a generate rule is disallowed")
-		}
-	case false:
-		if oldRuleHashes.IsSuperset(newRuleHashes) {
-			return nil
-		} else {
-			return errors.New("rule deletion - change of immutable fields for a generate rule is disallowed")
-		}
+	if !newGenerationHashes.IsSuperset(oldGenerationHashes) {
+		return "changes in the generate rule pattern could result in stale targets", nil
 	}
-	return nil
+
+	if !newRuleHashes.IsSuperset(oldRuleHashes) {
+		return "", errors.New("changes of the rule spec in a generate rule is disallowed")
+	}
+
+	return "", nil
 }
 
-func resetMutableFields(rule kyvernov1.Rule) *kyvernov1.Rule {
+func resetMutableFields(rule kyvernov1.Rule) (*kyvernov1.Rule, *kyvernov1.Generation) {
 	new := new(kyvernov1.Rule)
 	rule.DeepCopyInto(new)
-	new.Generation.Synchronize = true
-	new.Generation.SetData(nil)
-	new.Generation.ForEachGeneration = nil
-	new.Generation.OrphanDownstreamOnPolicyDelete = true
-	return new
+	generation := new.Generation
+	new.Generation = nil
+	generation.Synchronize = true
+	generation.SetData(nil)
+	generation.ForEachGeneration = nil
+	generation.OrphanDownstreamOnPolicyDelete = true
+	generation.GenerateExisting = nil
+
+	return new, generation
 }
 
-func buildHashes(rules []kyvernov1.Rule) (sets.Set[string], error) {
-	ruleHashes := sets.New[string]()
+func buildHashes(rules []kyvernov1.Rule) (ruleHashes sets.Set[string], generationHashes sets.Set[string], _ error) {
+	ruleHashes, generationHashes = sets.New[string](), sets.New[string]()
+
 	for _, rule := range rules {
-		r := resetMutableFields(rule)
-		data, err := json.Marshal(r)
+		if !rule.HasGenerate() {
+			continue
+		}
+		r, generation := resetMutableFields(rule)
+		data, err := json.Marshal(generation)
 		if err != nil {
-			return ruleHashes, fmt.Errorf("failed to create hash from the generate rule %v", err)
+			return ruleHashes, generationHashes, fmt.Errorf("failed to create hash from the generate rule %v", err)
 		}
 		hash := md5.Sum(data) //nolint:gosec
+		generationHashes.Insert(hex.EncodeToString(hash[:]))
+
+		data, err = json.Marshal(r)
+		if err != nil {
+			return ruleHashes, generationHashes, fmt.Errorf("failed to create hash from the generate rule %v", err)
+		}
+		hash = md5.Sum(data) //nolint:gosec
 		ruleHashes.Insert(hex.EncodeToString(hash[:]))
 	}
-	return ruleHashes, nil
+	return ruleHashes, generationHashes, nil
 }

pkg/validation/policy/validate.go

@@ -227,7 +227,8 @@ func Validate(policy, oldPolicy kyvernov1.PolicyInterface, client dclient.Interf
 		}
 	}
 
-	if err := immutableGenerateFields(policy, oldPolicy); err != nil {
+	if warning, err := immutableGenerateFields(policy, oldPolicy); warning != "" || err != nil {
+		warnings = append(warnings, fmt.Sprintf("no synchronization will be performed to the old target resource upon policy updates: %s", warning))
 		return warnings, err
 	}
 

pkg/validation/policy/validate_test.go

@@ -2209,12 +2209,13 @@ func Test_Validate_RuleImageExtractorsJMESPath(t *testing.T) {
 	assert.Equal(t, expectedErr.Error(), actualErr.Error())
 }
 
-func Test_ImmutableGenerateFields(t *testing.T) {
+func Test_GenerateFieldsUpdates(t *testing.T) {
 	tests := []struct {
-		name        string
-		oldPolicy   []byte
-		newPolicy   []byte
-		expectedErr bool
+		name          string
+		oldPolicy     []byte
+		newPolicy     []byte
+		expectedErr   bool
+		expectWarning bool
 	}{
 		{
 			name: "update-apiVersion",
@@ -2292,7 +2293,8 @@ func Test_ImmutableGenerateFields(t *testing.T) {
 					]
 				}
 			}`),
-			expectedErr: true,
+			expectedErr:   false,
+			expectWarning: true,
 		},
 		{
 			name: "update-kind",
@@ -2370,7 +2372,8 @@ func Test_ImmutableGenerateFields(t *testing.T) {
         ]
     }
 }`),
-			expectedErr: true,
+			expectedErr:   false,
+			expectWarning: true,
 		},
 		{
 			name: "update-namespace",
@@ -2448,7 +2451,8 @@ func Test_ImmutableGenerateFields(t *testing.T) {
 					]
 				}
 			}`),
-			expectedErr: true,
+			expectedErr:   false,
+			expectWarning: true,
 		},
 		{
 			name: "update-name",
@@ -2526,7 +2530,8 @@ func Test_ImmutableGenerateFields(t *testing.T) {
 					]
 				}
 			}`),
-			expectedErr: true,
+			expectedErr:   false,
+			expectWarning: true,
 		},
 		{
 			name: "update-sync-flag",
@@ -2604,7 +2609,8 @@ func Test_ImmutableGenerateFields(t *testing.T) {
 					]
 				}
 			}`),
-			expectedErr: false,
+			expectedErr:   false,
+			expectWarning: false,
 		},
 		{
 			name: "update-clone-name",
@@ -2682,7 +2688,8 @@ func Test_ImmutableGenerateFields(t *testing.T) {
 					]
 				}
 			}`),
-			expectedErr: true,
+			expectedErr:   false,
+			expectWarning: true,
 		},
 		{
 			name: "update-clone-namespace",
@@ -2760,7 +2767,8 @@ func Test_ImmutableGenerateFields(t *testing.T) {
 					]
 				}
 			}`),
-			expectedErr: true,
+			expectedErr:   false,
+			expectWarning: true,
 		},
 		{
 			name: "update-clone-namespace-unset-new",
@@ -2837,7 +2845,8 @@ func Test_ImmutableGenerateFields(t *testing.T) {
 					]
 				}
 			}`),
-			expectedErr: true,
+			expectedErr:   false,
+			expectWarning: true,
 		},
 		{
 			name: "update-cloneList-kinds",
@@ -2916,7 +2925,8 @@ func Test_ImmutableGenerateFields(t *testing.T) {
 					]
 				}
 			}`),
-			expectedErr: true,
+			expectedErr:   false,
+			expectWarning: true,
 		},
 		{
 			name: "update-cloneList-namespace",
@@ -2995,7 +3005,8 @@ func Test_ImmutableGenerateFields(t *testing.T) {
 					]
 				}
 			}`),
-			expectedErr: true,
+			expectedErr:   false,
+			expectWarning: true,
 		},
 		{
 			name: "update-cloneList-selector",
@@ -3085,7 +3096,8 @@ func Test_ImmutableGenerateFields(t *testing.T) {
 					]
 				}
 			}`),
-			expectedErr: true,
+			expectedErr:   false,
+			expectWarning: true,
 		},
 		{
 			name: "update-clone-List-selector-unset",
@@ -3170,7 +3182,8 @@ func Test_ImmutableGenerateFields(t *testing.T) {
 					]
 				}
 			}`),
-			expectedErr: true,
+			expectedErr:   false,
+			expectWarning: true,
 		},
 		{
 			name: "update-cloneList-selector-nochange",
@@ -3260,7 +3273,8 @@ func Test_ImmutableGenerateFields(t *testing.T) {
 					]
 				}
 			}`),
-			expectedErr: false,
+			expectedErr:   false,
+			expectWarning: false,
 		},
 	}
 
@@ -3271,8 +3285,10 @@ func Test_ImmutableGenerateFields(t *testing.T) {
 		err = json.Unmarshal(test.newPolicy, &new)
 		assert.Nil(t, err)
 
-		err = immutableGenerateFields(new, old)
+		warning, err := immutableGenerateFields(new, old)
+		golangassert.Assert(t, (warning != "") == test.expectWarning, test.name, err)
 		golangassert.Assert(t, (err != nil) == test.expectedErr, test.name, err)
+
 	}
 }
 

test/conformance/chainsaw/e2e-matrix.json

@@ -61,9 +61,9 @@
     "^generate$/^policy$/^standard$/^data$/^nosync$/^(pol-data-nosync-create-policy-invalid|pol-data-nosync-delete-downstream|pol-data-nosync-delete-policy|pol-data-nosync-delete-rule|pol-data-nosync-delete-rule-deprecated|pol-data-nosync-delete-trigger|pol-data-nosync-modify-downstream|pol-data-nosync-modify-rule|pol-data-nosync-update-trigger-no-match)\\[.*\\]$",
     "^generate$/^policy$/^standard$/^data$/^sync$/^(pol-data-sync-create-policy-invalid|pol-data-sync-create-policy-valid|pol-data-sync-delete-downstream|pol-data-sync-delete-policy|pol-data-sync-delete-rule|pol-data-sync-delete-rule-deprecated|pol-data-sync-delete-trigger|pol-data-sync-modify-downstream|pol-data-sync-modify-rule|pol-data-sync-modify-rule-deprecated|pol-data-sync-update-trigger-no-match)\\[.*\\]$",
     "^generate$/^policy$/^standard$/^existing$/^(match-trigger-namespace|match-trigger-namespace-deprecated|non-match-trigger-namespace|non-match-trigger-namespace-deprecated)\\[.*\\]$",
-    "^generate$/^validation$/^clusterpolicy$/^(cloneList|immutable-clone|immutable-clonelist|immutable-downstream|immutable-rule-spec|orphan|prevent-loop|target-namespace-scope|use-generate-existing-on-policy-update)\\[.*\\]$",
+    "^generate$/^validation$/^clusterpolicy$/^(cloneList|immutable-rule-spec|orphan|prevent-loop|target-namespace-scope|use-generate-existing-on-policy-update|warn-clone-change|warn-clonelist-change|warn-downstream-change)\\[.*\\]$",
     "^generate$/^validation$/^clusterpolicy$/^permissions$/^(no-permission|same-kind)\\[.*\\]$",
-    "^generate$/^validation$/^policy$/^(cloneList|immutable-clone|immutable-clonelist|immutable-downstream|immutable-rule-spec|permissions|prevent-loop|target-namespace-scope|use-generate-existing-on-policy-update)\\[.*\\]$"
+    "^generate$/^validation$/^policy$/^(cloneList|immutable-rule-spec|permissions|prevent-loop|target-namespace-scope|use-generate-existing-on-policy-update|warn-clone-change|warn-clonelist-change|warn-downstream-change)\\[.*\\]$"
   ],
   "generate-validating-admission-policy": [
     "^generate-validating-admission-policy$/^clusterpolicy$/^standard$/^generate$/^(block-ephemeral-containers|block-exec-in-pods|cpol-all-match-resource|cpol-any-exclude-namespace-match-resource|cpol-any-exclude-resource|cpol-any-exclude-resource-match-with-namespace-selector|cpol-any-exclude-resource-match-with-object-selector|cpol-any-match-multiple-resources|cpol-any-match-resource|cpol-any-match-resources-by-names|cpol-match-all-exclude-one|cpol-match-kind-with-wildcard|cpol-match-resource-in-specific-namespace|cpol-with-an-exception|cpol-with-an-exception-excluding-namespaces|cpol-with-two-exceptions)\\[.*\\]$",

test/conformance/chainsaw/generate/clusterpolicy/standard/clone/sync/cpol-clone-list-sync-update-source/3-0-update-cluster-policy.yaml

@@ -0,0 +1,31 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: sync-with-multi-clone-update
+spec:
+  rules:
+  - name: sync-secret
+    match:
+      any:
+      - resources:
+          kinds:
+          - Namespace
+    exclude:
+      any:
+      - resources:
+          namespaces:
+          - kube-system
+          - default
+          - kube-public
+          - kyverno
+    generate:
+      generateExisting: false
+      namespace: "{{request.object.metadata.name}}"
+      synchronize : true
+      cloneList:
+        namespace: default
+        kinds:
+          - v1/Secret
+        selector:
+          matchLabels:
+            allowedToBeCloned: "true"

test/conformance/chainsaw/generate/clusterpolicy/standard/clone/sync/cpol-clone-list-sync-update-source/4-0-update-source.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: bootstrap-config
+  namespace: default
+  labels:
+    allowedToBeCloned:  "true"
+data:
+  initial_lives: "100"

test/conformance/chainsaw/generate/clusterpolicy/standard/clone/sync/cpol-clone-list-sync-update-source/README.md

@@ -4,7 +4,8 @@ This test verifies the synchronize behavior of generated resource, if the select
 
 ## Expected Behavior
 
-This test ensures that update of source resource(ConfigMap) match selected using `allowedToBeCloned:  "true"` label get synchronized with target resource created by a ClusterPolicy `generate.cloneList` rule, otherwise the test fails. 
+1. update source resource (configmap) match selected using `allowedToBeCloned:  "true"` label, the change should be synced to the target configmap.
+2. remove configmap from the `cloneList.kinds` in the policy, update the source configmap, the change should not be synced to the previous cloned configmap
 
 ## Reference Issue(s)
 

test/conformance/chainsaw/generate/clusterpolicy/standard/clone/sync/cpol-clone-list-sync-update-source/chainsaw-test.yaml

@@ -7,28 +7,34 @@ spec:
   - name: step-00
     try:
     - apply:
-        file: permissions.yaml
+        file: 0-0-permissions.yaml
     - apply:
-        file: manifests.yaml
+        file: 0-1-manifests.yaml
     - apply:
-        file: cluster-policy.yaml
+        file: 0-2-cluster-policy.yaml
     - assert:
-        file: cluster-policy-ready.yaml
+        file: 0-3-cluster-policy-ready.yaml
   - name: step-01
     try:
     - apply:
-        file: ns.yaml
+        file: 1-0-ns.yaml
     - assert:
-        file: resource-assert.yaml
+        file: 1-1-resource-assert.yaml
   - name: step-02
     try:
     - apply:
-        file: ns.yaml
+        file: 2-0-update-source.yaml
     - assert:
-        file: resource-assert.yaml
+        file: 2-1-synchronized-target.yaml
   - name: step-03
     try:
     - apply:
-        file: update-source.yaml
+        file: 3-0-update-cluster-policy.yaml
     - assert:
-        file: synchronized-target.yaml
+        file: 0-3-cluster-policy-ready.yaml
+  - name: step-04
+    try:
+    - apply:
+        file: 4-0-update-source.yaml
+    - assert:
+        file: 2-1-synchronized-target.yaml

test/conformance/chainsaw/generate/clusterpolicy/standard/data/sync/cpol-data-sync-delete-rule/1-2-policy.yaml

@@ -30,6 +30,7 @@ spec:
         metadata:
           labels:
             somekey: somevalue
+            foo: bar
         data:
           ZK_ADDRESS: "192.168.10.10:2181,192.168.10.11:2181,192.168.10.12:2181"
           KAFKA_ADDRESS: "192.168.10.13:9092,192.168.10.14:9092,192.168.10.15:9092"

test/conformance/chainsaw/generate/clusterpolicy/standard/data/sync/cpol-data-sync-delete-rule/4-1-rule-update.yaml

@@ -0,0 +1,65 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: multiple-gens
+spec:
+  rules:
+  - name: k-kafka-address
+    match:
+      any:
+      - resources:
+          kinds:
+          - Namespace
+    exclude:
+      any:
+      - resources:
+          namespaces:
+          - kube-system
+          - default
+          - kube-public
+          - kyverno
+    generate:
+      generateExisting: false
+      synchronize: true
+      apiVersion: v1
+      kind: ConfigMap
+      name: zk-kafka-address-new
+      namespace: "{{request.object.metadata.name}}"
+      data:
+        kind: ConfigMap
+        metadata:
+          labels:
+            somekey: somevalue
+            foo: bar
+        data:
+          ZK_ADDRESS: "192.168.10.10:2181,192.168.10.11:2181,192.168.10.12:2181"
+          KAFKA_ADDRESS: "192.168.10.13:9092,192.168.10.14:9092,192.168.10.15:9092"
+  - name: super-secret
+    match:
+      any:
+      - resources:
+          kinds:
+          - Namespace
+    exclude:
+      any:
+      - resources:
+          namespaces:
+          - kube-system
+          - default
+          - kube-public
+          - kyverno
+    generate:
+      generateExisting: false
+      synchronize: true
+      apiVersion: v1
+      kind: Secret
+      name: supersecret
+      namespace: "{{request.object.metadata.name}}"
+      data:
+        kind: Secret
+        type: Opaque
+        metadata:
+          labels:
+            somekey: somesecretvalue
+        data:
+          mysupersecretkey: bXlzdXBlcnNlY3JldHZhbHVl

test/conformance/chainsaw/generate/clusterpolicy/standard/data/sync/cpol-data-sync-delete-rule/5-1-namespace-update.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: cpol-data-sync-delete-rule
+  labels:
+    foo: bar

test/conformance/chainsaw/generate/clusterpolicy/standard/data/sync/cpol-data-sync-delete-rule/5-2-configmap-new.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+data:
+  KAFKA_ADDRESS: 192.168.10.13:9092,192.168.10.14:9092,192.168.10.15:9092
+  ZK_ADDRESS: 192.168.10.10:2181,192.168.10.11:2181,192.168.10.12:2181
+kind: ConfigMap
+metadata:
+  labels:
+    somekey: somevalue
+  name: zk-kafka-address-new
+  namespace: cpol-data-sync-delete-rule

test/conformance/chainsaw/generate/clusterpolicy/standard/data/sync/cpol-data-sync-delete-rule/README.md

@@ -4,7 +4,9 @@ This test checks to ensure that deletion of a rule in a ClusterPolicy generate r
 
 ## Expected Behavior
 
-The downstream (generated) resource is expected to be deleted if the corresponding rule within a ClusterPolicy is deleted. If it is not deleted, the test fails. If it is deleted, the test passes.
+1. create the namespace that triggers the policy, two rules should be applied and generate a configmap and a secret correspondingly.
+2. update the configmap rule and trigger the policy again, a new configmap should be generated.
+3. delete the newly updated configmap rule, the new configmap should be deleted while the old configmap preserves.
 
 ## Reference Issue(s)
 

test/conformance/chainsaw/generate/clusterpolicy/standard/data/sync/cpol-data-sync-delete-rule/chainsaw-test.yaml

@@ -7,34 +7,42 @@ spec:
   - name: step-01
     try:
     - apply:
-        file: permissions.yaml
+        file: 1-1-permissions.yaml
     - apply:
-        file: policy.yaml
+        file: 1-2-policy.yaml
     - assert:
-        file: policy-ready.yaml
+        file: 1-3-policy-ready.yaml
   - name: step-02
     try:
     - apply:
-        file: chainsaw-step-02-apply-1-1.yaml
+        file: 2-1-namespace.yaml
   - name: step-03
     try:
     - assert:
-        file: secret.yaml
+        file: 3-1-secret.yaml
     - assert:
-        file: configmap.yaml
+        file: 3-2-configmap.yaml
   - name: step-04
     try:
     - apply:
-        file: delete-rule.yaml
+        file: 4-1-rule-update.yaml
     - assert:
-        file: policy-ready.yaml
+        file: 1-3-policy-ready.yaml
   - name: step-05
     try:
-    - sleep:
-        duration: 3s
+    - apply:
+        file: 5-1-namespace-update.yaml
+    - assert:
+        file: 5-2-configmap-new.yaml
   - name: step-06
     try:
+    - apply:
+        file: 6-1-delete-rule.yaml
     - assert:
-        file: secret.yaml
+        file: 1-3-policy-ready.yaml
+  - name: step-07
+    try:
+    - assert:
+        file: 3-2-configmap.yaml
     - error:
-        file: configmap.yaml
+        file: 5-2-configmap-new.yaml

test/conformance/chainsaw/generate/validation/clusterpolicy/immutable-clone/chainsaw-test.yaml

@@ -1,26 +0,0 @@
-apiVersion: chainsaw.kyverno.io/v1alpha1
-kind: Test
-metadata:
-  name: immutable-clone
-spec:
-  steps:
-  - name: step-01
-    try:
-    - apply:
-        file: permissions.yaml
-    - apply:
-        file: policy.yaml
-    - assert:
-        file: policy-ready.yaml
-  - name: step-02
-    try:
-    - apply:
-        expect:
-        - check:
-            ($error != null): true
-        file: update-name.yaml
-    - apply:
-        expect:
-        - check:
-            ($error != null): true
-        file: update-namespace.yaml

test/conformance/chainsaw/generate/validation/clusterpolicy/immutable-clonelist/chainsaw-test.yaml

@@ -1,29 +0,0 @@
-apiVersion: chainsaw.kyverno.io/v1alpha1
-kind: Test
-metadata:
-  name: immutable-clonelist
-spec:
-  steps:
-  - name: step-01
-    try:
-    - apply:
-        file: cluster-policy.yaml
-    - assert:
-        file: cluster-policy-ready.yaml
-  - name: step-02
-    try:
-    - apply:
-        expect:
-        - check:
-            ($error != null): true
-        file: update-ns.yaml
-    - apply:
-        expect:
-        - check:
-            ($error != null): true
-        file: update-kinds.yaml
-    - apply:
-        expect:
-        - check:
-            ($error != null): true
-        file: update-selector.yaml

test/conformance/chainsaw/generate/validation/clusterpolicy/immutable-downstream/chainsaw-test.yaml

@@ -1,34 +0,0 @@
-apiVersion: chainsaw.kyverno.io/v1alpha1
-kind: Test
-metadata:
-  name: immutable-downstream
-spec:
-  steps:
-  - name: step-01
-    try:
-    - apply:
-        file: chainsaw-step-01-apply-1-1.yaml
-    - assert:
-        file: chainsaw-step-01-assert-1-1.yaml
-  - name: step-02
-    try:
-    - apply:
-        expect:
-        - check:
-            ($error != null): true
-        file: update-name.yaml
-    - apply:
-        expect:
-        - check:
-            ($error != null): true
-        file: update-apiversion.yaml
-    - apply:
-        expect:
-        - check:
-            ($error != null): true
-        file: update-namespace.yaml
-    - apply:
-        expect:
-        - check:
-            ($error != null): true
-        file: update-kind.yaml

test/conformance/chainsaw/generate/validation/clusterpolicy/warn-clone-change/chainsaw-test.yaml

@@ -0,0 +1,22 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  name: immutable-clone
+spec:
+  steps:
+  - name: step-01
+    try:
+    - apply:
+        file: permissions.yaml
+    - apply:
+        file: policy.yaml
+    - assert:
+        file: policy-ready.yaml
+  - name: step-02
+    try:
+    - script:
+        content: > 
+          kubectl apply -f update-name.yaml 2>&1 | grep -q "Warning: no synchronization will be performed to the old target resource upon policy updates"
+    - script:
+        content: > 
+          kubectl apply -f update-namespace.yaml 2>&1 | grep -q "Warning: no synchronization will be performed to the old target resource upon policy updates"

test/conformance/chainsaw/generate/validation/clusterpolicy/warn-clonelist-change/chainsaw-test.yaml

@@ -0,0 +1,23 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  name: immutable-clonelist
+spec:
+  steps:
+  - name: step-01
+    try:
+    - apply:
+        file: cluster-policy.yaml
+    - assert:
+        file: cluster-policy-ready.yaml
+  - name: step-02
+    try:
+    - script:
+        content: > 
+          kubectl apply -f update-ns.yaml 2>&1 | grep -q "Warning: no synchronization will be performed to the old target resource upon policy updates"
+    - script:
+        content: > 
+          kubectl apply -f update-kinds.yaml 2>&1 | grep -q "Warning: no synchronization will be performed to the old target resource upon policy updates"
+    - script:
+        content: > 
+          kubectl apply -f update-selector.yaml 2>&1 | grep -q "Warning: no synchronization will be performed to the old target resource upon policy updates"

test/conformance/chainsaw/generate/validation/clusterpolicy/warn-downstream-change/chainsaw-test.yaml

@@ -0,0 +1,26 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  name: immutable-downstream
+spec:
+  steps:
+  - name: step-01
+    try:
+    - apply:
+        file: chainsaw-step-01-apply-1-1.yaml
+    - assert:
+        file: chainsaw-step-01-assert-1-1.yaml
+  - name: step-02
+    try:
+    - script:
+        content: > 
+          kubectl apply -f update-name.yaml 2>&1 | grep -q "Warning: no synchronization will be performed to the old target resource upon policy updates"
+    - script:
+        content: > 
+          kubectl apply -f update-namespace.yaml 2>&1 | grep -q "Warning: no synchronization will be performed to the old target resource upon policy updates"
+    - script:
+        content: > 
+          kubectl apply -f update-apiversion.yaml 2>&1 | grep -q "Warning: no synchronization will be performed to the old target resource upon policy updates"
+    - script:
+        content: > 
+          kubectl apply -f update-kind.yaml 2>&1 | grep -q "Warning: no synchronization will be performed to the old target resource upon policy updates"

test/conformance/chainsaw/generate/validation/policy/immutable-clone/update-namespace.yaml

@@ -1,22 +0,0 @@
-apiVersion: kyverno.io/v2beta1
-kind: Policy
-metadata:
-  name: generate-update-clone
-  namespace: default
-spec:
-  rules:
-  - name: clone-secret
-    match:
-      any:
-      - resources:
-          kinds:
-          - ConfigMap
-    generate:
-      apiVersion: v1
-      kind: Secret
-      name: regcred
-      namespace: default
-      synchronize: true
-      clone:
-        namespace: ichangethis
-        name: regcred

test/conformance/chainsaw/generate/validation/policy/immutable-clonelist/chainsaw-test.yaml

@@ -1,29 +0,0 @@
-apiVersion: chainsaw.kyverno.io/v1alpha1
-kind: Test
-metadata:
-  name: immutable-clonelist
-spec:
-  steps:
-  - name: step-01
-    try:
-    - apply:
-        file: policy.yaml
-    - assert:
-        file: policy-ready.yaml
-  - name: step-02
-    try:
-    - apply:
-        expect:
-        - check:
-            ($error != null): true
-        file: update-ns.yaml
-    - apply:
-        expect:
-        - check:
-            ($error != null): true
-        file: update-kinds.yaml
-    - apply:
-        expect:
-        - check:
-            ($error != null): true
-        file: update-selector.yaml

test/conformance/chainsaw/generate/validation/policy/immutable-clonelist/update-ns.yaml

@@ -1,25 +0,0 @@
-apiVersion: kyverno.io/v1
-kind: Policy
-metadata:
-  name: generate-update-clonelist
-  namespace: default
-spec:
-  generateExisting: false
-  rules:
-  - name: sync-secret
-    match:
-      any:
-      - resources:
-          kinds:
-          - ConfigMap
-    generate:
-      namespace: default
-      synchronize : true
-      cloneList:
-        namespace: update-clonelist-ns
-        kinds:
-          - v1/Secret
-          - v1/ConfigMap
-        selector:
-          matchLabels:
-            allowedToBeCloned: "true"

test/conformance/chainsaw/generate/validation/policy/immutable-downstream/chainsaw-test.yaml

@@ -1,34 +0,0 @@
-apiVersion: chainsaw.kyverno.io/v1alpha1
-kind: Test
-metadata:
-  name: immutable-downstream
-spec:
-  steps:
-  - name: step-01
-    try:
-    - apply:
-        file: chainsaw-step-01-apply-1-1.yaml
-    - assert:
-        file: chainsaw-step-01-assert-1-1.yaml
-  - name: step-02
-    try:
-    - apply:
-        expect:
-        - check:
-            ($error != null): true
-        file: update-name.yaml
-    - apply:
-        expect:
-        - check:
-            ($error != null): true
-        file: update-apiversion.yaml
-    - apply:
-        expect:
-        - check:
-            ($error != null): true
-        file: update-namespace.yaml
-    - apply:
-        expect:
-        - check:
-            ($error != null): true
-        file: update-kind.yaml

test/conformance/chainsaw/generate/validation/policy/immutable-downstream/update-namespace.yaml

@@ -1,28 +0,0 @@
-apiVersion: kyverno.io/v1
-kind: Policy
-metadata:
-  name: generate-update-downstream-rule
-  namespace: default
-spec:
-  generateExisting: false
-  rules:
-  - name: k-kafka-address
-    match:
-      any:
-      - resources:
-          kinds:
-          - ConfigMap
-    generate:
-      synchronize: true
-      apiVersion: v1
-      kind: ConfigMap
-      name: zk-kafka-address
-      namespace: ichangedthis
-      data:
-        kind: ConfigMap
-        metadata:
-          labels:
-            somekey: somevalue
-        data:
-          ZK_ADDRESS: "192.168.10.10:2181,192.168.10.11:2181,192.168.10.12:2181"
-          KAFKA_ADDRESS: "192.168.10.13:9092,192.168.10.14:9092,192.168.10.15:9092"

test/conformance/chainsaw/generate/validation/policy/warn-clone-change/chainsaw-test.yaml

@@ -12,13 +12,6 @@ spec:
         file: policy-ready.yaml
   - name: step-02
     try:
-    - apply:
-        expect:
-        - check:
-            ($error != null): true
-        file: update-namespace.yaml
-    - apply:
-        expect:
-        - check:
-            ($error != null): true
-        file: update-name.yaml
+    - script:
+        content: > 
+          kubectl apply -f update-name.yaml 2>&1 | grep -q "Warning: no synchronization will be performed to the old target resource upon policy updates"

test/conformance/chainsaw/generate/validation/policy/warn-clonelist-change/chainsaw-test.yaml

@@ -0,0 +1,22 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  name: immutable-clonelist
+spec:
+  steps:
+  - name: step-01
+    try:
+    - apply:
+        file: policy.yaml
+    - assert:
+        file: policy-ready.yaml
+  - name: step-02
+    try:
+    - script:
+        content: > 
+          kubectl apply -f update-kinds.yaml 2>&1 | grep -q "Warning: no synchronization will be performed to the old target resource upon policy updates"
+  - name: step-03
+    try:
+    - script:
+        content: > 
+          kubectl apply -f update-selector.yaml 2>&1 | grep -q "Warning: no synchronization will be performed to the old target resource upon policy updates"

test/conformance/chainsaw/generate/validation/policy/warn-clonelist-change/update-selector.yaml

@@ -11,7 +11,7 @@ spec:
       any:
       - resources:
           kinds:
-          - 
+          - ConfigMap
     generate:
       namespace: default
       synchronize : true

test/conformance/chainsaw/generate/validation/policy/warn-downstream-change/chainsaw-test.yaml

@@ -0,0 +1,23 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  name: immutable-downstream
+spec:
+  steps:
+  - name: step-01
+    try:
+    - apply:
+        file: chainsaw-step-01-apply-1-1.yaml
+    - assert:
+        file: chainsaw-step-01-assert-1-1.yaml
+  - name: step-02
+    try:
+    - script:
+        content: > 
+          kubectl apply -f update-name.yaml 2>&1 | grep -q "Warning: no synchronization will be performed to the old target resource upon policy updates"
+    - script:
+        content: > 
+          kubectl apply -f update-apiversion.yaml 2>&1 | grep -q "Warning: no synchronization will be performed to the old target resource upon policy updates"
+    - script:
+        content: > 
+          kubectl apply -f update-kind.yaml 2>&1 | grep -q "Warning: no synchronization will be performed to the old target resource upon policy updates"