fix: Support subresources as the trigger in generate rules (#6760)

* fix: check background scanning only for validation policies

Signed-off-by: Vyom-Yadav <jackhammervyom@gmail.com>

* fix: Support subresources as the trigger in generate rules

Signed-off-by: Vyom-Yadav <jackhammervyom@gmail.com>

---------

Signed-off-by: Vyom-Yadav <jackhammervyom@gmail.com>

pkg/background/generate/generate.go

@@ -33,6 +33,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	corev1listers "k8s.io/client-go/listers/core/v1"
 	"k8s.io/client-go/tools/cache"
 )
@@ -123,22 +124,63 @@ func (c *GenerateController) ProcessUR(ur *kyvernov1beta1.UpdateRequest) error {
 const doesNotApply = "policy does not apply to resource"
 
 func (c *GenerateController) getTrigger(spec kyvernov1beta1.UpdateRequestSpec) (*unstructured.Unstructured, error) {
-	if spec.Context.AdmissionRequestInfo.Operation == admissionv1.Delete {
-		request := spec.Context.AdmissionRequestInfo.AdmissionRequest
-		_, oldResource, err := admissionutils.ExtractResources(nil, *request)
-		if err != nil {
-			return nil, fmt.Errorf("failed to load resource from context: %w", err)
-		}
-		labels := oldResource.GetLabels()
-		if labels[common.GeneratePolicyLabel] != "" {
-			// non-trigger deletion, get trigger from ur spec
-			c.log.V(4).Info("non-trigger resource is deleted, fetching the trigger from the UR spec", "trigger", spec.Resource.String())
-			return common.GetResource(c.client, spec, c.log)
-		}
-		return &oldResource, nil
+	admissionRequest := spec.Context.AdmissionRequestInfo.AdmissionRequest
+	if admissionRequest == nil {
+		return common.GetResource(c.client, spec, c.log)
 	} else {
+		operation := spec.Context.AdmissionRequestInfo.Operation
+		if operation == admissionv1.Delete {
+			return getTriggerForDeleteOperation(spec, c)
+		} else if operation == admissionv1.Create {
+			return getTriggerForCreateOperation(spec, c)
+		} else {
+			newResource, oldResource, err := admissionutils.ExtractResources(nil, *admissionRequest)
+			if err != nil {
+				c.log.Error(err, "failed to extract resources from admission review request")
+				return nil, err
+			}
+
+			trigger := &newResource
+			if newResource.Object == nil {
+				trigger = &oldResource
+			}
+			return trigger, nil
+		}
+	}
+}
+
+func getTriggerForDeleteOperation(spec kyvernov1beta1.UpdateRequestSpec, c *GenerateController) (*unstructured.Unstructured, error) {
+	request := spec.Context.AdmissionRequestInfo.AdmissionRequest
+	_, oldResource, err := admissionutils.ExtractResources(nil, *request)
+	if err != nil {
+		return nil, fmt.Errorf("failed to load resource from context: %w", err)
+	}
+	labels := oldResource.GetLabels()
+	if labels[common.GeneratePolicyLabel] != "" {
+		// non-trigger deletion, get trigger from ur spec
+		c.log.V(4).Info("non-trigger resource is deleted, fetching the trigger from the UR spec", "trigger", spec.Resource.String())
 		return common.GetResource(c.client, spec, c.log)
 	}
+	return &oldResource, nil
+}
+
+func getTriggerForCreateOperation(spec kyvernov1beta1.UpdateRequestSpec, c *GenerateController) (*unstructured.Unstructured, error) {
+	admissionRequest := spec.Context.AdmissionRequestInfo.AdmissionRequest
+	trigger, err := common.GetResource(c.client, spec, c.log)
+	if err != nil || trigger == nil {
+		if admissionRequest.SubResource == "" {
+			return nil, err
+		} else {
+			c.log.V(4).Info("trigger resource not found for subresource, reverting to resource in AdmissionReviewRequest", "subresource", admissionRequest.SubResource)
+			newResource, _, err := admissionutils.ExtractResources(nil, *admissionRequest)
+			if err != nil {
+				c.log.Error(err, "failed to extract resources from admission review request")
+				return nil, err
+			}
+			trigger = &newResource
+		}
+	}
+	return trigger, err
 }
 
 func (c *GenerateController) applyGenerate(resource unstructured.Unstructured, ur kyvernov1beta1.UpdateRequest, namespaceLabels map[string]string) ([]kyvernov1.ResourceSpec, error) {
@@ -161,6 +203,16 @@ func (c *GenerateController) applyGenerate(resource unstructured.Unstructured, u
 		return nil, err
 	}
 
+	admissionRequest := ur.Spec.Context.AdmissionRequestInfo.AdmissionRequest
+	if admissionRequest != nil {
+		var gvk schema.GroupVersionKind
+		gvk, err = c.client.Discovery().GetGVKFromGVR(schema.GroupVersionResource(admissionRequest.Resource))
+		if err != nil {
+			return nil, err
+		}
+		policyContext = policyContext.WithResourceKind(gvk, admissionRequest.SubResource)
+	}
+
 	// check if the policy still applies to the resource
 	engineResponse := c.engine.Generate(context.Background(), policyContext)
 	if len(engineResponse.PolicyResponse.Rules) == 0 {

pkg/background/mutate/mutate.go

@@ -15,6 +15,7 @@ import (
 	engineapi "github.com/kyverno/kyverno/pkg/engine/api"
 	"github.com/kyverno/kyverno/pkg/event"
 	"github.com/kyverno/kyverno/pkg/utils"
+	admissionutils "github.com/kyverno/kyverno/pkg/utils/admission"
 	engineutils "github.com/kyverno/kyverno/pkg/utils/engine"
 	"go.uber.org/multierr"
 	yamlv2 "gopkg.in/yaml.v2"
@@ -104,26 +105,27 @@ func (c *MutateExistingController) ProcessUR(ur *kyvernov1beta1.UpdateRequest) e
 						continue
 					} else {
 						logger.WithName(rule.Name).Info("trigger resource not found for subresource, reverting to resource in AdmissionReviewRequest", "subresource", admissionRequest.SubResource)
-						triggerBytes := admissionRequest.Object.Raw
-						trigger = &unstructured.Unstructured{}
-						if err := trigger.UnmarshalJSON(triggerBytes); err != nil {
-							logger.WithName(rule.Name).Error(err, "failed to convert trigger resource")
+						newResource, _, err := admissionutils.ExtractResources(nil, *admissionRequest)
+						if err != nil {
+							logger.WithName(rule.Name).Error(err, "failed to extract resources from admission review request")
 							errs = append(errs, err)
 							continue
 						}
+						trigger = &newResource
 					}
 				}
 			} else {
-				triggerBytes := admissionRequest.Object.Raw
-				if triggerBytes == nil {
-					triggerBytes = admissionRequest.OldObject.Raw
-				}
-				trigger = &unstructured.Unstructured{}
-				if err := trigger.UnmarshalJSON(triggerBytes); err != nil {
-					logger.WithName(rule.Name).Error(err, "failed to convert trigger resource")
+				newResource, oldResource, err := admissionutils.ExtractResources(nil, *admissionRequest)
+				if err != nil {
+					logger.WithName(rule.Name).Error(err, "failed to extract resources from admission review request")
 					errs = append(errs, err)
 					continue
 				}
+
+				trigger = &newResource
+				if newResource.Object == nil {
+					trigger = &oldResource
+				}
 			}
 		}
 

pkg/policy/validate.go

@@ -1251,7 +1251,7 @@ func validateKinds(kinds []string, mock, backgroundScanningEnabled, isValidation
 			if len(gvrss) == 0 {
 				return fmt.Errorf("unable to convert GVK to GVR for kinds %s", k)
 			}
-			if backgroundScanningEnabled {
+			if isValidationPolicy && backgroundScanningEnabled {
 				for gvrs := range gvrss {
 					if gvrs.SubResource != "" {
 						return fmt.Errorf("background scan enabled with subresource %s", k)

test/conformance/kuttl/generate/e2e/generate-on-subresource-trigger/01-cluster-policy.yaml

@@ -0,0 +1,8 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+apply:
+- policy.yaml
+- namespace.yaml
+assert:
+- policy-ready.yaml
+- namespace-ready.yaml

test/conformance/kuttl/generate/e2e/generate-on-subresource-trigger/02-script.yaml

@@ -0,0 +1,6 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - command: kubectl run nginx --image=nginx -n test-generate-exec
+  - command: kubectl wait --for=condition=Ready pod/nginx -n test-generate-exec
+  - command: kubectl exec -n test-generate-exec nginx -it -- ls

test/conformance/kuttl/generate/e2e/generate-on-subresource-trigger/03-assert.yaml

@@ -0,0 +1,18 @@
+apiVersion: v1
+data:
+  KAFKA_ADDRESS: 192.168.10.13:9092,192.168.10.14:9092,192.168.10.15:9092
+  ZK_ADDRESS: 192.168.10.10:2181,192.168.10.11:2181,192.168.10.12:2181
+kind: ConfigMap
+metadata:
+  labels:
+    app.kubernetes.io/managed-by: kyverno
+    generate.kyverno.io/policy-name: zk-kafka-address
+    generate.kyverno.io/policy-namespace: ""
+    generate.kyverno.io/rule-name: k-kafka-address
+    generate.kyverno.io/trigger-apiversion: v1
+    generate.kyverno.io/trigger-kind: PodExecOptions
+    generate.kyverno.io/trigger-name: ""
+    generate.kyverno.io/trigger-namespace: test-generate-exec
+    somekey: somevalue
+  name: zk-kafka-address
+  namespace: test-generate-exec

test/conformance/kuttl/generate/e2e/generate-on-subresource-trigger/99-cleanup.yaml

@@ -0,0 +1,7 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - command: kubectl delete cpol zk-kafka-address --force --wait=true --ignore-not-found=true
+  - command: kubectl delete pod nginx -n test-generate-exec --wait=true --ignore-not-found=true
+  - command: kubectl delete cm zk-kafka-address -n test-generate-exec --wait=true --ignore-not-found=true
+  - command: kubectl delete ns test-generate-exec --wait=true --ignore-not-found=true

test/conformance/kuttl/generate/e2e/generate-on-subresource-trigger/README.md

@@ -0,0 +1,11 @@
+## Description
+
+This test assures generation of resource with a sub-resource acting as a trigger.
+
+## Expected Behavior
+
+The test passes and `configmap` `zk-kafka-address` is created.
+
+## Reference Issue(s)
+
+6399

test/conformance/kuttl/generate/e2e/generate-on-subresource-trigger/namespace-ready.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: test-generate-exec
+status:
+  phase: Active

test/conformance/kuttl/generate/e2e/generate-on-subresource-trigger/namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: test-generate-exec

test/conformance/kuttl/generate/e2e/generate-on-subresource-trigger/policy-ready.yaml

@@ -0,0 +1,9 @@
+apiVersion: kyverno.io/v2beta1
+kind: ClusterPolicy
+metadata:
+  name: zk-kafka-address
+status:
+  conditions:
+  - reason: Succeeded
+    status: "True"
+    type: Ready

test/conformance/kuttl/generate/e2e/generate-on-subresource-trigger/policy.yaml

@@ -0,0 +1,29 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: zk-kafka-address
+spec:
+  # generateExisting does not work for sub-resources
+  generateExisting: false
+  rules:
+    - name: k-kafka-address
+      match:
+        any:
+          - resources:
+              kinds:
+                - "Pod/exec"
+      generate:
+        # synchronization does not work for sub-resources
+        synchronize: false
+        apiVersion: v1
+        kind: ConfigMap
+        name: zk-kafka-address
+        namespace: "{{request.namespace}}"
+        data:
+          kind: ConfigMap
+          metadata:
+            labels:
+              somekey: somevalue
+          data:
+            ZK_ADDRESS: "192.168.10.10:2181,192.168.10.11:2181,192.168.10.12:2181"
+            KAFKA_ADDRESS: "192.168.10.13:9092,192.168.10.14:9092,192.168.10.15:9092"