Added omit-events flag to allow disabling of event emission (#7010)

* added comma seperated flag

Signed-off-by: Vishal Choudhary <sendtovishalchoudhary@gmail.com>

* reason added in logs

Signed-off-by: Vishal Choudhary <sendtovishalchoudhary@gmail.com>

* added requested changes

Signed-off-by: Vishal Choudhary <sendtovishalchoudhary@gmail.com>

* kuttl test init

Signed-off-by: Vishal Choudhary <sendtovishalchoudhary@gmail.com>

* updated kuttl tests

Signed-off-by: Vishal Choudhary <sendtovishalchoudhary@gmail.com>

* updated behavior

Signed-off-by: Vishal Choudhary <sendtovishalchoudhary@gmail.com>

* fixed flawed behavior

Signed-off-by: Vishal Choudhary <sendtovishalchoudhary@gmail.com>

* updated test location and added readme

Signed-off-by: Vishal Choudhary <sendtovishalchoudhary@gmail.com>

* tests

Signed-off-by: Vishal Choudhary <sendtovishalchoudhary@gmail.com>

* updated step

Signed-off-by: Vishal Choudhary <sendtovishalchoudhary@gmail.com>

* omit events

Signed-off-by: Vishal Choudhary <sendtovishalchoudhary@gmail.com>

---------

Signed-off-by: Vishal Choudhary <sendtovishalchoudhary@gmail.com>
Co-authored-by: shuting <shuting@nirmata.com>

cmd/background-controller/main.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"flag"
 	"os"
+	"strings"
 	"sync"
 	"time"
 
@@ -84,10 +85,12 @@ func main() {
 	var (
 		genWorkers      int
 		maxQueuedEvents int
+		omitEvents      string
 	)
 	flagset := flag.NewFlagSet("updaterequest-controller", flag.ExitOnError)
 	flagset.IntVar(&genWorkers, "genWorkers", 10, "Workers for the background controller.")
 	flagset.IntVar(&maxQueuedEvents, "maxQueuedEvents", 1000, "Maximum events to be queued.")
+	flagset.StringVar(&omitEvents, "omit-events", "", "Set this flag to a comma sperated list of PolicyViolation, PolicyApplied, PolicyError, PolicySkipped to disable events, e.g. --omit-events=PolicyApplied,PolicyViolation")
 	// config
 	appConfig := internal.NewConfiguration(
 		internal.WithProfiling(),
@@ -113,11 +116,16 @@ func main() {
 	kyamlopenapi.Schema()
 	// informer factories
 	kyvernoInformer := kyvernoinformer.NewSharedInformerFactory(setup.KyvernoClient, resyncPeriod)
+	emitEventsValues := strings.Split(omitEvents, ",")
+	if omitEvents == "" {
+		emitEventsValues = []string{}
+	}
 	eventGenerator := event.NewEventGenerator(
 		setup.KyvernoDynamicClient,
 		kyvernoInformer.Kyverno().V1().ClusterPolicies(),
 		kyvernoInformer.Kyverno().V1().Policies(),
 		maxQueuedEvents,
+		emitEventsValues,
 		logging.WithName("EventGenerator"),
 	)
 	// this controller only subscribe to events, nothing is returned...

cmd/kyverno/main.go

@@ -7,6 +7,7 @@ import (
 	"flag"
 	"fmt"
 	"os"
+	"strings"
 	"sync"
 	"time"
 
@@ -181,6 +182,7 @@ func main() {
 		serverIP                     string
 		webhookTimeout               int
 		maxQueuedEvents              int
+		omitEvents                   string
 		autoUpdateWebhooks           bool
 		webhookRegistrationTimeout   time.Duration
 		admissionReports             bool
@@ -192,6 +194,7 @@ func main() {
 	flagset.BoolVar(&dumpPayload, "dumpPayload", false, "Set this flag to activate/deactivate debug mode.")
 	flagset.IntVar(&webhookTimeout, "webhookTimeout", webhookcontroller.DefaultWebhookTimeout, "Timeout for webhook configurations.")
 	flagset.IntVar(&maxQueuedEvents, "maxQueuedEvents", 1000, "Maximum events to be queued.")
+	flagset.StringVar(&omitEvents, "omit-events", "", "Set this flag to a comma sperated list of PolicyViolation, PolicyApplied, PolicyError, PolicySkipped to disable events, e.g. --omit-events=PolicyApplied,PolicyViolation")
 	flagset.StringVar(&serverIP, "serverIP", "", "IP address where Kyverno controller runs. Only required if out-of-cluster.")
 	flagset.BoolVar(&autoUpdateWebhooks, "autoUpdateWebhooks", true, "Set this flag to 'false' to disable auto-configuration of the webhook.")
 	flagset.DurationVar(&webhookRegistrationTimeout, "webhookRegistrationTimeout", 120*time.Second, "Timeout for webhook registration, e.g., 30s, 1m, 5m.")
@@ -252,11 +255,16 @@ func main() {
 		serverIP,
 	)
 	policyCache := policycache.NewCache()
+	omitEventsValues := strings.Split(omitEvents, ",")
+	if omitEvents == "" {
+		omitEventsValues = []string{}
+	}
 	eventGenerator := event.NewEventGenerator(
 		setup.KyvernoDynamicClient,
 		kyvernoInformer.Kyverno().V1().ClusterPolicies(),
 		kyvernoInformer.Kyverno().V1().Policies(),
 		maxQueuedEvents,
+		omitEventsValues,
 		logging.WithName("EventGenerator"),
 	)
 	// this controller only subscribe to events, nothing is returned...

cmd/reports-controller/main.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"flag"
 	"os"
+	"strings"
 	"sync"
 	"time"
 
@@ -165,6 +166,7 @@ func main() {
 		backgroundScanWorkers  int
 		backgroundScanInterval time.Duration
 		maxQueuedEvents        int
+		omitEvents             string
 		skipResourceFilters    bool
 	)
 	flagset := flag.NewFlagSet("reports-controller", flag.ExitOnError)
@@ -174,6 +176,7 @@ func main() {
 	flagset.IntVar(&backgroundScanWorkers, "backgroundScanWorkers", backgroundscancontroller.Workers, "Configure the number of background scan workers.")
 	flagset.DurationVar(&backgroundScanInterval, "backgroundScanInterval", time.Hour, "Configure background scan interval.")
 	flagset.IntVar(&maxQueuedEvents, "maxQueuedEvents", 1000, "Maximum events to be queued.")
+	flagset.StringVar(&omitEvents, "omit-events", "", "Set this flag to a comma sperated list of PolicyViolation, PolicyApplied, PolicyError, PolicySkipped to disable events, e.g. --omit-events=PolicyApplied,PolicyViolation")
 	flagset.BoolVar(&skipResourceFilters, "skipResourceFilters", true, "If true, resource filters wont be considered.")
 	// config
 	appConfig := internal.NewConfiguration(
@@ -207,11 +210,16 @@ func main() {
 	setup.Logger.Info("background scan interval", "duration", backgroundScanInterval.String())
 	// informer factories
 	kyvernoInformer := kyvernoinformer.NewSharedInformerFactory(setup.KyvernoClient, resyncPeriod)
+	omitEventsValues := strings.Split(omitEvents, ",")
+	if omitEvents == "" {
+		omitEventsValues = []string{}
+	}
 	eventGenerator := event.NewEventGenerator(
 		setup.KyvernoDynamicClient,
 		kyvernoInformer.Kyverno().V1().ClusterPolicies(),
 		kyvernoInformer.Kyverno().V1().Policies(),
 		maxQueuedEvents,
+		omitEventsValues,
 		logging.WithName("EventGenerator"),
 	)
 	// engine

pkg/event/controller.go

@@ -43,6 +43,8 @@ type generator struct {
 
 	maxQueuedEvents int
 
+	omitEvents []string
+
 	log logr.Logger
 }
 
@@ -64,6 +66,7 @@ func NewEventGenerator(
 	cpInformer kyvernov1informers.ClusterPolicyInformer,
 	pInformer kyvernov1informers.PolicyInformer,
 	maxQueuedEvents int,
+	omitEvents []string,
 	log logr.Logger,
 ) Controller {
 	gen := generator{
@@ -76,6 +79,7 @@ func NewEventGenerator(
 		genPolicyRecorder:      NewRecorder(GeneratePolicyController, client.GetEventsInterface()),
 		mutateExistingRecorder: NewRecorder(MutateExistingController, client.GetEventsInterface()),
 		maxQueuedEvents:        maxQueuedEvents,
+		omitEvents:             omitEvents,
 		log:                    log,
 	}
 	return &gen
@@ -96,7 +100,19 @@ func (gen *generator) Add(infos ...Info) {
 			logger.V(3).Info("skipping event creation for resource without a name", "kind", info.Kind, "name", info.Name, "namespace", info.Namespace)
 			continue
 		}
-		gen.queue.Add(info)
+
+		shouldEmitEvent := true
+		for _, eventReason := range gen.omitEvents {
+			if info.Reason == Reason(eventReason) {
+				shouldEmitEvent = false
+				logger.V(6).Info("omitting event", "kind", info.Kind, "name", info.Name, "namespace", info.Namespace, "reason", info.Reason)
+			}
+		}
+
+		if shouldEmitEvent {
+			gen.queue.Add(info)
+			logger.V(6).Info("creating event", "kind", info.Kind, "name", info.Name, "namespace", info.Namespace, "reason", info.Reason)
+		}
 	}
 }
 

test/conformance/kuttl/flags/standard/emit-events/01-admission-controller-apply.yaml

@@ -0,0 +1,6 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+apply:
+- admission-controller.yaml
+assert:
+- admission-controller-assert.yaml

test/conformance/kuttl/flags/standard/emit-events/02-policy.yaml

@@ -0,0 +1,6 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+apply:
+- policy.yaml
+assert:
+- policy-assert.yaml

test/conformance/kuttl/flags/standard/emit-events/03-resource.yaml

@@ -0,0 +1,6 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+apply:
+  - file: resource.yaml
+  - file: resource-fail.yaml
+    shouldFail: true

test/conformance/kuttl/flags/standard/emit-events/04-event.yaml

@@ -0,0 +1,4 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+apply:
+- file: event-assert.yaml

test/conformance/kuttl/flags/standard/emit-events/05-script.yaml

@@ -0,0 +1,12 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+- script: |
+    if kubectl logs deployment/kyverno-admission-controller -n kyverno | grep "reason=\"PolicyViolation\"" 
+    then 
+        echo "Test succeeded. PolicyViolation event was not created."
+        exit 0
+    else 
+        echo "Tested failed. PolicyViolation event should have been created."
+        exit 1
+    fi

test/conformance/kuttl/flags/standard/emit-events/README.md

@@ -0,0 +1,18 @@
+## Description
+
+This test updates the deployment with flag `--omit-events=PolicyApplied` set
+Then it creates a policy, and a resource.
+The resource is expected to be accepted.
+A `PolicyApplied` event should be created.
+Then it creates a respource that is expected to be rejected
+A `PolicyViolation` event should not be emitted as the flag does not include that.
+
+## Steps
+
+1.  Update the deployment of admission controller to add this ar`--omit-events=PolicyApplied`.  
+2.  - Create a policy
+    - Assert the policy becomes ready
+3.  - Create a resource,
+4.  - Asset a `PolicyApplied` event is created
+5.  Try creating a resource with a script that is expected to fail.
+6.  Exit the script with `0` if it returns an error

test/conformance/kuttl/flags/standard/emit-events/admission-controller-assert.yaml

@@ -0,0 +1,9 @@
+apiVersion: kyverno.io/v1
+kind: Policy
+metadata:
+  name: kyverno-admission-controller
+status:
+  conditions:
+    - reason: Succeeded
+      status: "True"
+      type: Ready

test/conformance/kuttl/flags/standard/emit-events/admission-controller.yaml

@@ -0,0 +1,170 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: kyverno-admission-controller
+  namespace: kyverno
+  labels:
+    app.kubernetes.io/component: admission-controller
+    app.kubernetes.io/instance: kyverno
+    app.kubernetes.io/part-of: kyverno
+    app.kubernetes.io/version: latest
+spec:
+  replicas: 
+  strategy:
+    rollingUpdate:
+      maxSurge: 1
+      maxUnavailable: 40%
+    type: RollingUpdate
+  selector:
+    matchLabels:
+      app.kubernetes.io/component: admission-controller
+      app.kubernetes.io/instance: kyverno
+      app.kubernetes.io/part-of: kyverno
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/component: admission-controller
+        app.kubernetes.io/instance: kyverno
+        app.kubernetes.io/part-of: kyverno
+        app.kubernetes.io/version: latest
+    spec:
+      dnsPolicy: ClusterFirst
+      serviceAccountName: kyverno-admission-controller
+      initContainers:
+        - name: kyverno-pre
+          image: "ghcr.io/kyverno/kyvernopre:latest"
+          imagePullPolicy: IfNotPresent
+          args:
+            - --loggingFormat=text
+            - --v=2
+          resources:
+            limits:
+              cpu: 100m
+              memory: 256Mi
+            requests:
+              cpu: 10m
+              memory: 64Mi
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+              - ALL
+            privileged: false
+            readOnlyRootFilesystem: true
+            runAsNonRoot: true
+            seccompProfile:
+              type: RuntimeDefault
+          env:
+          - name: METRICS_CONFIG
+            value: kyverno-metrics
+          - name: KYVERNO_NAMESPACE
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.namespace
+          - name: KYVERNO_POD_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.name
+          - name: KYVERNO_DEPLOYMENT
+            value: kyverno
+      containers:
+        - name: kyverno
+          image: "ghcr.io/kyverno/kyverno:latest"
+          imagePullPolicy: IfNotPresent
+          args:
+            - --omit-events=PolicyViolation
+            - --backgroundServiceAccountName=system:serviceaccount:kyverno:kyverno-background-controller
+            - --servicePort=443
+            - --loggingFormat=text
+            - --v=2
+            - --disableMetrics=false
+            - --otelConfig=prometheus
+            - --metricsPort=8000
+            - --admissionReports=true
+            - --autoUpdateWebhooks=true
+            - --enableConfigMapCaching=true
+            - --dumpPayload=false
+            - --forceFailurePolicyIgnore=false
+            - --enablePolicyException=false
+            - --exceptionNamespace=
+            - --protectManagedResources=false
+            - --allowInsecureRegistry=false
+            - --registryCredentialHelpers=default,google,amazon,azure,github
+          resources: 
+            limits:
+              memory: 384Mi
+            requests:
+              cpu: 100m
+              memory: 128Mi
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+              - ALL
+            privileged: false
+            readOnlyRootFilesystem: true
+            runAsNonRoot: true
+            seccompProfile:
+              type: RuntimeDefault
+          ports:
+          - containerPort: 9443
+            name: https
+            protocol: TCP
+          - containerPort: 8000
+            name: metrics-port
+            protocol: TCP
+          env:
+          - name: INIT_CONFIG
+            value: kyverno
+          - name: METRICS_CONFIG
+            value: kyverno-metrics
+          - name: KYVERNO_NAMESPACE
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.namespace
+          - name: KYVERNO_POD_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.name
+          - name: KYVERNO_SERVICEACCOUNT_NAME
+            value: kyverno-admission-controller
+          - name: KYVERNO_SVC
+            value: kyverno-svc
+          - name: TUF_ROOT
+            value: /.sigstore
+          - name: KYVERNO_DEPLOYMENT
+            value: kyverno-admission-controller
+          startupProbe:
+            failureThreshold: 20
+            httpGet:
+              path: /health/liveness
+              port: 9443
+              scheme: HTTPS
+            initialDelaySeconds: 2
+            periodSeconds: 6
+          livenessProbe:
+            failureThreshold: 2
+            httpGet:
+              path: /health/liveness
+              port: 9443
+              scheme: HTTPS
+            initialDelaySeconds: 15
+            periodSeconds: 30
+            successThreshold: 1
+            timeoutSeconds: 5
+          readinessProbe:
+            failureThreshold: 6
+            httpGet:
+              path: /health/readiness
+              port: 9443
+              scheme: HTTPS
+            initialDelaySeconds: 5
+            periodSeconds: 10
+            successThreshold: 1
+            timeoutSeconds: 5
+          volumeMounts:
+            - mountPath: /.sigstore
+              name: sigstore
+      volumes:
+      - name: sigstore
+        emptyDir: {}

test/conformance/kuttl/flags/standard/emit-events/event-assert.yaml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: Event
+metadata: {}
+involvedObject:
+  apiVersion: kyverno.io/v1
+  kind: Policy
+  name: require-labels
+type: Normal
+reason: PolicyApplied
+source:
+  component: kyverno-admission

test/conformance/kuttl/flags/standard/emit-events/policy-assert.yaml

@@ -0,0 +1,9 @@
+apiVersion: kyverno.io/v1
+kind: Policy
+metadata:
+  name: require-labels
+status:
+  conditions:
+    - reason: Succeeded
+      status: "True"
+      type: Ready

test/conformance/kuttl/flags/standard/emit-events/policy.yaml

@@ -0,0 +1,20 @@
+apiVersion: kyverno.io/v1
+kind: Policy
+metadata:
+  name: require-labels
+spec:
+  validationFailureAction: Enforce
+  background: false
+  rules:
+    - name: require-team
+      match:
+        any:
+          - resources:
+              kinds:
+                - ConfigMap
+      validate:
+        message: 'The label `team` is required.'
+        pattern:
+          metadata:
+            labels:
+              team: '?*'

test/conformance/kuttl/flags/standard/emit-events/resource-fail.yaml

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: bar
+  labels:
+    foo: bar
+  

test/conformance/kuttl/flags/standard/emit-events/resource.yaml

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: foo
+  labels:
+    team: kyverno
+