feat: add lazy loading feature flag (#7680)

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

charts/kyverno/README.md

@@ -298,6 +298,7 @@ The chart values are organised per component.
 | features.backgroundScan.backgroundScanInterval | string | `"1h"` | Background scan interval |
 | features.backgroundScan.skipResourceFilters | bool | `true` | Skips resource filters in background scan |
 | features.configMapCaching.enabled | bool | `true` | Enables the feature |
+| features.deferredLoading.enabled | bool | `true` | Enables the feature |
 | features.dumpPayload.enabled | bool | `false` | Enables the feature |
 | features.forceFailurePolicyIgnore.enabled | bool | `false` | Enables the feature |
 | features.logging.format | string | `"text"` | Logging format |

charts/kyverno/templates/_helpers.tpl

@@ -28,6 +28,9 @@
 {{- with .configMapCaching -}}
   {{- $flags = append $flags (print "--enableConfigMapCaching=" .enabled) -}}
 {{- end -}}
+{{- with .deferredLoading -}}
+  {{- $flags = append $flags (print "--enableDeferredLoading=" .enabled) -}}
+{{- end -}}
 {{- with .dumpPayload -}}
   {{- $flags = append $flags (print "--dumpPayload=" .enabled) -}}
 {{- end -}}

charts/kyverno/templates/admission-controller/deployment.yaml

@@ -155,6 +155,7 @@ spec:
               "admissionReports"
               "autoUpdateWebhooks"
               "configMapCaching"
+              "deferredLoading"
               "dumpPayload"
               "forceFailurePolicyIgnore"
               "logging"

charts/kyverno/templates/background-controller/deployment.yaml

@@ -109,6 +109,7 @@ spec:
             {{- end }}
             {{- include "kyverno.features.flags" (pick (mergeOverwrite .Values.features .Values.backgroundController.featuresOverride)
               "configMapCaching"
+              "deferredLoading"
               "logging"
               "omitEvents"
               "policyExceptions"

charts/kyverno/templates/cleanup-controller/deployment.yaml

@@ -106,6 +106,7 @@ spec:
             {{- end }}
             {{- end }}
             {{- include "kyverno.features.flags" (pick (mergeOverwrite .Values.features .Values.cleanupController.featuresOverride)
+              "deferredLoading"
               "dumpPayload"
               "logging"
             ) | nindent 12 }}

charts/kyverno/templates/reports-controller/deployment.yaml

@@ -112,6 +112,7 @@ spec:
               "aggregateReports"
               "backgroundScan"
               "configMapCaching"
+              "deferredLoading"
               "logging"
               "omitEvents"
               "policyExceptions"

charts/kyverno/values.yaml

@@ -349,6 +349,9 @@ features:
   configMapCaching:
     # -- Enables the feature
     enabled: true
+  deferredLoading:
+    # -- Enables the feature
+    enabled: true
   dumpPayload:
     # -- Enables the feature
     enabled: false

cmd/background-controller/main.go

@@ -99,6 +99,7 @@ func main() {
 		internal.WithKubeconfig(),
 		internal.WithPolicyExceptions(),
 		internal.WithConfigMapCaching(),
+		internal.WithDeferredLoading(),
 		internal.WithRegistryClient(),
 		internal.WithLeaderElection(),
 		internal.WithKyvernoClient(),

cmd/cleanup-controller/main.go

@@ -67,6 +67,7 @@ func main() {
 		internal.WithKyvernoClient(),
 		internal.WithKyvernoDynamicClient(),
 		internal.WithConfigMapCaching(),
+		internal.WithDeferredLoading(),
 		internal.WithFlagSets(flagset),
 	)
 	// parse flags

cmd/internal/config.go

@@ -11,6 +11,7 @@ type Configuration interface {
 	UsesKubeconfig() bool
 	UsesPolicyExceptions() bool
 	UsesConfigMapCaching() bool
+	UsesDeferredLoading() bool
 	UsesCosign() bool
 	UsesRegistryClient() bool
 	UsesLeaderElection() bool
@@ -68,6 +69,12 @@ func WithConfigMapCaching() ConfigurationOption {
 	}
 }
 
+func WithDeferredLoading() ConfigurationOption {
+	return func(c *configuration) {
+		c.usesDeferredLoading = true
+	}
+}
+
 func WithCosign() ConfigurationOption {
 	return func(c *configuration) {
 		c.usesCosign = true
@@ -131,6 +138,7 @@ type configuration struct {
 	usesKubeconfig           bool
 	usesPolicyExceptions     bool
 	usesConfigMapCaching     bool
+	usesDeferredLoading      bool
 	usesCosign               bool
 	usesRegistryClient       bool
 	usesLeaderElection       bool
@@ -166,6 +174,10 @@ func (c *configuration) UsesConfigMapCaching() bool {
 	return c.usesConfigMapCaching
 }
 
+func (c *configuration) UsesDeferredLoading() bool {
+	return c.usesDeferredLoading
+}
+
 func (c *configuration) UsesCosign() bool {
 	return c.usesCosign
 }

cmd/internal/flag.go

@@ -7,6 +7,7 @@ import (
 	"github.com/go-logr/logr"
 	"github.com/kyverno/kyverno/pkg/leaderelection"
 	"github.com/kyverno/kyverno/pkg/logging"
+	"github.com/kyverno/kyverno/pkg/toggle"
 )
 
 var (
@@ -87,6 +88,10 @@ func initConfigMapCachingFlags() {
 	flag.BoolVar(&enableConfigMapCaching, "enableConfigMapCaching", true, "Enable config maps caching.")
 }
 
+func initDeferredLoadingFlags() {
+	flag.Func(toggle.EnableDeferredLoadingFlagName, toggle.EnableDeferredLoadingDescription, toggle.EnableDeferredLoading.Parse)
+}
+
 func initCosignFlags() {
 	flag.StringVar(&imageSignatureRepository, "imageSignatureRepository", "", "(DEPRECATED, will be removed in 1.12) Alternate repository for image signatures. Can be overridden per rule via `verifyImages.Repository`.")
 }
@@ -160,6 +165,10 @@ func initFlags(config Configuration, opts ...Option) {
 	if config.UsesConfigMapCaching() {
 		initConfigMapCachingFlags()
 	}
+	// deferred loading
+	if config.UsesDeferredLoading() {
+		initDeferredLoadingFlags()
+	}
 	// cosign
 	if config.UsesCosign() {
 		initCosignFlags()

cmd/kyverno/main.go

@@ -212,6 +212,7 @@ func main() {
 		internal.WithKubeconfig(),
 		internal.WithPolicyExceptions(),
 		internal.WithConfigMapCaching(),
+		internal.WithDeferredLoading(),
 		internal.WithCosign(),
 		internal.WithRegistryClient(),
 		internal.WithLeaderElection(),

cmd/reports-controller/main.go

@@ -189,6 +189,7 @@ func main() {
 		internal.WithKubeconfig(),
 		internal.WithPolicyExceptions(),
 		internal.WithConfigMapCaching(),
+		internal.WithDeferredLoading(),
 		internal.WithCosign(),
 		internal.WithRegistryClient(),
 		internal.WithLeaderElection(),

config/install-latest-testing.yaml

@@ -38799,6 +38799,7 @@ spec:
             - --admissionReports=true
             - --autoUpdateWebhooks=true
             - --enableConfigMapCaching=true
+            - --enableDeferredLoading=true
             - --dumpPayload=false
             - --forceFailurePolicyIgnore=false
             - --loggingFormat=text
@@ -38945,6 +38946,7 @@ spec:
             - --otelConfig=prometheus
             - --metricsPort=8000
             - --enableConfigMapCaching=true
+            - --enableDeferredLoading=true
             - --loggingFormat=text
             - --v=2
             - --enablePolicyException=false
@@ -39041,6 +39043,7 @@ spec:
             - --disableMetrics=false
             - --otelConfig=prometheus
             - --metricsPort=8000
+            - --enableDeferredLoading=true
             - --dumpPayload=false
             - --loggingFormat=text
             - --v=2
@@ -39173,6 +39176,7 @@ spec:
             - --backgroundScanInterval=1h
             - --skipResourceFilters=true
             - --enableConfigMapCaching=true
+            - --enableDeferredLoading=true
             - --loggingFormat=text
             - --v=2
             - --enablePolicyException=false

pkg/engine/context/loaders/apicall.go

@@ -49,17 +49,14 @@ func (a *apiLoader) LoadData() error {
 	if err != nil {
 		return fmt.Errorf("failed to initiaize APICal: %w", err)
 	}
-
 	if a.data == nil {
 		var err error
 		if a.data, err = executor.Fetch(a.ctx); err != nil {
 			return fmt.Errorf("failed to fetch data for APICall: %w", err)
 		}
 	}
-
 	if _, err := executor.Store(a.data); err != nil {
 		return fmt.Errorf("failed to store data for APICall: %w", err)
 	}
-
 	return nil
 }

pkg/engine/factories/contextloaderfactory.go

@@ -11,6 +11,7 @@ import (
 	"github.com/kyverno/kyverno/pkg/engine/context/loaders"
 	"github.com/kyverno/kyverno/pkg/engine/jmespath"
 	"github.com/kyverno/kyverno/pkg/logging"
+	"github.com/kyverno/kyverno/pkg/toggle"
 )
 
 type ContextLoaderFactoryOptions func(*contextLoader)
@@ -54,20 +55,24 @@ func (l *contextLoader) Load(
 		}
 	}
 	for _, entry := range contextEntries {
-		deferredLoader, err := l.newDeferredLoader(ctx, jp, client, rclientFactory, entry, jsonContext)
+		loader, err := l.newLoader(ctx, jp, client, rclientFactory, entry, jsonContext)
 		if err != nil {
 			return fmt.Errorf("failed to create deferred loader for context entry %s", entry.Name)
 		}
-		if deferredLoader != nil {
+		if loader != nil {
-			if err := jsonContext.AddDeferredLoader(entry.Name, deferredLoader); err != nil {
+			if toggle.FromContext(ctx).EnableDeferredLoading() {
-				return err
+				if err := jsonContext.AddDeferredLoader(entry.Name, loader); err != nil {
+					return err
+				}
+			} else {
+				return loader.LoadData()
 			}
 		}
 	}
 	return nil
 }
 
-func (l *contextLoader) newDeferredLoader(
+func (l *contextLoader) newLoader(
 	ctx context.Context,
 	jp jmespath.Interface,
 	client engineapi.RawClient,
@@ -103,6 +108,5 @@ func (l *contextLoader) newDeferredLoader(
 		l := loaders.NewVariableLoader(l.logger, entry, jsonContext, jp)
 		return l, nil
 	}
-
 	return nil, fmt.Errorf("missing ConfigMap|APICall|ImageRegistry|Variable in context entry %s", entry.Name)
 }

pkg/toggle/context.go

@@ -9,6 +9,7 @@ var defaults Toggles = defaultToggles{}
 type Toggles interface {
 	ProtectManagedResources() bool
 	ForceFailurePolicyIgnore() bool
+	EnableDeferredLoading() bool
 }
 
 type defaultToggles struct{}
@@ -21,15 +22,24 @@ func (defaultToggles) ForceFailurePolicyIgnore() bool {
 	return ForceFailurePolicyIgnore.enabled()
 }
 
+func (defaultToggles) EnableDeferredLoading() bool {
+	return EnableDeferredLoading.enabled()
+}
+
 type contextKey struct{}
 
 func NewContext(ctx context.Context, toggles Toggles) context.Context {
+	if ctx == nil {
+		return nil
+	}
 	return context.WithValue(ctx, contextKey{}, toggles)
 }
 
 func FromContext(ctx context.Context) Toggles {
-	if toggles, ok := ctx.Value(contextKey{}).(Toggles); ok {
+	if ctx != nil {
-		return toggles
+		if toggles, ok := ctx.Value(contextKey{}).(Toggles); ok {
+			return toggles
+		}
 	}
 	return defaults
 }

pkg/toggle/toggle.go

@@ -16,11 +16,17 @@ const (
 	ForceFailurePolicyIgnoreDescription = "Set the flag to 'true', to force set Failure Policy to 'ignore'."
 	forceFailurePolicyIgnoreEnvVar      = "FLAG_FORCE_FAILURE_POLICY_IGNORE"
 	defaultForceFailurePolicyIgnore     = false
+	// enable deferred context loading
+	EnableDeferredLoadingFlagName    = "enableDeferredLoading"
+	EnableDeferredLoadingDescription = "enable deferred loading of context variables"
+	enableDeferredLoadingEnvVar      = "FLAG_ENABLE_DEFERRED_LOADING"
+	defaultEnableDeferredLoading     = true
 )
 
 var (
 	ProtectManagedResources  = newToggle(defaultProtectManagedResources, protectManagedResourcesEnvVar)
 	ForceFailurePolicyIgnore = newToggle(defaultForceFailurePolicyIgnore, forceFailurePolicyIgnoreEnvVar)
+	EnableDeferredLoading    = newToggle(defaultEnableDeferredLoading, enableDeferredLoadingEnvVar)
 )
 
 type ToggleFlag interface {