refactor: add specific loaders from #7597 (#7671)

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

pkg/engine/api/context.go

@@ -95,7 +95,7 @@ func LoadAPIData(ctx context.Context, jp jmespath.Interface, logger logr.Logger,
 	if err != nil {
 		return fmt.Errorf("failed to initialize APICall: %w", err)
 	}
-	if _, err := executor.Execute(ctx); err != nil {
+	if _, err := executor.FetchAndLoad(ctx); err != nil {
 		return fmt.Errorf("failed to execute APICall: %w", err)
 	}
 	return nil

pkg/engine/api/contextloader.go

@@ -12,6 +12,8 @@ type RegistryClientFactory interface {
 	GetClient(ctx context.Context, creds *kyvernov1.ImageRegistryCredentials) (RegistryClient, error)
 }
 
+type Initializer = func(jsonContext enginecontext.Interface) error
+
 // ContextLoaderFactory provides a ContextLoader given a policy context and rule name
 type ContextLoaderFactory = func(policy kyvernov1.PolicyInterface, rule kyvernov1.Rule) ContextLoader
 

pkg/engine/api/resolver_test.go

@@ -72,6 +72,7 @@ func Test_namespacedResourceResolverChain_Get(t *testing.T) {
 		wantErr error
 		wantCm  *corev1.ConfigMap
 	}{{
+		name: "Test0",
 		fields: fields{
 			resolvers: []ConfigmapResolver{
 				dummyResolver{},
@@ -80,6 +81,7 @@ func Test_namespacedResourceResolverChain_Get(t *testing.T) {
 			},
 		},
 	}, {
+		name: "Test1",
 		fields: fields{
 			resolvers: []ConfigmapResolver{
 				dummyResolver{
@@ -95,6 +97,7 @@ func Test_namespacedResourceResolverChain_Get(t *testing.T) {
 		},
 		wantErr: errors.New("3"),
 	}, {
+		name: "Test2",
 		fields: fields{
 			resolvers: []ConfigmapResolver{
 				dummyResolver{
@@ -107,6 +110,7 @@ func Test_namespacedResourceResolverChain_Get(t *testing.T) {
 			},
 		},
 	}, {
+		name: "Test3",
 		fields: fields{
 			resolvers: []ConfigmapResolver{
 				dummyResolver{
@@ -123,9 +127,8 @@ func Test_namespacedResourceResolverChain_Get(t *testing.T) {
 		t.Run(tt.name, func(t *testing.T) {
 			resolver, _ := NewNamespacedResourceResolver(tt.fields.resolvers...)
 			got, err := resolver.Get(context.TODO(), tt.args.namespace, tt.args.name)
-			if !reflect.DeepEqual(err, tt.wantErr) { //nolint:deepequalerrors
-				t.Errorf("ConfigmapResolver.Get() error = %v, wantErr %v", err, tt.wantErr)
-				return
+			if !checkError(tt.wantErr, err) {
+				t.Errorf("ConfigmapResolver.Get() %s error = %v, wantErr %v", tt.name, err, tt.wantErr)
 			}
 			if !reflect.DeepEqual(got, tt.wantCm) {
 				t.Errorf("ConfigmapResolver.Get() = %v, want %v", got, tt.wantCm)
@@ -133,3 +136,13 @@ func Test_namespacedResourceResolverChain_Get(t *testing.T) {
 		})
 	}
 }
+
+func checkError(wantErr, err error) bool {
+	if wantErr != nil {
+		if err == nil {
+			return false
+		}
+		return wantErr.Error() == err.Error()
+	}
+	return err == nil
+}

pkg/engine/apicall/apiCall.go

@@ -51,23 +51,38 @@ func New(
 	}, nil
 }
 
-func (a *apiCall) Execute(ctx context.Context) ([]byte, error) {
+func (a *apiCall) FetchAndLoad(ctx context.Context) ([]byte, error) {
+	data, err := a.Fetch(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	results, err := a.Store(data)
+	if err != nil {
+		return nil, err
+	}
+
+	return results, nil
+}
+
+func (a *apiCall) Fetch(ctx context.Context) ([]byte, error) {
 	call, err := variables.SubstituteAllInType(a.logger, a.jsonCtx, a.entry.APICall)
 	if err != nil {
 		return nil, fmt.Errorf("failed to substitute variables in context entry %s %s: %v", a.entry.Name, a.entry.APICall.URLPath, err)
 	}
-
 	data, err := a.execute(ctx, call)
 	if err != nil {
 		return nil, err
 	}
+	return data, nil
+}
 
-	result, err := a.transformAndStore(data)
+func (a *apiCall) Store(data []byte) ([]byte, error) {
+	results, err := a.transformAndStore(data)
 	if err != nil {
 		return nil, err
 	}
-
-	return result, nil
+	return results, nil
 }
 
 func (a *apiCall) execute(ctx context.Context, call *kyvernov1.APICall) ([]byte, error) {

pkg/engine/apicall/apiCall_test.go

@@ -56,20 +56,20 @@ func Test_serviceGetRequest(t *testing.T) {
 
 	call, err := New(logr.Discard(), jp, entry, ctx, nil)
 	assert.NilError(t, err)
-	_, err = call.Execute(context.TODO())
+	_, err = call.FetchAndLoad(context.TODO())
 	assert.ErrorContains(t, err, "invalid request type")
 
 	entry.APICall.Method = "GET"
 	call, err = New(logr.Discard(), jp, entry, ctx, nil)
 	assert.NilError(t, err)
-	_, err = call.Execute(context.TODO())
+	_, err = call.FetchAndLoad(context.TODO())
 	assert.ErrorContains(t, err, "HTTP 404")
 
 	entry.APICall.Service.URL = s.URL + "/resource"
 	call, err = New(logr.Discard(), jp, entry, ctx, nil)
 	assert.NilError(t, err)
 
-	data, err := call.Execute(context.TODO())
+	data, err := call.FetchAndLoad(context.TODO())
 	assert.NilError(t, err)
 	assert.Assert(t, data != nil, "nil data")
 	assert.Equal(t, string(serverResponse), string(data))
@@ -93,7 +93,7 @@ func Test_servicePostRequest(t *testing.T) {
 	ctx := enginecontext.NewContext(jp)
 	call, err := New(logr.Discard(), jp, entry, ctx, nil)
 	assert.NilError(t, err)
-	data, err := call.Execute(context.TODO())
+	data, err := call.FetchAndLoad(context.TODO())
 	assert.NilError(t, err)
 	assert.Equal(t, "{}\n", string(data))
 
@@ -141,7 +141,7 @@ func Test_servicePostRequest(t *testing.T) {
 
 	call, err = New(logr.Discard(), jp, entry, ctx, nil)
 	assert.NilError(t, err)
-	data, err = call.Execute(context.TODO())
+	data, err = call.FetchAndLoad(context.TODO())
 	assert.NilError(t, err)
 
 	expectedResults := `{"images":["https://ghcr.io/tomcat/tomcat:9","https://ghcr.io/vault/vault:v3","https://ghcr.io/busybox/busybox:latest"]}`

pkg/engine/context/loader.go

@@ -0,0 +1,16 @@
+package context
+
+// Loader fetches or produces data and loads it into the context. A loader is created for each
+// context entry (e.g. `context.variable`, `context.apiCall`, etc.)
+// Loaders are invoked lazily based on variable lookups. Loaders may be invoked multiple times to
+// handle checkpoints and restores that occur when processing loops. A loader that fetches remote
+// data should be able to handle multiple invocations in an optimal manner by mantaining internal
+// state and caching remote data. For example, if an API call is made the data retrieved can be
+// stored so that it can be saved in the outer context when a restore is performed.
+type Loader interface {
+	// Load data fetches or produces data and stores it in the context
+	LoadData() error
+	// Has loaded indicates if the loader has previously
+	// executed and stored data in a context
+	HasLoaded() bool
+}

pkg/engine/context/loaders/apicall.go

@@ -0,0 +1,65 @@
+package loaders
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/go-logr/logr"
+	kyvernov1 "github.com/kyverno/kyverno/api/kyverno/v1"
+	engineapi "github.com/kyverno/kyverno/pkg/engine/api"
+	"github.com/kyverno/kyverno/pkg/engine/apicall"
+	enginecontext "github.com/kyverno/kyverno/pkg/engine/context"
+	"github.com/kyverno/kyverno/pkg/engine/jmespath"
+)
+
+type apiLoader struct {
+	ctx       context.Context //nolint:containedctx
+	logger    logr.Logger
+	entry     kyvernov1.ContextEntry
+	enginectx enginecontext.Interface
+	jp        jmespath.Interface
+	client    engineapi.RawClient
+	data      []byte
+}
+
+func NewAPILoader(
+	ctx context.Context,
+	logger logr.Logger,
+	entry kyvernov1.ContextEntry,
+	enginectx enginecontext.Interface,
+	jp jmespath.Interface,
+	client engineapi.RawClient,
+) enginecontext.Loader {
+	return &apiLoader{
+		ctx:       ctx,
+		logger:    logger,
+		entry:     entry,
+		enginectx: enginectx,
+		jp:        jp,
+		client:    client,
+	}
+}
+
+func (a *apiLoader) HasLoaded() bool {
+	return a.data != nil
+}
+
+func (a *apiLoader) LoadData() error {
+	executor, err := apicall.New(a.logger, a.jp, a.entry, a.enginectx, a.client)
+	if err != nil {
+		return fmt.Errorf("failed to initiaize APICal: %w", err)
+	}
+
+	if a.data == nil {
+		var err error
+		if a.data, err = executor.Fetch(a.ctx); err != nil {
+			return fmt.Errorf("failed to fetch data for APICall: %w", err)
+		}
+	}
+
+	if _, err := executor.Store(a.data); err != nil {
+		return fmt.Errorf("failed to store data for APICall: %w", err)
+	}
+
+	return nil
+}

pkg/engine/context/loaders/configmap.go

@@ -0,0 +1,95 @@
+package loaders
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+
+	"github.com/go-logr/logr"
+	kyvernov1 "github.com/kyverno/kyverno/api/kyverno/v1"
+	engineapi "github.com/kyverno/kyverno/pkg/engine/api"
+	enginecontext "github.com/kyverno/kyverno/pkg/engine/context"
+	"github.com/kyverno/kyverno/pkg/engine/variables"
+)
+
+type configMapLoader struct {
+	ctx       context.Context //nolint:containedctx
+	logger    logr.Logger
+	entry     kyvernov1.ContextEntry
+	resolver  engineapi.ConfigmapResolver
+	enginectx enginecontext.Interface
+	data      []byte
+}
+
+func NewConfigMapLoader(
+	ctx context.Context,
+	logger logr.Logger,
+	entry kyvernov1.ContextEntry,
+	resolver engineapi.ConfigmapResolver,
+	enginectx enginecontext.Interface,
+) enginecontext.Loader {
+	return &configMapLoader{
+		ctx:       ctx,
+		logger:    logger,
+		entry:     entry,
+		resolver:  resolver,
+		enginectx: enginectx,
+	}
+}
+
+func (cml *configMapLoader) HasLoaded() bool {
+	return cml.data != nil
+}
+
+func (cml *configMapLoader) LoadData() error {
+	if cml.resolver == nil {
+		return fmt.Errorf("a ConfigmapResolver is required")
+	}
+
+	if cml.data == nil {
+		data, err := cml.fetchConfigMap()
+		if err != nil {
+			return fmt.Errorf("failed to retrieve config map for context entry %s: %v", cml.entry.Name, err)
+		}
+
+		cml.data = data
+	}
+
+	if err := cml.enginectx.AddContextEntry(cml.entry.Name, cml.data); err != nil {
+		return fmt.Errorf("failed to add config map for context entry %s: %v", cml.entry.Name, err)
+	}
+
+	return nil
+}
+
+func (cml *configMapLoader) fetchConfigMap() ([]byte, error) {
+	logger := cml.logger
+	entryName := cml.entry.Name
+	cmName := cml.entry.ConfigMap.Name
+	cmNamespace := cml.entry.ConfigMap.Namespace
+
+	contextData := make(map[string]interface{})
+	name, err := variables.SubstituteAll(logger, cml.enginectx, cml.entry.ConfigMap.Name)
+	if err != nil {
+		return nil, fmt.Errorf("failed to substitute variables in context %s configMap.name %s: %v", entryName, cmName, err)
+	}
+	namespace, err := variables.SubstituteAll(logger, cml.enginectx, cml.entry.ConfigMap.Namespace)
+	if err != nil {
+		return nil, fmt.Errorf("failed to substitute variables in context %s configMap.namespace %s: %v", entryName, cmNamespace, err)
+	}
+	if namespace == "" {
+		namespace = "default"
+	}
+	obj, err := cml.resolver.Get(cml.ctx, namespace.(string), name.(string))
+	if err != nil {
+		return nil, fmt.Errorf("failed to get configmap %s/%s : %v", namespace, name, err)
+	}
+	// extract configmap data
+	contextData["data"] = obj.Data
+	contextData["metadata"] = obj.ObjectMeta
+	data, err := json.Marshal(contextData)
+	if err != nil {
+		return nil, fmt.Errorf("failed to unmarshal configmap %s/%s: %v", namespace, name, err)
+	}
+	return data, nil
+}

pkg/engine/context/loaders/imagedata.go

@@ -0,0 +1,152 @@
+package loaders
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+
+	"github.com/go-logr/logr"
+	kyvernov1 "github.com/kyverno/kyverno/api/kyverno/v1"
+	engineapi "github.com/kyverno/kyverno/pkg/engine/api"
+	enginecontext "github.com/kyverno/kyverno/pkg/engine/context"
+	"github.com/kyverno/kyverno/pkg/engine/jmespath"
+	"github.com/kyverno/kyverno/pkg/engine/variables"
+)
+
+type imageDataLoader struct {
+	ctx            context.Context //nolint:containedctx
+	logger         logr.Logger
+	entry          kyvernov1.ContextEntry
+	enginectx      enginecontext.Interface
+	jp             jmespath.Interface
+	rclientFactory engineapi.RegistryClientFactory
+	data           []byte
+}
+
+func NewImageDataLoader(
+	ctx context.Context,
+	logger logr.Logger,
+	entry kyvernov1.ContextEntry,
+	enginectx enginecontext.Interface,
+	jp jmespath.Interface,
+	rclientFactory engineapi.RegistryClientFactory,
+) enginecontext.Loader {
+	return &imageDataLoader{
+		ctx:            ctx,
+		logger:         logger,
+		entry:          entry,
+		enginectx:      enginectx,
+		jp:             jp,
+		rclientFactory: rclientFactory,
+	}
+}
+
+func (idl *imageDataLoader) LoadData() error {
+	return idl.loadImageData()
+}
+
+func (cml *imageDataLoader) HasLoaded() bool {
+	return cml.data != nil
+}
+
+func (idl *imageDataLoader) loadImageData() error {
+	if idl.data == nil {
+		imageData, err := idl.fetchImageData()
+		if err != nil {
+			return err
+		}
+
+		idl.data, err = json.Marshal(imageData)
+		if err != nil {
+			return err
+		}
+	}
+
+	if err := idl.enginectx.AddContextEntry(idl.entry.Name, idl.data); err != nil {
+		return fmt.Errorf("failed to add resource data to context: contextEntry: %v, error: %v", idl.entry, err)
+	}
+
+	return nil
+}
+
+func (idl *imageDataLoader) fetchImageData() (interface{}, error) {
+	entry := idl.entry
+	ref, err := variables.SubstituteAll(idl.logger, idl.enginectx, entry.ImageRegistry.Reference)
+	if err != nil {
+		return nil, fmt.Errorf("ailed to substitute variables in context entry %s %s: %v", entry.Name, entry.ImageRegistry.Reference, err)
+	}
+
+	refString, ok := ref.(string)
+	if !ok {
+		return nil, fmt.Errorf("invalid image reference %s, image reference must be a string", ref)
+	}
+
+	path, err := variables.SubstituteAll(idl.logger, idl.enginectx, entry.ImageRegistry.JMESPath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to substitute variables in context entry %s %s: %v", entry.Name, entry.ImageRegistry.JMESPath, err)
+	}
+
+	client, err := idl.rclientFactory.GetClient(idl.ctx, entry.ImageRegistry.ImageRegistryCredentials)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get registry client %s: %v", entry.Name, err)
+	}
+
+	imageData, err := idl.fetchImageDataMap(client, refString)
+	if err != nil {
+		return nil, err
+	}
+
+	if path != "" {
+		imageData, err = applyJMESPath(idl.jp, path.(string), imageData)
+		if err != nil {
+			return nil, fmt.Errorf("failed to apply JMESPath (%s) results to context entry %s, error: %v", entry.ImageRegistry.JMESPath, entry.Name, err)
+		}
+	}
+
+	return imageData, nil
+}
+
+// FetchImageDataMap fetches image information from the remote registry.
+func (idl *imageDataLoader) fetchImageDataMap(client engineapi.ImageDataClient, ref string) (interface{}, error) {
+	desc, err := client.ForRef(context.Background(), ref)
+	if err != nil {
+		return nil, fmt.Errorf("failed to fetch image descriptor: %s, error: %v", ref, err)
+	}
+
+	var manifest interface{}
+	if err := json.Unmarshal(desc.Manifest, &manifest); err != nil {
+		return nil, fmt.Errorf("failed to decode manifest for image reference: %s, error: %v", ref, err)
+	}
+
+	var configData interface{}
+	if err := json.Unmarshal(desc.Config, &configData); err != nil {
+		return nil, fmt.Errorf("failed to decode config for image reference: %s, error: %v", ref, err)
+	}
+
+	data := map[string]interface{}{
+		"image":         desc.Image,
+		"resolvedImage": desc.ResolvedImage,
+		"registry":      desc.Registry,
+		"repository":    desc.Repository,
+		"identifier":    desc.Identifier,
+		"manifest":      manifest,
+		"configData":    configData,
+	}
+
+	// we need to do the conversion from struct types to an interface type so that jmespath
+	// evaluation works correctly. go-jmespath cannot handle function calls like max/sum
+	// for types like integers for eg. the conversion to untyped allows the stdlib json
+	// to convert all the types to types that are compatible with jmespath.
+	jsonDoc, err := json.Marshal(data)
+	if err != nil {
+		return nil, err
+	}
+
+	var untyped interface{}
+	err = json.Unmarshal(jsonDoc, &untyped)
+	if err != nil {
+		return nil, err
+	}
+
+	return untyped, nil
+}

pkg/engine/context/loaders/variable.go

@@ -0,0 +1,120 @@
+package loaders
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/go-logr/logr"
+	kyvernov1 "github.com/kyverno/kyverno/api/kyverno/v1"
+	enginecontext "github.com/kyverno/kyverno/pkg/engine/context"
+	"github.com/kyverno/kyverno/pkg/engine/jmespath"
+	"github.com/kyverno/kyverno/pkg/engine/variables"
+)
+
+type variableLoader struct {
+	logger    logr.Logger
+	entry     kyvernov1.ContextEntry
+	enginectx enginecontext.Interface
+	jp        jmespath.Interface
+	data      []byte
+}
+
+func NewVariableLoader(
+	logger logr.Logger,
+	entry kyvernov1.ContextEntry,
+	enginectx enginecontext.Interface,
+	jp jmespath.Interface,
+) enginecontext.Loader {
+	return &variableLoader{
+		logger:    logger,
+		entry:     entry,
+		enginectx: enginectx,
+		jp:        jp,
+	}
+}
+
+func (vl *variableLoader) HasLoaded() bool {
+	return vl.data != nil
+}
+
+func (vl *variableLoader) LoadData() error {
+	return vl.loadVariable()
+}
+
+func (vl *variableLoader) loadVariable() (err error) {
+	logger := vl.logger
+	ctx := vl.enginectx
+	entry := vl.entry
+
+	path := ""
+	if entry.Variable.JMESPath != "" {
+		jp, err := variables.SubstituteAll(logger, ctx, entry.Variable.JMESPath)
+		if err != nil {
+			return fmt.Errorf("failed to substitute variables in context entry %s %s: %v", entry.Name, entry.Variable.JMESPath, err)
+		}
+		path = jp.(string)
+		logger.V(4).Info("evaluated jmespath", "variable name", entry.Name, "jmespath", path)
+	}
+
+	var defaultValue interface{} = nil
+	if entry.Variable.Default != nil {
+		value, err := variables.DocumentToUntyped(entry.Variable.Default)
+		if err != nil {
+			return fmt.Errorf("invalid default for variable %s", entry.Name)
+		}
+		defaultValue, err = variables.SubstituteAll(logger, ctx, value)
+		if err != nil {
+			return fmt.Errorf("failed to substitute variables in context entry %s %s: %v", entry.Name, entry.Variable.Default, err)
+		}
+		logger.V(4).Info("evaluated default value", "variable name", entry.Name, "jmespath", defaultValue)
+	}
+
+	var output interface{} = defaultValue
+	if entry.Variable.Value != nil {
+		value, _ := variables.DocumentToUntyped(entry.Variable.Value)
+		variable, err := variables.SubstituteAll(logger, ctx, value)
+		if err != nil {
+			return fmt.Errorf("failed to substitute variables in context entry %s %s: %v", entry.Name, entry.Variable.Value, err)
+		}
+		if path != "" {
+			variable, err := applyJMESPath(vl.jp, path, variable)
+			if err == nil {
+				output = variable
+			} else if defaultValue == nil {
+				return fmt.Errorf("failed to apply jmespath %s to variable %v: %v", path, variable, err)
+			}
+		} else {
+			output = variable
+		}
+	} else {
+		if path != "" {
+			if variable, err := ctx.Query(path); err == nil {
+				if variable != nil {
+					output = variable
+				}
+			} else if defaultValue == nil {
+				return fmt.Errorf("failed to apply jmespath %s to variable %v: %v", path, variable, err)
+			}
+		}
+	}
+
+	logger.V(4).Info("evaluated output", "variable name", entry.Name, "output", output)
+	if output == nil {
+		return fmt.Errorf("failed to add context entry for variable %s since it evaluated to nil", entry.Name)
+	}
+
+	vl.data, err = json.Marshal(output)
+	if err != nil {
+		return fmt.Errorf("failed to add context entry for variable %s: %v", entry.Name, err)
+	}
+
+	return ctx.ReplaceContextEntry(entry.Name, vl.data)
+}
+
+func applyJMESPath(jp jmespath.Interface, query string, data interface{}) (interface{}, error) {
+	q, err := jp.Query(query)
+	if err != nil {
+		return nil, fmt.Errorf("failed to compile JMESPath: %s, error: %v", query, err)
+	}
+	return q.Search(data)
+}