kyverno/pkg/generate/generate.go

package generate

import (
	contextdefault "context"
	"encoding/json"
	"fmt"
	"reflect"
	"time"

	"github.com/go-logr/logr"
	kyverno "github.com/kyverno/kyverno/pkg/api/kyverno/v1"
	"github.com/kyverno/kyverno/pkg/config"
	dclient "github.com/kyverno/kyverno/pkg/dclient"
	"github.com/kyverno/kyverno/pkg/engine"
	"github.com/kyverno/kyverno/pkg/engine/context"
	"github.com/kyverno/kyverno/pkg/engine/validate"
	"github.com/kyverno/kyverno/pkg/engine/variables"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
)

func (c *Controller) processGR(gr *kyverno.GenerateRequest) error {
	logger := c.log.WithValues("name", gr.Name, "policy", gr.Spec.Policy, "kind", gr.Spec.Resource.Kind, "apiVersion", gr.Spec.Resource.APIVersion, "namespace", gr.Spec.Resource.Namespace, "name", gr.Spec.Resource.Name)
	var err error
	var resource *unstructured.Unstructured
	var genResources []kyverno.ResourceSpec

	// 1 - Check if the resource exists
	resource, err = getResource(c.client, gr.Spec.Resource)
	if err != nil {
		// Dont update status
		logger.V(3).Info("resource does not exist or is pending creation, re-queueing", "details", err.Error())
		return err
	}

	// trigger resource is being terminated
	if resource == nil {
		return nil
	}

	// 2 - Apply the generate policy on the resource
	genResources, err = c.applyGenerate(*resource, *gr)

	// 3 - Report Events
	events := failedEvents(err, *gr, *resource)
	c.eventGen.Add(events...)

	// 4 - Update Status
	return updateStatus(c.statusControl, *gr, err, genResources)
}

func (c *Controller) applyGenerate(resource unstructured.Unstructured, gr kyverno.GenerateRequest) ([]kyverno.ResourceSpec, error) {
	logger := c.log.WithValues("name", gr.Name, "policy", gr.Spec.Policy, "kind", gr.Spec.Resource.Kind, "apiVersion", gr.Spec.Resource.APIVersion, "namespace", gr.Spec.Resource.Namespace, "name", gr.Spec.Resource.Name)
	// Get the list of rules to be applied
	// get policy
	// build context
	ctx := context.NewContext()

	logger.V(3).Info("applying generate policy rule")

	policyObj, err := c.policyLister.Get(gr.Spec.Policy)
	if err != nil {
		if apierrors.IsNotFound(err) {
			for _, e := range gr.Status.GeneratedResources {
				resp, err := c.client.GetResource(e.APIVersion, e.Kind, e.Namespace, e.Name)
				if err != nil && !apierrors.IsNotFound(err) {
					logger.Error(err, "failed to find generated resource", "name", e.Name)
					continue
				}

				if resp != nil && resp.GetLabels()["policy.kyverno.io/synchronize"] == "enable" {
					if err := c.client.DeleteResource(resp.GetAPIVersion(), resp.GetKind(), resp.GetNamespace(), resp.GetName(), false); err != nil {
						logger.Error(err, "generated resource is not deleted", "Resource", e.Name)
					}
				}
			}

			return nil, nil
		}

		logger.Error(err, "error in fetching policy")
		return nil, err
	}

	resourceRaw, err := resource.MarshalJSON()
	if err != nil {
		logger.Error(err, "failed to marshal resource")
		return nil, err
	}

	err = ctx.AddResource(resourceRaw)
	if err != nil {
		logger.Error(err, "failed to load resource in context")
		return nil, err
	}

	err = ctx.AddUserInfo(gr.Spec.Context.UserRequestInfo)
	if err != nil {
		logger.Error(err, "failed to load SA in context")
		return nil, err
	}

	err = ctx.AddSA(gr.Spec.Context.UserRequestInfo.AdmissionUserInfo.Username)
	if err != nil {
		logger.Error(err, "failed to load UserInfo in context")
		return nil, err
	}

	policyContext := engine.PolicyContext{
		NewResource:      resource,
		Policy:           *policyObj,
		Context:          ctx,
		AdmissionInfo:    gr.Spec.Context.UserRequestInfo,
		ExcludeGroupRole: c.Config.GetExcludeGroupRole(),
		ResourceCache:    c.resCache,
		JSONContext:      ctx,
	}

	// check if the policy still applies to the resource
	engineResponse := engine.Generate(policyContext)
	if len(engineResponse.PolicyResponse.Rules) == 0 {
		logger.V(4).Info("policy does not apply to resource")
		return nil, fmt.Errorf("policy %s, does not apply to resource %v", gr.Spec.Policy, gr.Spec.Resource)
	}

	// Removing GR if rule is failed. Used when the generate condition failed but gr exist
	for _, r := range engineResponse.PolicyResponse.Rules {
		if !r.Success {

			logger.V(4).Info("querying all generate requests")
			grList, err := c.kyvernoClient.KyvernoV1().GenerateRequests(config.KyvernoNamespace).List(contextdefault.TODO(), metav1.ListOptions{})
			if err != nil {
				logger.Error(err, "failed to list generate requests")
				continue
			}

			for _, v := range grList.Items {
				if engineResponse.PolicyResponse.Policy == v.Spec.Policy && engineResponse.PolicyResponse.Resource.Name == v.Spec.Resource.Name && engineResponse.PolicyResponse.Resource.Kind == v.Spec.Resource.Kind && engineResponse.PolicyResponse.Resource.Namespace == v.Spec.Resource.Namespace {
					err := c.kyvernoClient.KyvernoV1().GenerateRequests(config.KyvernoNamespace).Delete(contextdefault.TODO(), v.GetName(), metav1.DeleteOptions{})
					if err != nil {
						logger.Error(err, " failed to delete generate request")
					}
				}
			}
		}
	}

	// Apply the generate rule on resource
	return c.applyGeneratePolicy(logger, policyContext, gr)
}

func updateStatus(statusControl StatusControlInterface, gr kyverno.GenerateRequest, err error, genResources []kyverno.ResourceSpec) error {
	if err != nil {
		return statusControl.Failed(gr, err.Error(), genResources)
	}

	// Generate request successfully processed
	return statusControl.Success(gr, genResources)
}

func (c *Controller) applyGeneratePolicy(log logr.Logger, policyContext engine.PolicyContext, gr kyverno.GenerateRequest) ([]kyverno.ResourceSpec, error) {
	// List of generatedResources
	var genResources []kyverno.ResourceSpec
	// Get the response as the actions to be performed on the resource
	// - - substitute values
	policy := policyContext.Policy
	resource := policyContext.NewResource
	ctx := policyContext.Context

	resCache := policyContext.ResourceCache
	jsonContext := policyContext.JSONContext
	// To manage existing resources, we compare the creation time for the default resource to be generated and policy creation time

	ruleNameToProcessingTime := make(map[string]time.Duration)
	for _, rule := range policy.Spec.Rules {
		if !rule.HasGenerate() {
			continue
		}

		startTime := time.Now()
		processExisting := false

		if len(rule.MatchResources.Kinds) > 0 {
			if len(rule.MatchResources.Annotations) == 0 && rule.MatchResources.Selector == nil {
				processExisting = func() bool {
					rcreationTime := resource.GetCreationTimestamp()
					pcreationTime := policy.GetCreationTimestamp()
					return rcreationTime.Before(&pcreationTime)
				}()
			}
		}

		// add configmap json data to context
		if err := engine.AddResourceToContext(log, rule.Context, resCache, jsonContext); err != nil {
			log.Info("cannot add configmaps to context", "reason", err.Error())
			return nil, err
		}

		genResource, err := applyRule(log, c.client, rule, resource, ctx, policy.Name, gr, processExisting)
		if err != nil {
			log.Error(err, "failed to apply generate rule", "policy", policy.Name,
				"rule", rule.Name, "resource", resource.GetName())
			return nil, err
		}

		ruleNameToProcessingTime[rule.Name] = time.Since(startTime)
		genResources = append(genResources, genResource)
	}

	if gr.Status.State == "" && len(genResources) > 0 {
		log.V(3).Info("updating policy status", "policy", policy.Name, "data", ruleNameToProcessingTime)
		c.policyStatusListener.Update(generateSyncStats{
			policyName:               policy.Name,
			ruleNameToProcessingTime: ruleNameToProcessingTime,
		})
	}

	return genResources, nil
}

type generateSyncStats struct {
	policyName               string
	ruleNameToProcessingTime map[string]time.Duration
}

func (vc generateSyncStats) PolicyName() string {
	return vc.policyName
}

func (vc generateSyncStats) UpdateStatus(status kyverno.PolicyStatus) kyverno.PolicyStatus {

	for i := range status.Rules {
		if executionTime, exist := vc.ruleNameToProcessingTime[status.Rules[i].Name]; exist {
			status.ResourcesGeneratedCount++
			status.Rules[i].ResourcesGeneratedCount++
			averageOver := int64(status.Rules[i].AppliedCount + status.Rules[i].FailedCount)
			status.Rules[i].ExecutionTime = updateGenerateExecutionTime(
				executionTime,
				status.Rules[i].ExecutionTime,
				averageOver,
			).String()
		}
	}

	return status
}

func updateGenerateExecutionTime(newTime time.Duration, oldAverageTimeString string, averageOver int64) time.Duration {
	if averageOver == 0 {
		return newTime
	}
	oldAverageExecutionTime, _ := time.ParseDuration(oldAverageTimeString)
	numerator := (oldAverageExecutionTime.Nanoseconds() * averageOver) + newTime.Nanoseconds()
	denominator := averageOver
	newAverageTimeInNanoSeconds := numerator / denominator
	return time.Duration(newAverageTimeInNanoSeconds) * time.Nanosecond
}

func applyRule(log logr.Logger, client *dclient.Client, rule kyverno.Rule, resource unstructured.Unstructured, ctx context.EvalInterface, policy string, gr kyverno.GenerateRequest, processExisting bool) (kyverno.ResourceSpec, error) {
	var rdata map[string]interface{}
	var err error
	var mode ResourceMode
	var noGenResource kyverno.ResourceSpec
	// convert to unstructured Resource
	genUnst, err := getUnstrRule(rule.Generation.DeepCopy())
	if err != nil {
		return noGenResource, err
	}
	// Variable substitutions
	// format : {{<variable_name}}
	// - if there is variables that are not defined the context -> results in error and rule is not applied
	// - valid variables are replaced with the values
	object, err := variables.SubstituteVars(log, ctx, genUnst.Object)
	if err != nil {
		return noGenResource, err
	}
	genUnst.Object, _ = object.(map[string]interface{})

	genKind, _, err := unstructured.NestedString(genUnst.Object, "kind")
	if err != nil {
		return noGenResource, err
	}
	genName, _, err := unstructured.NestedString(genUnst.Object, "name")
	if err != nil {
		return noGenResource, err
	}
	genNamespace, _, err := unstructured.NestedString(genUnst.Object, "namespace")
	if err != nil {
		return noGenResource, err
	}

	genAPIVersion, _, err := unstructured.NestedString(genUnst.Object, "apiVersion")
	if err != nil {
		return noGenResource, err
	}
	// Resource to be generated
	newGenResource := kyverno.ResourceSpec{
		APIVersion: genAPIVersion,
		Kind:       genKind,
		Namespace:  genNamespace,
		Name:       genName,
	}
	genData, _, err := unstructured.NestedMap(genUnst.Object, "data")
	if err != nil {
		return noGenResource, err
	}
	genCopy, _, err := unstructured.NestedMap(genUnst.Object, "clone")
	if err != nil {
		return noGenResource, err
	}
	if genData != nil {
		rdata, mode, err = manageData(log, genAPIVersion, genKind, genNamespace, genName, genData, client, resource)
	} else {
		rdata, mode, err = manageClone(log, genAPIVersion, genKind, genNamespace, genName, genCopy, client, resource)
	}

	logger := log.WithValues("genKind", genKind, "genAPIVersion", genAPIVersion, "genNamespace", genNamespace, "genName", genName)

	if err != nil {
		return noGenResource, err
	}

	if rdata == nil {
		// existing resource contains the configuration
		return newGenResource, nil
	}
	if processExisting {
		return noGenResource, nil
	}

	// build the resource template
	newResource := &unstructured.Unstructured{}
	newResource.SetUnstructuredContent(rdata)
	newResource.SetName(genName)
	newResource.SetNamespace(genNamespace)
	if newResource.GetKind() == "" {
		newResource.SetKind(genKind)
	}

	newResource.SetAPIVersion(genAPIVersion)
	// manage labels
	// - app.kubernetes.io/managed-by: kyverno
	// - kyverno.io/generated-by: kind/namespace/name (trigger resource)
	manageLabels(newResource, resource)
	// Add Synchronize label
	label := newResource.GetLabels()
	label["policy.kyverno.io/policy-name"] = policy
	label["policy.kyverno.io/gr-name"] = gr.Name
	newResource.SetLabels(label)
	if mode == Create {
		if rule.Generation.Synchronize {
			label["policy.kyverno.io/synchronize"] = "enable"
		} else {
			label["policy.kyverno.io/synchronize"] = "disable"
		}
		// Reset resource version
		newResource.SetResourceVersion("")
		// Create the resource
		_, err = client.CreateResource(genAPIVersion, genKind, genNamespace, newResource, false)
		if err != nil {
			return noGenResource, err
		}

		logger.V(2).Info("created resource")

	} else if mode == Update {
		var isUpdate bool
		label := newResource.GetLabels()
		isUpdate = false
		if rule.Generation.Synchronize {
			if label["policy.kyverno.io/synchronize"] == "enable" {
				isUpdate = true
			}
		} else {
			if label["policy.kyverno.io/synchronize"] == "enable" {
				isUpdate = true
			}
		}
		if rule.Generation.Synchronize {
			label["policy.kyverno.io/synchronize"] = "enable"
		} else {
			label["policy.kyverno.io/synchronize"] = "disable"
		}
		if isUpdate {
			logger.V(4).Info("updating existing resource")
			newResource.SetLabels(label)
			_, err := client.UpdateResource(genAPIVersion, genKind, genNamespace, newResource, false)
			if err != nil {
				logger.Error(err, "updating existing resource")
				return noGenResource, err
			}
			logger.V(2).Info("updated generated resource")
		} else {
			resource := &unstructured.Unstructured{}
			resource.SetUnstructuredContent(rdata)
			resource.SetLabels(label)
			_, err := client.UpdateResource(genAPIVersion, genKind, genNamespace, resource, false)
			if err != nil {
				logger.Error(err, "updating existing resource")
				return noGenResource, err
			}
			logger.V(2).Info("updated generated resource")
		}

		logger.V(2).Info("Synchronize resource is disabled")
	}
	return newGenResource, nil
}

func manageData(log logr.Logger, apiVersion, kind, namespace, name string, data map[string]interface{}, client *dclient.Client, resource unstructured.Unstructured) (map[string]interface{}, ResourceMode, error) {
	// check if resource to be generated exists
	obj, err := client.GetResource(apiVersion, kind, namespace, name)
	if err != nil {
		if apierrors.IsNotFound(err) {
			log.V(3).Info("resource does not exist, will try to create", "genKind", kind, "genAPIVersion", apiVersion, "genNamespace", namespace, "genName", name)
			return data, Create, nil
		}
		//something wrong while fetching resource
		// client-errors
		return nil, Skip, err
	}
	updateObj := &unstructured.Unstructured{}
	updateObj.SetUnstructuredContent(data)
	updateObj.SetResourceVersion(obj.GetResourceVersion())
	return updateObj.UnstructuredContent(), Update, nil
}

func manageClone(log logr.Logger, apiVersion, kind, namespace, name string, clone map[string]interface{}, client *dclient.Client, resource unstructured.Unstructured) (map[string]interface{}, ResourceMode, error) {
	newRNs, _, err := unstructured.NestedString(clone, "namespace")
	if err != nil {
		return nil, Skip, err
	}
	newRName, _, err := unstructured.NestedString(clone, "name")
	if err != nil {
		return nil, Skip, err
	}

	// Short-circuit if the resource to be generated and the clone is the same
	if newRNs == namespace && newRName == name {
		// attempting to clone it self, this will fail -> short-ciruit it
		return nil, Skip, nil
	}

	// check if the resource as reference in clone exists?
	obj, err := client.GetResource(apiVersion, kind, newRNs, newRName)
	if err != nil {
		return nil, Skip, fmt.Errorf("reference clone resource %s/%s/%s/%s not found. %v", apiVersion, kind, newRNs, newRName, err)
	}

	// check if resource to be generated exists
	newResource, err := client.GetResource(apiVersion, kind, namespace, name)
	if err == nil {
		obj.SetUID(newResource.GetUID())
		obj.SetSelfLink(newResource.GetSelfLink())
		obj.SetCreationTimestamp(newResource.GetCreationTimestamp())
		obj.SetManagedFields(newResource.GetManagedFields())
		obj.SetResourceVersion(newResource.GetResourceVersion())
		if reflect.DeepEqual(obj, newResource) {
			return nil, Skip, nil
		}
		return obj.UnstructuredContent(), Update, nil
	}

	// create the resource based on the reference clone
	return obj.UnstructuredContent(), Create, nil

}

// ResourceMode defines the mode for generated resource
type ResourceMode string

const (
	//Skip : failed to process rule, will not update the resource
	Skip ResourceMode = "SKIP"
	//Create : create a new resource
	Create = "CREATE"
	//Update : update/overwrite the new resource
	Update = "UPDATE"
)

func checkResource(log logr.Logger, newResourceSpec interface{}, resource *unstructured.Unstructured) error {
	// check if the resource spec if a subset of the resource
	if path, err := validate.ValidateResourceWithPattern(log, resource.Object, newResourceSpec); err != nil {
		log.Error(err, "Failed to match the resource ", "path", path)
		return err
	}
	return nil
}

func getUnstrRule(rule *kyverno.Generation) (*unstructured.Unstructured, error) {
	ruleData, err := json.Marshal(rule)
	if err != nil {
		return nil, err
	}
	return ConvertToUnstructured(ruleData)
}

//ConvertToUnstructured converts the resource to unstructured format
func ConvertToUnstructured(data []byte) (*unstructured.Unstructured, error) {
	resource := &unstructured.Unstructured{}
	err := resource.UnmarshalJSON(data)
	if err != nil {
		return nil, err
	}
	return resource, nil
}