feat: stop adding autogen annotation (#3379)

Signed-off-by: Charles-Edouard Brétéché <charled.breteche@gmail.com>

Co-authored-by: shuting <shuting@nirmata.com>

cmd/kyverno/main.go

@@ -62,6 +62,7 @@ var (
 	profile                      bool
 	disableMetricsExport         bool
 	autoUpdateWebhooks           bool
+	autogenInternals             bool
 	policyControllerResyncPeriod time.Duration
 	imagePullSecrets             string
 	imageSignatureRepository     string
@@ -102,6 +103,7 @@ func main() {
 	flag.BoolVar(&autoUpdateWebhooks, "autoUpdateWebhooks", true, "Set this flag to 'false' to disable auto-configuration of the webhook.")
 	flag.Float64Var(&clientRateLimitQPS, "clientRateLimitQPS", 0, "Configure the maximum QPS to the master from Kyverno. Uses the client default if zero.")
 	flag.IntVar(&clientRateLimitBurst, "clientRateLimitBurst", 0, "Configure the maximum burst for throttle. Uses the client default if zero.")
+	flag.BoolVar(&autogenInternals, "autogenInternals", false, "Enables autogen internal policies. When this is 'true' policy rules should not be mutated.")
 
 	flag.DurationVar(&webhookRegistrationTimeout, "webhookRegistrationTimeout", 120*time.Second, "Timeout for webhook registration, e.g., 30s, 1m, 5m.")
 	if err := flag.Set("v", "2"); err != nil {
@@ -318,6 +320,7 @@ func main() {
 		log.Log.WithName("PolicyController"),
 		policyControllerResyncPeriod,
 		promConfig,
+		autogenInternals,
 	)
 
 	if err != nil {
@@ -475,6 +478,7 @@ func main() {
 		openAPIController,
 		grc,
 		promConfig,
+		autogenInternals,
 	)
 
 	if err != nil {

pkg/engine/utils/utils.go

@@ -38,6 +38,9 @@ func (ri RuleType) String() string {
 // ApplyPatches patches given resource with given patches and returns patched document
 // return original resource if any error occurs
 func ApplyPatches(resource []byte, patches [][]byte) ([]byte, error) {
+	if len(patches) == 0 {
+		return resource, nil
+	}
 	joinedPatches := JoinPatches(patches)
 	patch, err := jsonpatch.DecodePatch(joinedPatches)
 	if err != nil {

pkg/kyverno/apply/apply_command.go

@@ -107,7 +107,7 @@ func Command() *cobra.Command {
 	var resourcePaths []string
 	var cluster, policyReport, stdin, registryAccess bool
 	var mutateLogPath, variablesString, valuesFile, namespace string
-
+	var autogenInternals bool
 	cmd = &cobra.Command{
 		Use:     "apply",
 		Short:   "applies policies on resources",
@@ -122,7 +122,7 @@ func Command() *cobra.Command {
 				}
 			}()
 
-			rc, resources, skipInvalidPolicies, pvInfos, err := applyCommandHelper(resourcePaths, cluster, policyReport, mutateLogPath, variablesString, valuesFile, namespace, policyPaths, stdin, registryAccess)
+			rc, resources, skipInvalidPolicies, pvInfos, err := applyCommandHelper(resourcePaths, cluster, policyReport, mutateLogPath, variablesString, valuesFile, namespace, policyPaths, autogenInternals, stdin, registryAccess)
 			if err != nil {
 				return err
 			}
@@ -141,12 +141,13 @@ func Command() *cobra.Command {
 	cmd.Flags().BoolVarP(&policyReport, "policy-report", "", false, "Generates policy report when passed (default policyviolation r")
 	cmd.Flags().StringVarP(&namespace, "namespace", "n", "", "Optional Policy parameter passed with cluster flag")
 	cmd.Flags().BoolVarP(&stdin, "stdin", "i", false, "Optional mutate policy parameter to pipe directly through to kubectl")
+	cmd.Flags().BoolVarP(&autogenInternals, "autogenInternals", "", false, "Use autogen internals")
 	cmd.Flags().BoolVarP(&registryAccess, "registry", "", false, "If set to true, access the image registry using local docker credentials to populate external data")
 	return cmd
 }
 
 func applyCommandHelper(resourcePaths []string, cluster bool, policyReport bool, mutateLogPath string,
-	variablesString string, valuesFile string, namespace string, policyPaths []string, stdin, registryAccess bool) (rc *common.ResultCounts, resources []*unstructured.Unstructured, skipInvalidPolicies SkippedInvalidPolicies, pvInfos []policyreport.Info, err error) {
+	variablesString string, valuesFile string, namespace string, policyPaths []string, autogenInternals bool, stdin bool, registryAccess bool) (rc *common.ResultCounts, resources []*unstructured.Unstructured, skipInvalidPolicies SkippedInvalidPolicies, pvInfos []policyreport.Info, err error) {
 	store.SetMock(true)
 	store.SetRegistryAccess(registryAccess)
 	kubernetesConfig := genericclioptions.NewConfigFlags(true)
@@ -223,7 +224,7 @@ func applyCommandHelper(resourcePaths []string, cluster bool, policyReport bool,
 		}
 	}
 
-	mutatedPolicies, err := common.MutatePolicies(policies)
+	mutatedPolicies, err := common.MutatePolicies(policies, autogenInternals)
 	if err != nil {
 		if !sanitizederror.IsErrorSanitized(err) {
 			return rc, resources, skipInvalidPolicies, pvInfos, sanitizederror.NewWithError("failed to mutate policy", err)

pkg/kyverno/apply/apply_command_test.go

@@ -72,7 +72,7 @@ func Test_Apply(t *testing.T) {
 	}
 
 	for _, tc := range testcases {
-		_, _, _, info, _ := applyCommandHelper(tc.ResourcePaths, false, true, "", "", "", "", tc.PolicyPaths, false, false)
+		_, _, _, info, _ := applyCommandHelper(tc.ResourcePaths, false, true, "", "", "", "", tc.PolicyPaths, false, false, false)
 		resps := buildPolicyReports(info)
 		for i, resp := range resps {
 			compareSummary(tc.expectedPolicyReports[i].Summary, resp.UnstructuredContent()["summary"].(map[string]interface{}))

pkg/kyverno/common/common.go

@@ -174,8 +174,8 @@ func GetPolicies(paths []string) (policies []*v1.ClusterPolicy, errors []error)
 }
 
 // MutatePolicy - applies mutation to a policy
-func MutatePolicy(policy *v1.ClusterPolicy, logger logr.Logger) (*v1.ClusterPolicy, error) {
+func MutatePolicy(policy *v1.ClusterPolicy, autogenInternals bool, logger logr.Logger) (*v1.ClusterPolicy, error) {
-	patches, _ := policymutation.GenerateJSONPatchesForDefaults(policy, logger)
+	patches, _ := policymutation.GenerateJSONPatchesForDefaults(policy, autogenInternals, logger)
 	if len(patches) == 0 {
 		return policy, nil
 	}
@@ -438,12 +438,12 @@ func GetVariable(variablesString, valuesFile string, fs billy.Filesystem, isGit
 }
 
 // MutatePolicies - function to apply mutation on policies
-func MutatePolicies(policies []*v1.ClusterPolicy) ([]*v1.ClusterPolicy, error) {
+func MutatePolicies(policies []*v1.ClusterPolicy, autogenInternals bool) ([]*v1.ClusterPolicy, error) {
 	newPolicies := make([]*v1.ClusterPolicy, 0)
 	logger := log.Log.WithName("apply")
 
 	for _, policy := range policies {
-		p, err := MutatePolicy(policy, logger)
+		p, err := MutatePolicy(policy, autogenInternals, logger)
 		if err != nil {
 			if !sanitizederror.IsErrorSanitized(err) {
 				return nil, sanitizederror.NewWithError("failed to mutate policy.", err)

pkg/kyverno/test/test_command.go

@@ -801,7 +801,7 @@ func applyPoliciesFromPath(fs billy.Filesystem, policyBytes []byte, isGit bool,
 	}
 	policies = filteredPolicies
 
-	mutatedPolicies, err := common.MutatePolicies(policies)
+	mutatedPolicies, err := common.MutatePolicies(policies, false)
 	if err != nil {
 		if !sanitizederror.IsErrorSanitized(err) {
 			return sanitizederror.NewWithError("failed to mutate policy", err)

pkg/kyverno/validate/command.go

@@ -27,6 +27,7 @@ import (
 func Command() *cobra.Command {
 	var outputType string
 	var crdPaths []string
+	var autogenInternals bool
 	cmd := &cobra.Command{
 		Use:     "validate",
 		Short:   "Validates kyverno policies",
@@ -78,7 +79,7 @@ func Command() *cobra.Command {
 				}
 			}
 
-			err = validatePolicies(policies, v1crd, openAPIController, outputType)
+			err = validatePolicies(policies, autogenInternals, v1crd, openAPIController, outputType)
 			if err != nil {
 				return sanitizederror.NewWithError("failed to validate policies", err)
 			}
@@ -87,6 +88,7 @@ func Command() *cobra.Command {
 	}
 	cmd.Flags().StringVarP(&outputType, "output", "o", "", "Prints the mutated policy in yaml or json format")
 	cmd.Flags().StringArrayVarP(&crdPaths, "crd", "c", []string{}, "Path to CRD files")
+	cmd.Flags().BoolVarP(&autogenInternals, "autogenInternals", "", false, "Use autogen internals")
 	return cmd
 }
 
@@ -153,7 +155,7 @@ func validatePolicyAccordingToPolicyCRD(policy *v1.ClusterPolicy, v1crd apiexten
 	return
 }
 
-func validatePolicies(policies []*v1.ClusterPolicy, v1crd apiextensions.CustomResourceDefinitionSpec, openAPIController *openapi.Controller, outputType string) error {
+func validatePolicies(policies []*v1.ClusterPolicy, autogenInternals bool, v1crd apiextensions.CustomResourceDefinitionSpec, openAPIController *openapi.Controller, outputType string) error {
 	invalidPolicyFound := false
 	for _, policy := range policies {
 		err, errorList := validatePolicyAccordingToPolicyCRD(policy, v1crd)
@@ -178,7 +180,7 @@ func validatePolicies(policies []*v1.ClusterPolicy, v1crd apiextensions.CustomRe
 			fmt.Printf("Policy %s is valid.\n\n", policy.Name)
 			if outputType != "" {
 				logger := log.Log.WithName("validate")
-				p, err := common.MutatePolicy(policy, logger)
+				p, err := common.MutatePolicy(policy, autogenInternals, logger)
 				if err != nil {
 					if !sanitizederror.IsErrorSanitized(err) {
 						return sanitizederror.NewWithError("failed to mutate policy.", err)

pkg/policy/policy_controller.go

@@ -103,6 +103,8 @@ type PolicyController struct {
 	log logr.Logger
 
 	promConfig *metrics.PromConfig
+
+	autogenInternals bool
 }
 
 // NewPolicyController create a new PolicyController
@@ -120,7 +122,9 @@ func NewPolicyController(
 	namespaces informers.NamespaceInformer,
 	log logr.Logger,
 	reconcilePeriod time.Duration,
-	promConfig *metrics.PromConfig) (*PolicyController, error) {
+	promConfig *metrics.PromConfig,
+	autogenInternals bool,
+) (*PolicyController, error) {
 
 	// Event broad caster
 	eventBroadcaster := record.NewBroadcaster()
@@ -145,6 +149,7 @@ func NewPolicyController(
 		reconcilePeriod:    reconcilePeriod,
 		promConfig:         promConfig,
 		log:                log,
+		autogenInternals:   autogenInternals,
 	}
 
 	pc.pLister = pInformer.Lister()
@@ -193,7 +198,7 @@ func (pc *PolicyController) addPolicy(obj interface{}) {
 	go pc.registerPolicyChangesMetricAddPolicy(logger, p)
 
 	if p.Spec.Background == nil || p.Spec.ValidationFailureAction == "" || missingAutoGenRules(p, logger) {
-		pol, _ := common.MutatePolicy(p, logger)
+		pol, _ := common.MutatePolicy(p, pc.autogenInternals, logger)
 		pol.SetGroupVersionKind(schema.GroupVersionKind{Group: "kyverno.io", Version: "v1", Kind: "ClusterPolicy"})
 		_, err := pc.client.UpdateResource("kyverno.io/v1", "ClusterPolicy", "", pol, false)
 		if err != nil {
@@ -220,7 +225,7 @@ func (pc *PolicyController) updatePolicy(old, cur interface{}) {
 	go pc.registerPolicyChangesMetricUpdatePolicy(logger, oldP, curP)
 
 	if curP.Spec.Background == nil || curP.Spec.ValidationFailureAction == "" || missingAutoGenRules(curP, logger) {
-		pol, _ := common.MutatePolicy(curP, logger)
+		pol, _ := common.MutatePolicy(curP, pc.autogenInternals, logger)
 		pol.SetGroupVersionKind(schema.GroupVersionKind{Group: "kyverno.io", Version: "v1", Kind: "ClusterPolicy"})
 		_, err := pc.client.UpdateResource("kyverno.io/v1", "ClusterPolicy", "", pol, false)
 		if err != nil {
@@ -292,7 +297,7 @@ func (pc *PolicyController) addNsPolicy(obj interface{}) {
 
 	pol := ConvertPolicyToClusterPolicy(p)
 	if pol.Spec.Background == nil || pol.Spec.ValidationFailureAction == "" || missingAutoGenRules(pol, logger) {
-		nsPol, _ := common.MutatePolicy(pol, logger)
+		nsPol, _ := common.MutatePolicy(pol, pc.autogenInternals, logger)
 		nsPol.SetGroupVersionKind(schema.GroupVersionKind{Group: "kyverno.io", Version: "v1", Kind: "Policy"})
 		_, err := pc.client.UpdateResource("kyverno.io/v1", "Policy", p.Namespace, nsPol, false)
 		if err != nil {
@@ -319,7 +324,7 @@ func (pc *PolicyController) updateNsPolicy(old, cur interface{}) {
 	ncurP := ConvertPolicyToClusterPolicy(curP)
 
 	if ncurP.Spec.Background == nil || ncurP.Spec.ValidationFailureAction == "" || missingAutoGenRules(ncurP, logger) {
-		nsPol, _ := common.MutatePolicy(ncurP, logger)
+		nsPol, _ := common.MutatePolicy(ncurP, pc.autogenInternals, logger)
 		nsPol.SetGroupVersionKind(schema.GroupVersionKind{Group: "kyverno.io", Version: "v1", Kind: "Policy"})
 		_, err := pc.client.UpdateResource("kyverno.io/v1", "Policy", ncurP.GetNamespace(), nsPol, false)
 		if err != nil {

pkg/policymutation/policymutation.go

@@ -17,7 +17,7 @@ import (
 // - ValidationFailureAction
 // - Background
 // - auto-gen annotation and rules
-func GenerateJSONPatchesForDefaults(policy *kyverno.ClusterPolicy, log logr.Logger) ([]byte, []string) {
+func GenerateJSONPatchesForDefaults(policy *kyverno.ClusterPolicy, autogenInternals bool, log logr.Logger) ([]byte, []string) {
 	var patches [][]byte
 	var updateMsgs []string
 
@@ -38,7 +38,7 @@ func GenerateJSONPatchesForDefaults(policy *kyverno.ClusterPolicy, log logr.Logg
 		updateMsgs = append(updateMsgs, updateMsg)
 	}
 
-	patch, errs := GeneratePodControllerRule(*policy, log)
+	patch, errs := GeneratePodControllerRule(*policy, autogenInternals, log)
 	if len(errs) > 0 {
 		var errMsgs []string
 		for _, err := range errs {
@@ -248,7 +248,7 @@ func defaultFailurePolicy(spec *kyverno.Spec, log logr.Logger) ([]byte, string)
 //             make sure all fields are applicable to pod controllers
 
 // GeneratePodControllerRule returns two patches: rulePatches and annotation patch(if necessary)
-func GeneratePodControllerRule(policy kyverno.ClusterPolicy, log logr.Logger) (patches [][]byte, errs []error) {
+func GeneratePodControllerRule(policy kyverno.ClusterPolicy, autogenInternals bool, log logr.Logger) (patches [][]byte, errs []error) {
 	applyAutoGen, desiredControllers := autogen.CanAutoGen(&policy.Spec, log)
 
 	if !applyAutoGen {
@@ -262,11 +262,13 @@ func GeneratePodControllerRule(policy kyverno.ClusterPolicy, log logr.Logger) (p
 	// - predefined controllers are invalid, overwrite the value
 	if !ok || !applyAutoGen {
 		actualControllers = desiredControllers
-		annPatch, err := defaultPodControllerAnnotation(ann, actualControllers)
+		if !autogenInternals {
-		if err != nil {
+			annPatch, err := defaultPodControllerAnnotation(ann, actualControllers)
-			errs = append(errs, fmt.Errorf("failed to generate pod controller annotation for policy '%s': %v", policy.Name, err))
+			if err != nil {
-		} else {
+				errs = append(errs, fmt.Errorf("failed to generate pod controller annotation for policy '%s': %v", policy.Name, err))
-			patches = append(patches, annPatch)
+			} else {
+				patches = append(patches, annPatch)
+			}
 		}
 	} else {
 		if !applyAutoGen {

pkg/webhooks/policymutation.go

@@ -42,7 +42,7 @@ func (ws *WebhookServer) policyMutation(request *v1beta1.AdmissionRequest) *v1be
 	defer logger.V(3).Info("finished policy change mutation", "time", time.Since(startTime).String())
 
 	// Generate JSON Patches for defaults
-	patches, updateMsgs := policymutation.GenerateJSONPatchesForDefaults(policy, logger)
+	patches, updateMsgs := policymutation.GenerateJSONPatchesForDefaults(policy, ws.autogenInternals, logger)
 	if len(patches) != 0 {
 		patchType := v1beta1.PatchTypeJSONPatch
 		return &v1beta1.AdmissionResponse{

pkg/webhooks/policymutation_test.go

@@ -34,7 +34,7 @@ func TestGeneratePodControllerRule_NilAnnotation(t *testing.T) {
 
 	var policy kyverno.ClusterPolicy
 	assert.Assert(t, json.Unmarshal(policyRaw, &policy))
-	patches, errs := policymutation.GeneratePodControllerRule(policy, log.Log)
+	patches, errs := policymutation.GeneratePodControllerRule(policy, false, log.Log)
 	assert.Assert(t, len(errs) == 0)
 
 	p, err := utils.ApplyPatches(policyRaw, patches)
@@ -67,7 +67,7 @@ func TestGeneratePodControllerRule_PredefinedAnnotation(t *testing.T) {
 
 	var policy kyverno.ClusterPolicy
 	assert.Assert(t, json.Unmarshal(policyRaw, &policy))
-	patches, errs := policymutation.GeneratePodControllerRule(policy, log.Log)
+	patches, errs := policymutation.GeneratePodControllerRule(policy, false, log.Log)
 	assert.Assert(t, len(errs) == 0)
 	assert.Assert(t, len(patches) == 1)
 }
@@ -118,7 +118,7 @@ func TestGeneratePodControllerRule_DisableFeature(t *testing.T) {
 
 	var policy kyverno.ClusterPolicy
 	assert.Assert(t, json.Unmarshal(policyRaw, &policy))
-	patches, errs := policymutation.GeneratePodControllerRule(policy, log.Log)
+	patches, errs := policymutation.GeneratePodControllerRule(policy, false, log.Log)
 	assert.Assert(t, len(errs) == 0)
 	assert.Assert(t, len(patches) == 0)
 }
@@ -169,7 +169,7 @@ func TestGeneratePodControllerRule_Mutate(t *testing.T) {
 
 	var policy kyverno.ClusterPolicy
 	assert.Assert(t, json.Unmarshal(policyRaw, &policy))
-	patches, errs := policymutation.GeneratePodControllerRule(policy, log.Log)
+	patches, errs := policymutation.GeneratePodControllerRule(policy, false, log.Log)
 	assert.Assert(t, len(errs) == 0)
 
 	p, err := utils.ApplyPatches(policyRaw, patches)
@@ -303,7 +303,7 @@ func TestGeneratePodControllerRule_ExistOtherAnnotation(t *testing.T) {
 
 	var policy kyverno.ClusterPolicy
 	assert.Assert(t, json.Unmarshal(policyRaw, &policy))
-	patches, errs := policymutation.GeneratePodControllerRule(policy, log.Log)
+	patches, errs := policymutation.GeneratePodControllerRule(policy, false, log.Log)
 	assert.Assert(t, len(errs) == 0)
 
 	p, err := utils.ApplyPatches(policyRaw, patches)
@@ -375,7 +375,7 @@ func TestGeneratePodControllerRule_ValidateAnyPattern(t *testing.T) {
 
 	var policy kyverno.ClusterPolicy
 	assert.Assert(t, json.Unmarshal(policyRaw, &policy))
-	patches, errs := policymutation.GeneratePodControllerRule(policy, log.Log)
+	patches, errs := policymutation.GeneratePodControllerRule(policy, false, log.Log)
 	assert.Assert(t, len(errs) == 0)
 
 	p, err := utils.ApplyPatches(policyRaw, patches)
@@ -514,7 +514,7 @@ func TestGeneratePodControllerRule_ValidatePattern(t *testing.T) {
 	var policy kyverno.ClusterPolicy
 	// var policy, generatePolicy unstructured.Unstructured
 	assert.Assert(t, json.Unmarshal(policyRaw, &policy))
-	patches, errs := policymutation.GeneratePodControllerRule(policy, log.Log)
+	patches, errs := policymutation.GeneratePodControllerRule(policy, false, log.Log)
 	assert.Assert(t, len(errs) == 0)
 
 	p, err := utils.ApplyPatches(policyRaw, patches)

pkg/webhooks/server.go

@@ -124,6 +124,8 @@ type WebhookServer struct {
 	grController *generate.Controller
 
 	promConfig *metrics.PromConfig
+
+	autogenInternals bool
 }
 
 // NewWebhookServer creates new instance of WebhookServer accordingly to given configuration
@@ -152,6 +154,7 @@ func NewWebhookServer(
 	openAPIController *openapi.Controller,
 	grc *generate.Controller,
 	promConfig *metrics.PromConfig,
+	autogenInternals bool,
 ) (*WebhookServer, error) {
 
 	if tlsPair == nil {
@@ -196,6 +199,7 @@ func NewWebhookServer(
 		log:               log,
 		openAPIController: openAPIController,
 		promConfig:        promConfig,
+		autogenInternals:  autogenInternals,
 	}
 
 	mux := httprouter.New()