Bugfix/1.1.6 adjust resync and cleanup unused (#884)

* - support wildcards for namespaces

* do not annotate resource, unless policy is an autogen policy

* close HTTP body

* improve messages

* remove policy store

Policy store was not fully implemented and simply provided a way
to list all polices and get a policy by name, which can be done via
standard client-go interfaces.

We need to revisit and design a better PolicyStore that provides fast
lookups for matching policies based on names, namespaces, etc.

* handle wildcard namespaces in background processing

* fix unit tests 1) remove platform dependent path usage 2) remove policy store

* add test case for mutate with wildcard namespaces

* adjust all resync periods

* remove unused data fields

* add pattern for match

cmd/initContainer/main.go

@@ -53,7 +53,7 @@ func main() {
 
 	// DYNAMIC CLIENT
 	// - client for all registered resources
-	client, err := client.NewClient(clientConfig, 10*time.Second, stopCh, log.Log)
+	client, err := client.NewClient(clientConfig, 15*time.Minute, stopCh, log.Log)
 	if err != nil {
 		setupLog.Error(err, "Failed to create client")
 		os.Exit(1)

pkg/constant/constant.go

@@ -3,10 +3,10 @@ package constant
 import "time"
 
 const (
-	CRDControllerResync             = 10 * time.Minute
+	CRDControllerResync             = 15 * time.Minute
-	PolicyViolationControllerResync = 5 * time.Minute
+	PolicyViolationControllerResync = 15 * time.Minute
-	PolicyControllerResync          = time.Second
+	PolicyControllerResync          = 15 * time.Minute
-	EventControllerResync           = time.Second
+	EventControllerResync           = 15 * time.Minute
-	GenerateControllerResync        = time.Second
+	GenerateControllerResync        = 15 * time.Minute
-	GenerateRequestControllerResync = time.Second
+	GenerateRequestControllerResync = 15 * time.Minute
 )

pkg/kyverno/apply/command.go

@@ -94,7 +94,7 @@ func Command() *cobra.Command {
 				if err != nil {
 					return err
 				}
-				dClient, err = client.NewClient(restConfig, 10*time.Second, make(chan struct{}), log.Log)
+				dClient, err = client.NewClient(restConfig, 5*time.Minute, make(chan struct{}), log.Log)
 				if err != nil {
 					return err
 				}

pkg/openapi/crdSync.go

@@ -16,7 +16,7 @@ import (
 	"github.com/googleapis/gnostic/compiler"
 
 	openapi_v2 "github.com/googleapis/gnostic/OpenAPIv2"
-	log "sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/log"
 
 	"github.com/nirmata/kyverno/pkg/constant"
 	client "github.com/nirmata/kyverno/pkg/dclient"
@@ -28,7 +28,7 @@ type crdSync struct {
 	controller *Controller
 }
 
-// crdDefinitionPrior represents CRD's version prior to 1.16
+// crdDefinitionPrior represents CRDs version prior to 1.16
 var crdDefinitionPrior struct {
 	Spec struct {
 		Names struct {
@@ -40,7 +40,7 @@ var crdDefinitionPrior struct {
 	} `json:"spec"`
 }
 
-// crdDefinitionNew represents CRD in version 1.16+
+// crdDefinitionNew represents CRDs version 1.16+
 var crdDefinitionNew struct {
 	Spec struct {
 		Names struct {
@@ -55,9 +55,6 @@ var crdDefinitionNew struct {
 	} `json:"spec"`
 }
 
-var crdVersion struct {
-}
-
 func NewCRDSync(client *client.Client, controller *Controller) *crdSync {
 	if controller == nil {
 		panic(fmt.Errorf("nil controller sent into crd sync"))
@@ -72,12 +69,12 @@ func NewCRDSync(client *client.Client, controller *Controller) *crdSync {
 func (c *crdSync) Run(workers int, stopCh <-chan struct{}) {
 	newDoc, err := c.client.DiscoveryClient.OpenAPISchema()
 	if err != nil {
-		log.Log.Error(err, "cannot get openapi schema")
+		log.Log.Error(err, "cannot get OpenAPI schema")
 	}
 
 	err = c.controller.useOpenApiDocument(newDoc)
 	if err != nil {
-		log.Log.Error(err, "Could not set custom OpenApi document")
+		log.Log.Error(err, "Could not set custom OpenAPI document")
 	}
 
 	// Sync CRD before kyverno starts
@@ -110,12 +107,17 @@ func (c *crdSync) sync() {
 }
 
 func (o *Controller) deleteCRDFromPreviousSync() {
-	for _, crd := range o.crdList {
+	for k := range o.kindToDefinitionName {
-		delete(o.kindToDefinitionName, crd)
+		delete(o.kindToDefinitionName, k)
-		delete(o.definitions, crd)
 	}
 
-	o.crdList = []string{}
+	o.kindToDefinitionName = make(map[string]string, 0)
+
+	for k := range o.definitions {
+		delete(o.definitions, k)
+	}
+
+	o.definitions = make(map[string]*openapi_v2.Schema, 0)
 }
 
 func (o *Controller) parseCRD(crd unstructured.Unstructured) {
@@ -164,8 +166,6 @@ func (o *Controller) parseCRD(crd unstructured.Unstructured) {
 		return
 	}
 
-	o.crdList = append(o.crdList, crdName)
-
 	o.kindToDefinitionName[crdName] = crdName
 	o.definitions[crdName] = parsedSchema
 }

pkg/openapi/validation.go

@@ -27,10 +27,8 @@ import (
 
 type Controller struct {
 	mutex                sync.RWMutex
-	document             *openapi_v2.Document
 	definitions          map[string]*openapi_v2.Schema
 	kindToDefinitionName map[string]string
-	crdList              []string
 	models               proto.Models
 }
 
@@ -82,7 +80,7 @@ func (o *Controller) ValidateResource(patchedResource unstructured.Unstructured,
 	schema := o.models.LookupModel(kind)
 	if schema == nil {
 		// Check if kind is a CRD
-		schema, err = o.getSchemaFromDefinitions(kind)
+		schema, err = o.getCRDSchema(kind)
 		if err != nil || schema == nil {
 			return fmt.Errorf("pre-validation: couldn't find model %s", kind)
 		}
@@ -144,22 +142,20 @@ func (o *Controller) ValidatePolicyMutation(policy v1.ClusterPolicy) error {
 	return nil
 }
 
-func (o *Controller) useOpenApiDocument(customDoc *openapi_v2.Document) error {
+func (o *Controller) useOpenApiDocument(doc *openapi_v2.Document) error {
 	o.mutex.Lock()
 	defer o.mutex.Unlock()
 
-	o.document = customDoc
-
 	o.definitions = make(map[string]*openapi_v2.Schema)
 	o.kindToDefinitionName = make(map[string]string)
-	for _, definition := range o.document.GetDefinitions().AdditionalProperties {
+	for _, definition := range doc.GetDefinitions().AdditionalProperties {
 		o.definitions[definition.GetName()] = definition.GetValue()
 		path := strings.Split(definition.GetName(), ".")
 		o.kindToDefinitionName[path[len(path)-1]] = definition.GetName()
 	}
 
 	var err error
-	o.models, err = proto.NewOpenAPIData(o.document)
+	o.models, err = proto.NewOpenAPIData(doc)
 	if err != nil {
 		return err
 	}
@@ -178,7 +174,7 @@ func getSchemaDocument() (*openapi_v2.Document, error) {
 }
 
 // For crd, we do not store definition in document
-func (o *Controller) getSchemaFromDefinitions(kind string) (proto.Schema, error) {
+func (o *Controller) getCRDSchema(kind string) (proto.Schema, error) {
 	if kind == "" {
 		return nil, errors.New("invalid kind")
 	}

pkg/policy/apply.go

@@ -17,6 +17,12 @@ import (
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 )
 
+const (
+	// JSON patch uses ~1 for / characters
+	// see: https://tools.ietf.org/html/rfc6901#section-3
+	PodTemplateAnnotationApplied = "pod-policies.kyverno.io~1autogen-applied"
+)
+
 // applyPolicy applies policy on a resource
 //TODO: generation rules
 func applyPolicy(policy kyverno.ClusterPolicy, resource unstructured.Unstructured, logger logr.Logger) (responses []response.EngineResponse) {
@@ -138,7 +144,8 @@ func dropKyvernoAnnotation(patches [][]byte, log logr.Logger) (resultPathes [][]
 		}
 
 		value := fmt.Sprintf("%v", data.Value)
-		if strings.Contains(value, engine.PodTemplateAnnotation) {
+		if strings.Contains(value, engine.PodTemplateAnnotation) ||
+			strings.Contains(value, PodTemplateAnnotationApplied) {
 			continue
 		}