522 supporting crd validation

pkg/dclient/utils.go

@@ -3,6 +3,8 @@ package client
 import (
 	"strings"
 
+	openapi_v2 "github.com/googleapis/gnostic/OpenAPIv2"
+
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
@@ -74,6 +76,10 @@ func (c *fakeDiscoveryClient) GetGVRFromKind(kind string) schema.GroupVersionRes
 	return c.getGVR(resource)
 }
 
+func (c *fakeDiscoveryClient) OpenAPISchema() (*openapi_v2.Document, error) {
+	return nil, nil
+}
+
 func newUnstructured(apiVersion, kind, namespace, name string) *unstructured.Unstructured {
 	return &unstructured.Unstructured{
 		Object: map[string]interface{}{

pkg/openapi/crdSync.go

@@ -1,14 +1,34 @@
 package openapi
 
 import (
+	"encoding/json"
 	"time"
 
 	"github.com/golang/glog"
 
+	"gopkg.in/yaml.v2"
+
+	"github.com/googleapis/gnostic/compiler"
+
+	openapi_v2 "github.com/googleapis/gnostic/OpenAPIv2"
+
 	client "github.com/nirmata/kyverno/pkg/dclient"
 	"k8s.io/apimachinery/pkg/util/wait"
 )
 
+type crdDefinition struct {
+	Spec struct {
+		Names struct {
+			Kind string `json:"kind"`
+		} `json:"names"`
+		Versions []struct {
+			Schema struct {
+				OpenAPIV3Schema interface{} `json:"openAPIV3Schema"`
+			} `json:"schema"`
+		} `json:"versions"`
+	} `json:"spec"`
+}
+
 type crdSync struct {
 	client *client.Client
 }
@@ -20,20 +40,54 @@ func NewCRDSync(client *client.Client) *crdSync {
 }
 
 func (c *crdSync) Run(workers int, stopCh <-chan struct{}) {
-	for i := 0; i < workers; i++ {
-		go wait.Until(c.syncCrd, time.Second*10, stopCh)
-	}
-	<-stopCh
-}
-
-func (c *crdSync) syncCrd() {
 	newDoc, err := c.client.DiscoveryClient.OpenAPISchema()
 	if err != nil {
 		glog.V(4).Infof("cannot get openapi schema: %v", err)
 	}
 
-	err = useCustomOpenApiDocument(newDoc)
+	err = useOpenApiDocument(newDoc)
 	if err != nil {
 		glog.V(4).Infof("Could not set custom OpenApi document: %v\n", err)
 	}
+
+	for i := 0; i < workers; i++ {
+		go wait.Until(c.sync, time.Second*10, stopCh)
+	}
+	<-stopCh
+}
+
+func (c *crdSync) sync() {
+	openApiGlobalState.mutex.Lock()
+	defer openApiGlobalState.mutex.Unlock()
+
+	crds, err := c.client.ListResource("CustomResourceDefinition", "", nil)
+	if err != nil {
+		glog.V(4).Infof("could not fetch crd's from server: %v", err)
+		return
+	}
+
+	for _, crd := range crds.Items {
+		var crdDefinition crdDefinition
+		crdRaw, _ := json.Marshal(crd.Object)
+		_ = json.Unmarshal(crdRaw, &crdDefinition)
+
+		crdName := crdDefinition.Spec.Names.Kind
+		if len(crdDefinition.Spec.Versions) < 1 {
+			glog.V(4).Infof("could not parse crd schema, no versions present")
+			continue
+		}
+
+		var schema yaml.MapSlice
+		schemaRaw, _ := json.Marshal(crdDefinition.Spec.Versions[0].Schema.OpenAPIV3Schema)
+		_ = yaml.Unmarshal(schemaRaw, &schema)
+
+		parsedSchema, err := openapi_v2.NewSchema(schema, compiler.NewContext("schema", nil))
+		if err != nil {
+			glog.V(4).Infof("could not parse crd schema:%v", err)
+			continue
+		}
+
+		openApiGlobalState.kindToDefinitionName[crdName] = crdName
+		openApiGlobalState.definitions[crdName] = parsedSchema
+	}
 }

pkg/openapi/validation.go

@@ -40,7 +40,7 @@ func init() {
 			panic(err)
 		}
 
-		err = useCustomOpenApiDocument(defaultDoc)
+		err = useOpenApiDocument(defaultDoc)
 		if err != nil {
 			panic(err)
 		}
@@ -75,15 +75,18 @@ func ValidatePolicyMutation(policy v1.ClusterPolicy) error {
 	for kind, rules := range kindToRules {
 		newPolicy := policy
 		newPolicy.Spec.Rules = rules
-
 		resource, _ := generateEmptyResource(openApiGlobalState.definitions[openApiGlobalState.kindToDefinitionName[kind]]).(map[string]interface{})
+		if resource == nil {
+			glog.V(4).Infof("Cannot Validate policy: openApi definition now found for %v", kind)
+			return nil
+		}
 		newResource := unstructured.Unstructured{Object: resource}
 		newResource.SetKind(kind)
 
 		ctx := context.NewContext()
 		err := ctx.AddSA("kyvernoDummyUsername")
 		if err != nil {
-			glog.Infof("Failed to load service account in context:%v", err)
+			glog.V(4).Infof("Failed to load service account in context:%v", err)
 		}
 
 		policyContext := engine.PolicyContext{
@@ -101,7 +104,7 @@ func ValidatePolicyMutation(policy v1.ClusterPolicy) error {
 			}
 			return fmt.Errorf(strings.Join(errMessages, "\n"))
 		}
-		err = ValidateResource(resp.PatchedResource.UnstructuredContent(), kind)
+		err = ValidateResource(*resp.PatchedResource.DeepCopy(), kind)
 		if err != nil {
 			return err
 		}
@@ -110,9 +113,16 @@ func ValidatePolicyMutation(policy v1.ClusterPolicy) error {
 	return nil
 }
 
-func ValidateResource(patchedResource interface{}, kind string) error {
+// For crd, we do not store definition in document
+func getSchemaFromDefinitions(kind string) (proto.Schema, error) {
+	path := proto.NewPath(kind)
+	return (&proto.Definitions{}).ParseSchema(openApiGlobalState.definitions[kind], &path)
+}
+
+func ValidateResource(patchedResource unstructured.Unstructured, kind string) error {
 	openApiGlobalState.mutex.RLock()
 	defer openApiGlobalState.mutex.RUnlock()
+	var err error
 
 	if !openApiGlobalState.isSet {
 		glog.V(4).Info("Cannot Validate resource: Validation global state not set")
@@ -122,10 +132,14 @@ func ValidateResource(patchedResource interface{}, kind string) error {
 	kind = openApiGlobalState.kindToDefinitionName[kind]
 	schema := openApiGlobalState.models.LookupModel(kind)
 	if schema == nil {
-		return fmt.Errorf("pre-validation: couldn't find model %s", kind)
+		schema, err = getSchemaFromDefinitions(kind)
+		if err != nil || schema == nil {
+			return fmt.Errorf("pre-validation: couldn't find model %s", kind)
+		}
+		delete(patchedResource.Object, "kind")
 	}
 
-	if errs := validation.ValidateModel(patchedResource, schema, kind); len(errs) > 0 {
+	if errs := validation.ValidateModel(patchedResource.UnstructuredContent(), schema, kind); len(errs) > 0 {
 		var errorMessages []string
 		for i := range errs {
 			errorMessages = append(errorMessages, errs[i].Error())
@@ -137,7 +151,7 @@ func ValidateResource(patchedResource interface{}, kind string) error {
 	return nil
 }
 
-func useCustomOpenApiDocument(customDoc *openapi_v2.Document) error {
+func useOpenApiDocument(customDoc *openapi_v2.Document) error {
 	openApiGlobalState.mutex.Lock()
 	defer openApiGlobalState.mutex.Unlock()
 
@@ -181,7 +195,11 @@ func generateEmptyResource(kindSchema *openapi_v2.Schema) interface{} {
 	}
 
 	if len(types) != 1 {
-		return nil
+		if len(kindSchema.GetProperties().GetAdditionalProperties()) > 0 {
+			types = []string{"object"}
+		} else {
+			return nil
+		}
 	}
 
 	switch types[0] {

pkg/openapi/validation_test.go

@@ -41,6 +41,10 @@ func Test_ValidateMutationPolicy(t *testing.T) {
 			policy:      []byte(`{"apiVersion":"kyverno.io/v1","kind":"ClusterPolicy","metadata":{"name":"mutate-pod-disable-automoutingapicred"},"spec":{"rules":[{"name":"pod-disable-automoutingapicred","match":{"resources":{"kinds":["Pod"]}},"mutate":{"overlay":{"spec":{"(serviceAccountName)":"*","automountServiceAccountToken":80}}}}]}}`),
 			errMessage:  `ValidationError(io.k8s.api.core.v1.Pod.spec.automountServiceAccountToken): invalid type for io.k8s.api.core.v1.PodSpec.automountServiceAccountToken: got "integer", expected "boolean"`,
 		},
+		{
+			description: "Testing Policies with substitute variables",
+			policy:      []byte(`{"apiVersion":"kyverno.io/v1","kind":"ClusterPolicy","metadata":{"name":"add-ns-access-controls","annotations":{"policies.kyverno.io/category":"Workload Isolation","policies.kyverno.io/description":"Create roles and role bindings for a new namespace"}},"spec":{"background":false,"rules":[{"name":"add-sa-annotation","match":{"resources":{"kinds":["Namespace"]}},"mutate":{"overlay":{"metadata":{"annotations":{"nirmata.io/ns-creator":"{{serviceAccountName}}"}}}}},{"name":"generate-owner-role","match":{"resources":{"kinds":["Namespace"]}},"preconditions":[{"key":"{{request.userInfo.username}}","operator":"NotEqual","value":""},{"key":"{{serviceAccountName}}","operator":"NotEqual","value":""},{"key":"{{serviceAccountNamespace}}","operator":"NotEqual","value":""}],"generate":{"kind":"ClusterRole","name":"ns-owner-{{request.object.metadata.name}}-{{request.userInfo.username}}","data":{"metadata":{"annotations":{"nirmata.io/ns-creator":"{{serviceAccountName}}"}},"rules":[{"apiGroups":[""],"resources":["namespaces"],"verbs":["delete"],"resourceNames":["{{request.object.metadata.name}}"]}]}}},{"name":"generate-owner-role-binding","match":{"resources":{"kinds":["Namespace"]}},"preconditions":[{"key":"{{request.userInfo.username}}","operator":"NotEqual","value":""},{"key":"{{serviceAccountName}}","operator":"NotEqual","value":""},{"key":"{{serviceAccountNamespace}}","operator":"NotEqual","value":""}],"generate":{"kind":"ClusterRoleBinding","name":"ns-owner-{{request.object.metadata.name}}-{{request.userInfo.username}}-binding","data":{"metadata":{"annotations":{"nirmata.io/ns-creator":"{{serviceAccountName}}"}},"roleRef":{"apiGroup":"rbac.authorization.k8s.io","kind":"ClusterRole","name":"ns-owner-{{request.object.metadata.name}}-{{request.userInfo.username}}"},"subjects":[{"kind":"ServiceAccount","name":"{{serviceAccountName}}","namespace":"{{serviceAccountNamespace}}"}]}}},{"name":"generate-admin-role-binding","match":{"resources":{"kinds":["Namespace"]}},"preconditions":[{"key":"{{request.userInfo.username}}","operator":"NotEqual","value":""},{"key":"{{serviceAccountName}}","operator":"NotEqual","value":""},{"key":"{{serviceAccountNamespace}}","operator":"NotEqual","value":""}],"generate":{"kind":"RoleBinding","name":"ns-admin-{{request.object.metadata.name}}-{{request.userInfo.username}}-binding","namespace":"{{request.object.metadata.name}}","data":{"metadata":{"annotations":{"nirmata.io/ns-creator":"{{serviceAccountName}}"}},"roleRef":{"apiGroup":"rbac.authorization.k8s.io","kind":"ClusterRole","name":"admin"},"subjects":[{"kind":"ServiceAccount","name":"{{serviceAccountName}}","namespace":"{{serviceAccountNamespace}}"}]}}}]}}`),
+		},
 	}
 
 	for i, tc := range tcs {

pkg/webhooks/mutation.go

@@ -103,7 +103,7 @@ func (ws *WebhookServer) HandleMutation(request *v1beta1.AdmissionRequest, resou
 			glog.V(4).Infof("Failed to apply policy %s on resource %s/%s\n", policy.Name, resource.GetNamespace(), resource.GetName())
 			continue
 		}
-		err := openapi.ValidateResource(engineResponse.PatchedResource.UnstructuredContent(), engineResponse.PatchedResource.GetKind())
+		err := openapi.ValidateResource(*engineResponse.PatchedResource.DeepCopy(), engineResponse.PatchedResource.GetKind())
 		if err != nil {
 			glog.V(4).Infoln(err)
 			continue