From a0f9ad1361d6c09e6e7b982709c57a048353b7b9 Mon Sep 17 00:00:00 2001 From: shravan Date: Wed, 4 Mar 2020 18:56:59 +0530 Subject: [PATCH 1/8] 522 save commit --- pkg/dclient/client.go | 7 + pkg/openapi/crdSync.go | 38 +++++ pkg/openapi/validation.go | 247 +++++++++++++++++++++++++++++++++ pkg/openapi/validation_test.go | 65 +++++++++ 4 files changed, 357 insertions(+) create mode 100644 pkg/openapi/crdSync.go create mode 100644 pkg/openapi/validation.go create mode 100644 pkg/openapi/validation_test.go diff --git a/pkg/dclient/client.go b/pkg/dclient/client.go index 9bb212369e..6cacaeaf9a 100644 --- a/pkg/dclient/client.go +++ b/pkg/dclient/client.go @@ -5,6 +5,8 @@ import ( "strings" "time" + openapi_v2 "github.com/googleapis/gnostic/OpenAPIv2" + "github.com/golang/glog" "github.com/nirmata/kyverno/pkg/config" apps "k8s.io/api/apps/v1" @@ -215,6 +217,7 @@ func convertToCSR(obj *unstructured.Unstructured) (*certificates.CertificateSign type IDiscovery interface { GetGVRFromKind(kind string) schema.GroupVersionResource GetServerVersion() (*version.Info, error) + OpenAPISchema() (*openapi_v2.Document, error) } // SetDiscovery sets the discovery client implementation @@ -246,6 +249,10 @@ func (c ServerPreferredResources) Poll(resync time.Duration, stopCh <-chan struc } } +func (c ServerPreferredResources) OpenAPISchema() (*openapi_v2.Document, error) { + return c.cachedClient.OpenAPISchema() +} + //GetGVRFromKind get the Group Version Resource from kind // if kind is not found in first attempt we invalidate the cache, // the retry will then fetch the new registered resources and check again diff --git a/pkg/openapi/crdSync.go b/pkg/openapi/crdSync.go new file mode 100644 index 0000000000..1ad171514e --- /dev/null +++ b/pkg/openapi/crdSync.go @@ -0,0 +1,38 @@ +package openapi + +import ( + "time" + + "github.com/golang/glog" + + client "github.com/nirmata/kyverno/pkg/dclient" + "k8s.io/apimachinery/pkg/util/wait" +) + +type crdSync struct { + client *client.Client +} + +func NewCRDSync(client *client.Client) *crdSync { + return &crdSync{ + client: client, + } +} + +func (c *crdSync) Run(workers int, stopCh <-chan struct{}) { + for i := 0; i < workers; i++ { + go wait.Until(c.syncCrd, time.Second*10, stopCh) + } +} + +func (c *crdSync) syncCrd() { + newDoc, err := c.client.DiscoveryClient.OpenAPISchema() + if err != nil { + glog.V(4).Infof("cannot get openapi schema: %v", err) + } + + err = useCustomOpenApiDocument(newDoc) + if err != nil { + glog.V(4).Infof("Could not set custom OpenApi document: %v\n", err) + } +} diff --git a/pkg/openapi/validation.go b/pkg/openapi/validation.go new file mode 100644 index 0000000000..701b056c61 --- /dev/null +++ b/pkg/openapi/validation.go @@ -0,0 +1,247 @@ +package openapi + +import ( + "fmt" + "strconv" + "strings" + "sync" + + "github.com/nirmata/kyverno/data" + + "github.com/golang/glog" + + "github.com/nirmata/kyverno/pkg/engine" + "github.com/nirmata/kyverno/pkg/engine/context" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + v1 "github.com/nirmata/kyverno/pkg/api/kyverno/v1" + + openapi_v2 "github.com/googleapis/gnostic/OpenAPIv2" + "github.com/googleapis/gnostic/compiler" + "k8s.io/kube-openapi/pkg/util/proto" + "k8s.io/kube-openapi/pkg/util/proto/validation" + + "gopkg.in/yaml.v2" +) + +var openApiGlobalState struct { + mutex sync.RWMutex + document *openapi_v2.Document + definitions map[string]*openapi_v2.Schema + kindToDefinitionName map[string]string + models proto.Models + isSet bool +} + +func init() { + if !openApiGlobalState.isSet { + defaultDoc, err := getSchemaDocument() + if err != nil { + panic(err) + } + + err = useCustomOpenApiDocument(defaultDoc) + if err != nil { + panic(err) + } + } +} + +func ValidatePolicyMutation(policy v1.ClusterPolicy) error { + openApiGlobalState.mutex.RLock() + defer openApiGlobalState.mutex.RUnlock() + + if !openApiGlobalState.isSet { + glog.V(4).Info("Cannot Validate policy: Validation global state not set") + return nil + } + + var kindToRules = make(map[string][]v1.Rule) + for _, rule := range policy.Spec.Rules { + if rule.HasMutate() { + rule.MatchResources = v1.MatchResources{ + UserInfo: v1.UserInfo{}, + ResourceDescription: v1.ResourceDescription{ + Kinds: rule.MatchResources.Kinds, + }, + } + rule.ExcludeResources = v1.ExcludeResources{} + for _, kind := range rule.MatchResources.Kinds { + kindToRules[kind] = append(kindToRules[kind], rule) + } + } + } + + for kind, rules := range kindToRules { + newPolicy := policy + newPolicy.Spec.Rules = rules + + resource, _ := generateEmptyResource(openApiGlobalState.definitions[openApiGlobalState.kindToDefinitionName[kind]]).(map[string]interface{}) + newResource := unstructured.Unstructured{Object: resource} + newResource.SetKind(kind) + policyContext := engine.PolicyContext{ + Policy: newPolicy, + NewResource: newResource, + Context: context.NewContext(), + } + resp := engine.Mutate(policyContext) + if len(resp.GetSuccessRules()) != len(rules) { + var errMessages []string + for _, rule := range resp.PolicyResponse.Rules { + if !rule.Success { + errMessages = append(errMessages, fmt.Sprintf("Invalid rule : %v, %v", rule.Name, rule.Message)) + } + } + return fmt.Errorf(strings.Join(errMessages, "\n")) + } + err := ValidateResource(resp.PatchedResource.UnstructuredContent(), kind) + if err != nil { + return err + } + } + + return nil +} + +func ValidateResource(patchedResource interface{}, kind string) error { + openApiGlobalState.mutex.RLock() + defer openApiGlobalState.mutex.RUnlock() + + if !openApiGlobalState.isSet { + glog.V(4).Info("Cannot Validate resource: Validation global state not set") + return nil + } + + kind = openApiGlobalState.kindToDefinitionName[kind] + schema := openApiGlobalState.models.LookupModel(kind) + if schema == nil { + return fmt.Errorf("pre-validation: couldn't find model %s", kind) + } + + if errs := validation.ValidateModel(patchedResource, schema, kind); len(errs) > 0 { + var errorMessages []string + for i := range errs { + errorMessages = append(errorMessages, errs[i].Error()) + } + + return fmt.Errorf(strings.Join(errorMessages, "\n\n")) + } + + return nil +} + +func useCustomOpenApiDocument(customDoc *openapi_v2.Document) error { + openApiGlobalState.mutex.Lock() + defer openApiGlobalState.mutex.Unlock() + + openApiGlobalState.document = customDoc + + openApiGlobalState.definitions = make(map[string]*openapi_v2.Schema) + openApiGlobalState.kindToDefinitionName = make(map[string]string) + for _, definition := range openApiGlobalState.document.GetDefinitions().AdditionalProperties { + openApiGlobalState.definitions[definition.GetName()] = definition.GetValue() + path := strings.Split(definition.GetName(), ".") + openApiGlobalState.kindToDefinitionName[path[len(path)-1]] = definition.GetName() + } + + var err error + openApiGlobalState.models, err = proto.NewOpenAPIData(openApiGlobalState.document) + if err != nil { + return err + } + + openApiGlobalState.isSet = true + + return nil +} + +func getSchemaDocument() (*openapi_v2.Document, error) { + var spec yaml.MapSlice + err := yaml.Unmarshal([]byte(data.SwaggerDoc), &spec) + if err != nil { + return nil, err + } + + return openapi_v2.NewDocument(spec, compiler.NewContext("$root", nil)) +} + +func generateEmptyResource(kindSchema *openapi_v2.Schema) interface{} { + + types := kindSchema.GetType().GetValue() + + if kindSchema.GetXRef() != "" { + return generateEmptyResource(openApiGlobalState.definitions[strings.TrimPrefix(kindSchema.GetXRef(), "#/definitions/")]) + } + + if len(types) != 1 { + return nil + } + + switch types[0] { + case "object": + var props = make(map[string]interface{}) + properties := kindSchema.GetProperties().GetAdditionalProperties() + if len(properties) == 0 { + return props + } + + var wg sync.WaitGroup + var mutex sync.Mutex + wg.Add(len(properties)) + for _, property := range properties { + go func(property *openapi_v2.NamedSchema) { + prop := generateEmptyResource(property.GetValue()) + mutex.Lock() + props[property.GetName()] = prop + mutex.Unlock() + wg.Done() + }(property) + } + wg.Wait() + return props + case "array": + var array []interface{} + for _, schema := range kindSchema.GetItems().GetSchema() { + array = append(array, generateEmptyResource(schema)) + } + return array + case "string": + if kindSchema.GetDefault() != nil { + return string(kindSchema.GetDefault().Value.Value) + } + if kindSchema.GetExample() != nil { + return string(kindSchema.GetExample().GetValue().Value) + } + return "" + case "integer": + if kindSchema.GetDefault() != nil { + val, _ := strconv.Atoi(string(kindSchema.GetDefault().Value.Value)) + return val + } + if kindSchema.GetExample() != nil { + val, _ := strconv.Atoi(string(kindSchema.GetExample().GetValue().Value)) + return val + } + return 0 + case "number": + if kindSchema.GetDefault() != nil { + val, _ := strconv.Atoi(string(kindSchema.GetDefault().Value.Value)) + return val + } + if kindSchema.GetExample() != nil { + val, _ := strconv.Atoi(string(kindSchema.GetExample().GetValue().Value)) + return val + } + return 0 + case "boolean": + if kindSchema.GetDefault() != nil { + return string(kindSchema.GetDefault().Value.Value) == "true" + } + if kindSchema.GetExample() != nil { + return string(kindSchema.GetExample().GetValue().Value) == "true" + } + return false + } + + return nil +} diff --git a/pkg/openapi/validation_test.go b/pkg/openapi/validation_test.go new file mode 100644 index 0000000000..4c6c8ac2a6 --- /dev/null +++ b/pkg/openapi/validation_test.go @@ -0,0 +1,65 @@ +package openapi + +import ( + "encoding/json" + "testing" + + v1 "github.com/nirmata/kyverno/pkg/api/kyverno/v1" +) + +func Test_ValidateMutationPolicy(t *testing.T) { + err := setValidationGlobalState() + if err != nil { + t.Fatalf("Could not set global state") + } + + tcs := []struct { + description string + policy []byte + errMessage string + }{ + { + description: "Policy with mutating imagePullPolicy Overlay", + policy: []byte(`{"apiVersion":"kyverno.io/v1","kind":"ClusterPolicy","metadata":{"name":"set-image-pull-policy-2"},"spec":{"rules":[{"name":"set-image-pull-policy-2","match":{"resources":{"kinds":["Pod"]}},"mutate":{"overlay":{"spec":{"containers":[{"(image)":"*","imagePullPolicy":"Always"}]}}}}]}}`), + }, + { + description: "Policy with mutating imagePullPolicy Overlay, field does not exist", + policy: []byte(`{"apiVersion":"kyverno.io/v1","kind":"ClusterPolicy","metadata":{"name":"set-image-pull-policy-2"},"spec":{"rules":[{"name":"set-image-pull-policy-2","match":{"resources":{"kinds":["Pod"]}},"mutate":{"overlay":{"spec":{"containers":[{"(image)":"*","nonExistantField":"Always"}]}}}}]}}`), + errMessage: `ValidationError(io.k8s.api.core.v1.Pod.spec.containers[0]): unknown field "nonExistantField" in io.k8s.api.core.v1.Container`, + }, + { + description: "Policy with mutating imagePullPolicy Overlay, type of value is different (does not throw error since all numbers are also strings according to swagger)", + policy: []byte(`{"apiVersion":"kyverno.io/v1","kind":"ClusterPolicy","metadata":{"name":"set-image-pull-policy-2"},"spec":{"rules":[{"name":"set-image-pull-policy-2","match":{"resources":{"kinds":["Pod"]}},"mutate":{"overlay":{"spec":{"containers":[{"(image)":"*","imagePullPolicy":80}]}}}}]}}`), + }, + { + description: "Policy with patches", + policy: []byte(`{"apiVersion":"kyverno.io/v1","kind":"ClusterPolicy","metadata":{"name":"policy-endpoints"},"spec":{"rules":[{"name":"pEP","match":{"resources":{"kinds":["Endpoints"],"selector":{"matchLabels":{"label":"test"}}}},"mutate":{"patches":[{"path":"/subsets/0/ports/0/port","op":"replace","value":9663},{"path":"/metadata/labels/isMutated","op":"add","value":"true"}]}}]}}`), + }, + { + description: "Policy with patches, value converted from number to string", + policy: []byte(`{"apiVersion":"kyverno.io/v1","kind":"ClusterPolicy","metadata":{"name":"policy-endpoints"},"spec":{"rules":[{"name":"pEP","match":{"resources":{"kinds":["Endpoints"],"selector":{"matchLabels":{"label":"test"}}}},"mutate":{"patches":[{"path":"/subsets/0/ports/0/port","op":"replace","value":"9663"},{"path":"/metadata/labels/isMutated","op":"add","value":"true"}]}}]}}`), + errMessage: `ValidationError(io.k8s.api.core.v1.Endpoints.subsets[0].ports[0].port): invalid type for io.k8s.api.core.v1.EndpointPort.port: got "string", expected "integer"`, + }, + { + description: "Policy where boolean is been converted to number", + policy: []byte(`{"apiVersion":"kyverno.io/v1","kind":"ClusterPolicy","metadata":{"name":"mutate-pod-disable-automoutingapicred"},"spec":{"rules":[{"name":"pod-disable-automoutingapicred","match":{"resources":{"kinds":["Pod"]}},"mutate":{"overlay":{"spec":{"(serviceAccountName)":"*","automountServiceAccountToken":80}}}}]}}`), + errMessage: `ValidationError(io.k8s.api.core.v1.Pod.spec.automountServiceAccountToken): invalid type for io.k8s.api.core.v1.PodSpec.automountServiceAccountToken: got "integer", expected "boolean"`, + }, + } + + for i, tc := range tcs { + policy := v1.ClusterPolicy{} + _ = json.Unmarshal(tc.policy, &policy) + + var errMessage string + err := ValidatePolicyMutation(policy) + if err != nil { + errMessage = err.Error() + } + + if errMessage != tc.errMessage { + t.Errorf("\nTestcase [%v] failed:\nExpected Error: %v\nGot Error: %v", i+1, tc.errMessage, errMessage) + } + } + +} From 888d2ae171151dad497ccbe538fa3f7cb43b9a34 Mon Sep 17 00:00:00 2001 From: shravan Date: Wed, 4 Mar 2020 19:16:26 +0530 Subject: [PATCH 2/8] 522 save commit --- cmd/kyverno/main.go | 6 ++++++ pkg/openapi/crdSync.go | 1 + pkg/policy/validate.go | 6 ++++++ 3 files changed, 13 insertions(+) diff --git a/cmd/kyverno/main.go b/cmd/kyverno/main.go index 62cefceeb9..3fcc02a047 100644 --- a/cmd/kyverno/main.go +++ b/cmd/kyverno/main.go @@ -5,6 +5,8 @@ import ( "flag" "time" + "github.com/nirmata/kyverno/pkg/openapi" + "github.com/golang/glog" "github.com/nirmata/kyverno/pkg/checker" kyvernoclient "github.com/nirmata/kyverno/pkg/client/clientset/versioned" @@ -200,6 +202,9 @@ func main() { glog.Fatalf("Failed registering Admission Webhooks: %v\n", err) } + // Sync openAPI definitions of resources + openApiSync := openapi.NewCRDSync(client) + // WEBHOOOK // - https server to provide endpoints called based on rules defined in Mutating & Validation webhook configuration // - reports the results based on the response from the policy engine: @@ -238,6 +243,7 @@ func main() { go grc.Run(1, stopCh) go grcc.Run(1, stopCh) go pvgen.Run(1, stopCh) + go openApiSync.Run(1, stopCh) // verifys if the admission control is enabled and active // resync: 60 seconds diff --git a/pkg/openapi/crdSync.go b/pkg/openapi/crdSync.go index 1ad171514e..f32c845b23 100644 --- a/pkg/openapi/crdSync.go +++ b/pkg/openapi/crdSync.go @@ -23,6 +23,7 @@ func (c *crdSync) Run(workers int, stopCh <-chan struct{}) { for i := 0; i < workers; i++ { go wait.Until(c.syncCrd, time.Second*10, stopCh) } + <-stopCh } func (c *crdSync) syncCrd() { diff --git a/pkg/policy/validate.go b/pkg/policy/validate.go index bc44d7524f..4789687afa 100644 --- a/pkg/policy/validate.go +++ b/pkg/policy/validate.go @@ -8,6 +8,8 @@ import ( "strconv" "strings" + "github.com/nirmata/kyverno/pkg/openapi" + kyverno "github.com/nirmata/kyverno/pkg/api/kyverno/v1" "github.com/nirmata/kyverno/pkg/engine/anchor" rbacv1 "k8s.io/api/rbac/v1" @@ -80,6 +82,10 @@ func Validate(p kyverno.ClusterPolicy) error { } } + if err := openapi.ValidatePolicyMutation(p); err != nil { + return err + } + return nil } From 044d55600a6eefa7903d9ed203283177c0fd0984 Mon Sep 17 00:00:00 2001 From: shravan Date: Wed, 4 Mar 2020 19:27:08 +0530 Subject: [PATCH 3/8] 522 fixed tests and added validation of mutated resources --- pkg/openapi/validation.go | 12 ++++++------ pkg/openapi/validation_test.go | 4 ---- pkg/webhooks/mutation.go | 7 +++++++ 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/pkg/openapi/validation.go b/pkg/openapi/validation.go index 701b056c61..e11c33bbcc 100644 --- a/pkg/openapi/validation.go +++ b/pkg/openapi/validation.go @@ -216,23 +216,23 @@ func generateEmptyResource(kindSchema *openapi_v2.Schema) interface{} { case "integer": if kindSchema.GetDefault() != nil { val, _ := strconv.Atoi(string(kindSchema.GetDefault().Value.Value)) - return val + return int64(val) } if kindSchema.GetExample() != nil { val, _ := strconv.Atoi(string(kindSchema.GetExample().GetValue().Value)) - return val + return int64(val) } - return 0 + return int64(0) case "number": if kindSchema.GetDefault() != nil { val, _ := strconv.Atoi(string(kindSchema.GetDefault().Value.Value)) - return val + return int64(val) } if kindSchema.GetExample() != nil { val, _ := strconv.Atoi(string(kindSchema.GetExample().GetValue().Value)) - return val + return int64(val) } - return 0 + return int64(0) case "boolean": if kindSchema.GetDefault() != nil { return string(kindSchema.GetDefault().Value.Value) == "true" diff --git a/pkg/openapi/validation_test.go b/pkg/openapi/validation_test.go index 4c6c8ac2a6..6723c13b0c 100644 --- a/pkg/openapi/validation_test.go +++ b/pkg/openapi/validation_test.go @@ -8,10 +8,6 @@ import ( ) func Test_ValidateMutationPolicy(t *testing.T) { - err := setValidationGlobalState() - if err != nil { - t.Fatalf("Could not set global state") - } tcs := []struct { description string diff --git a/pkg/webhooks/mutation.go b/pkg/webhooks/mutation.go index 650fce8374..d6df3fcb34 100644 --- a/pkg/webhooks/mutation.go +++ b/pkg/webhooks/mutation.go @@ -3,6 +3,8 @@ package webhooks import ( "time" + "github.com/nirmata/kyverno/pkg/openapi" + "github.com/golang/glog" kyverno "github.com/nirmata/kyverno/pkg/api/kyverno/v1" "github.com/nirmata/kyverno/pkg/engine" @@ -101,6 +103,11 @@ func (ws *WebhookServer) HandleMutation(request *v1beta1.AdmissionRequest, resou glog.V(4).Infof("Failed to apply policy %s on resource %s/%s\n", policy.Name, resource.GetNamespace(), resource.GetName()) continue } + err := openapi.ValidateResource(engineResponse.PatchedResource.UnstructuredContent(), engineResponse.PatchedResource.GetKind()) + if err != nil { + glog.V(4).Infoln(err) + continue + } // gather patches patches = append(patches, engineResponse.GetPatches()...) glog.V(4).Infof("Mutation from policy %s has applied successfully to %s %s/%s", policy.Name, request.Kind.Kind, resource.GetNamespace(), resource.GetName()) From b27a62b6bf1afe98b232eb724b4fd11c1aabf86f Mon Sep 17 00:00:00 2001 From: shravan Date: Wed, 4 Mar 2020 19:38:33 +0530 Subject: [PATCH 4/8] 522 added service account name in context --- pkg/openapi/validation.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/pkg/openapi/validation.go b/pkg/openapi/validation.go index e11c33bbcc..221aa13631 100644 --- a/pkg/openapi/validation.go +++ b/pkg/openapi/validation.go @@ -79,10 +79,17 @@ func ValidatePolicyMutation(policy v1.ClusterPolicy) error { resource, _ := generateEmptyResource(openApiGlobalState.definitions[openApiGlobalState.kindToDefinitionName[kind]]).(map[string]interface{}) newResource := unstructured.Unstructured{Object: resource} newResource.SetKind(kind) + + ctx := context.NewContext() + err := ctx.AddSA("kyvernoDummyUsername") + if err != nil { + glog.Infof("Failed to load service account in context:%v", err) + } + policyContext := engine.PolicyContext{ Policy: newPolicy, NewResource: newResource, - Context: context.NewContext(), + Context: ctx, } resp := engine.Mutate(policyContext) if len(resp.GetSuccessRules()) != len(rules) { @@ -94,7 +101,7 @@ func ValidatePolicyMutation(policy v1.ClusterPolicy) error { } return fmt.Errorf(strings.Join(errMessages, "\n")) } - err := ValidateResource(resp.PatchedResource.UnstructuredContent(), kind) + err = ValidateResource(resp.PatchedResource.UnstructuredContent(), kind) if err != nil { return err } From 7aa1e1515b91d76778183e5992c93398bb11941f Mon Sep 17 00:00:00 2001 From: shravan Date: Thu, 5 Mar 2020 22:50:32 +0530 Subject: [PATCH 5/8] 522 supporting crd validation --- pkg/dclient/utils.go | 6 +++ pkg/openapi/crdSync.go | 70 ++++++++++++++++++++++++++++++---- pkg/openapi/validation.go | 36 ++++++++++++----- pkg/openapi/validation_test.go | 4 ++ pkg/webhooks/mutation.go | 2 +- 5 files changed, 100 insertions(+), 18 deletions(-) diff --git a/pkg/dclient/utils.go b/pkg/dclient/utils.go index a8a0275b95..8b170eb094 100644 --- a/pkg/dclient/utils.go +++ b/pkg/dclient/utils.go @@ -3,6 +3,8 @@ package client import ( "strings" + openapi_v2 "github.com/googleapis/gnostic/OpenAPIv2" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" @@ -74,6 +76,10 @@ func (c *fakeDiscoveryClient) GetGVRFromKind(kind string) schema.GroupVersionRes return c.getGVR(resource) } +func (c *fakeDiscoveryClient) OpenAPISchema() (*openapi_v2.Document, error) { + return nil, nil +} + func newUnstructured(apiVersion, kind, namespace, name string) *unstructured.Unstructured { return &unstructured.Unstructured{ Object: map[string]interface{}{ diff --git a/pkg/openapi/crdSync.go b/pkg/openapi/crdSync.go index f32c845b23..955a8abb28 100644 --- a/pkg/openapi/crdSync.go +++ b/pkg/openapi/crdSync.go @@ -1,14 +1,34 @@ package openapi import ( + "encoding/json" "time" "github.com/golang/glog" + "gopkg.in/yaml.v2" + + "github.com/googleapis/gnostic/compiler" + + openapi_v2 "github.com/googleapis/gnostic/OpenAPIv2" + client "github.com/nirmata/kyverno/pkg/dclient" "k8s.io/apimachinery/pkg/util/wait" ) +type crdDefinition struct { + Spec struct { + Names struct { + Kind string `json:"kind"` + } `json:"names"` + Versions []struct { + Schema struct { + OpenAPIV3Schema interface{} `json:"openAPIV3Schema"` + } `json:"schema"` + } `json:"versions"` + } `json:"spec"` +} + type crdSync struct { client *client.Client } @@ -20,20 +40,54 @@ func NewCRDSync(client *client.Client) *crdSync { } func (c *crdSync) Run(workers int, stopCh <-chan struct{}) { - for i := 0; i < workers; i++ { - go wait.Until(c.syncCrd, time.Second*10, stopCh) - } - <-stopCh -} - -func (c *crdSync) syncCrd() { newDoc, err := c.client.DiscoveryClient.OpenAPISchema() if err != nil { glog.V(4).Infof("cannot get openapi schema: %v", err) } - err = useCustomOpenApiDocument(newDoc) + err = useOpenApiDocument(newDoc) if err != nil { glog.V(4).Infof("Could not set custom OpenApi document: %v\n", err) } + + for i := 0; i < workers; i++ { + go wait.Until(c.sync, time.Second*10, stopCh) + } + <-stopCh +} + +func (c *crdSync) sync() { + openApiGlobalState.mutex.Lock() + defer openApiGlobalState.mutex.Unlock() + + crds, err := c.client.ListResource("CustomResourceDefinition", "", nil) + if err != nil { + glog.V(4).Infof("could not fetch crd's from server: %v", err) + return + } + + for _, crd := range crds.Items { + var crdDefinition crdDefinition + crdRaw, _ := json.Marshal(crd.Object) + _ = json.Unmarshal(crdRaw, &crdDefinition) + + crdName := crdDefinition.Spec.Names.Kind + if len(crdDefinition.Spec.Versions) < 1 { + glog.V(4).Infof("could not parse crd schema, no versions present") + continue + } + + var schema yaml.MapSlice + schemaRaw, _ := json.Marshal(crdDefinition.Spec.Versions[0].Schema.OpenAPIV3Schema) + _ = yaml.Unmarshal(schemaRaw, &schema) + + parsedSchema, err := openapi_v2.NewSchema(schema, compiler.NewContext("schema", nil)) + if err != nil { + glog.V(4).Infof("could not parse crd schema:%v", err) + continue + } + + openApiGlobalState.kindToDefinitionName[crdName] = crdName + openApiGlobalState.definitions[crdName] = parsedSchema + } } diff --git a/pkg/openapi/validation.go b/pkg/openapi/validation.go index 221aa13631..38b58e92e0 100644 --- a/pkg/openapi/validation.go +++ b/pkg/openapi/validation.go @@ -40,7 +40,7 @@ func init() { panic(err) } - err = useCustomOpenApiDocument(defaultDoc) + err = useOpenApiDocument(defaultDoc) if err != nil { panic(err) } @@ -75,15 +75,18 @@ func ValidatePolicyMutation(policy v1.ClusterPolicy) error { for kind, rules := range kindToRules { newPolicy := policy newPolicy.Spec.Rules = rules - resource, _ := generateEmptyResource(openApiGlobalState.definitions[openApiGlobalState.kindToDefinitionName[kind]]).(map[string]interface{}) + if resource == nil { + glog.V(4).Infof("Cannot Validate policy: openApi definition now found for %v", kind) + return nil + } newResource := unstructured.Unstructured{Object: resource} newResource.SetKind(kind) ctx := context.NewContext() err := ctx.AddSA("kyvernoDummyUsername") if err != nil { - glog.Infof("Failed to load service account in context:%v", err) + glog.V(4).Infof("Failed to load service account in context:%v", err) } policyContext := engine.PolicyContext{ @@ -101,7 +104,7 @@ func ValidatePolicyMutation(policy v1.ClusterPolicy) error { } return fmt.Errorf(strings.Join(errMessages, "\n")) } - err = ValidateResource(resp.PatchedResource.UnstructuredContent(), kind) + err = ValidateResource(*resp.PatchedResource.DeepCopy(), kind) if err != nil { return err } @@ -110,9 +113,16 @@ func ValidatePolicyMutation(policy v1.ClusterPolicy) error { return nil } -func ValidateResource(patchedResource interface{}, kind string) error { +// For crd, we do not store definition in document +func getSchemaFromDefinitions(kind string) (proto.Schema, error) { + path := proto.NewPath(kind) + return (&proto.Definitions{}).ParseSchema(openApiGlobalState.definitions[kind], &path) +} + +func ValidateResource(patchedResource unstructured.Unstructured, kind string) error { openApiGlobalState.mutex.RLock() defer openApiGlobalState.mutex.RUnlock() + var err error if !openApiGlobalState.isSet { glog.V(4).Info("Cannot Validate resource: Validation global state not set") @@ -122,10 +132,14 @@ func ValidateResource(patchedResource interface{}, kind string) error { kind = openApiGlobalState.kindToDefinitionName[kind] schema := openApiGlobalState.models.LookupModel(kind) if schema == nil { - return fmt.Errorf("pre-validation: couldn't find model %s", kind) + schema, err = getSchemaFromDefinitions(kind) + if err != nil || schema == nil { + return fmt.Errorf("pre-validation: couldn't find model %s", kind) + } + delete(patchedResource.Object, "kind") } - if errs := validation.ValidateModel(patchedResource, schema, kind); len(errs) > 0 { + if errs := validation.ValidateModel(patchedResource.UnstructuredContent(), schema, kind); len(errs) > 0 { var errorMessages []string for i := range errs { errorMessages = append(errorMessages, errs[i].Error()) @@ -137,7 +151,7 @@ func ValidateResource(patchedResource interface{}, kind string) error { return nil } -func useCustomOpenApiDocument(customDoc *openapi_v2.Document) error { +func useOpenApiDocument(customDoc *openapi_v2.Document) error { openApiGlobalState.mutex.Lock() defer openApiGlobalState.mutex.Unlock() @@ -181,7 +195,11 @@ func generateEmptyResource(kindSchema *openapi_v2.Schema) interface{} { } if len(types) != 1 { - return nil + if len(kindSchema.GetProperties().GetAdditionalProperties()) > 0 { + types = []string{"object"} + } else { + return nil + } } switch types[0] { diff --git a/pkg/openapi/validation_test.go b/pkg/openapi/validation_test.go index 6723c13b0c..e534478908 100644 --- a/pkg/openapi/validation_test.go +++ b/pkg/openapi/validation_test.go @@ -41,6 +41,10 @@ func Test_ValidateMutationPolicy(t *testing.T) { policy: []byte(`{"apiVersion":"kyverno.io/v1","kind":"ClusterPolicy","metadata":{"name":"mutate-pod-disable-automoutingapicred"},"spec":{"rules":[{"name":"pod-disable-automoutingapicred","match":{"resources":{"kinds":["Pod"]}},"mutate":{"overlay":{"spec":{"(serviceAccountName)":"*","automountServiceAccountToken":80}}}}]}}`), errMessage: `ValidationError(io.k8s.api.core.v1.Pod.spec.automountServiceAccountToken): invalid type for io.k8s.api.core.v1.PodSpec.automountServiceAccountToken: got "integer", expected "boolean"`, }, + { + description: "Testing Policies with substitute variables", + policy: []byte(`{"apiVersion":"kyverno.io/v1","kind":"ClusterPolicy","metadata":{"name":"add-ns-access-controls","annotations":{"policies.kyverno.io/category":"Workload Isolation","policies.kyverno.io/description":"Create roles and role bindings for a new namespace"}},"spec":{"background":false,"rules":[{"name":"add-sa-annotation","match":{"resources":{"kinds":["Namespace"]}},"mutate":{"overlay":{"metadata":{"annotations":{"nirmata.io/ns-creator":"{{serviceAccountName}}"}}}}},{"name":"generate-owner-role","match":{"resources":{"kinds":["Namespace"]}},"preconditions":[{"key":"{{request.userInfo.username}}","operator":"NotEqual","value":""},{"key":"{{serviceAccountName}}","operator":"NotEqual","value":""},{"key":"{{serviceAccountNamespace}}","operator":"NotEqual","value":""}],"generate":{"kind":"ClusterRole","name":"ns-owner-{{request.object.metadata.name}}-{{request.userInfo.username}}","data":{"metadata":{"annotations":{"nirmata.io/ns-creator":"{{serviceAccountName}}"}},"rules":[{"apiGroups":[""],"resources":["namespaces"],"verbs":["delete"],"resourceNames":["{{request.object.metadata.name}}"]}]}}},{"name":"generate-owner-role-binding","match":{"resources":{"kinds":["Namespace"]}},"preconditions":[{"key":"{{request.userInfo.username}}","operator":"NotEqual","value":""},{"key":"{{serviceAccountName}}","operator":"NotEqual","value":""},{"key":"{{serviceAccountNamespace}}","operator":"NotEqual","value":""}],"generate":{"kind":"ClusterRoleBinding","name":"ns-owner-{{request.object.metadata.name}}-{{request.userInfo.username}}-binding","data":{"metadata":{"annotations":{"nirmata.io/ns-creator":"{{serviceAccountName}}"}},"roleRef":{"apiGroup":"rbac.authorization.k8s.io","kind":"ClusterRole","name":"ns-owner-{{request.object.metadata.name}}-{{request.userInfo.username}}"},"subjects":[{"kind":"ServiceAccount","name":"{{serviceAccountName}}","namespace":"{{serviceAccountNamespace}}"}]}}},{"name":"generate-admin-role-binding","match":{"resources":{"kinds":["Namespace"]}},"preconditions":[{"key":"{{request.userInfo.username}}","operator":"NotEqual","value":""},{"key":"{{serviceAccountName}}","operator":"NotEqual","value":""},{"key":"{{serviceAccountNamespace}}","operator":"NotEqual","value":""}],"generate":{"kind":"RoleBinding","name":"ns-admin-{{request.object.metadata.name}}-{{request.userInfo.username}}-binding","namespace":"{{request.object.metadata.name}}","data":{"metadata":{"annotations":{"nirmata.io/ns-creator":"{{serviceAccountName}}"}},"roleRef":{"apiGroup":"rbac.authorization.k8s.io","kind":"ClusterRole","name":"admin"},"subjects":[{"kind":"ServiceAccount","name":"{{serviceAccountName}}","namespace":"{{serviceAccountNamespace}}"}]}}}]}}`), + }, } for i, tc := range tcs { diff --git a/pkg/webhooks/mutation.go b/pkg/webhooks/mutation.go index d6df3fcb34..459e3742ed 100644 --- a/pkg/webhooks/mutation.go +++ b/pkg/webhooks/mutation.go @@ -103,7 +103,7 @@ func (ws *WebhookServer) HandleMutation(request *v1beta1.AdmissionRequest, resou glog.V(4).Infof("Failed to apply policy %s on resource %s/%s\n", policy.Name, resource.GetNamespace(), resource.GetName()) continue } - err := openapi.ValidateResource(engineResponse.PatchedResource.UnstructuredContent(), engineResponse.PatchedResource.GetKind()) + err := openapi.ValidateResource(*engineResponse.PatchedResource.DeepCopy(), engineResponse.PatchedResource.GetKind()) if err != nil { glog.V(4).Infoln(err) continue From 4db0cf7a877ae31b247824ef441f928eef9b1743 Mon Sep 17 00:00:00 2001 From: shravan Date: Fri, 6 Mar 2020 01:09:38 +0530 Subject: [PATCH 6/8] 522 adding force mutate function --- pkg/engine/mutate/overlay.go | 6 ++-- pkg/engine/mutation.go | 70 ++++++++++++++++++++++++++++++++++++ pkg/openapi/crdSync.go | 63 ++++++++++++++++++++------------ pkg/openapi/validation.go | 63 ++++++++++---------------------- 4 files changed, 133 insertions(+), 69 deletions(-) diff --git a/pkg/engine/mutate/overlay.go b/pkg/engine/mutate/overlay.go index 21e5e7b77e..3b76f30c74 100644 --- a/pkg/engine/mutate/overlay.go +++ b/pkg/engine/mutate/overlay.go @@ -116,7 +116,7 @@ func processOverlayPatches(resource, overlay interface{}) ([][]byte, overlayErro } } - patchBytes, err := mutateResourceWithOverlay(resource, overlay) + patchBytes, err := MutateResourceWithOverlay(resource, overlay) if err != nil { return patchBytes, newOverlayError(overlayFailure, err.Error()) } @@ -124,8 +124,8 @@ func processOverlayPatches(resource, overlay interface{}) ([][]byte, overlayErro return patchBytes, overlayError{} } -// mutateResourceWithOverlay is a start of overlaying process -func mutateResourceWithOverlay(resource, pattern interface{}) ([][]byte, error) { +// MutateResourceWithOverlay is a start of overlaying process +func MutateResourceWithOverlay(resource, pattern interface{}) ([][]byte, error) { // It assumes that mutation is started from root, so "/" is passed return applyOverlay(resource, pattern, "/") } diff --git a/pkg/engine/mutation.go b/pkg/engine/mutation.go index d1fab46211..98a63eb268 100644 --- a/pkg/engine/mutation.go +++ b/pkg/engine/mutation.go @@ -1,10 +1,15 @@ package engine import ( + "fmt" "reflect" "strings" "time" + "github.com/nirmata/kyverno/pkg/engine/utils" + + "github.com/nirmata/kyverno/pkg/engine/context" + "github.com/golang/glog" kyverno "github.com/nirmata/kyverno/pkg/api/kyverno/v1" "github.com/nirmata/kyverno/pkg/engine/mutate" @@ -122,6 +127,71 @@ func Mutate(policyContext PolicyContext) (resp response.EngineResponse) { resp.PatchedResource = patchedResource return resp } + +func mutateResourceWithOverlay(resource unstructured.Unstructured, overlay interface{}) (unstructured.Unstructured, error) { + patches, err := mutate.MutateResourceWithOverlay(resource.UnstructuredContent(), overlay) + if err != nil { + return unstructured.Unstructured{}, err + } + if len(patches) == 0 { + return resource, nil + } + + // convert to RAW + resourceRaw, err := resource.MarshalJSON() + if err != nil { + return unstructured.Unstructured{}, err + } + + var patchResource []byte + patchResource, err = utils.ApplyPatches(resourceRaw, patches) + if err != nil { + return unstructured.Unstructured{}, err + } + + resource = unstructured.Unstructured{} + err = resource.UnmarshalJSON(patchResource) + if err != nil { + return unstructured.Unstructured{}, err + } + + return resource, nil +} + +// ForceMutate does not check any conditions, it simply mutates the given resource +func ForceMutate(ctx context.EvalInterface, policy kyverno.ClusterPolicy, resource unstructured.Unstructured) (unstructured.Unstructured, error) { + var err error + for _, rule := range policy.Spec.Rules { + if !rule.HasMutate() { + continue + } + + mutation := rule.Mutation.DeepCopy() + + if mutation.Overlay != nil { + overlay := mutation.Overlay + if overlay, err = variables.SubstituteVars(ctx, overlay); err != nil { + return unstructured.Unstructured{}, err + } + + resource, err = mutateResourceWithOverlay(resource, overlay) + if err != nil { + return unstructured.Unstructured{}, fmt.Errorf("could not mutate resource with overlay on rule %v:%v", rule.Name, err) + } + } + + if rule.Mutation.Patches != nil { + var resp response.RuleResponse + resp, resource = mutate.ProcessPatches(rule, resource) + if !resp.Success { + return unstructured.Unstructured{}, fmt.Errorf(resp.Message) + } + } + } + + return resource, nil +} + func incrementAppliedRuleCount(resp *response.EngineResponse) { resp.PolicyResponse.RulesAppliedCount++ } diff --git a/pkg/openapi/crdSync.go b/pkg/openapi/crdSync.go index 955a8abb28..da428c2955 100644 --- a/pkg/openapi/crdSync.go +++ b/pkg/openapi/crdSync.go @@ -4,6 +4,8 @@ import ( "encoding/json" "time" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "github.com/golang/glog" "gopkg.in/yaml.v2" @@ -66,28 +68,45 @@ func (c *crdSync) sync() { return } + deleteCRDFromPreviousSync() + for _, crd := range crds.Items { - var crdDefinition crdDefinition - crdRaw, _ := json.Marshal(crd.Object) - _ = json.Unmarshal(crdRaw, &crdDefinition) - - crdName := crdDefinition.Spec.Names.Kind - if len(crdDefinition.Spec.Versions) < 1 { - glog.V(4).Infof("could not parse crd schema, no versions present") - continue - } - - var schema yaml.MapSlice - schemaRaw, _ := json.Marshal(crdDefinition.Spec.Versions[0].Schema.OpenAPIV3Schema) - _ = yaml.Unmarshal(schemaRaw, &schema) - - parsedSchema, err := openapi_v2.NewSchema(schema, compiler.NewContext("schema", nil)) - if err != nil { - glog.V(4).Infof("could not parse crd schema:%v", err) - continue - } - - openApiGlobalState.kindToDefinitionName[crdName] = crdName - openApiGlobalState.definitions[crdName] = parsedSchema + parseCRD(crd) } } + +func deleteCRDFromPreviousSync() { + for _, crd := range openApiGlobalState.crdList { + delete(openApiGlobalState.kindToDefinitionName, crd) + delete(openApiGlobalState.definitions, crd) + } + + openApiGlobalState.crdList = []string{} +} + +func parseCRD(crd unstructured.Unstructured) { + var crdDefinition crdDefinition + crdRaw, _ := json.Marshal(crd.Object) + _ = json.Unmarshal(crdRaw, &crdDefinition) + + crdName := crdDefinition.Spec.Names.Kind + if len(crdDefinition.Spec.Versions) < 1 { + glog.V(4).Infof("could not parse crd schema, no versions present") + return + } + + var schema yaml.MapSlice + schemaRaw, _ := json.Marshal(crdDefinition.Spec.Versions[0].Schema.OpenAPIV3Schema) + _ = yaml.Unmarshal(schemaRaw, &schema) + + parsedSchema, err := openapi_v2.NewSchema(schema, compiler.NewContext("schema", nil)) + if err != nil { + glog.V(4).Infof("could not parse crd schema:%v", err) + return + } + + openApiGlobalState.crdList = append(openApiGlobalState.crdList, crdName) + + openApiGlobalState.kindToDefinitionName[crdName] = crdName + openApiGlobalState.definitions[crdName] = parsedSchema +} diff --git a/pkg/openapi/validation.go b/pkg/openapi/validation.go index 38b58e92e0..c8fc9084b5 100644 --- a/pkg/openapi/validation.go +++ b/pkg/openapi/validation.go @@ -29,21 +29,19 @@ var openApiGlobalState struct { document *openapi_v2.Document definitions map[string]*openapi_v2.Schema kindToDefinitionName map[string]string + crdList []string models proto.Models - isSet bool } func init() { - if !openApiGlobalState.isSet { - defaultDoc, err := getSchemaDocument() - if err != nil { - panic(err) - } + defaultDoc, err := getSchemaDocument() + if err != nil { + panic(err) + } - err = useOpenApiDocument(defaultDoc) - if err != nil { - panic(err) - } + err = useOpenApiDocument(defaultDoc) + if err != nil { + panic(err) } } @@ -51,11 +49,6 @@ func ValidatePolicyMutation(policy v1.ClusterPolicy) error { openApiGlobalState.mutex.RLock() defer openApiGlobalState.mutex.RUnlock() - if !openApiGlobalState.isSet { - glog.V(4).Info("Cannot Validate policy: Validation global state not set") - return nil - } - var kindToRules = make(map[string][]v1.Rule) for _, rule := range policy.Spec.Rules { if rule.HasMutate() { @@ -73,7 +66,7 @@ func ValidatePolicyMutation(policy v1.ClusterPolicy) error { } for kind, rules := range kindToRules { - newPolicy := policy + newPolicy := *policy.DeepCopy() newPolicy.Spec.Rules = rules resource, _ := generateEmptyResource(openApiGlobalState.definitions[openApiGlobalState.kindToDefinitionName[kind]]).(map[string]interface{}) if resource == nil { @@ -89,22 +82,11 @@ func ValidatePolicyMutation(policy v1.ClusterPolicy) error { glog.V(4).Infof("Failed to load service account in context:%v", err) } - policyContext := engine.PolicyContext{ - Policy: newPolicy, - NewResource: newResource, - Context: ctx, + patchedResource, err := engine.ForceMutate(ctx, *newPolicy.DeepCopy(), newResource) + if err != nil { + return err } - resp := engine.Mutate(policyContext) - if len(resp.GetSuccessRules()) != len(rules) { - var errMessages []string - for _, rule := range resp.PolicyResponse.Rules { - if !rule.Success { - errMessages = append(errMessages, fmt.Sprintf("Invalid rule : %v, %v", rule.Name, rule.Message)) - } - } - return fmt.Errorf(strings.Join(errMessages, "\n")) - } - err = ValidateResource(*resp.PatchedResource.DeepCopy(), kind) + err = ValidateResource(*patchedResource.DeepCopy(), kind) if err != nil { return err } @@ -113,22 +95,11 @@ func ValidatePolicyMutation(policy v1.ClusterPolicy) error { return nil } -// For crd, we do not store definition in document -func getSchemaFromDefinitions(kind string) (proto.Schema, error) { - path := proto.NewPath(kind) - return (&proto.Definitions{}).ParseSchema(openApiGlobalState.definitions[kind], &path) -} - func ValidateResource(patchedResource unstructured.Unstructured, kind string) error { openApiGlobalState.mutex.RLock() defer openApiGlobalState.mutex.RUnlock() var err error - if !openApiGlobalState.isSet { - glog.V(4).Info("Cannot Validate resource: Validation global state not set") - return nil - } - kind = openApiGlobalState.kindToDefinitionName[kind] schema := openApiGlobalState.models.LookupModel(kind) if schema == nil { @@ -171,8 +142,6 @@ func useOpenApiDocument(customDoc *openapi_v2.Document) error { return err } - openApiGlobalState.isSet = true - return nil } @@ -186,6 +155,12 @@ func getSchemaDocument() (*openapi_v2.Document, error) { return openapi_v2.NewDocument(spec, compiler.NewContext("$root", nil)) } +// For crd, we do not store definition in document +func getSchemaFromDefinitions(kind string) (proto.Schema, error) { + path := proto.NewPath(kind) + return (&proto.Definitions{}).ParseSchema(openApiGlobalState.definitions[kind], &path) +} + func generateEmptyResource(kindSchema *openapi_v2.Schema) interface{} { types := kindSchema.GetType().GetValue() From 4831aa86aee5189efe4ce83b7d8e08a6e4050720 Mon Sep 17 00:00:00 2001 From: shravan Date: Fri, 6 Mar 2020 01:52:03 +0530 Subject: [PATCH 7/8] 522 dealing with substitute variables --- pkg/engine/mutation.go | 34 ++++++++++++++++++++++++++++++++-- pkg/openapi/validation.go | 9 +-------- pkg/openapi/validation_test.go | 4 ++-- 3 files changed, 35 insertions(+), 12 deletions(-) diff --git a/pkg/engine/mutation.go b/pkg/engine/mutation.go index 98a63eb268..625b0b2371 100644 --- a/pkg/engine/mutation.go +++ b/pkg/engine/mutation.go @@ -1,8 +1,10 @@ package engine import ( + "encoding/json" "fmt" "reflect" + "regexp" "strings" "time" @@ -170,8 +172,12 @@ func ForceMutate(ctx context.EvalInterface, policy kyverno.ClusterPolicy, resour if mutation.Overlay != nil { overlay := mutation.Overlay - if overlay, err = variables.SubstituteVars(ctx, overlay); err != nil { - return unstructured.Unstructured{}, err + if ctx != nil { + if overlay, err = variables.SubstituteVars(ctx, overlay); err != nil { + return unstructured.Unstructured{}, err + } + } else { + overlay = replaceSubstituteVariables(overlay) } resource, err = mutateResourceWithOverlay(resource, overlay) @@ -192,6 +198,30 @@ func ForceMutate(ctx context.EvalInterface, policy kyverno.ClusterPolicy, resour return resource, nil } +func replaceSubstituteVariables(overlay interface{}) interface{} { + overlayRaw, err := json.Marshal(overlay) + if err != nil { + return overlay + } + + regex := regexp.MustCompile(`\{\{([^{}]*)\}\}`) + for { + if len(regex.FindAllStringSubmatch(string(overlayRaw), -1)) > 0 { + overlayRaw = regex.ReplaceAll(overlayRaw, []byte(`placeholderValue`)) + } else { + break + } + } + + var output interface{} + err = json.Unmarshal(overlayRaw, &output) + if err != nil { + return overlay + } + + return output +} + func incrementAppliedRuleCount(resp *response.EngineResponse) { resp.PolicyResponse.RulesAppliedCount++ } diff --git a/pkg/openapi/validation.go b/pkg/openapi/validation.go index c8fc9084b5..5f89d8499a 100644 --- a/pkg/openapi/validation.go +++ b/pkg/openapi/validation.go @@ -11,7 +11,6 @@ import ( "github.com/golang/glog" "github.com/nirmata/kyverno/pkg/engine" - "github.com/nirmata/kyverno/pkg/engine/context" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" v1 "github.com/nirmata/kyverno/pkg/api/kyverno/v1" @@ -76,13 +75,7 @@ func ValidatePolicyMutation(policy v1.ClusterPolicy) error { newResource := unstructured.Unstructured{Object: resource} newResource.SetKind(kind) - ctx := context.NewContext() - err := ctx.AddSA("kyvernoDummyUsername") - if err != nil { - glog.V(4).Infof("Failed to load service account in context:%v", err) - } - - patchedResource, err := engine.ForceMutate(ctx, *newPolicy.DeepCopy(), newResource) + patchedResource, err := engine.ForceMutate(nil, *newPolicy.DeepCopy(), newResource) if err != nil { return err } diff --git a/pkg/openapi/validation_test.go b/pkg/openapi/validation_test.go index e534478908..813c2ed15b 100644 --- a/pkg/openapi/validation_test.go +++ b/pkg/openapi/validation_test.go @@ -42,8 +42,8 @@ func Test_ValidateMutationPolicy(t *testing.T) { errMessage: `ValidationError(io.k8s.api.core.v1.Pod.spec.automountServiceAccountToken): invalid type for io.k8s.api.core.v1.PodSpec.automountServiceAccountToken: got "integer", expected "boolean"`, }, { - description: "Testing Policies with substitute variables", - policy: []byte(`{"apiVersion":"kyverno.io/v1","kind":"ClusterPolicy","metadata":{"name":"add-ns-access-controls","annotations":{"policies.kyverno.io/category":"Workload Isolation","policies.kyverno.io/description":"Create roles and role bindings for a new namespace"}},"spec":{"background":false,"rules":[{"name":"add-sa-annotation","match":{"resources":{"kinds":["Namespace"]}},"mutate":{"overlay":{"metadata":{"annotations":{"nirmata.io/ns-creator":"{{serviceAccountName}}"}}}}},{"name":"generate-owner-role","match":{"resources":{"kinds":["Namespace"]}},"preconditions":[{"key":"{{request.userInfo.username}}","operator":"NotEqual","value":""},{"key":"{{serviceAccountName}}","operator":"NotEqual","value":""},{"key":"{{serviceAccountNamespace}}","operator":"NotEqual","value":""}],"generate":{"kind":"ClusterRole","name":"ns-owner-{{request.object.metadata.name}}-{{request.userInfo.username}}","data":{"metadata":{"annotations":{"nirmata.io/ns-creator":"{{serviceAccountName}}"}},"rules":[{"apiGroups":[""],"resources":["namespaces"],"verbs":["delete"],"resourceNames":["{{request.object.metadata.name}}"]}]}}},{"name":"generate-owner-role-binding","match":{"resources":{"kinds":["Namespace"]}},"preconditions":[{"key":"{{request.userInfo.username}}","operator":"NotEqual","value":""},{"key":"{{serviceAccountName}}","operator":"NotEqual","value":""},{"key":"{{serviceAccountNamespace}}","operator":"NotEqual","value":""}],"generate":{"kind":"ClusterRoleBinding","name":"ns-owner-{{request.object.metadata.name}}-{{request.userInfo.username}}-binding","data":{"metadata":{"annotations":{"nirmata.io/ns-creator":"{{serviceAccountName}}"}},"roleRef":{"apiGroup":"rbac.authorization.k8s.io","kind":"ClusterRole","name":"ns-owner-{{request.object.metadata.name}}-{{request.userInfo.username}}"},"subjects":[{"kind":"ServiceAccount","name":"{{serviceAccountName}}","namespace":"{{serviceAccountNamespace}}"}]}}},{"name":"generate-admin-role-binding","match":{"resources":{"kinds":["Namespace"]}},"preconditions":[{"key":"{{request.userInfo.username}}","operator":"NotEqual","value":""},{"key":"{{serviceAccountName}}","operator":"NotEqual","value":""},{"key":"{{serviceAccountNamespace}}","operator":"NotEqual","value":""}],"generate":{"kind":"RoleBinding","name":"ns-admin-{{request.object.metadata.name}}-{{request.userInfo.username}}-binding","namespace":"{{request.object.metadata.name}}","data":{"metadata":{"annotations":{"nirmata.io/ns-creator":"{{serviceAccountName}}"}},"roleRef":{"apiGroup":"rbac.authorization.k8s.io","kind":"ClusterRole","name":"admin"},"subjects":[{"kind":"ServiceAccount","name":"{{serviceAccountName}}","namespace":"{{serviceAccountNamespace}}"}]}}}]}}`), + description: "Dealing with nested variables", + policy: []byte(`{"apiVersion":"kyverno.io/v1","kind":"ClusterPolicy","metadata":{"name":"add-ns-access-controls","annotations":{"policies.kyverno.io/category":"Workload Isolation","policies.kyverno.io/description":"Create roles and role bindings for a new namespace"}},"spec":{"background":false,"rules":[{"name":"add-sa-annotation","match":{"resources":{"kinds":["Namespace"]}},"mutate":{"overlay":{"metadata":{"annotations":{"nirmata.io/ns-creator":"{{serviceAccountName-{{something}}}}"}}}}},{"name":"generate-owner-role","match":{"resources":{"kinds":["Namespace"]}},"preconditions":[{"key":"{{request.userInfo.username}}","operator":"NotEqual","value":""},{"key":"{{serviceAccountName}}","operator":"NotEqual","value":""},{"key":"{{serviceAccountNamespace}}","operator":"NotEqual","value":""}],"generate":{"kind":"ClusterRole","name":"ns-owner-{{request.object.metadata.name{{something}}}}-{{request.userInfo.username}}","data":{"metadata":{"annotations":{"nirmata.io/ns-creator":"{{serviceAccountName}}"}},"rules":[{"apiGroups":[""],"resources":["namespaces"],"verbs":["delete"],"resourceNames":["{{request.object.metadata.name}}"]}]}}},{"name":"generate-owner-role-binding","match":{"resources":{"kinds":["Namespace"]}},"preconditions":[{"key":"{{request.userInfo.username}}","operator":"NotEqual","value":""},{"key":"{{serviceAccountName}}","operator":"NotEqual","value":""},{"key":"{{serviceAccountNamespace}}","operator":"NotEqual","value":""}],"generate":{"kind":"ClusterRoleBinding","name":"ns-owner-{{request.object.metadata.name}}-{{request.userInfo.username}}-binding","data":{"metadata":{"annotations":{"nirmata.io/ns-creator":"{{serviceAccountName}}"}},"roleRef":{"apiGroup":"rbac.authorization.k8s.io","kind":"ClusterRole","name":"ns-owner-{{request.object.metadata.name}}-{{request.userInfo.username}}"},"subjects":[{"kind":"ServiceAccount","name":"{{serviceAccountName}}","namespace":"{{serviceAccountNamespace}}"}]}}},{"name":"generate-admin-role-binding","match":{"resources":{"kinds":["Namespace"]}},"preconditions":[{"key":"{{request.userInfo.username}}","operator":"NotEqual","value":""},{"key":"{{serviceAccountName}}","operator":"NotEqual","value":""},{"key":"{{serviceAccountNamespace}}","operator":"NotEqual","value":""}],"generate":{"kind":"RoleBinding","name":"ns-admin-{{request.object.metadata.name}}-{{request.userInfo.username}}-binding","namespace":"{{request.object.metadata.name}}","data":{"metadata":{"annotations":{"nirmata.io/ns-creator":"{{serviceAccountName}}"}},"roleRef":{"apiGroup":"rbac.authorization.k8s.io","kind":"ClusterRole","name":"admin"},"subjects":[{"kind":"ServiceAccount","name":"{{serviceAccountName}}","namespace":"{{serviceAccountNamespace}}"}]}}}]}}`), }, } From a0baf182f231d8fa563bad5ddbfffaa4e796bcbe Mon Sep 17 00:00:00 2001 From: shravan Date: Fri, 6 Mar 2020 01:54:36 +0530 Subject: [PATCH 8/8] 522 moving force mutate to a seperate file --- pkg/engine/forceMutate.go | 107 ++++++++++++++++++++++++++++++++++++++ pkg/engine/mutation.go | 100 ----------------------------------- 2 files changed, 107 insertions(+), 100 deletions(-) create mode 100644 pkg/engine/forceMutate.go diff --git a/pkg/engine/forceMutate.go b/pkg/engine/forceMutate.go new file mode 100644 index 0000000000..14ba42c468 --- /dev/null +++ b/pkg/engine/forceMutate.go @@ -0,0 +1,107 @@ +package engine + +import ( + "encoding/json" + "fmt" + "regexp" + + kyverno "github.com/nirmata/kyverno/pkg/api/kyverno/v1" + "github.com/nirmata/kyverno/pkg/engine/context" + "github.com/nirmata/kyverno/pkg/engine/mutate" + "github.com/nirmata/kyverno/pkg/engine/response" + "github.com/nirmata/kyverno/pkg/engine/utils" + "github.com/nirmata/kyverno/pkg/engine/variables" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +func mutateResourceWithOverlay(resource unstructured.Unstructured, overlay interface{}) (unstructured.Unstructured, error) { + patches, err := mutate.MutateResourceWithOverlay(resource.UnstructuredContent(), overlay) + if err != nil { + return unstructured.Unstructured{}, err + } + if len(patches) == 0 { + return resource, nil + } + + // convert to RAW + resourceRaw, err := resource.MarshalJSON() + if err != nil { + return unstructured.Unstructured{}, err + } + + var patchResource []byte + patchResource, err = utils.ApplyPatches(resourceRaw, patches) + if err != nil { + return unstructured.Unstructured{}, err + } + + resource = unstructured.Unstructured{} + err = resource.UnmarshalJSON(patchResource) + if err != nil { + return unstructured.Unstructured{}, err + } + + return resource, nil +} + +// ForceMutate does not check any conditions, it simply mutates the given resource +func ForceMutate(ctx context.EvalInterface, policy kyverno.ClusterPolicy, resource unstructured.Unstructured) (unstructured.Unstructured, error) { + var err error + for _, rule := range policy.Spec.Rules { + if !rule.HasMutate() { + continue + } + + mutation := rule.Mutation.DeepCopy() + + if mutation.Overlay != nil { + overlay := mutation.Overlay + if ctx != nil { + if overlay, err = variables.SubstituteVars(ctx, overlay); err != nil { + return unstructured.Unstructured{}, err + } + } else { + overlay = replaceSubstituteVariables(overlay) + } + + resource, err = mutateResourceWithOverlay(resource, overlay) + if err != nil { + return unstructured.Unstructured{}, fmt.Errorf("could not mutate resource with overlay on rule %v:%v", rule.Name, err) + } + } + + if rule.Mutation.Patches != nil { + var resp response.RuleResponse + resp, resource = mutate.ProcessPatches(rule, resource) + if !resp.Success { + return unstructured.Unstructured{}, fmt.Errorf(resp.Message) + } + } + } + + return resource, nil +} + +func replaceSubstituteVariables(overlay interface{}) interface{} { + overlayRaw, err := json.Marshal(overlay) + if err != nil { + return overlay + } + + regex := regexp.MustCompile(`\{\{([^{}]*)\}\}`) + for { + if len(regex.FindAllStringSubmatch(string(overlayRaw), -1)) > 0 { + overlayRaw = regex.ReplaceAll(overlayRaw, []byte(`placeholderValue`)) + } else { + break + } + } + + var output interface{} + err = json.Unmarshal(overlayRaw, &output) + if err != nil { + return overlay + } + + return output +} diff --git a/pkg/engine/mutation.go b/pkg/engine/mutation.go index 625b0b2371..d1fab46211 100644 --- a/pkg/engine/mutation.go +++ b/pkg/engine/mutation.go @@ -1,17 +1,10 @@ package engine import ( - "encoding/json" - "fmt" "reflect" - "regexp" "strings" "time" - "github.com/nirmata/kyverno/pkg/engine/utils" - - "github.com/nirmata/kyverno/pkg/engine/context" - "github.com/golang/glog" kyverno "github.com/nirmata/kyverno/pkg/api/kyverno/v1" "github.com/nirmata/kyverno/pkg/engine/mutate" @@ -129,99 +122,6 @@ func Mutate(policyContext PolicyContext) (resp response.EngineResponse) { resp.PatchedResource = patchedResource return resp } - -func mutateResourceWithOverlay(resource unstructured.Unstructured, overlay interface{}) (unstructured.Unstructured, error) { - patches, err := mutate.MutateResourceWithOverlay(resource.UnstructuredContent(), overlay) - if err != nil { - return unstructured.Unstructured{}, err - } - if len(patches) == 0 { - return resource, nil - } - - // convert to RAW - resourceRaw, err := resource.MarshalJSON() - if err != nil { - return unstructured.Unstructured{}, err - } - - var patchResource []byte - patchResource, err = utils.ApplyPatches(resourceRaw, patches) - if err != nil { - return unstructured.Unstructured{}, err - } - - resource = unstructured.Unstructured{} - err = resource.UnmarshalJSON(patchResource) - if err != nil { - return unstructured.Unstructured{}, err - } - - return resource, nil -} - -// ForceMutate does not check any conditions, it simply mutates the given resource -func ForceMutate(ctx context.EvalInterface, policy kyverno.ClusterPolicy, resource unstructured.Unstructured) (unstructured.Unstructured, error) { - var err error - for _, rule := range policy.Spec.Rules { - if !rule.HasMutate() { - continue - } - - mutation := rule.Mutation.DeepCopy() - - if mutation.Overlay != nil { - overlay := mutation.Overlay - if ctx != nil { - if overlay, err = variables.SubstituteVars(ctx, overlay); err != nil { - return unstructured.Unstructured{}, err - } - } else { - overlay = replaceSubstituteVariables(overlay) - } - - resource, err = mutateResourceWithOverlay(resource, overlay) - if err != nil { - return unstructured.Unstructured{}, fmt.Errorf("could not mutate resource with overlay on rule %v:%v", rule.Name, err) - } - } - - if rule.Mutation.Patches != nil { - var resp response.RuleResponse - resp, resource = mutate.ProcessPatches(rule, resource) - if !resp.Success { - return unstructured.Unstructured{}, fmt.Errorf(resp.Message) - } - } - } - - return resource, nil -} - -func replaceSubstituteVariables(overlay interface{}) interface{} { - overlayRaw, err := json.Marshal(overlay) - if err != nil { - return overlay - } - - regex := regexp.MustCompile(`\{\{([^{}]*)\}\}`) - for { - if len(regex.FindAllStringSubmatch(string(overlayRaw), -1)) > 0 { - overlayRaw = regex.ReplaceAll(overlayRaw, []byte(`placeholderValue`)) - } else { - break - } - } - - var output interface{} - err = json.Unmarshal(overlayRaw, &output) - if err != nil { - return overlay - } - - return output -} - func incrementAppliedRuleCount(resp *response.EngineResponse) { resp.PolicyResponse.RulesAppliedCount++ }