From 302090cd86e0e90c99101dff1ed068616f354677 Mon Sep 17 00:00:00 2001 From: Maxim Goncharenko Date: Tue, 21 May 2019 18:27:56 +0300 Subject: [PATCH 1/7] Implemented base for Mutation Overlay --- pkg/engine/mutation.go | 13 +-- pkg/engine/mutation/overlay.go | 6 - pkg/engine/overlay.go | 133 ++++++++++++++++++++++ pkg/engine/overlay_test.go | 11 ++ pkg/engine/{mutation => }/patches.go | 2 +- pkg/engine/{mutation => }/patches_test.go | 2 +- pkg/engine/utils.go | 12 ++ pkg/engine/validation.go | 21 +--- pkg/engine/validation_test.go | 6 +- pkg/webhooks/server.go | 5 +- 10 files changed, 171 insertions(+), 40 deletions(-) delete mode 100644 pkg/engine/mutation/overlay.go create mode 100644 pkg/engine/overlay.go create mode 100644 pkg/engine/overlay_test.go rename pkg/engine/{mutation => }/patches.go (99%) rename pkg/engine/{mutation => }/patches_test.go (99%) diff --git a/pkg/engine/mutation.go b/pkg/engine/mutation.go index 1474c4364f..02bdfdeb9a 100644 --- a/pkg/engine/mutation.go +++ b/pkg/engine/mutation.go @@ -4,15 +4,14 @@ import ( "log" kubepolicy "github.com/nirmata/kube-policy/pkg/apis/policy/v1alpha1" - "github.com/nirmata/kube-policy/pkg/engine/mutation" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // Mutate performs mutation. Overlay first and then mutation patches // TODO: return events and violations -func Mutate(policy kubepolicy.Policy, rawResource []byte, gvk metav1.GroupVersionKind) ([]mutation.PatchBytes, []byte) { - var policyPatches []mutation.PatchBytes - var processedPatches []mutation.PatchBytes +func Mutate(policy kubepolicy.Policy, rawResource []byte, gvk metav1.GroupVersionKind) ([]PatchBytes, []byte) { + var policyPatches []PatchBytes + var processedPatches []PatchBytes var err error patchedDocument := rawResource @@ -30,18 +29,18 @@ func Mutate(policy kubepolicy.Policy, rawResource []byte, gvk metav1.GroupVersio // Process Overlay if rule.Mutation.Overlay != nil { - overlayPatches, err := mutation.ProcessOverlay(rule.Mutation.Overlay, rawResource) + //overlayPatches, err := ProcessOverlay(rule.Mutation.Overlay, rawResource) if err != nil { log.Printf("Overlay application has failed for rule %s in policy %s, err: %v\n", rule.Name, policy.ObjectMeta.Name, err) } else { - policyPatches = append(policyPatches, overlayPatches...) + //policyPatches = append(policyPatches, overlayPatches...) } } // Process Patches if rule.Mutation.Patches != nil { - processedPatches, patchedDocument, err = mutation.ProcessPatches(rule.Mutation.Patches, patchedDocument) + processedPatches, patchedDocument, err = ProcessPatches(rule.Mutation.Patches, patchedDocument) if err != nil { log.Printf("Patches application has failed for rule %s in policy %s, err: %v\n", rule.Name, policy.ObjectMeta.Name, err) } else { diff --git a/pkg/engine/mutation/overlay.go b/pkg/engine/mutation/overlay.go deleted file mode 100644 index 308596e1bc..0000000000 --- a/pkg/engine/mutation/overlay.go +++ /dev/null @@ -1,6 +0,0 @@ -package mutation - -func ProcessOverlay(overlay interface{}, rawResource []byte) ([]PatchBytes, error) { - // TODO: Overlay to be implemented - return nil, nil -} diff --git a/pkg/engine/overlay.go b/pkg/engine/overlay.go new file mode 100644 index 0000000000..4bac976dc3 --- /dev/null +++ b/pkg/engine/overlay.go @@ -0,0 +1,133 @@ +package engine + +import ( + "encoding/json" + "fmt" + "log" + "strconv" + + kubepolicy "github.com/nirmata/kube-policy/pkg/apis/policy/v1alpha1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// ProcessOverlay handles validating admission request +// Checks the target resourse for rules defined in the policy +func ProcessOverlay(policy kubepolicy.Policy, rawResource []byte, gvk metav1.GroupVersionKind) ([]PatchBytes, []byte) { + var resource interface{} + json.Unmarshal(rawResource, &resource) + + for _, rule := range policy.Spec.Rules { + if rule.Mutation == nil || rule.Mutation.Overlay == nil { + continue + } + + ok := ResourceMeetsDescription(rawResource, rule.ResourceDescription, gvk) + if !ok { + log.Printf("Rule \"%s\" is not applicable to resource\n", rule.Name) + continue + } + + overlay := *rule.Mutation.Overlay + if err, _ := applyOverlay(resource, overlay, ""); err != nil { + //return fmt.Errorf("%s: %s", *rule.Validation.Message, err.Error()) + } + } + + return nil, nil +} + +func applyOverlay(resource, overlay interface{}, path string) ([]PatchBytes, error) { + switch typedOverlay := overlay.(type) { + case map[string]interface{}: + typedResource := resource.(map[string]interface{}) + + for key, value := range typedOverlay { + path += "/" + key + resourcePart, ok := typedResource[key] + + if ok { + applyOverlay(resourcePart, value, path) + } else { + createSubtree(value, path) + } + } + case []interface{}: + typedResource := resource.([]interface{}) + applyOverlayToArray(typedResource, typedOverlay, path) + case string: + typedResource, ok := resource.(string) + if !ok { + return nil, fmt.Errorf("Expected string, found %T", resource) + } + replaceResource(typedResource, typedOverlay, path) + case float64: + typedResource, ok := resource.(float64) + if !ok { + return nil, fmt.Errorf("Expected string, found %T", resource) + } + replaceResource(typedResource, typedOverlay, path) + case int64: + typedResource, ok := resource.(int64) + if !ok { + return nil, fmt.Errorf("Expected string, found %T", resource) + } + replaceResource(typedResource, typedOverlay, path) + } + + return nil, nil +} + +func applyOverlayToArray(resource, overlay []interface{}, path string) { + switch overlay[0].(type) { + case map[string]interface{}: + for _, overlayElement := range overlay { + typedOverlay := overlayElement.(map[string]interface{}) + + anchors := GetAnchorsFromMap(typedOverlay) + + if len(anchors) > 0 { + // Try to replace + for i, resourceElement := range resource { + path += "/" + strconv.Itoa(i) + typedResource := resourceElement.(map[string]interface{}) + if !skipArrayObject(typedResource, anchors) { + replaceResource(typedResource, typedOverlay, path) + } + } + } else { + // Add new item to the front + path += "/0" + createSubtree(typedOverlay, path) + } + } + default: + path += "/0" + for _, value := range overlay { + createSubtree(value, path) + } + } +} + +func skipArrayObject(object, anchors map[string]interface{}) bool { + for key, pattern := range anchors { + key = key[1 : len(key)-1] + + value, ok := object[key] + if !ok { + return true + } + + return value != pattern + } + + return false +} + +func replaceResource(resource, overlay interface{}, path string) { + +} + +func createSubtree(overlayPart interface{}, path string) []PatchBytes { + + return nil +} diff --git a/pkg/engine/overlay_test.go b/pkg/engine/overlay_test.go new file mode 100644 index 0000000000..a35f183ae6 --- /dev/null +++ b/pkg/engine/overlay_test.go @@ -0,0 +1,11 @@ +package engine + +import ( + "testing" + + "gotest.tools/assert" +) + +func TestApplyOverlay_BaseCase(t *testing.T) { + assert.Assert(t, true) +} diff --git a/pkg/engine/mutation/patches.go b/pkg/engine/patches.go similarity index 99% rename from pkg/engine/mutation/patches.go rename to pkg/engine/patches.go index 068125c4fa..0d56a08432 100644 --- a/pkg/engine/mutation/patches.go +++ b/pkg/engine/patches.go @@ -1,4 +1,4 @@ -package mutation +package engine import ( "encoding/json" diff --git a/pkg/engine/mutation/patches_test.go b/pkg/engine/patches_test.go similarity index 99% rename from pkg/engine/mutation/patches_test.go rename to pkg/engine/patches_test.go index e4d26e5a1a..ea27ec77c3 100644 --- a/pkg/engine/mutation/patches_test.go +++ b/pkg/engine/patches_test.go @@ -1,4 +1,4 @@ -package mutation +package engine import ( "testing" diff --git a/pkg/engine/utils.go b/pkg/engine/utils.go index 0646bf33e3..40fb634b2b 100644 --- a/pkg/engine/utils.go +++ b/pkg/engine/utils.go @@ -104,3 +104,15 @@ func ParseRegexPolicyResourceName(policyResourceName string) (string, bool) { } return strings.Trim(regex[1], " "), true } + +func GetAnchorsFromMap(anchorsMap map[string]interface{}) map[string]interface{} { + result := make(map[string]interface{}) + + for key, value := range anchorsMap { + if wrappedWithParentheses(key) { + result[key] = value + } + } + + return result +} diff --git a/pkg/engine/validation.go b/pkg/engine/validation.go index c3f4d788cd..63e7f97e25 100644 --- a/pkg/engine/validation.go +++ b/pkg/engine/validation.go @@ -96,10 +96,7 @@ func validateArray(resourcePart, patternPart interface{}) error { switch pattern := patternArray[0].(type) { case map[string]interface{}: - anchors, err := getAnchorsFromMap(pattern) - if err != nil { - return err - } + anchors := GetAnchorsFromMap(pattern) for _, value := range resourceArray { resource, ok := value.(map[string]interface{}) @@ -107,7 +104,7 @@ func validateArray(resourcePart, patternPart interface{}) error { return fmt.Errorf("expected array, found %T", resourcePart) } - if skipArrayObject(resource, anchors) { + if skipValidatingObject(resource, anchors) { continue } @@ -177,19 +174,7 @@ func validateMapElement(resourcePart, patternPart interface{}) error { return nil } -func getAnchorsFromMap(pattern map[string]interface{}) (map[string]interface{}, error) { - result := make(map[string]interface{}) - - for key, value := range pattern { - if wrappedWithParentheses(key) { - result[key] = value - } - } - - return result, nil -} - -func skipArrayObject(object, anchors map[string]interface{}) bool { +func skipValidatingObject(object, anchors map[string]interface{}) bool { for key, pattern := range anchors { key = key[1 : len(key)-1] diff --git a/pkg/engine/validation_test.go b/pkg/engine/validation_test.go index eac1a884bf..3f3f483c8e 100644 --- a/pkg/engine/validation_test.go +++ b/pkg/engine/validation_test.go @@ -306,8 +306,7 @@ func TestGetAnchorsFromMap_ThereAreAnchors(t *testing.T) { var unmarshalled map[string]interface{} json.Unmarshal(rawMap, &unmarshalled) - actualMap, err := getAnchorsFromMap(unmarshalled) - assert.NilError(t, err) + actualMap := GetAnchorsFromMap(unmarshalled) assert.Equal(t, len(actualMap), 2) assert.Equal(t, actualMap["(name)"].(string), "nirmata-*") assert.Equal(t, actualMap["(namespace)"].(string), "kube-?olicy") @@ -319,8 +318,7 @@ func TestGetAnchorsFromMap_ThereAreNoAnchors(t *testing.T) { var unmarshalled map[string]interface{} json.Unmarshal(rawMap, &unmarshalled) - actualMap, err := getAnchorsFromMap(unmarshalled) - assert.NilError(t, err) + actualMap := GetAnchorsFromMap(unmarshalled) assert.Assert(t, len(actualMap) == 0) } diff --git a/pkg/webhooks/server.go b/pkg/webhooks/server.go index 209c9d4b7f..cd7c6e0499 100644 --- a/pkg/webhooks/server.go +++ b/pkg/webhooks/server.go @@ -16,7 +16,6 @@ import ( policylister "github.com/nirmata/kube-policy/pkg/client/listers/policy/v1alpha1" "github.com/nirmata/kube-policy/pkg/config" engine "github.com/nirmata/kube-policy/pkg/engine" - "github.com/nirmata/kube-policy/pkg/engine/mutation" tlsutils "github.com/nirmata/kube-policy/pkg/tls" v1beta1 "k8s.io/api/admission/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -143,7 +142,7 @@ func (ws *WebhookServer) HandleMutation(request *v1beta1.AdmissionRequest) *v1be return nil } - var allPatches []mutation.PatchBytes + var allPatches []engine.PatchBytes for _, policy := range policies { ws.logger.Printf("Applying policy %s with %d rules\n", policy.ObjectMeta.Name, len(policy.Spec.Rules)) @@ -160,7 +159,7 @@ func (ws *WebhookServer) HandleMutation(request *v1beta1.AdmissionRequest) *v1be patchType := v1beta1.PatchTypeJSONPatch return &v1beta1.AdmissionResponse{ Allowed: true, - Patch: mutation.JoinPatches(allPatches), + Patch: engine.JoinPatches(allPatches), PatchType: &patchType, } } From ab31d980b68cc430ecc8f1654178a5d606c8f584 Mon Sep 17 00:00:00 2001 From: kacejot Date: Wed, 22 May 2019 18:28:38 +0100 Subject: [PATCH 2/7] Updated mutation base due to spec --- pkg/engine/mutation.go | 2 +- pkg/engine/overlay.go | 33 ++++++++++++++------------------- pkg/engine/overlay_test.go | 16 +++++++++++++++- 3 files changed, 30 insertions(+), 21 deletions(-) diff --git a/pkg/engine/mutation.go b/pkg/engine/mutation.go index 02bdfdeb9a..276f4bcd36 100644 --- a/pkg/engine/mutation.go +++ b/pkg/engine/mutation.go @@ -3,7 +3,7 @@ package engine import ( "log" - kubepolicy "github.com/nirmata/kube-policy/pkg/apis/policy/v1alpha1" + kubepolicy "github.com/nirmata/kyverno/pkg/apis/policy/v1alpha1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) diff --git a/pkg/engine/overlay.go b/pkg/engine/overlay.go index 4bac976dc3..8c2ef40ac6 100644 --- a/pkg/engine/overlay.go +++ b/pkg/engine/overlay.go @@ -2,11 +2,11 @@ package engine import ( "encoding/json" - "fmt" "log" + "reflect" "strconv" - kubepolicy "github.com/nirmata/kube-policy/pkg/apis/policy/v1alpha1" + kubepolicy "github.com/nirmata/kyverno/pkg/apis/policy/v1alpha1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -37,6 +37,12 @@ func ProcessOverlay(policy kubepolicy.Policy, rawResource []byte, gvk metav1.Gro } func applyOverlay(resource, overlay interface{}, path string) ([]PatchBytes, error) { + // resource item exists but has different type - replace + // all subtree within this path by overlay + if reflect.TypeOf(resource) != reflect.TypeOf(overlay) { + replaceResource(resource, overlay, path) + } + switch typedOverlay := overlay.(type) { case map[string]interface{}: typedResource := resource.(map[string]interface{}) @@ -55,23 +61,11 @@ func applyOverlay(resource, overlay interface{}, path string) ([]PatchBytes, err typedResource := resource.([]interface{}) applyOverlayToArray(typedResource, typedOverlay, path) case string: - typedResource, ok := resource.(string) - if !ok { - return nil, fmt.Errorf("Expected string, found %T", resource) - } - replaceResource(typedResource, typedOverlay, path) + replaceResource(resource, overlay, path) case float64: - typedResource, ok := resource.(float64) - if !ok { - return nil, fmt.Errorf("Expected string, found %T", resource) - } - replaceResource(typedResource, typedOverlay, path) + replaceResource(resource, overlay, path) case int64: - typedResource, ok := resource.(int64) - if !ok { - return nil, fmt.Errorf("Expected string, found %T", resource) - } - replaceResource(typedResource, typedOverlay, path) + replaceResource(resource, overlay, path) } return nil, nil @@ -82,7 +76,6 @@ func applyOverlayToArray(resource, overlay []interface{}, path string) { case map[string]interface{}: for _, overlayElement := range overlay { typedOverlay := overlayElement.(map[string]interface{}) - anchors := GetAnchorsFromMap(typedOverlay) if len(anchors) > 0 { @@ -117,7 +110,9 @@ func skipArrayObject(object, anchors map[string]interface{}) bool { return true } - return value != pattern + if value != pattern { + return true + } } return false diff --git a/pkg/engine/overlay_test.go b/pkg/engine/overlay_test.go index a35f183ae6..ab508039b6 100644 --- a/pkg/engine/overlay_test.go +++ b/pkg/engine/overlay_test.go @@ -1,11 +1,25 @@ package engine import ( + "encoding/json" + "fmt" + "reflect" "testing" "gotest.tools/assert" ) func TestApplyOverlay_BaseCase(t *testing.T) { - assert.Assert(t, true) + resource1Raw := []byte(`{ "dictionary": { "key1": "val1", "key2": "val2", "array": [ 1, 2 ] } }`) + resource2Raw := []byte(`{ "dictionary": "somestring" }`) + + var resource1, resource2 interface{} + + json.Unmarshal(resource1Raw, &resource1) + json.Unmarshal(resource2Raw, &resource2) + + fmt.Printf("First resource type: %v", reflect.TypeOf(resource1)) + fmt.Printf("Second resource type: %v", reflect.TypeOf(resource2)) + + assert.Assert(t, reflect.TypeOf(resource1) == reflect.TypeOf(resource2)) } From f776e26dccb01a4d37a10cbbc396ee21ff938ecc Mon Sep 17 00:00:00 2001 From: kacejot Date: Wed, 22 May 2019 18:29:10 +0100 Subject: [PATCH 3/7] Updated code due to changes in structure --- pkg/webhooks/server.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pkg/webhooks/server.go b/pkg/webhooks/server.go index f611050762..0c5156ed38 100644 --- a/pkg/webhooks/server.go +++ b/pkg/webhooks/server.go @@ -12,11 +12,12 @@ import ( "os" "time" - kubeClient "github.com/nirmata/kube-policy/kubeclient" - policylister "github.com/nirmata/kube-policy/pkg/client/listers/policy/v1alpha1" - "github.com/nirmata/kube-policy/pkg/config" - engine "github.com/nirmata/kube-policy/pkg/engine" - tlsutils "github.com/nirmata/kube-policy/pkg/tls" + "github.com/nirmata/kyverno/client" + "github.com/nirmata/kyverno/pkg/client/listers/policy/v1alpha1" + "github.com/nirmata/kyverno/pkg/config" + engine "github.com/nirmata/kyverno/pkg/engine" + "github.com/nirmata/kyverno/pkg/sharedinformer" + tlsutils "github.com/nirmata/kyverno/pkg/tls" v1beta1 "k8s.io/api/admission/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" From afb6b068e4751ed48a9f871b6875317dd9cd39bd Mon Sep 17 00:00:00 2001 From: kacejot Date: Wed, 22 May 2019 18:29:57 +0100 Subject: [PATCH 4/7] Updated .gitignore to ingore kyverno binary --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 6068efc5ee..b838b00397 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,7 @@ vendor pkg/client pkg/apis/policy/v1alpha1/zz_generated.deepcopy.go certs -kube-policy +kyverno Gopkg.lock Dockerfile .vscode From 09e0065d4cdd0000d204cd620f0f98958f6e4d68 Mon Sep 17 00:00:00 2001 From: kacejot Date: Wed, 22 May 2019 22:34:25 +0100 Subject: [PATCH 5/7] Finished mutating overlay. Added several tests --- pkg/engine/overlay.go | 309 +++++++++++++++++++++++++++++++++---- pkg/engine/overlay_test.go | 60 +++++-- 2 files changed, 329 insertions(+), 40 deletions(-) diff --git a/pkg/engine/overlay.go b/pkg/engine/overlay.go index 8c2ef40ac6..1fe49b5141 100644 --- a/pkg/engine/overlay.go +++ b/pkg/engine/overlay.go @@ -2,9 +2,11 @@ package engine import ( "encoding/json" + "fmt" "log" "reflect" - "strconv" + + jsonpatch "github.com/evanphx/json-patch" kubepolicy "github.com/nirmata/kyverno/pkg/apis/policy/v1alpha1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -37,10 +39,18 @@ func ProcessOverlay(policy kubepolicy.Policy, rawResource []byte, gvk metav1.Gro } func applyOverlay(resource, overlay interface{}, path string) ([]PatchBytes, error) { + var appliedPatches []PatchBytes + // resource item exists but has different type - replace // all subtree within this path by overlay if reflect.TypeOf(resource) != reflect.TypeOf(overlay) { - replaceResource(resource, overlay, path) + patch, err := replaceSubtree(overlay, path) + if err != nil { + return nil, err + } + + appliedPatches = append(appliedPatches, patch) + return appliedPatches, nil } switch typedOverlay := overlay.(type) { @@ -48,57 +58,169 @@ func applyOverlay(resource, overlay interface{}, path string) ([]PatchBytes, err typedResource := resource.(map[string]interface{}) for key, value := range typedOverlay { - path += "/" + key + if wrappedWithParentheses(key) { + key = key[1 : len(key)-1] + } + currentPath := path + key + "/" resourcePart, ok := typedResource[key] if ok { - applyOverlay(resourcePart, value, path) + patches, err := applyOverlay(resourcePart, value, currentPath) + if err != nil { + return nil, err + } + + appliedPatches = append(appliedPatches, patches...) + } else { - createSubtree(value, path) + patch, err := insertSubtree(value, currentPath) + if err != nil { + return nil, err + } + + appliedPatches = append(appliedPatches, patch) } } case []interface{}: typedResource := resource.([]interface{}) - applyOverlayToArray(typedResource, typedOverlay, path) + patches, err := applyOverlayToArray(typedResource, typedOverlay, path) + if err != nil { + return nil, err + } + + appliedPatches = append(appliedPatches, patches...) case string: - replaceResource(resource, overlay, path) + patch, err := replaceSubtree(overlay, path) + if err != nil { + return nil, err + } + + appliedPatches = append(appliedPatches, patch) case float64: - replaceResource(resource, overlay, path) + patch, err := replaceSubtree(overlay, path) + if err != nil { + return nil, err + } + + appliedPatches = append(appliedPatches, patch) case int64: - replaceResource(resource, overlay, path) + patch, err := replaceSubtree(overlay, path) + if err != nil { + return nil, err + } + + appliedPatches = append(appliedPatches, patch) } - return nil, nil + return appliedPatches, nil } -func applyOverlayToArray(resource, overlay []interface{}, path string) { +func applyOverlayToArray(resource, overlay []interface{}, path string) ([]PatchBytes, error) { + var appliedPatches []PatchBytes + if len(overlay) == 0 { + return nil, fmt.Errorf("overlay does not support empty arrays") + } + + if len(resource) == 0 { + patches, err := fillEmptyArray(overlay, path) + if err != nil { + return nil, err + } + + return patches, nil + } + + if reflect.TypeOf(resource[0]) != reflect.TypeOf(overlay[0]) { + return nil, fmt.Errorf("overlay array and resource array have elements of different types: %T and %T", overlay[0], resource[0]) + } + switch overlay[0].(type) { case map[string]interface{}: for _, overlayElement := range overlay { typedOverlay := overlayElement.(map[string]interface{}) anchors := GetAnchorsFromMap(typedOverlay) - if len(anchors) > 0 { - // Try to replace - for i, resourceElement := range resource { - path += "/" + strconv.Itoa(i) - typedResource := resourceElement.(map[string]interface{}) + currentPath := path + "0/" + for _, resourceElement := range resource { + typedResource := resourceElement.(map[string]interface{}) + if len(anchors) > 0 { if !skipArrayObject(typedResource, anchors) { - replaceResource(typedResource, typedOverlay, path) + patches, err := applyOverlay(resourceElement, overlayElement, currentPath) + if err != nil { + return nil, err + } + + appliedPatches = append(appliedPatches, patches...) + } + } else { + if hasNestedAnchors(overlayElement) { + patches, err := applyOverlay(resourceElement, overlayElement, currentPath) + if err != nil { + return nil, err + } + appliedPatches = append(appliedPatches, patches...) + } else { + patch, err := insertSubtree(overlayElement, currentPath) + if err != nil { + return nil, err + } + appliedPatches = append(appliedPatches, patch) } } - } else { - // Add new item to the front - path += "/0" - createSubtree(typedOverlay, path) + } + + } + default: + path += "0/" + for _, value := range overlay { + patch, err := insertSubtree(value, path) + if err != nil { + return nil, err + } + appliedPatches = append(appliedPatches, patch) + } + } + + return appliedPatches, nil +} + +// In case of empty resource array +// append all non-anchor items to front +func fillEmptyArray(overlay []interface{}, path string) ([]PatchBytes, error) { + var appliedPatches []PatchBytes + if len(overlay) == 0 { + return nil, fmt.Errorf("overlay does not support empty arrays") + } + + path += "0/" + + switch overlay[0].(type) { + case map[string]interface{}: + for _, overlayElement := range overlay { + typedOverlay := overlayElement.(map[string]interface{}) + anchors := GetAnchorsFromMap(typedOverlay) + + if len(anchors) == 0 { + patch, err := insertSubtree(overlayElement, path) + if err != nil { + return nil, err + } + + appliedPatches = append(appliedPatches, patch) } } default: - path += "/0" - for _, value := range overlay { - createSubtree(value, path) + for _, overlayElement := range overlay { + patch, err := insertSubtree(overlayElement, path) + if err != nil { + return nil, err + } + + appliedPatches = append(appliedPatches, patch) } } + + return appliedPatches, nil } func skipArrayObject(object, anchors map[string]interface{}) bool { @@ -118,11 +240,140 @@ func skipArrayObject(object, anchors map[string]interface{}) bool { return false } -func replaceResource(resource, overlay interface{}, path string) { - +func insertSubtree(overlay interface{}, path string) ([]byte, error) { + return processSubtree(overlay, path, "add") } -func createSubtree(overlayPart interface{}, path string) []PatchBytes { - - return nil +func replaceSubtree(overlay interface{}, path string) ([]byte, error) { + return processSubtree(overlay, path, "replace") +} + +func processSubtree(overlay interface{}, path string, op string) ([]byte, error) { + if len(path) > 1 && path[len(path)-1] == '/' { + path = path[:len(path)-1] + } + + if path == "" { + path = "/" + } + + value := prepareJSONValue(overlay) + patchStr := fmt.Sprintf(`{ "op": "%s", "path": "%s", "value": %s }`, op, path, value) + + // check the patch + _, err := jsonpatch.DecodePatch([]byte("[" + patchStr + "]")) + if err != nil { + return nil, err + } + + return []byte(patchStr), nil +} + +func prepareJSONValue(overlay interface{}) string { + switch typed := overlay.(type) { + case map[string]interface{}: + if len(typed) == 0 { + return "" + } + + if hasOnlyAnchors(overlay) { + return "" + } + + result := "" + for key, value := range typed { + jsonValue := prepareJSONValue(value) + + pair := fmt.Sprintf(`"%s":%s`, key, jsonValue) + + if result != "" { + result += ", " + } + + result += pair + } + + result = fmt.Sprintf(`{ %s }`, result) + return result + case []interface{}: + if len(typed) == 0 { + return "" + } + + if hasOnlyAnchors(overlay) { + return "" + } + + result := "" + for _, value := range typed { + jsonValue := prepareJSONValue(value) + + if result != "" { + result += ", " + } + + result += jsonValue + } + + result = fmt.Sprintf(`[ %s ]`, result) + return result + case string: + return fmt.Sprintf(`"%s"`, typed) + case float64: + return fmt.Sprintf("%f", typed) + case int64: + return fmt.Sprintf("%d", typed) + default: + return "" + } +} + +func hasOnlyAnchors(overlay interface{}) bool { + switch typed := overlay.(type) { + case map[string]interface{}: + if anchors := GetAnchorsFromMap(typed); len(anchors) == len(typed) { + return true + } + + for _, value := range typed { + if !hasOnlyAnchors(value) { + return false + } + } + + return true + case string: + return false + case float64: + return false + case int64: + return false + default: + return false + } +} + +func hasNestedAnchors(overlay interface{}) bool { + switch typed := overlay.(type) { + case map[string]interface{}: + if anchors := GetAnchorsFromMap(typed); len(anchors) > 0 { + return true + } + + for _, value := range typed { + if hasNestedAnchors(value) { + return true + } + } + + return false + case string: + return false + case float64: + return false + case int64: + return false + default: + return false + } } diff --git a/pkg/engine/overlay_test.go b/pkg/engine/overlay_test.go index ab508039b6..086fbe7511 100644 --- a/pkg/engine/overlay_test.go +++ b/pkg/engine/overlay_test.go @@ -2,24 +2,62 @@ package engine import ( "encoding/json" - "fmt" - "reflect" + "log" "testing" + jsonpatch "github.com/evanphx/json-patch" "gotest.tools/assert" ) -func TestApplyOverlay_BaseCase(t *testing.T) { - resource1Raw := []byte(`{ "dictionary": { "key1": "val1", "key2": "val2", "array": [ 1, 2 ] } }`) - resource2Raw := []byte(`{ "dictionary": "somestring" }`) +func TestApplyOverlay_NestedListWithAnchor(t *testing.T) { + resourceRaw := []byte(`{ "apiVersion": "v1", "kind": "Endpoints", "metadata": { "name": "test-endpoint", "labels": { "label": "test" } }, "subsets": [ { "addresses": [ { "ip": "192.168.10.171" } ], "ports": [ { "name": "secure-connection", "port": 443, "protocol": "TCP" } ] } ] }`) + overlayRaw := []byte(`{ "subsets": [ { "ports": [ { "(name)": "secure-connection", "port": 444, "protocol": "UDP" } ] } ] }`) - var resource1, resource2 interface{} + var resource, overlay interface{} - json.Unmarshal(resource1Raw, &resource1) - json.Unmarshal(resource2Raw, &resource2) + json.Unmarshal(resourceRaw, &resource) + json.Unmarshal(overlayRaw, &overlay) - fmt.Printf("First resource type: %v", reflect.TypeOf(resource1)) - fmt.Printf("Second resource type: %v", reflect.TypeOf(resource2)) + patches, err := applyOverlay(resource, overlay, "/") + assert.NilError(t, err) + assert.Assert(t, patches != nil) - assert.Assert(t, reflect.TypeOf(resource1) == reflect.TypeOf(resource2)) + patch := JoinPatches(patches) + decoded, err := jsonpatch.DecodePatch(patch) + assert.NilError(t, err) + assert.Assert(t, decoded != nil) + + patched, err := decoded.Apply(resourceRaw) + assert.NilError(t, err) + assert.Assert(t, patched != nil) + + expectedResult := []byte(`{"apiVersion":"v1","kind":"Endpoints","metadata":{"name":"test-endpoint","labels":{"label":"test"}},"subsets":[{"addresses":[{"ip":"192.168.10.171"}],"ports":[{"name":"secure-connection","port":444.000000,"protocol":"UDP"}]}]}`) + assert.Equal(t, string(expectedResult), string(patched)) +} + +func TestApplyOverlay_InsertIntoArray(t *testing.T) { + resourceRaw := []byte(`{ "apiVersion": "v1", "kind": "Endpoints", "metadata": { "name": "test-endpoint", "labels": { "label": "test" } }, "subsets": [ { "addresses": [ { "ip": "192.168.10.171" } ], "ports": [ { "name": "secure-connection", "port": 443, "protocol": "TCP" } ] } ] }`) + overlayRaw := []byte(`{ "subsets": [ { "addresses": [ { "ip": "192.168.10.172" }, { "ip": "192.168.10.173" } ], "ports": [ { "name": "insecure-connection", "port": 80, "protocol": "UDP" } ] } ] }`) + + var resource, overlay interface{} + + json.Unmarshal(resourceRaw, &resource) + json.Unmarshal(overlayRaw, &overlay) + + patches, err := applyOverlay(resource, overlay, "/") + assert.NilError(t, err) + assert.Assert(t, patches != nil) + + patch := JoinPatches(patches) + + decoded, err := jsonpatch.DecodePatch(patch) + assert.NilError(t, err) + assert.Assert(t, decoded != nil) + + patched, err := decoded.Apply(resourceRaw) + assert.NilError(t, err) + assert.Assert(t, patched != nil) + + log.Fatalf("%s", patched) + //assert.Equal(t, string(expectedResult), string(patched)) } From 1219063ff87559c3041a652e86ce42e6e491cb5b Mon Sep 17 00:00:00 2001 From: kacejot Date: Wed, 22 May 2019 22:43:19 +0100 Subject: [PATCH 6/7] Updated unit tests --- pkg/engine/overlay_test.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg/engine/overlay_test.go b/pkg/engine/overlay_test.go index 086fbe7511..f3dcebd4fc 100644 --- a/pkg/engine/overlay_test.go +++ b/pkg/engine/overlay_test.go @@ -2,7 +2,6 @@ package engine import ( "encoding/json" - "log" "testing" jsonpatch "github.com/evanphx/json-patch" @@ -58,6 +57,6 @@ func TestApplyOverlay_InsertIntoArray(t *testing.T) { assert.NilError(t, err) assert.Assert(t, patched != nil) - log.Fatalf("%s", patched) - //assert.Equal(t, string(expectedResult), string(patched)) + expectedResult := []byte(`{"apiVersion":"v1","kind":"Endpoints","metadata":{"name":"test-endpoint","labels":{"label":"test"}},"subsets":[{"addresses":[{"ip":"192.168.10.172"},{"ip":"192.168.10.173"}],"ports":[{"name":"insecure-connection","port":80.000000,"protocol":"UDP"}]},{"addresses":[{"ip":"192.168.10.171"}],"ports":[{"name":"secure-connection","port":443,"protocol":"TCP"}]}]}`) + assert.Equal(t, string(expectedResult), string(patched)) } From 101870fb5e79c4050596b9ca524d20df5a700dec Mon Sep 17 00:00:00 2001 From: kacejot Date: Wed, 22 May 2019 22:54:38 +0100 Subject: [PATCH 7/7] Added Overlay logic to mutation handling --- pkg/engine/mutation.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/engine/mutation.go b/pkg/engine/mutation.go index 276f4bcd36..3d387d6f55 100644 --- a/pkg/engine/mutation.go +++ b/pkg/engine/mutation.go @@ -29,11 +29,11 @@ func Mutate(policy kubepolicy.Policy, rawResource []byte, gvk metav1.GroupVersio // Process Overlay if rule.Mutation.Overlay != nil { - //overlayPatches, err := ProcessOverlay(rule.Mutation.Overlay, rawResource) + overlayPatches, err := ProcessOverlay(policy, rawResource, gvk) if err != nil { log.Printf("Overlay application has failed for rule %s in policy %s, err: %v\n", rule.Name, policy.ObjectMeta.Name, err) } else { - //policyPatches = append(policyPatches, overlayPatches...) + policyPatches = append(policyPatches, overlayPatches...) } }