mirrors/kyverno
mirrors/kyverno
1
0
Fork 0
mirror of https://github.com/kyverno/kyverno.git synced 2025-03-13 19:28:55 +00:00
Code Activity

fix: use structured jsonpatch instead of byte arrays (#7186)

Browse source 
Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>
This commit is contained in:
Charles-Edouard Brétéché 2023-05-13 10:56:54 +02:00 • committed by GitHub
parent 073309a8ae
commit 79a255a1e6
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
15 changed files with 220 additions and 172 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
pkg/background/mutate/mutate.go
View file

@ -2,7 +2,6 @@ package mutate
import (
"context"
"encoding/json"
"fmt"
"github.com/go-logr/logr"
@ -265,22 +264,11 @@ func addAnnotation(policy kyvernov1.PolicyInterface, patched *unstructured.Unstr
var rulePatches []utils.RulePatch
for _, patch := range r.Patches() {
var patchmap map[string]interface{}
if err := json.Unmarshal(patch, &patchmap); err != nil {
return nil, fmt.Errorf("failed to parse JSON patch bytes: %v", err)
}
rp := struct {
RuleName string `json:"rulename"`
Op string `json:"op"`
Path string `json:"path"`
}{
rulePatches = append(rulePatches, utils.RulePatch{
RuleName: r.Name(),
Op: patchmap["op"].(string),
Path: patchmap["path"].(string),
}
rulePatches = append(rulePatches, rp)
Op: patch.Operation,
Path: patch.Path,
})
}
annotationContent := make(map[string]string)

5
pkg/engine/api/engineresponse.go
View file

@ -7,6 +7,7 @@ import (
datautils "github.com/kyverno/kyverno/pkg/utils/data"
utils "github.com/kyverno/kyverno/pkg/utils/match"
"github.com/kyverno/kyverno/pkg/utils/wildcard"
"github.com/mattbaird/jsonpatch"
"k8s.io/api/admissionregistration/v1alpha1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
)
@ -153,8 +154,8 @@ func (er EngineResponse) IsValidatingAdmissionPolicy() bool {
}
// GetPatches returns all the patches joined
func (er EngineResponse) GetPatches() [][]byte {
var patches [][]byte
func (er EngineResponse) GetPatches() []jsonpatch.JsonPatchOperation {
var patches []jsonpatch.JsonPatchOperation
for _, r := range er.PolicyResponse.Rules {
patches = append(patches, r.Patches()...)
}

53
pkg/engine/api/engineresponse_test.go
View file

@ -5,6 +5,7 @@ import (
"testing"
kyvernov1 "github.com/kyverno/kyverno/api/kyverno/v1"
"github.com/mattbaird/jsonpatch"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
)
@ -917,7 +918,7 @@ func TestEngineResponse_GetPatches(t *testing.T) {
tests := []struct {
name string
fields fields
want [][]byte
want []jsonpatch.JsonPatchOperation
}{{}, {
fields: fields{
PolicyResponse: PolicyResponse{
@ -941,22 +942,62 @@ func TestEngineResponse_GetPatches(t *testing.T) {
PolicyResponse: PolicyResponse{
Rules: []RuleResponse{
{},
*RuleResponse{}.WithPatches([][]byte{{0, 1, 2}, {3, 4, 5}}...),
*RuleResponse{}.WithPatches([]jsonpatch.JsonPatchOperation{{
Operation: "add",
Path: "/1",
Value: 0,
}, {
Operation: "add",
Path: "/2",
Value: 1,
}}...),
},
},
},
want: [][]byte{{0, 1, 2}, {3, 4, 5}},
want: []jsonpatch.JsonPatchOperation{{
Operation: "add",
Path: "/1",
Value: 0,
}, {
Operation: "add",
Path: "/2",
Value: 1,
}},
}, {
fields: fields{
PolicyResponse: PolicyResponse{
Rules: []RuleResponse{
{},
*RuleResponse{}.WithPatches([][]byte{{0, 1, 2}, {3, 4, 5}}...),
*RuleResponse{}.WithPatches([][]byte{{7, 8, 9}}...),
*RuleResponse{}.WithPatches([]jsonpatch.JsonPatchOperation{{
Operation: "add",
Path: "/1",
Value: 0,
}, {
Operation: "add",
Path: "/2",
Value: 1,
}}...),
*RuleResponse{}.WithPatches([]jsonpatch.JsonPatchOperation{{
Operation: "add",
Path: "/3",
Value: 2,
}}...),
},
},
},
want: [][]byte{{0, 1, 2}, {3, 4, 5}, {7, 8, 9}},
want: []jsonpatch.JsonPatchOperation{{
Operation: "add",
Path: "/1",
Value: 0,
}, {
Operation: "add",
Path: "/2",
Value: 1,
}, {
Operation: "add",
Path: "/3",
Value: 2,
}},
}}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {

26
pkg/engine/api/imageverifymetadata.go
View file

@ -6,7 +6,7 @@ import (
"strings"
"github.com/go-logr/logr"
jsonutils "github.com/kyverno/kyverno/pkg/utils/json"
"github.com/mattbaird/jsonpatch"
)
const ImageVerifyAnnotationKey = "kyverno.io/verify-images"
@ -43,27 +43,27 @@ func ParseImageMetadata(jsonData string) (*ImageVerificationMetadata, error) {
}, nil
}
func (ivm *ImageVerificationMetadata) Patches(hasAnnotations bool, log logr.Logger) ([][]byte, error) {
func (ivm *ImageVerificationMetadata) Patches(hasAnnotations bool, log logr.Logger) ([]jsonpatch.JsonPatchOperation, error) {
if data, err := json.Marshal(ivm.Data); err != nil {
return nil, fmt.Errorf("failed to marshal metadata value: %v: %w", data, err)
} else {
var patches [][]byte
var patches []jsonpatch.JsonPatchOperation
if !hasAnnotations {
patch := jsonutils.NewPatchOperation("/metadata/annotations", "add", map[string]string{})
patchBytes, err := patch.Marshal()
if err != nil {
return nil, err
patch := jsonpatch.JsonPatchOperation{
Operation: "add",
Path: "/metadata/annotations",
Value: map[string]string{},
}
log.V(4).Info("adding annotation patch", "patch", patch)
patches = append(patches, patchBytes)
patches = append(patches, patch)
}
patch := jsonutils.NewPatchOperation(makeAnnotationKeyForJSONPatch(), "add", string(data))
patchBytes, err := patch.Marshal()
if err != nil {
return nil, err
patch := jsonpatch.JsonPatchOperation{
Operation: "add",
Path: makeAnnotationKeyForJSONPatch(),
Value: string(data),
}
log.V(4).Info("adding image verification patch", "patch", patch)
patches = append(patches, patchBytes)
patches = append(patches, patch)
return patches, nil
}
}

22
pkg/engine/api/imageverifymetadata_test.go
View file

@ -5,6 +5,7 @@ import (
"testing"
"github.com/go-logr/logr"
"github.com/stretchr/testify/assert"
)
func TestImageVerificationMetadata_IsVerified(t *testing.T) {
@ -306,7 +307,7 @@ func TestImageVerificationMetadata_Patches(t *testing.T) {
name string
fields fields
args args
want [][]byte
want []string
wantErr bool
}{{
fields: fields{
@ -318,9 +319,9 @@ func TestImageVerificationMetadata_Patches(t *testing.T) {
hasAnnotations: false,
log: logr.Discard(),
},
want: [][]byte{
[]byte(`{"path":"/metadata/annotations","op":"add","value":{}}`),
[]byte(`{"path":"/metadata/annotations/kyverno.io~1verify-images","op":"add","value":"{\"test\":true}"}`),
want: []string{
`{"op":"add","path":"/metadata/annotations","value":{}}`,
`{"op":"add","path":"/metadata/annotations/kyverno.io~1verify-images","value":"{\"test\":true}"}`,
},
}, {
fields: fields{
@ -332,16 +333,16 @@ func TestImageVerificationMetadata_Patches(t *testing.T) {
hasAnnotations: true,
log: logr.Discard(),
},
want: [][]byte{
[]byte(`{"path":"/metadata/annotations/kyverno.io~1verify-images","op":"add","value":"{\"test\":true}"}`),
want: []string{
`{"op":"add","path":"/metadata/annotations/kyverno.io~1verify-images","value":"{\"test\":true}"}`,
},
}, {
args: args{
hasAnnotations: true,
log: logr.Discard(),
},
want: [][]byte{
[]byte(`{"path":"/metadata/annotations/kyverno.io~1verify-images","op":"add","value":"null"}`),
want: []string{
`{"op":"add","path":"/metadata/annotations/kyverno.io~1verify-images","value":"null"}`,
},
}}
for _, tt := range tests {
@ -354,8 +355,9 @@ func TestImageVerificationMetadata_Patches(t *testing.T) {
t.Errorf("ImageVerificationMetadata.Patches() error = %v, wantErr %v", err, tt.wantErr)
return
}
if !reflect.DeepEqual(got, tt.want) {
t.Errorf("ImageVerificationMetadata.Patches() = %v, want %v", got, tt.want)
assert.Equal(t, len(got), len(tt.want))
for i := range got {
assert.Equal(t, got[i].Json(), tt.want[i])
}
})
}

7
pkg/engine/api/ruleresponse.go
View file

@ -5,6 +5,7 @@ import (
kyvernov2alpha1 "github.com/kyverno/kyverno/api/kyverno/v2alpha1"
pssutils "github.com/kyverno/kyverno/pkg/pss/utils"
"github.com/mattbaird/jsonpatch"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/pod-security-admission/api"
@ -31,7 +32,7 @@ type RuleResponse struct {
// status rule status
status RuleStatus
// patches are JSON patches, for mutation rules
patches [][]byte
patches []jsonpatch.JsonPatchOperation
// stats contains rule statistics
stats ExecutionStats
// generatedResource is the generated by the generate rules of a policy
@ -102,7 +103,7 @@ func (r RuleResponse) WithGeneratedResource(resource unstructured.Unstructured)
return &r
}
func (r RuleResponse) WithPatches(patches ...[]byte) *RuleResponse {
func (r RuleResponse) WithPatches(patches ...jsonpatch.JsonPatchOperation) *RuleResponse {
r.patches = patches
return &r
}
@ -136,7 +137,7 @@ func (r *RuleResponse) GeneratedResource() unstructured.Unstructured {
return r.generatedResource
}
func (r *RuleResponse) Patches() [][]byte {
func (r *RuleResponse) Patches() []jsonpatch.JsonPatchOperation {
return r.patches
}

3
pkg/engine/handlers/mutation/common.go
View file

@ -9,7 +9,6 @@ import (
engineapi "github.com/kyverno/kyverno/pkg/engine/api"
"github.com/kyverno/kyverno/pkg/engine/internal"
"github.com/kyverno/kyverno/pkg/engine/mutate"
"github.com/kyverno/kyverno/pkg/engine/mutate/patch"
engineutils "github.com/kyverno/kyverno/pkg/engine/utils"
"github.com/kyverno/kyverno/pkg/utils/api"
"github.com/mattbaird/jsonpatch"
@ -153,7 +152,7 @@ func buildRuleResponse(rule *kyvernov1.Rule, mutateResp *mutate.Response, info r
mutateResp.Status,
)
if mutateResp.Status == engineapi.RuleStatusPass {
resp = resp.WithPatches(patch.ConvertPatches(mutateResp.Patches...)...)
resp = resp.WithPatches(mutateResp.Patches...)
if len(rule.Mutation.Targets) != 0 {
resp = resp.WithPatchedTarget(&mutateResp.PatchedResource, info.parentResourceGVR, info.subresource)
}

8
pkg/engine/image_verify_test.go
View file

@ -16,10 +16,12 @@ import (
"github.com/kyverno/kyverno/pkg/engine/context/resolvers"
"github.com/kyverno/kyverno/pkg/engine/internal"
"github.com/kyverno/kyverno/pkg/engine/jmespath"
"github.com/kyverno/kyverno/pkg/engine/mutate/patch"
"github.com/kyverno/kyverno/pkg/engine/policycontext"
engineutils "github.com/kyverno/kyverno/pkg/engine/utils"
"github.com/kyverno/kyverno/pkg/registryclient"
kubeutils "github.com/kyverno/kyverno/pkg/utils/kube"
"github.com/mattbaird/jsonpatch"
"gotest.tools/assert"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
kubefake "k8s.io/client-go/kubernetes/fake"
@ -479,7 +481,7 @@ func Test_SignatureGoodSigned(t *testing.T) {
assert.Equal(t, engineResp.PolicyResponse.Rules[0].Status(), engineapi.RuleStatusPass, engineResp.PolicyResponse.Rules[0].Message())
assert.Equal(t, len(engineResp.PolicyResponse.Rules[0].Patches()), 1)
patch := engineResp.PolicyResponse.Rules[0].Patches()[0]
assert.Equal(t, string(patch), "{\"op\":\"replace\",\"path\":\"/spec/containers/0/image\",\"value\":\"ghcr.io/kyverno/test-verify-image:signed@sha256:b31bfb4d0213f254d361e0079deaaebefa4f82ba7aa76ef82e90b4935ad5b105\"}")
assert.Equal(t, patch.Json(), "{\"op\":\"replace\",\"path\":\"/spec/containers/0/image\",\"value\":\"ghcr.io/kyverno/test-verify-image:signed@sha256:b31bfb4d0213f254d361e0079deaaebefa4f82ba7aa76ef82e90b4935ad5b105\"}")
}
func Test_SignatureUnsigned(t *testing.T) {
@ -797,8 +799,8 @@ func Test_MarkImageVerified(t *testing.T) {
assert.Equal(t, verified, true)
}
func testApplyPatches(t *testing.T, patches [][]byte) unstructured.Unstructured {
patchedResource, err := engineutils.ApplyPatches([]byte(testResource), patches)
func testApplyPatches(t *testing.T, patches []jsonpatch.JsonPatchOperation) unstructured.Unstructured {
patchedResource, err := engineutils.ApplyPatches([]byte(testResource), patch.ConvertPatches(patches...))
assert.NilError(t, err)
assert.Assert(t, patchedResource != nil)

27
pkg/engine/internal/imageverifier.go
View file

@ -2,7 +2,6 @@ package internal
import (
"context"
"encoding/json"
"errors"
"fmt"
"net"
@ -21,6 +20,7 @@ import (
apiutils "github.com/kyverno/kyverno/pkg/utils/api"
"github.com/kyverno/kyverno/pkg/utils/jsonpointer"
"github.com/kyverno/kyverno/pkg/utils/wildcard"
"github.com/mattbaird/jsonpatch"
"go.uber.org/multierr"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
)
@ -149,12 +149,12 @@ func buildStatementMap(statements []map[string]interface{}) (map[string][]map[st
return results, predicateTypes
}
func makeAddDigestPatch(imageInfo apiutils.ImageInfo, digest string) ([]byte, error) {
patch := make(map[string]interface{})
patch["op"] = "replace"
patch["path"] = imageInfo.Pointer
patch["value"] = imageInfo.String() + "@" + digest
return json.Marshal(patch)
func makeAddDigestPatch(imageInfo apiutils.ImageInfo, digest string) jsonpatch.JsonPatchOperation {
return jsonpatch.JsonPatchOperation{
Operation: "replace",
Path: imageInfo.Pointer,
Value: imageInfo.String() + "@" + digest,
}
}
func EvaluateConditions(
@ -224,7 +224,7 @@ func (iv *ImageVerifier) Verify(
if ruleResp == nil {
ruleResp = engineapi.RulePass(iv.rule.Name, engineapi.ImageVerify, "mutated image digest")
}
ruleResp = ruleResp.WithPatches(patch)
ruleResp = ruleResp.WithPatches(*patch)
imageInfo.Digest = retrievedDigest
image = imageInfo.String()
}
@ -565,7 +565,7 @@ func (iv *ImageVerifier) checkAttestations(a kyvernov1.Attestation, s map[string
return EvaluateConditions(a.Conditions, iv.policyContext.JSONContext(), s, iv.logger)
}
func (iv *ImageVerifier) handleMutateDigest(ctx context.Context, digest string, imageInfo apiutils.ImageInfo) ([]byte, string, error) {
func (iv *ImageVerifier) handleMutateDigest(ctx context.Context, digest string, imageInfo apiutils.ImageInfo) (*jsonpatch.JsonPatchOperation, string, error) {
if imageInfo.Digest != "" {
return nil, "", nil
}
@ -576,10 +576,7 @@ func (iv *ImageVerifier) handleMutateDigest(ctx context.Context, digest string,
}
digest = desc.Digest.String()
}
patch, err := makeAddDigestPatch(imageInfo, digest)
if err != nil {
return nil, "", fmt.Errorf("failed to create image digest patch: %w", err)
}
iv.logger.V(4).Info("adding digest patch", "image", imageInfo.String(), "patch", string(patch))
return patch, digest, nil
patch := makeAddDigestPatch(imageInfo, digest)
iv.logger.V(4).Info("adding digest patch", "image", imageInfo.String(), "patch", patch.Json())
return &patch, digest, nil
}

9
pkg/engine/mutate/mutation_test.go
View file

@ -10,7 +10,6 @@ import (
engineapi "github.com/kyverno/kyverno/pkg/engine/api"
"github.com/kyverno/kyverno/pkg/engine/context"
"github.com/kyverno/kyverno/pkg/engine/jmespath"
"github.com/kyverno/kyverno/pkg/engine/mutate/patch"
kubeutils "github.com/kyverno/kyverno/pkg/utils/kube"
"gotest.tools/assert"
"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions"
@ -58,9 +57,7 @@ func applyPatches(rule *types.Rule, resource unstructured.Unstructured) (*engine
"",
engineapi.Mutation,
mutateResp.Message,
).WithPatches(
patch.ConvertPatches(mutateResp.Patches...)...,
), mutateResp.PatchedResource
).WithPatches(mutateResp.Patches...), mutateResp.PatchedResource
}
func TestProcessPatches_EmptyPatches(t *testing.T) {
@ -167,7 +164,7 @@ func TestProcessPatches_AddAndRemovePathsDontExist_ContinueOnError_NotEmptyResul
rr, _ := applyPatches(rule, *resourceUnstructured)
assert.Equal(t, rr.Status(), engineapi.RuleStatusPass)
assert.Assert(t, len(rr.Patches()) != 0)
assertEqStringAndData(t, `{"path":"/metadata/labels/label3","op":"add","value":"label3Value"}`, rr.Patches()[0])
assertEqStringAndData(t, `{"path":"/metadata/labels/label3","op":"add","value":"label3Value"}`, []byte(rr.Patches()[0].Json()))
}
func TestProcessPatches_RemovePathDoesntExist_EmptyResult(t *testing.T) {
@ -193,7 +190,7 @@ func TestProcessPatches_RemovePathDoesntExist_NotEmptyResult(t *testing.T) {
rr, _ := applyPatches(rule, *resourceUnstructured)
assert.Equal(t, rr.Status(), engineapi.RuleStatusPass)
assert.Assert(t, len(rr.Patches()) == 1)
assertEqStringAndData(t, `{"path":"/metadata/labels/label2","op":"add","value":"label2Value"}`, rr.Patches()[0])
assertEqStringAndData(t, `{"path":"/metadata/labels/label2","op":"add","value":"label2Value"}`, []byte(rr.Patches()[0].Json()))
}
func assertEqStringAndData(t *testing.T, str string, data []byte) {

63
pkg/engine/mutation_test.go
View file

@ -95,7 +95,7 @@ func Test_VariableSubstitutionPatchStrategicMerge(t *testing.T) {
]
}
}`)
expectedPatch := []byte(`{"op":"add","path":"/metadata/labels","value":{"appname":"check-root-user"}}`)
expectedPatch := `{"op":"add","path":"/metadata/labels","value":{"appname":"check-root-user"}}`
var policy kyverno.ClusterPolicy
err := json.Unmarshal(policyRaw, &policy)
@ -120,8 +120,8 @@ func Test_VariableSubstitutionPatchStrategicMerge(t *testing.T) {
assert.Equal(t, len(er.PolicyResponse.Rules), 1)
assert.Equal(t, len(er.PolicyResponse.Rules[0].Patches()), 1)
t.Log(string(er.PolicyResponse.Rules[0].Patches()[0]))
if !reflect.DeepEqual(expectedPatch, er.PolicyResponse.Rules[0].Patches()[0]) {
t.Log(er.PolicyResponse.Rules[0].Patches()[0].Json())
if !reflect.DeepEqual(expectedPatch, er.PolicyResponse.Rules[0].Patches()[0].Json()) {
t.Error("patches dont match")
}
}
@ -251,7 +251,7 @@ func Test_variableSubstitutionCLI(t *testing.T) {
}
}`)
expectedPatch := []byte(`{"op":"add","path":"/metadata/labels","value":{"my-environment-name":"dev1"}}`)
expectedPatch := `{"op":"add","path":"/metadata/labels","value":{"my-environment-name":"dev1"}}`
var policy kyverno.ClusterPolicy
err := json.Unmarshal(policyRaw, &policy)
@ -292,8 +292,8 @@ func Test_variableSubstitutionCLI(t *testing.T) {
assert.Equal(t, len(er.PolicyResponse.Rules), 1)
assert.Equal(t, len(er.PolicyResponse.Rules[0].Patches()), 1)
t.Log(string(expectedPatch))
t.Log(string(er.PolicyResponse.Rules[0].Patches()[0]))
if !reflect.DeepEqual(expectedPatch, er.PolicyResponse.Rules[0].Patches()[0]) {
t.Log(er.PolicyResponse.Rules[0].Patches()[0].Json())
if !reflect.DeepEqual(expectedPatch, er.PolicyResponse.Rules[0].Patches()[0].Json()) {
t.Error("patches don't match")
}
}
@ -400,8 +400,8 @@ func Test_chained_rules(t *testing.T) {
assert.Equal(t, len(er.PolicyResponse.Rules[0].Patches()), 1)
assert.Equal(t, len(er.PolicyResponse.Rules[1].Patches()), 1)
assert.Equal(t, string(er.PolicyResponse.Rules[0].Patches()[0]), `{"op":"replace","path":"/spec/containers/0/image","value":"myregistry.corp.com/foo/bash:5.0"}`)
assert.Equal(t, string(er.PolicyResponse.Rules[1].Patches()[0]), `{"op":"replace","path":"/spec/containers/0/image","value":"otherregistry.corp.com/foo/bash:5.0"}`)
assert.Equal(t, er.PolicyResponse.Rules[0].Patches()[0].Json(), `{"op":"replace","path":"/spec/containers/0/image","value":"myregistry.corp.com/foo/bash:5.0"}`)
assert.Equal(t, er.PolicyResponse.Rules[1].Patches()[0].Json(), `{"op":"replace","path":"/spec/containers/0/image","value":"otherregistry.corp.com/foo/bash:5.0"}`)
}
func Test_precondition(t *testing.T) {
@ -460,7 +460,7 @@ func Test_precondition(t *testing.T) {
]
}
}`)
expectedPatch := []byte(`{"op":"add","path":"/metadata/labels/my-added-label","value":"test"}`)
expectedPatch := `{"op":"add","path":"/metadata/labels/my-added-label","value":"test"}`
var policy kyverno.ClusterPolicy
err := json.Unmarshal(policyRaw, &policy)
@ -480,8 +480,8 @@ func Test_precondition(t *testing.T) {
er := testMutate(context.TODO(), nil, registryclient.NewOrDie(), policyContext, enginetest.ContextLoaderFactory(nil, nil))
t.Log(string(expectedPatch))
t.Log(string(er.PolicyResponse.Rules[0].Patches()[0]))
if !reflect.DeepEqual(expectedPatch, er.PolicyResponse.Rules[0].Patches()[0]) {
t.Log(er.PolicyResponse.Rules[0].Patches()[0].Json())
if !reflect.DeepEqual(expectedPatch, er.PolicyResponse.Rules[0].Patches()[0].Json()) {
t.Error("patches don't match")
}
}
@ -556,7 +556,7 @@ func Test_nonZeroIndexNumberPatchesJson6902(t *testing.T) {
}
}`)
expectedPatch := []byte(`{"op":"add","path":"/subsets/0/addresses/1","value":{"ip":"192.168.42.172"}}`)
expectedPatch := `{"op":"add","path":"/subsets/0/addresses/1","value":{"ip":"192.168.42.172"}}`
var policy kyverno.ClusterPolicy
err := json.Unmarshal(policyraw, &policy)
@ -576,8 +576,8 @@ func Test_nonZeroIndexNumberPatchesJson6902(t *testing.T) {
er := testMutate(context.TODO(), nil, registryclient.NewOrDie(), policyContext, enginetest.ContextLoaderFactory(nil, nil))
t.Log(string(expectedPatch))
t.Log(string(er.PolicyResponse.Rules[0].Patches()[0]))
if !reflect.DeepEqual(expectedPatch, er.PolicyResponse.Rules[0].Patches()[0]) {
t.Log(er.PolicyResponse.Rules[0].Patches()[0].Json())
if !reflect.DeepEqual(expectedPatch, er.PolicyResponse.Rules[0].Patches()[0].Json()) {
t.Error("patches don't match")
}
}
@ -1581,7 +1581,7 @@ func Test_mutate_existing_resources(t *testing.T) {
for _, rr := range er.PolicyResponse.Rules {
for i, p := range rr.Patches() {
assert.Equal(t, test.patches[i], string(p), "test %s failed:\nGot %s\nExpected: %s", test.name, rr.Patches()[i], test.patches[i])
assert.Equal(t, test.patches[i], p.Json(), "test %s failed:\nGot %s\nExpected: %s", test.name, rr.Patches()[i], test.patches[i])
assert.Equal(t, rr.Status(), engineapi.RuleStatusPass, rr.Status())
}
}
@ -1660,8 +1660,8 @@ func Test_RuleSelectorMutate(t *testing.T) {
}
}`)
expectedPatch1 := []byte(`{"op":"add","path":"/metadata/labels","value":{"app":"root"}}`)
expectedPatch2 := []byte(`{"op":"add","path":"/metadata/labels/appname","value":"check-root-user"}`)
expectedPatch1 := `{"op":"add","path":"/metadata/labels","value":{"app":"root"}}`
expectedPatch2 := `{"op":"add","path":"/metadata/labels/appname","value":"check-root-user"}`
var policy kyverno.ClusterPolicy
err := json.Unmarshal(policyRaw, &policy)
@ -1687,10 +1687,10 @@ func Test_RuleSelectorMutate(t *testing.T) {
assert.Equal(t, len(er.PolicyResponse.Rules[0].Patches()), 1)
assert.Equal(t, len(er.PolicyResponse.Rules[1].Patches()), 1)
if !reflect.DeepEqual(expectedPatch1, er.PolicyResponse.Rules[0].Patches()[0]) {
if !reflect.DeepEqual(expectedPatch1, er.PolicyResponse.Rules[0].Patches()[0].Json()) {
t.Error("rule 1 patches dont match")
}
if !reflect.DeepEqual(expectedPatch2, er.PolicyResponse.Rules[1].Patches()[0]) {
if !reflect.DeepEqual(expectedPatch2, er.PolicyResponse.Rules[1].Patches()[0].Json()) {
t.Errorf("rule 2 patches dont match")
}
@ -1701,7 +1701,7 @@ func Test_RuleSelectorMutate(t *testing.T) {
assert.Equal(t, len(er.PolicyResponse.Rules), 1)
assert.Equal(t, len(er.PolicyResponse.Rules[0].Patches()), 1)
if !reflect.DeepEqual(expectedPatch1, er.PolicyResponse.Rules[0].Patches()[0]) {
if !reflect.DeepEqual(expectedPatch1, er.PolicyResponse.Rules[0].Patches()[0].Json()) {
t.Error("rule 1 patches dont match")
}
}
@ -1713,7 +1713,7 @@ func Test_SpecialCharacters(t *testing.T) {
name string
policyRaw []byte
documentRaw []byte
want [][]byte
want []string
}{
{
name: "regex_replace",
@ -1790,8 +1790,8 @@ func Test_SpecialCharacters(t *testing.T) {
}
}
}`),
want: [][]byte{
[]byte(`{"op":"replace","path":"/metadata/labels/retention","value":"days_30"}`),
want: []string{
`{"op":"replace","path":"/metadata/labels/retention","value":"days_30"}`,
},
},
{
@ -1869,8 +1869,8 @@ func Test_SpecialCharacters(t *testing.T) {
}
}
}`),
want: [][]byte{
[]byte(`{"op":"replace","path":"/metadata/labels/corp.com~1retention","value":"days_30"}`),
want: []string{
`{"op":"replace","path":"/metadata/labels/corp.com~1retention","value":"days_30"}`,
},
},
{
@ -1948,8 +1948,8 @@ func Test_SpecialCharacters(t *testing.T) {
}
}
}`),
want: [][]byte{
[]byte(`{"op":"replace","path":"/metadata/labels/corp-retention","value":"days_30"}`),
want: []string{
`{"op":"replace","path":"/metadata/labels/corp-retention","value":"days_30"}`,
},
},
{
@ -2026,8 +2026,8 @@ func Test_SpecialCharacters(t *testing.T) {
}
}
}`),
want: [][]byte{
[]byte(`{"op":"replace","path":"/metadata/labels/deploy-zone","value":"EU-CENTRAL-1"}`),
want: []string{
`{"op":"replace","path":"/metadata/labels/deploy-zone","value":"EU-CENTRAL-1"}`,
},
},
}
@ -2062,8 +2062,9 @@ func Test_SpecialCharacters(t *testing.T) {
// Mutate and make sure that we got the expected amount of rules.
patches := testMutate(context.TODO(), nil, registryclient.NewOrDie(), policyContext, nil).GetPatches()
if !reflect.DeepEqual(patches, tt.want) {
t.Errorf("Mutate() got patches %s, expected %s", patches, tt.want)
assert.Equal(t, len(patches), len(tt.want))
for i := range patches {
assert.Equal(t, patches[i].Json(), tt.want[i])
}
})
}

69
pkg/utils/annotations.go
View file

@ -1,12 +1,11 @@
package utils
import (
"encoding/json"
"strings"
"github.com/go-logr/logr"
engineapi "github.com/kyverno/kyverno/pkg/engine/api"
jsonutils "github.com/kyverno/kyverno/pkg/utils/json"
"github.com/mattbaird/jsonpatch"
yamlv2 "gopkg.in/yaml.v2"
)
@ -33,9 +32,9 @@ var OperationToPastTense = map[string]string{
"test": "tested",
}
func GenerateAnnotationPatches(engineResponses []engineapi.EngineResponse, log logr.Logger) [][]byte {
func GenerateAnnotationPatches(engineResponses []engineapi.EngineResponse, log logr.Logger) []jsonpatch.JsonPatchOperation {
var annotations map[string]string
var patchBytes [][]byte
var patchBytes []jsonpatch.JsonPatchOperation
for _, er := range engineResponses {
if ann := er.PatchedResource.GetAnnotations(); ann != nil {
annotations = ann
@ -45,7 +44,7 @@ func GenerateAnnotationPatches(engineResponses []engineapi.EngineResponse, log l
if annotations == nil {
annotations = make(map[string]string)
}
var patchResponse jsonutils.PatchOperation
var patchResponse jsonpatch.JsonPatchOperation
value := annotationFromEngineResponses(engineResponses, log)
if value == nil {
// no patches or error while processing patches
@ -54,38 +53,47 @@ func GenerateAnnotationPatches(engineResponses []engineapi.EngineResponse, log l
if _, ok := annotations[strings.ReplaceAll(policyAnnotation, "~1", "/")]; ok {
// create update patch string
if _, ok := annotations["policies.kyverno.io/patches"]; ok {
patchResponse = jsonutils.NewPatchOperation("/metadata/annotations/"+oldAnnotation, "remove", nil)
patchResponse = jsonpatch.JsonPatchOperation{
Operation: "remove",
Path: "/metadata/annotations/" + oldAnnotation,
Value: nil,
}
delete(annotations, "policies.kyverno.io/patches")
patchByte, _ := json.Marshal(patchResponse)
patchBytes = append(patchBytes, patchByte)
patchBytes = append(patchBytes, patchResponse)
}
patchResponse = jsonutils.NewPatchOperation("/metadata/annotations/"+policyAnnotation, "replace", string(value))
patchByte, _ := json.Marshal(patchResponse)
patchBytes = append(patchBytes, patchByte)
patchResponse = jsonpatch.JsonPatchOperation{
Operation: "replace",
Path: "/metadata/annotations/" + policyAnnotation,
Value: string(value),
}
patchBytes = append(patchBytes, patchResponse)
} else {
// mutate rule has annotation patches
if len(annotations) > 0 {
if _, ok := annotations["policies.kyverno.io/patches"]; ok {
patchResponse = jsonutils.NewPatchOperation("/metadata/annotations/"+oldAnnotation, "remove", nil)
patchResponse = jsonpatch.JsonPatchOperation{
Operation: "remove",
Path: "/metadata/annotations/" + oldAnnotation,
Value: nil,
}
delete(annotations, "policies.kyverno.io/patches")
patchByte, _ := json.Marshal(patchResponse)
patchBytes = append(patchBytes, patchByte)
patchBytes = append(patchBytes, patchResponse)
}
patchResponse = jsonutils.NewPatchOperation("/metadata/annotations/"+policyAnnotation, "add", string(value))
patchByte, _ := json.Marshal(patchResponse)
patchBytes = append(patchBytes, patchByte)
patchResponse = jsonpatch.JsonPatchOperation{
Operation: "add",
Path: "/metadata/annotations/" + policyAnnotation,
Value: string(value),
}
patchBytes = append(patchBytes, patchResponse)
} else {
// insert 'policies.kyverno.patches' entry in annotation map
annotations[strings.ReplaceAll(policyAnnotation, "~1", "/")] = string(value)
patchResponse = jsonutils.NewPatchOperation("/metadata/annotations", "add", annotations)
patchByte, _ := json.Marshal(patchResponse)
patchBytes = append(patchBytes, patchByte)
}
}
for _, patchByte := range patchBytes {
err := jsonutils.CheckPatch(patchByte)
if err != nil {
log.Error(err, "failed to build JSON patch for annotation", "patch", string(patchByte))
patchResponse = jsonpatch.JsonPatchOperation{
Operation: "add",
Path: "/metadata/annotations",
Value: annotations,
}
patchBytes = append(patchBytes, patchResponse)
}
}
return patchBytes
@ -123,15 +131,10 @@ func annotationFromPolicyResponse(policyResponse engineapi.PolicyResponse, log l
var RulePatches []RulePatch
for _, ruleInfo := range policyResponse.Rules {
for _, patch := range ruleInfo.Patches() {
var patchmap map[string]interface{}
if err := json.Unmarshal(patch, &patchmap); err != nil {
log.Error(err, "Failed to parse JSON patch bytes")
continue
}
rp := RulePatch{
RuleName: ruleInfo.Name(),
Op: patchmap["op"].(string),
Path: patchmap["path"].(string),
Op: patch.Operation,
Path: patch.Path,
}
RulePatches = append(RulePatches, rp)
log.V(4).Info("annotation value prepared", "patches", RulePatches)

64
pkg/utils/annotations_test.go
View file

@ -6,17 +6,13 @@ import (
kyvernov1 "github.com/kyverno/kyverno/api/kyverno/v1"
engineapi "github.com/kyverno/kyverno/pkg/engine/api"
"github.com/kyverno/kyverno/pkg/logging"
"github.com/mattbaird/jsonpatch"
"gotest.tools/assert"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
)
func newPolicyResponse(rule string, patchesStr []string, status engineapi.RuleStatus) engineapi.PolicyResponse {
var patches [][]byte
for _, p := range patchesStr {
patches = append(patches, []byte(p))
}
func newPolicyResponse(rule string, patches []jsonpatch.JsonPatchOperation, status engineapi.RuleStatus) engineapi.PolicyResponse {
return engineapi.PolicyResponse{
Rules: []engineapi.RuleResponse{
*engineapi.NewRuleResponse(rule, engineapi.Mutation, "", status).WithPatches(patches...),
@ -24,7 +20,7 @@ func newPolicyResponse(rule string, patchesStr []string, status engineapi.RuleSt
}
}
func newEngineResponse(policy, rule string, patchesStr []string, status engineapi.RuleStatus, annotation map[string]interface{}) engineapi.EngineResponse {
func newEngineResponse(policy, rule string, patchesStr []jsonpatch.JsonPatchOperation, status engineapi.RuleStatus, annotation map[string]interface{}) engineapi.EngineResponse {
p := &kyvernov1.ClusterPolicy{
ObjectMeta: metav1.ObjectMeta{
Name: policy,
@ -43,33 +39,45 @@ func newEngineResponse(policy, rule string, patchesStr []string, status engineap
}
func Test_empty_annotation(t *testing.T) {
patchStr := `{ "op": "replace", "path": "/spec/containers/0/imagePullPolicy", "value": "IfNotPresent" }`
engineResponse := newEngineResponse("mutate-container", "default-imagepullpolicy", []string{patchStr}, engineapi.RuleStatusPass, nil)
patchStr := jsonpatch.JsonPatchOperation{
Operation: "replace",
Path: "/spec/containers/0/imagePullPolicy",
Value: "IfNotPresent",
}
engineResponse := newEngineResponse("mutate-container", "default-imagepullpolicy", []jsonpatch.JsonPatchOperation{patchStr}, engineapi.RuleStatusPass, nil)
annPatches := GenerateAnnotationPatches([]engineapi.EngineResponse{engineResponse}, logging.GlobalLogger())
expectedPatches := `{"path":"/metadata/annotations","op":"add","value":{"policies.kyverno.io/last-applied-patches":"default-imagepullpolicy.mutate-container.kyverno.io: replaced /spec/containers/0/imagePullPolicy\n"}}`
assert.Equal(t, string(annPatches[0]), expectedPatches)
expectedPatches := `{"op":"add","path":"/metadata/annotations","value":{"policies.kyverno.io/last-applied-patches":"default-imagepullpolicy.mutate-container.kyverno.io: replaced /spec/containers/0/imagePullPolicy\n"}}`
assert.Equal(t, annPatches[0].Json(), expectedPatches)
}
func Test_exist_annotation(t *testing.T) {
annotation := map[string]interface{}{
"test": "annotation",
}
patchStr := `{ "op": "replace", "path": "/spec/containers/0/imagePullPolicy", "value": "IfNotPresent" }`
engineResponse := newEngineResponse("mutate-container", "default-imagepullpolicy", []string{patchStr}, engineapi.RuleStatusPass, annotation)
patchStr := jsonpatch.JsonPatchOperation{
Operation: "replace",
Path: "/spec/containers/0/imagePullPolicy",
Value: "IfNotPresent",
}
engineResponse := newEngineResponse("mutate-container", "default-imagepullpolicy", []jsonpatch.JsonPatchOperation{patchStr}, engineapi.RuleStatusPass, annotation)
annPatches := GenerateAnnotationPatches([]engineapi.EngineResponse{engineResponse}, logging.GlobalLogger())
expectedPatches := `{"path":"/metadata/annotations/policies.kyverno.io~1last-applied-patches","op":"add","value":"default-imagepullpolicy.mutate-container.kyverno.io: replaced /spec/containers/0/imagePullPolicy\n"}`
assert.Equal(t, string(annPatches[0]), expectedPatches)
expectedPatches := `{"op":"add","path":"/metadata/annotations/policies.kyverno.io~1last-applied-patches","value":"default-imagepullpolicy.mutate-container.kyverno.io: replaced /spec/containers/0/imagePullPolicy\n"}`
assert.Equal(t, annPatches[0].Json(), expectedPatches)
}
func Test_exist_kyverno_annotation(t *testing.T) {
annotation := map[string]interface{}{
"policies.kyverno.patches": "old-annotation",
}
patchStr := `{ "op": "replace", "path": "/spec/containers/0/imagePullPolicy", "value": "IfNotPresent" }`
engineResponse := newEngineResponse("mutate-container", "default-imagepullpolicy", []string{patchStr}, engineapi.RuleStatusPass, annotation)
patchStr := jsonpatch.JsonPatchOperation{
Operation: "replace",
Path: "/spec/containers/0/imagePullPolicy",
Value: "IfNotPresent",
}
engineResponse := newEngineResponse("mutate-container", "default-imagepullpolicy", []jsonpatch.JsonPatchOperation{patchStr}, engineapi.RuleStatusPass, annotation)
annPatches := GenerateAnnotationPatches([]engineapi.EngineResponse{engineResponse}, logging.GlobalLogger())
expectedPatches := `{"path":"/metadata/annotations/policies.kyverno.io~1last-applied-patches","op":"add","value":"default-imagepullpolicy.mutate-container.kyverno.io: replaced /spec/containers/0/imagePullPolicy\n"}`
assert.Equal(t, string(annPatches[0]), expectedPatches)
expectedPatches := `{"op":"add","path":"/metadata/annotations/policies.kyverno.io~1last-applied-patches","value":"default-imagepullpolicy.mutate-container.kyverno.io: replaced /spec/containers/0/imagePullPolicy\n"}`
assert.Equal(t, annPatches[0].Json(), expectedPatches)
}
func Test_annotation_nil_patch(t *testing.T) {
@ -79,7 +87,7 @@ func Test_annotation_nil_patch(t *testing.T) {
engineResponse := newEngineResponse("mutate-container", "default-imagepullpolicy", nil, engineapi.RuleStatusPass, annotation)
annPatches := GenerateAnnotationPatches([]engineapi.EngineResponse{engineResponse}, logging.GlobalLogger())
assert.Assert(t, annPatches == nil)
engineResponseNew := newEngineResponse("mutate-container", "default-imagepullpolicy", []string{""}, engineapi.RuleStatusPass, annotation)
engineResponseNew := newEngineResponse("mutate-container", "default-imagepullpolicy", []jsonpatch.JsonPatchOperation{}, engineapi.RuleStatusPass, annotation)
annPatchesNew := GenerateAnnotationPatches([]engineapi.EngineResponse{engineResponseNew}, logging.GlobalLogger())
assert.Assert(t, annPatchesNew == nil)
}
@ -97,11 +105,15 @@ func Test_exist_patches(t *testing.T) {
annotation := map[string]interface{}{
"policies.kyverno.io/patches": "present",
}
patchStr := `{ "op": "replace", "path": "/spec/containers/0/imagePullPolicy", "value": "IfNotPresent" }`
engineResponse := newEngineResponse("mutate-container", "default-imagepullpolicy", []string{patchStr}, engineapi.RuleStatusPass, annotation)
patchStr := jsonpatch.JsonPatchOperation{
Operation: "replace",
Path: "/spec/containers/0/imagePullPolicy",
Value: "IfNotPresent",
}
engineResponse := newEngineResponse("mutate-container", "default-imagepullpolicy", []jsonpatch.JsonPatchOperation{patchStr}, engineapi.RuleStatusPass, annotation)
annPatches := GenerateAnnotationPatches([]engineapi.EngineResponse{engineResponse}, logging.GlobalLogger())
expectedPatches1 := `{"path":"/metadata/annotations/policies.kyverno.io~1patches","op":"remove"}`
expectedPatches2 := `{"path":"/metadata/annotations/policies.kyverno.io~1last-applied-patches","op":"add","value":"default-imagepullpolicy.mutate-container.kyverno.io: replaced /spec/containers/0/imagePullPolicy\n"}`
assert.Equal(t, string(annPatches[0]), expectedPatches1)
assert.Equal(t, string(annPatches[1]), expectedPatches2)
expectedPatches1 := `{"op":"remove","path":"/metadata/annotations/policies.kyverno.io~1patches"}`
expectedPatches2 := `{"op":"add","path":"/metadata/annotations/policies.kyverno.io~1last-applied-patches","value":"default-imagepullpolicy.mutate-container.kyverno.io: replaced /spec/containers/0/imagePullPolicy\n"}`
assert.Equal(t, annPatches[0].Json(), expectedPatches1)
assert.Equal(t, annPatches[1].Json(), expectedPatches2)
}

6
pkg/webhooks/resource/imageverification/handler.go
View file

@ -11,6 +11,7 @@ import (
"github.com/kyverno/kyverno/pkg/config"
"github.com/kyverno/kyverno/pkg/engine"
engineapi "github.com/kyverno/kyverno/pkg/engine/api"
"github.com/kyverno/kyverno/pkg/engine/mutate/patch"
"github.com/kyverno/kyverno/pkg/event"
"github.com/kyverno/kyverno/pkg/tracing"
admissionutils "github.com/kyverno/kyverno/pkg/utils/admission"
@ -18,6 +19,7 @@ import (
jsonutils "github.com/kyverno/kyverno/pkg/utils/json"
reportutils "github.com/kyverno/kyverno/pkg/utils/report"
webhookutils "github.com/kyverno/kyverno/pkg/webhooks/utils"
"github.com/mattbaird/jsonpatch"
"go.opentelemetry.io/otel/trace"
admissionv1 "k8s.io/api/admission/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@ -84,7 +86,7 @@ func (h *imageVerificationHandler) handleVerifyImages(
return true, "", nil, nil
}
var engineResponses []engineapi.EngineResponse
var patches [][]byte
var patches []jsonpatch.JsonPatchOperation
verifiedImageData := engineapi.ImageVerificationMetadata{}
failurePolicy := kyvernov1.Ignore
@ -137,7 +139,7 @@ func (h *imageVerificationHandler) handleVerifyImages(
go h.handleAudit(ctx, policyContext.NewResource(), request, nil, engineResponses...)
warnings := webhookutils.GetWarningMessages(engineResponses)
return true, "", jsonutils.JoinPatches(patches...), warnings
return true, "", jsonutils.JoinPatches(patch.ConvertPatches(patches...)...), warnings
}
func hasAnnotations(context *engine.PolicyContext) bool {

10
pkg/webhooks/resource/mutation/mutation.go
View file

@ -9,6 +9,7 @@ import (
kyvernov1 "github.com/kyverno/kyverno/api/kyverno/v1"
"github.com/kyverno/kyverno/pkg/engine"
engineapi "github.com/kyverno/kyverno/pkg/engine/api"
"github.com/kyverno/kyverno/pkg/engine/mutate/patch"
"github.com/kyverno/kyverno/pkg/event"
"github.com/kyverno/kyverno/pkg/metrics"
"github.com/kyverno/kyverno/pkg/openapi"
@ -17,6 +18,7 @@ import (
engineutils "github.com/kyverno/kyverno/pkg/utils/engine"
jsonutils "github.com/kyverno/kyverno/pkg/utils/json"
webhookutils "github.com/kyverno/kyverno/pkg/webhooks/utils"
"github.com/mattbaird/jsonpatch"
"go.opentelemetry.io/otel/trace"
admissionv1 "k8s.io/api/admission/v1"
corev1listers "k8s.io/client-go/listers/core/v1"
@ -83,7 +85,7 @@ func (v *mutationHandler) applyMutations(
return nil, nil, nil
}
var patches [][]byte
var patches []jsonpatch.JsonPatchOperation
var engineResponses []engineapi.EngineResponse
for _, policy := range policies {
@ -136,10 +138,10 @@ func (v *mutationHandler) applyMutations(
logMutationResponse(patches, engineResponses, v.log)
// patches holds all the successful patches, if no patch is created, it returns nil
return jsonutils.JoinPatches(patches...), engineResponses, nil
return jsonutils.JoinPatches(patch.ConvertPatches(patches...)...), engineResponses, nil
}
func (h *mutationHandler) applyMutation(ctx context.Context, request admissionv1.AdmissionRequest, policyContext *engine.PolicyContext) (*engineapi.EngineResponse, [][]byte, error) {
func (h *mutationHandler) applyMutation(ctx context.Context, request admissionv1.AdmissionRequest, policyContext *engine.PolicyContext) (*engineapi.EngineResponse, []jsonpatch.JsonPatchOperation, error) {
if request.Kind.Kind != "Namespace" && request.Namespace != "" {
policyContext = policyContext.WithNamespaceLabels(engineutils.GetNamespaceSelectorsFromNamespaceLister(request.Kind.Kind, request.Namespace, h.nsLister, h.log))
}
@ -161,7 +163,7 @@ func (h *mutationHandler) applyMutation(ctx context.Context, request admissionv1
return &engineResponse, policyPatches, nil
}
func logMutationResponse(patches [][]byte, engineResponses []engineapi.EngineResponse, logger logr.Logger) {
func logMutationResponse(patches []jsonpatch.JsonPatchOperation, engineResponses []engineapi.EngineResponse, logger logr.Logger) {
if len(patches) != 0 {
logger.V(4).Info("created patches", "count", len(patches))
}