From 8b1066be29af5c858ee1a967a364f7f86ae3d955 Mon Sep 17 00:00:00 2001 From: shivkumar dudhani Date: Mon, 19 Aug 2019 16:40:10 -0700 Subject: [PATCH 01/14] initial commit --- pkg/engine/generation.go | 16 ++++++++++++++++ pkg/engine/mutation.go | 18 +++++++++++++++++- pkg/engine/validation.go | 16 ++++++++++++++++ pkg/policy/status.go | 14 ++++++++++++++ pkg/policyviolation/controller.go | 2 +- 5 files changed, 64 insertions(+), 2 deletions(-) create mode 100644 pkg/policy/status.go diff --git a/pkg/engine/generation.go b/pkg/engine/generation.go index 5577cdc768..cddc891743 100644 --- a/pkg/engine/generation.go +++ b/pkg/engine/generation.go @@ -3,6 +3,7 @@ package engine import ( "encoding/json" "errors" + "time" "github.com/golang/glog" kyverno "github.com/nirmata/kyverno/pkg/api/kyverno/v1alpha1" @@ -15,6 +16,20 @@ import ( //Generate apply generation rules on a resource func Generate(client *client.Client, policy kyverno.Policy, ns unstructured.Unstructured) []info.RuleInfo { + var executionTime time.Duration + var rulesAppliedCount int + startTime := time.Now() + glog.V(4).Infof("started applying generation rules of policy %q (%v)", policy.Name, startTime) + defer func() { + executionTime = time.Since(startTime) + glog.V(4).Infof("Finished applying generation rules policy %q (%v)", policy.Name, executionTime) + glog.V(4).Infof("Generation Rules appplied succesfully count %q for policy %q", rulesAppliedCount, policy.Name) + }() + succesfulRuleCount := func() { + // rules applied succesfully count + rulesAppliedCount++ + } + ris := []info.RuleInfo{} for _, rule := range policy.Spec.Rules { if rule.Generation == (kyverno.Generation{}) { @@ -30,6 +45,7 @@ func Generate(client *client.Client, policy kyverno.Policy, ns unstructured.Unst } else { ri.Addf("Generation succesfully.", rule.Name) glog.Infof("succesfully applied policy %s rule %s on resource %s/%s/%s", policy.Name, rule.Name, ns.GetKind(), ns.GetNamespace(), ns.GetName()) + succesfulRuleCount() } ris = append(ris, ri) } diff --git a/pkg/engine/mutation.go b/pkg/engine/mutation.go index 392913ee61..a23caa706c 100644 --- a/pkg/engine/mutation.go +++ b/pkg/engine/mutation.go @@ -2,6 +2,7 @@ package engine import ( "reflect" + "time" "github.com/golang/glog" kyverno "github.com/nirmata/kyverno/pkg/api/kyverno/v1alpha1" @@ -12,6 +13,20 @@ import ( // Mutate performs mutation. Overlay first and then mutation patches //TODO: check if gvk needs to be passed or can be set in resource func Mutate(policy kyverno.Policy, resource unstructured.Unstructured) ([][]byte, []info.RuleInfo) { + var executionTime time.Duration + var rulesAppliedCount int + startTime := time.Now() + glog.V(4).Infof("started applying mutation rules of policy %q (%v)", policy.Name, startTime) + defer func() { + executionTime = time.Since(startTime) + glog.V(4).Infof("Finished applying mutation rules policy %q (%v)", policy.Name, executionTime) + glog.V(4).Infof("Mutation Rules appplied succesfully count %q for policy %q", rulesAppliedCount, policy.Name) + }() + succesfulRuleCount := func() { + // rules applied succesfully count + rulesAppliedCount++ + } + //TODO: convert rawResource to unstructured to avoid unmarhalling all the time for get some resource information var patches [][]byte var ruleInfos []info.RuleInfo @@ -46,12 +61,12 @@ func Mutate(policy kyverno.Policy, resource unstructured.Unstructured) ([][]byte glog.V(4).Infof("overlay applied succesfully on resource %s/%s", resource.GetNamespace(), resource.GetName()) ruleInfo.Add("Overlay succesfully applied") - // update rule information // strip slashes from string patch := JoinPatches(oPatches) ruleInfo.Changes = string(patch) patches = append(patches, oPatches...) + succesfulRuleCount() } else { glog.V(4).Infof("failed to apply overlay: %v", err) ruleInfo.Fail() @@ -72,6 +87,7 @@ func Mutate(policy kyverno.Policy, resource unstructured.Unstructured) ([][]byte glog.V(4).Infof("patches applied succesfully on resource %s/%s", resource.GetNamespace(), resource.GetName()) ruleInfo.Addf("Patches succesfully applied.") patches = append(patches, jsonPatches...) + succesfulRuleCount() } } ruleInfos = append(ruleInfos, ruleInfo) diff --git a/pkg/engine/validation.go b/pkg/engine/validation.go index 665c986feb..9e65c6e43d 100644 --- a/pkg/engine/validation.go +++ b/pkg/engine/validation.go @@ -8,6 +8,7 @@ import ( "reflect" "strconv" "strings" + "time" "github.com/golang/glog" kyverno "github.com/nirmata/kyverno/pkg/api/kyverno/v1alpha1" @@ -18,6 +19,20 @@ import ( // Validate handles validating admission request // Checks the target resources for rules defined in the policy func Validate(policy kyverno.Policy, resource unstructured.Unstructured) ([]info.RuleInfo, error) { + var executionTime time.Duration + var rulesAppliedCount int + startTime := time.Now() + glog.V(4).Infof("started applying validation rules of policy %q (%v)", policy.Name, startTime) + defer func() { + executionTime = time.Since(startTime) + glog.V(4).Infof("Finished applying validation rules policy %q (%v)", policy.Name, executionTime) + glog.V(4).Infof("Validation Rules appplied succesfully count %q for policy %q", rulesAppliedCount, policy.Name) + }() + succesfulRuleCount := func() { + // rules applied succesfully count + rulesAppliedCount++ + } + //TODO: convert rawResource to unstructured to avoid unmarhalling all the time for get some resource information //TODO: pass unstructured instead of rawResource ? @@ -57,6 +72,7 @@ func Validate(policy kyverno.Policy, resource unstructured.Unstructured) ([]info } else { ruleInfo.Add("Pattern succesfully validated") glog.V(4).Infof("pattern validated succesfully on resource %s/%s", resource.GetNamespace(), resource.GetName()) + succesfulRuleCount() } ruleInfos = append(ruleInfos, ruleInfo) } diff --git a/pkg/policy/status.go b/pkg/policy/status.go new file mode 100644 index 0000000000..7a49726b98 --- /dev/null +++ b/pkg/policy/status.go @@ -0,0 +1,14 @@ +package policy + +import "time" + +type PolicyStatus struct { + // average time required to process the policy rules on a resource + avgExecutionTime time.Duration + // Count of rules that were applied succesfully + rulesAppliedCount int + // Count of resources for whom update/create api requests were blocked as the resoruce did not satisfy the policy rules + resourcesBlockedCount int + // Count of the resource for whom the mutation rules were applied succesfully + resourcesMutatedCount int +} diff --git a/pkg/policyviolation/controller.go b/pkg/policyviolation/controller.go index 045f59b857..53c34b2cf8 100644 --- a/pkg/policyviolation/controller.go +++ b/pkg/policyviolation/controller.go @@ -238,13 +238,13 @@ func (pvc *PolicyViolationController) syncActiveResource(curPv *kyverno.PolicyVi return err } glog.V(4).Infof("removing policy violation %s as the corresponding resource %s/%s/%s does not exist anymore", curPv.Name, rspec.Kind, rspec.Namespace, rspec.Name) + return nil } if err != nil { glog.V(4).Infof("error while retrieved resource %s/%s/%s: %v", rspec.Kind, rspec.Namespace, rspec.Name, err) return err } //TODO- if the policy is not present, remove the policy violation - return nil } From 0b5cc03b2d0fb580901936a4ac2809d589a0a2b6 Mon Sep 17 00:00:00 2001 From: shivkumar dudhani Date: Mon, 19 Aug 2019 18:57:19 -0700 Subject: [PATCH 02/14] engineResponse to contain stats --- pkg/engine/generation.go | 14 ++++++++------ pkg/engine/mutation.go | 24 +++++++++++++----------- pkg/engine/utils.go | 11 +++++++++++ pkg/engine/validation.go | 19 +++++++++++-------- pkg/namespace/generation.go | 4 ++-- 5 files changed, 45 insertions(+), 27 deletions(-) diff --git a/pkg/engine/generation.go b/pkg/engine/generation.go index b90311822e..9578d39674 100644 --- a/pkg/engine/generation.go +++ b/pkg/engine/generation.go @@ -18,19 +18,20 @@ import ( ) //Generate apply generation rules on a resource -func Generate(client *client.Client, policy kyverno.Policy, ns unstructured.Unstructured) []info.RuleInfo { +func Generate(client *client.Client, policy kyverno.Policy, ns unstructured.Unstructured) EngineResponse { + var response EngineResponse var executionTime time.Duration - var rulesAppliedCount int startTime := time.Now() glog.V(4).Infof("started applying generation rules of policy %q (%v)", policy.Name, startTime) defer func() { executionTime = time.Since(startTime) - glog.V(4).Infof("Finished applying generation rules policy %q (%v)", policy.Name, executionTime) - glog.V(4).Infof("Generation Rules appplied succesfully count %q for policy %q", rulesAppliedCount, policy.Name) + response.ExecutionTime = time.Since(startTime) + glog.V(4).Infof("Finished applying generation rules policy %q (%v)", policy.Name, response.ExecutionTime) + glog.V(4).Infof("Mutation Rules appplied succesfully count %q for policy %q", response.RulesAppliedCount, policy.Name) }() succesfulRuleCount := func() { // rules applied succesfully count - rulesAppliedCount++ + response.RulesAppliedCount++ } ris := []info.RuleInfo{} @@ -52,7 +53,8 @@ func Generate(client *client.Client, policy kyverno.Policy, ns unstructured.Unst } ris = append(ris, ri) } - return ris + response.RuleInfos = ris + return response } func applyRuleGenerator(client *client.Client, ns unstructured.Unstructured, gen kyverno.Generation, policyCreationTime metav1.Time) error { diff --git a/pkg/engine/mutation.go b/pkg/engine/mutation.go index 7cade297b1..88ae7d7644 100644 --- a/pkg/engine/mutation.go +++ b/pkg/engine/mutation.go @@ -13,22 +13,23 @@ import ( // Mutate performs mutation. Overlay first and then mutation patches func Mutate(policy kyverno.Policy, resource unstructured.Unstructured) EngineResponse { + var response EngineResponse var allPatches, rulePatches [][]byte var err error var errs []error ris := []info.RuleInfo{} var executionTime time.Duration - var rulesAppliedCount int startTime := time.Now() glog.V(4).Infof("started applying mutation rules of policy %q (%v)", policy.Name, startTime) defer func() { executionTime = time.Since(startTime) - glog.V(4).Infof("Finished applying mutation rules policy %q (%v)", policy.Name, executionTime) - glog.V(4).Infof("Mutation Rules appplied succesfully count %q for policy %q", rulesAppliedCount, policy.Name) + response.ExecutionTime = time.Since(startTime) + glog.V(4).Infof("Finished applying mutation rules policy %q (%v)", policy.Name, response.ExecutionTime) + glog.V(4).Infof("Mutation Rules appplied succesfully count %q for policy %q", response.RulesAppliedCount, policy.Name) }() succesfulRuleCount := func() { // rules applied succesfully count - rulesAppliedCount++ + response.RulesAppliedCount++ } patchedDocument, err := resource.MarshalJSON() @@ -38,7 +39,8 @@ func Mutate(policy kyverno.Policy, resource unstructured.Unstructured) EngineRes if err != nil { glog.V(4).Infof("unable to marshal resource : %v", err) - return EngineResponse{PatchedResource: resource} + response.PatchedResource = resource + return response } for _, rule := range policy.Spec.Rules { @@ -114,12 +116,12 @@ func Mutate(policy kyverno.Policy, resource unstructured.Unstructured) EngineRes patchedResource, err := ConvertToUnstructured(patchedDocument) if err != nil { glog.Errorf("Failed to convert patched resource to unstructuredtype, err%v\n:", err) - return EngineResponse{PatchedResource: resource} + response.PatchedResource = resource + return response } - return EngineResponse{ - Patches: allPatches, - PatchedResource: *patchedResource, - RuleInfos: ris, - } + response.Patches = allPatches + response.PatchedResource = *patchedResource + response.RuleInfos = ris + return response } diff --git a/pkg/engine/utils.go b/pkg/engine/utils.go index 9096a74869..ff2f3f9745 100644 --- a/pkg/engine/utils.go +++ b/pkg/engine/utils.go @@ -5,6 +5,7 @@ import ( "fmt" "strconv" "strings" + "time" "github.com/golang/glog" @@ -18,10 +19,20 @@ import ( "k8s.io/apimachinery/pkg/labels" ) +//EngineResponse provides the response to the application of a policy rule set on a resource type EngineResponse struct { Patches [][]byte PatchedResource unstructured.Unstructured RuleInfos []info.RuleInfo + EngineStats +} + +//EngineStats stores in the statistics for a single application of resource +type EngineStats struct { + // average time required to process the policy rules on a resource + ExecutionTime time.Duration + // Count of rules that were applied succesfully + RulesAppliedCount int } // //ListResourcesThatApplyToPolicy returns list of resources that are filtered by policy rules diff --git a/pkg/engine/validation.go b/pkg/engine/validation.go index 42d917f998..9dcaca2ae1 100644 --- a/pkg/engine/validation.go +++ b/pkg/engine/validation.go @@ -19,29 +19,32 @@ import ( // Validate handles validating admission request // Checks the target resources for rules defined in the policy func Validate(policy kyverno.Policy, resource unstructured.Unstructured) EngineResponse { + var response EngineResponse var executionTime time.Duration - var rulesAppliedCount int startTime := time.Now() glog.V(4).Infof("started applying validation rules of policy %q (%v)", policy.Name, startTime) defer func() { executionTime = time.Since(startTime) - glog.V(4).Infof("Finished applying validation rules policy %q (%v)", policy.Name, executionTime) - glog.V(4).Infof("Validation Rules appplied succesfully count %q for policy %q", rulesAppliedCount, policy.Name) + response.ExecutionTime = time.Since(startTime) + glog.V(4).Infof("Finished applying mutation rules policy %q (%v)", policy.Name, response.ExecutionTime) + glog.V(4).Infof("Mutation Rules appplied succesfully count %q for policy %q", response.RulesAppliedCount, policy.Name) }() succesfulRuleCount := func() { // rules applied succesfully count - rulesAppliedCount++ + response.RulesAppliedCount++ } resourceRaw, err := resource.MarshalJSON() if err != nil { glog.V(4).Infof("Skip processing validating rule, unable to marshal resource : %v\n", err) - return EngineResponse{PatchedResource: resource} + response.PatchedResource = resource + return response } var resourceInt interface{} if err := json.Unmarshal(resourceRaw, &resourceInt); err != nil { glog.V(4).Infof("unable to unmarshal resource : %v\n", err) - return EngineResponse{PatchedResource: resource} + response.PatchedResource = resource + return response } var ruleInfos []info.RuleInfo @@ -73,8 +76,8 @@ func Validate(policy kyverno.Policy, resource unstructured.Unstructured) EngineR } ruleInfos = append(ruleInfos, ruleInfo) } - - return EngineResponse{RuleInfos: ruleInfos} + response.RuleInfos = ruleInfos + return response } // validateResourceWithPattern is a start of element-by-element validation process diff --git a/pkg/namespace/generation.go b/pkg/namespace/generation.go index 18a6a2908d..ca715e20ce 100644 --- a/pkg/namespace/generation.go +++ b/pkg/namespace/generation.go @@ -148,8 +148,8 @@ func applyPolicy(client *client.Client, resource unstructured.Unstructured, poli glog.V(4).Infof("Finished applying %s on resource %s/%s/%s (%v)", policy.Name, resource.GetKind(), resource.GetNamespace(), resource.GetName(), time.Since(startTime)) }() policyInfo := info.NewPolicyInfo(policy.Name, resource.GetKind(), resource.GetName(), resource.GetNamespace(), policy.Spec.ValidationFailureAction) - ruleInfos := engine.Generate(client, policy, resource) - policyInfo.AddRuleInfos(ruleInfos) + engineResponse := engine.Generate(client, policy, resource) + policyInfo.AddRuleInfos(engineResponse.RuleInfos) return policyInfo } From e507fb642292803c408aeddfe5f7c2419629ae1d Mon Sep 17 00:00:00 2001 From: shivkumar dudhani Date: Tue, 20 Aug 2019 12:51:25 -0700 Subject: [PATCH 03/14] recieve stats + update violation status move to aggregator --- main.go | 2 +- pkg/engine/generation.go | 6 +- pkg/engine/mutation.go | 16 +++--- pkg/engine/validation.go | 14 ++--- pkg/policy/controller.go | 18 +++--- pkg/policy/status.go | 109 ++++++++++++++++++++++++++++++++++++- pkg/webhooks/mutation.go | 26 +++++++++ pkg/webhooks/server.go | 7 ++- pkg/webhooks/validation.go | 21 +++++++ 9 files changed, 187 insertions(+), 32 deletions(-) diff --git a/main.go b/main.go index 6db33a255e..2c2a950354 100644 --- a/main.go +++ b/main.go @@ -110,7 +110,7 @@ func main() { if err = webhookRegistrationClient.Register(); err != nil { glog.Fatalf("Failed registering Admission Webhooks: %v\n", err) } - server, err := webhooks.NewWebhookServer(pclient, client, tlsPair, pInformer.Kyverno().V1alpha1().Policies(), pInformer.Kyverno().V1alpha1().PolicyViolations(), egen, webhookRegistrationClient, filterK8Resources) + server, err := webhooks.NewWebhookServer(pclient, client, tlsPair, pInformer.Kyverno().V1alpha1().Policies(), pInformer.Kyverno().V1alpha1().PolicyViolations(), egen, webhookRegistrationClient, pc.GetPolicyStatusAggregator(), filterK8Resources) if err != nil { glog.Fatalf("Unable to create webhook server: %v\n", err) } diff --git a/pkg/engine/generation.go b/pkg/engine/generation.go index 9578d39674..9b46aa755f 100644 --- a/pkg/engine/generation.go +++ b/pkg/engine/generation.go @@ -20,16 +20,14 @@ import ( //Generate apply generation rules on a resource func Generate(client *client.Client, policy kyverno.Policy, ns unstructured.Unstructured) EngineResponse { var response EngineResponse - var executionTime time.Duration startTime := time.Now() glog.V(4).Infof("started applying generation rules of policy %q (%v)", policy.Name, startTime) defer func() { - executionTime = time.Since(startTime) response.ExecutionTime = time.Since(startTime) glog.V(4).Infof("Finished applying generation rules policy %q (%v)", policy.Name, response.ExecutionTime) glog.V(4).Infof("Mutation Rules appplied succesfully count %q for policy %q", response.RulesAppliedCount, policy.Name) }() - succesfulRuleCount := func() { + incrementAppliedRuleCount := func() { // rules applied succesfully count response.RulesAppliedCount++ } @@ -49,9 +47,9 @@ func Generate(client *client.Client, policy kyverno.Policy, ns unstructured.Unst } else { ri.Addf("Generation succesfully.", rule.Name) glog.Infof("succesfully applied policy %s rule %s on resource %s/%s/%s", policy.Name, rule.Name, ns.GetKind(), ns.GetNamespace(), ns.GetName()) - succesfulRuleCount() } ris = append(ris, ri) + incrementAppliedRuleCount() } response.RuleInfos = ris return response diff --git a/pkg/engine/mutation.go b/pkg/engine/mutation.go index 88ae7d7644..fd453f4d16 100644 --- a/pkg/engine/mutation.go +++ b/pkg/engine/mutation.go @@ -12,22 +12,20 @@ import ( // Mutate performs mutation. Overlay first and then mutation patches -func Mutate(policy kyverno.Policy, resource unstructured.Unstructured) EngineResponse { - var response EngineResponse +func Mutate(policy kyverno.Policy, resource unstructured.Unstructured) (response EngineResponse) { + // var response EngineResponse var allPatches, rulePatches [][]byte var err error var errs []error ris := []info.RuleInfo{} - var executionTime time.Duration startTime := time.Now() glog.V(4).Infof("started applying mutation rules of policy %q (%v)", policy.Name, startTime) defer func() { - executionTime = time.Since(startTime) response.ExecutionTime = time.Since(startTime) - glog.V(4).Infof("Finished applying mutation rules policy %q (%v)", policy.Name, response.ExecutionTime) - glog.V(4).Infof("Mutation Rules appplied succesfully count %q for policy %q", response.RulesAppliedCount, policy.Name) + glog.V(4).Infof("finished applying mutation rules policy %v (%v)", policy.Name, response.ExecutionTime) + glog.V(4).Infof("Mutation Rules appplied succesfully count %v for policy %q", response.RulesAppliedCount, policy.Name) }() - succesfulRuleCount := func() { + incrementAppliedRuleCount := func() { // rules applied succesfully count response.RulesAppliedCount++ } @@ -78,12 +76,12 @@ func Mutate(policy kyverno.Policy, resource unstructured.Unstructured) EngineRes allPatches = append(allPatches, rulePatches...) glog.V(4).Infof("overlay applied succesfully on resource %s/%s", resource.GetNamespace(), resource.GetName()) - succesfulRuleCount() } else { glog.V(4).Infof("failed to apply overlay: %v", err) ruleInfo.Fail() ruleInfo.Addf("failed to apply overlay: %v", err) } + incrementAppliedRuleCount() } // Process Patches @@ -101,8 +99,8 @@ func Mutate(policy kyverno.Policy, resource unstructured.Unstructured) EngineRes ruleInfo.Patches = rulePatches allPatches = append(allPatches, rulePatches...) - succesfulRuleCount() } + incrementAppliedRuleCount() } patchedDocument, err = ApplyPatches(patchedDocument, rulePatches) diff --git a/pkg/engine/validation.go b/pkg/engine/validation.go index 9dcaca2ae1..f43c59c2c1 100644 --- a/pkg/engine/validation.go +++ b/pkg/engine/validation.go @@ -18,18 +18,16 @@ import ( // Validate handles validating admission request // Checks the target resources for rules defined in the policy -func Validate(policy kyverno.Policy, resource unstructured.Unstructured) EngineResponse { - var response EngineResponse - var executionTime time.Duration +func Validate(policy kyverno.Policy, resource unstructured.Unstructured) (response EngineResponse) { + // var response EngineResponse startTime := time.Now() glog.V(4).Infof("started applying validation rules of policy %q (%v)", policy.Name, startTime) defer func() { - executionTime = time.Since(startTime) response.ExecutionTime = time.Since(startTime) - glog.V(4).Infof("Finished applying mutation rules policy %q (%v)", policy.Name, response.ExecutionTime) - glog.V(4).Infof("Mutation Rules appplied succesfully count %q for policy %q", response.RulesAppliedCount, policy.Name) + glog.V(4).Infof("Finished applying validation rules policy %v (%v)", policy.Name, response.ExecutionTime) + glog.V(4).Infof("Validation Rules appplied succesfully count %v for policy %q", response.RulesAppliedCount, policy.Name) }() - succesfulRuleCount := func() { + incrementAppliedRuleCount := func() { // rules applied succesfully count response.RulesAppliedCount++ } @@ -72,8 +70,8 @@ func Validate(policy kyverno.Policy, resource unstructured.Unstructured) EngineR } else { ruleInfo.Add("Pattern succesfully validated") glog.V(4).Infof("pattern validated succesfully on resource %s/%s", resource.GetNamespace(), resource.GetName()) - succesfulRuleCount() } + incrementAppliedRuleCount() ruleInfos = append(ruleInfos, ruleInfo) } response.RuleInfos = ruleInfos diff --git a/pkg/policy/controller.go b/pkg/policy/controller.go index 142f28594f..b7bd93cef8 100644 --- a/pkg/policy/controller.go +++ b/pkg/policy/controller.go @@ -68,6 +68,8 @@ type PolicyController struct { rm resourceManager // filter the resources defined in the list filterK8Resources []utils.K8Resource + // recieves stats and aggregates details + statusAggregator *PolicyStatusAggregator } // NewPolicyController create a new PolicyController @@ -116,6 +118,9 @@ func NewPolicyController(kyvernoClient *kyvernoclient.Clientset, client *client. //TODO: pass the time in seconds instead of converting it internally pc.rm = NewResourceManager(30) + // aggregator + pc.statusAggregator = NewPolicyStatAggregator(kyvernoClient) + return &pc, nil } @@ -335,6 +340,9 @@ func (pc *PolicyController) Run(workers int, stopCh <-chan struct{}) { for i := 0; i < workers; i++ { go wait.Until(pc.worker, time.Second, stopCh) } + // policy status aggregator + //TODO: workers required for aggergation + pc.statusAggregator.Run(1, stopCh) <-stopCh } @@ -403,7 +411,8 @@ func (pc *PolicyController) syncPolicy(key string) error { policyInfos := pc.processExistingResources(*p) // report errors pc.report(policyInfos) - return pc.syncStatusOnly(p, pvList) + return pc.statusAggregator.UpdateViolationCount(p, pvList) + // return pc.syncStatusOnly(p, pvList) } //syncStatusOnly updates the policy status subresource @@ -422,13 +431,6 @@ func (pc *PolicyController) syncStatusOnly(p *kyverno.Policy, pvList []*kyverno. return err } -func calculateStatus(pvList []*kyverno.PolicyViolation) kyverno.PolicyStatus { - violationCount := len(pvList) - status := kyverno.PolicyStatus{ - Violations: violationCount, - } - return status -} func (pc *PolicyController) getPolicyViolationsForPolicy(p *kyverno.Policy) ([]*kyverno.PolicyViolation, error) { // List all PolicyViolation to find those we own but that no longer match our // selector. They will be orphaned by ClaimPolicyViolation(). diff --git a/pkg/policy/status.go b/pkg/policy/status.go index 7a49726b98..acc1ed3dbf 100644 --- a/pkg/policy/status.go +++ b/pkg/policy/status.go @@ -1,6 +1,15 @@ package policy -import "time" +import ( + "reflect" + "time" + + "github.com/golang/glog" + kyverno "github.com/nirmata/kyverno/pkg/api/kyverno/v1alpha1" + kyvernoclient "github.com/nirmata/kyverno/pkg/client/clientset/versioned" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/wait" +) type PolicyStatus struct { // average time required to process the policy rules on a resource @@ -12,3 +21,101 @@ type PolicyStatus struct { // Count of the resource for whom the mutation rules were applied succesfully resourcesMutatedCount int } + +type PolicyStatusAggregator struct { + // time since we start aggregating the stats + startTime time.Time + // channel to recieve stats + ch chan PolicyStat + // update polict status + psControl PStatusControlInterface +} + +//NewPolicyStatAggregator returns a new policy status +func NewPolicyStatAggregator(client *kyvernoclient.Clientset) *PolicyStatusAggregator { + psa := PolicyStatusAggregator{ + startTime: time.Now(), + ch: make(chan PolicyStat), + } + psa.psControl = PSControl{Client: client} + //TODO: add WaitGroup + return &psa +} + +func (psa *PolicyStatusAggregator) Run(workers int, stopCh <-chan struct{}) { + defer utilruntime.HandleCrash() + glog.V(4).Info("Started aggregator for policy status stats") + defer func() { + glog.V(4).Info("Shutting down aggregator for policy status stats") + }() + for i := 0; i < workers; i++ { + go wait.Until(psa.aggregate, time.Second, stopCh) + } +} +func (psa *PolicyStatusAggregator) aggregate() { + for r := range psa.ch { + glog.V(4).Infof("recieved policy stats %v", r) + } +} + +type PolicyStatusInterface interface { + SendStat(stat PolicyStat) + UpdateViolationCount(p *kyverno.Policy, pvList []*kyverno.PolicyViolation) error +} + +type PolicyStat struct { + PolicyName string + MutationExecutionTime time.Duration + ValidationExecutionTime time.Duration + RulesAppliedCount int + ResourceBlocked bool +} + +//SendStat sends the stat information for aggregation +func (psa *PolicyStatusAggregator) SendStat(stat PolicyStat) { + glog.V(4).Infof("sending policy stats: %v", stat) + // Send over channel + psa.ch <- stat +} + +//UpdateViolationCount updates the active violation count +func (psa *PolicyStatusAggregator) UpdateViolationCount(p *kyverno.Policy, pvList []*kyverno.PolicyViolation) error { + newStatus := calculateStatus(pvList) + if reflect.DeepEqual(newStatus, p.Status) { + // no update to status + return nil + } + // update status + newPolicy := p + newPolicy.Status = newStatus + + return psa.psControl.UpdatePolicyStatus(newPolicy) +} + +func calculateStatus(pvList []*kyverno.PolicyViolation) kyverno.PolicyStatus { + violationCount := len(pvList) + status := kyverno.PolicyStatus{ + Violations: violationCount, + } + return status +} + +//GetPolicyStatusAggregator returns interface to send policy status stats +func (pc *PolicyController) GetPolicyStatusAggregator() PolicyStatusInterface { + return pc.statusAggregator +} + +//PStatusControlInterface Provides interface to operate on policy status +type PStatusControlInterface interface { + UpdatePolicyStatus(newPolicy *kyverno.Policy) error +} + +type PSControl struct { + Client kyvernoclient.Interface +} + +//UpdatePolicyStatus update policy status +func (c PSControl) UpdatePolicyStatus(newPolicy *kyverno.Policy) error { + _, err := c.Client.KyvernoV1alpha1().Policies().UpdateStatus(newPolicy) + return err +} diff --git a/pkg/webhooks/mutation.go b/pkg/webhooks/mutation.go index d1e9247af6..f706020627 100644 --- a/pkg/webhooks/mutation.go +++ b/pkg/webhooks/mutation.go @@ -4,6 +4,7 @@ import ( "github.com/golang/glog" engine "github.com/nirmata/kyverno/pkg/engine" "github.com/nirmata/kyverno/pkg/info" + policyctr "github.com/nirmata/kyverno/pkg/policy" "github.com/nirmata/kyverno/pkg/utils" v1beta1 "k8s.io/api/admission/v1beta1" "k8s.io/apimachinery/pkg/labels" @@ -14,6 +15,23 @@ import ( func (ws *WebhookServer) HandleMutation(request *v1beta1.AdmissionRequest) (bool, engine.EngineResponse) { var patches [][]byte var policyInfos []info.PolicyInfo + var policyStats []policyctr.PolicyStat + // gather stats from the engine response + gatherStat := func(policyName string, er engine.EngineResponse) { + ps := policyctr.PolicyStat{} + ps.PolicyName = policyName + ps.MutationExecutionTime = er.ExecutionTime + ps.RulesAppliedCount = er.RulesAppliedCount + policyStats = append(policyStats, ps) + } + // send stats for aggregation + sendStat := func(blocked bool) { + for _, stat := range policyStats { + stat.ResourceBlocked = blocked + //SEND + ws.policyStatus.SendStat(stat) + } + } glog.V(5).Infof("Receive request in mutating webhook: Kind=%s, Namespace=%s Name=%s UID=%s patchOperation=%s", request.Kind.Kind, request.Namespace, request.Name, request.UID, request.Operation) @@ -54,6 +72,10 @@ func (ws *WebhookServer) HandleMutation(request *v1beta1.AdmissionRequest) (bool engineResponse = engine.Mutate(*policy, *resource) policyInfo.AddRuleInfos(engineResponse.RuleInfos) + // Gather policy application statistics + gatherStat(policy.Name, engineResponse) + + // ps := policyctr.NewPolicyStat(policy.Name, engineResponse.ExecutionTime, nil, engineResponse.RulesAppliedCount) if !policyInfo.IsSuccessful() { glog.V(4).Infof("Failed to apply policy %s on resource %s/%s\n", policy.Name, resource.GetNamespace(), resource.GetName()) @@ -67,6 +89,7 @@ func (ws *WebhookServer) HandleMutation(request *v1beta1.AdmissionRequest) (bool patches = append(patches, engineResponse.Patches...) policyInfos = append(policyInfos, policyInfo) glog.V(4).Infof("Mutation from policy %s has applied succesfully to %s %s/%s", policy.Name, request.Kind.Kind, resource.GetNamespace(), resource.GetName()) + } // ADD ANNOTATIONS @@ -80,11 +103,14 @@ func (ws *WebhookServer) HandleMutation(request *v1beta1.AdmissionRequest) (bool } ok, msg := isAdmSuccesful(policyInfos) + // Send policy engine Stats if ok { + sendStat(false) engineResponse.Patches = patches return true, engineResponse } + sendStat(true) glog.Errorf("Failed to mutate the resource: %s\n", msg) return false, engineResponse } diff --git a/pkg/webhooks/server.go b/pkg/webhooks/server.go index 2bfb048c45..af6ae8a880 100644 --- a/pkg/webhooks/server.go +++ b/pkg/webhooks/server.go @@ -19,6 +19,7 @@ import ( "github.com/nirmata/kyverno/pkg/config" client "github.com/nirmata/kyverno/pkg/dclient" "github.com/nirmata/kyverno/pkg/event" + "github.com/nirmata/kyverno/pkg/policy" tlsutils "github.com/nirmata/kyverno/pkg/tls" "github.com/nirmata/kyverno/pkg/utils" v1beta1 "k8s.io/api/admission/v1beta1" @@ -37,7 +38,9 @@ type WebhookServer struct { pvListerSynced cache.InformerSynced eventGen event.Interface webhookRegistrationClient *WebhookRegistrationClient - filterK8Resources []utils.K8Resource + // API to send policy stats for aggregation + policyStatus policy.PolicyStatusInterface + filterK8Resources []utils.K8Resource } // NewWebhookServer creates new instance of WebhookServer accordingly to given configuration @@ -50,6 +53,7 @@ func NewWebhookServer( pvInormer kyvernoinformer.PolicyViolationInformer, eventGen event.Interface, webhookRegistrationClient *WebhookRegistrationClient, + policyStatus policy.PolicyStatusInterface, filterK8Resources string) (*WebhookServer, error) { if tlsPair == nil { @@ -73,6 +77,7 @@ func NewWebhookServer( pvListerSynced: pInformer.Informer().HasSynced, eventGen: eventGen, webhookRegistrationClient: webhookRegistrationClient, + policyStatus: policyStatus, filterK8Resources: utils.ParseKinds(filterK8Resources), } mux := http.NewServeMux() diff --git a/pkg/webhooks/validation.go b/pkg/webhooks/validation.go index 6e2953b324..088694bd67 100644 --- a/pkg/webhooks/validation.go +++ b/pkg/webhooks/validation.go @@ -4,6 +4,7 @@ import ( "github.com/golang/glog" engine "github.com/nirmata/kyverno/pkg/engine" "github.com/nirmata/kyverno/pkg/info" + policyctr "github.com/nirmata/kyverno/pkg/policy" "github.com/nirmata/kyverno/pkg/policyviolation" "github.com/nirmata/kyverno/pkg/utils" v1beta1 "k8s.io/api/admission/v1beta1" @@ -17,6 +18,23 @@ import ( // If there are no errors in validating rule we apply generation rules func (ws *WebhookServer) HandleValidation(request *v1beta1.AdmissionRequest, resource unstructured.Unstructured) *v1beta1.AdmissionResponse { var policyInfos []info.PolicyInfo + var policyStats []policyctr.PolicyStat + // gather stats from the engine response + gatherStat := func(policyName string, er engine.EngineResponse) { + ps := policyctr.PolicyStat{} + ps.PolicyName = policyName + ps.ValidationExecutionTime = er.ExecutionTime + ps.RulesAppliedCount = er.RulesAppliedCount + policyStats = append(policyStats, ps) + } + // send stats for aggregation + sendStat := func(blocked bool) { + for _, stat := range policyStats { + stat.ResourceBlocked = blocked + //SEND + ws.policyStatus.SendStat(stat) + } + } glog.V(5).Infof("Receive request in validating webhook: Kind=%s, Namespace=%s Name=%s UID=%s patchOperation=%s", request.Kind.Kind, request.Namespace, request.Name, request.UID, request.Operation) @@ -55,6 +73,7 @@ func (ws *WebhookServer) HandleValidation(request *v1beta1.AdmissionRequest, res if len(engineResponse.RuleInfos) == 0 { continue } + gatherStat(policy.Name, engineResponse) if len(engineResponse.RuleInfos) > 0 { glog.V(4).Infof("Validation from policy %s has applied succesfully to %s %s/%s", policy.Name, request.Kind.Kind, resource.GetNamespace(), resource.GetName()) @@ -87,6 +106,7 @@ func (ws *WebhookServer) HandleValidation(request *v1beta1.AdmissionRequest, res // Even if one the policy being applied ok, msg := isAdmSuccesful(policyInfos) if !ok && toBlock(policyInfos) { + sendStat(true) return &v1beta1.AdmissionResponse{ Allowed: false, Result: &metav1.Status{ @@ -98,6 +118,7 @@ func (ws *WebhookServer) HandleValidation(request *v1beta1.AdmissionRequest, res // ADD POLICY VIOLATIONS policyviolation.GeneratePolicyViolations(ws.pvListerSynced, ws.pvLister, ws.kyvernoClient, policyInfos) + sendStat(false) return &v1beta1.AdmissionResponse{ Allowed: true, } From bcad9ada2df1a6c98991ca51eb03265a408bfd9e Mon Sep 17 00:00:00 2001 From: shivkumar dudhani Date: Tue, 20 Aug 2019 13:35:03 -0700 Subject: [PATCH 04/14] introduce locking for policy status updates --- pkg/policy/controller.go | 5 +-- pkg/policy/status.go | 66 +++++++++++++++++++++++++++++++++++++--- 2 files changed, 64 insertions(+), 7 deletions(-) diff --git a/pkg/policy/controller.go b/pkg/policy/controller.go index b7bd93cef8..0457423250 100644 --- a/pkg/policy/controller.go +++ b/pkg/policy/controller.go @@ -119,7 +119,7 @@ func NewPolicyController(kyvernoClient *kyvernoclient.Clientset, client *client. pc.rm = NewResourceManager(30) // aggregator - pc.statusAggregator = NewPolicyStatAggregator(kyvernoClient) + pc.statusAggregator = NewPolicyStatAggregator(kyvernoClient, pInformer.Lister()) return &pc, nil } @@ -411,7 +411,8 @@ func (pc *PolicyController) syncPolicy(key string) error { policyInfos := pc.processExistingResources(*p) // report errors pc.report(policyInfos) - return pc.statusAggregator.UpdateViolationCount(p, pvList) + // fetch the policy again via the aggreagator to remain consistent + return pc.statusAggregator.UpdateViolationCount(p.Name, pvList) // return pc.syncStatusOnly(p, pvList) } diff --git a/pkg/policy/status.go b/pkg/policy/status.go index acc1ed3dbf..fe8a77512a 100644 --- a/pkg/policy/status.go +++ b/pkg/policy/status.go @@ -2,11 +2,13 @@ package policy import ( "reflect" + "sync" "time" "github.com/golang/glog" kyverno "github.com/nirmata/kyverno/pkg/api/kyverno/v1alpha1" kyvernoclient "github.com/nirmata/kyverno/pkg/client/clientset/versioned" + kyvernolister "github.com/nirmata/kyverno/pkg/client/listers/kyverno/v1alpha1" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" ) @@ -29,14 +31,20 @@ type PolicyStatusAggregator struct { ch chan PolicyStat // update polict status psControl PStatusControlInterface + // pLister can list/get policy from the shared informer's store + pLister kyvernolister.PolicyLister + // UpdateViolationCount and SendStat can update same policy status + // we need to sync the updates using policyName + policyUpdateData sync.Map } //NewPolicyStatAggregator returns a new policy status -func NewPolicyStatAggregator(client *kyvernoclient.Clientset) *PolicyStatusAggregator { +func NewPolicyStatAggregator(client *kyvernoclient.Clientset, pLister kyvernolister.PolicyLister) *PolicyStatusAggregator { psa := PolicyStatusAggregator{ startTime: time.Now(), ch: make(chan PolicyStat), } + psa.pLister = pLister psa.psControl = PSControl{Client: client} //TODO: add WaitGroup return &psa @@ -53,14 +61,46 @@ func (psa *PolicyStatusAggregator) Run(workers int, stopCh <-chan struct{}) { } } func (psa *PolicyStatusAggregator) aggregate() { + // As mutation and validation are handled seperately + // ideally we need to combine the exection time from both for a policy + // but its tricky to detect here the type of rules policy contains + // so we dont combine the results, but instead compute the execution time for + // mutation & validation rules seperately for r := range psa.ch { glog.V(4).Infof("recieved policy stats %v", r) + if err := psa.updateStats(r); err != nil { + glog.Info("Failed to update stats for policy %s: %v", r.PolicyName, err) + } } + +} + +func (psa *PolicyStatusAggregator) updateStats(stats PolicyStat) error { + func() { + glog.V(4).Infof("lock updates for policy name %s", stats.PolicyName) + // Lock the update for policy + psa.policyUpdateData.Store(stats.PolicyName, struct{}{}) + }() + defer func() { + glog.V(4).Infof("Unlock updates for policy name %s", stats.PolicyName) + psa.policyUpdateData.Delete(stats.PolicyName) + }() + // get policy + policy, err := psa.pLister.Get(stats.PolicyName) + if err != nil { + glog.V(4).Infof("failed to get policy %s. Unable to update violation count: %v", stats.PolicyName, err) + return err + } + glog.V(4).Infof("updating stats for policy %s", policy.Name) + // update the statistics + // policy.Status + + return nil } type PolicyStatusInterface interface { SendStat(stat PolicyStat) - UpdateViolationCount(p *kyverno.Policy, pvList []*kyverno.PolicyViolation) error + UpdateViolationCount(policyName string, pvList []*kyverno.PolicyViolation) error } type PolicyStat struct { @@ -79,14 +119,30 @@ func (psa *PolicyStatusAggregator) SendStat(stat PolicyStat) { } //UpdateViolationCount updates the active violation count -func (psa *PolicyStatusAggregator) UpdateViolationCount(p *kyverno.Policy, pvList []*kyverno.PolicyViolation) error { +func (psa *PolicyStatusAggregator) UpdateViolationCount(policyName string, pvList []*kyverno.PolicyViolation) error { + func() { + glog.V(4).Infof("lock updates for policy name %s", policyName) + // Lock the update for policy + psa.policyUpdateData.Store(policyName, struct{}{}) + }() + defer func() { + glog.V(4).Infof("Unlock updates for policy name %s", policyName) + psa.policyUpdateData.Delete(policyName) + }() + // get policy + policy, err := psa.pLister.Get(policyName) + if err != nil { + glog.V(4).Infof("failed to get policy %s. Unable to update violation count: %v", policyName, err) + return err + } + newStatus := calculateStatus(pvList) - if reflect.DeepEqual(newStatus, p.Status) { + if reflect.DeepEqual(newStatus, policy.Status) { // no update to status return nil } // update status - newPolicy := p + newPolicy := policy newPolicy.Status = newStatus return psa.psControl.UpdatePolicyStatus(newPolicy) From 3f876e6f4681bf7b906834781685ac5371c99160 Mon Sep 17 00:00:00 2001 From: shivkumar dudhani Date: Tue, 20 Aug 2019 15:13:52 -0700 Subject: [PATCH 05/14] update status v1 --- pkg/api/kyverno/v1alpha1/types.go | 12 ++++- pkg/policy/controller.go | 2 +- pkg/policy/status.go | 83 ++++++++++++++++++++++--------- 3 files changed, 72 insertions(+), 25 deletions(-) diff --git a/pkg/api/kyverno/v1alpha1/types.go b/pkg/api/kyverno/v1alpha1/types.go index 5597f97e23..691a0c1f11 100644 --- a/pkg/api/kyverno/v1alpha1/types.go +++ b/pkg/api/kyverno/v1alpha1/types.go @@ -1,6 +1,8 @@ package v1alpha1 import ( + "time" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -89,7 +91,15 @@ type CloneFrom struct { //PolicyStatus provides status for violations type PolicyStatus struct { - Violations int `json:"violations"` + ViolationCount int `json:"violationCount"` + // Count of rules that were applied + RulesAppliedCount int `json:"rulesAppliedCount"` + // Count of resources for whom update/create api requests were blocked as the resoruce did not satisfy the policy rules + ResourcesBlockedCount int `json:"resourcesBlockedCount"` + // average time required to process the policy Mutation rules on a resource + AvgExecutionTimeMutation time.Duration `json:"averageMutationExecutionTime"` + // average time required to process the policy Validation rules on a resource + AvgExecutionTimeValidation time.Duration `json:"averageValidationExecutionTime"` } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object diff --git a/pkg/policy/controller.go b/pkg/policy/controller.go index 0457423250..02cc1c072e 100644 --- a/pkg/policy/controller.go +++ b/pkg/policy/controller.go @@ -119,7 +119,7 @@ func NewPolicyController(kyvernoClient *kyvernoclient.Clientset, client *client. pc.rm = NewResourceManager(30) // aggregator - pc.statusAggregator = NewPolicyStatAggregator(kyvernoClient, pInformer.Lister()) + pc.statusAggregator = NewPolicyStatAggregator(kyvernoClient, pInformer) return &pc, nil } diff --git a/pkg/policy/status.go b/pkg/policy/status.go index fe8a77512a..f19961dba5 100644 --- a/pkg/policy/status.go +++ b/pkg/policy/status.go @@ -1,6 +1,7 @@ package policy import ( + "fmt" "reflect" "sync" "time" @@ -8,48 +9,55 @@ import ( "github.com/golang/glog" kyverno "github.com/nirmata/kyverno/pkg/api/kyverno/v1alpha1" kyvernoclient "github.com/nirmata/kyverno/pkg/client/clientset/versioned" + kyvernoinformer "github.com/nirmata/kyverno/pkg/client/informers/externalversions/kyverno/v1alpha1" kyvernolister "github.com/nirmata/kyverno/pkg/client/listers/kyverno/v1alpha1" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/tools/cache" ) -type PolicyStatus struct { - // average time required to process the policy rules on a resource - avgExecutionTime time.Duration - // Count of rules that were applied succesfully - rulesAppliedCount int - // Count of resources for whom update/create api requests were blocked as the resoruce did not satisfy the policy rules - resourcesBlockedCount int - // Count of the resource for whom the mutation rules were applied succesfully - resourcesMutatedCount int -} +// type PolicyStatus struct { +// // average time required to process the policy rules on a resource +// avgExecutionTime time.Duration +// // Count of rules that were applied succesfully +// rulesAppliedCount int +// // Count of resources for whom update/create api requests were blocked as the resoruce did not satisfy the policy rules +// resourcesBlockedCount int +// // Count of the resource for whom the mutation rules were applied succesfully +// resourcesMutatedCount int +// } +//PolicyStatusAggregator stores information abt aggregation type PolicyStatusAggregator struct { // time since we start aggregating the stats startTime time.Time // channel to recieve stats ch chan PolicyStat - // update polict status + // update policy status psControl PStatusControlInterface // pLister can list/get policy from the shared informer's store pLister kyvernolister.PolicyLister + // pListerSynced returns true if the Policy store has been synced at least once + pListerSynced cache.InformerSynced // UpdateViolationCount and SendStat can update same policy status // we need to sync the updates using policyName policyUpdateData sync.Map } //NewPolicyStatAggregator returns a new policy status -func NewPolicyStatAggregator(client *kyvernoclient.Clientset, pLister kyvernolister.PolicyLister) *PolicyStatusAggregator { +func NewPolicyStatAggregator(client *kyvernoclient.Clientset, pInformer kyvernoinformer.PolicyInformer) *PolicyStatusAggregator { psa := PolicyStatusAggregator{ startTime: time.Now(), ch: make(chan PolicyStat), } - psa.pLister = pLister + psa.pLister = pInformer.Lister() + psa.pListerSynced = pInformer.Informer().HasSynced psa.psControl = PSControl{Client: client} //TODO: add WaitGroup return &psa } +//Run begins aggregator func (psa *PolicyStatusAggregator) Run(workers int, stopCh <-chan struct{}) { defer utilruntime.HandleCrash() glog.V(4).Info("Started aggregator for policy status stats") @@ -69,7 +77,7 @@ func (psa *PolicyStatusAggregator) aggregate() { for r := range psa.ch { glog.V(4).Infof("recieved policy stats %v", r) if err := psa.updateStats(r); err != nil { - glog.Info("Failed to update stats for policy %s: %v", r.PolicyName, err) + glog.Infof("Failed to update stats for policy %s: %v", r.PolicyName, err) } } @@ -77,32 +85,59 @@ func (psa *PolicyStatusAggregator) aggregate() { func (psa *PolicyStatusAggregator) updateStats(stats PolicyStat) error { func() { - glog.V(4).Infof("lock updates for policy name %s", stats.PolicyName) + glog.V(4).Infof("lock updates for policy %s", stats.PolicyName) // Lock the update for policy psa.policyUpdateData.Store(stats.PolicyName, struct{}{}) }() defer func() { - glog.V(4).Infof("Unlock updates for policy name %s", stats.PolicyName) + glog.V(4).Infof("Unlock updates for policy %s", stats.PolicyName) psa.policyUpdateData.Delete(stats.PolicyName) }() + + // //wait for cache sync + // if !cache.WaitForCacheSync(nil, psa.pListerSynced) { + // glog.Infof("unable to sync cache for policy informer") + // return nil + // } // get policy policy, err := psa.pLister.Get(stats.PolicyName) if err != nil { glog.V(4).Infof("failed to get policy %s. Unable to update violation count: %v", stats.PolicyName, err) return err } + newpolicy := policy + fmt.Println(newpolicy.ResourceVersion) + newpolicy.Status = kyverno.PolicyStatus{} glog.V(4).Infof("updating stats for policy %s", policy.Name) - // update the statistics - // policy.Status - - return nil + // rules applied count + newpolicy.Status.RulesAppliedCount = newpolicy.Status.RulesAppliedCount + stats.RulesAppliedCount + // resource blocked count + if stats.ResourceBlocked { + policy.Status.ResourcesBlockedCount++ + } + var zeroDuration time.Duration + if newpolicy.Status.AvgExecutionTimeMutation != zeroDuration { + // avg execution time for mutation rules + newpolicy.Status.AvgExecutionTimeMutation = (newpolicy.Status.AvgExecutionTimeMutation + stats.MutationExecutionTime) / 2 + } else { + newpolicy.Status.AvgExecutionTimeMutation = stats.MutationExecutionTime + } + if policy.Status.AvgExecutionTimeValidation != zeroDuration { + // avg execution time for validation rules + newpolicy.Status.AvgExecutionTimeValidation = (newpolicy.Status.AvgExecutionTimeValidation + stats.ValidationExecutionTime) / 2 + } else { + newpolicy.Status.AvgExecutionTimeValidation = stats.ValidationExecutionTime + } + return psa.psControl.UpdatePolicyStatus(newpolicy) } +//PolicyStatusInterface provides methods to modify policyStatus type PolicyStatusInterface interface { SendStat(stat PolicyStat) UpdateViolationCount(policyName string, pvList []*kyverno.PolicyViolation) error } +//PolicyStat stored stats for policy type PolicyStat struct { PolicyName string MutationExecutionTime time.Duration @@ -121,12 +156,12 @@ func (psa *PolicyStatusAggregator) SendStat(stat PolicyStat) { //UpdateViolationCount updates the active violation count func (psa *PolicyStatusAggregator) UpdateViolationCount(policyName string, pvList []*kyverno.PolicyViolation) error { func() { - glog.V(4).Infof("lock updates for policy name %s", policyName) + glog.V(4).Infof("lock updates for policy %s", policyName) // Lock the update for policy psa.policyUpdateData.Store(policyName, struct{}{}) }() defer func() { - glog.V(4).Infof("Unlock updates for policy name %s", policyName) + glog.V(4).Infof("Unlock updates for policy %s", policyName) psa.policyUpdateData.Delete(policyName) }() // get policy @@ -139,6 +174,7 @@ func (psa *PolicyStatusAggregator) UpdateViolationCount(policyName string, pvLis newStatus := calculateStatus(pvList) if reflect.DeepEqual(newStatus, policy.Status) { // no update to status + glog.V(4).Infof("no changes in policy violation count for policy %s", policy.Name) return nil } // update status @@ -151,7 +187,7 @@ func (psa *PolicyStatusAggregator) UpdateViolationCount(policyName string, pvLis func calculateStatus(pvList []*kyverno.PolicyViolation) kyverno.PolicyStatus { violationCount := len(pvList) status := kyverno.PolicyStatus{ - Violations: violationCount, + ViolationCount: violationCount, } return status } @@ -166,6 +202,7 @@ type PStatusControlInterface interface { UpdatePolicyStatus(newPolicy *kyverno.Policy) error } +//PSControl allows update for policy status type PSControl struct { Client kyvernoclient.Interface } From dc47132ade87df1f713da32726659123983abf53 Mon Sep 17 00:00:00 2001 From: shivkumar dudhani Date: Tue, 20 Aug 2019 16:40:20 -0700 Subject: [PATCH 06/14] update policy status --- pkg/api/kyverno/v1alpha1/types.go | 6 +- pkg/policy/controller.go | 26 +++- pkg/policy/status.go | 211 +++++++++++------------------- pkg/utils/util.go | 8 ++ pkg/webhooks/mutation.go | 6 +- pkg/webhooks/validation.go | 6 +- 6 files changed, 117 insertions(+), 146 deletions(-) diff --git a/pkg/api/kyverno/v1alpha1/types.go b/pkg/api/kyverno/v1alpha1/types.go index 691a0c1f11..44769bd446 100644 --- a/pkg/api/kyverno/v1alpha1/types.go +++ b/pkg/api/kyverno/v1alpha1/types.go @@ -1,8 +1,6 @@ package v1alpha1 import ( - "time" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -97,9 +95,9 @@ type PolicyStatus struct { // Count of resources for whom update/create api requests were blocked as the resoruce did not satisfy the policy rules ResourcesBlockedCount int `json:"resourcesBlockedCount"` // average time required to process the policy Mutation rules on a resource - AvgExecutionTimeMutation time.Duration `json:"averageMutationExecutionTime"` + AvgExecutionTimeMutation string `json:"averageMutationExecutionTime"` // average time required to process the policy Validation rules on a resource - AvgExecutionTimeValidation time.Duration `json:"averageValidationExecutionTime"` + AvgExecutionTimeValidation string `json:"averageValidationExecutionTime"` } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object diff --git a/pkg/policy/controller.go b/pkg/policy/controller.go index 02cc1c072e..fccbed13c5 100644 --- a/pkg/policy/controller.go +++ b/pkg/policy/controller.go @@ -119,7 +119,8 @@ func NewPolicyController(kyvernoClient *kyvernoclient.Clientset, client *client. pc.rm = NewResourceManager(30) // aggregator - pc.statusAggregator = NewPolicyStatAggregator(kyvernoClient, pInformer) + // pc.statusAggregator = NewPolicyStatAggregator(kyvernoClient, pInformer) + pc.statusAggregator = NewPolicyStatAggregator(kyvernoClient) return &pc, nil } @@ -392,6 +393,8 @@ func (pc *PolicyController) syncPolicy(key string) error { policy, err := pc.pLister.Get(key) if errors.IsNotFound(err) { glog.V(2).Infof("Policy %v has been deleted", key) + // remove the recorded stats for the policy + pc.statusAggregator.RemovePolicyStats(key) return nil } @@ -412,15 +415,15 @@ func (pc *PolicyController) syncPolicy(key string) error { // report errors pc.report(policyInfos) // fetch the policy again via the aggreagator to remain consistent - return pc.statusAggregator.UpdateViolationCount(p.Name, pvList) - // return pc.syncStatusOnly(p, pvList) + // return pc.statusAggregator.UpdateViolationCount(p.Name, pvList) + return pc.syncStatusOnly(p, pvList) } //syncStatusOnly updates the policy status subresource // status: // - violations : (count of the resources that violate this policy ) func (pc *PolicyController) syncStatusOnly(p *kyverno.Policy, pvList []*kyverno.PolicyViolation) error { - newStatus := calculateStatus(pvList) + newStatus := pc.calculateStatus(p.Name, pvList) if reflect.DeepEqual(newStatus, p.Status) { // no update to status return nil @@ -432,6 +435,21 @@ func (pc *PolicyController) syncStatusOnly(p *kyverno.Policy, pvList []*kyverno. return err } +func (pc *PolicyController) calculateStatus(policyName string, pvList []*kyverno.PolicyViolation) kyverno.PolicyStatus { + violationCount := len(pvList) + status := kyverno.PolicyStatus{ + ViolationCount: violationCount, + } + // get stats + stats := pc.statusAggregator.GetPolicyStats(policyName) + if stats != (PolicyStatInfo{}) { + status.RulesAppliedCount = stats.RulesAppliedCount + status.ResourcesBlockedCount = stats.ResourceBlocked + status.AvgExecutionTimeMutation = stats.MutationExecutionTime.String() + status.AvgExecutionTimeValidation = stats.ValidationExecutionTime.String() + } + return status +} func (pc *PolicyController) getPolicyViolationsForPolicy(p *kyverno.Policy) ([]*kyverno.PolicyViolation, error) { // List all PolicyViolation to find those we own but that no longer match our // selector. They will be orphaned by ClaimPolicyViolation(). diff --git a/pkg/policy/status.go b/pkg/policy/status.go index f19961dba5..60a6a6dbf0 100644 --- a/pkg/policy/status.go +++ b/pkg/policy/status.go @@ -1,59 +1,38 @@ package policy import ( - "fmt" - "reflect" "sync" "time" "github.com/golang/glog" - kyverno "github.com/nirmata/kyverno/pkg/api/kyverno/v1alpha1" kyvernoclient "github.com/nirmata/kyverno/pkg/client/clientset/versioned" - kyvernoinformer "github.com/nirmata/kyverno/pkg/client/informers/externalversions/kyverno/v1alpha1" - kyvernolister "github.com/nirmata/kyverno/pkg/client/listers/kyverno/v1alpha1" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/tools/cache" ) -// type PolicyStatus struct { -// // average time required to process the policy rules on a resource -// avgExecutionTime time.Duration -// // Count of rules that were applied succesfully -// rulesAppliedCount int -// // Count of resources for whom update/create api requests were blocked as the resoruce did not satisfy the policy rules -// resourcesBlockedCount int -// // Count of the resource for whom the mutation rules were applied succesfully -// resourcesMutatedCount int -// } - //PolicyStatusAggregator stores information abt aggregation type PolicyStatusAggregator struct { // time since we start aggregating the stats startTime time.Time // channel to recieve stats ch chan PolicyStat - // update policy status - psControl PStatusControlInterface - // pLister can list/get policy from the shared informer's store - pLister kyvernolister.PolicyLister - // pListerSynced returns true if the Policy store has been synced at least once - pListerSynced cache.InformerSynced - // UpdateViolationCount and SendStat can update same policy status - // we need to sync the updates using policyName - policyUpdateData sync.Map + //TODO: lock based on key, possibly sync.Map ? + //sync RW for policyData + mux sync.RWMutex + // stores aggregated stats for policy + policyData map[string]PolicyStatInfo } //NewPolicyStatAggregator returns a new policy status -func NewPolicyStatAggregator(client *kyvernoclient.Clientset, pInformer kyvernoinformer.PolicyInformer) *PolicyStatusAggregator { +func NewPolicyStatAggregator(client *kyvernoclient.Clientset, + +// pInformer kyvernoinformer.PolicyInformer +) *PolicyStatusAggregator { psa := PolicyStatusAggregator{ - startTime: time.Now(), - ch: make(chan PolicyStat), + startTime: time.Now(), + ch: make(chan PolicyStat), + policyData: map[string]PolicyStatInfo{}, } - psa.pLister = pInformer.Lister() - psa.pListerSynced = pInformer.Informer().HasSynced - psa.psControl = PSControl{Client: client} - //TODO: add WaitGroup return &psa } @@ -65,10 +44,11 @@ func (psa *PolicyStatusAggregator) Run(workers int, stopCh <-chan struct{}) { glog.V(4).Info("Shutting down aggregator for policy status stats") }() for i := 0; i < workers; i++ { - go wait.Until(psa.aggregate, time.Second, stopCh) + go wait.Until(psa.process, time.Second, stopCh) } } -func (psa *PolicyStatusAggregator) aggregate() { + +func (psa *PolicyStatusAggregator) process() { // As mutation and validation are handled seperately // ideally we need to combine the exection time from both for a policy // but its tricky to detect here the type of rules policy contains @@ -76,74 +56,96 @@ func (psa *PolicyStatusAggregator) aggregate() { // mutation & validation rules seperately for r := range psa.ch { glog.V(4).Infof("recieved policy stats %v", r) - if err := psa.updateStats(r); err != nil { - glog.Infof("Failed to update stats for policy %s: %v", r.PolicyName, err) - } + // if err := psa.updateStats(r); err != nil { + // glog.Infof("Failed to update stats for policy %s: %v", r.PolicyName, err) + // } + psa.aggregate(r) } - } -func (psa *PolicyStatusAggregator) updateStats(stats PolicyStat) error { +func (psa *PolicyStatusAggregator) aggregate(ps PolicyStat) { func() { - glog.V(4).Infof("lock updates for policy %s", stats.PolicyName) - // Lock the update for policy - psa.policyUpdateData.Store(stats.PolicyName, struct{}{}) + glog.V(4).Infof("write lock update policy %s", ps.PolicyName) + psa.mux.Lock() }() defer func() { - glog.V(4).Infof("Unlock updates for policy %s", stats.PolicyName) - psa.policyUpdateData.Delete(stats.PolicyName) + glog.V(4).Infof("write Unlock update policy %s", ps.PolicyName) + psa.mux.Unlock() }() - - // //wait for cache sync - // if !cache.WaitForCacheSync(nil, psa.pListerSynced) { - // glog.Infof("unable to sync cache for policy informer") - // return nil - // } - // get policy - policy, err := psa.pLister.Get(stats.PolicyName) - if err != nil { - glog.V(4).Infof("failed to get policy %s. Unable to update violation count: %v", stats.PolicyName, err) - return err + info, ok := psa.policyData[ps.PolicyName] + if !ok { + psa.policyData[ps.PolicyName] = ps.Stats + glog.V(4).Infof("added stats for policy %s", ps.PolicyName) + return } - newpolicy := policy - fmt.Println(newpolicy.ResourceVersion) - newpolicy.Status = kyverno.PolicyStatus{} - glog.V(4).Infof("updating stats for policy %s", policy.Name) - // rules applied count - newpolicy.Status.RulesAppliedCount = newpolicy.Status.RulesAppliedCount + stats.RulesAppliedCount - // resource blocked count - if stats.ResourceBlocked { - policy.Status.ResourcesBlockedCount++ + // aggregate + info.RulesAppliedCount = info.RulesAppliedCount + ps.Stats.RulesAppliedCount + if ps.Stats.ResourceBlocked == 1 { + info.ResourceBlocked++ } var zeroDuration time.Duration - if newpolicy.Status.AvgExecutionTimeMutation != zeroDuration { - // avg execution time for mutation rules - newpolicy.Status.AvgExecutionTimeMutation = (newpolicy.Status.AvgExecutionTimeMutation + stats.MutationExecutionTime) / 2 + if info.MutationExecutionTime != zeroDuration { + info.MutationExecutionTime = (info.MutationExecutionTime + ps.Stats.MutationExecutionTime) / 2 + glog.V(4).Infof("updated avg mutation time %v", info.MutationExecutionTime) } else { - newpolicy.Status.AvgExecutionTimeMutation = stats.MutationExecutionTime + info.MutationExecutionTime = ps.Stats.MutationExecutionTime } - if policy.Status.AvgExecutionTimeValidation != zeroDuration { - // avg execution time for validation rules - newpolicy.Status.AvgExecutionTimeValidation = (newpolicy.Status.AvgExecutionTimeValidation + stats.ValidationExecutionTime) / 2 + if info.ValidationExecutionTime != zeroDuration { + info.ValidationExecutionTime = (info.ValidationExecutionTime + ps.Stats.ValidationExecutionTime) / 2 + glog.V(4).Infof("updated avg validation time %v", info.ValidationExecutionTime) } else { - newpolicy.Status.AvgExecutionTimeValidation = stats.ValidationExecutionTime + info.ValidationExecutionTime = ps.Stats.ValidationExecutionTime } - return psa.psControl.UpdatePolicyStatus(newpolicy) + // update + psa.policyData[ps.PolicyName] = info + glog.V(4).Infof("updated stats for policy %s", ps.PolicyName) +} + +//GetPolicyStats returns the policy stats +func (psa *PolicyStatusAggregator) GetPolicyStats(policyName string) PolicyStatInfo { + func() { + glog.V(4).Infof("read lock update policy %s", policyName) + psa.mux.RLock() + }() + defer func() { + glog.V(4).Infof("read Unlock update policy %s", policyName) + psa.mux.RUnlock() + }() + glog.V(4).Infof("read stats for policy %s", policyName) + return psa.policyData[policyName] +} + +//RemovePolicyStats rmves policy stats records +func (psa *PolicyStatusAggregator) RemovePolicyStats(policyName string) { + func() { + glog.V(4).Infof("write lock update policy %s", policyName) + psa.mux.Lock() + }() + defer func() { + glog.V(4).Infof("write Unlock update policy %s", policyName) + psa.mux.Unlock() + }() + glog.V(4).Infof("removing stats for policy %s", policyName) + delete(psa.policyData, policyName) } //PolicyStatusInterface provides methods to modify policyStatus type PolicyStatusInterface interface { SendStat(stat PolicyStat) - UpdateViolationCount(policyName string, pvList []*kyverno.PolicyViolation) error + // UpdateViolationCount(policyName string, pvList []*kyverno.PolicyViolation) error } //PolicyStat stored stats for policy type PolicyStat struct { - PolicyName string + PolicyName string + Stats PolicyStatInfo +} + +type PolicyStatInfo struct { MutationExecutionTime time.Duration ValidationExecutionTime time.Duration RulesAppliedCount int - ResourceBlocked bool + ResourceBlocked int } //SendStat sends the stat information for aggregation @@ -153,62 +155,7 @@ func (psa *PolicyStatusAggregator) SendStat(stat PolicyStat) { psa.ch <- stat } -//UpdateViolationCount updates the active violation count -func (psa *PolicyStatusAggregator) UpdateViolationCount(policyName string, pvList []*kyverno.PolicyViolation) error { - func() { - glog.V(4).Infof("lock updates for policy %s", policyName) - // Lock the update for policy - psa.policyUpdateData.Store(policyName, struct{}{}) - }() - defer func() { - glog.V(4).Infof("Unlock updates for policy %s", policyName) - psa.policyUpdateData.Delete(policyName) - }() - // get policy - policy, err := psa.pLister.Get(policyName) - if err != nil { - glog.V(4).Infof("failed to get policy %s. Unable to update violation count: %v", policyName, err) - return err - } - - newStatus := calculateStatus(pvList) - if reflect.DeepEqual(newStatus, policy.Status) { - // no update to status - glog.V(4).Infof("no changes in policy violation count for policy %s", policy.Name) - return nil - } - // update status - newPolicy := policy - newPolicy.Status = newStatus - - return psa.psControl.UpdatePolicyStatus(newPolicy) -} - -func calculateStatus(pvList []*kyverno.PolicyViolation) kyverno.PolicyStatus { - violationCount := len(pvList) - status := kyverno.PolicyStatus{ - ViolationCount: violationCount, - } - return status -} - //GetPolicyStatusAggregator returns interface to send policy status stats func (pc *PolicyController) GetPolicyStatusAggregator() PolicyStatusInterface { return pc.statusAggregator } - -//PStatusControlInterface Provides interface to operate on policy status -type PStatusControlInterface interface { - UpdatePolicyStatus(newPolicy *kyverno.Policy) error -} - -//PSControl allows update for policy status -type PSControl struct { - Client kyvernoclient.Interface -} - -//UpdatePolicyStatus update policy status -func (c PSControl) UpdatePolicyStatus(newPolicy *kyverno.Policy) error { - _, err := c.Client.KyvernoV1alpha1().Policies().UpdateStatus(newPolicy) - return err -} diff --git a/pkg/utils/util.go b/pkg/utils/util.go index e77e1a55db..eb548c328d 100644 --- a/pkg/utils/util.go +++ b/pkg/utils/util.go @@ -86,3 +86,11 @@ func NewKubeClient(config *rest.Config) (kubernetes.Interface, error) { } return kclient, nil } + +//Btoi converts boolean to int +func Btoi(b bool) int { + if b { + return 1 + } + return 0 +} diff --git a/pkg/webhooks/mutation.go b/pkg/webhooks/mutation.go index f706020627..620ab3242c 100644 --- a/pkg/webhooks/mutation.go +++ b/pkg/webhooks/mutation.go @@ -20,14 +20,14 @@ func (ws *WebhookServer) HandleMutation(request *v1beta1.AdmissionRequest) (bool gatherStat := func(policyName string, er engine.EngineResponse) { ps := policyctr.PolicyStat{} ps.PolicyName = policyName - ps.MutationExecutionTime = er.ExecutionTime - ps.RulesAppliedCount = er.RulesAppliedCount + ps.Stats.MutationExecutionTime = er.ExecutionTime + ps.Stats.RulesAppliedCount = er.RulesAppliedCount policyStats = append(policyStats, ps) } // send stats for aggregation sendStat := func(blocked bool) { for _, stat := range policyStats { - stat.ResourceBlocked = blocked + stat.Stats.ResourceBlocked = utils.Btoi(blocked) //SEND ws.policyStatus.SendStat(stat) } diff --git a/pkg/webhooks/validation.go b/pkg/webhooks/validation.go index 088694bd67..14105ed0f3 100644 --- a/pkg/webhooks/validation.go +++ b/pkg/webhooks/validation.go @@ -23,14 +23,14 @@ func (ws *WebhookServer) HandleValidation(request *v1beta1.AdmissionRequest, res gatherStat := func(policyName string, er engine.EngineResponse) { ps := policyctr.PolicyStat{} ps.PolicyName = policyName - ps.ValidationExecutionTime = er.ExecutionTime - ps.RulesAppliedCount = er.RulesAppliedCount + ps.Stats.ValidationExecutionTime = er.ExecutionTime + ps.Stats.RulesAppliedCount = er.RulesAppliedCount policyStats = append(policyStats, ps) } // send stats for aggregation sendStat := func(blocked bool) { for _, stat := range policyStats { - stat.ResourceBlocked = blocked + stat.Stats.ResourceBlocked = utils.Btoi(blocked) //SEND ws.policyStatus.SendStat(stat) } From 4f309480afadd8e5de6d88996ed6c2a5442164aa Mon Sep 17 00:00:00 2001 From: shivkumar dudhani Date: Tue, 20 Aug 2019 16:57:19 -0700 Subject: [PATCH 07/14] report stats from existing resources --- pkg/policy/apply.go | 42 +++++++++++++++++++++++++++++++++++++++--- pkg/policy/existing.go | 6 +++--- 2 files changed, 42 insertions(+), 6 deletions(-) diff --git a/pkg/policy/apply.go b/pkg/policy/apply.go index ec779c330a..6a1b5db9ee 100644 --- a/pkg/policy/apply.go +++ b/pkg/policy/apply.go @@ -14,7 +14,20 @@ import ( // applyPolicy applies policy on a resource //TODO: generation rules -func applyPolicy(policy kyverno.Policy, resource unstructured.Unstructured) (info.PolicyInfo, error) { +func applyPolicy(policy kyverno.Policy, resource unstructured.Unstructured, policyStatus PolicyStatusInterface) (info.PolicyInfo, error) { + var ps PolicyStat + gatherStat := func(policyName string, er engine.EngineResponse) { + // ps := policyctr.PolicyStat{} + ps.PolicyName = policyName + ps.Stats.ValidationExecutionTime = er.ExecutionTime + ps.Stats.RulesAppliedCount = er.RulesAppliedCount + } + // send stats for aggregation + sendStat := func(blocked bool) { + //SEND + policyStatus.SendStat(ps) + } + startTime := time.Now() glog.V(4).Infof("Started apply policy %s on resource %s/%s/%s (%v)", policy.Name, resource.GetKind(), resource.GetNamespace(), resource.GetName(), startTime) defer func() { @@ -24,7 +37,7 @@ func applyPolicy(policy kyverno.Policy, resource unstructured.Unstructured) (inf policyInfo := info.NewPolicyInfo(policy.Name, resource.GetKind(), resource.GetName(), resource.GetNamespace(), policy.Spec.ValidationFailureAction) //MUTATION - mruleInfos, err := mutation(policy, resource) + mruleInfos, err := mutation(policy, resource, policyStatus) policyInfo.AddRuleInfos(mruleInfos) if err != nil { return policyInfo, err @@ -35,13 +48,36 @@ func applyPolicy(policy kyverno.Policy, resource unstructured.Unstructured) (inf if len(engineResponse.RuleInfos) != 0 { policyInfo.AddRuleInfos(engineResponse.RuleInfos) } + // gather stats + gatherStat(policy.Name, engineResponse) + //send stats + sendStat(false) //TODO: GENERATION return policyInfo, nil } -func mutation(policy kyverno.Policy, resource unstructured.Unstructured) ([]info.RuleInfo, error) { +func mutation(policy kyverno.Policy, resource unstructured.Unstructured, policyStatus PolicyStatusInterface) ([]info.RuleInfo, error) { + var ps PolicyStat + // gather stats from the engine response + gatherStat := func(policyName string, er engine.EngineResponse) { + // ps := policyctr.PolicyStat{} + ps.PolicyName = policyName + ps.Stats.MutationExecutionTime = er.ExecutionTime + ps.Stats.RulesAppliedCount = er.RulesAppliedCount + } + // send stats for aggregation + sendStat := func(blocked bool) { + //SEND + policyStatus.SendStat(ps) + } + engineResponse := engine.Mutate(policy, resource) + // gather stats + gatherStat(policy.Name, engineResponse) + //send stats + sendStat(false) + patches := engineResponse.Patches ruleInfos := engineResponse.RuleInfos if len(ruleInfos) == 0 { diff --git a/pkg/policy/existing.go b/pkg/policy/existing.go index eaa5d0657d..092d657736 100644 --- a/pkg/policy/existing.go +++ b/pkg/policy/existing.go @@ -29,7 +29,7 @@ func (pc *PolicyController) processExistingResources(policy kyverno.Policy) []in } // apply the policy on each glog.V(4).Infof("apply policy %s with resource version %s on resource %s/%s/%s with resource version %s", policy.Name, policy.ResourceVersion, resource.GetKind(), resource.GetNamespace(), resource.GetName(), resource.GetResourceVersion()) - policyInfo := applyPolicyOnResource(policy, resource) + policyInfo := applyPolicyOnResource(policy, resource, pc.statusAggregator) policyInfos = append(policyInfos, *policyInfo) // post-processing, register the resource as processed pc.rm.RegisterResource(policy.GetName(), policy.GetResourceVersion(), resource.GetKind(), resource.GetNamespace(), resource.GetName(), resource.GetResourceVersion()) @@ -37,8 +37,8 @@ func (pc *PolicyController) processExistingResources(policy kyverno.Policy) []in return policyInfos } -func applyPolicyOnResource(policy kyverno.Policy, resource unstructured.Unstructured) *info.PolicyInfo { - policyInfo, err := applyPolicy(policy, resource) +func applyPolicyOnResource(policy kyverno.Policy, resource unstructured.Unstructured, policyStatus PolicyStatusInterface) *info.PolicyInfo { + policyInfo, err := applyPolicy(policy, resource, policyStatus) if err != nil { glog.V(4).Infof("failed to process policy %s on resource %s/%s/%s: %v", policy.GetName(), resource.GetKind(), resource.GetNamespace(), resource.GetName(), err) return nil From 292a644bf8e13c0615f2e646b43f9749511a11dc Mon Sep 17 00:00:00 2001 From: shivkumar dudhani Date: Tue, 20 Aug 2019 17:35:40 -0700 Subject: [PATCH 08/14] generate stats for generate rules --- pkg/api/kyverno/v1alpha1/types.go | 6 ++++-- pkg/engine/generation.go | 5 ++--- pkg/namespace/controller.go | 6 +++++- pkg/namespace/generation.go | 24 ++++++++++++++++++++++-- pkg/policy/controller.go | 1 + pkg/policy/status.go | 10 +++++++--- 6 files changed, 41 insertions(+), 11 deletions(-) diff --git a/pkg/api/kyverno/v1alpha1/types.go b/pkg/api/kyverno/v1alpha1/types.go index 44769bd446..29fb9506a4 100644 --- a/pkg/api/kyverno/v1alpha1/types.go +++ b/pkg/api/kyverno/v1alpha1/types.go @@ -95,9 +95,11 @@ type PolicyStatus struct { // Count of resources for whom update/create api requests were blocked as the resoruce did not satisfy the policy rules ResourcesBlockedCount int `json:"resourcesBlockedCount"` // average time required to process the policy Mutation rules on a resource - AvgExecutionTimeMutation string `json:"averageMutationExecutionTime"` + AvgExecutionTimeMutation string `json:"averageMutationRulesExecutionTime"` // average time required to process the policy Validation rules on a resource - AvgExecutionTimeValidation string `json:"averageValidationExecutionTime"` + AvgExecutionTimeValidation string `json:"averageValidationRulesExecutionTime"` + // average time required to process the policy Validation rules on a resource + AvgExecutionTimeGeneration string `json:"averageGenerationRulesExecutionTime"` } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object diff --git a/pkg/engine/generation.go b/pkg/engine/generation.go index 9b46aa755f..5012121d37 100644 --- a/pkg/engine/generation.go +++ b/pkg/engine/generation.go @@ -18,14 +18,13 @@ import ( ) //Generate apply generation rules on a resource -func Generate(client *client.Client, policy kyverno.Policy, ns unstructured.Unstructured) EngineResponse { - var response EngineResponse +func Generate(client *client.Client, policy kyverno.Policy, ns unstructured.Unstructured) (response EngineResponse) { startTime := time.Now() glog.V(4).Infof("started applying generation rules of policy %q (%v)", policy.Name, startTime) defer func() { response.ExecutionTime = time.Since(startTime) glog.V(4).Infof("Finished applying generation rules policy %q (%v)", policy.Name, response.ExecutionTime) - glog.V(4).Infof("Mutation Rules appplied succesfully count %q for policy %q", response.RulesAppliedCount, policy.Name) + glog.V(4).Infof("Generation Rules appplied count %q for policy %q", response.RulesAppliedCount, policy.Name) }() incrementAppliedRuleCount := func() { // rules applied succesfully count diff --git a/pkg/namespace/controller.go b/pkg/namespace/controller.go index 78e6340a3f..38547fdb15 100644 --- a/pkg/namespace/controller.go +++ b/pkg/namespace/controller.go @@ -8,6 +8,7 @@ import ( "github.com/golang/glog" client "github.com/nirmata/kyverno/pkg/dclient" "github.com/nirmata/kyverno/pkg/event" + "github.com/nirmata/kyverno/pkg/policy" "k8s.io/apimachinery/pkg/api/errors" kyvernoclient "github.com/nirmata/kyverno/pkg/client/clientset/versioned" @@ -44,7 +45,8 @@ type NamespaceController struct { pvListerSynced cache.InformerSynced // pvLister can list/get policy violation from the shared informer's store pvLister kyvernolister.PolicyViolationLister - + // API to send policy stats for aggregation + policyStatus policy.PolicyStatusInterface // eventGen provides interface to generate evenets eventGen event.Interface // Namespaces that need to be synced @@ -59,6 +61,7 @@ func NewNamespaceController(kyvernoClient *kyvernoclient.Clientset, nsInformer v1Informer.NamespaceInformer, pInformer kyvernoinformer.PolicyInformer, pvInformer kyvernoinformer.PolicyViolationInformer, + policyStatus policy.PolicyStatusInterface, eventGen event.Interface) *NamespaceController { //TODO: do we need to event recorder for this controller? // create the controller @@ -83,6 +86,7 @@ func NewNamespaceController(kyvernoClient *kyvernoclient.Clientset, nsc.pLister = pInformer.Lister() nsc.pvListerSynced = pInformer.Informer().HasSynced nsc.pvLister = pvInformer.Lister() + nsc.policyStatus = policyStatus // resource manager // rebuild after 300 seconds/ 5 mins diff --git a/pkg/namespace/generation.go b/pkg/namespace/generation.go index ca715e20ce..d36bc7e24a 100644 --- a/pkg/namespace/generation.go +++ b/pkg/namespace/generation.go @@ -6,6 +6,7 @@ import ( client "github.com/nirmata/kyverno/pkg/dclient" "github.com/nirmata/kyverno/pkg/engine" + "github.com/nirmata/kyverno/pkg/policy" "github.com/golang/glog" @@ -13,7 +14,9 @@ import ( kyvernolister "github.com/nirmata/kyverno/pkg/client/listers/kyverno/v1alpha1" "github.com/nirmata/kyverno/pkg/info" + policyctr "github.com/nirmata/kyverno/pkg/policy" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" @@ -108,7 +111,7 @@ func (nsc *NamespaceController) processNamespace(namespace corev1.Namespace) []i glog.V(4).Infof("policy %s with resource version %s already processed on resource %s/%s/%s with resource version %s", policy.Name, policy.ResourceVersion, ns.GetKind(), ns.GetNamespace(), ns.GetName(), ns.GetResourceVersion()) continue } - policyInfo := applyPolicy(nsc.client, ns, *policy) + policyInfo := applyPolicy(nsc.client, ns, *policy, nsc.policyStatus) policyInfos = append(policyInfos, policyInfo) // post-processing, register the resource as processed nsc.rm.RegisterResource(policy.GetName(), policy.GetResourceVersion(), ns.GetKind(), ns.GetNamespace(), ns.GetName(), ns.GetResourceVersion()) @@ -141,7 +144,20 @@ func listpolicies(ns unstructured.Unstructured, pLister kyvernolister.PolicyList return filteredpolicies } -func applyPolicy(client *client.Client, resource unstructured.Unstructured, policy kyverno.Policy) info.PolicyInfo { +func applyPolicy(client *client.Client, resource unstructured.Unstructured, policy kyverno.Policy, policyStatus policy.PolicyStatusInterface) info.PolicyInfo { + var ps policyctr.PolicyStat + gatherStat := func(policyName string, er engine.EngineResponse) { + // ps := policyctr.PolicyStat{} + ps.PolicyName = policyName + ps.Stats.GenerationExecutionTime = er.ExecutionTime + ps.Stats.RulesAppliedCount = er.RulesAppliedCount + } + // send stats for aggregation + sendStat := func(blocked bool) { + //SEND + policyStatus.SendStat(ps) + } + startTime := time.Now() glog.V(4).Infof("Started apply policy %s on resource %s/%s/%s (%v)", policy.Name, resource.GetKind(), resource.GetNamespace(), resource.GetName(), startTime) defer func() { @@ -150,6 +166,10 @@ func applyPolicy(client *client.Client, resource unstructured.Unstructured, poli policyInfo := info.NewPolicyInfo(policy.Name, resource.GetKind(), resource.GetName(), resource.GetNamespace(), policy.Spec.ValidationFailureAction) engineResponse := engine.Generate(client, policy, resource) policyInfo.AddRuleInfos(engineResponse.RuleInfos) + // gather stats + gatherStat(policy.Name, engineResponse) + //send stats + sendStat(false) return policyInfo } diff --git a/pkg/policy/controller.go b/pkg/policy/controller.go index fccbed13c5..40a80cefee 100644 --- a/pkg/policy/controller.go +++ b/pkg/policy/controller.go @@ -447,6 +447,7 @@ func (pc *PolicyController) calculateStatus(policyName string, pvList []*kyverno status.ResourcesBlockedCount = stats.ResourceBlocked status.AvgExecutionTimeMutation = stats.MutationExecutionTime.String() status.AvgExecutionTimeValidation = stats.ValidationExecutionTime.String() + status.AvgExecutionTimeGeneration = stats.GenerationExecutionTime.String() } return status } diff --git a/pkg/policy/status.go b/pkg/policy/status.go index 60a6a6dbf0..133300320b 100644 --- a/pkg/policy/status.go +++ b/pkg/policy/status.go @@ -56,9 +56,6 @@ func (psa *PolicyStatusAggregator) process() { // mutation & validation rules seperately for r := range psa.ch { glog.V(4).Infof("recieved policy stats %v", r) - // if err := psa.updateStats(r); err != nil { - // glog.Infof("Failed to update stats for policy %s: %v", r.PolicyName, err) - // } psa.aggregate(r) } } @@ -96,6 +93,12 @@ func (psa *PolicyStatusAggregator) aggregate(ps PolicyStat) { } else { info.ValidationExecutionTime = ps.Stats.ValidationExecutionTime } + if info.GenerationExecutionTime != zeroDuration { + info.GenerationExecutionTime = (info.GenerationExecutionTime + ps.Stats.GenerationExecutionTime) / 2 + glog.V(4).Infof("updated avg generation time %v", info.GenerationExecutionTime) + } else { + info.GenerationExecutionTime = ps.Stats.GenerationExecutionTime + } // update psa.policyData[ps.PolicyName] = info glog.V(4).Infof("updated stats for policy %s", ps.PolicyName) @@ -144,6 +147,7 @@ type PolicyStat struct { type PolicyStatInfo struct { MutationExecutionTime time.Duration ValidationExecutionTime time.Duration + GenerationExecutionTime time.Duration RulesAppliedCount int ResourceBlocked int } From a4310a38dd0fc766d3c96b8e9aa868cb5fb109af Mon Sep 17 00:00:00 2001 From: shivkumar dudhani Date: Tue, 20 Aug 2019 17:35:55 -0700 Subject: [PATCH 09/14] add missing files --- main.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/main.go b/main.go index 2c2a950354..6d61e72d9a 100644 --- a/main.go +++ b/main.go @@ -67,7 +67,8 @@ func main() { // POLICY CONTROLLER // - reconciliation policy and policy violation // - process policy on existing resources - // - status: violation count + // - status aggregator: recieves stats when a policy is applied + // : updates the policy status pc, err := policy.NewPolicyController(pclient, client, pInformer.Kyverno().V1alpha1().Policies(), pInformer.Kyverno().V1alpha1().PolicyViolations(), egen) if err != nil { @@ -92,7 +93,7 @@ func main() { // GENERATE CONTROLLER // - watches for Namespace resource and generates resource based on the policy generate rule - nsc := namespace.NewNamespaceController(pclient, client, kubeInformer.Core().V1().Namespaces(), pInformer.Kyverno().V1alpha1().Policies(), pInformer.Kyverno().V1alpha1().PolicyViolations(), egen) + nsc := namespace.NewNamespaceController(pclient, client, kubeInformer.Core().V1().Namespaces(), pInformer.Kyverno().V1alpha1().Policies(), pInformer.Kyverno().V1alpha1().PolicyViolations(), pc.GetPolicyStatusAggregator(), egen) tlsPair, err := initTLSPemPair(clientConfig, client) if err != nil { From e7082153102a4c24b66bf6de9d20433a356bbed3 Mon Sep 17 00:00:00 2001 From: shivkumar dudhani Date: Tue, 20 Aug 2019 23:43:30 -0700 Subject: [PATCH 10/14] add Policy kind as known Kind --- pkg/api/kyverno/v1alpha1/register.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/api/kyverno/v1alpha1/register.go b/pkg/api/kyverno/v1alpha1/register.go index 86f19d5512..11e0a7e1db 100644 --- a/pkg/api/kyverno/v1alpha1/register.go +++ b/pkg/api/kyverno/v1alpha1/register.go @@ -29,6 +29,8 @@ var ( // Adds the list of known types to Scheme. func addKnownTypes(scheme *runtime.Scheme) error { scheme.AddKnownTypes(SchemeGroupVersion, + &Policy{}, + &PolicyList{}, &PolicyViolation{}, &PolicyViolationList{}, ) From ed9c88cd07f5c3e9592968a435d75d94bcb3956f Mon Sep 17 00:00:00 2001 From: shivkumar dudhani Date: Tue, 20 Aug 2019 23:51:34 -0700 Subject: [PATCH 11/14] update comments --- main.go | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/main.go b/main.go index 6d61e72d9a..f7aedf3bc1 100644 --- a/main.go +++ b/main.go @@ -31,6 +31,7 @@ var ( func main() { defer glog.Flush() printVersionInfo() + // profile cpu and memory consuption prof = enableProfiling(cpu, memory) // CLIENT CONFIG @@ -47,6 +48,7 @@ func main() { if err != nil { glog.Fatalf("Error creating client: %v\n", err) } + // DYNAMIC CLIENT // - client for all registered resources client, err := client.NewClient(clientConfig) @@ -60,15 +62,16 @@ func main() { // - PolicyVolation // - cache resync time: 10 seconds pInformer := kyvernoinformer.NewSharedInformerFactoryWithOptions(pclient, 10*time.Second) + // EVENT GENERATOR - // - generate event with retry + // - generate event with retry mechanism egen := event.NewEventGenerator(client, pInformer.Kyverno().V1alpha1().Policies()) // POLICY CONTROLLER // - reconciliation policy and policy violation // - process policy on existing resources // - status aggregator: recieves stats when a policy is applied - // : updates the policy status + // & updates the policy status pc, err := policy.NewPolicyController(pclient, client, pInformer.Kyverno().V1alpha1().Policies(), pInformer.Kyverno().V1alpha1().PolicyViolations(), egen) if err != nil { @@ -76,6 +79,7 @@ func main() { } // POLICY VIOLATION CONTROLLER + // policy violation cleanup if the corresponding resource is deleted // status: lastUpdatTime pvc, err := policyviolation.NewPolicyViolationController(client, pclient, pInformer.Kyverno().V1alpha1().Policies(), pInformer.Kyverno().V1alpha1().PolicyViolations()) if err != nil { @@ -100,17 +104,25 @@ func main() { glog.Fatalf("Failed to initialize TLS key/certificate pair: %v\n", err) } - // WEBHOOK REGISTRATION - // -- validationwebhookconfiguration (Policy) - // -- mutatingwebhookconfiguration (All resources) webhookRegistrationClient, err := webhooks.NewWebhookRegistrationClient(clientConfig, client, serverIP, int32(webhookTimeout)) if err != nil { glog.Fatalf("Unable to register admission webhooks on cluster: %v\n", err) } - + // WEBHOOK REGISTRATION + // - validationwebhookconfiguration (Policy) + // - mutatingwebhookconfiguration (All resources) + // webhook confgiuration is also generated dynamically in the policy controller + // based on the policy resources created if err = webhookRegistrationClient.Register(); err != nil { glog.Fatalf("Failed registering Admission Webhooks: %v\n", err) } + + // WEBHOOOK + // - https server to provide endpoints called based on rules defined in Mutating & Validation webhook configuration + // - reports the results based on the response from the policy engine: + // -- annotations on resources with update details on mutation JSON patches + // -- generate policy violation resource + // -- generate events on policy and resource server, err := webhooks.NewWebhookServer(pclient, client, tlsPair, pInformer.Kyverno().V1alpha1().Policies(), pInformer.Kyverno().V1alpha1().PolicyViolations(), egen, webhookRegistrationClient, pc.GetPolicyStatusAggregator(), filterK8Resources) if err != nil { glog.Fatalf("Unable to create webhook server: %v\n", err) @@ -118,10 +130,7 @@ func main() { stopCh := signals.SetupSignalHandler() - if err = webhookRegistrationClient.Register(); err != nil { - glog.Fatalf("Failed registering Admission Webhooks: %v\n", err) - } - + // Start the components pInformer.Start(stopCh) kubeInformer.Start(stopCh) go pc.Run(1, stopCh) From f1960876a28a4532c48d1591fb390f72bd0b83b3 Mon Sep 17 00:00:00 2001 From: shivkumar dudhani Date: Tue, 20 Aug 2019 23:56:47 -0700 Subject: [PATCH 12/14] update comments --- main.go | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/main.go b/main.go index f7aedf3bc1..b7db798d13 100644 --- a/main.go +++ b/main.go @@ -56,6 +56,12 @@ func main() { glog.Fatalf("Error creating client: %v\n", err) } + // KUBERNETES CLIENT + kubeClient, err := utils.NewKubeClient(clientConfig) + if err != nil { + glog.Fatalf("Error creating kubernetes client: %v\n", err) + } + // KYVERNO CRD INFORMER // watches CRD resources: // - Policy @@ -72,7 +78,6 @@ func main() { // - process policy on existing resources // - status aggregator: recieves stats when a policy is applied // & updates the policy status - pc, err := policy.NewPolicyController(pclient, client, pInformer.Kyverno().V1alpha1().Policies(), pInformer.Kyverno().V1alpha1().PolicyViolations(), egen) if err != nil { glog.Fatalf("error creating policy controller: %v\n", err) @@ -86,24 +91,22 @@ func main() { glog.Fatalf("error creating policy violation controller: %v\n", err) } - // NAMESPACE INFORMER + // KUBERNETES RESOURCES INFORMER // watches namespace resource // - cache resync time: 10 seconds - kubeClient, err := utils.NewKubeClient(clientConfig) - if err != nil { - glog.Fatalf("Error creating kubernetes client: %v\n", err) - } kubeInformer := kubeinformers.NewSharedInformerFactoryWithOptions(kubeClient, 10*time.Second) // GENERATE CONTROLLER // - watches for Namespace resource and generates resource based on the policy generate rule nsc := namespace.NewNamespaceController(pclient, client, kubeInformer.Core().V1().Namespaces(), pInformer.Kyverno().V1alpha1().Policies(), pInformer.Kyverno().V1alpha1().PolicyViolations(), pc.GetPolicyStatusAggregator(), egen) + // CONFIGURE CERTIFICATES tlsPair, err := initTLSPemPair(clientConfig, client) if err != nil { glog.Fatalf("Failed to initialize TLS key/certificate pair: %v\n", err) } + // WERBHOOK REGISTRATION CLIENT webhookRegistrationClient, err := webhooks.NewWebhookRegistrationClient(clientConfig, client, serverIP, int32(webhookTimeout)) if err != nil { glog.Fatalf("Unable to register admission webhooks on cluster: %v\n", err) From d8c315e3391c4371941f6858cec3bf417f9a286b Mon Sep 17 00:00:00 2001 From: shivkumar dudhani Date: Wed, 21 Aug 2019 01:07:32 -0700 Subject: [PATCH 13/14] fix import cylce after merge + seperate webhookconfig client --- main.go | 17 +++++---- pkg/namespace/generation.go | 28 ++++++-------- pkg/policy/controller.go | 37 ++++++++++++++----- .../registration.go | 2 +- .../registration_test.go | 2 +- pkg/webhooks/server.go | 5 ++- 6 files changed, 53 insertions(+), 38 deletions(-) rename pkg/{webhooks => webhookconfig}/registration.go (99%) rename pkg/{webhooks => webhookconfig}/registration_test.go (99%) diff --git a/main.go b/main.go index 0cb3132940..c95261538d 100644 --- a/main.go +++ b/main.go @@ -14,8 +14,9 @@ import ( "github.com/nirmata/kyverno/pkg/policy" "github.com/nirmata/kyverno/pkg/policyviolation" "github.com/nirmata/kyverno/pkg/utils" + "github.com/nirmata/kyverno/pkg/webhookconfig" "github.com/nirmata/kyverno/pkg/webhooks" - kubeinformer "k8s.io/client-go/informers" + kubeinformers "k8s.io/client-go/informers" "k8s.io/sample-controller/pkg/signals" ) @@ -65,6 +66,12 @@ func main() { glog.Fatalf("Error creating kubernetes client: %v\n", err) } + // WERBHOOK REGISTRATION CLIENT + webhookRegistrationClient, err := webhookconfig.NewWebhookRegistrationClient(clientConfig, client, serverIP, int32(webhookTimeout)) + if err != nil { + glog.Fatalf("Unable to register admission webhooks on cluster: %v\n", err) + } + // KYVERNO CRD INFORMER // watches CRD resources: // - Policy @@ -72,7 +79,7 @@ func main() { // - cache resync time: 10 seconds pInformer := kyvernoinformer.NewSharedInformerFactoryWithOptions(pclient, 10*time.Second) - // KUBERNETES RESOURCES INFORMER + // KUBERNETES RESOURCES INFORMER // watches namespace resource // - cache resync time: 10 seconds kubeInformer := kubeinformers.NewSharedInformerFactoryWithOptions(kubeClient, 10*time.Second) @@ -109,12 +116,6 @@ func main() { glog.Fatalf("Failed to initialize TLS key/certificate pair: %v\n", err) } - // WERBHOOK REGISTRATION CLIENT - webhookRegistrationClient, err := webhooks.NewWebhookRegistrationClient(clientConfig, client, serverIP, int32(webhookTimeout)) - if err != nil { - glog.Fatalf("Unable to register admission webhooks on cluster: %v\n", err) - } - // WEBHOOK REGISTRATION // - validationwebhookconfiguration (Policy) // - mutatingwebhookconfiguration (All resources) diff --git a/pkg/namespace/generation.go b/pkg/namespace/generation.go index d36bc7e24a..997795eac8 100644 --- a/pkg/namespace/generation.go +++ b/pkg/namespace/generation.go @@ -4,19 +4,14 @@ import ( "sync" "time" + "github.com/golang/glog" + kyverno "github.com/nirmata/kyverno/pkg/api/kyverno/v1alpha1" + kyvernolister "github.com/nirmata/kyverno/pkg/client/listers/kyverno/v1alpha1" client "github.com/nirmata/kyverno/pkg/dclient" "github.com/nirmata/kyverno/pkg/engine" - "github.com/nirmata/kyverno/pkg/policy" - - "github.com/golang/glog" - - kyverno "github.com/nirmata/kyverno/pkg/api/kyverno/v1alpha1" - - kyvernolister "github.com/nirmata/kyverno/pkg/client/listers/kyverno/v1alpha1" "github.com/nirmata/kyverno/pkg/info" - policyctr "github.com/nirmata/kyverno/pkg/policy" + "github.com/nirmata/kyverno/pkg/policy" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" @@ -144,10 +139,9 @@ func listpolicies(ns unstructured.Unstructured, pLister kyvernolister.PolicyList return filteredpolicies } -func applyPolicy(client *client.Client, resource unstructured.Unstructured, policy kyverno.Policy, policyStatus policy.PolicyStatusInterface) info.PolicyInfo { - var ps policyctr.PolicyStat +func applyPolicy(client *client.Client, resource unstructured.Unstructured, p kyverno.Policy, policyStatus policy.PolicyStatusInterface) info.PolicyInfo { + var ps policy.PolicyStat gatherStat := func(policyName string, er engine.EngineResponse) { - // ps := policyctr.PolicyStat{} ps.PolicyName = policyName ps.Stats.GenerationExecutionTime = er.ExecutionTime ps.Stats.RulesAppliedCount = er.RulesAppliedCount @@ -159,15 +153,15 @@ func applyPolicy(client *client.Client, resource unstructured.Unstructured, poli } startTime := time.Now() - glog.V(4).Infof("Started apply policy %s on resource %s/%s/%s (%v)", policy.Name, resource.GetKind(), resource.GetNamespace(), resource.GetName(), startTime) + glog.V(4).Infof("Started apply policy %s on resource %s/%s/%s (%v)", p.Name, resource.GetKind(), resource.GetNamespace(), resource.GetName(), startTime) defer func() { - glog.V(4).Infof("Finished applying %s on resource %s/%s/%s (%v)", policy.Name, resource.GetKind(), resource.GetNamespace(), resource.GetName(), time.Since(startTime)) + glog.V(4).Infof("Finished applying %s on resource %s/%s/%s (%v)", p.Name, resource.GetKind(), resource.GetNamespace(), resource.GetName(), time.Since(startTime)) }() - policyInfo := info.NewPolicyInfo(policy.Name, resource.GetKind(), resource.GetName(), resource.GetNamespace(), policy.Spec.ValidationFailureAction) - engineResponse := engine.Generate(client, policy, resource) + policyInfo := info.NewPolicyInfo(p.Name, resource.GetKind(), resource.GetName(), resource.GetNamespace(), p.Spec.ValidationFailureAction) + engineResponse := engine.Generate(client, p, resource) policyInfo.AddRuleInfos(engineResponse.RuleInfos) // gather stats - gatherStat(policy.Name, engineResponse) + gatherStat(p.Name, engineResponse) //send stats sendStat(false) diff --git a/pkg/policy/controller.go b/pkg/policy/controller.go index a9277ff12f..76d31796ec 100644 --- a/pkg/policy/controller.go +++ b/pkg/policy/controller.go @@ -17,7 +17,7 @@ import ( client "github.com/nirmata/kyverno/pkg/dclient" "github.com/nirmata/kyverno/pkg/event" "github.com/nirmata/kyverno/pkg/utils" - "github.com/nirmata/kyverno/pkg/webhooks" + "github.com/nirmata/kyverno/pkg/webhookconfig" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -71,7 +71,7 @@ type PolicyController struct { // mutationwebhookLister can list/get mutatingwebhookconfigurations mutationwebhookLister webhooklister.MutatingWebhookConfigurationLister // WebhookRegistrationClient - webhookRegistrationClient *webhooks.WebhookRegistrationClient + webhookRegistrationClient *webhookconfig.WebhookRegistrationClient // Resource manager, manages the mapping for already processed resource rm resourceManager // filter the resources defined in the list @@ -82,7 +82,7 @@ type PolicyController struct { // NewPolicyController create a new PolicyController func NewPolicyController(kyvernoClient *kyvernoclient.Clientset, client *client.Client, pInformer kyvernoinformer.PolicyInformer, pvInformer kyvernoinformer.PolicyViolationInformer, - eventGen event.Interface, webhookInformer webhookinformer.MutatingWebhookConfigurationInformer, webhookRegistrationClient *webhooks.WebhookRegistrationClient) (*PolicyController, error) { + eventGen event.Interface, webhookInformer webhookinformer.MutatingWebhookConfigurationInformer, webhookRegistrationClient *webhookconfig.WebhookRegistrationClient) (*PolicyController, error) { // Event broad caster eventBroadcaster := record.NewBroadcaster() eventBroadcaster.StartLogging(glog.Infof) @@ -405,11 +405,11 @@ func (pc *PolicyController) syncPolicy(key string) error { policy, err := pc.pLister.Get(key) if errors.IsNotFound(err) { glog.V(2).Infof("Policy %v has been deleted", key) - - // remove the recorded stats for the policy + + // remove the recorded stats for the policy pc.statusAggregator.RemovePolicyStats(key) - // remove webhook configurations if there are not policies - if err := pc.handleWebhookRegistration(true, nil); err != nil { + // remove webhook configurations if there are not policies + if err := pc.handleWebhookRegistration(true, nil); err != nil { glog.Errorln(err) } return nil @@ -466,14 +466,14 @@ func (pc *PolicyController) handleWebhookRegistration(delete bool, policy *kyver if policies == nil { glog.V(3).Infoln("No policy found in the cluster, deregistering webhook") pc.webhookRegistrationClient.DeregisterMutatingWebhook() - } else if !webhooks.HasMutateOrValidatePolicies(policies) { + } else if !HasMutateOrValidatePolicies(policies) { glog.V(3).Infoln("No muatate/validate policy found in the cluster, deregistering webhook") pc.webhookRegistrationClient.DeregisterMutatingWebhook() } return nil } - if webhookList == nil && webhooks.HasMutateOrValidate(*policy) { + if webhookList == nil && HasMutateOrValidate(*policy) { glog.V(3).Infoln("Found policy without mutatingwebhook, registering webhook") pc.webhookRegistrationClient.RegisterMutatingWebhook() } @@ -934,3 +934,22 @@ func joinPatches(patches ...[]byte) []byte { result = append(result, []byte("\n]")...) return result } + +func HasMutateOrValidatePolicies(policies []*kyverno.Policy) bool { + for _, policy := range policies { + if HasMutateOrValidate(*policy) { + return true + } + } + return false +} + +func HasMutateOrValidate(policy kyverno.Policy) bool { + for _, rule := range policy.Spec.Rules { + if !reflect.DeepEqual(rule.Mutation, kyverno.Mutation{}) || !reflect.DeepEqual(rule.Validation, kyverno.Validation{}) { + glog.Infoln(rule.Name) + return true + } + } + return false +} diff --git a/pkg/webhooks/registration.go b/pkg/webhookconfig/registration.go similarity index 99% rename from pkg/webhooks/registration.go rename to pkg/webhookconfig/registration.go index bbdbf94afb..f4cdfa8154 100644 --- a/pkg/webhooks/registration.go +++ b/pkg/webhookconfig/registration.go @@ -1,4 +1,4 @@ -package webhooks +package webhookconfig import ( "errors" diff --git a/pkg/webhooks/registration_test.go b/pkg/webhookconfig/registration_test.go similarity index 99% rename from pkg/webhooks/registration_test.go rename to pkg/webhookconfig/registration_test.go index eb3cad3f78..2188007f44 100644 --- a/pkg/webhooks/registration_test.go +++ b/pkg/webhookconfig/registration_test.go @@ -1,4 +1,4 @@ -package webhooks +package webhookconfig import ( "bytes" diff --git a/pkg/webhooks/server.go b/pkg/webhooks/server.go index af6ae8a880..28483b47b2 100644 --- a/pkg/webhooks/server.go +++ b/pkg/webhooks/server.go @@ -22,6 +22,7 @@ import ( "github.com/nirmata/kyverno/pkg/policy" tlsutils "github.com/nirmata/kyverno/pkg/tls" "github.com/nirmata/kyverno/pkg/utils" + "github.com/nirmata/kyverno/pkg/webhookconfig" v1beta1 "k8s.io/api/admission/v1beta1" "k8s.io/client-go/tools/cache" ) @@ -37,7 +38,7 @@ type WebhookServer struct { pListerSynced cache.InformerSynced pvListerSynced cache.InformerSynced eventGen event.Interface - webhookRegistrationClient *WebhookRegistrationClient + webhookRegistrationClient *webhookconfig.WebhookRegistrationClient // API to send policy stats for aggregation policyStatus policy.PolicyStatusInterface filterK8Resources []utils.K8Resource @@ -52,7 +53,7 @@ func NewWebhookServer( pInformer kyvernoinformer.PolicyInformer, pvInormer kyvernoinformer.PolicyViolationInformer, eventGen event.Interface, - webhookRegistrationClient *WebhookRegistrationClient, + webhookRegistrationClient *webhookconfig.WebhookRegistrationClient, policyStatus policy.PolicyStatusInterface, filterK8Resources string) (*WebhookServer, error) { From 042bc645497ce6713bfca286f8bacd73ef7387b6 Mon Sep 17 00:00:00 2001 From: shivkumar dudhani Date: Wed, 21 Aug 2019 12:03:53 -0700 Subject: [PATCH 14/14] fix test build errors + skip testrunner --- pkg/engine/generation.go | 8 +- pkg/engine/utils_test.go | 942 +++++++----------- pkg/testrunner/test.go | 62 +- pkg/testrunner/testcase.go | 6 +- pkg/testrunner/testrunner_test.go | 2 + pkg/testrunner/utils.go | 10 + .../resources/CAFile | 0 7 files changed, 384 insertions(+), 646 deletions(-) rename pkg/{webhooks => webhookconfig}/resources/CAFile (100%) diff --git a/pkg/engine/generation.go b/pkg/engine/generation.go index 5012121d37..90d6a9c3bf 100644 --- a/pkg/engine/generation.go +++ b/pkg/engine/generation.go @@ -41,10 +41,10 @@ func Generate(client *client.Client, policy kyverno.Policy, ns unstructured.Unst err := applyRuleGenerator(client, ns, rule.Generation, policy.GetCreationTimestamp()) if err != nil { ri.Fail() - ri.Addf("Failed to apply rule generator, err %v.", rule.Name, err) + ri.Addf("Failed to apply rule %s generator, err %v.", rule.Name, err) glog.Infof("failed to apply policy %s rule %s on resource %s/%s/%s: %v", policy.Name, rule.Name, ns.GetKind(), ns.GetNamespace(), ns.GetName(), err) } else { - ri.Addf("Generation succesfully.", rule.Name) + ri.Addf("Generation succesfully for rule %s", rule.Name) glog.Infof("succesfully applied policy %s rule %s on resource %s/%s/%s", policy.Name, rule.Name, ns.GetKind(), ns.GetNamespace(), ns.GetName()) } ris = append(ris, ri) @@ -101,10 +101,10 @@ func applyRuleGenerator(client *client.Client, ns unstructured.Unstructured, gen // 2> If clone already exists return resource, err = client.GetResource(gen.Kind, gen.Clone.Namespace, gen.Clone.Name) if err != nil { - glog.V(4).Infof("generate rule: clone reference resource %s/%s/%s not present: %v", gen.Kind, gen.Kind, gen.Clone.Namespace, gen.Clone.Name, err) + glog.V(4).Infof("generate rule: clone reference resource %s/%s/%s not present: %v", gen.Kind, gen.Clone.Namespace, gen.Clone.Name, err) return err } - glog.V(4).Infof("generate rule: clone reference resource %s/%s/%s present", gen.Kind, gen.Kind, gen.Clone.Namespace, gen.Clone.Name) + glog.V(4).Infof("generate rule: clone reference resource %s/%s/%s present", gen.Kind, gen.Clone.Namespace, gen.Clone.Name) rdata = resource.UnstructuredContent() } if processExisting { diff --git a/pkg/engine/utils_test.go b/pkg/engine/utils_test.go index 81379246c5..66b192b0f8 100644 --- a/pkg/engine/utils_test.go +++ b/pkg/engine/utils_test.go @@ -3,105 +3,232 @@ package engine import ( "testing" - types "github.com/nirmata/kyverno/pkg/api/kyverno/v1alpha1" + kyverno "github.com/nirmata/kyverno/pkg/api/kyverno/v1alpha1" "gotest.tools/assert" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -func TestResourceMeetsDescription_Kind(t *testing.T) { - resourceName := "test-config-map" - resourceDescription := types.ResourceDescription{ - Kinds: []string{"ConfigMap"}, - Name: resourceName, +// Match multiple kinds +func TestResourceDescriptionMatch_MultipleKind(t *testing.T) { + rawResource := []byte(`{ + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": { + "name": "nginx-deployment", + "labels": { + "app": "nginx" + } + }, + "spec": { + "replicas": 3, + "selector": { + "matchLabels": { + "app": "nginx" + } + }, + "template": { + "metadata": { + "labels": { + "app": "nginx" + } + }, + "spec": { + "containers": [ + { + "name": "nginx", + "image": "nginx:1.7.9", + "ports": [ + { + "containerPort": 80 + } + ] + } + ] + } + } + } + }`) + resource, err := ConvertToUnstructured(rawResource) + if err != nil { + t.Errorf("unable to convert raw resource to unstructured: %v", err) + + } + resourceDescription := kyverno.ResourceDescription{ + Kinds: []string{"Deployment", "Pods"}, Selector: &metav1.LabelSelector{ MatchLabels: nil, MatchExpressions: nil, }, } - excludeResourcesResourceDesc := types.ResourceDescription{} - groupVersionKind := metav1.GroupVersionKind{Kind: "ConfigMap"} + rule := kyverno.Rule{MatchResources: kyverno.MatchResources{resourceDescription}} - rawResource := []byte(`{ - "metadata":{ - "name":"test-config-map", - "namespace":"default", - "creationTimestamp":null, - "labels":{ - "label1":"test1", - "label2":"test2" - } - } - }`) - - assert.Assert(t, ResourceMeetsDescription(rawResource, resourceDescription, excludeResourcesResourceDesc, groupVersionKind)) - resourceDescription.Kinds[0] = "Deployment" - assert.Assert(t, false == ResourceMeetsDescription(rawResource, resourceDescription, excludeResourcesResourceDesc, groupVersionKind)) - resourceDescription.Kinds[0] = "ConfigMap" - groupVersionKind.Kind = "Deployment" - assert.Assert(t, false == ResourceMeetsDescription(rawResource, resourceDescription, excludeResourcesResourceDesc, groupVersionKind)) + assert.Assert(t, MatchesResourceDescription(*resource, rule)) } -func TestResourceMeetsDescription_Name(t *testing.T) { - resourceName := "test-config-map" - resourceDescription := types.ResourceDescription{ - Kinds: []string{"ConfigMap"}, - Name: resourceName, +// Match resource name +func TestResourceDescriptionMatch_Name(t *testing.T) { + rawResource := []byte(`{ + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": { + "name": "nginx-deployment", + "labels": { + "app": "nginx" + } + }, + "spec": { + "replicas": 3, + "selector": { + "matchLabels": { + "app": "nginx" + } + }, + "template": { + "metadata": { + "labels": { + "app": "nginx" + } + }, + "spec": { + "containers": [ + { + "name": "nginx", + "image": "nginx:1.7.9", + "ports": [ + { + "containerPort": 80 + } + ] + } + ] + } + } + } + }`) + resource, err := ConvertToUnstructured(rawResource) + if err != nil { + t.Errorf("unable to convert raw resource to unstructured: %v", err) + + } + resourceDescription := kyverno.ResourceDescription{ + Kinds: []string{"Deployment"}, + Name: "nginx-deployment", Selector: &metav1.LabelSelector{ MatchLabels: nil, MatchExpressions: nil, }, } - excludeResourcesResourceDesc := types.ResourceDescription{} + rule := kyverno.Rule{MatchResources: kyverno.MatchResources{resourceDescription}} - groupVersionKind := metav1.GroupVersionKind{Kind: "ConfigMap"} - - rawResource := []byte(`{ - "metadata":{ - "name":"test-config-map", - "namespace":"default", - "creationTimestamp":null, - "labels":{ - "label1":"test1", - "label2":"test2" - } - } - }`) - - assert.Assert(t, ResourceMeetsDescription(rawResource, resourceDescription, excludeResourcesResourceDesc, groupVersionKind)) - resourceDescription.Name = "test-config-map-new" - assert.Assert(t, false == ResourceMeetsDescription(rawResource, resourceDescription, excludeResourcesResourceDesc, groupVersionKind)) - - rawResource = []byte(`{ - "metadata":{ - "name":"test-config-map-new", - "namespace":"default", - "creationTimestamp":null, - "labels":{ - "label1":"test1", - "label2":"test2" - } - } - }`) - assert.Assert(t, ResourceMeetsDescription(rawResource, resourceDescription, excludeResourcesResourceDesc, groupVersionKind)) - - rawResource = []byte(`{ - "metadata":{ - "name":"", - "namespace":"default", - "creationTimestamp":null, - "labels":{ - "label1":"test1", - "label2":"test2" - } - } - }`) - assert.Assert(t, false == ResourceMeetsDescription(rawResource, resourceDescription, excludeResourcesResourceDesc, groupVersionKind)) + assert.Assert(t, MatchesResourceDescription(*resource, rule)) } -func TestResourceMeetsDescription_MatchExpressions(t *testing.T) { - resourceName := "test-config-map" - resourceDescription := types.ResourceDescription{ - Kinds: []string{"ConfigMap"}, - Name: resourceName, +// Match resource regex +func TestResourceDescriptionMatch_Name_Regex(t *testing.T) { + rawResource := []byte(`{ + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": { + "name": "nginx-deployment", + "labels": { + "app": "nginx" + } + }, + "spec": { + "replicas": 3, + "selector": { + "matchLabels": { + "app": "nginx" + } + }, + "template": { + "metadata": { + "labels": { + "app": "nginx" + } + }, + "spec": { + "containers": [ + { + "name": "nginx", + "image": "nginx:1.7.9", + "ports": [ + { + "containerPort": 80 + } + ] + } + ] + } + } + } + }`) + resource, err := ConvertToUnstructured(rawResource) + if err != nil { + t.Errorf("unable to convert raw resource to unstructured: %v", err) + + } + resourceDescription := kyverno.ResourceDescription{ + Kinds: []string{"Deployment"}, + Name: "nginx-*", + Selector: &metav1.LabelSelector{ + MatchLabels: nil, + MatchExpressions: nil, + }, + } + rule := kyverno.Rule{MatchResources: kyverno.MatchResources{resourceDescription}} + + assert.Assert(t, MatchesResourceDescription(*resource, rule)) +} + +// Match expressions for labels to not match +func TestResourceDescriptionMatch_Label_Expression_NotMatch(t *testing.T) { + rawResource := []byte(`{ + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": { + "name": "nginx-deployment", + "labels": { + "app": "nginx" + } + }, + "spec": { + "replicas": 3, + "selector": { + "matchLabels": { + "app": "nginx" + } + }, + "template": { + "metadata": { + "labels": { + "app": "nginx" + } + }, + "spec": { + "containers": [ + { + "name": "nginx", + "image": "nginx:1.7.9", + "ports": [ + { + "containerPort": 80 + } + ] + } + ] + } + } + } + }`) + resource, err := ConvertToUnstructured(rawResource) + if err != nil { + t.Errorf("unable to convert raw resource to unstructured: %v", err) + + } + resourceDescription := kyverno.ResourceDescription{ + Kinds: []string{"Deployment"}, + Name: "nginx-*", Selector: &metav1.LabelSelector{ MatchLabels: nil, MatchExpressions: []metav1.LabelSelectorRequirement{ @@ -112,561 +239,158 @@ func TestResourceMeetsDescription_MatchExpressions(t *testing.T) { "sometest1", }, }, - metav1.LabelSelectorRequirement{ - Key: "label1", - Operator: "In", - Values: []string{ - "test1", - "test8", - "test201", - }, - }, - metav1.LabelSelectorRequirement{ - Key: "label3", - Operator: "DoesNotExist", - Values: nil, - }, - metav1.LabelSelectorRequirement{ - Key: "label2", - Operator: "In", - Values: []string{ - "test2", - }, - }, }, }, } - excludeResourcesResourceDesc := types.ResourceDescription{} + rule := kyverno.Rule{MatchResources: kyverno.MatchResources{resourceDescription}} - groupVersionKind := metav1.GroupVersionKind{Kind: "ConfigMap"} - rawResource := []byte(`{ - "metadata":{ - "name":"test-config-map", - "namespace":"default", - "creationTimestamp":null, - "labels":{ - "label1":"test1", - "label2":"test2" - } - } - }`) - - assert.Assert(t, ResourceMeetsDescription(rawResource, resourceDescription, excludeResourcesResourceDesc, groupVersionKind)) - - rawResource = []byte(`{ - "metadata":{ - "name":"test-config-map", - "namespace":"default", - "creationTimestamp":null, - "labels":{ - "label1":"test1234567890", - "label2":"test2" - } - } - }`) - - assert.Assert(t, false == ResourceMeetsDescription(rawResource, resourceDescription, excludeResourcesResourceDesc, groupVersionKind)) + assert.Assert(t, MatchesResourceDescription(*resource, rule)) } -func TestResourceMeetsDescription_MatchLabels(t *testing.T) { - resourceName := "test-config-map" - resourceDescription := types.ResourceDescription{ - Kinds: []string{"ConfigMap"}, - Name: resourceName, - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "label1": "test1", - "label2": "test2", - }, - MatchExpressions: nil, - }, - } - groupVersionKind := metav1.GroupVersionKind{Kind: "ConfigMap"} - excludeResourcesResourceDesc := types.ResourceDescription{} - +// Match label expression in matching set +func TestResourceDescriptionMatch_Label_Expression_Match(t *testing.T) { rawResource := []byte(`{ - "metadata":{ - "name":"test-config-map", - "namespace":"default", - "creationTimestamp":null, - "labels":{ - "label1":"test1", - "label2":"test2" - } - } - }`) - assert.Assert(t, ResourceMeetsDescription(rawResource, resourceDescription, excludeResourcesResourceDesc, groupVersionKind)) - - rawResource = []byte(`{ - "metadata":{ - "name":"test-config-map", - "namespace":"default", - "creationTimestamp":null, - "labels":{ - "label3":"test1", - "label2":"test2" - } - } - }`) - assert.Assert(t, false == ResourceMeetsDescription(rawResource, resourceDescription, excludeResourcesResourceDesc, groupVersionKind)) - - resourceDescription = types.ResourceDescription{ - Kinds: []string{"ConfigMap"}, - Name: resourceName, - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "label3": "test1", - "label2": "test2", - }, - MatchExpressions: nil, + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": { + "name": "nginx-deployment", + "labels": { + "app": "nginx" + } }, + "spec": { + "replicas": 3, + "selector": { + "matchLabels": { + "app": "nginx" + } + }, + "template": { + "metadata": { + "labels": { + "app": "nginx" + } + }, + "spec": { + "containers": [ + { + "name": "nginx", + "image": "nginx:1.7.9", + "ports": [ + { + "containerPort": 80 + } + ] + } + ] + } + } + } + }`) + resource, err := ConvertToUnstructured(rawResource) + if err != nil { + t.Errorf("unable to convert raw resource to unstructured: %v", err) + } - - assert.Assert(t, ResourceMeetsDescription(rawResource, resourceDescription, excludeResourcesResourceDesc, groupVersionKind)) -} - -func TestResourceMeetsDescription_MatchLabelsAndMatchExpressions(t *testing.T) { - resourceName := "test-config-map" - resourceDescription := types.ResourceDescription{ - Kinds: []string{"ConfigMap"}, - Name: resourceName, + resourceDescription := kyverno.ResourceDescription{ + Kinds: []string{"Deployment"}, + Name: "nginx-*", Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "label1": "test1", - }, + MatchLabels: nil, MatchExpressions: []metav1.LabelSelectorRequirement{ metav1.LabelSelectorRequirement{ - Key: "label2", - Operator: "In", - Values: []string{ - "test2", - }, - }, - }, - }, - } - groupVersionKind := metav1.GroupVersionKind{Kind: "ConfigMap"} - excludeResourcesResourceDesc := types.ResourceDescription{} - - rawResource := []byte(`{ - "metadata":{ - "name":"test-config-map", - "namespace":"default", - "creationTimestamp":null, - "labels":{ - "label1":"test1", - "label2":"test2" - } - } - }`) - - assert.Assert(t, ResourceMeetsDescription(rawResource, resourceDescription, excludeResourcesResourceDesc, groupVersionKind)) - - resourceDescription = types.ResourceDescription{ - Kinds: []string{"ConfigMap"}, - Name: resourceName, - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "label1": "test1", - }, - MatchExpressions: []metav1.LabelSelectorRequirement{ - metav1.LabelSelectorRequirement{ - Key: "label2", + Key: "app", Operator: "NotIn", Values: []string{ - "sometest1", + "nginx1", + "nginx2", }, }, }, }, } + rule := kyverno.Rule{MatchResources: kyverno.MatchResources{resourceDescription}} - rawResource = []byte(`{ - "metadata":{ - "name":"test-config-map", - "namespace":"default", - "creationTimestamp":null, - "labels":{ - "label1":"test1", - "label2":"test2" - } - } - }`) - assert.Assert(t, ResourceMeetsDescription(rawResource, resourceDescription, excludeResourcesResourceDesc, groupVersionKind)) - - resourceDescription = types.ResourceDescription{ - Kinds: []string{"ConfigMap"}, - Name: resourceName, - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "label1": "test1", - }, - MatchExpressions: []metav1.LabelSelectorRequirement{ - metav1.LabelSelectorRequirement{ - Key: "label2", - Operator: "In", - Values: []string{ - "sometest1", - }, - }, - }, - }, - } - - assert.Assert(t, false == ResourceMeetsDescription(rawResource, resourceDescription, excludeResourcesResourceDesc, groupVersionKind)) - - resourceDescription = types.ResourceDescription{ - Kinds: []string{"ConfigMap"}, - Name: resourceName, - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "label1": "test1", - "label3": "test3", - }, - MatchExpressions: []metav1.LabelSelectorRequirement{ - metav1.LabelSelectorRequirement{ - Key: "label2", - Operator: "In", - Values: []string{ - "test2", - }, - }, - }, - }, - } - - assert.Assert(t, false == ResourceMeetsDescription(rawResource, resourceDescription, excludeResourcesResourceDesc, groupVersionKind)) + assert.Assert(t, MatchesResourceDescription(*resource, rule)) } -// func TestResourceMeetsDescription_Kind(t *testing.T) { -// resourceName := "test-config-map" -// resourceDescription := types.ResourceDescription{ -// Kinds: []string{"ConfigMap"}, -// Name: &resourceName, -// Selector: &metav1.LabelSelector{ -// MatchLabels: nil, -// MatchExpressions: nil, -// }, -// } -// excludeResourcesResourceDesc := types.ResourceDescription{} -// groupVersionKind := metav1.GroupVersionKind{Kind: "ConfigMap"} -// rawResource := []byte(`{ -// "metadata":{ -// "name":"test-config-map", -// "namespace":"default", -// "creationTimestamp":null, -// "labels":{ -// "label1":"test1", -// "label2":"test2" -// } -// } -// }`) +// check for exclude conditions +func TestResourceDescriptionExclude_Label_Expression_Match(t *testing.T) { + rawResource := []byte(`{ + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": { + "name": "nginx-deployment", + "labels": { + "app": "nginx", + "block": "true" + } + }, + "spec": { + "replicas": 3, + "selector": { + "matchLabels": { + "app": "nginx" + } + }, + "template": { + "metadata": { + "labels": { + "app": "nginx" + } + }, + "spec": { + "containers": [ + { + "name": "nginx", + "image": "nginx:1.7.9", + "ports": [ + { + "containerPort": 80 + } + ] + } + ] + } + } + } + }`) + resource, err := ConvertToUnstructured(rawResource) + if err != nil { + t.Errorf("unable to convert raw resource to unstructured: %v", err) -// assert.Assert(t, ResourceMeetsDescription(rawResource, resourceDescription, excludeResourcesResourceDesc, groupVersionKind)) -// resourceDescription.Kinds[0] = "Deployment" -// assert.Assert(t, false == ResourceMeetsDescription(rawResource, resourceDescription, excludeResourcesResourceDesc, groupVersionKind)) -// resourceDescription.Kinds[0] = "ConfigMap" -// groupVersionKind.Kind = "Deployment" -// assert.Assert(t, false == ResourceMeetsDescription(rawResource, resourceDescription, excludeResourcesResourceDesc, groupVersionKind)) -// } + } + resourceDescription := kyverno.ResourceDescription{ + Kinds: []string{"Deployment"}, + Name: "nginx-*", + Selector: &metav1.LabelSelector{ + MatchLabels: nil, + MatchExpressions: []metav1.LabelSelectorRequirement{ + metav1.LabelSelectorRequirement{ + Key: "app", + Operator: "NotIn", + Values: []string{ + "nginx1", + "nginx2", + }, + }, + }, + }, + } -// func TestResourceMeetsDescription_Name(t *testing.T) { -// resourceName := "test-config-map" -// resourceDescription := types.ResourceDescription{ -// Kinds: []string{"ConfigMap"}, -// Name: &resourceName, -// Selector: &metav1.LabelSelector{ -// MatchLabels: nil, -// MatchExpressions: nil, -// }, -// } -// excludeResourcesResourceDesc := types.ResourceDescription{} + resourceDescriptionExclude := kyverno.ResourceDescription{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "block": "true", + }, + }, + } -// groupVersionKind := metav1.GroupVersionKind{Kind: "ConfigMap"} + rule := kyverno.Rule{MatchResources: kyverno.MatchResources{resourceDescription}, + ExcludeResources: kyverno.ExcludeResources{resourceDescriptionExclude}} -// rawResource := []byte(`{ -// "metadata":{ -// "name":"test-config-map", -// "namespace":"default", -// "creationTimestamp":null, -// "labels":{ -// "label1":"test1", -// "label2":"test2" -// } -// } -// }`) - -// assert.Assert(t, ResourceMeetsDescription(rawResource, resourceDescription, excludeResourcesResourceDesc, groupVersionKind)) -// resourceName = "test-config-map-new" -// assert.Assert(t, false == ResourceMeetsDescription(rawResource, resourceDescription, excludeResourcesResourceDesc, groupVersionKind)) - -// rawResource = []byte(`{ -// "metadata":{ -// "name":"test-config-map-new", -// "namespace":"default", -// "creationTimestamp":null, -// "labels":{ -// "label1":"test1", -// "label2":"test2" -// } -// } -// }`) -// assert.Assert(t, ResourceMeetsDescription(rawResource, resourceDescription, excludeResourcesResourceDesc, groupVersionKind)) - -// rawResource = []byte(`{ -// "metadata":{ -// "name":"", -// "namespace":"default", -// "creationTimestamp":null, -// "labels":{ -// "label1":"test1", -// "label2":"test2" -// } -// } -// }`) -// assert.Assert(t, false == ResourceMeetsDescription(rawResource, resourceDescription, excludeResourcesResourceDesc, groupVersionKind)) -// } - -// func TestResourceMeetsDescription_MatchExpressions(t *testing.T) { -// resourceName := "test-config-map" -// resourceDescription := types.ResourceDescription{ -// Kinds: []string{"ConfigMap"}, -// Name: &resourceName, -// Selector: &metav1.LabelSelector{ -// MatchLabels: nil, -// MatchExpressions: []metav1.LabelSelectorRequirement{ -// metav1.LabelSelectorRequirement{ -// Key: "label2", -// Operator: "NotIn", -// Values: []string{ -// "sometest1", -// }, -// }, -// metav1.LabelSelectorRequirement{ -// Key: "label1", -// Operator: "In", -// Values: []string{ -// "test1", -// "test8", -// "test201", -// }, -// }, -// metav1.LabelSelectorRequirement{ -// Key: "label3", -// Operator: "DoesNotExist", -// Values: nil, -// }, -// metav1.LabelSelectorRequirement{ -// Key: "label2", -// Operator: "In", -// Values: []string{ -// "test2", -// }, -// }, -// }, -// }, -// } -// excludeResourcesResourceDesc := types.ResourceDescription{} - -// groupVersionKind := metav1.GroupVersionKind{Kind: "ConfigMap"} -// rawResource := []byte(`{ -// "metadata":{ -// "name":"test-config-map", -// "namespace":"default", -// "creationTimestamp":null, -// "labels":{ -// "label1":"test1", -// "label2":"test2" -// } -// } -// }`) - -// assert.Assert(t, ResourceMeetsDescription(rawResource, resourceDescription, excludeResourcesResourceDesc, groupVersionKind)) - -// rawResource = []byte(`{ -// "metadata":{ -// "name":"test-config-map", -// "namespace":"default", -// "creationTimestamp":null, -// "labels":{ -// "label1":"test1234567890", -// "label2":"test2" -// } -// } -// }`) - -// assert.Assert(t, false == ResourceMeetsDescription(rawResource, resourceDescription, excludeResourcesResourceDesc, groupVersionKind)) -// } - -// func TestResourceMeetsDescription_MatchLabels(t *testing.T) { -// resourceName := "test-config-map" -// resourceDescription := types.ResourceDescription{ -// Kinds: []string{"ConfigMap"}, -// Name: &resourceName, -// Selector: &metav1.LabelSelector{ -// MatchLabels: map[string]string{ -// "label1": "test1", -// "label2": "test2", -// }, -// MatchExpressions: nil, -// }, -// } -// groupVersionKind := metav1.GroupVersionKind{Kind: "ConfigMap"} -// excludeResourcesResourceDesc := types.ResourceDescription{} - -// rawResource := []byte(`{ -// "metadata":{ -// "name":"test-config-map", -// "namespace":"default", -// "creationTimestamp":null, -// "labels":{ -// "label1":"test1", -// "label2":"test2" -// } -// } -// }`) -// assert.Assert(t, ResourceMeetsDescription(rawResource, resourceDescription, excludeResourcesResourceDesc, groupVersionKind)) - -// rawResource = []byte(`{ -// "metadata":{ -// "name":"test-config-map", -// "namespace":"default", -// "creationTimestamp":null, -// "labels":{ -// "label3":"test1", -// "label2":"test2" -// } -// } -// }`) -// assert.Assert(t, false == ResourceMeetsDescription(rawResource, resourceDescription, excludeResourcesResourceDesc, groupVersionKind)) - -// resourceDescription = types.ResourceDescription{ -// Kinds: []string{"ConfigMap"}, -// Name: &resourceName, -// Selector: &metav1.LabelSelector{ -// MatchLabels: map[string]string{ -// "label3": "test1", -// "label2": "test2", -// }, -// MatchExpressions: nil, -// }, -// } - -// assert.Assert(t, ResourceMeetsDescription(rawResource, resourceDescription, excludeResourcesResourceDesc, groupVersionKind)) -// } - -// func TestResourceMeetsDescription_MatchLabelsAndMatchExpressions(t *testing.T) { -// resourceName := "test-config-map" -// resourceDescription := types.ResourceDescription{ -// Kinds: []string{"ConfigMap"}, -// Name: &resourceName, -// Selector: &metav1.LabelSelector{ -// MatchLabels: map[string]string{ -// "label1": "test1", -// }, -// MatchExpressions: []metav1.LabelSelectorRequirement{ -// metav1.LabelSelectorRequirement{ -// Key: "label2", -// Operator: "In", -// Values: []string{ -// "test2", -// }, -// }, -// }, -// }, -// } -// groupVersionKind := metav1.GroupVersionKind{Kind: "ConfigMap"} -// excludeResourcesResourceDesc := types.ResourceDescription{} - -// rawResource := []byte(`{ -// "metadata":{ -// "name":"test-config-map", -// "namespace":"default", -// "creationTimestamp":null, -// "labels":{ -// "label1":"test1", -// "label2":"test2" -// } -// } -// }`) - -// assert.Assert(t, ResourceMeetsDescription(rawResource, resourceDescription, excludeResourcesResourceDesc, groupVersionKind)) - -// resourceDescription = types.ResourceDescription{ -// Kinds: []string{"ConfigMap"}, -// Name: &resourceName, -// Selector: &metav1.LabelSelector{ -// MatchLabels: map[string]string{ -// "label1": "test1", -// }, -// MatchExpressions: []metav1.LabelSelectorRequirement{ -// metav1.LabelSelectorRequirement{ -// Key: "label2", -// Operator: "NotIn", -// Values: []string{ -// "sometest1", -// }, -// }, -// }, -// }, -// } - -// rawResource = []byte(`{ -// "metadata":{ -// "name":"test-config-map", -// "namespace":"default", -// "creationTimestamp":null, -// "labels":{ -// "label1":"test1", -// "label2":"test2" -// } -// } -// }`) -// assert.Assert(t, ResourceMeetsDescription(rawResource, resourceDescription, excludeResourcesResourceDesc, groupVersionKind)) - -// resourceDescription = types.ResourceDescription{ -// Kinds: []string{"ConfigMap"}, -// Name: &resourceName, -// Selector: &metav1.LabelSelector{ -// MatchLabels: map[string]string{ -// "label1": "test1", -// }, -// MatchExpressions: []metav1.LabelSelectorRequirement{ -// metav1.LabelSelectorRequirement{ -// Key: "label2", -// Operator: "In", -// Values: []string{ -// "sometest1", -// }, -// }, -// }, -// }, -// } - -// assert.Assert(t, false == ResourceMeetsDescription(rawResource, resourceDescription, excludeResourcesResourceDesc, groupVersionKind)) - -// resourceDescription = types.ResourceDescription{ -// Kinds: []string{"ConfigMap"}, -// Name: &resourceName, -// Selector: &metav1.LabelSelector{ -// MatchLabels: map[string]string{ -// "label1": "test1", -// "label3": "test3", -// }, -// MatchExpressions: []metav1.LabelSelectorRequirement{ -// metav1.LabelSelectorRequirement{ -// Key: "label2", -// Operator: "In", -// Values: []string{ -// "test2", -// }, -// }, -// }, -// }, -// } - -// assert.Assert(t, false == ResourceMeetsDescription(rawResource, resourceDescription, excludeResourcesResourceDesc, groupVersionKind)) -// } + assert.Assert(t, !MatchesResourceDescription(*resource, rule)) +} func TestWrappedWithParentheses_StringIsWrappedWithParentheses(t *testing.T) { str := "(something)" diff --git a/pkg/testrunner/test.go b/pkg/testrunner/test.go index bd2104f97b..b1cc4e0144 100644 --- a/pkg/testrunner/test.go +++ b/pkg/testrunner/test.go @@ -2,15 +2,14 @@ package testrunner import ( "fmt" + "reflect" "strconv" "testing" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - ospath "path" "github.com/golang/glog" - pt "github.com/nirmata/kyverno/pkg/apis/policy/v1alpha1" + kyverno "github.com/nirmata/kyverno/pkg/api/kyverno/v1alpha1" client "github.com/nirmata/kyverno/pkg/dclient" "github.com/nirmata/kyverno/pkg/engine" "github.com/nirmata/kyverno/pkg/info" @@ -22,7 +21,7 @@ type test struct { t *testing.T testCase *testCase // input - policy *pt.Policy + policy *kyverno.Policy tResource *resourceInfo loadResources []*resourceInfo // expected @@ -64,7 +63,7 @@ func (t *test) run() { t.checkGenerationResult(client, policyInfo) } -func (t *test) checkMutationResult(pr *resourceInfo, policyInfo *info.PolicyInfo) { +func (t *test) checkMutationResult(pr *resourceInfo, policyInfo info.PolicyInfo) { if t.testCase.Expected.Mutation == nil { glog.Info("No Mutation check defined") return @@ -91,12 +90,12 @@ func (t *test) overAllPass(result bool, expected string) { } } -func (t *test) compareRules(ruleInfos []*info.RuleInfo, rules []tRules) { +func (t *test) compareRules(ruleInfos []info.RuleInfo, rules []tRules) { // Compare the rules specified in the expected against the actual rule info returned by the apply policy for _, eRule := range rules { // Look-up the rule from the policy info rule := lookUpRule(eRule.Name, ruleInfos) - if rule == nil { + if reflect.DeepEqual(rule, info.RuleInfo{}) { t.t.Errorf("Rule with name %s not found", eRule.Name) continue } @@ -118,16 +117,17 @@ func (t *test) compareRules(ruleInfos []*info.RuleInfo, rules []tRules) { } } -func lookUpRule(name string, ruleInfos []*info.RuleInfo) *info.RuleInfo { +func lookUpRule(name string, ruleInfos []info.RuleInfo) info.RuleInfo { + for _, r := range ruleInfos { if r.Name == name { return r } } - return nil + return info.RuleInfo{} } -func (t *test) checkValidationResult(policyInfo *info.PolicyInfo) { +func (t *test) checkValidationResult(policyInfo info.PolicyInfo) { if t.testCase.Expected.Validation == nil { glog.Info("No Validation check defined") return @@ -137,7 +137,7 @@ func (t *test) checkValidationResult(policyInfo *info.PolicyInfo) { t.compareRules(policyInfo.Rules, t.testCase.Expected.Validation.Rules) } -func (t *test) checkGenerationResult(client *client.Client, policyInfo *info.PolicyInfo) { +func (t *test) checkGenerationResult(client *client.Client, policyInfo info.PolicyInfo) { if t.testCase.Expected.Generation == nil { glog.Info("No Generate check defined") return @@ -162,11 +162,12 @@ func (t *test) checkGenerationResult(client *client.Client, policyInfo *info.Pol } } -func (t *test) applyPolicy(policy *pt.Policy, +func (t *test) applyPolicy(policy *kyverno.Policy, tresource *resourceInfo, - client *client.Client) (*resourceInfo, *info.PolicyInfo, error) { + client *client.Client) (*resourceInfo, info.PolicyInfo, error) { // apply policy on the trigger resource // Mutate + var zeroPolicyInfo info.PolicyInfo var err error rawResource := tresource.rawResource rname := engine.ParseNameFromObject(rawResource) @@ -177,42 +178,43 @@ func (t *test) applyPolicy(policy *pt.Policy, rname, rns, policy.Spec.ValidationFailureAction) + + resource, err := ConvertToUnstructured(rawResource) + if err != nil { + return nil, zeroPolicyInfo, err + } + // Apply Mutation Rules - patches, ruleInfos := engine.Mutate(*policy, rawResource, *tresource.gvk) - policyInfo.AddRuleInfos(ruleInfos) + engineResponse := engine.Mutate(*policy, *resource) + // patches, ruleInfos := engine.Mutate(*policy, rawResource, *tresource.gvk) + policyInfo.AddRuleInfos(engineResponse.RuleInfos) // TODO: only validate if there are no errors in mutate, why? if policyInfo.IsSuccessful() { - if len(patches) != 0 { - rawResource, err = engine.ApplyPatches(rawResource, patches) + if len(engineResponse.Patches) != 0 { + rawResource, err = engine.ApplyPatches(rawResource, engineResponse.Patches) if err != nil { - return nil, nil, err + return nil, zeroPolicyInfo, err } } } // Validate - ruleInfos, err = engine.Validate(*policy, rawResource, *tresource.gvk) - policyInfo.AddRuleInfos(ruleInfos) + engineResponse = engine.Validate(*policy, *resource) + policyInfo.AddRuleInfos(engineResponse.RuleInfos) if err != nil { - return nil, nil, err + return nil, zeroPolicyInfo, err } if rkind == "Namespace" { if client != nil { - // convert []byte to unstructured - unstr := unstructured.Unstructured{} - err := unstr.UnmarshalJSON(rawResource) - if err != nil { - glog.Error(err) - } - ruleInfos := engine.Generate(client, policy, unstr) - policyInfo.AddRuleInfos(ruleInfos) + engineResponse := engine.Generate(client, *policy, *resource) + policyInfo.AddRuleInfos(engineResponse.RuleInfos) } } // Generate // transform the patched Resource into resource Info ri, err := extractResourceRaw(rawResource) if err != nil { - return nil, nil, err + return nil, zeroPolicyInfo, err } // return the results return ri, policyInfo, nil diff --git a/pkg/testrunner/testcase.go b/pkg/testrunner/testcase.go index 80363c7b1e..99ba30c733 100644 --- a/pkg/testrunner/testcase.go +++ b/pkg/testrunner/testcase.go @@ -7,7 +7,7 @@ import ( ospath "path" "github.com/golang/glog" - pt "github.com/nirmata/kyverno/pkg/apis/policy/v1alpha1" + kyverno "github.com/nirmata/kyverno/pkg/api/kyverno/v1alpha1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" yaml "k8s.io/apimachinery/pkg/util/yaml" @@ -117,8 +117,8 @@ func (tc *testCase) loadTriggerResource(ap string) (*resourceInfo, error) { } // Loads a single policy -func (tc *testCase) loadPolicy(file string) (*pt.Policy, error) { - p := &pt.Policy{} +func (tc *testCase) loadPolicy(file string) (*kyverno.Policy, error) { + p := &kyverno.Policy{} data, err := LoadFile(file) if err != nil { return nil, err diff --git a/pkg/testrunner/testrunner_test.go b/pkg/testrunner/testrunner_test.go index 9d1155e0c1..25ab75111f 100644 --- a/pkg/testrunner/testrunner_test.go +++ b/pkg/testrunner/testrunner_test.go @@ -3,5 +3,7 @@ package testrunner import "testing" func TestCLI(t *testing.T) { + //https://github.com/nirmata/kyverno/issues/301 + t.Skip("skipping testrunner as this needs a re-design") runner(t, "/test/scenarios/cli") } diff --git a/pkg/testrunner/utils.go b/pkg/testrunner/utils.go index eadd4c3f42..947a4b5ad3 100644 --- a/pkg/testrunner/utils.go +++ b/pkg/testrunner/utils.go @@ -121,3 +121,13 @@ func ParseNamespaceFromObject(bytes []byte) string { } return "" } + +func ConvertToUnstructured(data []byte) (*unstructured.Unstructured, error) { + resource := &unstructured.Unstructured{} + err := resource.UnmarshalJSON(data) + if err != nil { + glog.V(4).Infof("failed to unmarshall resource: %v", err) + return nil, err + } + return resource, nil +} diff --git a/pkg/webhooks/resources/CAFile b/pkg/webhookconfig/resources/CAFile similarity index 100% rename from pkg/webhooks/resources/CAFile rename to pkg/webhookconfig/resources/CAFile