From e0f617b383f6fb0dafa3ecafb89cd36998c3d02e Mon Sep 17 00:00:00 2001 From: shuting Date: Tue, 1 Sep 2020 09:11:20 -0700 Subject: [PATCH] 810 support cronJob for auto-gen (#1089) * add watch policy to clusterrole kyverno:customresources * - improve auto-gen policy application logic - remove unused code * move method to common util * auto-gen rule for cronJob * update doc * set CronJob as default auto-gen pod controller * - update doc; - fix test * remove unused code --- documentation/writing-policies-autogen.md | 2 +- pkg/engine/mutation.go | 8 +- pkg/engine/utils.go | 25 ++++- pkg/engine/validation.go | 4 +- pkg/kyverno/apply/command.go | 11 +-- pkg/kyverno/common/common.go | 63 +----------- pkg/policy/existing.go | 42 +------- pkg/policymutation/cronjob.go | 96 ++++++++++++++++++ pkg/policymutation/policymutation.go | 105 +++++++++++--------- pkg/policymutation/policymutation_test.go | 114 ++++++++++++++++++++++ pkg/utils/loadpolicy.go | 69 +++++++++++++ pkg/webhooks/policymutation_test.go | 88 +++++++++++++++-- 12 files changed, 462 insertions(+), 165 deletions(-) create mode 100644 pkg/policymutation/cronjob.go create mode 100644 pkg/policymutation/policymutation_test.go create mode 100644 pkg/utils/loadpolicy.go diff --git a/documentation/writing-policies-autogen.md b/documentation/writing-policies-autogen.md index 790806b0ba..58b32a3807 100644 --- a/documentation/writing-policies-autogen.md +++ b/documentation/writing-policies-autogen.md @@ -12,7 +12,7 @@ Kyverno solves this issue by supporting automatic generation of policy rules for This auto-generation behavior is controlled by the `pod-policies.kyverno.io/autogen-controllers` annotation. -By default, Kyverno inserts an annotation `pod-policies.kyverno.io/autogen-controllers=all`, to generate an additional rule that is applied to pod controllers: DaemonSet, Deployment, Job, StatefulSet. +By default, Kyverno inserts an annotation `pod-policies.kyverno.io/autogen-controllers=DaemonSet,Deployment,Job,StatefulSet,CrobJob`, to generate additional rules that are applied to these pod controllers. You can change the annotation `pod-policies.kyverno.io/autogen-controllers` to customize the target pod controllers for the auto-generated rules. For example, Kyverno generates a rule for a `Deployment` if the annotation of policy is defined as `pod-policies.kyverno.io/autogen-controllers=Deployment`. diff --git a/pkg/engine/mutation.go b/pkg/engine/mutation.go index 713b144a83..13d84dba80 100644 --- a/pkg/engine/mutation.go +++ b/pkg/engine/mutation.go @@ -13,12 +13,12 @@ import ( ) const ( + // PodControllerCronJob represent CronJob string + PodControllerCronJob = "CronJob" //PodControllers stores the list of Pod-controllers in csv string - PodControllers = "DaemonSet,Deployment,Job,StatefulSet" + PodControllers = "DaemonSet,Deployment,Job,StatefulSet,CronJob" //PodControllersAnnotation defines the annotation key for Pod-Controllers PodControllersAnnotation = "pod-policies.kyverno.io/autogen-controllers" - //PodTemplateAnnotation defines the annotation key for Pod-Template - PodTemplateAnnotation = "pod-policies.kyverno.io/autogen-applied" ) // Mutate performs mutation. Overlay first and then mutation patches @@ -35,7 +35,7 @@ func Mutate(policyContext PolicyContext) (resp response.EngineResponse) { startMutateResultResponse(&resp, policy, patchedResource) defer endMutateResultResponse(logger, &resp, startTime) - if policy.HasAutoGenAnnotation() && excludePod(patchedResource) { + if SkipPolicyApplication(policy, patchedResource) { logger.V(5).Info("Skip applying policy, Pod has ownerRef set", "policy", policy.GetName()) resp.PatchedResource = patchedResource return diff --git a/pkg/engine/utils.go b/pkg/engine/utils.go index caae9d4534..6acba14321 100644 --- a/pkg/engine/utils.go +++ b/pkg/engine/utils.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "reflect" + "strings" "time" "github.com/nirmata/kyverno/pkg/utils" @@ -249,9 +250,10 @@ func copyConditions(original []kyverno.Condition) []kyverno.Condition { return copy } -// excludePod checks if a Pod has ownerRef set -func excludePod(resource unstructured.Unstructured) bool { - if resource.GetKind() == "Pod" { +// excludeResource checks if the resource has ownerRef set +func excludeResource(resource unstructured.Unstructured) bool { + kind := resource.GetKind() + if kind == "Pod" || kind == "Job" { if len(resource.GetOwnerReferences()) > 0 { return true } @@ -259,3 +261,20 @@ func excludePod(resource unstructured.Unstructured) bool { return false } + +// SkipPolicyApplication returns true: +// - if the policy has auto-gen annotation && resource == Pod +// - if the auto-gen contains cronJob && resource == Job +func SkipPolicyApplication(policy kyverno.ClusterPolicy, resource unstructured.Unstructured) bool { + if policy.HasAutoGenAnnotation() && excludeResource(resource) { + return true + } + + if podControllers, ok := policy.GetAnnotations()[PodControllersAnnotation]; ok { + if strings.Contains(podControllers, "CronJob") && excludeResource(resource) { + return true + } + } + + return false +} diff --git a/pkg/engine/validation.go b/pkg/engine/validation.go index 91e2ca59c6..6c16bd6406 100644 --- a/pkg/engine/validation.go +++ b/pkg/engine/validation.go @@ -104,7 +104,7 @@ func incrementAppliedCount(resp *response.EngineResponse) { func isRequestDenied(log logr.Logger, ctx context.EvalInterface, policy kyverno.ClusterPolicy, resource unstructured.Unstructured, admissionInfo kyverno.RequestInfo, excludeGroupRole []string) *response.EngineResponse { resp := &response.EngineResponse{} - if policy.HasAutoGenAnnotation() && excludePod(resource) { + if SkipPolicyApplication(policy, resource) { log.V(5).Info("Skip applying policy, Pod has ownerRef set", "policy", policy.GetName()) return resp } @@ -150,7 +150,7 @@ func isRequestDenied(log logr.Logger, ctx context.EvalInterface, policy kyverno. func validateResource(log logr.Logger, ctx context.EvalInterface, policy kyverno.ClusterPolicy, resource unstructured.Unstructured, admissionInfo kyverno.RequestInfo, excludeGroupRole []string) *response.EngineResponse { resp := &response.EngineResponse{} - if policy.HasAutoGenAnnotation() && excludePod(resource) { + if SkipPolicyApplication(policy, resource) { log.V(5).Info("Skip applying policy, Pod has ownerRef set", "policy", policy.GetName()) return resp } diff --git a/pkg/kyverno/apply/command.go b/pkg/kyverno/apply/command.go index b2b4001332..6aac56ceec 100644 --- a/pkg/kyverno/apply/command.go +++ b/pkg/kyverno/apply/command.go @@ -42,13 +42,12 @@ import ( log "sigs.k8s.io/controller-runtime/pkg/log" ) - type resultCounts struct { - pass int - fail int - warn int + pass int + fail int + warn int error int - skip int + skip int } func Command() *cobra.Command { @@ -325,7 +324,7 @@ func getResource(path string) ([]*unstructured.Unstructured, error) { return nil, err } - files, splitDocError := common.SplitYAMLDocuments(file) + files, splitDocError := utils.SplitYAMLDocuments(file) if splitDocError != nil { return nil, splitDocError } diff --git a/pkg/kyverno/common/common.go b/pkg/kyverno/common/common.go index 62153fc296..0d994499b6 100644 --- a/pkg/kyverno/common/common.go +++ b/pkg/kyverno/common/common.go @@ -1,11 +1,8 @@ package common import ( - "bufio" - "bytes" "encoding/json" "fmt" - "io" "io/ioutil" "os" "path/filepath" @@ -21,7 +18,7 @@ import ( "github.com/nirmata/kyverno/pkg/kyverno/sanitizedError" "github.com/nirmata/kyverno/pkg/openapi" "github.com/nirmata/kyverno/pkg/policymutation" - "k8s.io/apimachinery/pkg/util/yaml" + "github.com/nirmata/kyverno/pkg/utils" log "sigs.k8s.io/controller-runtime/pkg/log" ) @@ -56,7 +53,7 @@ func GetPolicies(paths []string) (policies []*v1.ClusterPolicy, error error) { policies = append(policies, policiesFromDir...) } else { - getPolicies, getErrors := GetPolicy(path) + getPolicies, getErrors := utils.GetPolicy(path) var errString string for _, err := range getErrors { if err != nil { @@ -76,62 +73,6 @@ func GetPolicies(paths []string) (policies []*v1.ClusterPolicy, error error) { return policies, nil } -// GetPolicy - Extracts policies from a YAML -func GetPolicy(path string) (clusterPolicies []*v1.ClusterPolicy, errors []error) { - file, err := ioutil.ReadFile(path) - if err != nil { - errors = append(errors, fmt.Errorf(fmt.Sprintf("failed to load file: %v. error: %v", path, err))) - return clusterPolicies, errors - } - - policies, err := SplitYAMLDocuments(file) - if err != nil { - errors = append(errors, err) - return clusterPolicies, errors - } - - for _, thisPolicyBytes := range policies { - policyBytes, err := yaml.ToJSON(thisPolicyBytes) - if err != nil { - errors = append(errors, fmt.Errorf(fmt.Sprintf("failed to convert json. error: %v", err))) - continue - } - - policy := &v1.ClusterPolicy{} - if err := json.Unmarshal(policyBytes, policy); err != nil { - errors = append(errors, fmt.Errorf(fmt.Sprintf("failed to decode policy in %s. error: %v", path, err))) - continue - } - - if policy.TypeMeta.Kind != "ClusterPolicy" { - errors = append(errors, fmt.Errorf(fmt.Sprintf("resource %v is not a cluster policy", policy.Name))) - continue - } - clusterPolicies = append(clusterPolicies, policy) - } - - return clusterPolicies, errors -} - -// SplitYAMLDocuments reads the YAML bytes per-document, unmarshals the TypeMeta information from each document -// and returns a map between the GroupVersionKind of the document and the document bytes -func SplitYAMLDocuments(yamlBytes []byte) (policies [][]byte, error error) { - buf := bytes.NewBuffer(yamlBytes) - reader := yaml.NewYAMLReader(bufio.NewReader(buf)) - for { - // Read one YAML document at a time, until io.EOF is returned - b, err := reader.Read() - if err == io.EOF || len(b) == 0 { - break - } else if err != nil { - return policies, fmt.Errorf("unable to read yaml") - } - - policies = append(policies, b) - } - return policies, error -} - //GetPoliciesValidation - validating policies func GetPoliciesValidation(policyPaths []string) ([]*v1.ClusterPolicy, *openapi.Controller, error) { policies, err := GetPolicies(policyPaths) diff --git a/pkg/policy/existing.go b/pkg/policy/existing.go index 11298d1879..a819aeabb5 100644 --- a/pkg/policy/existing.go +++ b/pkg/policy/existing.go @@ -36,16 +36,6 @@ func (pc *PolicyController) processExistingResources(policy *kyverno.ClusterPoli continue } - // skip reporting violation on pod which has annotation pod-policies.kyverno.io/autogen-applied - ann := policy.GetAnnotations() - if annValue, ok := ann[engine.PodControllersAnnotation]; ok { - if annValue != "none" { - if skipPodApplication(resource, logger) { - continue - } - } - } - // apply the policy on each engineResponse := applyPolicy(*policy, resource, logger, pc.configHandler.GetExcludeGroupRole()) // get engine response for mutation & validation independently @@ -84,22 +74,14 @@ func (pc *PolicyController) listResources(policy *kyverno.ClusterPolicy) map[str } } - if policy.HasAutoGenAnnotation() { - return excludePod(resourceMap, pc.log) - } - - return resourceMap + return excludeAutoGenResources(*policy, resourceMap, pc.log) } -// excludePod filter out the pods with ownerReference -func excludePod(resourceMap map[string]unstructured.Unstructured, log logr.Logger) map[string]unstructured.Unstructured { +// excludeAutoGenResources filter out the pods / jobs with ownerReference +func excludeAutoGenResources(policy kyverno.ClusterPolicy, resourceMap map[string]unstructured.Unstructured, log logr.Logger) map[string]unstructured.Unstructured { for uid, r := range resourceMap { - if r.GetKind() != "Pod" { - continue - } - - if len(r.GetOwnerReferences()) > 0 { - log.V(4).Info("exclude Pod", "namespace", r.GetNamespace(), "name", r.GetName()) + if engine.SkipPolicyApplication(policy, r) { + log.V(4).Info("exclude resource", "namespace", r.GetNamespace(), "kind", r.GetKind(), "name", r.GetName()) delete(resourceMap, uid) } } @@ -401,17 +383,3 @@ func (rm *ResourceManager) ProcessResource(policy, pv, kind, ns, name, rv string func buildKey(policy, pv, kind, ns, name, rv string) string { return policy + "/" + pv + "/" + kind + "/" + ns + "/" + name + "/" + rv } - -func skipPodApplication(resource unstructured.Unstructured, log logr.Logger) bool { - if resource.GetKind() != "Pod" { - return false - } - - annotation := resource.GetAnnotations() - if _, ok := annotation[engine.PodTemplateAnnotation]; ok { - log.V(4).Info("Policies already processed on pod controllers, skip processing policy on Pod", "kind", resource.GetKind(), "namespace", resource.GetNamespace(), "name", resource.GetName()) - return true - } - - return false -} diff --git a/pkg/policymutation/cronjob.go b/pkg/policymutation/cronjob.go new file mode 100644 index 0000000000..bd88cbce5d --- /dev/null +++ b/pkg/policymutation/cronjob.go @@ -0,0 +1,96 @@ +package policymutation + +import ( + "fmt" + "strings" + + "github.com/go-logr/logr" + kyverno "github.com/nirmata/kyverno/pkg/api/kyverno/v1" + "github.com/nirmata/kyverno/pkg/engine" +) + +func generateCronJobRule(rule kyverno.Rule, controllers string, log logr.Logger) kyvernoRule { + logger := log.WithName("handleCronJob") + + hasCronJob := strings.Contains(controllers, engine.PodControllerCronJob) || strings.Contains(controllers, "all") + if !hasCronJob { + return kyvernoRule{} + } + + logger.V(3).Info("") + jobRule := generateRuleForControllers(rule, "Job", logger) + + cronJobRule := &jobRule + cronJobRule.Name = fmt.Sprintf("autogen-cronjob-%s", rule.Name) + cronJobRule.MatchResources.Kinds = []string{engine.PodControllerCronJob} + if (jobRule.ExcludeResources) != nil && (len(jobRule.ExcludeResources.Kinds) > 0) { + cronJobRule.ExcludeResources.Kinds = []string{engine.PodControllerCronJob} + } + + if (jobRule.Mutation != nil) && (jobRule.Mutation.Overlay != nil) { + newMutation := &kyverno.Mutation{ + Overlay: map[string]interface{}{ + "spec": map[string]interface{}{ + "jobTemplate": jobRule.Mutation.Overlay, + }, + }, + } + + cronJobRule.Mutation = newMutation.DeepCopy() + return *cronJobRule + } + + if (jobRule.Validation != nil) && (jobRule.Validation.Pattern != nil) { + newValidate := &kyverno.Validation{ + Message: rule.Validation.Message, + Pattern: map[string]interface{}{ + "spec": map[string]interface{}{ + "jobTemplate": jobRule.Validation.Pattern, + }, + }, + } + cronJobRule.Validation = newValidate.DeepCopy() + return *cronJobRule + } + + if (jobRule.Validation != nil) && (len(jobRule.Validation.AnyPattern) != 0) { + var patterns []interface{} + for _, pattern := range jobRule.Validation.AnyPattern { + newPattern := map[string]interface{}{ + "spec": map[string]interface{}{ + "jobTemplate": pattern, + }, + } + + patterns = append(patterns, newPattern) + } + + cronJobRule.Validation = &kyverno.Validation{ + Message: rule.Validation.Message, + AnyPattern: patterns, + } + return *cronJobRule + } + + return kyvernoRule{} +} + +// stripCronJob removes CronJob from controllers +func stripCronJob(controllers string) string { + var newControllers []string + + controllerArr := strings.Split(controllers, ",") + for _, c := range controllerArr { + if c == engine.PodControllerCronJob { + continue + } + + newControllers = append(newControllers, c) + } + + if len(newControllers) == 0 { + return "" + } + + return strings.Join(newControllers, ",") +} diff --git a/pkg/policymutation/policymutation.go b/pkg/policymutation/policymutation.go index c0cf128c70..58b47b9d1f 100644 --- a/pkg/policymutation/policymutation.go +++ b/pkg/policymutation/policymutation.go @@ -121,7 +121,7 @@ func GeneratePodControllerRule(policy kyverno.ClusterPolicy, log logr.Logger) (p // scenario A if !ok { - controllers = "DaemonSet,Deployment,Job,StatefulSet" + controllers = engine.PodControllers annPatch, err := defaultPodControllerAnnotation(ann) if err != nil { errs = append(errs, fmt.Errorf("failed to generate pod controller annotation for policy '%s': %v", policy.Name, err)) @@ -173,8 +173,6 @@ func createRuleMap(rules []kyverno.Rule) map[string]kyvernoRule { // generateRulePatches generates rule for podControllers based on scenario A and C func generateRulePatches(policy kyverno.ClusterPolicy, controllers string, log logr.Logger) (rulePatches [][]byte, errs []error) { - var genRule kyvernoRule - insertIdx := len(policy.Spec.Rules) ruleMap := createRuleMap(policy.Spec.Rules) @@ -186,48 +184,62 @@ func generateRulePatches(policy kyverno.ClusterPolicy, controllers string, log l for _, rule := range policy.Spec.Rules { patchPostion := insertIdx - genRule = generateRuleForControllers(rule, controllers, log) - if reflect.DeepEqual(genRule, kyvernoRule{}) { - continue - } + convertToPatches := func(genRule kyvernoRule, patchPostion int) []byte { + operation := "add" + if existingAutoGenRule, alreadyExists := ruleMap[genRule.Name]; alreadyExists { + existingAutoGenRuleRaw, _ := json.Marshal(existingAutoGenRule) + genRuleRaw, _ := json.Marshal(genRule) - operation := "add" - if existingAutoGenRule, alreadyExists := ruleMap[genRule.Name]; alreadyExists { - existingAutoGenRuleRaw, _ := json.Marshal(existingAutoGenRule) - genRuleRaw, _ := json.Marshal(genRule) - - if string(existingAutoGenRuleRaw) == string(genRuleRaw) { - continue + if string(existingAutoGenRuleRaw) == string(genRuleRaw) { + return nil + } + operation = "replace" + patchPostion = ruleIndex[genRule.Name] } - operation = "replace" - patchPostion = ruleIndex[genRule.Name] + + // generate patch bytes + jsonPatch := struct { + Path string `json:"path"` + Op string `json:"op"` + Value interface{} `json:"value"` + }{ + fmt.Sprintf("/spec/rules/%s", strconv.Itoa(patchPostion)), + operation, + genRule, + } + pbytes, err := json.Marshal(jsonPatch) + if err != nil { + errs = append(errs, err) + return nil + } + + // check the patch + if _, err := jsonpatch.DecodePatch([]byte("[" + string(pbytes) + "]")); err != nil { + errs = append(errs, err) + return nil + } + + return pbytes } - // generate patch bytes - jsonPatch := struct { - Path string `json:"path"` - Op string `json:"op"` - Value interface{} `json:"value"` - }{ - fmt.Sprintf("/spec/rules/%s", strconv.Itoa(patchPostion)), - operation, - genRule, - } - pbytes, err := json.Marshal(jsonPatch) - if err != nil { - errs = append(errs, err) - continue + // handle all other controllers other than CronJob + genRule := generateRuleForControllers(rule, stripCronJob(controllers), log) + if !reflect.DeepEqual(genRule, kyvernoRule{}) { + pbytes := convertToPatches(genRule, patchPostion) + rulePatches = append(rulePatches, pbytes) + insertIdx++ + patchPostion = insertIdx } - // check the patch - if _, err := jsonpatch.DecodePatch([]byte("[" + string(pbytes) + "]")); err != nil { - errs = append(errs, err) - continue + // handle CronJob, it appends an additional rule + genRule = generateCronJobRule(rule, controllers, log) + if !reflect.DeepEqual(genRule, kyvernoRule{}) { + pbytes := convertToPatches(genRule, patchPostion) + rulePatches = append(rulePatches, pbytes) + insertIdx++ } - - rulePatches = append(rulePatches, pbytes) - insertIdx++ } + return } @@ -249,10 +261,15 @@ type kyvernoRule struct { } func generateRuleForControllers(rule kyverno.Rule, controllers string, log logr.Logger) kyvernoRule { - if strings.HasPrefix(rule.Name, "autogen-") { + logger := log.WithName("generateRuleForControllers") + + if strings.HasPrefix(rule.Name, "autogen-") || controllers == "" { + logger.V(5).Info("skip generateRuleForControllers") return kyvernoRule{} } + logger.V(3).Info("processing rule", "rulename", rule.Name) + match := rule.MatchResources exclude := rule.ExcludeResources if !utils.ContainsString(match.ResourceDescription.Kinds, "Pod") || @@ -284,11 +301,11 @@ func generateRuleForControllers(rule kyverno.Rule, controllers string, log logr. if skipAutoGeneration { if match.ResourceDescription.Name != "" || match.ResourceDescription.Selector != nil || exclude.ResourceDescription.Name != "" || exclude.ResourceDescription.Selector != nil { - log.Info("skip generating rule on pod controllers: Name / Selector in resource decription may not be applicable.", "rule", rule.Name) + logger.Info("skip generating rule on pod controllers: Name / Selector in resource decription may not be applicable.", "rule", rule.Name) return kyvernoRule{} } if controllers == "all" { - controllers = engine.PodControllers + controllers = "DaemonSet,Deployment,Job,StatefulSet" } else { controllers = strings.Join(controllersValidated, ",") } @@ -354,12 +371,12 @@ func generateRuleForControllers(rule kyverno.Rule, controllers string, log logr. return kyvernoRule{} } -// defaultPodControllerAnnotation generates annotation "pod-policies.kyverno.io/autogen-controllers=all" -// ann passes in the annotation of the policy +// defaultPodControllerAnnotation inserts an annotation +// "pod-policies.kyverno.io/autogen-controllers=DaemonSet,Deployment,Job,StatefulSet" to policy func defaultPodControllerAnnotation(ann map[string]string) ([]byte, error) { if ann == nil { ann = make(map[string]string) - ann[engine.PodControllersAnnotation] = "DaemonSet,Deployment,Job,StatefulSet" + ann[engine.PodControllersAnnotation] = engine.PodControllers jsonPatch := struct { Path string `json:"path"` Op string `json:"op"` @@ -384,7 +401,7 @@ func defaultPodControllerAnnotation(ann map[string]string) ([]byte, error) { }{ "/metadata/annotations/pod-policies.kyverno.io~1autogen-controllers", "add", - "DaemonSet,Deployment,Job,StatefulSet", + engine.PodControllers, } patchByte, err := json.Marshal(jsonPatch) diff --git a/pkg/policymutation/policymutation_test.go b/pkg/policymutation/policymutation_test.go new file mode 100644 index 0000000000..b9a5f70c7a --- /dev/null +++ b/pkg/policymutation/policymutation_test.go @@ -0,0 +1,114 @@ +package policymutation + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/nirmata/kyverno/pkg/engine" + "github.com/nirmata/kyverno/pkg/utils" + "gotest.tools/assert" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +func currentDir() (string, error) { + homedir, err := os.UserHomeDir() + if err != nil { + return "", nil + } + + return filepath.Join(homedir, "github.com/nirmata/kyverno"), nil +} + +func Test_CronJobOnly(t *testing.T) { + + controllers := engine.PodControllerCronJob + dir, err := os.Getwd() + baseDir := filepath.Dir(filepath.Dir(dir)) + assert.NilError(t, err) + + policies, errs := utils.GetPolicy(baseDir + "/samples/best_practices/disallow_bind_mounts.yaml") + if len(errs) != 0 { + t.Log(errs) + } + + policy := policies[0] + policy.SetAnnotations(map[string]string{ + engine.PodControllersAnnotation: controllers, + }) + + rulePatches, errs := generateRulePatches(*policy, controllers, log.Log) + if len(errs) != 0 { + t.Log(errs) + } + + expectedPatches := [][]byte{ + []byte(`{"path":"/spec/rules/1","op":"add","value":{"name":"autogen-cronjob-validate-hostPath","match":{"resources":{"kinds":["CronJob"]}},"validate":{"message":"Host path volumes are not allowed","pattern":{"spec":{"jobTemplate":{"spec":{"template":{"spec":{"=(volumes)":[{"X(hostPath)":null}]}}}}}}}}}`), + } + + assert.DeepEqual(t, rulePatches, expectedPatches) +} + +func Test_CronJob_hasExclude(t *testing.T) { + + controllers := engine.PodControllerCronJob + dir, err := os.Getwd() + baseDir := filepath.Dir(filepath.Dir(dir)) + assert.NilError(t, err) + + policies, errs := utils.GetPolicy(baseDir + "/samples/best_practices/disallow_bind_mounts.yaml") + if len(errs) != 0 { + t.Log(errs) + } + + policy := policies[0] + policy.SetAnnotations(map[string]string{ + engine.PodControllersAnnotation: controllers, + }) + + rule := policy.Spec.Rules[0].DeepCopy() + rule.ExcludeResources.Kinds = []string{"Pod"} + rule.ExcludeResources.Namespaces = []string{"test"} + policy.Spec.Rules[0] = *rule + + rulePatches, errs := generateRulePatches(*policy, controllers, log.Log) + if len(errs) != 0 { + t.Log(errs) + } + + expectedPatches := [][]byte{ + []byte(`{"path":"/spec/rules/1","op":"add","value":{"name":"autogen-cronjob-validate-hostPath","match":{"resources":{"kinds":["CronJob"]}},"exclude":{"resources":{"kinds":["CronJob"],"namespaces":["test"]}},"validate":{"message":"Host path volumes are not allowed","pattern":{"spec":{"jobTemplate":{"spec":{"template":{"spec":{"=(volumes)":[{"X(hostPath)":null}]}}}}}}}}}`), + } + + assert.DeepEqual(t, rulePatches, expectedPatches) +} + +func Test_CronJobAndDeployment(t *testing.T) { + controllers := strings.Join([]string{engine.PodControllerCronJob, "Deployment"}, ",") + dir, err := os.Getwd() + baseDir := filepath.Dir(filepath.Dir(dir)) + assert.NilError(t, err) + + policies, errs := utils.GetPolicy(baseDir + "/samples/best_practices/disallow_bind_mounts.yaml") + if len(errs) != 0 { + t.Log(errs) + } + + policy := policies[0] + policy.SetAnnotations(map[string]string{ + engine.PodControllersAnnotation: controllers, + }) + + rulePatches, errs := generateRulePatches(*policy, controllers, log.Log) + if len(errs) != 0 { + t.Log(errs) + } + + expectedPatches := [][]byte{ + []byte(`{"path":"/spec/rules/1","op":"add","value":{"name":"autogen-validate-hostPath","match":{"resources":{"kinds":["Deployment"]}},"validate":{"message":"Host path volumes are not allowed","pattern":{"spec":{"template":{"spec":{"=(volumes)":[{"X(hostPath)":null}]}}}}}}}`), + []byte(`{"path":"/spec/rules/2","op":"add","value":{"name":"autogen-cronjob-validate-hostPath","match":{"resources":{"kinds":["CronJob"]}},"validate":{"message":"Host path volumes are not allowed","pattern":{"spec":{"jobTemplate":{"spec":{"template":{"spec":{"=(volumes)":[{"X(hostPath)":null}]}}}}}}}}}`), + } + + assert.DeepEqual(t, rulePatches, expectedPatches) +} diff --git a/pkg/utils/loadpolicy.go b/pkg/utils/loadpolicy.go new file mode 100644 index 0000000000..175b26e642 --- /dev/null +++ b/pkg/utils/loadpolicy.go @@ -0,0 +1,69 @@ +package utils + +import ( + "bufio" + "bytes" + "encoding/json" + "fmt" + "io" + "io/ioutil" + + v1 "github.com/nirmata/kyverno/pkg/api/kyverno/v1" + "k8s.io/apimachinery/pkg/util/yaml" +) + +// GetPolicy - Extracts policies from a YAML +func GetPolicy(path string) (clusterPolicies []*v1.ClusterPolicy, errors []error) { + file, err := ioutil.ReadFile(path) + if err != nil { + errors = append(errors, fmt.Errorf(fmt.Sprintf("failed to load file: %v. error: %v", path, err))) + return clusterPolicies, errors + } + + policies, err := SplitYAMLDocuments(file) + if err != nil { + errors = append(errors, err) + return clusterPolicies, errors + } + + for _, thisPolicyBytes := range policies { + policyBytes, err := yaml.ToJSON(thisPolicyBytes) + if err != nil { + errors = append(errors, fmt.Errorf(fmt.Sprintf("failed to convert json. error: %v", err))) + continue + } + + policy := &v1.ClusterPolicy{} + if err := json.Unmarshal(policyBytes, policy); err != nil { + errors = append(errors, fmt.Errorf(fmt.Sprintf("failed to decode policy in %s. error: %v", path, err))) + continue + } + + if policy.TypeMeta.Kind != "ClusterPolicy" { + errors = append(errors, fmt.Errorf(fmt.Sprintf("resource %v is not a cluster policy", policy.Name))) + continue + } + clusterPolicies = append(clusterPolicies, policy) + } + + return clusterPolicies, errors +} + +// SplitYAMLDocuments reads the YAML bytes per-document, unmarshals the TypeMeta information from each document +// and returns a map between the GroupVersionKind of the document and the document bytes +func SplitYAMLDocuments(yamlBytes []byte) (policies [][]byte, error error) { + buf := bytes.NewBuffer(yamlBytes) + reader := yaml.NewYAMLReader(bufio.NewReader(buf)) + for { + // Read one YAML document at a time, until io.EOF is returned + b, err := reader.Read() + if err == io.EOF || len(b) == 0 { + break + } else if err != nil { + return policies, fmt.Errorf("unable to read yaml") + } + + policies = append(policies, b) + } + return policies, nil +} diff --git a/pkg/webhooks/policymutation_test.go b/pkg/webhooks/policymutation_test.go index 56c62e58d2..e1b2e001b5 100644 --- a/pkg/webhooks/policymutation_test.go +++ b/pkg/webhooks/policymutation_test.go @@ -2,12 +2,13 @@ package webhooks import ( "encoding/json" - "reflect" "testing" kyverno "github.com/nirmata/kyverno/pkg/api/kyverno/v1" "github.com/nirmata/kyverno/pkg/engine/utils" "github.com/nirmata/kyverno/pkg/policymutation" + + assertnew "github.com/stretchr/testify/assert" "gotest.tools/assert" "sigs.k8s.io/controller-runtime/pkg/log" ) @@ -16,7 +17,10 @@ func compareJSONAsMap(t *testing.T, expected, actual []byte) { var expectedMap, actualMap map[string]interface{} assert.NilError(t, json.Unmarshal(expected, &expectedMap)) assert.NilError(t, json.Unmarshal(actual, &actualMap)) - assert.Assert(t, reflect.DeepEqual(expectedMap, actualMap)) + + if !assertnew.Equal(t, expectedMap, actualMap) { + t.FailNow() + } } func TestGeneratePodControllerRule_NilAnnotation(t *testing.T) { @@ -42,7 +46,7 @@ func TestGeneratePodControllerRule_NilAnnotation(t *testing.T) { "metadata": { "name": "add-safe-to-evict", "annotations": { - "pod-policies.kyverno.io/autogen-controllers": "DaemonSet,Deployment,Job,StatefulSet" + "pod-policies.kyverno.io/autogen-controllers": "DaemonSet,Deployment,Job,StatefulSet,CronJob" } } }`) @@ -243,11 +247,47 @@ func TestGeneratePodControllerRule_Mutate(t *testing.T) { } } } + }, + { + "name": "autogen-cronjob-annotate-empty-dir", + "match": { + "resources": { + "kinds": [ + "CronJob" + ] + } + }, + "mutate": { + "overlay": { + "spec": { + "jobTemplate": { + "spec": { + "template": { + "metadata": { + "annotations": { + "+(cluster-autoscaler.kubernetes.io/safe-to-evict)": "true" + } + }, + "spec": { + "volumes": [ + { + "(emptyDir)": { + } + } + ] + } + } + } + } + } + } + } } ] } }`) - compareJSONAsMap(t, p, expectedPolicy) + + compareJSONAsMap(t, expectedPolicy, p) } func TestGeneratePodControllerRule_ExistOtherAnnotation(t *testing.T) { policyRaw := []byte(`{ @@ -275,7 +315,7 @@ func TestGeneratePodControllerRule_ExistOtherAnnotation(t *testing.T) { "metadata": { "name": "add-safe-to-evict", "annotations": { - "pod-policies.kyverno.io/autogen-controllers": "DaemonSet,Deployment,Job,StatefulSet", + "pod-policies.kyverno.io/autogen-controllers": "DaemonSet,Deployment,Job,StatefulSet,CronJob", "test": "annotation" } } @@ -472,6 +512,7 @@ func TestGeneratePodControllerRule_ValidatePattern(t *testing.T) { }`) var policy kyverno.ClusterPolicy + // var policy, generatePolicy unstructured.Unstructured assert.Assert(t, json.Unmarshal(policyRaw, &policy)) patches, errs := policymutation.GeneratePodControllerRule(policy, log.Log) assert.Assert(t, len(errs) == 0) @@ -484,7 +525,7 @@ func TestGeneratePodControllerRule_ValidatePattern(t *testing.T) { "kind": "ClusterPolicy", "metadata": { "annotations": { - "pod-policies.kyverno.io/autogen-controllers": "DaemonSet,Deployment,Job,StatefulSet" + "pod-policies.kyverno.io/autogen-controllers": "DaemonSet,Deployment,Job,StatefulSet,CronJob" }, "name": "add-safe-to-evict" }, @@ -544,9 +585,42 @@ func TestGeneratePodControllerRule_ValidatePattern(t *testing.T) { } } } + }, + { + "name": "autogen-cronjob-validate-docker-sock-mount", + "match": { + "resources": { + "kinds": [ + "CronJob" + ] + } + }, + "validate": { + "message": "Use of the Docker Unix socket is not allowed", + "pattern": { + "spec": { + "jobTemplate": { + "spec": { + "template": { + "spec": { + "=(volumes)": [ + { + "=(hostPath)": { + "path": "!/var/run/docker.sock" + } + } + ] + } + } + } + } + } + } + } } ] } }`) - compareJSONAsMap(t, p, expectedPolicy) + + compareJSONAsMap(t, expectedPolicy, p) }