Merge pull request #1030 from NoSkillGirl/feature/add_cli_variables

Feature/add cli variables

cmd/kyverno/main.go

@@ -278,7 +278,7 @@ func main() {
 	// Sync openAPI definitions of resources
 	openAPISync := openapi.NewCRDSync(client, openAPIController)
 
-	supportMudateValidate := utils.HigherThanKubernetesVersion(client, log.Log, 1, 14, 0)
+	supportMutateValidate := utils.HigherThanKubernetesVersion(client, log.Log, 1, 14, 0)
 
 	// WEBHOOK
 	// - https server to provide endpoints called based on rules defined in Mutating & Validation webhook configuration
@@ -304,7 +304,7 @@ func main() {
 		grgen,
 		rWebhookWatcher,
 		auditHandler,
-		supportMudateValidate,
+		supportMutateValidate,
 		cleanUp,
 		log.Log.WithName("WebhookServer"),
 		openAPIController,

documentation/kyverno-cli.md

@@ -88,5 +88,113 @@ Saving the mutated resource in a file/directory:
 kyverno apply /path/to/policy.yaml --resource /path/to/resource.yaml -o <file path/directory path>
 ```
 
+Apply policy with variables:
+
+Use --set flag to pass the values for variables in a policy while applying on a resource.
+
+```
+kyverno apply /path/to/policy.yaml --resource /path/to/resource.yaml --set <variable1>=<value1>,<variable2>=<value2>
+```
+
+Use --values_file for applying multiple policies on multiple resources and pass a file containing variables and its values.
+
+```
+kyverno apply /path/to/policy1.yaml /path/to/policy2.yaml --resource /path/to/resource1.yaml --resource /path/to/resource2.yaml -f /path/to/value.yaml
+```
+
+Format of value.yaml :
+
+```
+policies:
+  - name: <policy1 name>
+    resources:
+      - name: <resource1 name>
+        values:
+          <variable1 in policy1>: <value>
+          <variable2 in policy1>: <value>
+      - name: <resource2 name>
+        values:
+          <variable1 in policy1>: <value>
+          <variable2 in policy1>: <value>
+  - name: <policy2 name>
+    resources:
+      - name: <resource1 name>
+        values:
+          <variable1 in policy2>: <value>
+          <variable2 in policy2>: <value>
+      - name: <resource2 name>
+        values:
+          <variable1 in policy2>: <value>
+          <variable2 in policy2>: <value>
+```
+
+Example:
+
+Policy file(add_network_policy.yaml):
+
+```
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: add-networkpolicy
+  annotations:
+    policies.kyverno.io/category: Workload Management
+    policies.kyverno.io/description: By default, Kubernetes allows communications across 
+      all pods within a cluster. Network policies and, a CNI that supports network policies, 
+      must be used to restrict communinications. A default NetworkPolicy should be configured 
+      for each namespace to default deny all ingress traffic to the pods in the namespace. 
+      Application teams can then configure additional NetworkPolicy resources to allow 
+      desired traffic to application pods from select sources.
+spec:
+  rules:
+  - name: default-deny-ingress
+    match:
+      resources: 
+        kinds:
+        - Namespace
+        name: "*"
+    generate: 
+      kind: NetworkPolicy
+      name: default-deny-ingress
+      namespace: "{{request.object.metadata.name}}"
+      synchronize : true
+      data:
+        spec:
+          # select all pods in the namespace
+          podSelector: {}
+          policyTypes: 
+          - Ingress
+```
+Resource file(required_default_network_policy.yaml) :
+
+```
+kind: Namespace
+apiVersion: v1
+metadata: 
+    name: "devtest"
+```
+Applying policy on resource using set/-s flag:
+
+```
+kyverno apply /path/to/add_network_policy.yaml --resource /path/to/required_default_network_policy.yaml -s request.object.metadata.name=devtest
+```
+
+Applying policy on resource using --values_file/-f flag:
+
+yaml file with variables(value.yaml) :
+
+```
+policies:
+  - name: default-deny-ingress
+    resources:
+      - name: devtest
+        values:
+          request.namespace: devtest
+```
+
+```
+kyverno apply /path/to/add_network_policy.yaml --resource /path/to/required_default_network_policy.yaml -f /path/to/value.yaml
+```
+
 
 <small>*Read Next >> [Sample Policies](/samples/README.md)*</small>

pkg/kyverno/apply/command.go

@@ -5,8 +5,13 @@ import (
 	"errors"
 	"fmt"
 	"io/ioutil"
+
+	"github.com/nirmata/kyverno/pkg/engine/context"
+	"k8s.io/apimachinery/pkg/util/yaml"
+
 	"os"
 	"path/filepath"
+
 	"strings"
 	"time"
 
@@ -50,7 +55,24 @@ func Command() *cobra.Command {
 	var cmd *cobra.Command
 	var resourcePaths []string
 	var cluster bool
-	var mutateLogPath string
+	var mutateLogPath, variablesString, valuesFile string
+	variables := make(map[string]string)
+
+	type Resource struct {
+		Name   string            `json:"name"`
+		Values map[string]string `json:"values"`
+	}
+
+	type Policy struct {
+		Name      string     `json:"name"`
+		Resources []Resource `json:"resources"`
+	}
+
+	type Values struct {
+		Policies []Policy `json:"policies"`
+	}
+
+	valuesMap := make(map[string]map[string]*Resource)
 
 	kubernetesConfig := genericclioptions.NewConfigFlags(true)
 
@@ -63,26 +85,63 @@ func Command() *cobra.Command {
 				if err != nil {
 					if !sanitizedError.IsErrorSanitized(err) {
 						log.Log.Error(err, "failed to sanitize")
-						err = fmt.Errorf("Internal error")
+						err = fmt.Errorf("internal error")
 					}
 				}
 			}()
 
+			if valuesFile != "" && variablesString != "" {
+				return sanitizedError.NewWithError("pass the values either using set flag or values_file flag", err)
+			}
+
+			if valuesFile != "" {
+				yamlFile, err := ioutil.ReadFile(valuesFile)
+				if err != nil {
+					return sanitizedError.NewWithError("unable to read yaml", err)
+				}
+
+				valuesBytes, err := yaml.ToJSON(yamlFile)
+				if err != nil {
+					return sanitizedError.NewWithError("failed to convert json", err)
+				}
+
+				values := &Values{}
+				if err := json.Unmarshal(valuesBytes, values); err != nil {
+					return sanitizedError.NewWithError("failed to decode yaml", err)
+				}
+
+				for _, p := range values.Policies {
+					pmap := make(map[string]*Resource)
+					for _, r := range p.Resources {
+						pmap[r.Name] = &r
+					}
+					valuesMap[p.Name] = pmap
+				}
+			}
+
+			if variablesString != "" {
+				kvpairs := strings.Split(strings.Trim(variablesString, " "), ",")
+				for _, kvpair := range kvpairs {
+					kvs := strings.Split(strings.Trim(kvpair, " "), "=")
+					variables[strings.Trim(kvs[0], " ")] = strings.Trim(kvs[1], " ")
+				}
+			}
+
 			if len(resourcePaths) == 0 && !cluster {
 				return sanitizedError.NewWithError(fmt.Sprintf("resource file(s) or cluster required"), err)
 			}
 
-			var mutatelogPathIsDir bool
+			var mutateLogPathIsDir bool
 			if mutateLogPath != "" {
 				spath := strings.Split(mutateLogPath, "/")
 				sfileName := strings.Split(spath[len(spath)-1], ".")
 				if sfileName[len(sfileName)-1] == "yml" || sfileName[len(sfileName)-1] == "yaml" {
-					mutatelogPathIsDir = false
+					mutateLogPathIsDir = false
 				} else {
-					mutatelogPathIsDir = true
+					mutateLogPathIsDir = true
 				}
 
-				err = createFileOrFolder(mutateLogPath, mutatelogPathIsDir)
+				err = createFileOrFolder(mutateLogPath, mutateLogPathIsDir)
 				if err != nil {
 					if !sanitizedError.IsErrorSanitized(err) {
 						return sanitizedError.NewWithError("failed to create file/folder.", err)
@@ -143,14 +202,32 @@ func Command() *cobra.Command {
 					continue
 				}
 
-				if common.PolicyHasVariables(*policy) {
+				if common.PolicyHasVariables(*policy) && variablesString == "" && valuesFile == "" {
 					rc.skip += len(resources)
-					fmt.Printf("\nskipping policy %s as policies with variables are not supported\n", policy.Name)
+					fmt.Printf("\nskipping policy %s as it has variables. pass the values for the variables using set/values_file flag", policy.Name)
 					continue
 				}
 
+				if common.PolicyHasVariables(*policy) && variablesString == "" && valuesFile == "" {
+					return sanitizedError.NewWithError(fmt.Sprintf("policy %s have variables. pass the values for the variables using set/values_file flag", policy.Name), err)
+				}
+
 				for _, resource := range resources {
-					applyPolicyOnResource(policy, resource, rc)
+					// get values from file for this policy resource combination
+					thisPolicyResouceValues := make(map[string]string)
+					if len(valuesMap[policy.GetName()]) != 0 && valuesMap[policy.GetName()][resource.GetName()] != nil {
+						thisPolicyResouceValues = valuesMap[policy.GetName()][resource.GetName()].Values
+					}
+
+					for k, v := range variables {
+						thisPolicyResouceValues[k] = v
+					}
+
+					if common.PolicyHasVariables(*policy) && len(thisPolicyResouceValues) == 0 {
+						return sanitizedError.NewWithError(fmt.Sprintf("policy %s have variables. pass the values for the variables using set/values_file flag", policy.Name), err)
+					}
+
+					err = applyPolicyOnResource(policy, resource, mutateLogPath, mutateLogPathIsDir, thisPolicyResouceValues, rc)
 					if err != nil {
 						return sanitizedError.NewWithError(fmt.Errorf("failed to apply policy %v on resource %v", policy.Name, resource.GetName()).Error(), err)
 					}
@@ -171,6 +248,8 @@ func Command() *cobra.Command {
 	cmd.Flags().StringArrayVarP(&resourcePaths, "resource", "r", []string{}, "Path to resource files")
 	cmd.Flags().BoolVarP(&cluster, "cluster", "c", false, "Checks if policies should be applied to cluster in the current context")
 	cmd.Flags().StringVarP(&mutateLogPath, "output", "o", "", "Prints the mutated resources in provided file/directory")
+	cmd.Flags().StringVarP(&variablesString, "set", "s", "", "Variables that are required")
+	cmd.Flags().StringVarP(&valuesFile, "values_file", "f", "", "File containing values for policy variables")
 	return cmd
 }
 
@@ -300,13 +379,29 @@ func getResource(path string) ([]*unstructured.Unstructured, error) {
 }
 
 // applyPolicyOnResource - function to apply policy on resource
-func applyPolicyOnResource(policy *v1.ClusterPolicy, resource *unstructured.Unstructured, rc *resultCounts) {
+func applyPolicyOnResource(policy *v1.ClusterPolicy, resource *unstructured.Unstructured, mutateLogPath string, mutateLogPathIsDir bool, variables map[string]string, rc *resultCounts) error {
 	responseError := false
 
 	resPath := fmt.Sprintf("%s/%s/%s", resource.GetNamespace(), resource.GetKind(), resource.GetName())
 	log.Log.V(3).Info("applying policy on resource", "policy", policy.Name, "resource", resPath)
 
-	mutateResponse := engine.Mutate(engine.PolicyContext{Policy: *policy, NewResource: *resource})
+	// build context
+	ctx := context.NewContext()
+	for key, value := range variables {
+		startString := ""
+		endString := ""
+		for _, k := range strings.Split(key, ".") {
+			startString += fmt.Sprintf(`{"%s":`, k)
+			endString += `}`
+		}
+
+		midString := fmt.Sprintf(`"%s"`, value)
+		finalString := startString + midString + endString
+		var jsonData = []byte(finalString)
+		ctx.AddJSON(jsonData)
+	}
+
+	mutateResponse := engine.Mutate(engine.PolicyContext{Policy: *policy, NewResource: *resource, Context: ctx})
 	if !mutateResponse.IsSuccessful() {
 		fmt.Printf("Failed to apply mutate policy %s -> resource %s", policy.Name, resPath)
 		for i, r := range mutateResponse.PolicyResponse.Rules {
@@ -326,10 +421,13 @@ func applyPolicyOnResource(policy *v1.ClusterPolicy, resource *unstructured.Unst
 				fmt.Printf("\n" + mutatedResource)
 				fmt.Printf("\n")
 			}
+
+		} else {
+			fmt.Printf("\n\nMutation:\nMutation skipped. Resource not matches the policy\n")
 		}
 	}
 
-	validateResponse := engine.Validate(engine.PolicyContext{Policy: *policy, NewResource: mutateResponse.PatchedResource})
+	validateResponse := engine.Validate(engine.PolicyContext{Policy: *policy, NewResource: mutateResponse.PatchedResource, Context: ctx})
 	if !validateResponse.IsSuccessful() {
 		fmt.Printf("\npolicy %s -> resource %s failed: \n", policy.Name, resPath)
 		for i, r := range validateResponse.PolicyResponse.Rules {
@@ -367,6 +465,7 @@ func applyPolicyOnResource(policy *v1.ClusterPolicy, resource *unstructured.Unst
 	} else {
 		rc.pass++
 	}
+	return nil
 }
 
 // mutatePolicies - function to apply mutation on policies
@@ -388,15 +487,15 @@ func mutatePolices(policies []*v1.ClusterPolicy) ([]*v1.ClusterPolicy, error) {
 }
 
 // printMutatedOutput - function to print output in provided file or directory
-func printMutatedOutput(mutatelogPath string, mutatelogPathIsDir bool, yaml string, fileName string) error {
+func printMutatedOutput(mutateLogPath string, mutateLogPathIsDir bool, yaml string, fileName string) error {
 	var f *os.File
 	var err error
 	yaml = yaml + ("\n---\n\n")
 
-	if !mutatelogPathIsDir {
-		f, err = os.OpenFile(mutatelogPath, os.O_APPEND|os.O_WRONLY, 0644)
+	if !mutateLogPathIsDir {
+		f, err = os.OpenFile(mutateLogPath, os.O_APPEND|os.O_WRONLY, 0644)
 	} else {
-		f, err = os.OpenFile(mutatelogPath+"/"+fileName+".yaml", os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+		f, err = os.OpenFile(mutateLogPath+"/"+fileName+".yaml", os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
 	}
 
 	if err != nil {
@@ -414,19 +513,19 @@ func printMutatedOutput(mutatelogPath string, mutatelogPathIsDir bool, yaml stri
 }
 
 // createFileOrFolder - creating file or folder according to path provided
-func createFileOrFolder(mutatelogPath string, mutatelogPathIsDir bool) error {
-	mutatelogPath = filepath.Clean(mutatelogPath)
-	_, err := os.Stat(mutatelogPath)
+func createFileOrFolder(mutateLogPath string, mutateLogPathIsDir bool) error {
+	mutateLogPath = filepath.Clean(mutateLogPath)
+	_, err := os.Stat(mutateLogPath)
 
 	if err != nil {
 		if os.IsNotExist(err) {
-			if !mutatelogPathIsDir {
+			if !mutateLogPathIsDir {
 				// check the folder existance, then create the file
 				var folderPath string
-				s := strings.Split(mutatelogPath, "/")
+				s := strings.Split(mutateLogPath, "/")
 
 				if len(s) > 1 {
-					folderPath = mutatelogPath[:len(mutatelogPath)-len(s[len(s)-1])-1]
+					folderPath = mutateLogPath[:len(mutateLogPath)-len(s[len(s)-1])-1]
 					_, err := os.Stat(folderPath)
 					fmt.Println(err)
 					if os.IsNotExist(err) {
@@ -438,7 +537,7 @@ func createFileOrFolder(mutatelogPath string, mutatelogPathIsDir bool) error {
 
 				}
 
-				file, err := os.OpenFile(mutatelogPath, os.O_RDONLY|os.O_CREATE, 0644)
+				file, err := os.OpenFile(mutateLogPath, os.O_RDONLY|os.O_CREATE, 0644)
 				if err != nil {
 					return sanitizedError.NewWithError(fmt.Sprintf("failed to create file"), err)
 				}
@@ -449,7 +548,7 @@ func createFileOrFolder(mutatelogPath string, mutatelogPathIsDir bool) error {
 				}
 
 			} else {
-				errDir := os.MkdirAll(mutatelogPath, 0755)
+				errDir := os.MkdirAll(mutateLogPath, 0755)
 				if errDir != nil {
 					return sanitizedError.NewWithError(fmt.Sprintf("failed to create directory"), err)
 				}

pkg/kyverno/common/common.go

@@ -69,11 +69,6 @@ func GetPolicies(paths []string) (policies []*v1.ClusterPolicy, error error) {
 		}
 	}
 
-	for i := range policies {
-		setFalse := false
-		policies[i].Spec.Background = &setFalse
-	}
-
 	return policies, nil
 }
 
@@ -151,13 +146,6 @@ func GetPoliciesValidation(policyPaths []string) ([]*v1.ClusterPolicy, *openapi.
 	return policies, openAPIController, nil
 }
 
-// PolicyHasVariables - check for variables in policy
-func PolicyHasVariables(policy v1.ClusterPolicy) bool {
-	policyRaw, _ := json.Marshal(policy)
-	regex := regexp.MustCompile(`\{\{([^{}]*)\}\}`)
-	return len(regex.FindAllStringSubmatch(string(policyRaw), -1)) > 0
-}
-
 // MutatePolicy - applies mutation to a policy
 func MutatePolicy(policy *v1.ClusterPolicy, logger logr.Logger) (*v1.ClusterPolicy, error) {
 	patches, _ := policymutation.GenerateJSONPatchesForDefaults(policy, logger)
@@ -199,3 +187,10 @@ func MutatePolicy(policy *v1.ClusterPolicy, logger logr.Logger) (*v1.ClusterPoli
 
 	return &p, nil
 }
+
+// PolicyHasVariables - check for variables in policy
+func PolicyHasVariables(policy v1.ClusterPolicy) bool {
+	policyRaw, _ := json.Marshal(policy)
+	regex := regexp.MustCompile(`\{\{([^{}]*)\}\}`)
+	return len(regex.FindAllStringSubmatch(string(policyRaw), -1)) > 0
+}

pkg/kyverno/validate/command.go

@@ -49,12 +49,6 @@ func Command() *cobra.Command {
 
 			invalidPolicyFound := false
 			for _, policy := range policies {
-				if common.PolicyHasVariables(*policy) {
-					invalidPolicyFound = true
-					fmt.Printf("Policy %s is invalid.\n", policy.Name)
-					log.Log.Error(errors.New("'validate' does not support policies with variables"), "Policy "+policy.Name+" is invalid")
-					continue
-				}
 				err := policy2.Validate(utils.MarshalPolicy(*policy), nil, true, openAPIController)
 				if err != nil {
 					fmt.Printf("Policy %s is invalid.\n", policy.Name)

pkg/policymutation/policymutation.go

@@ -79,7 +79,7 @@ func defaultvalidationFailureAction(policy *kyverno.ClusterPolicy, log logr.Logg
 	// set ValidationFailureAction to "audit" if not specified
 	Audit := common.Audit
 	if policy.Spec.ValidationFailureAction == "" {
-		log.V(4).Info("setting defautl value", "spec.validationFailureAction", Audit)
+		log.V(4).Info("setting default value", "spec.validationFailureAction", Audit)
 
 		jsonPatch := struct {
 			Path  string `json:"path"`

pkg/webhooks/policymutation.go

@@ -18,7 +18,7 @@ func (ws *WebhookServer) policyMutation(request *v1beta1.AdmissionRequest) *v1be
 
 	//TODO: can this happen? wont this be picked by OpenAPI spec schema ?
 	if err := json.Unmarshal(raw, &policy); err != nil {
-		logger.Error(err, "faield to unmarshall policy admission request")
+		logger.Error(err, "failed to unmarshall policy admission request")
 		return &v1beta1.AdmissionResponse{
 			Allowed: true,
 			Result: &metav1.Status{

pkg/webhooks/server.go

@@ -110,7 +110,7 @@ type WebhookServer struct {
 	log               logr.Logger
 	openAPIController *openapi.Controller
 
-	supportMudateValidate bool
+	supportMutateValidate bool
 }
 
 // NewWebhookServer creates new instance of WebhookServer accordingly to given configuration
@@ -133,7 +133,7 @@ func NewWebhookServer(
 	grGenerator *generate.Generator,
 	resourceWebhookWatcher *webhookconfig.ResourceWebhookRegister,
 	auditHandler AuditHandler,
-	supportMudateValidate bool,
+	supportMutateValidate bool,
 	cleanUp chan<- struct{},
 	log logr.Logger,
 	openAPIController *openapi.Controller,
@@ -177,11 +177,11 @@ func NewWebhookServer(
 		auditHandler:              auditHandler,
 		log:                       log,
 		openAPIController:         openAPIController,
-		supportMudateValidate:     supportMudateValidate,
+		supportMutateValidate:     supportMutateValidate,
 	}
 
 	mux := httprouter.New()
-	mux.HandlerFunc("POST", config.MutatingWebhookServicePath, ws.handlerFunc(ws.resourceMutation, true))
+	mux.HandlerFunc("POST", config.MutatingWebhookServicePath, ws.handlerFunc(ws.ResourceMutation, true))
 	mux.HandlerFunc("POST", config.ValidatingWebhookServicePath, ws.handlerFunc(ws.resourceValidation, true))
 	mux.HandlerFunc("POST", config.PolicyMutatingWebhookServicePath, ws.handlerFunc(ws.policyMutation, true))
 	mux.HandlerFunc("POST", config.PolicyValidatingWebhookServicePath, ws.handlerFunc(ws.policyValidation, true))
@@ -260,9 +260,9 @@ func writeResponse(rw http.ResponseWriter, admissionReview *v1beta1.AdmissionRev
 	}
 }
 
-func (ws *WebhookServer) resourceMutation(request *v1beta1.AdmissionRequest) *v1beta1.AdmissionResponse {
+func (ws *WebhookServer) ResourceMutation(request *v1beta1.AdmissionRequest) *v1beta1.AdmissionResponse {
 
-	logger := ws.log.WithName("resourceMutation").WithValues("uid", request.UID, "kind", request.Kind.Kind, "namespace", request.Namespace, "name", request.Name, "operation", request.Operation)
+	logger := ws.log.WithName("ResourceMutation").WithValues("uid", request.UID, "kind", request.Kind.Kind, "namespace", request.Namespace, "name", request.Name, "operation", request.Operation)
 
 	if excludeKyvernoResources(request.Kind.Kind) {
 		return &v1beta1.AdmissionResponse{
@@ -329,7 +329,7 @@ func (ws *WebhookServer) resourceMutation(request *v1beta1.AdmissionRequest) *v1
 	var patches []byte
 	patchedResource := request.Object.Raw
 
-	if ws.supportMudateValidate {
+	if ws.supportMutateValidate {
 		// MUTATION
 		// mutation failure should not block the resource creation
 		// any mutation failure is reported as the violation
@@ -368,7 +368,9 @@ func (ws *WebhookServer) resourceMutation(request *v1beta1.AdmissionRequest) *v1
 	// Success -> Generate Request CR created successfully
 	// Failed -> Failed to create Generate Request CR
 
-	go ws.HandleGenerate(request.DeepCopy(), generatePolicies, ctx, userRequestInfo, ws.configHandler)
+	if request.Operation == v1beta1.Create || request.Operation == v1beta1.Update {
+		go ws.HandleGenerate(request.DeepCopy(), generatePolicies, ctx, userRequestInfo, ws.configHandler)
+	}
 
 	// Succesful processing of mutation & validation rules in policy
 	patchType := v1beta1.PatchTypeJSONPatch
@@ -398,7 +400,7 @@ func (ws *WebhookServer) resourceValidation(request *v1beta1.AdmissionRequest) *
 		}
 	}
 
-	if !ws.supportMudateValidate {
+	if !ws.supportMutateValidate {
 		logger.Info("mutate and validate rules are not supported prior to Kubernetes 1.14.0")
 		return &v1beta1.AdmissionResponse{
 			Allowed: true,