Update Kyverno test command (#1608)

* fix link (#1566)

Signed-off-by: vyankatesh <vyankatesh@neualto.com>

* update icon in chart.yaml

Signed-off-by: Shuting Zhao <shutting06@gmail.com>
Signed-off-by: vyankatesh <vyankatesh@neualto.com>

* Adding default policies for restricted mode and adding notes to helm install (#1556)

* Adding default policies for restricted mode, taking validationFailureAction from values.yaml and adding notes on helm install

Signed-off-by: Raj Das <mail.rajdas@gmail.com>

* Adding emoji

Signed-off-by: Raj Das <mail.rajdas@gmail.com>

* Update NOTES.txt

* minor fix

Signed-off-by: Raj Das <mail.rajdas@gmail.com>

* adding to readme

Signed-off-by: Raj Das <mail.rajdas@gmail.com>
Signed-off-by: vyankatesh <vyankatesh@neualto.com>

* update links and formatting in PR template (#1573)

* update links and formatting in PR template

Signed-off-by: Chip Zoller <chipzoller@gmail.com>

* update policy submission request template

Signed-off-by: Chip Zoller <chipzoller@gmail.com>
Signed-off-by: vyankatesh <vyankatesh@neualto.com>

* fix: restricting empty value to pass through the validation checks (#1574)

Signed-off-by: Yashvardhan Kukreja <yash.kukreja.98@gmail.com>
Signed-off-by: vyankatesh <vyankatesh@neualto.com>

* Actually fix contributor link in PR template (#1575)

* update links and formatting in PR template

Signed-off-by: Chip Zoller <chipzoller@gmail.com>

* update policy submission request template

Signed-off-by: Chip Zoller <chipzoller@gmail.com>

* actually fix contrib guidelines

Signed-off-by: Chip Zoller <chipzoller@gmail.com>

* actually fix contrib guidelines

Signed-off-by: Chip Zoller <chipzoller@gmail.com>
Signed-off-by: vyankatesh <vyankatesh@neualto.com>

* code improvement (#1567)

* code improvement

Signed-off-by: NoSkillGirl <singhpooja240393@gmail.com>

* added if conditions

Signed-off-by: NoSkillGirl <singhpooja240393@gmail.com>

* fixed unit test cases

Signed-off-by: NoSkillGirl <singhpooja240393@gmail.com>
Signed-off-by: vyankatesh <vyankatesh@neualto.com>

* feat(operators): support subset checking for in and notin (#1555)

* feat(operators): support subset checking for in and notin

Signed-off-by: Arsh Sharma <arshsharma461@gmail.com>

* feat(operators): fixed NotIn function

Signed-off-by: Arsh Sharma <arshsharma461@gmail.com>
Signed-off-by: vyankatesh <vyankatesh@neualto.com>

* panic fix (#1601)

Signed-off-by: NoSkillGirl <singhpooja240393@gmail.com>
Signed-off-by: vyankatesh <vyankatesh@neualto.com>

* update kyverno cli test cmd

Signed-off-by: vyankatesh <vyankatesh@neualto.com>

* code indentation

Signed-off-by: vyankatesh <vyankatesh@neualto.com>

* change  help text

Signed-off-by: vyankatesh <vyankatesh@neualto.com>

Co-authored-by: Dekel <dekelb@users.noreply.github.com>
Co-authored-by: Shuting Zhao <shutting06@gmail.com>
Co-authored-by: Raj Babu Das <mail.rajdas@gmail.com>
Co-authored-by: Chip Zoller <chipzoller@gmail.com>
Co-authored-by: Yashvardhan Kukreja <yash.kukreja.98@gmail.com>
Co-authored-by: Pooja Singh <36136335+NoSkillGirl@users.noreply.github.com>
Co-authored-by: Arsh Sharma <56963264+RinkiyaKeDad@users.noreply.github.com>
Co-authored-by: vyankatesh <vyankatesh@neualto.com>

pkg/kyverno/apply/command.go

@@ -147,7 +147,7 @@ func applyCommandHelper(resourcePaths []string, cluster bool, policyReport bool,
 		return validateEngineResponses, rc, resources, skippedPolicies, sanitizederror.NewWithError("pass the values either using set flag or values_file flag", err)
 	}
 
-	variables, valuesMap, err := common.GetVariable(variablesString, valuesFile)
+	variables, valuesMap, err := common.GetVariable(variablesString, valuesFile, fs, false, "")
 	if err != nil {
 		if !sanitizederror.IsErrorSanitized(err) {
 			return validateEngineResponses, rc, resources, skippedPolicies, sanitizederror.NewWithError("failed to decode yaml", err)
@@ -180,7 +180,7 @@ func applyCommandHelper(resourcePaths []string, cluster bool, policyReport bool,
 		return validateEngineResponses, rc, resources, skippedPolicies, sanitizederror.NewWithError("a stdin pipe can be used for either policies or resources, not both", err)
 	}
 
-	policies, err := common.GetPoliciesFromPaths(fs, policyPaths, false)
+	policies, err := common.GetPoliciesFromPaths(fs, policyPaths, false, "")
 	if err != nil {
 		fmt.Printf("Error: failed to load policies\nCause: %s\n", err)
 		os.Exit(1)
@@ -205,7 +205,7 @@ func applyCommandHelper(resourcePaths []string, cluster bool, policyReport bool,
 		}
 	}
 
-	resources, err = common.GetResourceAccordingToResourcePath(fs, resourcePaths, cluster, mutatedPolicies, dClient, namespace, policyReport, false)
+	resources, err = common.GetResourceAccordingToResourcePath(fs, resourcePaths, cluster, mutatedPolicies, dClient, namespace, policyReport, false, "")
 	if err != nil {
 		fmt.Printf("Error: failed to load resources\nCause: %s\n", err)
 		os.Exit(1)

pkg/kyverno/common/common.go

@@ -300,9 +300,11 @@ func RemoveDuplicateVariables(matches [][]string) string {
 }
 
 // GetVariable - get the variables from console/file
-func GetVariable(variablesString, valuesFile string) (map[string]string, map[string]map[string]Resource, error) {
+func GetVariable(variablesString, valuesFile string, fs billy.Filesystem, isGit bool, policyresoucePath string) (map[string]string, map[string]map[string]Resource, error) {
 	valuesMap := make(map[string]map[string]Resource)
 	variables := make(map[string]string)
+	var yamlFile []byte
+	var err error
 	if variablesString != "" {
 		kvpairs := strings.Split(strings.Trim(variablesString, " "), ",")
 		for _, kvpair := range kvpairs {
@@ -311,7 +313,16 @@ func GetVariable(variablesString, valuesFile string) (map[string]string, map[str
 		}
 	}
 	if valuesFile != "" {
-		yamlFile, err := ioutil.ReadFile(valuesFile)
+		if isGit {
+			filep, err := fs.Open(filepath.Join(policyresoucePath, valuesFile))
+			if err != nil {
+				fmt.Printf("Unable to open variable file: %s. error: %s", valuesFile, err)
+			}
+			yamlFile, err = ioutil.ReadAll(filep)
+		} else {
+			yamlFile, err = ioutil.ReadFile(valuesFile)
+		}
+
 		if err != nil {
 			return variables, valuesMap, sanitizederror.NewWithError("unable to read yaml", err)
 		}
@@ -500,14 +511,19 @@ func PrintMutatedOutput(mutateLogPath string, mutateLogPathIsDir bool, yaml stri
 }
 
 // GetPoliciesFromPaths - get policies according to the resource path
-func GetPoliciesFromPaths(fs billy.Filesystem, dirPath []string, isGit bool) (policies []*v1.ClusterPolicy, err error) {
+func GetPoliciesFromPaths(fs billy.Filesystem, dirPath []string, isGit bool, policyresoucePath string) (policies []*v1.ClusterPolicy, err error) {
 	var errors []error
 	if isGit {
 		for _, pp := range dirPath {
-			filep, err := fs.Open(pp)
+			filep, err := fs.Open(filepath.Join(policyresoucePath, pp))
+			if err != nil {
+				fmt.Printf("Error: file not available with path %s: %v", filep.Name(), err.Error())
+				continue
+			}
 			bytes, err := ioutil.ReadAll(filep)
 			if err != nil {
 				fmt.Printf("Error: failed to read file %s: %v", filep.Name(), err.Error())
+				continue
 			}
 			policyBytes, err := yaml.ToJSON(bytes)
 			if err != nil {
@@ -558,9 +574,9 @@ func GetPoliciesFromPaths(fs billy.Filesystem, dirPath []string, isGit bool) (po
 
 // GetResourceAccordingToResourcePath - get resources according to the resource path
 func GetResourceAccordingToResourcePath(fs billy.Filesystem, resourcePaths []string,
-	cluster bool, policies []*v1.ClusterPolicy, dClient *client.Client, namespace string, policyReport bool, isGit bool) (resources []*unstructured.Unstructured, err error) {
+	cluster bool, policies []*v1.ClusterPolicy, dClient *client.Client, namespace string, policyReport bool, isGit bool, policyresoucePath string) (resources []*unstructured.Unstructured, err error) {
 	if isGit {
-		resources, err = GetResourcesWithTest(fs, policies, resourcePaths, isGit)
+		resources, err = GetResourcesWithTest(fs, policies, resourcePaths, isGit, policyresoucePath)
 		if err != nil {
 			return nil, sanitizederror.NewWithError("failed to extract the resources", err)
 		}

pkg/kyverno/common/fetch.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
+	"path/filepath"
 	"strings"
 
 	"github.com/go-git/go-billy/v5"
@@ -100,7 +101,7 @@ func GetResources(policies []*v1.ClusterPolicy, resourcePaths []string, dClient
 }
 
 // GetResourcesWithTest with gets matched resources by the given policies
-func GetResourcesWithTest(fs billy.Filesystem, policies []*v1.ClusterPolicy, resourcePaths []string, isGit bool) ([]*unstructured.Unstructured, error) {
+func GetResourcesWithTest(fs billy.Filesystem, policies []*v1.ClusterPolicy, resourcePaths []string, isGit bool, policyresoucePath string) ([]*unstructured.Unstructured, error) {
 	resources := make([]*unstructured.Unstructured, 0)
 	var resourceTypesMap = make(map[string]bool)
 	var resourceTypes []string
@@ -119,7 +120,7 @@ func GetResourcesWithTest(fs billy.Filesystem, policies []*v1.ClusterPolicy, res
 			var resourceBytes []byte
 			var err error
 			if isGit {
-				filep, err := fs.Open(resourcePath)
+				filep, err := fs.Open(filepath.Join(policyresoucePath, resourcePath))
 				if err != nil {
 					fmt.Printf("Unable to open resource file: %s. error: %s", resourcePath, err)
 					continue

pkg/kyverno/test/command.go

@@ -34,11 +34,11 @@ import (
 
 // Command returns version command
 func Command() *cobra.Command {
-
+	var cmd *cobra.Command
-	var valuesFile string
+	var valuesFile, fileName string
-	return &cobra.Command{
+	cmd = &cobra.Command{
 		Use:   "test",
-		Short: "Shows current test of kyverno",
+		Short: "run tests from directory",
 		RunE: func(cmd *cobra.Command, dirPath []string) (err error) {
 			defer func() {
 				if err != nil {
@@ -48,7 +48,7 @@ func Command() *cobra.Command {
 					}
 				}
 			}()
-			err = testCommandExecute(dirPath, valuesFile)
+			err = testCommandExecute(dirPath, valuesFile, fileName)
 			if err != nil {
 				log.Log.V(3).Info("a directory is required")
 				return err
@@ -56,6 +56,8 @@ func Command() *cobra.Command {
 			return nil
 		},
 	}
+	cmd.Flags().StringVarP(&fileName, "file-name", "f", "test.yaml", "test filename")
+	return cmd
 }
 
 type Test struct {
@@ -103,7 +105,7 @@ type Values struct {
 	Policies []Policy `json:"policies"`
 }
 
-func testCommandExecute(dirPath []string, valuesFile string) (err error) {
+func testCommandExecute(dirPath []string, valuesFile string, fileName string) (err error) {
 	var errors []error
 	fs := memfs.New()
 
@@ -134,45 +136,35 @@ func testCommandExecute(dirPath []string, valuesFile string) (err error) {
 		sort.Strings(policyYamls)
 		for _, yamlFilePath := range policyYamls {
 			file, err := fs.Open(yamlFilePath)
-			bytes, err := ioutil.ReadAll(file)
+			if strings.Contains(file.Name(), fileName) {
-			if err != nil {
+				policyresoucePath := strings.Trim(yamlFilePath, fileName)
-				sanitizederror.NewWithError("Error: failed to read file", err)
+				bytes, err := ioutil.ReadAll(file)
+				if err != nil {
+					sanitizederror.NewWithError("Error: failed to read file", err)
+					continue
+				}
+				policyBytes, err := yaml.ToJSON(bytes)
+				if err != nil {
+					sanitizederror.NewWithError("failed to convert to JSON", err)
+					continue
+				}
+				if err := applyPoliciesFromPath(fs, policyBytes, valuesFile, true, policyresoucePath); err != nil {
+					return sanitizederror.NewWithError("failed to apply test command", err)
+				}
 			}
-			policyBytes, err := yaml.ToJSON(bytes)
 			if err != nil {
-				sanitizederror.NewWithError("failed to convert to JSON", err)
+				sanitizederror.NewWithError("Error: failed to open file", err)
 				continue
 			}
-			if err := applyPoliciesFromPath(fs, policyBytes, valuesFile, true); err != nil {
-				return sanitizederror.NewWithError("failed to apply test command", err)
-			}
 		}
 	} else {
 		path := filepath.Clean(dirPath[0])
-		fileDesc, err := os.Stat(path)
 		if err != nil {
 			errors = append(errors, err)
 		}
-		if fileDesc.IsDir() {
+		err := getLocalDirTestFiles(fs, path, fileName, valuesFile)
-			files, err := ioutil.ReadDir(path)
+		if err != nil {
-			if err != nil {
+			errors = append(errors, err)
-				errors = append(errors, fmt.Errorf("failed to read %v: %v", path, err.Error()))
-			}
-			for _, file := range files {
-				fmt.Printf("\napplying  test on file  %s...", file.Name())
-
-				yamlFile, err := ioutil.ReadFile(filepath.Join(path, file.Name()))
-				if err != nil {
-					return sanitizederror.NewWithError("unable to read yaml", err)
-				}
-				valuesBytes, err := yaml.ToJSON(yamlFile)
-				if err != nil {
-					return sanitizederror.NewWithError("failed to convert json", err)
-				}
-				if err := applyPoliciesFromPath(fs, valuesBytes, valuesFile, false); err != nil {
-					return sanitizederror.NewWithError("failed to apply test command", err)
-				}
-			}
 		}
 		if len(errors) > 0 && log.Log.V(1).Enabled() {
 			fmt.Printf("ignoring errors: \n")
@@ -184,6 +176,36 @@ func testCommandExecute(dirPath []string, valuesFile string) (err error) {
 	return nil
 }
 
+func getLocalDirTestFiles(fs billy.Filesystem, path, fileName, valuesFile string) error {
+	files, err := ioutil.ReadDir(path)
+	if err != nil {
+		return fmt.Errorf("failed to read %v: %v", path, err.Error())
+	}
+	for _, file := range files {
+		if file.IsDir() {
+			getLocalDirTestFiles(fs, filepath.Join(path, file.Name()), fileName, valuesFile)
+			continue
+		}
+		if strings.Contains(file.Name(), fileName) {
+			yamlFile, err := ioutil.ReadFile(filepath.Join(path, file.Name()))
+			if err != nil {
+				sanitizederror.NewWithError("unable to read yaml", err)
+				continue
+			}
+			valuesBytes, err := yaml.ToJSON(yamlFile)
+			if err != nil {
+				sanitizederror.NewWithError("failed to convert json", err)
+				continue
+			}
+			if err := applyPoliciesFromPath(fs, valuesBytes, valuesFile, false, path); err != nil {
+				sanitizederror.NewWithError("failed to apply test command", err)
+				continue
+			}
+		}
+	}
+	return nil
+}
+
 func buildPolicyResults(resps []*response.EngineResponse) map[string][]interface{} {
 	results := make(map[string][]interface{})
 	infos := policyreport.GeneratePRsFromEngineResponse(resps, log.Log)
@@ -210,7 +232,18 @@ func buildPolicyResults(resps []*response.EngineResponse) map[string][]interface
 	return results
 }
 
-func applyPoliciesFromPath(fs billy.Filesystem, policyBytes []byte, valuesFile string, isGit bool) (err error) {
+func getPolicyResouceFullPath(path []string, policyresoucePath string, isGit bool) []string {
+	var pol []string
+	if !isGit {
+		for _, p := range path {
+			pol = append(pol, filepath.Join(policyresoucePath, p))
+		}
+		return pol
+	}
+	return path
+}
+
+func applyPoliciesFromPath(fs billy.Filesystem, policyBytes []byte, valuesFile string, isGit bool, policyresoucePath string) (err error) {
 	openAPIController, err := openapi.NewOpenAPIController()
 	engineResponses := make([]*response.EngineResponse, 0)
 	validateEngineResponses := make([]*response.EngineResponse, 0)
@@ -222,14 +255,21 @@ func applyPoliciesFromPath(fs billy.Filesystem, policyBytes []byte, valuesFile s
 	if err := json.Unmarshal(policyBytes, values); err != nil {
 		return sanitizederror.NewWithError("failed to decode yaml", err)
 	}
-	_, valuesMap, err := common.GetVariable(variablesString, values.Variables)
+
+	fmt.Printf("\nExecuting %s...", values.Name)
+
+	_, valuesMap, err := common.GetVariable(variablesString, values.Variables, fs, isGit, policyresoucePath)
 	if err != nil {
 		if !sanitizederror.IsErrorSanitized(err) {
 			return sanitizederror.NewWithError("failed to decode yaml", err)
 		}
 		return err
 	}
-	policies, err := common.GetPoliciesFromPaths(fs, values.Policies, isGit)
+
+	fullPolicyPath := getPolicyResouceFullPath(values.Policies, policyresoucePath, isGit)
+	fullResourcePath := getPolicyResouceFullPath(values.Resources, policyresoucePath, isGit)
+
+	policies, err := common.GetPoliciesFromPaths(fs, fullPolicyPath, isGit, policyresoucePath)
 	if err != nil {
 		fmt.Printf("Error: failed to load policies\nCause: %s\n", err)
 		os.Exit(1)
@@ -240,7 +280,7 @@ func applyPoliciesFromPath(fs billy.Filesystem, policyBytes []byte, valuesFile s
 			return sanitizederror.NewWithError("failed to mutate policy", err)
 		}
 	}
-	resources, err := common.GetResourceAccordingToResourcePath(fs, values.Resources, false, mutatedPolicies, dClient, "", false, isGit)
+	resources, err := common.GetResourceAccordingToResourcePath(fs, fullResourcePath, false, mutatedPolicies, dClient, "", false, isGit, policyresoucePath)
 	if err != nil {
 		fmt.Printf("Error: failed to load resources\nCause: %s\n", err)
 		os.Exit(1)
@@ -259,7 +299,6 @@ func applyPoliciesFromPath(fs billy.Filesystem, policyBytes []byte, valuesFile s
 	for _, policy := range mutatedPolicies {
 		err := policy2.Validate(policy, nil, true, openAPIController)
 		if err != nil {
-			fmt.Println("valuesMap1")
 			log.Log.V(3).Info(fmt.Sprintf("skipping policy %v as it is not valid", policy.Name), "error", err)
 			continue
 		}

pkg/kyverno/test/git.go

@@ -1,11 +1,12 @@
 package test
 
 import (
+	"os"
+	"path/filepath"
+
 	"github.com/go-git/go-billy/v5"
 	"github.com/go-git/go-git/v5"
 	"github.com/go-git/go-git/v5/storage/memory"
-	"os"
-	"path/filepath"
 )
 
 func clone(path string, fs billy.Filesystem) (*git.Repository, error) {
@@ -25,6 +26,12 @@ func listYAMLs(fs billy.Filesystem, path string) ([]string, error) {
 	for _, fi := range fis {
 		name := filepath.Join(path, fi.Name())
 		if fi.IsDir() {
+			moreYAMLs, err := listYAMLs(fs, name)
+			if err != nil {
+				return nil, err
+			}
+
+			yamls = append(yamls, moreYAMLs...)
 			continue
 		}