mirror of
https://github.com/kyverno/kyverno.git
synced 2025-03-09 09:26:54 +00:00
1020 lines
35 KiB
Go
1020 lines
35 KiB
Go
package test
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"net/url"
|
|
"os"
|
|
"path"
|
|
"path/filepath"
|
|
"regexp"
|
|
"sort"
|
|
"strings"
|
|
|
|
"github.com/go-git/go-billy/v5"
|
|
"github.com/go-git/go-billy/v5/memfs"
|
|
kyvernov1 "github.com/kyverno/kyverno/api/kyverno/v1"
|
|
"github.com/kyverno/kyverno/api/kyverno/v1beta1"
|
|
policyreportv1alpha2 "github.com/kyverno/kyverno/api/policyreport/v1alpha2"
|
|
"github.com/kyverno/kyverno/cmd/cli/kubectl-kyverno/test/api"
|
|
"github.com/kyverno/kyverno/cmd/cli/kubectl-kyverno/test/manifest"
|
|
"github.com/kyverno/kyverno/cmd/cli/kubectl-kyverno/utils/common"
|
|
sanitizederror "github.com/kyverno/kyverno/cmd/cli/kubectl-kyverno/utils/sanitizedError"
|
|
"github.com/kyverno/kyverno/cmd/cli/kubectl-kyverno/utils/store"
|
|
"github.com/kyverno/kyverno/pkg/autogen"
|
|
"github.com/kyverno/kyverno/pkg/background/generate"
|
|
"github.com/kyverno/kyverno/pkg/clients/dclient"
|
|
engineapi "github.com/kyverno/kyverno/pkg/engine/api"
|
|
"github.com/kyverno/kyverno/pkg/openapi"
|
|
policy2 "github.com/kyverno/kyverno/pkg/policy"
|
|
gitutils "github.com/kyverno/kyverno/pkg/utils/git"
|
|
"github.com/spf13/cobra"
|
|
"golang.org/x/exp/slices"
|
|
corev1 "k8s.io/api/core/v1"
|
|
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
|
"k8s.io/apimachinery/pkg/util/yaml"
|
|
"sigs.k8s.io/controller-runtime/pkg/log"
|
|
)
|
|
|
|
// Command returns version command
|
|
func Command() *cobra.Command {
|
|
var cmd *cobra.Command
|
|
var testCase string
|
|
var fileName, gitBranch string
|
|
var registryAccess, failOnly, removeColor, manifestValidate, manifestMutate bool
|
|
cmd = &cobra.Command{
|
|
Use: "test <path_to_folder_Containing_test.yamls> [flags]\n kyverno test <path_to_gitRepository_with_dir> --git-branch <branchName>\n kyverno test --manifest-mutate > kyverno-test.yaml\n kyverno test --manifest-validate > kyverno-test.yaml",
|
|
// Args: cobra.ExactArgs(1),
|
|
Short: "Run tests from directory.",
|
|
Long: longHelp,
|
|
Example: exampleHelp,
|
|
RunE: func(cmd *cobra.Command, dirPath []string) (err error) {
|
|
defer func() {
|
|
if err != nil {
|
|
if !sanitizederror.IsErrorSanitized(err) {
|
|
log.Log.Error(err, "failed to sanitize")
|
|
err = fmt.Errorf("internal error")
|
|
}
|
|
}
|
|
}()
|
|
if manifestMutate {
|
|
manifest.PrintMutate()
|
|
} else if manifestValidate {
|
|
manifest.PrintValidate()
|
|
} else {
|
|
store.SetRegistryAccess(registryAccess)
|
|
_, err = testCommandExecute(dirPath, fileName, gitBranch, testCase, failOnly, removeColor, false)
|
|
if err != nil {
|
|
log.Log.V(3).Info("a directory is required")
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
},
|
|
}
|
|
cmd.Flags().StringVarP(&fileName, "file-name", "f", "kyverno-test.yaml", "test filename")
|
|
cmd.Flags().StringVarP(&gitBranch, "git-branch", "b", "", "test github repository branch")
|
|
cmd.Flags().StringVarP(&testCase, "test-case-selector", "t", "", `run some specific test cases by passing a string argument in double quotes to this flag like - "policy=<policy_name>, rule=<rule_name>, resource=<resource_name". The argument could be any combination of policy, rule and resource.`)
|
|
cmd.Flags().BoolVarP(&manifestMutate, "manifest-mutate", "", false, "prints out a template test manifest for a mutate policy")
|
|
cmd.Flags().BoolVarP(&manifestValidate, "manifest-validate", "", false, "prints out a template test manifest for a validate policy")
|
|
cmd.Flags().BoolVarP(®istryAccess, "registry", "", false, "If set to true, access the image registry using local docker credentials to populate external data")
|
|
cmd.Flags().BoolVarP(&failOnly, "fail-only", "", false, "If set to true, display all the failing test only as output for the test command")
|
|
cmd.Flags().BoolVarP(&removeColor, "remove-color", "", false, "Remove any color from output")
|
|
return cmd
|
|
}
|
|
|
|
type Table struct {
|
|
ID int `header:"#"`
|
|
Policy string `header:"policy"`
|
|
Rule string `header:"rule"`
|
|
Resource string `header:"resource"`
|
|
Result string `header:"result"`
|
|
}
|
|
|
|
type resultCounts struct {
|
|
Skip int
|
|
Pass int
|
|
Fail int
|
|
}
|
|
|
|
var ftable []Table
|
|
|
|
func testCommandExecute(
|
|
dirPath []string,
|
|
fileName string,
|
|
gitBranch string,
|
|
testCase string,
|
|
failOnly bool,
|
|
removeColor bool,
|
|
auditWarn bool,
|
|
) (rc *resultCounts, err error) {
|
|
// check input dir
|
|
if len(dirPath) == 0 {
|
|
return rc, sanitizederror.NewWithError("a directory is required", err)
|
|
}
|
|
// parse filter
|
|
filter := parseFilter(testCase)
|
|
// init openapi manager
|
|
openApiManager, err := openapi.NewManager(log.Log)
|
|
if err != nil {
|
|
return rc, fmt.Errorf("unable to create open api controller, %w", err)
|
|
}
|
|
var errors []error
|
|
fs := memfs.New()
|
|
rc = &resultCounts{}
|
|
var testYamlCount int
|
|
if strings.Contains(dirPath[0], "https://") {
|
|
gitURL, err := url.Parse(dirPath[0])
|
|
if err != nil {
|
|
return rc, sanitizederror.NewWithError("failed to parse URL", err)
|
|
}
|
|
|
|
pathElems := strings.Split(gitURL.Path[1:], "/")
|
|
if len(pathElems) <= 1 {
|
|
err := fmt.Errorf("invalid URL path %s - expected https://github.com/:owner/:repository/:branch (without --git-branch flag) OR https://github.com/:owner/:repository/:directory (with --git-branch flag)", gitURL.Path)
|
|
fmt.Printf("Error: failed to parse URL \nCause: %s\n", err)
|
|
os.Exit(1)
|
|
}
|
|
|
|
gitURL.Path = strings.Join([]string{pathElems[0], pathElems[1]}, "/")
|
|
repoURL := gitURL.String()
|
|
|
|
var gitPathToYamls string
|
|
if gitBranch == "" {
|
|
gitPathToYamls = "/"
|
|
|
|
if string(dirPath[0][len(dirPath[0])-1]) == "/" {
|
|
gitBranch = strings.ReplaceAll(dirPath[0], repoURL+"/", "")
|
|
} else {
|
|
gitBranch = strings.ReplaceAll(dirPath[0], repoURL, "")
|
|
}
|
|
|
|
if gitBranch == "" {
|
|
gitBranch = "main"
|
|
} else if string(gitBranch[0]) == "/" {
|
|
gitBranch = gitBranch[1:]
|
|
}
|
|
} else {
|
|
if string(dirPath[0][len(dirPath[0])-1]) == "/" {
|
|
gitPathToYamls = strings.ReplaceAll(dirPath[0], repoURL+"/", "/")
|
|
} else {
|
|
gitPathToYamls = strings.ReplaceAll(dirPath[0], repoURL, "/")
|
|
}
|
|
}
|
|
|
|
_, cloneErr := gitutils.Clone(repoURL, fs, gitBranch)
|
|
if cloneErr != nil {
|
|
fmt.Printf("Error: failed to clone repository \nCause: %s\n", cloneErr)
|
|
log.Log.V(3).Info(fmt.Sprintf("failed to clone repository %v as it is not valid", repoURL), "error", cloneErr)
|
|
os.Exit(1)
|
|
}
|
|
|
|
policyYamls, err := gitutils.ListYamls(fs, gitPathToYamls)
|
|
if err != nil {
|
|
return rc, sanitizederror.NewWithError("failed to list YAMLs in repository", err)
|
|
}
|
|
sort.Strings(policyYamls)
|
|
|
|
for _, yamlFilePath := range policyYamls {
|
|
file, err := fs.Open(yamlFilePath)
|
|
if err != nil {
|
|
errors = append(errors, sanitizederror.NewWithError("Error: failed to open file", err))
|
|
continue
|
|
}
|
|
|
|
if path.Base(file.Name()) == fileName {
|
|
testYamlCount++
|
|
policyresoucePath := strings.Trim(yamlFilePath, fileName)
|
|
bytes, err := io.ReadAll(file)
|
|
if err != nil {
|
|
errors = append(errors, sanitizederror.NewWithError("Error: failed to read file", err))
|
|
continue
|
|
}
|
|
|
|
policyBytes, err := yaml.ToJSON(bytes)
|
|
if err != nil {
|
|
errors = append(errors, sanitizederror.NewWithError("failed to convert to JSON", err))
|
|
continue
|
|
}
|
|
if err := applyPoliciesFromPath(fs, policyBytes, true, policyresoucePath, rc, openApiManager, filter, failOnly, removeColor, auditWarn); err != nil {
|
|
return rc, sanitizederror.NewWithError("failed to apply test command", err)
|
|
}
|
|
}
|
|
}
|
|
|
|
if testYamlCount == 0 {
|
|
fmt.Printf("\n No test yamls available \n")
|
|
}
|
|
} else {
|
|
var testFiles int
|
|
path := filepath.Clean(dirPath[0])
|
|
errors = getLocalDirTestFiles(fs, path, fileName, rc, &testFiles, openApiManager, filter, failOnly, removeColor, auditWarn)
|
|
|
|
if testFiles == 0 {
|
|
fmt.Printf("\n No test files found. Please provide test YAML files named kyverno-test.yaml \n")
|
|
}
|
|
}
|
|
|
|
if len(errors) > 0 && log.Log.V(1).Enabled() {
|
|
fmt.Printf("test errors: \n")
|
|
for _, e := range errors {
|
|
fmt.Printf(" %v \n", e.Error())
|
|
}
|
|
}
|
|
|
|
if !failOnly {
|
|
fmt.Printf("\nTest Summary: %d tests passed and %d tests failed\n", rc.Pass+rc.Skip, rc.Fail)
|
|
} else {
|
|
fmt.Printf("\nTest Summary: %d out of %d tests failed\n", rc.Fail, rc.Pass+rc.Skip+rc.Fail)
|
|
}
|
|
fmt.Printf("\n")
|
|
|
|
if rc.Fail > 0 && !failOnly {
|
|
printFailedTestResult(removeColor)
|
|
os.Exit(1)
|
|
}
|
|
os.Exit(0)
|
|
return rc, nil
|
|
}
|
|
|
|
func getLocalDirTestFiles(
|
|
fs billy.Filesystem,
|
|
path string,
|
|
fileName string,
|
|
rc *resultCounts,
|
|
testFiles *int,
|
|
openApiManager openapi.Manager,
|
|
filter filter,
|
|
failOnly bool,
|
|
removeColor bool,
|
|
auditWarn bool,
|
|
) []error {
|
|
var errors []error
|
|
|
|
files, err := os.ReadDir(path)
|
|
if err != nil {
|
|
return []error{fmt.Errorf("failed to read %v: %v", path, err.Error())}
|
|
}
|
|
for _, file := range files {
|
|
if file.IsDir() {
|
|
getLocalDirTestFiles(fs, filepath.Join(path, file.Name()), fileName, rc, testFiles, openApiManager, filter, failOnly, removeColor, auditWarn)
|
|
continue
|
|
}
|
|
if file.Name() == fileName {
|
|
*testFiles++
|
|
// We accept the risk of including files here as we read the test dir only.
|
|
yamlFile, err := os.ReadFile(filepath.Join(path, file.Name())) // #nosec G304
|
|
if err != nil {
|
|
errors = append(errors, sanitizederror.NewWithError("unable to read yaml", err))
|
|
continue
|
|
}
|
|
valuesBytes, err := yaml.ToJSON(yamlFile)
|
|
if err != nil {
|
|
errors = append(errors, sanitizederror.NewWithError("failed to convert json", err))
|
|
continue
|
|
}
|
|
if err := applyPoliciesFromPath(fs, valuesBytes, false, path, rc, openApiManager, filter, failOnly, removeColor, auditWarn); err != nil {
|
|
errors = append(errors, sanitizederror.NewWithError(fmt.Sprintf("failed to apply test command from file %s", file.Name()), err))
|
|
continue
|
|
}
|
|
}
|
|
}
|
|
return errors
|
|
}
|
|
|
|
func buildPolicyResults(
|
|
engineResponses []engineapi.EngineResponse,
|
|
testResults []api.TestResults,
|
|
policyResourcePath string,
|
|
fs billy.Filesystem,
|
|
isGit bool,
|
|
auditWarn bool,
|
|
) (map[string]policyreportv1alpha2.PolicyReportResult, []api.TestResults) {
|
|
results := map[string]policyreportv1alpha2.PolicyReportResult{}
|
|
|
|
for _, resp := range engineResponses {
|
|
policyName := resp.Policy.GetName()
|
|
resourceName := resp.Resource.GetName()
|
|
resourceKind := resp.Resource.GetKind()
|
|
resourceNamespace := resp.Resource.GetNamespace()
|
|
policyNamespace := resp.Policy.GetNamespace()
|
|
|
|
var rules []string
|
|
for _, rule := range resp.PolicyResponse.Rules {
|
|
rules = append(rules, rule.Name())
|
|
}
|
|
|
|
result := policyreportv1alpha2.PolicyReportResult{
|
|
Policy: policyName,
|
|
Resources: []corev1.ObjectReference{
|
|
{
|
|
Name: resourceName,
|
|
},
|
|
},
|
|
Message: buildMessage(resp),
|
|
}
|
|
|
|
var patchedResourcePath []string
|
|
for i, test := range testResults {
|
|
var userDefinedPolicyNamespace string
|
|
var userDefinedPolicyName string
|
|
found, err := isNamespacedPolicy(test.Policy)
|
|
if err != nil {
|
|
log.Log.V(3).Info("error while checking the policy is namespaced or not", "policy: ", test.Policy, "error: ", err)
|
|
continue
|
|
}
|
|
|
|
if found {
|
|
userDefinedPolicyNamespace, userDefinedPolicyName = getUserDefinedPolicyNameAndNamespace(test.Policy)
|
|
test.Policy = userDefinedPolicyName
|
|
}
|
|
|
|
if test.Resources != nil {
|
|
if test.Policy == policyName {
|
|
// results[].namespace value implicit set same as metadata.namespace until and unless
|
|
// user provides explicit values for results[].namespace in test yaml file.
|
|
if test.Namespace == "" {
|
|
test.Namespace = resourceNamespace
|
|
testResults[i].Namespace = resourceNamespace
|
|
}
|
|
for _, resource := range test.Resources {
|
|
if resource == resourceName {
|
|
var resultsKey string
|
|
resultsKey = GetResultKeyAccordingToTestResults(userDefinedPolicyNamespace, test.Policy, test.Rule, test.Namespace, test.Kind, resource)
|
|
if !slices.Contains(rules, test.Rule) {
|
|
if !slices.Contains(rules, "autogen-"+test.Rule) {
|
|
if !slices.Contains(rules, "autogen-cronjob-"+test.Rule) {
|
|
result.Result = policyreportv1alpha2.StatusSkip
|
|
} else {
|
|
testResults[i].AutoGeneratedRule = "autogen-cronjob"
|
|
test.Rule = "autogen-cronjob-" + test.Rule
|
|
resultsKey = GetResultKeyAccordingToTestResults(userDefinedPolicyNamespace, test.Policy, test.Rule, test.Namespace, test.Kind, resource)
|
|
}
|
|
} else {
|
|
testResults[i].AutoGeneratedRule = "autogen"
|
|
test.Rule = "autogen-" + test.Rule
|
|
resultsKey = GetResultKeyAccordingToTestResults(userDefinedPolicyNamespace, test.Policy, test.Rule, test.Namespace, test.Kind, resource)
|
|
}
|
|
|
|
if results[resultsKey].Result == "" {
|
|
result.Result = policyreportv1alpha2.StatusSkip
|
|
results[resultsKey] = result
|
|
}
|
|
}
|
|
|
|
patchedResourcePath = append(patchedResourcePath, test.PatchedResource)
|
|
if _, ok := results[resultsKey]; !ok {
|
|
results[resultsKey] = result
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if test.Resource != "" {
|
|
if test.Policy == policyName && test.Resource == resourceName {
|
|
var resultsKey string
|
|
resultsKey = GetResultKeyAccordingToTestResults(userDefinedPolicyNamespace, test.Policy, test.Rule, test.Namespace, test.Kind, test.Resource)
|
|
if !slices.Contains(rules, test.Rule) {
|
|
if !slices.Contains(rules, "autogen-"+test.Rule) {
|
|
if !slices.Contains(rules, "autogen-cronjob-"+test.Rule) {
|
|
result.Result = policyreportv1alpha2.StatusSkip
|
|
} else {
|
|
testResults[i].AutoGeneratedRule = "autogen-cronjob"
|
|
test.Rule = "autogen-cronjob-" + test.Rule
|
|
resultsKey = GetResultKeyAccordingToTestResults(userDefinedPolicyNamespace, test.Policy, test.Rule, test.Namespace, test.Kind, test.Resource)
|
|
}
|
|
} else {
|
|
testResults[i].AutoGeneratedRule = "autogen"
|
|
test.Rule = "autogen-" + test.Rule
|
|
resultsKey = GetResultKeyAccordingToTestResults(userDefinedPolicyNamespace, test.Policy, test.Rule, test.Namespace, test.Kind, test.Resource)
|
|
}
|
|
|
|
if results[resultsKey].Result == "" {
|
|
result.Result = policyreportv1alpha2.StatusSkip
|
|
results[resultsKey] = result
|
|
}
|
|
}
|
|
|
|
patchedResourcePath = append(patchedResourcePath, test.PatchedResource)
|
|
if _, ok := results[resultsKey]; !ok {
|
|
results[resultsKey] = result
|
|
}
|
|
}
|
|
}
|
|
|
|
for _, rule := range resp.PolicyResponse.Rules {
|
|
if rule.RuleType() != engineapi.Generation || test.Rule != rule.Name() {
|
|
continue
|
|
}
|
|
|
|
var resultsKey []string
|
|
var resultKey string
|
|
var result policyreportv1alpha2.PolicyReportResult
|
|
resultsKey = GetAllPossibleResultsKey(policyNamespace, policyName, rule.Name(), resourceNamespace, resourceKind, resourceName)
|
|
for _, key := range resultsKey {
|
|
if val, ok := results[key]; ok {
|
|
result = val
|
|
resultKey = key
|
|
} else {
|
|
continue
|
|
}
|
|
|
|
if rule.Status() == engineapi.RuleStatusSkip {
|
|
result.Result = policyreportv1alpha2.StatusSkip
|
|
} else if rule.Status() == engineapi.RuleStatusError {
|
|
result.Result = policyreportv1alpha2.StatusError
|
|
} else {
|
|
var x string
|
|
result.Result = policyreportv1alpha2.StatusFail
|
|
x = getAndCompareResource(test.GeneratedResource, rule.GeneratedResource(), isGit, policyResourcePath, fs, true)
|
|
if x == "pass" {
|
|
result.Result = policyreportv1alpha2.StatusPass
|
|
}
|
|
}
|
|
results[resultKey] = result
|
|
}
|
|
}
|
|
|
|
for _, rule := range resp.PolicyResponse.Rules {
|
|
if rule.RuleType() != engineapi.Mutation || test.Rule != rule.Name() {
|
|
continue
|
|
}
|
|
|
|
var resultsKey []string
|
|
var resultKey string
|
|
var result policyreportv1alpha2.PolicyReportResult
|
|
resultsKey = GetAllPossibleResultsKey(policyNamespace, policyName, rule.Name(), resourceNamespace, resourceKind, resourceName)
|
|
for _, key := range resultsKey {
|
|
if val, ok := results[key]; ok {
|
|
result = val
|
|
resultKey = key
|
|
} else {
|
|
continue
|
|
}
|
|
|
|
if rule.Status() == engineapi.RuleStatusSkip {
|
|
result.Result = policyreportv1alpha2.StatusSkip
|
|
} else if rule.Status() == engineapi.RuleStatusError {
|
|
result.Result = policyreportv1alpha2.StatusError
|
|
} else {
|
|
var x string
|
|
for _, path := range patchedResourcePath {
|
|
result.Result = policyreportv1alpha2.StatusFail
|
|
x = getAndCompareResource(path, resp.PatchedResource, isGit, policyResourcePath, fs, false)
|
|
if x == "pass" {
|
|
result.Result = policyreportv1alpha2.StatusPass
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
results[resultKey] = result
|
|
}
|
|
}
|
|
|
|
for _, rule := range resp.PolicyResponse.Rules {
|
|
if rule.RuleType() != engineapi.Validation && rule.RuleType() != engineapi.ImageVerify || test.Rule != rule.Name() {
|
|
continue
|
|
}
|
|
|
|
var resultsKey []string
|
|
var resultKey string
|
|
var result policyreportv1alpha2.PolicyReportResult
|
|
resultsKey = GetAllPossibleResultsKey(policyNamespace, policyName, rule.Name(), resourceNamespace, resourceKind, resourceName)
|
|
for _, key := range resultsKey {
|
|
if val, ok := results[key]; ok {
|
|
result = val
|
|
resultKey = key
|
|
} else {
|
|
continue
|
|
}
|
|
|
|
ann := resp.Policy.GetAnnotations()
|
|
if rule.Status() == engineapi.RuleStatusSkip {
|
|
result.Result = policyreportv1alpha2.StatusSkip
|
|
} else if rule.Status() == engineapi.RuleStatusError {
|
|
result.Result = policyreportv1alpha2.StatusError
|
|
} else if rule.Status() == engineapi.RuleStatusPass {
|
|
result.Result = policyreportv1alpha2.StatusPass
|
|
} else if rule.Status() == engineapi.RuleStatusFail {
|
|
if scored, ok := ann[kyvernov1.AnnotationPolicyScored]; ok && scored == "false" {
|
|
result.Result = policyreportv1alpha2.StatusWarn
|
|
} else if auditWarn && resp.GetValidationFailureAction().Audit() {
|
|
result.Result = policyreportv1alpha2.StatusWarn
|
|
} else {
|
|
result.Result = policyreportv1alpha2.StatusFail
|
|
}
|
|
} else {
|
|
fmt.Println(rule)
|
|
}
|
|
|
|
results[resultKey] = result
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return results, testResults
|
|
}
|
|
|
|
func GetAllPossibleResultsKey(policyNamespace, policy, rule, resourceNamespace, kind, resource string) []string {
|
|
var resultsKey []string
|
|
resultKey1 := fmt.Sprintf("%s-%s-%s-%s", policy, rule, kind, resource)
|
|
resultKey2 := fmt.Sprintf("%s-%s-%s-%s-%s", policy, rule, resourceNamespace, kind, resource)
|
|
resultKey3 := fmt.Sprintf("%s-%s-%s-%s-%s", policyNamespace, policy, rule, kind, resource)
|
|
resultKey4 := fmt.Sprintf("%s-%s-%s-%s-%s-%s", policyNamespace, policy, rule, resourceNamespace, kind, resource)
|
|
resultsKey = append(resultsKey, resultKey1, resultKey2, resultKey3, resultKey4)
|
|
return resultsKey
|
|
}
|
|
|
|
func GetResultKeyAccordingToTestResults(policyNs, policy, rule, resourceNs, kind, resource string) string {
|
|
var resultKey string
|
|
resultKey = fmt.Sprintf("%s-%s-%s-%s", policy, rule, kind, resource)
|
|
|
|
if policyNs != "" && resourceNs != "" {
|
|
resultKey = fmt.Sprintf("%s-%s-%s-%s-%s-%s", policyNs, policy, rule, resourceNs, kind, resource)
|
|
} else if policyNs != "" {
|
|
resultKey = fmt.Sprintf("%s-%s-%s-%s-%s", policyNs, policy, rule, kind, resource)
|
|
} else if resourceNs != "" {
|
|
resultKey = fmt.Sprintf("%s-%s-%s-%s-%s", policy, rule, resourceNs, kind, resource)
|
|
}
|
|
return resultKey
|
|
}
|
|
|
|
func isNamespacedPolicy(policyNames string) (bool, error) {
|
|
return regexp.MatchString("^[a-z]*/[a-z]*", policyNames)
|
|
}
|
|
|
|
func getUserDefinedPolicyNameAndNamespace(policyName string) (string, string) {
|
|
if strings.Contains(policyName, "/") {
|
|
parts := strings.Split(policyName, "/")
|
|
namespace := parts[0]
|
|
policy := parts[1]
|
|
return namespace, policy
|
|
}
|
|
return "", policyName
|
|
}
|
|
|
|
// getAndCompareResource --> Get the patchedResource or generatedResource from the path provided by user
|
|
// And compare this resource with engine generated resource.
|
|
func getAndCompareResource(path string, engineResource unstructured.Unstructured, isGit bool, policyResourcePath string, fs billy.Filesystem, isGenerate bool) string {
|
|
var status string
|
|
resourceType := "patchedResource"
|
|
if isGenerate {
|
|
resourceType = "generatedResource"
|
|
}
|
|
|
|
userResource, err := common.GetResourceFromPath(fs, path, isGit, policyResourcePath, resourceType)
|
|
if err != nil {
|
|
fmt.Printf("Error: failed to load resources\nCause: %s\n", err)
|
|
return ""
|
|
}
|
|
matched, err := generate.ValidateResourceWithPattern(log.Log, engineResource.UnstructuredContent(), userResource.UnstructuredContent())
|
|
if err != nil {
|
|
log.Log.V(3).Info(resourceType+" mismatch", "error", err.Error())
|
|
status = "fail"
|
|
} else if matched == "" {
|
|
status = "pass"
|
|
}
|
|
return status
|
|
}
|
|
|
|
func buildMessage(resp engineapi.EngineResponse) string {
|
|
var bldr strings.Builder
|
|
for _, ruleResp := range resp.PolicyResponse.Rules {
|
|
fmt.Fprintf(&bldr, " %s: %s \n", ruleResp.Name(), ruleResp.Status())
|
|
fmt.Fprintf(&bldr, " %s \n", ruleResp.Message())
|
|
}
|
|
|
|
return bldr.String()
|
|
}
|
|
|
|
func getFullPath(paths []string, policyResourcePath string, isGit bool) []string {
|
|
var pols []string
|
|
var pol string
|
|
if !isGit {
|
|
for _, path := range paths {
|
|
pol = filepath.Join(policyResourcePath, path)
|
|
pols = append(pols, pol)
|
|
}
|
|
return pols
|
|
}
|
|
return paths
|
|
}
|
|
|
|
func applyPoliciesFromPath(
|
|
fs billy.Filesystem,
|
|
policyBytes []byte,
|
|
isGit bool,
|
|
policyResourcePath string,
|
|
rc *resultCounts,
|
|
openApiManager openapi.Manager,
|
|
filter filter,
|
|
failOnly bool,
|
|
removeColor bool,
|
|
auditWarn bool,
|
|
) (err error) {
|
|
engineResponses := make([]engineapi.EngineResponse, 0)
|
|
var dClient dclient.Interface
|
|
values := &api.Test{}
|
|
var variablesString string
|
|
var resultCounts common.ResultCounts
|
|
|
|
store.SetMock(true)
|
|
if err := json.Unmarshal(policyBytes, values); err != nil {
|
|
return sanitizederror.NewWithError("failed to decode yaml", err)
|
|
}
|
|
|
|
var filteredResults []api.TestResults
|
|
for _, res := range values.Results {
|
|
if filter(res) {
|
|
filteredResults = append(filteredResults, res)
|
|
}
|
|
}
|
|
values.Results = filteredResults
|
|
|
|
if len(values.Results) == 0 {
|
|
return nil
|
|
}
|
|
|
|
fmt.Printf("\nExecuting %s...\n", values.Name)
|
|
valuesFile := values.Variables
|
|
userInfoFile := values.UserInfo
|
|
|
|
variables, globalValMap, valuesMap, namespaceSelectorMap, subresources, err := common.GetVariable(variablesString, values.Variables, fs, isGit, policyResourcePath)
|
|
if err != nil {
|
|
if !sanitizederror.IsErrorSanitized(err) {
|
|
return sanitizederror.NewWithError("failed to decode yaml", err)
|
|
}
|
|
return err
|
|
}
|
|
|
|
// get the user info as request info from a different file
|
|
var userInfo v1beta1.RequestInfo
|
|
|
|
if userInfoFile != "" {
|
|
userInfo, err = common.GetUserInfoFromPath(fs, userInfoFile, isGit, policyResourcePath)
|
|
if err != nil {
|
|
fmt.Printf("Error: failed to load request info\nCause: %s\n", err)
|
|
os.Exit(1)
|
|
}
|
|
}
|
|
|
|
policyFullPath := getFullPath(values.Policies, policyResourcePath, isGit)
|
|
resourceFullPath := getFullPath(values.Resources, policyResourcePath, isGit)
|
|
|
|
for i, result := range values.Results {
|
|
arrPatchedResource := []string{result.PatchedResource}
|
|
arrGeneratedResource := []string{result.GeneratedResource}
|
|
arrCloneSourceResource := []string{result.CloneSourceResource}
|
|
|
|
patchedResourceFullPath := getFullPath(arrPatchedResource, policyResourcePath, isGit)
|
|
generatedResourceFullPath := getFullPath(arrGeneratedResource, policyResourcePath, isGit)
|
|
CloneSourceResourceFullPath := getFullPath(arrCloneSourceResource, policyResourcePath, isGit)
|
|
|
|
values.Results[i].PatchedResource = patchedResourceFullPath[0]
|
|
values.Results[i].GeneratedResource = generatedResourceFullPath[0]
|
|
values.Results[i].CloneSourceResource = CloneSourceResourceFullPath[0]
|
|
}
|
|
|
|
policies, err := common.GetPoliciesFromPaths(fs, policyFullPath, isGit, policyResourcePath)
|
|
if err != nil {
|
|
fmt.Printf("Error: failed to load policies\nCause: %s\n", err)
|
|
os.Exit(1)
|
|
}
|
|
|
|
var filteredPolicies []kyvernov1.PolicyInterface
|
|
for _, p := range policies {
|
|
for _, res := range values.Results {
|
|
if p.GetName() == res.Policy {
|
|
filteredPolicies = append(filteredPolicies, p)
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
ruleToCloneSourceResource := map[string]string{}
|
|
for _, p := range filteredPolicies {
|
|
var filteredRules []kyvernov1.Rule
|
|
|
|
for _, rule := range autogen.ComputeRules(p) {
|
|
for _, res := range values.Results {
|
|
if rule.Name == res.Rule {
|
|
filteredRules = append(filteredRules, rule)
|
|
if rule.HasGenerate() {
|
|
ruleUnstr, err := generate.GetUnstrRule(rule.Generation.DeepCopy())
|
|
if err != nil {
|
|
fmt.Printf("Error: failed to get unstructured rule\nCause: %s\n", err)
|
|
break
|
|
}
|
|
|
|
genClone, _, err := unstructured.NestedMap(ruleUnstr.Object, "clone")
|
|
if err != nil {
|
|
fmt.Printf("Error: failed to read data\nCause: %s\n", err)
|
|
break
|
|
}
|
|
|
|
if len(genClone) != 0 {
|
|
ruleToCloneSourceResource[rule.Name] = res.CloneSourceResource
|
|
}
|
|
}
|
|
break
|
|
}
|
|
}
|
|
}
|
|
p.GetSpec().SetRules(filteredRules)
|
|
}
|
|
policies = filteredPolicies
|
|
|
|
err = common.PrintMutatedPolicy(policies)
|
|
if err != nil {
|
|
return sanitizederror.NewWithError("failed to print mutated policy", err)
|
|
}
|
|
|
|
resources, err := common.GetResourceAccordingToResourcePath(fs, resourceFullPath, false, policies, dClient, "", false, isGit, policyResourcePath)
|
|
if err != nil {
|
|
fmt.Printf("Error: failed to load resources\nCause: %s\n", err)
|
|
os.Exit(1)
|
|
}
|
|
|
|
checkableResources := selectResourcesForCheck(resources, values)
|
|
|
|
msgPolicies := "1 policy"
|
|
if len(policies) > 1 {
|
|
msgPolicies = fmt.Sprintf("%d policies", len(policies))
|
|
}
|
|
|
|
msgResources := "1 resource"
|
|
if len(checkableResources) > 1 {
|
|
msgResources = fmt.Sprintf("%d resources", len(checkableResources))
|
|
}
|
|
|
|
if len(policies) > 0 && len(checkableResources) > 0 {
|
|
fmt.Printf("applying %s to %s... \n", msgPolicies, msgResources)
|
|
}
|
|
|
|
for _, policy := range policies {
|
|
_, err := policy2.Validate(policy, nil, nil, true, openApiManager)
|
|
if err != nil {
|
|
log.Log.Error(err, "skipping invalid policy", "name", policy.GetName())
|
|
continue
|
|
}
|
|
|
|
matches := common.HasVariables(policy)
|
|
variable := common.RemoveDuplicateAndObjectVariables(matches)
|
|
|
|
if len(variable) > 0 {
|
|
if len(variables) == 0 {
|
|
// check policy in variable file
|
|
if valuesFile == "" || valuesMap[policy.GetName()] == nil {
|
|
fmt.Printf("test skipped for policy %v (as required variables are not provided by the users) \n \n", policy.GetName())
|
|
}
|
|
}
|
|
}
|
|
|
|
kindOnwhichPolicyIsApplied := common.GetKindsFromPolicy(policy, subresources, dClient)
|
|
|
|
for _, resource := range checkableResources {
|
|
thisPolicyResourceValues, err := common.CheckVariableForPolicy(valuesMap, globalValMap, policy.GetName(), resource.GetName(), resource.GetKind(), variables, kindOnwhichPolicyIsApplied, variable)
|
|
if err != nil {
|
|
return sanitizederror.NewWithError(fmt.Sprintf("policy `%s` have variables. pass the values for the variables for resource `%s` using set/values_file flag", policy.GetName(), resource.GetName()), err)
|
|
}
|
|
applyPolicyConfig := common.ApplyPolicyConfig{
|
|
Policy: policy,
|
|
Resource: resource,
|
|
MutateLogPath: "",
|
|
Variables: thisPolicyResourceValues,
|
|
UserInfo: userInfo,
|
|
PolicyReport: true,
|
|
NamespaceSelectorMap: namespaceSelectorMap,
|
|
Rc: &resultCounts,
|
|
RuleToCloneSourceResource: ruleToCloneSourceResource,
|
|
Client: dClient,
|
|
Subresources: subresources,
|
|
}
|
|
ers, err := common.ApplyPolicyOnResource(applyPolicyConfig)
|
|
if err != nil {
|
|
return sanitizederror.NewWithError(fmt.Errorf("failed to apply policy %v on resource %v", policy.GetName(), resource.GetName()).Error(), err)
|
|
}
|
|
engineResponses = append(engineResponses, ers...)
|
|
}
|
|
}
|
|
resultsMap, testResults := buildPolicyResults(engineResponses, values.Results, policyResourcePath, fs, isGit, auditWarn)
|
|
resultErr := printTestResult(resultsMap, testResults, rc, failOnly, removeColor)
|
|
if resultErr != nil {
|
|
return sanitizederror.NewWithError("failed to print test result:", resultErr)
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
func selectResourcesForCheck(resources []*unstructured.Unstructured, values *api.Test) []*unstructured.Unstructured {
|
|
res, _, _ := selectResourcesForCheckInternal(resources, values)
|
|
return res
|
|
}
|
|
|
|
// selectResourcesForCheckInternal internal method to test duplicates and unused
|
|
func selectResourcesForCheckInternal(resources []*unstructured.Unstructured, values *api.Test) ([]*unstructured.Unstructured, int, int) {
|
|
var duplicates int
|
|
var unused int
|
|
uniqResources := make(map[string]*unstructured.Unstructured)
|
|
|
|
for i := range resources {
|
|
r := resources[i]
|
|
key := fmt.Sprintf("%s/%s/%s", r.GetKind(), r.GetName(), r.GetNamespace())
|
|
if _, ok := uniqResources[key]; ok {
|
|
fmt.Println("skipping duplicate resource, resource :", r)
|
|
duplicates++
|
|
} else {
|
|
uniqResources[key] = r
|
|
}
|
|
}
|
|
|
|
selectedResources := map[string]*unstructured.Unstructured{}
|
|
for key := range uniqResources {
|
|
r := uniqResources[key]
|
|
for _, res := range values.Results {
|
|
if res.Kind == r.GetKind() {
|
|
for _, testr := range res.Resources {
|
|
if r.GetName() == testr {
|
|
selectedResources[key] = r
|
|
}
|
|
}
|
|
if r.GetName() == res.Resource {
|
|
selectedResources[key] = r
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
var checkableResources []*unstructured.Unstructured
|
|
|
|
for key := range selectedResources {
|
|
checkableResources = append(checkableResources, selectedResources[key])
|
|
delete(uniqResources, key)
|
|
}
|
|
for _, r := range uniqResources {
|
|
fmt.Println("skipping unused resource, resource :", r)
|
|
unused++
|
|
}
|
|
return checkableResources, duplicates, unused
|
|
}
|
|
|
|
func printTestResult(resps map[string]policyreportv1alpha2.PolicyReportResult, testResults []api.TestResults, rc *resultCounts, failOnly, removeColor bool) error {
|
|
printer := newTablePrinter(removeColor)
|
|
var table []Table
|
|
|
|
var countDeprecatedResource int
|
|
testCount := 1
|
|
for _, v := range testResults {
|
|
res := new(Table)
|
|
res.ID = testCount
|
|
if v.Resources == nil {
|
|
testCount++
|
|
}
|
|
res.Policy = colorize(removeColor, boldFgCyan, v.Policy)
|
|
res.Rule = colorize(removeColor, boldFgCyan, v.Rule)
|
|
|
|
if v.Resources != nil {
|
|
for _, resource := range v.Resources {
|
|
res.ID = testCount
|
|
testCount++
|
|
res.Resource = colorize(removeColor, boldFgCyan, v.Namespace) + "/" + colorize(removeColor, boldFgCyan, v.Kind) + "/" + colorize(removeColor, boldFgCyan, resource)
|
|
var ruleNameInResultKey string
|
|
if v.AutoGeneratedRule != "" {
|
|
ruleNameInResultKey = fmt.Sprintf("%s-%s", v.AutoGeneratedRule, v.Rule)
|
|
} else {
|
|
ruleNameInResultKey = v.Rule
|
|
}
|
|
|
|
resultKey := fmt.Sprintf("%s-%s-%s-%s", v.Policy, ruleNameInResultKey, v.Kind, resource)
|
|
found, _ := isNamespacedPolicy(v.Policy)
|
|
var ns string
|
|
ns, v.Policy = getUserDefinedPolicyNameAndNamespace(v.Policy)
|
|
if found && v.Namespace != "" {
|
|
resultKey = fmt.Sprintf("%s-%s-%s-%s-%s-%s", ns, v.Policy, ruleNameInResultKey, v.Namespace, v.Kind, resource)
|
|
} else if found {
|
|
resultKey = fmt.Sprintf("%s-%s-%s-%s-%s", ns, v.Policy, ruleNameInResultKey, v.Kind, resource)
|
|
res.Policy = colorize(removeColor, boldFgCyan, ns) + "/" + colorize(removeColor, boldFgCyan, v.Policy)
|
|
res.Resource = colorize(removeColor, boldFgCyan, v.Namespace) + "/" + colorize(removeColor, boldFgCyan, v.Kind) + "/" + colorize(removeColor, boldFgCyan, resource)
|
|
} else if v.Namespace != "" {
|
|
res.Resource = colorize(removeColor, boldFgCyan, v.Namespace) + "/" + colorize(removeColor, boldFgCyan, v.Kind) + "/" + colorize(removeColor, boldFgCyan, resource)
|
|
resultKey = fmt.Sprintf("%s-%s-%s-%s-%s", v.Policy, ruleNameInResultKey, v.Namespace, v.Kind, resource)
|
|
}
|
|
|
|
var testRes policyreportv1alpha2.PolicyReportResult
|
|
if val, ok := resps[resultKey]; ok {
|
|
testRes = val
|
|
} else {
|
|
log.Log.V(2).Info("result not found", "key", resultKey)
|
|
res.Result = colorize(removeColor, boldYellow, "Not found")
|
|
rc.Fail++
|
|
table = append(table, *res)
|
|
ftable = append(ftable, *res)
|
|
continue
|
|
}
|
|
|
|
if v.Result == "" && v.Status != "" {
|
|
v.Result = v.Status
|
|
}
|
|
|
|
if testRes.Result == v.Result {
|
|
res.Result = colorize(removeColor, boldGreen, "Pass")
|
|
if testRes.Result == policyreportv1alpha2.StatusSkip {
|
|
rc.Skip++
|
|
} else {
|
|
rc.Pass++
|
|
}
|
|
} else {
|
|
log.Log.V(2).Info("result mismatch", "expected", v.Result, "received", testRes.Result, "key", resultKey)
|
|
res.Result = colorize(removeColor, boldRed, "Fail")
|
|
rc.Fail++
|
|
ftable = append(ftable, *res)
|
|
}
|
|
|
|
if failOnly {
|
|
if res.Result == boldRed.Sprintf("Fail") || res.Result == "Fail" {
|
|
table = append(table, *res)
|
|
}
|
|
} else {
|
|
table = append(table, *res)
|
|
}
|
|
}
|
|
} else if v.Resource != "" {
|
|
countDeprecatedResource++
|
|
res.Resource = colorize(removeColor, boldFgCyan, v.Namespace) + "/" + colorize(removeColor, boldFgCyan, v.Kind) + "/" + colorize(removeColor, boldFgCyan, v.Resource)
|
|
var ruleNameInResultKey string
|
|
if v.AutoGeneratedRule != "" {
|
|
ruleNameInResultKey = fmt.Sprintf("%s-%s", v.AutoGeneratedRule, v.Rule)
|
|
} else {
|
|
ruleNameInResultKey = v.Rule
|
|
}
|
|
|
|
resultKey := fmt.Sprintf("%s-%s-%s-%s", v.Policy, ruleNameInResultKey, v.Kind, v.Resource)
|
|
found, _ := isNamespacedPolicy(v.Policy)
|
|
var ns string
|
|
ns, v.Policy = getUserDefinedPolicyNameAndNamespace(v.Policy)
|
|
if found && v.Namespace != "" {
|
|
resultKey = fmt.Sprintf("%s-%s-%s-%s-%s-%s", ns, v.Policy, ruleNameInResultKey, v.Namespace, v.Kind, v.Resource)
|
|
} else if found {
|
|
resultKey = fmt.Sprintf("%s-%s-%s-%s-%s", ns, v.Policy, ruleNameInResultKey, v.Kind, v.Resource)
|
|
res.Policy = colorize(removeColor, boldFgCyan, ns) + "/" + colorize(removeColor, boldFgCyan, v.Policy)
|
|
res.Resource = colorize(removeColor, boldFgCyan, v.Namespace) + "/" + colorize(removeColor, boldFgCyan, v.Kind) + "/" + colorize(removeColor, boldFgCyan, v.Resource)
|
|
} else if v.Namespace != "" {
|
|
res.Resource = colorize(removeColor, boldFgCyan, v.Namespace) + "/" + colorize(removeColor, boldFgCyan, v.Kind) + "/" + colorize(removeColor, boldFgCyan, v.Resource)
|
|
resultKey = fmt.Sprintf("%s-%s-%s-%s-%s", v.Policy, ruleNameInResultKey, v.Namespace, v.Kind, v.Resource)
|
|
}
|
|
|
|
var testRes policyreportv1alpha2.PolicyReportResult
|
|
if val, ok := resps[resultKey]; ok {
|
|
testRes = val
|
|
} else {
|
|
log.Log.V(2).Info("result not found", "key", resultKey)
|
|
res.Result = colorize(removeColor, boldYellow, "Not found")
|
|
rc.Fail++
|
|
table = append(table, *res)
|
|
ftable = append(ftable, *res)
|
|
continue
|
|
}
|
|
|
|
if v.Result == "" && v.Status != "" {
|
|
v.Result = v.Status
|
|
}
|
|
|
|
if testRes.Result == v.Result {
|
|
res.Result = colorize(removeColor, boldGreen, "Pass")
|
|
if testRes.Result == policyreportv1alpha2.StatusSkip {
|
|
rc.Skip++
|
|
} else {
|
|
rc.Pass++
|
|
}
|
|
} else {
|
|
log.Log.V(2).Info("result mismatch", "expected", v.Result, "received", testRes.Result, "key", resultKey)
|
|
res.Result = colorize(removeColor, boldRed, "Fail")
|
|
rc.Fail++
|
|
ftable = append(ftable, *res)
|
|
}
|
|
|
|
if failOnly {
|
|
if res.Result == boldRed.Sprintf("Fail") || res.Result == "Fail" {
|
|
table = append(table, *res)
|
|
}
|
|
} else {
|
|
table = append(table, *res)
|
|
}
|
|
}
|
|
}
|
|
fmt.Printf("\n")
|
|
printer.Print(table)
|
|
return nil
|
|
}
|
|
|
|
func printFailedTestResult(removeColor bool) {
|
|
printer := newTablePrinter(removeColor)
|
|
for i, v := range ftable {
|
|
v.ID = i + 1
|
|
}
|
|
fmt.Printf("Aggregated Failed Test Cases : ")
|
|
fmt.Printf("\n")
|
|
printer.Print(ftable)
|
|
}
|