To support gitURLs for "apply" command (#4502)

* apply cmd support gitURL

Signed-off-by: viveksahu26 <vivekkumarsahu650@gmail.com>

* Added e2e test cases

Signed-off-by: viveksahu26 <vivekkumarsahu650@gmail.com>

* fix e2e test case

Signed-off-by: viveksahu26 <vivekkumarsahu650@gmail.com>

* added functions to make it simple

Signed-off-by: viveksahu26 <vivekkumarsahu650@gmail.com>

* fixed tests

Signed-off-by: viveksahu26 <vivekkumarsahu650@gmail.com>

* added more e2e test

Signed-off-by: viveksahu26 <vivekkumarsahu650@gmail.com>

* typo mistake

Signed-off-by: viveksahu26 <vivekkumarsahu650@gmail.com>

* removed uncovert linters

Signed-off-by: viveksahu26 <vivekkumarsahu650@gmail.com>

* renamed variables, functions

Signed-off-by: viveksahu26 <vivekkumarsahu650@gmail.com>

* test: to check urls are git source or not.

Signed-off-by: viveksahu26 <vivekkumarsahu650@gmail.com>

* remove misleading message

Signed-off-by: viveksahu26 <vivekkumarsahu650@gmail.com>

Signed-off-by: viveksahu26 <vivekkumarsahu650@gmail.com>
Co-authored-by: Charles-Edouard Brétéché <charled.breteche@gmail.com>

cmd/cli/kubectl-kyverno/apply/apply_command.go

@@ -3,13 +3,17 @@ package apply
 import (
 	"context"
 	"fmt"
+	"net/url"
 	"os"
 	"path/filepath"
+	"sort"
 	"strings"
 	"time"
 
 	"github.com/go-git/go-billy/v5/memfs"
+	kyvernov1 "github.com/kyverno/kyverno/api/kyverno/v1"
 	"github.com/kyverno/kyverno/api/kyverno/v1beta1"
+	"github.com/kyverno/kyverno/cmd/cli/kubectl-kyverno/test"
 	"github.com/kyverno/kyverno/cmd/cli/kubectl-kyverno/utils/common"
 	sanitizederror "github.com/kyverno/kyverno/cmd/cli/kubectl-kyverno/utils/sanitizedError"
 	"github.com/kyverno/kyverno/cmd/cli/kubectl-kyverno/utils/store"
@@ -59,6 +63,7 @@ type ApplyCommandConfig struct {
 	AuditWarn       bool
 	ResourcePaths   []string
 	PolicyPaths     []string
+	GitBranch       string
 }
 
 var applyHelp = `
@@ -72,6 +77,9 @@ To apply on a folder of resources:
 To apply on a cluster:
         kyverno apply /path/to/policy.yaml /path/to/folderOfPolicies --cluster
 
+To apply policies from a gitSourceURL on a cluster:
+	Example: Taking github.com as a gitSourceURL here. Some other standards  gitSourceURL are: gitlab.com , bitbucket.org , etc.
+		kyverno apply https://github.com/kyverno/policies/openshift/ --git-branch main --cluster
 
 To apply policy with variables:
 
@@ -165,6 +173,7 @@ func Command() *cobra.Command {
 	cmd.Flags().BoolVarP(&applyCommandConfig.RegistryAccess, "registry", "", false, "If set to true, access the image registry using local docker credentials to populate external data")
 	cmd.Flags().StringVarP(&applyCommandConfig.KubeConfig, "kubeconfig", "", "", "path to kubeconfig file with authorization and master location information")
 	cmd.Flags().StringVarP(&applyCommandConfig.Context, "context", "", "", "The name of the kubeconfig context to use")
+	cmd.Flags().StringVarP(&applyCommandConfig.GitBranch, "git-branch", "b", "", "test git repository branch")
 	cmd.Flags().BoolVarP(&applyCommandConfig.AuditWarn, "audit-warn", "", false, "If set to true, will flag audit policies as warnings instead of failures")
 	return cmd
 }
@@ -222,7 +231,40 @@ func (c *ApplyCommandConfig) applyCommandHelper() (rc *common.ResultCounts, reso
 		return rc, resources, skipInvalidPolicies, pvInfos, sanitizederror.NewWithError("a stdin pipe can be used for either policies or resources, not both", err)
 	}
 
-	policies, err := common.GetPoliciesFromPaths(fs, c.PolicyPaths, false, "")
+	isGit := common.IsGitSourcePath(c.PolicyPaths)
+	var policies []kyvernov1.PolicyInterface
+	gitSourceURL, err := url.Parse(c.PolicyPaths[0])
+	if err != nil {
+		fmt.Printf("Error: failed to load policies\nCause: %s\n", err)
+		os.Exit(1)
+	}
+
+	pathElems := strings.Split(gitSourceURL.Path[1:], "/")
+	if len(pathElems) <= 1 {
+		err := fmt.Errorf("invalid URL path %s - expected https://<any_git_source_domain>/:owner/:repository/:branch (without --git-branch flag) OR https://<any_git_source_domain>/:owner/:repository/:directory (with --git-branch flag)", gitSourceURL.Path)
+		fmt.Printf("Error: failed to parse URL \nCause: %s\n", err)
+		os.Exit(1)
+	}
+
+	gitSourceURL.Path = strings.Join([]string{pathElems[0], pathElems[1]}, "/")
+	repoURL := gitSourceURL.String()
+	var gitPathToYamls string
+	if isGit {
+		c.GitBranch, gitPathToYamls = common.GetGitBranchOrPolicyPaths(c.GitBranch, repoURL, c.PolicyPaths)
+		_, cloneErr := test.Clone(repoURL, fs, c.GitBranch)
+		if cloneErr != nil {
+			fmt.Printf("Error: failed to clone repository \nCause: %s\n", cloneErr)
+			log.Log.V(3).Info(fmt.Sprintf("failed to clone repository  %v as it is not valid", repoURL), "error", cloneErr)
+			os.Exit(1)
+		}
+		policyYamls, err := test.ListYAMLs(fs, gitPathToYamls)
+		if err != nil {
+			return rc, resources, skipInvalidPolicies, pvInfos, sanitizederror.NewWithError("failed to list YAMLs in repository", err)
+		}
+		c.PolicyPaths = policyYamls
+		sort.Strings(policyYamls)
+	}
+	policies, err = common.GetPoliciesFromPaths(fs, c.PolicyPaths, isGit, "")
 	if err != nil {
 		fmt.Printf("Error: failed to load policies\nCause: %s\n", err)
 		os.Exit(1)

cmd/cli/kubectl-kyverno/apply/apply_command_test.go

@@ -10,7 +10,9 @@ import (
 func Test_Apply(t *testing.T) {
 	type TestCase struct {
 		PolicyPaths           []string
+		GitBranch             string
 		ResourcePaths         []string
+		Cluster               bool
 		expectedPolicyReports []preport.PolicyReport
 		config                ApplyCommandConfig
 	}
@@ -70,6 +72,25 @@ func Test_Apply(t *testing.T) {
 				},
 			},
 		},
+		{
+			config: ApplyCommandConfig{
+				PolicyPaths:  []string{"https://github.com/kyverno/policies/openshift/team-validate-ns-name/"},
+				GitBranch:    "main",
+				PolicyReport: true,
+				Cluster:      true,
+			},
+			expectedPolicyReports: []preport.PolicyReport{
+				{
+					Summary: preport.PolicyReportSummary{
+						Pass:  9,
+						Fail:  0,
+						Skip:  0,
+						Error: 0,
+						Warn:  0,
+					},
+				},
+			},
+		},
 		{
 			config: ApplyCommandConfig{
 				PolicyPaths:   []string{"../../../../test/best_practices/disallow_latest_tag.yaml"},

cmd/cli/kubectl-kyverno/test/git.go

@@ -11,7 +11,7 @@ import (
 	"github.com/go-git/go-git/v5/storage/memory"
 )
 
-func clone(path string, fs billy.Filesystem, branch string) (*git.Repository, error) {
+func Clone(path string, fs billy.Filesystem, branch string) (*git.Repository, error) {
 	return git.Clone(memory.NewStorage(), fs, &git.CloneOptions{
 		URL:           path,
 		ReferenceName: plumbing.ReferenceName(fmt.Sprintf("refs/heads/%s", branch)),
@@ -21,7 +21,7 @@ func clone(path string, fs billy.Filesystem, branch string) (*git.Repository, er
 	})
 }
 
-func listYAMLs(fs billy.Filesystem, path string) ([]string, error) {
+func ListYAMLs(fs billy.Filesystem, path string) ([]string, error) {
 	path = filepath.Clean(path)
 
 	if _, err := fs.Stat(path); err != nil {
@@ -37,7 +37,7 @@ func listYAMLs(fs billy.Filesystem, path string) ([]string, error) {
 	for _, fi := range fis {
 		name := filepath.Join(path, fi.Name())
 		if fi.IsDir() {
-			moreYAMLs, err := listYAMLs(fs, name)
+			moreYAMLs, err := ListYAMLs(fs, name)
 			if err != nil {
 				return nil, err
 			}

cmd/cli/kubectl-kyverno/test/test_command.go

@@ -404,14 +404,14 @@ func testCommandExecute(dirPath []string, fileName string, gitBranch string, tes
 			}
 		}
 
-		_, cloneErr := clone(repoURL, fs, gitBranch)
+		_, cloneErr := Clone(repoURL, fs, gitBranch)
 		if cloneErr != nil {
 			fmt.Printf("Error: failed to clone repository \nCause: %s\n", cloneErr)
 			log.Log.V(3).Info(fmt.Sprintf("failed to clone repository  %v as it is not valid", repoURL), "error", cloneErr)
 			os.Exit(1)
 		}
 
-		policyYamls, err := listYAMLs(fs, gitPathToYamls)
+		policyYamls, err := ListYAMLs(fs, gitPathToYamls)
 		if err != nil {
 			return rc, sanitizederror.NewWithError("failed to list YAMLs in repository", err)
 		}

cmd/cli/kubectl-kyverno/utils/common/common.go

@@ -1120,3 +1120,31 @@ func GetUserInfoFromPath(fs billy.Filesystem, path string, isGit bool, policyRes
 	}
 	return *userInfo, *subjectInfo, nil
 }
+
+func IsGitSourcePath(policyPaths []string) bool {
+	return strings.Contains(policyPaths[0], "https://")
+}
+
+func GetGitBranchOrPolicyPaths(gitBranch, repoURL string, policyPaths []string) (string, string) {
+	var gitPathToYamls string
+	if gitBranch == "" {
+		gitPathToYamls = "/"
+		if string(policyPaths[0][len(policyPaths[0])-1]) == "/" {
+			gitBranch = strings.ReplaceAll(policyPaths[0], repoURL+"/", "")
+		} else {
+			gitBranch = strings.ReplaceAll(policyPaths[0], repoURL, "")
+		}
+		if gitBranch == "" {
+			gitBranch = "main"
+		} else if string(gitBranch[0]) == "/" {
+			gitBranch = gitBranch[1:]
+		}
+		return gitBranch, gitPathToYamls
+	}
+	if string(policyPaths[0][len(policyPaths[0])-1]) == "/" {
+		gitPathToYamls = strings.ReplaceAll(policyPaths[0], repoURL+"/", "/")
+	} else {
+		gitPathToYamls = strings.ReplaceAll(policyPaths[0], repoURL, "/")
+	}
+	return gitBranch, gitPathToYamls
+}

cmd/cli/kubectl-kyverno/utils/common/common_test.go

@@ -115,3 +115,75 @@ func Test_NamespaceSelector(t *testing.T) {
 		assert.Equal(t, int64(rc.Error), int64(tc.result.Error))
 	}
 }
+
+func Test_IsGitSourcePath(t *testing.T) {
+	type TestCase struct {
+		path    []string
+		actual  bool
+		desired bool
+	}
+	testcases := []TestCase{
+		{
+			path:    []string{"https://github.com/kyverno/policies/openshift/team-validate-ns-name/"},
+			desired: true,
+		},
+		{
+			path:    []string{"/kyverno/policies/openshift/team-validate-ns-name/"},
+			desired: false,
+		},
+		{
+			path:    []string{"https://bitbucket.org/kyverno/policies/openshift/team-validate-ns-name"},
+			desired: true,
+		},
+		{
+			path:    []string{"https://anydomain.com/kyverno/policies/openshift/team-validate-ns-name"},
+			desired: true,
+		},
+	}
+	for _, tc := range testcases {
+		tc.actual = IsGitSourcePath(tc.path)
+		if tc.actual != tc.desired {
+			t.Errorf("%s is not a git URL", tc.path)
+		}
+	}
+}
+
+func Test_GetGitBranchOrPolicyPaths(t *testing.T) {
+	type TestCase struct {
+		gitBranch                             string
+		repoURL                               string
+		policyPath                            []string
+		desiredBranch, actualBranch           string
+		desiredPathToYAMLs, actualPathToYAMLs string
+	}
+	testcases := []TestCase{
+		{
+			gitBranch:          "main",
+			repoURL:            "https://github.com/kyverno/policies",
+			policyPath:         []string{"https://github.com/kyverno/policies/openshift/team-validate-ns-name/"},
+			desiredBranch:      "main",
+			desiredPathToYAMLs: "/openshift/team-validate-ns-name/",
+		},
+		{
+			gitBranch:          "",
+			repoURL:            "https://github.com/kyverno/policies",
+			policyPath:         []string{"https://github.com/kyverno/policies/"},
+			desiredBranch:      "main",
+			desiredPathToYAMLs: "/",
+		},
+		{
+			gitBranch:          "",
+			repoURL:            "https://github.com/kyverno/policies",
+			policyPath:         []string{"https://github.com/kyverno/policies"},
+			desiredBranch:      "main",
+			desiredPathToYAMLs: "/",
+		},
+	}
+
+	for _, tc := range testcases {
+		tc.actualBranch, tc.actualPathToYAMLs = GetGitBranchOrPolicyPaths(tc.gitBranch, tc.repoURL, tc.policyPath)
+		if tc.actualBranch != tc.desiredBranch || tc.actualPathToYAMLs != tc.desiredPathToYAMLs {
+			t.Errorf("Want %q got %q  OR Want %q got %q", tc.desiredBranch, tc.actualBranch, tc.desiredPathToYAMLs, tc.actualPathToYAMLs)
+		}
+	}
+}