feat: oci pull/push support for policie(s) (#5026)

Signed-off-by: Batuhan Apaydın <batuhan.apaydin@trendyol.com>

Signed-off-by: Batuhan Apaydın <batuhan.apaydin@trendyol.com>
Co-authored-by: Charles-Edouard Brétéché <charled.breteche@gmail.com>

.gitignore

@@ -6,7 +6,6 @@ coverage.txt
 .idea
 cmd/initContainer/kyvernopre
 cmd/kyverno/kyverno
-kubectl-kyverno
 /release
 .DS_Store
 .tools

cmd/cli/kubectl-kyverno/main.go

@@ -3,9 +3,11 @@ package main
 import (
 	"flag"
 	"os"
+	"strconv"
 
 	"github.com/kyverno/kyverno/cmd/cli/kubectl-kyverno/apply"
 	"github.com/kyverno/kyverno/cmd/cli/kubectl-kyverno/jp"
+	"github.com/kyverno/kyverno/cmd/cli/kubectl-kyverno/oci"
 	"github.com/kyverno/kyverno/cmd/cli/kubectl-kyverno/test"
 	"github.com/kyverno/kyverno/cmd/cli/kubectl-kyverno/version"
 	"github.com/spf13/cobra"
@@ -14,10 +16,13 @@ import (
 	log "sigs.k8s.io/controller-runtime/pkg/log"
 )
 
+const EnableExperimentalEnv = "KYVERNO_EXPERIMENTAL"
+
 // CLI ...
 func main() {
 	cli := &cobra.Command{
 		Use:   "kyverno",
+		Long:  `To enable experimental commands, KYVERNO_EXPERIMENTAL should be configured with true or 1.`,
 		Short: "Kubernetes Native Policy Management",
 	}
 
@@ -30,6 +35,10 @@ func main() {
 		jp.Command(),
 	}
 
+	if enableExperimental() {
+		commands = append(commands, oci.Command())
+	}
+
 	cli.AddCommand(commands...)
 
 	if err := cli.Execute(); err != nil {
@@ -37,6 +46,13 @@ func main() {
 	}
 }
 
+func enableExperimental() bool {
+	if b, err := strconv.ParseBool(os.Getenv(EnableExperimentalEnv)); err == nil {
+		return b
+	}
+	return false
+}
+
 func configurelog(cli *cobra.Command) {
 	// clear flags initialized in static dependencies
 	if flag.CommandLine.Lookup("log_dir") != nil {

cmd/cli/kubectl-kyverno/oci/oci.go

@@ -0,0 +1,50 @@
+package oci
+
+import (
+	"io"
+
+	"github.com/awslabs/amazon-ecr-credential-helper/ecr-login"
+	"github.com/chrismellard/docker-credential-acr-env/pkg/credhelper"
+	"github.com/google/go-containerregistry/pkg/authn"
+	"github.com/google/go-containerregistry/pkg/authn/github"
+	"github.com/google/go-containerregistry/pkg/v1/google"
+	"github.com/google/go-containerregistry/pkg/v1/remote"
+	"github.com/spf13/cobra"
+)
+
+const (
+	policyConfigMediaType = "application/vnd.cncf.kyverno.config.v1+json"
+	policyLayerMediaType  = "application/vnd.cncf.kyverno.policy.layer.v1+yaml"
+)
+
+var (
+	amazonKeychain = authn.NewKeychainFromHelper(ecr.NewECRHelper(ecr.WithLogger(io.Discard)))
+	azureKeychain  = authn.NewKeychainFromHelper(credhelper.NewACRCredentialsHelper())
+	keychain       = authn.NewMultiKeychain(
+		authn.DefaultKeychain,
+		google.Keychain,
+		github.Keychain,
+		amazonKeychain,
+		azureKeychain,
+	)
+
+	Get      = remote.Get
+	Write    = remote.Write
+	imageRef string
+)
+
+func Command() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:     "oci",
+		Long:    `This command is one of the supported experimental commands, and its behaviour might be changed any time`,
+		Short:   "pulls/pushes images that include policie(s) from/to OCI registries",
+		Example: "",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return cmd.Help()
+		},
+	}
+	cmd.PersistentFlags().StringVarP(&imageRef, "image", "i", "", "image reference to push to")
+	cmd.AddCommand(ociPullCommand())
+	cmd.AddCommand(ociPushCommand())
+	return cmd
+}

cmd/cli/kubectl-kyverno/oci/oci_pull.go

@@ -0,0 +1,114 @@
+package oci
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+
+	securejoin "github.com/cyphar/filepath-securejoin"
+	"github.com/google/go-containerregistry/pkg/name"
+	"github.com/google/go-containerregistry/pkg/v1/remote"
+	"github.com/spf13/cobra"
+	"sigs.k8s.io/yaml"
+)
+
+var dir string
+
+func ociPullCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "pull",
+		Long:  "This command is one of the supported experimental commands, and its behaviour might be changed any time",
+		Short: "pulls policie(s) that are included in an OCI image from OCI registry and saves them to a local directory",
+		Example: `# pull policy from an OCI image and save it to the specific directory
+kyverno oci pull -i <imgref> -d policies`,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			if imageRef == "" {
+				return errors.New("image reference is required")
+			}
+
+			dir = filepath.Clean(dir)
+			if !filepath.IsAbs(dir) {
+				cwd, err := os.Getwd()
+				if err != nil {
+					return err
+				}
+
+				dir, err = securejoin.SecureJoin(cwd, dir)
+				if err != nil {
+					return err
+				}
+			}
+
+			fi, err := os.Lstat(dir)
+			// Dir does not need to exist, as it can later be created.
+			if err != nil && errors.Is(err, os.ErrNotExist) {
+				if err := os.MkdirAll(dir, 0o750); err != nil {
+					return fmt.Errorf("unable to create directory %s: %w", dir, err)
+				}
+			}
+
+			if err == nil && !fi.IsDir() {
+				return fmt.Errorf("dir '%s' must be a directory", dir)
+			}
+
+			ref, err := name.ParseReference(imageRef)
+			if err != nil {
+				return fmt.Errorf("parsing image reference: %v", err)
+			}
+
+			do := []remote.Option{
+				remote.WithContext(cmd.Context()),
+				remote.WithAuthFromKeychain(keychain),
+			}
+
+			rmt, err := Get(ref, do...)
+			if err != nil {
+				return fmt.Errorf("getting image: %v", err)
+			}
+
+			img, err := rmt.Image()
+			if err != nil {
+				return fmt.Errorf("getting image: %v", err)
+			}
+
+			l, err := img.Layers()
+			if err != nil {
+				return fmt.Errorf("getting image layers: %v", err)
+			}
+
+			for _, layer := range l {
+				lmt, err := layer.MediaType()
+				if err != nil {
+					return fmt.Errorf("getting layer media type: %v", err)
+				}
+
+				if lmt == policyLayerMediaType {
+					blob, err := layer.Compressed()
+					if err != nil {
+						return fmt.Errorf("getting layer blob: %v", err)
+					}
+					defer blob.Close()
+
+					var policy map[string]interface{}
+					b, err := io.ReadAll(blob)
+					if err != nil {
+						return fmt.Errorf("reading layer blob: %v", err)
+					}
+					if err := yaml.Unmarshal(b, &policy); err != nil {
+						return fmt.Errorf("unmarshaling layer blob: %v", err)
+					}
+
+					fn := policy["metadata"].(map[string]interface{})["name"].(string) + ".yaml"
+					if err := os.WriteFile(filepath.Join(dir, fn), b, 0o600); err != nil {
+						return fmt.Errorf("creating file: %v", err)
+					}
+				}
+			}
+			return nil
+		},
+	}
+	cmd.Flags().StringVarP(&dir, "directory", "d", ".", "path to a directory")
+	return cmd
+}

cmd/cli/kubectl-kyverno/oci/oci_push.go

@@ -0,0 +1,114 @@
+package oci
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/google/go-containerregistry/pkg/name"
+	"github.com/google/go-containerregistry/pkg/v1/empty"
+	"github.com/google/go-containerregistry/pkg/v1/mutate"
+	"github.com/google/go-containerregistry/pkg/v1/remote"
+	"github.com/google/go-containerregistry/pkg/v1/static"
+	"github.com/google/go-containerregistry/pkg/v1/types"
+	"github.com/spf13/cobra"
+	"gopkg.in/yaml.v3"
+)
+
+var policyRef string
+
+func ociPushCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "push",
+		Long:  "This command is one of the supported experimental commands in Kyverno CLI, and its behaviour might be changed any time.",
+		Short: "push policie(s) that are included in an OCI image to OCI registry",
+		Example: `# push policy to an OCI image from a given policy file
+kyverno oci push -p policy.yaml -i <imgref>
+
+# push multiple policies to an OCI image from a given directory that includes policies
+kyverno oci push -p policies. -i <imgref>`,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			if imageRef == "" {
+				return errors.New("image reference is required")
+			}
+
+			var p []string
+			f, err := os.Stat(policyRef)
+			if os.IsNotExist(err) {
+				return fmt.Errorf("policy file or directory  %s does not exist", policyRef)
+			}
+
+			if f.IsDir() {
+				err = filepath.Walk(policyRef, func(path string, info os.FileInfo, err error) error {
+					if !info.IsDir() {
+						p = append(p, path)
+					}
+
+					if m := info.Mode(); !(m.IsRegular() || m.IsDir()) {
+						return nil
+					}
+
+					return nil
+				})
+			} else {
+				p = append(p, policyRef)
+			}
+
+			if err != nil {
+				return fmt.Errorf("unable to read policy file or directory %s: %w", policyRef, err)
+			}
+
+			fmt.Println("Policies will be pushing: ", p)
+
+			img := mutate.MediaType(empty.Image, types.OCIManifestSchema1)
+			img = mutate.ConfigMediaType(img, policyConfigMediaType)
+			for _, policy := range p {
+				policyBytes, err := os.ReadFile(filepath.Clean(policy))
+				if err != nil {
+					return fmt.Errorf("failed to read policy file %s: %v", policy, err)
+				}
+
+				var policyMap map[string]interface{}
+				if err = yaml.Unmarshal(policyBytes, &policyMap); err != nil {
+					return fmt.Errorf("failed to unmarshal policy file %s: %v", policy, err)
+				}
+
+				annotations := map[string]string{}
+				for k, v := range policyMap["metadata"].(map[string]interface{})["annotations"].(map[string]interface{}) {
+					annotations[k] = v.(string)
+				}
+
+				ref, err := name.ParseReference(imageRef)
+				if err != nil {
+					return fmt.Errorf("parsing image reference: %v", err)
+				}
+
+				do := []remote.Option{
+					remote.WithContext(cmd.Context()),
+					remote.WithAuthFromKeychain(keychain),
+				}
+
+				policyLayer := static.NewLayer(policyBytes, policyLayerMediaType)
+				img, err = mutate.Append(img, mutate.Addendum{
+					Layer:       policyLayer,
+					Annotations: annotations,
+				})
+
+				if err != nil {
+					return fmt.Errorf("mutating image: %v", err)
+				}
+
+				fmt.Fprintf(os.Stderr, "Uploading Kyverno policy file [%s] to [%s] with mediaType [%s].\n", policy, ref.Name(), policyLayerMediaType)
+				if err = Write(ref, img, do...); err != nil {
+					return fmt.Errorf("writing image: %v", err)
+				}
+
+				fmt.Fprintf(os.Stderr, "Kyverno policy file [%s] successfully uploaded to [%s]\n", policy, ref.Name())
+			}
+			return nil
+		},
+	}
+	cmd.Flags().StringVarP(&policyRef, "policy", "p", "", "path to policie(s)")
+	return cmd
+}

go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/blang/semver/v4 v4.0.0
 	github.com/cenkalti/backoff v2.2.1+incompatible
 	github.com/chrismellard/docker-credential-acr-env v0.0.0-20221002210726-e883f69e0206
+	github.com/cyphar/filepath-securejoin v0.2.3
 	github.com/distribution/distribution v2.8.1+incompatible
 	github.com/evanphx/json-patch v5.6.0+incompatible
 	github.com/evanphx/json-patch/v5 v5.6.0

go.sum

@@ -541,6 +541,8 @@ github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ
 github.com/cyberphone/json-canonicalization v0.0.0-20210303052042-6bc126869bf4/go.mod h1:uzvlm1mxhHkdfqitSA92i7Se+S9ksOn3a3qmv/kyOCw=
 github.com/cyberphone/json-canonicalization v0.0.0-20210823021906-dc406ceaf94b h1:lMzA7yYThpwx7iYNpTeiQnRH6h5JSfSYMJdz+pxZOW8=
 github.com/cyberphone/json-canonicalization v0.0.0-20210823021906-dc406ceaf94b/go.mod h1:uzvlm1mxhHkdfqitSA92i7Se+S9ksOn3a3qmv/kyOCw=
+github.com/cyphar/filepath-securejoin v0.2.3 h1:YX6ebbZCZP7VkM3scTTokDgBL2TY741X51MTk3ycuNI=
+github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=
 github.com/daixiang0/gci v0.2.8/go.mod h1:+4dZ7TISfSmqfAGv59ePaHfNzgGtIkHAhhdKggP1JAc=
 github.com/danieljoos/wincred v1.0.2/go.mod h1:SnuYRW9lp1oJrZX/dXJqr0cPK5gYXqx3EJbmjhLdK9U=
 github.com/danieljoos/wincred v1.1.0/go.mod h1:XYlo+eRTsVA9aHGp7NGjFkPla4m+DCL7hqDjlFjiygg=