From cbbd8488c873d67437726cb564ea05dba93c2004 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Batuhan=20Apayd=C4=B1n?= <batuhan.apaydin@trendyol.com>
Date: Mon, 24 Oct 2022 21:47:20 +0300
Subject: [PATCH] feat: oci pull/push support for policie(s) (#5026)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Batuhan Apaydın <batuhan.apaydin@trendyol.com>
Signed-off-by: Batuhan Apaydın <batuhan.apaydin@trendyol.com>
Co-authored-by: Charles-Edouard Brétéché <charled.breteche@gmail.com>
---
.gitignore | 1 -
cmd/cli/kubectl-kyverno/main.go | 16 ++++
cmd/cli/kubectl-kyverno/oci/oci.go | 50 +++++++++++
cmd/cli/kubectl-kyverno/oci/oci_pull.go | 114 ++++++++++++++++++++++++
cmd/cli/kubectl-kyverno/oci/oci_push.go | 114 ++++++++++++++++++++++++
go.mod | 1 +
go.sum | 2 +
7 files changed, 297 insertions(+), 1 deletion(-)
create mode 100644 cmd/cli/kubectl-kyverno/oci/oci.go
create mode 100644 cmd/cli/kubectl-kyverno/oci/oci_pull.go
create mode 100644 cmd/cli/kubectl-kyverno/oci/oci_push.go
diff --git a/.gitignore b/.gitignore
index 6f7d7c261b..7a1bf3bb55 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,7 +6,6 @@ coverage.txt
.idea
cmd/initContainer/kyvernopre
cmd/kyverno/kyverno
-kubectl-kyverno
/release
.DS_Store
.tools
diff --git a/cmd/cli/kubectl-kyverno/main.go b/cmd/cli/kubectl-kyverno/main.go
index 407123a47f..079108c947 100644
--- a/cmd/cli/kubectl-kyverno/main.go
+++ b/cmd/cli/kubectl-kyverno/main.go
@@ -3,9 +3,11 @@ package main
import (
"flag"
"os"
+ "strconv"
"github.com/kyverno/kyverno/cmd/cli/kubectl-kyverno/apply"
"github.com/kyverno/kyverno/cmd/cli/kubectl-kyverno/jp"
+ "github.com/kyverno/kyverno/cmd/cli/kubectl-kyverno/oci"
"github.com/kyverno/kyverno/cmd/cli/kubectl-kyverno/test"
"github.com/kyverno/kyverno/cmd/cli/kubectl-kyverno/version"
"github.com/spf13/cobra"
@@ -14,10 +16,13 @@ import (
log "sigs.k8s.io/controller-runtime/pkg/log"
)
+const EnableExperimentalEnv = "KYVERNO_EXPERIMENTAL"
+
// CLI ...
func main() {
cli := &cobra.Command{
Use: "kyverno",
+ Long: `To enable experimental commands, KYVERNO_EXPERIMENTAL should be configured with true or 1.`,
Short: "Kubernetes Native Policy Management",
}
@@ -30,6 +35,10 @@ func main() {
jp.Command(),
}
+ if enableExperimental() {
+ commands = append(commands, oci.Command())
+ }
+
cli.AddCommand(commands...)
if err := cli.Execute(); err != nil {
@@ -37,6 +46,13 @@ func main() {
}
}
+func enableExperimental() bool {
+ if b, err := strconv.ParseBool(os.Getenv(EnableExperimentalEnv)); err == nil {
+ return b
+ }
+ return false
+}
+
func configurelog(cli *cobra.Command) {
// clear flags initialized in static dependencies
if flag.CommandLine.Lookup("log_dir") != nil {
diff --git a/cmd/cli/kubectl-kyverno/oci/oci.go b/cmd/cli/kubectl-kyverno/oci/oci.go
new file mode 100644
index 0000000000..130e99357d
--- /dev/null
+++ b/cmd/cli/kubectl-kyverno/oci/oci.go
@@ -0,0 +1,50 @@
+package oci
+
+import (
+ "io"
+
+ "github.com/awslabs/amazon-ecr-credential-helper/ecr-login"
+ "github.com/chrismellard/docker-credential-acr-env/pkg/credhelper"
+ "github.com/google/go-containerregistry/pkg/authn"
+ "github.com/google/go-containerregistry/pkg/authn/github"
+ "github.com/google/go-containerregistry/pkg/v1/google"
+ "github.com/google/go-containerregistry/pkg/v1/remote"
+ "github.com/spf13/cobra"
+)
+
+const (
+ policyConfigMediaType = "application/vnd.cncf.kyverno.config.v1+json"
+ policyLayerMediaType = "application/vnd.cncf.kyverno.policy.layer.v1+yaml"
+)
+
+var (
+ amazonKeychain = authn.NewKeychainFromHelper(ecr.NewECRHelper(ecr.WithLogger(io.Discard)))
+ azureKeychain = authn.NewKeychainFromHelper(credhelper.NewACRCredentialsHelper())
+ keychain = authn.NewMultiKeychain(
+ authn.DefaultKeychain,
+ google.Keychain,
+ github.Keychain,
+ amazonKeychain,
+ azureKeychain,
+ )
+
+ Get = remote.Get
+ Write = remote.Write
+ imageRef string
+)
+
+func Command() *cobra.Command {
+ cmd := &cobra.Command{
+ Use: "oci",
+ Long: `This command is one of the supported experimental commands, and its behaviour might be changed any time`,
+ Short: "pulls/pushes images that include policie(s) from/to OCI registries",
+ Example: "",
+ RunE: func(cmd *cobra.Command, args []string) error {
+ return cmd.Help()
+ },
+ }
+ cmd.PersistentFlags().StringVarP(&imageRef, "image", "i", "", "image reference to push to")
+ cmd.AddCommand(ociPullCommand())
+ cmd.AddCommand(ociPushCommand())
+ return cmd
+}
diff --git a/cmd/cli/kubectl-kyverno/oci/oci_pull.go b/cmd/cli/kubectl-kyverno/oci/oci_pull.go
new file mode 100644
index 0000000000..cf07ac2c66
--- /dev/null
+++ b/cmd/cli/kubectl-kyverno/oci/oci_pull.go
@@ -0,0 +1,114 @@
+package oci
+
+import (
+ "errors"
+ "fmt"
+ "io"
+ "os"
+ "path/filepath"
+
+ securejoin "github.com/cyphar/filepath-securejoin"
+ "github.com/google/go-containerregistry/pkg/name"
+ "github.com/google/go-containerregistry/pkg/v1/remote"
+ "github.com/spf13/cobra"
+ "sigs.k8s.io/yaml"
+)
+
+var dir string
+
+func ociPullCommand() *cobra.Command {
+ cmd := &cobra.Command{
+ Use: "pull",
+ Long: "This command is one of the supported experimental commands, and its behaviour might be changed any time",
+ Short: "pulls policie(s) that are included in an OCI image from OCI registry and saves them to a local directory",
+ Example: `# pull policy from an OCI image and save it to the specific directory
+kyverno oci pull -i <imgref> -d policies`,
+ RunE: func(cmd *cobra.Command, args []string) error {
+ if imageRef == "" {
+ return errors.New("image reference is required")
+ }
+
+ dir = filepath.Clean(dir)
+ if !filepath.IsAbs(dir) {
+ cwd, err := os.Getwd()
+ if err != nil {
+ return err
+ }
+
+ dir, err = securejoin.SecureJoin(cwd, dir)
+ if err != nil {
+ return err
+ }
+ }
+
+ fi, err := os.Lstat(dir)
+ // Dir does not need to exist, as it can later be created.
+ if err != nil && errors.Is(err, os.ErrNotExist) {
+ if err := os.MkdirAll(dir, 0o750); err != nil {
+ return fmt.Errorf("unable to create directory %s: %w", dir, err)
+ }
+ }
+
+ if err == nil && !fi.IsDir() {
+ return fmt.Errorf("dir '%s' must be a directory", dir)
+ }
+
+ ref, err := name.ParseReference(imageRef)
+ if err != nil {
+ return fmt.Errorf("parsing image reference: %v", err)
+ }
+
+ do := []remote.Option{
+ remote.WithContext(cmd.Context()),
+ remote.WithAuthFromKeychain(keychain),
+ }
+
+ rmt, err := Get(ref, do...)
+ if err != nil {
+ return fmt.Errorf("getting image: %v", err)
+ }
+
+ img, err := rmt.Image()
+ if err != nil {
+ return fmt.Errorf("getting image: %v", err)
+ }
+
+ l, err := img.Layers()
+ if err != nil {
+ return fmt.Errorf("getting image layers: %v", err)
+ }
+
+ for _, layer := range l {
+ lmt, err := layer.MediaType()
+ if err != nil {
+ return fmt.Errorf("getting layer media type: %v", err)
+ }
+
+ if lmt == policyLayerMediaType {
+ blob, err := layer.Compressed()
+ if err != nil {
+ return fmt.Errorf("getting layer blob: %v", err)
+ }
+ defer blob.Close()
+
+ var policy map[string]interface{}
+ b, err := io.ReadAll(blob)
+ if err != nil {
+ return fmt.Errorf("reading layer blob: %v", err)
+ }
+ if err := yaml.Unmarshal(b, &policy); err != nil {
+ return fmt.Errorf("unmarshaling layer blob: %v", err)
+ }
+
+ fn := policy["metadata"].(map[string]interface{})["name"].(string) + ".yaml"
+ if err := os.WriteFile(filepath.Join(dir, fn), b, 0o600); err != nil {
+ return fmt.Errorf("creating file: %v", err)
+ }
+ }
+ }
+ return nil
+ },
+ }
+ cmd.Flags().StringVarP(&dir, "directory", "d", ".", "path to a directory")
+ return cmd
+}
diff --git a/cmd/cli/kubectl-kyverno/oci/oci_push.go b/cmd/cli/kubectl-kyverno/oci/oci_push.go
new file mode 100644
index 0000000000..d0286b2dcb
--- /dev/null
+++ b/cmd/cli/kubectl-kyverno/oci/oci_push.go
@@ -0,0 +1,114 @@
+package oci
+
+import (
+ "errors"
+ "fmt"
+ "os"
+ "path/filepath"
+
+ "github.com/google/go-containerregistry/pkg/name"
+ "github.com/google/go-containerregistry/pkg/v1/empty"
+ "github.com/google/go-containerregistry/pkg/v1/mutate"
+ "github.com/google/go-containerregistry/pkg/v1/remote"
+ "github.com/google/go-containerregistry/pkg/v1/static"
+ "github.com/google/go-containerregistry/pkg/v1/types"
+ "github.com/spf13/cobra"
+ "gopkg.in/yaml.v3"
+)
+
+var policyRef string
+
+func ociPushCommand() *cobra.Command {
+ cmd := &cobra.Command{
+ Use: "push",
+ Long: "This command is one of the supported experimental commands in Kyverno CLI, and its behaviour might be changed any time.",
+ Short: "push policie(s) that are included in an OCI image to OCI registry",
+ Example: `# push policy to an OCI image from a given policy file
+kyverno oci push -p policy.yaml -i <imgref>
+
+# push multiple policies to an OCI image from a given directory that includes policies
+kyverno oci push -p policies. -i <imgref>`,
+ RunE: func(cmd *cobra.Command, args []string) error {
+ if imageRef == "" {
+ return errors.New("image reference is required")
+ }
+
+ var p []string
+ f, err := os.Stat(policyRef)
+ if os.IsNotExist(err) {
+ return fmt.Errorf("policy file or directory %s does not exist", policyRef)
+ }
+
+ if f.IsDir() {
+ err = filepath.Walk(policyRef, func(path string, info os.FileInfo, err error) error {
+ if !info.IsDir() {
+ p = append(p, path)
+ }
+
+ if m := info.Mode(); !(m.IsRegular() || m.IsDir()) {
+ return nil
+ }
+
+ return nil
+ })
+ } else {
+ p = append(p, policyRef)
+ }
+
+ if err != nil {
+ return fmt.Errorf("unable to read policy file or directory %s: %w", policyRef, err)
+ }
+
+ fmt.Println("Policies will be pushing: ", p)
+
+ img := mutate.MediaType(empty.Image, types.OCIManifestSchema1)
+ img = mutate.ConfigMediaType(img, policyConfigMediaType)
+ for _, policy := range p {
+ policyBytes, err := os.ReadFile(filepath.Clean(policy))
+ if err != nil {
+ return fmt.Errorf("failed to read policy file %s: %v", policy, err)
+ }
+
+ var policyMap map[string]interface{}
+ if err = yaml.Unmarshal(policyBytes, &policyMap); err != nil {
+ return fmt.Errorf("failed to unmarshal policy file %s: %v", policy, err)
+ }
+
+ annotations := map[string]string{}
+ for k, v := range policyMap["metadata"].(map[string]interface{})["annotations"].(map[string]interface{}) {
+ annotations[k] = v.(string)
+ }
+
+ ref, err := name.ParseReference(imageRef)
+ if err != nil {
+ return fmt.Errorf("parsing image reference: %v", err)
+ }
+
+ do := []remote.Option{
+ remote.WithContext(cmd.Context()),
+ remote.WithAuthFromKeychain(keychain),
+ }
+
+ policyLayer := static.NewLayer(policyBytes, policyLayerMediaType)
+ img, err = mutate.Append(img, mutate.Addendum{
+ Layer: policyLayer,
+ Annotations: annotations,
+ })
+
+ if err != nil {
+ return fmt.Errorf("mutating image: %v", err)
+ }
+
+ fmt.Fprintf(os.Stderr, "Uploading Kyverno policy file [%s] to [%s] with mediaType [%s].\n", policy, ref.Name(), policyLayerMediaType)
+ if err = Write(ref, img, do...); err != nil {
+ return fmt.Errorf("writing image: %v", err)
+ }
+
+ fmt.Fprintf(os.Stderr, "Kyverno policy file [%s] successfully uploaded to [%s]\n", policy, ref.Name())
+ }
+ return nil
+ },
+ }
+ cmd.Flags().StringVarP(&policyRef, "policy", "p", "", "path to policie(s)")
+ return cmd
+}
diff --git a/go.mod b/go.mod
index 3f2f8304ac..409012aab4 100644
--- a/go.mod
+++ b/go.mod
@@ -9,6 +9,7 @@ require (
github.com/blang/semver/v4 v4.0.0
github.com/cenkalti/backoff v2.2.1+incompatible
github.com/chrismellard/docker-credential-acr-env v0.0.0-20221002210726-e883f69e0206
+ github.com/cyphar/filepath-securejoin v0.2.3
github.com/distribution/distribution v2.8.1+incompatible
github.com/evanphx/json-patch v5.6.0+incompatible
github.com/evanphx/json-patch/v5 v5.6.0
diff --git a/go.sum b/go.sum
index 8e6c261bc7..1e0e54c6e7 100644
--- a/go.sum
+++ b/go.sum
@@ -541,6 +541,8 @@ github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ
github.com/cyberphone/json-canonicalization v0.0.0-20210303052042-6bc126869bf4/go.mod h1:uzvlm1mxhHkdfqitSA92i7Se+S9ksOn3a3qmv/kyOCw=
github.com/cyberphone/json-canonicalization v0.0.0-20210823021906-dc406ceaf94b h1:lMzA7yYThpwx7iYNpTeiQnRH6h5JSfSYMJdz+pxZOW8=
github.com/cyberphone/json-canonicalization v0.0.0-20210823021906-dc406ceaf94b/go.mod h1:uzvlm1mxhHkdfqitSA92i7Se+S9ksOn3a3qmv/kyOCw=
+github.com/cyphar/filepath-securejoin v0.2.3 h1:YX6ebbZCZP7VkM3scTTokDgBL2TY741X51MTk3ycuNI=
+github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=
github.com/daixiang0/gci v0.2.8/go.mod h1:+4dZ7TISfSmqfAGv59ePaHfNzgGtIkHAhhdKggP1JAc=
github.com/danieljoos/wincred v1.0.2/go.mod h1:SnuYRW9lp1oJrZX/dXJqr0cPK5gYXqx3EJbmjhLdK9U=
github.com/danieljoos/wincred v1.1.0/go.mod h1:XYlo+eRTsVA9aHGp7NGjFkPla4m+DCL7hqDjlFjiygg=