Feature: Add support for allowing insecure registries. (#3983)

Now you can work with self signed registries by updating your deployment with adding `--allowInsecureRegistry` to the `args` field.

Signed-off-by: Anton Popovichenko <anton.popovichenko@mendix.com>

Co-authored-by: Charles-Edouard Brétéché <charled.breteche@gmail.com>

cmd/cli/kubectl-kyverno/utils/store/store.go

@@ -30,7 +30,7 @@ func GetForeachElement() int {
 
 func SetRegistryAccess(access bool) {
 	if access {
-		registryclient.InitializeLocal()
+		registryclient.DefaultClient.UseLocalKeychain()
 	}
 	RegistryAccess = access
 }

cmd/kyverno/main.go

@@ -65,6 +65,7 @@ var (
 	policyControllerResyncPeriod time.Duration
 	imagePullSecrets             string
 	imageSignatureRepository     string
+	allowInsecureRegistry        bool
 	clientRateLimitQPS           float64
 	clientRateLimitBurst         int
 	webhookRegistrationTimeout   time.Duration
@@ -84,6 +85,7 @@ func main() {
 	flag.DurationVar(&policyControllerResyncPeriod, "backgroundScan", time.Hour, "Perform background scan every given interval, e.g., 30s, 15m, 1h.")
 	flag.StringVar(&imagePullSecrets, "imagePullSecrets", "", "Secret resource names for image registry access credentials.")
 	flag.StringVar(&imageSignatureRepository, "imageSignatureRepository", "", "Alternate repository for image signatures. Can be overridden per rule via `verifyImages.Repository`.")
+	flag.BoolVar(&allowInsecureRegistry, "allowInsecureRegistry", false, "Whether to allow insecure connections to registries. Don't use this for anything but testing.")
 	flag.BoolVar(&autoUpdateWebhooks, "autoUpdateWebhooks", true, "Set this flag to 'false' to disable auto-configuration of the webhook.")
 	flag.Float64Var(&clientRateLimitQPS, "clientRateLimitQPS", 0, "Configure the maximum QPS to the Kubernetes API server from Kyverno. Uses the client default if zero.")
 	flag.IntVar(&clientRateLimitBurst, "clientRateLimitBurst", 0, "Configure the maximum burst for throttle. Uses the client default if zero.")
@@ -157,14 +159,31 @@ func main() {
 	kyvernoV1beta1 := kyvernoInformer.Kyverno().V1beta1()
 	kyvernoV1alpha2 := kyvernoInformer.Kyverno().V1alpha2()
 
+	var registryOptions []registryclient.Option
+
 	// load image registry secrets
 	secrets := strings.Split(imagePullSecrets, ",")
 	if imagePullSecrets != "" && len(secrets) > 0 {
 		setupLog.Info("initializing registry credentials", "secrets", secrets)
-		if err := registryclient.Initialize(kubeClient, config.KyvernoNamespace(), "", secrets); err != nil {
-			setupLog.Error(err, "failed to initialize image pull secrets")
-			os.Exit(1)
-		}
+		registryOptions = append(
+			registryOptions,
+			registryclient.WithKeychainPullSecrets(kubeClient, config.KyvernoNamespace(), "", secrets),
+		)
+	}
+
+	if allowInsecureRegistry {
+		setupLog.Info("initializing registry with allowing insecure connections to registries")
+		registryOptions = append(
+			registryOptions,
+			registryclient.WithAllowInsecureRegistry(),
+		)
+	}
+
+	// initialize default registry client with our settings
+	registryclient.DefaultClient, err = registryclient.InitClient(registryOptions...)
+	if err != nil {
+		setupLog.Error(err, "failed to initialize registry client")
+		os.Exit(1)
 	}
 
 	if imageSignatureRepository != "" {

pkg/cosign/cosign.go

@@ -113,14 +113,7 @@ func buildCosignOptions(opts Options) (*cosign.CheckOpts, error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "constructing client options")
 	}
-
-	o, err := registryclient.GetOptions()
-	if err != nil {
-		return nil, errors.Wrap(err, "getting remote options")
-	}
-
-	remoteOpts = append(remoteOpts, remote.WithRemoteOptions(o))
-
+	remoteOpts = append(remoteOpts, registryclient.BuildRemoteOption(registryclient.DefaultClient))
 	cosignOpts := &cosign.CheckOpts{
 		Annotations:        map[string]interface{}{},
 		RegistryClientOpts: remoteOpts,

pkg/engine/imageVerify.go

@@ -43,6 +43,11 @@ func VerifyAndPatchImages(policyContext *PolicyContext) (*response.EngineRespons
 	policyContext.JSONContext.Checkpoint()
 	defer policyContext.JSONContext.Restore()
 
+	// update image registry secrets
+	if err := registryclient.DefaultClient.RefreshKeychainPullSecrets(); err != nil {
+		logger.Error(err, "failed to update image pull secrets")
+	}
+
 	ivm := &ImageVerificationMetadata{}
 	rules := autogen.ComputeRules(policyContext.Policy)
 	for i := range rules {
@@ -206,11 +211,11 @@ func (iv *imageVerifier) handleMutateDigest(digest string, imageInfo apiutils.Im
 	}
 
 	if digest == "" {
-		var err error
-		digest, err = fetchImageDigest(imageInfo.String())
+		desc, err := registryclient.DefaultClient.FetchImageDescriptor(imageInfo.String())
 		if err != nil {
 			return nil, "", err
 		}
+		digest = desc.Digest.String()
 	}
 
 	patch, err := makeAddDigestPatch(imageInfo, digest)
@@ -240,14 +245,6 @@ func hasImageVerifiedAnnotationChanged(ctx *PolicyContext, log logr.Logger) bool
 	return result
 }
 
-func fetchImageDigest(ref string) (string, error) {
-	_, desc, err := registryclient.Get(ref)
-	if err != nil {
-		return "", err
-	}
-	return desc.Digest.String(), nil
-}
-
 func imageMatches(image string, imagePatterns []string) bool {
 	for _, imagePattern := range imagePatterns {
 		if wildcard.Match(imagePattern, image) {

pkg/engine/jsonContext.go

@@ -133,6 +133,9 @@ func loadVariable(logger logr.Logger, entry kyvernov1.ContextEntry, ctx *PolicyC
 }
 
 func loadImageData(logger logr.Logger, entry kyvernov1.ContextEntry, ctx *PolicyContext) error {
+	if err := registryclient.DefaultClient.RefreshKeychainPullSecrets(); err != nil {
+		return fmt.Errorf("unable to load image registry credentials, %w", err)
+	}
 	imageData, err := fetchImageData(logger, entry, ctx)
 	if err != nil {
 		return err
@@ -175,7 +178,7 @@ func fetchImageData(logger logr.Logger, entry kyvernov1.ContextEntry, ctx *Polic
 
 // FetchImageDataMap fetches image information from the remote registry.
 func fetchImageDataMap(ref string) (interface{}, error) {
-	parsedRef, desc, err := registryclient.Get(ref)
+	desc, err := registryclient.DefaultClient.FetchImageDescriptor(ref)
 	if err != nil {
 		return nil, err
 	}
@@ -201,12 +204,13 @@ func fetchImageDataMap(ref string) (interface{}, error) {
 	if err := json.Unmarshal(rawConfig, &configData); err != nil {
 		return nil, fmt.Errorf("failed to decode config for image reference: %s, error: %v", ref, err)
 	}
+
 	data := map[string]interface{}{
 		"image":         ref,
-		"resolvedImage": fmt.Sprintf("%s@%s", parsedRef.Context().Name(), desc.Digest.String()),
-		"registry":      parsedRef.Context().RegistryStr(),
-		"repository":    parsedRef.Context().RepositoryStr(),
-		"identifier":    parsedRef.Identifier(),
+		"resolvedImage": fmt.Sprintf("%s@%s", desc.Ref.Context().Name(), desc.Digest.String()),
+		"registry":      desc.Ref.Context().RegistryStr(),
+		"repository":    desc.Ref.Context().RepositoryStr(),
+		"identifier":    desc.Ref.Identifier(),
 		"manifest":      manifest,
 		"configData":    configData,
 	}

pkg/registryclient/client.go

@@ -2,8 +2,10 @@ package registryclient
 
 import (
 	"context"
+	"crypto/tls"
 	"fmt"
 	"io/ioutil"
+	"net/http"
 
 	ecr "github.com/awslabs/amazon-ecr-credential-helper/ecr-login"
 	"github.com/chrismellard/docker-credential-acr-env/pkg/credhelper"
@@ -12,81 +14,165 @@ import (
 	kauth "github.com/google/go-containerregistry/pkg/authn/kubernetes"
 	"github.com/google/go-containerregistry/pkg/name"
 	"github.com/google/go-containerregistry/pkg/v1/google"
-	"github.com/google/go-containerregistry/pkg/v1/remote"
+	gcrremote "github.com/google/go-containerregistry/pkg/v1/remote"
 	"github.com/pkg/errors"
+	"github.com/sigstore/cosign/pkg/oci/remote"
 	"k8s.io/client-go/kubernetes"
 )
 
-var (
-	isLocal         bool
-	secrets         []string
-	kubeClient      kubernetes.Interface
-	namespace       string
-	serviceAccount  string
-	defaultKeychain = authn.NewMultiKeychain(
+// DefaultClient is default registry client.
+var DefaultClient, _ = InitClient()
+
+// Client provides registry related objects.
+type Client interface {
+	// Keychain provides keychain object.
+	Keychain() authn.Keychain
+
+	// Transport provides transport object.
+	Transport() *http.Transport
+
+	// FetchImageDescriptor fetches Descriptor from registry with given imageRef
+	// and provides access to metadata about remote artifact.
+	FetchImageDescriptor(imageRef string) (*gcrremote.Descriptor, error)
+
+	// UseLocalKeychain updates keychain with the default local keychain.
+	UseLocalKeychain()
+
+	// RefreshKeychainPullSecrets loads fresh data from pull secrets and updates Keychain.
+	// If pull secrets are empty - returns.
+	RefreshKeychainPullSecrets() error
+}
+
+// InitClient initialize registry client with given options.
+func InitClient(options ...Option) (Client, error) {
+	baseKeychain := authn.NewMultiKeychain(
 		authn.DefaultKeychain,
 		google.Keychain,
 		authn.NewKeychainFromHelper(ecr.NewECRHelper(ecr.WithLogger(ioutil.Discard))),
 		authn.NewKeychainFromHelper(credhelper.NewACRCredentialsHelper()),
 		github.Keychain,
 	)
-)
+	c := &client{
+		keychain:     baseKeychain,
+		baseKeychain: baseKeychain,
+		transport:    gcrremote.DefaultTransport,
+	}
 
-// InitializeLocal loads the docker credentials and initializes the default auth method for container registry API calls
-func InitializeLocal() {
-	isLocal = true
+	for _, opt := range options {
+		if err := opt(c); err != nil {
+			return nil, err
+		}
+	}
+
+	return c, nil
 }
 
-// Initialize loads the image pull secrets and initializes the default auth method for container registry API calls
-func Initialize(client kubernetes.Interface, ns, sa string, imagePullSecrets []string) error {
-	isLocal = false
-	kubeClient = client
-	namespace = ns
-	serviceAccount = sa
-	secrets = imagePullSecrets
-	_, err := getKeychain()
-	return err
+// Option is an option to initialize registry client.
+type Option func(*client) error
+
+// WithKeychainPullSecrets provides initialize registry client option that allows to use pull secrets.
+func WithKeychainPullSecrets(kubClient kubernetes.Interface, namespace, serviceAccount string, imagePullSecrets []string) Option {
+	return func(c *client) error {
+		refresher := func(c *client) error {
+			freshKeychain, err := generateKeychainForPullSecrets(kubClient, namespace, serviceAccount, imagePullSecrets)
+			if err != nil {
+				return err
+			}
+
+			c.keychain = authn.NewMultiKeychain(
+				c.baseKeychain,
+				freshKeychain,
+			)
+
+			return nil
+		}
+
+		c.pullSecretRefresher = refresher
+		return refresher(c)
+	}
 }
 
-func getKeychain() (authn.Keychain, error) {
-	if isLocal {
-		return authn.DefaultKeychain, nil
+// WithKeychainPullSecrets provides initialize registry client option that allows to use insecure registries.
+func WithAllowInsecureRegistry() Option {
+	return func(c *client) error {
+		c.transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
+		return nil
 	}
-	if len(secrets) == 0 {
-		return defaultKeychain, nil
+}
+
+type client struct {
+	keychain  authn.Keychain
+	transport *http.Transport
+
+	baseKeychain        authn.Keychain
+	pullSecretRefresher func(*client) error
+}
+
+// Keychain provides keychain object.
+func (c *client) Keychain() authn.Keychain {
+	return c.keychain
+}
+
+// Transport provides transport object.
+func (c *client) Transport() *http.Transport {
+	return c.transport
+}
+
+// UseLocalKeychain updates keychain with the default local keychain.
+func (c *client) UseLocalKeychain() {
+	c.keychain = authn.DefaultKeychain
+	c.baseKeychain = authn.DefaultKeychain
+}
+
+// FetchImageDescriptor fetches Descriptor from registry with given imageRef
+// and provides access to metadata about remote artifact.
+func (c *client) FetchImageDescriptor(imageRef string) (*gcrremote.Descriptor, error) {
+	parsedRef, err := name.ParseReference(imageRef)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse image reference: %s, error: %v", imageRef, err)
 	}
+
+	desc, err := gcrremote.Get(parsedRef, gcrremote.WithAuthFromKeychain(c.keychain))
+	if err != nil {
+		return nil, fmt.Errorf("failed to fetch image reference: %s, error: %v", imageRef, err)
+	}
+
+	return desc, nil
+}
+
+// RefreshKeychainPullSecrets loads fresh data from pull secrets and updates Keychain.
+// If pull secrets are empty - returns.
+func (c *client) RefreshKeychainPullSecrets() error {
+	if c.pullSecretRefresher == nil {
+		return nil
+	}
+
+	return c.pullSecretRefresher(c)
+}
+
+// generateKeychainForPullSecrets generates keychain by fetching secrets data from imagePullSecrets.
+func generateKeychainForPullSecrets(
+	client kubernetes.Interface,
+	namespace, serviceAccount string,
+	imagePullSecrets []string,
+) (authn.Keychain, error) {
 	kcOpts := kauth.Options{
 		Namespace:          namespace,
 		ServiceAccountName: serviceAccount,
-		ImagePullSecrets:   secrets,
+		ImagePullSecrets:   imagePullSecrets,
 	}
-	kc, err := kauth.New(context.Background(), kubeClient, kcOpts)
+
+	kc, err := kauth.New(context.Background(), client, kcOpts) // uses k8s client to fetch secrets data
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to initialize registry keychain")
 	}
-	return authn.NewMultiKeychain(defaultKeychain, kc), nil
+	return kc, err
 }
 
-func Get(ref string) (name.Reference, *remote.Descriptor, error) {
-	parsedRef, err := name.ParseReference(ref)
-	if err != nil {
-		return nil, nil, fmt.Errorf("failed to parse image reference: %s, error: %v", ref, err)
-	}
-	kc, err := getKeychain()
-	if err != nil {
-		return nil, nil, err
-	}
-	desc, err := remote.Get(parsedRef, remote.WithAuthFromKeychain(kc))
-	if err != nil {
-		return nil, nil, fmt.Errorf("failed to fetch image reference: %s, error: %v", ref, err)
-	}
-	return parsedRef, desc, nil
-}
-
-func GetOptions() (remote.Option, error) {
-	kc, err := getKeychain()
-	if err != nil {
-		return nil, err
-	}
-	return remote.WithAuthFromKeychain(kc), nil
+// BuildRemoteOption builds remote.Option based on client.
+func BuildRemoteOption(c Client) remote.Option {
+	return remote.WithRemoteOptions(
+		gcrremote.WithAuthFromKeychain(c.Keychain()),
+		gcrremote.WithTransport(c.Transport()),
+	)
 }

pkg/registryclient/client_test.go

@@ -0,0 +1,37 @@
+package registryclient
+
+import (
+	"crypto/tls"
+	"net/http"
+	"testing"
+
+	"github.com/google/go-containerregistry/pkg/v1/remote"
+	"gotest.tools/assert"
+)
+
+// Make sure that client conforms Client interface.
+var _ Client = &client{}
+
+func TestInitClientWithEmptyOptions(t *testing.T) {
+	expClient := &client{
+		transport: remote.DefaultTransport,
+	}
+	c, err := InitClient()
+	assert.NilError(t, err)
+	assert.Assert(t, expClient.transport == c.Transport())
+	assert.Assert(t, c.Keychain() != nil)
+}
+
+func TestInitClientWithInsecureRegistryOption(t *testing.T) {
+	expClient := &client{
+		transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}},
+	}
+	c, err := InitClient(WithAllowInsecureRegistry())
+
+	expInsecureSkipVerify := expClient.transport.TLSClientConfig.InsecureSkipVerify
+	gotInsecureSkipVerify := c.Transport().TLSClientConfig.InsecureSkipVerify
+
+	assert.NilError(t, err)
+	assert.Assert(t, expInsecureSkipVerify == gotInsecureSkipVerify)
+	assert.Assert(t, c.Keychain() != nil)
+}