Added kubeconfig flag support (#4308)

* Added kubeconfig flag support

Signed-off-by: Ayushman Mishra <ayushvidushi01@gmail.com>

* removed swp file

Signed-off-by: Ayushman Mishra <ayushvidushi01@gmail.com>

* changed

Signed-off-by: Ayushman Mishra <ayushvidushi01@gmail.com>

Signed-off-by: Ayushman Mishra <ayushvidushi01@gmail.com>
Co-authored-by: shuting <shuting@nirmata.com>

cmd/initContainer/main.go

@@ -27,13 +27,13 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/rest"
 	"k8s.io/klog/v2"
 	"k8s.io/klog/v2/klogr"
 	"sigs.k8s.io/controller-runtime/pkg/log"
 )
 
 var (
+	kubeconfig           string
 	setupLog             = log.Log.WithName("setup")
 	clientRateLimitQPS   float64
 	clientRateLimitBurst int
@@ -65,6 +65,7 @@ func main() {
 
 	klog.InitFlags(nil) // add the block above before invoking klog.InitFlags()
 	log.SetLogger(klogr.New())
+	flag.StringVar(&kubeconfig, "kubeconfig", "", "Path to a kubeconfig. Only required if out-of-cluster.")
 	flag.Float64Var(&clientRateLimitQPS, "clientRateLimitQPS", 0, "Configure the maximum QPS to the Kubernetes API server from Kyverno. Uses the client default if zero.")
 	flag.IntVar(&clientRateLimitBurst, "clientRateLimitBurst", 0, "Configure the maximum burst for throttle. Uses the client default if zero.")
 	if err := flag.Set("v", "2"); err != nil {
@@ -76,13 +77,9 @@ func main() {
 	// os signal handler
 	stopCh := signal.SetupSignalHandler()
 	// create client config
-	clientConfig, err := rest.InClusterConfig()
+	clientConfig, err := config.CreateClientConfig(kubeconfig, clientRateLimitQPS, clientRateLimitBurst)
 	if err != nil {
-		setupLog.Error(err, "Failed to create clientConfig")
+		setupLog.Error(err, "Failed to build kubeconfig")
-		os.Exit(1)
-	}
-	if err := config.ConfigureClientConfig(clientConfig, clientRateLimitQPS, clientRateLimitBurst); err != nil {
-		setupLog.Error(err, "Failed to create clientConfig")
 		os.Exit(1)
 	}
 

cmd/kyverno/main.go

@@ -44,7 +44,6 @@ import (
 	_ "go.uber.org/automaxprocs" // #nosec
 	kubeinformers "k8s.io/client-go/informers"
 	"k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/cache"
 	"k8s.io/klog/v2"
 	"k8s.io/klog/v2/klogr"
@@ -56,6 +55,7 @@ const resyncPeriod = 15 * time.Minute
 var (
 	// TODO: this has been added to backward support command line arguments
 	// will be removed in future and the configuration will be set only via configmaps
+	kubeconfig                   string
 	serverIP                     string
 	profilePort                  string
 	metricsPort                  string
@@ -91,6 +91,7 @@ func main() {
 	flag.IntVar(&webhookTimeout, "webhookTimeout", int(webhookconfig.DefaultWebhookTimeout), "Timeout for webhook configurations.")
 	flag.IntVar(&genWorkers, "genWorkers", 10, "Workers for generate controller.")
 	flag.IntVar(&maxQueuedEvents, "maxQueuedEvents", 1000, "Maximum events to be queued.")
+	flag.StringVar(&kubeconfig, "kubeconfig", "", "Path to a kubeconfig. Only required if out-of-cluster.")
 	flag.StringVar(&serverIP, "serverIP", "", "IP address where Kyverno controller runs. Only required if out-of-cluster.")
 	flag.BoolVar(&profile, "profile", false, "Set this flag to 'true', to enable profiling.")
 	flag.StringVar(&profilePort, "profilePort", "6060", "Enable profiling at given port, defaults to 6060.")
@@ -124,13 +125,9 @@ func main() {
 	debug := serverIP != ""
 
 	// clients
-	clientConfig, err := rest.InClusterConfig()
+	clientConfig, err := config.CreateClientConfig(kubeconfig, clientRateLimitQPS, clientRateLimitBurst)
 	if err != nil {
-		setupLog.Error(err, "Failed to create clientConfig")
+		setupLog.Error(err, "Failed to build kubeconfig")
-		os.Exit(1)
-	}
-	if err := config.ConfigureClientConfig(clientConfig, clientRateLimitQPS, clientRateLimitBurst); err != nil {
-		setupLog.Error(err, "Failed to create clientConfig")
 		os.Exit(1)
 	}
 	kyvernoClient, err := kyvernoclient.NewForConfig(clientConfig)

pkg/config/client.go

@@ -5,14 +5,30 @@ import (
 	"math"
 
 	rest "k8s.io/client-go/rest"
+	clientcmd "k8s.io/client-go/tools/clientcmd"
 )
 
-// ConfigureClientConfig creates client config and applies rate limit QPS and burst
+// CreateClientConfig creates client config and applies rate limit QPS and burst
-func ConfigureClientConfig(clientConfig *rest.Config, qps float64, burst int) error {
+func CreateClientConfig(kubeconfig string, qps float64, burst int) (*rest.Config, error) {
+	clientConfig, err := createClientConfig(kubeconfig)
+	if err != nil {
+		return nil, err
+	}
+
 	if qps > math.MaxFloat32 {
-		return fmt.Errorf("client rate limit QPS must not be higher than %e", math.MaxFloat32)
+		return nil, fmt.Errorf("client rate limit QPS must not be higher than %e", math.MaxFloat32)
 	}
 	clientConfig.Burst = burst
 	clientConfig.QPS = float32(qps)
-	return nil
+	return clientConfig, nil
+}
+
+// createClientConfig creates client config
+func createClientConfig(kubeconfig string) (*rest.Config, error) {
+	if kubeconfig == "" {
+		logger.Info("Using in-cluster configuration")
+		return rest.InClusterConfig()
+	}
+	logger.V(4).Info("Using specified kubeconfig", "kubeconfig", kubeconfig)
+	return clientcmd.BuildConfigFromFlags("", kubeconfig)
 }

pkg/config/config_test.go

@@ -2,17 +2,20 @@ package config_test
 
 import (
 	"math"
+	"os"
 	"testing"
 
-	"gotest.tools/assert"
-	"k8s.io/client-go/rest"
-
 	"github.com/kyverno/kyverno/pkg/config"
+	"gotest.tools/assert"
+	"k8s.io/apimachinery/pkg/runtime"
+	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
+	clientcmdlatest "k8s.io/client-go/tools/clientcmd/api/latest"
 )
 
 func Test_CreateClientConfig_WithKubeConfig(t *testing.T) {
-	c := &rest.Config{}
+	cf := createMinimalKubeconfig(t)
-	err := config.ConfigureClientConfig(c, 0, 0)
+	defer os.Remove(cf)
+	_, err := config.CreateClientConfig(cf, 0, 0)
 	assert.NilError(t, err)
 }
 
@@ -21,8 +24,9 @@ func Test_CreateClientConfig_SetBurstQPS(t *testing.T) {
 		qps   = 55
 		burst = 99
 	)
-	c := &rest.Config{}
+	cf := createMinimalKubeconfig(t)
-	err := config.ConfigureClientConfig(c, qps, burst)
+	defer os.Remove(cf)
+	c, err := config.CreateClientConfig(cf, qps, burst)
 	assert.NilError(t, err)
 	assert.Equal(t, float32(qps), c.QPS)
 	assert.Equal(t, burst, c.Burst)
@@ -30,7 +34,34 @@ func Test_CreateClientConfig_SetBurstQPS(t *testing.T) {
 
 func Test_CreateClientConfig_LimitQPStoFloat32(t *testing.T) {
 	qps := float64(math.MaxFloat32) * 2
-	c := &rest.Config{}
+	cf := createMinimalKubeconfig(t)
-	err := config.ConfigureClientConfig(c, qps, 0)
+	defer os.Remove(cf)
+	_, err := config.CreateClientConfig(cf, qps, 0)
 	assert.ErrorContains(t, err, "QPS")
 }
+func createMinimalKubeconfig(t *testing.T) string {
+	t.Helper()
+
+	minimalConfig := clientcmdapi.Config{
+		Clusters: map[string]*clientcmdapi.Cluster{
+			"test": {Server: "http://localhost:7777"},
+		},
+		AuthInfos: map[string]*clientcmdapi.AuthInfo{
+			"test": {},
+		},
+		Contexts: map[string]*clientcmdapi.Context{
+			"test": {AuthInfo: "test", Cluster: "test"},
+		},
+		CurrentContext: "test",
+	}
+
+	f, err := os.CreateTemp("", "")
+	assert.NilError(t, err)
+	enc, err := runtime.Encode(clientcmdlatest.Codec, &minimalConfig)
+	assert.NilError(t, err)
+	_, err = f.Write(enc)
+	assert.NilError(t, err)
+	assert.NilError(t, f.Close())
+
+	return f.Name()
+}