From e5a9148a75d868eb235eb9f3d9055ae5bfc1e5d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Charles-Edouard=20Br=C3=A9t=C3=A9ch=C3=A9?= Date: Fri, 14 Apr 2023 11:00:52 +0200 Subject: [PATCH] refactor: move clients creation in internal package (#6924) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * refactor: move clients creation in internal package Signed-off-by: Charles-Edouard Brétéché * dclient Signed-off-by: Charles-Edouard Brétéché * metadata Signed-off-by: Charles-Edouard Brétéché --------- Signed-off-by: Charles-Edouard Brétéché --- cmd/background-controller/main.go | 27 ++++------ cmd/cleanup-controller/main.go | 17 +++---- cmd/internal/client.go | 24 ++++----- cmd/internal/config.go | 82 +++++++++++++++++++++++++++---- cmd/internal/setup.go | 40 +++++++++++++++ cmd/kyverno-init/main.go | 12 ++--- cmd/kyverno/main.go | 47 +++++++----------- cmd/reports-controller/main.go | 33 +++++-------- pkg/metrics/common_types.go | 4 +- 9 files changed, 179 insertions(+), 107 deletions(-) diff --git a/cmd/background-controller/main.go b/cmd/background-controller/main.go index 455bf1479f..07b3fe423c 100644 --- a/cmd/background-controller/main.go +++ b/cmd/background-controller/main.go @@ -13,8 +13,6 @@ import ( "github.com/kyverno/kyverno/pkg/client/clientset/versioned" kyvernoinformer "github.com/kyverno/kyverno/pkg/client/informers/externalversions" "github.com/kyverno/kyverno/pkg/clients/dclient" - dynamicclient "github.com/kyverno/kyverno/pkg/clients/dynamic" - kyvernoclient "github.com/kyverno/kyverno/pkg/clients/kyverno" "github.com/kyverno/kyverno/pkg/config" policymetricscontroller "github.com/kyverno/kyverno/pkg/controllers/metrics/policy" engineapi "github.com/kyverno/kyverno/pkg/engine/api" @@ -100,6 +98,9 @@ func main() { internal.WithConfigMapCaching(), internal.WithRegistryClient(), internal.WithLeaderElection(), + internal.WithKyvernoClient(), + internal.WithDynamicClient(), + internal.WithKyvernoDynamicClient(), internal.WithFlagSets(flagset), ) // parse flags @@ -107,21 +108,13 @@ func main() { // setup signalCtx, setup, sdown := internal.Setup(appConfig, "kyverno-background-controller", false) defer sdown() - // create instrumented clients - kyvernoClient := internal.CreateKyvernoClient(setup.Logger, kyvernoclient.WithMetrics(setup.MetricsManager, metrics.KyvernoClient), kyvernoclient.WithTracing()) - dynamicClient := internal.CreateDynamicClient(setup.Logger, dynamicclient.WithMetrics(setup.MetricsManager, metrics.KyvernoClient), dynamicclient.WithTracing()) - dClient, err := dclient.NewClient(signalCtx, dynamicClient, setup.KubeClient, 15*time.Minute) - if err != nil { - setup.Logger.Error(err, "failed to create dynamic client") - os.Exit(1) - } // THIS IS AN UGLY FIX // ELSE KYAML IS NOT THREAD SAFE kyamlopenapi.Schema() // informer factories - kyvernoInformer := kyvernoinformer.NewSharedInformerFactory(kyvernoClient, resyncPeriod) + kyvernoInformer := kyvernoinformer.NewSharedInformerFactory(setup.KyvernoClient, resyncPeriod) eventGenerator := event.NewEventGenerator( - dClient, + setup.KyvernoDynamicClient, kyvernoInformer.Kyverno().V1().ClusterPolicies(), kyvernoInformer.Kyverno().V1().Policies(), maxQueuedEvents, @@ -141,10 +134,10 @@ func main() { setup.Configuration, setup.MetricsConfiguration, setup.Jp, - dClient, + setup.KyvernoDynamicClient, setup.RegistryClient, setup.KubeClient, - kyvernoClient, + setup.KyvernoClient, ) // start informers and wait for cache sync if !internal.StartInformersAndWaitForCacheSync(signalCtx, setup.Logger, kyvernoInformer) { @@ -165,15 +158,15 @@ func main() { logger := setup.Logger.WithName("leader") // create leader factories kubeInformer := kubeinformers.NewSharedInformerFactory(setup.KubeClient, resyncPeriod) - kyvernoInformer := kyvernoinformer.NewSharedInformerFactory(kyvernoClient, resyncPeriod) + kyvernoInformer := kyvernoinformer.NewSharedInformerFactory(setup.KyvernoClient, resyncPeriod) // create leader controllers leaderControllers, err := createrLeaderControllers( engine, genWorkers, kubeInformer, kyvernoInformer, - kyvernoClient, - dClient, + setup.KyvernoClient, + setup.KyvernoDynamicClient, setup.RegistryClient, setup.Configuration, setup.MetricsManager, diff --git a/cmd/cleanup-controller/main.go b/cmd/cleanup-controller/main.go index 498264a32b..fda71bc5fe 100644 --- a/cmd/cleanup-controller/main.go +++ b/cmd/cleanup-controller/main.go @@ -12,15 +12,12 @@ import ( cleanuphandlers "github.com/kyverno/kyverno/cmd/cleanup-controller/handlers/cleanup" "github.com/kyverno/kyverno/cmd/internal" kyvernoinformer "github.com/kyverno/kyverno/pkg/client/informers/externalversions" - dynamicclient "github.com/kyverno/kyverno/pkg/clients/dynamic" - kyvernoclient "github.com/kyverno/kyverno/pkg/clients/kyverno" "github.com/kyverno/kyverno/pkg/config" "github.com/kyverno/kyverno/pkg/controllers/certmanager" "github.com/kyverno/kyverno/pkg/controllers/cleanup" genericloggingcontroller "github.com/kyverno/kyverno/pkg/controllers/generic/logging" genericwebhookcontroller "github.com/kyverno/kyverno/pkg/controllers/generic/webhook" "github.com/kyverno/kyverno/pkg/leaderelection" - "github.com/kyverno/kyverno/pkg/metrics" "github.com/kyverno/kyverno/pkg/tls" "github.com/kyverno/kyverno/pkg/webhooks" admissionregistrationv1 "k8s.io/api/admissionregistration/v1" @@ -66,6 +63,8 @@ func main() { internal.WithTracing(), internal.WithKubeconfig(), internal.WithLeaderElection(), + internal.WithKyvernoClient(), + internal.WithKyvernoDynamicClient(), internal.WithFlagSets(flagset), ) // parse flags @@ -73,8 +72,6 @@ func main() { // setup ctx, setup, sdown := internal.Setup(appConfig, "kyverno-cleanup-controller", false) defer sdown() - // create instrumented clients - kyvernoClient := internal.CreateKyvernoClient(setup.Logger, kyvernoclient.WithMetrics(setup.MetricsManager, metrics.KubeClient), kyvernoclient.WithTracing()) // setup leader election le, err := leaderelection.New( setup.Logger.WithName("leader-election"), @@ -87,7 +84,7 @@ func main() { logger := setup.Logger.WithName("leader") // informer factories kubeInformer := kubeinformers.NewSharedInformerFactoryWithOptions(setup.KubeClient, resyncPeriod) - kyvernoInformer := kyvernoinformer.NewSharedInformerFactory(kyvernoClient, resyncPeriod) + kyvernoInformer := kyvernoinformer.NewSharedInformerFactory(setup.KyvernoClient, resyncPeriod) kubeKyvernoInformer := kubeinformers.NewSharedInformerFactoryWithOptions(setup.KubeClient, resyncPeriod, kubeinformers.WithNamespace(config.KyvernoNamespace())) // listers secretLister := kubeKyvernoInformer.Core().V1().Secrets().Lister().Secrets(config.KyvernoNamespace()) @@ -169,12 +166,10 @@ func main() { setup.Logger.Error(err, "failed to initialize leader election") os.Exit(1) } - dynamicClient := internal.CreateDynamicClient(setup.Logger, dynamicclient.WithMetrics(setup.MetricsManager, metrics.KyvernoClient), dynamicclient.WithTracing()) - dClient := internal.CreateDClient(setup.Logger, ctx, dynamicClient, setup.KubeClient, 15*time.Minute) // informer factories kubeInformer := kubeinformers.NewSharedInformerFactoryWithOptions(setup.KubeClient, resyncPeriod) kubeKyvernoInformer := kubeinformers.NewSharedInformerFactoryWithOptions(setup.KubeClient, resyncPeriod, kubeinformers.WithNamespace(config.KyvernoNamespace())) - kyvernoInformer := kyvernoinformer.NewSharedInformerFactory(kyvernoClient, resyncPeriod) + kyvernoInformer := kyvernoinformer.NewSharedInformerFactory(setup.KyvernoClient, resyncPeriod) // listers secretLister := kubeKyvernoInformer.Core().V1().Secrets().Lister().Secrets(config.KyvernoNamespace()) cpolLister := kyvernoInformer.Kyverno().V2alpha1().ClusterCleanupPolicies().Lister() @@ -198,8 +193,8 @@ func main() { os.Exit(1) } // create handlers - admissionHandlers := admissionhandlers.New(dClient) - cleanupHandlers := cleanuphandlers.New(setup.Logger.WithName("cleanup-handler"), dClient, cpolLister, polLister, nsLister, setup.Jp) + admissionHandlers := admissionhandlers.New(setup.KyvernoDynamicClient) + cleanupHandlers := cleanuphandlers.New(setup.Logger.WithName("cleanup-handler"), setup.KyvernoDynamicClient, cpolLister, polLister, nsLister, setup.Jp) // create server server := NewServer( func() ([]byte, []byte, error) { diff --git a/cmd/internal/client.go b/cmd/internal/client.go index 7ec0b341b3..b35cf47887 100644 --- a/cmd/internal/client.go +++ b/cmd/internal/client.go @@ -44,7 +44,7 @@ func createKubernetesClient(logger logr.Logger, opts ...kube.NewOption) kubernet return client } -func CreateKyvernoClient(logger logr.Logger, opts ...kyverno.NewOption) versioned.Interface { +func createKyvernoClient(logger logr.Logger, opts ...kyverno.NewOption) versioned.Interface { logger = logger.WithName("kyverno-client") logger.Info("create kyverno client...", "kubeconfig", kubeconfig, "qps", clientRateLimitQPS, "burst", clientRateLimitBurst) client, err := kyverno.NewForConfig(createClientConfig(logger), opts...) @@ -52,7 +52,7 @@ func CreateKyvernoClient(logger logr.Logger, opts ...kyverno.NewOption) versione return client } -func CreateDynamicClient(logger logr.Logger, opts ...dyn.NewOption) dynamic.Interface { +func createDynamicClient(logger logr.Logger, opts ...dyn.NewOption) dynamic.Interface { logger = logger.WithName("dynamic-client") logger.Info("create dynamic client...", "kubeconfig", kubeconfig, "qps", clientRateLimitQPS, "burst", clientRateLimitBurst) client, err := dyn.NewForConfig(createClientConfig(logger), opts...) @@ -60,7 +60,7 @@ func CreateDynamicClient(logger logr.Logger, opts ...dyn.NewOption) dynamic.Inte return client } -func CreateMetadataClient(logger logr.Logger, opts ...meta.NewOption) metadata.Interface { +func createMetadataClient(logger logr.Logger, opts ...meta.NewOption) metadata.Interface { logger = logger.WithName("metadata-client") logger.Info("create metadata client...", "kubeconfig", kubeconfig, "qps", clientRateLimitQPS, "burst", clientRateLimitBurst) client, err := meta.NewForConfig(createClientConfig(logger), opts...) @@ -68,7 +68,7 @@ func CreateMetadataClient(logger logr.Logger, opts ...meta.NewOption) metadata.I return client } -func CreateApiServerClient(logger logr.Logger, opts ...apisrv.NewOption) apiserver.Interface { +func createApiServerClient(logger logr.Logger, opts ...apisrv.NewOption) apiserver.Interface { logger = logger.WithName("apiserver-client") logger.Info("create apiserver client...", "kubeconfig", kubeconfig, "qps", clientRateLimitQPS, "burst", clientRateLimitBurst) client, err := apisrv.NewForConfig(createClientConfig(logger), opts...) @@ -76,6 +76,14 @@ func CreateApiServerClient(logger logr.Logger, opts ...apisrv.NewOption) apiserv return client } +func createKyvernoDynamicClient(logger logr.Logger, ctx context.Context, dyn dynamic.Interface, kube kubernetes.Interface, resync time.Duration) dclient.Interface { + logger = logger.WithName("d-client") + logger.Info("create the kyverno dynamic client...", "kubeconfig", kubeconfig, "qps", clientRateLimitQPS, "burst", clientRateLimitBurst) + client, err := dclient.NewClient(ctx, dyn, kube, resync) + checkError(logger, err, "failed to create d client") + return client +} + func CreateAggregatorClient(logger logr.Logger, opts ...agg.NewOption) aggregator.Interface { logger = logger.WithName("aggregator-client") logger.Info("create aggregator client...", "kubeconfig", kubeconfig, "qps", clientRateLimitQPS, "burst", clientRateLimitBurst) @@ -83,11 +91,3 @@ func CreateAggregatorClient(logger logr.Logger, opts ...agg.NewOption) aggregato checkError(logger, err, "failed to create aggregator client") return client } - -func CreateDClient(logger logr.Logger, ctx context.Context, dyn dynamic.Interface, kube kubernetes.Interface, resync time.Duration) dclient.Interface { - logger = logger.WithName("d-client") - logger.Info("create the kyverno dynamic client...", "kubeconfig", kubeconfig, "qps", clientRateLimitQPS, "burst", clientRateLimitBurst) - client, err := dclient.NewClient(ctx, dyn, kube, resync) - checkError(logger, err, "failed to create d client") - return client -} diff --git a/cmd/internal/config.go b/cmd/internal/config.go index 30d9181432..b722680333 100644 --- a/cmd/internal/config.go +++ b/cmd/internal/config.go @@ -14,6 +14,11 @@ type Configuration interface { UsesCosign() bool UsesRegistryClient() bool UsesLeaderElection() bool + UsesKyvernoClient() bool + UsesDynamicClient() bool + UsesApiServerClient() bool + UsesMetadataClient() bool + UsesKyvernoDynamicClient() bool FlagSets() []*flag.FlagSet } @@ -81,6 +86,38 @@ func WithLeaderElection() ConfigurationOption { } } +func WithKyvernoClient() ConfigurationOption { + return func(c *configuration) { + c.usesKyvernoClient = true + } +} + +func WithDynamicClient() ConfigurationOption { + return func(c *configuration) { + c.usesDynamicClient = true + } +} + +func WithApiServerClient() ConfigurationOption { + return func(c *configuration) { + c.usesApiServerClient = true + } +} + +func WithMetadataClient() ConfigurationOption { + return func(c *configuration) { + c.usesMetadataClient = true + } +} + +func WithKyvernoDynamicClient() ConfigurationOption { + return func(c *configuration) { + // requires dynamic client + c.usesDynamicClient = true + c.usesKyvernoDynamicClient = true + } +} + func WithFlagSets(flagsets ...*flag.FlagSet) ConfigurationOption { return func(c *configuration) { c.flagSets = append(c.flagSets, flagsets...) @@ -88,16 +125,21 @@ func WithFlagSets(flagsets ...*flag.FlagSet) ConfigurationOption { } type configuration struct { - usesMetrics bool - usesTracing bool - usesProfiling bool - usesKubeconfig bool - usesPolicyExceptions bool - usesConfigMapCaching bool - usesCosign bool - usesRegistryClient bool - usesLeaderElection bool - flagSets []*flag.FlagSet + usesMetrics bool + usesTracing bool + usesProfiling bool + usesKubeconfig bool + usesPolicyExceptions bool + usesConfigMapCaching bool + usesCosign bool + usesRegistryClient bool + usesLeaderElection bool + usesKyvernoClient bool + usesDynamicClient bool + usesApiServerClient bool + usesMetadataClient bool + usesKyvernoDynamicClient bool + flagSets []*flag.FlagSet } func (c *configuration) UsesMetrics() bool { @@ -136,6 +178,26 @@ func (c *configuration) UsesLeaderElection() bool { return c.usesLeaderElection } +func (c *configuration) UsesKyvernoClient() bool { + return c.usesKyvernoClient +} + +func (c *configuration) UsesDynamicClient() bool { + return c.usesDynamicClient +} + +func (c *configuration) UsesApiServerClient() bool { + return c.usesApiServerClient +} + +func (c *configuration) UsesMetadataClient() bool { + return c.usesMetadataClient +} + +func (c *configuration) UsesKyvernoDynamicClient() bool { + return c.usesKyvernoDynamicClient +} + func (c *configuration) FlagSets() []*flag.FlagSet { return c.flagSets } diff --git a/cmd/internal/setup.go b/cmd/internal/setup.go index 6743b7d11f..fdec87ac25 100644 --- a/cmd/internal/setup.go +++ b/cmd/internal/setup.go @@ -2,14 +2,24 @@ package internal import ( "context" + "time" "github.com/go-logr/logr" + "github.com/kyverno/kyverno/pkg/client/clientset/versioned" + apiserverclient "github.com/kyverno/kyverno/pkg/clients/apiserver" + "github.com/kyverno/kyverno/pkg/clients/dclient" + dynamicclient "github.com/kyverno/kyverno/pkg/clients/dynamic" kubeclient "github.com/kyverno/kyverno/pkg/clients/kube" + kyvernoclient "github.com/kyverno/kyverno/pkg/clients/kyverno" + metadataclient "github.com/kyverno/kyverno/pkg/clients/metadata" "github.com/kyverno/kyverno/pkg/config" "github.com/kyverno/kyverno/pkg/engine/jmespath" "github.com/kyverno/kyverno/pkg/metrics" "github.com/kyverno/kyverno/pkg/registryclient" + apiserver "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/metadata" ) func shutdown(logger logr.Logger, sdowns ...context.CancelFunc) context.CancelFunc { @@ -32,6 +42,11 @@ type SetupResult struct { KubeClient kubernetes.Interface LeaderElectionClient kubernetes.Interface RegistryClient registryclient.Client + KyvernoClient versioned.Interface + DynamicClient dynamic.Interface + ApiServerClient apiserver.Interface + MetadataClient metadata.Interface + KyvernoDynamicClient dclient.Interface } func Setup(config Configuration, name string, skipResourceFilters bool) (context.Context, SetupResult, context.CancelFunc) { @@ -55,6 +70,26 @@ func Setup(config Configuration, name string, skipResourceFilters bool) (context if config.UsesLeaderElection() { leaderElectionClient = createKubernetesClient(logger, kubeclient.WithMetrics(metricsManager, metrics.KubeClient), kubeclient.WithTracing()) } + var kyvernoClient versioned.Interface + if config.UsesKyvernoClient() { + kyvernoClient = createKyvernoClient(logger, kyvernoclient.WithMetrics(metricsManager, metrics.KyvernoClient), kyvernoclient.WithTracing()) + } + var dynamicClient dynamic.Interface + if config.UsesDynamicClient() { + dynamicClient = createDynamicClient(logger, dynamicclient.WithMetrics(metricsManager, metrics.DynamicClient), dynamicclient.WithTracing()) + } + var apiServerClient apiserver.Interface + if config.UsesApiServerClient() { + apiServerClient = createApiServerClient(logger, apiserverclient.WithMetrics(metricsManager, metrics.ApiServerClient), apiserverclient.WithTracing()) + } + var dClient dclient.Interface + if config.UsesKyvernoDynamicClient() { + dClient = createKyvernoDynamicClient(logger, ctx, dynamicClient, client, 15*time.Minute) + } + var metadataClient metadata.Interface + if config.UsesMetadataClient() { + metadataClient = createMetadataClient(logger, metadataclient.WithMetrics(metricsManager, metrics.MetadataClient), metadataclient.WithTracing()) + } return ctx, SetupResult{ Logger: logger, @@ -65,6 +100,11 @@ func Setup(config Configuration, name string, skipResourceFilters bool) (context KubeClient: client, LeaderElectionClient: leaderElectionClient, RegistryClient: registryClient, + KyvernoClient: kyvernoClient, + DynamicClient: dynamicClient, + ApiServerClient: apiServerClient, + MetadataClient: metadataClient, + KyvernoDynamicClient: dClient, }, shutdown(logger.WithName("shutdown"), sdownMaxProcs, sdownMetrics, sdownTracing, sdownSignals) } diff --git a/cmd/kyverno-init/main.go b/cmd/kyverno-init/main.go index 01639ed971..2ebf54f1ac 100644 --- a/cmd/kyverno-init/main.go +++ b/cmd/kyverno-init/main.go @@ -7,7 +7,6 @@ import ( "context" "os" "sync" - "time" "github.com/kyverno/kyverno/cmd/internal" kyvernoclient "github.com/kyverno/kyverno/pkg/client/clientset/versioned" @@ -32,6 +31,9 @@ func main() { // config appConfig := internal.NewConfiguration( internal.WithKubeconfig(), + internal.WithKyvernoClient(), + internal.WithDynamicClient(), + internal.WithKyvernoDynamicClient(), ) // parse flags internal.ParseFlags(appConfig) @@ -42,10 +44,6 @@ func main() { // setup maxprocs ctx, setup, sdown := internal.Setup(appConfig, "kyverno-init-controller", false) defer sdown() - // create clients - dynamicClient := internal.CreateDynamicClient(setup.Logger) - kyvernoClient := internal.CreateKyvernoClient(setup.Logger) - client := internal.CreateDClient(setup.Logger, ctx, dynamicClient, setup.KubeClient, 15*time.Minute) // Exit for unsupported version of kubernetes cluster if !kubeutils.HigherThanKubernetesVersion(setup.KubeClient.Discovery(), logging.GlobalLogger(), 1, 16, 0) { os.Exit(1) @@ -92,8 +90,8 @@ func main() { in := gen(done, ctx.Done(), requests...) // process requests // processing routine count : 2 - p1 := process(client, kyvernoClient, done, ctx.Done(), in) - p2 := process(client, kyvernoClient, done, ctx.Done(), in) + p1 := process(setup.KyvernoDynamicClient, setup.KyvernoClient, done, ctx.Done(), in) + p2 := process(setup.KyvernoDynamicClient, setup.KyvernoClient, done, ctx.Done(), in) // merge results from processing routines for err := range merge(done, ctx.Done(), p1, p2) { if err != nil { diff --git a/cmd/kyverno/main.go b/cmd/kyverno/main.go index 6cdb1f9c3a..2f4f874bde 100644 --- a/cmd/kyverno/main.go +++ b/cmd/kyverno/main.go @@ -14,10 +14,7 @@ import ( "github.com/kyverno/kyverno/cmd/internal" "github.com/kyverno/kyverno/pkg/client/clientset/versioned" kyvernoinformer "github.com/kyverno/kyverno/pkg/client/informers/externalversions" - apiserverclient "github.com/kyverno/kyverno/pkg/clients/apiserver" "github.com/kyverno/kyverno/pkg/clients/dclient" - dynamicclient "github.com/kyverno/kyverno/pkg/clients/dynamic" - kyvernoclient "github.com/kyverno/kyverno/pkg/clients/kyverno" "github.com/kyverno/kyverno/pkg/config" "github.com/kyverno/kyverno/pkg/controllers/certmanager" genericloggingcontroller "github.com/kyverno/kyverno/pkg/controllers/generic/logging" @@ -30,7 +27,6 @@ import ( "github.com/kyverno/kyverno/pkg/event" "github.com/kyverno/kyverno/pkg/leaderelection" "github.com/kyverno/kyverno/pkg/logging" - "github.com/kyverno/kyverno/pkg/metrics" "github.com/kyverno/kyverno/pkg/openapi" "github.com/kyverno/kyverno/pkg/policycache" "github.com/kyverno/kyverno/pkg/tls" @@ -218,6 +214,10 @@ func main() { internal.WithCosign(), internal.WithRegistryClient(), internal.WithLeaderElection(), + internal.WithKyvernoClient(), + internal.WithDynamicClient(), + internal.WithKyvernoDynamicClient(), + internal.WithApiServerClient(), internal.WithFlagSets(flagset), ) // parse flags @@ -227,27 +227,18 @@ func main() { defer sdown() // show version showWarnings(setup.Logger) - // create instrumented clients - kyvernoClient := internal.CreateKyvernoClient(setup.Logger, kyvernoclient.WithMetrics(setup.MetricsManager, metrics.KyvernoClient), kyvernoclient.WithTracing()) - dynamicClient := internal.CreateDynamicClient(setup.Logger, dynamicclient.WithMetrics(setup.MetricsManager, metrics.KyvernoClient), dynamicclient.WithTracing()) - apiserverClient := internal.CreateApiServerClient(setup.Logger, apiserverclient.WithMetrics(setup.MetricsManager, metrics.KubeClient), apiserverclient.WithTracing()) - dClient, err := dclient.NewClient(signalCtx, dynamicClient, setup.KubeClient, 15*time.Minute) - if err != nil { - setup.Logger.Error(err, "failed to create dynamic client") - os.Exit(1) - } // THIS IS AN UGLY FIX // ELSE KYAML IS NOT THREAD SAFE kyamlopenapi.Schema() // check we can run - if err := sanityChecks(apiserverClient); err != nil { + if err := sanityChecks(setup.ApiServerClient); err != nil { setup.Logger.Error(err, "sanity checks failed") os.Exit(1) } // informer factories kubeInformer := kubeinformers.NewSharedInformerFactory(setup.KubeClient, resyncPeriod) kubeKyvernoInformer := kubeinformers.NewSharedInformerFactoryWithOptions(setup.KubeClient, resyncPeriod, kubeinformers.WithNamespace(config.KyvernoNamespace())) - kyvernoInformer := kyvernoinformer.NewSharedInformerFactory(kyvernoClient, resyncPeriod) + kyvernoInformer := kyvernoinformer.NewSharedInformerFactory(setup.KyvernoClient, resyncPeriod) secretLister := kubeKyvernoInformer.Core().V1().Secrets().Lister().Secrets(config.KyvernoNamespace()) openApiManager, err := openapi.NewManager(setup.Logger.WithName("openapi")) if err != nil { @@ -265,7 +256,7 @@ func main() { ) policyCache := policycache.NewCache() eventGenerator := event.NewEventGenerator( - dClient, + setup.KyvernoDynamicClient, kyvernoInformer.Kyverno().V1().ClusterPolicies(), kyvernoInformer.Kyverno().V1().Policies(), maxQueuedEvents, @@ -304,10 +295,10 @@ func main() { setup.Configuration, setup.MetricsConfiguration, setup.Jp, - dClient, + setup.KyvernoDynamicClient, setup.RegistryClient, setup.KubeClient, - kyvernoClient, + setup.KyvernoClient, ) // create non leader controllers nonLeaderControllers, nonLeaderBootstrap := createNonLeaderControllers( @@ -316,8 +307,8 @@ func main() { kubeInformer, kyvernoInformer, setup.KubeClient, - kyvernoClient, - dClient, + setup.KyvernoClient, + setup.KyvernoDynamicClient, setup.Configuration, policyCache, openApiManager, @@ -349,7 +340,7 @@ func main() { // create leader factories kubeInformer := kubeinformers.NewSharedInformerFactory(setup.KubeClient, resyncPeriod) kubeKyvernoInformer := kubeinformers.NewSharedInformerFactoryWithOptions(setup.KubeClient, resyncPeriod, kubeinformers.WithNamespace(config.KyvernoNamespace())) - kyvernoInformer := kyvernoinformer.NewSharedInformerFactory(kyvernoClient, resyncPeriod) + kyvernoInformer := kyvernoinformer.NewSharedInformerFactory(setup.KyvernoClient, resyncPeriod) // create leader controllers leaderControllers, warmup, err := createrLeaderControllers( admissionReports, @@ -360,8 +351,8 @@ func main() { kubeKyvernoInformer, kyvernoInformer, setup.KubeClient, - kyvernoClient, - dClient, + setup.KyvernoClient, + setup.KyvernoDynamicClient, certRenewer, runtime, int32(servicePort), @@ -411,17 +402,17 @@ func main() { }() // create webhooks server urgen := webhookgenerate.NewGenerator( - kyvernoClient, + setup.KyvernoClient, kyvernoInformer.Kyverno().V1beta1().UpdateRequests(), ) policyHandlers := webhookspolicy.NewHandlers( - dClient, + setup.KyvernoDynamicClient, openApiManager, ) resourceHandlers := webhooksresource.NewHandlers( engine, - dClient, - kyvernoClient, + setup.KyvernoDynamicClient, + setup.KyvernoClient, setup.RegistryClient, setup.Configuration, setup.MetricsManager, @@ -463,7 +454,7 @@ func main() { runtime, kubeInformer.Rbac().V1().RoleBindings().Lister(), kubeInformer.Rbac().V1().ClusterRoleBindings().Lister(), - dClient.Discovery(), + setup.KyvernoDynamicClient.Discovery(), ) // start informers and wait for cache sync // we need to call start again because we potentially registered new informers diff --git a/cmd/reports-controller/main.go b/cmd/reports-controller/main.go index 77fe2b789b..f34fbde475 100644 --- a/cmd/reports-controller/main.go +++ b/cmd/reports-controller/main.go @@ -12,9 +12,6 @@ import ( "github.com/kyverno/kyverno/pkg/client/clientset/versioned" kyvernoinformer "github.com/kyverno/kyverno/pkg/client/informers/externalversions" "github.com/kyverno/kyverno/pkg/clients/dclient" - dynamicclient "github.com/kyverno/kyverno/pkg/clients/dynamic" - kyvernoclient "github.com/kyverno/kyverno/pkg/clients/kyverno" - metadataclient "github.com/kyverno/kyverno/pkg/clients/metadata" "github.com/kyverno/kyverno/pkg/config" admissionreportcontroller "github.com/kyverno/kyverno/pkg/controllers/report/admission" aggregatereportcontroller "github.com/kyverno/kyverno/pkg/controllers/report/aggregate" @@ -25,7 +22,6 @@ import ( "github.com/kyverno/kyverno/pkg/event" "github.com/kyverno/kyverno/pkg/leaderelection" "github.com/kyverno/kyverno/pkg/logging" - "github.com/kyverno/kyverno/pkg/metrics" "github.com/kyverno/kyverno/pkg/registryclient" kubeinformers "k8s.io/client-go/informers" metadatainformers "k8s.io/client-go/metadata/metadatainformer" @@ -190,6 +186,10 @@ func main() { internal.WithCosign(), internal.WithRegistryClient(), internal.WithLeaderElection(), + internal.WithKyvernoClient(), + internal.WithDynamicClient(), + internal.WithMetadataClient(), + internal.WithKyvernoDynamicClient(), internal.WithFlagSets(flagset), ) // parse flags @@ -201,22 +201,13 @@ func main() { // setup ctx, setup, sdown := internal.Setup(appConfig, "kyverno-reports-controller", skipResourceFilters) defer sdown() - // create instrumented clients - kyvernoClient := internal.CreateKyvernoClient(setup.Logger, kyvernoclient.WithMetrics(setup.MetricsManager, metrics.KyvernoClient), kyvernoclient.WithTracing()) - metadataClient := internal.CreateMetadataClient(setup.Logger, metadataclient.WithMetrics(setup.MetricsManager, metrics.KyvernoClient), metadataclient.WithTracing()) - dynamicClient := internal.CreateDynamicClient(setup.Logger, dynamicclient.WithMetrics(setup.MetricsManager, metrics.KyvernoClient), dynamicclient.WithTracing()) - dClient, err := dclient.NewClient(ctx, dynamicClient, setup.KubeClient, 15*time.Minute) - if err != nil { - setup.Logger.Error(err, "failed to create dynamic client") - os.Exit(1) - } // THIS IS AN UGLY FIX // ELSE KYAML IS NOT THREAD SAFE kyamlopenapi.Schema() // informer factories - kyvernoInformer := kyvernoinformer.NewSharedInformerFactory(kyvernoClient, resyncPeriod) + kyvernoInformer := kyvernoinformer.NewSharedInformerFactory(setup.KyvernoClient, resyncPeriod) eventGenerator := event.NewEventGenerator( - dClient, + setup.KyvernoDynamicClient, kyvernoInformer.Kyverno().V1().ClusterPolicies(), kyvernoInformer.Kyverno().V1().Policies(), maxQueuedEvents, @@ -229,10 +220,10 @@ func main() { setup.Configuration, setup.MetricsConfiguration, setup.Jp, - dClient, + setup.KyvernoDynamicClient, setup.RegistryClient, setup.KubeClient, - kyvernoClient, + setup.KyvernoClient, ) // start informers and wait for cache sync if !internal.StartInformersAndWaitForCacheSync(ctx, setup.Logger, kyvernoInformer) { @@ -255,8 +246,8 @@ func main() { // create leader factories kubeInformer := kubeinformers.NewSharedInformerFactory(setup.KubeClient, resyncPeriod) kubeKyvernoInformer := kubeinformers.NewSharedInformerFactoryWithOptions(setup.KubeClient, resyncPeriod, kubeinformers.WithNamespace(config.KyvernoNamespace())) - kyvernoInformer := kyvernoinformer.NewSharedInformerFactory(kyvernoClient, resyncPeriod) - metadataInformer := metadatainformers.NewSharedInformerFactory(metadataClient, 15*time.Minute) + kyvernoInformer := kyvernoinformer.NewSharedInformerFactory(setup.KyvernoClient, resyncPeriod) + metadataInformer := metadatainformers.NewSharedInformerFactory(setup.MetadataClient, 15*time.Minute) // create leader controllers leaderControllers, warmup, err := createrLeaderControllers( engine, @@ -267,8 +258,8 @@ func main() { kubeInformer, kyvernoInformer, metadataInformer, - kyvernoClient, - dClient, + setup.KyvernoClient, + setup.KyvernoDynamicClient, setup.RegistryClient, setup.Configuration, setup.Jp, diff --git a/pkg/metrics/common_types.go b/pkg/metrics/common_types.go index 7e725e2710..ccd9777bf0 100644 --- a/pkg/metrics/common_types.go +++ b/pkg/metrics/common_types.go @@ -74,8 +74,10 @@ const ( type ClientType string const ( - KubeDynamicClient ClientType = "dynamic" + DynamicClient ClientType = "dynamic" KubeClient ClientType = "kubeclient" KyvernoClient ClientType = "kyverno" + MetadataClient ClientType = "metadata" + ApiServerClient ClientType = "apiserver" PolicyReportClient ClientType = "policyreport" )