From 807b16b87c5cace4905dee14112df07692c5b7da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Charles-Edouard=20Br=C3=A9t=C3=A9ch=C3=A9?= Date: Tue, 13 Dec 2022 16:43:17 +0100 Subject: [PATCH] feat: add certs controller to cleanup policies (#5671) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add certs controller to cleanup policies Signed-off-by: Charles-Edouard Brétéché * fix Signed-off-by: Charles-Edouard Brétéché * webhook controller Signed-off-by: Charles-Edouard Brétéché Signed-off-by: Charles-Edouard Brétéché --- charts/kyverno/README.md | 1 + .../cleanup-controller/clusterrole.yaml | 11 + .../templates/cleanup-controller/role.yaml | 9 + .../templates/cleanup-controller/secret.yaml | 42 +--- charts/kyverno/templates/secret.yaml | 2 + charts/kyverno/values.yaml | 4 + cmd/cleanup-controller/controller.go | 190 ++++++++++++++++++ cmd/cleanup-controller/main.go | 40 +++- 8 files changed, 259 insertions(+), 40 deletions(-) create mode 100644 cmd/cleanup-controller/controller.go diff --git a/charts/kyverno/README.md b/charts/kyverno/README.md index c0ad2287fc..f79d11ef8c 100644 --- a/charts/kyverno/README.md +++ b/charts/kyverno/README.md @@ -219,6 +219,7 @@ The command removes all the Kubernetes components associated with the chart and | cleanupController.rbac.create | bool | `true` | Create RBAC resources | | cleanupController.rbac.serviceAccount.name | string | `nil` | Service account name | | cleanupController.rbac.clusterRole.extraResources | list | `[]` | Extra resource permissions to add in the cluster role | +| cleanupController.createSelfSignedCert | bool | `false` | Create self-signed certificates at deployment time. The certificates won't be automatically renewed if this is set to `true`. | | cleanupController.image.registry | string | `nil` | Image registry | | cleanupController.image.repository | string | `"ghcr.io/kyverno/cleanup-controller"` | Image repository | | cleanupController.image.tag | string | `nil` | Image tag Defaults to appVersion in Chart.yaml if omitted | diff --git a/charts/kyverno/templates/cleanup-controller/clusterrole.yaml b/charts/kyverno/templates/cleanup-controller/clusterrole.yaml index 95afc4b59e..3e03f0d343 100644 --- a/charts/kyverno/templates/cleanup-controller/clusterrole.yaml +++ b/charts/kyverno/templates/cleanup-controller/clusterrole.yaml @@ -7,6 +7,17 @@ metadata: labels: {{- include "kyverno.cleanup-controller.labels" . | nindent 4 }} rules: + - apiGroups: + - admissionregistration.k8s.io + resources: + - validatingwebhookconfigurations + verbs: + - create + - delete + - get + - list + - update + - watch - apiGroups: - '' resources: diff --git a/charts/kyverno/templates/cleanup-controller/role.yaml b/charts/kyverno/templates/cleanup-controller/role.yaml index c34ef1daf2..4a6219a031 100644 --- a/charts/kyverno/templates/cleanup-controller/role.yaml +++ b/charts/kyverno/templates/cleanup-controller/role.yaml @@ -12,6 +12,15 @@ rules: - '' resources: - secrets + verbs: + - get + - list + - watch + - create + - update +- apiGroups: + - '' + resources: - configmaps verbs: - get diff --git a/charts/kyverno/templates/cleanup-controller/secret.yaml b/charts/kyverno/templates/cleanup-controller/secret.yaml index 9c3527ea91..d709e59c5a 100644 --- a/charts/kyverno/templates/cleanup-controller/secret.yaml +++ b/charts/kyverno/templates/cleanup-controller/secret.yaml @@ -1,4 +1,5 @@ {{- if .Values.cleanupController.enabled -}} +{{- if .Values.cleanupController.createSelfSignedCert -}} {{- $ca := genCA (printf "*.%s.svc" (include "kyverno.namespace" .)) 1024 -}} {{- $svcName := (printf "%s.%s.svc" (include "kyverno.cleanup-controller.name" .) (include "kyverno.namespace" .)) -}} {{- $cert := genSignedCert $svcName nil (list $svcName) 1024 $ca -}} @@ -6,9 +7,9 @@ apiVersion: v1 kind: Secret metadata: name: {{ template "kyverno.cleanup-controller.name" . }}.{{ template "kyverno.namespace" . }}.svc.kyverno-tls-ca + namespace: {{ template "kyverno.namespace" . }} labels: {{- include "kyverno.cleanup-controller.labels" . | nindent 4 }} - namespace: {{ template "kyverno.namespace" . }} type: kubernetes.io/tls data: tls.key: {{ $ca.Key | b64enc }} @@ -18,45 +19,14 @@ apiVersion: v1 kind: Secret metadata: name: {{ template "kyverno.cleanup-controller.name" . }}.{{ template "kyverno.namespace" . }}.svc.kyverno-tls-pair + namespace: {{ template "kyverno.namespace" . }} labels: {{- include "kyverno.cleanup-controller.labels" . | nindent 4 }} - namespace: {{ template "kyverno.namespace" . }} + annotations: + self-signed-cert: "true" type: kubernetes.io/tls data: tls.key: {{ $cert.Key | b64enc }} tls.crt: {{ $cert.Cert | b64enc }} ---- -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration -metadata: - name: {{ template "kyverno.cleanup-controller.name" . }} - labels: - {{- include "kyverno.cleanup-controller.labels" . | nindent 4 }} -webhooks: -- admissionReviewVersions: - - v1beta1 - clientConfig: - caBundle: {{ $ca.Cert | b64enc }} - service: - name: {{ template "kyverno.cleanup-controller.name" . }} - namespace: {{ template "kyverno.namespace" . }} - path: /validate - port: 443 - failurePolicy: Fail - matchPolicy: Equivalent - name: {{ printf "%s.%s.svc" (include "kyverno.cleanup-controller.name" .) (include "kyverno.namespace" .) }} - rules: - - apiGroups: - - kyverno.io - apiVersions: - - v2alpha1 - operations: - - CREATE - - UPDATE - resources: - - clustercleanuppolicies/* - - cleanuppolicies/* - scope: '*' - sideEffects: None - timeoutSeconds: 10 +{{- end -}} {{- end -}} diff --git a/charts/kyverno/templates/secret.yaml b/charts/kyverno/templates/secret.yaml index c254b8963c..1592ca741d 100644 --- a/charts/kyverno/templates/secret.yaml +++ b/charts/kyverno/templates/secret.yaml @@ -6,6 +6,7 @@ apiVersion: v1 kind: Secret metadata: name: {{ template "kyverno.serviceName" . }}.{{ template "kyverno.namespace" . }}.svc.kyverno-tls-ca + namespace: {{ template "kyverno.namespace" . }} labels: {{ include "kyverno.labels" . | nindent 4 }} app: kyverno type: kubernetes.io/tls @@ -17,6 +18,7 @@ apiVersion: v1 kind: Secret metadata: name: {{ template "kyverno.serviceName" . }}.{{ template "kyverno.namespace" . }}.svc.kyverno-tls-pair + namespace: {{ template "kyverno.namespace" . }} labels: {{ include "kyverno.labels" . | nindent 4 }} app: kyverno annotations: diff --git a/charts/kyverno/values.yaml b/charts/kyverno/values.yaml index af4535ce83..5d959b31e7 100644 --- a/charts/kyverno/values.yaml +++ b/charts/kyverno/values.yaml @@ -506,6 +506,10 @@ cleanupController: # resources: # - pods + # -- Create self-signed certificates at deployment time. + # The certificates won't be automatically renewed if this is set to `true`. + createSelfSignedCert: false + image: # -- Image registry registry: diff --git a/cmd/cleanup-controller/controller.go b/cmd/cleanup-controller/controller.go new file mode 100644 index 0000000000..f7eb8ec57b --- /dev/null +++ b/cmd/cleanup-controller/controller.go @@ -0,0 +1,190 @@ +package main + +import ( + "context" + "fmt" + "time" + + "github.com/go-logr/logr" + kyvernov1 "github.com/kyverno/kyverno/api/kyverno/v1" + "github.com/kyverno/kyverno/pkg/config" + "github.com/kyverno/kyverno/pkg/controllers" + "github.com/kyverno/kyverno/pkg/logging" + "github.com/kyverno/kyverno/pkg/tls" + controllerutils "github.com/kyverno/kyverno/pkg/utils/controller" + admissionregistrationv1 "k8s.io/api/admissionregistration/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + admissionregistrationv1informers "k8s.io/client-go/informers/admissionregistration/v1" + corev1informers "k8s.io/client-go/informers/core/v1" + admissionregistrationv1listers "k8s.io/client-go/listers/admissionregistration/v1" + corev1listers "k8s.io/client-go/listers/core/v1" + "k8s.io/client-go/util/workqueue" +) + +const ( + // Workers is the number of workers for this controller + Workers = 2 + ControllerName = "webhook-controller" + maxRetries = 10 + managedByLabel = "webhook.kyverno.io/managed-by" +) + +var ( + none = admissionregistrationv1.SideEffectClassNone + fail = admissionregistrationv1.Fail +) + +var logger = logging.ControllerLogger(ControllerName) + +type controller struct { + // clients + vwcClient controllerutils.ObjectClient[*admissionregistrationv1.ValidatingWebhookConfiguration] + + // listers + vwcLister admissionregistrationv1listers.ValidatingWebhookConfigurationLister + secretLister corev1listers.SecretNamespaceLister + + // queue + queue workqueue.RateLimitingInterface + + // config + webhookName string + server string +} + +func NewController( + vwcClient controllerutils.ObjectClient[*admissionregistrationv1.ValidatingWebhookConfiguration], + vwcInformer admissionregistrationv1informers.ValidatingWebhookConfigurationInformer, + secretInformer corev1informers.SecretInformer, + webhookName string, + server string, +) controllers.Controller { + queue := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), ControllerName) + c := controller{ + vwcClient: vwcClient, + vwcLister: vwcInformer.Lister(), + secretLister: secretInformer.Lister().Secrets(config.KyvernoNamespace()), + queue: queue, + webhookName: webhookName, + server: server, + } + controllerutils.AddDefaultEventHandlers(logger, vwcInformer.Informer(), queue) + controllerutils.AddEventHandlersT( + secretInformer.Informer(), + func(obj *corev1.Secret) { + if obj.GetNamespace() == config.KyvernoNamespace() && obj.GetName() == tls.GenerateRootCASecretName() { + c.enqueue() + } + }, + func(_, obj *corev1.Secret) { + if obj.GetNamespace() == config.KyvernoNamespace() && obj.GetName() == tls.GenerateRootCASecretName() { + c.enqueue() + } + }, + func(obj *corev1.Secret) { + if obj.GetNamespace() == config.KyvernoNamespace() && obj.GetName() == tls.GenerateRootCASecretName() { + c.enqueue() + } + }, + ) + return &c +} + +func (c *controller) Run(ctx context.Context, workers int) { + c.enqueue() + controllerutils.Run(ctx, logger, ControllerName, time.Second, c.queue, workers, maxRetries, c.reconcile) +} + +func (c *controller) enqueue() { + c.queue.Add(c.webhookName) +} + +func (c *controller) reconcile(ctx context.Context, logger logr.Logger, key, _, _ string) error { + if key != c.webhookName { + return nil + } + caData, err := tls.ReadRootCASecret(c.secretLister) + if err != nil { + return err + } + desired, err := c.build(caData) + if err != nil { + return err + } + observed, err := c.vwcLister.Get(desired.Name) + if err != nil { + if apierrors.IsNotFound(err) { + _, err := c.vwcClient.Create(ctx, desired, metav1.CreateOptions{}) + return err + } + return err + } + _, err = controllerutils.Update(ctx, observed, c.vwcClient, func(w *admissionregistrationv1.ValidatingWebhookConfiguration) error { + w.Labels = desired.Labels + w.OwnerReferences = desired.OwnerReferences + w.Webhooks = desired.Webhooks + return nil + }) + return err +} + +func objectMeta(name string, owner ...metav1.OwnerReference) metav1.ObjectMeta { + return metav1.ObjectMeta{ + Name: name, + Labels: map[string]string{ + managedByLabel: kyvernov1.ValueKyvernoApp, + }, + OwnerReferences: owner, + } +} + +func (c *controller) build(caBundle []byte) (*admissionregistrationv1.ValidatingWebhookConfiguration, error) { + return &admissionregistrationv1.ValidatingWebhookConfiguration{ + ObjectMeta: objectMeta(c.webhookName), + Webhooks: []admissionregistrationv1.ValidatingWebhook{{ + Name: fmt.Sprintf("%s.%s.svc", config.KyvernoServiceName(), config.KyvernoNamespace()), + ClientConfig: c.clientConfig(caBundle, validatingWebhookServicePath), + Rules: []admissionregistrationv1.RuleWithOperations{{ + Rule: admissionregistrationv1.Rule{ + APIGroups: []string{ + "kyverno.io", + }, + APIVersions: []string{ + "v2alpha1", + }, + Resources: []string{ + "cleanuppolicies/*", + "clustercleanuppolicies/*", + }, + }, + Operations: []admissionregistrationv1.OperationType{ + admissionregistrationv1.Create, + admissionregistrationv1.Update, + }, + }}, + FailurePolicy: &fail, + SideEffects: &none, + AdmissionReviewVersions: []string{"v1"}, + }}, + }, + nil +} + +func (c *controller) clientConfig(caBundle []byte, path string) admissionregistrationv1.WebhookClientConfig { + clientConfig := admissionregistrationv1.WebhookClientConfig{ + CABundle: caBundle, + } + if c.server == "" { + clientConfig.Service = &admissionregistrationv1.ServiceReference{ + Namespace: config.KyvernoNamespace(), + Name: config.KyvernoServiceName(), + Path: &path, + } + } else { + url := fmt.Sprintf("https://%s%s", c.server, path) + clientConfig.URL = &url + } + return clientConfig +} diff --git a/cmd/cleanup-controller/main.go b/cmd/cleanup-controller/main.go index b2d8a5dd29..990a69739e 100644 --- a/cmd/cleanup-controller/main.go +++ b/cmd/cleanup-controller/main.go @@ -16,6 +16,7 @@ import ( kubeclient "github.com/kyverno/kyverno/pkg/clients/kube" kyvernoclient "github.com/kyverno/kyverno/pkg/clients/kyverno" "github.com/kyverno/kyverno/pkg/config" + "github.com/kyverno/kyverno/pkg/controllers/certmanager" "github.com/kyverno/kyverno/pkg/controllers/cleanup" "github.com/kyverno/kyverno/pkg/leaderelection" "github.com/kyverno/kyverno/pkg/metrics" @@ -32,7 +33,6 @@ const ( // TODO: // - helm review labels / selectors // - implement probes -// - better certs management // - supports certs in cronjob type probes struct{} @@ -88,8 +88,38 @@ func main() { // informer factories kubeInformer := kubeinformers.NewSharedInformerFactoryWithOptions(kubeClient, resyncPeriod) kyvernoInformer := kyvernoinformer.NewSharedInformerFactory(kyvernoClient, resyncPeriod) + kubeKyvernoInformer := kubeinformers.NewSharedInformerFactoryWithOptions(kubeClient, resyncPeriod, kubeinformers.WithNamespace(config.KyvernoNamespace())) + // listers + secretLister := kubeKyvernoInformer.Core().V1().Secrets().Lister().Secrets(config.KyvernoNamespace()) // controllers - controller := internal.NewController( + renewer := tls.NewCertRenewer( + kubeClient.CoreV1().Secrets(config.KyvernoNamespace()), + secretLister, + tls.CertRenewalInterval, + tls.CAValidityDuration, + tls.TLSValidityDuration, + "", + ) + certController := internal.NewController( + certmanager.ControllerName, + certmanager.NewController( + kubeKyvernoInformer.Core().V1().Secrets(), + renewer, + ), + certmanager.Workers, + ) + webhookController := internal.NewController( + ControllerName, + NewController( + kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations(), + kubeInformer.Admissionregistration().V1().ValidatingWebhookConfigurations(), + kubeKyvernoInformer.Core().V1().Secrets(), + "kyverno-cleanup-policies", + "", + ), + Workers, + ) + cleanupController := internal.NewController( cleanup.ControllerName, cleanup.NewController( kubeClient, @@ -101,13 +131,15 @@ func main() { cleanup.Workers, ) // start informers and wait for cache sync - if !internal.StartInformersAndWaitForCacheSync(ctx, kyvernoInformer, kubeInformer) { + if !internal.StartInformersAndWaitForCacheSync(ctx, kyvernoInformer, kubeInformer, kubeKyvernoInformer) { logger.Error(errors.New("failed to wait for cache sync"), "failed to wait for cache sync") os.Exit(1) } // start leader controllers var wg sync.WaitGroup - controller.Run(ctx, logger.WithName("cleanup-controller"), &wg) + certController.Run(ctx, logger, &wg) + webhookController.Run(ctx, logger, &wg) + cleanupController.Run(ctx, logger, &wg) // wait all controllers shut down wg.Wait() },