refactor: make cert manager a real controller (#4792)

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

Co-authored-by: Prateek Pandey <prateek.pandey@nirmata.com>

cmd/kyverno/main.go

@@ -639,11 +639,6 @@ func main() {
 			logger := logger.WithName("leader")
 			// when losing the lead we just terminate the pod
 			defer signalCancel()
-			// init tls secret
-			if err := certRenewer.InitTLSPemPair(); err != nil {
-				logger.Error(err, "tls initialization error")
-				os.Exit(1)
-			}
 			// validate config
 			if err := webhookCfg.ValidateWebhookConfigurations(config.KyvernoNamespace(), config.KyvernoConfigMapName()); err != nil {
 				logger.Error(err, "invalid format of the Kyverno init ConfigMap, please correct the format of 'data.webhooks'")
@@ -682,6 +677,10 @@ func main() {
 				// TODO: shall we just exit ?
 				logger.Error(errors.New("failed to wait for cache sync"), "failed to wait for cache sync")
 			}
+			// start leader controllers
+			for _, controller := range leaderControllers {
+				go controller.run(signalCtx, logger.WithName("controllers"))
+			}
 			// bootstrap
 			if autoUpdateWebhooks {
 				go webhookCfg.UpdateWebhookConfigurations(configuration)
@@ -692,10 +691,6 @@ func main() {
 				os.Exit(1)
 			}
 			webhookCfg.UpdateWebhookChan <- true
-			// start leader controllers
-			for _, controller := range leaderControllers {
-				go controller.run(signalCtx, logger.WithName("controllers"))
-			}
 			// wait until we loose the lead (or signal context is canceled)
 			<-ctx.Done()
 		},

pkg/controllers/certmanager/controller.go

@@ -2,91 +2,111 @@ package certmanager
 
 import (
 	"context"
-	"os"
-	"reflect"
 	"time"
 
+	"github.com/go-logr/logr"
 	"github.com/kyverno/kyverno/pkg/common"
 	"github.com/kyverno/kyverno/pkg/config"
 	"github.com/kyverno/kyverno/pkg/controllers"
 	"github.com/kyverno/kyverno/pkg/tls"
+	controllerutils "github.com/kyverno/kyverno/pkg/utils/controller"
 	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
 	corev1informers "k8s.io/client-go/informers/core/v1"
 	corev1listers "k8s.io/client-go/listers/core/v1"
-	"k8s.io/client-go/tools/cache"
+	"k8s.io/client-go/util/workqueue"
 )
 
 const (
 	// Workers is the number of workers for this controller
 	Workers        = 1
 	ControllerName = "certmanager-controller"
+	maxRetries     = 10
 )
 
 type controller struct {
-	renewer      *tls.CertRenewer
+	renewer *tls.CertRenewer
+
+	// listers
 	secretLister corev1listers.SecretLister
-	secretQueue  chan bool
+
+	// queue
+	queue         workqueue.RateLimitingInterface
+	secretEnqueue controllerutils.EnqueueFunc
 }
 
 func NewController(secretInformer corev1informers.SecretInformer, certRenewer *tls.CertRenewer) controllers.Controller {
-	manager := &controller{
+	queue := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), ControllerName)
-		renewer:      certRenewer,
+	c := controller{
-		secretLister: secretInformer.Lister(),
+		renewer:       certRenewer,
-		secretQueue:  make(chan bool, 1),
+		secretLister:  secretInformer.Lister(),
+		queue:         queue,
+		secretEnqueue: controllerutils.AddDefaultEventHandlers(logger.V(3), secretInformer.Informer(), queue),
 	}
-	secretInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
+	return &c
-		AddFunc:    manager.addSecretFunc,
-		UpdateFunc: manager.updateSecretFunc,
-	})
-	return manager
 }
 
-func (m *controller) Run(ctx context.Context, workers int) {
+func (c *controller) Run(ctx context.Context, workers int) {
-	logger.Info("start managing certificate")
+	go c.ticker(ctx)
+	// we need to enqueue our secrets in case they don't exist yet in the cluster
+	// this way we ensure the reconcile happens (hence renewal/creation)
+	if err := c.secretEnqueue(&corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: config.KyvernoNamespace(),
+			Name:      tls.GenerateTLSPairSecretName(),
+		},
+	}); err != nil {
+		logger.Error(err, "failed to enqueue secret", "name", tls.GenerateTLSPairSecretName())
+	}
+	if err := c.secretEnqueue(&corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: config.KyvernoNamespace(),
+			Name:      tls.GenerateRootCASecretName(),
+		},
+	}); err != nil {
+		logger.Error(err, "failed to enqueue CA secret", "name", tls.GenerateRootCASecretName())
+	}
+	controllerutils.Run(ctx, ControllerName, logger.V(3), c.queue, workers, maxRetries, c.reconcile)
+}
+
+func (c *controller) reconcile(ctx context.Context, logger logr.Logger, key, namespace, name string) error {
+	if namespace != config.KyvernoNamespace() {
+		return nil
+	}
+	if name != tls.GenerateTLSPairSecretName() && name != tls.GenerateRootCASecretName() {
+		return nil
+	}
+	return c.renewCertificates()
+}
+
+func (c *controller) ticker(ctx context.Context) {
 	certsRenewalTicker := time.NewTicker(tls.CertRenewalInterval)
 	defer certsRenewalTicker.Stop()
 	for {
 		select {
 		case <-certsRenewalTicker.C:
-			if err := m.renewCertificates(); err != nil {
+			list, err := c.secretLister.List(labels.Everything())
-				logger.Error(err, "unable to renew certificates, force restarting")
+			if err == nil {
-				os.Exit(1)
+				for _, secret := range list {
-			}
+					if err := c.secretEnqueue(secret); err != nil {
-		case <-m.secretQueue:
+						logger.Error(err, "failed to enqueue secret", "name", secret.Name)
-			if err := m.renewCertificates(); err != nil {
+					}
-				logger.Error(err, "unable to renew certificates, force restarting")
+				}
-				os.Exit(1)
+			} else {
+				logger.Error(err, "falied to list secrets")
 			}
 		case <-ctx.Done():
-			logger.V(2).Info("stopping cert renewer")
 			return
 		}
 	}
 }
 
-func (m *controller) addSecretFunc(obj interface{}) {
+func (c *controller) renewCertificates() error {
-	secret := obj.(*corev1.Secret)
+	if err := common.RetryFunc(time.Second, 5*time.Second, c.renewer.RenewCA, "failed to renew CA", logger)(); err != nil {
-	if secret.GetNamespace() == config.KyvernoNamespace() && secret.GetName() == tls.GenerateTLSPairSecretName() {
-		m.secretQueue <- true
-	}
-}
-
-func (m *controller) updateSecretFunc(oldObj interface{}, newObj interface{}) {
-	old := oldObj.(*corev1.Secret)
-	new := newObj.(*corev1.Secret)
-	if new.GetNamespace() == config.KyvernoNamespace() && new.GetName() == tls.GenerateTLSPairSecretName() {
-		if !reflect.DeepEqual(old.DeepCopy().Data, new.DeepCopy().Data) {
-			m.secretQueue <- true
-			logger.V(4).Info("secret updated, reconciling webhook configurations")
-		}
-	}
-}
-
-func (m *controller) renewCertificates() error {
-	if err := common.RetryFunc(time.Second, 5*time.Second, m.renewer.RenewCA, "failed to renew CA", logger)(); err != nil {
 		return err
 	}
-	if err := common.RetryFunc(time.Second, 5*time.Second, m.renewer.RenewTLS, "failed to renew TLS", logger)(); err != nil {
+	if err := common.RetryFunc(time.Second, 5*time.Second, c.renewer.RenewTLS, "failed to renew TLS", logger)(); err != nil {
 		return err
 	}
 	return nil