refactor: health check system (#5176)

* refactor: health check system

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* filter

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* fix

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* fix

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>
Co-authored-by: Vyankatesh Kudtarkar <vyankateshkd@gmail.com>

cmd/kyverno/main.go

@@ -621,7 +621,6 @@ func main() {
 	runtime := runtimeutils.NewRuntime(
 		logger.WithName("runtime-checks"),
 		serverIP,
-		kubeKyvernoInformer.Coordination().V1().Leases(),
 		kubeKyvernoInformer.Apps().V1().Deployments(),
 		certRenewer,
 	)

pkg/controllers/webhook/controller.go

@@ -44,10 +44,10 @@ const (
 	ControllerName            = "webhook-controller"
 	DefaultWebhookTimeout     = 10
 	AnnotationLastRequestTime = "kyverno.io/last-request-time"
-	IdleDeadline              = tickerInterval * 5
+	IdleDeadline              = tickerInterval * 10
 	maxRetries                = 10
 	managedByLabel            = "webhook.kyverno.io/managed-by"
-	tickerInterval            = 30 * time.Second
+	tickerInterval            = 10 * time.Second
 )
 
 var (
@@ -74,7 +74,7 @@ type controller struct {
 	secretClient    controllerutils.GetClient[*corev1.Secret]
 	mwcClient       controllerutils.ObjectClient[*admissionregistrationv1.MutatingWebhookConfiguration]
 	vwcClient       controllerutils.ObjectClient[*admissionregistrationv1.ValidatingWebhookConfiguration]
-	leaseClient     controllerutils.UpdateClient[*coordinationv1.Lease]
+	leaseClient     controllerutils.ObjectClient[*coordinationv1.Lease]
 	kyvernoClient   versioned.Interface
 
 	// listers
@@ -106,7 +106,7 @@ func NewController(
 	secretClient controllerutils.GetClient[*corev1.Secret],
 	mwcClient controllerutils.ObjectClient[*admissionregistrationv1.MutatingWebhookConfiguration],
 	vwcClient controllerutils.ObjectClient[*admissionregistrationv1.ValidatingWebhookConfiguration],
-	leaseClient controllerutils.UpdateClient[*coordinationv1.Lease],
+	leaseClient controllerutils.ObjectClient[*coordinationv1.Lease],
 	kyvernoClient versioned.Interface,
 	mwcInformer admissionregistrationv1informers.MutatingWebhookConfigurationInformer,
 	vwcInformer admissionregistrationv1informers.ValidatingWebhookConfigurationInformer,
@@ -216,24 +216,32 @@ func (c *controller) watchdog(ctx context.Context, logger logr.Logger) {
 		case <-ticker.C:
 			lease, err := c.getLease()
 			if err != nil {
-				logger.Error(err, "failed to get lease")
+				if apierrors.IsNotFound(err) {
+					_, err = c.leaseClient.Create(ctx, &coordinationv1.Lease{
+						ObjectMeta: metav1.ObjectMeta{
+							Name:      "kyverno-health",
+							Namespace: config.KyvernoNamespace(),
+							Labels: map[string]string{
+								"app.kubernetes.io/name": kyvernov1.ValueKyvernoApp,
+							},
+							Annotations: map[string]string{
+								AnnotationLastRequestTime: time.Now().Format(time.RFC3339),
+							},
+						},
+					}, metav1.CreateOptions{})
+					if err != nil {
+						logger.Error(err, "failed to create lease")
+					}
+				} else {
+					logger.Error(err, "failed to get lease")
+				}
 			} else {
-				if _, err := controllerutils.Update(
-					ctx,
-					lease,
-					c.leaseClient,
-					func(lease *coordinationv1.Lease) error {
-						if lease.Annotations == nil {
-							lease.Annotations = map[string]string{}
-						}
-						lease.Annotations[AnnotationLastRequestTime] = time.Now().Format(time.RFC3339)
-						if lease.Labels == nil {
-							lease.Labels = map[string]string{}
-						}
-						lease.Labels["app.kubernetes.io/name"] = kyvernov1.ValueKyvernoApp
-						return nil
-					},
-				); err != nil {
+				lease := lease.DeepCopy()
+				lease.Labels = map[string]string{
+					"app.kubernetes.io/name": kyvernov1.ValueKyvernoApp,
+				}
+				_, err = c.leaseClient.Update(ctx, lease, metav1.UpdateOptions{})
+				if err != nil {
 					logger.Error(err, "failed to update lease")
 				}
 			}
@@ -811,7 +819,7 @@ func (c *controller) getAllPolicies() ([]kyvernov1.PolicyInterface, error) {
 }
 
 func (c *controller) getLease() (*coordinationv1.Lease, error) {
-	return c.leaseLister.Leases(config.KyvernoNamespace()).Get("kyverno")
+	return c.leaseLister.Leases(config.KyvernoNamespace()).Get("kyverno-health")
 }
 
 // mergeWebhook merges the matching kinds of the policy to webhook.rule

pkg/utils/runtime/utils.go

@@ -1,24 +1,13 @@
 package runtime
 
 import (
-	"time"
-
 	"github.com/go-logr/logr"
 	"github.com/kyverno/kyverno/pkg/config"
 	"github.com/kyverno/kyverno/pkg/tls"
 	appsv1 "k8s.io/api/apps/v1"
-	coordinationv1 "k8s.io/api/coordination/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	appsv1informers "k8s.io/client-go/informers/apps/v1"
-	coordinationv1informers "k8s.io/client-go/informers/coordination/v1"
 	appsv1listers "k8s.io/client-go/listers/apps/v1"
-	coordinationv1listers "k8s.io/client-go/listers/coordination/v1"
-)
-
-const (
-	AnnotationLastRequestTime = "kyverno.io/last-request-time"
-	IdleDeadline              = tickerInterval * 5
-	tickerInterval            = 30 * time.Second
 )
 
 type Runtime interface {
@@ -31,7 +20,6 @@ type Runtime interface {
 
 type runtime struct {
 	serverIP         string
-	leaseLister      coordinationv1listers.LeaseLister
 	deploymentLister appsv1listers.DeploymentLister
 	certValidator    tls.CertValidator
 	logger           logr.Logger
@@ -40,14 +28,12 @@ type runtime struct {
 func NewRuntime(
 	logger logr.Logger,
 	serverIP string,
-	leaseInformer coordinationv1informers.LeaseInformer,
 	deploymentInformer appsv1informers.DeploymentInformer,
 	certValidator tls.CertValidator,
 ) Runtime {
 	return &runtime{
 		logger:           logger,
 		serverIP:         serverIP,
-		leaseLister:      leaseInformer.Lister(),
 		deploymentLister: deploymentInformer.Lister(),
 		certValidator:    certValidator,
 	}
@@ -58,11 +44,11 @@ func (c *runtime) IsDebug() bool {
 }
 
 func (c *runtime) IsLive() bool {
-	return c.check()
+	return true
 }
 
 func (c *runtime) IsReady() bool {
-	return c.check() && c.validateCertificates()
+	return c.validateCertificates()
 }
 
 func (c *runtime) IsRollingUpdate() bool {
@@ -103,31 +89,10 @@ func (c *runtime) IsGoingDown() bool {
 	return false
 }
 
-func (c *runtime) getLease() (*coordinationv1.Lease, error) {
-	return c.leaseLister.Leases(config.KyvernoNamespace()).Get("kyverno")
-}
-
 func (c *runtime) getDeployment() (*appsv1.Deployment, error) {
 	return c.deploymentLister.Deployments(config.KyvernoNamespace()).Get(config.KyvernoDeploymentName())
 }
 
-func (c *runtime) check() bool {
-	lease, err := c.getLease()
-	if err != nil {
-		c.logger.Error(err, "failed to get lease")
-		return false
-	}
-	annotations := lease.GetAnnotations()
-	if annotations == nil {
-		return false
-	}
-	annTime, err := time.Parse(time.RFC3339, annotations[AnnotationLastRequestTime])
-	if err != nil {
-		return false
-	}
-	return time.Now().Before(annTime.Add(IdleDeadline))
-}
-
 func (c *runtime) validateCertificates() bool {
 	validity, err := c.certValidator.ValidateCert()
 	if err != nil {

pkg/webhooks/handlers/admission.go

@@ -11,6 +11,7 @@ import (
 	"github.com/kyverno/kyverno/pkg/config"
 	"github.com/kyverno/kyverno/pkg/tracing"
 	admissionutils "github.com/kyverno/kyverno/pkg/utils/admission"
+	jsonutils "github.com/kyverno/kyverno/pkg/utils/json"
 	"go.opentelemetry.io/otel/attribute"
 	admissionv1 "k8s.io/api/admission/v1"
 )
@@ -101,6 +102,15 @@ func Filter(c config.Configuration, inner AdmissionHandler) AdmissionHandler {
 
 func Verify() AdmissionHandler {
 	return func(logger logr.Logger, request *admissionv1.AdmissionRequest, startTime time.Time) *admissionv1.AdmissionResponse {
-		return admissionutils.Response(true)
+		if request.Name != "kyverno-health" || request.Namespace != config.KyvernoNamespace() {
+			return admissionutils.ResponseSuccess()
+		}
+		patch := jsonutils.NewPatchOperation("/metadata/annotations/"+"kyverno.io~1last-request-time", "replace", time.Now().Format(time.RFC3339))
+		bytes, err := patch.ToPatchBytes()
+		if err != nil {
+			logger.Error(err, "failed to build patch bytes")
+			return admissionutils.ResponseFailure(err.Error())
+		}
+		return admissionutils.ResponseSuccessWithPatch(bytes)
 	}
 }

pkg/webhooks/server.go

@@ -239,6 +239,7 @@ func (s *server) cleanup(ctx context.Context) {
 			}
 		}
 		deleteLease("kyvernopre-lock")
+		deleteLease("kyverno-health")
 		deleteVwc(config.ValidatingWebhookConfigurationName)
 		deleteVwc(config.PolicyValidatingWebhookConfigurationName)
 		deleteMwc(config.MutatingWebhookConfigurationName)