mirror of
https://github.com/kyverno/kyverno.git
synced 2025-03-31 03:45:17 +00:00
adds lease objects for storing last-request-time and set-status annotations in deployment (#3447)
* funcs to patch last request time and status Signed-off-by: Mritunjay Sharma <mritunjaysharma394@gmail.com> * instead of patch, updating status Signed-off-by: Mritunjay Sharma <mritunjaysharma394@gmail.com> * added lease object appraoch Signed-off-by: Mritunjay Sharma <mritunjaysharma394@gmail.com> * cleanup Signed-off-by: Mritunjay Sharma <mritunjaysharma394@gmail.com> * attempt to solve panic issue Signed-off-by: Mritunjay Sharma <mritunjaysharma394@gmail.com> * fixes lease updates for both annotations Signed-off-by: Mritunjay Sharma <mritunjaysharma394@gmail.com> * minor cleanups in log messages Signed-off-by: Mritunjay Sharma <mritunjaysharma394@gmail.com> * clean up Signed-off-by: Mritunjay Sharma <mritunjaysharma394@gmail.com> * add object selector Signed-off-by: Mritunjay Sharma <mritunjaysharma394@gmail.com> * fixed leases and object selector Signed-off-by: Mritunjay Sharma <mritunjaysharma394@gmail.com>
This commit is contained in:
Mritunjay Kumar Sharma
GitHub
parent
d1bf3d4742
commit
e303dddf86
3 changed files with 99 additions and 78 deletions
pkg/webhookconfig
|
|
@ -1,6 +1,7 @@
|
|||
package webhookconfig
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
|
@ -10,9 +11,9 @@ import (
|
|||
"github.com/kyverno/kyverno/pkg/event"
|
||||
"github.com/kyverno/kyverno/pkg/tls"
|
||||
"github.com/pkg/errors"
|
||||
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
appsv1 "k8s.io/client-go/kubernetes/typed/apps/v1"
|
||||
coordinationv1 "k8s.io/client-go/kubernetes/typed/coordination/v1"
|
||||
)
|
||||
|
||||
//maxRetryCount defines the max deadline count
|
||||
|
|
@ -41,8 +42,8 @@ const (
|
|||
// not compare other details like the webhook settings.
|
||||
//
|
||||
type Monitor struct {
|
||||
// deployClient is used to manage Kyverno deployment
|
||||
deployClient appsv1.DeploymentInterface
|
||||
// leaseClient is used to manage Kyverno lease
|
||||
leaseClient coordinationv1.LeaseInterface
|
||||
|
||||
// lastSeenRequestTime records the timestamp
|
||||
// of the latest received admission request
|
||||
|
|
@ -55,7 +56,7 @@ type Monitor struct {
|
|||
// NewMonitor returns a new instance of webhook monitor
|
||||
func NewMonitor(kubeClient kubernetes.Interface, log logr.Logger) (*Monitor, error) {
|
||||
monitor := &Monitor{
|
||||
deployClient: kubeClient.AppsV1().Deployments(config.KyvernoNamespace),
|
||||
leaseClient: kubeClient.CoordinationV1().Leases(config.KyvernoNamespace),
|
||||
lastSeenRequestTime: time.Now(),
|
||||
log: log,
|
||||
}
|
||||
|
|
@ -83,7 +84,7 @@ func (t *Monitor) Run(register *Register, certRenewer *tls.CertRenewer, eventGen
|
|||
logger := t.log.WithName("webhookMonitor")
|
||||
|
||||
logger.V(3).Info("starting webhook monitor", "interval", idleCheckInterval.String())
|
||||
status := newStatusControl(t.deployClient, eventGen, logger.WithName("WebhookStatusControl"))
|
||||
status := newStatusControl(t.leaseClient, eventGen, logger.WithName("WebhookStatusControl"))
|
||||
|
||||
ticker := time.NewTicker(tickerInterval)
|
||||
defer ticker.Stop()
|
||||
|
|
@ -121,7 +122,7 @@ func (t *Monitor) Run(register *Register, certRenewer *tls.CertRenewer, eventGen
|
|||
}()
|
||||
|
||||
timeDiff := time.Since(t.Time())
|
||||
lastRequestTimeFromAnn := lastRequestTimeFromAnnotation(register, t.log.WithName("lastRequestTimeFromAnnotation"))
|
||||
lastRequestTimeFromAnn := lastRequestTimeFromAnnotation(t.leaseClient, t.log.WithName("lastRequestTimeFromAnnotation"))
|
||||
if lastRequestTimeFromAnn == nil {
|
||||
if err := status.UpdateLastRequestTimestmap(t.Time()); err != nil {
|
||||
logger.Error(err, "failed to annotate deployment for lastRequestTime")
|
||||
|
|
@ -200,27 +201,22 @@ func registerWebhookIfNotPresent(register *Register, logger logr.Logger) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
func lastRequestTimeFromAnnotation(register *Register, logger logr.Logger) *time.Time {
|
||||
_, deploy, err := register.GetKubePolicyDeployment()
|
||||
func lastRequestTimeFromAnnotation(leaseClient coordinationv1.LeaseInterface, logger logr.Logger) *time.Time {
|
||||
|
||||
lease, err := leaseClient.Get(context.TODO(), "kyverno", metav1.GetOptions{})
|
||||
if err != nil {
|
||||
logger.Info("unable to get Kyverno deployment", "reason", err.Error())
|
||||
return nil
|
||||
logger.Info("Lease 'kyverno' not found. Starting clean-up...")
|
||||
}
|
||||
|
||||
timeStamp, ok, err := unstructured.NestedString(deploy.UnstructuredContent(), "metadata", "annotations", annLastRequestTime)
|
||||
if err != nil {
|
||||
logger.Info("unable to get annotation", "reason", err.Error())
|
||||
return nil
|
||||
}
|
||||
|
||||
if !ok {
|
||||
timeStamp := lease.GetAnnotations()
|
||||
if timeStamp == nil {
|
||||
logger.Info("timestamp not set in the annotation, setting")
|
||||
return nil
|
||||
}
|
||||
|
||||
annTime, err := time.Parse(time.RFC3339, timeStamp)
|
||||
annTime, err := time.Parse(time.RFC3339, timeStamp[annLastRequestTime])
|
||||
if err != nil {
|
||||
logger.Error(err, "failed to parse timestamp annotation", "timeStamp", timeStamp)
|
||||
logger.Error(err, "failed to parse timestamp annotation", "timeStamp", timeStamp[annLastRequestTime])
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -279,10 +279,10 @@ func (wrc *Register) ValidateWebhookConfigurations(namespace, name string) error
|
|||
return json.Unmarshal([]byte(webhooks), &webhookCfgs)
|
||||
}
|
||||
|
||||
// cleanupKyvernoResource returns true if Kyverno deployment is terminating
|
||||
// cleanupKyvernoResource returns true if Kyverno lease is terminating
|
||||
func (wrc *Register) cleanupKyvernoResource() bool {
|
||||
logger := wrc.log.WithName("cleanupKyvernoResource")
|
||||
deploy, err := wrc.client.GetResource("", "Deployment", deployNamespace, deployName)
|
||||
deploy, err := wrc.client.GetResource("", "Deployment", config.KyvernoNamespace, config.KyvernoDeploymentName)
|
||||
if err != nil {
|
||||
if errorsapi.IsNotFound(err) {
|
||||
logger.Info("Kyverno deployment not found, cleanup Kyverno resources")
|
||||
|
|
@ -523,6 +523,26 @@ func getPolicyValidatingWebhookConfigurationName(serverIP string) string {
|
|||
}
|
||||
|
||||
func (wrc *Register) constructVerifyMutatingWebhookConfig(caData []byte) *admregapi.MutatingWebhookConfiguration {
|
||||
genWebHook := generateMutatingWebhook(
|
||||
config.VerifyMutatingWebhookName,
|
||||
config.VerifyMutatingWebhookServicePath,
|
||||
caData,
|
||||
true,
|
||||
wrc.timeoutSeconds,
|
||||
admregapi.Rule{
|
||||
Resources: []string{"leases"},
|
||||
APIGroups: []string{"coordination.k8s.io"},
|
||||
APIVersions: []string{"v1"},
|
||||
},
|
||||
[]admregapi.OperationType{admregapi.Update},
|
||||
admregapi.Ignore,
|
||||
)
|
||||
|
||||
genWebHook.ObjectSelector = &v1.LabelSelector{
|
||||
MatchLabels: map[string]string{
|
||||
"app.kubernetes.io/name": "kyverno",
|
||||
},
|
||||
}
|
||||
return &admregapi.MutatingWebhookConfiguration{
|
||||
ObjectMeta: v1.ObjectMeta{
|
||||
Name: config.VerifyMutatingWebhookConfigurationName,
|
||||
|
|
@ -531,20 +551,7 @@ func (wrc *Register) constructVerifyMutatingWebhookConfig(caData []byte) *admreg
|
|||
},
|
||||
},
|
||||
Webhooks: []admregapi.MutatingWebhook{
|
||||
generateMutatingWebhook(
|
||||
config.VerifyMutatingWebhookName,
|
||||
config.VerifyMutatingWebhookServicePath,
|
||||
caData,
|
||||
true,
|
||||
wrc.timeoutSeconds,
|
||||
admregapi.Rule{
|
||||
Resources: []string{"deployments"},
|
||||
APIGroups: []string{"apps"},
|
||||
APIVersions: []string{"v1"},
|
||||
},
|
||||
[]admregapi.OperationType{admregapi.Update},
|
||||
admregapi.Ignore,
|
||||
),
|
||||
genWebHook,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
|
@ -553,25 +560,31 @@ func (wrc *Register) constructDebugVerifyMutatingWebhookConfig(caData []byte) *a
|
|||
logger := wrc.log
|
||||
url := fmt.Sprintf("https://%s%s", wrc.serverIP, config.VerifyMutatingWebhookServicePath)
|
||||
logger.V(4).Info("Debug VerifyMutatingWebhookConfig is registered with url", "url", url)
|
||||
genWebHook := generateDebugMutatingWebhook(
|
||||
config.VerifyMutatingWebhookName,
|
||||
url,
|
||||
caData,
|
||||
true,
|
||||
wrc.timeoutSeconds,
|
||||
admregapi.Rule{
|
||||
Resources: []string{"leases"},
|
||||
APIGroups: []string{"coordination.k8s.io"},
|
||||
APIVersions: []string{"v1"},
|
||||
},
|
||||
[]admregapi.OperationType{admregapi.Update},
|
||||
admregapi.Ignore,
|
||||
)
|
||||
genWebHook.ObjectSelector = &v1.LabelSelector{
|
||||
MatchLabels: map[string]string{
|
||||
"app.kubernetes.io/name": "kyverno",
|
||||
},
|
||||
}
|
||||
return &admregapi.MutatingWebhookConfiguration{
|
||||
ObjectMeta: v1.ObjectMeta{
|
||||
Name: config.VerifyMutatingWebhookConfigurationDebugName,
|
||||
},
|
||||
Webhooks: []admregapi.MutatingWebhook{
|
||||
generateDebugMutatingWebhook(
|
||||
config.VerifyMutatingWebhookName,
|
||||
url,
|
||||
caData,
|
||||
true,
|
||||
wrc.timeoutSeconds,
|
||||
admregapi.Rule{
|
||||
Resources: []string{"deployments"},
|
||||
APIGroups: []string{"apps"},
|
||||
APIVersions: []string{"v1"},
|
||||
},
|
||||
[]admregapi.OperationType{admregapi.Update},
|
||||
admregapi.Ignore,
|
||||
),
|
||||
genWebHook,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,11 +10,11 @@ import (
|
|||
"github.com/kyverno/kyverno/pkg/event"
|
||||
"github.com/pkg/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
appsv1 "k8s.io/client-go/kubernetes/typed/apps/v1"
|
||||
coordinationv1 "k8s.io/client-go/kubernetes/typed/coordination/v1"
|
||||
)
|
||||
|
||||
var deployName string = config.KyvernoDeploymentName
|
||||
var deployNamespace string = config.KyvernoNamespace
|
||||
var leaseName string = "kyverno"
|
||||
var leaseNamespace string = config.KyvernoNamespace
|
||||
|
||||
const (
|
||||
annWebhookStatus string = "kyverno.io/webhookActive"
|
||||
|
|
@ -23,9 +23,9 @@ const (
|
|||
|
||||
//statusControl controls the webhook status
|
||||
type statusControl struct {
|
||||
deployClient appsv1.DeploymentInterface
|
||||
eventGen event.Interface
|
||||
log logr.Logger
|
||||
eventGen event.Interface
|
||||
log logr.Logger
|
||||
leaseClient coordinationv1.LeaseInterface
|
||||
}
|
||||
|
||||
//success ...
|
||||
|
|
@ -39,47 +39,48 @@ func (vc statusControl) failure() error {
|
|||
}
|
||||
|
||||
// NewStatusControl creates a new webhook status control
|
||||
func newStatusControl(deployClient appsv1.DeploymentInterface, eventGen event.Interface, log logr.Logger) *statusControl {
|
||||
func newStatusControl(leaseClient coordinationv1.LeaseInterface, eventGen event.Interface, log logr.Logger) *statusControl {
|
||||
return &statusControl{
|
||||
deployClient: deployClient,
|
||||
eventGen: eventGen,
|
||||
log: log,
|
||||
eventGen: eventGen,
|
||||
log: log,
|
||||
leaseClient: leaseClient,
|
||||
}
|
||||
}
|
||||
|
||||
func (vc statusControl) setStatus(status string) error {
|
||||
logger := vc.log.WithValues("name", deployName, "namespace", deployNamespace)
|
||||
logger := vc.log.WithValues("name", leaseName, "namespace", leaseNamespace)
|
||||
var ann map[string]string
|
||||
var err error
|
||||
deploy, err := vc.deployClient.Get(context.TODO(), deployName, metav1.GetOptions{})
|
||||
|
||||
lease, err := vc.leaseClient.Get(context.TODO(), "kyverno", metav1.GetOptions{})
|
||||
if err != nil {
|
||||
logger.Error(err, "failed to get deployment")
|
||||
vc.log.WithName("UpdateLastRequestTimestmap").Error(err, "Lease 'kyverno' not found. Starting clean-up...")
|
||||
return err
|
||||
}
|
||||
|
||||
ann = deploy.GetAnnotations()
|
||||
ann = lease.GetAnnotations()
|
||||
if ann == nil {
|
||||
ann = map[string]string{}
|
||||
ann[annWebhookStatus] = status
|
||||
}
|
||||
|
||||
deployStatus, ok := ann[annWebhookStatus]
|
||||
leaseStatus, ok := ann[annWebhookStatus]
|
||||
if ok {
|
||||
if deployStatus == status {
|
||||
if leaseStatus == status {
|
||||
logger.V(4).Info(fmt.Sprintf("annotation %s already set to '%s'", annWebhookStatus, status))
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
ann[annWebhookStatus] = status
|
||||
deploy.SetAnnotations(ann)
|
||||
lease.SetAnnotations(ann)
|
||||
|
||||
_, err = vc.deployClient.Update(context.TODO(), deploy, metav1.UpdateOptions{})
|
||||
_, err = vc.leaseClient.Update(context.TODO(), lease, metav1.UpdateOptions{})
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "key %s, val %s", annWebhookStatus, status)
|
||||
}
|
||||
|
||||
logger.Info("updated deployment annotation", "key", annWebhookStatus, "val", status)
|
||||
logger.Info("updated lease annotation", "key", annWebhookStatus, "val", status)
|
||||
|
||||
// create event on kyverno deployment
|
||||
createStatusUpdateEvent(status, vc.eventGen)
|
||||
|
|
@ -88,22 +89,31 @@ func (vc statusControl) setStatus(status string) error {
|
|||
|
||||
func createStatusUpdateEvent(status string, eventGen event.Interface) {
|
||||
e := event.Info{}
|
||||
e.Kind = "Deployment"
|
||||
e.Namespace = deployNamespace
|
||||
e.Name = deployName
|
||||
e.Kind = "Lease"
|
||||
e.Namespace = leaseNamespace
|
||||
e.Name = leaseName
|
||||
e.Reason = "Update"
|
||||
e.Message = fmt.Sprintf("admission control webhook active status changed to %s", status)
|
||||
eventGen.Add(e)
|
||||
}
|
||||
|
||||
func (vc statusControl) UpdateLastRequestTimestmap(new time.Time) error {
|
||||
deploy, err := vc.deployClient.Get(context.TODO(), deployName, metav1.GetOptions{})
|
||||
|
||||
lease, err := vc.leaseClient.Get(context.TODO(), leaseName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
vc.log.WithName("UpdateLastRequestTimestmap").Error(err, "failed to get deployment")
|
||||
vc.log.WithName("UpdateLastRequestTimestmap").Error(err, "Lease 'kyverno' not found. Starting clean-up...")
|
||||
return err
|
||||
}
|
||||
|
||||
annotation := deploy.GetAnnotations()
|
||||
//add label to lease
|
||||
label := lease.GetLabels()
|
||||
if len(label) == 0 {
|
||||
label = make(map[string]string)
|
||||
label["app.kubernetes.io/name"] = "kyverno"
|
||||
}
|
||||
lease.SetLabels(label)
|
||||
|
||||
annotation := lease.GetAnnotations()
|
||||
if annotation == nil {
|
||||
annotation = make(map[string]string)
|
||||
}
|
||||
|
|
@ -114,10 +124,12 @@ func (vc statusControl) UpdateLastRequestTimestmap(new time.Time) error {
|
|||
}
|
||||
|
||||
annotation[annLastRequestTime] = string(t)
|
||||
deploy.SetAnnotations(annotation)
|
||||
_, err = vc.deployClient.Update(context.TODO(), deploy, metav1.UpdateOptions{})
|
||||
lease.SetAnnotations(annotation)
|
||||
|
||||
//update annotations in lease
|
||||
_, err = vc.leaseClient.Update(context.TODO(), lease, metav1.UpdateOptions{})
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to update annotation %s for deployment %s in namespace %s", annLastRequestTime, deploy.GetName(), deploy.GetNamespace())
|
||||
return errors.Wrapf(err, "failed to update annotation %s for deployment %s in namespace %s", annLastRequestTime, lease.GetName(), lease.GetNamespace())
|
||||
}
|
||||
|
||||
return nil
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue