create resource mutating webhook after verifying webhook is active

cmd/kyverno/main.go

@@ -74,10 +74,18 @@ func main() {
 		glog.Fatalf("Error creating kubernetes client: %v\n", err)
 	}
 
+	// KUBERNETES RESOURCES INFORMER
+	// watches namespace resource
+	// - cache resync time: 10 seconds
+	kubeInformer := kubeinformers.NewSharedInformerFactoryWithOptions(
+		kubeClient,
+		10*time.Second)
+
 	// WERBHOOK REGISTRATION CLIENT
 	webhookRegistrationClient := webhookconfig.NewWebhookRegistrationClient(
 		clientConfig,
 		client,
+		kubeInformer.Admissionregistration().V1beta1().MutatingWebhookConfigurations(),
 		serverIP,
 		int32(webhookTimeout))
 
@@ -90,13 +98,6 @@ func main() {
 		pclient,
 		10*time.Second)
 
-	// KUBERNETES RESOURCES INFORMER
-	// watches namespace resource
-	// - cache resync time: 10 seconds
-	kubeInformer := kubeinformers.NewSharedInformerFactoryWithOptions(
-		kubeClient,
-		10*time.Second)
-
 	// Configuration Data
 	// dyamically load the configuration from configMap
 	// - resource filters
@@ -183,9 +184,9 @@ func main() {
 	}
 
 	// WEBHOOK REGISTRATION
-	// - validationwebhookconfiguration (Policy)
+	// - mutating,validatingwebhookconfiguration (Policy)
-	// - mutatingwebhookconfiguration (All resources)
+	// - verifymutatingwebhookconfiguration (Kyverno Deployment)
-	// webhook confgiuration is also generated dynamically in the policy controller
+	// resource webhook confgiuration is generated dynamically in the webhook server and policy controller
 	// based on the policy resources created
 	if err = webhookRegistrationClient.Register(); err != nil {
 		glog.Fatalf("Failed registering Admission Webhooks: %v\n", err)

pkg/checker/checker.go

@@ -12,7 +12,11 @@ import (
 )
 
 //MaxRetryCount defines the max deadline count
-const MaxRetryCount int = 3
+const (
+	MaxRetryCount   int           = 3
+	DefaultDeadline time.Duration = 60 * time.Second
+	DefaultResync   time.Duration = 60 * time.Second
+)
 
 // LastReqTime
 type LastReqTime struct {
@@ -54,13 +58,13 @@ func checkIfPolicyWithMutateAndGenerateExists(pLister kyvernolister.ClusterPolic
 }
 
 //Run runs the checker and verify the resource update
-func (t *LastReqTime) Run(pLister kyvernolister.ClusterPolicyLister,eventGen event.Interface, client *dclient.Client, defaultResync time.Duration, deadline time.Duration, stopCh <-chan struct{}) {
+func (t *LastReqTime) Run(pLister kyvernolister.ClusterPolicyLister, eventGen event.Interface, client *dclient.Client, defaultResync time.Duration, deadline time.Duration, stopCh <-chan struct{}) {
 	glog.V(2).Infof("starting default resync for webhook checker with resync time %d", defaultResync)
 	maxDeadline := deadline * time.Duration(MaxRetryCount)
 	ticker := time.NewTicker(defaultResync)
 	var statuscontrol StatusInterface
 	/// interface to update and increment kyverno webhook status via annotations
-	statuscontrol = NewVerifyControl(client,eventGen)
+	statuscontrol = NewVerifyControl(client, eventGen)
 	// send the initial update status
 	if checkIfPolicyWithMutateAndGenerateExists(pLister) {
 		if err := statuscontrol.SuccessStatus(); err != nil {

pkg/policy/controller.go

@@ -475,7 +475,7 @@ func (pc *PolicyController) syncPolicy(key string) error {
 		return err
 	}
 
-	if err := pc.createResourceMutatingWebhookConfigurationIfRequired(*policy); err != nil {
+	if err := pc.webhookRegistrationClient.CreateResourceMutatingWebhookConfigurationIfRequired(*policy); err != nil {
 		glog.V(4).Infof("failed to create resource mutating webhook configurations, policies wont be applied on resources: %v", err)
 		glog.Errorln(err)
 	}

pkg/policy/webhookregistration.go

@@ -50,27 +50,6 @@ func (pc *PolicyController) removeResourceWebhookConfiguration() error {
 	return nil
 }
 
-func (pc *PolicyController) createResourceMutatingWebhookConfigurationIfRequired(policy kyverno.ClusterPolicy) error {
-	// if the policy contains mutating & validation rules and it config does not exist we create one
-	if policy.HasMutateOrValidate() {
-		// check cache
-		configName := pc.webhookRegistrationClient.GetResourceMutatingWebhookConfigName()
-		config, err := pc.mWebhookConfigLister.Get(configName)
-		if err != nil {
-			glog.V(4).Infof("failed to list mutating webhook configuration: %v", err)
-			return err
-		}
-		if config != nil {
-			// mutating webhoook configuration already exists
-			return nil
-		}
-		if err := pc.webhookRegistrationClient.CreateResourceMutatingWebhookConfiguration(); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
 func hasMutateOrValidatePolicies(policies []*kyverno.ClusterPolicy) bool {
 	for _, policy := range policies {
 		if (*policy).HasMutateOrValidate() {

pkg/webhookconfig/registration.go

@@ -2,14 +2,20 @@ package webhookconfig
 
 import (
 	"errors"
+	"fmt"
 	"sync"
 	"time"
 
 	"github.com/golang/glog"
+	kyverno "github.com/nirmata/kyverno/pkg/api/kyverno/v1"
+	"github.com/nirmata/kyverno/pkg/checker"
 	"github.com/nirmata/kyverno/pkg/config"
 	client "github.com/nirmata/kyverno/pkg/dclient"
 	admregapi "k8s.io/api/admissionregistration/v1beta1"
 	errorsapi "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/util/wait"
+	mconfiginformer "k8s.io/client-go/informers/admissionregistration/v1beta1"
+	mconfiglister "k8s.io/client-go/listers/admissionregistration/v1beta1"
 	rest "k8s.io/client-go/rest"
 )
 
@@ -22,22 +28,29 @@ const (
 type WebhookRegistrationClient struct {
 	client       *client.Client
 	clientConfig *rest.Config
+	// list/get mutatingwebhookconfigurations
+	mWebhookConfigLister mconfiglister.MutatingWebhookConfigurationLister
 	// serverIP should be used if running Kyverno out of clutser
 	serverIP       string
 	timeoutSeconds int32
+
+	LastReqTime *checker.LastReqTime
 }
 
 // NewWebhookRegistrationClient creates new WebhookRegistrationClient instance
 func NewWebhookRegistrationClient(
 	clientConfig *rest.Config,
 	client *client.Client,
+	mconfigwebhookinformer mconfiginformer.MutatingWebhookConfigurationInformer,
 	serverIP string,
 	webhookTimeout int32) *WebhookRegistrationClient {
 	return &WebhookRegistrationClient{
-		clientConfig:   clientConfig,
+		clientConfig:         clientConfig,
-		client:         client,
+		client:               client,
-		serverIP:       serverIP,
+		mWebhookConfigLister: mconfigwebhookinformer.Lister(),
-		timeoutSeconds: webhookTimeout,
+		serverIP:             serverIP,
+		timeoutSeconds:       webhookTimeout,
+		LastReqTime:          checker.NewLastReqTime(),
 	}
 }
 
@@ -82,10 +95,39 @@ func (wrc *WebhookRegistrationClient) RemoveWebhookConfigurations(cleanUp chan<-
 	close(cleanUp)
 }
 
+// CreateResourceMutatingWebhookConfigurationIfRequired creates a Mutatingwebhookconfiguration for all resource types
+// if there's no mutatingwebhookcfg for existing policy
+func (wrc *WebhookRegistrationClient) CreateResourceMutatingWebhookConfigurationIfRequired(policy kyverno.ClusterPolicy) error {
+	// if the policy contains mutating & validation rules and it config does not exist we create one
+	if policy.HasMutateOrValidate() {
+		// check cache
+		configName := wrc.GetResourceMutatingWebhookConfigName()
+		config, err := wrc.mWebhookConfigLister.Get(configName)
+		if !errorsapi.IsNotFound(err) {
+			glog.V(4).Infof("failed to list mutating webhook configuration: %v", err)
+			return err
+		}
+
+		if config != nil {
+			// mutating webhoook configuration already exists
+			return nil
+		}
+
+		if err := wrc.createResourceMutatingWebhookConfiguration(); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 //CreateResourceMutatingWebhookConfiguration create a Mutatingwebhookconfiguration resource for all resource type
 // used to forward request to kyverno webhooks to apply policeis
 // Mutationg webhook is be used for Mutating & Validating purpose
-func (wrc *WebhookRegistrationClient) CreateResourceMutatingWebhookConfiguration() error {
+func (wrc *WebhookRegistrationClient) createResourceMutatingWebhookConfiguration() error {
+	if err := wrc.waitForWebhookSuccess(); err != nil {
+		return fmt.Errorf("inactive webhook: %v", err)
+	}
+
 	var caData []byte
 	var config *admregapi.MutatingWebhookConfiguration
 
@@ -283,3 +325,28 @@ func (wrc *WebhookRegistrationClient) removePolicyValidatingWebhookConfiguration
 		glog.V(4).Infof("succesfully deleted policy webhook configuration %s", validatingConfig)
 	}
 }
+
+// waitForWebhookSuccess checks webhook status by checking the
+// LastReqTime set in webhook server, returns immediately if active
+// if not, then waits for webhook to be active, times out in 60s
+func (wrc *WebhookRegistrationClient) waitForWebhookSuccess() error {
+	// as default timeout for webhook checker is set to 60s
+	// resource webhook creation times out in a cycle of(60s) webhook checker
+	backoff := wait.Backoff{
+		Duration: time.Second,
+		Factor:   2,
+		Steps:    7,
+	}
+
+	count := 0
+	return wait.ExponentialBackoff(backoff, func() (bool, error) {
+		timeDiff := time.Since(wrc.LastReqTime.Time())
+		if timeDiff < checker.DefaultDeadline {
+			glog.Infof("Verified webhook status, creating resource mutating webhook configuration")
+			return true, nil
+		}
+		glog.V(4).Infof("webhook is inactive, retrying #%d", count)
+		count++
+		return false, nil
+	})
+}

pkg/webhooks/policyvalidation.go

@@ -37,11 +37,9 @@ func (ws *WebhookServer) handlePolicyValidation(request *v1beta1.AdmissionReques
 	}
 
 	if admissionResp.Allowed {
-		if policy.HasMutateOrValidate() {
+		// create mutating resource mutatingwebhookconfiguration if not present
-			// create mutating resource mutatingwebhookconfiguration if not present
+		if err := ws.webhookRegistrationClient.CreateResourceMutatingWebhookConfigurationIfRequired(*policy); err != nil {
-			if err := ws.webhookRegistrationClient.CreateResourceMutatingWebhookConfiguration(); err != nil {
+			glog.Error("failed to created resource mutating webhook configuration, policies wont be applied on the resource")
-				glog.Error("failed to created resource mutating webhook configuration, policies wont be applied on the resource")
-			}
 		}
 	}
 	return admissionResp

pkg/webhooks/server.go

@@ -110,7 +110,7 @@ func NewWebhookServer(
 		policyStatus:              policyStatus,
 		configHandler:             configHandler,
 		cleanUp:                   cleanUp,
-		lastReqTime:               checker.NewLastReqTime(),
+		lastReqTime:               webhookRegistrationClient.LastReqTime,
 		pvGenerator:               pvGenerator,
 		pMetaStore:                pMetaStore,
 	}
@@ -263,7 +263,7 @@ func (ws *WebhookServer) RunAsync(stopCh <-chan struct{}) {
 	// resync: 60 seconds
 	// deadline: 60 seconds (send request)
 	// max deadline: deadline*3 (set the deployment annotation as false)
-	go ws.lastReqTime.Run(ws.pLister, ws.eventGen, ws.client, 60*time.Second, 60*time.Second, stopCh)
+	go ws.lastReqTime.Run(ws.pLister, ws.eventGen, ws.client, checker.DefaultResync, checker.DefaultDeadline, stopCh)
 }
 
 // Stop TLS server and returns control after the server is shut down