Register webhooks only once service endpoint is ready (#1741)

* Register webhooks only once service endpoint is ready

Fixes #1740

Signed-off-by: Trey Dockendorf <tdockendorf@osc.edu>

* Wait for webhook to become ready in main loop

Signed-off-by: Trey Dockendorf <tdockendorf@osc.edu>

* Better error handling and logging around checking endpoint

Signed-off-by: Trey Dockendorf <tdockendorf@osc.edu>

* Log soft failure as info, remove redundant return

Signed-off-by: Trey Dockendorf <tdockendorf@osc.edu>

.github/workflows/e2e.yaml

@@ -73,6 +73,7 @@ jobs:
           echo ">>> Check kyverno"
           kubectl get pods -n kyverno
           ${GITHUB_WORKSPACE}/scripts/verify-deployment.sh -n kyverno  kyverno
+          sleep 20
           echo ">>> Run Kyverno e2e test"
           make test-e2e
 

cmd/kyverno/main.go

@@ -297,10 +297,27 @@ func main() {
 	}
 
 	// Register webhookCfg
-	if err = webhookCfg.Register(); err != nil {
-		setupLog.Error(err, "Failed to register admission control webhooks")
+	go func() {
+		registerTimeout := time.After(30 * time.Second)
+		registerTicker := time.NewTicker(time.Second)
+		defer registerTicker.Stop()
+		var err error
+	loop:
+		for {
+			select {
+			case <-registerTicker.C:
+				err = webhookCfg.Register()
+				if err != nil {
+					setupLog.Info("Failed to register admission control webhooks")
+				} else {
+					break loop
+				}
+			case <-registerTimeout:
+				setupLog.Error(err, "Timeout registering admission control webhooks")
 				os.Exit(1)
 			}
+		}
+	}()
 
 	openAPIController, err := openapi.NewOpenAPIController()
 	if err != nil {

pkg/webhookconfig/registration.go

@@ -13,9 +13,11 @@ import (
 	"github.com/kyverno/kyverno/pkg/resourcecache"
 	"github.com/kyverno/kyverno/pkg/tls"
 	admregapi "k8s.io/api/admissionregistration/v1beta1"
+	corev1 "k8s.io/api/core/v1"
 	errorsapi "k8s.io/apimachinery/pkg/api/errors"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
 	rest "k8s.io/client-go/rest"
 )
 
@@ -63,6 +65,9 @@ func (wrc *Register) Register() error {
 	if wrc.serverIP != "" {
 		logger.Info("Registering webhook", "url", fmt.Sprintf("https://%s", wrc.serverIP))
 	}
+	if err := wrc.checkEndpoint(); err != nil {
+		return err
+	}
 
 	wrc.removeWebhookConfigurations()
 
@@ -470,3 +475,29 @@ func (wrc *Register) removeSecrets() {
 		}
 	}
 }
+
+func (wrc *Register) checkEndpoint() error {
+	obj, err := wrc.client.GetResource("", "Endpoints", config.KyvernoNamespace, config.KyvernoServiceName)
+	if err != nil {
+		wrc.log.Error(err, "failed to get endpoint", "ns", config.KyvernoNamespace, "name", config.KyvernoServiceName)
+		return err
+	}
+	var endpoint corev1.Endpoints
+	err = runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), &endpoint)
+	if err != nil {
+		wrc.log.Error(err, "failed to convert endpoint from unstructured", "ns", config.KyvernoNamespace, "name", config.KyvernoServiceName)
+		return err
+	}
+	for _, subset := range endpoint.Subsets {
+		if len(subset.Addresses) == 0 {
+			continue
+		}
+		if subset.Addresses[0].IP != "" {
+			wrc.log.Info("Endpoint ready", "ns", config.KyvernoNamespace, "name", config.KyvernoServiceName)
+			return nil
+		}
+	}
+	err = fmt.Errorf("Endpoint not ready")
+	wrc.log.Error(err, "Endpoint not ready", "ns", config.KyvernoNamespace, "name", config.KyvernoServiceName)
+	return err
+}