handle discovery errors for metrics API group (#1494)

Signed-off-by: Jim Bugwadia <jim@nirmata.com>

cmd/initContainer/main.go

@@ -46,6 +46,7 @@ func main() {
 	if err := flag.Set("v", "2"); err != nil {
 		klog.Fatalf("failed to set log level: %v", err)
 	}
+
 	flag.Parse()
 
 	// os signal handler
@@ -109,9 +110,10 @@ func main() {
 			log.Log.Error(err, "failed to cleanup resource")
 		}
 	}
+
 	// if there is any failure then we fail process
 	if failure {
-		log.Log.Info("failed to cleanup webhook configurations")
+		log.Log.Info("failed to cleanup prior configurations")
 		os.Exit(1)
 	}
 }
@@ -131,6 +133,7 @@ func executeRequest(client *client.Client, req request) error {
 	case policyViolation, clusterPolicyViolation:
 		return removeViolationCRD(client)
 	}
+
 	return nil
 }
 
@@ -140,6 +143,7 @@ func createClientConfig(kubeconfig string) (*rest.Config, error) {
 		logger.Info("Using in-cluster configuration")
 		return rest.InClusterConfig()
 	}
+
 	logger.Info(fmt.Sprintf("Using configuration from '%s'", kubeconfig))
 	return clientcmd.BuildConfigFromFlags("", kubeconfig)
 }

cmd/kyverno/main.go

@@ -123,7 +123,7 @@ func main() {
 
 	// CRD CHECK
 	// - verify if Kyverno CRDs are available
-	if !utils.CRDInstalled(client.DiscoveryClient, log.Log) {
+	if !utils.CRDsInstalled(client.DiscoveryClient) {
 		setupLog.Error(fmt.Errorf("CRDs not installed"), "Failed to access Kyverno CRDs")
 		os.Exit(1)
 	}

pkg/dclient/client.go

@@ -147,6 +147,7 @@ func (c *Client) ListResource(apiVersion string, kind string, namespace string,
 	if lselector != nil {
 		options = meta.ListOptions{LabelSelector: helperv1.FormatLabelSelector(lselector)}
 	}
+
 	return c.getResourceInterface(apiVersion, kind, namespace).List(context.TODO(), options)
 }
 
@@ -325,31 +326,38 @@ func (c ServerPreferredResources) FindResource(apiVersion string, kind string) (
 }
 
 func (c ServerPreferredResources) findResource(apiVersion string, kind string) (*meta.APIResource, schema.GroupVersionResource, error) {
-	var serverresources []*meta.APIResourceList
+	var serverResources []*meta.APIResourceList
 	var err error
 	if apiVersion == "" {
-		serverresources, err = c.cachedClient.ServerPreferredResources()
+		serverResources, err = c.cachedClient.ServerPreferredResources()
 	} else {
-		serverresources, err = c.cachedClient.ServerResources()
+		_, serverResources, err = c.cachedClient.ServerGroupsAndResources()
 	}
 
 	if err != nil {
-		c.log.Error(err, "failed to get registered preferred resources")
+		if discovery.IsGroupDiscoveryFailedError(err) {
-		return nil, schema.GroupVersionResource{}, err
+			logDiscoveryErrors(err, c)
+		} else if isMetricsServerUnavailable(kind, err) {
+			c.log.V(3).Info("failed to find preferred resource version", "error", err.Error())
+		} else {
+			c.log.Error(err, "failed to find preferred resource version")
+			return nil, schema.GroupVersionResource{}, err
+		}
 	}
 
-	for _, serverresource := range serverresources {
+	for _, serverResource := range serverResources {
 
-		if apiVersion != "" && serverresource.GroupVersion != apiVersion {
+		if apiVersion != "" && serverResource.GroupVersion != apiVersion {
 			continue
 		}
-		for _, resource := range serverresource.APIResources {
+
+		for _, resource := range serverResource.APIResources {
 
 			// skip the resource names with "/", to avoid comparison with subresources
 			if resource.Kind == kind && !strings.Contains(resource.Name, "/") {
-				gv, err := schema.ParseGroupVersion(serverresource.GroupVersion)
+				gv, err := schema.ParseGroupVersion(serverResource.GroupVersion)
 				if err != nil {
-					c.log.Error(err, "failed to parse groupVersion", "groupVersion", serverresource.GroupVersion)
+					c.log.Error(err, "failed to parse groupVersion", "groupVersion", serverResource.GroupVersion)
 					return nil, schema.GroupVersionResource{}, err
 				}
 
@@ -360,3 +368,24 @@ func (c ServerPreferredResources) findResource(apiVersion string, kind string) (
 
 	return nil, schema.GroupVersionResource{}, fmt.Errorf("kind '%s' not found in apiVersion '%s'", kind, apiVersion)
 }
+
+func logDiscoveryErrors(err error, c ServerPreferredResources) {
+	discoveryError := err.(*discovery.ErrGroupDiscoveryFailed)
+	for gv, e := range discoveryError.Groups {
+		if gv.Group == "custom.metrics.k8s.io" || gv.Group == "metrics.k8s.io" {
+			// These error occur when Prometheus is installed as an external metrics server
+			// See: https://github.com/kyverno/kyverno/issues/1490
+			c.log.V(3).Info("failed to retrieve metrics API group", "gv", gv)
+			continue
+		}
+
+		c.log.Error(e, "failed to retrieve API group", "gv", gv)
+	}
+}
+
+func isMetricsServerUnavailable(kind string, err error) bool {
+	// error message is defined at:
+	// https://github.com/kubernetes/apimachinery/blob/2456ebdaba229616fab2161a615148884b46644b/pkg/api/errors/errors.go#L432
+	return strings.HasPrefix(kind, "metrics.k8s.io/") &&
+		strings.Contains(err.Error(), "the server is currently unable to handle the request")
+}

pkg/generate/generate_controller.go

@@ -3,8 +3,6 @@ package generate
 import (
 	"time"
 
-	"sigs.k8s.io/controller-runtime/pkg/log"
-
 	"github.com/go-logr/logr"
 	kyverno "github.com/kyverno/kyverno/pkg/api/kyverno/v1"
 	kyvernoclient "github.com/kyverno/kyverno/pkg/client/clientset/versioned"
@@ -276,8 +274,6 @@ func (c *Controller) Run(workers int, stopCh <-chan struct{}) {
 // worker runs a worker thread that just dequeues items, processes them, and marks them done.
 // It enforces that the syncHandler is never invoked concurrently with the same key.
 func (c *Controller) worker() {
-	log.Log.Info("starting new worker...")
-
 	for c.processNextWorkItem() {
 	}
 }

pkg/utils/util.go

@@ -2,10 +2,9 @@ package utils
 
 import (
 	"fmt"
-	"reflect"
 	"regexp"
+	"sigs.k8s.io/controller-runtime/pkg/log"
 	"strconv"
-	"strings"
 
 	"github.com/go-logr/logr"
 	client "github.com/kyverno/kyverno/pkg/dclient"
@@ -57,35 +56,30 @@ func NewKubeClient(config *rest.Config) (kubernetes.Interface, error) {
 	return kclient, nil
 }
 
-//CRDInstalled to check if the CRD is installed or not
+// CRDsInstalled checks if the Kyverno CRDs are installed or not
-func CRDInstalled(discovery client.IDiscovery, log logr.Logger) bool {
+func CRDsInstalled(discovery client.IDiscovery) bool {
-	logger := log.WithName("CRDInstalled")
-	check := func(kind string) bool {
-		gvr, err := discovery.GetGVRFromKind(kind)
-		if err != nil {
-			if isServerUnavailable(err) {
-				logger.Info("**WARNING** unable to check CRD status", "kind", kind, "error", err.Error())
-				return true
-			}
-
-			logger.Error(err, "failed to check CRD status", "kind", kind)
-			return false
-		}
-
-		if reflect.DeepEqual(gvr, schema.GroupVersionResource{}) {
-			logger.Info("CRD not installed", "kind", kind)
-			return false
-		}
-		logger.Info("CRD found", "kind", kind)
-		return true
-	}
-
 	kyvernoCRDs := []string{"ClusterPolicy", "ClusterPolicyReport", "PolicyReport", "ClusterReportChangeRequest", "ReportChangeRequest"}
 	for _, crd := range kyvernoCRDs {
-		if !check(crd) {
+		if !isCRDInstalled(discovery, crd) {
 			return false
 		}
 	}
+
+	return true
+}
+
+func isCRDInstalled(discoveryClient client.IDiscovery, kind string) bool {
+	gvr, err := discoveryClient.GetGVRFromKind(kind)
+	if gvr.Empty() {
+		if err == nil {
+			err = fmt.Errorf("not found")
+		}
+
+		log.Log.Error(err, "failed to retrieve CRD", "kind", kind)
+		return false
+	}
+
+	log.Log.Info("CRD found", "gvr", gvr.String())
 	return true
 }
 
@@ -193,9 +187,3 @@ func SliceContains(slice []string, values ...string) bool {
 
 	return false
 }
-
-func isServerUnavailable(err error) bool {
-	// error message -
-	// https://github.com/kubernetes/apimachinery/blob/2456ebdaba229616fab2161a615148884b46644b/pkg/api/errors/errors.go#L432
-	return strings.Contains(err.Error(), "the server is currently unable to handle the request")
-}