Merge pull request #2488 from NoSkillGirl/2208/e2e_test_cases_debugging

Added retry for processing generate requests

.github/workflows/e2e.yaml

@@ -82,7 +82,6 @@ jobs:
           kubectl get pods -n kyverno
           ${GITHUB_WORKSPACE}/scripts/verify-deployment.sh -n kyverno  kyverno
           sleep 20
-          
           echo ">>> Expose the Kyverno's service's metric server to the host"
           kubectl port-forward svc/kyverno-svc-metrics -n kyverno 8000:8000 &
           echo ">>> Run Kyverno e2e test"

pkg/generate/cleanup/cleanup.go

@@ -1,6 +1,8 @@
 package cleanup
 
 import (
+	"strconv"
+
 	"github.com/go-logr/logr"
 	kyverno "github.com/kyverno/kyverno/pkg/api/kyverno/v1"
 	dclient "github.com/kyverno/kyverno/pkg/dclient"
@@ -14,13 +16,30 @@ func (c *Controller) processGR(gr kyverno.GenerateRequest) error {
 
 	// 2- The trigger resource is deleted, then delete the generated resources
 	if !ownerResourceExists(logger, c.client, gr) {
-		if err := deleteGeneratedResources(logger, c.client, gr); err != nil {
-			return err
+		deleteGR := false
+		// check retry count in annotaion
+		grAnnotations := gr.Annotations
+		if val, ok := grAnnotations["generate.kyverno.io/retry-count"]; ok {
+			retryCount, err := strconv.ParseUint(val, 10, 32)
+			if err != nil {
+				logger.Error(err, "unable to convert retry-count")
+				return err
+			}
+
+			if retryCount >= 5 {
+				deleteGR = true
+			}
+		}
+
+		if deleteGR {
+			if err := deleteGeneratedResources(logger, c.client, gr); err != nil {
+				return err
+			}
+			// - trigger-resource is deleted
+			// - generated-resources are deleted
+			// - > Now delete the GenerateRequest CR
+			return c.control.Delete(gr.Name)
 		}
-		// - trigger-resource is deleted
-		// - generated-resources are deleted
-		// - > Now delete the GenerateRequest CR
-		return c.control.Delete(gr.Name)
 	}
 	return nil
 }

pkg/generate/cleanup/controller.go

@@ -170,6 +170,7 @@ func (c *Controller) deletePolicy(obj interface{}) {
 		}
 
 		for _, gr := range grs {
+			logger.V(4).Info("enqueue the gr for cleanup", "gr name", gr.Name)
 			c.addGR(gr)
 		}
 	}

pkg/generate/generate.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"fmt"
 	"reflect"
+	"strconv"
 	"strings"
 	"time"
 
@@ -38,7 +39,47 @@ func (c *Controller) processGR(gr *kyverno.GenerateRequest) error {
 	resource, err = getResource(c.client, gr.Spec.Resource, c.log)
 	if err != nil {
 		// Don't update status
-		logger.V(3).Info("resource does not exist or is pending creation, re-queueing", "details", err.Error())
+		// re-queueing the GR by updating the annotation
+		// retry - 5 times
+		logger.V(3).Info("resource does not exist or is pending creation, re-queueing", "details", err.Error(), "retry")
+		updateAnnotation := true
+		grAnnotations := gr.Annotations
+
+		if len(grAnnotations) == 0 {
+			grAnnotations = make(map[string]string)
+			grAnnotations["generate.kyverno.io/retry-count"] = "1"
+		} else {
+			if val, ok := grAnnotations["generate.kyverno.io/retry-count"]; ok {
+				sleepCountInt64, err := strconv.ParseUint(val, 10, 32)
+				if err != nil {
+					logger.Error(err, "unable to convert retry-count")
+					return err
+				}
+
+				sleepCountInt := int(sleepCountInt64) + 1
+				if sleepCountInt > 5 {
+					updateAnnotation = false
+				} else {
+					time.Sleep(time.Second * time.Duration(sleepCountInt))
+					incrementedCountString := strconv.Itoa(sleepCountInt)
+					grAnnotations["generate.kyverno.io/retry-count"] = incrementedCountString
+				}
+
+			} else {
+				time.Sleep(time.Second * 1)
+				grAnnotations["generate.kyverno.io/retry-count"] = "1"
+			}
+		}
+
+		if updateAnnotation {
+			gr.SetAnnotations(grAnnotations)
+			_, err := c.kyvernoClient.KyvernoV1().GenerateRequests(config.KyvernoNamespace).Update(contextdefault.TODO(), gr, metav1.UpdateOptions{})
+			if err != nil {
+				logger.Error(err, "failed to update annotation in generate request for the resource", "generate request", gr.Name)
+				return err
+			}
+		}
+
 		return err
 	}