527 getting generate stats from sync

pkg/generate/controller.go

@@ -58,6 +58,8 @@ type Controller struct {
 	//TODO: list of generic informers
 	// only support Namespaces for re-evalutation on resource updates
 	nsInformer informers.GenericInformer
+
+	policyStatus *policyStatus.Sync
 }
 
 //NewController returns an instance of the Generate-Request Controller
@@ -80,8 +82,9 @@ func NewController(
 		// as we dont want a deleted GR to be re-queue
 		queue:           workqueue.NewNamedRateLimitingQueue(workqueue.NewItemExponentialFailureRateLimiter(1, 30), "generate-request"),
 		dynamicInformer: dynamicInformer,
+		policyStatus:    policyStatus,
 	}
-	c.statusControl = StatusControl{client: kyvernoclient, policyStatus: policyStatus}
+	c.statusControl = StatusControl{client: kyvernoclient}
 
 	pInformer.Informer().AddEventHandlerWithResyncPeriod(cache.ResourceEventHandlerFuncs{
 		UpdateFunc: c.updatePolicy, // We only handle updates to policy

pkg/generate/generate.go

@@ -3,6 +3,9 @@ package generate
 import (
 	"encoding/json"
 	"fmt"
+	"time"
+
+	"github.com/nirmata/kyverno/pkg/policyStatus"
 
 	"github.com/golang/glog"
 	kyverno "github.com/nirmata/kyverno/pkg/api/kyverno/v1"
@@ -80,7 +83,7 @@ func (c *Controller) applyGenerate(resource unstructured.Unstructured, gr kyvern
 	}
 
 	// Apply the generate rule on resource
-	return applyGeneratePolicy(c.client, policyContext)
+	return c.applyGeneratePolicy(policyContext, gr)
 }
 
 func updateStatus(statusControl StatusControlInterface, gr kyverno.GenerateRequest, err error, genResources []kyverno.ResourceSpec) error {
@@ -92,7 +95,7 @@ func updateStatus(statusControl StatusControlInterface, gr kyverno.GenerateReque
 	return statusControl.Success(gr, genResources)
 }
 
-func applyGeneratePolicy(client *dclient.Client, policyContext engine.PolicyContext) ([]kyverno.ResourceSpec, error) {
+func (c *Controller) applyGeneratePolicy(policyContext engine.PolicyContext, gr kyverno.GenerateRequest) ([]kyverno.ResourceSpec, error) {
 	// List of generatedResources
 	var genResources []kyverno.ResourceSpec
 	// Get the response as the actions to be performed on the resource
@@ -107,20 +110,77 @@ func applyGeneratePolicy(client *dclient.Client, policyContext engine.PolicyCont
 		return rcreationTime.Before(&pcreationTime)
 	}()
 
+	ruleNameToProcessingTime := make(map[string]time.Duration)
 	for _, rule := range policy.Spec.Rules {
 		if !rule.HasGenerate() {
 			continue
 		}
-		genResource, err := applyRule(client, rule, resource, ctx, processExisting)
+
+		startTime := time.Now()
+		genResource, err := applyRule(c.client, rule, resource, ctx, processExisting)
 		if err != nil {
 			return nil, err
 		}
+
+		ruleNameToProcessingTime[rule.Name] = time.Since(startTime)
 		genResources = append(genResources, genResource)
 	}
 
+	if gr.Status.State == "" {
+		go func() {
+			c.policyStatus.Listener <- &generateSyncStats{
+				policyName:               policy.Name,
+				ruleNameToProcessingTime: ruleNameToProcessingTime,
+			}
+		}()
+	}
+
 	return genResources, nil
 }
 
+type generateSyncStats struct {
+	policyName               string
+	ruleNameToProcessingTime map[string]time.Duration
+}
+
+func (vc *generateSyncStats) UpdateStatus(s *policyStatus.Sync) {
+	s.Cache.Mutex.Lock()
+	status, exist := s.Cache.Data[vc.policyName]
+	if !exist {
+		policy, _ := s.PolicyStore.Get(vc.policyName)
+		if policy != nil {
+			status = policy.Status
+		}
+	}
+
+	for i := range status.Rules {
+		if executionTime, exist := vc.ruleNameToProcessingTime[status.Rules[i].Name]; exist {
+			status.ResourcesGeneratedCount += 1
+			status.Rules[i].ResourcesGeneratedCount += 1
+			averageOver := int64(status.Rules[i].AppliedCount + status.Rules[i].FailedCount)
+			status.Rules[i].ExecutionTime = updateGenerateExecutionTime(
+				executionTime,
+				status.Rules[i].ExecutionTime,
+				averageOver,
+			).String()
+		}
+	}
+
+	s.Cache.Data[vc.policyName] = status
+	s.Cache.Mutex.Unlock()
+}
+
+func updateGenerateExecutionTime(newTime time.Duration, oldAverageTimeString string, averageOver int64) time.Duration {
+	if averageOver == 0 {
+		return newTime
+	}
+	oldAverageExecutionTime, _ := time.ParseDuration(oldAverageTimeString)
+	numerator := (oldAverageExecutionTime.Nanoseconds() * averageOver) + newTime.Nanoseconds()
+	denominator := averageOver
+	newAverageTimeInNanoSeconds := numerator / denominator
+	return time.Duration(newAverageTimeInNanoSeconds) * time.Nanosecond
+}
+
 func applyRule(client *dclient.Client, rule kyverno.Rule, resource unstructured.Unstructured, ctx context.EvalInterface, processExisting bool) (kyverno.ResourceSpec, error) {
 	var rdata map[string]interface{}
 	var err error

pkg/generate/policyStatus_test.go

@@ -45,7 +45,7 @@ func Test_Stats(t *testing.T) {
 	s := policyStatus.NewSync(nil, &dummyStore{})
 
 	for _, generateCountStat := range testCase.generatedCountStats {
-		receiver := &generatedResourceCount{
+		receiver := &generateSyncStats{
 			generateRequest: generateCountStat,
 		}
 		receiver.UpdateStatus(s)

pkg/generate/status.go

@@ -4,7 +4,6 @@ import (
 	"github.com/golang/glog"
 	kyverno "github.com/nirmata/kyverno/pkg/api/kyverno/v1"
 	kyvernoclient "github.com/nirmata/kyverno/pkg/client/clientset/versioned"
-	"github.com/nirmata/kyverno/pkg/policyStatus"
 )
 
 //StatusControlInterface provides interface to update status subresource
@@ -15,8 +14,7 @@ type StatusControlInterface interface {
 
 // StatusControl is default implementaation of GRStatusControlInterface
 type StatusControl struct {
-	client       kyvernoclient.Interface
-	policyStatus *policyStatus.Sync
+	client kyvernoclient.Interface
 }
 
 //Failed sets gr status.state to failed with message
@@ -36,19 +34,11 @@ func (sc StatusControl) Failed(gr kyverno.GenerateRequest, message string, genRe
 
 // Success sets the gr status.state to completed and clears message
 func (sc StatusControl) Success(gr kyverno.GenerateRequest, genResources []kyverno.ResourceSpec) error {
-	oldState := gr.Status.State
-
 	gr.Status.State = kyverno.Completed
 	gr.Status.Message = ""
 	// Update Generated Resources
 	gr.Status.GeneratedResources = genResources
 
-	if oldState != kyverno.Completed {
-		go func() {
-			sc.policyStatus.Listener <- updatePolicyStatusWithGeneratedResourceCount(gr)
-		}()
-	}
-
 	_, err := sc.client.KyvernoV1().GenerateRequests("kyverno").UpdateStatus(&gr)
 	if err != nil {
 		glog.V(4).Infof("FAILED: updated gr %s status to %s", gr.Name, string(kyverno.Completed))
@@ -57,29 +47,3 @@ func (sc StatusControl) Success(gr kyverno.GenerateRequest, genResources []kyver
 	glog.V(4).Infof("updated gr %s status to %s", gr.Name, string(kyverno.Completed))
 	return nil
 }
-
-type generatedResourceCount struct {
-	generateRequest kyverno.GenerateRequest
-}
-
-func updatePolicyStatusWithGeneratedResourceCount(generateRequest kyverno.GenerateRequest) *generatedResourceCount {
-	return &generatedResourceCount{
-		generateRequest: generateRequest,
-	}
-}
-
-func (vc *generatedResourceCount) UpdateStatus(s *policyStatus.Sync) {
-	s.Cache.Mutex.Lock()
-	status, exist := s.Cache.Data[vc.generateRequest.Spec.Policy]
-	if !exist {
-		policy, _ := s.PolicyStore.Get(vc.generateRequest.Spec.Policy)
-		if policy != nil {
-			status = policy.Status
-		}
-	}
-
-	status.ResourcesGeneratedCount += len(vc.generateRequest.Status.GeneratedResources)
-
-	s.Cache.Data[vc.generateRequest.Spec.Policy] = status
-	s.Cache.Mutex.Unlock()
-}

pkg/policyStatus/main.go

@@ -52,7 +52,6 @@ func (s *Sync) Run(workers int, stopCh <-chan struct{}) {
 
 	wait.Until(s.updatePolicyStatus, 2*time.Second, stopCh)
 	<-stopCh
-	s.updatePolicyStatus()
 }
 
 func (s *Sync) updateStatusCache(stopCh <-chan struct{}) {