MLCronJob status update (#1526)

.circleci/continue_config.yml

@@ -16,7 +16,7 @@ parameters:
 executors:
   golang-executor:
     docker:
-      - image: gcr.io/gcr-for-testing/golang:1.20.11
+      - image: gcr.io/gcr-for-testing/golang:1.20.12
   machine-executor:
     machine:
       image: ubuntu-2204:current

CHANGELOG.md

@@ -29,6 +29,7 @@
 - (Feature) (ML) Extension Storage Condition
 - (Improvement) (ML) Switch to fsnotify for file watching for MacOS support
 - (Feature) (ML) Unify Images, Resources and Lifecycle
+- (Improvement) (ML) CronJob status update
 
 ## [1.2.35](https://github.com/arangodb/kube-arangodb/tree/1.2.35) (2023-11-06)
 - (Maintenance) Update go-driver to v1.6.0, update IsNotFound() checks

Makefile

@@ -48,7 +48,7 @@ endif
 
 TEST_BUILD ?= 0
 GOBUILDARGS ?=
-GOBASEVERSION := 1.20.11
+GOBASEVERSION := 1.20.12
 GOVERSION := $(GOBASEVERSION)-alpine3.17
 DISTRIBUTION := alpine:3.15
 GOBUILDTAGS := $(RELEASE_MODE)
@@ -779,6 +779,7 @@ set-api-version/%:
 synchronize: synchronize-v2alpha1-with-v1
 
 synchronize-v2alpha1-with-v1:
+	@echo ">> Please use only COMMUNITY mode! Current RELEASE_MODE=$(RELEASE_MODE)"
 	@rm -f pkg/apis/deployment/v1/zz_generated.deepcopy.go pkg/apis/deployment/v2alpha1/zz_generated.deepcopy.go
 	@for file in $$(find "$(ROOT)/pkg/apis/deployment/v1/" -type f -exec $(REALPATH) --relative-to "$(ROOT)/pkg/apis/deployment/v1/" {} \;); do if [ ! -d "$(ROOT)/pkg/apis/deployment/v2alpha1/$$(dirname $${file})" ]; then mkdir -p "$(ROOT)/pkg/apis/deployment/v2alpha1/$$(dirname $${file})"; fi; done
 	@for file in $$(find "$(ROOT)/pkg/apis/deployment/v1/" -type f -exec $(REALPATH) --relative-to "$(ROOT)/pkg/apis/deployment/v1/" {} \;); do cat "$(ROOT)/pkg/apis/deployment/v1/$${file}" | $(SED) "s#package v1#package v2alpha1#g" | $(SED) 's#ArangoDeploymentVersion = "v1"#ArangoDeploymentVersion = "v2alpha1"#g' > "$(ROOT)/pkg/apis/deployment/v2alpha1/$${file}"; done

docs/api/ArangoMLCronJob.V1Alpha1.md

@@ -4,7 +4,7 @@
 
 ### .spec
 
-Type: `batch.CronJobSpec` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/cronjob_spec.go#L32)</sup>
+Type: `batch.CronJobSpec` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/cronjob_spec.go#L33)</sup>
 
 Links:
 * [Kubernetes Documentation](https://godoc.org/k8s.io/api/batch/v1beta1#CronJobSpec)
@@ -13,7 +13,7 @@ Links:
 
 ### .status
 
-Type: `batch.CronJobStatus` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/cronjob_status.go#L36)</sup>
+Type: `batch.CronJobStatus` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/cronjob_status.go#L38)</sup>
 
 Links:
 * [Kubernetes Documentation](https://godoc.org/k8s.io/api/batch/v1beta1#CronJobStatus)
@@ -22,7 +22,31 @@ Links:
 
 ### .status.conditions
 
-Type: `api.Conditions` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/cronjob_status.go#L32)</sup>
+Type: `api.Conditions` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/cronjob_status.go#L34)</sup>
 
 Conditions specific to the entire cron job
 
+***
+
+### .status.ref.name
+
+Type: `string` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/shared/v1/object.go#L46)</sup>
+
+Name of the object
+
+***
+
+### .status.ref.namespace
+
+Type: `string` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/shared/v1/object.go#L49)</sup>
+
+Namespace of the object. Should default to the namespace of the parent object
+
+***
+
+### .status.ref.uid
+
+Type: `string` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/shared/v1/object.go#L52)</sup>
+
+UID keeps the information about object UID
+

pkg/apis/ml/v1alpha1/cronjob_spec.go

@@ -21,17 +21,27 @@
 package v1alpha1
 
 import (
-	batchApi "k8s.io/api/batch/v1beta1"
+	batch "k8s.io/api/batch/v1"
 
 	"github.com/arangodb/kube-arangodb/pkg/apis/shared"
+	"github.com/arangodb/kube-arangodb/pkg/util/errors"
 )
 
 type ArangoMLCronJobSpec struct {
 	// +doc/type: batch.CronJobSpec
 	// +doc/link: Kubernetes Documentation|https://godoc.org/k8s.io/api/batch/v1beta1#CronJobSpec
-	*batchApi.CronJobSpec `json:",inline"`
+	*batch.CronJobSpec `json:",inline"`
 }
 
 func (a *ArangoMLCronJobSpec) Validate() error {
-	return shared.WithErrors(shared.PrefixResourceErrors("spec"))
+	if a == nil {
+		return errors.Newf("Spec is not defined")
+	}
+
+	var err []error
+	if a.CronJobSpec == nil {
+		err = append(err, shared.PrefixResourceErrors("spec", errors.Newf("CronJobSpec is not defined")))
+	}
+
+	return shared.WithErrors(err...)
 }

pkg/apis/ml/v1alpha1/cronjob_status.go

@@ -21,9 +21,11 @@
 package v1alpha1
 
 import (
-	batchApi "k8s.io/api/batch/v1beta1"
+	batch "k8s.io/api/batch/v1"
 
 	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+	"github.com/arangodb/kube-arangodb/pkg/apis/shared"
+	sharedApi "github.com/arangodb/kube-arangodb/pkg/apis/shared/v1"
 )
 
 type ArangoMLCronJobStatus struct {
@@ -33,5 +35,12 @@ type ArangoMLCronJobStatus struct {
 
 	// +doc/type: batch.CronJobStatus
 	// +doc/link: Kubernetes Documentation|https://godoc.org/k8s.io/api/batch/v1beta1#CronJobStatus
-	batchApi.CronJobStatus `json:",inline"`
+	*batch.CronJobStatus `json:",inline"`
+
+	// Ref keeps the reference to the CronJob
+	Ref *sharedApi.Object `json:"ref,omitempty"`
+}
+
+func (a *ArangoMLCronJobStatus) Validate() error {
+	return shared.WithErrors(shared.PrefixResourceErrors("spec"))
 }

pkg/apis/ml/v1alpha1/extension_conditions.go

@@ -29,4 +29,5 @@ const (
 	ExtensionMetadataServiceValidCondition api.ConditionType = "MetadataServiceValid"
 	ExtensionServiceAccountReadyCondition  api.ConditionType = "ServiceAccountReady"
 	LicenseValidCondition                  api.ConditionType = "LicenseValid"
+	CronJobSyncedCondition                 api.ConditionType = "CronJobSynced"
 )

pkg/apis/ml/v1alpha1/zz_generated.deepcopy.go

@@ -28,7 +28,7 @@ package v1alpha1
 import (
 	v1 "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
 	sharedv1 "github.com/arangodb/kube-arangodb/pkg/apis/shared/v1"
-	v1beta1 "k8s.io/api/batch/v1beta1"
+	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
 	runtime "k8s.io/apimachinery/pkg/runtime"
 )
@@ -199,7 +199,7 @@ func (in *ArangoMLCronJobSpec) DeepCopyInto(out *ArangoMLCronJobSpec) {
 	*out = *in
 	if in.CronJobSpec != nil {
 		in, out := &in.CronJobSpec, &out.CronJobSpec
-		*out = new(v1beta1.CronJobSpec)
+		*out = new(batchv1.CronJobSpec)
 		(*in).DeepCopyInto(*out)
 	}
 	return
@@ -225,7 +225,16 @@ func (in *ArangoMLCronJobStatus) DeepCopyInto(out *ArangoMLCronJobStatus) {
 			(*in)[i].DeepCopyInto(&(*out)[i])
 		}
 	}
-	in.CronJobStatus.DeepCopyInto(&out.CronJobStatus)
+	if in.CronJobStatus != nil {
+		in, out := &in.CronJobStatus, &out.CronJobStatus
+		*out = new(batchv1.CronJobStatus)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.Ref != nil {
+		in, out := &in.Ref, &out.Ref
+		*out = new(sharedv1.Object)
+		(*in).DeepCopyInto(*out)
+	}
 	return
 }
 

pkg/operatorV2/operator_worker.go

@@ -22,6 +22,7 @@ package operator
 
 import (
 	"context"
+	"fmt"
 
 	"github.com/arangodb/kube-arangodb/pkg/operatorV2/operation"
 	"github.com/arangodb/kube-arangodb/pkg/util/errors"
@@ -109,7 +110,9 @@ func (o *operator) processObject(obj interface{}) error {
 		o.workqueue.AddRateLimited(key)
 
 		if !IsReconcile(err) {
-			return errors.Newf("error syncing '%s': %s, re-queuing", key, err.Error())
+			message := fmt.Sprintf("error syncing '%s': %s, re-queuing", key, err.Error())
+			loggerWorker.Debug(message)
+			return errors.Newf(message)
 		}
 
 		return nil

pkg/util/tests/kubernetes.go

@@ -96,6 +96,12 @@ type KubernetesObject interface {
 func CreateObjects(t *testing.T, k8s kubernetes.Interface, arango arangoClientSet.Interface, objects ...interface{}) func(t *testing.T) {
 	for _, object := range objects {
 		switch v := object.(type) {
+		case **batch.CronJob:
+			require.NotNil(t, v)
+
+			vl := *v
+			_, err := k8s.BatchV1().CronJobs(vl.GetNamespace()).Create(context.Background(), vl, meta.CreateOptions{})
+			require.NoError(t, err)
 		case **batch.Job:
 			require.NotNil(t, v)
 
@@ -174,6 +180,12 @@ func CreateObjects(t *testing.T, k8s kubernetes.Interface, arango arangoClientSe
 			vl := *v
 			_, err := k8s.RbacV1().RoleBindings(vl.GetNamespace()).Create(context.Background(), vl, meta.CreateOptions{})
 			require.NoError(t, err)
+		case **mlApi.ArangoMLCronJob:
+			require.NotNil(t, v)
+
+			vl := *v
+			_, err := arango.MlV1alpha1().ArangoMLCronJobs(vl.GetNamespace()).Create(context.Background(), vl, meta.CreateOptions{})
+			require.NoError(t, err)
 		default:
 			require.Fail(t, fmt.Sprintf("Unable to create object: %s", reflect.TypeOf(v).String()))
 		}
@@ -187,6 +199,12 @@ func CreateObjects(t *testing.T, k8s kubernetes.Interface, arango arangoClientSe
 func UpdateObjects(t *testing.T, k8s kubernetes.Interface, arango arangoClientSet.Interface, objects ...interface{}) func(t *testing.T) {
 	for _, object := range objects {
 		switch v := object.(type) {
+		case **batch.CronJob:
+			require.NotNil(t, v)
+
+			vl := *v
+			_, err := k8s.BatchV1().CronJobs(vl.GetNamespace()).Update(context.Background(), vl, meta.UpdateOptions{})
+			require.NoError(t, err)
 		case **batch.Job:
 			require.NotNil(t, v)
 
@@ -241,6 +259,12 @@ func UpdateObjects(t *testing.T, k8s kubernetes.Interface, arango arangoClientSe
 			vl := *v
 			_, err := arango.MlV1alpha1().ArangoMLStorages(vl.GetNamespace()).Update(context.Background(), vl, meta.UpdateOptions{})
 			require.NoError(t, err)
+		case **mlApi.ArangoMLCronJob:
+			require.NotNil(t, v)
+
+			vl := *v
+			_, err := arango.MlV1alpha1().ArangoMLCronJobs(vl.GetNamespace()).Update(context.Background(), vl, meta.UpdateOptions{})
+			require.NoError(t, err)
 		case **rbac.ClusterRole:
 			require.NotNil(t, v)
 
@@ -275,9 +299,112 @@ func UpdateObjects(t *testing.T, k8s kubernetes.Interface, arango arangoClientSe
 	}
 }
 
+func DeleteObjects(t *testing.T, k8s kubernetes.Interface, arango arangoClientSet.Interface, objects ...interface{}) func(t *testing.T) {
+	for _, object := range objects {
+		switch v := object.(type) {
+		case **batch.CronJob:
+			require.NotNil(t, v)
+
+			vl := *v
+			require.NoError(t, k8s.BatchV1().CronJobs(vl.GetNamespace()).Delete(context.Background(), vl.GetName(), meta.DeleteOptions{}))
+		case **batch.Job:
+			require.NotNil(t, v)
+
+			vl := *v
+			require.NoError(t, k8s.BatchV1().Jobs(vl.GetNamespace()).Delete(context.Background(), vl.GetName(), meta.DeleteOptions{}))
+		case **core.Pod:
+			require.NotNil(t, v)
+
+			vl := *v
+			require.NoError(t, k8s.CoreV1().Pods(vl.GetNamespace()).Delete(context.Background(), vl.GetName(), meta.DeleteOptions{}))
+		case **core.Secret:
+			require.NotNil(t, v)
+
+			vl := *v
+			require.NoError(t, k8s.CoreV1().Secrets(vl.GetNamespace()).Delete(context.Background(), vl.GetName(), meta.DeleteOptions{}))
+		case **core.ServiceAccount:
+			require.NotNil(t, v)
+
+			vl := *v
+			require.NoError(t, k8s.CoreV1().ServiceAccounts(vl.GetNamespace()).Delete(context.Background(), vl.GetName(), meta.DeleteOptions{}))
+		case **api.ArangoDeployment:
+			require.NotNil(t, v)
+
+			vl := *v
+			require.NoError(t, arango.DatabaseV1().ArangoDeployments(vl.GetNamespace()).Delete(context.Background(), vl.GetName(), meta.DeleteOptions{}))
+		case **api.ArangoClusterSynchronization:
+			require.NotNil(t, v)
+
+			vl := *v
+			require.NoError(t, arango.DatabaseV1().ArangoClusterSynchronizations(vl.GetNamespace()).Delete(context.Background(), vl.GetName(), meta.DeleteOptions{}))
+		case **backupApi.ArangoBackup:
+			require.NotNil(t, v)
+
+			vl := *v
+			require.NoError(t, arango.BackupV1().ArangoBackups(vl.GetNamespace()).Delete(context.Background(), vl.GetName(), meta.DeleteOptions{}))
+		case **mlApi.ArangoMLExtension:
+			require.NotNil(t, v)
+
+			vl := *v
+			require.NoError(t, arango.MlV1alpha1().ArangoMLExtensions(vl.GetNamespace()).Delete(context.Background(), vl.GetName(), meta.DeleteOptions{}))
+		case **mlApi.ArangoMLStorage:
+			require.NotNil(t, v)
+
+			vl := *v
+			require.NoError(t, arango.MlV1alpha1().ArangoMLStorages(vl.GetNamespace()).Delete(context.Background(), vl.GetName(), meta.DeleteOptions{}))
+		case **mlApi.ArangoMLCronJob:
+			require.NotNil(t, v)
+
+			vl := *v
+			require.NoError(t, arango.MlV1alpha1().ArangoMLCronJobs(vl.GetNamespace()).Delete(context.Background(), vl.GetName(), meta.DeleteOptions{}))
+		case **rbac.ClusterRole:
+			require.NotNil(t, v)
+
+			vl := *v
+			require.NoError(t, k8s.RbacV1().ClusterRoles().Delete(context.Background(), vl.GetName(), meta.DeleteOptions{}))
+		case **rbac.ClusterRoleBinding:
+			require.NotNil(t, v)
+
+			vl := *v
+			require.NoError(t, k8s.RbacV1().ClusterRoleBindings().Delete(context.Background(), vl.GetName(), meta.DeleteOptions{}))
+		case **rbac.Role:
+			require.NotNil(t, v)
+
+			vl := *v
+			require.NoError(t, k8s.RbacV1().Roles(vl.GetNamespace()).Delete(context.Background(), vl.GetName(), meta.DeleteOptions{}))
+		case **rbac.RoleBinding:
+			require.NotNil(t, v)
+
+			vl := *v
+			require.NoError(t, k8s.RbacV1().RoleBindings(vl.GetNamespace()).Delete(context.Background(), vl.GetName(), meta.DeleteOptions{}))
+		default:
+			require.Fail(t, fmt.Sprintf("Unable to create object: %s", reflect.TypeOf(v).String()))
+		}
+	}
+
+	return func(t *testing.T) {
+		RefreshObjects(t, k8s, arango, objects...)
+	}
+}
+
 func RefreshObjects(t *testing.T, k8s kubernetes.Interface, arango arangoClientSet.Interface, objects ...interface{}) {
 	for _, object := range objects {
 		switch v := object.(type) {
+		case **batch.CronJob:
+			require.NotNil(t, v)
+
+			vl := *v
+
+			vn, err := k8s.BatchV1().CronJobs(vl.GetNamespace()).Get(context.Background(), vl.GetName(), meta.GetOptions{})
+			if err != nil {
+				if kerrors.IsNotFound(err) {
+					*v = nil
+				} else {
+					require.NoError(t, err)
+				}
+			} else {
+				*v = vn
+			}
 		case **batch.Job:
 			require.NotNil(t, v)
 
@@ -413,6 +540,21 @@ func RefreshObjects(t *testing.T, k8s kubernetes.Interface, arango arangoClientS
 			} else {
 				*v = vn
 			}
+		case **mlApi.ArangoMLCronJob:
+			require.NotNil(t, v)
+
+			vl := *v
+
+			vn, err := arango.MlV1alpha1().ArangoMLCronJobs(vl.GetNamespace()).Get(context.Background(), vl.GetName(), meta.GetOptions{})
+			if err != nil {
+				if kerrors.IsNotFound(err) {
+					*v = nil
+				} else {
+					require.NoError(t, err)
+				}
+			} else {
+				*v = vn
+			}
 		case **rbac.ClusterRole:
 			require.NotNil(t, v)
 
@@ -483,6 +625,12 @@ type MetaObjectMod[T meta.Object] func(t *testing.T, obj T)
 
 func SetMetaBasedOnType(t *testing.T, object meta.Object) {
 	switch v := object.(type) {
+	case *batch.CronJob:
+		v.Kind = "CronJob"
+		v.APIVersion = "batch/v1"
+		v.SetSelfLink(fmt.Sprintf("/api/batch/v1/cronjobs/%s/%s",
+			object.GetNamespace(),
+			object.GetName()))
 	case *batch.Job:
 		v.Kind = "Job"
 		v.APIVersion = "batch/v1"
@@ -571,6 +719,14 @@ func SetMetaBasedOnType(t *testing.T, object meta.Object) {
 		v.SetSelfLink(fmt.Sprintf("/api/rbac.authorization.k8s.io/v1/rolebingings/%s/%s",
 			object.GetNamespace(),
 			object.GetName()))
+	case *mlApi.ArangoMLCronJob:
+		v.Kind = ml.ArangoMLCronJobResourceKind
+		v.APIVersion = mlApi.SchemeGroupVersion.String()
+		v.SetSelfLink(fmt.Sprintf("/api/%s/%s/%s/%s",
+			mlApi.SchemeGroupVersion.String(),
+			ml.ArangoMLCronJobResourcePlural,
+			object.GetNamespace(),
+			object.GetName()))
 	default:
 		require.Fail(t, fmt.Sprintf("Unable to create object: %s", reflect.TypeOf(v).String()))
 	}
@@ -618,6 +774,10 @@ func NewItem(t *testing.T, o operation.Operation, object meta.Object) operation.
 	}
 
 	switch v := object.(type) {
+	case *batch.CronJob:
+		item.Group = "batch"
+		item.Version = "v1"
+		item.Kind = "CronJob"
 	case *batch.Job:
 		item.Group = "batch"
 		item.Version = "v1"
@@ -670,6 +830,10 @@ func NewItem(t *testing.T, o operation.Operation, object meta.Object) operation.
 		item.Group = "rbac.authorization.k8s.io"
 		item.Version = "v1"
 		item.Kind = "RoleBinding"
+	case *mlApi.ArangoMLCronJob:
+		item.Group = ml.ArangoMLGroupName
+		item.Version = mlApi.ArangoMLVersion
+		item.Kind = ml.ArangoMLCronJobResourceKind
 	default:
 		require.Fail(t, fmt.Sprintf("Unable to create object: %s", reflect.TypeOf(v).String()))
 	}