[Feature] [ML] Metadata Service Implementation (#1510)

CHANGELOG.md

@@ -21,6 +21,7 @@
 - (Feature) Improve K8S Mock for UT
 - (Feature) (ML) Introduce basic Conditions
 - (Improvement) Raise memory requests for init containers to 50mi
+- (Feature) (ML) Metadata Service Implementation
 
 ## [1.2.35](https://github.com/arangodb/kube-arangodb/tree/1.2.35) (2023-11-06)
 - (Maintenance) Update go-driver to v1.6.0, update IsNotFound() checks

docs/api/ArangoMLExtension.V1Alpha1.md

@@ -2,6 +2,24 @@
 
 ## Spec
 
+### .spec.metadataService.local.arangoMLFeatureStore
+
+Type: `string` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/extension_spec_metadata_service.go#L65)</sup>
+
+ArangoMLFeatureStoreDatabase define Database name to be used as MetadataService Backend in ArangoMLFeatureStoreDatabase
+
+Default Value: `arangomlfeaturestore`
+
+***
+
+### .spec.metadataService.local.arangoPipeDatabase
+
+Type: `string` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/extension_spec_metadata_service.go#L61)</sup>
+
+ArangoPipeDatabase define Database name to be used as MetadataService Backend in ArangoPipe
+
+Default Value: `arangopipe`
+
 ## Status
 
 ### .status.conditions
@@ -10,3 +28,43 @@ Type: `api.Conditions` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/
 
 Conditions specific to the entire extension
 
+***
+
+### .status.metadataService.local.arangoMLFeatureStore
+
+Type: `string` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/extension_status_metadata_service.go#L38)</sup>
+
+ArangoMLFeatureStoreDatabase define Database name to be used as MetadataService Backend
+
+***
+
+### .status.metadataService.local.arangoPipe
+
+Type: `string` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/extension_status_metadata_service.go#L35)</sup>
+
+ArangoPipeDatabase define Database name to be used as MetadataService Backend
+
+***
+
+### .status.metadataService.secret.name
+
+Type: `string` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/batchjob_spec.go#L29)</sup>
+
+Name of the object
+
+***
+
+### .status.metadataService.secret.namespace
+
+Type: `string` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/batchjob_status.go#L5)</sup>
+
+Namespace of the object. Should default to the namespace of the parent object
+
+***
+
+### .status.metadataService.secret.uid
+
+Type: `string` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/batchjob_status.go#L7)</sup>
+
+UID keeps the information about object UID
+

docs/api/ArangoMLStorage.V1Alpha1.md

@@ -23,7 +23,7 @@ Required
 
 ### .spec.backend.s3.caSecret.name
 
-Type: `string` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/batchjob_status.go#L12)</sup>
+Type: `string` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/batchjob_spec.go#L29)</sup>
 
 Name of the object
 
@@ -31,15 +31,23 @@ Name of the object
 
 ### .spec.backend.s3.caSecret.namespace
 
-Type: `string` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/batchjob_status.go#L13)</sup>
+Type: `string` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/batchjob_status.go#L5)</sup>
 
 Namespace of the object. Should default to the namespace of the parent object
 
 ***
 
+### .spec.backend.s3.caSecret.uid
+
+Type: `string` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/batchjob_status.go#L7)</sup>
+
+UID keeps the information about object UID
+
+***
+
 ### .spec.backend.s3.credentialsSecret.name
 
-Type: `string` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/batchjob_status.go#L12)</sup>
+Type: `string` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/batchjob_spec.go#L29)</sup>
 
 Name of the object
 
@@ -47,12 +55,20 @@ Name of the object
 
 ### .spec.backend.s3.credentialsSecret.namespace
 
-Type: `string` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/batchjob_status.go#L13)</sup>
+Type: `string` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/batchjob_status.go#L5)</sup>
 
 Namespace of the object. Should default to the namespace of the parent object
 
 ***
 
+### .spec.backend.s3.credentialsSecret.uid
+
+Type: `string` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/batchjob_status.go#L7)</sup>
+
+UID keeps the information about object UID
+
+***
+
 ### .spec.backend.s3.endpoint
 
 Type: `string` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.35/pkg/apis/ml/v1alpha1/storage_spec_backend_s3.go#L34)</sup>

pkg/apis/ml/v1alpha1/batchjob.go

@@ -22,6 +22,8 @@ package v1alpha1
 
 import (
 	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/arangodb/kube-arangodb/pkg/apis/ml"
 )
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
@@ -46,6 +48,18 @@ type ArangoMLBatchJob struct {
 	Status ArangoMLBatchJobStatus `json:"status"`
 }
 
+// AsOwner creates an OwnerReference for the given BatchJob
+func (d *ArangoMLBatchJob) AsOwner() meta.OwnerReference {
+	trueVar := true
+	return meta.OwnerReference{
+		APIVersion: SchemeGroupVersion.String(),
+		Kind:       ml.ArangoMLBatchJobResourceKind,
+		Name:       d.Name,
+		UID:        d.UID,
+		Controller: &trueVar,
+	}
+}
+
 func (a *ArangoMLBatchJob) GetStatus() ArangoMLBatchJobStatus {
 	return a.Status
 }

pkg/apis/ml/v1alpha1/conditions.go

@@ -23,5 +23,6 @@ package v1alpha1
 import api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
 
 const (
+	ReadyCondition     api.ConditionType = "Ready"
 	SpecValidCondition api.ConditionType = "SpecValid"
 )

pkg/apis/ml/v1alpha1/cronjob.go

@@ -22,6 +22,8 @@ package v1alpha1
 
 import (
 	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/arangodb/kube-arangodb/pkg/apis/ml"
 )
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
@@ -46,6 +48,18 @@ type ArangoMLCronJob struct {
 	Status ArangoMLCronJobStatus `json:"status"`
 }
 
+// AsOwner creates an OwnerReference for the given CronJob
+func (d *ArangoMLCronJob) AsOwner() meta.OwnerReference {
+	trueVar := true
+	return meta.OwnerReference{
+		APIVersion: SchemeGroupVersion.String(),
+		Kind:       ml.ArangoMLCronJobResourceKind,
+		Name:       d.Name,
+		UID:        d.UID,
+		Controller: &trueVar,
+	}
+}
+
 func (a *ArangoMLCronJob) GetStatus() ArangoMLCronJobStatus {
 	return a.Status
 }

pkg/apis/ml/v1alpha1/extension.go

@@ -22,6 +22,8 @@ package v1alpha1
 
 import (
 	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/arangodb/kube-arangodb/pkg/apis/ml"
 )
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
@@ -46,6 +48,18 @@ type ArangoMLExtension struct {
 	Status ArangoMLExtensionStatus `json:"status"`
 }
 
+// AsOwner creates an OwnerReference for the given Extension
+func (d *ArangoMLExtension) AsOwner() meta.OwnerReference {
+	trueVar := true
+	return meta.OwnerReference{
+		APIVersion: SchemeGroupVersion.String(),
+		Kind:       ml.ArangoMLExtensionResourceKind,
+		Name:       d.Name,
+		UID:        d.UID,
+		Controller: &trueVar,
+	}
+}
+
 func (a *ArangoMLExtension) GetStatus() ArangoMLExtensionStatus {
 	return a.Status
 }

pkg/apis/ml/v1alpha1/extension_conditions.go

@@ -24,4 +24,6 @@ import api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
 
 const (
 	ExtensionDeploymentFoundCondition api.ConditionType = "DeploymentFound"
+
+	ExtensionMetadataServiceValidCondition api.ConditionType = "MetadataServiceValid"
 )

pkg/apis/ml/v1alpha1/extension_spec.go

@@ -23,8 +23,21 @@ package v1alpha1
 import "github.com/arangodb/kube-arangodb/pkg/apis/shared"
 
 type ArangoMLExtensionSpec struct {
+	// MetadataService keeps the MetadataService configuration
+	// +doc/immutable: This setting cannot be changed after the MetadataService has been created.
+	MetadataService *ArangoMLExtensionSpecMetadataService `json:"metadataService,omitempty"`
+}
+
+func (a *ArangoMLExtensionSpec) GetMetadataService() *ArangoMLExtensionSpecMetadataService {
+	if a == nil || a.MetadataService == nil {
+		return nil
+	}
+
+	return a.MetadataService
 }
 
 func (a *ArangoMLExtensionSpec) Validate() error {
-	return shared.WithErrors(shared.PrefixResourceErrors("spec"))
+	return shared.WithErrors(shared.PrefixResourceErrors("spec",
+		shared.PrefixResourceErrors("metadataService", a.GetMetadataService().Validate()),
+	))
 }

pkg/apis/ml/v1alpha1/extension_spec_metadata_service.go

@@ -0,0 +1,99 @@
+//
+// DISCLAIMER
+//
+// Copyright 2023 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package v1alpha1
+
+import (
+	"github.com/arangodb/kube-arangodb/pkg/apis/shared"
+	"github.com/arangodb/kube-arangodb/pkg/util"
+	"github.com/arangodb/kube-arangodb/pkg/util/errors"
+)
+
+const (
+	ArangoMLExtensionSpecMetadataServiceLocalDefaultArangoPipeDatabase           = "arangopipe"
+	ArangoMLExtensionSpecMetadataServiceLocalDefaultArangoMLFeatureStoreDatabase = "arangomlfeaturestore"
+)
+
+type ArangoMLExtensionSpecMetadataService struct {
+	// Local define to use Local ArangoDeployment as the Metadata Service
+	Local *ArangoMLExtensionSpecMetadataServiceLocal `json:"local,omitempty"`
+}
+
+func (a *ArangoMLExtensionSpecMetadataService) GetLocal() *ArangoMLExtensionSpecMetadataServiceLocal {
+	if a == nil || a.Local == nil {
+		return nil
+	}
+
+	return a.Local
+}
+
+func (a *ArangoMLExtensionSpecMetadataService) Validate() error {
+	// If Nil then we use default
+	if a == nil {
+		return nil
+	}
+
+	return shared.WithErrors(
+		shared.PrefixResourceErrors("local", a.GetLocal().Validate()),
+	)
+}
+
+type ArangoMLExtensionSpecMetadataServiceLocal struct {
+	// ArangoPipeDatabase define Database name to be used as MetadataService Backend in ArangoPipe
+	// +doc/default: arangopipe
+	ArangoPipeDatabase *string `json:"arangoPipeDatabase,omitempty"`
+
+	// ArangoMLFeatureStoreDatabase define Database name to be used as MetadataService Backend in ArangoMLFeatureStoreDatabase
+	// +doc/default: arangomlfeaturestore
+	ArangoMLFeatureStoreDatabase *string `json:"arangoMLFeatureStore,omitempty"`
+}
+
+func (a *ArangoMLExtensionSpecMetadataServiceLocal) GetArangoPipeDatabase() string {
+	if a == nil {
+		return ArangoMLExtensionSpecMetadataServiceLocalDefaultArangoPipeDatabase
+	}
+
+	if d := a.ArangoPipeDatabase; d == nil {
+		return ArangoMLExtensionSpecMetadataServiceLocalDefaultArangoPipeDatabase
+	} else {
+		return *d
+	}
+}
+
+func (a *ArangoMLExtensionSpecMetadataServiceLocal) GetArangoMLFeatureStoreDatabase() string {
+	if a == nil {
+		return ArangoMLExtensionSpecMetadataServiceLocalDefaultArangoMLFeatureStoreDatabase
+	}
+
+	if d := a.ArangoMLFeatureStoreDatabase; d == nil {
+		return ArangoMLExtensionSpecMetadataServiceLocalDefaultArangoMLFeatureStoreDatabase
+	} else {
+		return *d
+	}
+}
+
+func (a *ArangoMLExtensionSpecMetadataServiceLocal) Validate() error {
+	// If Nil then we use default
+
+	return shared.WithErrors(
+		shared.PrefixResourceErrors("arangoPipeDatabase", util.BoolSwitch(a.GetArangoPipeDatabase() != ArangoMLExtensionSpecMetadataServiceLocalDefaultArangoPipeDatabase, errors.Newf("Database name is hardcoded"), nil)),
+		shared.PrefixResourceErrors("arangoMLFeatureStore", util.BoolSwitch(a.GetArangoMLFeatureStoreDatabase() != ArangoMLExtensionSpecMetadataServiceLocalDefaultArangoMLFeatureStoreDatabase, errors.Newf("Database name is hardcoded"), nil)),
+	)
+}

pkg/apis/ml/v1alpha1/extension_status.go

@@ -26,4 +26,7 @@ type ArangoMLExtensionStatus struct {
 	// Conditions specific to the entire extension
 	// +doc/type: api.Conditions
 	Conditions api.ConditionList `json:"conditions,omitempty"`
+
+	// MetadataService keeps the MetadataService configuration
+	MetadataService *ArangoMLExtensionStatusMetadataService `json:"metadataService,omitempty"`
 }

pkg/apis/ml/v1alpha1/extension_status_metadata_service.go

@@ -0,0 +1,39 @@
+//
+// DISCLAIMER
+//
+// Copyright 2023 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package v1alpha1
+
+import shared "github.com/arangodb/kube-arangodb/pkg/apis/shared/v1"
+
+type ArangoMLExtensionStatusMetadataService struct {
+	// Local define the Local ArangoDeployment Metadata Service configuration
+	Local *ArangoMLExtensionStatusMetadataServiceLocal `json:"local,omitempty"`
+
+	// Secret define the Secret specification to store all the details
+	Secret *shared.Object `json:"secret,omitempty"`
+}
+
+type ArangoMLExtensionStatusMetadataServiceLocal struct {
+	// ArangoPipeDatabase define Database name to be used as MetadataService Backend
+	ArangoPipeDatabase string `json:"arangoPipe"`
+
+	// ArangoMLFeatureStoreDatabase define Database name to be used as MetadataService Backend
+	ArangoMLFeatureStoreDatabase string `json:"arangoMLFeatureStore,omitempty"`
+}

pkg/apis/ml/v1alpha1/storage.go

@@ -22,6 +22,8 @@ package v1alpha1
 
 import (
 	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/arangodb/kube-arangodb/pkg/apis/ml"
 )
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
@@ -46,6 +48,18 @@ type ArangoMLStorage struct {
 	Status ArangoMLStorageStatus `json:"status"`
 }
 
+// AsOwner creates an OwnerReference for the given Storage
+func (d *ArangoMLStorage) AsOwner() meta.OwnerReference {
+	trueVar := true
+	return meta.OwnerReference{
+		APIVersion: SchemeGroupVersion.String(),
+		Kind:       ml.ArangoMLStorageResourceKind,
+		Name:       d.Name,
+		UID:        d.UID,
+		Controller: &trueVar,
+	}
+}
+
 func (a *ArangoMLStorage) GetStatus() ArangoMLStorageStatus {
 	return a.Status
 }

pkg/apis/ml/v1alpha1/zz_generated.deepcopy.go

@@ -237,7 +237,7 @@ func (in *ArangoMLExtension) DeepCopyInto(out *ArangoMLExtension) {
 	*out = *in
 	out.TypeMeta = in.TypeMeta
 	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
-	out.Spec = in.Spec
+	in.Spec.DeepCopyInto(&out.Spec)
 	in.Status.DeepCopyInto(&out.Status)
 	return
 }
@@ -296,6 +296,11 @@ func (in *ArangoMLExtensionList) DeepCopyObject() runtime.Object {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ArangoMLExtensionSpec) DeepCopyInto(out *ArangoMLExtensionSpec) {
 	*out = *in
+	if in.MetadataService != nil {
+		in, out := &in.MetadataService, &out.MetadataService
+		*out = new(ArangoMLExtensionSpecMetadataService)
+		(*in).DeepCopyInto(*out)
+	}
 	return
 }
 
@@ -309,6 +314,53 @@ func (in *ArangoMLExtensionSpec) DeepCopy() *ArangoMLExtensionSpec {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ArangoMLExtensionSpecMetadataService) DeepCopyInto(out *ArangoMLExtensionSpecMetadataService) {
+	*out = *in
+	if in.Local != nil {
+		in, out := &in.Local, &out.Local
+		*out = new(ArangoMLExtensionSpecMetadataServiceLocal)
+		(*in).DeepCopyInto(*out)
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ArangoMLExtensionSpecMetadataService.
+func (in *ArangoMLExtensionSpecMetadataService) DeepCopy() *ArangoMLExtensionSpecMetadataService {
+	if in == nil {
+		return nil
+	}
+	out := new(ArangoMLExtensionSpecMetadataService)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ArangoMLExtensionSpecMetadataServiceLocal) DeepCopyInto(out *ArangoMLExtensionSpecMetadataServiceLocal) {
+	*out = *in
+	if in.ArangoPipeDatabase != nil {
+		in, out := &in.ArangoPipeDatabase, &out.ArangoPipeDatabase
+		*out = new(string)
+		**out = **in
+	}
+	if in.ArangoMLFeatureStoreDatabase != nil {
+		in, out := &in.ArangoMLFeatureStoreDatabase, &out.ArangoMLFeatureStoreDatabase
+		*out = new(string)
+		**out = **in
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ArangoMLExtensionSpecMetadataServiceLocal.
+func (in *ArangoMLExtensionSpecMetadataServiceLocal) DeepCopy() *ArangoMLExtensionSpecMetadataServiceLocal {
+	if in == nil {
+		return nil
+	}
+	out := new(ArangoMLExtensionSpecMetadataServiceLocal)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ArangoMLExtensionStatus) DeepCopyInto(out *ArangoMLExtensionStatus) {
 	*out = *in
@@ -319,6 +371,11 @@ func (in *ArangoMLExtensionStatus) DeepCopyInto(out *ArangoMLExtensionStatus) {
 			(*in)[i].DeepCopyInto(&(*out)[i])
 		}
 	}
+	if in.MetadataService != nil {
+		in, out := &in.MetadataService, &out.MetadataService
+		*out = new(ArangoMLExtensionStatusMetadataService)
+		(*in).DeepCopyInto(*out)
+	}
 	return
 }
 
@@ -332,6 +389,48 @@ func (in *ArangoMLExtensionStatus) DeepCopy() *ArangoMLExtensionStatus {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ArangoMLExtensionStatusMetadataService) DeepCopyInto(out *ArangoMLExtensionStatusMetadataService) {
+	*out = *in
+	if in.Local != nil {
+		in, out := &in.Local, &out.Local
+		*out = new(ArangoMLExtensionStatusMetadataServiceLocal)
+		**out = **in
+	}
+	if in.Secret != nil {
+		in, out := &in.Secret, &out.Secret
+		*out = new(sharedv1.Object)
+		(*in).DeepCopyInto(*out)
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ArangoMLExtensionStatusMetadataService.
+func (in *ArangoMLExtensionStatusMetadataService) DeepCopy() *ArangoMLExtensionStatusMetadataService {
+	if in == nil {
+		return nil
+	}
+	out := new(ArangoMLExtensionStatusMetadataService)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ArangoMLExtensionStatusMetadataServiceLocal) DeepCopyInto(out *ArangoMLExtensionStatusMetadataServiceLocal) {
+	*out = *in
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ArangoMLExtensionStatusMetadataServiceLocal.
+func (in *ArangoMLExtensionStatusMetadataServiceLocal) DeepCopy() *ArangoMLExtensionStatusMetadataServiceLocal {
+	if in == nil {
+		return nil
+	}
+	out := new(ArangoMLExtensionStatusMetadataServiceLocal)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ArangoMLStorage) DeepCopyInto(out *ArangoMLStorage) {
 	*out = *in

pkg/apis/shared/v1/object.go

@@ -22,6 +22,7 @@ package v1
 
 import (
 	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
 
 	"github.com/arangodb/kube-arangodb/pkg/apis/shared"
 )
@@ -32,6 +33,9 @@ type Object struct {
 
 	// Namespace of the object. Should default to the namespace of the parent object
 	Namespace *string `json:"namespace,omitempty"`
+
+	// UID keeps the information about object UID
+	UID *types.UID `json:"uid,omitempty"`
 }
 
 func (o *Object) IsEmpty() bool {
@@ -57,6 +61,16 @@ func (o *Object) GetNamespace(obj meta.Object) string {
 	return obj.GetNamespace()
 }
 
+func (o *Object) GetUID() types.UID {
+	if o != nil {
+		if n := o.UID; n != nil {
+			return *n
+		}
+	}
+
+	return ""
+}
+
 func (o *Object) Validate() error {
 	if o == nil {
 		o = &Object{}
@@ -67,6 +81,9 @@ func (o *Object) Validate() error {
 	if o.Namespace != nil {
 		errs = append(errs, shared.PrefixResourceErrors("namespace", AsKubernetesResourceName(o.Namespace).Validate()))
 	}
+	if u := o.UID; u != nil {
+		errs = append(errs, shared.PrefixResourceErrors("uid", shared.ValidateUID(*u)))
+	}
 
 	return shared.WithErrors(errs...)
 }

pkg/apis/shared/v1/zz_generated.deepcopy.go

@@ -25,6 +25,10 @@
 
 package v1
 
+import (
+	types "k8s.io/apimachinery/pkg/types"
+)
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in HashList) DeepCopyInto(out *HashList) {
 	{
@@ -53,6 +57,11 @@ func (in *Object) DeepCopyInto(out *Object) {
 		*out = new(string)
 		**out = **in
 	}
+	if in.UID != nil {
+		in, out := &in.UID, &out.UID
+		*out = new(types.UID)
+		**out = **in
+	}
 	return
 }
 

pkg/apis/shared/validate.go

@@ -1,7 +1,7 @@
 //
 // DISCLAIMER
 //
-// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -23,7 +23,10 @@ package shared
 import (
 	"regexp"
 
+	"k8s.io/apimachinery/pkg/types"
+
 	"github.com/arangodb/kube-arangodb/pkg/util/errors"
+	"github.com/arangodb/kube-arangodb/pkg/util/strings"
 )
 
 var (
@@ -54,3 +57,19 @@ func ValidateOptionalResourceName(name string) error {
 	}
 	return nil
 }
+
+// ValidateUID validates if it is valid Kubernetes UID
+func ValidateUID(uid types.UID) error {
+	v := strings.Split(string(uid), "-")
+
+	if len(v) != 0 &&
+		len(v[0]) != 6 &&
+		len(v[1]) != 4 &&
+		len(v[2]) != 4 &&
+		len(v[3]) != 4 &&
+		len(v[4]) != 6 {
+		return errors.Newf("Invalid UID: %s", uid)
+	}
+
+	return nil
+}

pkg/apis/shared/validate_test.go

@@ -0,0 +1,33 @@
+//
+// DISCLAIMER
+//
+// Copyright 2023 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package shared
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"k8s.io/apimachinery/pkg/util/uuid"
+)
+
+func Test_ValidateUID(t *testing.T) {
+	require.Error(t, ValidateUID(""))
+	require.NoError(t, ValidateUID(uuid.NewUUID()))
+}

pkg/crd/crds/ml-extension.schema.generated.yaml

@@ -2,6 +2,21 @@ v1alpha1:
   openAPIV3Schema:
     properties:
       spec:
+        properties:
+          metadataService:
+            description: MetadataService keeps the MetadataService configuration
+            properties:
+              local:
+                description: Local define to use Local ArangoDeployment as the Metadata Service
+                properties:
+                  arangoMLFeatureStore:
+                    description: ArangoMLFeatureStoreDatabase define Database name to be used as MetadataService Backend in ArangoMLFeatureStoreDatabase
+                    type: string
+                  arangoPipeDatabase:
+                    description: ArangoPipeDatabase define Database name to be used as MetadataService Backend in ArangoPipe
+                    type: string
+                type: object
+            type: object
         type: object
     type: object
     x-kubernetes-preserve-unknown-fields: true

pkg/crd/crds/ml-storage.schema.generated.yaml

@@ -28,6 +28,8 @@ v1alpha1:
                         type: string
                       namespace:
                         type: string
+                      uid:
+                        type: string
                     type: object
                   credentialsSecret:
                     description: |-
@@ -38,6 +40,8 @@ v1alpha1:
                         type: string
                       namespace:
                         type: string
+                      uid:
+                        type: string
                     type: object
                   endpoint:
                     description: |-

pkg/deployment/deployment_inspector.go

@@ -506,8 +506,9 @@ func (d *Deployment) refreshMaintenanceTTL(ctx context.Context) {
 
 	condition, ok := status.Conditions.Get(api.ConditionTypeMaintenance)
 	maintenance := agencyState.Supervision.Maintenance
+	hotBackup := agencyState.Target.HotBackup.Create.Exists()
 
-	if !ok || !condition.IsTrue() {
+	if !ok || !condition.IsTrue() || hotBackup {
 		return
 	}
 

pkg/operatorV2/errors_reconcile.go

@@ -0,0 +1,49 @@
+//
+// DISCLAIMER
+//
+// Copyright 2023 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package operator
+
+import "fmt"
+
+func Reconcile(msg string, args ...interface{}) error {
+	return reconcile{
+		message: fmt.Sprintf(msg, args...),
+	}
+}
+
+type reconcile struct {
+	message string
+}
+
+func (r reconcile) Error() string {
+	return r.message
+}
+
+func IsReconcile(err error) bool {
+	if err == nil {
+		return false
+	}
+
+	if _, ok := err.(reconcile); ok {
+		return true
+	}
+
+	return false
+}

pkg/operatorV2/handle.go

@@ -20,7 +20,32 @@
 
 package operator
 
-import "context"
+import (
+	"context"
+
+	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+)
+
+func WithCondition(conditions *api.ConditionList, condition api.ConditionType, changed bool, err error) (bool, error) {
+	if changed || err != nil {
+		// Condition should be false
+		if conditions.Update(condition, false, "Not ready", "Not ready") {
+			changed = true
+		}
+	} else {
+		if conditions.Update(condition, true, "Ready", "Ready") {
+			changed = true
+		}
+	}
+
+	if err == nil || IsStop(err) {
+		if changed {
+			err = Reconcile("Condition changed")
+		}
+	}
+
+	return changed, err
+}
 
 type HandleP0Func func(ctx context.Context) (bool, error)
 
@@ -66,6 +91,15 @@ func HandleP1[P1 interface{}](ctx context.Context, p1 P1, handler ...HandleP1Fun
 	return isChanged, nil
 }
 
+func HandleP1WithStop[P1 interface{}](ctx context.Context, p1 P1, handler ...HandleP1Func[P1]) (bool, error) {
+	changed, err := HandleP1[P1](ctx, p1, handler...)
+	if IsStop(err) {
+		return changed, nil
+	}
+
+	return changed, err
+}
+
 func HandleP2[P1, P2 interface{}](ctx context.Context, p1 P1, p2 P2, handler ...HandleP2Func[P1, P2]) (bool, error) {
 	isChanged := false
 	for _, h := range handler {
@@ -82,6 +116,15 @@ func HandleP2[P1, P2 interface{}](ctx context.Context, p1 P1, p2 P2, handler ...
 	return isChanged, nil
 }
 
+func HandleP2WithStop[P1, P2 interface{}](ctx context.Context, p1 P1, p2 P2, handler ...HandleP2Func[P1, P2]) (bool, error) {
+	changed, err := HandleP2[P1, P2](ctx, p1, p2, handler...)
+	if IsStop(err) {
+		return changed, nil
+	}
+
+	return changed, err
+}
+
 func HandleP3[P1, P2, P3 interface{}](ctx context.Context, p1 P1, p2 P2, p3 P3, handler ...HandleP3Func[P1, P2, P3]) (bool, error) {
 	isChanged := false
 	for _, h := range handler {
@@ -98,6 +141,20 @@ func HandleP3[P1, P2, P3 interface{}](ctx context.Context, p1 P1, p2 P2, p3 P3,
 	return isChanged, nil
 }
 
+func HandleP3WithStop[P1, P2, P3 interface{}](ctx context.Context, p1 P1, p2 P2, p3 P3, handler ...HandleP3Func[P1, P2, P3]) (bool, error) {
+	changed, err := HandleP3[P1, P2, P3](ctx, p1, p2, p3, handler...)
+	if IsStop(err) {
+		return changed, nil
+	}
+
+	return changed, err
+}
+
+func HandleP3WithCondition[P1, P2, P3 interface{}](ctx context.Context, conditions *api.ConditionList, condition api.ConditionType, p1 P1, p2 P2, p3 P3, handler ...HandleP3Func[P1, P2, P3]) (bool, error) {
+	changed, err := HandleP3[P1, P2, P3](ctx, p1, p2, p3, handler...)
+	return WithCondition(conditions, condition, changed, err)
+}
+
 func HandleP4[P1, P2, P3, P4 interface{}](ctx context.Context, p1 P1, p2 P2, p3 P3, p4 P4, handler ...HandleP4Func[P1, P2, P3, P4]) (bool, error) {
 	isChanged := false
 	for _, h := range handler {
@@ -114,6 +171,20 @@ func HandleP4[P1, P2, P3, P4 interface{}](ctx context.Context, p1 P1, p2 P2, p3
 	return isChanged, nil
 }
 
+func HandleP4WithStop[P1, P2, P3, P4 interface{}](ctx context.Context, p1 P1, p2 P2, p3 P3, p4 P4, handler ...HandleP4Func[P1, P2, P3, P4]) (bool, error) {
+	changed, err := HandleP4[P1, P2, P3, P4](ctx, p1, p2, p3, p4, handler...)
+	if IsStop(err) {
+		return changed, nil
+	}
+
+	return changed, err
+}
+
+func HandleP4WithCondition[P1, P2, P3, P4 interface{}](ctx context.Context, conditions *api.ConditionList, condition api.ConditionType, p1 P1, p2 P2, p3 P3, p4 P4, handler ...HandleP4Func[P1, P2, P3, P4]) (bool, error) {
+	changed, err := HandleP4[P1, P2, P3, P4](ctx, p1, p2, p3, p4, handler...)
+	return WithCondition(conditions, condition, changed, err)
+}
+
 func HandleP9[P1, P2, P3, P4, P5, P6, P7, P8, P9 interface{}](ctx context.Context, p1 P1, p2 P2, p3 P3, p4 P4, p5 P5, p6 P6, p7 P7, p8 P8, p9 P9, handler ...HandleP9Func[P1, P2, P3, P4, P5, P6, P7, P8, P9]) (bool, error) {
 	isChanged := false
 	for _, h := range handler {
@@ -129,3 +200,12 @@ func HandleP9[P1, P2, P3, P4, P5, P6, P7, P8, P9 interface{}](ctx context.Contex
 
 	return isChanged, nil
 }
+
+func HandleP9WithStop[P1, P2, P3, P4, P5, P6, P7, P8, P9 interface{}](ctx context.Context, p1 P1, p2 P2, p3 P3, p4 P4, p5 P5, p6 P6, p7 P7, p8 P8, p9 P9, handler ...HandleP9Func[P1, P2, P3, P4, P5, P6, P7, P8, P9]) (bool, error) {
+	changed, err := HandleP9[P1, P2, P3, P4, P5, P6, P7, P8, P9](ctx, p1, p2, p3, p4, p5, p6, p7, p8, p9, handler...)
+	if IsStop(err) {
+		return changed, nil
+	}
+
+	return changed, err
+}

pkg/operatorV2/operator_worker.go

@@ -107,7 +107,12 @@ func (o *operator) processObject(obj interface{}) error {
 
 	if err = o.processItem(item); err != nil {
 		o.workqueue.AddRateLimited(key)
-		return errors.Newf("error syncing '%s': %s, re-queuing", key, err.Error())
+
+		if !IsReconcile(err) {
+			return errors.Newf("error syncing '%s': %s, re-queuing", key, err.Error())
+		}
+
+		return nil
 	}
 
 	loggerWorker.Trace("Processed Item Action: %s, Type: %s/%s/%s, Namespace: %s, Name: %s",

pkg/util/tests/kubernetes.go

@@ -27,6 +27,7 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/require"
+	core "k8s.io/api/core/v1"
 	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/uuid"
 	"k8s.io/client-go/kubernetes"
@@ -38,9 +39,31 @@ import (
 	"github.com/arangodb/kube-arangodb/pkg/apis/ml"
 	mlApi "github.com/arangodb/kube-arangodb/pkg/apis/ml/v1alpha1"
 	arangoClientSet "github.com/arangodb/kube-arangodb/pkg/generated/clientset/versioned"
+	operator "github.com/arangodb/kube-arangodb/pkg/operatorV2"
 	"github.com/arangodb/kube-arangodb/pkg/operatorV2/operation"
+	"github.com/arangodb/kube-arangodb/pkg/util/errors"
 )
 
+func Handle(handler operator.Handler, item operation.Item) error {
+	return HandleWithMax(handler, item, 128)
+}
+
+func HandleWithMax(handler operator.Handler, item operation.Item, max int) error {
+	for i := 0; i < max; i++ {
+		if err := handler.Handle(context.Background(), item); err != nil {
+			if operator.IsReconcile(err) {
+				continue
+			}
+
+			return err
+		}
+
+		return nil
+	}
+
+	return errors.Newf("Max retries reached")
+}
+
 type KubernetesObject interface {
 	meta.Object
 	meta.Type
@@ -49,6 +72,12 @@ type KubernetesObject interface {
 func CreateObjects(t *testing.T, k8s kubernetes.Interface, arango arangoClientSet.Interface, objects ...interface{}) func(t *testing.T) {
 	for _, object := range objects {
 		switch v := object.(type) {
+		case **core.Secret:
+			require.NotNil(t, v)
+
+			vl := *v
+			_, err := k8s.CoreV1().Secrets(vl.GetNamespace()).Create(context.Background(), vl, meta.CreateOptions{})
+			require.NoError(t, err)
 		case **api.ArangoDeployment:
 			require.NotNil(t, v)
 
@@ -86,6 +115,15 @@ func CreateObjects(t *testing.T, k8s kubernetes.Interface, arango arangoClientSe
 func RefreshObjects(t *testing.T, k8s kubernetes.Interface, arango arangoClientSet.Interface, objects ...interface{}) {
 	for _, object := range objects {
 		switch v := object.(type) {
+		case **core.Secret:
+			require.NotNil(t, v)
+
+			vl := *v
+
+			vn, err := k8s.CoreV1().Secrets(vl.GetNamespace()).Get(context.Background(), vl.GetName(), meta.GetOptions{})
+			require.NoError(t, err)
+
+			*v = vn
 		case **api.ArangoDeployment:
 			require.NotNil(t, v)
 
@@ -132,6 +170,12 @@ type MetaObjectMod[T meta.Object] func(t *testing.T, obj T)
 
 func SetMetaBasedOnType(t *testing.T, object meta.Object) {
 	switch v := object.(type) {
+	case *core.Secret:
+		v.Kind = "Secret"
+		v.APIVersion = "v1"
+		v.SetSelfLink(fmt.Sprintf("/api/v1/secrets/%s/%s",
+			object.GetNamespace(),
+			object.GetName()))
 	case *api.ArangoDeployment:
 		v.Kind = deployment.ArangoDeploymentResourceKind
 		v.APIVersion = api.SchemeGroupVersion.String()
@@ -200,6 +244,10 @@ func NewItem(t *testing.T, o operation.Operation, object meta.Object) operation.
 	}
 
 	switch v := object.(type) {
+	case *core.Secret:
+		item.Group = ""
+		item.Version = "v1"
+		item.Kind = "Secret"
 	case *api.ArangoDeployment:
 		item.Group = deployment.ArangoDeploymentGroupName
 		item.Version = api.ArangoDeploymentVersion

pkg/util/tests/kubernetes_test.go

@@ -25,30 +25,38 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/require"
+	core "k8s.io/api/core/v1"
 	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	backupApi "github.com/arangodb/kube-arangodb/pkg/apis/backup/v1"
 	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
 	mlApi "github.com/arangodb/kube-arangodb/pkg/apis/ml/v1alpha1"
+	"github.com/arangodb/kube-arangodb/pkg/operatorV2/operation"
 	"github.com/arangodb/kube-arangodb/pkg/util/kclient"
 )
 
 func NewMetaObjectRun[T meta.Object](t *testing.T) {
 	var obj T
 	t.Run(reflect.TypeOf(obj).String(), func(t *testing.T) {
-		c := kclient.NewFakeClient()
+		t.Run("Item", func(t *testing.T) {
+			NewItem(t, operation.Update, NewMetaObject[T](t, "test", "test"))
+		})
+		t.Run("K8S", func(t *testing.T) {
+			c := kclient.NewFakeClient()
 
-		obj := NewMetaObject[T](t, "test", "test")
+			obj := NewMetaObject[T](t, "test", "test")
 
-		require.NotNil(t, obj)
+			require.NotNil(t, obj)
 
-		refresh := CreateObjects(t, c.Kubernetes(), c.Arango(), &obj)
+			refresh := CreateObjects(t, c.Kubernetes(), c.Arango(), &obj)
 
-		refresh(t)
+			refresh(t)
+		})
 	})
 }
 
 func Test_NewMetaObject(t *testing.T) {
+	NewMetaObjectRun[*core.Secret](t)
 	NewMetaObjectRun[*api.ArangoDeployment](t)
 	NewMetaObjectRun[*api.ArangoClusterSynchronization](t)
 	NewMetaObjectRun[*backupApi.ArangoBackup](t)