diff --git a/chart/kube-arangodb/templates/deployment-operator/role.yaml b/chart/kube-arangodb/templates/deployment-operator/role.yaml index 3549ff08e..fb1ce3c4e 100644 --- a/chart/kube-arangodb/templates/deployment-operator/role.yaml +++ b/chart/kube-arangodb/templates/deployment-operator/role.yaml @@ -17,14 +17,14 @@ rules: resources: ["arangodeployments"] verbs: ["*"] - apiGroups: [""] - resources: ["pods", "services", "endpoints", "persistentvolumeclaims", "events", "secrets"] + resources: ["pods", "services", "endpoints", "persistentvolumeclaims", "events", "secrets", "serviceaccounts"] verbs: ["*"] - apiGroups: ["apps"] resources: ["deployments", "replicasets"] verbs: ["get"] - apiGroups: ["policy"] resources: ["poddisruptionbudgets"] - verbs: ["get", "create", "delete"] + verbs: ["*"] - apiGroups: ["backup.arangodb.com"] resources: ["arangobackuppolicies", "arangobackups"] verbs: ["get", "list", "watch"] diff --git a/docs/Manual/Deployment/Kubernetes/DeploymentResource.md b/docs/Manual/Deployment/Kubernetes/DeploymentResource.md index e1ba81d4f..b1eb1a419 100644 --- a/docs/Manual/Deployment/Kubernetes/DeploymentResource.md +++ b/docs/Manual/Deployment/Kubernetes/DeploymentResource.md @@ -100,6 +100,10 @@ Possible values are: This setting specifies the list of image pull secrets for the docker image to use for all ArangoDB servers. +### `spec.annotations: map[string]string` + +This setting set specified annotations to all ArangoDeployment owned resources (pods, services, PVC's, PDB's). + ### `spec.storageEngine: string` This setting specifies the type of storage engine used for all servers @@ -517,6 +521,10 @@ rules: If you are using a different service account, please grant these rights to that service account. +### `spec..annotations: map[string]string` + +This setting set annotations overrides for pods in this group. Annotations are merged with `spec.annotations`. + ### `spec..priorityClassName: string` Priority class name for pods of this group. Will be forwarded to the pod spec. [Kubernetes documentation](https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/) diff --git a/pkg/apis/backup/v1/backup_policy.go b/pkg/apis/backup/v1/backup_policy.go index 3980130c0..1800c30bd 100644 --- a/pkg/apis/backup/v1/backup_policy.go +++ b/pkg/apis/backup/v1/backup_policy.go @@ -70,7 +70,8 @@ func (a *ArangoBackupPolicy) NewBackup(d *deployment.ArangoDeployment) *ArangoBa Name: fmt.Sprintf("%s-%s", d.Name, utils.RandomString(8)), Namespace: a.Namespace, - Labels: d.Labels, + Labels: d.Labels, + Annotations: d.Annotations, Finalizers: []string{ FinalizerArangoBackup, diff --git a/pkg/apis/backup/v1alpha/backup_policy.go b/pkg/apis/backup/v1alpha/backup_policy.go index 27f747bcb..5ac50229f 100644 --- a/pkg/apis/backup/v1alpha/backup_policy.go +++ b/pkg/apis/backup/v1alpha/backup_policy.go @@ -70,7 +70,8 @@ func (a *ArangoBackupPolicy) NewBackup(d *deployment.ArangoDeployment) *ArangoBa Name: fmt.Sprintf("%s-%s", d.Name, utils.RandomString(8)), Namespace: a.Namespace, - Labels: d.Labels, + Labels: d.Labels, + Annotations: d.Annotations, Finalizers: []string{ FinalizerArangoBackup, diff --git a/pkg/apis/deployment/v1/deployment_spec.go b/pkg/apis/deployment/v1/deployment_spec.go index 701fc9726..af8d5d80d 100644 --- a/pkg/apis/deployment/v1/deployment_spec.go +++ b/pkg/apis/deployment/v1/deployment_spec.go @@ -56,9 +56,12 @@ type DeploymentSpec struct { DowntimeAllowed *bool `json:"downtimeAllowed,omitempty"` DisableIPv6 *bool `json:"disableIPv6,omitempty"` - NetworkAttachedVolumes *bool `json:"networkAttachedVolumes,omitempty"` + NetworkAttachedVolumes *bool `json:"networkAttachedVolumes,omitempty"` - RestoreFrom *string `json:"restoreFrom,omitempty"` + // Annotations specified the annotations added to all resources + Annotations map[string]string `json:"annotations,omitempty"` + + RestoreFrom *string `json:"restoreFrom,omitempty"` ExternalAccess ExternalAccessSpec `json:"externalAccess"` RocksDB RocksDBSpec `json:"rocksdb"` @@ -105,6 +108,11 @@ func (s DeploymentSpec) GetEnvironment() Environment { return EnvironmentOrDefault(s.Environment) } +// GetAnnotations returns the annotations of this group +func (s DeploymentSpec) GetAnnotations() map[string]string { + return s.Annotations +} + // GetStorageEngine returns the value of storageEngine. func (s DeploymentSpec) GetStorageEngine() StorageEngine { return StorageEngineOrDefault(s.StorageEngine) diff --git a/pkg/apis/deployment/v1/server_group.go b/pkg/apis/deployment/v1/server_group.go index 773cd1199..80937d78f 100644 --- a/pkg/apis/deployment/v1/server_group.go +++ b/pkg/apis/deployment/v1/server_group.go @@ -22,17 +22,32 @@ package v1 -import time "time" +import "time" type ServerGroup int const ( + ServerGroupUnknown ServerGroup = 0 ServerGroupSingle ServerGroup = 1 ServerGroupAgents ServerGroup = 2 ServerGroupDBServers ServerGroup = 3 ServerGroupCoordinators ServerGroup = 4 ServerGroupSyncMasters ServerGroup = 5 ServerGroupSyncWorkers ServerGroup = 6 + + ServerGroupSingleString = "single" + ServerGroupAgentsString = "agent" + ServerGroupDBServersString = "dbserver" + ServerGroupCoordinatorsString = "coordinator" + ServerGroupSyncMastersString = "syncmaster" + ServerGroupSyncWorkersString = "syncworker" + + ServerGroupSingleAbbreviatedString = "sngl" + ServerGroupAgentsAbbreviatedString = "agnt" + ServerGroupDBServersAbbreviatedString = "prmr" + ServerGroupCoordinatorsAbbreviatedString = "crdn" + ServerGroupSyncMastersAbbreviatedString = "syma" + ServerGroupSyncWorkersAbbreviatedString = "sywo" ) var ( @@ -51,17 +66,17 @@ var ( func (g ServerGroup) AsRole() string { switch g { case ServerGroupSingle: - return "single" + return ServerGroupSingleString case ServerGroupAgents: - return "agent" + return ServerGroupAgentsString case ServerGroupDBServers: - return "dbserver" + return ServerGroupDBServersString case ServerGroupCoordinators: - return "coordinator" + return ServerGroupCoordinatorsString case ServerGroupSyncMasters: - return "syncmaster" + return ServerGroupSyncMastersString case ServerGroupSyncWorkers: - return "syncworker" + return ServerGroupSyncWorkersString default: return "?" } @@ -71,17 +86,17 @@ func (g ServerGroup) AsRole() string { func (g ServerGroup) AsRoleAbbreviated() string { switch g { case ServerGroupSingle: - return "sngl" + return ServerGroupSingleAbbreviatedString case ServerGroupAgents: - return "agnt" + return ServerGroupAgentsAbbreviatedString case ServerGroupDBServers: - return "prmr" + return ServerGroupDBServersAbbreviatedString case ServerGroupCoordinators: - return "crdn" + return ServerGroupCoordinatorsAbbreviatedString case ServerGroupSyncMasters: - return "syma" + return ServerGroupSyncMastersAbbreviatedString case ServerGroupSyncWorkers: - return "sywo" + return ServerGroupSyncWorkersAbbreviatedString default: return "?" } @@ -140,3 +155,43 @@ func (g ServerGroup) IsExportMetrics() bool { return false } } + +// ServerGroupFromAbbreviatedRole returns ServerGroup from abbreviated role +func ServerGroupFromAbbreviatedRole(label string) ServerGroup { + switch label { + case ServerGroupSingleAbbreviatedString: + return ServerGroupSingle + case ServerGroupAgentsAbbreviatedString: + return ServerGroupAgents + case ServerGroupDBServersAbbreviatedString: + return ServerGroupDBServers + case ServerGroupCoordinatorsAbbreviatedString: + return ServerGroupCoordinators + case ServerGroupSyncMastersAbbreviatedString: + return ServerGroupSyncMasters + case ServerGroupSyncWorkersAbbreviatedString: + return ServerGroupSyncWorkers + default: + return ServerGroupUnknown + } +} + +// ServerGroupFromAbbreviatedRole returns ServerGroup from role +func ServerGroupFromRole(label string) ServerGroup { + switch label { + case ServerGroupSingleString: + return ServerGroupSingle + case ServerGroupAgentsString: + return ServerGroupAgents + case ServerGroupDBServersString: + return ServerGroupDBServers + case ServerGroupCoordinatorsString: + return ServerGroupCoordinators + case ServerGroupSyncMastersString: + return ServerGroupSyncMasters + case ServerGroupSyncWorkersString: + return ServerGroupSyncWorkers + default: + return ServerGroupUnknown + } +} diff --git a/pkg/apis/deployment/v1/server_group_spec.go b/pkg/apis/deployment/v1/server_group_spec.go index 853c69152..9f362a2ef 100644 --- a/pkg/apis/deployment/v1/server_group_spec.go +++ b/pkg/apis/deployment/v1/server_group_spec.go @@ -52,6 +52,8 @@ type ServerGroupSpec struct { Resources v1.ResourceRequirements `json:"resources,omitempty"` // Tolerations specifies the tolerations added to Pods in this group. Tolerations []v1.Toleration `json:"tolerations,omitempty"` + // Annotations specified the annotations added to Pods in this group. + Annotations map[string]string `json:"annotations,omitempty"` // ServiceAccountName specifies the name of the service account used for Pods in this group. ServiceAccountName *string `json:"serviceAccountName,omitempty"` // NodeSelector speficies a set of selectors for nodes @@ -133,6 +135,11 @@ func (s ServerGroupSpec) GetNodeSelector() map[string]string { return s.NodeSelector } +// GetAnnotations returns the annotations of this group +func (s ServerGroupSpec) GetAnnotations() map[string]string { + return s.Annotations +} + // GetArgs returns the value of args. func (s ServerGroupSpec) GetArgs() []string { return s.Args diff --git a/pkg/apis/deployment/v1/zz_generated.deepcopy.go b/pkg/apis/deployment/v1/zz_generated.deepcopy.go index 233fbbb64..b3caa83fd 100644 --- a/pkg/apis/deployment/v1/zz_generated.deepcopy.go +++ b/pkg/apis/deployment/v1/zz_generated.deepcopy.go @@ -27,7 +27,7 @@ package v1 import ( time "time" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -270,7 +270,7 @@ func (in *DeploymentSpec) DeepCopyInto(out *DeploymentSpec) { } if in.ImagePullPolicy != nil { in, out := &in.ImagePullPolicy, &out.ImagePullPolicy - *out = new(v1.PullPolicy) + *out = new(corev1.PullPolicy) **out = **in } if in.ImagePullSecrets != nil { @@ -293,6 +293,13 @@ func (in *DeploymentSpec) DeepCopyInto(out *DeploymentSpec) { *out = new(bool) **out = **in } + if in.Annotations != nil { + in, out := &in.Annotations, &out.Annotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } if in.RestoreFrom != nil { in, out := &in.RestoreFrom, &out.RestoreFrom *out = new(string) @@ -563,7 +570,7 @@ func (in *MemberStatus) DeepCopyInto(out *MemberStatus) { } if in.SideCarSpecs != nil { in, out := &in.SideCarSpecs, &out.SideCarSpecs - *out = make(map[string]v1.Container, len(*in)) + *out = make(map[string]corev1.Container, len(*in)) for key, val := range *in { (*out)[key] = *val.DeepCopy() } @@ -834,11 +841,18 @@ func (in *ServerGroupSpec) DeepCopyInto(out *ServerGroupSpec) { in.Resources.DeepCopyInto(&out.Resources) if in.Tolerations != nil { in, out := &in.Tolerations, &out.Tolerations - *out = make([]v1.Toleration, len(*in)) + *out = make([]corev1.Toleration, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.Annotations != nil { + in, out := &in.Annotations, &out.Annotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } if in.ServiceAccountName != nil { in, out := &in.ServiceAccountName, &out.ServiceAccountName *out = new(string) @@ -858,12 +872,12 @@ func (in *ServerGroupSpec) DeepCopyInto(out *ServerGroupSpec) { } if in.VolumeClaimTemplate != nil { in, out := &in.VolumeClaimTemplate, &out.VolumeClaimTemplate - *out = new(v1.PersistentVolumeClaim) + *out = new(corev1.PersistentVolumeClaim) (*in).DeepCopyInto(*out) } if in.Sidecars != nil { in, out := &in.Sidecars, &out.Sidecars - *out = make([]v1.Container, len(*in)) + *out = make([]corev1.Container, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } diff --git a/pkg/apis/deployment/v1alpha/deployment_spec.go b/pkg/apis/deployment/v1alpha/deployment_spec.go index b851c25ba..7e7cf162f 100644 --- a/pkg/apis/deployment/v1alpha/deployment_spec.go +++ b/pkg/apis/deployment/v1alpha/deployment_spec.go @@ -56,9 +56,12 @@ type DeploymentSpec struct { DowntimeAllowed *bool `json:"downtimeAllowed,omitempty"` DisableIPv6 *bool `json:"disableIPv6,omitempty"` - NetworkAttachedVolumes *bool `json:"networkAttachedVolumes,omitempty"` + NetworkAttachedVolumes *bool `json:"networkAttachedVolumes,omitempty"` - RestoreFrom *string `json:"restoreFrom,omitempty"` + // Annotations specified the annotations added to Pods in this group. + Annotations map[string]string `json:"annotations,omitempty"` + + RestoreFrom *string `json:"restoreFrom,omitempty"` ExternalAccess ExternalAccessSpec `json:"externalAccess"` RocksDB RocksDBSpec `json:"rocksdb"` @@ -105,6 +108,11 @@ func (s DeploymentSpec) GetEnvironment() Environment { return EnvironmentOrDefault(s.Environment) } +// GetAnnotations returns the annotations of this group +func (s DeploymentSpec) GetAnnotations() map[string]string { + return s.Annotations +} + // GetStorageEngine returns the value of storageEngine. func (s DeploymentSpec) GetStorageEngine() StorageEngine { return StorageEngineOrDefault(s.StorageEngine) diff --git a/pkg/apis/deployment/v1alpha/server_group.go b/pkg/apis/deployment/v1alpha/server_group.go index d6ceefe16..3c75f4d6f 100644 --- a/pkg/apis/deployment/v1alpha/server_group.go +++ b/pkg/apis/deployment/v1alpha/server_group.go @@ -27,12 +27,27 @@ import time "time" type ServerGroup int const ( + ServerGroupUnknown ServerGroup = 0 ServerGroupSingle ServerGroup = 1 ServerGroupAgents ServerGroup = 2 ServerGroupDBServers ServerGroup = 3 ServerGroupCoordinators ServerGroup = 4 ServerGroupSyncMasters ServerGroup = 5 ServerGroupSyncWorkers ServerGroup = 6 + + ServerGroupSingleString = "single" + ServerGroupAgentsString = "agent" + ServerGroupDBServersString = "dbserver" + ServerGroupCoordinatorsString = "coordinator" + ServerGroupSyncMastersString = "syncmaster" + ServerGroupSyncWorkersString = "syncworker" + + ServerGroupSingleAbbreviatedString = "sngl" + ServerGroupAgentsAbbreviatedString = "agnt" + ServerGroupDBServersAbbreviatedString = "prmr" + ServerGroupCoordinatorsAbbreviatedString = "crdn" + ServerGroupSyncMastersAbbreviatedString = "syma" + ServerGroupSyncWorkersAbbreviatedString = "sywo" ) var ( @@ -51,17 +66,17 @@ var ( func (g ServerGroup) AsRole() string { switch g { case ServerGroupSingle: - return "single" + return ServerGroupSingleString case ServerGroupAgents: - return "agent" + return ServerGroupAgentsString case ServerGroupDBServers: - return "dbserver" + return ServerGroupDBServersString case ServerGroupCoordinators: - return "coordinator" + return ServerGroupCoordinatorsString case ServerGroupSyncMasters: - return "syncmaster" + return ServerGroupSyncMastersString case ServerGroupSyncWorkers: - return "syncworker" + return ServerGroupSyncWorkersString default: return "?" } @@ -71,17 +86,17 @@ func (g ServerGroup) AsRole() string { func (g ServerGroup) AsRoleAbbreviated() string { switch g { case ServerGroupSingle: - return "sngl" + return ServerGroupSingleAbbreviatedString case ServerGroupAgents: - return "agnt" + return ServerGroupAgentsAbbreviatedString case ServerGroupDBServers: - return "prmr" + return ServerGroupDBServersAbbreviatedString case ServerGroupCoordinators: - return "crdn" + return ServerGroupCoordinatorsAbbreviatedString case ServerGroupSyncMasters: - return "syma" + return ServerGroupSyncMastersAbbreviatedString case ServerGroupSyncWorkers: - return "sywo" + return ServerGroupSyncWorkersAbbreviatedString default: return "?" } @@ -140,3 +155,43 @@ func (g ServerGroup) IsExportMetrics() bool { return false } } + +// ServerGroupFromAbbreviatedRole returns ServerGroup from abbreviated role +func ServerGroupFromAbbreviatedRole(label string) ServerGroup { + switch label { + case ServerGroupSingleAbbreviatedString: + return ServerGroupSingle + case ServerGroupAgentsAbbreviatedString: + return ServerGroupAgents + case ServerGroupDBServersAbbreviatedString: + return ServerGroupDBServers + case ServerGroupCoordinatorsAbbreviatedString: + return ServerGroupCoordinators + case ServerGroupSyncMastersAbbreviatedString: + return ServerGroupSyncMasters + case ServerGroupSyncWorkersAbbreviatedString: + return ServerGroupSyncWorkers + default: + return ServerGroupUnknown + } +} + +// ServerGroupFromAbbreviatedRole returns ServerGroup from role +func ServerGroupFromRole(label string) ServerGroup { + switch label { + case ServerGroupSingleString: + return ServerGroupSingle + case ServerGroupAgentsString: + return ServerGroupAgents + case ServerGroupDBServersString: + return ServerGroupDBServers + case ServerGroupCoordinatorsString: + return ServerGroupCoordinators + case ServerGroupSyncMastersString: + return ServerGroupSyncMasters + case ServerGroupSyncWorkersString: + return ServerGroupSyncWorkers + default: + return ServerGroupUnknown + } +} diff --git a/pkg/apis/deployment/v1alpha/server_group_spec.go b/pkg/apis/deployment/v1alpha/server_group_spec.go index 5218bc417..28bb2af61 100644 --- a/pkg/apis/deployment/v1alpha/server_group_spec.go +++ b/pkg/apis/deployment/v1alpha/server_group_spec.go @@ -52,6 +52,8 @@ type ServerGroupSpec struct { Resources v1.ResourceRequirements `json:"resources,omitempty"` // Tolerations specifies the tolerations added to Pods in this group. Tolerations []v1.Toleration `json:"tolerations,omitempty"` + // Annotations specified the annotations added to Pods in this group. + Annotations map[string]string `json:"annotations,omitempty"` // ServiceAccountName specifies the name of the service account used for Pods in this group. ServiceAccountName *string `json:"serviceAccountName,omitempty"` // NodeSelector speficies a set of selectors for nodes @@ -133,6 +135,11 @@ func (s ServerGroupSpec) GetNodeSelector() map[string]string { return s.NodeSelector } +// GetAnnotations returns the annotations of this group +func (s ServerGroupSpec) GetAnnotations() map[string]string { + return s.Annotations +} + // GetArgs returns the value of args. func (s ServerGroupSpec) GetArgs() []string { return s.Args diff --git a/pkg/apis/deployment/v1alpha/zz_generated.deepcopy.go b/pkg/apis/deployment/v1alpha/zz_generated.deepcopy.go index f8c593311..1cd3518cc 100644 --- a/pkg/apis/deployment/v1alpha/zz_generated.deepcopy.go +++ b/pkg/apis/deployment/v1alpha/zz_generated.deepcopy.go @@ -293,6 +293,13 @@ func (in *DeploymentSpec) DeepCopyInto(out *DeploymentSpec) { *out = new(bool) **out = **in } + if in.Annotations != nil { + in, out := &in.Annotations, &out.Annotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } if in.RestoreFrom != nil { in, out := &in.RestoreFrom, &out.RestoreFrom *out = new(string) @@ -839,6 +846,13 @@ func (in *ServerGroupSpec) DeepCopyInto(out *ServerGroupSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.Annotations != nil { + in, out := &in.Annotations, &out.Annotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } if in.ServiceAccountName != nil { in, out := &in.ServiceAccountName, &out.ServiceAccountName *out = new(string) diff --git a/pkg/deployment/deployment.go b/pkg/deployment/deployment.go index 5b4e36c70..c38bde99f 100644 --- a/pkg/deployment/deployment.go +++ b/pkg/deployment/deployment.go @@ -244,6 +244,10 @@ func (d *Deployment) run() { log.Info().Msg("start running...") } + if err := d.resources.EnsureAnnotations(); err != nil { + log.Warn().Err(err).Msg("unable to update annotations") + } + d.lookForServiceMonitorCRD() inspectionInterval := maxInspectionInterval diff --git a/pkg/deployment/deployment_inspector.go b/pkg/deployment/deployment_inspector.go index 137764bd9..fc6ae9052 100644 --- a/pkg/deployment/deployment_inspector.go +++ b/pkg/deployment/deployment_inspector.go @@ -172,6 +172,11 @@ func (d *Deployment) inspectDeployment(lastInterval util.Interval) util.Interval d.CreateEvent(k8sutil.NewErrorEvent("PDB creation failed", err, d.apiObject)) } + if err := d.resources.EnsureAnnotations(); err != nil { + hasError = true + d.CreateEvent(k8sutil.NewErrorEvent("Annotation update failed", err, d.apiObject)) + } + // Create access packages if err := d.createAccessPackages(); err != nil { hasError = true diff --git a/pkg/deployment/resources/annotations.go b/pkg/deployment/resources/annotations.go new file mode 100644 index 000000000..f20cdaf26 --- /dev/null +++ b/pkg/deployment/resources/annotations.go @@ -0,0 +1,340 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Adam Janikowski +// + +package resources + +import ( + "time" + + "github.com/arangodb/kube-arangodb/pkg/apis/deployment" + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" + "github.com/arangodb/kube-arangodb/pkg/backup/utils" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + "github.com/rs/zerolog/log" + core "k8s.io/api/core/v1" + policy "k8s.io/api/policy/v1beta1" + meta "k8s.io/apimachinery/pkg/apis/meta/v1" + typedCore "k8s.io/client-go/kubernetes/typed/core/v1" + policyTyped "k8s.io/client-go/kubernetes/typed/policy/v1beta1" +) + +func (r *Resources) EnsureAnnotations() error { + kubecli := r.context.GetKubeCli() + + log.Info().Msgf("Ensuring annotations") + + if err := ensureSecretsAnnotations(kubecli.CoreV1().Secrets(r.context.GetNamespace()), + deployment.ArangoDeploymentResourceKind, + r.context.GetAPIObject().GetName(), + r.context.GetAPIObject().GetNamespace(), + r.context.GetSpec().Annotations); err != nil { + return err + } + + if err := ensureServiceAccountsAnnotations(kubecli.CoreV1().ServiceAccounts(r.context.GetNamespace()), + deployment.ArangoDeploymentResourceKind, + r.context.GetAPIObject().GetName(), + r.context.GetAPIObject().GetNamespace(), + r.context.GetSpec().Annotations); err != nil { + return err + } + + if err := ensureServicesAnnotations(kubecli.CoreV1().Services(r.context.GetNamespace()), + deployment.ArangoDeploymentResourceKind, + r.context.GetAPIObject().GetName(), + r.context.GetAPIObject().GetNamespace(), + r.context.GetSpec().Annotations); err != nil { + return err + } + + if err := ensurePdbsAnnotations(kubecli.PolicyV1beta1().PodDisruptionBudgets(r.context.GetNamespace()), + deployment.ArangoDeploymentResourceKind, + r.context.GetAPIObject().GetName(), + r.context.GetAPIObject().GetNamespace(), + r.context.GetSpec().Annotations); err != nil { + return err + } + + if err := ensurePvcsAnnotations(kubecli.CoreV1().PersistentVolumeClaims(r.context.GetNamespace()), + deployment.ArangoDeploymentResourceKind, + r.context.GetAPIObject().GetName(), + r.context.GetAPIObject().GetNamespace(), + r.context.GetSpec().Annotations); err != nil { + return err + } + + if err := ensurePodsAnnotations(kubecli.CoreV1().Pods(r.context.GetNamespace()), + deployment.ArangoDeploymentResourceKind, + r.context.GetAPIObject().GetName(), + r.context.GetAPIObject().GetNamespace(), + r.context.GetSpec().Annotations, + r.context.GetSpec()); err != nil { + return err + } + + return nil +} + +func ensureSecretsAnnotations(client typedCore.SecretInterface, kind, name, namespace string, annotations map[string]string) error { + secrets, err := k8sutil.GetSecretsForParent(client, + kind, + name, + namespace) + if err != nil { + return err + } + + for _, secret := range secrets { + if !k8sutil.CompareAnnotations(secret.GetAnnotations(), annotations) { + log.Info().Msgf("Replacing annotations for Secret %s", secret.Name) + if err = setSecretAnnotations(client, secret, annotations); err != nil { + return err + } + } + } + + return nil +} + +func setSecretAnnotations(client typedCore.SecretInterface, secret *core.Secret, annotations map[string]string) error { + return utils.Retry(5, 200*time.Millisecond, func() error { + currentSecret, err := client.Get(secret.Name, meta.GetOptions{}) + if err != nil { + return err + } + + currentSecret.Annotations = k8sutil.MergeAnnotations(k8sutil.GetSecuredAnnotations(currentSecret.Annotations), annotations) + + _, err = client.Update(currentSecret) + if err != nil { + return err + } + + return nil + }) +} + +func ensureServiceAccountsAnnotations(client typedCore.ServiceAccountInterface, kind, name, namespace string, annotations map[string]string) error { + serviceAccounts, err := k8sutil.GetServiceAccountsForParent(client, + kind, + name, + namespace) + if err != nil { + return err + } + + for _, serviceAccount := range serviceAccounts { + if !k8sutil.CompareAnnotations(serviceAccount.GetAnnotations(), annotations) { + log.Info().Msgf("Replacing annotations for ServiceAccount %s", serviceAccount.Name) + if err = setServiceAccountAnnotations(client, serviceAccount, annotations); err != nil { + return err + } + } + } + + return nil +} + +func setServiceAccountAnnotations(client typedCore.ServiceAccountInterface, serviceAccount *core.ServiceAccount, annotations map[string]string) error { + return utils.Retry(5, 200*time.Millisecond, func() error { + currentServiceAccount, err := client.Get(serviceAccount.Name, meta.GetOptions{}) + if err != nil { + return err + } + + currentServiceAccount.Annotations = k8sutil.MergeAnnotations(k8sutil.GetSecuredAnnotations(currentServiceAccount.Annotations), annotations) + + _, err = client.Update(currentServiceAccount) + if err != nil { + return err + } + + return nil + }) +} + +func ensureServicesAnnotations(client typedCore.ServiceInterface, kind, name, namespace string, annotations map[string]string) error { + services, err := k8sutil.GetServicesForParent(client, + kind, + name, + namespace) + if err != nil { + return err + } + + for _, service := range services { + if !k8sutil.CompareAnnotations(service.GetAnnotations(), annotations) { + log.Info().Msgf("Replacing annotations for Service %s", service.Name) + if err = setServiceAnnotations(client, service, annotations); err != nil { + return err + } + } + } + + return nil +} + +func setServiceAnnotations(client typedCore.ServiceInterface, service *core.Service, annotations map[string]string) error { + return utils.Retry(5, 200*time.Millisecond, func() error { + currentService, err := client.Get(service.Name, meta.GetOptions{}) + if err != nil { + return err + } + + currentService.Annotations = k8sutil.MergeAnnotations(k8sutil.GetSecuredAnnotations(currentService.Annotations), annotations) + + _, err = client.Update(currentService) + if err != nil { + return err + } + + return nil + }) +} + +func ensurePdbsAnnotations(client policyTyped.PodDisruptionBudgetInterface, kind, name, namespace string, annotations map[string]string) error { + podDisruptionBudgets, err := k8sutil.GetPDBForParent(client, + kind, + name, + namespace) + if err != nil { + return err + } + + for _, podDisruptionBudget := range podDisruptionBudgets { + if !k8sutil.CompareAnnotations(podDisruptionBudget.GetAnnotations(), annotations) { + log.Info().Msgf("Replacing annotations for PDB %s", podDisruptionBudget.Name) + if err = setPdbAnnotations(client, podDisruptionBudget, annotations); err != nil { + return err + } + } + } + + return nil +} + +func setPdbAnnotations(client policyTyped.PodDisruptionBudgetInterface, podDisruptionBudget *policy.PodDisruptionBudget, annotations map[string]string) error { + return utils.Retry(5, 200*time.Millisecond, func() error { + currentPodDistributionBudget, err := client.Get(podDisruptionBudget.Name, meta.GetOptions{}) + if err != nil { + return err + } + + currentPodDistributionBudget.Annotations = k8sutil.MergeAnnotations(k8sutil.GetSecuredAnnotations(currentPodDistributionBudget.Annotations), annotations) + + _, err = client.Update(currentPodDistributionBudget) + if err != nil { + return err + } + + return nil + }) +} + +func ensurePvcsAnnotations(client typedCore.PersistentVolumeClaimInterface, kind, name, namespace string, annotations map[string]string) error { + persistentVolumeClaims, err := k8sutil.GetPVCForParent(client, + kind, + name, + namespace) + if err != nil { + return err + } + + for _, persistentVolumeClaim := range persistentVolumeClaims { + if !k8sutil.CompareAnnotations(persistentVolumeClaim.GetAnnotations(), annotations) { + log.Info().Msgf("Replacing annotations for PVC %s", persistentVolumeClaim.Name) + if err = setPvcAnnotations(client, persistentVolumeClaim, annotations); err != nil { + return err + } + } + } + + return nil +} + +func setPvcAnnotations(client typedCore.PersistentVolumeClaimInterface, persistentVolumeClaim *core.PersistentVolumeClaim, annotations map[string]string) error { + return utils.Retry(5, 200*time.Millisecond, func() error { + currentVolumeClaim, err := client.Get(persistentVolumeClaim.Name, meta.GetOptions{}) + if err != nil { + return err + } + + currentVolumeClaim.Annotations = k8sutil.MergeAnnotations(k8sutil.GetSecuredAnnotations(currentVolumeClaim.Annotations), annotations) + + _, err = client.Update(currentVolumeClaim) + if err != nil { + return err + } + + return nil + }) +} + +func getPodGroup(pod *core.Pod) api.ServerGroup { + if pod.Labels == nil { + return api.ServerGroupUnknown + } + + return api.ServerGroupFromRole(pod.Labels[k8sutil.LabelKeyRole]) +} + +func ensurePodsAnnotations(client typedCore.PodInterface, kind, name, namespace string, annotations map[string]string, spec api.DeploymentSpec) error { + pods, err := k8sutil.GetPodsForParent(client, + kind, + name, + namespace) + if err != nil { + return err + } + + for _, pod := range pods { + group := getPodGroup(pod) + mergedAnnotations := k8sutil.MergeAnnotations(annotations, spec.GetServerGroupSpec(group).Annotations) + + if !k8sutil.CompareAnnotations(pod.GetAnnotations(), mergedAnnotations) { + log.Info().Msgf("Replacing annotations for Pod %s", pod.Name) + if err = setPodAnnotations(client, pod, mergedAnnotations); err != nil { + return err + } + } + } + + return nil +} + +func setPodAnnotations(client typedCore.PodInterface, pod *core.Pod, annotations map[string]string) error { + return utils.Retry(5, 200*time.Millisecond, func() error { + currentPod, err := client.Get(pod.Name, meta.GetOptions{}) + if err != nil { + return err + } + + currentPod.Annotations = k8sutil.MergeAnnotations(k8sutil.GetSecuredAnnotations(currentPod.Annotations), annotations) + + _, err = client.Update(currentPod) + if err != nil { + return err + } + + return nil + }) +} diff --git a/pkg/util/k8sutil/annotations.go b/pkg/util/k8sutil/annotations.go new file mode 100644 index 000000000..756db0879 --- /dev/null +++ b/pkg/util/k8sutil/annotations.go @@ -0,0 +1,187 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Adam Janikowski +// + +package k8sutil + +import ( + core "k8s.io/api/core/v1" + policy "k8s.io/api/policy/v1beta1" + meta "k8s.io/apimachinery/pkg/apis/meta/v1" + typedCore "k8s.io/client-go/kubernetes/typed/core/v1" + policyTyped "k8s.io/client-go/kubernetes/typed/policy/v1beta1" +) + +func IsChildResource(kind, name, namespace string, resource meta.Object) bool { + if resource == nil { + return false + } + + if namespace != resource.GetNamespace() { + return false + } + + ownerRef := resource.GetOwnerReferences() + + if ownerRef == nil || len(ownerRef) == 0 { + return false + } + + for _, owner := range ownerRef { + if owner.Kind != kind { + continue + } + + if owner.Name != name { + continue + } + + return true + } + + return false +} + +func GetSecretsForParent(client typedCore.SecretInterface, kind, name, namespace string) ([]*core.Secret, error) { + secrets, err := client.List(meta.ListOptions{}) + if err != nil { + return nil, err + } + + if len(secrets.Items) == 0 { + return []*core.Secret{}, nil + } + + childSecrets := make([]*core.Secret, 0, len(secrets.Items)) + + for _, secret := range secrets.Items { + if IsChildResource(kind, name, namespace, &secret) { + childSecrets = append(childSecrets, secret.DeepCopy()) + } + } + + return childSecrets, nil +} + +func GetPDBForParent(client policyTyped.PodDisruptionBudgetInterface, kind, name, namespace string) ([]*policy.PodDisruptionBudget, error) { + pdbs, err := client.List(meta.ListOptions{}) + if err != nil { + return nil, err + } + + if len(pdbs.Items) == 0 { + return []*policy.PodDisruptionBudget{}, nil + } + + childPdbs := make([]*policy.PodDisruptionBudget, 0, len(pdbs.Items)) + + for _, pdb := range pdbs.Items { + if IsChildResource(kind, name, namespace, &pdb) { + childPdbs = append(childPdbs, pdb.DeepCopy()) + } + } + + return childPdbs, nil +} + +func GetPVCForParent(client typedCore.PersistentVolumeClaimInterface, kind, name, namespace string) ([]*core.PersistentVolumeClaim, error) { + pvcs, err := client.List(meta.ListOptions{}) + if err != nil { + return nil, err + } + + if len(pvcs.Items) == 0 { + return []*core.PersistentVolumeClaim{}, nil + } + + childPvcs := make([]*core.PersistentVolumeClaim, 0, len(pvcs.Items)) + + for _, pvc := range pvcs.Items { + if IsChildResource(kind, name, namespace, &pvc) { + childPvcs = append(childPvcs, pvc.DeepCopy()) + } + } + + return childPvcs, nil +} + +func GetServicesForParent(client typedCore.ServiceInterface, kind, name, namespace string) ([]*core.Service, error) { + services, err := client.List(meta.ListOptions{}) + if err != nil { + return nil, err + } + + if len(services.Items) == 0 { + return []*core.Service{}, nil + } + + childServices := make([]*core.Service, 0, len(services.Items)) + + for _, service := range services.Items { + if IsChildResource(kind, name, namespace, &service) { + childServices = append(childServices, service.DeepCopy()) + } + } + + return childServices, nil +} + +func GetServiceAccountsForParent(client typedCore.ServiceAccountInterface, kind, name, namespace string) ([]*core.ServiceAccount, error) { + serviceAccounts, err := client.List(meta.ListOptions{}) + if err != nil { + return nil, err + } + + if len(serviceAccounts.Items) == 0 { + return []*core.ServiceAccount{}, nil + } + + childServiceAccounts := make([]*core.ServiceAccount, 0, len(serviceAccounts.Items)) + + for _, serviceAccount := range serviceAccounts.Items { + if IsChildResource(kind, name, namespace, &serviceAccount) { + childServiceAccounts = append(childServiceAccounts, serviceAccount.DeepCopy()) + } + } + + return childServiceAccounts, nil +} + +func GetPodsForParent(client typedCore.PodInterface, kind, name, namespace string) ([]*core.Pod, error) { + podList, err := client.List(meta.ListOptions{}) + if err != nil { + return nil, err + } + + if len(podList.Items) == 0 { + return []*core.Pod{}, nil + } + + pods := make([]*core.Pod, 0, len(podList.Items)) + + for _, pod := range podList.Items { + if IsChildResource(kind, name, namespace, &pod) { + pods = append(pods, pod.DeepCopy()) + } + } + + return pods, nil +} diff --git a/pkg/util/k8sutil/map.go b/pkg/util/k8sutil/map.go new file mode 100644 index 000000000..923eb7631 --- /dev/null +++ b/pkg/util/k8sutil/map.go @@ -0,0 +1,150 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Adam Janikowski +// + +package k8sutil + +import "regexp" + +const ( + kubernetesAnnotationMatch = ".*kubernetes\\.io/.*" + arangoAnnotationMatch = ".*arangodb\\.com/" +) + +var ( + kubernetesAnnotationRegex *regexp.Regexp + arangoAnnotationRegex *regexp.Regexp +) + +func init() { + r, err := regexp.Compile(kubernetesAnnotationMatch) + if err != nil { + panic(err) + } + + kubernetesAnnotationRegex = r + + r, err = regexp.Compile(arangoAnnotationMatch) + if err != nil { + panic(err) + } + + arangoAnnotationRegex = r +} + +// MergeAnnotations into one annotations map +func MergeAnnotations(annotations ...map[string]string) map[string]string { + ret := map[string]string{} + + for _, annotationMap := range annotations { + if annotationMap == nil { + continue + } + + for annotation, value := range annotationMap { + ret[annotation] = value + } + } + + return ret +} + +// IsSecuredAnnotation check if annotation key is from secured namespace +func IsSecuredAnnotation(key string) bool { + return kubernetesAnnotationRegex.MatchString(key) || arangoAnnotationRegex.MatchString(key) +} + +func GetSecuredAnnotations(annotations map[string]string) map[string]string { + if annotations == nil { + return map[string]string{} + } + + filteredAnnotations := map[string]string{} + + for key, value := range annotations { + if !IsSecuredAnnotation(key) { + continue + } + + filteredAnnotations[key] = value + } + + return filteredAnnotations +} + +func filterActualAnnotations(actual, expected map[string]string) map[string]string { + if actual == nil { + return nil + } + + if expected == nil { + expected = map[string]string{} + } + + actualFiltered := map[string]string{} + + for key, value := range actual { + if _, ok := expected[key]; IsSecuredAnnotation(key) && !ok { + continue + } + + actualFiltered[key] = value + } + + return actualFiltered +} + +// CompareAnnotations will compare annotations, but will ignore secured annotations which are present in +// actual but not specified in expected map +func CompareAnnotations(actual, expected map[string]string) bool { + actualFiltered := filterActualAnnotations(actual, expected) + + if actualFiltered == nil && expected == nil { + return true + } + + if (actualFiltered == nil && expected != nil && len(expected) == 0) || + (expected == nil && actualFiltered != nil && len(actualFiltered) == 0) { + return true + } + + if actualFiltered == nil || expected == nil { + return false + } + + if len(actualFiltered) != len(expected) { + return false + } + + for key, value := range expected { + existingValue, existing := actualFiltered[key] + + if !existing { + return false + } + + if existingValue != value { + return false + } + } + + return true +} diff --git a/tests/annotations_test.go b/tests/annotations_test.go new file mode 100644 index 000000000..58dab4e4b --- /dev/null +++ b/tests/annotations_test.go @@ -0,0 +1,340 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Adam Janikowski +// + +package tests + +import ( + "testing" + "time" + + "github.com/arangodb/kube-arangodb/pkg/apis/deployment" + "github.com/arangodb/kube-arangodb/pkg/generated/clientset/versioned" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + "github.com/dchest/uniuri" + "github.com/rs/zerolog/log" + "github.com/stretchr/testify/require" + core "k8s.io/api/core/v1" + meta "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" + "github.com/arangodb/kube-arangodb/pkg/client" +) + +func addAnnotation(t *testing.T, kubeClient kubernetes.Interface, arangoClient versioned.Interface, depl *api.ArangoDeployment, annotations map[string]string) { + object, err := arangoClient.DatabaseV1().ArangoDeployments(depl.GetNamespace()).Get(depl.GetName(), meta.GetOptions{}) + require.NoError(t, err) + + object.Spec.Annotations = annotations + object.Spec.Coordinators.Annotations = depl.Spec.Coordinators.Annotations + + _, err = arangoClient.DatabaseV1().ArangoDeployments(depl.GetNamespace()).Update(object) + require.NoError(t, err) + + ensureAnnotations(t, kubeClient, object) +} + +func ensureAnnotationsTimeout(t *testing.T, client kubernetes.Interface, depl *api.ArangoDeployment) func() error { + return func() error { + if err := ensureSecretAnnotations(t, client, depl); err == nil || !isInterrupt(err) { + return err + } + + if err := ensurePDBAnnotation(t, client, depl); err == nil || !isInterrupt(err) { + return err + } + + if err := ensurePVCAnnotation(t, client, depl); err == nil || !isInterrupt(err) { + return err + } + + if err := ensureServiceAnnotation(t, client, depl); err == nil || !isInterrupt(err) { + return err + } + + if err := ensureServiceAccountAnnotation(t, client, depl); err == nil || !isInterrupt(err) { + return err + } + + if err := ensurePodAnnotations(t, client, depl); err == nil || !isInterrupt(err) { + return err + } + + return interrupt{} + } +} + +func ensureSecretAnnotations(t *testing.T, client kubernetes.Interface, depl *api.ArangoDeployment) error { + secrets, err := k8sutil.GetSecretsForParent(client.CoreV1().Secrets(depl.Namespace), deployment.ArangoDeploymentResourceKind, depl.Name, depl.Namespace) + require.NoError(t, err) + require.True(t, len(secrets) > 0) + for _, secret := range secrets { + if !k8sutil.CompareAnnotations(secret.GetAnnotations(), depl.Spec.Annotations) { + log.Info().Msgf("Annotations for Secret does not match on %s", secret.Name) + return nil + } + } + + return interrupt{} +} + +func getPodGroup(pod *core.Pod) api.ServerGroup { + if pod.Labels == nil { + return api.ServerGroupUnknown + } + + return api.ServerGroupFromRole(pod.Labels[k8sutil.LabelKeyRole]) +} + +func ensurePodAnnotations(t *testing.T, client kubernetes.Interface, depl *api.ArangoDeployment) error { + pods, err := k8sutil.GetPodsForParent(client.CoreV1().Pods(depl.Namespace), deployment.ArangoDeploymentResourceKind, depl.Name, depl.Namespace) + require.NoError(t, err) + require.True(t, len(pods) > 0) + for _, pod := range pods { + group := getPodGroup(pod) + combinedAnnotations := k8sutil.MergeAnnotations(depl.Spec.Annotations, depl.Spec.GetServerGroupSpec(group).Annotations) + if !k8sutil.CompareAnnotations(pod.GetAnnotations(), combinedAnnotations) { + log.Info().Msgf("Annotations for Pod does not match on %s", pod.Name) + return nil + } + } + + return interrupt{} +} + +func ensurePDBAnnotation(t *testing.T, client kubernetes.Interface, depl *api.ArangoDeployment) error { + podDisruptionBudgets, err := k8sutil.GetPDBForParent(client.PolicyV1beta1().PodDisruptionBudgets(depl.Namespace), deployment.ArangoDeploymentResourceKind, depl.Name, depl.Namespace) + require.NoError(t, err) + require.True(t, len(podDisruptionBudgets) > 0) + for _, podDisruptionBudget := range podDisruptionBudgets { + if !k8sutil.CompareAnnotations(podDisruptionBudget.GetAnnotations(), depl.Spec.Annotations) { + log.Info().Msgf("Annotations for PDB does not match on %s", podDisruptionBudget.Name) + return nil + } + } + + return interrupt{} +} + +func ensurePVCAnnotation(t *testing.T, client kubernetes.Interface, depl *api.ArangoDeployment) error { + persistentVolumeClaims, err := k8sutil.GetPVCForParent(client.CoreV1().PersistentVolumeClaims(depl.Namespace), deployment.ArangoDeploymentResourceKind, depl.Name, depl.Namespace) + require.NoError(t, err) + require.True(t, len(persistentVolumeClaims) > 0) + for _, persistentVolumeClaim := range persistentVolumeClaims { + if !k8sutil.CompareAnnotations(persistentVolumeClaim.GetAnnotations(), depl.Spec.Annotations) { + log.Info().Msgf("Annotations for PVC does not match on %s", persistentVolumeClaim.Name) + return nil + } + } + + return interrupt{} +} + +func ensureServiceAnnotation(t *testing.T, client kubernetes.Interface, depl *api.ArangoDeployment) error { + services, err := k8sutil.GetServicesForParent(client.CoreV1().Services(depl.Namespace), deployment.ArangoDeploymentResourceKind, depl.Name, depl.Namespace) + require.NoError(t, err) + require.True(t, len(services) > 0) + for _, service := range services { + if !k8sutil.CompareAnnotations(service.GetAnnotations(), depl.Spec.Annotations) { + log.Info().Msgf("Annotations for Service does not match on %s", service.Name) + return nil + } + } + + return interrupt{} +} + +func ensureServiceAccountAnnotation(t *testing.T, client kubernetes.Interface, depl *api.ArangoDeployment) error { + serviceAccounts, err := k8sutil.GetServiceAccountsForParent(client.CoreV1().ServiceAccounts(depl.Namespace), deployment.ArangoDeploymentResourceKind, depl.Name, depl.Namespace) + require.NoError(t, err) + for _, serviceAccount := range serviceAccounts { + if !k8sutil.CompareAnnotations(serviceAccount.GetAnnotations(), depl.Spec.Annotations) { + log.Info().Msgf("Annotations for Service Account does not match on %s", serviceAccount.Name) + return nil + } + } + + return interrupt{} +} + +func ensureAnnotations(t *testing.T, client kubernetes.Interface, depl *api.ArangoDeployment) { + if err := timeout(2*time.Second, 5*time.Minute, ensureAnnotationsTimeout(t, client, depl)); err != nil { + require.NoError(t, err) + } +} + +func TestAnnotations(t *testing.T) { + longOrSkip(t) + c := client.MustNewInCluster() + kubecli := mustNewKubeClient(t) + ns := getNamespace(t) + + // Prepare deployment config + depl := newDeployment("test-annotations-" + uniuri.NewLen(4)) + depl.Spec.Mode = api.NewMode(api.DeploymentModeCluster) + depl.Spec.Environment = api.NewEnvironment(api.EnvironmentProduction) + depl.Spec.SetDefaults(depl.GetName()) + + // Create deployment + depl, err := c.DatabaseV1().ArangoDeployments(ns).Create(depl) + require.NoError(t, err) + + defer deferedCleanupDeployment(c, depl.GetName(), ns) + + // Wait for deployment to be ready + if _, err := waitUntilDeployment(c, depl.GetName(), ns, deploymentIsReady()); err != nil { + t.Fatalf("Deployment not running in time: %v", err) + } + + t.Run("Add annotation", func(t *testing.T) { + annotations := map[string]string{ + "annotation": uniuri.NewLen(8), + } + + addAnnotation(t, kubecli, c, depl, annotations) + + addAnnotation(t, kubecli, c, depl, nil) + }) + + t.Run("Add kubernetes annotation", func(t *testing.T) { + key := "kubernetes.io/test-only-annotation" + + annotations := map[string]string{ + key: uniuri.NewLen(8), + "annotation": uniuri.NewLen(8), + } + + addAnnotation(t, kubecli, c, depl, annotations) + + addAnnotation(t, kubecli, c, depl, nil) + + secrets, err := k8sutil.GetSecretsForParent(kubecli.CoreV1().Secrets(depl.Namespace), + deployment.ArangoDeploymentResourceKind, + depl.Name, + depl.Namespace) + require.NoError(t, err) + require.True(t, len(secrets) > 0) + + for _, secret := range secrets { + require.NotNil(t, secret.Annotations) + + _, ok := secret.Annotations[key] + + require.True(t, ok) + } + }) + + t.Run("Add arangodb annotation", func(t *testing.T) { + key := "arangodb.com/test-only-annotation" + + annotations := map[string]string{ + key: uniuri.NewLen(8), + "annotation": uniuri.NewLen(8), + } + + addAnnotation(t, kubecli, c, depl, annotations) + + addAnnotation(t, kubecli, c, depl, nil) + + secrets, err := k8sutil.GetSecretsForParent(kubecli.CoreV1().Secrets(depl.Namespace), + deployment.ArangoDeploymentResourceKind, + depl.Name, + depl.Namespace) + require.NoError(t, err) + require.True(t, len(secrets) > 0) + + for _, secret := range secrets { + require.NotNil(t, secret.Annotations) + + _, ok := secret.Annotations[key] + + require.True(t, ok) + } + }) + + t.Run("Replace annotation", func(t *testing.T) { + annotations := map[string]string{ + "annotation": uniuri.NewLen(8), + } + + addAnnotation(t, kubecli, c, depl, annotations) + + annotations["annotation"] = uniuri.NewLen(16) + + addAnnotation(t, kubecli, c, depl, annotations) + + addAnnotation(t, kubecli, c, depl, nil) + }) + + t.Run("Add annotations", func(t *testing.T) { + annotations := map[string]string{ + "annotation": uniuri.NewLen(8), + "annotation2": uniuri.NewLen(16), + } + + addAnnotation(t, kubecli, c, depl, annotations) + + addAnnotation(t, kubecli, c, depl, nil) + }) + + t.Run("Add annotations for group", func(t *testing.T) { + annotations := map[string]string{ + "annotation": uniuri.NewLen(8), + "annotation2": uniuri.NewLen(16), + } + + depl.Spec.Coordinators.Annotations = map[string]string{ + "coordinator-only": uniuri.NewLen(32), + "annotation": uniuri.NewLen(8), + } + + addAnnotation(t, kubecli, c, depl, annotations) + + pods, err := k8sutil.GetPodsForParent(kubecli.CoreV1().Pods(depl.Namespace), + deployment.ArangoDeploymentResourceKind, + depl.Name, + depl.Namespace) + require.NoError(t, err) + require.True(t, len(pods) > 0) + + for _, pod := range pods { + require.NotNil(t, pod.Annotations) + + value, ok := pod.Annotations["annotation"] + _, coordOnly := pod.Annotations["coordinator-only"] + + require.True(t, ok) + + if getPodGroup(pod) == api.ServerGroupCoordinators { + require.Equal(t, depl.Spec.Coordinators.Annotations["annotation"], value) + require.True(t, coordOnly) + } else { + require.Equal(t, annotations["annotation"], value) + require.False(t, coordOnly) + } + } + + depl.Spec.Coordinators.Annotations = nil + + addAnnotation(t, kubecli, c, depl, nil) + }) +}