1
0
Fork 0
mirror of https://github.com/arangodb/kube-arangodb.git synced 2024-12-14 11:57:37 +00:00

[Feature] Compare Generic (#1480)

This commit is contained in:
Adam Janikowski 2023-11-13 20:18:58 +01:00 committed by GitHub
parent 5ebc821941
commit 934039fb41
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
25 changed files with 681 additions and 352 deletions

View file

@ -9,6 +9,7 @@
- (Documentation) Do not use field type name for field URL hash
- (Maintenance) Bump Go to 1.20.11
- (Feature) License ArangoDeployment Fetcher
- (Feature) K8S Resources Compare Generic
## [1.2.35](https://github.com/arangodb/kube-arangodb/tree/1.2.35) (2023-11-06)
- (Maintenance) Update go-driver to v1.6.0, update IsNotFound() checks

View file

@ -63,13 +63,6 @@ type ArangoMemberPodTemplate struct {
Endpoint *string `json:"endpoint,omitempty"`
}
func (a *ArangoMemberPodTemplate) GetChecksum() string {
if a == nil {
return ""
}
return a.Checksum
}
func (a *ArangoMemberPodTemplate) Equals(b *ArangoMemberPodTemplate) bool {
if a == nil && b == nil {
return true
@ -100,3 +93,45 @@ func (a *ArangoMemberPodTemplate) EqualPodSpecChecksum(checksum string) bool {
}
return checksum == a.PodSpecChecksum
}
func (a *ArangoMemberPodTemplate) GetTemplate() *core.PodTemplateSpec {
if a == nil {
return nil
}
return a.PodSpec.DeepCopy()
}
func (a *ArangoMemberPodTemplate) SetTemplate(t *core.PodTemplateSpec) {
if a == nil {
return
}
a.PodSpec = t.DeepCopy()
}
func (a *ArangoMemberPodTemplate) GetTemplateChecksum() string {
if a == nil {
return ""
}
return a.PodSpecChecksum
}
func (a *ArangoMemberPodTemplate) SetTemplateChecksum(s string) {
if a == nil {
return
}
a.PodSpecChecksum = s
}
func (a *ArangoMemberPodTemplate) SetChecksum(s string) {
if a == nil {
return
}
a.Checksum = s
}
func (a *ArangoMemberPodTemplate) GetChecksum() string {
if a == nil {
return ""
}
return a.Checksum
}

View file

@ -30,6 +30,7 @@ import (
"github.com/arangodb/kube-arangodb/pkg/deployment/actions"
"github.com/arangodb/kube-arangodb/pkg/deployment/reconcile/shared"
"github.com/arangodb/kube-arangodb/pkg/deployment/rotation"
"github.com/arangodb/kube-arangodb/pkg/util/compare"
"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
)
@ -231,7 +232,7 @@ func (r *Reconciler) updateMemberRotationConditions(apiObject k8sutil.APIObject,
return nil, err
} else {
switch m {
case rotation.EnforcedRotation:
case compare.EnforcedRotation:
if reason != "" {
r.log.Bool("enforced", true).Info(reason)
} else {
@ -239,7 +240,7 @@ func (r *Reconciler) updateMemberRotationConditions(apiObject k8sutil.APIObject,
}
// We need to do enforced rotation
return restartMemberConditionAction(group, member.ID, reason), nil
case rotation.InPlaceRotation:
case compare.InPlaceRotation:
if member.Conditions.IsTrue(api.ConditionTypeUpdateFailed) {
if !(member.Conditions.IsTrue(api.ConditionTypePendingRestart) || member.Conditions.IsTrue(api.ConditionTypeRestart)) {
return api.Plan{pendingRestartMemberConditionAction(group, member.ID, reason)}, nil
@ -249,11 +250,11 @@ func (r *Reconciler) updateMemberRotationConditions(apiObject k8sutil.APIObject,
return nil, nil
}
return api.Plan{shared.UpdateMemberConditionActionV2(reason, api.ConditionTypePendingUpdate, group, member.ID, true, reason, "", "")}, nil
case rotation.SilentRotation:
case compare.SilentRotation:
// Propagate changes without restart, but apply plan if required
plan = append(plan, actions.NewAction(api.ActionTypeArangoMemberUpdatePodStatus, group, member, "Propagating status of pod").AddParam(ActionTypeArangoMemberUpdatePodStatusChecksum, checksum))
return plan, nil
case rotation.GracefulRotation:
case compare.GracefulRotation:
if reason != "" {
r.log.Bool("enforced", false).Info(reason)
} else {

View file

@ -36,6 +36,7 @@ import (
"github.com/arangodb/kube-arangodb/pkg/deployment/resources"
"github.com/arangodb/kube-arangodb/pkg/deployment/rotation"
"github.com/arangodb/kube-arangodb/pkg/util"
"github.com/arangodb/kube-arangodb/pkg/util/compare"
"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
)
@ -213,7 +214,7 @@ func (r *Reconciler) createUpdatePlanInternal(apiObject k8sutil.APIObject, spec
if mode, p, checksum, reason, err := rotation.IsRotationRequired(context.ACS(), spec, m.Member, m.Group, p, arangoMember.Spec.Template, arangoMember.Status.Template); err != nil {
r.planLogger.Err(err).Str("member", m.Member.ID).Error("Error while generating update plan")
continue
} else if mode != rotation.InPlaceRotation {
} else if mode != compare.InPlaceRotation {
return api.Plan{
shared.RemoveMemberConditionActionV2(reason, api.ConditionTypePendingUpdate, m.Group, m.Member.ID),
shared.UpdateMemberConditionActionV2(reason, api.ConditionTypeUpdating, m.Group, m.Member.ID, true, reason, "", ""),

View file

@ -1,7 +1,7 @@
//
// DISCLAIMER
//
// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -25,36 +25,37 @@ import (
api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
"github.com/arangodb/kube-arangodb/pkg/util"
"github.com/arangodb/kube-arangodb/pkg/util/compare"
)
func podCompare(_ api.DeploymentSpec, _ api.ServerGroup, spec, status *core.PodSpec) comparePodFunc {
return func(builder api.ActionBuilder) (mode Mode, plan api.Plan, err error) {
if spec.SchedulerName != status.SchedulerName {
status.SchedulerName = spec.SchedulerName
mode = mode.And(SilentRotation)
func podCompare(_ api.DeploymentSpec, _ api.ServerGroup, spec, status *core.PodTemplateSpec) compare.Func {
return func(builder api.ActionBuilder) (mode compare.Mode, plan api.Plan, err error) {
if spec.Spec.SchedulerName != status.Spec.SchedulerName {
status.Spec.SchedulerName = spec.Spec.SchedulerName
mode = mode.And(compare.SilentRotation)
}
if !util.CompareInt64p(spec.TerminationGracePeriodSeconds, status.TerminationGracePeriodSeconds) {
status.TerminationGracePeriodSeconds = spec.TerminationGracePeriodSeconds
mode = mode.And(SilentRotation)
if !util.CompareInt64p(spec.Spec.TerminationGracePeriodSeconds, status.Spec.TerminationGracePeriodSeconds) {
status.Spec.TerminationGracePeriodSeconds = spec.Spec.TerminationGracePeriodSeconds
mode = mode.And(compare.SilentRotation)
}
return
}
}
func affinityCompare(_ api.DeploymentSpec, _ api.ServerGroup, spec, status *core.PodSpec) comparePodFunc {
return func(builder api.ActionBuilder) (mode Mode, plan api.Plan, e error) {
if specC, err := util.SHA256FromJSON(spec.Affinity); err != nil {
func affinityCompare(_ api.DeploymentSpec, _ api.ServerGroup, spec, status *core.PodTemplateSpec) compare.Func {
return func(builder api.ActionBuilder) (mode compare.Mode, plan api.Plan, e error) {
if specC, err := util.SHA256FromJSON(spec.Spec.Affinity); err != nil {
e = err
return
} else {
if statusC, err := util.SHA256FromJSON(status.Affinity); err != nil {
if statusC, err := util.SHA256FromJSON(status.Spec.Affinity); err != nil {
e = err
return
} else if specC != statusC {
mode = mode.And(SilentRotation)
status.Affinity = spec.Affinity.DeepCopy()
mode = mode.And(compare.SilentRotation)
status.Spec.Affinity = spec.Spec.Affinity.DeepCopy()
return
} else {
return

View file

@ -1,7 +1,7 @@
//
// DISCLAIMER
//
// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -30,8 +30,9 @@ import (
api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
"github.com/arangodb/kube-arangodb/pkg/deployment/features"
"github.com/arangodb/kube-arangodb/pkg/util"
"github.com/arangodb/kube-arangodb/pkg/util/compare"
"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
strings2 "github.com/arangodb/kube-arangodb/pkg/util/strings"
arangoStrings "github.com/arangodb/kube-arangodb/pkg/util/strings"
)
const (
@ -39,37 +40,36 @@ const (
ContainerImage = "image"
)
func containersCompare(ds api.DeploymentSpec, g api.ServerGroup, spec, status *core.PodSpec) comparePodFunc {
return func(builder api.ActionBuilder) (mode Mode, plan api.Plan, err error) {
a, b := spec.Containers, status.Containers
func containersCompare(ds api.DeploymentSpec, g api.ServerGroup, spec, status *core.PodTemplateSpec) compare.Func {
return compare.SubElementsP2(func(in *core.PodTemplateSpec) *[]core.Container {
return &in.Spec.Containers
}, func(ds api.DeploymentSpec, g api.ServerGroup, specContainers, statusContainers *[]core.Container) compare.Func {
return compare.ArrayExtractorP2(func(ds api.DeploymentSpec, g api.ServerGroup, specContainer, statusContainer *core.Container) compare.Func {
return func(builder api.ActionBuilder) (mode compare.Mode, plan api.Plan, err error) {
if specContainer.Name != statusContainer.Name {
return compare.SkippedRotation, nil, nil
}
if len(a) == 0 || len(a) != len(b) {
// If the number of the containers is different or is zero then skip rotation.
return SkippedRotation, nil, nil
}
for id := range a {
if ac, bc := &a[id], &b[id]; ac.Name == bc.Name {
if ac.Name == api.ServerGroupReservedContainerNameServer {
if isOnlyLogLevelChanged(ac.Command, bc.Command) {
if specContainer.Name == api.ServerGroupReservedContainerNameServer {
if isOnlyLogLevelChanged(specContainer.Command, statusContainer.Command) {
plan = append(plan, builder.NewAction(api.ActionTypeRuntimeContainerArgsLogLevelUpdate).
AddParam(ContainerName, ac.Name))
AddParam(ContainerName, specContainer.Name))
bc.Command = ac.Command
mode = mode.And(InPlaceRotation)
statusContainer.Command = specContainer.Command
mode = mode.And(compare.InPlaceRotation)
}
g := podContainerFuncGenerator(ds, g, ac, bc)
g := compare.NewFuncGenP2(ds, g, specContainer, statusContainer)
if m, p, err := comparePodContainer(builder, g(compareServerContainerVolumeMounts), g(compareServerContainerProbes), g(compareServerContainerEnvs)); err != nil {
if m, p, err := compare.Evaluate(builder, g(compareServerContainerVolumeMounts), g(compareServerContainerProbes), g(compareServerContainerEnvs)); err != nil {
log.Err(err).Msg("Error while getting pod diff")
return SkippedRotation, nil, err
return compare.SkippedRotation, nil, err
} else {
mode = mode.And(m)
plan = append(plan, p...)
}
if !equality.Semantic.DeepEqual(ac.EnvFrom, bc.EnvFrom) {
if !equality.Semantic.DeepEqual(specContainer.EnvFrom, statusContainer.EnvFrom) {
// Check EnvFromSource differences.
filter := func(a, b map[string]core.EnvFromSource) (map[string]core.EnvFromSource, map[string]core.EnvFromSource) {
delete(a, features.ConfigMapName())
@ -77,81 +77,81 @@ func containersCompare(ds api.DeploymentSpec, g api.ServerGroup, spec, status *c
return a, b
}
if areEnvsFromEqual(ac.EnvFrom, bc.EnvFrom, filter) {
if areEnvsFromEqual(specContainer.EnvFrom, statusContainer.EnvFrom, filter) {
// Envs are the same after filtering, but it were different before filtering, so it can be replaced.
bc.EnvFrom = ac.EnvFrom
mode = mode.And(SilentRotation)
statusContainer.EnvFrom = specContainer.EnvFrom
mode = mode.And(compare.SilentRotation)
}
}
if !equality.Semantic.DeepEqual(ac.Ports, bc.Ports) {
bc.Ports = ac.Ports
mode = mode.And(SilentRotation)
if !equality.Semantic.DeepEqual(specContainer.Ports, statusContainer.Ports) {
statusContainer.Ports = specContainer.Ports
mode = mode.And(compare.SilentRotation)
}
} else {
if ac.Image != bc.Image {
if specContainer.Image != statusContainer.Image {
// Image changed
plan = append(plan, builder.NewAction(api.ActionTypeRuntimeContainerImageUpdate).AddParam(ContainerName, ac.Name).AddParam(ContainerImage, ac.Image))
plan = append(plan, builder.NewAction(api.ActionTypeRuntimeContainerImageUpdate).AddParam(ContainerName, specContainer.Name).AddParam(ContainerImage, specContainer.Image))
bc.Image = ac.Image
mode = mode.And(InPlaceRotation)
statusContainer.Image = specContainer.Image
mode = mode.And(compare.InPlaceRotation)
}
g := podContainerFuncGenerator(ds, g, ac, bc)
g := compare.NewFuncGenP2(ds, g, specContainer, statusContainer)
if m, p, err := comparePodContainer(builder, g(compareAnyContainerVolumeMounts), g(compareAnyContainerEnvs)); err != nil {
if m, p, err := compare.Evaluate(builder, g(compareAnyContainerVolumeMounts), g(compareAnyContainerEnvs)); err != nil {
log.Err(err).Msg("Error while getting pod diff")
return SkippedRotation, nil, err
return compare.SkippedRotation, nil, err
} else {
mode = mode.And(m)
plan = append(plan, p...)
}
}
if api.IsReservedServerGroupContainerName(ac.Name) {
mode = mode.And(internalContainerLifecycleCompare(ac, bc))
if api.IsReservedServerGroupContainerName(specContainer.Name) {
mode = mode.And(internalContainerLifecycleCompare(specContainer, statusContainer))
}
}
}
return
}
return
}
})(ds, g, specContainers, statusContainers)
})(ds, g, spec, status)
}
func initContainersCompare(deploymentSpec api.DeploymentSpec, group api.ServerGroup, spec, status *core.PodSpec) comparePodFunc {
return func(builder api.ActionBuilder) (Mode, api.Plan, error) {
func initContainersCompare(deploymentSpec api.DeploymentSpec, group api.ServerGroup, spec, status *core.PodTemplateSpec) compare.Func {
return func(builder api.ActionBuilder) (compare.Mode, api.Plan, error) {
gs := deploymentSpec.GetServerGroupSpec(group)
equal, err := util.CompareJSON(spec.InitContainers, status.InitContainers)
equal, err := util.CompareJSON(spec.Spec.InitContainers, status.Spec.InitContainers)
if err != nil {
return SkippedRotation, nil, err
return compare.SkippedRotation, nil, err
}
// if equal nothing to do
if equal {
return SkippedRotation, nil, nil
return compare.SkippedRotation, nil, nil
}
switch gs.InitContainers.GetMode().Get() {
case api.ServerGroupInitContainerIgnoreMode:
// Just copy spec to status if different
if !equal {
status.InitContainers = spec.InitContainers
return SilentRotation, nil, err
status.Spec.InitContainers = spec.Spec.InitContainers
return compare.SilentRotation, nil, err
} else {
return SkippedRotation, nil, err
return compare.SkippedRotation, nil, err
}
default:
statusInitContainers, specInitContainers := filterReservedInitContainers(status.InitContainers), filterReservedInitContainers(spec.InitContainers)
statusInitContainers, specInitContainers := filterReservedInitContainers(status.Spec.InitContainers), filterReservedInitContainers(spec.Spec.InitContainers)
if equal, err := util.CompareJSON(specInitContainers, statusInitContainers); err != nil {
return SkippedRotation, nil, err
return compare.SkippedRotation, nil, err
} else if equal {
status.InitContainers = spec.InitContainers
return SilentRotation, nil, nil
status.Spec.InitContainers = spec.Spec.InitContainers
return compare.SilentRotation, nil, nil
}
}
return SkippedRotation, nil, nil
return compare.SkippedRotation, nil, nil
}
}
@ -173,7 +173,7 @@ func filterReservedInitContainers(c []core.Container) []core.Container {
// isOnlyLogLevelChanged returns true when status and spec log level arguments are different.
// If any other argument than --log.level is different false is returned.
func isOnlyLogLevelChanged(specArgs, statusArgs []string) bool {
diff := strings2.DiffStrings(specArgs, statusArgs)
diff := arangoStrings.DiffStrings(specArgs, statusArgs)
if len(diff) == 0 {
return false
}
@ -187,27 +187,27 @@ func isOnlyLogLevelChanged(specArgs, statusArgs []string) bool {
return true
}
func internalContainerLifecycleCompare(spec, status *core.Container) Mode {
func internalContainerLifecycleCompare(spec, status *core.Container) compare.Mode {
if spec.Lifecycle == nil && status.Lifecycle == nil {
return SkippedRotation
return compare.SkippedRotation
}
if spec.Lifecycle == nil {
status.Lifecycle = nil
return SilentRotation
return compare.SilentRotation
}
if status.Lifecycle == nil {
status.Lifecycle = spec.Lifecycle
return SilentRotation
return compare.SilentRotation
}
if !equality.Semantic.DeepEqual(spec.Lifecycle, status.Lifecycle) {
status.Lifecycle = spec.Lifecycle.DeepCopy()
return SilentRotation
return compare.SilentRotation
}
return SkippedRotation
return compare.SkippedRotation
}
func areProbesEqual(a, b *core.Probe) bool {

View file

@ -1,7 +1,7 @@
//
// DISCLAIMER
//
// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -29,6 +29,7 @@ import (
api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
"github.com/arangodb/kube-arangodb/pkg/apis/shared"
"github.com/arangodb/kube-arangodb/pkg/deployment/actions"
"github.com/arangodb/kube-arangodb/pkg/util/compare"
)
func Test_ArangoDContainers_SidecarImages(t *testing.T) {
@ -39,7 +40,7 @@ func Test_ArangoDContainers_SidecarImages(t *testing.T) {
status: buildPodSpec(addContainer(shared.ServerContainerName), addSidecarWithImage("sidecar", "local:2.0")),
TestCaseOverride: TestCaseOverride{
expectedMode: InPlaceRotation,
expectedMode: compare.InPlaceRotation,
expectedPlan: api.Plan{
actions.NewClusterAction(api.ActionTypeRuntimeContainerImageUpdate),
},
@ -51,7 +52,7 @@ func Test_ArangoDContainers_SidecarImages(t *testing.T) {
status: buildPodSpec(addSidecarWithImage("sidecar1", "local:1.0"), addSidecarWithImage("sidecar", "local:2.0")),
TestCaseOverride: TestCaseOverride{
expectedMode: InPlaceRotation,
expectedMode: compare.InPlaceRotation,
expectedPlan: api.Plan{
actions.NewClusterAction(api.ActionTypeRuntimeContainerImageUpdate),
},
@ -75,7 +76,7 @@ func Test_InitContainers(t *testing.T) {
})),
TestCaseOverride: TestCaseOverride{
expectedMode: SkippedRotation,
expectedMode: compare.SkippedRotation,
},
groupSpec: buildGroupSpec(func(depl *api.ServerGroupSpec) {
@ -94,7 +95,7 @@ func Test_InitContainers(t *testing.T) {
})),
TestCaseOverride: TestCaseOverride{
expectedMode: SilentRotation,
expectedMode: compare.SilentRotation,
},
groupSpec: buildGroupSpec(func(depl *api.ServerGroupSpec) {
@ -120,7 +121,7 @@ func Test_InitContainers(t *testing.T) {
})),
TestCaseOverride: TestCaseOverride{
expectedMode: GracefulRotation,
expectedMode: compare.GracefulRotation,
},
groupSpec: buildGroupSpec(func(depl *api.ServerGroupSpec) {
@ -143,7 +144,7 @@ func Test_InitContainers(t *testing.T) {
})),
TestCaseOverride: TestCaseOverride{
expectedMode: SilentRotation,
expectedMode: compare.SilentRotation,
},
groupSpec: buildGroupSpec(func(depl *api.ServerGroupSpec) {
@ -164,7 +165,7 @@ func Test_InitContainers(t *testing.T) {
})),
TestCaseOverride: TestCaseOverride{
expectedMode: SilentRotation,
expectedMode: compare.SilentRotation,
},
groupSpec: buildGroupSpec(func(depl *api.ServerGroupSpec) {
@ -189,7 +190,7 @@ func Test_InitContainers(t *testing.T) {
})),
TestCaseOverride: TestCaseOverride{
expectedMode: SilentRotation,
expectedMode: compare.SilentRotation,
},
groupSpec: buildGroupSpec(func(depl *api.ServerGroupSpec) {
@ -214,7 +215,7 @@ func Test_InitContainers(t *testing.T) {
})),
TestCaseOverride: TestCaseOverride{
expectedMode: GracefulRotation,
expectedMode: compare.GracefulRotation,
},
groupSpec: buildGroupSpec(func(depl *api.ServerGroupSpec) {
@ -229,11 +230,11 @@ func Test_InitContainers(t *testing.T) {
})
}
func logLevelTestCaseGen(name string, mode Mode, spec, status []string) TestCase {
func logLevelTestCaseGen(name string, mode compare.Mode, spec, status []string) TestCase {
var c TestCase
c.name = name
c.expectedMode = mode
if c.expectedMode == InPlaceRotation {
if c.expectedMode == compare.InPlaceRotation {
c.expectedPlan = api.Plan{
actions.NewClusterAction(api.ActionTypeRuntimeContainerArgsLogLevelUpdate),
}
@ -247,27 +248,27 @@ func logLevelTestCaseGen(name string, mode Mode, spec, status []string) TestCase
func Test_Container_LogArgs(t *testing.T) {
testCases := []TestCase{
logLevelTestCaseGen("Only log level arguments of the ArangoDB server have been changed",
InPlaceRotation,
compare.InPlaceRotation,
[]string{"--log.level=INFO", "--log.level=requests=error"},
[]string{"--log.level=INFO"}),
logLevelTestCaseGen("ArangoDB server arguments have not been changed",
SkippedRotation,
compare.SkippedRotation,
[]string{"--log.level=INFO"},
[]string{"--log.level=INFO"}),
logLevelTestCaseGen("Multi ArangoDB server arguments have not been changed",
SkippedRotation,
compare.SkippedRotation,
[]string{"--log.level=INFO", "--log.level=requests=debug"},
[]string{"--log.level=INFO", "--log.level=requests=debug"}),
logLevelTestCaseGen("Not only log argument changed",
GracefulRotation,
compare.GracefulRotation,
[]string{"--log.level=INFO", "--server.endpoint=localhost"},
[]string{"--log.level=INFO"}),
logLevelTestCaseGen("Change of order with existing arg & switch to DEBUG",
InPlaceRotation,
compare.InPlaceRotation,
[]string{"--log.level=DEBUG", "--foo"},
[]string{"--foo", "--log.level=INFO"}),
logLevelTestCaseGen("Removal of arg",
InPlaceRotation,
compare.InPlaceRotation,
[]string{"--foo", "--log.level=INFO"},
[]string{"--foo"}),
}
@ -283,7 +284,7 @@ func Test_Container_Args(t *testing.T) {
[]string{"--log.level=INFO", "--log.level=requests=error"})),
status: buildPodSpec(addContainerWithCommand("sidecar", []string{"--log.level=INFO"})),
TestCaseOverride: TestCaseOverride{
expectedMode: GracefulRotation,
expectedMode: compare.GracefulRotation,
},
},
}
@ -314,7 +315,7 @@ func Test_Container_Ports(t *testing.T) {
}
})),
TestCaseOverride: TestCaseOverride{
expectedMode: SilentRotation,
expectedMode: compare.SilentRotation,
},
},
{
@ -338,7 +339,7 @@ func Test_Container_Ports(t *testing.T) {
}
})),
TestCaseOverride: TestCaseOverride{
expectedMode: GracefulRotation,
expectedMode: compare.GracefulRotation,
},
},
}

View file

@ -1,7 +1,7 @@
//
// DISCLAIMER
//
// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -28,18 +28,19 @@ import (
api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
"github.com/arangodb/kube-arangodb/pkg/deployment/resources"
"github.com/arangodb/kube-arangodb/pkg/deployment/topology"
"github.com/arangodb/kube-arangodb/pkg/util/compare"
"github.com/arangodb/kube-arangodb/pkg/util/constants"
)
func compareServerContainerEnvs(ds api.DeploymentSpec, g api.ServerGroup, spec, status *core.Container) comparePodContainerFunc {
return func(builder api.ActionBuilder) (mode Mode, plan api.Plan, err error) {
func compareServerContainerEnvs(ds api.DeploymentSpec, g api.ServerGroup, spec, status *core.Container) compare.Func {
return func(builder api.ActionBuilder) (mode compare.Mode, plan api.Plan, err error) {
specV := mapEnvs(spec)
statusV := mapEnvs(status)
diff := getEnvDiffFromPods(specV, statusV)
if len(diff) == 0 {
return SkippedRotation, nil, nil
return compare.SkippedRotation, nil, nil
}
for k := range diff {
@ -53,24 +54,24 @@ func compareServerContainerEnvs(ds api.DeploymentSpec, g api.ServerGroup, spec,
// Lifecycle envs can change without restart
continue
default:
return GracefulRotation, nil, nil
return compare.GracefulRotation, nil, nil
}
}
status.Env = spec.Env
return SilentRotation, nil, nil
return compare.SilentRotation, nil, nil
}
}
func compareAnyContainerEnvs(ds api.DeploymentSpec, g api.ServerGroup, spec, status *core.Container) comparePodContainerFunc {
return func(builder api.ActionBuilder) (mode Mode, plan api.Plan, err error) {
func compareAnyContainerEnvs(ds api.DeploymentSpec, g api.ServerGroup, spec, status *core.Container) compare.Func {
return func(builder api.ActionBuilder) (mode compare.Mode, plan api.Plan, err error) {
specV := mapEnvs(spec)
statusV := mapEnvs(status)
diff := getEnvDiffFromPods(specV, statusV)
if len(diff) == 0 {
return SkippedRotation, nil, nil
return compare.SkippedRotation, nil, nil
}
for k := range diff {
@ -79,12 +80,12 @@ func compareAnyContainerEnvs(ds api.DeploymentSpec, g api.ServerGroup, spec, sta
// Lifecycle envs can change without restart
continue
default:
return GracefulRotation, nil, nil
return compare.GracefulRotation, nil, nil
}
}
status.Env = spec.Env
return SilentRotation, nil, nil
return compare.SilentRotation, nil, nil
}
}

View file

@ -1,7 +1,7 @@
//
// DISCLAIMER
//
// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -25,13 +25,14 @@ import (
"k8s.io/apimachinery/pkg/api/equality"
api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
"github.com/arangodb/kube-arangodb/pkg/util/compare"
)
func compareServerContainerProbes(ds api.DeploymentSpec, g api.ServerGroup, spec, status *core.Container) comparePodContainerFunc {
return func(builder api.ActionBuilder) (mode Mode, plan api.Plan, err error) {
func compareServerContainerProbes(ds api.DeploymentSpec, g api.ServerGroup, spec, status *core.Container) compare.Func {
return func(builder api.ActionBuilder) (mode compare.Mode, plan api.Plan, err error) {
if !areProbesEqual(spec.StartupProbe, status.StartupProbe) {
status.StartupProbe = spec.StartupProbe
mode = mode.And(SilentRotation)
mode = mode.And(compare.SilentRotation)
}
if !areProbesEqual(spec.ReadinessProbe, status.ReadinessProbe) {
@ -42,7 +43,7 @@ func compareServerContainerProbes(ds api.DeploymentSpec, g api.ServerGroup, spec
if equality.Semantic.DeepDerivative(spec.ReadinessProbe, q) {
status.ReadinessProbe = spec.ReadinessProbe
mode = mode.And(SilentRotation)
mode = mode.And(compare.SilentRotation)
}
}
}
@ -55,7 +56,7 @@ func compareServerContainerProbes(ds api.DeploymentSpec, g api.ServerGroup, spec
if equality.Semantic.DeepDerivative(spec.LivenessProbe, q) {
status.LivenessProbe = spec.LivenessProbe
mode = mode.And(SilentRotation)
mode = mode.And(compare.SilentRotation)
}
}
}

View file

@ -27,6 +27,7 @@ import (
"github.com/arangodb/kube-arangodb/pkg/deployment/topology"
"github.com/arangodb/kube-arangodb/pkg/util"
compare2 "github.com/arangodb/kube-arangodb/pkg/util/compare"
)
func Test_ArangoD_SchedulerName(t *testing.T) {
@ -41,7 +42,7 @@ func Test_ArangoD_SchedulerName(t *testing.T) {
}),
TestCaseOverride: TestCaseOverride{
expectedMode: SilentRotation,
expectedMode: compare2.SilentRotation,
},
},
{
@ -54,7 +55,7 @@ func Test_ArangoD_SchedulerName(t *testing.T) {
}),
TestCaseOverride: TestCaseOverride{
expectedMode: SilentRotation,
expectedMode: compare2.SilentRotation,
},
},
{
@ -67,7 +68,7 @@ func Test_ArangoD_SchedulerName(t *testing.T) {
}),
TestCaseOverride: TestCaseOverride{
expectedMode: SkippedRotation,
expectedMode: compare2.SkippedRotation,
},
},
}
@ -87,7 +88,7 @@ func Test_ArangoD_TerminationGracePeriodSeconds(t *testing.T) {
}),
TestCaseOverride: TestCaseOverride{
expectedMode: SilentRotation,
expectedMode: compare2.SilentRotation,
},
},
{
@ -100,7 +101,7 @@ func Test_ArangoD_TerminationGracePeriodSeconds(t *testing.T) {
}),
TestCaseOverride: TestCaseOverride{
expectedMode: SilentRotation,
expectedMode: compare2.SilentRotation,
},
},
{
@ -113,7 +114,7 @@ func Test_ArangoD_TerminationGracePeriodSeconds(t *testing.T) {
}),
TestCaseOverride: TestCaseOverride{
expectedMode: SilentRotation,
expectedMode: compare2.SilentRotation,
},
},
}
@ -150,7 +151,7 @@ func Test_ArangoD_Affinity(t *testing.T) {
}),
TestCaseOverride: TestCaseOverride{
expectedMode: SilentRotation,
expectedMode: compare2.SilentRotation,
},
},
{
@ -180,7 +181,7 @@ func Test_ArangoD_Affinity(t *testing.T) {
}),
TestCaseOverride: TestCaseOverride{
expectedMode: SilentRotation,
expectedMode: compare2.SilentRotation,
},
},
{
@ -229,7 +230,7 @@ func Test_ArangoD_Affinity(t *testing.T) {
}),
TestCaseOverride: TestCaseOverride{
expectedMode: SilentRotation,
expectedMode: compare2.SilentRotation,
},
},
{
@ -278,7 +279,7 @@ func Test_ArangoD_Affinity(t *testing.T) {
}),
TestCaseOverride: TestCaseOverride{
expectedMode: SilentRotation,
expectedMode: compare2.SilentRotation,
},
},
{
@ -327,7 +328,7 @@ func Test_ArangoD_Affinity(t *testing.T) {
}),
TestCaseOverride: TestCaseOverride{
expectedMode: SilentRotation,
expectedMode: compare2.SilentRotation,
},
},
}
@ -351,7 +352,7 @@ func Test_ArangoD_Labels(t *testing.T) {
}),
TestCaseOverride: TestCaseOverride{
expectedMode: SkippedRotation,
expectedMode: compare2.SkippedRotation,
},
},
{
@ -368,7 +369,7 @@ func Test_ArangoD_Labels(t *testing.T) {
}),
TestCaseOverride: TestCaseOverride{
expectedMode: SkippedRotation,
expectedMode: compare2.SkippedRotation,
},
},
{
@ -387,7 +388,7 @@ func Test_ArangoD_Labels(t *testing.T) {
}),
TestCaseOverride: TestCaseOverride{
expectedMode: SkippedRotation,
expectedMode: compare2.SkippedRotation,
},
},
}
@ -414,7 +415,7 @@ func Test_ArangoD_Envs_Zone(t *testing.T) {
})),
TestCaseOverride: TestCaseOverride{
expectedMode: SilentRotation,
expectedMode: compare2.SilentRotation,
},
},
{
@ -434,7 +435,7 @@ func Test_ArangoD_Envs_Zone(t *testing.T) {
})),
TestCaseOverride: TestCaseOverride{
expectedMode: SilentRotation,
expectedMode: compare2.SilentRotation,
},
},
{
@ -459,7 +460,7 @@ func Test_ArangoD_Envs_Zone(t *testing.T) {
})),
TestCaseOverride: TestCaseOverride{
expectedMode: SilentRotation,
expectedMode: compare2.SilentRotation,
},
},
{
@ -488,7 +489,7 @@ func Test_ArangoD_Envs_Zone(t *testing.T) {
})),
TestCaseOverride: TestCaseOverride{
expectedMode: GracefulRotation,
expectedMode: compare2.GracefulRotation,
},
},
}

View file

@ -26,15 +26,16 @@ import (
core "k8s.io/api/core/v1"
api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
"github.com/arangodb/kube-arangodb/pkg/util/compare"
)
func comparePodTolerations(_ api.DeploymentSpec, _ api.ServerGroup, spec, status *core.PodSpec) comparePodFunc {
return func(builder api.ActionBuilder) (mode Mode, plan api.Plan, err error) {
if !reflect.DeepEqual(spec.Tolerations, status.Tolerations) {
func comparePodTolerations(_ api.DeploymentSpec, _ api.ServerGroup, spec, status *core.PodTemplateSpec) compare.Func {
return func(builder api.ActionBuilder) (mode compare.Mode, plan api.Plan, err error) {
if !reflect.DeepEqual(spec.Spec.Tolerations, status.Spec.Tolerations) {
plan = append(plan, builder.NewAction(api.ActionTypeRuntimeContainerSyncTolerations))
status.Tolerations = spec.Tolerations
mode = mode.And(InPlaceRotation)
status.Spec.Tolerations = spec.Spec.Tolerations
mode = mode.And(compare.InPlaceRotation)
return
}

View file

@ -1,7 +1,7 @@
//
// DISCLAIMER
//
// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -27,17 +27,18 @@ import (
api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
"github.com/arangodb/kube-arangodb/pkg/apis/shared"
"github.com/arangodb/kube-arangodb/pkg/util/compare"
)
func compareServerContainerVolumeMounts(ds api.DeploymentSpec, g api.ServerGroup, spec, status *core.Container) comparePodContainerFunc {
return func(builder api.ActionBuilder) (mode Mode, plan api.Plan, err error) {
func compareServerContainerVolumeMounts(ds api.DeploymentSpec, g api.ServerGroup, spec, status *core.Container) compare.Func {
return func(builder api.ActionBuilder) (mode compare.Mode, plan api.Plan, err error) {
specV := mapVolumeMounts(spec)
statusV := mapVolumeMounts(status)
diff := getVolumeMountsDiffFromPods(specV, statusV)
if len(diff) == 0 {
return SkippedRotation, nil, nil
return compare.SkippedRotation, nil, nil
}
for k, v := range diff {
@ -46,34 +47,34 @@ func compareServerContainerVolumeMounts(ds api.DeploymentSpec, g api.ServerGroup
// We are fine, should be just replaced
if v.a == nil {
// we remove volume
return GracefulRotation, nil, nil
return compare.GracefulRotation, nil, nil
}
if ds.Mode.Get().ServingGroup() == g {
// Always enforce on serving group
return GracefulRotation, nil, nil
return compare.GracefulRotation, nil, nil
}
case shared.LifecycleVolumeName:
// Do nothing
default:
return GracefulRotation, nil, nil
return compare.GracefulRotation, nil, nil
}
}
status.VolumeMounts = spec.VolumeMounts
return SilentRotation, nil, nil
return compare.SilentRotation, nil, nil
}
}
func compareAnyContainerVolumeMounts(ds api.DeploymentSpec, g api.ServerGroup, spec, status *core.Container) comparePodContainerFunc {
return func(builder api.ActionBuilder) (mode Mode, plan api.Plan, err error) {
func compareAnyContainerVolumeMounts(ds api.DeploymentSpec, g api.ServerGroup, spec, status *core.Container) compare.Func {
return func(builder api.ActionBuilder) (mode compare.Mode, plan api.Plan, err error) {
specV := mapVolumeMounts(spec)
statusV := mapVolumeMounts(status)
diff := getVolumeMountsDiffFromPods(specV, statusV)
if len(diff) == 0 {
return SkippedRotation, nil, nil
return compare.SkippedRotation, nil, nil
}
for k := range diff {
@ -81,12 +82,12 @@ func compareAnyContainerVolumeMounts(ds api.DeploymentSpec, g api.ServerGroup, s
case shared.LifecycleVolumeName:
// Do nothing
default:
return GracefulRotation, nil, nil
return compare.GracefulRotation, nil, nil
}
}
status.VolumeMounts = spec.VolumeMounts
return SilentRotation, nil, nil
return compare.SilentRotation, nil, nil
}
}

View file

@ -1,7 +1,7 @@
//
// DISCLAIMER
//
// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -27,21 +27,22 @@ import (
api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
"github.com/arangodb/kube-arangodb/pkg/apis/shared"
"github.com/arangodb/kube-arangodb/pkg/util/compare"
)
type volumeDiff struct {
a, b *core.Volume
}
func comparePodVolumes(ds api.DeploymentSpec, g api.ServerGroup, spec, status *core.PodSpec) comparePodFunc {
return func(builder api.ActionBuilder) (mode Mode, plan api.Plan, err error) {
specV := mapVolumes(spec)
statusV := mapVolumes(status)
func comparePodVolumes(ds api.DeploymentSpec, g api.ServerGroup, spec, status *core.PodTemplateSpec) compare.Func {
return func(builder api.ActionBuilder) (mode compare.Mode, plan api.Plan, err error) {
specV := mapVolumes(spec.Spec)
statusV := mapVolumes(status.Spec)
diff := getVolumesDiffFromPods(specV, statusV)
if len(diff) == 0 {
return SkippedRotation, nil, nil
return compare.SkippedRotation, nil, nil
}
for k, v := range diff {
@ -50,20 +51,20 @@ func comparePodVolumes(ds api.DeploymentSpec, g api.ServerGroup, spec, status *c
// We are fine, should be just replaced
if v.a == nil {
// we remove volume
return GracefulRotation, nil, nil
return compare.GracefulRotation, nil, nil
}
if ds.Mode.Get().ServingGroup() == g {
// Always enforce on serving group
return GracefulRotation, nil, nil
return compare.GracefulRotation, nil, nil
}
default:
return GracefulRotation, nil, nil
return compare.GracefulRotation, nil, nil
}
}
status.Volumes = spec.Volumes
return SilentRotation, nil, nil
status.Spec.Volumes = spec.Spec.Volumes
return compare.SilentRotation, nil, nil
}
}
@ -97,7 +98,7 @@ func getVolumesDiffFromPods(a, b map[string]*core.Volume) map[string]volumeDiff
return d
}
func mapVolumes(a *core.PodSpec) map[string]*core.Volume {
func mapVolumes(a core.PodSpec) map[string]*core.Volume {
n := make(map[string]*core.Volume, len(a.Volumes))
for id := range a.Volumes {

View file

@ -1,7 +1,7 @@
//
// DISCLAIMER
//
// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -27,6 +27,7 @@ import (
api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
"github.com/arangodb/kube-arangodb/pkg/apis/shared"
"github.com/arangodb/kube-arangodb/pkg/util/compare"
)
func Test_ArangoD_Volumes(t *testing.T) {
@ -39,7 +40,7 @@ func Test_ArangoD_Volumes(t *testing.T) {
deploymentSpec: buildDeployment(),
TestCaseOverride: TestCaseOverride{
expectedMode: SkippedRotation,
expectedMode: compare.SkippedRotation,
},
},
{
@ -48,7 +49,7 @@ func Test_ArangoD_Volumes(t *testing.T) {
status: buildPodSpec(addVolume("data", addVolumeConfigMapSource(&core.ConfigMapVolumeSource{}))),
TestCaseOverride: TestCaseOverride{
expectedMode: SkippedRotation,
expectedMode: compare.SkippedRotation,
},
},
{
@ -61,7 +62,7 @@ func Test_ArangoD_Volumes(t *testing.T) {
}))),
TestCaseOverride: TestCaseOverride{
expectedMode: GracefulRotation,
expectedMode: compare.GracefulRotation,
},
},
{
@ -70,7 +71,7 @@ func Test_ArangoD_Volumes(t *testing.T) {
status: buildPodSpec(),
TestCaseOverride: TestCaseOverride{
expectedMode: GracefulRotation,
expectedMode: compare.GracefulRotation,
},
},
{
@ -83,7 +84,7 @@ func Test_ArangoD_Volumes(t *testing.T) {
}))),
TestCaseOverride: TestCaseOverride{
expectedMode: GracefulRotation,
expectedMode: compare.GracefulRotation,
},
},
{
@ -98,23 +99,23 @@ func Test_ArangoD_Volumes(t *testing.T) {
overrides: map[api.DeploymentMode]map[api.ServerGroup]TestCaseOverride{
api.DeploymentModeSingle: {
api.ServerGroupSingle: {
expectedMode: GracefulRotation,
expectedMode: compare.GracefulRotation,
},
},
api.DeploymentModeActiveFailover: {
api.ServerGroupSingle: {
expectedMode: GracefulRotation,
expectedMode: compare.GracefulRotation,
},
},
api.DeploymentModeCluster: {
api.ServerGroupCoordinators: {
expectedMode: GracefulRotation,
expectedMode: compare.GracefulRotation,
},
},
},
TestCaseOverride: TestCaseOverride{
expectedMode: SilentRotation,
expectedMode: compare.SilentRotation,
},
},
{
@ -125,23 +126,23 @@ func Test_ArangoD_Volumes(t *testing.T) {
overrides: map[api.DeploymentMode]map[api.ServerGroup]TestCaseOverride{
api.DeploymentModeSingle: {
api.ServerGroupSingle: {
expectedMode: GracefulRotation,
expectedMode: compare.GracefulRotation,
},
},
api.DeploymentModeActiveFailover: {
api.ServerGroupSingle: {
expectedMode: GracefulRotation,
expectedMode: compare.GracefulRotation,
},
},
api.DeploymentModeCluster: {
api.ServerGroupCoordinators: {
expectedMode: GracefulRotation,
expectedMode: compare.GracefulRotation,
},
},
},
TestCaseOverride: TestCaseOverride{
expectedMode: SilentRotation,
expectedMode: compare.SilentRotation,
},
},
{
@ -154,7 +155,7 @@ func Test_ArangoD_Volumes(t *testing.T) {
}))),
TestCaseOverride: TestCaseOverride{
expectedMode: GracefulRotation,
expectedMode: compare.GracefulRotation,
},
},
}
@ -170,7 +171,7 @@ func Test_ArangoD_VolumeMounts(t *testing.T) {
status: buildPodSpec(addContainer("server")),
TestCaseOverride: TestCaseOverride{
expectedMode: SkippedRotation,
expectedMode: compare.SkippedRotation,
},
},
{
@ -183,7 +184,7 @@ func Test_ArangoD_VolumeMounts(t *testing.T) {
}))),
TestCaseOverride: TestCaseOverride{
expectedMode: SkippedRotation,
expectedMode: compare.SkippedRotation,
},
},
{
@ -192,7 +193,7 @@ func Test_ArangoD_VolumeMounts(t *testing.T) {
status: buildPodSpec(addContainer("server", addVolumeMount("mount2"))),
TestCaseOverride: TestCaseOverride{
expectedMode: GracefulRotation,
expectedMode: compare.GracefulRotation,
},
},
{
@ -201,7 +202,7 @@ func Test_ArangoD_VolumeMounts(t *testing.T) {
status: buildPodSpec(addContainer("server")),
TestCaseOverride: TestCaseOverride{
expectedMode: GracefulRotation,
expectedMode: compare.GracefulRotation,
},
},
{
@ -210,7 +211,7 @@ func Test_ArangoD_VolumeMounts(t *testing.T) {
status: buildPodSpec(addContainer("server", addVolumeMount("mount"))),
TestCaseOverride: TestCaseOverride{
expectedMode: GracefulRotation,
expectedMode: compare.GracefulRotation,
},
},
{
@ -219,7 +220,7 @@ func Test_ArangoD_VolumeMounts(t *testing.T) {
status: buildPodSpec(addContainer("server", addVolumeMount("mount"))),
TestCaseOverride: TestCaseOverride{
expectedMode: GracefulRotation,
expectedMode: compare.GracefulRotation,
},
},
{
@ -228,7 +229,7 @@ func Test_ArangoD_VolumeMounts(t *testing.T) {
status: buildPodSpec(addContainer("server", addVolumeMount(shared.ArangoDTimezoneVolumeName))),
TestCaseOverride: TestCaseOverride{
expectedMode: GracefulRotation,
expectedMode: compare.GracefulRotation,
},
},
{
@ -239,23 +240,23 @@ func Test_ArangoD_VolumeMounts(t *testing.T) {
overrides: map[api.DeploymentMode]map[api.ServerGroup]TestCaseOverride{
api.DeploymentModeSingle: {
api.ServerGroupSingle: {
expectedMode: GracefulRotation,
expectedMode: compare.GracefulRotation,
},
},
api.DeploymentModeActiveFailover: {
api.ServerGroupSingle: {
expectedMode: GracefulRotation,
expectedMode: compare.GracefulRotation,
},
},
api.DeploymentModeCluster: {
api.ServerGroupCoordinators: {
expectedMode: GracefulRotation,
expectedMode: compare.GracefulRotation,
},
},
},
TestCaseOverride: TestCaseOverride{
expectedMode: SilentRotation,
expectedMode: compare.SilentRotation,
},
},
}

View file

@ -29,49 +29,27 @@ import (
api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
"github.com/arangodb/kube-arangodb/pkg/deployment/acs/sutil"
"github.com/arangodb/kube-arangodb/pkg/handlers/utils"
"github.com/arangodb/kube-arangodb/pkg/util/compare"
"github.com/arangodb/kube-arangodb/pkg/util/constants"
)
type Mode int
const (
SkippedRotation Mode = iota
// SilentRotation Propagates changes without restart. Returned plan is executed in High actions
SilentRotation
// InPlaceRotation Silently accept changes. Returned plan is executed in Normal actions
InPlaceRotation
// GracefulRotation Schedule pod restart. Returned plan is ignored
GracefulRotation
// EnforcedRotation Enforce pod restart. Returned plan is ignored
EnforcedRotation
)
// And returns the higher value of the rotation mode.
func (m Mode) And(b Mode) Mode {
if m > b {
return m
}
return b
}
// CheckPossible returns true if rotation is possible
func CheckPossible(member api.MemberStatus) bool {
return !member.Conditions.IsTrue(api.ConditionTypeTerminated)
}
func IsRotationRequired(acs sutil.ACS, spec api.DeploymentSpec, member api.MemberStatus, group api.ServerGroup, pod *core.Pod, specTemplate, statusTemplate *api.ArangoMemberPodTemplate) (mode Mode, plan api.Plan, specChecksum string, reason string, err error) {
func IsRotationRequired(acs sutil.ACS, spec api.DeploymentSpec, member api.MemberStatus, group api.ServerGroup, pod *core.Pod, specTemplate, statusTemplate *api.ArangoMemberPodTemplate) (mode compare.Mode, plan api.Plan, specChecksum string, reason string, err error) {
// Determine if rotation is required based on plan and actions
// Set default mode for return value
mode = SkippedRotation
mode = compare.SkippedRotation
// We are under termination
if pod != nil {
if member.Conditions.IsTrue(api.ConditionTypeTerminating) || pod.DeletionTimestamp != nil {
if l := utils.StringList(pod.Finalizers); l.Has(constants.FinalizerPodGracefulShutdown) && !l.Has(constants.FinalizerDelayPodTermination) {
reason = "Recreation enforced by deleted state"
mode = EnforcedRotation
mode = compare.EnforcedRotation
}
return
@ -85,7 +63,7 @@ func IsRotationRequired(acs sutil.ACS, spec api.DeploymentSpec, member api.Membe
if spec.MemberPropagationMode.Get() == api.DeploymentMemberPropagationModeAlways && member.Conditions.IsTrue(api.ConditionTypePendingRestart) {
reason = "Restart is pending"
mode = EnforcedRotation
mode = compare.EnforcedRotation
return
}
@ -93,20 +71,20 @@ func IsRotationRequired(acs sutil.ACS, spec api.DeploymentSpec, member api.Membe
if pod != nil {
if member.Pod.GetUID() != pod.UID {
reason = "Pod UID does not match, this pod is not managed by Operator. Recreating"
mode = EnforcedRotation
mode = compare.EnforcedRotation
return
}
if _, ok := pod.Annotations[deployment.ArangoDeploymentPodRotateAnnotation]; ok {
reason = "Recreation enforced by annotation"
mode = EnforcedRotation
mode = compare.EnforcedRotation
return
}
}
if p := member.Pod; p != nil && p.SpecVersion == "" {
reason = "Pod Spec Version is nil - recreating pod"
mode = EnforcedRotation
mode = compare.EnforcedRotation
return
}
@ -118,19 +96,19 @@ func IsRotationRequired(acs sutil.ACS, spec api.DeploymentSpec, member api.Membe
// Check if any of resize events are in place
if member.Conditions.IsTrue(api.ConditionTypePendingTLSRotation) {
reason = "TLS Rotation pending"
mode = EnforcedRotation
mode = compare.EnforcedRotation
return
}
if member.Conditions.Check(api.ConditionTypePVCResizePending).Exists().LastTransition(3 * time.Minute).Evaluate() {
reason = "PVC Resize pending for more than 3 min"
mode = EnforcedRotation
mode = compare.EnforcedRotation
return
}
if mode, plan, err := compare(spec, member, group, specTemplate, statusTemplate); err != nil {
return SkippedRotation, nil, specTemplate.Checksum, "", err
} else if mode == SkippedRotation {
if mode, plan, err := compareFunc(spec, member, group, specTemplate, statusTemplate); err != nil {
return compare.SkippedRotation, nil, specTemplate.Checksum, "", err
} else if mode == compare.SkippedRotation {
return mode, plan, specTemplate.Checksum, "No rotation needed", nil
} else {
return mode, plan, specTemplate.Checksum, "Pod needs rotation", nil

View file

@ -1,7 +1,7 @@
//
// DISCLAIMER
//
// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -21,124 +21,32 @@
package rotation
import (
"crypto/sha256"
"encoding/json"
"fmt"
jd "github.com/josephburnett/jd/lib"
"github.com/rs/zerolog/log"
core "k8s.io/api/core/v1"
api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
"github.com/arangodb/kube-arangodb/pkg/deployment/actions"
"github.com/arangodb/kube-arangodb/pkg/deployment/resources"
"github.com/arangodb/kube-arangodb/pkg/util/compare"
)
type comparePodFuncGen func(deploymentSpec api.DeploymentSpec, group api.ServerGroup, spec, status *core.PodSpec) comparePodFunc
type comparePodFunc func(builder api.ActionBuilder) (mode Mode, plan api.Plan, err error)
func podFuncGenerator(deploymentSpec api.DeploymentSpec, group api.ServerGroup, spec, status *core.PodSpec) func(c comparePodFuncGen) comparePodFunc {
return func(c comparePodFuncGen) comparePodFunc {
return c(deploymentSpec, group, spec, status)
}
}
type comparePodContainerFuncGen func(deploymentSpec api.DeploymentSpec, group api.ServerGroup, spec, status *core.Container) comparePodContainerFunc
type comparePodContainerFunc func(builder api.ActionBuilder) (mode Mode, plan api.Plan, err error)
func podContainerFuncGenerator(deploymentSpec api.DeploymentSpec, group api.ServerGroup, spec, status *core.Container) func(c comparePodContainerFuncGen) comparePodContainerFunc {
return func(c comparePodContainerFuncGen) comparePodContainerFunc {
return c(deploymentSpec, group, spec, status)
}
}
func comparePodContainer(builder api.ActionBuilder, f ...comparePodContainerFunc) (mode Mode, plan api.Plan, err error) {
for _, q := range f {
if m, p, err := q(builder); err != nil {
return 0, nil, err
} else {
mode = mode.And(m)
plan = append(plan, p...)
}
}
return
}
func comparePod(builder api.ActionBuilder, f ...comparePodFunc) (mode Mode, plan api.Plan, err error) {
for _, q := range f {
if m, p, err := q(builder); err != nil {
return 0, nil, err
} else {
mode = mode.And(m)
plan = append(plan, p...)
}
}
return
}
func compare(deploymentSpec api.DeploymentSpec, member api.MemberStatus, group api.ServerGroup,
spec, status *api.ArangoMemberPodTemplate) (mode Mode, plan api.Plan, err error) {
if spec.Checksum == status.Checksum {
return SkippedRotation, nil, nil
}
// If checksums are different and rotation is not needed and there are no changes between containers
// then silent rotation must be applied to adjust status checksum.
mode = SilentRotation
podStatus := status.PodSpec.DeepCopy()
// Try to fill fields
b := actions.NewActionBuilderWrap(group, member)
g := podFuncGenerator(deploymentSpec, group, &spec.PodSpec.Spec, &podStatus.Spec)
if m, p, err := comparePod(b, g(podCompare), g(affinityCompare), g(comparePodVolumes), g(containersCompare), g(initContainersCompare), g(comparePodTolerations)); err != nil {
log.Err(err).Msg("Error while getting pod diff")
return SkippedRotation, nil, err
} else {
mode = mode.And(m)
plan = append(plan, p...)
}
checksum, err := resources.ChecksumArangoPod(deploymentSpec.GetServerGroupSpec(group), resources.CreatePodFromTemplate(podStatus))
if err != nil {
log.Err(err).Msg("Error while getting pod checksum")
return SkippedRotation, nil, err
}
newStatus, err := api.GetArangoMemberPodTemplate(podStatus, checksum)
if err != nil {
log.Err(err).Msg("Error while getting template")
return SkippedRotation, nil, err
}
if spec.RotationNeeded(newStatus) {
line := logger.Str("id", member.ID)
specBytes, errA := json.Marshal(spec.PodSpec)
if errA == nil {
line = line.Str("spec", string(specBytes))
}
statusBytes, errB := json.Marshal(newStatus.PodSpec)
if errB == nil {
line = line.Str("status", string(statusBytes))
}
if errA == nil && errB == nil {
if specData, err := jd.ReadJsonString(string(specBytes)); err == nil && specData != nil {
if statusData, err := jd.ReadJsonString(string(statusBytes)); err == nil && statusData != nil {
line = line.Str("diff", specData.Diff(statusData).Render())
}
func compareFunc(deploymentSpec api.DeploymentSpec, member api.MemberStatus, group api.ServerGroup,
spec, status *api.ArangoMemberPodTemplate) (mode compare.Mode, plan api.Plan, err error) {
return compare.P2[core.PodTemplateSpec, api.DeploymentSpec, api.ServerGroup](logger,
deploymentSpec, group,
actions.NewActionBuilderWrap(group, member),
func(in *core.PodTemplateSpec) (string, error) {
data, err := json.Marshal(in.Spec)
if err != nil {
return "", err
}
}
line.Info("Pod needs rotation - templates does not match")
checksum := fmt.Sprintf("%0x", sha256.Sum256(data))
return GracefulRotation, nil, nil
}
return
return checksum, nil
},
spec, status,
podCompare, affinityCompare, comparePodVolumes, containersCompare, initContainersCompare, comparePodTolerations)
}

View file

@ -1,7 +1,7 @@
//
// DISCLAIMER
//
// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -30,6 +30,7 @@ import (
api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
"github.com/arangodb/kube-arangodb/pkg/deployment/resources"
"github.com/arangodb/kube-arangodb/pkg/util/compare"
"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
)
@ -43,7 +44,7 @@ var podLifecycleChange000Spec []byte
//go:embed testdata/pod_lifecycle_change.000.status.json
var podLifecycleChange000Status []byte
func runPredefinedTests(t *testing.T, spec, status []byte) (mode Mode, plan api.Plan, err error) {
func runPredefinedTests(t *testing.T, spec, status []byte) (mode compare.Mode, plan api.Plan, err error) {
var specO, statusO core.PodTemplateSpec
require.NoError(t, json.Unmarshal(spec, &specO))
@ -63,7 +64,7 @@ func runPredefinedTests(t *testing.T, spec, status []byte) (mode Mode, plan api.
statusT, err := api.GetArangoMemberPodTemplate(&statusO, statusC)
require.NoError(t, err)
return compare(obj, member, api.ServerGroupUnknown, specT, statusT)
return compareFunc(obj, member, api.ServerGroupUnknown, specT, statusT)
}
func Test_PredefinedTests(t *testing.T) {
@ -71,6 +72,6 @@ func Test_PredefinedTests(t *testing.T) {
mode, plan, err := runPredefinedTests(t, podLifecycleChange000Spec, podLifecycleChange000Status)
require.NoError(t, err)
require.Empty(t, plan)
require.Equal(t, SilentRotation, mode)
require.Equal(t, compare.SilentRotation, mode)
})
}

View file

@ -1,7 +1,7 @@
//
// DISCLAIMER
//
// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -28,10 +28,11 @@ import (
api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
"github.com/arangodb/kube-arangodb/pkg/deployment/resources"
compare2 "github.com/arangodb/kube-arangodb/pkg/util/compare"
)
type TestCaseOverride struct {
expectedMode Mode
expectedMode compare2.Mode
expectedPlan api.Plan
expectedErr string
}
@ -98,7 +99,7 @@ func runTestCasesForModeAndGroup(t *testing.T, m api.DeploymentMode, g api.Serve
pspec := newTemplateFromSpec(t, tc.spec, g, *ds)
pstatus := newTemplateFromSpec(t, tc.status, g, *ds)
mode, plan, err := compare(*ds, api.MemberStatus{ID: "id"}, g, pspec, pstatus)
mode, plan, err := compareFunc(*ds, api.MemberStatus{ID: "id"}, g, pspec, pstatus)
q := tc.TestCaseOverride
@ -113,7 +114,7 @@ func runTestCasesForModeAndGroup(t *testing.T, m api.DeploymentMode, g api.Serve
require.Equal(t, q.expectedMode, mode)
switch mode {
case InPlaceRotation:
case compare2.InPlaceRotation:
require.Len(t, plan, len(q.expectedPlan))
for i := range plan {

View file

@ -0,0 +1,98 @@
//
// DISCLAIMER
//
// Copyright 2023 ArangoDB GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Copyright holder is ArangoDB GmbH, Cologne, Germany
//
package compare
import (
"reflect"
api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
"github.com/arangodb/kube-arangodb/pkg/logging"
)
// P2 runs compare of the provided templates with 2 generic parameters
func P2[T interface{}, P1, P2 interface{}](
log logging.Logger,
p1 P1, p2 P2,
actionBuilder api.ActionBuilder,
checksum Checksum[T],
spec, status Template[T],
evaluators ...GenP2[T, P1, P2]) (mode Mode, plan api.Plan, err error) {
if spec.GetChecksum() == status.GetChecksum() {
return SkippedRotation, nil, nil
}
mode = SilentRotation
// Try to fill fields
newStatus := status.GetTemplate()
currentSpec := spec.GetTemplate()
g := NewFuncGenP2[T](p1, p2, currentSpec, newStatus)
evaluatorsFunc := make([]Func, len(evaluators))
for id := range evaluators {
evaluatorsFunc[id] = g(evaluators[id])
}
if m, p, err := Evaluate(actionBuilder, evaluatorsFunc...); err != nil {
log.Err(err).Error("Error while getting diff")
return SkippedRotation, nil, err
} else {
mode = mode.And(m)
plan = append(plan, p...)
}
// Diff has been generated! Proceed with calculations
checksumString, err := checksum(newStatus)
if err != nil {
log.Err(err).Error("Error while getting checksum")
return SkippedRotation, nil, err
}
if checksumString != spec.GetChecksum() {
line := log
// Rotate anyway!
specData, statusData, diff, err := Diff(currentSpec, newStatus)
if err == nil {
if diff != "" {
line = line.Str("diff", diff)
}
if specData != "" {
line = line.Str("spec", specData)
}
if statusData != "" {
line = line.Str("status", statusData)
}
}
line.Info("Resource %s needs rotation - templates does not match", reflect.TypeOf(currentSpec).String())
return GracefulRotation, nil, nil
}
return
}

53
pkg/util/compare/diff.go Normal file
View file

@ -0,0 +1,53 @@
//
// DISCLAIMER
//
// Copyright 2023 ArangoDB GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Copyright holder is ArangoDB GmbH, Cologne, Germany
//
package compare
import (
"encoding/json"
jd "github.com/josephburnett/jd/lib"
)
func Diff[T interface{}](spec, status *T) (specData, statusData, diff string, outErr error) {
specBytes, err := json.Marshal(spec)
if err != nil {
return "", "", "", err
}
specData = string(specBytes)
statusBytes, err := json.Marshal(status)
if err != nil {
return "", "", "", err
}
statusData = string(statusBytes)
if specData, err := jd.ReadJsonString(string(specBytes)); err != nil {
return "", "", "", err
} else if specData != nil {
if statusData, err := jd.ReadJsonString(string(statusBytes)); err != nil {
return "", "", "", err
} else if statusData != nil {
diff = specData.Diff(statusData).Render()
}
}
return
}

31
pkg/util/compare/gen.go Normal file
View file

@ -0,0 +1,31 @@
//
// DISCLAIMER
//
// Copyright 2023 ArangoDB GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Copyright holder is ArangoDB GmbH, Cologne, Germany
//
package compare
type GenP2[T, P1, P2 interface{}] func(p1 P1, p2 P2, spec, status *T) Func
type FuncGenP2[T, P1, P2 interface{}] func(in GenP2[T, P1, P2]) Func
func NewFuncGenP2[T, P1, P2 interface{}](p1 P1, p2 P2, spec, status *T) FuncGenP2[T, P1, P2] {
return func(in GenP2[T, P1, P2]) Func {
return in(p1, p2, spec, status)
}
}

View file

@ -0,0 +1,70 @@
//
// DISCLAIMER
//
// Copyright 2023 ArangoDB GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Copyright holder is ArangoDB GmbH, Cologne, Germany
//
package compare
import (
api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
)
type Template[T interface{}] interface {
GetTemplate() *T
SetTemplate(*T)
GetTemplateChecksum() string
SetTemplateChecksum(string)
GetChecksum() string
SetChecksum(string)
}
type Checksum[T interface{}] func(in *T) (string, error)
type FuncGen[T interface{}] func(spec, status *T) Func
type Func func(builder api.ActionBuilder) (mode Mode, plan api.Plan, err error)
func Merge(f ...Func) Func {
return func(builder api.ActionBuilder) (mode Mode, plan api.Plan, err error) {
for _, q := range f {
if m, p, err := q(builder); err != nil {
return 0, nil, err
} else {
mode = mode.And(m)
plan = append(plan, p...)
}
}
return
}
}
func Evaluate(builder api.ActionBuilder, f ...Func) (mode Mode, plan api.Plan, err error) {
for _, q := range f {
if m, p, err := q(builder); err != nil {
return 0, nil, err
} else {
mode = mode.And(m)
plan = append(plan, p...)
}
}
return
}

54
pkg/util/compare/mode.go Normal file
View file

@ -0,0 +1,54 @@
//
// DISCLAIMER
//
// Copyright 2023 ArangoDB GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Copyright holder is ArangoDB GmbH, Cologne, Germany
//
package compare
import (
api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
)
type Mode int
const (
SkippedRotation Mode = iota
// SilentRotation Propagates changes without restart. Returned plan is executed in High actions
SilentRotation
// InPlaceRotation Silently accept changes. Returned plan is executed in Normal actions
InPlaceRotation
// GracefulRotation Schedule pod restart. Returned plan is ignored
GracefulRotation
// EnforcedRotation Enforce pod restart. Returned plan is ignored
EnforcedRotation
)
// And returns the higher value of the rotation mode.
func (m Mode) And(b Mode) Mode {
if m > b {
return m
}
return b
}
func (m Mode) Func() Func {
return func(builder api.ActionBuilder) (mode Mode, plan api.Plan, err error) {
return m, nil, nil
}
}

63
pkg/util/compare/sub.go Normal file
View file

@ -0,0 +1,63 @@
//
// DISCLAIMER
//
// Copyright 2023 ArangoDB GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Copyright holder is ArangoDB GmbH, Cologne, Germany
//
package compare
type SubElementExtractor[T1, T2 interface{}] func(in *T1) *T2
func SubElementsP2[T1, T2, P1, P2 interface{}](extractor SubElementExtractor[T1, T2], gens ...GenP2[T2, P1, P2]) GenP2[T1, P1, P2] {
return func(p1 P1, p2 P2, spec, status *T1) Func {
specF := extractor(spec)
statusF := extractor(status)
funcs := make([]Func, len(gens))
for id := range gens {
funcs[id] = gens[id](p1, p2, specF, statusF)
}
return Merge(funcs...)
}
}
func ArrayExtractorP2[T, P1, P2 interface{}](gens ...GenP2[T, P1, P2]) GenP2[[]T, P1, P2] {
return func(p1 P1, p2 P2, spec, status *[]T) Func {
if spec == nil || status == nil {
return SkippedRotation.Func()
}
specA := *spec
statusA := *status
if len(specA) != len(statusA) {
return SkippedRotation.Func()
}
funcs := make([]Func, 0, len(specA)*len(gens))
// Iterate over ids
for id := range specA {
for _, gen := range gens {
funcs = append(funcs, gen(p1, p2, &specA[id], &statusA[id]))
}
}
return Merge(funcs...)
}
}

View file

@ -0,0 +1,25 @@
//
// DISCLAIMER
//
// Copyright 2023 ArangoDB GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Copyright holder is ArangoDB GmbH, Cologne, Germany
//
package k8sutil
type DeepCopy[T interface{}] interface {
DeepCopy() T
}