From fe97fc3cc08712a6aca47b45d02f57c333c48d09 Mon Sep 17 00:00:00 2001 From: Adam Janikowski <12255597+ajanikow@users.noreply.github.com> Date: Mon, 9 Sep 2024 20:00:31 +0200 Subject: [PATCH] [Feature] [Gateway] Dynamic Configuration (#1718) --- CHANGELOG.md | 1 + docs/api/ArangoDeployment.V1.md | 13 +- .../deployment/v1/deployment_spec_gateway.go | 14 ++ .../deployment/v1/zz_generated.deepcopy.go | 5 + .../v2alpha1/deployment_spec_gateway.go | 14 ++ .../v2alpha1/zz_generated.deepcopy.go | 5 + .../database-deployment.schema.generated.yaml | 10 + .../resources/config_map_gateway.go | 35 ++-- .../resources/config_map_gateway_member.go | 50 +++++ pkg/deployment/resources/config_maps.go | 3 + .../resources/config_maps_member.go | 172 ++++++++++++++++++ pkg/deployment/resources/gateway/dynamic.go | 93 ++++++++++ .../resources/gateway/gateway_config.go | 53 +++++- pkg/deployment/resources/gateway/marshal.go | 41 +++++ pkg/deployment/resources/pod_creator.go | 7 +- .../resources/pod_creator_gateway.go | 35 ++-- .../pod_creator_gateway_container.go | 26 ++- pkg/generated/timezones/timezones_test.go | 8 +- pkg/util/checksum.go | 8 + pkg/util/dict.go | 6 + 20 files changed, 552 insertions(+), 47 deletions(-) create mode 100644 pkg/deployment/resources/config_map_gateway_member.go create mode 100644 pkg/deployment/resources/config_maps_member.go create mode 100644 pkg/deployment/resources/gateway/dynamic.go create mode 100644 pkg/deployment/resources/gateway/marshal.go diff --git a/CHANGELOG.md b/CHANGELOG.md index cd801c513..9338e16d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ - (Feature) (Gateway) ArangoDB JWT Auth Integration - (Feature) Scheduler Handler - (Feature) (Gateway) ArangoDB Auth Token +- (Feature) (Gateway) Dynamic Configuration ## [1.2.42](https://github.com/arangodb/kube-arangodb/tree/1.2.42) (2024-07-23) - (Maintenance) Go 1.22.4 & Kubernetes 1.29.6 libraries diff --git a/docs/api/ArangoDeployment.V1.md b/docs/api/ArangoDeployment.V1.md index 7c3763641..8ca8d7f04 100644 --- a/docs/api/ArangoDeployment.V1.md +++ b/docs/api/ArangoDeployment.V1.md @@ -3043,6 +3043,17 @@ Type: `boolean` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1. *** +### .spec.gateway.dynamic + +Type: `boolean` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.42/pkg/apis/deployment/v1/deployment_spec_gateway.go#L38) + +Dynamic setting enables/disables support dynamic configuration of the gateway in the cluster. +When enabled, gateway config will be reloaded by ConfigMap live updates. + +Default Value: `false` + +*** + ### .spec.gateway.enabled Type: `boolean` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.42/pkg/apis/deployment/v1/deployment_spec_gateway.go#L33) @@ -3056,7 +3067,7 @@ Default Value: `false` ### .spec.gateway.image -Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.42/pkg/apis/deployment/v1/deployment_spec_gateway.go#L37) +Type: `string` [\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.42/pkg/apis/deployment/v1/deployment_spec_gateway.go#L42) Image is the image to use for the gateway. By default, the image is determined by the operator. diff --git a/pkg/apis/deployment/v1/deployment_spec_gateway.go b/pkg/apis/deployment/v1/deployment_spec_gateway.go index c194f1880..7cc263783 100644 --- a/pkg/apis/deployment/v1/deployment_spec_gateway.go +++ b/pkg/apis/deployment/v1/deployment_spec_gateway.go @@ -32,6 +32,11 @@ type DeploymentSpecGateway struct { // +doc/default: false Enabled *bool `json:"enabled,omitempty"` + // Dynamic setting enables/disables support dynamic configuration of the gateway in the cluster. + // When enabled, gateway config will be reloaded by ConfigMap live updates. + // +doc/default: false + Dynamic *bool `json:"dynamic,omitempty"` + // Image is the image to use for the gateway. // By default, the image is determined by the operator. Image *string `json:"image"` @@ -49,6 +54,15 @@ func (d *DeploymentSpecGateway) IsEnabled() bool { return *d.Enabled } +// IsDynamic returns whether the gateway dynamic config is enabled. +func (d *DeploymentSpecGateway) IsDynamic() bool { + if d == nil || d.Dynamic == nil { + return false + } + + return *d.Dynamic +} + func (d *DeploymentSpecGateway) GetSidecar() *schedulerApi.IntegrationSidecar { if d == nil || d.Sidecar == nil { return nil diff --git a/pkg/apis/deployment/v1/zz_generated.deepcopy.go b/pkg/apis/deployment/v1/zz_generated.deepcopy.go index 5993516d7..5c7fe4b2d 100644 --- a/pkg/apis/deployment/v1/zz_generated.deepcopy.go +++ b/pkg/apis/deployment/v1/zz_generated.deepcopy.go @@ -1180,6 +1180,11 @@ func (in *DeploymentSpecGateway) DeepCopyInto(out *DeploymentSpecGateway) { *out = new(bool) **out = **in } + if in.Dynamic != nil { + in, out := &in.Dynamic, &out.Dynamic + *out = new(bool) + **out = **in + } if in.Image != nil { in, out := &in.Image, &out.Image *out = new(string) diff --git a/pkg/apis/deployment/v2alpha1/deployment_spec_gateway.go b/pkg/apis/deployment/v2alpha1/deployment_spec_gateway.go index 166916462..fe897b2b4 100644 --- a/pkg/apis/deployment/v2alpha1/deployment_spec_gateway.go +++ b/pkg/apis/deployment/v2alpha1/deployment_spec_gateway.go @@ -32,6 +32,11 @@ type DeploymentSpecGateway struct { // +doc/default: false Enabled *bool `json:"enabled,omitempty"` + // Dynamic setting enables/disables support dynamic configuration of the gateway in the cluster. + // When enabled, gateway config will be reloaded by ConfigMap live updates. + // +doc/default: false + Dynamic *bool `json:"dynamic,omitempty"` + // Image is the image to use for the gateway. // By default, the image is determined by the operator. Image *string `json:"image"` @@ -49,6 +54,15 @@ func (d *DeploymentSpecGateway) IsEnabled() bool { return *d.Enabled } +// IsDynamic returns whether the gateway dynamic config is enabled. +func (d *DeploymentSpecGateway) IsDynamic() bool { + if d == nil || d.Dynamic == nil { + return false + } + + return *d.Dynamic +} + func (d *DeploymentSpecGateway) GetSidecar() *schedulerApi.IntegrationSidecar { if d == nil || d.Sidecar == nil { return nil diff --git a/pkg/apis/deployment/v2alpha1/zz_generated.deepcopy.go b/pkg/apis/deployment/v2alpha1/zz_generated.deepcopy.go index f0ed3e2e3..12de5471e 100644 --- a/pkg/apis/deployment/v2alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/deployment/v2alpha1/zz_generated.deepcopy.go @@ -1180,6 +1180,11 @@ func (in *DeploymentSpecGateway) DeepCopyInto(out *DeploymentSpecGateway) { *out = new(bool) **out = **in } + if in.Dynamic != nil { + in, out := &in.Dynamic, &out.Dynamic + *out = new(bool) + **out = **in + } if in.Image != nil { in, out := &in.Image, &out.Image *out = new(string) diff --git a/pkg/crd/crds/database-deployment.schema.generated.yaml b/pkg/crd/crds/database-deployment.schema.generated.yaml index af86e39dc..2ee177b8e 100644 --- a/pkg/crd/crds/database-deployment.schema.generated.yaml +++ b/pkg/crd/crds/database-deployment.schema.generated.yaml @@ -6567,6 +6567,11 @@ v1: gateway: description: Gateway defined main Gateway configuration. properties: + dynamic: + description: |- + Dynamic setting enables/disables support dynamic configuration of the gateway in the cluster. + When enabled, gateway config will be reloaded by ConfigMap live updates. + type: boolean enabled: description: |- Enabled setting enables/disables support for gateway in the cluster. @@ -23084,6 +23089,11 @@ v1alpha: gateway: description: Gateway defined main Gateway configuration. properties: + dynamic: + description: |- + Dynamic setting enables/disables support dynamic configuration of the gateway in the cluster. + When enabled, gateway config will be reloaded by ConfigMap live updates. + type: boolean enabled: description: |- Enabled setting enables/disables support for gateway in the cluster. diff --git a/pkg/deployment/resources/config_map_gateway.go b/pkg/deployment/resources/config_map_gateway.go index 57c54fc98..f539e40a4 100644 --- a/pkg/deployment/resources/config_map_gateway.go +++ b/pkg/deployment/resources/config_map_gateway.go @@ -55,7 +55,26 @@ func (r *Resources) ensureGatewayConfig(ctx context.Context, cachedStatus inspec return errors.WithStack(errors.Wrapf(err, "Failed to generate gateway config")) } - gatewayCfgYaml, gatewayCfgChecksum, _, err := cfg.RenderYAML() + gatewayCfgYaml, _, _, err := cfg.RenderYAML() + if err != nil { + return errors.WithStack(errors.Wrapf(err, "Failed to render gateway config")) + } + + gatewayCfgCDSYaml, _, _, err := cfg.RenderCDSYAML() + if err != nil { + return errors.WithStack(errors.Wrapf(err, "Failed to render gateway cds config")) + } + + gatewayCfgLDSYaml, _, _, err := cfg.RenderLDSYAML() + if err != nil { + return errors.WithStack(errors.Wrapf(err, "Failed to render gateway lds config")) + } + + elements, err := r.renderConfigMap(map[string]string{ + GatewayConfigFileName: string(gatewayCfgYaml), + GatewayCDSConfigFileName: string(gatewayCfgCDSYaml), + GatewayLDSConfigFileName: string(gatewayCfgLDSYaml), + }) if err != nil { return errors.WithStack(errors.Wrapf(err, "Failed to render gateway config")) } @@ -66,10 +85,7 @@ func (r *Resources) ensureGatewayConfig(ctx context.Context, cachedStatus inspec ObjectMeta: meta.ObjectMeta{ Name: configMapName, }, - Data: map[string]string{ - GatewayConfigFileName: string(gatewayCfgYaml), - GatewayConfigChecksumFileName: gatewayCfgChecksum, - }, + Data: elements, } owner := r.context.GetAPIObject().AsOwner() @@ -88,17 +104,14 @@ func (r *Resources) ensureGatewayConfig(ctx context.Context, cachedStatus inspec return errors.Reconcile() } else { // CM Exists, checks checksum - if key is not in the map we return empty string - if existingSha, existingChecksumSha := util.SHA256FromString(cm.Data[GatewayConfigFileName]), cm.Data[GatewayConfigChecksumFileName]; existingSha != gatewayCfgChecksum || existingChecksumSha != gatewayCfgChecksum { + if currentSha, expectedSha := util.Optional(cm.Data, ConfigMapChecksumKey, ""), util.Optional(elements, ConfigMapChecksumKey, ""); currentSha != expectedSha || currentSha == "" { // We need to do the update if _, changed, err := patcher.Patcher[*core.ConfigMap](ctx, cachedStatus.ConfigMapsModInterface().V1(), cm, meta.PatchOptions{}, - patcher.PatchConfigMapData(map[string]string{ - GatewayConfigFileName: string(gatewayCfgYaml), - GatewayConfigChecksumFileName: gatewayCfgChecksum, - })); err != nil { + patcher.PatchConfigMapData(elements)); err != nil { log.Err(err).Debug("Failed to patch GatewayConfig ConfigMap") return errors.WithStack(err) } else if changed { - log.Str("service", cm.GetName()).Str("before", existingSha).Str("after", gatewayCfgChecksum).Info("Updated GatewayConfig") + log.Str("configmap", cm.GetName()).Str("before", currentSha).Str("after", expectedSha).Info("Updated GatewayConfig") } } } diff --git a/pkg/deployment/resources/config_map_gateway_member.go b/pkg/deployment/resources/config_map_gateway_member.go new file mode 100644 index 000000000..2ca0c509e --- /dev/null +++ b/pkg/deployment/resources/config_map_gateway_member.go @@ -0,0 +1,50 @@ +// +// DISCLAIMER +// +// Copyright 2024 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// + +package resources + +import ( + "context" + + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" + "github.com/arangodb/kube-arangodb/pkg/deployment/resources/gateway" + inspectorInterface "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector" +) + +func (r *Resources) ensureMemberConfigGatewayConfig(ctx context.Context, cachedStatus inspectorInterface.Inspector, member api.DeploymentStatusMemberElement) (map[string]string, error) { + if member.Group != api.ServerGroupGateways { + return nil, nil + } + + data, _, _, err := gateway.NodeDynamicConfig("arangodb", member.Member.ID, &gateway.DynamicConfig{ + Path: GatewayVolumeMountDir, + File: GatewayCDSConfigFileName, + }, &gateway.DynamicConfig{ + Path: GatewayVolumeMountDir, + File: GatewayLDSConfigFileName, + }) + if err != nil { + return nil, err + } + + return map[string]string{ + GatewayDynamicConfigFileName: string(data), + }, nil +} diff --git a/pkg/deployment/resources/config_maps.go b/pkg/deployment/resources/config_maps.go index e6fec0239..495ce477c 100644 --- a/pkg/deployment/resources/config_maps.go +++ b/pkg/deployment/resources/config_maps.go @@ -54,6 +54,9 @@ func (r *Resources) EnsureConfigMaps(ctx context.Context, cachedStatus inspector if err := reconcileRequired.WithError(r.ensureGatewayConfig(ctx, cachedStatus, configMaps)); err != nil { return errors.Section(err, "Gateway ConfigMap") } + if err := reconcileRequired.WithError(r.ensureMemberConfig(ctx, cachedStatus, configMaps)); err != nil { + return errors.Section(err, "Member ConfigMap") + } } return reconcileRequired.Reconcile(ctx) } diff --git a/pkg/deployment/resources/config_maps_member.go b/pkg/deployment/resources/config_maps_member.go new file mode 100644 index 000000000..f65dc2bec --- /dev/null +++ b/pkg/deployment/resources/config_maps_member.go @@ -0,0 +1,172 @@ +// +// DISCLAIMER +// +// Copyright 2024 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// + +package resources + +import ( + "context" + + core "k8s.io/api/core/v1" + meta "k8s.io/apimachinery/pkg/apis/meta/v1" + + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" + "github.com/arangodb/kube-arangodb/pkg/util" + "github.com/arangodb/kube-arangodb/pkg/util/assertion" + "github.com/arangodb/kube-arangodb/pkg/util/errors" + "github.com/arangodb/kube-arangodb/pkg/util/globals" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + inspectorInterface "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector" + configMapsV1 "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector/configmap/v1" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/kerrors" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/patcher" +) + +const ( + ConfigMapChecksumKey = "CHECKSUM" + + MemberConfigVolumeMountDir = "/etc/member/" + MemberConfigVolumeName = "member-config" + + MemberConfigChecksumENV = "MEMBER_CONFIG_CHECKSUM" +) + +type memberConfigMapRenderer func(ctx context.Context, cachedStatus inspectorInterface.Inspector, member api.DeploymentStatusMemberElement) (map[string]string, error) + +func (r *Resources) ensureMemberConfig(ctx context.Context, cachedStatus inspectorInterface.Inspector, configMaps configMapsV1.ModInterface) error { + status := r.context.GetStatus() + + log := r.log.Str("section", "member-config-render") + + reconcileRequired := k8sutil.NewReconcile(cachedStatus) + + members := status.Members.AsList() + + if err := reconcileRequired.ParallelAll(len(members), func(id int) error { + memberName := members[id].Member.ArangoMemberName(r.context.GetAPIObject().GetName(), members[id].Group) + + am, ok := cachedStatus.ArangoMember().V1().GetSimple(memberName) + if !ok { + return errors.Errorf("ArangoMember %s not found", memberName) + } + + switch members[id].Group.Type() { + case api.ServerGroupTypeGateway, api.ServerGroupTypeArangoSync, api.ServerGroupTypeArangoD: + elements, err := r.renderMemberConfigElements(ctx, cachedStatus, members[id], r.ensureMemberConfigGatewayConfig) + if err != nil { + return err + } + + if len(elements) == 0 { + // CM should be gone + if obj, ok := cachedStatus.ConfigMap().V1().GetSimple(memberName); !ok { + return nil + } else { + if err := cachedStatus.ConfigMapsModInterface().V1().Delete(ctx, memberName, meta.DeleteOptions{ + Preconditions: meta.NewUIDPreconditions(string(obj.GetUID())), + }); err != nil { + if !kerrors.IsNotFound(err) { + return err + } + } + } + } else { + // We expect CM + if obj, ok := cachedStatus.ConfigMap().V1().GetSimple(memberName); !ok { + // Let's Create ConfigMap + obj = &core.ConfigMap{ + ObjectMeta: meta.ObjectMeta{ + Name: memberName, + }, + Data: elements, + } + + err = globals.GetGlobalTimeouts().Kubernetes().RunWithTimeout(ctx, func(ctxChild context.Context) error { + return k8sutil.CreateConfigMap(ctxChild, configMaps, obj, util.NewType(am.AsOwner())) + }) + if kerrors.IsAlreadyExists(err) { + // CM added while we tried it also + return nil + } else if err != nil { + // Failed to create + return errors.WithStack(err) + } + + return errors.Reconcile() + } else { + // CM Exists, checks checksum - if key is not in the map we return empty string + if currentSha, expectedSha := util.Optional(obj.Data, ConfigMapChecksumKey, ""), util.Optional(elements, ConfigMapChecksumKey, ""); currentSha != expectedSha || currentSha == "" { + // We need to do the update + if _, changed, err := patcher.Patcher[*core.ConfigMap](ctx, cachedStatus.ConfigMapsModInterface().V1(), obj, meta.PatchOptions{}, + patcher.PatchConfigMapData(elements)); err != nil { + log.Err(err).Debug("Failed to patch GatewayConfig ConfigMap") + return errors.WithStack(err) + } else if changed { + log.Str("service", obj.GetName()).Str("before", currentSha).Str("after", expectedSha).Info("Updated Member Config") + } + } + } + } + return nil + default: + assertion.InvalidGroupKey.Assert(true, "Unable to create Member ConfigMap an unknown group: %s", members[id].Group.AsRole()) + return nil + } + }); err != nil { + return errors.Section(err, "Member ConfigMap") + } + + return nil +} + +func (r *Resources) renderConfigMap(elements ...map[string]string) (map[string]string, error) { + result := map[string]string{} + + for _, r := range elements { + for k, v := range r { + if _, ok := result[k]; ok { + return nil, errors.Errorf("Key %s already defined", k) + } + + result[k] = v + } + } + + if len(result) == 0 { + return nil, nil + } + + result[ConfigMapChecksumKey] = util.SHA256FromStringMap(result) + + return result, nil +} + +func (r *Resources) renderMemberConfigElements(ctx context.Context, cachedStatus inspectorInterface.Inspector, member api.DeploymentStatusMemberElement, renders ...memberConfigMapRenderer) (map[string]string, error) { + var elements = make([]map[string]string, len(renders)) + + for _, r := range renders { + if els, err := r(ctx, cachedStatus, member); err != nil { + return nil, errors.Wrapf(err, "Unable to render CM for %s", member.Member.ID) + } else { + elements = append(elements, els) + } + } + + return r.renderConfigMap(elements...) +} diff --git a/pkg/deployment/resources/gateway/dynamic.go b/pkg/deployment/resources/gateway/dynamic.go new file mode 100644 index 000000000..6fb8ea7db --- /dev/null +++ b/pkg/deployment/resources/gateway/dynamic.go @@ -0,0 +1,93 @@ +// +// DISCLAIMER +// +// Copyright 2024 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// + +package gateway + +import ( + "path" + + bootstrapAPI "github.com/envoyproxy/go-control-plane/envoy/config/bootstrap/v3" + coreAPI "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + discoveryApi "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + proto "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" +) + +type DynamicConfig struct { + Path, File string +} + +func (d *DynamicConfig) AsConfigSource() *coreAPI.ConfigSource { + if d == nil { + return nil + } + + return &coreAPI.ConfigSource{ + ConfigSourceSpecifier: &coreAPI.ConfigSource_PathConfigSource{ + PathConfigSource: &coreAPI.PathConfigSource{ + Path: path.Join(d.Path, d.File), + WatchedDirectory: &coreAPI.WatchedDirectory{ + Path: d.Path, + }, + }, + }, + } +} + +func NodeDynamicConfig(cluster, id string, cds, lds *DynamicConfig) ([]byte, string, *bootstrapAPI.Bootstrap, error) { + var b = bootstrapAPI.Bootstrap{ + Node: &coreAPI.Node{ + Id: id, + Cluster: cluster, + }, + } + + if v := cds; v != nil { + if b.DynamicResources == nil { + b.DynamicResources = &bootstrapAPI.Bootstrap_DynamicResources{} + } + + b.DynamicResources.CdsConfig = v.AsConfigSource() + } + + if v := lds; v != nil { + if b.DynamicResources == nil { + b.DynamicResources = &bootstrapAPI.Bootstrap_DynamicResources{} + } + + b.DynamicResources.LdsConfig = v.AsConfigSource() + } + + return Marshal(&b) +} + +func DynamicConfigResponse[T proto.Message](in ...T) (*discoveryApi.DiscoveryResponse, error) { + resources := make([]*anypb.Any, len(in)) + for id := range in { + if a, err := anypb.New(in[id]); err != nil { + return nil, err + } else { + resources[id] = a + } + } + return &discoveryApi.DiscoveryResponse{ + Resources: resources, + }, nil +} diff --git a/pkg/deployment/resources/gateway/gateway_config.go b/pkg/deployment/resources/gateway/gateway_config.go index 5206aaafd..e7c360e61 100644 --- a/pkg/deployment/resources/gateway/gateway_config.go +++ b/pkg/deployment/resources/gateway/gateway_config.go @@ -36,11 +36,11 @@ import ( tlsInspectorApi "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/listener/tls_inspector/v3" httpConnectionManagerAPI "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" upstreamHttpApi "github.com/envoyproxy/go-control-plane/envoy/extensions/upstreams/http/v3" + discoveryApi "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" "github.com/golang/protobuf/ptypes/any" - "google.golang.org/protobuf/encoding/protojson" + "github.com/golang/protobuf/ptypes/wrappers" "google.golang.org/protobuf/types/known/anypb" "google.golang.org/protobuf/types/known/durationpb" - "sigs.k8s.io/yaml" shared "github.com/arangodb/kube-arangodb/pkg/apis/shared" "github.com/arangodb/kube-arangodb/pkg/util" @@ -74,15 +74,51 @@ func (c Config) RenderYAML() ([]byte, string, *bootstrapAPI.Bootstrap, error) { return nil, "", nil, err } - data, err := protojson.MarshalOptions{ - UseProtoNames: true, - }.Marshal(cfg) + return Marshal(cfg) +} + +func (c Config) RenderCDSYAML() ([]byte, string, *discoveryApi.DiscoveryResponse, error) { + cfg, err := c.RenderCDS() if err != nil { return nil, "", nil, err } - data, err = yaml.JSONToYAML(data) - return data, util.SHA256(data), cfg, err + return Marshal(cfg) +} + +func (c Config) RenderLDSYAML() ([]byte, string, *discoveryApi.DiscoveryResponse, error) { + cfg, err := c.RenderLDS() + if err != nil { + return nil, "", nil, err + } + + return Marshal(cfg) +} + +func (c Config) RenderCDS() (*discoveryApi.DiscoveryResponse, error) { + if err := c.Validate(); err != nil { + return nil, errors.Wrapf(err, "Validation failed") + } + + clusters, err := c.RenderClusters() + if err != nil { + return nil, errors.Wrapf(err, "Unable to render clusters") + } + + return DynamicConfigResponse(clusters...) +} + +func (c Config) RenderLDS() (*discoveryApi.DiscoveryResponse, error) { + if err := c.Validate(); err != nil { + return nil, errors.Wrapf(err, "Validation failed") + } + + listener, err := c.RenderListener() + if err != nil { + return nil, errors.Wrapf(err, "Unable to render listener") + } + + return DynamicConfigResponse(listener) } func (c Config) Render() (*bootstrapAPI.Bootstrap, error) { @@ -268,6 +304,9 @@ func (c Config) RenderFilters() ([]*listenerAPI.Filter, error) { Routes: routes, }, }, + ValidateClusters: &wrappers.BoolValue{ + Value: false, + }, }, }, HttpFilters: append(httpFilters, &httpConnectionManagerAPI.HttpFilter{ diff --git a/pkg/deployment/resources/gateway/marshal.go b/pkg/deployment/resources/gateway/marshal.go new file mode 100644 index 000000000..69bb5b95a --- /dev/null +++ b/pkg/deployment/resources/gateway/marshal.go @@ -0,0 +1,41 @@ +// +// DISCLAIMER +// +// Copyright 2024 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// + +package gateway + +import ( + "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/proto" + "sigs.k8s.io/yaml" + + "github.com/arangodb/kube-arangodb/pkg/util" +) + +func Marshal[T proto.Message](in T) ([]byte, string, T, error) { + data, err := protojson.MarshalOptions{ + UseProtoNames: true, + }.Marshal(in) + if err != nil { + return nil, "", util.Default[T](), err + } + + data, err = yaml.JSONToYAML(data) + return data, util.SHA256(data), in, err +} diff --git a/pkg/deployment/resources/pod_creator.go b/pkg/deployment/resources/pod_creator.go index 2b1782f2c..518f43b6a 100644 --- a/pkg/deployment/resources/pod_creator.go +++ b/pkg/deployment/resources/pod_creator.go @@ -26,6 +26,7 @@ import ( "encoding/json" "fmt" "net" + "path" "path/filepath" "strconv" "sync" @@ -288,7 +289,11 @@ func createArangoSyncArgs(apiObject meta.Object, spec api.DeploymentSpec, group func createArangoGatewayArgs(input pod.Input, additionalOptions ...k8sutil.OptionPair) []string { options := k8sutil.CreateOptionPairs(64) - options.Add("--config-path", GatewayConfigFilePath) + if input.Deployment.Gateway.IsDynamic() { + options.Add("--config-path", path.Join(MemberConfigVolumeMountDir, GatewayDynamicConfigFileName)) + } else { + options.Add("--config-path", path.Join(GatewayVolumeMountDir, GatewayConfigFileName)) + } options.Append(additionalOptions...) diff --git a/pkg/deployment/resources/pod_creator_gateway.go b/pkg/deployment/resources/pod_creator_gateway.go index 55df1a4b3..0450b007e 100644 --- a/pkg/deployment/resources/pod_creator_gateway.go +++ b/pkg/deployment/resources/pod_creator_gateway.go @@ -30,13 +30,14 @@ import ( ) const ( - ArangoGatewayExecutor string = "/usr/local/bin/envoy" - GatewayVolumeMountDir = "/etc/gateway/" - GatewayVolumeName = "gateway" - GatewayConfigFileName = "gateway.yaml" - GatewayConfigChecksumFileName = "gateway.checksum" - GatewayConfigChecksumENV = "GATEWAY_CONFIG_CHECKSUM" - GatewayConfigFilePath = GatewayVolumeMountDir + GatewayConfigFileName + ArangoGatewayExecutor = "/usr/local/bin/envoy" + GatewayVolumeMountDir = "/etc/gateway/" + GatewayVolumeName = "gateway" + GatewayConfigFileName = "gateway.yaml" + GatewayDynamicConfigFileName = "gateway.dynamic.yaml" + GatewayCDSConfigFileName = "gateway.dynamic.cds.yaml" + GatewayLDSConfigFileName = "gateway.dynamic.lds.yaml" + GatewayConfigChecksumENV = "GATEWAY_CONFIG_CHECKSUM" ) func GetGatewayConfigMapName(name string) string { @@ -47,7 +48,17 @@ func createGatewayVolumes(input pod.Input) pod.Volumes { volumes := pod.NewVolumes() volumes.AddVolume(k8sutil.CreateVolumeWithConfigMap(GatewayVolumeName, GetGatewayConfigMapName(input.ApiObject.GetName()))) - volumes.AddVolumeMount(GatewayVolumeMount()) + volumes.AddVolume(k8sutil.CreateVolumeWithConfigMap(MemberConfigVolumeName, input.ArangoMember.GetName())) + volumes.AddVolumeMount(core.VolumeMount{ + Name: GatewayVolumeName, + MountPath: GatewayVolumeMountDir, + ReadOnly: true, + }) + volumes.AddVolumeMount(core.VolumeMount{ + Name: MemberConfigVolumeName, + MountPath: MemberConfigVolumeMountDir, + ReadOnly: true, + }) // TLS volumes.Append(pod.TLS(), input) @@ -57,11 +68,3 @@ func createGatewayVolumes(input pod.Input) pod.Volumes { return volumes } - -func GatewayVolumeMount() core.VolumeMount { - return core.VolumeMount{ - Name: GatewayVolumeName, - MountPath: GatewayVolumeMountDir, - ReadOnly: true, - } -} diff --git a/pkg/deployment/resources/pod_creator_gateway_container.go b/pkg/deployment/resources/pod_creator_gateway_container.go index cc4a3767c..2a9989427 100644 --- a/pkg/deployment/resources/pod_creator_gateway_container.go +++ b/pkg/deployment/resources/pod_creator_gateway_container.go @@ -131,18 +131,26 @@ func (a *ArangoGatewayContainer) GetEnvs() ([]core.EnvVar, []core.EnvFromSource) envs.Add(true, k8sutil.GetLifecycleEnv()...) - var cmChecksum = "" - - if cm, ok := a.cachedStatus.ConfigMap().V1().GetSimple(GetGatewayConfigMapName(a.input.ApiObject.GetName())); ok { - if v, ok := cm.Data[GatewayConfigChecksumFileName]; ok { - cmChecksum = v + if cm, ok := a.cachedStatus.ConfigMap().V1().GetSimple(GetGatewayConfigMapName(a.input.ArangoMember.GetName())); ok { + if v, ok := cm.Data[ConfigMapChecksumKey]; ok { + envs.Add(true, core.EnvVar{ + Name: MemberConfigChecksumENV, + Value: v, + }) } } - envs.Add(true, core.EnvVar{ - Name: GatewayConfigChecksumENV, - Value: cmChecksum, - }) + if !a.spec.Gateway.IsDynamic() { + if cm, ok := a.cachedStatus.ConfigMap().V1().GetSimple(GetGatewayConfigMapName(a.input.ApiObject.GetName())); ok { + if v, ok := cm.Data[ConfigMapChecksumKey]; ok { + envs.Add(true, core.EnvVar{ + Name: GatewayConfigChecksumENV, + Value: v, + }) + } + } + + } if len(a.groupSpec.Envs) > 0 { for _, env := range a.groupSpec.Envs { diff --git a/pkg/generated/timezones/timezones_test.go b/pkg/generated/timezones/timezones_test.go index 983e415b2..f809c8fe7 100644 --- a/pkg/generated/timezones/timezones_test.go +++ b/pkg/generated/timezones/timezones_test.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany +// Copyright 2016-2024 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -28,6 +28,10 @@ import ( ) func Test_Timezone(t *testing.T) { + // Ensure we use static time for comparison + testTime, err := time.Parse(time.RFC3339, "2024-09-01T00:00:00Z") + require.NoError(t, err) + for tz, tq := range timezones { t.Run(tz, func(t *testing.T) { t.Run("Check fields", func(t *testing.T) { @@ -49,7 +53,7 @@ func Test_Timezone(t *testing.T) { l, err := time.LoadLocationFromTZData("", tz) require.NoError(t, err) - z, offset := time.Now().In(l).Zone() + z, offset := testTime.In(l).Zone() require.Equal(t, tq.Zone, z) require.Equal(t, int(tq.Offset/time.Second), offset) diff --git a/pkg/util/checksum.go b/pkg/util/checksum.go index 144bbe291..17a7ab9f5 100644 --- a/pkg/util/checksum.go +++ b/pkg/util/checksum.go @@ -50,6 +50,14 @@ func SHA256FromStringArray(data ...string) string { return SHA256FromString(strings.Join(data, "|")) } +func SHA256FromStringMap(data map[string]string) string { + return SHA256FromExtract(func(t KV[string, string]) string { + return fmt.Sprintf("%s:%s", t.K, SHA256FromString(t.V)) + }, ExtractWithSort(data, func(i, j string) bool { + return i < j + })...) +} + func SHA256FromString(data string) string { return SHA256([]byte(data)) } diff --git a/pkg/util/dict.go b/pkg/util/dict.go index 38af1a980..dcf083da5 100644 --- a/pkg/util/dict.go +++ b/pkg/util/dict.go @@ -43,6 +43,12 @@ func Extract[K comparable, V any](in map[K]V) []KV[K, V] { return r } +func ExtractWithSort[K comparable, V any](in map[K]V, cmp func(i, j K) bool) []KV[K, V] { + return Sort(Extract(in), func(i, j KV[K, V]) bool { + return cmp(i.K, j.K) + }) +} + func Sort[IN any](in []IN, cmp func(i, j IN) bool) []IN { r := make([]IN, len(in)) copy(r, in)