1
0
Fork 0
mirror of https://github.com/arangodb/kube-arangodb.git synced 2024-12-14 11:57:37 +00:00

[Feature] Propagate env variables to members (#1100)

This commit is contained in:
Tomasz Mielech 2022-09-19 16:48:54 +02:00 committed by GitHub
parent 05f9757bf3
commit a4e26f3ce4
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
15 changed files with 349 additions and 25 deletions

View file

@ -28,7 +28,8 @@
- (Feature) Sensitive information protection
- (Bugfix) Propagate SecurityContext to the ID Containers
- (Bugfix) Fix for enabling all features
- (Feature) Propagate feature and predefined env variables to members
## [1.2.15](https://github.com/arangodb/kube-arangodb/tree/1.2.15) (2022-07-20)
- (Bugfix) Ensure pod names not too long
- (Refactor) Use cached member's clients

View file

@ -27,7 +27,7 @@ rules:
verbs: ["*"]
{{- end }}
- apiGroups: [""]
resources: ["pods", "services", "endpoints", "persistentvolumeclaims", "events", "secrets", "serviceaccounts"]
resources: ["pods", "services", "endpoints", "persistentvolumeclaims", "events", "secrets", "serviceaccounts", "configmaps"]
verbs: ["*"]
- apiGroups: ["apps"]
resources: ["deployments", "replicasets"]

View file

@ -27,6 +27,7 @@ import (
"net"
"net/http"
"os"
"reflect"
"strconv"
"strings"
"time"
@ -120,6 +121,8 @@ var (
versionOnly bool // Run only version endpoint, explicitly disabled with other
enableK2KClusterSync bool // Run k2kClusterSync operator
operatorFeatureConfigMap string // ConfigMap name
scalingIntegrationEnabled bool
alpineImage, metricsExporterImage, arangoImage string
@ -320,6 +323,9 @@ func executeMain(cmd *cobra.Command, args []string) {
if err != nil {
logger.Err(err).Fatal("Failed to create operator config & deps")
}
if err := ensureFeaturesConfigMap(context.Background(), client.Kubernetes().CoreV1().ConfigMaps(namespace), cfg); err != nil {
logger.Err(err).Error("Failed to create features config map")
}
o, err := operator.NewOperator(cfg, deps)
if err != nil {
logger.Err(err).Fatal("Failed to create operator")
@ -531,3 +537,53 @@ func createRecorder(kubecli kubernetes.Interface, name, namespace string) record
apps.AddToScheme(combinedScheme)
return eventBroadcaster.NewRecorder(combinedScheme, core.EventSource{Component: name})
}
// ensureFeaturesConfigMap creates or updates config map with enabled features.
func ensureFeaturesConfigMap(ctx context.Context, client typedCore.ConfigMapInterface, cfg operator.Config) error {
ft := features.GetFeatureMap()
featuresCM := make(map[string]string, len(ft))
for k, v := range ft {
if v {
featuresCM[k] = features.Enabled
} else {
featuresCM[k] = features.Disabled
}
}
nctx, c := globals.GetGlobalTimeouts().Kubernetes().WithTimeout(ctx)
defer c()
if cm, err := client.Get(nctx, features.ConfigMapName(), meta.GetOptions{}); err != nil {
if !deploymentApi.IsNotFound(err) {
return err
}
nctx, c := globals.GetGlobalTimeouts().Kubernetes().WithTimeout(ctx)
defer c()
if _, err := client.Create(nctx, &core.ConfigMap{
ObjectMeta: meta.ObjectMeta{
Name: features.ConfigMapName(),
Namespace: cfg.Namespace,
},
Data: make(map[string]string),
}, meta.CreateOptions{}); err != nil {
return err
}
return nil
} else if !reflect.DeepEqual(cm.Data, featuresCM) {
q := cm.DeepCopy()
q.Data = featuresCM
nctx, c := globals.GetGlobalTimeouts().Kubernetes().WithTimeout(ctx)
defer c()
if _, err := client.Update(nctx, q, meta.UpdateOptions{}); err != nil {
return err
}
return nil
}
return nil
}

View file

@ -172,7 +172,6 @@ func TestEnsurePod_ArangoDB_Volumes(t *testing.T) {
},
},
},
{
Name: "DBserver POD with Volume Mount",
ArangoDeployment: &api.ArangoDeployment{

View file

@ -26,6 +26,7 @@ import (
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"testing"
@ -44,9 +45,11 @@ import (
"github.com/arangodb/kube-arangodb/pkg/apis/shared"
"github.com/arangodb/kube-arangodb/pkg/deployment/acs"
"github.com/arangodb/kube-arangodb/pkg/deployment/client"
"github.com/arangodb/kube-arangodb/pkg/deployment/features"
"github.com/arangodb/kube-arangodb/pkg/deployment/resources"
"github.com/arangodb/kube-arangodb/pkg/deployment/resources/inspector"
arangofake "github.com/arangodb/kube-arangodb/pkg/generated/clientset/versioned/fake"
"github.com/arangodb/kube-arangodb/pkg/util"
"github.com/arangodb/kube-arangodb/pkg/util/arangod/conn"
"github.com/arangodb/kube-arangodb/pkg/util/constants"
"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
@ -608,6 +611,8 @@ func (testCase *testCaseStruct) createTestPodData(deployment *Deployment, group
deployment.currentObject.Status.Members.Update(member, group)
}
testCase.createTestEnvVariables(deployment, group)
}
func finalizers(group api.ServerGroup) []string {
@ -786,3 +791,81 @@ func addLifecycle(name string, uuidRequired bool, license string, group api.Serv
}
}
}
func (testCase *testCaseStruct) createTestEnvVariables(deployment *Deployment, group api.ServerGroup) {
if group == api.ServerGroupSyncMasters || group == api.ServerGroupSyncWorkers {
return
}
// Set up environment variables.
for i, container := range testCase.ExpectedPod.Spec.Containers {
if container.Name != api.ServerGroupReservedContainerNameServer {
continue
}
testCase.ExpectedPod.Spec.Containers[i].EnvFrom = []core.EnvFromSource{
{
ConfigMapRef: &core.ConfigMapEnvSource{
LocalObjectReference: core.LocalObjectReference{
Name: features.ConfigMapName(),
},
Optional: util.NewBool(true),
},
},
}
var version, enterprise string
if len(deployment.currentObjectStatus.Images) > 0 {
version = string(deployment.currentObjectStatus.Images[0].ArangoDBVersion)
enterprise = strconv.FormatBool(deployment.currentObjectStatus.Images[0].Enterprise)
}
if version == "" {
version = testVersion
}
if enterprise == "" {
enterprise = "false"
}
if !isEnvExist(testCase.ExpectedPod.Spec.Containers[i].Env, resources.ArangoDBOverrideServerGroupEnv) {
testCase.ExpectedPod.Spec.Containers[i].Env = append(testCase.ExpectedPod.Spec.Containers[i].Env,
core.EnvVar{
Name: resources.ArangoDBOverrideServerGroupEnv,
Value: group.AsRole(),
})
}
if !isEnvExist(testCase.ExpectedPod.Spec.Containers[i].Env, resources.ArangoDBOverrideDeploymentModeEnv) {
testCase.ExpectedPod.Spec.Containers[i].Env = append(testCase.ExpectedPod.Spec.Containers[i].Env,
core.EnvVar{
Name: resources.ArangoDBOverrideDeploymentModeEnv,
Value: string(testCase.ArangoDeployment.Spec.GetMode()),
})
}
if !isEnvExist(testCase.ExpectedPod.Spec.Containers[i].Env, resources.ArangoDBOverrideVersionEnv) {
testCase.ExpectedPod.Spec.Containers[i].Env = append(testCase.ExpectedPod.Spec.Containers[i].Env,
core.EnvVar{
Name: resources.ArangoDBOverrideVersionEnv,
Value: version,
})
}
if !isEnvExist(testCase.ExpectedPod.Spec.Containers[i].Env, resources.ArangoDBOverrideEnterpriseEnv) {
testCase.ExpectedPod.Spec.Containers[i].Env = append(testCase.ExpectedPod.Spec.Containers[i].Env,
core.EnvVar{
Name: resources.ArangoDBOverrideEnterpriseEnv,
Value: enterprise,
})
}
}
}
func isEnvExist(envs []core.EnvVar, name string) bool {
for _, env := range envs {
if env.Name == name {
return true
}
}
return false
}

View file

@ -0,0 +1,31 @@
//
// DISCLAIMER
//
// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Copyright holder is ArangoDB GmbH, Cologne, Germany
//
package features
const (
DefaultFeaturesConfigMap = "arangodb-operator-feature-config-map"
)
var configMapName = DefaultFeaturesConfigMap
func ConfigMapName() string {
return configMapName
}

View file

@ -22,6 +22,11 @@ package features
import "github.com/arangodb/go-driver"
const (
Enabled = "true"
Disabled = "false"
)
var _ Feature = &feature{}
type Feature interface {

View file

@ -22,13 +22,18 @@ package features
import (
"fmt"
"os"
"sync"
"github.com/spf13/cobra"
"github.com/arangodb/go-driver"
"github.com/arangodb/kube-arangodb/pkg/util"
)
const prefixArg = "deployment.feature"
var features = map[string]Feature{}
var featuresLock sync.Mutex
var enableAll = false
@ -54,6 +59,9 @@ var internalCMD = &cobra.Command{
Run: cmdRun,
}
// Init initializes all registered features.
// If a feature is not provided via process's argument, then it is taken from environment variable
// or from enabled by default setting.
func Init(cmd *cobra.Command) error {
featuresLock.Lock()
defer featuresLock.Unlock()
@ -62,7 +70,8 @@ func Init(cmd *cobra.Command) error {
f := cmd.Flags()
f.BoolVar(&enableAll, "deployment.feature.all", false, "Enable ALL Features")
featureArgName := GetFeatureArgName("all")
f.BoolVar(&enableAll, featureArgName, isEnabledFeatureFromEnv(featureArgName), "Enable ALL Features")
for _, feature := range features {
z := ""
@ -79,21 +88,25 @@ func Init(cmd *cobra.Command) error {
}
}
featureName := fmt.Sprintf("deployment.feature.%s", feature.Name())
f.BoolVar(feature.EnabledPointer(), featureName, feature.EnabledByDefault(), z)
featureArgName = GetFeatureArgName(feature.Name())
enabled := feature.EnabledByDefault() || isEnabledFeatureFromEnv(featureArgName)
f.BoolVar(feature.EnabledPointer(), featureArgName, enabled, z)
if ok, reason := feature.Deprecated(); ok {
if err := f.MarkDeprecated(featureName, reason); err != nil {
if err := f.MarkDeprecated(featureArgName, reason); err != nil {
return err
}
}
if feature.Hidden() {
if err := f.MarkHidden(featureName); err != nil {
if err := f.MarkHidden(featureArgName); err != nil {
return err
}
}
}
f.StringVar(&configMapName, "features-config-map-name", DefaultFeaturesConfigMap, "Name of the Feature Map ConfigMap")
return nil
}
@ -127,6 +140,39 @@ func cmdRun(_ *cobra.Command, _ []string) {
}
}
// Supported returns false when:
// - feature is disabled.
// - a given version is lower than minimum feature version.
// - feature expects enterprise but a given enterprise arg is not true.
func Supported(f Feature, v driver.Version, enterprise bool) bool {
return f.Enabled() && ((f.EnterpriseRequired() && enterprise) || !f.EnterpriseRequired()) && v.CompareTo(f.Version()) >= 0
if !f.Enabled() {
return false
}
if f.EnterpriseRequired() && !enterprise {
// This feature requires enterprise version but current version is not enterprise.
return false
}
return v.CompareTo(f.Version()) >= 0
}
// GetFeatureMap returns all features' arguments names.
func GetFeatureMap() map[string]bool {
args := make(map[string]bool, len(features))
for _, f := range features {
args[GetFeatureArgName(f.Name())] = f.Enabled()
}
return args
}
// GetFeatureArgName returns feature process argument name.
func GetFeatureArgName(featureName string) string {
return fmt.Sprintf("%s.%s", prefixArg, featureName)
}
// isEnabledFeatureFromEnv returns true if argument is enabled as an environment variable.
func isEnabledFeatureFromEnv(arg string) bool {
return os.Getenv(util.NormalizeEnv(arg)) == Enabled
}

View file

@ -396,8 +396,9 @@ func (a *ContainerIdentity) GetArgs() ([]string, error) {
return nil, nil
}
func (a *ContainerIdentity) GetEnvs() []core.EnvVar {
return nil
// GetEnvs returns environment variables for identity containers.
func (a *ContainerIdentity) GetEnvs() ([]core.EnvVar, []core.EnvFromSource) {
return nil, nil
}
func (a *ContainerIdentity) GetExecutor() string {
@ -461,7 +462,8 @@ func (a *ArangoDIdentity) GetArgs() ([]string, error) {
return options.Copy().Sort().AsArgs(), nil
}
func (a *ArangoDIdentity) GetEnvs() []core.EnvVar {
// GetEnvs returns environment variables for Arango identity containers.
func (a *ArangoDIdentity) GetEnvs() ([]core.EnvVar, []core.EnvFromSource) {
env := make([]core.EnvVar, 0)
// Add advanced check for license
@ -471,10 +473,10 @@ func (a *ArangoDIdentity) GetEnvs() []core.EnvVar {
}
if len(env) > 0 {
return env
return env, nil
}
return nil
return nil, nil
}
// GetVolumeMounts returns volume mount for the ArangoD data.

View file

@ -25,6 +25,7 @@ import (
"fmt"
"math"
"os"
"strconv"
core "k8s.io/api/core/v1"
@ -44,6 +45,10 @@ const (
ArangoDExecutor = "/usr/sbin/arangod"
ArangoDBOverrideDetectedTotalMemoryEnv = "ARANGODB_OVERRIDE_DETECTED_TOTAL_MEMORY"
ArangoDBOverrideDetectedNumberOfCoresEnv = "ARANGODB_OVERRIDE_DETECTED_NUMBER_OF_CORES"
ArangoDBOverrideServerGroupEnv = "ARANGODB_OVERRIDE_SERVER_GROUP"
ArangoDBOverrideDeploymentModeEnv = "ARANGODB_OVERRIDE_DEPLOYMENT_MODE"
ArangoDBOverrideVersionEnv = "ARANGODB_OVERRIDE_VERSION"
ArangoDBOverrideEnterpriseEnv = "ARANGODB_OVERRIDE_ENTERPRISE"
)
var _ interfaces.PodCreator = &MemberArangoDPod{}
@ -173,7 +178,8 @@ func (a *ArangoDContainer) GetImage() string {
}
}
func (a *ArangoDContainer) GetEnvs() []core.EnvVar {
// GetEnvs returns environment variables for ArangoDB containers.
func (a *ArangoDContainer) GetEnvs() ([]core.EnvVar, []core.EnvFromSource) {
envs := NewEnvBuilder()
if a.spec.License.HasSecretName() && a.imageInfo.ArangoDBVersion.CompareTo("3.9.0") < 0 {
@ -217,7 +223,36 @@ func (a *ArangoDContainer) GetEnvs() []core.EnvVar {
envs.Add(true, pod.Topology().Envs(a.member.AsInput())...)
return envs.GetEnvList()
envs.Add(true, core.EnvVar{
Name: ArangoDBOverrideServerGroupEnv,
Value: a.input.Group.AsRole(),
})
envs.Add(true, core.EnvVar{
Name: ArangoDBOverrideDeploymentModeEnv,
Value: string(a.input.Deployment.GetMode()),
})
envs.Add(true, core.EnvVar{
Name: ArangoDBOverrideVersionEnv,
Value: string(a.input.Version),
})
envs.Add(true, core.EnvVar{
Name: ArangoDBOverrideEnterpriseEnv,
Value: strconv.FormatBool(a.input.Enterprise),
})
envFromSource := []core.EnvFromSource{
{
ConfigMapRef: &core.ConfigMapEnvSource{
LocalObjectReference: core.LocalObjectReference{
Name: features.ConfigMapName(),
},
// Optional in case if operator could not create it when process started.
Optional: util.NewBool(true),
},
},
}
return envs.GetEnvList(), envFromSource
}
func (a *ArangoDContainer) GetResourceRequirements() core.ResourceRequirements {

View file

@ -151,7 +151,7 @@ func (a *ArangoSyncContainer) GetImage() string {
return a.imageInfo.Image
}
func (a *ArangoSyncContainer) GetEnvs() []core.EnvVar {
func (a *ArangoSyncContainer) GetEnvs() ([]core.EnvVar, []core.EnvFromSource) {
envs := NewEnvBuilder()
if a.spec.Sync.Monitoring.GetTokenSecretName() != "" {
@ -180,7 +180,7 @@ func (a *ArangoSyncContainer) GetEnvs() []core.EnvVar {
}
}
return envs.GetEnvList()
return envs.GetEnvList(), nil
}
func (a *ArangoSyncContainer) GetVolumeMounts() []core.VolumeMount {

View file

@ -28,6 +28,8 @@ import (
"k8s.io/apimachinery/pkg/api/equality"
api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
"github.com/arangodb/kube-arangodb/pkg/deployment/features"
"github.com/arangodb/kube-arangodb/pkg/deployment/resources"
"github.com/arangodb/kube-arangodb/pkg/deployment/topology"
"github.com/arangodb/kube-arangodb/pkg/util"
)
@ -68,17 +70,36 @@ func containersCompare(ds api.DeploymentSpec, g api.ServerGroup, spec, status *c
}
if !equality.Semantic.DeepEqual(ac.Env, bc.Env) {
if areEnvsEqual(ac.Env, bc.Env, func(a, b map[string]core.EnvVar) (map[string]core.EnvVar, map[string]core.EnvVar) {
delete(a, topology.ArangoDBZone)
delete(b, topology.ArangoDBZone)
filter := func(a, b map[string]core.EnvVar) (map[string]core.EnvVar, map[string]core.EnvVar) {
for _, excludedEnv := range getExcludedEnv() {
delete(a, excludedEnv)
delete(b, excludedEnv)
}
return a, b
}) {
}
if areEnvsEqual(ac.Env, bc.Env, filter) {
// Envs are the same after filtering, but it were different before filtering, so it can be replaced.
bc.Env = ac.Env
mode = mode.And(SilentRotation)
}
}
if !equality.Semantic.DeepEqual(ac.EnvFrom, bc.EnvFrom) {
// Check EnvFromSource differences.
filter := func(a, b map[string]core.EnvFromSource) (map[string]core.EnvFromSource, map[string]core.EnvFromSource) {
delete(a, features.ConfigMapName())
delete(b, features.ConfigMapName())
return a, b
}
if areEnvsFromEqual(ac.EnvFrom, bc.EnvFrom, filter) {
// Envs are the same after filtering, but it were different before filtering, so it can be replaced.
bc.EnvFrom = ac.EnvFrom
mode = mode.And(SilentRotation)
}
}
if !equality.Semantic.DeepEqual(ac.Ports, bc.Ports) {
bc.Ports = ac.Ports
mode = mode.And(SilentRotation)
@ -230,3 +251,37 @@ func areProbesEqual(a, b *core.Probe) bool {
}
return equality.Semantic.DeepEqual(a, b)
}
// areEnvsFromEqual returns true when environment variables from source are the same after filtering.
func areEnvsFromEqual(a, b []core.EnvFromSource, rules ...func(a, b map[string]core.EnvFromSource) (map[string]core.EnvFromSource, map[string]core.EnvFromSource)) bool {
am := createEnvsFromMap(a)
bm := createEnvsFromMap(b)
for _, r := range rules {
am, bm = r(am, bm)
}
return equality.Semantic.DeepEqual(am, bm)
}
// createEnvsFromMap returns map from list.
func createEnvsFromMap(e []core.EnvFromSource) map[string]core.EnvFromSource {
m := map[string]core.EnvFromSource{}
for _, q := range e {
if q.ConfigMapRef != nil {
m[q.ConfigMapRef.Name] = q
} else if q.SecretRef != nil {
m[q.SecretRef.Name] = q
}
}
return m
}
// getExcludedEnv returns environment variables which should not be compared when pod's rotation is considered.
func getExcludedEnv() []string {
return []string{topology.ArangoDBZone, resources.ArangoDBOverrideServerGroupEnv,
resources.ArangoDBOverrideDeploymentModeEnv, resources.ArangoDBOverrideVersionEnv,
resources.ArangoDBOverrideEnterpriseEnv}
}

View file

@ -20,7 +20,10 @@
package util
import "os"
import (
"os"
"strings"
)
// EnvironmentVariable is a wrapper to get environment variables
type EnvironmentVariable string
@ -55,3 +58,9 @@ func (e EnvironmentVariable) GetOrDefault(d string) string {
return d
}
// NormalizeEnv normalizes environment variables.
func NormalizeEnv(env string) string {
r := strings.NewReplacer(".", "_", "-", "_")
return strings.ToUpper(r.Replace(env))
}

View file

@ -73,7 +73,7 @@ type ContainerCreator interface {
GetLifecycle() (*core.Lifecycle, error)
GetImagePullPolicy() core.PullPolicy
GetImage() string
GetEnvs() []core.EnvVar
GetEnvs() ([]core.EnvVar, []core.EnvFromSource)
GetSecurityContext() *core.SecurityContext
GetPorts() []core.ContainerPort
GetVolumeMounts() []core.VolumeMount

View file

@ -496,12 +496,14 @@ func NewContainer(containerCreator interfaces.ContainerCreator) (core.Container,
return core.Container{}, err
}
env, envFrom := containerCreator.GetEnvs()
return core.Container{
Name: containerCreator.GetName(),
Image: containerCreator.GetImage(),
Command: append([]string{containerCreator.GetExecutor()}, args...),
Ports: containerCreator.GetPorts(),
Env: containerCreator.GetEnvs(),
Env: env,
EnvFrom: envFrom,
Resources: containerCreator.GetResourceRequirements(),
LivenessProbe: liveness,
ReadinessProbe: readiness,