diff --git a/examples/cluster-with-sync.yaml b/examples/cluster-with-sync.yaml new file mode 100644 index 000000000..41aee5993 --- /dev/null +++ b/examples/cluster-with-sync.yaml @@ -0,0 +1,14 @@ +apiVersion: "database.arangodb.com/v1alpha" +kind: "ArangoDeployment" +metadata: + name: "cluster-with-sync" +spec: + mode: Cluster + image: ewoutp/arangodb:3.3.8 + tls: + altNames: ["kube-01", "kube-02", "kube-03"] + sync: + enabled: true + auth: + clientCASecretName: client-auth-ca + diff --git a/pkg/apis/deployment/v1alpha/deployment_spec.go b/pkg/apis/deployment/v1alpha/deployment_spec.go index d25fb33de..fea5ef888 100644 --- a/pkg/apis/deployment/v1alpha/deployment_spec.go +++ b/pkg/apis/deployment/v1alpha/deployment_spec.go @@ -144,7 +144,7 @@ func (s *DeploymentSpec) SetDefaults(deploymentName string) { s.RocksDB.SetDefaults() s.Authentication.SetDefaults(deploymentName + "-jwt") s.TLS.SetDefaults(deploymentName + "-ca") - s.Sync.SetDefaults(s.GetImage(), s.GetImagePullPolicy(), deploymentName+"-sync-jwt", deploymentName+"-sync-ca") + s.Sync.SetDefaults(s.GetImage(), s.GetImagePullPolicy(), deploymentName+"-sync-jwt", deploymentName+"-sync-client-auth-ca", deploymentName+"-sync-ca") s.Single.SetDefaults(ServerGroupSingle, s.GetMode().HasSingleServers(), s.GetMode()) s.Agents.SetDefaults(ServerGroupAgents, s.GetMode().HasAgents(), s.GetMode()) s.DBServers.SetDefaults(ServerGroupDBServers, s.GetMode().HasDBServers(), s.GetMode()) diff --git a/pkg/apis/deployment/v1alpha/image_info.go b/pkg/apis/deployment/v1alpha/image_info.go index 0b26f25c7..34c7f8a88 100644 --- a/pkg/apis/deployment/v1alpha/image_info.go +++ b/pkg/apis/deployment/v1alpha/image_info.go @@ -29,6 +29,7 @@ type ImageInfo struct { Image string `json:"image"` // Human provided name of the image ImageID string `json:"image-id,omitempty"` // Unique ID (with SHA256) of the image ArangoDBVersion driver.Version `json:"arangodb-version,omitempty"` // ArangoDB version within the image + Enterprise bool `json:"enterprise,omitempty"` // If set, this is an enterprise image } // ImageInfoList is a list of image infos diff --git a/pkg/apis/deployment/v1alpha/sync_authentication_spec.go b/pkg/apis/deployment/v1alpha/sync_authentication_spec.go new file mode 100644 index 000000000..e8c9e3242 --- /dev/null +++ b/pkg/apis/deployment/v1alpha/sync_authentication_spec.go @@ -0,0 +1,87 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package v1alpha + +import ( + "github.com/arangodb/kube-arangodb/pkg/util" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" +) + +// SyncAuthenticationSpec holds dc2dc sync authentication specific configuration settings +type SyncAuthenticationSpec struct { + JWTSecretName *string `json:"jwtSecretName,omitempty"` // JWT secret for sync masters + ClientCASecretName *string `json:"clientCASecretName,omitempty"` // Secret containing client authentication CA +} + +// GetJWTSecretName returns the value of jwtSecretName. +func (s SyncAuthenticationSpec) GetJWTSecretName() string { + return util.StringOrDefault(s.JWTSecretName) +} + +// GetClientCASecretName returns the value of clientCASecretName. +func (s SyncAuthenticationSpec) GetClientCASecretName() string { + return util.StringOrDefault(s.ClientCASecretName) +} + +// Validate the given spec +func (s SyncAuthenticationSpec) Validate() error { + if err := k8sutil.ValidateResourceName(s.GetJWTSecretName()); err != nil { + return maskAny(err) + } + if err := k8sutil.ValidateResourceName(s.GetClientCASecretName()); err != nil { + return maskAny(err) + } + return nil +} + +// SetDefaults fills in missing defaults +func (s *SyncAuthenticationSpec) SetDefaults(defaultJWTSecretName, defaultClientCASecretName string) { + if s.GetJWTSecretName() == "" { + // Note that we don't check for nil here, since even a specified, but empty + // string should result in the default value. + s.JWTSecretName = util.NewString(defaultJWTSecretName) + } + if s.GetClientCASecretName() == "" { + // Note that we don't check for nil here, since even a specified, but empty + // string should result in the default value. + s.ClientCASecretName = util.NewString(defaultClientCASecretName) + } +} + +// SetDefaultsFrom fills unspecified fields with a value from given source spec. +func (s *SyncAuthenticationSpec) SetDefaultsFrom(source SyncAuthenticationSpec) { + if s.JWTSecretName == nil { + s.JWTSecretName = util.NewStringOrNil(source.JWTSecretName) + } + if s.ClientCASecretName == nil { + s.ClientCASecretName = util.NewStringOrNil(source.ClientCASecretName) + } +} + +// ResetImmutableFields replaces all immutable fields in the given target with values from the source spec. +// It returns a list of fields that have been reset. +// Field names are relative to given field prefix. +func (s SyncAuthenticationSpec) ResetImmutableFields(fieldPrefix string, target *SyncAuthenticationSpec) []string { + var resetFields []string + return resetFields +} diff --git a/pkg/apis/deployment/v1alpha/sync_spec.go b/pkg/apis/deployment/v1alpha/sync_spec.go index 56aad0f64..c7c771ce7 100644 --- a/pkg/apis/deployment/v1alpha/sync_spec.go +++ b/pkg/apis/deployment/v1alpha/sync_spec.go @@ -35,9 +35,9 @@ type SyncSpec struct { Image *string `json:"image,omitempty"` ImagePullPolicy *v1.PullPolicy `json:"imagePullPolicy,omitempty"` - Authentication AuthenticationSpec `json:"auth"` - TLS TLSSpec `json:"tls"` - Monitoring MonitoringSpec `json:"monitoring"` + Authentication SyncAuthenticationSpec `json:"auth"` + TLS TLSSpec `json:"tls"` + Monitoring MonitoringSpec `json:"monitoring"` } // IsEnabled returns the value of enabled. @@ -63,10 +63,10 @@ func (s SyncSpec) Validate(mode DeploymentMode) error { if s.GetImage() == "" { return maskAny(errors.Wrapf(ValidationError, "image must be set")) } - if err := s.Authentication.Validate(s.IsEnabled()); err != nil { - return maskAny(err) - } if s.IsEnabled() { + if err := s.Authentication.Validate(); err != nil { + return maskAny(err) + } if err := s.TLS.Validate(); err != nil { return maskAny(err) } @@ -78,15 +78,15 @@ func (s SyncSpec) Validate(mode DeploymentMode) error { } // SetDefaults fills in missing defaults -func (s *SyncSpec) SetDefaults(defaultImage string, defaulPullPolicy v1.PullPolicy, defaultJWTSecretName, defaultCASecretName string) { +func (s *SyncSpec) SetDefaults(defaultImage string, defaulPullPolicy v1.PullPolicy, defaultJWTSecretName, defaultClientAuthCASecretName, defaultTLSCASecretName string) { if s.GetImage() == "" { s.Image = util.NewString(defaultImage) } if s.GetImagePullPolicy() == "" { s.ImagePullPolicy = util.NewPullPolicy(defaulPullPolicy) } - s.Authentication.SetDefaults(defaultJWTSecretName) - s.TLS.SetDefaults(defaultCASecretName) + s.Authentication.SetDefaults(defaultJWTSecretName, defaultClientAuthCASecretName) + s.TLS.SetDefaults(defaultTLSCASecretName) s.Monitoring.SetDefaults() } diff --git a/pkg/apis/deployment/v1alpha/zz_generated.deepcopy.go b/pkg/apis/deployment/v1alpha/zz_generated.deepcopy.go index 969c9d303..eddb062ba 100644 --- a/pkg/apis/deployment/v1alpha/zz_generated.deepcopy.go +++ b/pkg/apis/deployment/v1alpha/zz_generated.deepcopy.go @@ -603,6 +603,40 @@ func (in *ServerGroupSpec) DeepCopy() *ServerGroupSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SyncAuthenticationSpec) DeepCopyInto(out *SyncAuthenticationSpec) { + *out = *in + if in.JWTSecretName != nil { + in, out := &in.JWTSecretName, &out.JWTSecretName + if *in == nil { + *out = nil + } else { + *out = new(string) + **out = **in + } + } + if in.ClientCASecretName != nil { + in, out := &in.ClientCASecretName, &out.ClientCASecretName + if *in == nil { + *out = nil + } else { + *out = new(string) + **out = **in + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SyncAuthenticationSpec. +func (in *SyncAuthenticationSpec) DeepCopy() *SyncAuthenticationSpec { + if in == nil { + return nil + } + out := new(SyncAuthenticationSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SyncSpec) DeepCopyInto(out *SyncSpec) { *out = *in diff --git a/pkg/deployment/deployment_inspector.go b/pkg/deployment/deployment_inspector.go index c4c9f3063..b7fe8acb2 100644 --- a/pkg/deployment/deployment_inspector.go +++ b/pkg/deployment/deployment_inspector.go @@ -108,6 +108,10 @@ func (d *Deployment) inspectDeployment(lastInterval time.Duration) time.Duration } // Ensure all resources are created + if err := d.resources.EnsureSecrets(); err != nil { + hasError = true + d.CreateEvent(k8sutil.NewErrorEvent("Secret creation failed", err, d.apiObject)) + } if err := d.resources.EnsureServices(); err != nil { hasError = true d.CreateEvent(k8sutil.NewErrorEvent("Service creation failed", err, d.apiObject)) diff --git a/pkg/deployment/images.go b/pkg/deployment/images.go index 504a4ca7b..daaa56836 100644 --- a/pkg/deployment/images.go +++ b/pkg/deployment/images.go @@ -137,6 +137,7 @@ func (ib *imagesBuilder) fetchArangoDBImageIDAndVersion(ctx context.Context, ima return true, nil } version := v.Version + enterprise := strings.ToLower(v.License) == "enterprise" // We have all the info we need now, kill the pod and store the image info. if err := ib.KubeCli.CoreV1().Pods(ns).Delete(podName, nil); err != nil && !k8sutil.IsNotFound(err) { @@ -148,6 +149,7 @@ func (ib *imagesBuilder) fetchArangoDBImageIDAndVersion(ctx context.Context, ima Image: image, ImageID: imageID, ArangoDBVersion: version, + Enterprise: enterprise, } ib.Status.Images.AddOrUpdate(info) if err := ib.UpdateCRStatus(ib.Status); err != nil { diff --git a/pkg/deployment/reconcile/plan_builder.go b/pkg/deployment/reconcile/plan_builder.go index a3c51ff34..54dfb3028 100644 --- a/pkg/deployment/reconcile/plan_builder.go +++ b/pkg/deployment/reconcile/plan_builder.go @@ -176,6 +176,10 @@ func createPlan(log zerolog.Logger, apiObject metav1.Object, // Only make changes when phase is created continue } + if group == api.ServerGroupSyncWorkers { + // SyncWorkers have no externally created TLS keyfile + continue + } // Load keyfile keyfile, err := getTLSKeyfile(group, m) if err != nil { diff --git a/pkg/deployment/resources/pod_creator.go b/pkg/deployment/resources/pod_creator.go index 4da74078f..03035b729 100644 --- a/pkg/deployment/resources/pod_creator.go +++ b/pkg/deployment/resources/pod_creator.go @@ -217,9 +217,72 @@ func createArangodArgs(apiObject metav1.Object, deplSpec api.DeploymentSpec, gro } // createArangoSyncArgs creates command line arguments for an arangosync server in the given group. -func createArangoSyncArgs(spec api.DeploymentSpec, group api.ServerGroup, groupSpec api.ServerGroupSpec, agents api.MemberStatusList, id string) []string { - // TODO - return nil +func createArangoSyncArgs(apiObject metav1.Object, spec api.DeploymentSpec, group api.ServerGroup, groupSpec api.ServerGroupSpec, agents api.MemberStatusList, id string) []string { + options := make([]optionPair, 0, 64) + var runCmd string + var port int + + /*if config.DebugCluster { + options = append(options, + optionPair{"--log.level", "debug"}) + }*/ + if spec.Sync.Monitoring.GetTokenSecretName() != "" { + options = append(options, + optionPair{"--monitoring.token", "$(" + constants.EnvArangoSyncMonitoringToken + ")"}, + ) + } + masterSecretPath := filepath.Join(k8sutil.MasterJWTSecretVolumeMountDir, constants.SecretKeyJWT) + options = append(options, + optionPair{"--master.jwt-secret", masterSecretPath}, + ) + switch group { + case api.ServerGroupSyncMasters: + runCmd = "master" + port = k8sutil.ArangoSyncMasterPort + keyPath := filepath.Join(k8sutil.TLSKeyfileVolumeMountDir, constants.SecretTLSKeyfile) + clientCAPath := filepath.Join(k8sutil.ClientAuthCAVolumeMountDir, constants.SecretCACertificate) + options = append(options, + optionPair{"--server.keyfile", keyPath}, + optionPair{"--server.client-cafile", clientCAPath}, + optionPair{"--mq.type", "direct"}, + ) + if spec.IsAuthenticated() { + clusterSecretPath := filepath.Join(k8sutil.ClusterJWTSecretVolumeMountDir, constants.SecretKeyJWT) + options = append(options, + optionPair{"--cluster.jwt-secret", clusterSecretPath}, + ) + } + dbServiceName := k8sutil.CreateDatabaseClientServiceName(apiObject.GetName()) + scheme := "http" + if spec.IsSecure() { + scheme = "https" + } + options = append(options, + optionPair{"--cluster.endpoint", fmt.Sprintf("%s://%s:%d", scheme, dbServiceName, k8sutil.ArangoPort)}) + case api.ServerGroupSyncWorkers: + runCmd = "worker" + port = k8sutil.ArangoSyncWorkerPort + syncServiceName := k8sutil.CreateSyncMasterClientServiceName(apiObject.GetName()) + options = append(options, + optionPair{"--master.endpoint", fmt.Sprintf("https://%s:%d", syncServiceName, k8sutil.ArangoSyncMasterPort)}) + } + serverEndpoint := "https://" + net.JoinHostPort(k8sutil.CreatePodDNSName(apiObject, group.AsRole(), id), strconv.Itoa(port)) + options = append(options, + optionPair{"--server.endpoint", serverEndpoint}, + optionPair{"--server.port", strconv.Itoa(port)}, + ) + + args := make([]string, 0, 2+len(options)+len(groupSpec.Args)) + sort.Slice(options, func(i, j int) bool { + return options[i].CompareTo(options[j]) < 0 + }) + args = append(args, "run", runCmd) + for _, o := range options { + args = append(args, o.Key+"="+o.Value) + } + args = append(args, groupSpec.Args...) + + return args } // createLivenessProbe creates configuration for a liveness probe of a server in the given group. @@ -246,6 +309,10 @@ func (r *Resources) createLivenessProbe(spec api.DeploymentSpec, group api.Serve return nil, nil case api.ServerGroupSyncMasters, api.ServerGroupSyncWorkers: authorization := "" + port := k8sutil.ArangoSyncMasterPort + if group == api.ServerGroupSyncWorkers { + port = k8sutil.ArangoSyncWorkerPort + } if spec.Sync.Monitoring.GetTokenSecretName() != "" { // Use monitoring token token, err := r.getSyncMonitoringToken(spec) @@ -274,6 +341,7 @@ func (r *Resources) createLivenessProbe(spec api.DeploymentSpec, group api.Serve LocalPath: "/_api/version", Secure: spec.IsSecure(), Authorization: authorization, + Port: port, }, nil default: return nil, nil @@ -377,12 +445,53 @@ func (r *Resources) createPodForMember(spec api.DeploymentSpec, group api.Server // Find image ID info, found := status.Images.GetByImage(spec.Sync.GetImage()) if !found { - log.Debug().Str("image", spec.Sync.GetImage()).Msg("Image ID is not known yet for image") + log.Debug().Str("image", spec.Sync.GetImage()).Msg("Image ID is not known yet for sync image") return nil } + if !info.Enterprise { + log.Debug().Str("image", spec.Sync.GetImage()).Msg("Image is not an enterprise image") + return maskAny(fmt.Errorf("Image '%s' does not contain an Enterprise version of ArangoDB", spec.Sync.GetImage())) + } + var tlsKeyfileSecretName, clientAuthCASecretName, masterJWTSecretName, clusterJWTSecretName string + // Check master JWT secret + masterJWTSecretName = spec.Sync.Authentication.GetJWTSecretName() + if err := k8sutil.ValidateJWTSecret(kubecli.CoreV1(), masterJWTSecretName, ns); err != nil { + return maskAny(errors.Wrapf(err, "Master JWT secret validation failed")) + } + if group == api.ServerGroupSyncMasters { + // Create TLS secret + tlsKeyfileSecretName = k8sutil.CreateTLSKeyfileSecretName(apiObject.GetName(), role, m.ID) + serverNames := []string{ + k8sutil.CreateSyncMasterClientServiceName(apiObject.GetName()), + k8sutil.CreatePodDNSName(apiObject, role, m.ID), + } + owner := apiObject.AsOwner() + if err := createServerCertificate(log, kubecli.CoreV1(), serverNames, spec.TLS, tlsKeyfileSecretName, ns, &owner); err != nil && !k8sutil.IsAlreadyExists(err) { + return maskAny(errors.Wrapf(err, "Failed to create TLS keyfile secret")) + } + // Check cluster JWT secret + if spec.IsAuthenticated() { + clusterJWTSecretName = spec.Authentication.GetJWTSecretName() + if err := k8sutil.ValidateJWTSecret(kubecli.CoreV1(), clusterJWTSecretName, ns); err != nil { + return maskAny(errors.Wrapf(err, "Cluster JWT secret validation failed")) + } + } + // Check client-auth CA certificate secret + clientAuthCASecretName = spec.Sync.Authentication.GetClientCASecretName() + if err := k8sutil.ValidateCACertificateSecret(kubecli.CoreV1(), clientAuthCASecretName, ns); err != nil { + return maskAny(errors.Wrapf(err, "Client authentication CA certificate secret validation failed")) + } + } + // Prepare arguments - args := createArangoSyncArgs(spec, group, groupSpec, status.Members.Agents, m.ID) + args := createArangoSyncArgs(apiObject, spec, group, groupSpec, status.Members.Agents, m.ID) env := make(map[string]k8sutil.EnvValue) + if spec.Sync.Monitoring.GetTokenSecretName() != "" { + env[constants.EnvArangoSyncMonitoringToken] = k8sutil.EnvValue{ + SecretName: spec.Sync.Monitoring.GetTokenSecretName(), + SecretKey: constants.SecretKeyJWT, + } + } livenessProbe, err := r.createLivenessProbe(spec, group) if err != nil { return maskAny(err) @@ -391,7 +500,8 @@ func (r *Resources) createPodForMember(spec api.DeploymentSpec, group api.Server if group == api.ServerGroupSyncWorkers { affinityWithRole = api.ServerGroupDBServers.AsRole() } - if err := k8sutil.CreateArangoSyncPod(kubecli, spec.IsDevelopment(), apiObject, role, m.ID, m.PodName, info.ImageID, spec.Sync.GetImagePullPolicy(), args, env, livenessProbe, affinityWithRole); err != nil { + if err := k8sutil.CreateArangoSyncPod(kubecli, spec.IsDevelopment(), apiObject, role, m.ID, m.PodName, info.ImageID, spec.Sync.GetImagePullPolicy(), args, env, + livenessProbe, tlsKeyfileSecretName, clientAuthCASecretName, masterJWTSecretName, clusterJWTSecretName, affinityWithRole); err != nil { return maskAny(err) } log.Debug().Str("pod-name", m.PodName).Msg("Created pod") diff --git a/pkg/deployment/resources/secrets.go b/pkg/deployment/resources/secrets.go index 44aa75c4a..b59e0aeb3 100644 --- a/pkg/deployment/resources/secrets.go +++ b/pkg/deployment/resources/secrets.go @@ -47,6 +47,9 @@ func (r *Resources) EnsureSecrets() error { } } if spec.Sync.IsEnabled() { + if err := r.ensureJWTSecret(spec.Sync.Authentication.GetJWTSecretName()); err != nil { + return maskAny(err) + } if err := r.ensureCACertificateSecret(spec.Sync.TLS); err != nil { return maskAny(err) } diff --git a/pkg/util/constants/constants.go b/pkg/util/constants/constants.go index b11f14dad..99fba7220 100644 --- a/pkg/util/constants/constants.go +++ b/pkg/util/constants/constants.go @@ -28,11 +28,12 @@ const ( EnvOperatorPodNamespace = "MY_POD_NAMESPACE" EnvOperatorPodIP = "MY_POD_IP" - EnvArangodJWTSecret = "ARANGOD_JWT_SECRET" // Contains JWT secret for the ArangoDB cluster - EnvArangoSyncJWTSecret = "ARANGOSYNC_JWT_SECRET" // Contains JWT secret for the ArangoSync masters + EnvArangodJWTSecret = "ARANGOD_JWT_SECRET" // Contains JWT secret for the ArangoDB cluster + EnvArangoSyncMonitoringToken = "ARANGOSYNC_MONITORING_TOKEN" // Constains monitoring token for ArangoSync servers SecretEncryptionKey = "key" // Key in a Secret.Data used to store an 32-byte encryption key SecretKeyJWT = "token" // Key inside a Secret used to hold a JW token + SecretKeyMonitoring = "token" // Key inside a Secret used to hold a monitoring token SecretCACertificate = "ca.crt" // Key in Secret.data used to store a PEM encoded CA certificate (public key) SecretCAKey = "ca.key" // Key in Secret.data used to store a PEM encoded CA private key diff --git a/pkg/util/k8sutil/constants.go b/pkg/util/k8sutil/constants.go index 86b8de7dc..d4524d201 100644 --- a/pkg/util/k8sutil/constants.go +++ b/pkg/util/k8sutil/constants.go @@ -24,7 +24,9 @@ package k8sutil const ( // Arango constants - ArangoPort = 8529 + ArangoPort = 8529 + ArangoSyncMasterPort = 8629 + ArangoSyncWorkerPort = 8729 // K8s constants ClusterIPNone = "None" diff --git a/pkg/util/k8sutil/pods.go b/pkg/util/k8sutil/pods.go index 8700e0f5d..417edba52 100644 --- a/pkg/util/k8sutil/pods.go +++ b/pkg/util/k8sutil/pods.go @@ -37,10 +37,16 @@ const ( alpineImage = "alpine" arangodVolumeName = "arangod-data" tlsKeyfileVolumeName = "tls-keyfile" + clientAuthCAVolumeName = "client-auth-ca" + clusterJWTSecretVolumeName = "cluster-jwt" + masterJWTSecretVolumeName = "master-jwt" rocksdbEncryptionVolumeName = "rocksdb-encryption" ArangodVolumeMountDir = "/data" RocksDBEncryptionVolumeMountDir = "/secrets/rocksdb/encryption" TLSKeyfileVolumeMountDir = "/secrets/tls" + ClientAuthCAVolumeMountDir = "/secrets/client-auth/ca" + ClusterJWTSecretVolumeMountDir = "/secrets/cluster/jwt" + MasterJWTSecretVolumeMountDir = "/secrets/master/jwt" ) // EnvValue is a helper structure for environment variable sources. @@ -159,6 +165,36 @@ func tlsKeyfileVolumeMounts() []v1.VolumeMount { } } +// clientAuthCACertificateVolumeMounts creates a volume mount structure for a client-auth CA certificate (ca.crt). +func clientAuthCACertificateVolumeMounts() []v1.VolumeMount { + return []v1.VolumeMount{ + { + Name: clientAuthCAVolumeName, + MountPath: ClientAuthCAVolumeMountDir, + }, + } +} + +// masterJWTVolumeMounts creates a volume mount structure for a master JWT secret (token). +func masterJWTVolumeMounts() []v1.VolumeMount { + return []v1.VolumeMount{ + { + Name: masterJWTSecretVolumeName, + MountPath: MasterJWTSecretVolumeMountDir, + }, + } +} + +// clusterJWTVolumeMounts creates a volume mount structure for a cluster JWT secret (token). +func clusterJWTVolumeMounts() []v1.VolumeMount { + return []v1.VolumeMount{ + { + Name: clusterJWTSecretVolumeName, + MountPath: ClusterJWTSecretVolumeMountDir, + }, + } +} + // rocksdbEncryptionVolumeMounts creates a volume mount structure for a RocksDB encryption key. func rocksdbEncryptionVolumeMounts() []v1.VolumeMount { return []v1.VolumeMount{ @@ -359,14 +395,78 @@ func CreateArangodPod(kubecli kubernetes.Interface, developmentMode bool, deploy // If the pod already exists, nil is returned. // If another error occurs, that error is returned. func CreateArangoSyncPod(kubecli kubernetes.Interface, developmentMode bool, deployment APIObject, role, id, podName, image string, imagePullPolicy v1.PullPolicy, - args []string, env map[string]EnvValue, livenessProbe *HTTPProbeConfig, affinityWithRole string) error { + args []string, env map[string]EnvValue, livenessProbe *HTTPProbeConfig, tlsKeyfileSecretName, clientAuthCASecretName, masterJWTSecretName, clusterJWTSecretName, affinityWithRole string) error { // Prepare basic pod p := newPod(deployment.GetName(), deployment.GetNamespace(), role, id, podName) // Add arangosync container c := arangosyncContainer("arangosync", image, imagePullPolicy, args, env, livenessProbe) + if tlsKeyfileSecretName != "" { + c.VolumeMounts = append(c.VolumeMounts, tlsKeyfileVolumeMounts()...) + } + if clientAuthCASecretName != "" { + c.VolumeMounts = append(c.VolumeMounts, clientAuthCACertificateVolumeMounts()...) + } + if masterJWTSecretName != "" { + c.VolumeMounts = append(c.VolumeMounts, masterJWTVolumeMounts()...) + } + if clusterJWTSecretName != "" { + c.VolumeMounts = append(c.VolumeMounts, clusterJWTVolumeMounts()...) + } p.Spec.Containers = append(p.Spec.Containers, c) + // TLS keyfile secret mount (if any) + if tlsKeyfileSecretName != "" { + vol := v1.Volume{ + Name: tlsKeyfileVolumeName, + VolumeSource: v1.VolumeSource{ + Secret: &v1.SecretVolumeSource{ + SecretName: tlsKeyfileSecretName, + }, + }, + } + p.Spec.Volumes = append(p.Spec.Volumes, vol) + } + + // Client Authentication certificate secret mount (if any) + if clientAuthCASecretName != "" { + vol := v1.Volume{ + Name: clientAuthCAVolumeName, + VolumeSource: v1.VolumeSource{ + Secret: &v1.SecretVolumeSource{ + SecretName: clientAuthCASecretName, + }, + }, + } + p.Spec.Volumes = append(p.Spec.Volumes, vol) + } + + // Master JWT secret mount (if any) + if masterJWTSecretName != "" { + vol := v1.Volume{ + Name: masterJWTSecretVolumeName, + VolumeSource: v1.VolumeSource{ + Secret: &v1.SecretVolumeSource{ + SecretName: masterJWTSecretName, + }, + }, + } + p.Spec.Volumes = append(p.Spec.Volumes, vol) + } + + // Cluster JWT secret mount (if any) + if clusterJWTSecretName != "" { + vol := v1.Volume{ + Name: clusterJWTSecretVolumeName, + VolumeSource: v1.VolumeSource{ + Secret: &v1.SecretVolumeSource{ + SecretName: clusterJWTSecretName, + }, + }, + } + p.Spec.Volumes = append(p.Spec.Volumes, vol) + } + // Add (anti-)affinity p.Spec.Affinity = createAffinity(deployment.GetName(), role, !developmentMode, affinityWithRole) diff --git a/pkg/util/k8sutil/probes.go b/pkg/util/k8sutil/probes.go index ca8b67d6c..6ef664cce 100644 --- a/pkg/util/k8sutil/probes.go +++ b/pkg/util/k8sutil/probes.go @@ -35,6 +35,8 @@ type HTTPProbeConfig struct { Secure bool // Value for an Authorization header (can be empty) Authorization string + // Port to inspect (defaults to ArangoPort) + Port int } // Create creates a probe from given config @@ -50,11 +52,15 @@ func (config HTTPProbeConfig) Create() *v1.Probe { Value: config.Authorization, }) } + port := config.Port + if port == 0 { + port = ArangoPort + } return &v1.Probe{ Handler: v1.Handler{ HTTPGet: &v1.HTTPGetAction{ Path: config.LocalPath, - Port: intstr.FromInt(ArangoPort), + Port: intstr.FromInt(port), Scheme: scheme, HTTPHeaders: headers, }, diff --git a/pkg/util/k8sutil/secrets.go b/pkg/util/k8sutil/secrets.go index 7dca11481..4383c2598 100644 --- a/pkg/util/k8sutil/secrets.go +++ b/pkg/util/k8sutil/secrets.go @@ -71,6 +71,21 @@ func CreateEncryptionKeySecret(cli corev1.CoreV1Interface, secretName, namespace return nil } +// ValidateCACertificateSecret checks that a secret with given name in given namespace +// exists and it contains a 'ca.crt' data field. +func ValidateCACertificateSecret(cli corev1.CoreV1Interface, secretName, namespace string) error { + s, err := cli.Secrets(namespace).Get(secretName, metav1.GetOptions{}) + if err != nil { + return maskAny(err) + } + // Check `ca.crt` field + _, found := s.Data[constants.SecretCACertificate] + if !found { + return maskAny(fmt.Errorf("No '%s' found in secret '%s'", constants.SecretCACertificate, secretName)) + } + return nil +} + // GetCASecret loads a secret with given name in the given namespace // and extracts the `ca.crt` & `ca.key` field. // If the secret does not exists or one of the fields is missing, @@ -151,6 +166,21 @@ func CreateTLSKeyfileSecret(cli corev1.CoreV1Interface, secretName, namespace st return nil } +// ValidateJWTSecret checks that a secret with given name in given namespace +// exists and it contains a 'token' data field. +func ValidateJWTSecret(cli corev1.CoreV1Interface, secretName, namespace string) error { + s, err := cli.Secrets(namespace).Get(secretName, metav1.GetOptions{}) + if err != nil { + return maskAny(err) + } + // Check `token` field + _, found := s.Data[constants.SecretKeyJWT] + if !found { + return maskAny(fmt.Errorf("No '%s' found in secret '%s'", constants.SecretKeyJWT, secretName)) + } + return nil +} + // GetJWTSecret loads the JWT secret from a Secret with given name. func GetJWTSecret(cli corev1.CoreV1Interface, secretName, namespace string) (string, error) { s, err := cli.Secrets(namespace).Get(secretName, metav1.GetOptions{}) diff --git a/pkg/util/k8sutil/services.go b/pkg/util/k8sutil/services.go index f513496e5..b3f6f3fb7 100644 --- a/pkg/util/k8sutil/services.go +++ b/pkg/util/k8sutil/services.go @@ -148,7 +148,7 @@ func CreateSyncMasterClientService(kubecli kubernetes.Interface, deployment meta v1.ServicePort{ Name: "server", Protocol: v1.ProtocolTCP, - Port: ArangoPort, + Port: ArangoSyncMasterPort, }, } publishNotReadyAddresses := true