From 0c1eeb67bcd3bf4debab5b0a3690d9dcd0314272 Mon Sep 17 00:00:00 2001 From: Adam Janikowski <12255597+ajanikow@users.noreply.github.com> Date: Mon, 15 Jun 2020 12:39:05 +0200 Subject: [PATCH] [Feature] Improved TLS rotation (#577) --- CHANGELOG.md | 2 + pkg/apis/deployment/v1/deployment_mode.go | 10 + pkg/apis/deployment/v1/deployment_status.go | 4 +- pkg/apis/deployment/v1/hashes.go | 33 + ...encryption_key_hashes.go => key_hashes.go} | 6 +- pkg/apis/deployment/v1/plan.go | 14 + pkg/apis/deployment/v1/tls_sni_spec.go | 16 - pkg/apis/deployment/v1/tls_spec.go | 28 +- .../deployment/v1/zz_generated.deepcopy.go | 74 +- pkg/deployment/deployment_inspector.go | 2 +- pkg/deployment/deployment_run_test.go | 6 +- pkg/deployment/pod/builder.go | 1 + pkg/deployment/pod/encryption.go | 4 + pkg/deployment/pod/jwt.go | 63 +- pkg/deployment/pod/sni.go | 4 + pkg/deployment/pod/tls.go | 90 +++ pkg/deployment/pod/upgrade.go | 4 + pkg/deployment/pod/volumes.go | 89 +++ .../action_encryption_status_update.go | 19 +- .../action_renew_tls_ca_certificate.go | 82 --- .../reconcile/action_renew_tls_certificate.go | 71 -- .../reconcile/action_tls_ca_append.go | 126 ++++ .../reconcile/action_tls_ca_clean.go | 124 ++++ .../reconcile/action_tls_ca_renew.go | 66 ++ .../reconcile/action_tls_keyfile_clean.go | 67 ++ .../reconcile/action_tls_keyfile_refresh.go | 93 +++ .../reconcile/action_tls_status_update.go | 86 +++ pkg/deployment/reconcile/plan_builder.go | 34 +- .../reconcile/plan_builder_encryption.go | 65 +- .../reconcile/plan_builder_rotate_upgrade.go | 5 - pkg/deployment/reconcile/plan_builder_test.go | 2 +- pkg/deployment/reconcile/plan_builder_tls.go | 697 ++++++++++++------ .../reconcile/plan_builder_tls_sni.go | 6 +- pkg/deployment/reconcile/plan_executor.go | 11 +- pkg/deployment/reconcile/timeouts.go | 31 +- pkg/deployment/reconcile/utils.go | 46 ++ pkg/deployment/resources/pod_creator.go | 90 +-- .../resources/pod_creator_arangod.go | 99 +-- pkg/deployment/resources/secret_hashes.go | 12 - pkg/deployment/resources/secrets.go | 105 ++- pkg/util/checksum.go | 32 + pkg/util/k8sutil/annotations.go | 18 + pkg/util/k8sutil/pods.go | 1 + 43 files changed, 1789 insertions(+), 649 deletions(-) create mode 100644 pkg/apis/deployment/v1/hashes.go rename pkg/apis/deployment/v1/{encryption_key_hashes.go => key_hashes.go} (81%) create mode 100644 pkg/deployment/pod/tls.go create mode 100644 pkg/deployment/pod/volumes.go delete mode 100644 pkg/deployment/reconcile/action_renew_tls_ca_certificate.go delete mode 100644 pkg/deployment/reconcile/action_renew_tls_certificate.go create mode 100644 pkg/deployment/reconcile/action_tls_ca_append.go create mode 100644 pkg/deployment/reconcile/action_tls_ca_clean.go create mode 100644 pkg/deployment/reconcile/action_tls_ca_renew.go create mode 100644 pkg/deployment/reconcile/action_tls_keyfile_clean.go create mode 100644 pkg/deployment/reconcile/action_tls_keyfile_refresh.go create mode 100644 pkg/deployment/reconcile/action_tls_status_update.go create mode 100644 pkg/deployment/reconcile/utils.go create mode 100644 pkg/util/checksum.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 944d5ea8b..d3ecde7d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## [master](https://github.com/arangodb/kube-arangodb/tree/master) (N/A) - Add Encryption Key rotation feature for ArangoDB EE 3.7+ +- Improve TLS CA and Keyfile rotation for CE and EE +- Add runtime TLS rotation for ArangoDB EE 3.7+ ## [1.0.3](https://github.com/arangodb/kube-arangodb/tree/1.0.3) (2020-05-25) - Prevent deletion of not known PVC's diff --git a/pkg/apis/deployment/v1/deployment_mode.go b/pkg/apis/deployment/v1/deployment_mode.go index b9249c2f7..dae683642 100644 --- a/pkg/apis/deployment/v1/deployment_mode.go +++ b/pkg/apis/deployment/v1/deployment_mode.go @@ -58,6 +58,16 @@ func (m *DeploymentMode) Get() DeploymentMode { return *m } +// String return string from mode +func (m *DeploymentMode) String() string { + return string(m.Get()) +} + +// Nww return pointer to mode +func (m DeploymentMode) New() *DeploymentMode { + return &m +} + // HasSingleServers returns true when the given mode is "Single" or "ActiveFailover". func (m DeploymentMode) HasSingleServers() bool { return m == DeploymentModeSingle || m == DeploymentModeActiveFailover diff --git a/pkg/apis/deployment/v1/deployment_status.go b/pkg/apis/deployment/v1/deployment_status.go index 864fb09f8..f734b6f70 100644 --- a/pkg/apis/deployment/v1/deployment_status.go +++ b/pkg/apis/deployment/v1/deployment_status.go @@ -70,8 +70,8 @@ type DeploymentStatus struct { // detect changes in secret values. SecretHashes *SecretHashes `json:"secret-hashes,omitempty"` - // CurrentEncryptionKeys keep list of currently applied encryption keys as SHA256 hash - CurrentEncryptionKeyHashes DeploymentStatusEncryptionKeyHashes `json:"currentEncryptionKeyHashes,omitempty"` + // Hashes keep status of hashes in deployment + Hashes DeploymentStatusHashes `json:"hashes,omitempty"` // ForceStatusReload if set to true forces a reload of the status from the custom resource. ForceStatusReload *bool `json:"force-status-reload,omitempty"` diff --git a/pkg/apis/deployment/v1/hashes.go b/pkg/apis/deployment/v1/hashes.go new file mode 100644 index 000000000..852998813 --- /dev/null +++ b/pkg/apis/deployment/v1/hashes.go @@ -0,0 +1,33 @@ +// +// DISCLAIMER +// +// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Adam Janikowski +// + +package v1 + +type DeploymentStatusHashes struct { + Encryption DeploymentStatusHashList `json:"encryption,omitempty"` + TLS DeploymentStatusHashesTLS `json:"tls,omitempty"` +} + +type DeploymentStatusHashesTLS struct { + CA *string `json:"ca,omitempty"` + Truststore DeploymentStatusHashList `json:"truststore,omitempty"` +} diff --git a/pkg/apis/deployment/v1/encryption_key_hashes.go b/pkg/apis/deployment/v1/key_hashes.go similarity index 81% rename from pkg/apis/deployment/v1/encryption_key_hashes.go rename to pkg/apis/deployment/v1/key_hashes.go index aee7ca919..40fd63c40 100644 --- a/pkg/apis/deployment/v1/encryption_key_hashes.go +++ b/pkg/apis/deployment/v1/key_hashes.go @@ -24,9 +24,9 @@ package v1 import "fmt" -type DeploymentStatusEncryptionKeyHashes []string +type DeploymentStatusHashList []string -func (d DeploymentStatusEncryptionKeyHashes) Contains(hash string) bool { +func (d DeploymentStatusHashList) Contains(hash string) bool { if len(d) == 0 { return false } @@ -40,6 +40,6 @@ func (d DeploymentStatusEncryptionKeyHashes) Contains(hash string) bool { return false } -func (d DeploymentStatusEncryptionKeyHashes) ContainsSHA256(hash string) bool { +func (d DeploymentStatusHashList) ContainsSHA256(hash string) bool { return d.Contains(fmt.Sprintf("sha256:%s", hash)) } diff --git a/pkg/apis/deployment/v1/plan.go b/pkg/apis/deployment/v1/plan.go index 438fafbb6..3b740f5d8 100644 --- a/pkg/apis/deployment/v1/plan.go +++ b/pkg/apis/deployment/v1/plan.go @@ -32,6 +32,10 @@ import ( // ActionType is a strongly typed name for a plan action item type ActionType string +func (a ActionType) String() string { + return string(a) +} + const ( // ActionTypeIdle causes a plan to be recalculated. ActionTypeIdle ActionType = "Idle" @@ -61,6 +65,16 @@ const ( ActionTypeRenewTLSCertificate ActionType = "RenewTLSCertificate" // ActionTypeRenewTLSCACertificate causes the TLS CA certificate of the entire deployment to be renewed. ActionTypeRenewTLSCACertificate ActionType = "RenewTLSCACertificate" + // ActionTypeAppendTLSCACertificate add TLS CA certificate to local truststore. + ActionTypeAppendTLSCACertificate ActionType = "AppendTLSCACertificate" + // ActionTypeCleanTLSCACertificate clean TLS CA certificate from local truststore. + ActionTypeCleanTLSCACertificate ActionType = "CleanTLSCACertificate" + // ActionTypeCleanTLSKeyfileCertificate clean server keyfile + ActionTypeCleanTLSKeyfileCertificate ActionType = "CleanTLSKeyfileCertificate" + // ActionTypeRefreshTLSKeyfileCertificate refresh server keyfile using API + ActionTypeRefreshTLSKeyfileCertificate ActionType = "RefreshTLSKeyfileCertificate" + // ActionTypeTLSKeyStatusUpdate update status with current data from deployment + ActionTypeTLSKeyStatusUpdate ActionType = "TLSKeyStatusUpdate" // ActionTypeUpdateTLSSNI update SNI inplace. ActionTypeUpdateTLSSNI ActionType = "UpdateTLSSNI" // ActionTypeSetCurrentImage causes status.CurrentImage to be updated to the image given in the action. diff --git a/pkg/apis/deployment/v1/tls_sni_spec.go b/pkg/apis/deployment/v1/tls_sni_spec.go index 08db99a67..018187306 100644 --- a/pkg/apis/deployment/v1/tls_sni_spec.go +++ b/pkg/apis/deployment/v1/tls_sni_spec.go @@ -27,25 +27,9 @@ import ( "github.com/pkg/errors" ) -type TLSSNIRotateMode string - -func (t *TLSSNIRotateMode) Get() TLSSNIRotateMode { - if t == nil { - return TLSSNIRotateModeInPlace - } - - return *t -} - -const ( - TLSSNIRotateModeInPlace TLSSNIRotateMode = "inplace" - TLSSNIRotateModeRecreate TLSSNIRotateMode = "recreate" -) - // TLSSNISpec holds TLS SNI additional certificates type TLSSNISpec struct { Mapping map[string][]string `json:"mapping,omitempty"` - Mode *TLSSNIRotateMode `json:"mode,omitempty"` } func (s TLSSNISpec) Validate() error { diff --git a/pkg/apis/deployment/v1/tls_spec.go b/pkg/apis/deployment/v1/tls_spec.go index de4381db4..23d56656b 100644 --- a/pkg/apis/deployment/v1/tls_spec.go +++ b/pkg/apis/deployment/v1/tls_spec.go @@ -31,16 +31,36 @@ import ( "github.com/arangodb/kube-arangodb/pkg/util/validation" ) +type TLSRotateMode string + +func (t *TLSRotateMode) Get() TLSRotateMode { + if t == nil { + return TLSRotateModeInPlace + } + + return *t +} + +func (t TLSRotateMode) New() *TLSRotateMode { + return &t +} + +const ( + TLSRotateModeInPlace TLSRotateMode = "inplace" + TLSRotateModeRecreate TLSRotateMode = "recreate" +) + const ( defaultTLSTTL = Duration("2610h") // About 3 month ) // TLSSpec holds TLS specific configuration settings type TLSSpec struct { - CASecretName *string `json:"caSecretName,omitempty"` - AltNames []string `json:"altNames,omitempty"` - TTL *Duration `json:"ttl,omitempty"` - SNI *TLSSNISpec `json:"sni,omitempty"` + CASecretName *string `json:"caSecretName,omitempty"` + AltNames []string `json:"altNames,omitempty"` + TTL *Duration `json:"ttl,omitempty"` + SNI *TLSSNISpec `json:"sni,omitempty"` + Mode *TLSRotateMode `json:"mode,omitempty"` } const ( diff --git a/pkg/apis/deployment/v1/zz_generated.deepcopy.go b/pkg/apis/deployment/v1/zz_generated.deepcopy.go index 9032360b9..6076d59b1 100644 --- a/pkg/apis/deployment/v1/zz_generated.deepcopy.go +++ b/pkg/apis/deployment/v1/zz_generated.deepcopy.go @@ -399,11 +399,7 @@ func (in *DeploymentStatus) DeepCopyInto(out *DeploymentStatus) { *out = new(SecretHashes) (*in).DeepCopyInto(*out) } - if in.CurrentEncryptionKeyHashes != nil { - in, out := &in.CurrentEncryptionKeyHashes, &out.CurrentEncryptionKeyHashes - *out = make(DeploymentStatusEncryptionKeyHashes, len(*in)) - copy(*out, *in) - } + in.Hashes.DeepCopyInto(&out.Hashes) if in.ForceStatusReload != nil { in, out := &in.ForceStatusReload, &out.ForceStatusReload *out = new(bool) @@ -423,25 +419,73 @@ func (in *DeploymentStatus) DeepCopy() *DeploymentStatus { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in DeploymentStatusEncryptionKeyHashes) DeepCopyInto(out *DeploymentStatusEncryptionKeyHashes) { +func (in DeploymentStatusHashList) DeepCopyInto(out *DeploymentStatusHashList) { { in := &in - *out = make(DeploymentStatusEncryptionKeyHashes, len(*in)) + *out = make(DeploymentStatusHashList, len(*in)) copy(*out, *in) return } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeploymentStatusEncryptionKeyHashes. -func (in DeploymentStatusEncryptionKeyHashes) DeepCopy() DeploymentStatusEncryptionKeyHashes { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeploymentStatusHashList. +func (in DeploymentStatusHashList) DeepCopy() DeploymentStatusHashList { if in == nil { return nil } - out := new(DeploymentStatusEncryptionKeyHashes) + out := new(DeploymentStatusHashList) in.DeepCopyInto(out) return *out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DeploymentStatusHashes) DeepCopyInto(out *DeploymentStatusHashes) { + *out = *in + if in.Encryption != nil { + in, out := &in.Encryption, &out.Encryption + *out = make(DeploymentStatusHashList, len(*in)) + copy(*out, *in) + } + in.TLS.DeepCopyInto(&out.TLS) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeploymentStatusHashes. +func (in *DeploymentStatusHashes) DeepCopy() *DeploymentStatusHashes { + if in == nil { + return nil + } + out := new(DeploymentStatusHashes) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DeploymentStatusHashesTLS) DeepCopyInto(out *DeploymentStatusHashesTLS) { + *out = *in + if in.CA != nil { + in, out := &in.CA, &out.CA + *out = new(string) + **out = **in + } + if in.Truststore != nil { + in, out := &in.Truststore, &out.Truststore + *out = make(DeploymentStatusHashList, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeploymentStatusHashesTLS. +func (in *DeploymentStatusHashesTLS) DeepCopy() *DeploymentStatusHashesTLS { + if in == nil { + return nil + } + out := new(DeploymentStatusHashesTLS) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DeploymentStatusMembers) DeepCopyInto(out *DeploymentStatusMembers) { *out = *in @@ -1361,11 +1405,6 @@ func (in *TLSSNISpec) DeepCopyInto(out *TLSSNISpec) { (*out)[key] = outVal } } - if in.Mode != nil { - in, out := &in.Mode, &out.Mode - *out = new(TLSSNIRotateMode) - **out = **in - } return } @@ -1402,6 +1441,11 @@ func (in *TLSSpec) DeepCopyInto(out *TLSSpec) { *out = new(TLSSNISpec) (*in).DeepCopyInto(*out) } + if in.Mode != nil { + in, out := &in.Mode, &out.Mode + *out = new(TLSRotateMode) + **out = **in + } return } diff --git a/pkg/deployment/deployment_inspector.go b/pkg/deployment/deployment_inspector.go index 1fa0aa89e..e373f1c74 100644 --- a/pkg/deployment/deployment_inspector.go +++ b/pkg/deployment/deployment_inspector.go @@ -150,7 +150,7 @@ func (d *Deployment) inspectDeploymentWithError(ctx context.Context, lastInterva } } - if err := d.resources.EnsureSecrets(cachedStatus); err != nil { + if err := d.resources.EnsureSecrets(d.deps.Log, cachedStatus); err != nil { return minInspectionInterval, errors.Wrapf(err, "Secret creation failed") } diff --git a/pkg/deployment/deployment_run_test.go b/pkg/deployment/deployment_run_test.go index eaa58625c..040850c75 100644 --- a/pkg/deployment/deployment_run_test.go +++ b/pkg/deployment/deployment_run_test.go @@ -27,6 +27,8 @@ import ( "fmt" "testing" + "github.com/rs/zerolog/log" + "github.com/arangodb/kube-arangodb/pkg/deployment/resources/inspector" "github.com/arangodb/kube-arangodb/pkg/util/errors" @@ -60,12 +62,12 @@ func runTestCase(t *testing.T, testCase testCaseStruct) { for { cache, err := inspector.NewInspector(d.GetKubeCli(), d.GetNamespace()) require.NoError(t, err) - err = d.resources.EnsureSecrets(cache) + err = d.resources.EnsureSecrets(log.Logger, cache) if err == nil { break } - if errs > 5 { + if errs > 25 { require.NoError(t, err) } diff --git a/pkg/deployment/pod/builder.go b/pkg/deployment/pod/builder.go index 732b04da8..e21c3de07 100644 --- a/pkg/deployment/pod/builder.go +++ b/pkg/deployment/pod/builder.go @@ -46,5 +46,6 @@ type Input struct { type Builder interface { Args(i Input) k8sutil.OptionPairs Volumes(i Input) ([]core.Volume, []core.VolumeMount) + Envs(i Input) []core.EnvVar Verify(i Input, cachedStatus inspector.Inspector) error } diff --git a/pkg/deployment/pod/encryption.go b/pkg/deployment/pod/encryption.go index da8c2e4e1..1c1f036b4 100644 --- a/pkg/deployment/pod/encryption.go +++ b/pkg/deployment/pod/encryption.go @@ -115,6 +115,10 @@ func Encryption() Builder { type encryption struct{} +func (e encryption) Envs(i Input) []core.EnvVar { + return nil +} + func (e encryption) Args(i Input) k8sutil.OptionPairs { if !IsEncryptionEnabled(i) { return nil diff --git a/pkg/deployment/pod/jwt.go b/pkg/deployment/pod/jwt.go index 3556c4391..a7fb7180c 100644 --- a/pkg/deployment/pod/jwt.go +++ b/pkg/deployment/pod/jwt.go @@ -23,28 +23,85 @@ package pod import ( + "path/filepath" + + "github.com/arangodb/go-driver" "github.com/arangodb/kube-arangodb/pkg/deployment/resources/inspector" + "github.com/arangodb/kube-arangodb/pkg/util/constants" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" "github.com/pkg/errors" core "k8s.io/api/core/v1" ) +func IsAuthenticated(i Input) bool { + return i.Deployment.IsAuthenticated() +} + +func VersionHasJWTSecretKeyfile(v driver.Version) bool { + if v.CompareTo("3.3.22") >= 0 && v.CompareTo("3.4.0") < 0 { + return true + } + if v.CompareTo("3.4.2") >= 0 { + return true + } + + return false +} + +func VersionHasJWTSecretKeyfolder(i Input) bool { + return i.Enterprise && i.Version.CompareTo("3.7.0") > 0 +} + func JWT() Builder { return jwt{} } type jwt struct{} -func (e jwt) Args(i Input) k8sutil.OptionPairs { +func (e jwt) Envs(i Input) []core.EnvVar { + if !IsAuthenticated(i) { + return nil + } + + if !VersionHasJWTSecretKeyfile(i.Version) { + return []core.EnvVar{k8sutil.CreateEnvSecretKeySelector(constants.EnvArangodJWTSecret, + i.Deployment.Authentication.GetJWTSecretName(), constants.SecretKeyToken)} + } + return nil } +func (e jwt) Args(i Input) k8sutil.OptionPairs { + if !IsAuthenticated(i) { + // Without authentication + return k8sutil.NewOptionPair(k8sutil.OptionPair{Key: "--server.authentication", Value: "false"}) + } + + options := k8sutil.CreateOptionPairs(2) + + options.Add("--server.authentication", "true") + + if VersionHasJWTSecretKeyfile(i.Version) { + keyPath := filepath.Join(k8sutil.ClusterJWTSecretVolumeMountDir, constants.SecretKeyToken) + options.Add("--server.jwt-secret-keyfile", keyPath) + } else { + options.Addf("--server.jwt-secret", "$(%s)", constants.EnvArangodJWTSecret) + } + + return options +} + func (e jwt) Volumes(i Input) ([]core.Volume, []core.VolumeMount) { - return nil, nil + if !IsAuthenticated(i) { + return nil, nil + } + + vol := k8sutil.CreateVolumeWithSecret(k8sutil.ClusterJWTSecretVolumeName, i.Deployment.Authentication.GetJWTSecretName()) + return []core.Volume{vol}, []core.VolumeMount{k8sutil.ClusterJWTVolumeMount()} } func (e jwt) Verify(i Input, cachedStatus inspector.Inspector) error { - if !i.Deployment.IsAuthenticated() { + if !IsAuthenticated(i) { return nil } diff --git a/pkg/deployment/pod/sni.go b/pkg/deployment/pod/sni.go index 9697a98d8..a0b67840c 100644 --- a/pkg/deployment/pod/sni.go +++ b/pkg/deployment/pod/sni.go @@ -57,6 +57,10 @@ func SNI() Builder { type sni struct{} +func (s sni) Envs(i Input) []core.EnvVar { + return nil +} + func (s sni) isSupported(i Input) bool { if !i.Deployment.TLS.IsSecure() { return false diff --git a/pkg/deployment/pod/tls.go b/pkg/deployment/pod/tls.go new file mode 100644 index 000000000..978e420e7 --- /dev/null +++ b/pkg/deployment/pod/tls.go @@ -0,0 +1,90 @@ +// +// DISCLAIMER +// +// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Adam Janikowski +// + +package pod + +import ( + "path/filepath" + + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" + + "github.com/arangodb/kube-arangodb/pkg/deployment/resources/inspector" + + "github.com/arangodb/kube-arangodb/pkg/util/constants" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + core "k8s.io/api/core/v1" +) + +func IsRuntimeTLSKeyfileUpdateSupported(i Input) bool { + return IsTLSEnabled(i) && i.Enterprise && + i.Version.CompareTo("3.7.0") >= 0 && + i.Deployment.TLS.Mode.Get() == api.TLSRotateModeInPlace +} + +func IsTLSEnabled(i Input) bool { + return i.Deployment.TLS.IsSecure() +} + +func GetTLSKeyfileSecretName(i Input) string { + return k8sutil.CreateTLSKeyfileSecretName(i.ApiObject.GetName(), i.Group.AsRole(), i.ID) +} + +func TLS() Builder { + return tls{} +} + +type tls struct{} + +func (s tls) Envs(i Input) []core.EnvVar { + return nil +} + +func (s tls) Verify(i Input, cachedStatus inspector.Inspector) error { + if !IsTLSEnabled(i) { + return nil + } + + return nil +} + +func (s tls) Volumes(i Input) ([]core.Volume, []core.VolumeMount) { + if !IsTLSEnabled(i) { + return nil, nil + } + + return []core.Volume{k8sutil.CreateVolumeWithSecret(k8sutil.TlsKeyfileVolumeName, GetTLSKeyfileSecretName(i))}, + []core.VolumeMount{k8sutil.TlsKeyfileVolumeMount()} +} + +func (s tls) Args(i Input) k8sutil.OptionPairs { + if !IsTLSEnabled(i) { + return nil + } + + opts := k8sutil.CreateOptionPairs() + + keyPath := filepath.Join(k8sutil.TLSKeyfileVolumeMountDir, constants.SecretTLSKeyfile) + opts.Add("--ssl.keyfile", keyPath) + opts.Add("--ssl.ecdh-curve", "") // This way arangod accepts curves other than P256 as well. + + return opts +} diff --git a/pkg/deployment/pod/upgrade.go b/pkg/deployment/pod/upgrade.go index 67266a0bd..753dede6f 100644 --- a/pkg/deployment/pod/upgrade.go +++ b/pkg/deployment/pod/upgrade.go @@ -35,6 +35,10 @@ func AutoUpgrade() Builder { type autoUpgrade struct{} +func (u autoUpgrade) Envs(i Input) []core.EnvVar { + return nil +} + func (u autoUpgrade) Verify(i Input, cachedStatus inspector.Inspector) error { return nil } diff --git a/pkg/deployment/pod/volumes.go b/pkg/deployment/pod/volumes.go new file mode 100644 index 000000000..0006a4df1 --- /dev/null +++ b/pkg/deployment/pod/volumes.go @@ -0,0 +1,89 @@ +// +// DISCLAIMER +// +// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Adam Janikowski +// +package pod + +import core "k8s.io/api/core/v1" + +func NewVolumes() Volumes { + return &volumes{ + volumes: []core.Volume{}, + volumeMounts: []core.VolumeMount{}, + } +} + +type Volumes interface { + Append(b Builder, i Input) + + AddVolume(volumes ...core.Volume) + AddVolumes(volumes []core.Volume) + + AddVolumeMount(mounts ...core.VolumeMount) + AddVolumeMounts(mounts []core.VolumeMount) + + Volumes() []core.Volume + VolumeMounts() []core.VolumeMount +} + +var _ Volumes = &volumes{} + +type volumes struct { + volumes []core.Volume + volumeMounts []core.VolumeMount +} + +func (v *volumes) Append(b Builder, i Input) { + vols, mounts := b.Volumes(i) + v.AddVolumes(vols) + v.AddVolumeMounts(mounts) +} + +func (v *volumes) AddVolume(volumes ...core.Volume) { + v.AddVolumes(volumes) +} + +func (v *volumes) AddVolumes(volumes []core.Volume) { + if len(volumes) == 0 { + return + } + + v.volumes = append(v.volumes, volumes...) +} + +func (v *volumes) AddVolumeMount(mounts ...core.VolumeMount) { + v.AddVolumeMounts(mounts) +} + +func (v *volumes) AddVolumeMounts(mounts []core.VolumeMount) { + if len(mounts) == 0 { + return + } + + v.volumeMounts = append(v.volumeMounts, mounts...) +} + +func (v *volumes) Volumes() []core.Volume { + return v.volumes +} + +func (v *volumes) VolumeMounts() []core.VolumeMount { + return v.volumeMounts +} diff --git a/pkg/deployment/reconcile/action_encryption_status_update.go b/pkg/deployment/reconcile/action_encryption_status_update.go index d96eaae87..4bcff13ef 100644 --- a/pkg/deployment/reconcile/action_encryption_status_update.go +++ b/pkg/deployment/reconcile/action_encryption_status_update.go @@ -24,7 +24,6 @@ package reconcile import ( "context" - "sort" "github.com/arangodb/kube-arangodb/pkg/util" meta "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -64,28 +63,20 @@ func (a *encryptionKeyStatusUpdateAction) Start(ctx context.Context) (bool, erro return true, nil } - keys := make([]string, 0, len(f.Data)) - - for key := range f.Data { - keys = append(keys, key) - } - - sort.Strings(keys) - - keyHashes := util.PrefixStringArray(keys, "sha256:") + keyHashes := secretKeysToListWithPrefix("sha256:", f) if err = a.actionCtx.WithStatusUpdate(func(s *api.DeploymentStatus) bool { if len(keyHashes) == 0 { - if s.CurrentEncryptionKeyHashes != nil { - s.CurrentEncryptionKeyHashes = nil + if s.Hashes.Encryption != nil { + s.Hashes.Encryption = nil return true } return false } - if !util.CompareStringArray(keyHashes, s.CurrentEncryptionKeyHashes) { - s.CurrentEncryptionKeyHashes = keyHashes + if !util.CompareStringArray(keyHashes, s.Hashes.Encryption) { + s.Hashes.Encryption = keyHashes return true } return false diff --git a/pkg/deployment/reconcile/action_renew_tls_ca_certificate.go b/pkg/deployment/reconcile/action_renew_tls_ca_certificate.go deleted file mode 100644 index ec8584720..000000000 --- a/pkg/deployment/reconcile/action_renew_tls_ca_certificate.go +++ /dev/null @@ -1,82 +0,0 @@ -// -// DISCLAIMER -// -// Copyright 2020 ArangoDB GmbH, Cologne, Germany -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Copyright holder is ArangoDB GmbH, Cologne, Germany -// -// Author Ewout Prangsma -// - -package reconcile - -import ( - "context" - "time" - - api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" - "github.com/rs/zerolog" -) - -func init() { - registerAction(api.ActionTypeRenewTLSCACertificate, newRenewTLSCACertificateAction) -} - -// newRenewTLSCACertificateAction creates a new Action that implements the given -// planned RenewTLSCACertificate action. -func newRenewTLSCACertificateAction(log zerolog.Logger, action api.Action, actionCtx ActionContext) Action { - a := &renewTLSCACertificateAction{} - - a.actionImpl = newActionImplDefRef(log, action, actionCtx, renewTLSCACertificateTimeout) - - return a -} - -// renewTLSCACertificateAction implements a RenewTLSCACertificate action. -type renewTLSCACertificateAction struct { - // actionImpl implement timeout and member id functions - actionImpl - - // actionEmptyCheckProgress implement check progress with empty implementation - actionEmptyCheckProgress -} - -// Start performs the start of the action. -// Returns true if the action is completely finished, false in case -// the start time needs to be recorded and a ready condition needs to be checked. -func (a *renewTLSCACertificateAction) Start(ctx context.Context) (bool, error) { - // Just delete the secret. - // It will be re-created. - if err := a.actionCtx.DeleteTLSCASecret(); err != nil { - return false, maskAny(err) - } - return true, nil -} - -// CheckProgress checks the progress of the action. -// Returns true if the action is completely finished, false otherwise. -func (a *renewTLSCACertificateAction) CheckProgress(ctx context.Context) (bool, bool, error) { - return true, false, nil -} - -// Timeout returns the amount of time after which this action will timeout. -func (a *renewTLSCACertificateAction) Timeout() time.Duration { - return renewTLSCACertificateTimeout -} - -// Return the MemberID used / created in this action -func (a *renewTLSCACertificateAction) MemberID() string { - return a.action.MemberID -} diff --git a/pkg/deployment/reconcile/action_renew_tls_certificate.go b/pkg/deployment/reconcile/action_renew_tls_certificate.go deleted file mode 100644 index 86725a928..000000000 --- a/pkg/deployment/reconcile/action_renew_tls_certificate.go +++ /dev/null @@ -1,71 +0,0 @@ -// -// DISCLAIMER -// -// Copyright 2020 ArangoDB GmbH, Cologne, Germany -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Copyright holder is ArangoDB GmbH, Cologne, Germany -// -// Author Ewout Prangsma -// - -package reconcile - -import ( - "context" - - api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" - "github.com/rs/zerolog" -) - -func init() { - registerAction(api.ActionTypeRenewTLSCertificate, newRenewTLSCertificateAction) -} - -// newRenewTLSCertificateAction creates a new Action that implements the given -// planned RenewTLSCertificate action. -func newRenewTLSCertificateAction(log zerolog.Logger, action api.Action, actionCtx ActionContext) Action { - a := &renewTLSCertificateAction{} - - a.actionImpl = newActionImplDefRef(log, action, actionCtx, renewTLSCertificateTimeout) - - return a -} - -// renewTLSCertificateAction implements a RenewTLSCertificate action. -type renewTLSCertificateAction struct { - // actionImpl implement timeout and member id functions - actionImpl - - // actionEmptyCheckProgress implement check progress with empty implementation - actionEmptyCheckProgress -} - -// Start performs the start of the action. -// Returns true if the action is completely finished, false in case -// the start time needs to be recorded and a ready condition needs to be checked. -func (a *renewTLSCertificateAction) Start(ctx context.Context) (bool, error) { - log := a.log - group := a.action.Group - m, ok := a.actionCtx.GetMemberStatusByID(a.action.MemberID) - if !ok { - log.Error().Msg("No such member") - } - // Just delete the secret. - // It will be re-created when the member restarts. - if err := a.actionCtx.DeleteTLSKeyfile(group, m); err != nil { - return false, maskAny(err) - } - return false, nil -} diff --git a/pkg/deployment/reconcile/action_tls_ca_append.go b/pkg/deployment/reconcile/action_tls_ca_append.go new file mode 100644 index 000000000..278450039 --- /dev/null +++ b/pkg/deployment/reconcile/action_tls_ca_append.go @@ -0,0 +1,126 @@ +// +// DISCLAIMER +// +// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Adam Janikowski +// + +package reconcile + +import ( + "context" + "encoding/base64" + + "github.com/arangodb/kube-arangodb/pkg/deployment/patch" + "github.com/arangodb/kube-arangodb/pkg/util" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + "github.com/pkg/errors" + "k8s.io/apimachinery/pkg/types" + + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" + "github.com/arangodb/kube-arangodb/pkg/deployment/resources" + "github.com/rs/zerolog" +) + +const ( + actionTypeAppendTLSCACertificateChecksum = "checksum" +) + +func init() { + registerAction(api.ActionTypeAppendTLSCACertificate, newAppendTLSCACertificateAction) +} + +func newAppendTLSCACertificateAction(log zerolog.Logger, action api.Action, actionCtx ActionContext) Action { + a := &appendTLSCACertificateAction{} + + a.actionImpl = newActionImplDefRef(log, action, actionCtx, operationTLSCACertificateTimeout) + + return a +} + +type appendTLSCACertificateAction struct { + actionImpl + + actionEmptyCheckProgress +} + +func (a *appendTLSCACertificateAction) Start(ctx context.Context) (bool, error) { + if !a.actionCtx.GetSpec().TLS.IsSecure() { + return true, nil + } + + certChecksum, exists := a.action.Params[actionTypeAppendTLSCACertificateChecksum] + if !exists { + a.log.Warn().Msgf("Key %s is missing in action", actionTypeAppendTLSCACertificateChecksum) + return true, nil + } + + caSecret, exists := a.actionCtx.GetCachedStatus().Secret(a.actionCtx.GetSpec().TLS.GetCASecretName()) + if !exists { + a.log.Warn().Msgf("Secret %s is missing", a.actionCtx.GetSpec().TLS.GetCASecretName()) + return true, nil + } + + caFolder, exists := a.actionCtx.GetCachedStatus().Secret(resources.GetCASecretName(a.actionCtx.GetAPIObject())) + if !exists { + a.log.Warn().Msgf("Secret %s is missing", resources.GetCASecretName(a.actionCtx.GetAPIObject())) + return true, nil + } + + ca, _, err := getKeyCertFromSecret(a.log, caSecret, resources.CACertName, resources.CAKeyName) + if err != nil { + a.log.Warn().Err(err).Msgf("Cert %s is invalid", resources.GetCASecretName(a.actionCtx.GetAPIObject())) + return true, nil + } + + caData, err := ca.ToPem() + if err != nil { + a.log.Warn().Err(err).Str("secret", resources.GetCASecretName(a.actionCtx.GetAPIObject())).Msgf("Unable to parse ca into pem") + return true, nil + } + + caSha := util.SHA256(caData) + + if caSha != certChecksum { + a.log.Warn().Msgf("Cert changed") + return true, nil + } + + if _, exists := caFolder.Data[caSha]; exists { + a.log.Warn().Msgf("Cert already exists") + return true, nil + } + + p := patch.NewPatch() + p.ItemAdd(patch.NewPath("data", caSha), base64.StdEncoding.EncodeToString(caData)) + + patch, err := p.Marshal() + if err != nil { + a.log.Error().Err(err).Msgf("Unable to encrypt patch") + return true, nil + } + + _, err = a.actionCtx.SecretsInterface().Patch(resources.GetCASecretName(a.actionCtx.GetAPIObject()), types.JSONPatchType, patch) + if err != nil { + if !k8sutil.IsInvalid(err) { + return false, errors.Wrapf(err, "Unable to update secret: %s", string(patch)) + } + } + + return true, nil +} diff --git a/pkg/deployment/reconcile/action_tls_ca_clean.go b/pkg/deployment/reconcile/action_tls_ca_clean.go new file mode 100644 index 000000000..9ee385945 --- /dev/null +++ b/pkg/deployment/reconcile/action_tls_ca_clean.go @@ -0,0 +1,124 @@ +// +// DISCLAIMER +// +// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Adam Janikowski +// + +package reconcile + +import ( + "context" + + "github.com/arangodb/kube-arangodb/pkg/deployment/patch" + "github.com/arangodb/kube-arangodb/pkg/util" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + "github.com/pkg/errors" + "k8s.io/apimachinery/pkg/types" + + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" + "github.com/arangodb/kube-arangodb/pkg/deployment/resources" + "github.com/rs/zerolog" +) + +func init() { + registerAction(api.ActionTypeCleanTLSCACertificate, newCleanTLSCACertificateAction) +} + +func newCleanTLSCACertificateAction(log zerolog.Logger, action api.Action, actionCtx ActionContext) Action { + a := &cleanTLSCACertificateAction{} + + a.actionImpl = newActionImplDefRef(log, action, actionCtx, operationTLSCACertificateTimeout) + + return a +} + +type cleanTLSCACertificateAction struct { + actionImpl + + actionEmptyCheckProgress +} + +func (a *cleanTLSCACertificateAction) Start(ctx context.Context) (bool, error) { + a.log.Info().Msgf("Clean TLS Ca") + if !a.actionCtx.GetSpec().TLS.IsSecure() { + a.log.Info().Msgf("Insecure deployment") + return true, nil + } + + certChecksum, exists := a.action.Params[actionTypeAppendTLSCACertificateChecksum] + if !exists { + a.log.Warn().Msgf("Key %s is missing in action", actionTypeAppendTLSCACertificateChecksum) + return true, nil + } + + caSecret, exists := a.actionCtx.GetCachedStatus().Secret(a.actionCtx.GetSpec().TLS.GetCASecretName()) + if !exists { + a.log.Warn().Msgf("Secret %s is missing", a.actionCtx.GetSpec().TLS.GetCASecretName()) + return true, nil + } + + caFolder, exists := a.actionCtx.GetCachedStatus().Secret(resources.GetCASecretName(a.actionCtx.GetAPIObject())) + if !exists { + a.log.Warn().Msgf("Secret %s is missing", resources.GetCASecretName(a.actionCtx.GetAPIObject())) + return true, nil + } + + ca, _, err := getKeyCertFromSecret(a.log, caSecret, resources.CACertName, resources.CAKeyName) + if err != nil { + a.log.Warn().Err(err).Msgf("Cert %s is invalid", resources.GetCASecretName(a.actionCtx.GetAPIObject())) + return true, nil + } + + caData, err := ca.ToPem() + if err != nil { + a.log.Warn().Err(err).Str("secret", resources.GetCASecretName(a.actionCtx.GetAPIObject())).Msgf("Unable to parse ca into pem") + return true, nil + } + + caSha := util.SHA256(caData) + + if caSha == certChecksum { + a.log.Warn().Msgf("Unable to remove current ca") + return true, nil + } + + if _, exists := caFolder.Data[certChecksum]; !exists { + a.log.Warn().Msgf("Cert missing") + return true, nil + } + + p := patch.NewPatch() + p.ItemRemove(patch.NewPath("data", certChecksum)) + + patch, err := p.Marshal() + if err != nil { + a.log.Error().Err(err).Msgf("Unable to encrypt patch") + return true, nil + } + + a.log.Info().Msgf("Removing key %s from truststore", certChecksum) + _, err = a.actionCtx.SecretsInterface().Patch(resources.GetCASecretName(a.actionCtx.GetAPIObject()), types.JSONPatchType, patch) + if err != nil { + if !k8sutil.IsInvalid(err) { + return false, errors.Wrapf(err, "Unable to update secret: %s", string(patch)) + } + } + + return true, nil +} diff --git a/pkg/deployment/reconcile/action_tls_ca_renew.go b/pkg/deployment/reconcile/action_tls_ca_renew.go new file mode 100644 index 000000000..1e2575952 --- /dev/null +++ b/pkg/deployment/reconcile/action_tls_ca_renew.go @@ -0,0 +1,66 @@ +// +// DISCLAIMER +// +// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Adam Janikowski +// + +package reconcile + +import ( + "context" + + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + "github.com/rs/zerolog" + meta "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func init() { + registerAction(api.ActionTypeRenewTLSCACertificate, newRenewTLSCACertificateAction) +} + +func newRenewTLSCACertificateAction(log zerolog.Logger, action api.Action, actionCtx ActionContext) Action { + a := &renewTLSCACertificateAction{} + + a.actionImpl = newActionImplDefRef(log, action, actionCtx, operationTLSCACertificateTimeout) + + return a +} + +type renewTLSCACertificateAction struct { + actionImpl + + actionEmptyCheckProgress +} + +func (a *renewTLSCACertificateAction) Start(ctx context.Context) (bool, error) { + if !a.actionCtx.GetSpec().TLS.IsSecure() { + return true, nil + } + + s := a.actionCtx.SecretsInterface() + if err := s.Delete(a.actionCtx.GetSpec().TLS.GetCASecretName(), &meta.DeleteOptions{}); err != nil { + if !k8sutil.IsNotFound(err) { + a.log.Warn().Err(err).Msgf("Unable to clean cert %s", a.actionCtx.GetSpec().TLS.GetCASecretName()) + return true, nil + } + } + + return true, nil +} diff --git a/pkg/deployment/reconcile/action_tls_keyfile_clean.go b/pkg/deployment/reconcile/action_tls_keyfile_clean.go new file mode 100644 index 000000000..e0e6568f6 --- /dev/null +++ b/pkg/deployment/reconcile/action_tls_keyfile_clean.go @@ -0,0 +1,67 @@ +// +// DISCLAIMER +// +// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Adam Janikowski +// + +package reconcile + +import ( + "context" + + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" + "github.com/rs/zerolog" +) + +func init() { + registerAction(api.ActionTypeCleanTLSKeyfileCertificate, newCleanTLSKeyfileCertificateAction) +} + +func newCleanTLSKeyfileCertificateAction(log zerolog.Logger, action api.Action, actionCtx ActionContext) Action { + a := &cleanTLSKeyfileCertificateAction{} + + a.actionImpl = newActionImplDefRef(log, action, actionCtx, operationTLSCACertificateTimeout) + + return a +} + +type cleanTLSKeyfileCertificateAction struct { + actionImpl + + actionEmptyCheckProgress +} + +func (a *cleanTLSKeyfileCertificateAction) Start(ctx context.Context) (bool, error) { + member, exists := a.actionCtx.GetMemberStatusByID(a.action.MemberID) + if !exists { + a.log.Warn().Msgf("Member does not exist") + return true, nil + } + + if err := a.actionCtx.DeleteTLSKeyfile(a.action.Group, member); err != nil { + a.log.Warn().Err(err).Msgf("Unable to remove keyfile") + if !k8sutil.IsNotFound(err) { + return false, err + } + } + + return true, nil +} diff --git a/pkg/deployment/reconcile/action_tls_keyfile_refresh.go b/pkg/deployment/reconcile/action_tls_keyfile_refresh.go new file mode 100644 index 000000000..003df5b53 --- /dev/null +++ b/pkg/deployment/reconcile/action_tls_keyfile_refresh.go @@ -0,0 +1,93 @@ +// +// DISCLAIMER +// +// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Adam Janikowski +// + +package reconcile + +import ( + "context" + + "github.com/arangodb/kube-arangodb/pkg/deployment/client" + "github.com/arangodb/kube-arangodb/pkg/util" + "github.com/arangodb/kube-arangodb/pkg/util/constants" + + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" + "github.com/rs/zerolog" +) + +func init() { + registerAction(api.ActionTypeRefreshTLSKeyfileCertificate, newRefreshTLSKeyfileCertificateAction) +} + +func newRefreshTLSKeyfileCertificateAction(log zerolog.Logger, action api.Action, actionCtx ActionContext) Action { + a := &refreshTLSKeyfileCertificateAction{} + + a.actionImpl = newActionImplDefRef(log, action, actionCtx, operationTLSCACertificateTimeout) + + return a +} + +type refreshTLSKeyfileCertificateAction struct { + actionImpl +} + +func (a *refreshTLSKeyfileCertificateAction) CheckProgress(ctx context.Context) (bool, bool, error) { + c, err := a.actionCtx.GetServerClient(ctx, a.action.Group, a.action.MemberID) + if err != nil { + a.log.Warn().Err(err).Msg("Unable to get client") + return true, false, nil + } + + s, exists := a.actionCtx.GetCachedStatus().Secret(k8sutil.CreateTLSKeyfileSecretName(a.actionCtx.GetAPIObject().GetName(), a.action.Group.AsRole(), a.action.MemberID)) + if !exists { + a.log.Warn().Msg("Keyfile secret is missing") + return true, false, nil + } + + keyfile, ok := s.Data[constants.SecretTLSKeyfile] + if !ok { + a.log.Warn().Msg("Keyfile secret is invalid") + return true, false, nil + } + + keyfileSha := util.SHA256(keyfile) + + client := client.NewClient(c.Connection()) + + e, err := client.RefreshTLS(ctx) + if err != nil { + a.log.Warn().Err(err).Msg("Unable to refresh TLS") + return true, false, nil + } + + if e.Result.KeyFile.Checksum == keyfileSha { + return true, false, nil + } + + return false, false, nil +} + +func (a *refreshTLSKeyfileCertificateAction) Start(ctx context.Context) (bool, error) { + ready, _, err := a.CheckProgress(ctx) + return ready, err +} diff --git a/pkg/deployment/reconcile/action_tls_status_update.go b/pkg/deployment/reconcile/action_tls_status_update.go new file mode 100644 index 000000000..247e51c6a --- /dev/null +++ b/pkg/deployment/reconcile/action_tls_status_update.go @@ -0,0 +1,86 @@ +// +// DISCLAIMER +// +// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Adam Janikowski +// + +package reconcile + +import ( + "context" + + "github.com/arangodb/kube-arangodb/pkg/deployment/resources" + "github.com/arangodb/kube-arangodb/pkg/util" + meta "k8s.io/apimachinery/pkg/apis/meta/v1" + + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" + "github.com/rs/zerolog" +) + +func init() { + registerAction(api.ActionTypeTLSKeyStatusUpdate, newTLSKeyStatusUpdate) +} + +func newTLSKeyStatusUpdate(log zerolog.Logger, action api.Action, actionCtx ActionContext) Action { + a := &tlsKeyStatusUpdateAction{} + + a.actionImpl = newActionImplDefRef(log, action, actionCtx, defaultTimeout) + + return a +} + +type tlsKeyStatusUpdateAction struct { + actionImpl + + actionEmptyCheckProgress +} + +func (a *tlsKeyStatusUpdateAction) Start(ctx context.Context) (bool, error) { + if !a.actionCtx.GetSpec().TLS.IsSecure() { + return true, nil + } + + f, err := a.actionCtx.SecretsInterface().Get(resources.GetCASecretName(a.actionCtx.GetAPIObject()), meta.GetOptions{}) + if err != nil { + a.log.Error().Err(err).Msgf("Unable to get folder info") + return true, nil + } + + keyHashes := secretKeysToListWithPrefix("sha256:", f) + + if err = a.actionCtx.WithStatusUpdate(func(s *api.DeploymentStatus) bool { + if len(keyHashes) == 1 { + if s.Hashes.TLS.CA == nil || *s.Hashes.TLS.CA != keyHashes[0] { + s.Hashes.TLS.CA = util.NewString(keyHashes[0]) + return true + } + } + + if !util.CompareStringArray(keyHashes, s.Hashes.TLS.Truststore) { + s.Hashes.TLS.Truststore = keyHashes + return true + } + + return false + }); err != nil { + return false, err + } + + return true, nil +} diff --git a/pkg/deployment/reconcile/plan_builder.go b/pkg/deployment/reconcile/plan_builder.go index 70d0c9d20..94ec722c0 100644 --- a/pkg/deployment/reconcile/plan_builder.go +++ b/pkg/deployment/reconcile/plan_builder.go @@ -193,6 +193,15 @@ func createPlan(ctx context.Context, log zerolog.Logger, apiObject k8sutil.APIOb } } + // Update status + if plan.IsEmpty() { + plan = pb.Apply(createEncryptionKeyStatusUpdate) + } + + if plan.IsEmpty() { + plan = pb.Apply(createTLSStatusUpdate) + } + // Check for scale up/down if plan.IsEmpty() { plan = pb.Apply(createScaleMemeberPlan) @@ -208,21 +217,28 @@ func createPlan(ctx context.Context, log zerolog.Logger, apiObject k8sutil.APIOb plan = pb.Apply(createEncryptionKey) } - // Check for the need to rotate TLS certificate of a members if plan.IsEmpty() { - plan = pb.Apply(createRotateTLSServerCertificatePlan) + plan = pb.Apply(createCARenewalPlan) } + if plan.IsEmpty() { + plan = pb.Apply(createCAAppendPlan) + } + + if plan.IsEmpty() { + plan = pb.Apply(createKeyfileRenewalPlan) + } + + // Check for the need to rotate TLS certificate of a members + //if plan.IsEmpty() { + // plan = pb.Apply(createRotateTLSServerCertificatePlan) + //} + // Check for changes storage classes or requirements if plan.IsEmpty() { plan = pb.Apply(createRotateServerStoragePlan) } - // Check for the need to rotate TLS CA certificate and all members - if plan.IsEmpty() { - plan = pb.Apply(createRotateTLSCAPlan) - } - if plan.IsEmpty() { plan = pb.Apply(createRotateTLSServerSNIPlan) } @@ -235,6 +251,10 @@ func createPlan(ctx context.Context, log zerolog.Logger, apiObject k8sutil.APIOb plan = pb.Apply(cleanEncryptionKey) } + if plan.IsEmpty() { + plan = pb.Apply(createCACleanPlan) + } + // Return plan return plan, true } diff --git a/pkg/deployment/reconcile/plan_builder_encryption.go b/pkg/deployment/reconcile/plan_builder_encryption.go index eb1d9d94d..a5ffb61ae 100644 --- a/pkg/deployment/reconcile/plan_builder_encryption.go +++ b/pkg/deployment/reconcile/plan_builder_encryption.go @@ -38,15 +38,23 @@ import ( "github.com/rs/zerolog" ) +func skipEncryptionPlan(spec api.DeploymentSpec, status api.DeploymentStatus) bool { + if !spec.RocksDB.IsEncrypted() { + return true + } + + if i := status.CurrentImage; i == nil || !i.Enterprise || i.ArangoDBVersion.CompareTo("3.7.0") < 0 { + return true + } + + return false +} + func createEncryptionKey(ctx context.Context, log zerolog.Logger, apiObject k8sutil.APIObject, spec api.DeploymentSpec, status api.DeploymentStatus, cachedStatus inspector.Inspector, context PlanBuilderContext) api.Plan { - if !spec.RocksDB.IsEncrypted() { - return nil - } - - if i := status.CurrentImage; i == nil || !i.Enterprise || i.ArangoDBVersion.CompareTo("3.7.0") < 0 { + if skipEncryptionPlan(spec, status) { return nil } @@ -128,30 +136,53 @@ func createEncryptionKey(ctx context.Context, return plan } - currentKeys := make([]string, 0, len(keyfolder.Data)) + return api.Plan{} +} - for key := range keyfolder.Data { - currentKeys = append(currentKeys, key) +func createEncryptionKeyStatusUpdate(ctx context.Context, + log zerolog.Logger, apiObject k8sutil.APIObject, + spec api.DeploymentSpec, status api.DeploymentStatus, + cachedStatus inspector.Inspector, context PlanBuilderContext) api.Plan { + if skipEncryptionPlan(spec, status) { + return nil } - currentKeyHashes := util.PrefixStringArray(currentKeys, "sha256:") - - if !util.CompareStringArray(currentKeyHashes, status.CurrentEncryptionKeyHashes) { + if createEncryptionKeyStatusUpdateRequired(ctx, log, apiObject, spec, status, cachedStatus, context) { return api.Plan{api.NewAction(api.ActionTypeEncryptionKeyStatusUpdate, api.ServerGroupUnknown, "")} } - return api.Plan{} + return nil + +} + +func createEncryptionKeyStatusUpdateRequired(ctx context.Context, + log zerolog.Logger, apiObject k8sutil.APIObject, + spec api.DeploymentSpec, status api.DeploymentStatus, + cachedStatus inspector.Inspector, context PlanBuilderContext) bool { + if skipEncryptionPlan(spec, status) { + return false + } + + keyfolder, exists := cachedStatus.Secret(pod.GetKeyfolderSecretName(context.GetName())) + if !exists { + log.Error().Msgf("Encryption key folder does not exist") + return false + } + + keyHashes := secretKeysToListWithPrefix("sha256:", keyfolder) + + if !util.CompareStringArray(keyHashes, status.Hashes.Encryption) { + return true + } + + return false } func cleanEncryptionKey(ctx context.Context, log zerolog.Logger, apiObject k8sutil.APIObject, spec api.DeploymentSpec, status api.DeploymentStatus, cachedStatus inspector.Inspector, context PlanBuilderContext) api.Plan { - if !spec.RocksDB.IsEncrypted() { - return nil - } - - if i := status.CurrentImage; i == nil || !i.Enterprise || i.ArangoDBVersion.CompareTo("3.7.0") < 0 { + if skipEncryptionPlan(spec, status) { return nil } diff --git a/pkg/deployment/reconcile/plan_builder_rotate_upgrade.go b/pkg/deployment/reconcile/plan_builder_rotate_upgrade.go index 79e541b67..8a3f460dd 100644 --- a/pkg/deployment/reconcile/plan_builder_rotate_upgrade.go +++ b/pkg/deployment/reconcile/plan_builder_rotate_upgrade.go @@ -76,7 +76,6 @@ func createRotateOrUpgradePlanInternal(log zerolog.Logger, apiObject k8sutil.API continue } - log.Debug().Msgf("Before upgrade") // Got pod, compare it with what it should be decision := podNeedsUpgrading(log, pod, spec, status.Images) if decision.UpgradeNeeded && !decision.UpgradeAllowed { @@ -89,14 +88,11 @@ func createRotateOrUpgradePlanInternal(log zerolog.Logger, apiObject k8sutil.API return nil } - log.Debug().Msgf("After upgrade") - if !newPlan.IsEmpty() { // Only rotate/upgrade 1 pod at a time continue } - log.Debug().Msgf("Before rotate") if decision.UpgradeNeeded { // Yes, upgrade is needed (and allowed) newPlan = createUpgradeMemberPlan(log, m, group, "Version upgrade", spec.GetImage(), status, @@ -108,7 +104,6 @@ func createRotateOrUpgradePlanInternal(log zerolog.Logger, apiObject k8sutil.API newPlan = createRotateMemberPlan(log, m, group, reason) } } - log.Debug().Msgf("After rotate") if !newPlan.IsEmpty() { // Only rotate/upgrade 1 pod at a time diff --git a/pkg/deployment/reconcile/plan_builder_test.go b/pkg/deployment/reconcile/plan_builder_test.go index abae76871..2b472f7e7 100644 --- a/pkg/deployment/reconcile/plan_builder_test.go +++ b/pkg/deployment/reconcile/plan_builder_test.go @@ -385,7 +385,7 @@ func TestCreatePlanClusterScale(t *testing.T) { var status api.DeploymentStatus addAgentsToStatus(t, &status, 3) - newPlan, changed := createPlan(ctx, log, depl, nil, spec, status, nil, c) + newPlan, changed := createPlan(ctx, log, depl, nil, spec, status, inspector.NewEmptyInspector(), c) assert.True(t, changed) require.Len(t, newPlan, 6) // Adding 3 dbservers & 3 coordinators (note: agents do not scale now) assert.Equal(t, api.ActionTypeAddMember, newPlan[0].Type) diff --git a/pkg/deployment/reconcile/plan_builder_tls.go b/pkg/deployment/reconcile/plan_builder_tls.go index 0976569ad..5de6f49b3 100644 --- a/pkg/deployment/reconcile/plan_builder_tls.go +++ b/pkg/deployment/reconcile/plan_builder_tls.go @@ -17,259 +17,506 @@ // // Copyright holder is ArangoDB GmbH, Cologne, Germany // -// Author Ewout Prangsma +// Author Adam Janikowski // package reconcile import ( + "bytes" "context" + "crypto/tls" "crypto/x509" "encoding/pem" - "net" + "fmt" + "net/http" + "net/url" + "reflect" "time" + "github.com/arangodb/kube-arangodb/pkg/deployment/client" + "github.com/arangodb/kube-arangodb/pkg/util/constants" + + "github.com/arangodb-helper/go-certificates" + "github.com/arangodb/kube-arangodb/pkg/deployment/resources" "github.com/arangodb/kube-arangodb/pkg/deployment/resources/inspector" + "github.com/arangodb/kube-arangodb/pkg/util" + "github.com/pkg/errors" + core "k8s.io/api/core/v1" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" "github.com/rs/zerolog" ) -// createRotateTLSServerCertificatePlan creates plan to rotate a server because of an (soon to be) expired TLS certificate. -func createRotateTLSServerCertificatePlan(ctx context.Context, - log zerolog.Logger, apiObject k8sutil.APIObject, - spec api.DeploymentSpec, status api.DeploymentStatus, - cachedStatus inspector.Inspector, context PlanBuilderContext) api.Plan { - if !spec.TLS.IsSecure() { - return nil - } - var plan api.Plan - status.Members.ForeachServerGroup(func(group api.ServerGroup, members api.MemberStatusList) error { - for _, m := range members { - if !plan.IsEmpty() { - // Only 1 change at a time - continue - } - if m.Phase != api.MemberPhaseCreated { - // Only make changes when phase is created - continue - } - if group == api.ServerGroupSyncWorkers { - // SyncWorkers have no externally created TLS keyfile - continue - } - // Load keyfile - secret, exists := cachedStatus.Secret(k8sutil.CreateTLSKeyfileSecretName(context.GetName(), group.AsRole(), m.ID)) - if !exists { - log.Warn(). - Str("role", group.AsRole()). - Str("id", m.ID). - Msg("Failed to get TLS secret") - continue - } +const CertificateRenewalMargin = 7 * 24 * time.Hour - keyfile, err := k8sutil.GetTLSKeyfileFromSecret(secret) - if err != nil { - log.Warn().Err(err). - Str("role", group.AsRole()). - Str("id", m.ID). - Msg("Failed to parse TLS secret") - continue - } +type Certificates []*x509.Certificate - tlsSpec := spec.TLS - if group.IsArangosync() { - tlsSpec = spec.Sync.TLS - } - renewalNeeded, reason := tlsKeyfileNeedsRenewal(log, keyfile, tlsSpec) - if renewalNeeded { - plan = append(append(plan, - api.NewAction(api.ActionTypeRenewTLSCertificate, group, m.ID, reason)), - createRotateMemberPlan(log, m, group, "TLS certificate renewal")..., - ) - } +func (c Certificates) Contains(cert *x509.Certificate) bool { + for _, localCert := range c { + if !localCert.Equal(cert) { + return false } - return nil - }) - return plan + } + + return true } -// createRotateTLSCAPlan creates plan to replace a TLS CA and rotate all server. -func createRotateTLSCAPlan(ctx context.Context, - log zerolog.Logger, apiObject k8sutil.APIObject, - spec api.DeploymentSpec, status api.DeploymentStatus, - cachedStatus inspector.Inspector, context PlanBuilderContext) api.Plan { - if !spec.TLS.IsSecure() { - return nil - } - - asOwner := apiObject.AsOwner() - - secretName := spec.TLS.GetCASecretName() - secret, exists := cachedStatus.Secret(secretName) - - if !exists { - log.Warn().Str("secret-name", secretName).Msg("TLS CA secret missing") - return nil - } - - cert, _, isOwned, err := k8sutil.GetCAFromSecret(secret, &asOwner) - if err != nil { - log.Warn().Err(err).Str("secret-name", secretName).Msg("Failed to fetch TLS CA secret") - return nil - } - if !isOwned { - // TLS CA is not owned by the deployment, we cannot change it - return nil - } - var plan api.Plan - if renewalNeeded, reason := tlsCANeedsRenewal(log, cert, spec.TLS); renewalNeeded { - if spec.IsDowntimeAllowed() { - var planSuffix api.Plan - plan = append(plan, - api.NewAction(api.ActionTypeRenewTLSCACertificate, 0, "", reason), - ) - status.Members.ForeachServerGroup(func(group api.ServerGroup, members api.MemberStatusList) error { - for _, m := range members { - if m.Phase != api.MemberPhaseCreated { - // Only make changes when phase is created - continue - } - if !group.IsArangod() { - // Sync master/worker is not applicable here - continue - } - plan = append(plan, - api.NewAction(api.ActionTypeRenewTLSCertificate, group, m.ID), - api.NewAction(api.ActionTypeRotateMember, group, m.ID, "TLS CA certificate changed"), - ) - planSuffix = append(planSuffix, - api.NewAction(api.ActionTypeWaitForMemberUp, group, m.ID, "TLS CA certificate changed"), - ) - } - return nil - }) - plan = append(plan, planSuffix...) - } else { - // Rotating the CA results in downtime. - // That is currently not allowed. - context.CreateEvent(k8sutil.NewDowntimeNotAllowedEvent(apiObject, "Rotate TLS CA")) - } - } - return plan -} - -// tlsKeyfileNeedsRenewal decides if the certificate in the given keyfile -// should be renewed. -func tlsKeyfileNeedsRenewal(log zerolog.Logger, keyfile string, spec api.TLSSpec) (bool, string) { - raw := []byte(keyfile) - // containsAll returns true when all elements in the expected list - // are in the actual list. - containsAll := func(actual []string, expected []string) bool { - for _, x := range expected { - found := false - for _, y := range actual { - if x == y { - found = true - break - } - } - if !found { - return false - } - } +func (c Certificates) ContainsAll(certs Certificates) bool { + if len(certs) == 0 { return true } - ipsToStringSlice := func(list []net.IP) []string { - result := make([]string, len(list)) - for i, x := range list { - result[i] = x.String() - } - return result - } - for { - var derBlock *pem.Block - derBlock, raw = pem.Decode(raw) - if derBlock == nil { - break - } - if derBlock.Type == "CERTIFICATE" { - cert, err := x509.ParseCertificate(derBlock.Bytes) - if err != nil { - // We do not understand the certificate, let's renew it - log.Warn().Err(err).Msg("Failed to parse x509 certificate. Renewing it") - return true, "Cannot parse x509 certificate: " + err.Error() - } - if cert.IsCA { - // Only look at the server certificate, not CA or intermediate - continue - } - // Check expiration date. Renewal at 2/3 of lifetime. - ttl := cert.NotAfter.Sub(cert.NotBefore) - expirationDate := cert.NotBefore.Add((ttl / 3) * 2) - if expirationDate.Before(time.Now()) { - // We should renew now - log.Debug(). - Str("not-before", cert.NotBefore.String()). - Str("not-after", cert.NotAfter.String()). - Str("expiration-date", expirationDate.String()). - Msg("TLS certificate renewal needed") - return true, "Server certificate about to expire" - } - // Check alternate names against spec - dnsNames, ipAddresses, emailAddress, err := spec.GetParsedAltNames() - if err == nil { - if !containsAll(cert.DNSNames, dnsNames) { - return true, "Some alternate DNS names are missing" - } - if !containsAll(ipsToStringSlice(cert.IPAddresses), ipAddresses) { - return true, "Some alternate IP addresses are missing" - } - if !containsAll(cert.EmailAddresses, emailAddress) { - return true, "Some alternate email addresses are missing" - } - } + + for _, cert := range certs { + if !c.Contains(cert) { + return false } } - return false, "" + + return true } -// tlsCANeedsRenewal decides if the given CA certificate -// should be renewed. -// Returns: shouldRenew, reason -func tlsCANeedsRenewal(log zerolog.Logger, cert string, spec api.TLSSpec) (bool, string) { - raw := []byte(cert) - for { - var derBlock *pem.Block - derBlock, raw = pem.Decode(raw) - if derBlock == nil { - break - } - if derBlock.Type == "CERTIFICATE" { - cert, err := x509.ParseCertificate(derBlock.Bytes) - if err != nil { - // We do not understand the certificate, let's renew it - log.Warn().Err(err).Msg("Failed to parse x509 certificate. Renewing it") - return true, "Cannot parse x509 certificate: " + err.Error() - } - if !cert.IsCA { - // Only look at the CA certificate - continue - } - // Check expiration date. Renewal at 90% of lifetime. - ttl := cert.NotAfter.Sub(cert.NotBefore) - expirationDate := cert.NotBefore.Add((ttl / 10) * 9) - if expirationDate.Before(time.Now()) { - // We should renew now - log.Debug(). - Str("not-before", cert.NotBefore.String()). - Str("not-after", cert.NotAfter.String()). - Str("expiration-date", expirationDate.String()). - Msg("TLS CA certificate renewal needed") - return true, "CA Certificate about to expire" - } +func (c Certificates) ToPem() ([]byte, error) { + bytes := bytes.NewBuffer([]byte{}) + + for _, cert := range c { + if err := pem.Encode(bytes, &pem.Block{Type: "CERTIFICATE", Bytes: cert.Raw}); err != nil { + return nil, err } } - return false, "" + + return bytes.Bytes(), nil +} + +func (c Certificates) AsCertPool() *x509.CertPool { + cp := x509.NewCertPool() + + for _, cert := range c { + cp.AddCert(cert) + } + + return cp +} + +func getCertsFromData(log zerolog.Logger, caPem []byte) Certificates { + certs := make([]*x509.Certificate, 0, 2) + + for { + pem, rest := pem.Decode(caPem) + if pem == nil { + break + } + + caPem = rest + + cert, err := x509.ParseCertificate(pem.Bytes) + if err != nil { + // This error should be ignored + log.Error().Err(err).Msg("Unable to parse certificate") + continue + } + + certs = append(certs, cert) + } + + return certs +} + +func getCertsFromSecret(log zerolog.Logger, secret *core.Secret) Certificates { + caPem, exists := secret.Data[core.ServiceAccountRootCAKey] + if !exists { + return nil + } + + return getCertsFromData(log, caPem) +} + +func getKeyCertFromCache(log zerolog.Logger, cachedStatus inspector.Inspector, spec api.DeploymentSpec, certName, keyName string) (Certificates, interface{}, error) { + caSecret, exists := cachedStatus.Secret(spec.TLS.GetCASecretName()) + if !exists { + return nil, nil, errors.Errorf("CA Secret does not exists") + } + + return getKeyCertFromSecret(log, caSecret, keyName, certName) +} + +func getKeyCertFromSecret(log zerolog.Logger, secret *core.Secret, certName, keyName string) (Certificates, interface{}, error) { + ca, exists := secret.Data[certName] + if !exists { + return nil, nil, errors.Errorf("Key %s missing in secret", certName) + } + + key, exists := secret.Data[keyName] + if !exists { + return nil, nil, errors.Errorf("Key %s missing in secret", keyName) + } + + cert, keys, err := certificates.LoadFromPEM(string(ca), string(key)) + if err != nil { + return nil, nil, err + } + + return cert, keys, nil +} + +// createTLSStatusUpdate creates plan to update ca info +func createTLSStatusUpdate(ctx context.Context, + log zerolog.Logger, apiObject k8sutil.APIObject, + spec api.DeploymentSpec, status api.DeploymentStatus, + cachedStatus inspector.Inspector, context PlanBuilderContext) api.Plan { + if !spec.TLS.IsSecure() { + return nil + } + + if createTLSStatusUpdateRequired(ctx, log, apiObject, spec, status, cachedStatus, context) { + return api.Plan{api.NewAction(api.ActionTypeTLSKeyStatusUpdate, api.ServerGroupUnknown, "", "Update status")} + } + + return nil +} + +func createTLSStatusUpdateRequired(ctx context.Context, + log zerolog.Logger, apiObject k8sutil.APIObject, + spec api.DeploymentSpec, status api.DeploymentStatus, + cachedStatus inspector.Inspector, context PlanBuilderContext) bool { + if !spec.TLS.IsSecure() { + return false + } + + trusted, exists := cachedStatus.Secret(resources.GetCASecretName(apiObject)) + if !exists { + log.Warn().Str("secret", resources.GetCASecretName(apiObject)).Msg("Folder with secrets does not exist") + return false + } + + keyHashes := secretKeysToListWithPrefix("sha256:", trusted) + + if len(keyHashes) == 0 { + return false + } + + if len(keyHashes) == 1 { + if status.Hashes.TLS.CA == nil { + return true + } + + if *status.Hashes.TLS.CA != keyHashes[0] { + return true + } + } + + if !util.CompareStringArray(status.Hashes.TLS.Truststore, keyHashes) { + return true + } + + return false +} + +// createCAAppendPlan creates plan to append CA +func createCAAppendPlan(ctx context.Context, + log zerolog.Logger, apiObject k8sutil.APIObject, + spec api.DeploymentSpec, status api.DeploymentStatus, + cachedStatus inspector.Inspector, context PlanBuilderContext) api.Plan { + if !spec.TLS.IsSecure() { + return nil + } + + caSecret, exists := cachedStatus.Secret(spec.TLS.GetCASecretName()) + if !exists { + log.Warn().Str("secret", spec.TLS.GetCASecretName()).Msg("CA Secret does not exists") + return nil + } + + ca, _, err := getKeyCertFromSecret(log, caSecret, resources.CACertName, resources.CAKeyName) + if err != nil { + log.Warn().Err(err).Str("secret", spec.TLS.GetCASecretName()).Msg("CA Secret does not contains Cert") + return nil + } + + if len(ca) == 0 { + log.Warn().Str("secret", spec.TLS.GetCASecretName()).Msg("CA does not contain any certs") + return nil + } + + trusted, exists := cachedStatus.Secret(resources.GetCASecretName(apiObject)) + if !exists { + log.Warn().Str("secret", resources.GetCASecretName(apiObject)).Msg("Folder with secrets does not exist") + return nil + } + + caData, err := ca.ToPem() + if err != nil { + log.Warn().Err(err).Str("secret", spec.TLS.GetCASecretName()).Msg("Unable to parse cert") + return nil + } + + certSha := util.SHA256(caData) + + if _, exists := trusted.Data[certSha]; !exists { + return api.Plan{api.NewAction(api.ActionTypeAppendTLSCACertificate, api.ServerGroupUnknown, "", "Append CA to truststore"). + AddParam(actionTypeAppendTLSCACertificateChecksum, certSha)} + } + + return nil +} + +// createCARenewalPlan creates plan to renew CA +func createCARenewalPlan(ctx context.Context, + log zerolog.Logger, apiObject k8sutil.APIObject, + spec api.DeploymentSpec, status api.DeploymentStatus, + cachedStatus inspector.Inspector, context PlanBuilderContext) api.Plan { + if !spec.TLS.IsSecure() { + return nil + } + + caSecret, exists := cachedStatus.Secret(spec.TLS.GetCASecretName()) + if !exists { + log.Warn().Str("secret", spec.TLS.GetCASecretName()).Msg("CA Secret does not exists") + return nil + } + + if !k8sutil.IsOwner(apiObject.AsOwner(), caSecret) { + log.Warn().Str("secret", spec.TLS.GetCASecretName()).Msg("CA Secret is not owned by Operator, we wont do anything") + return nil + } + + cas, _, err := getKeyCertFromSecret(log, caSecret, resources.CACertName, resources.CAKeyName) + if err != nil { + log.Warn().Err(err).Str("secret", spec.TLS.GetCASecretName()).Msg("CA Secret does not contains Cert") + return nil + } + + for _, ca := range cas { + if time.Now().Add(CertificateRenewalMargin).After(ca.NotAfter) { + // CA will expire soon, renewal needed + return api.Plan{api.NewAction(api.ActionTypeRenewTLSCACertificate, api.ServerGroupUnknown, "", "Renew CA Certificate")} + } + } + + return nil +} + +// createCACleanPlan creates plan to remove old CA's +func createCACleanPlan(ctx context.Context, + log zerolog.Logger, apiObject k8sutil.APIObject, + spec api.DeploymentSpec, status api.DeploymentStatus, + cachedStatus inspector.Inspector, context PlanBuilderContext) api.Plan { + if !spec.TLS.IsSecure() { + return nil + } + + caSecret, exists := cachedStatus.Secret(spec.TLS.GetCASecretName()) + if !exists { + log.Warn().Str("secret", spec.TLS.GetCASecretName()).Msg("CA Secret does not exists") + return nil + } + + ca, _, err := getKeyCertFromSecret(log, caSecret, resources.CACertName, resources.CAKeyName) + if err != nil { + log.Warn().Err(err).Str("secret", spec.TLS.GetCASecretName()).Msg("CA Secret does not contains Cert") + return nil + } + + if len(ca) == 0 { + log.Warn().Str("secret", spec.TLS.GetCASecretName()).Msg("CA does not contain any certs") + return nil + } + + trusted, exists := cachedStatus.Secret(resources.GetCASecretName(apiObject)) + if !exists { + log.Warn().Str("secret", resources.GetCASecretName(apiObject)).Msg("Folder with secrets does not exist") + return nil + } + + caData, err := ca.ToPem() + if err != nil { + log.Warn().Err(err).Str("secret", spec.TLS.GetCASecretName()).Msg("Unable to parse cert") + return nil + } + + certSha := util.SHA256(caData) + + for sha := range trusted.Data { + if certSha != sha { + return api.Plan{api.NewAction(api.ActionTypeCleanTLSCACertificate, api.ServerGroupUnknown, "", "Clean CA from truststore"). + AddParam(actionTypeAppendTLSCACertificateChecksum, sha)} + } + } + + return nil +} + +// createKeyfileRenewalPlan creates plan to renew server keyfile +func createKeyfileRenewalPlan(ctx context.Context, + log zerolog.Logger, apiObject k8sutil.APIObject, + spec api.DeploymentSpec, status api.DeploymentStatus, + cachedStatus inspector.Inspector, context PlanBuilderContext) api.Plan { + if !spec.TLS.IsSecure() { + return nil + } + + var plan api.Plan + + status.Members.ForeachServerGroup(func(group api.ServerGroup, members api.MemberStatusList) error { + if !group.IsArangod() { + return nil + } + + for _, member := range members { + if !plan.IsEmpty() { + return nil + } + + if renew, recreate := keyfileRenewalRequired(ctx, log, apiObject, spec, status, cachedStatus, context, group, member); renew { + log.Info().Msg("Renewal of keyfile required") + plan = append(plan, createKeyfileRotationPlan(log, spec, status, group, member, recreate)...) + } + } + + return nil + }) + + return plan +} + +func createKeyfileRenewalPlanMode( + spec api.DeploymentSpec, status api.DeploymentStatus, + member api.MemberStatus) api.TLSRotateMode { + if !spec.TLS.IsSecure() { + return api.TLSRotateModeRecreate + } + + if spec.TLS.Mode.Get() != api.TLSRotateModeInPlace { + return api.TLSRotateModeRecreate + } + + if i := status.CurrentImage; i == nil { + return api.TLSRotateModeRecreate + } else { + if !i.Enterprise || i.ArangoDBVersion.CompareTo("3.7.0") < 0 || i.ImageID != member.ImageID { + return api.TLSRotateModeRecreate + } + } + + return api.TLSRotateModeInPlace +} + +func createKeyfileRotationPlan(log zerolog.Logger, spec api.DeploymentSpec, status api.DeploymentStatus, group api.ServerGroup, member api.MemberStatus, recreate bool) api.Plan { + p := api.Plan{} + + if recreate { + p = append(p, + api.NewAction(api.ActionTypeCleanTLSKeyfileCertificate, group, member.ID, "Remove server keyfile and enforce renewal")) + } + + switch createKeyfileRenewalPlanMode(spec, status, member) { + case api.TLSRotateModeInPlace: + p = append(p, api.NewAction(api.ActionTypeRefreshTLSKeyfileCertificate, group, member.ID, "Renew Member Keyfile")) + default: + p = append(p, createRotateMemberPlan(log, member, group, "Restart server after keyfile removal")...) + } + return p +} + +func checkServerValidCertRequest(ctx context.Context, apiObject k8sutil.APIObject, group api.ServerGroup, member api.MemberStatus, ca Certificates) (*tls.ConnectionState, error) { + endpoint := fmt.Sprintf("https://%s:%d", k8sutil.CreatePodDNSName(apiObject, group.AsRole(), member.ID), k8sutil.ArangoPort) + + tlsConfig := &tls.Config{ + RootCAs: ca.AsCertPool(), + } + transport := &http.Transport{TLSClientConfig: tlsConfig} + client := &http.Client{Transport: transport, Timeout: time.Second} + + resp, err := client.Get(endpoint) + if err != nil { + return nil, err + } + + return resp.TLS, nil +} + +func keyfileRenewalRequired(ctx context.Context, + log zerolog.Logger, apiObject k8sutil.APIObject, + spec api.DeploymentSpec, status api.DeploymentStatus, + cachedStatus inspector.Inspector, context PlanBuilderContext, + group api.ServerGroup, member api.MemberStatus) (bool, bool) { + if !spec.TLS.IsSecure() { + return false, false + } + + caSecret, exists := cachedStatus.Secret(spec.TLS.GetCASecretName()) + if !exists { + log.Warn().Str("secret", spec.TLS.GetCASecretName()).Msg("CA Secret does not exists") + return false, false + } + + ca, _, err := getKeyCertFromSecret(log, caSecret, resources.CACertName, resources.CAKeyName) + if err != nil { + log.Warn().Err(err).Str("secret", spec.TLS.GetCASecretName()).Msg("CA Secret does not contains Cert") + return false, false + } + + res, err := checkServerValidCertRequest(ctx, apiObject, group, member, ca) + if err != nil { + switch v := err.(type) { + case *url.Error: + switch v.Err.(type) { + case x509.UnknownAuthorityError, x509.CertificateInvalidError: + return true, true + default: + log.Warn().Err(v.Err).Str("type", reflect.TypeOf(v.Err).String()).Msg("Validation of server cert failed") + } + default: + log.Warn().Err(err).Str("type", reflect.TypeOf(err).String()).Msg("Validation of server cert failed") + } + return false, false + } + + // Check if cert is not expired + for _, cert := range res.PeerCertificates { + if cert == nil { + continue + } + + if time.Now().Add(CertificateRenewalMargin).After(cert.NotAfter) { + return true, true + } + } + + // Ensure secret is propagated only on 3.7.0+ enterprise and inplace mode + if createKeyfileRenewalPlanMode(spec, status, member) == api.TLSRotateModeInPlace { + conn, err := context.GetServerClient(ctx, group, member.ID) + if err != nil { + log.Warn().Err(err).Msg("Unable to get client") + return false, false + } + + s, exists := cachedStatus.Secret(k8sutil.CreateTLSKeyfileSecretName(apiObject.GetName(), group.AsRole(), member.ID)) + if !exists { + log.Warn().Msg("Keyfile secret is missing") + return false, false + } + + c := client.NewClient(conn.Connection()) + tls, err := c.GetTLS(ctx) + if err != nil { + log.Warn().Err(err).Msg("Unable to get tls details") + return false, false + } + + keyfile, ok := s.Data[constants.SecretTLSKeyfile] + if !ok { + log.Warn().Msg("Keyfile secret is invalid") + return false, false + } + + keyfileSha := util.SHA256(keyfile) + + if tls.Result.KeyFile.Checksum != keyfileSha { + return true, false + } + } + + return false, false } diff --git a/pkg/deployment/reconcile/plan_builder_tls_sni.go b/pkg/deployment/reconcile/plan_builder_tls_sni.go index 42702f0e3..72e5e078e 100644 --- a/pkg/deployment/reconcile/plan_builder_tls_sni.go +++ b/pkg/deployment/reconcile/plan_builder_tls_sni.go @@ -90,10 +90,10 @@ func createRotateTLSServerSNIPlan(ctx context.Context, return nil } else if !ok { - switch sni.Mode.Get() { - case api.TLSSNIRotateModeRecreate: + switch spec.TLS.Mode.Get() { + case api.TLSRotateModeRecreate: plan = append(plan, createRotateMemberPlan(log, m, group, "SNI Secret needs update")...) - case api.TLSSNIRotateModeInPlace: + case api.TLSRotateModeInPlace: plan = append(plan, api.NewAction(api.ActionTypeUpdateTLSSNI, group, m.ID, "SNI Secret needs update")) default: diff --git a/pkg/deployment/reconcile/plan_executor.go b/pkg/deployment/reconcile/plan_executor.go index d663d5d7b..6e1ddd9b6 100644 --- a/pkg/deployment/reconcile/plan_executor.go +++ b/pkg/deployment/reconcile/plan_executor.go @@ -94,11 +94,8 @@ func (d *Reconciler) ExecutePlan(ctx context.Context, cachedStatus inspector.Ins } } log.Debug().Bool("ready", ready).Msg("Action Start completed") - if !ready { - // We need to check back soon - return true, nil - } - // Continue with next action + + return true, nil } else { // First action of plan has been started, check its progress ready, abort, err := action.CheckProgress(ctx) @@ -154,14 +151,14 @@ func (d *Reconciler) ExecutePlan(ctx context.Context, cachedStatus inspector.Ins // Timeout not yet expired, come back soon return true, nil } - // Continue with next action + return true, nil } } } // createAction create action object based on action type func (d *Reconciler) createAction(ctx context.Context, log zerolog.Logger, action api.Action, cachedStatus inspector.Inspector) Action { - actionCtx := newActionContext(log, d.context, cachedStatus) + actionCtx := newActionContext(log.With().Str("id", action.ID).Str("type", action.Type.String()).Logger(), d.context, cachedStatus) f, ok := getActionFactory(action.Type) if !ok { diff --git a/pkg/deployment/reconcile/timeouts.go b/pkg/deployment/reconcile/timeouts.go index 85307ef6c..8c2b1729a 100644 --- a/pkg/deployment/reconcile/timeouts.go +++ b/pkg/deployment/reconcile/timeouts.go @@ -25,21 +25,22 @@ package reconcile import "time" const ( - addMemberTimeout = time.Minute * 5 - cleanoutMemberTimeout = time.Hour * 12 - removeMemberTimeout = time.Minute * 15 - recreateMemberTimeout = time.Minute * 15 - renewTLSCertificateTimeout = time.Minute * 30 - renewTLSCACertificateTimeout = time.Minute * 30 - rotateMemberTimeout = time.Minute * 15 - pvcResizeTimeout = time.Minute * 15 - pvcResizedTimeout = time.Minute * 15 - backupRestoreTimeout = time.Minute * 15 - shutdownMemberTimeout = time.Minute * 30 - upgradeMemberTimeout = time.Hour * 6 - waitForMemberUpTimeout = time.Minute * 30 - tlsSNIUpdateTimeout = time.Minute * 10 - defaultTimeout = time.Minute * 10 + addMemberTimeout = time.Minute * 5 + cleanoutMemberTimeout = time.Hour * 12 + removeMemberTimeout = time.Minute * 15 + recreateMemberTimeout = time.Minute * 15 + renewTLSCertificateTimeout = time.Minute * 30 + renewTLSCACertificateTimeout = time.Minute * 30 + operationTLSCACertificateTimeout = time.Minute * 30 + rotateMemberTimeout = time.Minute * 15 + pvcResizeTimeout = time.Minute * 15 + pvcResizedTimeout = time.Minute * 15 + backupRestoreTimeout = time.Minute * 15 + shutdownMemberTimeout = time.Minute * 30 + upgradeMemberTimeout = time.Hour * 6 + waitForMemberUpTimeout = time.Minute * 30 + tlsSNIUpdateTimeout = time.Minute * 10 + defaultTimeout = time.Minute * 10 shutdownTimeout = time.Second * 15 ) diff --git a/pkg/deployment/reconcile/utils.go b/pkg/deployment/reconcile/utils.go new file mode 100644 index 000000000..d926b2a5d --- /dev/null +++ b/pkg/deployment/reconcile/utils.go @@ -0,0 +1,46 @@ +// +// DISCLAIMER +// +// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Adam Janikowski +// + +package reconcile + +import ( + "sort" + + "github.com/arangodb/kube-arangodb/pkg/util" + core "k8s.io/api/core/v1" +) + +func secretKeysToListWithPrefix(prefix string, s *core.Secret) []string { + return util.PrefixStringArray(secretKeysToList(s), prefix) +} + +func secretKeysToList(s *core.Secret) []string { + keys := make([]string, 0, len(s.Data)) + + for key := range s.Data { + keys = append(keys, key) + } + + sort.Strings(keys) + + return keys +} diff --git a/pkg/deployment/resources/pod_creator.go b/pkg/deployment/resources/pod_creator.go index a84a4b72c..869f94da5 100644 --- a/pkg/deployment/resources/pod_creator.go +++ b/pkg/deployment/resources/pod_creator.go @@ -54,20 +54,6 @@ func versionHasAdvertisedEndpoint(v driver.Version) bool { return v.CompareTo("3.4.0") >= 0 } -// versionHasJWTSecretKeyfile derives from the version number of arangod has -// the option --auth.jwt-secret-keyfile which can take the JWT secret from -// a file in the file system. -func versionHasJWTSecretKeyfile(v driver.Version) bool { - if v.CompareTo("3.3.22") >= 0 && v.CompareTo("3.4.0") < 0 { - return true - } - if v.CompareTo("3.4.2") >= 0 { - return true - } - - return false -} - // createArangodArgs creates command line arguments for an arangod server in the given group. func createArangodArgs(input pod.Input) []string { options := k8sutil.CreateOptionPairs(64) @@ -81,20 +67,7 @@ func createArangodArgs(input pod.Input) []string { options.Addf("--server.endpoint", "%s://%s:%d", scheme, input.Deployment.GetListenAddr(), k8sutil.ArangoPort) // Authentication - if input.Deployment.IsAuthenticated() { - // With authentication - options.Add("--server.authentication", "true") - - if versionHasJWTSecretKeyfile(input.Version) { - keyPath := filepath.Join(k8sutil.ClusterJWTSecretVolumeMountDir, constants.SecretKeyToken) - options.Add("--server.jwt-secret-keyfile", keyPath) - } else { - options.Addf("--server.jwt-secret", "$(%s)", constants.EnvArangodJWTSecret) - } - } else { - // Without authentication - options.Add("--server.authentication", "false") - } + options.Merge(pod.JWT().Args(input)) // Storage engine options.Add("--server.storage-engine", input.Deployment.GetStorageEngine().AsArangoArgument()) @@ -103,11 +76,7 @@ func createArangodArgs(input pod.Input) []string { options.Add("--log.level", "INFO") // TLS - if input.Deployment.IsSecure() { - keyPath := filepath.Join(k8sutil.TLSKeyfileVolumeMountDir, constants.SecretTLSKeyfile) - options.Add("--ssl.keyfile", keyPath) - options.Add("--ssl.ecdh-curve", "") // This way arangod accepts curves other than P256 as well. - } + options.Merge(pod.TLS().Args(input)) // RocksDB options.Merge(pod.Encryption().Args(input)) @@ -323,40 +292,19 @@ func (r *Resources) RenderPodForMember(cachedStatus inspector.Inspector, spec ap // Render pod if group.IsArangod() { // Prepare arguments - version := imageInfo.ArangoDBVersion autoUpgrade := m.Conditions.IsTrue(api.ConditionTypeAutoUpgrade) - tlsKeyfileSecretName := "" - if spec.IsSecure() { - tlsKeyfileSecretName = k8sutil.CreateTLSKeyfileSecretName(apiObject.GetName(), role, m.ID) - } - - rocksdbEncryptionSecretName := "" - if spec.RocksDB.IsEncrypted() { - rocksdbEncryptionSecretName = spec.RocksDB.Encryption.GetKeySecretName() - } - - var clusterJWTSecretName string - if spec.IsAuthenticated() { - if versionHasJWTSecretKeyfile(version) { - clusterJWTSecretName = spec.Authentication.GetJWTSecretName() - } - } - memberPod := MemberArangoDPod{ - status: m, - tlsKeyfileSecretName: tlsKeyfileSecretName, - rocksdbEncryptionSecretName: rocksdbEncryptionSecretName, - clusterJWTSecretName: clusterJWTSecretName, - groupSpec: groupSpec, - spec: spec, - group: group, - resources: r, - imageInfo: imageInfo, - context: r.context, - autoUpgrade: autoUpgrade, - deploymentStatus: status, - id: memberID, + status: m, + groupSpec: groupSpec, + spec: spec, + group: group, + resources: r, + imageInfo: imageInfo, + context: r.context, + autoUpgrade: autoUpgrade, + deploymentStatus: status, + id: memberID, } input := memberPod.AsInput() @@ -492,20 +440,6 @@ func (r *Resources) createPodForMember(spec api.DeploymentSpec, memberID string, if autoUpgrade { newPhase = api.MemberPhaseUpgrading } - if spec.IsSecure() { - tlsKeyfileSecretName := k8sutil.CreateTLSKeyfileSecretName(apiObject.GetName(), role, m.ID) - serverNames := []string{ - k8sutil.CreateDatabaseClientServiceDNSName(apiObject), - k8sutil.CreatePodDNSName(apiObject, role, m.ID), - } - if ip := spec.ExternalAccess.GetLoadBalancerIP(); ip != "" { - serverNames = append(serverNames, ip) - } - owner := apiObject.AsOwner() - if err := createTLSServerCertificate(log, secrets, serverNames, spec.TLS, tlsKeyfileSecretName, &owner); err != nil && !k8sutil.IsAlreadyExists(err) { - return maskAny(errors.Wrapf(err, "Failed to create TLS keyfile secret")) - } - } uid, checksum, err := CreateArangoPod(kubecli, apiObject, pod) if err != nil { diff --git a/pkg/deployment/resources/pod_creator_arangod.go b/pkg/deployment/resources/pod_creator_arangod.go index 011a16172..b509b24b6 100644 --- a/pkg/deployment/resources/pod_creator_arangod.go +++ b/pkg/deployment/resources/pod_creator_arangod.go @@ -50,22 +50,20 @@ var _ interfaces.PodCreator = &MemberArangoDPod{} var _ interfaces.ContainerCreator = &ArangoDContainer{} type MemberArangoDPod struct { - status api.MemberStatus - tlsKeyfileSecretName string - rocksdbEncryptionSecretName string - clusterJWTSecretName string - groupSpec api.ServerGroupSpec - spec api.DeploymentSpec - deploymentStatus api.DeploymentStatus - group api.ServerGroup - context Context - resources *Resources - imageInfo api.ImageInfo - autoUpgrade bool - id string + status api.MemberStatus + groupSpec api.ServerGroupSpec + spec api.DeploymentSpec + deploymentStatus api.DeploymentStatus + group api.ServerGroup + context Context + resources *Resources + imageInfo api.ImageInfo + autoUpgrade bool + id string } type ArangoDContainer struct { + member *MemberArangoDPod resources *Resources groupSpec api.ServerGroupSpec spec api.DeploymentSpec @@ -141,13 +139,8 @@ func (a *ArangoDContainer) GetImage() string { func (a *ArangoDContainer) GetEnvs() []core.EnvVar { envs := NewEnvBuilder() - if a.spec.IsAuthenticated() { - if !versionHasJWTSecretKeyfile(a.imageInfo.ArangoDBVersion) { - env := k8sutil.CreateEnvSecretKeySelector(constants.EnvArangodJWTSecret, - a.spec.Authentication.GetJWTSecretName(), constants.SecretKeyToken) - - envs.Add(true, env) - } + if env := pod.JWT().Envs(a.member.AsInput()); len(env) > 0 { + envs.Add(true, env...) } if a.spec.License.HasSecretName() { @@ -212,6 +205,7 @@ func (m *MemberArangoDPod) Init(pod *core.Pod) { func (m *MemberArangoDPod) Validate(cachedStatus inspector.Inspector) error { i := m.AsInput() + if err := pod.SNI().Verify(i, cachedStatus); err != nil { return err } @@ -224,6 +218,10 @@ func (m *MemberArangoDPod) Validate(cachedStatus inspector.Inspector) error { return err } + if err := pod.TLS().Verify(i, cachedStatus); err != nil { + return err + } + return nil } @@ -309,42 +307,28 @@ func (m *MemberArangoDPod) GetSidecars(pod *core.Pod) { } func (m *MemberArangoDPod) GetVolumes() ([]core.Volume, []core.VolumeMount) { - var volumes []core.Volume - var volumeMounts []core.VolumeMount + volumes := pod.NewVolumes() - volumeMounts = append(volumeMounts, k8sutil.ArangodVolumeMount()) + volumes.AddVolumeMount(k8sutil.ArangodVolumeMount()) if m.resources.context.GetLifecycleImage() != "" { - volumeMounts = append(volumeMounts, k8sutil.LifecycleVolumeMount()) + volumes.AddVolumeMount(k8sutil.LifecycleVolumeMount()) } if m.status.PersistentVolumeClaimName != "" { vol := k8sutil.CreateVolumeWithPersitantVolumeClaim(k8sutil.ArangodVolumeName, m.status.PersistentVolumeClaimName) - volumes = append(volumes, vol) + volumes.AddVolume(vol) } else { - volumes = append(volumes, k8sutil.CreateVolumeEmptyDir(k8sutil.ArangodVolumeName)) + volumes.AddVolume(k8sutil.CreateVolumeEmptyDir(k8sutil.ArangodVolumeName)) } - if m.tlsKeyfileSecretName != "" { - vol := k8sutil.CreateVolumeWithSecret(k8sutil.TlsKeyfileVolumeName, m.tlsKeyfileSecretName) - volumes = append(volumes, vol) - volumeMounts = append(volumeMounts, k8sutil.TlsKeyfileVolumeMount()) - } + // TLS + volumes.Append(pod.TLS(), m.AsInput()) // Encryption - { - encryptionVolumes, encryptionVolumeMounts := pod.Encryption().Volumes(m.AsInput()) - - if len(encryptionVolumes) > 0 { - volumes = append(volumes, encryptionVolumes...) - } - - if len(encryptionVolumeMounts) > 0 { - volumeMounts = append(volumeMounts, encryptionVolumeMounts...) - } - } + volumes.Append(pod.Encryption(), m.AsInput()) if m.spec.Metrics.IsEnabled() { switch m.spec.Metrics.Mode.Get() { @@ -357,43 +341,29 @@ func (m *MemberArangoDPod) GetVolumes() ([]core.Volume, []core.VolumeMount) { token := m.spec.Metrics.GetJWTTokenSecretName() if token != "" { vol := k8sutil.CreateVolumeWithSecret(k8sutil.ExporterJWTVolumeName, token) - volumes = append(volumes, vol) + volumes.AddVolume(vol) } } } - if m.clusterJWTSecretName != "" { - vol := k8sutil.CreateVolumeWithSecret(k8sutil.ClusterJWTSecretVolumeName, m.clusterJWTSecretName) - volumes = append(volumes, vol) - volumeMounts = append(volumeMounts, k8sutil.ClusterJWTVolumeMount()) - } + volumes.Append(pod.JWT(), m.AsInput()) if m.resources.context.GetLifecycleImage() != "" { - volumes = append(volumes, k8sutil.LifecycleVolume()) + volumes.AddVolume(k8sutil.LifecycleVolume()) } // SNI - { - sniVolumes, sniVolumeMounts := pod.SNI().Volumes(m.AsInput()) - - if len(sniVolumes) > 0 { - volumes = append(volumes, sniVolumes...) - } - - if len(sniVolumeMounts) > 0 { - volumeMounts = append(volumeMounts, sniVolumeMounts...) - } - } + volumes.Append(pod.SNI(), m.AsInput()) if len(m.groupSpec.Volumes) > 0 { - volumes = append(volumes, m.groupSpec.Volumes.Volumes()...) + volumes.AddVolume(m.groupSpec.Volumes.Volumes()...) } if len(m.groupSpec.VolumeMounts) > 0 { - volumeMounts = append(volumeMounts, m.groupSpec.VolumeMounts.VolumeMounts()...) + volumes.AddVolumeMount(m.groupSpec.VolumeMounts.VolumeMounts()...) } - return volumes, volumeMounts + return volumes.Volumes(), volumes.VolumeMounts() } func (m *MemberArangoDPod) IsDeploymentMode() bool { @@ -441,6 +411,7 @@ func (m *MemberArangoDPod) GetTolerations() []core.Toleration { func (m *MemberArangoDPod) GetContainerCreator() interfaces.ContainerCreator { return &ArangoDContainer{ + member: m, spec: m.spec, group: m.group, resources: m.resources, @@ -473,7 +444,7 @@ func (m *MemberArangoDPod) createMetricsExporterSidecar() *core.Container { c.VolumeMounts = append(c.VolumeMounts, k8sutil.ExporterJWTVolumeMount()) } - if m.tlsKeyfileSecretName != "" { + if pod.IsTLSEnabled(m.AsInput()) { c.VolumeMounts = append(c.VolumeMounts, k8sutil.TlsKeyfileVolumeMount()) } diff --git a/pkg/deployment/resources/secret_hashes.go b/pkg/deployment/resources/secret_hashes.go index 3d92ccbbd..0f49b73e1 100644 --- a/pkg/deployment/resources/secret_hashes.go +++ b/pkg/deployment/resources/secret_hashes.go @@ -176,18 +176,6 @@ func (r *Resources) ValidateSecretHashes(cachedStatus inspector.Inspector) error } } } - if spec.IsSecure() { - secretName := spec.TLS.GetCASecretName() - getExpectedHash := func() string { return getHashes().TLSCA } - setExpectedHash := func(h string) error { - return maskAny(updateHashes(func(dst *api.SecretHashes) { dst.TLSCA = h })) - } - if hashOK, err := validate(secretName, getExpectedHash, setExpectedHash, nil); err != nil { - return maskAny(err) - } else if !hashOK { - badSecretNames = append(badSecretNames, secretName) - } - } if spec.Sync.IsEnabled() { secretName := spec.Sync.TLS.GetCASecretName() getExpectedHash := func() string { return getHashes().SyncTLSCA } diff --git a/pkg/deployment/resources/secrets.go b/pkg/deployment/resources/secrets.go index acdd8b205..fa63f4c73 100644 --- a/pkg/deployment/resources/secrets.go +++ b/pkg/deployment/resources/secrets.go @@ -29,6 +29,8 @@ import ( "fmt" "time" + "github.com/rs/zerolog" + operatorErrors "github.com/arangodb/kube-arangodb/pkg/util/errors" "github.com/arangodb/kube-arangodb/pkg/deployment/resources/inspector" @@ -53,15 +55,25 @@ var ( inspectSecretsDurationGauges = metrics.MustRegisterGaugeVec(metricsComponent, "inspect_secrets_duration", "Amount of time taken by a single inspection of all Secrets for a deployment (in sec)", metrics.DeploymentName) ) +const ( + CAKeyName = "ca.key" + CACertName = "ca.crt" +) + +func GetCASecretName(apiObject k8sutil.APIObject) string { + return fmt.Sprintf("%s-truststore", apiObject.GetName()) +} + // EnsureSecrets creates all secrets needed to run the given deployment -func (r *Resources) EnsureSecrets(cachedStatus inspector.Inspector) error { +func (r *Resources) EnsureSecrets(log zerolog.Logger, cachedStatus inspector.Inspector) error { start := time.Now() spec := r.context.GetSpec() kubecli := r.context.GetKubeCli() ns := r.context.GetNamespace() secrets := kubecli.CoreV1().Secrets(ns) status, _ := r.context.GetStatus() - deploymentName := r.context.GetAPIObject().GetName() + apiObject := r.context.GetAPIObject() + deploymentName := apiObject.GetName() defer metrics.SetDuration(inspectSecretsDurationGauges.WithLabelValues(deploymentName), start) counterMetric := inspectedSecretsCounters.WithLabelValues(deploymentName) @@ -82,6 +94,40 @@ func (r *Resources) EnsureSecrets(cachedStatus inspector.Inspector) error { if err := r.ensureTLSCACertificateSecret(cachedStatus, secrets, spec.TLS); err != nil { return maskAny(err) } + + if err := r.ensureSecretWithEmptyKey(cachedStatus, secrets, GetCASecretName(r.context.GetAPIObject()), "empty"); err != nil { + return maskAny(err) + } + + if err := status.Members.ForeachServerGroup(func(group api.ServerGroup, list api.MemberStatusList) error { + if !group.IsArangod() { + return nil + } + + role := group.AsRole() + + for _, m := range list { + tlsKeyfileSecretName := k8sutil.CreateTLSKeyfileSecretName(apiObject.GetName(), role, m.ID) + if _, exists := cachedStatus.Secret(tlsKeyfileSecretName); !exists { + serverNames := []string{ + k8sutil.CreateDatabaseClientServiceDNSName(apiObject), + k8sutil.CreatePodDNSName(apiObject, role, m.ID), + } + if ip := spec.ExternalAccess.GetLoadBalancerIP(); ip != "" { + serverNames = append(serverNames, ip) + } + owner := apiObject.AsOwner() + if err := createTLSServerCertificate(log, secrets, serverNames, spec.TLS, tlsKeyfileSecretName, &owner); err != nil && !k8sutil.IsAlreadyExists(err) { + return maskAny(errors.Wrapf(err, "Failed to create TLS keyfile secret")) + } + + return operatorErrors.Reconcile() + } + } + return nil + }); err != nil { + return maskAny(err) + } } if spec.RocksDB.IsEncrypted() { if i := status.CurrentImage; i != nil && i.Enterprise && i.ArangoDBVersion.CompareTo("3.7.0") >= 0 { @@ -122,6 +168,61 @@ func (r *Resources) ensureTokenSecret(cachedStatus inspector.Inspector, secrets return nil } +func (r *Resources) ensureSecret(cachedStatus inspector.Inspector, secrets k8sutil.SecretInterface, secretName string) error { + if _, exists := cachedStatus.Secret(secretName); !exists { + return r.createSecret(secrets, secretName) + } + + return nil +} + +func (r *Resources) createSecret(secrets k8sutil.SecretInterface, secretName string) error { + // Create secret + secret := &core.Secret{ + ObjectMeta: meta.ObjectMeta{ + Name: secretName, + }, + } + // Attach secret to owner + owner := r.context.GetAPIObject().AsOwner() + k8sutil.AddOwnerRefToObject(secret, &owner) + if _, err := secrets.Create(secret); err != nil { + // Failed to create secret + return maskAny(err) + } + + return operatorErrors.Reconcile() +} + +func (r *Resources) ensureSecretWithEmptyKey(cachedStatus inspector.Inspector, secrets k8sutil.SecretInterface, secretName, keyName string) error { + if _, exists := cachedStatus.Secret(secretName); !exists { + return r.createSecretWithEmptyKey(secrets, secretName, keyName) + } + + return nil +} + +func (r *Resources) createSecretWithEmptyKey(secrets k8sutil.SecretInterface, secretName, keyName string) error { + // Create secret + secret := &core.Secret{ + ObjectMeta: meta.ObjectMeta{ + Name: secretName, + }, + Data: map[string][]byte{ + keyName: {}, + }, + } + // Attach secret to owner + owner := r.context.GetAPIObject().AsOwner() + k8sutil.AddOwnerRefToObject(secret, &owner) + if _, err := secrets.Create(secret); err != nil { + // Failed to create secret + return maskAny(err) + } + + return operatorErrors.Reconcile() +} + func (r *Resources) createTokenSecret(secrets k8sutil.SecretInterface, secretName string) error { tokenData := make([]byte, 32) rand.Read(tokenData) diff --git a/pkg/util/checksum.go b/pkg/util/checksum.go new file mode 100644 index 000000000..c16f45ac3 --- /dev/null +++ b/pkg/util/checksum.go @@ -0,0 +1,32 @@ +// +// DISCLAIMER +// +// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Adam Janikowski +// + +package util + +import ( + "crypto/sha256" + "fmt" +) + +func SHA256(data []byte) string { + return fmt.Sprintf("%0x", sha256.Sum256(data)) +} diff --git a/pkg/util/k8sutil/annotations.go b/pkg/util/k8sutil/annotations.go index 0882a0f99..7e1d0d94d 100644 --- a/pkg/util/k8sutil/annotations.go +++ b/pkg/util/k8sutil/annotations.go @@ -30,6 +30,24 @@ import ( policyTyped "k8s.io/client-go/kubernetes/typed/policy/v1beta1" ) +type OwnerRefObj interface { + GetOwnerReferences() []meta.OwnerReference +} + +func IsOwnerFromRef(owner, ref meta.OwnerReference) bool { + return owner.UID == ref.UID +} + +func IsOwner(ref meta.OwnerReference, object OwnerRefObj) bool { + for _, ownerRef := range object.GetOwnerReferences() { + if IsOwnerFromRef(ref, ownerRef) { + return true + } + } + + return false +} + func IsChildResource(kind, name, namespace string, resource meta.Object) bool { if resource == nil { return false diff --git a/pkg/util/k8sutil/pods.go b/pkg/util/k8sutil/pods.go index 2dbf27a12..3a77c9165 100644 --- a/pkg/util/k8sutil/pods.go +++ b/pkg/util/k8sutil/pods.go @@ -196,6 +196,7 @@ func TlsKeyfileVolumeMount() core.VolumeMount { return core.VolumeMount{ Name: TlsKeyfileVolumeName, MountPath: TLSKeyfileVolumeMountDir, + ReadOnly: true, } }