[Feature] Improved TLS rotation (#577)

CHANGELOG.md

@@ -2,6 +2,8 @@
 
 ## [master](https://github.com/arangodb/kube-arangodb/tree/master) (N/A)
 - Add Encryption Key rotation feature for ArangoDB EE 3.7+
+- Improve TLS CA and Keyfile rotation for CE and EE
+- Add runtime TLS rotation for ArangoDB EE 3.7+
 
 ## [1.0.3](https://github.com/arangodb/kube-arangodb/tree/1.0.3) (2020-05-25)
 - Prevent deletion of not known PVC's

pkg/apis/deployment/v1/deployment_mode.go

@@ -58,6 +58,16 @@ func (m *DeploymentMode) Get() DeploymentMode {
 	return *m
 }
 
+// String return string from mode
+func (m *DeploymentMode) String() string {
+	return string(m.Get())
+}
+
+// Nww return pointer to mode
+func (m DeploymentMode) New() *DeploymentMode {
+	return &m
+}
+
 // HasSingleServers returns true when the given mode is "Single" or "ActiveFailover".
 func (m DeploymentMode) HasSingleServers() bool {
 	return m == DeploymentModeSingle || m == DeploymentModeActiveFailover

pkg/apis/deployment/v1/deployment_status.go

@@ -70,8 +70,8 @@ type DeploymentStatus struct {
 	// detect changes in secret values.
 	SecretHashes *SecretHashes `json:"secret-hashes,omitempty"`
 
-	// CurrentEncryptionKeys keep list of currently applied encryption keys as SHA256 hash
-	CurrentEncryptionKeyHashes DeploymentStatusEncryptionKeyHashes `json:"currentEncryptionKeyHashes,omitempty"`
+	// Hashes keep status of hashes in deployment
+	Hashes DeploymentStatusHashes `json:"hashes,omitempty"`
 
 	// ForceStatusReload if set to true forces a reload of the status from the custom resource.
 	ForceStatusReload *bool `json:"force-status-reload,omitempty"`

pkg/apis/deployment/v1/hashes.go

@@ -0,0 +1,33 @@
+//
+// DISCLAIMER
+//
+// Copyright 2020 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Adam Janikowski
+//
+
+package v1
+
+type DeploymentStatusHashes struct {
+	Encryption DeploymentStatusHashList  `json:"encryption,omitempty"`
+	TLS        DeploymentStatusHashesTLS `json:"tls,omitempty"`
+}
+
+type DeploymentStatusHashesTLS struct {
+	CA         *string                  `json:"ca,omitempty"`
+	Truststore DeploymentStatusHashList `json:"truststore,omitempty"`
+}

pkg/apis/deployment/v1/key_hashes.go

@@ -24,9 +24,9 @@ package v1
 
 import "fmt"
 
-type DeploymentStatusEncryptionKeyHashes []string
+type DeploymentStatusHashList []string
 
-func (d DeploymentStatusEncryptionKeyHashes) Contains(hash string) bool {
+func (d DeploymentStatusHashList) Contains(hash string) bool {
 	if len(d) == 0 {
 		return false
 	}
@@ -40,6 +40,6 @@ func (d DeploymentStatusEncryptionKeyHashes) Contains(hash string) bool {
 	return false
 }
 
-func (d DeploymentStatusEncryptionKeyHashes) ContainsSHA256(hash string) bool {
+func (d DeploymentStatusHashList) ContainsSHA256(hash string) bool {
 	return d.Contains(fmt.Sprintf("sha256:%s", hash))
 }

pkg/apis/deployment/v1/plan.go

@@ -32,6 +32,10 @@ import (
 // ActionType is a strongly typed name for a plan action item
 type ActionType string
 
+func (a ActionType) String() string {
+	return string(a)
+}
+
 const (
 	// ActionTypeIdle causes a plan to be recalculated.
 	ActionTypeIdle ActionType = "Idle"
@@ -61,6 +65,16 @@ const (
 	ActionTypeRenewTLSCertificate ActionType = "RenewTLSCertificate"
 	// ActionTypeRenewTLSCACertificate causes the TLS CA certificate of the entire deployment to be renewed.
 	ActionTypeRenewTLSCACertificate ActionType = "RenewTLSCACertificate"
+	// ActionTypeAppendTLSCACertificate add TLS CA certificate to local truststore.
+	ActionTypeAppendTLSCACertificate ActionType = "AppendTLSCACertificate"
+	// ActionTypeCleanTLSCACertificate clean TLS CA certificate from local truststore.
+	ActionTypeCleanTLSCACertificate ActionType = "CleanTLSCACertificate"
+	// ActionTypeCleanTLSKeyfileCertificate clean server keyfile
+	ActionTypeCleanTLSKeyfileCertificate ActionType = "CleanTLSKeyfileCertificate"
+	// ActionTypeRefreshTLSKeyfileCertificate refresh server keyfile using API
+	ActionTypeRefreshTLSKeyfileCertificate ActionType = "RefreshTLSKeyfileCertificate"
+	// ActionTypeTLSKeyStatusUpdate update status with current data from deployment
+	ActionTypeTLSKeyStatusUpdate ActionType = "TLSKeyStatusUpdate"
 	// ActionTypeUpdateTLSSNI update SNI inplace.
 	ActionTypeUpdateTLSSNI ActionType = "UpdateTLSSNI"
 	// ActionTypeSetCurrentImage causes status.CurrentImage to be updated to the image given in the action.

pkg/apis/deployment/v1/tls_sni_spec.go

@@ -27,25 +27,9 @@ import (
 	"github.com/pkg/errors"
 )
 
-type TLSSNIRotateMode string
-
-func (t *TLSSNIRotateMode) Get() TLSSNIRotateMode {
-	if t == nil {
-		return TLSSNIRotateModeInPlace
-	}
-
-	return *t
-}
-
-const (
-	TLSSNIRotateModeInPlace  TLSSNIRotateMode = "inplace"
-	TLSSNIRotateModeRecreate TLSSNIRotateMode = "recreate"
-)
-
 // TLSSNISpec holds TLS SNI additional certificates
 type TLSSNISpec struct {
 	Mapping map[string][]string `json:"mapping,omitempty"`
-	Mode    *TLSSNIRotateMode   `json:"mode,omitempty"`
 }
 
 func (s TLSSNISpec) Validate() error {

pkg/apis/deployment/v1/tls_spec.go

@@ -31,6 +31,25 @@ import (
 	"github.com/arangodb/kube-arangodb/pkg/util/validation"
 )
 
+type TLSRotateMode string
+
+func (t *TLSRotateMode) Get() TLSRotateMode {
+	if t == nil {
+		return TLSRotateModeInPlace
+	}
+
+	return *t
+}
+
+func (t TLSRotateMode) New() *TLSRotateMode {
+	return &t
+}
+
+const (
+	TLSRotateModeInPlace  TLSRotateMode = "inplace"
+	TLSRotateModeRecreate TLSRotateMode = "recreate"
+)
+
 const (
 	defaultTLSTTL = Duration("2610h") // About 3 month
 )
@@ -41,6 +60,7 @@ type TLSSpec struct {
 	AltNames     []string       `json:"altNames,omitempty"`
 	TTL          *Duration      `json:"ttl,omitempty"`
 	SNI          *TLSSNISpec    `json:"sni,omitempty"`
+	Mode         *TLSRotateMode `json:"mode,omitempty"`
 }
 
 const (

pkg/apis/deployment/v1/zz_generated.deepcopy.go

@@ -399,11 +399,7 @@ func (in *DeploymentStatus) DeepCopyInto(out *DeploymentStatus) {
 		*out = new(SecretHashes)
 		(*in).DeepCopyInto(*out)
 	}
-	if in.CurrentEncryptionKeyHashes != nil {
-		in, out := &in.CurrentEncryptionKeyHashes, &out.CurrentEncryptionKeyHashes
-		*out = make(DeploymentStatusEncryptionKeyHashes, len(*in))
-		copy(*out, *in)
-	}
+	in.Hashes.DeepCopyInto(&out.Hashes)
 	if in.ForceStatusReload != nil {
 		in, out := &in.ForceStatusReload, &out.ForceStatusReload
 		*out = new(bool)
@@ -423,25 +419,73 @@ func (in *DeploymentStatus) DeepCopy() *DeploymentStatus {
 }
 
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in DeploymentStatusEncryptionKeyHashes) DeepCopyInto(out *DeploymentStatusEncryptionKeyHashes) {
+func (in DeploymentStatusHashList) DeepCopyInto(out *DeploymentStatusHashList) {
 	{
 		in := &in
-		*out = make(DeploymentStatusEncryptionKeyHashes, len(*in))
+		*out = make(DeploymentStatusHashList, len(*in))
 		copy(*out, *in)
 		return
 	}
 }
 
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeploymentStatusEncryptionKeyHashes.
-func (in DeploymentStatusEncryptionKeyHashes) DeepCopy() DeploymentStatusEncryptionKeyHashes {
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeploymentStatusHashList.
+func (in DeploymentStatusHashList) DeepCopy() DeploymentStatusHashList {
 	if in == nil {
 		return nil
 	}
-	out := new(DeploymentStatusEncryptionKeyHashes)
+	out := new(DeploymentStatusHashList)
 	in.DeepCopyInto(out)
 	return *out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *DeploymentStatusHashes) DeepCopyInto(out *DeploymentStatusHashes) {
+	*out = *in
+	if in.Encryption != nil {
+		in, out := &in.Encryption, &out.Encryption
+		*out = make(DeploymentStatusHashList, len(*in))
+		copy(*out, *in)
+	}
+	in.TLS.DeepCopyInto(&out.TLS)
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeploymentStatusHashes.
+func (in *DeploymentStatusHashes) DeepCopy() *DeploymentStatusHashes {
+	if in == nil {
+		return nil
+	}
+	out := new(DeploymentStatusHashes)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *DeploymentStatusHashesTLS) DeepCopyInto(out *DeploymentStatusHashesTLS) {
+	*out = *in
+	if in.CA != nil {
+		in, out := &in.CA, &out.CA
+		*out = new(string)
+		**out = **in
+	}
+	if in.Truststore != nil {
+		in, out := &in.Truststore, &out.Truststore
+		*out = make(DeploymentStatusHashList, len(*in))
+		copy(*out, *in)
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeploymentStatusHashesTLS.
+func (in *DeploymentStatusHashesTLS) DeepCopy() *DeploymentStatusHashesTLS {
+	if in == nil {
+		return nil
+	}
+	out := new(DeploymentStatusHashesTLS)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *DeploymentStatusMembers) DeepCopyInto(out *DeploymentStatusMembers) {
 	*out = *in
@@ -1361,11 +1405,6 @@ func (in *TLSSNISpec) DeepCopyInto(out *TLSSNISpec) {
 			(*out)[key] = outVal
 		}
 	}
-	if in.Mode != nil {
-		in, out := &in.Mode, &out.Mode
-		*out = new(TLSSNIRotateMode)
-		**out = **in
-	}
 	return
 }
 
@@ -1402,6 +1441,11 @@ func (in *TLSSpec) DeepCopyInto(out *TLSSpec) {
 		*out = new(TLSSNISpec)
 		(*in).DeepCopyInto(*out)
 	}
+	if in.Mode != nil {
+		in, out := &in.Mode, &out.Mode
+		*out = new(TLSRotateMode)
+		**out = **in
+	}
 	return
 }
 

pkg/deployment/deployment_inspector.go

@@ -150,7 +150,7 @@ func (d *Deployment) inspectDeploymentWithError(ctx context.Context, lastInterva
 		}
 	}
 
-	if err := d.resources.EnsureSecrets(cachedStatus); err != nil {
+	if err := d.resources.EnsureSecrets(d.deps.Log, cachedStatus); err != nil {
 		return minInspectionInterval, errors.Wrapf(err, "Secret creation failed")
 	}
 

pkg/deployment/deployment_run_test.go

@@ -27,6 +27,8 @@ import (
 	"fmt"
 	"testing"
 
+	"github.com/rs/zerolog/log"
+
 	"github.com/arangodb/kube-arangodb/pkg/deployment/resources/inspector"
 	"github.com/arangodb/kube-arangodb/pkg/util/errors"
 
@@ -60,12 +62,12 @@ func runTestCase(t *testing.T, testCase testCaseStruct) {
 		for {
 			cache, err := inspector.NewInspector(d.GetKubeCli(), d.GetNamespace())
 			require.NoError(t, err)
-			err = d.resources.EnsureSecrets(cache)
+			err = d.resources.EnsureSecrets(log.Logger, cache)
 			if err == nil {
 				break
 			}
 
-			if errs > 5 {
+			if errs > 25 {
 				require.NoError(t, err)
 			}
 

pkg/deployment/pod/builder.go

@@ -46,5 +46,6 @@ type Input struct {
 type Builder interface {
 	Args(i Input) k8sutil.OptionPairs
 	Volumes(i Input) ([]core.Volume, []core.VolumeMount)
+	Envs(i Input) []core.EnvVar
 	Verify(i Input, cachedStatus inspector.Inspector) error
 }

pkg/deployment/pod/encryption.go

@@ -115,6 +115,10 @@ func Encryption() Builder {
 
 type encryption struct{}
 
+func (e encryption) Envs(i Input) []core.EnvVar {
+	return nil
+}
+
 func (e encryption) Args(i Input) k8sutil.OptionPairs {
 	if !IsEncryptionEnabled(i) {
 		return nil

pkg/deployment/pod/jwt.go

@@ -23,28 +23,85 @@
 package pod
 
 import (
+	"path/filepath"
+
+	"github.com/arangodb/go-driver"
 	"github.com/arangodb/kube-arangodb/pkg/deployment/resources/inspector"
+	"github.com/arangodb/kube-arangodb/pkg/util/constants"
 	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
 	"github.com/pkg/errors"
 	core "k8s.io/api/core/v1"
 )
 
+func IsAuthenticated(i Input) bool {
+	return i.Deployment.IsAuthenticated()
+}
+
+func VersionHasJWTSecretKeyfile(v driver.Version) bool {
+	if v.CompareTo("3.3.22") >= 0 && v.CompareTo("3.4.0") < 0 {
+		return true
+	}
+	if v.CompareTo("3.4.2") >= 0 {
+		return true
+	}
+
+	return false
+}
+
+func VersionHasJWTSecretKeyfolder(i Input) bool {
+	return i.Enterprise && i.Version.CompareTo("3.7.0") > 0
+}
+
 func JWT() Builder {
 	return jwt{}
 }
 
 type jwt struct{}
 
-func (e jwt) Args(i Input) k8sutil.OptionPairs {
+func (e jwt) Envs(i Input) []core.EnvVar {
+	if !IsAuthenticated(i) {
+		return nil
+	}
+
+	if !VersionHasJWTSecretKeyfile(i.Version) {
+		return []core.EnvVar{k8sutil.CreateEnvSecretKeySelector(constants.EnvArangodJWTSecret,
+			i.Deployment.Authentication.GetJWTSecretName(), constants.SecretKeyToken)}
+	}
+
 	return nil
 }
 
+func (e jwt) Args(i Input) k8sutil.OptionPairs {
+	if !IsAuthenticated(i) {
+		// Without authentication
+		return k8sutil.NewOptionPair(k8sutil.OptionPair{Key: "--server.authentication", Value: "false"})
+	}
+
+	options := k8sutil.CreateOptionPairs(2)
+
+	options.Add("--server.authentication", "true")
+
+	if VersionHasJWTSecretKeyfile(i.Version) {
+		keyPath := filepath.Join(k8sutil.ClusterJWTSecretVolumeMountDir, constants.SecretKeyToken)
+		options.Add("--server.jwt-secret-keyfile", keyPath)
+	} else {
+		options.Addf("--server.jwt-secret", "$(%s)", constants.EnvArangodJWTSecret)
+	}
+
+	return options
+}
+
 func (e jwt) Volumes(i Input) ([]core.Volume, []core.VolumeMount) {
+	if !IsAuthenticated(i) {
 		return nil, nil
+	}
+
+	vol := k8sutil.CreateVolumeWithSecret(k8sutil.ClusterJWTSecretVolumeName, i.Deployment.Authentication.GetJWTSecretName())
+	return []core.Volume{vol}, []core.VolumeMount{k8sutil.ClusterJWTVolumeMount()}
 }
 
 func (e jwt) Verify(i Input, cachedStatus inspector.Inspector) error {
-	if !i.Deployment.IsAuthenticated() {
+	if !IsAuthenticated(i) {
 		return nil
 	}
 

pkg/deployment/pod/sni.go

@@ -57,6 +57,10 @@ func SNI() Builder {
 
 type sni struct{}
 
+func (s sni) Envs(i Input) []core.EnvVar {
+	return nil
+}
+
 func (s sni) isSupported(i Input) bool {
 	if !i.Deployment.TLS.IsSecure() {
 		return false

pkg/deployment/pod/tls.go

@@ -0,0 +1,90 @@
+//
+// DISCLAIMER
+//
+// Copyright 2020 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Adam Janikowski
+//
+
+package pod
+
+import (
+	"path/filepath"
+
+	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+
+	"github.com/arangodb/kube-arangodb/pkg/deployment/resources/inspector"
+
+	"github.com/arangodb/kube-arangodb/pkg/util/constants"
+	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
+	core "k8s.io/api/core/v1"
+)
+
+func IsRuntimeTLSKeyfileUpdateSupported(i Input) bool {
+	return IsTLSEnabled(i) && i.Enterprise &&
+		i.Version.CompareTo("3.7.0") >= 0 &&
+		i.Deployment.TLS.Mode.Get() == api.TLSRotateModeInPlace
+}
+
+func IsTLSEnabled(i Input) bool {
+	return i.Deployment.TLS.IsSecure()
+}
+
+func GetTLSKeyfileSecretName(i Input) string {
+	return k8sutil.CreateTLSKeyfileSecretName(i.ApiObject.GetName(), i.Group.AsRole(), i.ID)
+}
+
+func TLS() Builder {
+	return tls{}
+}
+
+type tls struct{}
+
+func (s tls) Envs(i Input) []core.EnvVar {
+	return nil
+}
+
+func (s tls) Verify(i Input, cachedStatus inspector.Inspector) error {
+	if !IsTLSEnabled(i) {
+		return nil
+	}
+
+	return nil
+}
+
+func (s tls) Volumes(i Input) ([]core.Volume, []core.VolumeMount) {
+	if !IsTLSEnabled(i) {
+		return nil, nil
+	}
+
+	return []core.Volume{k8sutil.CreateVolumeWithSecret(k8sutil.TlsKeyfileVolumeName, GetTLSKeyfileSecretName(i))},
+		[]core.VolumeMount{k8sutil.TlsKeyfileVolumeMount()}
+}
+
+func (s tls) Args(i Input) k8sutil.OptionPairs {
+	if !IsTLSEnabled(i) {
+		return nil
+	}
+
+	opts := k8sutil.CreateOptionPairs()
+
+	keyPath := filepath.Join(k8sutil.TLSKeyfileVolumeMountDir, constants.SecretTLSKeyfile)
+	opts.Add("--ssl.keyfile", keyPath)
+	opts.Add("--ssl.ecdh-curve", "") // This way arangod accepts curves other than P256 as well.
+
+	return opts
+}

pkg/deployment/pod/upgrade.go

@@ -35,6 +35,10 @@ func AutoUpgrade() Builder {
 
 type autoUpgrade struct{}
 
+func (u autoUpgrade) Envs(i Input) []core.EnvVar {
+	return nil
+}
+
 func (u autoUpgrade) Verify(i Input, cachedStatus inspector.Inspector) error {
 	return nil
 }

pkg/deployment/pod/volumes.go

@@ -0,0 +1,89 @@
+//
+// DISCLAIMER
+//
+// Copyright 2020 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Adam Janikowski
+//
+package pod
+
+import core "k8s.io/api/core/v1"
+
+func NewVolumes() Volumes {
+	return &volumes{
+		volumes:      []core.Volume{},
+		volumeMounts: []core.VolumeMount{},
+	}
+}
+
+type Volumes interface {
+	Append(b Builder, i Input)
+
+	AddVolume(volumes ...core.Volume)
+	AddVolumes(volumes []core.Volume)
+
+	AddVolumeMount(mounts ...core.VolumeMount)
+	AddVolumeMounts(mounts []core.VolumeMount)
+
+	Volumes() []core.Volume
+	VolumeMounts() []core.VolumeMount
+}
+
+var _ Volumes = &volumes{}
+
+type volumes struct {
+	volumes      []core.Volume
+	volumeMounts []core.VolumeMount
+}
+
+func (v *volumes) Append(b Builder, i Input) {
+	vols, mounts := b.Volumes(i)
+	v.AddVolumes(vols)
+	v.AddVolumeMounts(mounts)
+}
+
+func (v *volumes) AddVolume(volumes ...core.Volume) {
+	v.AddVolumes(volumes)
+}
+
+func (v *volumes) AddVolumes(volumes []core.Volume) {
+	if len(volumes) == 0 {
+		return
+	}
+
+	v.volumes = append(v.volumes, volumes...)
+}
+
+func (v *volumes) AddVolumeMount(mounts ...core.VolumeMount) {
+	v.AddVolumeMounts(mounts)
+}
+
+func (v *volumes) AddVolumeMounts(mounts []core.VolumeMount) {
+	if len(mounts) == 0 {
+		return
+	}
+
+	v.volumeMounts = append(v.volumeMounts, mounts...)
+}
+
+func (v *volumes) Volumes() []core.Volume {
+	return v.volumes
+}
+
+func (v *volumes) VolumeMounts() []core.VolumeMount {
+	return v.volumeMounts
+}

pkg/deployment/reconcile/action_encryption_status_update.go

@@ -24,7 +24,6 @@ package reconcile
 
 import (
 	"context"
-	"sort"
 
 	"github.com/arangodb/kube-arangodb/pkg/util"
 	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -64,28 +63,20 @@ func (a *encryptionKeyStatusUpdateAction) Start(ctx context.Context) (bool, erro
 		return true, nil
 	}
 
-	keys := make([]string, 0, len(f.Data))
-
-	for key := range f.Data {
-		keys = append(keys, key)
-	}
-
-	sort.Strings(keys)
-
-	keyHashes := util.PrefixStringArray(keys, "sha256:")
+	keyHashes := secretKeysToListWithPrefix("sha256:", f)
 
 	if err = a.actionCtx.WithStatusUpdate(func(s *api.DeploymentStatus) bool {
 		if len(keyHashes) == 0 {
-			if s.CurrentEncryptionKeyHashes != nil {
-				s.CurrentEncryptionKeyHashes = nil
+			if s.Hashes.Encryption != nil {
+				s.Hashes.Encryption = nil
 				return true
 			}
 
 			return false
 		}
 
-		if !util.CompareStringArray(keyHashes, s.CurrentEncryptionKeyHashes) {
-			s.CurrentEncryptionKeyHashes = keyHashes
+		if !util.CompareStringArray(keyHashes, s.Hashes.Encryption) {
+			s.Hashes.Encryption = keyHashes
 			return true
 		}
 		return false

pkg/deployment/reconcile/action_renew_tls_ca_certificate.go

@@ -1,82 +0,0 @@
-//
-// DISCLAIMER
-//
-// Copyright 2020 ArangoDB GmbH, Cologne, Germany
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-// Copyright holder is ArangoDB GmbH, Cologne, Germany
-//
-// Author Ewout Prangsma
-//
-
-package reconcile
-
-import (
-	"context"
-	"time"
-
-	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
-	"github.com/rs/zerolog"
-)
-
-func init() {
-	registerAction(api.ActionTypeRenewTLSCACertificate, newRenewTLSCACertificateAction)
-}
-
-// newRenewTLSCACertificateAction creates a new Action that implements the given
-// planned RenewTLSCACertificate action.
-func newRenewTLSCACertificateAction(log zerolog.Logger, action api.Action, actionCtx ActionContext) Action {
-	a := &renewTLSCACertificateAction{}
-
-	a.actionImpl = newActionImplDefRef(log, action, actionCtx, renewTLSCACertificateTimeout)
-
-	return a
-}
-
-// renewTLSCACertificateAction implements a RenewTLSCACertificate action.
-type renewTLSCACertificateAction struct {
-	// actionImpl implement timeout and member id functions
-	actionImpl
-
-	// actionEmptyCheckProgress implement check progress with empty implementation
-	actionEmptyCheckProgress
-}
-
-// Start performs the start of the action.
-// Returns true if the action is completely finished, false in case
-// the start time needs to be recorded and a ready condition needs to be checked.
-func (a *renewTLSCACertificateAction) Start(ctx context.Context) (bool, error) {
-	// Just delete the secret.
-	// It will be re-created.
-	if err := a.actionCtx.DeleteTLSCASecret(); err != nil {
-		return false, maskAny(err)
-	}
-	return true, nil
-}
-
-// CheckProgress checks the progress of the action.
-// Returns true if the action is completely finished, false otherwise.
-func (a *renewTLSCACertificateAction) CheckProgress(ctx context.Context) (bool, bool, error) {
-	return true, false, nil
-}
-
-// Timeout returns the amount of time after which this action will timeout.
-func (a *renewTLSCACertificateAction) Timeout() time.Duration {
-	return renewTLSCACertificateTimeout
-}
-
-// Return the MemberID used / created in this action
-func (a *renewTLSCACertificateAction) MemberID() string {
-	return a.action.MemberID
-}

pkg/deployment/reconcile/action_renew_tls_certificate.go

@@ -1,71 +0,0 @@
-//
-// DISCLAIMER
-//
-// Copyright 2020 ArangoDB GmbH, Cologne, Germany
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-// Copyright holder is ArangoDB GmbH, Cologne, Germany
-//
-// Author Ewout Prangsma
-//
-
-package reconcile
-
-import (
-	"context"
-
-	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
-	"github.com/rs/zerolog"
-)
-
-func init() {
-	registerAction(api.ActionTypeRenewTLSCertificate, newRenewTLSCertificateAction)
-}
-
-// newRenewTLSCertificateAction creates a new Action that implements the given
-// planned RenewTLSCertificate action.
-func newRenewTLSCertificateAction(log zerolog.Logger, action api.Action, actionCtx ActionContext) Action {
-	a := &renewTLSCertificateAction{}
-
-	a.actionImpl = newActionImplDefRef(log, action, actionCtx, renewTLSCertificateTimeout)
-
-	return a
-}
-
-// renewTLSCertificateAction implements a RenewTLSCertificate action.
-type renewTLSCertificateAction struct {
-	// actionImpl implement timeout and member id functions
-	actionImpl
-
-	// actionEmptyCheckProgress implement check progress with empty implementation
-	actionEmptyCheckProgress
-}
-
-// Start performs the start of the action.
-// Returns true if the action is completely finished, false in case
-// the start time needs to be recorded and a ready condition needs to be checked.
-func (a *renewTLSCertificateAction) Start(ctx context.Context) (bool, error) {
-	log := a.log
-	group := a.action.Group
-	m, ok := a.actionCtx.GetMemberStatusByID(a.action.MemberID)
-	if !ok {
-		log.Error().Msg("No such member")
-	}
-	// Just delete the secret.
-	// It will be re-created when the member restarts.
-	if err := a.actionCtx.DeleteTLSKeyfile(group, m); err != nil {
-		return false, maskAny(err)
-	}
-	return false, nil
-}

pkg/deployment/reconcile/action_tls_ca_append.go

@@ -0,0 +1,126 @@
+//
+// DISCLAIMER
+//
+// Copyright 2020 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Adam Janikowski
+//
+
+package reconcile
+
+import (
+	"context"
+	"encoding/base64"
+
+	"github.com/arangodb/kube-arangodb/pkg/deployment/patch"
+	"github.com/arangodb/kube-arangodb/pkg/util"
+	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
+	"github.com/pkg/errors"
+	"k8s.io/apimachinery/pkg/types"
+
+	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+	"github.com/arangodb/kube-arangodb/pkg/deployment/resources"
+	"github.com/rs/zerolog"
+)
+
+const (
+	actionTypeAppendTLSCACertificateChecksum = "checksum"
+)
+
+func init() {
+	registerAction(api.ActionTypeAppendTLSCACertificate, newAppendTLSCACertificateAction)
+}
+
+func newAppendTLSCACertificateAction(log zerolog.Logger, action api.Action, actionCtx ActionContext) Action {
+	a := &appendTLSCACertificateAction{}
+
+	a.actionImpl = newActionImplDefRef(log, action, actionCtx, operationTLSCACertificateTimeout)
+
+	return a
+}
+
+type appendTLSCACertificateAction struct {
+	actionImpl
+
+	actionEmptyCheckProgress
+}
+
+func (a *appendTLSCACertificateAction) Start(ctx context.Context) (bool, error) {
+	if !a.actionCtx.GetSpec().TLS.IsSecure() {
+		return true, nil
+	}
+
+	certChecksum, exists := a.action.Params[actionTypeAppendTLSCACertificateChecksum]
+	if !exists {
+		a.log.Warn().Msgf("Key %s is missing in action", actionTypeAppendTLSCACertificateChecksum)
+		return true, nil
+	}
+
+	caSecret, exists := a.actionCtx.GetCachedStatus().Secret(a.actionCtx.GetSpec().TLS.GetCASecretName())
+	if !exists {
+		a.log.Warn().Msgf("Secret %s is missing", a.actionCtx.GetSpec().TLS.GetCASecretName())
+		return true, nil
+	}
+
+	caFolder, exists := a.actionCtx.GetCachedStatus().Secret(resources.GetCASecretName(a.actionCtx.GetAPIObject()))
+	if !exists {
+		a.log.Warn().Msgf("Secret %s is missing", resources.GetCASecretName(a.actionCtx.GetAPIObject()))
+		return true, nil
+	}
+
+	ca, _, err := getKeyCertFromSecret(a.log, caSecret, resources.CACertName, resources.CAKeyName)
+	if err != nil {
+		a.log.Warn().Err(err).Msgf("Cert %s is invalid", resources.GetCASecretName(a.actionCtx.GetAPIObject()))
+		return true, nil
+	}
+
+	caData, err := ca.ToPem()
+	if err != nil {
+		a.log.Warn().Err(err).Str("secret", resources.GetCASecretName(a.actionCtx.GetAPIObject())).Msgf("Unable to parse ca into pem")
+		return true, nil
+	}
+
+	caSha := util.SHA256(caData)
+
+	if caSha != certChecksum {
+		a.log.Warn().Msgf("Cert changed")
+		return true, nil
+	}
+
+	if _, exists := caFolder.Data[caSha]; exists {
+		a.log.Warn().Msgf("Cert already exists")
+		return true, nil
+	}
+
+	p := patch.NewPatch()
+	p.ItemAdd(patch.NewPath("data", caSha), base64.StdEncoding.EncodeToString(caData))
+
+	patch, err := p.Marshal()
+	if err != nil {
+		a.log.Error().Err(err).Msgf("Unable to encrypt patch")
+		return true, nil
+	}
+
+	_, err = a.actionCtx.SecretsInterface().Patch(resources.GetCASecretName(a.actionCtx.GetAPIObject()), types.JSONPatchType, patch)
+	if err != nil {
+		if !k8sutil.IsInvalid(err) {
+			return false, errors.Wrapf(err, "Unable to update secret: %s", string(patch))
+		}
+	}
+
+	return true, nil
+}

pkg/deployment/reconcile/action_tls_ca_clean.go

@@ -0,0 +1,124 @@
+//
+// DISCLAIMER
+//
+// Copyright 2020 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Adam Janikowski
+//
+
+package reconcile
+
+import (
+	"context"
+
+	"github.com/arangodb/kube-arangodb/pkg/deployment/patch"
+	"github.com/arangodb/kube-arangodb/pkg/util"
+	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
+	"github.com/pkg/errors"
+	"k8s.io/apimachinery/pkg/types"
+
+	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+	"github.com/arangodb/kube-arangodb/pkg/deployment/resources"
+	"github.com/rs/zerolog"
+)
+
+func init() {
+	registerAction(api.ActionTypeCleanTLSCACertificate, newCleanTLSCACertificateAction)
+}
+
+func newCleanTLSCACertificateAction(log zerolog.Logger, action api.Action, actionCtx ActionContext) Action {
+	a := &cleanTLSCACertificateAction{}
+
+	a.actionImpl = newActionImplDefRef(log, action, actionCtx, operationTLSCACertificateTimeout)
+
+	return a
+}
+
+type cleanTLSCACertificateAction struct {
+	actionImpl
+
+	actionEmptyCheckProgress
+}
+
+func (a *cleanTLSCACertificateAction) Start(ctx context.Context) (bool, error) {
+	a.log.Info().Msgf("Clean TLS Ca")
+	if !a.actionCtx.GetSpec().TLS.IsSecure() {
+		a.log.Info().Msgf("Insecure deployment")
+		return true, nil
+	}
+
+	certChecksum, exists := a.action.Params[actionTypeAppendTLSCACertificateChecksum]
+	if !exists {
+		a.log.Warn().Msgf("Key %s is missing in action", actionTypeAppendTLSCACertificateChecksum)
+		return true, nil
+	}
+
+	caSecret, exists := a.actionCtx.GetCachedStatus().Secret(a.actionCtx.GetSpec().TLS.GetCASecretName())
+	if !exists {
+		a.log.Warn().Msgf("Secret %s is missing", a.actionCtx.GetSpec().TLS.GetCASecretName())
+		return true, nil
+	}
+
+	caFolder, exists := a.actionCtx.GetCachedStatus().Secret(resources.GetCASecretName(a.actionCtx.GetAPIObject()))
+	if !exists {
+		a.log.Warn().Msgf("Secret %s is missing", resources.GetCASecretName(a.actionCtx.GetAPIObject()))
+		return true, nil
+	}
+
+	ca, _, err := getKeyCertFromSecret(a.log, caSecret, resources.CACertName, resources.CAKeyName)
+	if err != nil {
+		a.log.Warn().Err(err).Msgf("Cert %s is invalid", resources.GetCASecretName(a.actionCtx.GetAPIObject()))
+		return true, nil
+	}
+
+	caData, err := ca.ToPem()
+	if err != nil {
+		a.log.Warn().Err(err).Str("secret", resources.GetCASecretName(a.actionCtx.GetAPIObject())).Msgf("Unable to parse ca into pem")
+		return true, nil
+	}
+
+	caSha := util.SHA256(caData)
+
+	if caSha == certChecksum {
+		a.log.Warn().Msgf("Unable to remove current ca")
+		return true, nil
+	}
+
+	if _, exists := caFolder.Data[certChecksum]; !exists {
+		a.log.Warn().Msgf("Cert missing")
+		return true, nil
+	}
+
+	p := patch.NewPatch()
+	p.ItemRemove(patch.NewPath("data", certChecksum))
+
+	patch, err := p.Marshal()
+	if err != nil {
+		a.log.Error().Err(err).Msgf("Unable to encrypt patch")
+		return true, nil
+	}
+
+	a.log.Info().Msgf("Removing key %s from truststore", certChecksum)
+	_, err = a.actionCtx.SecretsInterface().Patch(resources.GetCASecretName(a.actionCtx.GetAPIObject()), types.JSONPatchType, patch)
+	if err != nil {
+		if !k8sutil.IsInvalid(err) {
+			return false, errors.Wrapf(err, "Unable to update secret: %s", string(patch))
+		}
+	}
+
+	return true, nil
+}

pkg/deployment/reconcile/action_tls_ca_renew.go

@@ -0,0 +1,66 @@
+//
+// DISCLAIMER
+//
+// Copyright 2020 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Adam Janikowski
+//
+
+package reconcile
+
+import (
+	"context"
+
+	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
+	"github.com/rs/zerolog"
+	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func init() {
+	registerAction(api.ActionTypeRenewTLSCACertificate, newRenewTLSCACertificateAction)
+}
+
+func newRenewTLSCACertificateAction(log zerolog.Logger, action api.Action, actionCtx ActionContext) Action {
+	a := &renewTLSCACertificateAction{}
+
+	a.actionImpl = newActionImplDefRef(log, action, actionCtx, operationTLSCACertificateTimeout)
+
+	return a
+}
+
+type renewTLSCACertificateAction struct {
+	actionImpl
+
+	actionEmptyCheckProgress
+}
+
+func (a *renewTLSCACertificateAction) Start(ctx context.Context) (bool, error) {
+	if !a.actionCtx.GetSpec().TLS.IsSecure() {
+		return true, nil
+	}
+
+	s := a.actionCtx.SecretsInterface()
+	if err := s.Delete(a.actionCtx.GetSpec().TLS.GetCASecretName(), &meta.DeleteOptions{}); err != nil {
+		if !k8sutil.IsNotFound(err) {
+			a.log.Warn().Err(err).Msgf("Unable to clean cert %s", a.actionCtx.GetSpec().TLS.GetCASecretName())
+			return true, nil
+		}
+	}
+
+	return true, nil
+}

pkg/deployment/reconcile/action_tls_keyfile_clean.go

@@ -0,0 +1,67 @@
+//
+// DISCLAIMER
+//
+// Copyright 2020 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Adam Janikowski
+//
+
+package reconcile
+
+import (
+	"context"
+
+	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
+
+	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+	"github.com/rs/zerolog"
+)
+
+func init() {
+	registerAction(api.ActionTypeCleanTLSKeyfileCertificate, newCleanTLSKeyfileCertificateAction)
+}
+
+func newCleanTLSKeyfileCertificateAction(log zerolog.Logger, action api.Action, actionCtx ActionContext) Action {
+	a := &cleanTLSKeyfileCertificateAction{}
+
+	a.actionImpl = newActionImplDefRef(log, action, actionCtx, operationTLSCACertificateTimeout)
+
+	return a
+}
+
+type cleanTLSKeyfileCertificateAction struct {
+	actionImpl
+
+	actionEmptyCheckProgress
+}
+
+func (a *cleanTLSKeyfileCertificateAction) Start(ctx context.Context) (bool, error) {
+	member, exists := a.actionCtx.GetMemberStatusByID(a.action.MemberID)
+	if !exists {
+		a.log.Warn().Msgf("Member does not exist")
+		return true, nil
+	}
+
+	if err := a.actionCtx.DeleteTLSKeyfile(a.action.Group, member); err != nil {
+		a.log.Warn().Err(err).Msgf("Unable to remove keyfile")
+		if !k8sutil.IsNotFound(err) {
+			return false, err
+		}
+	}
+
+	return true, nil
+}

pkg/deployment/reconcile/action_tls_keyfile_refresh.go

@@ -0,0 +1,93 @@
+//
+// DISCLAIMER
+//
+// Copyright 2020 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Adam Janikowski
+//
+
+package reconcile
+
+import (
+	"context"
+
+	"github.com/arangodb/kube-arangodb/pkg/deployment/client"
+	"github.com/arangodb/kube-arangodb/pkg/util"
+	"github.com/arangodb/kube-arangodb/pkg/util/constants"
+
+	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
+
+	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+	"github.com/rs/zerolog"
+)
+
+func init() {
+	registerAction(api.ActionTypeRefreshTLSKeyfileCertificate, newRefreshTLSKeyfileCertificateAction)
+}
+
+func newRefreshTLSKeyfileCertificateAction(log zerolog.Logger, action api.Action, actionCtx ActionContext) Action {
+	a := &refreshTLSKeyfileCertificateAction{}
+
+	a.actionImpl = newActionImplDefRef(log, action, actionCtx, operationTLSCACertificateTimeout)
+
+	return a
+}
+
+type refreshTLSKeyfileCertificateAction struct {
+	actionImpl
+}
+
+func (a *refreshTLSKeyfileCertificateAction) CheckProgress(ctx context.Context) (bool, bool, error) {
+	c, err := a.actionCtx.GetServerClient(ctx, a.action.Group, a.action.MemberID)
+	if err != nil {
+		a.log.Warn().Err(err).Msg("Unable to get client")
+		return true, false, nil
+	}
+
+	s, exists := a.actionCtx.GetCachedStatus().Secret(k8sutil.CreateTLSKeyfileSecretName(a.actionCtx.GetAPIObject().GetName(), a.action.Group.AsRole(), a.action.MemberID))
+	if !exists {
+		a.log.Warn().Msg("Keyfile secret is missing")
+		return true, false, nil
+	}
+
+	keyfile, ok := s.Data[constants.SecretTLSKeyfile]
+	if !ok {
+		a.log.Warn().Msg("Keyfile secret is invalid")
+		return true, false, nil
+	}
+
+	keyfileSha := util.SHA256(keyfile)
+
+	client := client.NewClient(c.Connection())
+
+	e, err := client.RefreshTLS(ctx)
+	if err != nil {
+		a.log.Warn().Err(err).Msg("Unable to refresh TLS")
+		return true, false, nil
+	}
+
+	if e.Result.KeyFile.Checksum == keyfileSha {
+		return true, false, nil
+	}
+
+	return false, false, nil
+}
+
+func (a *refreshTLSKeyfileCertificateAction) Start(ctx context.Context) (bool, error) {
+	ready, _, err := a.CheckProgress(ctx)
+	return ready, err
+}

pkg/deployment/reconcile/action_tls_status_update.go

@@ -0,0 +1,86 @@
+//
+// DISCLAIMER
+//
+// Copyright 2020 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Adam Janikowski
+//
+
+package reconcile
+
+import (
+	"context"
+
+	"github.com/arangodb/kube-arangodb/pkg/deployment/resources"
+	"github.com/arangodb/kube-arangodb/pkg/util"
+	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+	"github.com/rs/zerolog"
+)
+
+func init() {
+	registerAction(api.ActionTypeTLSKeyStatusUpdate, newTLSKeyStatusUpdate)
+}
+
+func newTLSKeyStatusUpdate(log zerolog.Logger, action api.Action, actionCtx ActionContext) Action {
+	a := &tlsKeyStatusUpdateAction{}
+
+	a.actionImpl = newActionImplDefRef(log, action, actionCtx, defaultTimeout)
+
+	return a
+}
+
+type tlsKeyStatusUpdateAction struct {
+	actionImpl
+
+	actionEmptyCheckProgress
+}
+
+func (a *tlsKeyStatusUpdateAction) Start(ctx context.Context) (bool, error) {
+	if !a.actionCtx.GetSpec().TLS.IsSecure() {
+		return true, nil
+	}
+
+	f, err := a.actionCtx.SecretsInterface().Get(resources.GetCASecretName(a.actionCtx.GetAPIObject()), meta.GetOptions{})
+	if err != nil {
+		a.log.Error().Err(err).Msgf("Unable to get folder info")
+		return true, nil
+	}
+
+	keyHashes := secretKeysToListWithPrefix("sha256:", f)
+
+	if err = a.actionCtx.WithStatusUpdate(func(s *api.DeploymentStatus) bool {
+		if len(keyHashes) == 1 {
+			if s.Hashes.TLS.CA == nil || *s.Hashes.TLS.CA != keyHashes[0] {
+				s.Hashes.TLS.CA = util.NewString(keyHashes[0])
+				return true
+			}
+		}
+
+		if !util.CompareStringArray(keyHashes, s.Hashes.TLS.Truststore) {
+			s.Hashes.TLS.Truststore = keyHashes
+			return true
+		}
+
+		return false
+	}); err != nil {
+		return false, err
+	}
+
+	return true, nil
+}

pkg/deployment/reconcile/plan_builder.go

@@ -193,6 +193,15 @@ func createPlan(ctx context.Context, log zerolog.Logger, apiObject k8sutil.APIOb
 		}
 	}
 
+	// Update status
+	if plan.IsEmpty() {
+		plan = pb.Apply(createEncryptionKeyStatusUpdate)
+	}
+
+	if plan.IsEmpty() {
+		plan = pb.Apply(createTLSStatusUpdate)
+	}
+
 	// Check for scale up/down
 	if plan.IsEmpty() {
 		plan = pb.Apply(createScaleMemeberPlan)
@@ -208,21 +217,28 @@ func createPlan(ctx context.Context, log zerolog.Logger, apiObject k8sutil.APIOb
 		plan = pb.Apply(createEncryptionKey)
 	}
 
-	// Check for the need to rotate TLS certificate of a members
 	if plan.IsEmpty() {
-		plan = pb.Apply(createRotateTLSServerCertificatePlan)
+		plan = pb.Apply(createCARenewalPlan)
 	}
 
+	if plan.IsEmpty() {
+		plan = pb.Apply(createCAAppendPlan)
+	}
+
+	if plan.IsEmpty() {
+		plan = pb.Apply(createKeyfileRenewalPlan)
+	}
+
+	// Check for the need to rotate TLS certificate of a members
+	//if plan.IsEmpty() {
+	//	plan = pb.Apply(createRotateTLSServerCertificatePlan)
+	//}
+
 	// Check for changes storage classes or requirements
 	if plan.IsEmpty() {
 		plan = pb.Apply(createRotateServerStoragePlan)
 	}
 
-	// Check for the need to rotate TLS CA certificate and all members
-	if plan.IsEmpty() {
-		plan = pb.Apply(createRotateTLSCAPlan)
-	}
-
 	if plan.IsEmpty() {
 		plan = pb.Apply(createRotateTLSServerSNIPlan)
 	}
@@ -235,6 +251,10 @@ func createPlan(ctx context.Context, log zerolog.Logger, apiObject k8sutil.APIOb
 		plan = pb.Apply(cleanEncryptionKey)
 	}
 
+	if plan.IsEmpty() {
+		plan = pb.Apply(createCACleanPlan)
+	}
+
 	// Return plan
 	return plan, true
 }

pkg/deployment/reconcile/plan_builder_encryption.go

@@ -38,15 +38,23 @@ import (
 	"github.com/rs/zerolog"
 )
 
+func skipEncryptionPlan(spec api.DeploymentSpec, status api.DeploymentStatus) bool {
+	if !spec.RocksDB.IsEncrypted() {
+		return true
+	}
+
+	if i := status.CurrentImage; i == nil || !i.Enterprise || i.ArangoDBVersion.CompareTo("3.7.0") < 0 {
+		return true
+	}
+
+	return false
+}
+
 func createEncryptionKey(ctx context.Context,
 	log zerolog.Logger, apiObject k8sutil.APIObject,
 	spec api.DeploymentSpec, status api.DeploymentStatus,
 	cachedStatus inspector.Inspector, context PlanBuilderContext) api.Plan {
-	if !spec.RocksDB.IsEncrypted() {
-		return nil
-	}
-
-	if i := status.CurrentImage; i == nil || !i.Enterprise || i.ArangoDBVersion.CompareTo("3.7.0") < 0 {
+	if skipEncryptionPlan(spec, status) {
 		return nil
 	}
 
@@ -128,30 +136,53 @@ func createEncryptionKey(ctx context.Context,
 		return plan
 	}
 
-	currentKeys := make([]string, 0, len(keyfolder.Data))
+	return api.Plan{}
+}
 
-	for key := range keyfolder.Data {
-		currentKeys = append(currentKeys, key)
+func createEncryptionKeyStatusUpdate(ctx context.Context,
+	log zerolog.Logger, apiObject k8sutil.APIObject,
+	spec api.DeploymentSpec, status api.DeploymentStatus,
+	cachedStatus inspector.Inspector, context PlanBuilderContext) api.Plan {
+	if skipEncryptionPlan(spec, status) {
+		return nil
 	}
 
-	currentKeyHashes := util.PrefixStringArray(currentKeys, "sha256:")
-
-	if !util.CompareStringArray(currentKeyHashes, status.CurrentEncryptionKeyHashes) {
+	if createEncryptionKeyStatusUpdateRequired(ctx, log, apiObject, spec, status, cachedStatus, context) {
 		return api.Plan{api.NewAction(api.ActionTypeEncryptionKeyStatusUpdate, api.ServerGroupUnknown, "")}
 	}
 
-	return api.Plan{}
+	return nil
+
+}
+
+func createEncryptionKeyStatusUpdateRequired(ctx context.Context,
+	log zerolog.Logger, apiObject k8sutil.APIObject,
+	spec api.DeploymentSpec, status api.DeploymentStatus,
+	cachedStatus inspector.Inspector, context PlanBuilderContext) bool {
+	if skipEncryptionPlan(spec, status) {
+		return false
+	}
+
+	keyfolder, exists := cachedStatus.Secret(pod.GetKeyfolderSecretName(context.GetName()))
+	if !exists {
+		log.Error().Msgf("Encryption key folder does not exist")
+		return false
+	}
+
+	keyHashes := secretKeysToListWithPrefix("sha256:", keyfolder)
+
+	if !util.CompareStringArray(keyHashes, status.Hashes.Encryption) {
+		return true
+	}
+
+	return false
 }
 
 func cleanEncryptionKey(ctx context.Context,
 	log zerolog.Logger, apiObject k8sutil.APIObject,
 	spec api.DeploymentSpec, status api.DeploymentStatus,
 	cachedStatus inspector.Inspector, context PlanBuilderContext) api.Plan {
-	if !spec.RocksDB.IsEncrypted() {
-		return nil
-	}
-
-	if i := status.CurrentImage; i == nil || !i.Enterprise || i.ArangoDBVersion.CompareTo("3.7.0") < 0 {
+	if skipEncryptionPlan(spec, status) {
 		return nil
 	}
 

pkg/deployment/reconcile/plan_builder_rotate_upgrade.go

@@ -76,7 +76,6 @@ func createRotateOrUpgradePlanInternal(log zerolog.Logger, apiObject k8sutil.API
 				continue
 			}
 
-			log.Debug().Msgf("Before upgrade")
 			// Got pod, compare it with what it should be
 			decision := podNeedsUpgrading(log, pod, spec, status.Images)
 			if decision.UpgradeNeeded && !decision.UpgradeAllowed {
@@ -89,14 +88,11 @@ func createRotateOrUpgradePlanInternal(log zerolog.Logger, apiObject k8sutil.API
 				return nil
 			}
 
-			log.Debug().Msgf("After upgrade")
-
 			if !newPlan.IsEmpty() {
 				// Only rotate/upgrade 1 pod at a time
 				continue
 			}
 
-			log.Debug().Msgf("Before rotate")
 			if decision.UpgradeNeeded {
 				// Yes, upgrade is needed (and allowed)
 				newPlan = createUpgradeMemberPlan(log, m, group, "Version upgrade", spec.GetImage(), status,
@@ -108,7 +104,6 @@ func createRotateOrUpgradePlanInternal(log zerolog.Logger, apiObject k8sutil.API
 					newPlan = createRotateMemberPlan(log, m, group, reason)
 				}
 			}
-			log.Debug().Msgf("After rotate")
 
 			if !newPlan.IsEmpty() {
 				// Only rotate/upgrade 1 pod at a time

pkg/deployment/reconcile/plan_builder_test.go

@@ -385,7 +385,7 @@ func TestCreatePlanClusterScale(t *testing.T) {
 	var status api.DeploymentStatus
 	addAgentsToStatus(t, &status, 3)
 
-	newPlan, changed := createPlan(ctx, log, depl, nil, spec, status, nil, c)
+	newPlan, changed := createPlan(ctx, log, depl, nil, spec, status, inspector.NewEmptyInspector(), c)
 	assert.True(t, changed)
 	require.Len(t, newPlan, 6) // Adding 3 dbservers & 3 coordinators (note: agents do not scale now)
 	assert.Equal(t, api.ActionTypeAddMember, newPlan[0].Type)

pkg/deployment/reconcile/plan_builder_tls.go

@@ -17,259 +17,506 @@
 //
 // Copyright holder is ArangoDB GmbH, Cologne, Germany
 //
-// Author Ewout Prangsma
+// Author Adam Janikowski
 //
 
 package reconcile
 
 import (
+	"bytes"
 	"context"
+	"crypto/tls"
 	"crypto/x509"
 	"encoding/pem"
-	"net"
+	"fmt"
+	"net/http"
+	"net/url"
+	"reflect"
 	"time"
 
+	"github.com/arangodb/kube-arangodb/pkg/deployment/client"
+	"github.com/arangodb/kube-arangodb/pkg/util/constants"
+
+	"github.com/arangodb-helper/go-certificates"
+	"github.com/arangodb/kube-arangodb/pkg/deployment/resources"
 	"github.com/arangodb/kube-arangodb/pkg/deployment/resources/inspector"
+	"github.com/arangodb/kube-arangodb/pkg/util"
+	"github.com/pkg/errors"
+	core "k8s.io/api/core/v1"
 
 	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
 	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
 	"github.com/rs/zerolog"
 )
 
-// createRotateTLSServerCertificatePlan creates plan to rotate a server because of an (soon to be) expired TLS certificate.
-func createRotateTLSServerCertificatePlan(ctx context.Context,
-	log zerolog.Logger, apiObject k8sutil.APIObject,
-	spec api.DeploymentSpec, status api.DeploymentStatus,
-	cachedStatus inspector.Inspector, context PlanBuilderContext) api.Plan {
-	if !spec.TLS.IsSecure() {
-		return nil
-	}
-	var plan api.Plan
-	status.Members.ForeachServerGroup(func(group api.ServerGroup, members api.MemberStatusList) error {
-		for _, m := range members {
-			if !plan.IsEmpty() {
-				// Only 1 change at a time
-				continue
-			}
-			if m.Phase != api.MemberPhaseCreated {
-				// Only make changes when phase is created
-				continue
-			}
-			if group == api.ServerGroupSyncWorkers {
-				// SyncWorkers have no externally created TLS keyfile
-				continue
-			}
-			// Load keyfile
-			secret, exists := cachedStatus.Secret(k8sutil.CreateTLSKeyfileSecretName(context.GetName(), group.AsRole(), m.ID))
-			if !exists {
-				log.Warn().
-					Str("role", group.AsRole()).
-					Str("id", m.ID).
-					Msg("Failed to get TLS secret")
-				continue
-			}
+const CertificateRenewalMargin = 7 * 24 * time.Hour
 
-			keyfile, err := k8sutil.GetTLSKeyfileFromSecret(secret)
-			if err != nil {
-				log.Warn().Err(err).
-					Str("role", group.AsRole()).
-					Str("id", m.ID).
-					Msg("Failed to parse TLS secret")
-				continue
-			}
+type Certificates []*x509.Certificate
 
-			tlsSpec := spec.TLS
-			if group.IsArangosync() {
-				tlsSpec = spec.Sync.TLS
-			}
-			renewalNeeded, reason := tlsKeyfileNeedsRenewal(log, keyfile, tlsSpec)
-			if renewalNeeded {
-				plan = append(append(plan,
-					api.NewAction(api.ActionTypeRenewTLSCertificate, group, m.ID, reason)),
-					createRotateMemberPlan(log, m, group, "TLS certificate renewal")...,
-				)
-			}
-		}
-		return nil
-	})
-	return plan
-}
-
-// createRotateTLSCAPlan creates plan to replace a TLS CA and rotate all server.
-func createRotateTLSCAPlan(ctx context.Context,
-	log zerolog.Logger, apiObject k8sutil.APIObject,
-	spec api.DeploymentSpec, status api.DeploymentStatus,
-	cachedStatus inspector.Inspector, context PlanBuilderContext) api.Plan {
-	if !spec.TLS.IsSecure() {
-		return nil
-	}
-
-	asOwner := apiObject.AsOwner()
-
-	secretName := spec.TLS.GetCASecretName()
-	secret, exists := cachedStatus.Secret(secretName)
-
-	if !exists {
-		log.Warn().Str("secret-name", secretName).Msg("TLS CA secret missing")
-		return nil
-	}
-
-	cert, _, isOwned, err := k8sutil.GetCAFromSecret(secret, &asOwner)
-	if err != nil {
-		log.Warn().Err(err).Str("secret-name", secretName).Msg("Failed to fetch TLS CA secret")
-		return nil
-	}
-	if !isOwned {
-		// TLS CA is not owned by the deployment, we cannot change it
-		return nil
-	}
-	var plan api.Plan
-	if renewalNeeded, reason := tlsCANeedsRenewal(log, cert, spec.TLS); renewalNeeded {
-		if spec.IsDowntimeAllowed() {
-			var planSuffix api.Plan
-			plan = append(plan,
-				api.NewAction(api.ActionTypeRenewTLSCACertificate, 0, "", reason),
-			)
-			status.Members.ForeachServerGroup(func(group api.ServerGroup, members api.MemberStatusList) error {
-				for _, m := range members {
-					if m.Phase != api.MemberPhaseCreated {
-						// Only make changes when phase is created
-						continue
-					}
-					if !group.IsArangod() {
-						// Sync master/worker is not applicable here
-						continue
-					}
-					plan = append(plan,
-						api.NewAction(api.ActionTypeRenewTLSCertificate, group, m.ID),
-						api.NewAction(api.ActionTypeRotateMember, group, m.ID, "TLS CA certificate changed"),
-					)
-					planSuffix = append(planSuffix,
-						api.NewAction(api.ActionTypeWaitForMemberUp, group, m.ID, "TLS CA certificate changed"),
-					)
-				}
-				return nil
-			})
-			plan = append(plan, planSuffix...)
-		} else {
-			// Rotating the CA results in downtime.
-			// That is currently not allowed.
-			context.CreateEvent(k8sutil.NewDowntimeNotAllowedEvent(apiObject, "Rotate TLS CA"))
-		}
-	}
-	return plan
-}
-
-// tlsKeyfileNeedsRenewal decides if the certificate in the given keyfile
-// should be renewed.
-func tlsKeyfileNeedsRenewal(log zerolog.Logger, keyfile string, spec api.TLSSpec) (bool, string) {
-	raw := []byte(keyfile)
-	// containsAll returns true when all elements in the expected list
-	// are in the actual list.
-	containsAll := func(actual []string, expected []string) bool {
-		for _, x := range expected {
-			found := false
-			for _, y := range actual {
-				if x == y {
-					found = true
-					break
-				}
-			}
-			if !found {
+func (c Certificates) Contains(cert *x509.Certificate) bool {
+	for _, localCert := range c {
+		if !localCert.Equal(cert) {
 			return false
 		}
 	}
+
 	return true
-	}
-	ipsToStringSlice := func(list []net.IP) []string {
-		result := make([]string, len(list))
-		for i, x := range list {
-			result[i] = x.String()
-		}
-		return result
-	}
-	for {
-		var derBlock *pem.Block
-		derBlock, raw = pem.Decode(raw)
-		if derBlock == nil {
-			break
-		}
-		if derBlock.Type == "CERTIFICATE" {
-			cert, err := x509.ParseCertificate(derBlock.Bytes)
-			if err != nil {
-				// We do not understand the certificate, let's renew it
-				log.Warn().Err(err).Msg("Failed to parse x509 certificate. Renewing it")
-				return true, "Cannot parse x509 certificate: " + err.Error()
-			}
-			if cert.IsCA {
-				// Only look at the server certificate, not CA or intermediate
-				continue
-			}
-			// Check expiration date. Renewal at 2/3 of lifetime.
-			ttl := cert.NotAfter.Sub(cert.NotBefore)
-			expirationDate := cert.NotBefore.Add((ttl / 3) * 2)
-			if expirationDate.Before(time.Now()) {
-				// We should renew now
-				log.Debug().
-					Str("not-before", cert.NotBefore.String()).
-					Str("not-after", cert.NotAfter.String()).
-					Str("expiration-date", expirationDate.String()).
-					Msg("TLS certificate renewal needed")
-				return true, "Server certificate about to expire"
-			}
-			// Check alternate names against spec
-			dnsNames, ipAddresses, emailAddress, err := spec.GetParsedAltNames()
-			if err == nil {
-				if !containsAll(cert.DNSNames, dnsNames) {
-					return true, "Some alternate DNS names are missing"
-				}
-				if !containsAll(ipsToStringSlice(cert.IPAddresses), ipAddresses) {
-					return true, "Some alternate IP addresses are missing"
-				}
-				if !containsAll(cert.EmailAddresses, emailAddress) {
-					return true, "Some alternate email addresses are missing"
-				}
-			}
-		}
-	}
-	return false, ""
 }
 
-// tlsCANeedsRenewal decides if the given CA certificate
-// should be renewed.
-// Returns: shouldRenew, reason
-func tlsCANeedsRenewal(log zerolog.Logger, cert string, spec api.TLSSpec) (bool, string) {
-	raw := []byte(cert)
+func (c Certificates) ContainsAll(certs Certificates) bool {
+	if len(certs) == 0 {
+		return true
+	}
+
+	for _, cert := range certs {
+		if !c.Contains(cert) {
+			return false
+		}
+	}
+
+	return true
+}
+
+func (c Certificates) ToPem() ([]byte, error) {
+	bytes := bytes.NewBuffer([]byte{})
+
+	for _, cert := range c {
+		if err := pem.Encode(bytes, &pem.Block{Type: "CERTIFICATE", Bytes: cert.Raw}); err != nil {
+			return nil, err
+		}
+	}
+
+	return bytes.Bytes(), nil
+}
+
+func (c Certificates) AsCertPool() *x509.CertPool {
+	cp := x509.NewCertPool()
+
+	for _, cert := range c {
+		cp.AddCert(cert)
+	}
+
+	return cp
+}
+
+func getCertsFromData(log zerolog.Logger, caPem []byte) Certificates {
+	certs := make([]*x509.Certificate, 0, 2)
+
 	for {
-		var derBlock *pem.Block
-		derBlock, raw = pem.Decode(raw)
-		if derBlock == nil {
+		pem, rest := pem.Decode(caPem)
+		if pem == nil {
 			break
 		}
-		if derBlock.Type == "CERTIFICATE" {
-			cert, err := x509.ParseCertificate(derBlock.Bytes)
+
+		caPem = rest
+
+		cert, err := x509.ParseCertificate(pem.Bytes)
 		if err != nil {
-				// We do not understand the certificate, let's renew it
-				log.Warn().Err(err).Msg("Failed to parse x509 certificate. Renewing it")
-				return true, "Cannot parse x509 certificate: " + err.Error()
-			}
-			if !cert.IsCA {
-				// Only look at the CA certificate
+			// This error should be ignored
+			log.Error().Err(err).Msg("Unable to parse certificate")
 			continue
 		}
-			// Check expiration date. Renewal at 90% of lifetime.
-			ttl := cert.NotAfter.Sub(cert.NotBefore)
-			expirationDate := cert.NotBefore.Add((ttl / 10) * 9)
-			if expirationDate.Before(time.Now()) {
-				// We should renew now
-				log.Debug().
-					Str("not-before", cert.NotBefore.String()).
-					Str("not-after", cert.NotAfter.String()).
-					Str("expiration-date", expirationDate.String()).
-					Msg("TLS CA certificate renewal needed")
-				return true, "CA Certificate about to expire"
+
+		certs = append(certs, cert)
 	}
-		}
-	}
-	return false, ""
+
+	return certs
+}
+
+func getCertsFromSecret(log zerolog.Logger, secret *core.Secret) Certificates {
+	caPem, exists := secret.Data[core.ServiceAccountRootCAKey]
+	if !exists {
+		return nil
+	}
+
+	return getCertsFromData(log, caPem)
+}
+
+func getKeyCertFromCache(log zerolog.Logger, cachedStatus inspector.Inspector, spec api.DeploymentSpec, certName, keyName string) (Certificates, interface{}, error) {
+	caSecret, exists := cachedStatus.Secret(spec.TLS.GetCASecretName())
+	if !exists {
+		return nil, nil, errors.Errorf("CA Secret does not exists")
+	}
+
+	return getKeyCertFromSecret(log, caSecret, keyName, certName)
+}
+
+func getKeyCertFromSecret(log zerolog.Logger, secret *core.Secret, certName, keyName string) (Certificates, interface{}, error) {
+	ca, exists := secret.Data[certName]
+	if !exists {
+		return nil, nil, errors.Errorf("Key %s missing in secret", certName)
+	}
+
+	key, exists := secret.Data[keyName]
+	if !exists {
+		return nil, nil, errors.Errorf("Key %s missing in secret", keyName)
+	}
+
+	cert, keys, err := certificates.LoadFromPEM(string(ca), string(key))
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return cert, keys, nil
+}
+
+// createTLSStatusUpdate creates plan to update ca info
+func createTLSStatusUpdate(ctx context.Context,
+	log zerolog.Logger, apiObject k8sutil.APIObject,
+	spec api.DeploymentSpec, status api.DeploymentStatus,
+	cachedStatus inspector.Inspector, context PlanBuilderContext) api.Plan {
+	if !spec.TLS.IsSecure() {
+		return nil
+	}
+
+	if createTLSStatusUpdateRequired(ctx, log, apiObject, spec, status, cachedStatus, context) {
+		return api.Plan{api.NewAction(api.ActionTypeTLSKeyStatusUpdate, api.ServerGroupUnknown, "", "Update status")}
+	}
+
+	return nil
+}
+
+func createTLSStatusUpdateRequired(ctx context.Context,
+	log zerolog.Logger, apiObject k8sutil.APIObject,
+	spec api.DeploymentSpec, status api.DeploymentStatus,
+	cachedStatus inspector.Inspector, context PlanBuilderContext) bool {
+	if !spec.TLS.IsSecure() {
+		return false
+	}
+
+	trusted, exists := cachedStatus.Secret(resources.GetCASecretName(apiObject))
+	if !exists {
+		log.Warn().Str("secret", resources.GetCASecretName(apiObject)).Msg("Folder with secrets does not exist")
+		return false
+	}
+
+	keyHashes := secretKeysToListWithPrefix("sha256:", trusted)
+
+	if len(keyHashes) == 0 {
+		return false
+	}
+
+	if len(keyHashes) == 1 {
+		if status.Hashes.TLS.CA == nil {
+			return true
+		}
+
+		if *status.Hashes.TLS.CA != keyHashes[0] {
+			return true
+		}
+	}
+
+	if !util.CompareStringArray(status.Hashes.TLS.Truststore, keyHashes) {
+		return true
+	}
+
+	return false
+}
+
+// createCAAppendPlan creates plan to append CA
+func createCAAppendPlan(ctx context.Context,
+	log zerolog.Logger, apiObject k8sutil.APIObject,
+	spec api.DeploymentSpec, status api.DeploymentStatus,
+	cachedStatus inspector.Inspector, context PlanBuilderContext) api.Plan {
+	if !spec.TLS.IsSecure() {
+		return nil
+	}
+
+	caSecret, exists := cachedStatus.Secret(spec.TLS.GetCASecretName())
+	if !exists {
+		log.Warn().Str("secret", spec.TLS.GetCASecretName()).Msg("CA Secret does not exists")
+		return nil
+	}
+
+	ca, _, err := getKeyCertFromSecret(log, caSecret, resources.CACertName, resources.CAKeyName)
+	if err != nil {
+		log.Warn().Err(err).Str("secret", spec.TLS.GetCASecretName()).Msg("CA Secret does not contains Cert")
+		return nil
+	}
+
+	if len(ca) == 0 {
+		log.Warn().Str("secret", spec.TLS.GetCASecretName()).Msg("CA does not contain any certs")
+		return nil
+	}
+
+	trusted, exists := cachedStatus.Secret(resources.GetCASecretName(apiObject))
+	if !exists {
+		log.Warn().Str("secret", resources.GetCASecretName(apiObject)).Msg("Folder with secrets does not exist")
+		return nil
+	}
+
+	caData, err := ca.ToPem()
+	if err != nil {
+		log.Warn().Err(err).Str("secret", spec.TLS.GetCASecretName()).Msg("Unable to parse cert")
+		return nil
+	}
+
+	certSha := util.SHA256(caData)
+
+	if _, exists := trusted.Data[certSha]; !exists {
+		return api.Plan{api.NewAction(api.ActionTypeAppendTLSCACertificate, api.ServerGroupUnknown, "", "Append CA to truststore").
+			AddParam(actionTypeAppendTLSCACertificateChecksum, certSha)}
+	}
+
+	return nil
+}
+
+// createCARenewalPlan creates plan to renew CA
+func createCARenewalPlan(ctx context.Context,
+	log zerolog.Logger, apiObject k8sutil.APIObject,
+	spec api.DeploymentSpec, status api.DeploymentStatus,
+	cachedStatus inspector.Inspector, context PlanBuilderContext) api.Plan {
+	if !spec.TLS.IsSecure() {
+		return nil
+	}
+
+	caSecret, exists := cachedStatus.Secret(spec.TLS.GetCASecretName())
+	if !exists {
+		log.Warn().Str("secret", spec.TLS.GetCASecretName()).Msg("CA Secret does not exists")
+		return nil
+	}
+
+	if !k8sutil.IsOwner(apiObject.AsOwner(), caSecret) {
+		log.Warn().Str("secret", spec.TLS.GetCASecretName()).Msg("CA Secret is not owned by Operator, we wont do anything")
+		return nil
+	}
+
+	cas, _, err := getKeyCertFromSecret(log, caSecret, resources.CACertName, resources.CAKeyName)
+	if err != nil {
+		log.Warn().Err(err).Str("secret", spec.TLS.GetCASecretName()).Msg("CA Secret does not contains Cert")
+		return nil
+	}
+
+	for _, ca := range cas {
+		if time.Now().Add(CertificateRenewalMargin).After(ca.NotAfter) {
+			// CA will expire soon, renewal needed
+			return api.Plan{api.NewAction(api.ActionTypeRenewTLSCACertificate, api.ServerGroupUnknown, "", "Renew CA Certificate")}
+		}
+	}
+
+	return nil
+}
+
+// createCACleanPlan creates plan to remove old CA's
+func createCACleanPlan(ctx context.Context,
+	log zerolog.Logger, apiObject k8sutil.APIObject,
+	spec api.DeploymentSpec, status api.DeploymentStatus,
+	cachedStatus inspector.Inspector, context PlanBuilderContext) api.Plan {
+	if !spec.TLS.IsSecure() {
+		return nil
+	}
+
+	caSecret, exists := cachedStatus.Secret(spec.TLS.GetCASecretName())
+	if !exists {
+		log.Warn().Str("secret", spec.TLS.GetCASecretName()).Msg("CA Secret does not exists")
+		return nil
+	}
+
+	ca, _, err := getKeyCertFromSecret(log, caSecret, resources.CACertName, resources.CAKeyName)
+	if err != nil {
+		log.Warn().Err(err).Str("secret", spec.TLS.GetCASecretName()).Msg("CA Secret does not contains Cert")
+		return nil
+	}
+
+	if len(ca) == 0 {
+		log.Warn().Str("secret", spec.TLS.GetCASecretName()).Msg("CA does not contain any certs")
+		return nil
+	}
+
+	trusted, exists := cachedStatus.Secret(resources.GetCASecretName(apiObject))
+	if !exists {
+		log.Warn().Str("secret", resources.GetCASecretName(apiObject)).Msg("Folder with secrets does not exist")
+		return nil
+	}
+
+	caData, err := ca.ToPem()
+	if err != nil {
+		log.Warn().Err(err).Str("secret", spec.TLS.GetCASecretName()).Msg("Unable to parse cert")
+		return nil
+	}
+
+	certSha := util.SHA256(caData)
+
+	for sha := range trusted.Data {
+		if certSha != sha {
+			return api.Plan{api.NewAction(api.ActionTypeCleanTLSCACertificate, api.ServerGroupUnknown, "", "Clean CA from truststore").
+				AddParam(actionTypeAppendTLSCACertificateChecksum, sha)}
+		}
+	}
+
+	return nil
+}
+
+// createKeyfileRenewalPlan creates plan to renew server keyfile
+func createKeyfileRenewalPlan(ctx context.Context,
+	log zerolog.Logger, apiObject k8sutil.APIObject,
+	spec api.DeploymentSpec, status api.DeploymentStatus,
+	cachedStatus inspector.Inspector, context PlanBuilderContext) api.Plan {
+	if !spec.TLS.IsSecure() {
+		return nil
+	}
+
+	var plan api.Plan
+
+	status.Members.ForeachServerGroup(func(group api.ServerGroup, members api.MemberStatusList) error {
+		if !group.IsArangod() {
+			return nil
+		}
+
+		for _, member := range members {
+			if !plan.IsEmpty() {
+				return nil
+			}
+
+			if renew, recreate := keyfileRenewalRequired(ctx, log, apiObject, spec, status, cachedStatus, context, group, member); renew {
+				log.Info().Msg("Renewal of keyfile required")
+				plan = append(plan, createKeyfileRotationPlan(log, spec, status, group, member, recreate)...)
+			}
+		}
+
+		return nil
+	})
+
+	return plan
+}
+
+func createKeyfileRenewalPlanMode(
+	spec api.DeploymentSpec, status api.DeploymentStatus,
+	member api.MemberStatus) api.TLSRotateMode {
+	if !spec.TLS.IsSecure() {
+		return api.TLSRotateModeRecreate
+	}
+
+	if spec.TLS.Mode.Get() != api.TLSRotateModeInPlace {
+		return api.TLSRotateModeRecreate
+	}
+
+	if i := status.CurrentImage; i == nil {
+		return api.TLSRotateModeRecreate
+	} else {
+		if !i.Enterprise || i.ArangoDBVersion.CompareTo("3.7.0") < 0 || i.ImageID != member.ImageID {
+			return api.TLSRotateModeRecreate
+		}
+	}
+
+	return api.TLSRotateModeInPlace
+}
+
+func createKeyfileRotationPlan(log zerolog.Logger, spec api.DeploymentSpec, status api.DeploymentStatus, group api.ServerGroup, member api.MemberStatus, recreate bool) api.Plan {
+	p := api.Plan{}
+
+	if recreate {
+		p = append(p,
+			api.NewAction(api.ActionTypeCleanTLSKeyfileCertificate, group, member.ID, "Remove server keyfile and enforce renewal"))
+	}
+
+	switch createKeyfileRenewalPlanMode(spec, status, member) {
+	case api.TLSRotateModeInPlace:
+		p = append(p, api.NewAction(api.ActionTypeRefreshTLSKeyfileCertificate, group, member.ID, "Renew Member Keyfile"))
+	default:
+		p = append(p, createRotateMemberPlan(log, member, group, "Restart server after keyfile removal")...)
+	}
+	return p
+}
+
+func checkServerValidCertRequest(ctx context.Context, apiObject k8sutil.APIObject, group api.ServerGroup, member api.MemberStatus, ca Certificates) (*tls.ConnectionState, error) {
+	endpoint := fmt.Sprintf("https://%s:%d", k8sutil.CreatePodDNSName(apiObject, group.AsRole(), member.ID), k8sutil.ArangoPort)
+
+	tlsConfig := &tls.Config{
+		RootCAs: ca.AsCertPool(),
+	}
+	transport := &http.Transport{TLSClientConfig: tlsConfig}
+	client := &http.Client{Transport: transport, Timeout: time.Second}
+
+	resp, err := client.Get(endpoint)
+	if err != nil {
+		return nil, err
+	}
+
+	return resp.TLS, nil
+}
+
+func keyfileRenewalRequired(ctx context.Context,
+	log zerolog.Logger, apiObject k8sutil.APIObject,
+	spec api.DeploymentSpec, status api.DeploymentStatus,
+	cachedStatus inspector.Inspector, context PlanBuilderContext,
+	group api.ServerGroup, member api.MemberStatus) (bool, bool) {
+	if !spec.TLS.IsSecure() {
+		return false, false
+	}
+
+	caSecret, exists := cachedStatus.Secret(spec.TLS.GetCASecretName())
+	if !exists {
+		log.Warn().Str("secret", spec.TLS.GetCASecretName()).Msg("CA Secret does not exists")
+		return false, false
+	}
+
+	ca, _, err := getKeyCertFromSecret(log, caSecret, resources.CACertName, resources.CAKeyName)
+	if err != nil {
+		log.Warn().Err(err).Str("secret", spec.TLS.GetCASecretName()).Msg("CA Secret does not contains Cert")
+		return false, false
+	}
+
+	res, err := checkServerValidCertRequest(ctx, apiObject, group, member, ca)
+	if err != nil {
+		switch v := err.(type) {
+		case *url.Error:
+			switch v.Err.(type) {
+			case x509.UnknownAuthorityError, x509.CertificateInvalidError:
+				return true, true
+			default:
+				log.Warn().Err(v.Err).Str("type", reflect.TypeOf(v.Err).String()).Msg("Validation of server cert failed")
+			}
+		default:
+			log.Warn().Err(err).Str("type", reflect.TypeOf(err).String()).Msg("Validation of server cert failed")
+		}
+		return false, false
+	}
+
+	// Check if cert is not expired
+	for _, cert := range res.PeerCertificates {
+		if cert == nil {
+			continue
+		}
+
+		if time.Now().Add(CertificateRenewalMargin).After(cert.NotAfter) {
+			return true, true
+		}
+	}
+
+	// Ensure secret is propagated only on 3.7.0+ enterprise and inplace mode
+	if createKeyfileRenewalPlanMode(spec, status, member) == api.TLSRotateModeInPlace {
+		conn, err := context.GetServerClient(ctx, group, member.ID)
+		if err != nil {
+			log.Warn().Err(err).Msg("Unable to get client")
+			return false, false
+		}
+
+		s, exists := cachedStatus.Secret(k8sutil.CreateTLSKeyfileSecretName(apiObject.GetName(), group.AsRole(), member.ID))
+		if !exists {
+			log.Warn().Msg("Keyfile secret is missing")
+			return false, false
+		}
+
+		c := client.NewClient(conn.Connection())
+		tls, err := c.GetTLS(ctx)
+		if err != nil {
+			log.Warn().Err(err).Msg("Unable to get tls details")
+			return false, false
+		}
+
+		keyfile, ok := s.Data[constants.SecretTLSKeyfile]
+		if !ok {
+			log.Warn().Msg("Keyfile secret is invalid")
+			return false, false
+		}
+
+		keyfileSha := util.SHA256(keyfile)
+
+		if tls.Result.KeyFile.Checksum != keyfileSha {
+			return true, false
+		}
+	}
+
+	return false, false
 }

pkg/deployment/reconcile/plan_builder_tls_sni.go

@@ -90,10 +90,10 @@ func createRotateTLSServerSNIPlan(ctx context.Context,
 				return nil
 
 			} else if !ok {
-				switch sni.Mode.Get() {
-				case api.TLSSNIRotateModeRecreate:
+				switch spec.TLS.Mode.Get() {
+				case api.TLSRotateModeRecreate:
 					plan = append(plan, createRotateMemberPlan(log, m, group, "SNI Secret needs update")...)
-				case api.TLSSNIRotateModeInPlace:
+				case api.TLSRotateModeInPlace:
 					plan = append(plan,
 						api.NewAction(api.ActionTypeUpdateTLSSNI, group, m.ID, "SNI Secret needs update"))
 				default:

pkg/deployment/reconcile/plan_executor.go

@@ -94,11 +94,8 @@ func (d *Reconciler) ExecutePlan(ctx context.Context, cachedStatus inspector.Ins
 				}
 			}
 			log.Debug().Bool("ready", ready).Msg("Action Start completed")
-			if !ready {
-				// We need to check back soon
+
 			return true, nil
-			}
-			// Continue with next action
 		} else {
 			// First action of plan has been started, check its progress
 			ready, abort, err := action.CheckProgress(ctx)
@@ -154,14 +151,14 @@ func (d *Reconciler) ExecutePlan(ctx context.Context, cachedStatus inspector.Ins
 				// Timeout not yet expired, come back soon
 				return true, nil
 			}
-			// Continue with next action
+			return true, nil
 		}
 	}
 }
 
 // createAction create action object based on action type
 func (d *Reconciler) createAction(ctx context.Context, log zerolog.Logger, action api.Action, cachedStatus inspector.Inspector) Action {
-	actionCtx := newActionContext(log, d.context, cachedStatus)
+	actionCtx := newActionContext(log.With().Str("id", action.ID).Str("type", action.Type.String()).Logger(), d.context, cachedStatus)
 
 	f, ok := getActionFactory(action.Type)
 	if !ok {

pkg/deployment/reconcile/timeouts.go

@@ -31,6 +31,7 @@ const (
 	recreateMemberTimeout            = time.Minute * 15
 	renewTLSCertificateTimeout       = time.Minute * 30
 	renewTLSCACertificateTimeout     = time.Minute * 30
+	operationTLSCACertificateTimeout = time.Minute * 30
 	rotateMemberTimeout              = time.Minute * 15
 	pvcResizeTimeout                 = time.Minute * 15
 	pvcResizedTimeout                = time.Minute * 15

pkg/deployment/reconcile/utils.go

@@ -0,0 +1,46 @@
+//
+// DISCLAIMER
+//
+// Copyright 2020 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Adam Janikowski
+//
+
+package reconcile
+
+import (
+	"sort"
+
+	"github.com/arangodb/kube-arangodb/pkg/util"
+	core "k8s.io/api/core/v1"
+)
+
+func secretKeysToListWithPrefix(prefix string, s *core.Secret) []string {
+	return util.PrefixStringArray(secretKeysToList(s), prefix)
+}
+
+func secretKeysToList(s *core.Secret) []string {
+	keys := make([]string, 0, len(s.Data))
+
+	for key := range s.Data {
+		keys = append(keys, key)
+	}
+
+	sort.Strings(keys)
+
+	return keys
+}

pkg/deployment/resources/pod_creator.go

@@ -54,20 +54,6 @@ func versionHasAdvertisedEndpoint(v driver.Version) bool {
 	return v.CompareTo("3.4.0") >= 0
 }
 
-// versionHasJWTSecretKeyfile derives from the version number of arangod has
-// the option --auth.jwt-secret-keyfile which can take the JWT secret from
-// a file in the file system.
-func versionHasJWTSecretKeyfile(v driver.Version) bool {
-	if v.CompareTo("3.3.22") >= 0 && v.CompareTo("3.4.0") < 0 {
-		return true
-	}
-	if v.CompareTo("3.4.2") >= 0 {
-		return true
-	}
-
-	return false
-}
-
 // createArangodArgs creates command line arguments for an arangod server in the given group.
 func createArangodArgs(input pod.Input) []string {
 	options := k8sutil.CreateOptionPairs(64)
@@ -81,20 +67,7 @@ func createArangodArgs(input pod.Input) []string {
 	options.Addf("--server.endpoint", "%s://%s:%d", scheme, input.Deployment.GetListenAddr(), k8sutil.ArangoPort)
 
 	// Authentication
-	if input.Deployment.IsAuthenticated() {
-		// With authentication
-		options.Add("--server.authentication", "true")
-
-		if versionHasJWTSecretKeyfile(input.Version) {
-			keyPath := filepath.Join(k8sutil.ClusterJWTSecretVolumeMountDir, constants.SecretKeyToken)
-			options.Add("--server.jwt-secret-keyfile", keyPath)
-		} else {
-			options.Addf("--server.jwt-secret", "$(%s)", constants.EnvArangodJWTSecret)
-		}
-	} else {
-		// Without authentication
-		options.Add("--server.authentication", "false")
-	}
+	options.Merge(pod.JWT().Args(input))
 
 	// Storage engine
 	options.Add("--server.storage-engine", input.Deployment.GetStorageEngine().AsArangoArgument())
@@ -103,11 +76,7 @@ func createArangodArgs(input pod.Input) []string {
 	options.Add("--log.level", "INFO")
 
 	// TLS
-	if input.Deployment.IsSecure() {
-		keyPath := filepath.Join(k8sutil.TLSKeyfileVolumeMountDir, constants.SecretTLSKeyfile)
-		options.Add("--ssl.keyfile", keyPath)
-		options.Add("--ssl.ecdh-curve", "") // This way arangod accepts curves other than P256 as well.
-	}
+	options.Merge(pod.TLS().Args(input))
 
 	// RocksDB
 	options.Merge(pod.Encryption().Args(input))
@@ -323,31 +292,10 @@ func (r *Resources) RenderPodForMember(cachedStatus inspector.Inspector, spec ap
 	// Render pod
 	if group.IsArangod() {
 		// Prepare arguments
-		version := imageInfo.ArangoDBVersion
 		autoUpgrade := m.Conditions.IsTrue(api.ConditionTypeAutoUpgrade)
 
-		tlsKeyfileSecretName := ""
-		if spec.IsSecure() {
-			tlsKeyfileSecretName = k8sutil.CreateTLSKeyfileSecretName(apiObject.GetName(), role, m.ID)
-		}
-
-		rocksdbEncryptionSecretName := ""
-		if spec.RocksDB.IsEncrypted() {
-			rocksdbEncryptionSecretName = spec.RocksDB.Encryption.GetKeySecretName()
-		}
-
-		var clusterJWTSecretName string
-		if spec.IsAuthenticated() {
-			if versionHasJWTSecretKeyfile(version) {
-				clusterJWTSecretName = spec.Authentication.GetJWTSecretName()
-			}
-		}
-
 		memberPod := MemberArangoDPod{
 			status:           m,
-			tlsKeyfileSecretName:        tlsKeyfileSecretName,
-			rocksdbEncryptionSecretName: rocksdbEncryptionSecretName,
-			clusterJWTSecretName:        clusterJWTSecretName,
 			groupSpec:        groupSpec,
 			spec:             spec,
 			group:            group,
@@ -492,20 +440,6 @@ func (r *Resources) createPodForMember(spec api.DeploymentSpec, memberID string,
 		if autoUpgrade {
 			newPhase = api.MemberPhaseUpgrading
 		}
-		if spec.IsSecure() {
-			tlsKeyfileSecretName := k8sutil.CreateTLSKeyfileSecretName(apiObject.GetName(), role, m.ID)
-			serverNames := []string{
-				k8sutil.CreateDatabaseClientServiceDNSName(apiObject),
-				k8sutil.CreatePodDNSName(apiObject, role, m.ID),
-			}
-			if ip := spec.ExternalAccess.GetLoadBalancerIP(); ip != "" {
-				serverNames = append(serverNames, ip)
-			}
-			owner := apiObject.AsOwner()
-			if err := createTLSServerCertificate(log, secrets, serverNames, spec.TLS, tlsKeyfileSecretName, &owner); err != nil && !k8sutil.IsAlreadyExists(err) {
-				return maskAny(errors.Wrapf(err, "Failed to create TLS keyfile secret"))
-			}
-		}
 
 		uid, checksum, err := CreateArangoPod(kubecli, apiObject, pod)
 		if err != nil {

pkg/deployment/resources/pod_creator_arangod.go

@@ -51,9 +51,6 @@ var _ interfaces.ContainerCreator = &ArangoDContainer{}
 
 type MemberArangoDPod struct {
 	status           api.MemberStatus
-	tlsKeyfileSecretName        string
-	rocksdbEncryptionSecretName string
-	clusterJWTSecretName        string
 	groupSpec        api.ServerGroupSpec
 	spec             api.DeploymentSpec
 	deploymentStatus api.DeploymentStatus
@@ -66,6 +63,7 @@ type MemberArangoDPod struct {
 }
 
 type ArangoDContainer struct {
+	member    *MemberArangoDPod
 	resources *Resources
 	groupSpec api.ServerGroupSpec
 	spec      api.DeploymentSpec
@@ -141,13 +139,8 @@ func (a *ArangoDContainer) GetImage() string {
 func (a *ArangoDContainer) GetEnvs() []core.EnvVar {
 	envs := NewEnvBuilder()
 
-	if a.spec.IsAuthenticated() {
-		if !versionHasJWTSecretKeyfile(a.imageInfo.ArangoDBVersion) {
-			env := k8sutil.CreateEnvSecretKeySelector(constants.EnvArangodJWTSecret,
-				a.spec.Authentication.GetJWTSecretName(), constants.SecretKeyToken)
-
-			envs.Add(true, env)
-		}
+	if env := pod.JWT().Envs(a.member.AsInput()); len(env) > 0 {
+		envs.Add(true, env...)
 	}
 
 	if a.spec.License.HasSecretName() {
@@ -212,6 +205,7 @@ func (m *MemberArangoDPod) Init(pod *core.Pod) {
 
 func (m *MemberArangoDPod) Validate(cachedStatus inspector.Inspector) error {
 	i := m.AsInput()
+
 	if err := pod.SNI().Verify(i, cachedStatus); err != nil {
 		return err
 	}
@@ -224,6 +218,10 @@ func (m *MemberArangoDPod) Validate(cachedStatus inspector.Inspector) error {
 		return err
 	}
 
+	if err := pod.TLS().Verify(i, cachedStatus); err != nil {
+		return err
+	}
+
 	return nil
 }
 
@@ -309,42 +307,28 @@ func (m *MemberArangoDPod) GetSidecars(pod *core.Pod) {
 }
 
 func (m *MemberArangoDPod) GetVolumes() ([]core.Volume, []core.VolumeMount) {
-	var volumes []core.Volume
-	var volumeMounts []core.VolumeMount
+	volumes := pod.NewVolumes()
 
-	volumeMounts = append(volumeMounts, k8sutil.ArangodVolumeMount())
+	volumes.AddVolumeMount(k8sutil.ArangodVolumeMount())
 
 	if m.resources.context.GetLifecycleImage() != "" {
-		volumeMounts = append(volumeMounts, k8sutil.LifecycleVolumeMount())
+		volumes.AddVolumeMount(k8sutil.LifecycleVolumeMount())
 	}
 
 	if m.status.PersistentVolumeClaimName != "" {
 		vol := k8sutil.CreateVolumeWithPersitantVolumeClaim(k8sutil.ArangodVolumeName,
 			m.status.PersistentVolumeClaimName)
 
-		volumes = append(volumes, vol)
+		volumes.AddVolume(vol)
 	} else {
-		volumes = append(volumes, k8sutil.CreateVolumeEmptyDir(k8sutil.ArangodVolumeName))
+		volumes.AddVolume(k8sutil.CreateVolumeEmptyDir(k8sutil.ArangodVolumeName))
 	}
 
-	if m.tlsKeyfileSecretName != "" {
-		vol := k8sutil.CreateVolumeWithSecret(k8sutil.TlsKeyfileVolumeName, m.tlsKeyfileSecretName)
-		volumes = append(volumes, vol)
-		volumeMounts = append(volumeMounts, k8sutil.TlsKeyfileVolumeMount())
-	}
+	// TLS
+	volumes.Append(pod.TLS(), m.AsInput())
 
 	// Encryption
-	{
-		encryptionVolumes, encryptionVolumeMounts := pod.Encryption().Volumes(m.AsInput())
-
-		if len(encryptionVolumes) > 0 {
-			volumes = append(volumes, encryptionVolumes...)
-		}
-
-		if len(encryptionVolumeMounts) > 0 {
-			volumeMounts = append(volumeMounts, encryptionVolumeMounts...)
-		}
-	}
+	volumes.Append(pod.Encryption(), m.AsInput())
 
 	if m.spec.Metrics.IsEnabled() {
 		switch m.spec.Metrics.Mode.Get() {
@@ -357,43 +341,29 @@ func (m *MemberArangoDPod) GetVolumes() ([]core.Volume, []core.VolumeMount) {
 			token := m.spec.Metrics.GetJWTTokenSecretName()
 			if token != "" {
 				vol := k8sutil.CreateVolumeWithSecret(k8sutil.ExporterJWTVolumeName, token)
-				volumes = append(volumes, vol)
+				volumes.AddVolume(vol)
 			}
 		}
 	}
 
-	if m.clusterJWTSecretName != "" {
-		vol := k8sutil.CreateVolumeWithSecret(k8sutil.ClusterJWTSecretVolumeName, m.clusterJWTSecretName)
-		volumes = append(volumes, vol)
-		volumeMounts = append(volumeMounts, k8sutil.ClusterJWTVolumeMount())
-	}
+	volumes.Append(pod.JWT(), m.AsInput())
 
 	if m.resources.context.GetLifecycleImage() != "" {
-		volumes = append(volumes, k8sutil.LifecycleVolume())
+		volumes.AddVolume(k8sutil.LifecycleVolume())
 	}
 
 	// SNI
-	{
-		sniVolumes, sniVolumeMounts := pod.SNI().Volumes(m.AsInput())
-
-		if len(sniVolumes) > 0 {
-			volumes = append(volumes, sniVolumes...)
-		}
-
-		if len(sniVolumeMounts) > 0 {
-			volumeMounts = append(volumeMounts, sniVolumeMounts...)
-		}
-	}
+	volumes.Append(pod.SNI(), m.AsInput())
 
 	if len(m.groupSpec.Volumes) > 0 {
-		volumes = append(volumes, m.groupSpec.Volumes.Volumes()...)
+		volumes.AddVolume(m.groupSpec.Volumes.Volumes()...)
 	}
 
 	if len(m.groupSpec.VolumeMounts) > 0 {
-		volumeMounts = append(volumeMounts, m.groupSpec.VolumeMounts.VolumeMounts()...)
+		volumes.AddVolumeMount(m.groupSpec.VolumeMounts.VolumeMounts()...)
 	}
 
-	return volumes, volumeMounts
+	return volumes.Volumes(), volumes.VolumeMounts()
 }
 
 func (m *MemberArangoDPod) IsDeploymentMode() bool {
@@ -441,6 +411,7 @@ func (m *MemberArangoDPod) GetTolerations() []core.Toleration {
 
 func (m *MemberArangoDPod) GetContainerCreator() interfaces.ContainerCreator {
 	return &ArangoDContainer{
+		member:    m,
 		spec:      m.spec,
 		group:     m.group,
 		resources: m.resources,
@@ -473,7 +444,7 @@ func (m *MemberArangoDPod) createMetricsExporterSidecar() *core.Container {
 		c.VolumeMounts = append(c.VolumeMounts, k8sutil.ExporterJWTVolumeMount())
 	}
 
-	if m.tlsKeyfileSecretName != "" {
+	if pod.IsTLSEnabled(m.AsInput()) {
 		c.VolumeMounts = append(c.VolumeMounts, k8sutil.TlsKeyfileVolumeMount())
 	}
 

pkg/deployment/resources/secret_hashes.go

@@ -176,18 +176,6 @@ func (r *Resources) ValidateSecretHashes(cachedStatus inspector.Inspector) error
 			}
 		}
 	}
-	if spec.IsSecure() {
-		secretName := spec.TLS.GetCASecretName()
-		getExpectedHash := func() string { return getHashes().TLSCA }
-		setExpectedHash := func(h string) error {
-			return maskAny(updateHashes(func(dst *api.SecretHashes) { dst.TLSCA = h }))
-		}
-		if hashOK, err := validate(secretName, getExpectedHash, setExpectedHash, nil); err != nil {
-			return maskAny(err)
-		} else if !hashOK {
-			badSecretNames = append(badSecretNames, secretName)
-		}
-	}
 	if spec.Sync.IsEnabled() {
 		secretName := spec.Sync.TLS.GetCASecretName()
 		getExpectedHash := func() string { return getHashes().SyncTLSCA }

pkg/deployment/resources/secrets.go

@@ -29,6 +29,8 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/rs/zerolog"
+
 	operatorErrors "github.com/arangodb/kube-arangodb/pkg/util/errors"
 
 	"github.com/arangodb/kube-arangodb/pkg/deployment/resources/inspector"
@@ -53,15 +55,25 @@ var (
 	inspectSecretsDurationGauges = metrics.MustRegisterGaugeVec(metricsComponent, "inspect_secrets_duration", "Amount of time taken by a single inspection of all Secrets for a deployment (in sec)", metrics.DeploymentName)
 )
 
+const (
+	CAKeyName  = "ca.key"
+	CACertName = "ca.crt"
+)
+
+func GetCASecretName(apiObject k8sutil.APIObject) string {
+	return fmt.Sprintf("%s-truststore", apiObject.GetName())
+}
+
 // EnsureSecrets creates all secrets needed to run the given deployment
-func (r *Resources) EnsureSecrets(cachedStatus inspector.Inspector) error {
+func (r *Resources) EnsureSecrets(log zerolog.Logger, cachedStatus inspector.Inspector) error {
 	start := time.Now()
 	spec := r.context.GetSpec()
 	kubecli := r.context.GetKubeCli()
 	ns := r.context.GetNamespace()
 	secrets := kubecli.CoreV1().Secrets(ns)
 	status, _ := r.context.GetStatus()
-	deploymentName := r.context.GetAPIObject().GetName()
+	apiObject := r.context.GetAPIObject()
+	deploymentName := apiObject.GetName()
 	defer metrics.SetDuration(inspectSecretsDurationGauges.WithLabelValues(deploymentName), start)
 	counterMetric := inspectedSecretsCounters.WithLabelValues(deploymentName)
 
@@ -82,6 +94,40 @@ func (r *Resources) EnsureSecrets(cachedStatus inspector.Inspector) error {
 		if err := r.ensureTLSCACertificateSecret(cachedStatus, secrets, spec.TLS); err != nil {
 			return maskAny(err)
 		}
+
+		if err := r.ensureSecretWithEmptyKey(cachedStatus, secrets, GetCASecretName(r.context.GetAPIObject()), "empty"); err != nil {
+			return maskAny(err)
+		}
+
+		if err := status.Members.ForeachServerGroup(func(group api.ServerGroup, list api.MemberStatusList) error {
+			if !group.IsArangod() {
+				return nil
+			}
+
+			role := group.AsRole()
+
+			for _, m := range list {
+				tlsKeyfileSecretName := k8sutil.CreateTLSKeyfileSecretName(apiObject.GetName(), role, m.ID)
+				if _, exists := cachedStatus.Secret(tlsKeyfileSecretName); !exists {
+					serverNames := []string{
+						k8sutil.CreateDatabaseClientServiceDNSName(apiObject),
+						k8sutil.CreatePodDNSName(apiObject, role, m.ID),
+					}
+					if ip := spec.ExternalAccess.GetLoadBalancerIP(); ip != "" {
+						serverNames = append(serverNames, ip)
+					}
+					owner := apiObject.AsOwner()
+					if err := createTLSServerCertificate(log, secrets, serverNames, spec.TLS, tlsKeyfileSecretName, &owner); err != nil && !k8sutil.IsAlreadyExists(err) {
+						return maskAny(errors.Wrapf(err, "Failed to create TLS keyfile secret"))
+					}
+
+					return operatorErrors.Reconcile()
+				}
+			}
+			return nil
+		}); err != nil {
+			return maskAny(err)
+		}
 	}
 	if spec.RocksDB.IsEncrypted() {
 		if i := status.CurrentImage; i != nil && i.Enterprise && i.ArangoDBVersion.CompareTo("3.7.0") >= 0 {
@@ -122,6 +168,61 @@ func (r *Resources) ensureTokenSecret(cachedStatus inspector.Inspector, secrets
 	return nil
 }
 
+func (r *Resources) ensureSecret(cachedStatus inspector.Inspector, secrets k8sutil.SecretInterface, secretName string) error {
+	if _, exists := cachedStatus.Secret(secretName); !exists {
+		return r.createSecret(secrets, secretName)
+	}
+
+	return nil
+}
+
+func (r *Resources) createSecret(secrets k8sutil.SecretInterface, secretName string) error {
+	// Create secret
+	secret := &core.Secret{
+		ObjectMeta: meta.ObjectMeta{
+			Name: secretName,
+		},
+	}
+	// Attach secret to owner
+	owner := r.context.GetAPIObject().AsOwner()
+	k8sutil.AddOwnerRefToObject(secret, &owner)
+	if _, err := secrets.Create(secret); err != nil {
+		// Failed to create secret
+		return maskAny(err)
+	}
+
+	return operatorErrors.Reconcile()
+}
+
+func (r *Resources) ensureSecretWithEmptyKey(cachedStatus inspector.Inspector, secrets k8sutil.SecretInterface, secretName, keyName string) error {
+	if _, exists := cachedStatus.Secret(secretName); !exists {
+		return r.createSecretWithEmptyKey(secrets, secretName, keyName)
+	}
+
+	return nil
+}
+
+func (r *Resources) createSecretWithEmptyKey(secrets k8sutil.SecretInterface, secretName, keyName string) error {
+	// Create secret
+	secret := &core.Secret{
+		ObjectMeta: meta.ObjectMeta{
+			Name: secretName,
+		},
+		Data: map[string][]byte{
+			keyName: {},
+		},
+	}
+	// Attach secret to owner
+	owner := r.context.GetAPIObject().AsOwner()
+	k8sutil.AddOwnerRefToObject(secret, &owner)
+	if _, err := secrets.Create(secret); err != nil {
+		// Failed to create secret
+		return maskAny(err)
+	}
+
+	return operatorErrors.Reconcile()
+}
+
 func (r *Resources) createTokenSecret(secrets k8sutil.SecretInterface, secretName string) error {
 	tokenData := make([]byte, 32)
 	rand.Read(tokenData)

pkg/util/checksum.go

@@ -0,0 +1,32 @@
+//
+// DISCLAIMER
+//
+// Copyright 2020 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Adam Janikowski
+//
+
+package util
+
+import (
+	"crypto/sha256"
+	"fmt"
+)
+
+func SHA256(data []byte) string {
+	return fmt.Sprintf("%0x", sha256.Sum256(data))
+}

pkg/util/k8sutil/annotations.go

@@ -30,6 +30,24 @@ import (
 	policyTyped "k8s.io/client-go/kubernetes/typed/policy/v1beta1"
 )
 
+type OwnerRefObj interface {
+	GetOwnerReferences() []meta.OwnerReference
+}
+
+func IsOwnerFromRef(owner, ref meta.OwnerReference) bool {
+	return owner.UID == ref.UID
+}
+
+func IsOwner(ref meta.OwnerReference, object OwnerRefObj) bool {
+	for _, ownerRef := range object.GetOwnerReferences() {
+		if IsOwnerFromRef(ref, ownerRef) {
+			return true
+		}
+	}
+
+	return false
+}
+
 func IsChildResource(kind, name, namespace string, resource meta.Object) bool {
 	if resource == nil {
 		return false

pkg/util/k8sutil/pods.go

@@ -196,6 +196,7 @@ func TlsKeyfileVolumeMount() core.VolumeMount {
 	return core.VolumeMount{
 		Name:      TlsKeyfileVolumeName,
 		MountPath: TLSKeyfileVolumeMountDir,
+		ReadOnly:  true,
 	}
 }