[Feature] Encryption Key rotation (#570)

CHANGELOG.md

@@ -1,6 +1,7 @@
 # Change Log
 
 ## [master](https://github.com/arangodb/kube-arangodb/tree/master) (N/A)
+- Add Encryption Key rotation feature for ArangoDB EE 3.7+
 
 ## [1.0.3](https://github.com/arangodb/kube-arangodb/tree/1.0.3) (2020-05-25)
 - Prevent deletion of not known PVC's

pkg/apis/deployment/v1/deployment_spec.go

@@ -67,6 +67,8 @@ type DeploymentSpec struct {
 
 	RestoreFrom *string `json:"restoreFrom,omitempty"`
 
+	RestoreEncryptionSecret *string `json:"restoreEncryptionSecret,omitempty"`
+
 	// AllowUnsafeUpgrade determines if upgrade on missing member or with not in sync shards is allowed
 	AllowUnsafeUpgrade *bool `json:"allowUnsafeUpgrade,omitempty"`
 

pkg/apis/deployment/v1/plan.go

@@ -79,6 +79,12 @@ const (
 	ActionTypeBackupRestore ActionType = "BackupRestore"
 	// ActionTypeBackupRestoreClean restore plan
 	ActionTypeBackupRestoreClean ActionType = "BackupRestoreClean"
+	// ActionTypeEncryptionKeyAdd add new encryption key to list
+	ActionTypeEncryptionKeyAdd ActionType = "EncryptionKeyAdd"
+	// ActionTypeEncryptionKeyRemove removes encryption key to list
+	ActionTypeEncryptionKeyRemove ActionType = "EncryptionKeyRemove"
+	// ActionTypeEncryptionKeyRefresh refresh encryption keys
+	ActionTypeEncryptionKeyRefresh ActionType = "EncryptionKeyRefresh"
 )
 
 const (

pkg/apis/deployment/v1/zz_generated.deepcopy.go

@@ -317,6 +317,11 @@ func (in *DeploymentSpec) DeepCopyInto(out *DeploymentSpec) {
 		*out = new(string)
 		**out = **in
 	}
+	if in.RestoreEncryptionSecret != nil {
+		in, out := &in.RestoreEncryptionSecret, &out.RestoreEncryptionSecret
+		*out = new(string)
+		**out = **in
+	}
 	if in.AllowUnsafeUpgrade != nil {
 		in, out := &in.AllowUnsafeUpgrade, &out.AllowUnsafeUpgrade
 		*out = new(bool)

pkg/deployment/client/client.go

@@ -0,0 +1,152 @@
+//
+// DISCLAIMER
+//
+// Copyright 2020 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Adam Janikowski
+//
+
+package client
+
+import (
+	"context"
+	"net/http"
+
+	"github.com/arangodb/go-driver"
+)
+
+func NewClient(c driver.Connection) Client {
+	return &client{
+		c: c,
+	}
+}
+
+type Client interface {
+	GetTLS(ctx context.Context) (TLSDetails, error)
+	RefreshTLS(ctx context.Context) (TLSDetails, error)
+
+	GetEncryption(ctx context.Context) (EncryptionDetails, error)
+	RefreshEncryption(ctx context.Context) (EncryptionDetails, error)
+}
+
+type client struct {
+	c driver.Connection
+}
+
+func (c *client) parseTLSResponse(response driver.Response) (TLSDetails, error) {
+	if err := response.CheckStatus(http.StatusOK); err != nil {
+		return TLSDetails{}, err
+	}
+
+	var d TLSDetails
+
+	if err := response.ParseBody("", &d); err != nil {
+		return TLSDetails{}, err
+	}
+
+	return d, nil
+}
+
+func (c *client) parseEncryptionResponse(response driver.Response) (EncryptionDetails, error) {
+	if err := response.CheckStatus(http.StatusOK); err != nil {
+		return EncryptionDetails{}, err
+	}
+
+	var d EncryptionDetails
+
+	if err := response.ParseBody("", &d); err != nil {
+		return EncryptionDetails{}, err
+	}
+
+	return d, nil
+}
+
+func (c *client) GetTLS(ctx context.Context) (TLSDetails, error) {
+	r, err := c.c.NewRequest(http.MethodGet, "/_admin/server/tls")
+	if err != nil {
+		return TLSDetails{}, err
+	}
+
+	response, err := c.c.Do(ctx, r)
+	if err != nil {
+		return TLSDetails{}, err
+	}
+
+	d, err := c.parseTLSResponse(response)
+	if err != nil {
+		return TLSDetails{}, err
+	}
+
+	return d, nil
+}
+
+func (c *client) RefreshTLS(ctx context.Context) (TLSDetails, error) {
+	r, err := c.c.NewRequest(http.MethodPost, "/_admin/server/tls")
+	if err != nil {
+		return TLSDetails{}, err
+	}
+
+	response, err := c.c.Do(ctx, r)
+	if err != nil {
+		return TLSDetails{}, err
+	}
+
+	d, err := c.parseTLSResponse(response)
+	if err != nil {
+		return TLSDetails{}, err
+	}
+
+	return d, nil
+}
+
+func (c *client) GetEncryption(ctx context.Context) (EncryptionDetails, error) {
+	r, err := c.c.NewRequest(http.MethodGet, "/_admin/server/encryption")
+	if err != nil {
+		return EncryptionDetails{}, err
+	}
+
+	response, err := c.c.Do(ctx, r)
+	if err != nil {
+		return EncryptionDetails{}, err
+	}
+
+	d, err := c.parseEncryptionResponse(response)
+	if err != nil {
+		return EncryptionDetails{}, err
+	}
+
+	return d, nil
+}
+
+func (c *client) RefreshEncryption(ctx context.Context) (EncryptionDetails, error) {
+	r, err := c.c.NewRequest(http.MethodPost, "/_admin/server/encryption")
+	if err != nil {
+		return EncryptionDetails{}, err
+	}
+
+	response, err := c.c.Do(ctx, r)
+	if err != nil {
+		return EncryptionDetails{}, err
+	}
+
+	d, err := c.parseEncryptionResponse(response)
+	if err != nil {
+		return EncryptionDetails{}, err
+	}
+
+	return d, nil
+}

pkg/deployment/client/encryption.go

@@ -0,0 +1,56 @@
+//
+// DISCLAIMER
+//
+// Copyright 2020 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Adam Janikowski
+//
+
+package client
+
+type EncryptionKeyEntry struct {
+	Sha string `json:"sha256,omitempty"`
+}
+
+type EncryptionDetailsResult struct {
+	Keys []EncryptionKeyEntry `json:"encryption-keys,omitempty"`
+}
+
+func (e EncryptionDetailsResult) KeysPresent(m map[string][]byte) bool {
+	if len(e.Keys) != len(m) {
+		return false
+	}
+
+	for key := range m {
+		ok := false
+		for _, entry := range e.Keys {
+			if entry.Sha == key {
+				ok = true
+				break
+			}
+		}
+		if !ok {
+			return false
+		}
+	}
+
+	return true
+}
+
+type EncryptionDetails struct {
+	Result EncryptionDetailsResult `json:"result,omitempty"`
+}

pkg/deployment/client/tls.go

@@ -0,0 +1,38 @@
+//
+// DISCLAIMER
+//
+// Copyright 2020 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Adam Janikowski
+//
+
+package client
+
+type TLSKeyFile struct {
+	PrivateKeyHash string   `json:"privateKeySHA256,omitempty"`
+	Checksum       string   `json:"SHA256,omitempty"`
+	Certificates   []string `json:"certificates,omitempty"`
+}
+
+type TLSDetailsResult struct {
+	KeyFile TLSKeyFile            `json:"keyfile,omitempty"`
+	SNI     map[string]TLSKeyFile `json:"SNI,omitempty"`
+}
+
+type TLSDetails struct {
+	Result TLSDetailsResult `json:"result,omitempty"`
+}

pkg/deployment/context_impl.go

@@ -504,3 +504,7 @@ func (d *Deployment) WithStatusUpdate(action func(s *api.DeploymentStatus) bool,
 func (d *Deployment) SecretsInterface() k8sutil.SecretInterface {
 	return d.GetKubeCli().CoreV1().Secrets(d.GetNamespace())
 }
+
+func (d *Deployment) GetName() string {
+	return d.apiObject.GetName()
+}

pkg/deployment/deployment.go

@@ -83,8 +83,8 @@ type deploymentEvent struct {
 
 const (
 	deploymentEventQueueSize = 256
-	minInspectionInterval    = util.Interval(time.Second) // Ensure we inspect the generated resources no less than with this interval
-	maxInspectionInterval    = util.Interval(time.Minute) // Ensure we inspect the generated resources no less than with this interval
+	minInspectionInterval    = 250 * util.Interval(time.Millisecond) // Ensure we inspect the generated resources no less than with this interval
+	maxInspectionInterval    = 30 * util.Interval(time.Second)       // Ensure we inspect the generated resources no less than with this interval
 )
 
 // Deployment is the in process state of an ArangoDeployment.

pkg/deployment/deployment_encryption_test.go

@@ -0,0 +1,265 @@
+//
+// DISCLAIMER
+//
+// Copyright 2020 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Adam Janikowski
+//
+
+package deployment
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/arangodb/kube-arangodb/pkg/util/constants"
+
+	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/arangodb/kube-arangodb/pkg/util"
+
+	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+	core "k8s.io/api/core/v1"
+)
+
+func TestEnsurePod_ArangoDB_Encryption(t *testing.T) {
+	testCases := []testCaseStruct{
+		{
+			Name: "Agent CE 3.7.0 Pod with encrypted rocksdb",
+			ArangoDeployment: &api.ArangoDeployment{
+				Spec: api.DeploymentSpec{
+					Image:          util.NewString(testImage),
+					Authentication: noAuthentication,
+					TLS:            noTLS,
+					RocksDB:        rocksDBSpec,
+				},
+			},
+			Helper: func(t *testing.T, deployment *Deployment, testCase *testCaseStruct) {
+				deployment.status.last = api.DeploymentStatus{
+					Members: api.DeploymentStatusMembers{
+						Agents: api.MemberStatusList{
+							firstAgentStatus,
+						},
+					},
+					Images: createTestImagesWithVersion(false, "3.7.0"),
+				}
+
+				testCase.createTestPodData(deployment, api.ServerGroupAgents, firstAgentStatus)
+
+				secrets := deployment.GetKubeCli().CoreV1().Secrets(testNamespace)
+				key := make([]byte, 32)
+				k8sutil.CreateEncryptionKeySecret(secrets, testRocksDBEncryptionKey, key)
+			},
+			ExpectedEvent: "member agent is created",
+			ExpectedPod: core.Pod{
+				Spec: core.PodSpec{
+					Volumes: []core.Volume{
+						k8sutil.CreateVolumeEmptyDir(k8sutil.ArangodVolumeName),
+						k8sutil.CreateVolumeWithSecret(k8sutil.RocksdbEncryptionVolumeName, testRocksDBEncryptionKey),
+					},
+					Containers: []core.Container{
+						{
+							Name:  k8sutil.ServerContainerName,
+							Image: testImage,
+							Command: BuildTestAgentArgs(t, firstAgentStatus.ID,
+								AgentArgsWithTLS(firstAgentStatus.ID, false),
+								ArgsWithAuth(false),
+								ArgsWithEncryptionKey()),
+							Ports: createTestPorts(),
+							VolumeMounts: []core.VolumeMount{
+								k8sutil.ArangodVolumeMount(),
+								k8sutil.RocksdbEncryptionVolumeMount(),
+							},
+							Resources:       emptyResources,
+							LivenessProbe:   createTestLivenessProbe(false, "", k8sutil.ArangoPort),
+							ImagePullPolicy: core.PullIfNotPresent,
+							SecurityContext: securityContext.NewSecurityContext(),
+						},
+					},
+					RestartPolicy:                 core.RestartPolicyNever,
+					TerminationGracePeriodSeconds: &defaultAgentTerminationTimeout,
+					Hostname:                      testDeploymentName + "-" + api.ServerGroupAgentsString + "-" + firstAgentStatus.ID,
+					Subdomain:                     testDeploymentName + "-int",
+					Affinity: k8sutil.CreateAffinity(testDeploymentName, api.ServerGroupAgentsString,
+						false, ""),
+				},
+			},
+		},
+		{
+			Name: "DBserver CE 3.7.0 Pod with metrics exporter, lifecycle, tls, authentication, license, rocksDB encryption, secured liveness",
+			ArangoDeployment: &api.ArangoDeployment{
+				Spec: api.DeploymentSpec{
+					Image:          util.NewString(testImage),
+					Authentication: authenticationSpec,
+					TLS:            tlsSpec,
+					Metrics:        metricsSpec,
+					RocksDB:        rocksDBSpec,
+					Environment:    api.NewEnvironment(api.EnvironmentProduction),
+					License: api.LicenseSpec{
+						SecretName: util.NewString(testLicense),
+					},
+				},
+			},
+			Helper: func(t *testing.T, deployment *Deployment, testCase *testCaseStruct) {
+				deployment.status.last = api.DeploymentStatus{
+					Members: api.DeploymentStatusMembers{
+						DBServers: api.MemberStatusList{
+							firstDBServerStatus,
+						},
+					},
+					Images: createTestImagesWithVersion(false, "3.7.0"),
+				}
+
+				testCase.createTestPodData(deployment, api.ServerGroupDBServers, firstDBServerStatus)
+				testCase.ExpectedPod.ObjectMeta.Labels[k8sutil.LabelKeyArangoExporter] = testYes
+
+				secrets := deployment.GetKubeCli().CoreV1().Secrets(testNamespace)
+				key := make([]byte, 32)
+				k8sutil.CreateEncryptionKeySecret(secrets, testRocksDBEncryptionKey, key)
+
+				authorization, err := createTestToken(deployment, testCase, []string{"/_api/version"})
+				require.NoError(t, err)
+
+				testCase.ExpectedPod.Spec.Containers[0].LivenessProbe = createTestLivenessProbe(true,
+					authorization, k8sutil.ArangoPort)
+			},
+			config: Config{
+				LifecycleImage: testImageLifecycle,
+			},
+			ExpectedEvent: "member dbserver is created",
+			ExpectedPod: core.Pod{
+				Spec: core.PodSpec{
+					Volumes: []core.Volume{
+						k8sutil.CreateVolumeEmptyDir(k8sutil.ArangodVolumeName),
+						createTestTLSVolume(api.ServerGroupDBServersString, firstDBServerStatus.ID),
+						k8sutil.CreateVolumeWithSecret(k8sutil.RocksdbEncryptionVolumeName, testRocksDBEncryptionKey),
+						k8sutil.CreateVolumeWithSecret(k8sutil.ExporterJWTVolumeName, testExporterToken),
+						k8sutil.CreateVolumeWithSecret(k8sutil.ClusterJWTSecretVolumeName, testJWTSecretName),
+						k8sutil.LifecycleVolume(),
+					},
+					InitContainers: []core.Container{
+						createTestLifecycleContainer(emptyResources),
+					},
+					Containers: []core.Container{
+						{
+							Name:    k8sutil.ServerContainerName,
+							Image:   testImage,
+							Command: createTestCommandForDBServer(firstDBServerStatus.ID, true, true, true),
+							Env: []core.EnvVar{
+								k8sutil.CreateEnvSecretKeySelector(constants.EnvArangoLicenseKey,
+									testLicense, constants.SecretKeyToken),
+								k8sutil.CreateEnvFieldPath(constants.EnvOperatorPodName, "metadata.name"),
+								k8sutil.CreateEnvFieldPath(constants.EnvOperatorPodNamespace, "metadata.namespace"),
+								k8sutil.CreateEnvFieldPath(constants.EnvOperatorNodeName, "spec.nodeName"),
+								k8sutil.CreateEnvFieldPath(constants.EnvOperatorNodeNameArango, "spec.nodeName"),
+							},
+							Ports:           createTestPorts(),
+							Lifecycle:       createTestLifecycle(),
+							LivenessProbe:   createTestLivenessProbe(false, "", k8sutil.ArangoPort),
+							ImagePullPolicy: core.PullIfNotPresent,
+							SecurityContext: securityContext.NewSecurityContext(),
+							VolumeMounts: []core.VolumeMount{
+								k8sutil.ArangodVolumeMount(),
+								k8sutil.LifecycleVolumeMount(),
+								k8sutil.TlsKeyfileVolumeMount(),
+								k8sutil.RocksdbEncryptionVolumeMount(),
+								k8sutil.ClusterJWTVolumeMount(),
+							},
+							Resources: emptyResources,
+						},
+						func() core.Container {
+							c := testCreateExporterContainer(true, emptyResources)
+							c.VolumeMounts = append(c.VolumeMounts, k8sutil.TlsKeyfileVolumeMount())
+							return c
+						}(),
+					},
+					RestartPolicy:                 core.RestartPolicyNever,
+					TerminationGracePeriodSeconds: &defaultDBServerTerminationTimeout,
+					Hostname:                      testDeploymentName + "-" + api.ServerGroupDBServersString + "-" + firstDBServerStatus.ID,
+					Subdomain:                     testDeploymentName + "-int",
+					Affinity: k8sutil.CreateAffinity(testDeploymentName, api.ServerGroupDBServersString,
+						true, ""),
+				},
+			},
+		},
+		{
+			Name: "Agent EE 3.7.0 Pod with encrypted rocksdb",
+			ArangoDeployment: &api.ArangoDeployment{
+				Spec: api.DeploymentSpec{
+					Image:          util.NewString(testImage),
+					Authentication: noAuthentication,
+					TLS:            noTLS,
+					RocksDB:        rocksDBSpec,
+				},
+			},
+			Helper: func(t *testing.T, deployment *Deployment, testCase *testCaseStruct) {
+				deployment.status.last = api.DeploymentStatus{
+					Members: api.DeploymentStatusMembers{
+						Agents: api.MemberStatusList{
+							firstAgentStatus,
+						},
+					},
+					Images: createTestImagesWithVersion(true, "3.7.0"),
+				}
+
+				testCase.createTestPodData(deployment, api.ServerGroupAgents, firstAgentStatus)
+
+				secrets := deployment.GetKubeCli().CoreV1().Secrets(testNamespace)
+				key := make([]byte, 32)
+				k8sutil.CreateEncryptionKeySecret(secrets, testRocksDBEncryptionKey, key)
+			},
+			ExpectedEvent: "member agent is created",
+			ExpectedPod: core.Pod{
+				Spec: core.PodSpec{
+					Volumes: []core.Volume{
+						k8sutil.CreateVolumeEmptyDir(k8sutil.ArangodVolumeName),
+						k8sutil.CreateVolumeWithSecret(k8sutil.RocksdbEncryptionVolumeName, fmt.Sprintf("%s-encryption-folder", testDeploymentName)),
+					},
+					Containers: []core.Container{
+						{
+							Name:  k8sutil.ServerContainerName,
+							Image: testImage,
+							Command: BuildTestAgentArgs(t, firstAgentStatus.ID,
+								AgentArgsWithTLS(firstAgentStatus.ID, false),
+								ArgsWithAuth(false),
+								ArgsWithEncryptionFolder()),
+							Ports: createTestPorts(),
+							VolumeMounts: []core.VolumeMount{
+								k8sutil.ArangodVolumeMount(),
+								k8sutil.RocksdbEncryptionReadOnlyVolumeMount(),
+							},
+							Resources:       emptyResources,
+							LivenessProbe:   createTestLivenessProbe(false, "", k8sutil.ArangoPort),
+							ImagePullPolicy: core.PullIfNotPresent,
+							SecurityContext: securityContext.NewSecurityContext(),
+						},
+					},
+					RestartPolicy:                 core.RestartPolicyNever,
+					TerminationGracePeriodSeconds: &defaultAgentTerminationTimeout,
+					Hostname:                      testDeploymentName + "-" + api.ServerGroupAgentsString + "-" + firstAgentStatus.ID,
+					Subdomain:                     testDeploymentName + "-int",
+					Affinity: k8sutil.CreateAffinity(testDeploymentName, api.ServerGroupAgentsString,
+						false, ""),
+				},
+			},
+		},
+	}
+
+	runTestCases(t, testCases...)
+}

pkg/deployment/deployment_suite_args_test.go

@@ -0,0 +1,153 @@
+//
+// DISCLAIMER
+//
+// Copyright 2020 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Adam Janikowski
+//
+
+package deployment
+
+import (
+	"fmt"
+	"sort"
+	"testing"
+
+	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+	"github.com/arangodb/kube-arangodb/pkg/deployment/resources"
+)
+
+type TestArgs func(t *testing.T) map[string]string
+
+func buildTestArgs(t *testing.T, args ...TestArgs) map[string]string {
+	m := map[string]string{}
+
+	for _, arg := range args {
+		n := arg(t)
+		if len(n) == 0 {
+			continue
+		}
+
+		for key, value := range n {
+			m[key] = value
+		}
+	}
+
+	return m
+}
+
+func testArgsToList(m map[string]string) []string {
+	if len(m) == 0 {
+		return []string{}
+	}
+
+	r := make([]string, 0, len(m))
+	for key, value := range m {
+		r = append(r, fmt.Sprintf("--%s=%s", key, value))
+	}
+
+	sort.Strings(r)
+
+	return r
+}
+
+func buildTestArgsList(t *testing.T, args ...TestArgs) []string {
+	return testArgsToList(buildTestArgs(t, args...))
+}
+
+func BuildTestArgs(t *testing.T, args ...TestArgs) []string {
+	z := buildTestArgsList(t, args...)
+
+	return append([]string{resources.ArangoDExecutor}, z...)
+}
+
+func agentTestArgs(name string) TestArgs {
+	return func(t *testing.T) map[string]string {
+		return map[string]string{
+			"agency.activate":             "true",
+			"agency.disaster-recovery-id": name,
+			"agency.size":                 "3",
+			"agency.supervision":          "true",
+			"database.directory":          "/data",
+			"foxx.queues":                 "false",
+			"log.level":                   "INFO",
+			"log.output":                  "+",
+			"server.statistics":           "false",
+			"server.storage-engine":       "rocksdb",
+		}
+	}
+}
+
+func BuildTestAgentArgs(t *testing.T, name string, args ...TestArgs) []string {
+	return BuildTestArgs(t, append([]TestArgs{agentTestArgs(name)}, args...)...)
+}
+
+func ArgsWithAuth(auth bool) TestArgs {
+	return func(t *testing.T) map[string]string {
+		v := "true"
+		if !auth {
+			v = "false"
+		}
+
+		m := map[string]string{
+			"server.authentication": v,
+		}
+
+		if auth {
+			m["server.jwt-secret-keyfile"] = "/secrets/cluster/jwt/token"
+		}
+
+		return m
+	}
+}
+
+func AgentArgsWithTLS(name string, tls bool) TestArgs {
+	return func(t *testing.T) map[string]string {
+		p := "ssl"
+		if !tls {
+			p = "tcp"
+		}
+
+		m := map[string]string{
+			"agency.my-address": fmt.Sprintf("%s://%s-%s-%s.%s-int.%s.svc:8529", p, testDeploymentName, api.ServerGroupAgentsString, name, testDeploymentName, testNamespace),
+			"server.endpoint":   fmt.Sprintf("%s://[::]:8529", p),
+		}
+
+		if tls {
+			m["ssl.ecdh-curve"] = ""
+			m["ssl.keyfile"] = "/secrets/tls/tls.keyfile"
+		}
+
+		return m
+	}
+}
+
+func ArgsWithEncryptionKey() TestArgs {
+	return func(t *testing.T) map[string]string {
+		return map[string]string{
+			"rocksdb.encryption-keyfile": "/secrets/rocksdb/encryption/key",
+		}
+	}
+}
+
+func ArgsWithEncryptionFolder() TestArgs {
+	return func(t *testing.T) map[string]string {
+		return map[string]string{
+			"rocksdb.encryption-keyfolder": "/secrets/rocksdb/encryption",
+		}
+	}
+}

pkg/deployment/patch/item.go

@@ -0,0 +1,83 @@
+//
+// DISCLAIMER
+//
+// Copyright 2020 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Adam Janikowski
+//
+
+package patch
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+)
+
+type Operation string
+
+const (
+	AddOperation     Operation = "add"
+	ReplaceOperation Operation = "replace"
+	RemoveOperation  Operation = "remove"
+)
+
+var _ json.Marshaler = &Path{}
+
+func NewPath(items ...string) Path {
+	return items
+}
+
+type Path []string
+
+func (p Path) MarshalJSON() ([]byte, error) {
+	if len(p) == 0 {
+		return json.Marshal("/")
+	}
+	v := fmt.Sprintf("/%s", strings.Join(p, "/"))
+	return json.Marshal(v)
+}
+
+type Item struct {
+	Op    Operation   `json:"op"`
+	Path  Path        `json:"path"`
+	Value interface{} `json:"value,omitempty"`
+}
+
+func ItemAdd(path Path, value interface{}) Item {
+	return Item{
+		Op:    AddOperation,
+		Path:  path,
+		Value: value,
+	}
+}
+
+func ItemReplace(path Path, value interface{}) Item {
+	return Item{
+		Op:    ReplaceOperation,
+		Path:  path,
+		Value: value,
+	}
+}
+
+func ItemRemove(path Path) Item {
+	return Item{
+		Op:    RemoveOperation,
+		Path:  path,
+		Value: nil,
+	}
+}

pkg/deployment/patch/patch.go

@@ -0,0 +1,51 @@
+//
+// DISCLAIMER
+//
+// Copyright 2020 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Adam Janikowski
+//
+
+package patch
+
+import "encoding/json"
+
+func NewPatch(items ...Item) Patch {
+	return items
+}
+
+type Patch []Item
+
+func (p *Patch) Add(items ...Item) {
+	*p = append(*p, items...)
+}
+
+func (p *Patch) ItemAdd(path Path, value interface{}) {
+	p.Add(ItemAdd(path, value))
+}
+
+func (p *Patch) ItemReplace(path Path, value interface{}) {
+	p.Add(ItemReplace(path, value))
+}
+
+func (p *Patch) ItemRemove(path Path) {
+	p.Add(ItemRemove(path))
+}
+
+func (p Patch) Marshal() ([]byte, error) {
+	return json.Marshal(p)
+}

pkg/deployment/pod/encryption.go

@@ -0,0 +1,146 @@
+//
+// DISCLAIMER
+//
+// Copyright 2020 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Adam Janikowski
+//
+
+package pod
+
+import (
+	"crypto/sha256"
+	"fmt"
+	"path/filepath"
+
+	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+	"github.com/arangodb/kube-arangodb/pkg/util/constants"
+	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
+	"github.com/pkg/errors"
+	core "k8s.io/api/core/v1"
+	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func GroupEncryptionSupported(mode api.DeploymentMode, group api.ServerGroup) bool {
+	switch mode {
+	case api.DeploymentModeCluster:
+		switch group {
+		case api.ServerGroupDBServers, api.ServerGroupAgents:
+			return true
+		default:
+			return false
+		}
+
+	case api.DeploymentModeSingle:
+		return group == api.ServerGroupSingle
+
+	case api.DeploymentModeActiveFailover:
+		switch group {
+		case api.ServerGroupSingle, api.ServerGroupAgents:
+			return true
+		default:
+			return false
+		}
+	default:
+		return false
+	}
+}
+
+func GetEncryptionKey(secrets k8sutil.SecretInterface, name string) (string, []byte, error) {
+	keyfile, err := secrets.Get(name, meta.GetOptions{})
+	if err != nil {
+		return "", nil, errors.Wrapf(err, "Unable to fetch secret")
+	}
+
+	if len(keyfile.Data) == 0 {
+		return "", nil, errors.Errorf("Current encryption key is not valid")
+	}
+
+	d, ok := keyfile.Data[constants.SecretEncryptionKey]
+	if !ok || len(d) != 32 {
+		return "", nil, errors.Errorf("Current encryption key is not valid")
+	}
+
+	sha := fmt.Sprintf("%0x", sha256.Sum256(d))
+
+	return sha, d, nil
+}
+
+func GetKeyfolderSecretName(name string) string {
+	n := fmt.Sprintf("%s-encryption-folder", name)
+
+	return n
+}
+
+func IsEncryptionEnabled(i Input) bool {
+	return i.Deployment.RocksDB.IsEncrypted()
+}
+
+func MultiFileMode(i Input) bool {
+	return i.Enterprise && i.Version.CompareTo("3.7.0") >= 0
+}
+
+func Encryption() Builder {
+	return encryption{}
+}
+
+type encryption struct{}
+
+func (e encryption) Args(i Input) k8sutil.OptionPairs {
+	if !IsEncryptionEnabled(i) {
+		return nil
+	}
+	if !MultiFileMode(i) {
+		keyPath := filepath.Join(k8sutil.RocksDBEncryptionVolumeMountDir, constants.SecretEncryptionKey)
+		return k8sutil.NewOptionPair(k8sutil.OptionPair{
+			Key:   "--rocksdb.encryption-keyfile",
+			Value: keyPath,
+		})
+	} else {
+		return k8sutil.NewOptionPair(k8sutil.OptionPair{
+			Key:   "--rocksdb.encryption-keyfolder",
+			Value: k8sutil.RocksDBEncryptionVolumeMountDir,
+		})
+	}
+}
+
+func (e encryption) Volumes(i Input) ([]core.Volume, []core.VolumeMount) {
+	if !IsEncryptionEnabled(i) {
+		return nil, nil
+	}
+	if !MultiFileMode(i) {
+		vol := k8sutil.CreateVolumeWithSecret(k8sutil.RocksdbEncryptionVolumeName, i.Deployment.RocksDB.Encryption.GetKeySecretName())
+		return []core.Volume{vol}, []core.VolumeMount{k8sutil.RocksdbEncryptionVolumeMount()}
+	} else {
+		vol := k8sutil.CreateVolumeWithSecret(k8sutil.RocksdbEncryptionVolumeName, GetKeyfolderSecretName(i.ApiObject.GetName()))
+		return []core.Volume{vol}, []core.VolumeMount{k8sutil.RocksdbEncryptionReadOnlyVolumeMount()}
+	}
+}
+
+func (e encryption) Verify(i Input, s k8sutil.SecretInterface) error {
+	if !IsEncryptionEnabled(i) {
+		return nil
+	}
+	if !MultiFileMode(i) {
+		if err := k8sutil.ValidateEncryptionKeySecret(s, i.Deployment.RocksDB.Encryption.GetKeySecretName()); err != nil {
+			return errors.Wrapf(err, "RocksDB encryption key secret validation failed")
+		}
+		return nil
+	} else {
+		return nil
+	}
+}

pkg/deployment/reconcile/action_context.go

@@ -125,6 +125,8 @@ type ActionContext interface {
 	WithStatusUpdate(action func(s *api.DeploymentStatus) bool, force ...bool) error
 	// GetBackup receives information about a backup resource
 	GetBackup(backup string) (*backupApi.ArangoBackup, error)
+	// GetName receives information about a deployment name
+	GetName() string
 }
 
 // newActionContext creates a new ActionContext implementation.
@@ -141,6 +143,10 @@ type actionContext struct {
 	context Context
 }
 
+func (ac *actionContext) GetName() string {
+	return ac.context.GetName()
+}
+
 func (ac *actionContext) GetStatus() api.DeploymentStatus {
 	a, _ := ac.context.GetStatus()
 

pkg/deployment/reconcile/action_encryption_add.go

@@ -0,0 +1,106 @@
+//
+// DISCLAIMER
+//
+// Copyright 2020 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Adam Janikowski
+//
+
+package reconcile
+
+import (
+	"context"
+	"encoding/base64"
+
+	"github.com/arangodb/kube-arangodb/pkg/deployment/patch"
+	"github.com/arangodb/kube-arangodb/pkg/deployment/pod"
+	"github.com/pkg/errors"
+	"k8s.io/apimachinery/pkg/types"
+
+	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+	"github.com/rs/zerolog"
+)
+
+func ensureEncryptionSupport(actionCtx ActionContext) error {
+	if !actionCtx.GetSpec().RocksDB.IsEncrypted() {
+		return errors.Errorf("Encryption is disabled")
+	}
+
+	if image, ok := actionCtx.GetCurrentImageInfo(); !ok {
+		return errors.Errorf("Missing image info")
+	} else {
+		if !image.Enterprise {
+			return errors.Errorf("Supported only in enterprise")
+		}
+		if image.ArangoDBVersion.CompareTo("3.7.0") < 0 {
+			return errors.Errorf("Supported only in 3.7.0+")
+		}
+	}
+	return nil
+}
+
+func init() {
+	registerAction(api.ActionTypeEncryptionKeyAdd, newEncryptionKeyAdd)
+}
+
+func newEncryptionKeyAdd(log zerolog.Logger, action api.Action, actionCtx ActionContext) Action {
+	a := &encryptionKeyAddAction{}
+
+	a.actionImpl = newActionImplDefRef(log, action, actionCtx, defaultTimeout)
+
+	return a
+}
+
+type encryptionKeyAddAction struct {
+	actionImpl
+
+	actionEmptyCheckProgress
+}
+
+func (a *encryptionKeyAddAction) Start(ctx context.Context) (bool, error) {
+	if err := ensureEncryptionSupport(a.actionCtx); err != nil {
+		a.log.Error().Err(err).Msgf("Action not supported")
+		return true, nil
+	}
+
+	secret := a.actionCtx.GetSpec().RocksDB.Encryption.GetKeySecretName()
+	if s, ok := a.action.Params["secret"]; ok {
+		secret = s
+	}
+
+	sha, d, err := pod.GetEncryptionKey(a.actionCtx.SecretsInterface(), secret)
+	if err != nil {
+		a.log.Error().Err(err).Msgf("Unable to fetch current encryption key")
+		return true, nil
+	}
+
+	p := patch.NewPatch()
+	p.ItemAdd(patch.NewPath("data", sha), base64.StdEncoding.EncodeToString(d))
+
+	patch, err := p.Marshal()
+	if err != nil {
+		a.log.Error().Err(err).Msgf("Unable to encrypt patch")
+		return true, nil
+	}
+
+	_, err = a.actionCtx.SecretsInterface().Patch(pod.GetKeyfolderSecretName(a.actionCtx.GetAPIObject().GetName()), types.JSONPatchType, patch)
+	if err != nil {
+		return false, err
+	}
+
+	return true, nil
+}

pkg/deployment/reconcile/action_encryption_refresh.go

@@ -0,0 +1,82 @@
+//
+// DISCLAIMER
+//
+// Copyright 2020 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Adam Janikowski
+//
+
+package reconcile
+
+import (
+	"context"
+
+	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+	"github.com/arangodb/kube-arangodb/pkg/deployment/client"
+	"github.com/arangodb/kube-arangodb/pkg/deployment/pod"
+	"github.com/rs/zerolog"
+	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func init() {
+	registerAction(api.ActionTypeEncryptionKeyRefresh, newEncryptionKeyRefresh)
+}
+
+func newEncryptionKeyRefresh(log zerolog.Logger, action api.Action, actionCtx ActionContext) Action {
+	a := &encryptionKeyRefreshAction{}
+
+	a.actionImpl = newActionImplDefRef(log, action, actionCtx, defaultTimeout)
+
+	return a
+}
+
+type encryptionKeyRefreshAction struct {
+	actionImpl
+}
+
+func (a *encryptionKeyRefreshAction) Start(ctx context.Context) (bool, error) {
+	ready, _, err := a.CheckProgress(ctx)
+	return ready, err
+}
+
+func (a *encryptionKeyRefreshAction) CheckProgress(ctx context.Context) (bool, bool, error) {
+	keyfolder, err := a.actionCtx.SecretsInterface().Get(pod.GetKeyfolderSecretName(a.actionCtx.GetName()), meta.GetOptions{})
+	if err != nil {
+		a.log.Err(err).Msgf("Unable to fetch encryption folder")
+		return true, false, nil
+	}
+
+	c, err := a.actionCtx.GetServerClient(ctx, a.action.Group, a.action.MemberID)
+	if err != nil {
+		a.log.Warn().Err(err).Msg("Unable to get client")
+		return true, false, nil
+	}
+
+	client := client.NewClient(c.Connection())
+
+	e, err := client.RefreshEncryption(ctx)
+	if err != nil {
+		a.log.Warn().Err(err).Msg("Unable to refresh encryption")
+		return true, false, nil
+	}
+
+	if !e.Result.KeysPresent(keyfolder.Data) {
+		return false, false, nil
+	}
+
+	return true, false, nil
+}

pkg/deployment/reconcile/action_encryption_remove.go

@@ -0,0 +1,88 @@
+//
+// DISCLAIMER
+//
+// Copyright 2020 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Adam Janikowski
+//
+
+package reconcile
+
+import (
+	"context"
+
+	"github.com/arangodb/kube-arangodb/pkg/deployment/patch"
+	"github.com/arangodb/kube-arangodb/pkg/deployment/pod"
+	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
+	"github.com/pkg/errors"
+	"k8s.io/apimachinery/pkg/types"
+
+	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+	"github.com/rs/zerolog"
+)
+
+func init() {
+	registerAction(api.ActionTypeEncryptionKeyRemove, newEncryptionKeyRemove)
+}
+
+func newEncryptionKeyRemove(log zerolog.Logger, action api.Action, actionCtx ActionContext) Action {
+	a := &encryptionKeyRemoveAction{}
+
+	a.actionImpl = newActionImplDefRef(log, action, actionCtx, defaultTimeout)
+
+	return a
+}
+
+type encryptionKeyRemoveAction struct {
+	actionImpl
+
+	actionEmptyCheckProgress
+}
+
+func (a *encryptionKeyRemoveAction) Start(ctx context.Context) (bool, error) {
+	if err := ensureEncryptionSupport(a.actionCtx); err != nil {
+		a.log.Error().Err(err).Msgf("Action not supported")
+		return true, nil
+	}
+
+	if len(a.action.Params) == 0 {
+		return true, nil
+	}
+
+	key, ok := a.action.Params["key"]
+	if !ok {
+		return true, nil
+	}
+
+	p := patch.NewPatch()
+	p.ItemRemove(patch.NewPath("data", key))
+
+	patch, err := p.Marshal()
+	if err != nil {
+		a.log.Error().Err(err).Msgf("Unable to encrypt patch")
+		return true, nil
+	}
+
+	_, err = a.actionCtx.SecretsInterface().Patch(pod.GetKeyfolderSecretName(a.actionCtx.GetAPIObject().GetName()), types.JSONPatchType, patch)
+	if err != nil {
+		if !k8sutil.IsInvalid(err) {
+			return false, errors.Wrapf(err, "Unable to update secret: %s", string(patch))
+		}
+	}
+
+	return true, nil
+}

pkg/deployment/reconcile/action_wait_for_member_in_sync.go

@@ -63,6 +63,12 @@ func (a *actionWaitForMemberInSync) Start(ctx context.Context) (bool, error) {
 // CheckProgress checks the progress of the action.
 // Returns true if the action is completely finished, false otherwise.
 func (a *actionWaitForMemberInSync) CheckProgress(ctx context.Context) (bool, bool, error) {
+	member, ok := a.actionCtx.GetMemberStatusByID(a.MemberID())
+	if !ok || member.Phase == api.MemberPhaseFailed {
+		a.log.Debug().Msg("Member in failed phase")
+		return true, false, nil
+	}
+
 	ready, err := a.check(ctx)
 	if err != nil {
 		return false, false, err

pkg/deployment/reconcile/action_wait_for_member_up.go

@@ -67,6 +67,11 @@ func (a *actionWaitForMemberUp) Start(ctx context.Context) (bool, error) {
 // CheckProgress checks the progress of the action.
 // Returns true if the action is completely finished, false otherwise.
 func (a *actionWaitForMemberUp) CheckProgress(ctx context.Context) (bool, bool, error) {
+	member, ok := a.actionCtx.GetMemberStatusByID(a.MemberID())
+	if !ok || member.Phase == api.MemberPhaseFailed {
+		a.log.Debug().Msg("Member in failed phase")
+		return true, false, nil
+	}
 	if a.action.Group.IsArangosync() {
 		return a.checkProgressArangoSync(ctx)
 	}

pkg/deployment/reconcile/context.go

@@ -119,4 +119,6 @@ type Context interface {
 	SecretsInterface() k8sutil.SecretInterface
 	// GetBackup receives information about a backup resource
 	GetBackup(backup string) (*backupApi.ArangoBackup, error)
+	// GetName receives deployment name
+	GetName() string
 }

pkg/deployment/reconcile/helper_tls_sni.go

@@ -29,7 +29,7 @@ import (
 
 	"github.com/arangodb/go-driver"
 	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
-	"github.com/arangodb/kube-arangodb/pkg/deployment/tls"
+	"github.com/arangodb/kube-arangodb/pkg/deployment/client"
 	"github.com/arangodb/kube-arangodb/pkg/util/constants"
 	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
 	"github.com/pkg/errors"
@@ -70,7 +70,7 @@ func mapTLSSNIConfig(log zerolog.Logger, sni api.TLSSNISpec, secrets k8sutil.Sec
 }
 
 func compareTLSSNIConfig(ctx context.Context, c driver.Connection, m map[string]string, refresh bool) (bool, error) {
-	tlsClient := tls.NewClient(c)
+	tlsClient := client.NewClient(c)
 
 	f := tlsClient.GetTLS
 	if refresh {

pkg/deployment/reconcile/plan_builder.go

@@ -213,6 +213,11 @@ func createPlan(ctx context.Context, log zerolog.Logger, apiObject k8sutil.APIOb
 		}
 	}
 
+	// Add encryption keys
+	if plan.IsEmpty() {
+		plan = createEncryptionKey(ctx, log, spec, status, builderCtx)
+	}
+
 	// Check for the need to rotate TLS certificate of a members
 	if plan.IsEmpty() {
 		plan = createRotateTLSServerCertificatePlan(log, spec, status, builderCtx.GetTLSKeyfile)
@@ -236,6 +241,10 @@ func createPlan(ctx context.Context, log zerolog.Logger, apiObject k8sutil.APIOb
 		plan = createRestorePlan(ctx, log, spec, status, builderCtx)
 	}
 
+	if plan.IsEmpty() {
+		plan = cleanEncryptionKey(ctx, log, spec, status, builderCtx)
+	}
+
 	// Return plan
 	return plan, true
 }

pkg/deployment/reconcile/plan_builder_context.go

@@ -65,6 +65,8 @@ type PlanBuilderContext interface {
 	SecretsInterface() k8sutil.SecretInterface
 	// GetBackup receives information about a backup resource
 	GetBackup(backup string) (*backupApi.ArangoBackup, error)
+	// GetName receives deployment name
+	GetName() string
 }
 
 // newPlanBuilderContext creates a PlanBuilderContext from the given context

pkg/deployment/reconcile/plan_builder_encryption.go

@@ -0,0 +1,161 @@
+//
+// DISCLAIMER
+//
+// Copyright 2020 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Adam Janikowski
+//
+
+package reconcile
+
+import (
+	"context"
+
+	"github.com/arangodb/kube-arangodb/pkg/deployment/client"
+	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/arangodb/kube-arangodb/pkg/deployment/pod"
+
+	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+	"github.com/rs/zerolog"
+)
+
+func createEncryptionKey(ctx context.Context, log zerolog.Logger, spec api.DeploymentSpec, status api.DeploymentStatus, context PlanBuilderContext) api.Plan {
+	if !spec.RocksDB.IsEncrypted() {
+		return nil
+	}
+
+	if i := status.CurrentImage; i == nil || !i.Enterprise || i.ArangoDBVersion.CompareTo("3.7.0") < 0 {
+		return nil
+	}
+
+	name, _, err := pod.GetEncryptionKey(context.SecretsInterface(), spec.RocksDB.Encryption.GetKeySecretName())
+	if err != nil {
+		log.Err(err).Msgf("Unable to fetch encryption key")
+		return nil
+	}
+
+	keyfolder, err := context.SecretsInterface().Get(pod.GetKeyfolderSecretName(context.GetName()), meta.GetOptions{})
+	if err != nil {
+		log.Err(err).Msgf("Unable to fetch encryption folder")
+		return nil
+	}
+
+	if len(keyfolder.Data) == 0 {
+		keyfolder.Data = map[string][]byte{}
+	}
+
+	_, ok := keyfolder.Data[name]
+	if !ok {
+		return api.Plan{api.NewAction(api.ActionTypeEncryptionKeyAdd, api.ServerGroupUnknown, "")}
+	}
+
+	var plan api.Plan
+	status.Members.ForeachServerGroup(func(group api.ServerGroup, members api.MemberStatusList) error {
+		if !pod.GroupEncryptionSupported(spec.Mode.Get(), group) {
+			return nil
+		}
+
+		glog := log.With().Str("group", group.AsRole())
+
+		for _, m := range members {
+			if m.Phase != api.MemberPhaseCreated {
+				// Only make changes when phase is created
+				continue
+			}
+
+			if m.ArangoVersion.CompareTo("3.7.0") < 0 {
+				continue
+			}
+
+			mlog := glog.Str("member", m.ID).Logger()
+
+			c, err := context.GetServerClient(ctx, group, m.ID)
+			if err != nil {
+				mlog.Warn().Err(err).Msg("Unable to get client")
+				continue
+			}
+
+			client := client.NewClient(c.Connection())
+
+			e, err := client.GetEncryption(ctx)
+			if err != nil {
+				mlog.Error().Err(err).Msgf("Unable to fetch encryption keys")
+				continue
+			}
+
+			if !e.Result.KeysPresent(keyfolder.Data) {
+				plan = append(plan, api.NewAction(api.ActionTypeEncryptionKeyRefresh, group, m.ID))
+				mlog.Info().Msgf("Refresh of encryption keys required")
+				continue
+			}
+		}
+
+		return nil
+	})
+
+	if !plan.IsEmpty() {
+		return plan
+	}
+
+	return api.Plan{}
+}
+
+func cleanEncryptionKey(ctx context.Context, log zerolog.Logger, spec api.DeploymentSpec, status api.DeploymentStatus, context PlanBuilderContext) api.Plan {
+	if !spec.RocksDB.IsEncrypted() {
+		return nil
+	}
+
+	if i := status.CurrentImage; i == nil || !i.Enterprise || i.ArangoDBVersion.CompareTo("3.7.0") < 0 {
+		return nil
+	}
+
+	keyfolder, err := context.SecretsInterface().Get(pod.GetKeyfolderSecretName(context.GetName()), meta.GetOptions{})
+	if err != nil {
+		log.Err(err).Msgf("Unable to fetch encryption folder")
+		return nil
+	}
+
+	if len(keyfolder.Data) <= 1 {
+		return nil
+	}
+
+	name, _, err := pod.GetEncryptionKey(context.SecretsInterface(), spec.RocksDB.Encryption.GetKeySecretName())
+	if err != nil {
+		log.Err(err).Msgf("Unable to fetch encryption key")
+		return nil
+	}
+
+	if _, ok := keyfolder.Data[name]; !ok {
+		log.Err(err).Msgf("Key from encryption is not in keyfolder - do nothing")
+		return nil
+	}
+
+	var plan api.Plan
+
+	for key := range keyfolder.Data {
+		if key != name {
+			plan = append(plan, api.NewAction(api.ActionTypeEncryptionKeyRemove, api.ServerGroupUnknown, "").AddParam("key", key))
+		}
+	}
+
+	if !plan.IsEmpty() {
+		return plan
+	}
+
+	return api.Plan{}
+}

pkg/deployment/reconcile/plan_builder_restore.go

@@ -25,6 +25,10 @@ package reconcile
 import (
 	"context"
 
+	backupv1 "github.com/arangodb/kube-arangodb/pkg/apis/backup/v1"
+	"github.com/arangodb/kube-arangodb/pkg/deployment/pod"
+	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
+
 	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
 	"github.com/rs/zerolog"
 )
@@ -43,6 +47,10 @@ func createRestorePlan(ctx context.Context, log zerolog.Logger, spec api.Deploym
 			return nil
 		}
 
+		if p := createRestorePlanEncryption(ctx, log, spec, status, builderCtx, backup); !p.IsEmpty() {
+			return p
+		}
+
 		if backup.Status.Backup == nil {
 			log.Warn().Msg("Backup not yet ready")
 			return nil
@@ -55,3 +63,47 @@ func createRestorePlan(ctx context.Context, log zerolog.Logger, spec api.Deploym
 
 	return nil
 }
+
+func createRestorePlanEncryption(ctx context.Context, log zerolog.Logger, spec api.DeploymentSpec, status api.DeploymentStatus, builderCtx PlanBuilderContext, backup *backupv1.ArangoBackup) api.Plan {
+	if spec.RestoreEncryptionSecret != nil {
+		if !spec.RocksDB.IsEncrypted() {
+			return nil
+		}
+
+		if i := status.CurrentImage; i == nil || !i.Enterprise || i.ArangoDBVersion.CompareTo("3.7.0") < 0 {
+			return nil
+		}
+
+		secret := *spec.RestoreEncryptionSecret
+
+		// Additional logic to do restore with encryption key
+		keyfolder, err := builderCtx.SecretsInterface().Get(pod.GetKeyfolderSecretName(builderCtx.GetName()), meta.GetOptions{})
+		if err != nil {
+			log.Err(err).Msgf("Unable to fetch encryption folder")
+			return nil
+		}
+
+		if len(keyfolder.Data) == 0 {
+			return api.Plan{
+				api.NewAction(api.ActionTypeEncryptionKeyAdd, api.ServerGroupUnknown, "").AddParam("secret", secret),
+			}
+		}
+		name, _, err := pod.GetEncryptionKey(builderCtx.SecretsInterface(), secret)
+		if err != nil {
+			log.Err(err).Msgf("Unable to fetch encryption key")
+			return nil
+		}
+
+		if _, ok := keyfolder.Data[name]; !ok {
+			log.Err(err).Msgf("Key from encryption is not in keyfolder")
+
+			return api.Plan{
+				api.NewAction(api.ActionTypeEncryptionKeyAdd, api.ServerGroupUnknown, "").AddParam("secret", secret),
+			}
+		}
+
+		return nil
+	}
+
+	return nil
+}

pkg/deployment/reconcile/plan_builder_test.go

@@ -57,6 +57,10 @@ type testContext struct {
 	RecordedEvent    *k8sutil.Event
 }
 
+func (c *testContext) GetName() string {
+	panic("implement me")
+}
+
 func (c *testContext) GetBackup(backup string) (*backupApi.ArangoBackup, error) {
 	panic("implement me")
 }

pkg/deployment/reconcile/timeouts.go

@@ -39,6 +39,7 @@ const (
 	upgradeMemberTimeout         = time.Hour * 6
 	waitForMemberUpTimeout       = time.Minute * 30
 	tlsSNIUpdateTimeout          = time.Minute * 10
+	defaultTimeout               = time.Minute * 10
 
 	shutdownTimeout = time.Second * 15
 )

pkg/deployment/resources/pod_creator.go

@@ -107,10 +107,7 @@ func createArangodArgs(input pod.Input) []string {
 	}
 
 	// RocksDB
-	if input.Deployment.RocksDB.IsEncrypted() {
-		keyPath := filepath.Join(k8sutil.RocksDBEncryptionVolumeMountDir, constants.SecretEncryptionKey)
-		options.Add("--rocksdb.encryption-keyfile", keyPath)
-	}
+	options.Merge(pod.Encryption().Args(input))
 
 	options.Add("--database.directory", k8sutil.ArangodVolumeMountDir)
 	options.Add("--log.output", "+")

pkg/deployment/resources/pod_creator_arangod.go

@@ -322,10 +322,17 @@ func (m *MemberArangoDPod) GetVolumes() ([]core.Volume, []core.VolumeMount) {
 		volumeMounts = append(volumeMounts, k8sutil.TlsKeyfileVolumeMount())
 	}
 
-	if m.rocksdbEncryptionSecretName != "" {
-		vol := k8sutil.CreateVolumeWithSecret(k8sutil.RocksdbEncryptionVolumeName, m.rocksdbEncryptionSecretName)
-		volumes = append(volumes, vol)
-		volumeMounts = append(volumeMounts, k8sutil.RocksdbEncryptionVolumeMount())
+	// Encryption
+	{
+		encryptionVolumes, encryptionVolumeMounts := pod.Encryption().Volumes(m.AsInput())
+
+		if len(encryptionVolumes) > 0 {
+			volumes = append(volumes, encryptionVolumes...)
+		}
+
+		if len(encryptionVolumeMounts) > 0 {
+			volumeMounts = append(volumeMounts, encryptionVolumeMounts...)
+		}
 	}
 
 	if m.spec.Metrics.IsEnabled() {

pkg/deployment/resources/secret_hashes.go

@@ -110,6 +110,7 @@ func (r *Resources) ValidateSecretHashes() error {
 	log := r.log
 	var badSecretNames []string
 	status, lastVersion := r.context.GetStatus()
+	image := status.CurrentImage
 	getHashes := func() *api.SecretHashes {
 		if status.SecretHashes == nil {
 			status.SecretHashes = api.NewEmptySecretHashes()
@@ -148,15 +149,17 @@ func (r *Resources) ValidateSecretHashes() error {
 		}
 	}
 	if spec.RocksDB.IsEncrypted() {
-		secretName := spec.RocksDB.Encryption.GetKeySecretName()
-		getExpectedHash := func() string { return getHashes().RocksDBEncryptionKey }
-		setExpectedHash := func(h string) error {
-			return maskAny(updateHashes(func(dst *api.SecretHashes) { dst.RocksDBEncryptionKey = h }))
-		}
-		if hashOK, err := validate(secretName, getExpectedHash, setExpectedHash, nil); err != nil {
-			return maskAny(err)
-		} else if !hashOK {
-			badSecretNames = append(badSecretNames, secretName)
+		if image == nil || image.ArangoDBVersion.CompareTo("3.7.0") < 0 {
+			secretName := spec.RocksDB.Encryption.GetKeySecretName()
+			getExpectedHash := func() string { return getHashes().RocksDBEncryptionKey }
+			setExpectedHash := func(h string) error {
+				return maskAny(updateHashes(func(dst *api.SecretHashes) { dst.RocksDBEncryptionKey = h }))
+			}
+			if hashOK, err := validate(secretName, getExpectedHash, setExpectedHash, nil); err != nil {
+				return maskAny(err)
+			} else if !hashOK {
+				badSecretNames = append(badSecretNames, secretName)
+			}
 		}
 	}
 	if spec.IsSecure() {

pkg/deployment/resources/secrets.go

@@ -27,12 +27,15 @@ import (
 	"encoding/hex"
 	"time"
 
+	"github.com/arangodb/kube-arangodb/pkg/deployment/pod"
+	"github.com/pkg/errors"
+
 	"github.com/arangodb/kube-arangodb/pkg/util/constants"
 	jg "github.com/dgrijalva/jwt-go"
 	"k8s.io/apimachinery/pkg/api/equality"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
 	"github.com/arangodb/kube-arangodb/pkg/metrics"
@@ -51,6 +54,7 @@ func (r *Resources) EnsureSecrets() error {
 	ns := r.context.GetNamespace()
 	secrets := k8sutil.NewSecretCache(kubecli.CoreV1().Secrets(ns))
 	spec := r.context.GetSpec()
+	status, _ := r.context.GetStatus()
 	deploymentName := r.context.GetAPIObject().GetName()
 	defer metrics.SetDuration(inspectSecretsDurationGauges.WithLabelValues(deploymentName), start)
 	counterMetric := inspectedSecretsCounters.WithLabelValues(deploymentName)
@@ -73,6 +77,13 @@ func (r *Resources) EnsureSecrets() error {
 			return maskAny(err)
 		}
 	}
+	if spec.RocksDB.IsEncrypted() {
+		if i := status.CurrentImage; i != nil && i.Enterprise && i.ArangoDBVersion.CompareTo("3.7.0") >= 0 {
+			if err := r.ensureEncryptionKeyfolderSecret(secrets, spec.RocksDB.Encryption.GetKeySecretName(), pod.GetKeyfolderSecretName(deploymentName)); err != nil {
+				return maskAny(err)
+			}
+		}
+	}
 	if spec.Sync.IsEnabled() {
 		counterMetric.Inc()
 		if err := r.ensureTokenSecret(secrets, spec.Sync.Authentication.GetJWTSecretName()); err != nil {
@@ -98,7 +109,7 @@ func (r *Resources) EnsureSecrets() error {
 // of the deployment. If not, it will add such a secret with a random
 // token.
 func (r *Resources) ensureTokenSecret(secrets k8sutil.SecretInterface, secretName string) error {
-	if _, err := secrets.Get(secretName, metav1.GetOptions{}); k8sutil.IsNotFound(err) {
+	if _, err := secrets.Get(secretName, meta.GetOptions{}); k8sutil.IsNotFound(err) {
 		// Secret not found, create it
 		// Create token
 		tokenData := make([]byte, 32)
@@ -121,6 +132,28 @@ func (r *Resources) ensureTokenSecret(secrets k8sutil.SecretInterface, secretNam
 	return nil
 }
 
+func (r *Resources) ensureEncryptionKeyfolderSecret(secrets k8sutil.SecretInterface, keyfileSecretName, secretName string) error {
+	keyfile, err := secrets.Get(keyfileSecretName, meta.GetOptions{})
+	if err != nil {
+		return errors.Wrapf(err, "Unable to find original secret")
+	}
+
+	if len(keyfile.Data) == 0 {
+		return errors.Errorf("Missing key in secret")
+	}
+
+	d, ok := keyfile.Data[constants.SecretEncryptionKey]
+	if !ok {
+		return errors.Errorf("Missing key in secret")
+	}
+
+	owner := r.context.GetAPIObject().AsOwner()
+	if err = k8sutil.AppendKeyfileToKeyfolder(secrets, &owner, secretName, d); err != nil {
+		return errors.Wrapf(err, "Unable to create keyfolder secret")
+	}
+	return nil
+}
+
 var (
 	exporterTokenClaims = map[string]interface{}{
 		"iss":           "arangodb",
@@ -155,7 +188,7 @@ func (r *Resources) ensureExporterTokenSecret(secrets k8sutil.SecretInterface, t
 }
 
 func (r *Resources) ensureExporterTokenSecretCreateRequired(secrets k8sutil.SecretInterface, tokenSecretName, secretSecretName string) (bool, bool, error) {
-	if secret, err := secrets.Get(tokenSecretName, metav1.GetOptions{}); k8sutil.IsNotFound(err) {
+	if secret, err := secrets.Get(tokenSecretName, meta.GetOptions{}); k8sutil.IsNotFound(err) {
 		return true, false, nil
 	} else if err == nil {
 		// Check if claims are fine
@@ -192,7 +225,7 @@ func (r *Resources) ensureExporterTokenSecretCreateRequired(secrets k8sutil.Secr
 // ensureTLSCACertificateSecret checks if a secret with given name exists in the namespace
 // of the deployment. If not, it will add such a secret with a generated CA certificate.
 func (r *Resources) ensureTLSCACertificateSecret(secrets k8sutil.SecretInterface, spec api.TLSSpec) error {
-	if _, err := secrets.Get(spec.GetCASecretName(), metav1.GetOptions{}); k8sutil.IsNotFound(err) {
+	if _, err := secrets.Get(spec.GetCASecretName(), meta.GetOptions{}); k8sutil.IsNotFound(err) {
 		// Secret not found, create it
 		apiObject := r.context.GetAPIObject()
 		owner := apiObject.AsOwner()
@@ -214,7 +247,7 @@ func (r *Resources) ensureTLSCACertificateSecret(secrets k8sutil.SecretInterface
 // ensureClientAuthCACertificateSecret checks if a secret with given name exists in the namespace
 // of the deployment. If not, it will add such a secret with a generated CA certificate.
 func (r *Resources) ensureClientAuthCACertificateSecret(secrets k8sutil.SecretInterface, spec api.SyncAuthenticationSpec) error {
-	if _, err := secrets.Get(spec.GetClientCASecretName(), metav1.GetOptions{}); k8sutil.IsNotFound(err) {
+	if _, err := secrets.Get(spec.GetClientCASecretName(), meta.GetOptions{}); k8sutil.IsNotFound(err) {
 		// Secret not found, create it
 		apiObject := r.context.GetAPIObject()
 		owner := apiObject.AsOwner()

pkg/deployment/tls/client.go

@@ -1,112 +0,0 @@
-//
-// DISCLAIMER
-//
-// Copyright 2020 ArangoDB GmbH, Cologne, Germany
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-// Copyright holder is ArangoDB GmbH, Cologne, Germany
-//
-// Author Adam Janikowski
-//
-
-package tls
-
-import (
-	"context"
-	"net/http"
-
-	"github.com/arangodb/go-driver"
-)
-
-type KeyFile struct {
-	PrivateKeyChecksum string   `json:"privateKeySHA256,omitempty"`
-	Checksum           string   `json:"SHA256,omitempty"`
-	Certificates       []string `json:"certificates,omitempty"`
-}
-
-type DetailsResult struct {
-	KeyFile KeyFile            `json:"keyfile,omitempty"`
-	SNI     map[string]KeyFile `json:"SNI,omitempty"`
-}
-
-type Details struct {
-	Result DetailsResult `json:"result,omitempty"`
-}
-
-func NewClient(c driver.Connection) Client {
-	return &client{
-		c: c,
-	}
-}
-
-type Client interface {
-	GetTLS(ctx context.Context) (Details, error)
-	RefreshTLS(ctx context.Context) (Details, error)
-}
-
-type client struct {
-	c driver.Connection
-}
-
-func (c *client) parseResponse(response driver.Response) (Details, error) {
-	if err := response.CheckStatus(http.StatusOK); err != nil {
-		return Details{}, err
-	}
-
-	var d Details
-
-	if err := response.ParseBody("", &d); err != nil {
-		return Details{}, err
-	}
-
-	return d, nil
-}
-
-func (c *client) GetTLS(ctx context.Context) (Details, error) {
-	r, err := c.c.NewRequest(http.MethodGet, "/_admin/server/tls")
-	if err != nil {
-		return Details{}, err
-	}
-
-	response, err := c.c.Do(ctx, r)
-	if err != nil {
-		return Details{}, err
-	}
-
-	d, err := c.parseResponse(response)
-	if err != nil {
-		return Details{}, err
-	}
-
-	return d, nil
-}
-
-func (c *client) RefreshTLS(ctx context.Context) (Details, error) {
-	r, err := c.c.NewRequest(http.MethodPost, "/_admin/server/tls")
-	if err != nil {
-		return Details{}, err
-	}
-
-	response, err := c.c.Do(ctx, r)
-	if err != nil {
-		return Details{}, err
-	}
-
-	d, err := c.parseResponse(response)
-	if err != nil {
-		return Details{}, err
-	}
-
-	return d, nil
-}

pkg/util/k8sutil/errors.go

@@ -48,3 +48,9 @@ func IsConflict(err error) bool {
 func IsNotFound(err error) bool {
 	return apierrors.IsNotFound(errors.Cause(err))
 }
+
+// IsNotFound returns true if the given error is or is caused by a
+// kubernetes InvalidError,
+func IsInvalid(err error) bool {
+	return apierrors.IsInvalid(errors.Cause(err))
+}

pkg/util/k8sutil/pods.go

@@ -268,6 +268,15 @@ func RocksdbEncryptionVolumeMount() core.VolumeMount {
 	}
 }
 
+// RocksdbEncryptionReadOnlyVolumeMount creates a volume mount structure for a RocksDB encryption key.
+func RocksdbEncryptionReadOnlyVolumeMount() core.VolumeMount {
+	return core.VolumeMount{
+		Name:      RocksdbEncryptionVolumeName,
+		MountPath: RocksDBEncryptionVolumeMountDir,
+		ReadOnly:  true,
+	}
+}
+
 // ArangodInitContainer creates a container configured to initalize a UUID file.
 func ArangodInitContainer(name, id, engine, executable, operatorImage string, requireUUID bool, securityContext *core.SecurityContext) core.Container {
 	uuidFile := filepath.Join(ArangodVolumeMountDir, "UUID")

pkg/util/k8sutil/secrets.go

@@ -23,10 +23,13 @@
 package k8sutil
 
 import (
+	"crypto/sha256"
 	"fmt"
 
-	v1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	goErrors "github.com/pkg/errors"
+	core "k8s.io/api/core/v1"
+	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
 
 	"github.com/arangodb/kube-arangodb/pkg/util/constants"
 	jg "github.com/dgrijalva/jwt-go"
@@ -34,15 +37,17 @@ import (
 
 // SecretInterface has methods to work with Secret resources.
 type SecretInterface interface {
-	Create(*v1.Secret) (*v1.Secret, error)
-	Get(name string, options metav1.GetOptions) (*v1.Secret, error)
-	Delete(name string, options *metav1.DeleteOptions) error
+	Create(*core.Secret) (*core.Secret, error)
+	Update(*core.Secret) (*core.Secret, error)
+	Get(name string, options meta.GetOptions) (*core.Secret, error)
+	Delete(name string, options *meta.DeleteOptions) error
+	Patch(name string, pt types.PatchType, data []byte, subresources ...string) (*core.Secret, error)
 }
 
 // ValidateEncryptionKeySecret checks that a secret with given name in given namespace
 // exists and it contains a 'key' data field of exactly 32 bytes.
 func ValidateEncryptionKeySecret(secrets SecretInterface, secretName string) error {
-	s, err := secrets.Get(secretName, metav1.GetOptions{})
+	s, err := secrets.Get(secretName, meta.GetOptions{})
 	if err != nil {
 		return maskAny(err)
 	}
@@ -63,8 +68,8 @@ func CreateEncryptionKeySecret(secrets SecretInterface, secretName string, key [
 		return maskAny(fmt.Errorf("Key in secret '%s' is expected to be 32 bytes long, got %d", secretName, len(key)))
 	}
 	// Create secret
-	secret := &v1.Secret{
-		ObjectMeta: metav1.ObjectMeta{
+	secret := &core.Secret{
+		ObjectMeta: meta.ObjectMeta{
 			Name: secretName,
 		},
 		Data: map[string][]byte{
@@ -81,7 +86,7 @@ func CreateEncryptionKeySecret(secrets SecretInterface, secretName string, key [
 // ValidateCACertificateSecret checks that a secret with given name in given namespace
 // exists and it contains a 'ca.crt' data field.
 func ValidateCACertificateSecret(secrets SecretInterface, secretName string) error {
-	s, err := secrets.Get(secretName, metav1.GetOptions{})
+	s, err := secrets.Get(secretName, meta.GetOptions{})
 	if err != nil {
 		return maskAny(err)
 	}
@@ -99,7 +104,7 @@ func ValidateCACertificateSecret(secrets SecretInterface, secretName string) err
 // an error is returned.
 // Returns: certificate, error
 func GetCACertficateSecret(secrets SecretInterface, secretName string) (string, error) {
-	s, err := secrets.Get(secretName, metav1.GetOptions{})
+	s, err := secrets.Get(secretName, meta.GetOptions{})
 	if err != nil {
 		return "", maskAny(err)
 	}
@@ -116,8 +121,8 @@ func GetCACertficateSecret(secrets SecretInterface, secretName string) (string,
 // If the secret does not exists or one of the fields is missing,
 // an error is returned.
 // Returns: certificate, private-key, isOwnedByDeployment, error
-func GetCASecret(secrets SecretInterface, secretName string, ownerRef *metav1.OwnerReference) (string, string, bool, error) {
-	s, err := secrets.Get(secretName, metav1.GetOptions{})
+func GetCASecret(secrets SecretInterface, secretName string, ownerRef *meta.OwnerReference) (string, string, bool, error) {
+	s, err := secrets.Get(secretName, meta.GetOptions{})
 	if err != nil {
 		return "", "", false, maskAny(err)
 	}
@@ -143,10 +148,10 @@ func GetCASecret(secrets SecretInterface, secretName string, ownerRef *metav1.Ow
 }
 
 // CreateCASecret creates a secret used to store a PEM encoded CA certificate & private key.
-func CreateCASecret(secrets SecretInterface, secretName string, certificate, key string, ownerRef *metav1.OwnerReference) error {
+func CreateCASecret(secrets SecretInterface, secretName string, certificate, key string, ownerRef *meta.OwnerReference) error {
 	// Create secret
-	secret := &v1.Secret{
-		ObjectMeta: metav1.ObjectMeta{
+	secret := &core.Secret{
+		ObjectMeta: meta.ObjectMeta{
 			Name: secretName,
 		},
 		Data: map[string][]byte{
@@ -167,7 +172,7 @@ func CreateCASecret(secrets SecretInterface, secretName string, certificate, key
 // in the format ArangoDB accepts it for its `--ssl.keyfile` option.
 // Returns: keyfile (pem encoded), error
 func GetTLSKeyfileSecret(secrets SecretInterface, secretName string) (string, error) {
-	s, err := secrets.Get(secretName, metav1.GetOptions{})
+	s, err := secrets.Get(secretName, meta.GetOptions{})
 	if err != nil {
 		return "", maskAny(err)
 	}
@@ -181,10 +186,10 @@ func GetTLSKeyfileSecret(secrets SecretInterface, secretName string) (string, er
 
 // CreateTLSKeyfileSecret creates a secret used to store a PEM encoded keyfile
 // in the format ArangoDB accepts it for its `--ssl.keyfile` option.
-func CreateTLSKeyfileSecret(secrets SecretInterface, secretName string, keyfile string, ownerRef *metav1.OwnerReference) error {
+func CreateTLSKeyfileSecret(secrets SecretInterface, secretName string, keyfile string, ownerRef *meta.OwnerReference) error {
 	// Create secret
-	secret := &v1.Secret{
-		ObjectMeta: metav1.ObjectMeta{
+	secret := &core.Secret{
+		ObjectMeta: meta.ObjectMeta{
 			Name: secretName,
 		},
 		Data: map[string][]byte{
@@ -203,7 +208,7 @@ func CreateTLSKeyfileSecret(secrets SecretInterface, secretName string, keyfile
 // ValidateTokenSecret checks that a secret with given name in given namespace
 // exists and it contains a 'token' data field.
 func ValidateTokenSecret(secrets SecretInterface, secretName string) error {
-	s, err := secrets.Get(secretName, metav1.GetOptions{})
+	s, err := secrets.Get(secretName, meta.GetOptions{})
 	if err != nil {
 		return maskAny(err)
 	}
@@ -217,7 +222,7 @@ func ValidateTokenSecret(secrets SecretInterface, secretName string) error {
 
 // GetTokenSecret loads the token secret from a Secret with given name.
 func GetTokenSecret(secrets SecretInterface, secretName string) (string, error) {
-	s, err := secrets.Get(secretName, metav1.GetOptions{})
+	s, err := secrets.Get(secretName, meta.GetOptions{})
 	if err != nil {
 		return "", maskAny(err)
 	}
@@ -231,10 +236,10 @@ func GetTokenSecret(secrets SecretInterface, secretName string) (string, error)
 
 // CreateTokenSecret creates a secret with given name in given namespace
 // with a given token as value.
-func CreateTokenSecret(secrets SecretInterface, secretName, token string, ownerRef *metav1.OwnerReference) error {
+func CreateTokenSecret(secrets SecretInterface, secretName, token string, ownerRef *meta.OwnerReference) error {
 	// Create secret
-	secret := &v1.Secret{
-		ObjectMeta: metav1.ObjectMeta{
+	secret := &core.Secret{
+		ObjectMeta: meta.ObjectMeta{
 			Name: secretName,
 		},
 		Data: map[string][]byte{
@@ -250,9 +255,38 @@ func CreateTokenSecret(secrets SecretInterface, secretName, token string, ownerR
 	return nil
 }
 
+func AppendKeyfileToKeyfolder(secrets SecretInterface, ownerRef *meta.OwnerReference, secretName string, encryptionKey []byte) error {
+	encSha := fmt.Sprintf("%0x", sha256.Sum256(encryptionKey))
+	if _, err := secrets.Get(secretName, meta.GetOptions{}); err != nil {
+		if !IsNotFound(err) {
+			return goErrors.Wrapf(err, "Unable to get keyfolder")
+		}
+
+		// Create secret
+		secret := &core.Secret{
+			ObjectMeta: meta.ObjectMeta{
+				Name: secretName,
+			},
+			Data: map[string][]byte{
+				encSha: encryptionKey,
+			},
+		}
+		// Attach secret to owner
+		addOwnerRefToObject(secret, ownerRef)
+		if _, err := secrets.Create(secret); err != nil {
+			// Failed to create secret
+			return maskAny(err)
+		}
+
+		return nil
+	}
+
+	return nil
+}
+
 // CreateJWTFromSecret creates a JWT using the secret stored in secretSecretName and stores the
 // result in a new secret called tokenSecretName
-func CreateJWTFromSecret(secrets SecretInterface, tokenSecretName, secretSecretName string, claims map[string]interface{}, ownerRef *metav1.OwnerReference) error {
+func CreateJWTFromSecret(secrets SecretInterface, tokenSecretName, secretSecretName string, claims map[string]interface{}, ownerRef *meta.OwnerReference) error {
 
 	secret, err := GetTokenSecret(secrets, secretSecretName)
 	if err != nil {
@@ -273,10 +307,10 @@ func CreateJWTFromSecret(secrets SecretInterface, tokenSecretName, secretSecretN
 
 // CreateBasicAuthSecret creates a secret with given name in given namespace
 // with a given username and password as value.
-func CreateBasicAuthSecret(secrets SecretInterface, secretName, username, password string, ownerRef *metav1.OwnerReference) error {
+func CreateBasicAuthSecret(secrets SecretInterface, secretName, username, password string, ownerRef *meta.OwnerReference) error {
 	// Create secret
-	secret := &v1.Secret{
-		ObjectMeta: metav1.ObjectMeta{
+	secret := &core.Secret{
+		ObjectMeta: meta.ObjectMeta{
 			Name: secretName,
 		},
 		Data: map[string][]byte{
@@ -299,7 +333,7 @@ func CreateBasicAuthSecret(secrets SecretInterface, secretName, username, passwo
 // an error is returned.
 // Returns: username, password, error
 func GetBasicAuthSecret(secrets SecretInterface, secretName string) (string, string, error) {
-	s, err := secrets.Get(secretName, metav1.GetOptions{})
+	s, err := secrets.Get(secretName, meta.GetOptions{})
 	if err != nil {
 		return "", "", maskAny(err)
 	}
@@ -307,7 +341,7 @@ func GetBasicAuthSecret(secrets SecretInterface, secretName string) (string, str
 }
 
 // GetSecretAuthCredentials returns username and password from the secret
-func GetSecretAuthCredentials(secret *v1.Secret) (string, string, error) {
+func GetSecretAuthCredentials(secret *core.Secret) (string, string, error) {
 	username, found := secret.Data[constants.SecretUsername]
 	if !found {
 		return "", "", maskAny(fmt.Errorf("No '%s' found in secret '%s'", constants.SecretUsername, secret.Name))

pkg/util/k8sutil/secrets_cache.go

@@ -23,10 +23,12 @@
 package k8sutil
 
 import (
+	"github.com/pkg/errors"
 	v1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/types"
 	corev1 "k8s.io/client-go/kubernetes/typed/core/v1"
 )
 
@@ -37,6 +39,10 @@ type secretsCache struct {
 	cache []v1.Secret
 }
 
+func (sc *secretsCache) Patch(name string, pt types.PatchType, data []byte, subresources ...string) (*v1.Secret, error) {
+	return nil, errors.Errorf("Not implemented")
+}
+
 // NewSecretCache creates a cached version of the given SecretInterface.
 func NewSecretCache(cli corev1.SecretInterface) SecretInterface {
 	return &secretsCache{cli: cli}
@@ -58,6 +64,10 @@ func (sc *secretsCache) Create(s *v1.Secret) (*v1.Secret, error) {
 	return result, nil
 }
 
+func (sc *secretsCache) Update(s *v1.Secret) (*v1.Secret, error) {
+	return sc.Update(s)
+}
+
 func (sc *secretsCache) Delete(name string, options *metav1.DeleteOptions) error {
 	sc.cache = nil
 	err := sc.cli.Delete(name, options)