[Feature] TLS CA Secret Key (#1516)

CHANGELOG.md

@@ -24,6 +24,7 @@
 - (Feature) (ML) Metadata Service Implementation
 - (Feature) License Manager for ML Deployment
 - (Feature) (ML) Storage S3 sidecar implementation
+- (Feature) TLS CA Secret Key
 
 ## [1.2.35](https://github.com/arangodb/kube-arangodb/tree/1.2.35) (2023-11-06)
 - (Maintenance) Update go-driver to v1.6.0, update IsNotFound() checks

docs/generated/actions.md

@@ -55,6 +55,7 @@
 |        RebalancerGenerateV2        |   yes    |  10m0s  |    no    | Community & Enterprise |                                           Generates the Rebalancer plan                                            |
 |       RebuildOutSyncedShards       |    no    | 24h0m0s |    no    | Community & Enterprise |                               Run Rebuild Out Synced Shards procedure for DBServers                                |
 |           RecreateMember           |    no    |  15m0s  |    no    | Community & Enterprise |                                       Recreate member with same ID and Data                                        |
+|            RefreshTLSCA            |    no    |  30m0s  |    no    |    Enterprise Only     |                                                Refresh internal CA                                                 |
 |    RefreshTLSKeyfileCertificate    |    no    |  30m0s  |    no    |    Enterprise Only     |                                       Recreate Server TLS Certificate secret                                       |
 |            RemoveMember            |    no    |  15m0s  |    no    | Community & Enterprise |                                     Removes member from the Cluster and Status                                     |
 |          RemoveMemberPVC           |    no    |  15m0s  |    no    | Community & Enterprise |                                 Removes member PVC and enforce recreate procedure                                  |
@@ -150,6 +151,7 @@ spec:
       RebalancerGenerateV2: 10m0s
       RebuildOutSyncedShards: 24h0m0s
       RecreateMember: 15m0s
+      RefreshTLSCA: 30m0s
       RefreshTLSKeyfileCertificate: 30m0s
       RemoveMember: 15m0s
       RemoveMemberPVC: 15m0s

internal/actions.yaml

@@ -80,6 +80,10 @@ actions:
     enterprise: true
     description: Remove Certificate from CA TrustStore
     timeout: 30m
+  RefreshTLSCA:
+    enterprise: true
+    description: Refresh internal CA
+    timeout: 30m
   CleanTLSKeyfileCertificate:
     enterprise: true
     description: Remove old TLS certificate from server

pkg/apis/deployment/v1/actions.generated.go

@@ -173,6 +173,9 @@ const (
 	// ActionRecreateMemberDefaultTimeout define default timeout for action ActionRecreateMember
 	ActionRecreateMemberDefaultTimeout time.Duration = 900 * time.Second // 15m0s
 
+	// ActionRefreshTLSCADefaultTimeout define default timeout for action ActionRefreshTLSCA
+	ActionRefreshTLSCADefaultTimeout time.Duration = 1800 * time.Second // 30m0s
+
 	// ActionRefreshTLSKeyfileCertificateDefaultTimeout define default timeout for action ActionRefreshTLSKeyfileCertificate
 	ActionRefreshTLSKeyfileCertificateDefaultTimeout time.Duration = 1800 * time.Second // 30m0s
 
@@ -433,6 +436,9 @@ const (
 	// ActionTypeRecreateMember in scopes Normal. Recreate member with same ID and Data
 	ActionTypeRecreateMember ActionType = "RecreateMember"
 
+	// ActionTypeRefreshTLSCA in scopes Normal. Refresh internal CA
+	ActionTypeRefreshTLSCA ActionType = "RefreshTLSCA"
+
 	// ActionTypeRefreshTLSKeyfileCertificate in scopes Normal. Recreate Server TLS Certificate secret
 	ActionTypeRefreshTLSKeyfileCertificate ActionType = "RefreshTLSKeyfileCertificate"
 
@@ -643,6 +649,8 @@ func (a ActionType) DefaultTimeout() time.Duration {
 		return ActionRebuildOutSyncedShardsDefaultTimeout
 	case ActionTypeRecreateMember:
 		return ActionRecreateMemberDefaultTimeout
+	case ActionTypeRefreshTLSCA:
+		return ActionRefreshTLSCADefaultTimeout
 	case ActionTypeRefreshTLSKeyfileCertificate:
 		return ActionRefreshTLSKeyfileCertificateDefaultTimeout
 	case ActionTypeRemoveMember:
@@ -819,6 +827,8 @@ func (a ActionType) Priority() ActionPriority {
 		return ActionPriorityHigh
 	case ActionTypeRecreateMember:
 		return ActionPriorityNormal
+	case ActionTypeRefreshTLSCA:
+		return ActionPriorityNormal
 	case ActionTypeRefreshTLSKeyfileCertificate:
 		return ActionPriorityNormal
 	case ActionTypeRemoveMember:
@@ -1007,6 +1017,8 @@ func (a ActionType) Optional() bool {
 		return false
 	case ActionTypeRecreateMember:
 		return false
+	case ActionTypeRefreshTLSCA:
+		return false
 	case ActionTypeRefreshTLSKeyfileCertificate:
 		return false
 	case ActionTypeRemoveMember:

pkg/apis/deployment/v2alpha1/actions.generated.go

@@ -173,6 +173,9 @@ const (
 	// ActionRecreateMemberDefaultTimeout define default timeout for action ActionRecreateMember
 	ActionRecreateMemberDefaultTimeout time.Duration = 900 * time.Second // 15m0s
 
+	// ActionRefreshTLSCADefaultTimeout define default timeout for action ActionRefreshTLSCA
+	ActionRefreshTLSCADefaultTimeout time.Duration = 1800 * time.Second // 30m0s
+
 	// ActionRefreshTLSKeyfileCertificateDefaultTimeout define default timeout for action ActionRefreshTLSKeyfileCertificate
 	ActionRefreshTLSKeyfileCertificateDefaultTimeout time.Duration = 1800 * time.Second // 30m0s
 
@@ -433,6 +436,9 @@ const (
 	// ActionTypeRecreateMember in scopes Normal. Recreate member with same ID and Data
 	ActionTypeRecreateMember ActionType = "RecreateMember"
 
+	// ActionTypeRefreshTLSCA in scopes Normal. Refresh internal CA
+	ActionTypeRefreshTLSCA ActionType = "RefreshTLSCA"
+
 	// ActionTypeRefreshTLSKeyfileCertificate in scopes Normal. Recreate Server TLS Certificate secret
 	ActionTypeRefreshTLSKeyfileCertificate ActionType = "RefreshTLSKeyfileCertificate"
 
@@ -643,6 +649,8 @@ func (a ActionType) DefaultTimeout() time.Duration {
 		return ActionRebuildOutSyncedShardsDefaultTimeout
 	case ActionTypeRecreateMember:
 		return ActionRecreateMemberDefaultTimeout
+	case ActionTypeRefreshTLSCA:
+		return ActionRefreshTLSCADefaultTimeout
 	case ActionTypeRefreshTLSKeyfileCertificate:
 		return ActionRefreshTLSKeyfileCertificateDefaultTimeout
 	case ActionTypeRemoveMember:
@@ -819,6 +827,8 @@ func (a ActionType) Priority() ActionPriority {
 		return ActionPriorityHigh
 	case ActionTypeRecreateMember:
 		return ActionPriorityNormal
+	case ActionTypeRefreshTLSCA:
+		return ActionPriorityNormal
 	case ActionTypeRefreshTLSKeyfileCertificate:
 		return ActionPriorityNormal
 	case ActionTypeRemoveMember:
@@ -1007,6 +1017,8 @@ func (a ActionType) Optional() bool {
 		return false
 	case ActionTypeRecreateMember:
 		return false
+	case ActionTypeRefreshTLSCA:
+		return false
 	case ActionTypeRefreshTLSKeyfileCertificate:
 		return false
 	case ActionTypeRemoveMember:

pkg/deployment/reconcile/action.register.generated.go

@@ -165,6 +165,9 @@ var (
 	_ Action        = &actionRecreateMember{}
 	_ actionFactory = newRecreateMemberAction
 
+	_ Action        = &actionRefreshTLSCA{}
+	_ actionFactory = newRefreshTLSCAAction
+
 	_ Action        = &actionRefreshTLSKeyfileCertificate{}
 	_ actionFactory = newRefreshTLSKeyfileCertificateAction
 
@@ -951,6 +954,20 @@ func init() {
 		registerAction(action, function)
 	}
 
+	// RefreshTLSCA
+	{
+		// Get Action type
+		action := api.ActionTypeRefreshTLSCA
+
+		// Get Action defition
+		function := newRefreshTLSCAAction
+
+		// Wrap action main function
+
+		// Register action
+		registerAction(action, function)
+	}
+
 	// RefreshTLSKeyfileCertificate
 	{
 		// Get Action type

pkg/deployment/reconcile/action.register.generated_test.go

@@ -529,6 +529,16 @@ func Test_Actions(t *testing.T) {
 		})
 	})
 
+	t.Run("RefreshTLSCA", func(t *testing.T) {
+		ActionsExistence(t, api.ActionTypeRefreshTLSCA)
+		t.Run("Internal", func(t *testing.T) {
+			require.False(t, api.ActionTypeRefreshTLSCA.Internal())
+		})
+		t.Run("Optional", func(t *testing.T) {
+			require.False(t, api.ActionTypeRefreshTLSCA.Optional())
+		})
+	})
+
 	t.Run("RefreshTLSKeyfileCertificate", func(t *testing.T) {
 		ActionsExistence(t, api.ActionTypeRefreshTLSKeyfileCertificate)
 		t.Run("Internal", func(t *testing.T) {

pkg/deployment/reconcile/action_tls_ca_refresh.go

@@ -0,0 +1,93 @@
+//
+// DISCLAIMER
+//
+// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package reconcile
+
+import (
+	"context"
+	"encoding/base64"
+
+	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+
+	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
+	"github.com/arangodb/kube-arangodb/pkg/deployment/patch"
+	"github.com/arangodb/kube-arangodb/pkg/deployment/resources"
+	"github.com/arangodb/kube-arangodb/pkg/util/errors"
+	"github.com/arangodb/kube-arangodb/pkg/util/globals"
+	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil/kerrors"
+)
+
+// newRefreshTLSCAAction creates a new Action that implements the given
+// planned RefreshTLSCA action.
+func newRefreshTLSCAAction(action api.Action, actionCtx ActionContext) Action {
+	a := &actionRefreshTLSCA{}
+
+	a.actionImpl = newBaseActionImpl(action, actionCtx, &a.newMemberID)
+
+	return a
+}
+
+// actionRefreshTLSCA implements an RefreshTLSCAAction.
+type actionRefreshTLSCA struct {
+	// actionImpl implement timeout and member id functions
+	actionImpl
+
+	// actionEmptyCheckProgress implement check progress with empty implementation
+	actionEmptyCheckProgress
+
+	newMemberID string
+}
+
+// Start performs the start of the action.
+// Returns true if the action is completely finished, false in case
+// the start time needs to be recorded and a ready condition needs to be checked.
+func (a *actionRefreshTLSCA) Start(ctx context.Context) (bool, error) {
+	caFolder, exists := a.actionCtx.ACS().CurrentClusterCache().Secret().V1().GetSimple(resources.GetCASecretName(a.actionCtx.GetAPIObject()))
+	if !exists {
+		a.log.Warn("Secret %s is missing", resources.GetCASecretName(a.actionCtx.GetAPIObject()))
+		return true, nil
+	}
+
+	caLocalData := buildInternalCA(caFolder)
+
+	if string(caFolder.Data[resources.CACertName]) != caLocalData {
+		p := patch.NewPatch()
+		p.ItemAdd(patch.NewPath("data", resources.CACertName), base64.StdEncoding.EncodeToString([]byte(caLocalData)))
+
+		patch, err := p.Marshal()
+		if err != nil {
+			a.log.Err(err).Error("Unable to encrypt patch")
+			return true, nil
+		}
+
+		err = globals.GetGlobalTimeouts().Kubernetes().RunWithTimeout(ctx, func(ctxChild context.Context) error {
+			_, err := a.actionCtx.ACS().CurrentClusterCache().SecretsModInterface().V1().Patch(ctxChild, resources.GetCASecretName(a.actionCtx.GetAPIObject()), types.JSONPatchType, patch, meta.PatchOptions{})
+			return err
+		})
+		if err != nil {
+			if !kerrors.IsInvalid(err) {
+				return false, errors.Wrapf(err, "Unable to update secret: %s", string(patch))
+			}
+		}
+	}
+
+	return true, nil
+}

pkg/deployment/reconcile/plan_builder_tls.go

@@ -27,8 +27,11 @@ import (
 	"net/http"
 	"net/url"
 	"reflect"
+	"sort"
 	"time"
 
+	core "k8s.io/api/core/v1"
+
 	"github.com/arangodb/go-driver"
 
 	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
@@ -49,6 +52,22 @@ import (
 
 const CertificateRenewalMargin = 7 * 24 * time.Hour
 
+func buildInternalCA(secret *core.Secret) string {
+	// Let's check if ca.crt needs to be created
+	keys := make([]string, 0, len(secret.Data))
+	for k, v := range secret.Data {
+		if k == resources.CACertName {
+			continue
+		}
+
+		keys = append(keys, string(v))
+	}
+
+	sort.Strings(keys)
+
+	return strings.Join(keys, "\n")
+}
+
 func (r *Reconciler) createTLSStatusPropagatedFieldUpdate(ctx context.Context, apiObject k8sutil.APIObject,
 	spec api.DeploymentSpec, status api.DeploymentStatus,
 	context PlanBuilderContext, w WithPlanBuilder, builders ...planBuilder) api.Plan {
@@ -192,6 +211,10 @@ func (r *Reconciler) createCAAppendPlan(ctx context.Context, apiObject k8sutil.A
 			AddParam(checksum, certSha)}
 	}
 
+	if string(trusted.Data[resources.CACertName]) != buildInternalCA(trusted) {
+		return api.Plan{actions.NewClusterAction(api.ActionTypeRefreshTLSCA, "Refresh CA")}
+	}
+
 	return nil
 }
 
@@ -270,6 +293,10 @@ func (r *Reconciler) createCACleanPlan(ctx context.Context, apiObject k8sutil.AP
 	certSha := util.SHA256(caData)
 
 	for sha := range trusted.Data {
+		if sha == resources.CACertName {
+			continue
+		}
+
 		if certSha != sha {
 			return api.Plan{actions.NewClusterAction(api.ActionTypeCleanTLSCACertificate, "Clean CA from truststore").
 				AddParam(checksum, sha)}