1
0
Fork 0
mirror of https://github.com/arangodb/kube-arangodb.git synced 2024-12-14 11:57:37 +00:00

[Bugfix] Fix bootstrap phase (#663)

This commit is contained in:
Adam Janikowski 2020-11-10 12:12:06 +01:00 committed by GitHub
parent 8c2d4be3e8
commit 0df664e8bc
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
17 changed files with 388 additions and 190 deletions

View file

@ -1,6 +1,7 @@
# Change Log
## [master](https://github.com/arangodb/kube-arangodb/tree/master) (N/A)
- Fix Bootstrap phase and move it under Plan
## [1.1.1](https://github.com/arangodb/kube-arangodb/tree/1.1.1) (2020-11-04)
- Allow to mount EmptyDir

View file

@ -34,6 +34,10 @@ const (
// PasswordSecretName contains user password secret name
type PasswordSecretName string
func (p PasswordSecretName) Get() string {
return string(p)
}
const (
// PasswordSecretNameNone is magic value for no action
PasswordSecretNameNone PasswordSecretName = "None"

View file

@ -110,6 +110,8 @@ type DeploymentSpec struct {
Recovery *ArangoDeploymentRecoverySpec `json:"recovery,omitempty"`
Bootstrap BootstrapSpec `json:"bootstrap,omitempty"`
Timeouts *Timeouts `json:"timeouts,omitempty"`
}
// GetRestoreFrom returns the restore from string or empty string if not set

View file

@ -127,6 +127,10 @@ const (
ActionTypeEnableMaintenance ActionType = "EnableMaintenance"
// ActionTypeEnableMaintenance disables maintenance on cluster.
ActionTypeDisableMaintenance ActionType = "DisableMaintenance"
// ActionTypeBootstrapUpdate update bootstrap status to true
ActionTypeBootstrapUpdate ActionType = "BootstrapUpdate"
// ActionTypeBootstrapSetPassword set password to the bootstrapped user
ActionTypeBootstrapSetPassword ActionType = "BootstrapSetPassword"
)
const (
@ -182,9 +186,9 @@ func (a Action) AddParam(key, value string) Action {
}
// GetParam returns action parameter
func (a Action) GetParam(key string) (interface{}, bool) {
func (a Action) GetParam(key string) (string, bool) {
if a.Params == nil {
return nil, false
return "", false
}
i, ok := a.Params[key]

View file

@ -0,0 +1,55 @@
//
// DISCLAIMER
//
// Copyright 2020 ArangoDB GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Copyright holder is ArangoDB GmbH, Cologne, Germany
//
// Author Adam Janikowski
//
package v1
import (
"time"
meta "k8s.io/apimachinery/pkg/apis/meta/v1"
)
const (
addMemberTimeout = time.Minute * 5
)
type Timeouts struct {
AddMember *Timeout `json:"addMember,omitempty"`
}
func (t *Timeouts) Get() Timeouts {
if t == nil {
return Timeouts{}
}
return *t
}
type Timeout meta.Duration
func (t *Timeout) Get(d time.Duration) time.Duration {
if t == nil {
return d
}
return t.Duration
}

View file

@ -1,147 +0,0 @@
//
// DISCLAIMER
//
// Copyright 2020 ArangoDB GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Copyright holder is ArangoDB GmbH, Cologne, Germany
//
// Author Ewout Prangsma
//
package deployment
import (
"context"
"crypto/rand"
"encoding/hex"
"fmt"
api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/arangodb/go-driver"
)
// EnsureBootstrap executes the bootstrap once as soon as the deployment becomes ready
func (d *Deployment) EnsureBootstrap() error {
status, version := d.GetStatus()
if status.Conditions.IsTrue(api.ConditionTypeReady) {
if _, hasBootstrap := status.Conditions.Get(api.ConditionTypeBootstrapCompleted); !hasBootstrap {
return nil // The cluster was not initialised with ConditionTypeBoostrapCompleted == false
}
if status.Conditions.IsTrue(api.ConditionTypeBootstrapCompleted) {
return nil // Nothing to do, already bootstrapped
}
d.deps.Log.Info().Msgf("Bootstrap deployment %s", d.Name())
err := d.runBootstrap()
if err != nil {
status.Conditions.Update(api.ConditionTypeBootstrapCompleted, true, "Bootstrap failed", err.Error())
status.Conditions.Update(api.ConditionTypeBootstrapSucceded, false, "Bootstrap failed", err.Error())
} else {
status.Conditions.Update(api.ConditionTypeBootstrapCompleted, true, "Bootstrap successful", "The bootstrap process has been completed successfully")
status.Conditions.Update(api.ConditionTypeBootstrapSucceded, true, "Bootstrap successful", "The bootstrap process has been completed successfully")
}
if err = d.UpdateStatus(status, version); err != nil {
return maskAny(err)
}
d.deps.Log.Info().Msgf("Bootstrap completed for %s", d.Name())
}
return nil
}
// ensureRootUserPassword ensures the root user secret and returns the password specified or generated
func (d *Deployment) ensureUserPasswordSecret(secrets k8sutil.SecretInterface, username, secretName string) (string, error) {
if auth, err := secrets.Get(secretName, metav1.GetOptions{}); k8sutil.IsNotFound(err) {
// Create new one
tokenData := make([]byte, 32)
if _, err = rand.Read(tokenData); err != nil {
return "", err
}
token := hex.EncodeToString(tokenData)
owner := d.GetAPIObject().AsOwner()
if err := k8sutil.CreateBasicAuthSecret(secrets, secretName, username, token, &owner); err != nil {
return "", err
}
return token, nil
} else if err == nil {
user, pass, err := k8sutil.GetSecretAuthCredentials(auth)
if err == nil && user == username {
return pass, nil
}
return "", fmt.Errorf("invalid secret format in secret %s", secretName)
} else {
return "", err
}
}
// bootstrapUserPassword loads the password for the given user and updates the password stored in the database
func (d *Deployment) bootstrapUserPassword(client driver.Client, secrets k8sutil.SecretInterface, username, secretname string) error {
d.deps.Log.Debug().Msgf("Bootstrapping user %s, secret %s", username, secretname)
password, err := d.ensureUserPasswordSecret(secrets, username, secretname)
if err != nil {
return maskAny(err)
}
// Obtain the user
if user, err := client.User(context.TODO(), username); driver.IsNotFound(err) {
_, err := client.CreateUser(context.TODO(), username, &driver.UserOptions{Password: password})
return maskAny(err)
} else if err == nil {
return maskAny(user.Update(context.TODO(), driver.UserOptions{
Password: password,
}))
} else {
return err
}
}
// runBootstrap is run for a deployment once
func (d *Deployment) runBootstrap() error {
// execute the bootstrap code
// make sure that the bootstrap code is idempotent
ctx := context.Background()
client, err := d.clientCache.GetDatabase(ctx)
if err != nil {
return maskAny(err)
}
spec := d.GetSpec()
secrets := d.GetKubeCli().CoreV1().Secrets(d.Namespace())
for user, secret := range spec.Bootstrap.PasswordSecretNames {
if secret.IsNone() {
continue
}
if err := d.bootstrapUserPassword(client, secrets, user, string(secret)); err != nil {
return maskAny(err)
}
}
return nil
}

View file

@ -257,11 +257,6 @@ func (d *Deployment) inspectDeploymentWithError(ctx context.Context, lastInterva
return minInspectionInterval, errors.Wrapf(err, "AccessPackage creation failed")
}
// Ensure deployment bootstrap
if err := d.EnsureBootstrap(); err != nil {
return minInspectionInterval, errors.Wrapf(err, "Bootstrap failed")
}
// Inspect deployment for obsolete members
if err := d.resources.CleanupRemovedMembers(); err != nil {
return minInspectionInterval, errors.Wrapf(err, "Removed member cleanup failed")

View file

@ -42,7 +42,7 @@ type Action interface {
// Returns: ready, abort, error.
CheckProgress(ctx context.Context) (bool, bool, error)
// Timeout returns the amount of time after which this action will timeout.
Timeout() time.Duration
Timeout(deploymentSpec api.DeploymentSpec) time.Duration
// Return the MemberID used / created in this action
MemberID() string
}

View file

@ -24,6 +24,7 @@ package reconcile
import (
"context"
"time"
api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
"github.com/rs/zerolog"
@ -39,7 +40,9 @@ func init() {
func newAddMemberAction(log zerolog.Logger, action api.Action, actionCtx ActionContext) Action {
a := &actionAddMember{}
a.actionImpl = newActionImpl(log, action, actionCtx, addMemberTimeout, &a.newMemberID)
a.actionImpl = newBaseActionImpl(log, action, actionCtx, func(deploymentSpec api.DeploymentSpec) time.Duration {
return deploymentSpec.Timeouts.Get().AddMember.Get(addMemberTimeout)
}, &a.newMemberID)
return a
}

View file

@ -0,0 +1,148 @@
//
// DISCLAIMER
//
// Copyright 2020 ArangoDB GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Copyright holder is ArangoDB GmbH, Cologne, Germany
//
// Author Adam Janikowski
//
package reconcile
import (
"context"
"crypto/rand"
"encoding/hex"
"fmt"
"github.com/arangodb/go-driver"
api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
"github.com/arangodb/kube-arangodb/pkg/util"
"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
"github.com/rs/zerolog"
)
func init() {
registerAction(api.ActionTypeBootstrapSetPassword, newBootstrapSetPasswordAction)
}
func newBootstrapSetPasswordAction(log zerolog.Logger, action api.Action, actionCtx ActionContext) Action {
a := &actionBootstrapSetPassword{}
a.actionImpl = newActionImplDefRef(log, action, actionCtx, defaultTimeout)
return a
}
type actionBootstrapSetPassword struct {
// actionImpl implement timeout and member id functions
actionImpl
actionEmptyCheckProgress
}
func (a actionBootstrapSetPassword) Start(ctx context.Context) (bool, error) {
spec := a.actionCtx.GetSpec()
if user, ok := a.action.GetParam("user"); !ok {
a.log.Warn().Msgf("User param is not set in action")
return true, nil
} else {
if secret, ok := spec.Bootstrap.PasswordSecretNames[user]; !ok {
a.log.Warn().Msgf("User does not exist in password hashes")
return true, nil
} else {
ctx, c := context.WithTimeout(context.Background(), a.Timeout(spec))
defer c()
if password, err := a.setUserPassword(ctx, user, secret.Get()); err != nil {
return false, err
} else {
passwordSha := util.SHA256FromString(password)
if err := a.actionCtx.WithStatusUpdate(func(s *api.DeploymentStatus) bool {
if s.SecretHashes == nil {
s.SecretHashes = &api.SecretHashes{}
}
if s.SecretHashes.Users == nil {
s.SecretHashes.Users = map[string]string{}
}
if u, ok := s.SecretHashes.Users[user]; !ok || u != passwordSha {
s.SecretHashes.Users[user] = passwordSha
return true
}
return false
}); err != nil {
return false, err
}
}
}
}
return true, nil
}
func (a actionBootstrapSetPassword) setUserPassword(ctx context.Context, user, secret string) (string, error) {
a.log.Debug().Msgf("Bootstrapping user %s, secret %s", user, secret)
client, err := a.actionCtx.GetDatabaseClient(ctx)
if err != nil {
return "", maskAny(err)
}
password, err := a.ensureUserPasswordSecret(user, secret)
if err != nil {
return "", maskAny(err)
}
// Obtain the user
if u, err := client.User(context.Background(), user); driver.IsNotFound(err) {
_, err := client.CreateUser(context.Background(), user, &driver.UserOptions{Password: password})
return password, maskAny(err)
} else if err == nil {
return password, maskAny(u.Update(context.Background(), driver.UserOptions{
Password: password,
}))
} else {
return "", err
}
}
func (a actionBootstrapSetPassword) ensureUserPasswordSecret(user, secret string) (string, error) {
cache := a.actionCtx.GetCachedStatus()
if auth, ok := cache.Secret(secret); !ok {
// Create new one
tokenData := make([]byte, 32)
if _, err := rand.Read(tokenData); err != nil {
return "", err
}
token := hex.EncodeToString(tokenData)
owner := a.actionCtx.GetAPIObject().AsOwner()
if err := k8sutil.CreateBasicAuthSecret(a.actionCtx.SecretsInterface(), secret, user, token, &owner); err != nil {
return "", err
}
return token, nil
} else {
user, pass, err := k8sutil.GetSecretAuthCredentials(auth)
if err == nil && user == user {
return pass, nil
}
return "", fmt.Errorf("invalid secret format in secret %s", secret)
}
}

View file

@ -0,0 +1,68 @@
//
// DISCLAIMER
//
// Copyright 2020 ArangoDB GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Copyright holder is ArangoDB GmbH, Cologne, Germany
//
// Author Adam Janikowski
//
package reconcile
import (
"context"
"fmt"
api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
"github.com/rs/zerolog"
)
func init() {
registerAction(api.ActionTypeBootstrapUpdate, newBootstrapUpdateAction)
}
func newBootstrapUpdateAction(log zerolog.Logger, action api.Action, actionCtx ActionContext) Action {
a := &actionBootstrapUpdate{}
a.actionImpl = newActionImplDefRef(log, action, actionCtx, defaultTimeout)
return a
}
// actionBackupRestoreClean implements an BackupRestoreClean.
type actionBootstrapUpdate struct {
// actionImpl implement timeout and member id functions
actionImpl
actionEmptyCheckProgress
}
func (a actionBootstrapUpdate) Start(ctx context.Context) (bool, error) {
if err := a.actionCtx.WithStatusUpdate(func(status *api.DeploymentStatus) bool {
if errMessage, ok := a.action.GetParam("error"); ok {
status.Conditions.Update(api.ConditionTypeBootstrapCompleted, true, "Bootstrap failed", fmt.Sprintf("%s", errMessage))
status.Conditions.Update(api.ConditionTypeBootstrapSucceded, false, "Bootstrap failed", fmt.Sprintf("%s", errMessage))
} else {
status.Conditions.Update(api.ConditionTypeBootstrapCompleted, true, "Bootstrap successful", "The bootstrap process has been completed successfully")
status.Conditions.Update(api.ConditionTypeBootstrapSucceded, true, "Bootstrap successful", "The bootstrap process has been completed successfully")
}
return true
}, true); err != nil {
return false, err
}
return true, nil
}

View file

@ -30,6 +30,14 @@ import (
"github.com/rs/zerolog"
)
type TimeoutFetcher func(deploymentSpec api.DeploymentSpec) time.Duration
func NewTimeoutFetcher(t time.Duration) TimeoutFetcher {
return func(deploymentSpec api.DeploymentSpec) time.Duration {
return t
}
}
type actionEmptyCheckProgress struct {
}
@ -55,6 +63,18 @@ func newActionImpl(log zerolog.Logger, action api.Action, actionCtx ActionContex
panic("Action cannot have nil reference to member!")
}
return newBaseActionImpl(log, action, actionCtx, NewTimeoutFetcher(timeout), memberIDRef)
}
func newBaseActionImplDefRef(log zerolog.Logger, action api.Action, actionCtx ActionContext, timeout TimeoutFetcher) actionImpl {
return newBaseActionImpl(log, action, actionCtx, timeout, &action.MemberID)
}
func newBaseActionImpl(log zerolog.Logger, action api.Action, actionCtx ActionContext, timeout TimeoutFetcher, memberIDRef *string) actionImpl {
if memberIDRef == nil {
panic("Action cannot have nil reference to member!")
}
return actionImpl{
log: log,
action: action,
@ -69,13 +89,17 @@ type actionImpl struct {
action api.Action
actionCtx ActionContext
timeout time.Duration
timeout TimeoutFetcher
memberIDRef *string
}
// Timeout returns the amount of time after which this action will timeout.
func (a actionImpl) Timeout() time.Duration {
return a.timeout
func (a actionImpl) Timeout(deploymentSpec api.DeploymentSpec) time.Duration {
if a.timeout == nil {
return defaultTimeout
}
return a.timeout(deploymentSpec)
}
// Return the MemberID used / created in this action

View file

@ -279,6 +279,10 @@ func createPlan(ctx context.Context, log zerolog.Logger, apiObject k8sutil.APIOb
plan = pb.Apply(createTLSStatusPropagated)
}
if plan.IsEmpty() {
plan = pb.Apply(createBootstrapPlan)
}
// Return plan
return plan, true
}

View file

@ -0,0 +1,66 @@
//
// DISCLAIMER
//
// Copyright 2020 ArangoDB GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Copyright holder is ArangoDB GmbH, Cologne, Germany
//
// Author Adam Janikowski
//
package reconcile
import (
"context"
core "k8s.io/api/core/v1"
api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
"github.com/arangodb/kube-arangodb/pkg/deployment/resources/inspector"
"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
"github.com/rs/zerolog"
)
func createBootstrapPlan(ctx context.Context,
log zerolog.Logger, apiObject k8sutil.APIObject,
spec api.DeploymentSpec, status api.DeploymentStatus,
cachedStatus inspector.Inspector, context PlanBuilderContext) api.Plan {
if !status.Conditions.IsTrue(api.ConditionTypeReady) {
return nil
}
if condition, hasBootstrap := status.Conditions.Get(api.ConditionTypeBootstrapCompleted); !hasBootstrap || condition.Status == core.ConditionTrue {
return nil
}
for user, secret := range spec.Bootstrap.PasswordSecretNames {
if secret.IsNone() {
continue
}
if s := status.SecretHashes; s != nil {
if u := s.Users; u != nil {
if _, ok := u[user]; ok {
continue
}
}
}
return api.Plan{api.NewAction(api.ActionTypeBootstrapSetPassword, api.ServerGroupUnknown, "", "Updating password").AddParam("user", user)}
}
return api.Plan{api.NewAction(api.ActionTypeBootstrapUpdate, api.ServerGroupUnknown, "", "Finalizing bootstrap")}
}

View file

@ -136,7 +136,7 @@ func (d *Reconciler) ExecutePlan(ctx context.Context, cachedStatus inspector.Ins
d.context.CreateEvent(k8sutil.NewPlanAbortedEvent(d.context.GetAPIObject(), string(planAction.Type), planAction.MemberID, planAction.Group.AsRole()))
} else {
// Not ready yet & no abort, check timeout
deadline := planAction.CreationTime.Add(action.Timeout())
deadline := planAction.CreationTime.Add(action.Timeout(d.context.GetSpec()))
if time.Now().After(deadline) {
// Timeout has expired
deadlineExpired = true

View file

@ -25,7 +25,7 @@ package reconcile
import "time"
const (
addMemberTimeout = time.Minute * 5
addMemberTimeout = time.Minute * 10
cleanoutMemberTimeout = time.Hour * 12
removeMemberTimeout = time.Minute * 15
recreateMemberTimeout = time.Minute * 15

View file

@ -40,8 +40,6 @@ import (
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
)
@ -206,33 +204,6 @@ func (r *Resources) ValidateSecretHashes(cachedStatus inspector.Inspector) error
}
}
for username, secretName := range spec.Bootstrap.PasswordSecretNames {
if secretName.IsNone() || secretName.IsAuto() {
continue
}
_, err := r.context.GetKubeCli().CoreV1().Secrets(r.context.GetNamespace()).Get(string(secretName), metav1.GetOptions{})
if k8sutil.IsNotFound(err) {
// do nothing when secret was deleted
continue
}
getExpectedHash := func() string {
if v, ok := getHashes().Users[username]; ok {
return v
}
return ""
}
setExpectedHash := func(h string) error {
return maskAny(updateHashes(func(dst *api.SecretHashes) {
dst.Users[username] = h
}))
}
// If password changes it should not be set that deployment in 'SecretsChanged' state
validate(string(secretName), getExpectedHash, setExpectedHash, changeUserPassword)
}
if len(badSecretNames) > 0 {
// We have invalid hashes, set the SecretsChanged condition
if status.Conditions.Update(api.ConditionTypeSecretsChanged, true,