Feature/change user password (#463)

* Change password in a database for a user
* Add unit tests
* Change password for user when hash secret has changed
* Add integration test for changing root password
* Add disclaimer
* Change comment
* Run unit test for whole pkg/deployment/...
* Check password in the database
* Fix nil receiver
* Fix removing root secret password
* Fix unit test definition in Makefile
* Don't validate non-existing users' secrets

Makefile

@@ -308,8 +308,7 @@ run-unit-tests: $(SOURCES)
 		$(REPOPATH)/pkg/apis/deployment/v1alpha \
 		$(REPOPATH)/pkg/apis/replication/v1alpha \
 		$(REPOPATH)/pkg/apis/storage/v1alpha \
-		$(REPOPATH)/pkg/deployment/reconcile \
+		$(REPOPATH)/pkg/deployment/... \
-		$(REPOPATH)/pkg/deployment/resources \
 		$(REPOPATH)/pkg/storage \
 		$(REPOPATH)/pkg/util/k8sutil \
 		$(REPOPATH)/pkg/util/k8sutil/test \

pkg/apis/deployment/v1alpha/secret_hashes.go

@@ -34,6 +34,8 @@ type SecretHashes struct {
 	TLSCA string `json:"tls-ca,omitempty"`
 	// SyncTLSCA contains the hash of the sync.tls.caSecretName secret
 	SyncTLSCA string `json:"sync-tls-ca,omitempty"`
+	// User's map contains hashes for each user
+	Users map[string]string `json:"users,omitempty"`
 }
 
 // Equal compares two SecretHashes
@@ -47,5 +49,40 @@ func (sh *SecretHashes) Equal(other *SecretHashes) bool {
 	return sh.AuthJWT == other.AuthJWT &&
 		sh.RocksDBEncryptionKey == other.RocksDBEncryptionKey &&
 		sh.TLSCA == other.TLSCA &&
-		sh.SyncTLSCA == other.SyncTLSCA
+		sh.SyncTLSCA == other.SyncTLSCA &&
+		isStringMapEqual(sh.Users, other.Users)
+}
+
+// NewEmptySecretHashes creates new empty structure
+func NewEmptySecretHashes() *SecretHashes {
+	sh := &SecretHashes{}
+	sh.Users = make(map[string]string)
+	return sh
+}
+
+func isStringMapEqual(first map[string]string, second map[string]string) bool {
+	if first == nil && second == nil {
+		return true
+	}
+
+	if first == nil || second == nil {
+		return false
+	}
+
+	if len(first) != len(second) {
+		return false
+	}
+
+	for key, valueF := range first {
+		valueS, ok := second[key]
+		if !ok {
+			return false
+		}
+
+		if valueF != valueS {
+			return false
+		}
+	}
+
+	return true
 }

pkg/apis/deployment/v1alpha/secret_hashes_test.go

@@ -0,0 +1,164 @@
+//
+// DISCLAIMER
+//
+// Copyright 2019 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author tomasz@arangodb.con
+//
+
+package v1alpha
+
+import (
+	"github.com/magiconair/properties/assert"
+
+	"testing"
+)
+
+func TestSecretHashes_Equal(t *testing.T) {
+	// Arrange
+	sh := SecretHashes{}
+	testCases := []struct {
+		Name        string
+		CompareFrom *SecretHashes
+		CompareTo   *SecretHashes
+		Expected    bool
+	}{
+		{
+			Name:        "Parameter can not be nil",
+			CompareFrom: &SecretHashes{},
+			Expected:    false,
+		},
+		{
+			Name:        "The addresses are the same",
+			CompareFrom: &sh,
+			CompareTo:   &sh,
+			Expected:    true,
+		},
+		{
+			Name: "JWT token is different",
+			CompareFrom: &SecretHashes{
+				AuthJWT: "1",
+			},
+			CompareTo: &SecretHashes{
+				AuthJWT: "2",
+			},
+			Expected: false,
+		},
+		{
+			Name: "Users are different",
+			CompareFrom: &SecretHashes{
+				Users: map[string]string{
+					"root": "",
+				},
+			},
+			CompareTo: &SecretHashes{},
+			Expected:  false,
+		},
+		{
+			Name: "User's table size is different",
+			CompareFrom: &SecretHashes{
+				Users: map[string]string{
+					"root": "",
+				},
+			},
+			CompareTo: &SecretHashes{
+				Users: map[string]string{
+					"root": "",
+					"user": "",
+				},
+			},
+			Expected: false,
+		},
+		{
+			Name: "User's table has got different users",
+			CompareFrom: &SecretHashes{
+				Users: map[string]string{
+					"root": "",
+				},
+			},
+			CompareTo: &SecretHashes{
+				Users: map[string]string{
+					"user": "",
+				},
+			},
+			Expected: false,
+		},
+		{
+			Name: "User's table has got different hashes for users",
+			CompareFrom: &SecretHashes{
+				Users: map[string]string{
+					"root": "123",
+				},
+			},
+			CompareTo: &SecretHashes{
+				Users: map[string]string{
+					"root": "1234",
+				},
+			},
+			Expected: false,
+		},
+		{
+			Name: "Secret hashes are the same",
+			CompareFrom: &SecretHashes{
+				AuthJWT:              "1",
+				RocksDBEncryptionKey: "2",
+				TLSCA:                "3",
+				SyncTLSCA:            "4",
+				Users: map[string]string{
+					"root": "123",
+				},
+			},
+			CompareTo: &SecretHashes{
+				AuthJWT:              "1",
+				RocksDBEncryptionKey: "2",
+				TLSCA:                "3",
+				SyncTLSCA:            "4",
+				Users: map[string]string{
+					"root": "123",
+				},
+			},
+			Expected: true,
+		},
+		{
+			Name: "Secret hashes are the same without users",
+			CompareFrom: &SecretHashes{
+				AuthJWT:              "1",
+				RocksDBEncryptionKey: "2",
+				TLSCA:                "3",
+				SyncTLSCA:            "4",
+			},
+			CompareTo: &SecretHashes{
+				AuthJWT:              "1",
+				RocksDBEncryptionKey: "2",
+				TLSCA:                "3",
+				SyncTLSCA:            "4",
+			},
+			Expected: true,
+		},
+	}
+
+	for _, testCase := range testCases {
+		//nolint:scopelint
+		t.Run(testCase.Name, func(t *testing.T) {
+			// Act
+			expected := testCase.CompareFrom.Equal(testCase.CompareTo)
+
+			// Assert
+			assert.Equal(t, testCase.Expected, expected)
+		})
+	}
+}

pkg/apis/deployment/v1alpha/zz_generated.deepcopy.go

@@ -367,7 +367,7 @@ func (in *DeploymentStatus) DeepCopyInto(out *DeploymentStatus) {
 	if in.SecretHashes != nil {
 		in, out := &in.SecretHashes, &out.SecretHashes
 		*out = new(SecretHashes)
-		**out = **in
+		(*in).DeepCopyInto(*out)
 	}
 	if in.ForceStatusReload != nil {
 		in, out := &in.ForceStatusReload, &out.ForceStatusReload
@@ -757,6 +757,13 @@ func (in *RocksDBSpec) DeepCopy() *RocksDBSpec {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *SecretHashes) DeepCopyInto(out *SecretHashes) {
 	*out = *in
+	if in.Users != nil {
+		in, out := &in.Users, &out.Users
+		*out = make(map[string]string, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
 	return
 }
 

pkg/deployment/bootstrap.go

@@ -32,13 +32,7 @@ import (
 	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
-	driver "github.com/arangodb/go-driver"
+	"github.com/arangodb/go-driver"
-
-	"github.com/arangodb/kube-arangodb/pkg/util/constants"
-)
-
-const (
-	rootUserName = "root"
 )
 
 // EnsureBootstrap executes the bootstrap once as soon as the deployment becomes ready
@@ -81,7 +75,9 @@ func (d *Deployment) ensureUserPasswordSecret(secrets k8sutil.SecretInterface, u
 	if auth, err := secrets.Get(secretName, metav1.GetOptions{}); k8sutil.IsNotFound(err) {
 		// Create new one
 		tokenData := make([]byte, 32)
-		rand.Read(tokenData)
+		if _, err = rand.Read(tokenData); err != nil {
+			return "", err
+		}
 		token := hex.EncodeToString(tokenData)
 		owner := d.GetAPIObject().AsOwner()
 
@@ -91,12 +87,9 @@ func (d *Deployment) ensureUserPasswordSecret(secrets k8sutil.SecretInterface, u
 
 		return token, nil
 	} else if err == nil {
-		user, ok := auth.Data[constants.SecretUsername]
+		user, pass, err := k8sutil.GetSecretAuthCredentials(auth)
-		if ok && string(user) == username {
+		if err == nil && user == username {
-			pass, ok := auth.Data[constants.SecretPassword]
+			return pass, nil
-			if ok {
-				return string(pass), nil
-			}
 		}
 		return "", fmt.Errorf("invalid secret format in secret %s", secretName)
 	} else {

pkg/deployment/informers.go

@@ -24,7 +24,7 @@ package deployment
 
 import (
 	"k8s.io/api/core/v1"
-	v1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
+	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
 	"k8s.io/client-go/tools/cache"
 
 	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"

pkg/deployment/resources/secret_hashes.go

@@ -23,12 +23,17 @@
 package resources
 
 import (
+	"context"
 	"crypto/sha256"
 	"encoding/hex"
 	"fmt"
 	"sort"
 	"strings"
 
+	"github.com/arangodb/go-driver"
+
+	v1 "k8s.io/api/core/v1"
+
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha"
@@ -44,10 +49,14 @@ func (r *Resources) ValidateSecretHashes() error {
 	// validate performs a secret hash comparison for a single secret.
 	// Return true if all is good, false when the SecretChanged condition
 	// must be set.
-	validate := func(secretName string, getExpectedHash func() string, setExpectedHash func(string) error) (bool, error) {
+	validate := func(secretName string,
+		getExpectedHash func() string,
+		setExpectedHash func(string) error,
+		actionHashChanged func(Context, *v1.Secret) error) (bool, error) {
+
 		log := r.log.With().Str("secret-name", secretName).Logger()
 		expectedHash := getExpectedHash()
-		hash, err := r.getSecretHash(secretName)
+		secret, hash, err := r.getSecretHash(secretName)
 		if expectedHash == "" {
 			// No hash set yet, try to fill it
 			if k8sutil.IsNotFound(err) {
@@ -78,6 +87,18 @@ func (r *Resources) ValidateSecretHashes() error {
 				Str("expected-hash", expectedHash).
 				Str("new-hash", hash).
 				Msg("Secret has changed.")
+			if actionHashChanged != nil {
+				if err := actionHashChanged(r.context, secret); err != nil {
+					log.Debug().Msgf("failed to change secret. hash-changed-action returned error: %v", err)
+					return true, nil
+				}
+
+				if err := setExpectedHash(hash); err != nil {
+					log.Debug().Msg("Failed to change secret hash")
+					return true, maskAny(err)
+				}
+				return true, nil
+			}
 			// This is not good, return false so SecretsChanged condition will be set.
 			return false, nil
 		}
@@ -91,13 +112,13 @@ func (r *Resources) ValidateSecretHashes() error {
 	status, lastVersion := r.context.GetStatus()
 	getHashes := func() *api.SecretHashes {
 		if status.SecretHashes == nil {
-			status.SecretHashes = &api.SecretHashes{}
+			status.SecretHashes = api.NewEmptySecretHashes()
 		}
 		return status.SecretHashes
 	}
 	updateHashes := func(updater func(*api.SecretHashes)) error {
 		if status.SecretHashes == nil {
-			status.SecretHashes = &api.SecretHashes{}
+			status.SecretHashes = api.NewEmptySecretHashes()
 		}
 		updater(status.SecretHashes)
 		if err := r.context.UpdateStatus(status, lastVersion); err != nil {
@@ -114,7 +135,7 @@ func (r *Resources) ValidateSecretHashes() error {
 		setExpectedHash := func(h string) error {
 			return maskAny(updateHashes(func(dst *api.SecretHashes) { dst.AuthJWT = h }))
 		}
-		if hashOK, err := validate(secretName, getExpectedHash, setExpectedHash); err != nil {
+		if hashOK, err := validate(secretName, getExpectedHash, setExpectedHash, nil); err != nil {
 			return maskAny(err)
 		} else if !hashOK {
 			badSecretNames = append(badSecretNames, secretName)
@@ -126,7 +147,7 @@ func (r *Resources) ValidateSecretHashes() error {
 		setExpectedHash := func(h string) error {
 			return maskAny(updateHashes(func(dst *api.SecretHashes) { dst.RocksDBEncryptionKey = h }))
 		}
-		if hashOK, err := validate(secretName, getExpectedHash, setExpectedHash); err != nil {
+		if hashOK, err := validate(secretName, getExpectedHash, setExpectedHash, nil); err != nil {
 			return maskAny(err)
 		} else if !hashOK {
 			badSecretNames = append(badSecretNames, secretName)
@@ -138,7 +159,7 @@ func (r *Resources) ValidateSecretHashes() error {
 		setExpectedHash := func(h string) error {
 			return maskAny(updateHashes(func(dst *api.SecretHashes) { dst.TLSCA = h }))
 		}
-		if hashOK, err := validate(secretName, getExpectedHash, setExpectedHash); err != nil {
+		if hashOK, err := validate(secretName, getExpectedHash, setExpectedHash, nil); err != nil {
 			return maskAny(err)
 		} else if !hashOK {
 			badSecretNames = append(badSecretNames, secretName)
@@ -150,13 +171,40 @@ func (r *Resources) ValidateSecretHashes() error {
 		setExpectedHash := func(h string) error {
 			return maskAny(updateHashes(func(dst *api.SecretHashes) { dst.SyncTLSCA = h }))
 		}
-		if hashOK, err := validate(secretName, getExpectedHash, setExpectedHash); err != nil {
+		if hashOK, err := validate(secretName, getExpectedHash, setExpectedHash, nil); err != nil {
 			return maskAny(err)
 		} else if !hashOK {
 			badSecretNames = append(badSecretNames, secretName)
 		}
 	}
 
+	for username, secretName := range spec.Bootstrap.PasswordSecretNames {
+		if secretName.IsNone() || secretName.IsAuto() {
+			continue
+		}
+
+		_, err := r.context.GetKubeCli().CoreV1().Secrets(r.context.GetNamespace()).Get(string(secretName), metav1.GetOptions{})
+		if k8sutil.IsNotFound(err) {
+			// do nothing when secret was deleted
+			continue
+		}
+
+		getExpectedHash := func() string {
+			if v, ok := getHashes().Users[username]; ok {
+				return v
+			}
+			return ""
+		}
+		setExpectedHash := func(h string) error {
+			return maskAny(updateHashes(func(dst *api.SecretHashes) {
+				dst.Users[username] = h
+			}))
+		}
+
+		// If password changes it should not be set that deployment in 'SecretsChanged' state
+		validate(string(secretName), getExpectedHash, setExpectedHash, changeUserPassword)
+	}
+
 	if len(badSecretNames) > 0 {
 		// We have invalid hashes, set the SecretsChanged condition
 		if status.Conditions.Update(api.ConditionTypeSecretsChanged, true,
@@ -185,13 +233,47 @@ func (r *Resources) ValidateSecretHashes() error {
 	return nil
 }
 
+func changeUserPassword(c Context, secret *v1.Secret) error {
+	username, password, err := k8sutil.GetSecretAuthCredentials(secret)
+	if err != nil {
+		return nil
+	}
+
+	ctx := context.Background()
+	client, err := c.GetDatabaseClient(ctx)
+	if err != nil {
+		return maskAny(err)
+	}
+
+	user, err := client.User(ctx, username)
+	if err != nil {
+		if driver.IsNotFound(err) {
+			options := &driver.UserOptions{
+				Password: password,
+				Active:   new(bool),
+			}
+			*options.Active = true
+
+			_, err = client.CreateUser(ctx, username, options)
+			return maskAny(err)
+		}
+		return err
+	}
+
+	err = user.Update(ctx, driver.UserOptions{
+		Password: password,
+	})
+
+	return maskAny(err)
+}
+
 // getSecretHash fetches a secret with given name and returns a hash over its value.
-func (r *Resources) getSecretHash(secretName string) (string, error) {
+func (r *Resources) getSecretHash(secretName string) (*v1.Secret, string, error) {
 	kubecli := r.context.GetKubeCli()
 	ns := r.context.GetNamespace()
 	s, err := kubecli.CoreV1().Secrets(ns).Get(secretName, metav1.GetOptions{})
 	if err != nil {
-		return "", maskAny(err)
+		return nil, "", maskAny(err)
 	}
 	// Create hash of value
 	rows := make([]string, 0, len(s.Data))
@@ -203,5 +285,5 @@ func (r *Resources) getSecretHash(secretName string) (string, error) {
 	data := strings.Join(rows, "\n")
 	rawHash := sha256.Sum256([]byte(data))
 	hash := fmt.Sprintf("%0x", rawHash)
-	return hash, nil
+	return s, hash, nil
 }

pkg/util/k8sutil/names.go

@@ -74,11 +74,9 @@ func stripArangodPrefix(id string) string {
 // complies with kubernetes name requirements.
 // If the name is to long or contains invalid characters,
 // if will be adjusted and a hash with be added.
-func FixupResourceName(name string, maxLength ...int) string {
+func FixupResourceName(name string) string {
 	maxLen := 63
-	if len(maxLength) > 0 {
+
-		maxLen = maxLength[0]
-	}
 	sb := strings.Builder{}
 	needHash := len(name) > maxLen
 	for _, ch := range name {

pkg/util/k8sutil/secrets.go

@@ -302,14 +302,18 @@ func GetBasicAuthSecret(secrets SecretInterface, secretName string) (string, str
 	if err != nil {
 		return "", "", maskAny(err)
 	}
-	// Load `ca.crt` field
+	return GetSecretAuthCredentials(s)
-	username, found := s.Data[constants.SecretUsername]
+}
+
+// GetSecretAuthCredentials returns username and password from the secret
+func GetSecretAuthCredentials(secret *v1.Secret) (string, string, error) {
+	username, found := secret.Data[constants.SecretUsername]
 	if !found {
-		return "", "", maskAny(fmt.Errorf("No '%s' found in secret '%s'", constants.SecretUsername, secretName))
+		return "", "", maskAny(fmt.Errorf("No '%s' found in secret '%s'", constants.SecretUsername, secret.Name))
 	}
-	password, found := s.Data[constants.SecretPassword]
+	password, found := secret.Data[constants.SecretPassword]
 	if !found {
-		return "", "", maskAny(fmt.Errorf("No '%s' found in secret '%s'", constants.SecretPassword, secretName))
+		return "", "", maskAny(fmt.Errorf("No '%s' found in secret '%s'", constants.SecretPassword, secret.Name))
 	}
 	return string(username), string(password), nil
 }

tests/secret_hashes_test.go

@@ -0,0 +1,144 @@
+//
+// DISCLAIMER
+//
+// Copyright 2019 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author tomasz@arangodb.con
+//
+
+package tests
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/arangodb/go-driver"
+
+	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha"
+	"github.com/arangodb/kube-arangodb/pkg/client"
+	"github.com/arangodb/kube-arangodb/pkg/util/arangod"
+	"github.com/arangodb/kube-arangodb/pkg/util/constants"
+	"github.com/arangodb/kube-arangodb/pkg/util/retry"
+	"github.com/dchest/uniuri"
+	"github.com/pkg/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// TestSecretHashesRootUser checks if Status.SecretHashes.Users[root] changed after request for it
+func TestSecretHashesRootUser(t *testing.T) {
+	longOrSkip(t)
+	c := client.MustNewInCluster()
+	kubecli := mustNewKubeClient(t)
+	ns := getNamespace(t)
+
+	// Prepare deployment config
+	depl := newDeployment("test-auth-sng-def-" + uniuri.NewLen(4))
+	depl.Spec.Mode = api.NewMode(api.DeploymentModeSingle)
+	depl.Spec.SetDefaults(depl.GetName())
+	depl.Spec.Bootstrap.PasswordSecretNames[api.UserNameRoot] = api.PasswordSecretNameAuto
+
+	// Create deployment
+	apiObject, err := c.DatabaseV1alpha().ArangoDeployments(ns).Create(depl)
+	if err != nil {
+		t.Fatalf("Create deployment failed: %v", err)
+	}
+	defer deferedCleanupDeployment(c, depl.GetName(), ns)
+
+	// Wait for deployment to be ready
+	depl, err = waitUntilDeployment(c, depl.GetName(), ns, deploymentIsReady())
+	if err != nil {
+		t.Fatalf("Deployment not running in time: %v", err)
+	}
+
+	// Create a database client
+	ctx := arangod.WithRequireAuthentication(context.Background())
+	client := mustNewArangodDatabaseClient(ctx, kubecli, apiObject, t, nil)
+
+	// Wait for single server available
+	if err := waitUntilVersionUp(client, nil); err != nil {
+		t.Fatalf("Single server not running returning version in time: %v", err)
+	}
+
+	depl, err = waitUntilDeployment(c, depl.GetName(), ns, func(obj *api.ArangoDeployment) error {
+		// check if root secret password is set
+		secretHashes := obj.Status.SecretHashes
+		if secretHashes == nil {
+			return fmt.Errorf("field Status.SecretHashes is not set")
+		}
+
+		if secretHashes.Users == nil {
+			return fmt.Errorf("field Status.SecretHashes.Users is not set")
+		}
+
+		if hash, ok := secretHashes.Users[api.UserNameRoot]; !ok {
+			return fmt.Errorf("field Status.SecretHashes.Users[root] is not set")
+		} else if len(hash) == 0 {
+			return fmt.Errorf("field Status.SecretHashes.Users[root] is empty")
+		}
+
+		return nil
+	})
+
+	if err != nil {
+		t.Fatalf("Deployment is not set properly: %v", err)
+	}
+	rootHashSecret := depl.Status.SecretHashes.Users[api.UserNameRoot]
+
+	secretRootName := string(depl.Spec.Bootstrap.PasswordSecretNames[api.UserNameRoot])
+	secretRoot, err := waitUntilSecret(kubecli, secretRootName, ns, nil, time.Second)
+	if err != nil {
+		t.Fatalf("Root secret '%s' not found: %v", secretRootName, err)
+	}
+
+	secretRoot.Data[constants.SecretPassword] = []byte("1")
+	_, err = kubecli.CoreV1().Secrets(ns).Update(secretRoot)
+	if err != nil {
+		t.Fatalf("Root secret '%s' has not been changed: %v", secretRootName, err)
+	}
+
+	err = retry.Retry(func() error {
+		// check if root secret hash has changed
+		depl, err = c.DatabaseV1alpha().ArangoDeployments(ns).Get(depl.GetName(), metav1.GetOptions{})
+		if err != nil {
+			t.Fatalf("Failed to get deployment: %v", err)
+		}
+
+		if rootHashSecret == depl.Status.SecretHashes.Users[api.UserNameRoot] {
+			return maskAny(errors.New("field Status.SecretHashes.Users[root] has not been changed yet"))
+		}
+		return nil
+	}, deploymentReadyTimeout)
+	if err != nil {
+		t.Fatalf("%v", err)
+	}
+
+	// Check if password changed
+	auth := driver.BasicAuthentication(api.UserNameRoot, "1")
+	_, err = client.Connection().SetAuthentication(auth)
+	if err != nil {
+		t.Fatalf("The password for user '%s' has not been changed: %v", api.UserNameRoot, err)
+	}
+	_, err = client.Version(context.Background())
+	if err != nil {
+		t.Fatalf("can not get version after the password has been changed")
+	}
+
+	//Cleanup
+	removeDeployment(c, depl.GetName(), ns)
+}