Merge branch 'master' into deployment-phase

Jenkinsfile.groovy

@@ -26,6 +26,11 @@ def fetchParamsFromGitLog() {
         myParams[entry.key] = entry.value;
     }
 
+    // Is this a LONG test?
+    if ("${env.JOB_NAME}" == "kube-arangodb-long") {
+        myParams["LONG"] = true;
+    }
+
     // Fetch params configured in git commit messages 
     // Syntax: [ci OPT=value]
     // Example: [ci TESTOPTIONS="-test.run ^TestSimpleSingle$"]

Makefile

@@ -50,6 +50,7 @@ endif
 endif
 MANIFESTPATHDEPLOYMENT := manifests/arango-deployment$(MANIFESTSUFFIX).yaml
 MANIFESTPATHSTORAGE := manifests/arango-storage$(MANIFESTSUFFIX).yaml
+MANIFESTPATHTEST := manifests/arango-test$(MANIFESTSUFFIX).yaml
 ifndef DEPLOYMENTNAMESPACE
 	DEPLOYMENTNAMESPACE := default
 endif
@@ -75,7 +76,7 @@ TESTLENGTHOPTIONS := -test.short
 TESTTIMEOUT := 20m
 ifeq ($(LONG), 1)
 	TESTLENGTHOPTIONS :=
-	TESTTIMEOUT := 40m
+	TESTTIMEOUT := 60m
 endif
 ifdef VERBOSE
 	TESTVERBOSEOPTIONS := -v 
@@ -216,6 +217,7 @@ run-unit-tests: $(GOBUILDDIR) $(SOURCES)
 		golang:$(GOVERSION) \
 		go test $(TESTVERBOSEOPTIONS) \
 			$(REPOPATH)/pkg/apis/deployment/v1alpha \
+			$(REPOPATH)/pkg/apis/storage/v1alpha \
 			$(REPOPATH)/pkg/deployment/reconcile \
 			$(REPOPATH)/pkg/deployment/resources \
 			$(REPOPATH)/pkg/util/k8sutil \
@@ -251,6 +253,7 @@ endif
 	kubectl apply -f manifests/crd.yaml
 	kubectl apply -f $(MANIFESTPATHSTORAGE)
 	kubectl apply -f $(MANIFESTPATHDEPLOYMENT)
+	kubectl apply -f $(MANIFESTPATHTEST)
 	$(ROOTDIR)/scripts/kube_create_storage.sh $(DEPLOYMENTNAMESPACE)
 	$(ROOTDIR)/scripts/kube_run_tests.sh $(DEPLOYMENTNAMESPACE) $(TESTIMAGE) "$(ENTERPRISEIMAGE)" $(TESTTIMEOUT) $(TESTLENGTHOPTIONS)
 ifneq ($(DEPLOYMENTNAMESPACE), default)
@@ -312,6 +315,7 @@ minikube-start:
 
 .PHONY: delete-operator
 delete-operator:
+	kubectl delete -f $(MANIFESTPATHTEST) --ignore-not-found
 	kubectl delete -f $(MANIFESTPATHDEPLOYMENT) --ignore-not-found
 	kubectl delete -f $(MANIFESTPATHSTORAGE) --ignore-not-found
 
@@ -320,4 +324,5 @@ redeploy-operator: delete-operator manifests
 	kubectl apply -f manifests/crd.yaml
 	kubectl apply -f $(MANIFESTPATHSTORAGE)
 	kubectl apply -f $(MANIFESTPATHDEPLOYMENT)
+	kubectl apply -f $(MANIFESTPATHTEST)
 	kubectl get pods 

manifests/.gitignore

@@ -1,2 +1,3 @@
 arango-deployment-dev.yaml
-arango-storage-dev.yaml
+arango-storage-dev.yaml
+arango-test-dev.yaml

manifests/templates/deployment/rbac.yaml

@@ -69,4 +69,4 @@ subjects:
   name: {{ .Deployment.Operator.ServiceAccountName }}
   namespace: {{ .Deployment.Operator.Namespace }}
 
-{{- end -}}
+{{- end -}}

manifests/templates/test/rbac.yaml

@@ -0,0 +1,31 @@
+{{- if .RBAC -}}
+
+## Cluster role granting access to resources needed by the integration tests.
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRole
+metadata:
+  name: {{ .Test.RoleName }}
+rules:
+- apiGroups: [""]
+  resources: ["nodes"]
+  verbs: ["list"]
+
+---
+
+## Bind the cluster role granting access to ArangoLocalStorage resources
+## to the default service account of the configured namespace.
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ .Test.RoleBindingName }}
+  namespace: {{ .Test.Namespace }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ .Test.RoleName }}
+subjects:
+- kind: ServiceAccount
+  name: {{ .Test.ServiceAccountName }}
+  namespace: {{ .Test.Namespace }}
+
+{{- end -}}

pkg/apis/deployment/v1alpha/member_status.go

@@ -22,6 +22,12 @@
 
 package v1alpha
 
+import (
+	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
 // MemberStatus holds the current status of a single member (server)
 type MemberStatus struct {
 	// ID holds the unique ID of the member.
@@ -35,4 +41,40 @@ type MemberStatus struct {
 	PodName string `json:"podName,omitempty"`
 	// Conditions specific to this member
 	Conditions ConditionList `json:"conditions,omitempty"`
+	// RecentTerminatons holds the times when this member was recently terminated.
+	// First entry is the oldest. (do not add omitempty, since we want to be able to switch from a list to an empty list)
+	RecentTerminations []metav1.Time `json:"recent-terminations"`
+}
+
+// RemoveTerminationsBefore removes all recent terminations before the given timestamp.
+// It returns the number of terminations that have been removed.
+func (s *MemberStatus) RemoveTerminationsBefore(timestamp time.Time) int {
+	removed := 0
+	for {
+		if len(s.RecentTerminations) == 0 {
+			// Nothing left
+			return removed
+		}
+		if s.RecentTerminations[0].Time.Before(timestamp) {
+			// Let's remove it
+			s.RecentTerminations = s.RecentTerminations[1:]
+			removed++
+		} else {
+			// First (oldest) is not before given timestamp, we're done
+			return removed
+		}
+	}
+}
+
+// RecentTerminationsSince returns the number of terminations since the given timestamp.
+func (s MemberStatus) RecentTerminationsSince(timestamp time.Time) int {
+	count := 0
+	for idx := len(s.RecentTerminations) - 1; idx >= 0; idx-- {
+		if s.RecentTerminations[idx].Time.Before(timestamp) {
+			// This termination is before the timestamp, so we're done
+			return count
+		}
+		count++
+	}
+	return count
 }

pkg/apis/deployment/v1alpha/member_status_test.go

@@ -0,0 +1,53 @@
+//
+// DISCLAIMER
+//
+// Copyright 2018 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Ewout Prangsma
+//
+
+package v1alpha
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// TestMemberStatusRecentTerminations tests the functions related to MemberStatus.RecentTerminations.
+func TestMemberStatusRecentTerminations(t *testing.T) {
+	relTime := func(delta time.Duration) metav1.Time {
+		return metav1.Time{Time: time.Now().Add(delta)}
+	}
+
+	s := MemberStatus{}
+	assert.Equal(t, 0, s.RecentTerminationsSince(time.Now().Add(-time.Hour)))
+	assert.Equal(t, 0, s.RemoveTerminationsBefore(time.Now()))
+
+	s.RecentTerminations = []metav1.Time{metav1.Now()}
+	assert.Equal(t, 1, s.RecentTerminationsSince(time.Now().Add(-time.Minute)))
+	assert.Equal(t, 0, s.RecentTerminationsSince(time.Now().Add(time.Minute)))
+	assert.Equal(t, 0, s.RemoveTerminationsBefore(time.Now().Add(-time.Hour)))
+
+	s.RecentTerminations = []metav1.Time{relTime(-time.Hour), relTime(-time.Minute), relTime(time.Minute)}
+	assert.Equal(t, 3, s.RecentTerminationsSince(time.Now().Add(-time.Hour*2)))
+	assert.Equal(t, 2, s.RecentTerminationsSince(time.Now().Add(-time.Minute*2)))
+	assert.Equal(t, 2, s.RemoveTerminationsBefore(time.Now()))
+	assert.Len(t, s.RecentTerminations, 1)
+}

pkg/apis/deployment/v1alpha/zz_generated.deepcopy.go

@@ -371,6 +371,13 @@ func (in *MemberStatus) DeepCopyInto(out *MemberStatus) {
 			(*in)[i].DeepCopyInto(&(*out)[i])
 		}
 	}
+	if in.RecentTerminations != nil {
+		in, out := &in.RecentTerminations, &out.RecentTerminations
+		*out = make([]v1.Time, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
 	return
 }
 

pkg/apis/storage/v1alpha/local_storage_spec.go

@@ -45,6 +45,11 @@ func (s LocalStorageSpec) Validate() error {
 	if len(s.LocalPath) == 0 {
 		return maskAny(errors.Wrapf(ValidationError, "localPath cannot be empty"))
 	}
+	for _, p := range s.LocalPath {
+		if len(p) == 0 {
+			return maskAny(errors.Wrapf(ValidationError, "localPath cannot contain empty strings"))
+		}
+	}
 	return nil
 }
 

pkg/apis/storage/v1alpha/local_storage_spec_test.go

@@ -0,0 +1,58 @@
+//
+// DISCLAIMER
+//
+// Copyright 2018 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Jan Christoph Uhde
+//
+
+package v1alpha
+
+import (
+	"testing"
+
+	//"github.com/pkg/errors"
+	"github.com/stretchr/testify/assert"
+)
+
+// Test creation of local storage spec
+func TestLocalStorageSpecCreation(t *testing.T) {
+
+	class := StorageClassSpec{"SpecName", true}
+	local := LocalStorageSpec{StorageClass: class, LocalPath: []string{""}}
+	assert.Error(t, local.Validate())
+
+	class = StorageClassSpec{"spec-name", true}
+	local = LocalStorageSpec{StorageClass: class, LocalPath: []string{""}}
+	assert.Error(t, local.Validate(), "should fail as the empty sting is not a valid path")
+
+	class = StorageClassSpec{"spec-name", true}
+	local = LocalStorageSpec{StorageClass: class, LocalPath: []string{}}
+	assert.True(t, IsValidation(local.Validate()))
+}
+
+// Test reset of local storage spec
+func TestLocalStorageSpecReset(t *testing.T) {
+	class := StorageClassSpec{"spec-name", true}
+	source := LocalStorageSpec{StorageClass: class, LocalPath: []string{"/a/path", "/another/path"}}
+	target := LocalStorageSpec{}
+	resetImmutableFieldsResult := source.ResetImmutableFields(&target)
+	expected := []string{"storageClass.name", "localPath"}
+	assert.Equal(t, expected, resetImmutableFieldsResult)
+	assert.Equal(t, source.LocalPath, target.LocalPath)
+	assert.Equal(t, source.StorageClass.Name, target.StorageClass.Name)
+}

pkg/apis/storage/v1alpha/storage_class_spec_test.go

@@ -0,0 +1,57 @@
+//
+// DISCLAIMER
+//
+// Copyright 2018 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Jan Christoph Uhde
+//
+
+package v1alpha
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+// test creation of storage class spec
+func TestStorageClassSpecCreation(t *testing.T) {
+	storageClassSpec := StorageClassSpec{}
+	assert.Error(t, storageClassSpec.Validate(), "empty name name is not allowed")
+
+	storageClassSpec = StorageClassSpec{Name: "TheSpecName", IsDefault: true}
+	assert.Error(t, storageClassSpec.Validate(), "upper case letters are not allowed in resources")
+
+	storageClassSpec = StorageClassSpec{"the-spec-name", true}
+	assert.NoError(t, storageClassSpec.Validate())
+
+	storageClassSpec = StorageClassSpec{} // no proper name -> invalid
+	storageClassSpec.SetDefaults("foo")   // name is fixed -> vaild
+	assert.NoError(t, storageClassSpec.Validate())
+}
+
+// test reset of storage class spec
+func TestStorageClassSpecResetImmutableFileds(t *testing.T) {
+	specSource := StorageClassSpec{"source", true}
+	specTarget := StorageClassSpec{"target", true}
+
+	assert.Equal(t, "target", specTarget.Name)
+	rv := specSource.ResetImmutableFields("fieldPrefix-", &specTarget)
+	assert.Equal(t, "fieldPrefix-name", strings.Join(rv, ", "))
+	assert.Equal(t, "source", specTarget.Name)
+}

pkg/deployment/reconcile/action_rotate_member.go

@@ -110,6 +110,7 @@ func (a *actionRotateMember) CheckProgress(ctx context.Context) (bool, error) {
 	}
 	// Pod is now gone, update the member status
 	m.State = api.MemberStateNone
+	m.RecentTerminations = nil // Since we're rotating, we do not care about old terminations.
 	if err := a.actionCtx.UpdateMember(m); err != nil {
 		return false, maskAny(err)
 	}

pkg/deployment/reconcile/action_upgrade_member.go

@@ -120,6 +120,7 @@ func (a *actionUpgradeMember) CheckProgress(ctx context.Context) (bool, error) {
 	}
 	// Pod is now gone, update the member status
 	m.State = api.MemberStateNone
+	m.RecentTerminations = nil // Since we're upgrading, we do not care about old terminations.
 	if err := a.actionCtx.UpdateMember(m); err != nil {
 		return false, maskAny(err)
 	}

pkg/deployment/resources/pod_inspector.go

@@ -26,11 +26,12 @@ import (
 	"fmt"
 	"time"
 
-	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
 	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha"
 	"github.com/arangodb/kube-arangodb/pkg/metrics"
+	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
 )
 
 var (
@@ -81,12 +82,18 @@ func (r *Resources) InspectPods() error {
 			if memberStatus.Conditions.Update(api.ConditionTypeTerminated, true, "Pod Succeeded", "") {
 				log.Debug().Str("pod-name", p.GetName()).Msg("Updating member condition Terminated to true: Pod Succeeded")
 				updateMemberStatusNeeded = true
+				// Record termination time
+				now := metav1.Now()
+				memberStatus.RecentTerminations = append(memberStatus.RecentTerminations, now)
 			}
 		} else if k8sutil.IsPodFailed(&p) {
 			// Pod has terminated with at least 1 container with a non-zero exit code.
 			if memberStatus.Conditions.Update(api.ConditionTypeTerminated, true, "Pod Failed", "") {
 				log.Debug().Str("pod-name", p.GetName()).Msg("Updating member condition Terminated to true: Pod Failed")
 				updateMemberStatusNeeded = true
+				// Record termination time
+				now := metav1.Now()
+				memberStatus.RecentTerminations = append(memberStatus.RecentTerminations, now)
 			}
 		}
 		if k8sutil.IsPodReady(&p) {

tests/auth_test.go

@@ -49,7 +49,7 @@ func TestAuthenticationSingleDefaultSecret(t *testing.T) {
 	client := mustNewArangodDatabaseClient(ctx, kubecli, apiObject, t)
 
 	// Wait for single server available
-	if err := waitUntilVersionUp(client); err != nil {
+	if err := waitUntilVersionUp(client, nil); err != nil {
 		t.Fatalf("Single server not running returning version in time: %v", err)
 	}
 
@@ -97,7 +97,7 @@ func TestAuthenticationSingleCustomSecret(t *testing.T) {
 	client := mustNewArangodDatabaseClient(ctx, kubecli, apiObject, t)
 
 	// Wait for single server available
-	if err := waitUntilVersionUp(client); err != nil {
+	if err := waitUntilVersionUp(client, nil); err != nil {
 		t.Fatalf("Single server not running returning version in time: %v", err)
 	}
 
@@ -143,7 +143,7 @@ func TestAuthenticationNoneSingle(t *testing.T) {
 	client := mustNewArangodDatabaseClient(ctx, kubecli, apiObject, t)
 
 	// Wait for single server available
-	if err := waitUntilVersionUp(client); err != nil {
+	if err := waitUntilVersionUp(client, nil); err != nil {
 		t.Fatalf("Single server not running returning version in time: %v", err)
 	}
 
@@ -185,7 +185,7 @@ func TestAuthenticationClusterDefaultSecret(t *testing.T) {
 	client := mustNewArangodDatabaseClient(ctx, kubecli, apiObject, t)
 
 	// Wait for single server available
-	if err := waitUntilVersionUp(client); err != nil {
+	if err := waitUntilVersionUp(client, nil); err != nil {
 		t.Fatalf("Single server not running returning version in time: %v", err)
 	}
 
@@ -233,7 +233,7 @@ func TestAuthenticationClusterCustomSecret(t *testing.T) {
 	client := mustNewArangodDatabaseClient(ctx, kubecli, apiObject, t)
 
 	// Wait for single server available
-	if err := waitUntilVersionUp(client); err != nil {
+	if err := waitUntilVersionUp(client, nil); err != nil {
 		t.Fatalf("Single server not running returning version in time: %v", err)
 	}
 
@@ -279,7 +279,7 @@ func TestAuthenticationNoneCluster(t *testing.T) {
 	client := mustNewArangodDatabaseClient(ctx, kubecli, apiObject, t)
 
 	// Wait for single server available
-	if err := waitUntilVersionUp(client); err != nil {
+	if err := waitUntilVersionUp(client, nil); err != nil {
 		t.Fatalf("Single server not running returning version in time: %v", err)
 	}
 

tests/deployments_test.go

@@ -27,14 +27,10 @@ import (
 
 	"github.com/dchest/uniuri"
 
-	driver "github.com/arangodb/go-driver"
 	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha"
 	kubeArangoClient "github.com/arangodb/kube-arangodb/pkg/client"
-	arangod "github.com/arangodb/kube-arangodb/pkg/util/arangod"
 )
 
-// TODO - environements (provided from outside)
-
 // test deployment single server mmfiles
 func TestDeploymentSingleMMFiles(t *testing.T) {
 	deploymentSubTest(t, api.DeploymentModeSingle, api.StorageEngineMMFiles)
@@ -96,64 +92,8 @@ func deploymentSubTest(t *testing.T, mode api.DeploymentMode, engine api.Storage
 	ctx := context.Background()
 	DBClient := mustNewArangodDatabaseClient(ctx, k8sClient, deployment, t)
 
-	// deployment checks
-	switch mode := deployment.Spec.GetMode(); mode {
-	case api.DeploymentModeCluster:
-		// Wait for cluster to be completely ready
-		if err := waitUntilClusterHealth(DBClient, func(h driver.ClusterHealth) error {
-			return clusterHealthEqualsSpec(h, deployment.Spec)
-		}); err != nil {
-			t.Fatalf("Cluster not running in expected health in time: %v", err)
-		}
-	case api.DeploymentModeSingle:
-		if err := waitUntilVersionUp(DBClient); err != nil {
-			t.Fatalf("Single Server not running in time: %v", err)
-		}
-	case api.DeploymentModeResilientSingle:
-		if err := waitUntilVersionUp(DBClient); err != nil {
-			t.Fatalf("Single Server not running in time: %v", err)
-		}
-
-		members := deployment.Status.Members
-		singles := members.Single
-		agents := members.Agents
-
-		if len(singles) != 2 || len(agents) != 3 {
-			t.Fatalf("Wrong number of servers: single %d - agents %d", len(singles), len(agents))
-		}
-
-		for _, agent := range agents {
-			dbclient, err := arangod.CreateArangodClient(ctx, k8sClient.CoreV1(), deployment, api.ServerGroupAgents, agent.ID)
-			if err != nil {
-				t.Fatalf("Unable to create connection to: %s", agent.ID)
-			}
-
-			if err := waitUntilVersionUp(dbclient); err != nil {
-				t.Fatalf("Version check failed for: %s", agent.ID)
-			}
-		}
-
-		var goodResults, noLeaderResults int
-		for _, single := range singles {
-			dbclient, err := arangod.CreateArangodClient(ctx, k8sClient.CoreV1(), deployment, api.ServerGroupSingle, single.ID)
-			if err != nil {
-				t.Fatalf("Unable to create connection to: %s", single.ID)
-			}
-
-			if err := waitUntilVersionUp(dbclient, true); err == nil {
-				goodResults++
-			} else if driver.IsNoLeader(err) {
-				noLeaderResults++
-			} else {
-				t.Fatalf("Version check failed for: %s", single.ID)
-			}
-		}
-
-		if goodResults < 1 || noLeaderResults > 1 {
-			t.Fatalf("Wrong number of results: good %d - noleader %d", goodResults, noLeaderResults)
-		}
-	default:
-		t.Fatalf("DeploymentMode %v is not supported!", mode)
+	if err := waitUntilArangoDeploymentHealthy(deployment, DBClient, k8sClient, ""); err != nil {
+		t.Fatalf("Deployment not healthy in time: %v", err)
 	}
 
 	// Cleanup

tests/environments_test.go

@@ -0,0 +1,85 @@
+//
+// DISCLAIMER
+//
+// Copyright 2018 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Jan Christoph Uhde <jan@uhdejc.com>
+//
+package tests
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/dchest/uniuri"
+	"github.com/stretchr/testify/assert"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha"
+	kubeArangoClient "github.com/arangodb/kube-arangodb/pkg/client"
+	"github.com/arangodb/kube-arangodb/pkg/util"
+)
+
+// Test if deployment comes up in production environment.
+// LONG: The test ensures that the deployment fails if there are
+// less nodes available than servers required.
+func TestProduction(t *testing.T) {
+	longOrSkip(t)
+
+	mode := api.DeploymentModeCluster
+	engine := api.StorageEngineRocksDB
+
+	k8sNameSpace := getNamespace(t)
+	k8sClient := mustNewKubeClient(t)
+
+	nodeList, err := k8sClient.CoreV1().Nodes().List(metav1.ListOptions{})
+	if err != nil {
+		t.Fatalf("Unable to receive node list: %v", err)
+	}
+	numNodes := len(nodeList.Items)
+
+	deploymentClient := kubeArangoClient.MustNewInCluster()
+	deploymentTemplate := newDeployment(strings.Replace(fmt.Sprintf("tprod-%s-%s-%s", mode[:2], engine[:2], uniuri.NewLen(4)), ".", "", -1))
+	deploymentTemplate.Spec.Mode = api.NewMode(mode)
+	deploymentTemplate.Spec.StorageEngine = api.NewStorageEngine(engine)
+	deploymentTemplate.Spec.TLS = api.TLSSpec{}
+	deploymentTemplate.Spec.Environment = api.NewEnvironment(api.EnvironmentProduction)
+	deploymentTemplate.Spec.Image = util.NewString("arangodb/arangodb:3.3.4")
+	deploymentTemplate.Spec.DBServers.Count = util.NewInt(numNodes + 1)
+	deploymentTemplate.Spec.SetDefaults(deploymentTemplate.GetName()) // this must be last
+	assert.NoError(t, deploymentTemplate.Spec.Validate())
+
+	dbserverCount := *deploymentTemplate.Spec.DBServers.Count
+	if dbserverCount < 3 {
+		t.Fatalf("Not enough DBServers to run this test: server count %d", dbserverCount)
+	}
+
+	// Create deployment
+	_, err = deploymentClient.DatabaseV1alpha().ArangoDeployments(k8sNameSpace).Create(deploymentTemplate)
+	if err != nil {
+		// REVIEW - should the test already fail here
+		t.Fatalf("Create deployment failed: %v", err)
+	}
+
+	_, err = waitUntilDeployment(deploymentClient, deploymentTemplate.GetName(), k8sNameSpace, deploymentIsReady())
+	assert.Error(t, err, fmt.Sprintf("Deployment is up and running when it should not! There are not enough nodes(%d) for all DBServers(%d) in production modes.", numNodes, dbserverCount))
+
+	// Cleanup
+	removeDeployment(deploymentClient, deploymentTemplate.GetName(), k8sNameSpace)
+}

tests/immutable_test.go

@@ -65,7 +65,7 @@ func TestImmutableFields(t *testing.T) {
 	client := mustNewArangodDatabaseClient(ctx, kubecli, apiObject, t)
 
 	// Wait for single server to be completely ready
-	if err := waitUntilVersionUp(client); err != nil {
+	if err := waitUntilVersionUp(client, nil); err != nil {
 		t.Fatalf("Single server not up in time: %v", err)
 	}
 

tests/rocksdb_encryption_test.go

@@ -46,7 +46,7 @@ func TestRocksDBEncryptionSingle(t *testing.T) {
 	// Wait for deployment to be ready
 	apiObject, err := waitUntilDeployment(c, depl.GetName(), ns, deploymentIsReady())
 	if err != nil {
-		t.Errorf("Deployment not running in time: %v", err)
+		t.Fatalf("Deployment not running in time: %v", err)
 	}
 
 	// Create database client
@@ -54,7 +54,7 @@ func TestRocksDBEncryptionSingle(t *testing.T) {
 	client := mustNewArangodDatabaseClient(ctx, kubecli, apiObject, t)
 
 	// Wait for single server available
-	if err := waitUntilVersionUp(client); err != nil {
+	if err := waitUntilVersionUp(client, nil); err != nil {
 		t.Fatalf("Single server not running returning version in time: %v", err)
 	}
 

tests/simple_test.go

@@ -42,7 +42,7 @@ func TestSimpleSingle(t *testing.T) {
 	client := mustNewArangodDatabaseClient(ctx, kubecli, apiObject, t)
 
 	// Wait for single server available
-	if err := waitUntilVersionUp(client); err != nil {
+	if err := waitUntilVersionUp(client, nil); err != nil {
 		t.Fatalf("Single server not running returning version in time: %v", err)
 	}
 
@@ -83,7 +83,7 @@ func TestSimpleResilientSingle(t *testing.T) {
 	client := mustNewArangodDatabaseClient(ctx, kubecli, apiObject, t)
 
 	// Wait for single server available
-	if err := waitUntilVersionUp(client); err != nil {
+	if err := waitUntilVersionUp(client, nil); err != nil {
 		t.Fatalf("ResilientSingle servers not running returning version in time: %v", err)
 	}
 
@@ -124,7 +124,7 @@ func TestSimpleCluster(t *testing.T) {
 	client := mustNewArangodDatabaseClient(ctx, kubecli, apiObject, t)
 
 	// Wait for single server available
-	if err := waitUntilVersionUp(client); err != nil {
+	if err := waitUntilVersionUp(client, nil); err != nil {
 		t.Fatalf("Cluster not running returning version in time: %v", err)
 	}
 

tests/test_util.go

@@ -210,18 +210,21 @@ func waitUntilClusterHealth(cli driver.Client, predicate func(driver.ClusterHeal
 }
 
 // waitUntilVersionUp waits until the arango database responds to
-// an `/_api/version` request without an error.
-func waitUntilVersionUp(cli driver.Client, allowNoLeaderResponse ...bool) error {
+// an `/_api/version` request without an error. An additional Predicate
+// can do a check on the VersionInfo object returned by the server.
+func waitUntilVersionUp(cli driver.Client, predicate func(driver.VersionInfo) error, allowNoLeaderResponse ...bool) error {
 	var noLeaderErr error
 	allowNoLead := len(allowNoLeaderResponse) > 0 && allowNoLeaderResponse[0]
 	ctx := context.Background()
 
 	op := func() error {
-		if _, err := cli.Version(ctx); allowNoLead && driver.IsNoLeader(err) {
+		if version, err := cli.Version(ctx); allowNoLead && driver.IsNoLeader(err) {
 			noLeaderErr = err
 			return nil //return nil to make the retry below pass
 		} else if err != nil {
 			return maskAny(err)
+		} else if predicate != nil {
+			return predicate(version)
 		}
 		return nil
 	}
@@ -238,6 +241,16 @@ func waitUntilVersionUp(cli driver.Client, allowNoLeaderResponse ...bool) error
 	return nil
 }
 
+// creates predicate to be used in waitUntilVersionUp
+func createEqualVersionsPredicate(version driver.Version) func(driver.VersionInfo) error {
+	return func(infoFromServer driver.VersionInfo) error {
+		if version.CompareTo(infoFromServer.Version) != 0 {
+			return maskAny(fmt.Errorf("given version %v and version from server %v do not match", version, infoFromServer.Version))
+		}
+		return nil
+	}
+}
+
 // clusterHealthEqualsSpec returns nil when the given health matches
 // with the given deployment spec.
 func clusterHealthEqualsSpec(h driver.ClusterHealth, spec api.DeploymentSpec) error {
@@ -301,3 +314,79 @@ func removeSecret(cli kubernetes.Interface, secretName, ns string) error {
 	}
 	return nil
 }
+
+// check if a deployment is up and has reached a state where it is able to answer to /_api/version requests.
+// Optionally the returned version can be checked against a user provided version
+func waitUntilArangoDeploymentHealthy(deployment *api.ArangoDeployment, DBClient driver.Client, k8sClient kubernetes.Interface, versionString string) error {
+	// deployment checks
+	var checkVersionPredicate func(driver.VersionInfo) error
+	if len(versionString) > 0 {
+		checkVersionPredicate = createEqualVersionsPredicate(driver.Version(versionString))
+	}
+	switch mode := deployment.Spec.GetMode(); mode {
+	case api.DeploymentModeCluster:
+		// Wait for cluster to be completely ready
+		if err := waitUntilClusterHealth(DBClient, func(h driver.ClusterHealth) error {
+			return clusterHealthEqualsSpec(h, deployment.Spec)
+		}); err != nil {
+			return maskAny(fmt.Errorf("Cluster not running in expected health in time: %s", err))
+		}
+	case api.DeploymentModeSingle:
+		if err := waitUntilVersionUp(DBClient, checkVersionPredicate); err != nil {
+			return maskAny(fmt.Errorf("Single Server not running in time: %s", err))
+		}
+	case api.DeploymentModeResilientSingle:
+		if err := waitUntilVersionUp(DBClient, checkVersionPredicate); err != nil {
+			return maskAny(fmt.Errorf("Single Server not running in time: %s", err))
+		}
+
+		members := deployment.Status.Members
+		singles := members.Single
+		agents := members.Agents
+
+		if len(singles) != *deployment.Spec.Single.Count || len(agents) != *deployment.Spec.Agents.Count {
+			return maskAny(fmt.Errorf("Wrong number of servers: single %d - agents %d", len(singles), len(agents)))
+		}
+
+		ctx := context.Background()
+
+		//check agents
+		for _, agent := range agents {
+			dbclient, err := arangod.CreateArangodClient(ctx, k8sClient.CoreV1(), deployment, api.ServerGroupAgents, agent.ID)
+			if err != nil {
+				return maskAny(fmt.Errorf("Unable to create connection to: %s", agent.ID))
+			}
+
+			if err := waitUntilVersionUp(dbclient, checkVersionPredicate); err != nil {
+				return maskAny(fmt.Errorf("Version check failed for: %s", agent.ID))
+			}
+		}
+		//check single servers
+		{
+			var goodResults, noLeaderResults int
+			for _, single := range singles {
+				dbclient, err := arangod.CreateArangodClient(ctx, k8sClient.CoreV1(), deployment, api.ServerGroupSingle, single.ID)
+				if err != nil {
+					return maskAny(fmt.Errorf("Unable to create connection to: %s", single.ID))
+				}
+
+				if err := waitUntilVersionUp(dbclient, checkVersionPredicate, true); err == nil {
+					goodResults++
+				} else if driver.IsNoLeader(err) {
+					noLeaderResults++
+				} else {
+					return maskAny(fmt.Errorf("Version check failed for: %s", single.ID))
+				}
+			}
+
+			expectedGood := *deployment.Spec.Single.Count
+			expectedNoLeader := 0
+			if goodResults != expectedGood || noLeaderResults != expectedNoLeader {
+				return maskAny(fmt.Errorf("Wrong number of results: good %d (expected: %d)- noleader %d (expected %d)", goodResults, expectedGood, noLeaderResults, expectedNoLeader))
+			}
+		}
+	default:
+		return maskAny(fmt.Errorf("DeploymentMode %s is not supported!", mode))
+	}
+	return nil
+}

tests/upgrade_test.go

@@ -0,0 +1,139 @@
+//
+// DISCLAIMER
+//
+// Copyright 2018 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Jan Christoph Uhde <jan@uhdejc.com>
+//
+package tests
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/dchest/uniuri"
+
+	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha"
+	kubeArangoClient "github.com/arangodb/kube-arangodb/pkg/client"
+	"github.com/arangodb/kube-arangodb/pkg/util"
+)
+
+// test upgrade single server mmfiles 3.2 -> 3.3
+func TestUpgradeSingleMMFiles32to33(t *testing.T) {
+	upgradeSubTest(t, api.DeploymentModeSingle, api.StorageEngineMMFiles, "3.2.12", "3.3.4")
+}
+
+// // test upgrade single server rocksdb 3.3 -> 3.4
+// func TestUpgradeSingleRocksDB33to34(t *testing.T) {
+// 	upgradeSubTest(t, api.DeploymentModeSingle, api.StorageEngineRocksDB, "3.3.4", "3.4.0")
+// }
+
+// test upgrade resilient single server rocksdb 3.2 -> 3.3
+func TestUpgradeResilientSingleRocksDB32to33(t *testing.T) {
+	upgradeSubTest(t, api.DeploymentModeResilientSingle, api.StorageEngineRocksDB, "3.2.12", "3.3.4")
+}
+
+// // test upgrade resilient single server mmfiles 3.3 -> 3.4
+// func TestUpgradeResilientSingleMMFiles33to34(t *testing.T) {
+// 	upgradeSubTest(t, api.DeploymentModeResilientSingle, api.StorageEngineMMFiles, "3.3.0", "3.4.0")
+// }
+
+// test upgrade cluster rocksdb 3.2 -> 3.3
+func TestUpgradeClusterRocksDB32to33(t *testing.T) {
+	upgradeSubTest(t, api.DeploymentModeCluster, api.StorageEngineRocksDB, "3.2.12", "3.3.4")
+}
+
+// // test upgrade cluster mmfiles 3.3 -> 3.4
+// func TestUpgradeClusterMMFiles33to34(t *testing.T) {
+// 	upgradeSubTest(t, api.DeploymentModeCluster, api.StorageEngineRocksDB, "3.3.4", "3.4.0")
+// }
+
+// test downgrade single server mmfiles 3.3.3 -> 3.3.2
+func TestDowngradeSingleMMFiles333to332(t *testing.T) {
+	upgradeSubTest(t, api.DeploymentModeSingle, api.StorageEngineMMFiles, "3.3.3", "3.3.2")
+}
+
+// test downgrade resilient single server rocksdb 3.3.3 -> 3.3.2
+func TestDowngradeResilientSingleRocksDB333to332(t *testing.T) {
+	upgradeSubTest(t, api.DeploymentModeResilientSingle, api.StorageEngineRocksDB, "3.3.3", "3.3.2")
+}
+
+// test downgrade cluster rocksdb 3.3.3 -> 3.3.2
+func TestDowngradeClusterRocksDB332to332(t *testing.T) {
+	upgradeSubTest(t, api.DeploymentModeCluster, api.StorageEngineRocksDB, "3.3.3", "3.3.2")
+}
+
+func upgradeSubTest(t *testing.T, mode api.DeploymentMode, engine api.StorageEngine, fromVersion, toVersion string) error {
+	// check environment
+	longOrSkip(t)
+
+	k8sNameSpace := getNamespace(t)
+	k8sClient := mustNewKubeClient(t)
+	deploymentClient := kubeArangoClient.MustNewInCluster()
+
+	deploymentTemplate := newDeployment(strings.Replace(fmt.Sprintf("tu-%s-%s-%st%s-%s", mode[:2], engine[:2], fromVersion, toVersion, uniuri.NewLen(4)), ".", "", -1))
+	deploymentTemplate.Spec.Mode = api.NewMode(mode)
+	deploymentTemplate.Spec.StorageEngine = api.NewStorageEngine(engine)
+	deploymentTemplate.Spec.TLS = api.TLSSpec{} // should auto-generate cert
+	deploymentTemplate.Spec.Image = util.NewString("arangodb/arangodb:" + fromVersion)
+	deploymentTemplate.Spec.SetDefaults(deploymentTemplate.GetName()) // this must be last
+
+	// Create deployment
+	deployment, err := deploymentClient.DatabaseV1alpha().ArangoDeployments(k8sNameSpace).Create(deploymentTemplate)
+	if err != nil {
+		t.Fatalf("Create deployment failed: %v", err)
+	}
+
+	// Wait for deployment to be ready
+	deployment, err = waitUntilDeployment(deploymentClient, deploymentTemplate.GetName(), k8sNameSpace, deploymentIsReady())
+	if err != nil {
+		t.Fatalf("Deployment not running in time: %v", err)
+	}
+
+	// Create a database client
+	ctx := context.Background()
+	DBClient := mustNewArangodDatabaseClient(ctx, k8sClient, deployment, t)
+
+	if err := waitUntilArangoDeploymentHealthy(deployment, DBClient, k8sClient, ""); err != nil {
+		t.Fatalf("Deployment not healthy in time: %v", err)
+	}
+
+	// Try to change image version
+	deployment, err = updateDeployment(deploymentClient, deploymentTemplate.GetName(), k8sNameSpace,
+		func(spec *api.DeploymentSpec) {
+			spec.Image = util.NewString("arangodb/arangodb:" + toVersion)
+		})
+	if err != nil {
+		t.Fatalf("Failed to upgrade the Image from version : " + fromVersion + " to version: " + toVersion)
+	}
+
+	deployment, err = waitUntilDeployment(deploymentClient, deploymentTemplate.GetName(), k8sNameSpace, deploymentIsReady())
+	if err != nil {
+		t.Fatalf("Deployment not running in time: %v", err)
+	}
+
+	if err := waitUntilArangoDeploymentHealthy(deployment, DBClient, k8sClient, toVersion); err != nil {
+		t.Fatalf("Deployment not healthy in time: %v", err)
+	}
+
+	// Cleanup
+	removeDeployment(deploymentClient, deploymentTemplate.GetName(), k8sNameSpace)
+
+	return nil
+}

tools/manifests/manifest_builder.go

@@ -57,6 +57,9 @@ var (
 		"rbac.yaml",
 		"deployment.yaml",
 	}
+	testTemplateNames = []string{
+		"rbac.yaml",
+	}
 )
 
 func init() {
@@ -79,6 +82,7 @@ type TemplateOptions struct {
 	RBAC            bool
 	Deployment      ResourceOptions
 	Storage         ResourceOptions
+	Test            CommonOptions
 }
 
 type CommonOptions struct {
@@ -123,6 +127,7 @@ func main() {
 	templateNameSet := map[string][]string{
 		"deployment": deploymentTemplateNames,
 		"storage":    storageTemplateNames,
+		"test":       testTemplateNames,
 	}
 
 	// Process templates
@@ -160,6 +165,12 @@ func main() {
 			},
 			OperatorDeploymentName: "arango-storage-operator",
 		},
+		Test: CommonOptions{
+			Namespace:          options.Namespace,
+			RoleName:           "arango-operator-test",
+			RoleBindingName:    "arango-operator-test",
+			ServiceAccountName: "default",
+		},
 	}
 	for group, templateNames := range templateNameSet {
 		output := &bytes.Buffer{}