From 8d6565a6e259a57a1efd1a2bdb8bfd08a5040d24 Mon Sep 17 00:00:00 2001 From: Ewout Prangsma Date: Fri, 6 Apr 2018 10:40:16 +0200 Subject: [PATCH] Added environment variable used to cleanup failed tests --- Jenkinsfile.groovy | 1 + Makefile | 2 +- scripts/kube_run_tests.sh | 1 + tests/auth_test.go | 10 ++++++++-- tests/deployments_test.go | 3 +++ tests/environments_test.go | 1 + tests/immutable_test.go | 1 + tests/member_resilience_test.go | 3 +++ tests/resilience_test.go | 3 +++ tests/rocksdb_encryption_test.go | 1 + tests/scale_test.go | 2 ++ tests/test_util.go | 18 ++++++++++++++++++ tests/upgrade_test.go | 1 + 13 files changed, 44 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile.groovy b/Jenkinsfile.groovy index 599673891..4407bbde3 100644 --- a/Jenkinsfile.groovy +++ b/Jenkinsfile.groovy @@ -78,6 +78,7 @@ def buildTestSteps(Map myParams, String kubeConfigRoot, String kubeconfig) { timestamps { withCredentials([string(credentialsId: 'ENTERPRISEIMAGE', variable: 'DEFAULTENTERPRISEIMAGE')]) { withEnv([ + "CLEANDEPLOYMENTS=1", "DEPLOYMENTNAMESPACE=${myParams.TESTNAMESPACE}-${env.GIT_COMMIT}", "DOCKERNAMESPACE=${myParams.DOCKERNAMESPACE}", "ENTERPRISEIMAGE=${myParams.ENTERPRISEIMAGE}", diff --git a/Makefile b/Makefile index 4706a0574..090c4a3c9 100644 --- a/Makefile +++ b/Makefile @@ -80,7 +80,7 @@ TESTLENGTHOPTIONS := -test.short TESTTIMEOUT := 20m ifeq ($(LONG), 1) TESTLENGTHOPTIONS := - TESTTIMEOUT := 60m + TESTTIMEOUT := 120m endif ifdef VERBOSE TESTVERBOSEOPTIONS := -v diff --git a/scripts/kube_run_tests.sh b/scripts/kube_run_tests.sh index e8f657033..8e8ca4c9c 100755 --- a/scripts/kube_run_tests.sh +++ b/scripts/kube_run_tests.sh @@ -15,5 +15,6 @@ kubectl --namespace ${DEPLOYMENTNAMESPACE} \ --image=${IMAGEID} \ --env="ENTERPRISEIMAGE=${ENTERPRISEIMAGE}" \ --env="TEST_NAMESPACE=${DEPLOYMENTNAMESPACE}" \ + --env="CLEANDEPLOYMENTS=${CLEANDEPLOYMENTS}" \ -- \ -test.v -test.timeout $TESTTIMEOUT $TESTLENGTHOPTIONS $TESTOPTIONS diff --git a/tests/auth_test.go b/tests/auth_test.go index 1bc0ad9d6..aee75bc00 100644 --- a/tests/auth_test.go +++ b/tests/auth_test.go @@ -33,6 +33,7 @@ func TestAuthenticationSingleDefaultSecret(t *testing.T) { if err != nil { t.Fatalf("Create deployment failed: %v", err) } + defer deferedCleanupDeployment(c, depl.GetName(), ns) // Wait for deployment to be ready if _, err := waitUntilDeployment(c, depl.GetName(), ns, deploymentIsReady()); err != nil { @@ -86,6 +87,7 @@ func TestAuthenticationSingleCustomSecret(t *testing.T) { if err != nil { t.Fatalf("Create deployment failed: %v", err) } + defer deferedCleanupDeployment(c, depl.GetName(), ns) // Wait for deployment to be ready if _, err := waitUntilDeployment(c, depl.GetName(), ns, deploymentIsReady()); err != nil { @@ -106,7 +108,7 @@ func TestAuthenticationSingleCustomSecret(t *testing.T) { // Secret must still exist if _, err := waitUntilSecret(kubecli, depl.Spec.Authentication.GetJWTSecretName(), ns, nil, time.Second); err != nil { - t.Fatalf("JWT secret '%s' not found: %v", depl.Spec.Authentication.JWTSecretName, err) + t.Fatalf("JWT secret '%s' not found: %v", depl.Spec.Authentication.GetJWTSecretName(), err) } // Cleanup secret @@ -132,6 +134,7 @@ func TestAuthenticationNoneSingle(t *testing.T) { if err != nil { t.Fatalf("Create deployment failed: %v", err) } + defer deferedCleanupDeployment(c, depl.GetName(), ns) // Wait for deployment to be ready if _, err := waitUntilDeployment(c, depl.GetName(), ns, deploymentIsReady()); err != nil { @@ -169,6 +172,7 @@ func TestAuthenticationClusterDefaultSecret(t *testing.T) { if err != nil { t.Fatalf("Create deployment failed: %v", err) } + defer deferedCleanupDeployment(c, depl.GetName(), ns) // Wait for deployment to be ready if _, err := waitUntilDeployment(c, depl.GetName(), ns, deploymentIsReady()); err != nil { @@ -222,6 +226,7 @@ func TestAuthenticationClusterCustomSecret(t *testing.T) { if err != nil { t.Fatalf("Create deployment failed: %v", err) } + defer deferedCleanupDeployment(c, depl.GetName(), ns) // Wait for deployment to be ready if _, err := waitUntilDeployment(c, depl.GetName(), ns, deploymentIsReady()); err != nil { @@ -242,7 +247,7 @@ func TestAuthenticationClusterCustomSecret(t *testing.T) { // Secret must still exist if _, err := waitUntilSecret(kubecli, depl.Spec.Authentication.GetJWTSecretName(), ns, nil, time.Second); err != nil { - t.Fatalf("JWT secret '%s' not found: %v", depl.Spec.Authentication.JWTSecretName, err) + t.Fatalf("JWT secret '%s' not found: %v", depl.Spec.Authentication.GetJWTSecretName(), err) } // Cleanup secret @@ -268,6 +273,7 @@ func TestAuthenticationNoneCluster(t *testing.T) { if err != nil { t.Fatalf("Create deployment failed: %v", err) } + defer deferedCleanupDeployment(c, depl.GetName(), ns) // Wait for deployment to be ready if _, err := waitUntilDeployment(c, depl.GetName(), ns, deploymentIsReady()); err != nil { diff --git a/tests/deployments_test.go b/tests/deployments_test.go index 11871f137..075a63a46 100644 --- a/tests/deployments_test.go +++ b/tests/deployments_test.go @@ -84,6 +84,7 @@ func deploymentSubTest(t *testing.T, mode api.DeploymentMode, engine api.Storage // Create deployment _, err := deploymentClient.DatabaseV1alpha().ArangoDeployments(k8sNameSpace).Create(deploymentTemplate) require.NoError(t, err, fmt.Sprintf("Create deployment failed: %v", err)) + defer deferedCleanupDeployment(deploymentClient, deploymentTemplate.GetName(), k8sNameSpace) // Wait for deployment to be ready deployment, err := waitUntilDeployment(deploymentClient, deploymentTemplate.GetName(), k8sNameSpace, deploymentIsReady()) @@ -124,9 +125,11 @@ func TestMultiDeployment(t *testing.T) { // Create deployments _, err := deploymentClient.DatabaseV1alpha().ArangoDeployments(k8sNameSpace).Create(deploymentTemplate1) require.NoError(t, err, fmt.Sprintf("Deployment creation failed: %v", err)) + defer deferedCleanupDeployment(deploymentClient, deploymentTemplate1.GetName(), k8sNameSpace) _, err = deploymentClient.DatabaseV1alpha().ArangoDeployments(k8sNameSpace).Create(deploymentTemplate2) require.NoError(t, err, fmt.Sprintf("Deployment creation failed: %v", err)) + defer deferedCleanupDeployment(deploymentClient, deploymentTemplate2.GetName(), k8sNameSpace) // Wait for deployments to be ready deployment1, err := waitUntilDeployment(deploymentClient, deploymentTemplate1.GetName(), k8sNameSpace, deploymentIsReady()) diff --git a/tests/environments_test.go b/tests/environments_test.go index d8fd88c54..bd9adec4d 100644 --- a/tests/environments_test.go +++ b/tests/environments_test.go @@ -75,6 +75,7 @@ func TestProduction(t *testing.T) { // REVIEW - should the test already fail here t.Fatalf("Create deployment failed: %v", err) } + defer deferedCleanupDeployment(deploymentClient, deploymentTemplate.GetName(), k8sNameSpace) _, err = waitUntilDeployment(deploymentClient, deploymentTemplate.GetName(), k8sNameSpace, deploymentIsReady()) assert.Error(t, err, fmt.Sprintf("Deployment is up and running when it should not! There are not enough nodes(%d) for all DBServers(%d) in production modes.", numNodes, dbserverCount)) diff --git a/tests/immutable_test.go b/tests/immutable_test.go index 436fd2174..7ca148765 100644 --- a/tests/immutable_test.go +++ b/tests/immutable_test.go @@ -54,6 +54,7 @@ func TestImmutableFields(t *testing.T) { if err != nil { t.Fatalf("Create deployment failed: %v", err) } + defer deferedCleanupDeployment(c, depl.GetName(), ns) // Wait for deployment to be ready if _, err := waitUntilDeployment(c, depl.GetName(), ns, deploymentIsReady()); err != nil { diff --git a/tests/member_resilience_test.go b/tests/member_resilience_test.go index 34070913d..86aec609d 100644 --- a/tests/member_resilience_test.go +++ b/tests/member_resilience_test.go @@ -34,6 +34,7 @@ func TestMemberResilienceAgents(t *testing.T) { if err != nil { t.Fatalf("Create deployment failed: %v", err) } + defer deferedCleanupDeployment(c, depl.GetName(), ns) // Wait for deployment to be ready if _, err = waitUntilDeployment(c, depl.GetName(), ns, deploymentIsReady()); err != nil { @@ -134,6 +135,7 @@ func TestMemberResilienceCoordinators(t *testing.T) { if err != nil { t.Fatalf("Create deployment failed: %v", err) } + defer deferedCleanupDeployment(c, depl.GetName(), ns) // Wait for deployment to be ready if _, err = waitUntilDeployment(c, depl.GetName(), ns, deploymentIsReady()); err != nil { @@ -230,6 +232,7 @@ func TestMemberResilienceDBServers(t *testing.T) { if err != nil { t.Fatalf("Create deployment failed: %v", err) } + defer deferedCleanupDeployment(c, depl.GetName(), ns) // Wait for deployment to be ready if _, err = waitUntilDeployment(c, depl.GetName(), ns, deploymentIsReady()); err != nil { diff --git a/tests/resilience_test.go b/tests/resilience_test.go index 0faa162f1..b43ac71f1 100644 --- a/tests/resilience_test.go +++ b/tests/resilience_test.go @@ -35,6 +35,7 @@ func TestResiliencePod(t *testing.T) { if err != nil { t.Fatalf("Create deployment failed: %v", err) } + defer deferedCleanupDeployment(c, depl.GetName(), ns) // Wait for deployment to be ready if _, err = waitUntilDeployment(c, depl.GetName(), ns, deploymentIsReady()); err != nil { @@ -115,6 +116,7 @@ func TestResiliencePVC(t *testing.T) { if err != nil { t.Fatalf("Create deployment failed: %v", err) } + defer deferedCleanupDeployment(c, depl.GetName(), ns) // Wait for deployment to be ready if _, err = waitUntilDeployment(c, depl.GetName(), ns, deploymentIsReady()); err != nil { @@ -199,6 +201,7 @@ func TestResilienceService(t *testing.T) { if err != nil { t.Fatalf("Create deployment failed: %v", err) } + defer deferedCleanupDeployment(c, depl.GetName(), ns) // Wait for deployment to be ready if _, err = waitUntilDeployment(c, depl.GetName(), ns, deploymentIsReady()); err != nil { diff --git a/tests/rocksdb_encryption_test.go b/tests/rocksdb_encryption_test.go index fd990f40e..3f9d6e3d6 100644 --- a/tests/rocksdb_encryption_test.go +++ b/tests/rocksdb_encryption_test.go @@ -47,6 +47,7 @@ func TestRocksDBEncryptionSingle(t *testing.T) { if err != nil { t.Fatalf("Create deployment failed: %v", err) } + defer deferedCleanupDeployment(c, depl.GetName(), ns) // Wait for deployment to be ready apiObject, err := waitUntilDeployment(c, depl.GetName(), ns, deploymentIsReady()) diff --git a/tests/scale_test.go b/tests/scale_test.go index 9c77b0a47..b17f4c2c5 100644 --- a/tests/scale_test.go +++ b/tests/scale_test.go @@ -31,6 +31,7 @@ func TestScaleClusterNonTLS(t *testing.T) { if err != nil { t.Fatalf("Create deployment failed: %v", err) } + defer deferedCleanupDeployment(c, depl.GetName(), ns) // Wait for deployment to be ready if _, err := waitUntilDeployment(c, depl.GetName(), ns, deploymentIsReady()); err != nil { @@ -101,6 +102,7 @@ func TestScaleCluster(t *testing.T) { if err != nil { t.Fatalf("Create deployment failed: %v", err) } + defer deferedCleanupDeployment(c, depl.GetName(), ns) // Wait for deployment to be ready if _, err := waitUntilDeployment(c, depl.GetName(), ns, deploymentIsReady()); err != nil { diff --git a/tests/test_util.go b/tests/test_util.go index 2afc3366a..f13c262c8 100644 --- a/tests/test_util.go +++ b/tests/test_util.go @@ -71,6 +71,12 @@ func getEnterpriseImageOrSkip(t *testing.T) string { return image } +// shouldCleanDeployments returns true when deployments created +// by tests should be removed, even when the test fails. +func shouldCleanDeployments() bool { + return os.Getenv("CLEANDEPLOYMENTS") != "" +} + // mustNewKubeClient creates a kubernetes client // failing the test on errors. func mustNewKubeClient(t *testing.T) kubernetes.Interface { @@ -307,6 +313,18 @@ func removeDeployment(cli versioned.Interface, deploymentName, ns string) error return nil } +// deferedCleanupDeployment removes a deployment when shouldCleanDeployments return true. +// This function is intended to be used in a defer statement. +func deferedCleanupDeployment(cli versioned.Interface, deploymentName, ns string) error { + if !shouldCleanDeployments() { + return nil + } + if err := removeDeployment(cli, deploymentName, ns); err != nil { + return maskAny(err) + } + return nil +} + // removeSecret removes a secret func removeSecret(cli kubernetes.Interface, secretName, ns string) error { if err := cli.CoreV1().Secrets(ns).Delete(secretName, nil); err != nil && k8sutil.IsNotFound(err) { diff --git a/tests/upgrade_test.go b/tests/upgrade_test.go index 7e88f98c1..9e0c0ec99 100644 --- a/tests/upgrade_test.go +++ b/tests/upgrade_test.go @@ -99,6 +99,7 @@ func upgradeSubTest(t *testing.T, mode api.DeploymentMode, engine api.StorageEng if err != nil { t.Fatalf("Create deployment failed: %v", err) } + defer deferedCleanupDeployment(deploymentClient, deploymentTemplate.GetName(), k8sNameSpace) // Wait for deployment to be ready deployment, err = waitUntilDeployment(deploymentClient, deploymentTemplate.GetName(), k8sNameSpace, deploymentIsReady())