Run PrometheusVersionUpgrade tests separately (#5732)

As our compatibility matrix grows, the tests assuring we can upgrade one Prometheus version to the next are taking up to 1~2h to complete. We aim to accelerate our e2e tests that are required to pass on Pull Requests by moving this test to a separate CI, that runs once a day.

Signed-off-by: Arthur Silva Sens <arthur.sens@coralogix.com>

.github/workflows/e2e.yaml

@@ -90,6 +90,7 @@ jobs:
         EXCLUDE_THANOSRULER_TESTS=${{ matrix.thanosruler }}
         EXCLUDE_OPERATOR_UPGRADE_TESTS=${{ matrix.operatorUpgrade }}
         FEATURE_GATED_TESTS=${{ matrix.featureGated }}
+        EXCLUDE_PROMETHEUS_UPGRADE_TESTS=exclude
         make test-e2e
 
   # Added to summarize the matrix and allow easy branch protection rules setup

.github/workflows/test-prom-version-upgrade.yaml

@@ -0,0 +1,50 @@
+name: e2e
+on:
+  schedule:
+    - cron:  '37 15 * * *' # Every day 15:37
+
+jobs:
+  e2e-tests:
+    name: E2E tests
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: Import environment variables from file
+      run: cat ".github/env" >> $GITHUB_ENV
+    - name: Install Go
+      uses: actions/setup-go@v4
+      with:
+        go-version: '${{ env.golang-version }}'
+    - name: Build images
+      run: |
+        export SHELL=/bin/bash
+        make build image
+    - name: Start KinD
+      uses: helm/kind-action@v1.7.0
+      with:
+        version: ${{ env.kind-version }}
+        node_image: ${{ env.kind-image }}
+        wait: 300s
+        config: ./test/e2e/kind-conf.yaml
+        cluster_name: e2e
+    - name: Wait for cluster to finish bootstraping
+      run: |
+        kubectl wait --for=condition=Ready pods --all --all-namespaces --timeout=300s
+        kubectl cluster-info
+        kubectl get pods -A
+    - name: Load images
+      run: |
+        kind load docker-image -n e2e quay.io/prometheus-operator/prometheus-operator:$(git rev-parse --short HEAD)
+        kind load docker-image -n e2e quay.io/prometheus-operator/prometheus-config-reloader:$(git rev-parse --short HEAD)
+        kind load docker-image -n e2e quay.io/prometheus-operator/admission-webhook:$(git rev-parse --short HEAD)
+        kubectl apply -f scripts/kind-rbac.yaml
+    - name: Run tests
+      run: >
+        EXCLUDE_ALL_NS_TESTS=exclude
+        EXCLUDE_ALERTMANAGER_TESTS=exclude
+        EXCLUDE_PROMETHEUS_TESTS=exclude
+        EXCLUDE_PROMETHEUS_ALL_NS_TESTS=exclude
+        EXCLUDE_THANOSRULER_TESTS=exclude
+        EXCLUDE_OPERATOR_UPGRADE_TESTS=exclude
+        FEATURE_GATED_TESTS=exclude 
+        make test-e2e

test/e2e/main_test.go

@@ -67,6 +67,18 @@ func skipOperatorUpgradeTests(t *testing.T) {
 	}
 }
 
+func skipPromVersionUpgradeTests(t *testing.T) {
+	if os.Getenv("EXCLUDE_PROMETHEUS_UPGRADE_TESTS") != "" {
+		t.Skip("Skipping Prometheus Version upgrade tests")
+	}
+}
+
+func skipAllNSTests(t *testing.T) {
+	if os.Getenv("EXCLUDE_ALL_NS_TESTS") != "" {
+		t.Skip("Skipping AllNS upgrade tests")
+	}
+}
+
 // feature gated tests need to be explicitly included
 // not currently in use
 //
@@ -160,6 +172,8 @@ func TestMain(m *testing.M) {
 // TestAllNS tests the Prometheus Operator watching all namespaces in a
 // Kubernetes cluster.
 func TestAllNS(t *testing.T) {
+	skipAllNSTests(t)
+
 	testCtx := framework.NewTestCtx(t)
 	defer testCtx.Cleanup(t)
 
@@ -253,7 +267,6 @@ func testAllNSPrometheus(t *testing.T) {
 		"PromCreateDeleteCluster":                   testPromCreateDeleteCluster,
 		"PromScaleUpDownCluster":                    testPromScaleUpDownCluster,
 		"PromNoServiceMonitorSelector":              testPromNoServiceMonitorSelector,
-		"PromVersionMigration":                      testPromVersionMigration,
 		"PromResourceUpdate":                        testPromResourceUpdate,
 		"PromStorageLabelsAnnotations":              testPromStorageLabelsAnnotations,
 		"PromStorageUpdate":                         testPromStorageUpdate,
@@ -393,6 +406,18 @@ const (
 	prometheusOperatorServiceName = "prometheus-operator"
 )
 
+// TestPrometheusVersionUpgrade tests that all Prometheus versions in the compatibility matrix can be upgraded
+func TestPrometheusVersionUpgrade(t *testing.T) {
+	skipPromVersionUpgradeTests(t)
+	testFuncs := map[string]func(t *testing.T){
+		"PromVersionMigration": testPromVersionMigration,
+	}
+
+	for name, f := range testFuncs {
+		t.Run(name, f)
+	}
+}
+
 func testServerTLS(ctx context.Context, namespace string) func(t *testing.T) {
 	return func(t *testing.T) {
 		skipPrometheusTests(t)