Avoid CI timeouts in TestConfigGeneration (#3432)

This test generates the same configuration many times, for each
Prometheus version, to see if it is deterministic. As the compatibility
matrix grows, test times increase. Now, this sometimes fails in CI
because Travis kills jobs after 10 minutes of no output.

Run each version as a subtest, and run tests with `-v`, so that output
is produced after each version. This avoids the no-output timeout.

Parallelize testing for each Prometheus version.

When the tests are run with `-short` (as in `make test-unit`), only try
one hundred iterations. With the race detector on, as in that target, this takes
around 5 seconds. Without the race detector, short tests on this
package now run quick enough for fast iteration in an IDE.

Add an additional target and Travis job for running the long tests, but
without the race detector. This brings the run time for the full 1000
iterations per version to under a minute.

Signed-off-by: Matthias Rampke <matthias@rampke.de>

.travis.yml

@@ -21,6 +21,8 @@ jobs:
     script: cd cmd/po-rule-migration && go install
   - name: "Run unit tests"
     script: make test-unit
+  - name: "Run extended tests"
+    script: make test-long
   - name: "Run e2e tests (Alertmanager)"
     script: EXCLUDE_PROMETHEUS_TESTS=true EXCLUDE_THANOS_TESTS=true ./scripts/travis-e2e.sh
   - name: "Run e2e tests (Prometheus)"

Makefile

@@ -249,11 +249,15 @@ shellcheck: $(SHELLCHECK_BINARY)
 ###########
 
 .PHONY: test
-test: test-unit test-e2e
+test: test-unit test-long test-e2e
 
 .PHONY: test-unit
 test-unit:
-	go test -race $(TEST_RUN_ARGS) -short $(pkgs) -count=1
+	go test -race $(TEST_RUN_ARGS) -short $(pkgs) -count=1 -v
+
+.PHONY: test-long
+test-long:
+	go test $(TEST_RUN_ARGS) $(pkgs) -count=1 -v
 
 test/instrumented-sample-app/certs/cert.pem test/instrumented-sample-app/certs/key.pem:
 	cd test/instrumented-sample-app && make generate-certs

pkg/prometheus/promcfg_test.go

@@ -33,21 +33,29 @@ import (
 
 func TestConfigGeneration(t *testing.T) {
 	for _, v := range operator.PrometheusCompatibilityMatrix {
-		cfg, err := generateTestConfig(v)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		for i := 0; i < 1000; i++ {
-			testcfg, err := generateTestConfig(v)
+		t.Run(v, func(t *testing.T) {
+			t.Parallel()
+			cfg, err := generateTestConfig(v)
 			if err != nil {
 				t.Fatal(err)
 			}
 
-			if !bytes.Equal(cfg, testcfg) {
-				t.Fatalf("Config generation is not deterministic.\n\n\nFirst generation: \n\n%s\n\nDifferent generation: \n\n%s\n\n", string(cfg), string(testcfg))
+			reps := 1000
+			if testing.Short() {
+				reps = 100
 			}
-		}
+
+			for i := 0; i < reps; i++ {
+				testcfg, err := generateTestConfig(v)
+				if err != nil {
+					t.Fatal(err)
+				}
+
+				if !bytes.Equal(cfg, testcfg) {
+					t.Fatalf("Config generation is not deterministic.\n\n\nFirst generation: \n\n%s\n\nDifferent generation: \n\n%s\n\n", string(cfg), string(testcfg))
+				}
+			}
+		})
 	}
 }