feat: add experimental Prometheus 3 support (#6940)

* feat: add experimental Prometheus 3 support

Fixes #6939

Signed-off-by: Jayapriya Pai <slashpai9@gmail.com>

.github/workflows/e2e-prometheus3.yaml

@@ -0,0 +1,105 @@
+name: e2e-prometheus3
+on:
+  schedule:
+    - cron:  '30 14 * * *' # Every day 14:30
+jobs:
+  e2e-tests:
+    name: E2E experimental version tests
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        suite: [alertmanager, prometheus, prometheusAllNS, thanosruler, operatorUpgrade]
+        include:
+          - suite: alertmanager
+            prometheus: "exclude"
+            prometheusAllNS: "exclude"
+            alertmanager: ""
+            thanosruler: "exclude"
+            operatorUpgrade: "exclude"
+          - suite: prometheus
+            prometheus: ""
+            prometheusAllNS: "exclude"
+            alertmanager: "exclude"
+            thanosruler: "exclude"
+            operatorUpgrade: "exclude"
+          - suite: prometheusAllNS
+            prometheus: "exclude"
+            prometheusAllNS: ""
+            alertmanager: "exclude"
+            thanosruler: "exclude"
+            operatorUpgrade: "exclude"
+          - suite: thanosruler
+            prometheus: "exclude"
+            prometheusAllNS: "exclude"
+            alertmanager: "exclude"
+            thanosruler: ""
+            operatorUpgrade: "exclude"
+          - suite: operatorUpgrade
+            prometheus: "exclude"
+            prometheusAllNS: "exclude"
+            alertmanager: "exclude"
+            thanosruler: "exclude"
+            operatorUpgrade: ""
+    steps:
+    - uses: actions/checkout@v4
+    - name: Import environment variables from file
+      run: cat ".github/env" >> $GITHUB_ENV
+    - name: Install Go
+      uses: actions/setup-go@v5
+      with:
+        go-version: '${{ env.golang-version }}'
+        check-latest: true
+    - name: Build images
+      run: |
+        export SHELL=/bin/bash
+        make build image
+    - name: Start kind cluster
+      uses: helm/kind-action@v1.10.0
+      with:
+        version: ${{ env.kind-version }}
+        node_image: ${{ env.kind-image }}
+        wait: 300s
+        config: ./test/e2e/kind-conf.yaml
+        cluster_name: e2e
+    - name: Wait for cluster to finish bootstraping
+      run: |
+        echo "Waiting for all nodes to be ready..."
+        kubectl wait --for=condition=Ready nodes --all --timeout=120s
+        kubectl get nodes
+        echo "Waiting for all pods to be ready..."
+        kubectl wait --for=condition=Ready pods --all --all-namespaces --timeout=300s
+        kubectl get pods -A
+        echo "Cluster information"
+        kubectl cluster-info
+    - name: Load images
+      run: |
+        kind load docker-image -n e2e quay.io/prometheus-operator/prometheus-operator:$(git rev-parse --short HEAD)
+        kind load docker-image -n e2e quay.io/prometheus-operator/prometheus-config-reloader:$(git rev-parse --short HEAD)
+        kind load docker-image -n e2e quay.io/prometheus-operator/admission-webhook:$(git rev-parse --short HEAD)
+        kubectl apply -f scripts/kind-rbac.yaml
+    - name: Run tests
+      run: >
+        TEST_EXPERIMENTAL_PROMETHEUS=true
+        EXCLUDE_ALERTMANAGER_TESTS=${{ matrix.alertmanager }}
+        EXCLUDE_PROMETHEUS_TESTS=${{ matrix.prometheus }}
+        EXCLUDE_PROMETHEUS_ALL_NS_TESTS=${{ matrix.prometheusAllNS }}
+        EXCLUDE_THANOSRULER_TESTS=${{ matrix.thanosruler }}
+        EXCLUDE_OPERATOR_UPGRADE_TESTS=${{ matrix.operatorUpgrade }}
+        EXCLUDE_FEATURE_GATED_TESTS=exclude
+        EXCLUDE_PROMETHEUS_UPGRADE_TESTS=exclude
+        make test-e2e
+
+  # Added to summarize the matrix and allow easy branch protection rules setup
+  e2e-tests-result:
+    name: End-to-End Test Results
+    if: always()
+    needs:
+      - e2e-tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Mark the job as a success
+        if: needs.e2e-tests.result == 'success'
+        run: exit 0
+      - name: Mark the job as a failure
+        if: needs.e2e-tests.result != 'success'
+        run: exit 1

.github/workflows/test-prom-version-upgrade.yaml

@@ -41,6 +41,7 @@ jobs:
         kubectl apply -f scripts/kind-rbac.yaml
     - name: Run tests
       run: >
+        TEST_EXPERIMENTAL_PROMETHEUS=false
         EXCLUDE_ALL_NS_TESTS=exclude
         EXCLUDE_ALERTMANAGER_TESTS=exclude
         EXCLUDE_PROMETHEUS_TESTS=exclude

Documentation/compatibility.md

@@ -75,6 +75,7 @@ Prometheus Operator supports all Prometheus versions >= v2.0.0. The operator's e
 * v2.53.1
 * v2.54.0
 * v2.54.1
+* v3.0.0-beta.0
 ```
 
 The end-to-end tests are mostly tested against

cmd/po-docgen/compatibility.go

@@ -27,7 +27,7 @@ type compatibilityMatrix struct {
 
 func getCompatibilityMatrix() compatibilityMatrix {
 	return compatibilityMatrix{
-		PrometheusVersions:  operator.PrometheusCompatibilityMatrix,
+		PrometheusVersions:  append(operator.PrometheusCompatibilityMatrix, operator.PrometheusExperimentalVersions...),
 		DefaultPrometheus:   operator.DefaultPrometheusVersion,
 		DefaultAlertmanager: operator.DefaultAlertmanagerVersion,
 		DefaultThanos:       operator.DefaultThanosVersion,

pkg/operator/defaults.go

@@ -36,6 +36,8 @@ const (
 var (
 	// DefaultPrometheusVersion is a default image tag for the prometheus.
 	DefaultPrometheusVersion = PrometheusCompatibilityMatrix[len(PrometheusCompatibilityMatrix)-1]
+	// DefaultPrometheusExperimentalVersion is a default image tag for the prometheus experimental version (like Prometheus 3 beta).
+	DefaultPrometheusExperimentalVersion = PrometheusExperimentalVersions[len(PrometheusExperimentalVersions)-1]
 	// DefaultPrometheusBaseImage is a base container registry address for the prometheus.
 	DefaultPrometheusBaseImage = "quay.io/prometheus/prometheus"
 	// DefaultPrometheusImage is a default image pulling address for the prometheus.
@@ -95,4 +97,10 @@ var (
 		"v2.54.0",
 		"v2.54.1",
 	}
+
+	// Note: Issues in this version won't be supported by operator till its stable
+	// This is only added for users to try the unstable versions.
+	PrometheusExperimentalVersions = []string{
+		"v3.0.0-beta.0",
+	}
 )

pkg/prometheus/agent/daemonset.go

@@ -17,6 +17,7 @@ package prometheusagent
 import (
 	"fmt"
 
+	"github.com/blang/semver/v4"
 	"golang.org/x/exp/slices"
 	appsv1 "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
@@ -86,16 +87,22 @@ func makeDaemonSetSpec(
 ) (*appsv1.DaemonSetSpec, error) {
 	cpf := p.GetCommonPrometheusFields()
 
+	promVersion := operator.StringValOrDefault(cpf.Version, operator.DefaultPrometheusVersion)
+	version, err := semver.ParseTolerant(promVersion)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse Prometheus version: %w", err)
+	}
+
 	pImagePath, err := operator.BuildImagePathForAgent(
 		ptr.Deref(cpf.Image, ""),
 		c.PrometheusDefaultBaseImage,
-		operator.StringValOrDefault(cpf.Version, operator.DefaultPrometheusVersion),
+		promVersion,
 	)
 	if err != nil {
 		return nil, err
 	}
 
-	if !slices.Contains(cpf.EnableFeatures, "agent") {
+	if version.Major == 2 && !slices.Contains(cpf.EnableFeatures, "agent") {
 		cpf.EnableFeatures = append(cpf.EnableFeatures, "agent")
 	}
 

pkg/prometheus/agent/daemonset_test.go

@@ -23,6 +23,7 @@ import (
 	appsv1 "k8s.io/api/apps/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
+	monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
 	monitoringv1alpha1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1alpha1"
 	"github.com/prometheus-operator/prometheus-operator/pkg/operator"
 	prompkg "github.com/prometheus-operator/prometheus-operator/pkg/prometheus"
@@ -49,6 +50,39 @@ func TestListenTLSForDaemonSet(t *testing.T) {
 	testCorrectArgs(t, dset.Spec.Template.Spec.Containers[1].Args, dset.Spec.Template.Spec.Containers)
 }
 
+func TestPrometheusAgentCommandLineFlagForDaemonSet(t *testing.T) {
+	tests := []struct {
+		version       string
+		expectedArg   string
+		shouldContain bool
+	}{
+		{"v3.0.0", "--agent", true},
+		{"v3.0.0-beta.0", "--agent", true},
+		{"v2.53.0", "--agent", false},
+	}
+
+	for _, test := range tests {
+		sset, err := makeStatefulSetFromPrometheus(monitoringv1alpha1.PrometheusAgent{
+			Spec: monitoringv1alpha1.PrometheusAgentSpec{
+				CommonPrometheusFields: monitoringv1.CommonPrometheusFields{
+					Version: test.version,
+				},
+			},
+		})
+		require.NoError(t, err)
+
+		promArgs := sset.Spec.Template.Spec.Containers[0].Args
+		found := false
+		for _, flag := range promArgs {
+			if flag == test.expectedArg {
+				found = true
+				break
+			}
+		}
+		require.Equal(t, test.shouldContain, found)
+	}
+}
+
 func TestStartupProbeTimeoutSecondsForDaemonSet(t *testing.T) {
 	testcases := createTestCasesForTestStartupProbeTimeoutSeconds()
 

pkg/prometheus/agent/statefulset.go

@@ -17,6 +17,7 @@ package prometheusagent
 import (
 	"fmt"
 
+	"github.com/blang/semver/v4"
 	"golang.org/x/exp/slices"
 	appsv1 "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
@@ -148,16 +149,22 @@ func makeStatefulSetSpec(
 ) (*appsv1.StatefulSetSpec, error) {
 	cpf := p.GetCommonPrometheusFields()
 
+	promVersion := operator.StringValOrDefault(cpf.Version, operator.DefaultPrometheusVersion)
+	version, err := semver.ParseTolerant(promVersion)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse Prometheus version: %w", err)
+	}
+
 	pImagePath, err := operator.BuildImagePathForAgent(
 		ptr.Deref(cpf.Image, ""),
 		c.PrometheusDefaultBaseImage,
-		operator.StringValOrDefault(cpf.Version, operator.DefaultPrometheusVersion),
+		promVersion,
 	)
 	if err != nil {
 		return nil, err
 	}
 
-	if !slices.Contains(cpf.EnableFeatures, "agent") {
+	if version.Major == 2 && !slices.Contains(cpf.EnableFeatures, "agent") {
 		cpf.EnableFeatures = append(cpf.EnableFeatures, "agent")
 	}
 
@@ -362,6 +369,9 @@ func appendAgentArgs(
 	promArgs []monitoringv1.Argument,
 	cg *prompkg.ConfigGenerator,
 	walCompression *bool) []monitoringv1.Argument {
+	if cg.GetPrometheusVersion().Major == 3 {
+		promArgs = append(promArgs, monitoringv1.Argument{Name: "agent"})
+	}
 
 	promArgs = append(promArgs,
 		monitoringv1.Argument{Name: "storage.agent.path", Value: prompkg.StorageDir},

pkg/prometheus/agent/statefulset_test.go

@@ -88,6 +88,39 @@ func TestWALCompression(t *testing.T) {
 	}
 }
 
+func TestPrometheusAgentCommandLineFlag(t *testing.T) {
+	tests := []struct {
+		version       string
+		expectedArg   string
+		shouldContain bool
+	}{
+		{"v3.0.0", "--agent", true},
+		{"v3.0.0-beta.0", "--agent", true},
+		{"v2.53.0", "--agent", false},
+	}
+
+	for _, test := range tests {
+		sset, err := makeStatefulSetFromPrometheus(monitoringv1alpha1.PrometheusAgent{
+			Spec: monitoringv1alpha1.PrometheusAgentSpec{
+				CommonPrometheusFields: monitoringv1.CommonPrometheusFields{
+					Version: test.version,
+				},
+			},
+		})
+		require.NoError(t, err)
+
+		promArgs := sset.Spec.Template.Spec.Containers[0].Args
+		found := false
+		for _, flag := range promArgs {
+			if flag == test.expectedArg {
+				found = true
+				break
+			}
+		}
+		require.Equal(t, test.shouldContain, found)
+	}
+}
+
 func TestStartupProbeTimeoutSeconds(t *testing.T) {
 	testcases := createTestCasesForTestStartupProbeTimeoutSeconds()
 

pkg/prometheus/promcfg.go

@@ -109,7 +109,7 @@ func NewConfigGenerator(
 		return nil, fmt.Errorf("failed to parse Prometheus version: %w", err)
 	}
 
-	if version.Major != 2 {
+	if version.Major != 2 && version.Major != 3 {
 		return nil, fmt.Errorf("unsupported Prometheus major version %s: %w", version, err)
 	}
 
@@ -179,6 +179,11 @@ func getScrapeClassConfig(p monitoringv1.PrometheusInterface) (map[string]monito
 	return scrapeClasses, defaultScrapeClass, nil
 }
 
+// GetPrometheusVersion returns the currently configured Prometheus version.
+func (cg *ConfigGenerator) GetPrometheusVersion() semver.Version {
+	return cg.version
+}
+
 // WithKeyVals returns a new ConfigGenerator with the same characteristics as
 // the current object, expect that the keyvals are appended to the existing
 // logger.

test/framework/prometheus.go

@@ -178,6 +178,10 @@ func (f *Framework) CreateCertificateResources(namespace, certsDir string, prwtc
 }
 
 func (f *Framework) MakeBasicPrometheus(ns, name, group string, replicas int32) *monitoringv1.Prometheus {
+	promVersion := operator.DefaultPrometheusVersion
+	if os.Getenv("TEST_EXPERIMENTAL_PROMETHEUS") == "true" {
+		promVersion = operator.DefaultPrometheusExperimentalVersion
+	}
 	return &monitoringv1.Prometheus{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:        name,
@@ -187,7 +191,7 @@ func (f *Framework) MakeBasicPrometheus(ns, name, group string, replicas int32)
 		Spec: monitoringv1.PrometheusSpec{
 			CommonPrometheusFields: monitoringv1.CommonPrometheusFields{
 				Replicas: &replicas,
-				Version:  operator.DefaultPrometheusVersion,
+				Version:  promVersion,
 				ServiceMonitorSelector: &metav1.LabelSelector{
 					MatchLabels: map[string]string{
 						"group": group,