Merge pull request #1122 from mxinden/support-am-0.15

Support Alertmanager v0.15.0

Documentation/api.md

@@ -89,7 +89,7 @@ Specification of the desired behavior of the Alertmanager cluster. More info: ht
 | ----- | ----------- | ------ | -------- |
 | podMetadata | Standard object’s metadata. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/api-conventions.md#metadata Metadata Labels and Annotations gets propagated to the prometheus pods. | *[metav1.ObjectMeta](https://v1-6.docs.kubernetes.io/docs/api-reference/v1.6/#objectmeta-v1-meta) | false |
 | version | Version the cluster should be on. | string | false |
-| baseImage | Base image that is used to deploy pods. | string | false |
+| baseImage | Base image that is used to deploy pods, without tag. | string | false |
 | imagePullSecrets | An optional list of references to secrets in the same namespace to use for pulling prometheus and alertmanager images from registries see http://kubernetes.io/docs/user-guide/images#specifying-imagepullsecrets-on-a-pod | [][v1.LocalObjectReference](https://v1-6.docs.kubernetes.io/docs/api-reference/v1.6/#localobjectreference-v1-core) | false |
 | replicas | Size is the expected size of the alertmanager cluster. The controller will eventually make the size of the running cluster equal to the expected size. | *int32 | false |
 | storage | Storage is the definition of how storage will be used by the Alertmanager instances. | *[StorageSpec](#storagespec) | false |

example/prometheus-operator-crd/alertmanager.crd.yaml

@@ -521,7 +521,7 @@ spec:
                         - topologyKey
                       type: array
             baseImage:
-              description: Base image that is used to deploy pods.
+              description: Base image that is used to deploy pods, without tag.
               type: string
             containers:
               description: Containers allows injecting additional containers. This

pkg/alertmanager/statefulset.go

@@ -168,29 +168,30 @@ func makeStatefulSetSpec(a *monitoringv1.Alertmanager, config Config) (*appsv1.S
 
 	version, err := semver.Parse(versionStr)
 	if err != nil {
-		return nil, errors.Wrap(err, "parse version")
+		return nil, errors.Wrap(err, "failed to parse alertmanager version")
 	}
 
 	amArgs := []string{
-		fmt.Sprintf("-config.file=%s", alertmanagerConfFile),
-		fmt.Sprintf("-mesh.listen-address=:%d", 6783),
-		fmt.Sprintf("-storage.path=%s", alertmanagerStorageDir),
+		fmt.Sprintf("--config.file=%s", alertmanagerConfFile),
+		fmt.Sprintf("--cluster.listen-address=:%d", 6783),
+		fmt.Sprintf("--storage.path=%s", alertmanagerStorageDir),
 	}
 
 	if a.Spec.ListenLocal {
-		amArgs = append(amArgs, "-web.listen-address=127.0.0.1:9093")
+		amArgs = append(amArgs, "--web.listen-address=127.0.0.1:9093")
 	} else {
-		amArgs = append(amArgs, "-web.listen-address=:9093")
+		amArgs = append(amArgs, "--web.listen-address=:9093")
 	}
 
 	if a.Spec.ExternalURL != "" {
-		amArgs = append(amArgs, "-web.external-url="+a.Spec.ExternalURL)
+		amArgs = append(amArgs, "--web.external-url="+a.Spec.ExternalURL)
 	}
 
 	webRoutePrefix := "/"
 	if a.Spec.RoutePrefix != "" {
 		webRoutePrefix = a.Spec.RoutePrefix
 	}
+	amArgs = append(amArgs, fmt.Sprintf("--web.route-prefix=%v", webRoutePrefix))
 
 	localReloadURL := &url.URL{
 		Scheme: "http",
@@ -241,7 +242,7 @@ func makeStatefulSetSpec(a *monitoringv1.Alertmanager, config Config) (*appsv1.S
 	podLabels["alertmanager"] = a.Name
 
 	for i := int32(0); i < *a.Spec.Replicas; i++ {
-		amArgs = append(amArgs, fmt.Sprintf("-mesh.peer=%s-%d.%s.%s.svc", prefixedName(a.Name), i, governingServiceName, a.Namespace))
+		amArgs = append(amArgs, fmt.Sprintf("--cluster.peer=%s-%d.%s.%s.svc:6783", prefixedName(a.Name), i, governingServiceName, a.Namespace))
 	}
 
 	ports := []v1.ContainerPort{
@@ -275,17 +276,32 @@ func makeStatefulSetSpec(a *monitoringv1.Alertmanager, config Config) (*appsv1.S
 		securityContext = a.Spec.SecurityContext
 	}
 
+	// Adjust Alertmanager command line args to specified AM version
 	switch version.Major {
 	case 0:
-		if version.Minor >= 7 {
-			amArgs = append(amArgs, "-web.route-prefix="+webRoutePrefix)
-		}
-		if version.Minor >= 13 {
+		if version.Minor < 15 {
 			for i := range amArgs {
-				// starting with v0.13.0 of Alertmanager all flags are with double dashes.
-				amArgs[i] = "-" + amArgs[i]
+				// below Alertmanager v0.15.0 peer address port specification is not necessary
+				if strings.Contains(amArgs[i], "--cluster.peer") {
+					amArgs[i] = strings.TrimSuffix(amArgs[i], ":6783")
+				}
+
+				// below Alertmanager v0.15.0 high availability flags are prefixed with 'mesh' instead of 'cluster'
+				amArgs[i] = strings.Replace(amArgs[i], "--cluster.", "--mesh.", 1)
 			}
 		}
+		if version.Minor < 13 {
+			for i := range amArgs {
+				// below Alertmanager v0.13.0 all flags are with single dash.
+				amArgs[i] = strings.Replace(amArgs[i], "--", "-", 1)
+			}
+		}
+		if version.Minor < 7 {
+			// below Alertmanager v0.7.0 the flag 'web.route-prefix' does not exist
+			amArgs = filter(amArgs, func(s string) bool {
+				return !strings.Contains(s, "web.route-prefix")
+			})
+		}
 	default:
 		return nil, errors.Errorf("unsupported Alertmanager major version %s", version)
 	}
@@ -389,3 +405,13 @@ func subPathForStorage(s *monitoringv1.StorageSpec) string {
 
 	return "alertmanager-db"
 }
+
+func filter(strings []string, f func(string) bool) []string {
+	filteredStrings := make([]string, 0)
+	for _, s := range strings {
+		if f(s) {
+			filteredStrings = append(filteredStrings, s)
+		}
+	}
+	return filteredStrings
+}

pkg/alertmanager/statefulset_test.go

@@ -15,6 +15,7 @@
 package alertmanager
 
 import (
+	"fmt"
 	"reflect"
 	"strings"
 	"testing"
@@ -210,3 +211,152 @@ func TestListenLocal(t *testing.T) {
 		t.Fatal("Alertmanager container should only have one port defined")
 	}
 }
+
+// below Alertmanager v0.13.0 all flags are with single dash.
+func TestMakeStatefulSetSpecSingleDoubleDashedArgs(t *testing.T) {
+	tests := []struct {
+		version string
+		prefix  string
+		amount  int
+	}{
+		{"v0.12.0", "-", 1},
+		{"v0.13.0", "--", 2},
+	}
+
+	for _, test := range tests {
+		a := monitoringv1.Alertmanager{}
+		a.Spec.Version = test.version
+		replicas := int32(3)
+		a.Spec.Replicas = &replicas
+
+		statefulSet, err := makeStatefulSetSpec(&a, Config{})
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		amArgs := statefulSet.Template.Spec.Containers[0].Args
+
+		for _, arg := range amArgs {
+			if arg[:test.amount] != test.prefix {
+				t.Fatalf("expected all args to start with %v but got %v", test.prefix, arg)
+			}
+		}
+	}
+}
+
+// below Alertmanager v0.7.0 the flag 'web.route-prefix' does not exist
+func TestMakeStatefulSetSpecWebRoutePrefix(t *testing.T) {
+	tests := []struct {
+		version              string
+		expectWebRoutePrefix bool
+	}{
+		{"v0.6.0", false},
+		{"v0.7.0", true},
+	}
+
+	for _, test := range tests {
+		a := monitoringv1.Alertmanager{}
+		a.Spec.Version = test.version
+		replicas := int32(1)
+		a.Spec.Replicas = &replicas
+
+		statefulSet, err := makeStatefulSetSpec(&a, Config{})
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		amArgs := statefulSet.Template.Spec.Containers[0].Args
+
+		containsWebRoutePrefix := false
+
+		for _, arg := range amArgs {
+			if strings.Contains(arg, "-web.route-prefix") {
+				containsWebRoutePrefix = true
+			}
+		}
+
+		if containsWebRoutePrefix != test.expectWebRoutePrefix {
+			t.Fatalf("expected stateful set containing arg '-web.route-prefix' to be: %v", test.expectWebRoutePrefix)
+		}
+	}
+}
+
+// below Alertmanager v0.15.0 high availability flags are prefixed with 'mesh' instead of 'cluster'
+func TestMakeStatefulSetSpecMeshClusterFlags(t *testing.T) {
+	tests := []struct {
+		version       string
+		rightHAPrefix string
+		wrongHAPrefix string
+	}{
+		{"v0.14.0", "mesh", "cluster"},
+		{"v0.15.0", "cluster", "mesh"},
+	}
+
+	for _, test := range tests {
+		a := monitoringv1.Alertmanager{}
+		a.Spec.Version = test.version
+		replicas := int32(3)
+		a.Spec.Replicas = &replicas
+
+		statefulSet, err := makeStatefulSetSpec(&a, Config{})
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		haFlags := []string{"--%v.listen-address", "--%v.peer="}
+
+		amArgs := statefulSet.Template.Spec.Containers[0].Args
+
+		for _, flag := range haFlags {
+			if sliceContains(amArgs, fmt.Sprintf(flag, test.wrongHAPrefix)) {
+				t.Fatalf("expected Alertmanager args not to contain %v, but got %v", test.wrongHAPrefix, amArgs)
+			}
+			if !sliceContains(amArgs, fmt.Sprintf(flag, test.rightHAPrefix)) {
+				t.Fatalf("expected Alertmanager args to contain %v, but got %v", test.rightHAPrefix, amArgs)
+			}
+		}
+	}
+}
+
+// below Alertmanager v0.15.0 peer address port specification is not necessary
+func TestMakeStatefulSetSpecPeerFlagPort(t *testing.T) {
+	tests := []struct {
+		version    string
+		portNeeded bool
+	}{
+		{"v0.14.0", false},
+		{"v0.15.0", true},
+	}
+
+	for _, test := range tests {
+		a := monitoringv1.Alertmanager{}
+		a.Spec.Version = test.version
+		replicas := int32(3)
+		a.Spec.Replicas = &replicas
+
+		statefulSet, err := makeStatefulSetSpec(&a, Config{})
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		amArgs := statefulSet.Template.Spec.Containers[0].Args
+
+		for _, arg := range amArgs {
+			if strings.Contains(arg, ".peer") {
+				if strings.Contains(arg, ":6783") != test.portNeeded {
+					t.Fatalf("expected arg '%v' containing port specification to be: %v", arg, test.portNeeded)
+				}
+			}
+		}
+	}
+}
+
+func sliceContains(slice []string, match string) bool {
+	contains := false
+	for _, s := range slice {
+		if strings.Contains(s, match) {
+			contains = true
+		}
+	}
+	return contains
+}

pkg/client/monitoring/v1/openapi_generated.go

@@ -213,7 +213,7 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA
 						},
 						"baseImage": {
 							SchemaProps: spec.SchemaProps{
-								Description: "Base image that is used to deploy pods.",
+								Description: "Base image that is used to deploy pods, without tag.",
 								Type:        []string{"string"},
 								Format:      "",
 							},

pkg/client/monitoring/v1/types.go

@@ -396,7 +396,7 @@ type AlertmanagerSpec struct {
 	PodMetadata *metav1.ObjectMeta `json:"podMetadata,omitempty"`
 	// Version the cluster should be on.
 	Version string `json:"version,omitempty"`
-	// Base image that is used to deploy pods.
+	// Base image that is used to deploy pods, without tag.
 	BaseImage string `json:"baseImage,omitempty"`
 	// An optional list of references to secrets in the same namespace
 	// to use for pulling prometheus and alertmanager images from registries

test/e2e/alertmanager_test.go

@@ -130,28 +130,42 @@ func TestExposingAlertmanagerWithKubernetesAPI(t *testing.T) {
 func TestMeshInitialization(t *testing.T) {
 	t.Parallel()
 
-	ctx := framework.NewTestCtx(t)
-	defer ctx.Cleanup(t)
-	ns := ctx.CreateNamespace(t, framework.KubeClient)
-	ctx.SetupPrometheusRBAC(t, ns, framework.KubeClient)
+	// Starting with Alertmanager v0.15.0 hashicorp/memberlist is used for HA.
+	// Make sure both memberlist as well as mesh (< 0.15.0) work
+	amVersions := []string{"v0.14.0", "v0.15.0-rc.0"}
 
-	amClusterSize := 3
-	alertmanager := framework.MakeBasicAlertmanager("test", int32(amClusterSize))
-	alertmanagerService := framework.MakeAlertmanagerService(alertmanager.Name, "alertmanager-service", v1.ServiceTypeClusterIP)
+	for _, v := range amVersions {
+		version := v
+		t.Run(
+			fmt.Sprintf("amVersion%v", strings.Replace(version, ".", "-", -1)),
+			func(t *testing.T) {
+				t.Parallel()
+				ctx := framework.NewTestCtx(t)
+				defer ctx.Cleanup(t)
+				ns := ctx.CreateNamespace(t, framework.KubeClient)
+				ctx.SetupPrometheusRBAC(t, ns, framework.KubeClient)
 
-	if err := framework.CreateAlertmanagerAndWaitUntilReady(ns, alertmanager); err != nil {
-		t.Fatal(err)
-	}
+				amClusterSize := 3
+				alertmanager := framework.MakeBasicAlertmanager("test", int32(amClusterSize))
+				alertmanager.Spec.Version = version
+				alertmanagerService := framework.MakeAlertmanagerService(alertmanager.Name, "alertmanager-service", v1.ServiceTypeClusterIP)
 
-	if _, err := testFramework.CreateServiceAndWaitUntilReady(framework.KubeClient, ns, alertmanagerService); err != nil {
-		t.Fatal(err)
-	}
+				if err := framework.CreateAlertmanagerAndWaitUntilReady(ns, alertmanager); err != nil {
+					t.Fatal(err)
+				}
 
-	for i := 0; i < amClusterSize; i++ {
-		name := "alertmanager-" + alertmanager.Name + "-" + strconv.Itoa(i)
-		if err := framework.WaitForAlertmanagerInitializedMesh(ns, name, amClusterSize); err != nil {
-			t.Fatal(err)
-		}
+				if _, err := testFramework.CreateServiceAndWaitUntilReady(framework.KubeClient, ns, alertmanagerService); err != nil {
+					t.Fatal(err)
+				}
+
+				for i := 0; i < amClusterSize; i++ {
+					name := "alertmanager-" + alertmanager.Name + "-" + strconv.Itoa(i)
+					if err := framework.WaitForAlertmanagerInitializedMesh(ns, name, amClusterSize); err != nil {
+						t.Fatal(err)
+					}
+				}
+			},
+		)
 	}
 }
 

test/framework/alertmanager.go

@@ -199,12 +199,12 @@ func amImage(version string) string {
 }
 
 func (f *Framework) WaitForAlertmanagerInitializedMesh(ns, name string, amountPeers int) error {
-	return wait.Poll(time.Second, time.Second*20, func() (bool, error) {
+	return wait.Poll(time.Second, time.Minute*5, func() (bool, error) {
 		amStatus, err := f.GetAlertmanagerConfig(ns, name)
 		if err != nil {
 			return false, err
 		}
-		if len(amStatus.Data.MeshStatus.Peers) == amountPeers {
+		if amStatus.Data.getAmountPeers() == amountPeers {
 			return true, nil
 		}
 
@@ -249,10 +249,20 @@ type alertmanagerStatus struct {
 }
 
 type alertmanagerStatusData struct {
-	MeshStatus meshStatus `json:"meshStatus"`
-	ConfigYAML string     `json:"configYAML"`
+	ClusterStatus *clusterStatus `json:"clusterStatus,omitempty"`
+	MeshStatus    *clusterStatus `json:"meshStatus,omitempty"`
+	ConfigYAML    string         `json:"configYAML"`
 }
 
-type meshStatus struct {
+// Starting from AM v0.15.0 'MeshStatus' is called 'ClusterStatus'
+func (s *alertmanagerStatusData) getAmountPeers() int {
+	if s.MeshStatus != nil {
+		return len(s.MeshStatus.Peers)
+	} else {
+		return len(s.ClusterStatus.Peers)
+	}
+}
+
+type clusterStatus struct {
 	Peers []interface{} `json:"peers"`
 }

test/framework/context.go

@@ -31,7 +31,16 @@ type TestCtx struct {
 type finalizerFn func() error
 
 func (f *Framework) NewTestCtx(t *testing.T) TestCtx {
-	prefix := strings.TrimPrefix(strings.ToLower(t.Name()), "test")
+	// TestCtx is used among others for namespace names where '/' is forbidden
+	prefix := strings.TrimPrefix(
+		strings.Replace(
+			strings.ToLower(t.Name()),
+			"/",
+			"-",
+			-1,
+		),
+		"test",
+	)
 
 	id := prefix + "-" + strconv.FormatInt(time.Now().Unix(), 10)
 	return TestCtx{