thanos: support arbitrary object storage configuration

Thanos v0.2.0 introduced breaking changes that moved object storage configuration from a large set of flag and environment variables for each possible provider to one of two flags --objstore.config or --objstore.config-file.  This change updates prometheus-operator to support newer versions of thanos and decouples  prometheus-operator from the details of thanos object storage configuration.  Object storage configuration is stored in a user-created secret, about which prometheus-operator needs only to know the name.  Thanos is free to introduce support for arbitrary backends without updates to prometheus-operator.

Fixes #2133 and fixes #2225

pkg/apis/monitoring/v1/types.go

@@ -309,14 +309,19 @@ type ThanosSpec struct {
 	// Resources defines the resource requirements for the Thanos sidecar.
 	// If not provided, no requests/limits will be set
 	Resources v1.ResourceRequirements `json:"resources,omitempty"`
-	// GCS configures use of GCS in Thanos.
+	// Deprecated: GCS should be configured with an ObjectStorageConfig secret
+	// starting with Thanos v0.2.0. This field will be removed.
 	GCS *ThanosGCSSpec `json:"gcs,omitempty"`
-	// S3 configures use of S3 in Thanos.
+	// Deprecated: S3 should be configured with an ObjectStorageConfig secret
+	// starting with Thanos v0.2.0. This field will be removed.
 	S3 *ThanosS3Spec `json:"s3,omitempty"`
+	// ObjectStorageConfig configures object storage in Thanos.
+	ObjectStorageConfig *v1.SecretKeySelector `json:"objectStorageConfig,omitempty"`
 }
 
-// ThanosGCSSpec defines parameters for use of Google Cloud Storage (GCS) with
-// Thanos.
+// Deprecated: ThanosGCSSpec should be configured with an ObjectStorageConfig
+// secret starting with Thanos v0.2.0. ThanosGCSSpec will be removed.
+//
 // +k8s:openapi-gen=true
 type ThanosGCSSpec struct {
 	// Google Cloud Storage bucket name for stored blocks. If empty it won't
@@ -326,8 +331,9 @@ type ThanosGCSSpec struct {
 	SecretKey *v1.SecretKeySelector `json:"credentials,omitempty"`
 }
 
-// ThanosS3Spec defines parameters for of AWS Simple Storage Service (S3) with
-// Thanos. (S3 compatible services apply as well)
+// Deprecated: ThanosS3Spec should be configured with an ObjectStorageConfig
+// secret starting with Thanos v0.2.0. ThanosS3Spec will be removed.
+//
 // +k8s:openapi-gen=true
 type ThanosS3Spec struct {
 	// S3-Compatible API bucket name for stored blocks.

pkg/prometheus/statefulset.go

@@ -35,7 +35,7 @@ import (
 const (
 	governingServiceName     = "prometheus-operated"
 	DefaultPrometheusVersion = "v2.5.0"
-	DefaultThanosVersion     = "v0.1.0"
+	DefaultThanosVersion     = "v0.2.1"
 	defaultRetention         = "24h"
 	storageDir               = "/prometheus"
 	confDir                  = "/etc/prometheus/config"
@@ -681,6 +681,17 @@ func makeStatefulSetSpec(p monitoringv1.Prometheus, c *Config, ruleConfigMapName
 				},
 			},
 		}
+
+		if p.Spec.Thanos.ObjectStorageConfig != nil {
+			envVars = append(envVars, v1.EnvVar{
+				Name: "OBJSTORE_CONFIG",
+				ValueFrom: &v1.EnvVarSource{
+					SecretKeyRef: p.Spec.Thanos.ObjectStorageConfig,
+				},
+			})
+			thanosArgs = append(thanosArgs, "--objstore.config", "$(OBJSTORE_CONFIG)")
+		}
+
 		if p.Spec.Thanos.GCS != nil {
 			if p.Spec.Thanos.GCS.Bucket != nil {
 				thanosArgs = append(thanosArgs, fmt.Sprintf("--gcs.bucket=%s", *p.Spec.Thanos.GCS.Bucket))
@@ -841,9 +852,9 @@ func makeStatefulSetSpec(p monitoringv1.Prometheus, c *Config, ruleConfigMapName
 				NodeSelector:                  p.Spec.NodeSelector,
 				PriorityClassName:             p.Spec.PriorityClassName,
 				TerminationGracePeriodSeconds: &terminationGracePeriod,
-				Volumes:                       volumes,
-				Tolerations:                   p.Spec.Tolerations,
-				Affinity:                      p.Spec.Affinity,
+				Volumes:     volumes,
+				Tolerations: p.Spec.Tolerations,
+				Affinity:    p.Spec.Affinity,
 			},
 		},
 	}, nil