enhance: change thanos sidecar promethes.http-client params to prometheus.http-client-file and save config file in secret (#7029)

* enchance: change thanos sidecar promethes.http-client params to prometheus.http-client-file and save config file in secret

* Update pkg/prometheus/server/operator.go

---------

Co-authored-by: Simon Pasquier <spasquie@redhat.com>

pkg/prometheus/server/operator.go

@@ -783,6 +783,10 @@ func (c *Operator) sync(ctx context.Context, key string) error {
 		return fmt.Errorf("synchronizing web config secret failed: %w", err)
 	}
 
+	if err := c.createOrUpdateThanosConfigSecret(ctx, p); err != nil {
+		return fmt.Errorf("failed to reconcile Thanos config secret: %w", err)
+	}
+
 	// Create governing service if it doesn't exist.
 	svcClient := c.kclient.CoreV1().Services(p.Namespace)
 	if _, err := k8sutil.CreateOrUpdateService(ctx, svcClient, makeStatefulSetService(p, c.config)); err != nil {
@@ -1199,6 +1203,22 @@ func (c *Operator) createOrUpdateWebConfigSecret(ctx context.Context, p *monitor
 	return nil
 }
 
+func (c *Operator) createOrUpdateThanosConfigSecret(ctx context.Context, p *monitoringv1.Prometheus) error {
+	secret, err := buildPrometheusHTTPClientConfigSecret(p)
+	if err != nil {
+		return fmt.Errorf("failed to build Thanos HTTP client config secret: :%w", err)
+	}
+
+	operator.UpdateObject(
+		secret,
+		operator.WithLabels(c.config.Labels),
+		operator.WithAnnotations(c.config.Annotations),
+		operator.WithManagingOwner(p),
+	)
+
+	return k8sutil.CreateOrUpdateSecret(ctx, c.kclient.CoreV1().Secrets(secret.Namespace), secret)
+}
+
 func makeSelectorLabels(name string) map[string]string {
 	return map[string]string{
 		"app.kubernetes.io/managed-by":  "prometheus-operator",

pkg/prometheus/server/operator_test.go

@@ -15,6 +15,7 @@
 package prometheus
 
 import (
+	"context"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -22,6 +23,7 @@ import (
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes/fake"
 	"k8s.io/utils/ptr"
 
 	monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
@@ -227,3 +229,38 @@ func TestCreateStatefulSetInputHash(t *testing.T) {
 		})
 	}
 }
+
+func TestCreateThanosConfigSecret(t *testing.T) {
+	version := "v0.24.0"
+	ctx := context.Background()
+	for _, tc := range []struct {
+		name string
+		spec monitoringv1.PrometheusSpec
+	}{
+		{
+			name: "prometheus with thanos sidecar",
+			spec: monitoringv1.PrometheusSpec{
+				Thanos: &monitoringv1.ThanosSpec{
+					Version: &version,
+				},
+			},
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			p := &monitoringv1.Prometheus{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "test-create-thanos-config-secret",
+					Namespace: "test",
+				},
+				Spec: tc.spec,
+			}
+			o := Operator{kclient: fake.NewClientset()}
+			err := o.createOrUpdateThanosConfigSecret(ctx, p)
+			require.NoError(t, err)
+
+			get, err := o.kclient.CoreV1().Secrets("test").Get(ctx, thanosPrometheusHTTPClientConfigSecretName(p), metav1.GetOptions{})
+			require.NoError(t, err)
+			require.Equal(t, "tls_config:\n  insecure_skip_verify: true\n", string(get.Data[thanosPrometheusHTTPClientConfigFileName]))
+		})
+	}
+}

pkg/prometheus/server/statefulset.go

@@ -17,6 +17,7 @@ package prometheus
 import (
 	"fmt"
 	"path"
+	"path/filepath"
 
 	"github.com/blang/semver/v4"
 	appsv1 "k8s.io/api/apps/v1"
@@ -264,13 +265,14 @@ func makeStatefulSetSpec(
 
 	var additionalContainers, operatorInitContainers []v1.Container
 
-	thanosContainer, err := createThanosContainer(p, c)
+	thanosContainer, thanosVolumes, err := createThanosContainer(p, c)
 	if err != nil {
 		return nil, err
 	}
 
 	if thanosContainer != nil {
 		additionalContainers = append(additionalContainers, *thanosContainer)
+		volumes = append(volumes, thanosVolumes...)
 	}
 
 	if compactionDisabled(p) {
@@ -538,9 +540,9 @@ func appendServerVolumes(p *monitoringv1.Prometheus, volumes []v1.Volume, volume
 	return volumes, volumeMounts
 }
 
-func createThanosContainer(p *monitoringv1.Prometheus, c prompkg.Config) (*v1.Container, error) {
+func createThanosContainer(p *monitoringv1.Prometheus, c prompkg.Config) (*v1.Container, []v1.Volume, error) {
 	if p.Spec.Thanos == nil {
-		return nil, nil
+		return nil, nil, nil
 	}
 
 	var (
@@ -557,7 +559,7 @@ func createThanosContainer(p *monitoringv1.Prometheus, c prompkg.Config) (*v1.Co
 		ptr.Deref(thanos.SHA, ""),
 	)
 	if err != nil {
-		return nil, fmt.Errorf("failed to build image path: %w", err)
+		return nil, nil, fmt.Errorf("failed to build image path: %w", err)
 	}
 
 	var grpcBindAddress, httpBindAddress string
@@ -682,7 +684,7 @@ func createThanosContainer(p *monitoringv1.Prometheus, c prompkg.Config) (*v1.Co
 
 	thanosVersion, err := semver.ParseTolerant(ptr.Deref(thanos.Version, operator.DefaultThanosVersion))
 	if err != nil {
-		return nil, fmt.Errorf("failed to parse Thanos version: %w", err)
+		return nil, nil, fmt.Errorf("failed to parse Thanos version: %w", err)
 	}
 
 	if thanos.GetConfigTimeout != "" && thanosVersion.GTE(semver.MustParse("0.29.0")) {
@@ -691,17 +693,36 @@ func createThanosContainer(p *monitoringv1.Prometheus, c prompkg.Config) (*v1.Co
 	if thanos.GetConfigInterval != "" && thanosVersion.GTE(semver.MustParse("0.29.0")) {
 		thanosArgs = append(thanosArgs, monitoringv1.Argument{Name: "prometheus.get_config_interval", Value: string(thanos.GetConfigInterval)})
 	}
+
+	// set prometheus.http-client-config
+	// ref: https://thanos.io/tip/components/sidecar.md/#prometheus-http-client
+	var volumes []v1.Volume
 	if thanosVersion.GTE(semver.MustParse(thanosSupportedVersionHTTPClientFlag)) {
-		thanosArgs = append(thanosArgs, monitoringv1.Argument{Name: "prometheus.http-client", Value: `{"tls_config": {"insecure_skip_verify":true}}`})
+		thanosArgs = append(thanosArgs, monitoringv1.Argument{
+			Name:  "prometheus.http-client-file",
+			Value: filepath.Join(thanosConfigDir, thanosPrometheusHTTPClientConfigFileName),
+		})
+		container.VolumeMounts = append(container.VolumeMounts, v1.VolumeMount{
+			Name:      thanosPrometheusHTTPClientConfigSecretNameSuffix,
+			MountPath: thanosConfigDir,
+		})
+		volumes = append(volumes, v1.Volume{
+			Name: thanosPrometheusHTTPClientConfigSecretNameSuffix,
+			VolumeSource: v1.VolumeSource{
+				Secret: &v1.SecretVolumeSource{
+					SecretName: thanosPrometheusHTTPClientConfigSecretName(p),
+				},
+			},
+		})
 	}
 
 	containerArgs, err := operator.BuildArgs(thanosArgs, thanos.AdditionalArgs)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 	container.Args = append([]string{"sidecar"}, containerArgs...)
 
-	return container, nil
+	return container, volumes, nil
 }
 
 func queryLogFileVolumeMount(queryLogFile string) (v1.VolumeMount, bool) {

pkg/prometheus/server/statefulset_test.go

@@ -2472,7 +2472,7 @@ func TestThanosAdditionalArgsNoError(t *testing.T) {
 		"--grpc-address=:10901",
 		"--http-address=:10902",
 		"--log.level=info",
-		`--prometheus.http-client={"tls_config": {"insecure_skip_verify":true}}`,
+		"--prometheus.http-client-file=/etc/thanos/config/prometheus.http-client-file.yaml",
 		"--reloader.watch-interval=5m",
 	}
 
@@ -2955,6 +2955,41 @@ func TestIfThanosVersionDontHaveHttpClientFlag(t *testing.T) {
 	}
 }
 
+func TestThanosWithPrometheusHTTPClientConfigFile(t *testing.T) {
+	version := "0.24.0"
+
+	for _, tc := range []struct {
+		name string
+		spec monitoringv1.PrometheusSpec
+	}{
+		{
+			name: "thanos sidecar with prometheus.http-client-file",
+			spec: monitoringv1.PrometheusSpec{
+				Thanos: &monitoringv1.ThanosSpec{
+					Version: &version,
+				},
+			},
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			p := monitoringv1.Prometheus{Spec: tc.spec}
+			sset, err := makeStatefulSetFromPrometheus(p)
+			require.NoError(t, err)
+			for _, v := range sset.Spec.Template.Spec.Volumes {
+				if v.Name == thanosPrometheusHTTPClientConfigSecretNameSuffix {
+					require.Equal(t, v.VolumeSource.Secret.SecretName, thanosPrometheusHTTPClientConfigSecretName(&p))
+				}
+			}
+			for _, c := range sset.Spec.Template.Spec.Containers {
+				if c.Name == "thanos-sidecar" {
+					require.NotEmpty(t, c.VolumeMounts)
+					require.Equal(t, thanosPrometheusHTTPClientConfigSecretNameSuffix, c.VolumeMounts[0].Name)
+				}
+			}
+		})
+	}
+}
+
 func TestAutomountServiceAccountToken(t *testing.T) {
 	for _, tc := range []struct {
 		name                         string

pkg/prometheus/server/thanos_sidecar_config.go

@@ -0,0 +1,67 @@
+// Copyright 2020 The prometheus-operator Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package prometheus
+
+import (
+	"fmt"
+
+	"gopkg.in/yaml.v2"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
+	prompkg "github.com/prometheus-operator/prometheus-operator/pkg/prometheus"
+)
+
+const (
+	thanosConfigDir                                  = "/etc/thanos/config"
+	thanosPrometheusHTTPClientConfigFileName         = "prometheus.http-client-file.yaml"
+	thanosPrometheusHTTPClientConfigSecretNameSuffix = "thanos-prometheus-http-client-file"
+)
+
+// buildPrometheusHTTPClientConfigSecret returns a kubernetes secret with the HTTP configuration for the Thanos sidecar
+// to communicated with prometheus server.
+// https://thanos.io/tip/components/sidecar.md/#prometheus-http-client
+func buildPrometheusHTTPClientConfigSecret(p *monitoringv1.Prometheus) (*v1.Secret, error) {
+	dataYaml := yaml.MapSlice{}
+	dataYaml = append(dataYaml, yaml.MapItem{
+		Key: "tls_config",
+		Value: yaml.MapSlice{
+			{
+				Key:   "insecure_skip_verify",
+				Value: true,
+			},
+		},
+	})
+
+	data, err := yaml.Marshal(dataYaml)
+	if err != nil {
+		return nil, err
+	}
+
+	return &v1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      thanosPrometheusHTTPClientConfigSecretName(p),
+			Namespace: p.Namespace,
+		},
+		Data: map[string][]byte{
+			thanosPrometheusHTTPClientConfigFileName: data,
+		},
+	}, nil
+}
+
+func thanosPrometheusHTTPClientConfigSecretName(p monitoringv1.PrometheusInterface) string {
+	return fmt.Sprintf("%s-%s", prompkg.PrefixedName(p), thanosPrometheusHTTPClientConfigSecretNameSuffix)
+}