diff --git a/pkg/prometheus/server/operator.go b/pkg/prometheus/server/operator.go index 4417e9238..53fd2e6a0 100644 --- a/pkg/prometheus/server/operator.go +++ b/pkg/prometheus/server/operator.go @@ -783,6 +783,10 @@ func (c *Operator) sync(ctx context.Context, key string) error { return fmt.Errorf("synchronizing web config secret failed: %w", err) } + if err := c.createOrUpdateThanosConfigSecret(ctx, p); err != nil { + return fmt.Errorf("failed to reconcile Thanos config secret: %w", err) + } + // Create governing service if it doesn't exist. svcClient := c.kclient.CoreV1().Services(p.Namespace) if _, err := k8sutil.CreateOrUpdateService(ctx, svcClient, makeStatefulSetService(p, c.config)); err != nil { @@ -1199,6 +1203,22 @@ func (c *Operator) createOrUpdateWebConfigSecret(ctx context.Context, p *monitor return nil } +func (c *Operator) createOrUpdateThanosConfigSecret(ctx context.Context, p *monitoringv1.Prometheus) error { + secret, err := buildPrometheusHTTPClientConfigSecret(p) + if err != nil { + return fmt.Errorf("failed to build Thanos HTTP client config secret: :%w", err) + } + + operator.UpdateObject( + secret, + operator.WithLabels(c.config.Labels), + operator.WithAnnotations(c.config.Annotations), + operator.WithManagingOwner(p), + ) + + return k8sutil.CreateOrUpdateSecret(ctx, c.kclient.CoreV1().Secrets(secret.Namespace), secret) +} + func makeSelectorLabels(name string) map[string]string { return map[string]string{ "app.kubernetes.io/managed-by": "prometheus-operator", diff --git a/pkg/prometheus/server/operator_test.go b/pkg/prometheus/server/operator_test.go index c5a3fb7b7..1657f982a 100644 --- a/pkg/prometheus/server/operator_test.go +++ b/pkg/prometheus/server/operator_test.go @@ -15,6 +15,7 @@ package prometheus import ( + "context" "testing" "github.com/stretchr/testify/require" @@ -22,6 +23,7 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" "k8s.io/utils/ptr" monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" @@ -227,3 +229,38 @@ func TestCreateStatefulSetInputHash(t *testing.T) { }) } } + +func TestCreateThanosConfigSecret(t *testing.T) { + version := "v0.24.0" + ctx := context.Background() + for _, tc := range []struct { + name string + spec monitoringv1.PrometheusSpec + }{ + { + name: "prometheus with thanos sidecar", + spec: monitoringv1.PrometheusSpec{ + Thanos: &monitoringv1.ThanosSpec{ + Version: &version, + }, + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + p := &monitoringv1.Prometheus{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-create-thanos-config-secret", + Namespace: "test", + }, + Spec: tc.spec, + } + o := Operator{kclient: fake.NewClientset()} + err := o.createOrUpdateThanosConfigSecret(ctx, p) + require.NoError(t, err) + + get, err := o.kclient.CoreV1().Secrets("test").Get(ctx, thanosPrometheusHTTPClientConfigSecretName(p), metav1.GetOptions{}) + require.NoError(t, err) + require.Equal(t, "tls_config:\n insecure_skip_verify: true\n", string(get.Data[thanosPrometheusHTTPClientConfigFileName])) + }) + } +} diff --git a/pkg/prometheus/server/statefulset.go b/pkg/prometheus/server/statefulset.go index a30d6d216..588b7b98e 100644 --- a/pkg/prometheus/server/statefulset.go +++ b/pkg/prometheus/server/statefulset.go @@ -17,6 +17,7 @@ package prometheus import ( "fmt" "path" + "path/filepath" "github.com/blang/semver/v4" appsv1 "k8s.io/api/apps/v1" @@ -264,13 +265,14 @@ func makeStatefulSetSpec( var additionalContainers, operatorInitContainers []v1.Container - thanosContainer, err := createThanosContainer(p, c) + thanosContainer, thanosVolumes, err := createThanosContainer(p, c) if err != nil { return nil, err } if thanosContainer != nil { additionalContainers = append(additionalContainers, *thanosContainer) + volumes = append(volumes, thanosVolumes...) } if compactionDisabled(p) { @@ -538,9 +540,9 @@ func appendServerVolumes(p *monitoringv1.Prometheus, volumes []v1.Volume, volume return volumes, volumeMounts } -func createThanosContainer(p *monitoringv1.Prometheus, c prompkg.Config) (*v1.Container, error) { +func createThanosContainer(p *monitoringv1.Prometheus, c prompkg.Config) (*v1.Container, []v1.Volume, error) { if p.Spec.Thanos == nil { - return nil, nil + return nil, nil, nil } var ( @@ -557,7 +559,7 @@ func createThanosContainer(p *monitoringv1.Prometheus, c prompkg.Config) (*v1.Co ptr.Deref(thanos.SHA, ""), ) if err != nil { - return nil, fmt.Errorf("failed to build image path: %w", err) + return nil, nil, fmt.Errorf("failed to build image path: %w", err) } var grpcBindAddress, httpBindAddress string @@ -682,7 +684,7 @@ func createThanosContainer(p *monitoringv1.Prometheus, c prompkg.Config) (*v1.Co thanosVersion, err := semver.ParseTolerant(ptr.Deref(thanos.Version, operator.DefaultThanosVersion)) if err != nil { - return nil, fmt.Errorf("failed to parse Thanos version: %w", err) + return nil, nil, fmt.Errorf("failed to parse Thanos version: %w", err) } if thanos.GetConfigTimeout != "" && thanosVersion.GTE(semver.MustParse("0.29.0")) { @@ -691,17 +693,36 @@ func createThanosContainer(p *monitoringv1.Prometheus, c prompkg.Config) (*v1.Co if thanos.GetConfigInterval != "" && thanosVersion.GTE(semver.MustParse("0.29.0")) { thanosArgs = append(thanosArgs, monitoringv1.Argument{Name: "prometheus.get_config_interval", Value: string(thanos.GetConfigInterval)}) } + + // set prometheus.http-client-config + // ref: https://thanos.io/tip/components/sidecar.md/#prometheus-http-client + var volumes []v1.Volume if thanosVersion.GTE(semver.MustParse(thanosSupportedVersionHTTPClientFlag)) { - thanosArgs = append(thanosArgs, monitoringv1.Argument{Name: "prometheus.http-client", Value: `{"tls_config": {"insecure_skip_verify":true}}`}) + thanosArgs = append(thanosArgs, monitoringv1.Argument{ + Name: "prometheus.http-client-file", + Value: filepath.Join(thanosConfigDir, thanosPrometheusHTTPClientConfigFileName), + }) + container.VolumeMounts = append(container.VolumeMounts, v1.VolumeMount{ + Name: thanosPrometheusHTTPClientConfigSecretNameSuffix, + MountPath: thanosConfigDir, + }) + volumes = append(volumes, v1.Volume{ + Name: thanosPrometheusHTTPClientConfigSecretNameSuffix, + VolumeSource: v1.VolumeSource{ + Secret: &v1.SecretVolumeSource{ + SecretName: thanosPrometheusHTTPClientConfigSecretName(p), + }, + }, + }) } containerArgs, err := operator.BuildArgs(thanosArgs, thanos.AdditionalArgs) if err != nil { - return nil, err + return nil, nil, err } container.Args = append([]string{"sidecar"}, containerArgs...) - return container, nil + return container, volumes, nil } func queryLogFileVolumeMount(queryLogFile string) (v1.VolumeMount, bool) { diff --git a/pkg/prometheus/server/statefulset_test.go b/pkg/prometheus/server/statefulset_test.go index 82a0264bb..63158131a 100644 --- a/pkg/prometheus/server/statefulset_test.go +++ b/pkg/prometheus/server/statefulset_test.go @@ -2472,7 +2472,7 @@ func TestThanosAdditionalArgsNoError(t *testing.T) { "--grpc-address=:10901", "--http-address=:10902", "--log.level=info", - `--prometheus.http-client={"tls_config": {"insecure_skip_verify":true}}`, + "--prometheus.http-client-file=/etc/thanos/config/prometheus.http-client-file.yaml", "--reloader.watch-interval=5m", } @@ -2955,6 +2955,41 @@ func TestIfThanosVersionDontHaveHttpClientFlag(t *testing.T) { } } +func TestThanosWithPrometheusHTTPClientConfigFile(t *testing.T) { + version := "0.24.0" + + for _, tc := range []struct { + name string + spec monitoringv1.PrometheusSpec + }{ + { + name: "thanos sidecar with prometheus.http-client-file", + spec: monitoringv1.PrometheusSpec{ + Thanos: &monitoringv1.ThanosSpec{ + Version: &version, + }, + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + p := monitoringv1.Prometheus{Spec: tc.spec} + sset, err := makeStatefulSetFromPrometheus(p) + require.NoError(t, err) + for _, v := range sset.Spec.Template.Spec.Volumes { + if v.Name == thanosPrometheusHTTPClientConfigSecretNameSuffix { + require.Equal(t, v.VolumeSource.Secret.SecretName, thanosPrometheusHTTPClientConfigSecretName(&p)) + } + } + for _, c := range sset.Spec.Template.Spec.Containers { + if c.Name == "thanos-sidecar" { + require.NotEmpty(t, c.VolumeMounts) + require.Equal(t, thanosPrometheusHTTPClientConfigSecretNameSuffix, c.VolumeMounts[0].Name) + } + } + }) + } +} + func TestAutomountServiceAccountToken(t *testing.T) { for _, tc := range []struct { name string diff --git a/pkg/prometheus/server/thanos_sidecar_config.go b/pkg/prometheus/server/thanos_sidecar_config.go new file mode 100644 index 000000000..59ec351b5 --- /dev/null +++ b/pkg/prometheus/server/thanos_sidecar_config.go @@ -0,0 +1,67 @@ +// Copyright 2020 The prometheus-operator Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package prometheus + +import ( + "fmt" + + "gopkg.in/yaml.v2" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + prompkg "github.com/prometheus-operator/prometheus-operator/pkg/prometheus" +) + +const ( + thanosConfigDir = "/etc/thanos/config" + thanosPrometheusHTTPClientConfigFileName = "prometheus.http-client-file.yaml" + thanosPrometheusHTTPClientConfigSecretNameSuffix = "thanos-prometheus-http-client-file" +) + +// buildPrometheusHTTPClientConfigSecret returns a kubernetes secret with the HTTP configuration for the Thanos sidecar +// to communicated with prometheus server. +// https://thanos.io/tip/components/sidecar.md/#prometheus-http-client +func buildPrometheusHTTPClientConfigSecret(p *monitoringv1.Prometheus) (*v1.Secret, error) { + dataYaml := yaml.MapSlice{} + dataYaml = append(dataYaml, yaml.MapItem{ + Key: "tls_config", + Value: yaml.MapSlice{ + { + Key: "insecure_skip_verify", + Value: true, + }, + }, + }) + + data, err := yaml.Marshal(dataYaml) + if err != nil { + return nil, err + } + + return &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: thanosPrometheusHTTPClientConfigSecretName(p), + Namespace: p.Namespace, + }, + Data: map[string][]byte{ + thanosPrometheusHTTPClientConfigFileName: data, + }, + }, nil +} + +func thanosPrometheusHTTPClientConfigSecretName(p monitoringv1.PrometheusInterface) string { + return fmt.Sprintf("%s-%s", prompkg.PrefixedName(p), thanosPrometheusHTTPClientConfigSecretNameSuffix) +}