Add ListenLocal option for Thanos spec to listen on local loopback (#2728)

* Add ListenLocal option for Thanos spec to listen on local loopback

Resolves #2435

* Generated files

Documentation/api.md

@@ -604,5 +604,6 @@ ThanosSpec defines parameters for a Prometheus server within a Thanos deployment
 | baseImage | Thanos base image if other than default. | *string | false |
 | resources | Resources defines the resource requirements for the Thanos sidecar. If not provided, no requests/limits will be set | [v1.ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#resourcerequirements-v1-core) | false |
 | objectStorageConfig | ObjectStorageConfig configures object storage in Thanos. | *[v1.SecretKeySelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#secretkeyselector-v1-core) | false |
+| listenLocal | ListenLocal makes the Thanos sidecar listen on loopback, so that it does not bind against the Pod IP. | bool | false |
 
 [Back to TOC](#table-of-contents)

Documentation/thanos.md

@@ -71,6 +71,7 @@ spec:
       name: thanos-objstore-config
 ...
 ```
+Note: If you're using Istio you may need to also set `ListenLocal` on the Thanos spec due to Istio's forwarding of traffic to localhost.
 
 ## Thanos and kube-thanos
 

example/prometheus-operator-crd/prometheus.crd.yaml

@@ -4143,6 +4143,10 @@ spec:
                     to ensure the Prometheus Operator knows what version of Thanos
                     is being configured.
                   type: string
+                listenLocal:
+                  description: ListenLocal makes the Thanos sidecar listen on loopback,
+                    so that it does not bind against the Pod IP.
+                  type: boolean
                 objectStorageConfig:
                   description: SecretKeySelector selects a key of a Secret.
                   properties:

pkg/apis/monitoring/v1/openapi_generated.go

@@ -2862,6 +2862,13 @@ func schema_pkg_apis_monitoring_v1_ThanosSpec(ref common.ReferenceCallback) comm
 							Ref:         ref("k8s.io/api/core/v1.SecretKeySelector"),
 						},
 					},
+					"listenLocal": {
+						SchemaProps: spec.SchemaProps{
+							Description: "ListenLocal makes the Thanos sidecar listen on loopback, so that it does not bind against the Pod IP.",
+							Type:        []string{"boolean"},
+							Format:      "",
+						},
+					},
 				},
 			},
 		},

pkg/apis/monitoring/v1/types.go

@@ -360,6 +360,9 @@ type ThanosSpec struct {
 	Resources v1.ResourceRequirements `json:"resources,omitempty"`
 	// ObjectStorageConfig configures object storage in Thanos.
 	ObjectStorageConfig *v1.SecretKeySelector `json:"objectStorageConfig,omitempty"`
+	// ListenLocal makes the Thanos sidecar listen on loopback, so that it
+	// does not bind against the Pod IP.
+	ListenLocal bool `json:"listenLocal,omitempty"`
 }
 
 // RemoteWriteSpec defines the remote_write configuration for prometheus.

pkg/prometheus/statefulset.go

@@ -739,6 +739,10 @@ func makeStatefulSetSpec(p monitoringv1.Prometheus, c *Config, ruleConfigMapName
 		if p.Spec.Thanos.Image != nil && *p.Spec.Thanos.Image != "" {
 			thanosImage = *p.Spec.Thanos.Image
 		}
+		bindAddress := "[$(POD_IP)]"
+		if p.Spec.Thanos.ListenLocal {
+			bindAddress = "127.0.0.1"
+		}
 
 		container := v1.Container{
 			Name:  "thanos-sidecar",
@@ -747,8 +751,8 @@ func makeStatefulSetSpec(p monitoringv1.Prometheus, c *Config, ruleConfigMapName
 				"sidecar",
 				fmt.Sprintf("--prometheus.url=http://%s:9090%s", c.LocalHost, path.Clean(webRoutePrefix)),
 				fmt.Sprintf("--tsdb.path=%s", storageDir),
-				"--grpc-address=[$(POD_IP)]:10901",
-				"--http-address=[$(POD_IP)]:10902",
+				fmt.Sprintf("--grpc-address=%s:10901", bindAddress),
+				fmt.Sprintf("--http-address=%s:10902", bindAddress),
 			},
 			Env: []v1.EnvVar{
 				{

pkg/prometheus/statefulset_test.go

@@ -878,3 +878,30 @@ func TestWALCompression(t *testing.T) {
 		}
 	}
 }
+
+func TestThanosListenLocal(t *testing.T) {
+	sset, err := makeStatefulSet(monitoringv1.Prometheus{
+		Spec: monitoringv1.PrometheusSpec{
+			Thanos: &monitoringv1.ThanosSpec{
+				ListenLocal: true,
+			},
+		},
+	}, defaultTestConfig, nil, "")
+	if err != nil {
+		t.Fatalf("Unexpected error while making StatefulSet: %v", err)
+	}
+	foundGrpcFlag := false
+	foundHTTPFlag := false
+	for _, flag := range sset.Spec.Template.Spec.Containers[2].Args {
+		if flag == "--grpc-address=127.0.0.1:10901" {
+			foundGrpcFlag = true
+		}
+		if flag == "--http-address=127.0.0.1:10902" {
+			foundHTTPFlag = true
+		}
+	}
+
+	if !foundGrpcFlag || !foundHTTPFlag {
+		t.Fatal("Thanos not listening on loopback when it should.")
+	}
+}