test/e2e: Decrease config map data limit

The config map data limit seems to differ between environments. This
might result from different meta data sizes accross environments. Adding
a big buffer fixes the problem.

.travis.yml

@@ -10,7 +10,6 @@ before_install:
 jobs:
   include:
   - stage: Sanity check and tests
-
   # Check generated contents are up to date and code is formatted.
     script: make format generate-in-docker && git diff --exit-code
   - script: cd contrib/kube-prometheus && make test-in-docker
@@ -21,12 +20,10 @@ jobs:
   # Unit tests
   - script: make test-unit
   # E2e tests
-    script: ./scripts/travis-e2e.sh
+  - script: ./scripts/travis-e2e.sh
   - script: ./scripts/travis-e2e-helm.sh
 
-
   - stage: deploy
-
     script: skip
     deploy:
       provider: script
@@ -34,7 +31,5 @@ jobs:
       on:
         branch: master
 
-
   - stage: push-docker-image
-
     script: ./scripts/travis-push-docker-image.sh

pkg/prometheus/operator_test.go

@@ -36,11 +36,11 @@ func TestCreateStatefulSetChecksum(t *testing.T) {
 	p2.Spec.Version = "v1.7.2"
 	c := Config{}
 
-	p1Checksum, err := createSSetInputChecksum(p1, c)
+	p1Checksum, err := createSSetInputChecksum(p1, c, []string{})
 	if err != nil {
 		t.Fatal(err)
 	}
-	p2Checksum, err := createSSetInputChecksum(p2, c)
+	p2Checksum, err := createSSetInputChecksum(p2, c, []string{})
 	if err != nil {
 		t.Fatal(err)
 	}

pkg/prometheus/rules.go

@@ -32,6 +32,12 @@ import (
 	"github.com/pkg/errors"
 )
 
+// The maximum `Data` size of a config map seems to differ between
+// environments. This is probably due to different meta data sizes which count
+// into the overall maximum size of a config map. Thereby lets leave a
+// large buffer.
+var maxConfigMapDataSize = int(float64(v1.MaxSecretSize) * 0.5)
+
 func (c *Operator) createOrUpdateRuleConfigMaps(p *monitoringv1.Prometheus) ([]string, error) {
 	cClient := c.kclient.CoreV1().ConfigMaps(p.Namespace)
 
@@ -206,7 +212,7 @@ func sortKeyesOfStringMap(m map[string]string) []string {
 func makeRulesConfigMaps(p *monitoringv1.Prometheus, ruleFiles map[string]string) ([]v1.ConfigMap, error) {
 	//check if none of the rule files is too large for a single config map
 	for filename, file := range ruleFiles {
-		if len(file) > v1.MaxSecretSize {
+		if len(file) > maxConfigMapDataSize {
 			return nil, errors.Errorf(
 				"rule file '%v' is too large for a single Kubernetes config map",
 				filename,
@@ -222,7 +228,7 @@ func makeRulesConfigMaps(p *monitoringv1.Prometheus, ruleFiles map[string]string
 
 	for _, filename := range sortedNames {
 		// If rule file doesn't fit into current bucket, create new bucket
-		if bucketSize(buckets[currBucketIndex])+len(ruleFiles[filename]) > v1.MaxSecretSize {
+		if bucketSize(buckets[currBucketIndex])+len(ruleFiles[filename]) > maxConfigMapDataSize {
 			buckets = append(buckets, map[string]string{})
 			currBucketIndex++
 		}

pkg/prometheus/rules_test.go

@@ -64,7 +64,7 @@ func shouldSplitUpLargeSmallIntoTwo(t *testing.T) {
 	p := &monitoringv1.Prometheus{}
 	ruleFiles := map[string]string{}
 
-	ruleFiles["my-rule-file-1"] = strings.Repeat("a", v1.MaxSecretSize)
+	ruleFiles["my-rule-file-1"] = strings.Repeat("a", maxConfigMapDataSize)
 	ruleFiles["my-rule-file-2"] = "a"
 
 	configMaps, err := makeRulesConfigMaps(p, ruleFiles)

test/e2e/main_test.go

@@ -16,16 +16,14 @@ package e2e
 
 import (
 	"flag"
-	"fmt"
 	"log"
 	"os"
 	"testing"
 
-	"k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-
 	"github.com/coreos/prometheus-operator/pkg/k8sutil"
 	operatorFramework "github.com/coreos/prometheus-operator/test/framework"
+
+	"k8s.io/api/core/v1"
 )
 
 var framework *operatorFramework.Framework
@@ -68,34 +66,22 @@ func TestMain(m *testing.M) {
 	}
 
 	defer func() {
+		if code != 0 {
+			if err := framework.PrintEvents(); err != nil {
+				log.Printf("failed to print events: %v", err)
+			}
+			if err := framework.PrintPodLogs(framework.Namespace.Name, framework.OperatorPod.Name); err != nil {
+				log.Printf("failed to print Prometheus Operator logs: %v", err)
+			}
+		}
+
 		if err := framework.Teardown(); err != nil {
 			log.Printf("failed to teardown framework: %v\n", err)
 			code = 1
 		}
 
-		if code != 0 {
-			if err := printKubernetesEvents(); err != nil {
-				log.Printf("failed to print events: %v", err)
-			}
-		}
-
 		os.Exit(code)
 	}()
 
 	code = m.Run()
 }
-
-func printKubernetesEvents() error {
-	fmt.Println("Printing Kubernetes events for debugging:")
-	events, err := framework.KubeClient.CoreV1().Events("").List(metav1.ListOptions{})
-	if err != nil {
-		return err
-	}
-	if events != nil {
-		for _, e := range events.Items {
-			fmt.Printf("FirstTimestamp: '%v', Reason: '%v', Message: '%v'\n", e.FirstTimestamp, e.Reason, e.Message)
-		}
-	}
-
-	return nil
-}

test/e2e/prometheus_test.go

@@ -651,6 +651,14 @@ func TestPrometheusRulesExceedingConfigMapLimit(t *testing.T) {
 		t.Fatal(err)
 	}
 
+	defer func() {
+		if t.Failed() {
+			if err := framework.PrintPodLogs(ns, "prometheus-"+p.Name+"-0"); err != nil {
+				t.Fatal(err)
+			}
+		}
+	}()
+
 	pSVC := framework.MakePrometheusService(p.Name, "not-relevant", v1.ServiceTypeClusterIP)
 	if finalizerFn, err := testFramework.CreateServiceAndWaitUntilReady(framework.KubeClient, ns, pSVC); err != nil {
 		t.Fatal(errors.Wrap(err, "creating Prometheus service failed"))
@@ -658,16 +666,16 @@ func TestPrometheusRulesExceedingConfigMapLimit(t *testing.T) {
 		ctx.AddFinalizerFn(finalizerFn)
 	}
 
-	// Make sure both rule files ended up in the Prometheus Pod
 	for i := range prometheusRules {
-		err := framework.WaitForPrometheusFiringAlert(ns, pSVC.Name, "my-alert-"+strconv.Itoa(i))
+		_, err := framework.WaitForConfigMapExist(ns, "prometheus-"+p.Name+"-rulefiles-"+strconv.Itoa(i))
 		if err != nil {
 			t.Fatal(err)
 		}
 	}
 
+	// Make sure both rule files ended up in the Prometheus Pod
 	for i := range prometheusRules {
-		_, err := framework.WaitForConfigMapExist(ns, "prometheus-"+p.Name+"-rulefiles-"+strconv.Itoa(i))
+		err := framework.WaitForPrometheusFiringAlert(ns, pSVC.Name, "my-alert-"+strconv.Itoa(i))
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -686,6 +694,11 @@ func TestPrometheusRulesExceedingConfigMapLimit(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+
+	err = framework.WaitForPrometheusFiringAlert(ns, pSVC.Name, "my-alert-0")
+	if err != nil {
+		t.Fatal(err)
+	}
 }
 
 // generateHugePrometheusRule returns a Prometheus rule instance that would fill
@@ -698,8 +711,8 @@ func generateHugePrometheusRule(ns, identifier string) monitoringv1.PrometheusRu
 			Rules: []monitoringv1.Rule{},
 		},
 	}
-	// Approximating that each rule is 50 bytes and that the config map limit is 1024 * 1024 bytes
-	for i := 0; i < 15000; i++ {
+	// One rule marshaled as yaml is ~34 bytes long, the max is ~524288 bytes.
+	for i := 0; i < 12000; i++ {
 		groups[0].Rules = append(groups[0].Rules, monitoringv1.Rule{
 			Alert: alertName + "-" + identifier,
 			Expr:  "vector(1)",

test/framework/event.go

@@ -0,0 +1,37 @@
+// Copyright 2016 The prometheus-operator Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package framework
+
+import (
+	"fmt"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// PrintEvents prints the Kubernetes events to standard out.
+func (f *Framework) PrintEvents() error {
+	events, err := f.KubeClient.CoreV1().Events("").List(metav1.ListOptions{})
+	if err != nil {
+		return err
+	}
+	if events != nil {
+		fmt.Println("=== Kubernetes events:")
+		for _, e := range events.Items {
+			fmt.Printf("FirstTimestamp: '%v', Reason: '%v', Message: '%v'\n", e.FirstTimestamp, e.Reason, e.Message)
+		}
+	}
+
+	return nil
+}

test/framework/pod.go

@@ -0,0 +1,44 @@
+// Copyright 2016 The prometheus-operator Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package framework
+
+import (
+	"fmt"
+	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/pkg/errors"
+)
+
+// PrintPodLogs prints the logs of a specified Pod
+func (f *Framework) PrintPodLogs(ns, p string) error {
+	pod, err := f.KubeClient.CoreV1().Pods(ns).Get(p, metav1.GetOptions{})
+	if err != nil {
+		return errors.Wrapf(err, "failed to print logs of pod '%v': failed to get pod", p)
+	}
+
+	for _, c := range pod.Spec.Containers {
+		req := f.KubeClient.CoreV1().Pods(ns).GetLogs(p, &v1.PodLogOptions{Container: c.Name})
+		resp, err := req.DoRaw()
+		if err != nil {
+			return errors.Wrapf(err, "failed to retrieve logs of pod '%v'", p)
+		}
+
+		fmt.Printf("=== Logs of %v/%v/%v:", ns, p, c.Name)
+		fmt.Println(string(resp))
+	}
+
+	return nil
+}