diff --git a/.travis.yml b/.travis.yml index 2211be460..b4b4685cd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,6 @@ before_install: jobs: include: - stage: Sanity check and tests - # Check generated contents are up to date and code is formatted. script: make format generate-in-docker && git diff --exit-code - script: cd contrib/kube-prometheus && make test-in-docker @@ -21,12 +20,10 @@ jobs: # Unit tests - script: make test-unit # E2e tests - script: ./scripts/travis-e2e.sh + - script: ./scripts/travis-e2e.sh - script: ./scripts/travis-e2e-helm.sh - - stage: deploy - script: skip deploy: provider: script @@ -34,7 +31,5 @@ jobs: on: branch: master - - stage: push-docker-image - script: ./scripts/travis-push-docker-image.sh diff --git a/pkg/prometheus/operator_test.go b/pkg/prometheus/operator_test.go index c4b819bfa..a17d107a1 100644 --- a/pkg/prometheus/operator_test.go +++ b/pkg/prometheus/operator_test.go @@ -36,11 +36,11 @@ func TestCreateStatefulSetChecksum(t *testing.T) { p2.Spec.Version = "v1.7.2" c := Config{} - p1Checksum, err := createSSetInputChecksum(p1, c) + p1Checksum, err := createSSetInputChecksum(p1, c, []string{}) if err != nil { t.Fatal(err) } - p2Checksum, err := createSSetInputChecksum(p2, c) + p2Checksum, err := createSSetInputChecksum(p2, c, []string{}) if err != nil { t.Fatal(err) } diff --git a/pkg/prometheus/rules.go b/pkg/prometheus/rules.go index ba3cd566f..728d3e340 100644 --- a/pkg/prometheus/rules.go +++ b/pkg/prometheus/rules.go @@ -32,6 +32,12 @@ import ( "github.com/pkg/errors" ) +// The maximum `Data` size of a config map seems to differ between +// environments. This is probably due to different meta data sizes which count +// into the overall maximum size of a config map. Thereby lets leave a +// large buffer. +var maxConfigMapDataSize = int(float64(v1.MaxSecretSize) * 0.5) + func (c *Operator) createOrUpdateRuleConfigMaps(p *monitoringv1.Prometheus) ([]string, error) { cClient := c.kclient.CoreV1().ConfigMaps(p.Namespace) @@ -206,7 +212,7 @@ func sortKeyesOfStringMap(m map[string]string) []string { func makeRulesConfigMaps(p *monitoringv1.Prometheus, ruleFiles map[string]string) ([]v1.ConfigMap, error) { //check if none of the rule files is too large for a single config map for filename, file := range ruleFiles { - if len(file) > v1.MaxSecretSize { + if len(file) > maxConfigMapDataSize { return nil, errors.Errorf( "rule file '%v' is too large for a single Kubernetes config map", filename, @@ -222,7 +228,7 @@ func makeRulesConfigMaps(p *monitoringv1.Prometheus, ruleFiles map[string]string for _, filename := range sortedNames { // If rule file doesn't fit into current bucket, create new bucket - if bucketSize(buckets[currBucketIndex])+len(ruleFiles[filename]) > v1.MaxSecretSize { + if bucketSize(buckets[currBucketIndex])+len(ruleFiles[filename]) > maxConfigMapDataSize { buckets = append(buckets, map[string]string{}) currBucketIndex++ } diff --git a/pkg/prometheus/rules_test.go b/pkg/prometheus/rules_test.go index 82931c76a..d8cb6b916 100644 --- a/pkg/prometheus/rules_test.go +++ b/pkg/prometheus/rules_test.go @@ -64,7 +64,7 @@ func shouldSplitUpLargeSmallIntoTwo(t *testing.T) { p := &monitoringv1.Prometheus{} ruleFiles := map[string]string{} - ruleFiles["my-rule-file-1"] = strings.Repeat("a", v1.MaxSecretSize) + ruleFiles["my-rule-file-1"] = strings.Repeat("a", maxConfigMapDataSize) ruleFiles["my-rule-file-2"] = "a" configMaps, err := makeRulesConfigMaps(p, ruleFiles) diff --git a/test/e2e/main_test.go b/test/e2e/main_test.go index 2223ac2eb..40a2df165 100644 --- a/test/e2e/main_test.go +++ b/test/e2e/main_test.go @@ -16,16 +16,14 @@ package e2e import ( "flag" - "fmt" "log" "os" "testing" - "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "github.com/coreos/prometheus-operator/pkg/k8sutil" operatorFramework "github.com/coreos/prometheus-operator/test/framework" + + "k8s.io/api/core/v1" ) var framework *operatorFramework.Framework @@ -68,34 +66,22 @@ func TestMain(m *testing.M) { } defer func() { + if code != 0 { + if err := framework.PrintEvents(); err != nil { + log.Printf("failed to print events: %v", err) + } + if err := framework.PrintPodLogs(framework.Namespace.Name, framework.OperatorPod.Name); err != nil { + log.Printf("failed to print Prometheus Operator logs: %v", err) + } + } + if err := framework.Teardown(); err != nil { log.Printf("failed to teardown framework: %v\n", err) code = 1 } - if code != 0 { - if err := printKubernetesEvents(); err != nil { - log.Printf("failed to print events: %v", err) - } - } - os.Exit(code) }() code = m.Run() } - -func printKubernetesEvents() error { - fmt.Println("Printing Kubernetes events for debugging:") - events, err := framework.KubeClient.CoreV1().Events("").List(metav1.ListOptions{}) - if err != nil { - return err - } - if events != nil { - for _, e := range events.Items { - fmt.Printf("FirstTimestamp: '%v', Reason: '%v', Message: '%v'\n", e.FirstTimestamp, e.Reason, e.Message) - } - } - - return nil -} diff --git a/test/e2e/prometheus_test.go b/test/e2e/prometheus_test.go index 3c94841bf..eea9783ae 100644 --- a/test/e2e/prometheus_test.go +++ b/test/e2e/prometheus_test.go @@ -651,6 +651,14 @@ func TestPrometheusRulesExceedingConfigMapLimit(t *testing.T) { t.Fatal(err) } + defer func() { + if t.Failed() { + if err := framework.PrintPodLogs(ns, "prometheus-"+p.Name+"-0"); err != nil { + t.Fatal(err) + } + } + }() + pSVC := framework.MakePrometheusService(p.Name, "not-relevant", v1.ServiceTypeClusterIP) if finalizerFn, err := testFramework.CreateServiceAndWaitUntilReady(framework.KubeClient, ns, pSVC); err != nil { t.Fatal(errors.Wrap(err, "creating Prometheus service failed")) @@ -658,16 +666,16 @@ func TestPrometheusRulesExceedingConfigMapLimit(t *testing.T) { ctx.AddFinalizerFn(finalizerFn) } - // Make sure both rule files ended up in the Prometheus Pod for i := range prometheusRules { - err := framework.WaitForPrometheusFiringAlert(ns, pSVC.Name, "my-alert-"+strconv.Itoa(i)) + _, err := framework.WaitForConfigMapExist(ns, "prometheus-"+p.Name+"-rulefiles-"+strconv.Itoa(i)) if err != nil { t.Fatal(err) } } + // Make sure both rule files ended up in the Prometheus Pod for i := range prometheusRules { - _, err := framework.WaitForConfigMapExist(ns, "prometheus-"+p.Name+"-rulefiles-"+strconv.Itoa(i)) + err := framework.WaitForPrometheusFiringAlert(ns, pSVC.Name, "my-alert-"+strconv.Itoa(i)) if err != nil { t.Fatal(err) } @@ -686,6 +694,11 @@ func TestPrometheusRulesExceedingConfigMapLimit(t *testing.T) { if err != nil { t.Fatal(err) } + + err = framework.WaitForPrometheusFiringAlert(ns, pSVC.Name, "my-alert-0") + if err != nil { + t.Fatal(err) + } } // generateHugePrometheusRule returns a Prometheus rule instance that would fill @@ -698,8 +711,8 @@ func generateHugePrometheusRule(ns, identifier string) monitoringv1.PrometheusRu Rules: []monitoringv1.Rule{}, }, } - // Approximating that each rule is 50 bytes and that the config map limit is 1024 * 1024 bytes - for i := 0; i < 15000; i++ { + // One rule marshaled as yaml is ~34 bytes long, the max is ~524288 bytes. + for i := 0; i < 12000; i++ { groups[0].Rules = append(groups[0].Rules, monitoringv1.Rule{ Alert: alertName + "-" + identifier, Expr: "vector(1)", diff --git a/test/framework/event.go b/test/framework/event.go new file mode 100644 index 000000000..0a7c417fb --- /dev/null +++ b/test/framework/event.go @@ -0,0 +1,37 @@ +// Copyright 2016 The prometheus-operator Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package framework + +import ( + "fmt" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// PrintEvents prints the Kubernetes events to standard out. +func (f *Framework) PrintEvents() error { + events, err := f.KubeClient.CoreV1().Events("").List(metav1.ListOptions{}) + if err != nil { + return err + } + if events != nil { + fmt.Println("=== Kubernetes events:") + for _, e := range events.Items { + fmt.Printf("FirstTimestamp: '%v', Reason: '%v', Message: '%v'\n", e.FirstTimestamp, e.Reason, e.Message) + } + } + + return nil +} diff --git a/test/framework/pod.go b/test/framework/pod.go new file mode 100644 index 000000000..b5feb97eb --- /dev/null +++ b/test/framework/pod.go @@ -0,0 +1,44 @@ +// Copyright 2016 The prometheus-operator Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package framework + +import ( + "fmt" + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/pkg/errors" +) + +// PrintPodLogs prints the logs of a specified Pod +func (f *Framework) PrintPodLogs(ns, p string) error { + pod, err := f.KubeClient.CoreV1().Pods(ns).Get(p, metav1.GetOptions{}) + if err != nil { + return errors.Wrapf(err, "failed to print logs of pod '%v': failed to get pod", p) + } + + for _, c := range pod.Spec.Containers { + req := f.KubeClient.CoreV1().Pods(ns).GetLogs(p, &v1.PodLogOptions{Container: c.Name}) + resp, err := req.DoRaw() + if err != nil { + return errors.Wrapf(err, "failed to retrieve logs of pod '%v'", p) + } + + fmt.Printf("=== Logs of %v/%v/%v:", ns, p, c.Name) + fmt.Println(string(resp)) + } + + return nil +}