bundle.yaml: Bump Prometheus Operator memory request and limit (#1622)

When handling big Kubernetes objects, marshalling objects is memory
intense. This can be reproduced with the end-to-end test
`TestPrometheusRulesExceedingConfigMapLimit`. This patch doubles the
memory request and limit of the Prometheus Operator deployment to 100mb
and 200mb.

Documentation/user-guides/cluster-monitoring.md

@@ -70,10 +70,10 @@ spec:
         resources:
           limits:
             cpu: 200m
-            memory: 100Mi
+            memory: 200Mi
           requests:
             cpu: 100m
-            memory: 50Mi
+            memory: 100Mi
         securityContext:
           allowPrivilegeEscalation: false
           readOnlyRootFilesystem: true

Documentation/user-guides/getting-started.md

@@ -126,10 +126,10 @@ spec:
         resources:
           limits:
             cpu: 200m
-            memory: 100Mi
+            memory: 200Mi
           requests:
             cpu: 100m
-            memory: 50Mi
+            memory: 100Mi
         securityContext:
           allowPrivilegeEscalation: false
           readOnlyRootFilesystem: true

bundle.yaml

@@ -108,10 +108,10 @@ spec:
         resources:
           limits:
             cpu: 200m
-            memory: 100Mi
+            memory: 200Mi
           requests:
             cpu: 100m
-            memory: 50Mi
+            memory: 100Mi
         securityContext:
           allowPrivilegeEscalation: false
           readOnlyRootFilesystem: true

contrib/kube-prometheus/manifests/0prometheus-operator-deployment.yaml

@@ -29,10 +29,10 @@ spec:
         resources:
           limits:
             cpu: 200m
-            memory: 100Mi
+            memory: 200Mi
           requests:
             cpu: 100m
-            memory: 50Mi
+            memory: 100Mi
         securityContext:
           allowPrivilegeEscalation: false
           readOnlyRootFilesystem: true

example/non-rbac/prometheus-operator.yaml

@@ -29,10 +29,10 @@ spec:
         resources:
           limits:
             cpu: 200m
-            memory: 100Mi
+            memory: 200Mi
           requests:
             cpu: 100m
-            memory: 50Mi
+            memory: 100Mi
         securityContext:
           allowPrivilegeEscalation: false
           readOnlyRootFilesystem: true

example/rbac/prometheus-operator/prometheus-operator-deployment.yaml

@@ -29,10 +29,10 @@ spec:
         resources:
           limits:
             cpu: 200m
-            memory: 100Mi
+            memory: 200Mi
           requests:
             cpu: 100m
-            memory: 50Mi
+            memory: 100Mi
         securityContext:
           allowPrivilegeEscalation: false
           readOnlyRootFilesystem: true

jsonnet/prometheus-operator/prometheus-operator.libsonnet

@@ -127,8 +127,8 @@ local k = import 'ksonnet/ksonnet.beta.3/k.libsonnet';
         ]) +
         container.mixin.securityContext.withAllowPrivilegeEscalation(false) +
         container.mixin.securityContext.withReadOnlyRootFilesystem(true) +
-        container.mixin.resources.withRequests({ cpu: '100m', memory: '50Mi' }) +
-        container.mixin.resources.withLimits({ cpu: '200m', memory: '100Mi' });
+        container.mixin.resources.withRequests({ cpu: '100m', memory: '100Mi' }) +
+        container.mixin.resources.withLimits({ cpu: '200m', memory: '200Mi' });
 
       deployment.new('prometheus-operator', 1, operatorContainer, podLabels) +
       deployment.mixin.metadata.withNamespace($._config.namespace) +

test/e2e/main_test.go

@@ -20,10 +20,7 @@ import (
 	"os"
 	"testing"
 
-	"github.com/coreos/prometheus-operator/pkg/k8sutil"
 	operatorFramework "github.com/coreos/prometheus-operator/test/framework"
-
-	"k8s.io/api/core/v1"
 )
 
 var framework *operatorFramework.Framework
@@ -38,8 +35,8 @@ func TestMain(m *testing.M) {
 	flag.Parse()
 
 	var (
-		err  error
-		code int = 0
+		err      error
+		exitCode int
 	)
 
 	if framework, err = operatorFramework.New(*ns, *kubeconfig, *opImage); err != nil {
@@ -47,26 +44,8 @@ func TestMain(m *testing.M) {
 		os.Exit(1)
 	}
 
-	err = k8sutil.WaitForCRDReady(framework.MonClientV1.Prometheuses(v1.NamespaceAll).List)
-	if err != nil {
-		log.Printf("Prometheus CRD not ready: %v\n", err)
-		os.Exit(1)
-	}
-
-	err = k8sutil.WaitForCRDReady(framework.MonClientV1.ServiceMonitors(v1.NamespaceAll).List)
-	if err != nil {
-		log.Printf("ServiceMonitor CRD not ready: %v\n", err)
-		os.Exit(1)
-	}
-
-	err = k8sutil.WaitForCRDReady(framework.MonClientV1.Alertmanagers(v1.NamespaceAll).List)
-	if err != nil {
-		log.Printf("Alertmanagers CRD not ready: %v\n", err)
-		os.Exit(1)
-	}
-
 	defer func() {
-		if code != 0 {
+		if exitCode != 0 {
 			if err := framework.PrintEvents(); err != nil {
 				log.Printf("failed to print events: %v", err)
 			}
@@ -77,11 +56,31 @@ func TestMain(m *testing.M) {
 
 		if err := framework.Teardown(); err != nil {
 			log.Printf("failed to teardown framework: %v\n", err)
-			code = 1
+			exitCode = 1
 		}
 
-		os.Exit(code)
+		os.Exit(exitCode)
 	}()
 
-	code = m.Run()
+	exitCode = m.Run()
+
+	// Check if Prometheus Operator ever restarted.
+	restarts, err := framework.GetRestartCount(framework.Namespace.Name, framework.OperatorPod.Name)
+	if err != nil {
+		log.Printf("failed to retrieve restart count of Prometheus Operator pod: %v", err)
+		exitCode = 1
+	}
+	if len(restarts) != 1 {
+		log.Printf("expected to have 1 container but got %d", len(restarts))
+		exitCode = 1
+	}
+	for _, restart := range restarts {
+		if restart != 0 {
+			log.Printf(
+				"expected Prometheus Operator to never restart during entire test execution but got %d restarts",
+				restart,
+			)
+			exitCode = 1
+		}
+	}
 }

test/framework/alertmanager.go

@@ -17,7 +17,6 @@ package framework
 import (
 	"encoding/json"
 	"fmt"
-	"log"
 	"os"
 	"strings"
 	"time"
@@ -279,8 +278,12 @@ func (f *Framework) GetSilences(ns, n string) ([]amAPISil, error) {
 	return getSilencesResponse.Data, nil
 }
 
-func (f *Framework) WaitForAlertmanagerConfigToContainString(ns, amName string, expectedString string) error {
-	return wait.Poll(10*time.Second, time.Minute*5, func() (bool, error) {
+// WaitForAlertmanagerConfigToContainString retrieves the Alertmanager
+// configuration via the Alertmanager's API and checks if it contains the given
+// string.
+func (f *Framework) WaitForAlertmanagerConfigToContainString(ns, amName, expectedString string) error {
+	var pollError error
+	err := wait.Poll(10*time.Second, time.Minute*5, func() (bool, error) {
 		config, err := f.GetAlertmanagerConfig(ns, "alertmanager-"+amName+"-0")
 		if err != nil {
 			return false, err
@@ -290,10 +293,14 @@ func (f *Framework) WaitForAlertmanagerConfigToContainString(ns, amName string,
 			return true, nil
 		}
 
-		log.Printf("\n\nExpected:\n\n%#+v\n\nto contain:\n\n%#+v\n\n", config.Data.ConfigYAML, expectedString)
-
 		return false, nil
 	})
+
+	if err != nil {
+		return fmt.Errorf("failed to wait for alertmanager config to contain %q: %v: %v", expectedString, err, pollError)
+	}
+
+	return nil
 }
 
 type amAPICreateSilResp struct {

test/framework/framework.go

@@ -165,6 +165,26 @@ func (f *Framework) setupPrometheusOperator(opImage string) error {
 	}
 	f.OperatorPod = &pl.Items[0]
 
+	err = k8sutil.WaitForCRDReady(f.MonClientV1.Prometheuses(v1.NamespaceAll).List)
+	if err != nil {
+		return errors.Wrap(err, "Prometheus CRD not ready: %v\n")
+	}
+
+	err = k8sutil.WaitForCRDReady(f.MonClientV1.ServiceMonitors(v1.NamespaceAll).List)
+	if err != nil {
+		return errors.Wrap(err, "ServiceMonitor CRD not ready: %v\n")
+	}
+
+	err = k8sutil.WaitForCRDReady(f.MonClientV1.PrometheusRules(v1.NamespaceAll).List)
+	if err != nil {
+		return errors.Wrap(err, "PrometheusRule CRD not ready: %v\n")
+	}
+
+	err = k8sutil.WaitForCRDReady(f.MonClientV1.Alertmanagers(v1.NamespaceAll).List)
+	if err != nil {
+		return errors.Wrap(err, "Alertmanager CRD not ready: %v\n")
+	}
+
 	return nil
 }
 

test/framework/pod.go

@@ -42,3 +42,20 @@ func (f *Framework) PrintPodLogs(ns, p string) error {
 
 	return nil
 }
+
+// GetRestartCount returns a map of container names and their restart counts for
+// a given pod.
+func (f *Framework) GetRestartCount(ns, podName string) (map[string]int32, error) {
+	pod, err := f.KubeClient.CoreV1().Pods(ns).Get(podName, metav1.GetOptions{})
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to retrieve pod to get restart count")
+	}
+
+	restarts := map[string]int32{}
+
+	for _, status := range pod.Status.ContainerStatuses {
+		restarts[status.Name] = status.RestartCount
+	}
+
+	return restarts, nil
+}