mirrors/prometheus-operator
mirrors/prometheus-operator
1
0
Fork 0
mirror of https://github.com/prometheus-operator/prometheus-operator.git synced 2025-04-21 11:48:53 +00:00
Code Activity

prometheus: Split rule config map if it exceeds Kubernetes limit

Browse source 
`k8s.io/api/core/v1.MaxSecretSize` specifies the hard limit Kubernetes
has on the size of config maps [1]. If a set of Prometheus Rule Files
exceeds this limits, they are split onto multiple config maps. If there
is more than one rule config map for a Prometheus, its configuration
secret and its statefulset spec are adjusted accordingly.

The smallest entity to split by is a Prometheus rule file. This patch
does not split an individual rule file, if it exeeds the limit by
itself.

[1] https://github.com/kubernetes/kubernetes/pull/19909/
This commit is contained in:
Max Leonard Inden 2018-07-02 20:23:59 +02:00
parent cad1cb29a3
commit 06018c0445
No known key found for this signature in database
GPG key ID: 5403C5464810BC26
11 changed files with 508 additions and 182 deletions

29
pkg/prometheus/operator.go
View file

@ -737,7 +737,7 @@ func (c *Operator) sync(key string) error {
return err
}
err = c.createOrUpdateRuleConfigMap(p)
ruleConfigMapNames, err := c.createOrUpdateRuleConfigMaps(p)
if err != nil {
return err
}
@ -746,12 +746,12 @@ func (c *Operator) sync(key string) error {
// configuration themselves.
if p.Spec.ServiceMonitorSelector != nil {
// We just always regenerate the configuration to be safe.
if err := c.createOrUpdateConfigurationSecret(p); err != nil {
if err := c.createOrUpdateConfigurationSecret(p, ruleConfigMapNames); err != nil {
return errors.Wrap(err, "creating config failed")
}
}
// Create Secret if it doesn't exist.
// Create empty Secret if it doesn't exist. See comment above.
s, err := makeEmptyConfigurationSecret(p, c.config)
if err != nil {
return errors.Wrap(err, "generating empty config secret failed")
@ -778,14 +778,14 @@ func (c *Operator) sync(key string) error {
return errors.Wrap(err, "retrieving statefulset failed")
}
newSSetInputChecksum, err := createSSetInputChecksum(*p, c.config)
newSSetInputChecksum, err := createSSetInputChecksum(*p, c.config, ruleConfigMapNames)
if err != nil {
return err
}
if !exists {
level.Debug(c.logger).Log("msg", "no current Prometheus statefulset found")
sset, err := makeStatefulSet(*p, "", &c.config, newSSetInputChecksum)
sset, err := makeStatefulSet(*p, "", &c.config, ruleConfigMapNames, newSSetInputChecksum)
if err != nil {
return errors.Wrap(err, "making statefulset failed")
}
@ -803,7 +803,7 @@ func (c *Operator) sync(key string) error {
return nil
}
sset, err := makeStatefulSet(*p, obj.(*appsv1.StatefulSet).Spec.PodManagementPolicy, &c.config, newSSetInputChecksum)
sset, err := makeStatefulSet(*p, obj.(*appsv1.StatefulSet).Spec.PodManagementPolicy, &c.config, ruleConfigMapNames, newSSetInputChecksum)
if err != nil {
return errors.Wrap(err, "making statefulset failed")
}
@ -816,13 +816,13 @@ func (c *Operator) sync(key string) error {
return nil
}
// TODO: rename sSetInputChecksum
func createSSetInputChecksum(p monitoringv1.Prometheus, c Config) (string, error) {
func createSSetInputChecksum(p monitoringv1.Prometheus, c Config, ruleConfigMapNames []string) (string, error) {
json, err := json.Marshal(
struct {
P monitoringv1.Prometheus
C Config
}{p, c},
R []string
}{p, c, ruleConfigMapNames},
)
if err != nil {
return "", errors.Wrap(err, "failed to marshal Prometheus CRD and config to json")
@ -993,7 +993,7 @@ func (c *Operator) loadBasicAuthSecrets(mons map[string]*monitoringv1.ServiceMon
}
func (c *Operator) createOrUpdateConfigurationSecret(p *monitoringv1.Prometheus) error {
func (c *Operator) createOrUpdateConfigurationSecret(p *monitoringv1.Prometheus, ruleConfigMapNames []string) error {
smons, err := c.selectServiceMonitors(p)
if err != nil {
return errors.Wrap(err, "selecting ServiceMonitors failed")
@ -1023,7 +1023,14 @@ func (c *Operator) createOrUpdateConfigurationSecret(p *monitoringv1.Prometheus)
}
// Update secret based on the most recent configuration.
conf, err := generateConfig(p, smons, basicAuthSecrets, additionalScrapeConfigs, additionalAlertManagerConfigs)
conf, err := generateConfig(
p,
smons,
basicAuthSecrets,
additionalScrapeConfigs,
additionalAlertManagerConfigs,
ruleConfigMapNames,
)
if err != nil {
return errors.Wrap(err, "generating config failed")
}

17
pkg/prometheus/promcfg.go
View file

@ -86,7 +86,14 @@ func buildExternalLabels(p *v1.Prometheus) yaml.MapSlice {
return stringMapToMapSlice(m)
}
func generateConfig(p *v1.Prometheus, mons map[string]*v1.ServiceMonitor, basicAuthSecrets map[string]BasicAuthCredentials, additionalScrapeConfigs []byte, additionalAlertManagerConfigs []byte) ([]byte, error) {
func generateConfig(
p *v1.Prometheus,
mons map[string]*v1.ServiceMonitor,
basicAuthSecrets map[string]BasicAuthCredentials,
additionalScrapeConfigs []byte,
additionalAlertManagerConfigs []byte,
ruleConfigMapNames []string,
) ([]byte, error) {
versionStr := p.Spec.Version
if versionStr == "" {
versionStr = DefaultPrometheusVersion
@ -118,14 +125,18 @@ func generateConfig(p *v1.Prometheus, mons map[string]*v1.ServiceMonitor, basicA
},
})
ruleFilePaths := []string{}
for _, name := range ruleConfigMapNames {
ruleFilePaths = append(ruleFilePaths, rulesDir+"/"+name+"/*.yaml")
}
cfg = append(cfg, yaml.MapItem{
Key: "rule_files",
Value: []string{"/etc/prometheus/rules/*.yaml"},
Value: ruleFilePaths,
})
identifiers := make([]string, len(mons))
i := 0
for k, _ := range mons {
for k := range mons {
identifiers[i] = k
i++
}

8
pkg/prometheus/promcfg_test.go
View file

@ -16,6 +16,7 @@ package prometheus
import (
"bytes"
"fmt"
"testing"
yaml "gopkg.in/yaml.v2"
@ -109,6 +110,7 @@ func TestAlertmanagerBearerToken(t *testing.T) {
map[string]BasicAuthCredentials{},
nil,
nil,
nil,
)
if err != nil {
t.Fatal(err)
@ -123,8 +125,7 @@ func TestAlertmanagerBearerToken(t *testing.T) {
external_labels:
prometheus: default/test
prometheus_replica: $(POD_NAME)
rule_files:
- /etc/prometheus/rules/*.yaml
rule_files: []
scrape_configs: []
alerting:
alert_relabel_configs:
@ -153,7 +154,7 @@ alerting:
result := string(cfg)
if expected != result {
pretty.Compare(expected, result)
fmt.Println(pretty.Compare(expected, result))
t.Fatal("expected Prometheus configuration and actual configuration do not match")
}
}
@ -209,6 +210,7 @@ func generateTestConfig(version string) ([]byte, error) {
map[string]BasicAuthCredentials{},
nil,
nil,
nil,
)
}

199
pkg/prometheus/rules.go
View file

@ -18,12 +18,12 @@ import (
"crypto/sha256"
"fmt"
"sort"
"strconv"
"strings"
monitoringv1 "github.com/coreos/prometheus-operator/pkg/client/monitoring/v1"
"k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/tools/cache"
@ -32,67 +32,88 @@ import (
"github.com/pkg/errors"
)
func (c *Operator) createOrUpdateRuleConfigMap(p *monitoringv1.Prometheus) error {
func (c *Operator) createOrUpdateRuleConfigMaps(p *monitoringv1.Prometheus) ([]string, error) {
cClient := c.kclient.CoreV1().ConfigMaps(p.Namespace)
namespaces, err := c.selectRuleNamespaces(p)
if err != nil {
return err
return nil, err
}
rules, err := c.selectRules(p, namespaces)
if err != nil {
return err
return nil, err
}
newConfigMap := c.makeRulesConfigMap(p, rules)
currentConfigMap, err := cClient.Get(prometheusRuleConfigMapName(p.Name), metav1.GetOptions{})
if err != nil && !apierrors.IsNotFound(err) {
return err
newConfigMaps, err := makeRulesConfigMaps(p, rules)
if err != nil {
errors.Wrap(err, "failed to make rules config maps")
}
isNotFound := false
if apierrors.IsNotFound(err) {
newConfigMapNames := []string{}
for _, cm := range newConfigMaps {
newConfigMapNames = append(newConfigMapNames, cm.Name)
}
currentConfigMapList, err := cClient.List(prometheusRulesConfigMapSelector(p.Name))
if err != nil {
return nil, err
}
currentConfigMaps := currentConfigMapList.Items
if len(currentConfigMaps) == 0 {
level.Debug(c.logger).Log(
"msg", "no PrometheusRule configmap created yet",
"msg", "no PrometheusRule configmap found, creating new one",
"namespace", p.Namespace,
"prometheus", p.Name,
)
isNotFound = true
for _, cm := range newConfigMaps {
_, err = cClient.Create(&cm)
if err != nil {
return nil, errors.Wrapf(err, "failed to create config map '%v'", cm.Name)
}
}
return newConfigMapNames, nil
}
newChecksum := checksumRules(rules)
currentChecksum := checksumRules(currentConfigMap.Data)
newChecksum := checksumConfigMaps(newConfigMaps)
currentChecksum := checksumConfigMaps(currentConfigMaps)
if newChecksum == currentChecksum && !isNotFound {
if newChecksum == currentChecksum {
level.Debug(c.logger).Log(
"msg", "no PrometheusRule changes",
"namespace", p.Namespace,
"prometheus", p.Name,
)
return nil
return newConfigMapNames, nil
}
if isNotFound {
level.Debug(c.logger).Log(
"msg", "no PrometheusRule found, creating new one",
"namespace", p.Namespace,
"prometheus", p.Name,
)
_, err = cClient.Create(newConfigMap)
} else {
level.Debug(c.logger).Log(
"msg", "updating PrometheusRule",
"namespace", p.Namespace,
"prometheus", p.Name,
)
_, err = cClient.Update(newConfigMap)
}
if err != nil {
return err
// Simply deleting old config maps and creating new ones for now. Could be
// replaced by logic that only deletes obsolete config maps in the future.
for _, cm := range currentConfigMaps {
err := cClient.Delete(cm.Name, &metav1.DeleteOptions{})
if err != nil {
return nil, errors.Wrapf(err, "failed to delete current config map '%v'", cm.Name)
}
}
return nil
level.Debug(c.logger).Log(
"msg", "updating PrometheusRule",
"namespace", p.Namespace,
"prometheus", p.Name,
)
for _, cm := range newConfigMaps {
_, err = cClient.Create(&cm)
if err != nil {
return nil, errors.Wrapf(err, "failed to create new config map '%v'", cm.Name)
}
}
return newConfigMapNames, nil
}
func prometheusRulesConfigMapSelector(prometheusName string) metav1.ListOptions {
return metav1.ListOptions{LabelSelector: fmt.Sprintf("prometheus-name=%v", prometheusName)}
}
func (c *Operator) selectRuleNamespaces(p *monitoringv1.Prometheus) ([]string, error) {
@ -149,33 +170,96 @@ func (c *Operator) selectRules(p *monitoringv1.Prometheus, namespaces []string)
}
}
// sort rules map
rulenames := []string{}
for k := range rules {
rulenames = append(rulenames, k)
}
sort.Strings(rulenames)
sortedRules := map[string]string{}
for _, name := range rulenames {
sortedRules[name] = rules[name]
ruleNames := []string{}
for name := range rules {
ruleNames = append(ruleNames, name)
}
level.Debug(c.logger).Log(
"msg", "selected Rules",
"rules", strings.Join(rulenames, ","),
"rules", strings.Join(ruleNames, ","),
"namespace", p.Namespace,
"prometheus", p.Name,
)
return sortedRules, nil
return rules, nil
}
func (c *Operator) makeRulesConfigMap(p *monitoringv1.Prometheus, ruleFiles map[string]string) *v1.ConfigMap {
func sortKeyesOfStringMap(m map[string]string) []string {
keys := []string{}
for k := range m {
keys = append(keys, k)
}
sort.Strings(keys)
return keys
}
// makeRulesConfigMaps takes a Prometheus configuration and rule files and
// returns a list of Kubernetes config maps to be later on mounted into the
// Prometheus instance.
// If the total size of rule files exceeds the Kubernetes config map limit,
// they are split up via the simple first-fit [1] bin packing algorithm. In the
// future this can be replaced by a more sophisticated algorithm, but for now
// simplicity should be sufficient.
// [1] https://en.wikipedia.org/wiki/Bin_packing_problem#First-fit_algorithm
func makeRulesConfigMaps(p *monitoringv1.Prometheus, ruleFiles map[string]string) ([]v1.ConfigMap, error) {
//check if none of the rule files is too large for a single config map
for filename, file := range ruleFiles {
if len(file) > v1.MaxSecretSize {
return nil, errors.Errorf(
"rule file '%v' is too large for a single Kubernetes config map",
filename,
)
}
}
buckets := []map[string]string{
map[string]string{},
}
currBucketIndex := 0
sortedNames := sortKeyesOfStringMap(ruleFiles)
for _, filename := range sortedNames {
// If rule file doesn't fit into current bucket, create new bucket
if bucketSize(buckets[currBucketIndex])+len(ruleFiles[filename]) > v1.MaxSecretSize {
buckets = append(buckets, map[string]string{})
currBucketIndex++
}
buckets[currBucketIndex][filename] = ruleFiles[filename]
}
ruleFileConfigMaps := []v1.ConfigMap{}
for i, bucket := range buckets {
cm := makeRulesConfigMap(p, bucket)
cm.Name = cm.Name + "-" + strconv.Itoa(i)
ruleFileConfigMaps = append(ruleFileConfigMaps, cm)
}
return ruleFileConfigMaps, nil
}
func bucketSize(bucket map[string]string) int {
totalSize := 0
for _, v := range bucket {
totalSize += len(v)
}
return totalSize
}
func makeRulesConfigMap(p *monitoringv1.Prometheus, ruleFiles map[string]string) v1.ConfigMap {
boolTrue := true
return &v1.ConfigMap{
labels := map[string]string{"prometheus-name": p.Name}
for k, v := range managedByOperatorLabels {
labels[k] = v
}
return v1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: prometheusRuleConfigMapName(p.Name),
Labels: managedByOperatorLabels,
Labels: labels,
OwnerReferences: []metav1.OwnerReference{
{
APIVersion: p.APIVersion,
@ -191,10 +275,19 @@ func (c *Operator) makeRulesConfigMap(p *monitoringv1.Prometheus, ruleFiles map[
}
}
func checksumRules(files map[string]string) string {
var sum string
for name, value := range files {
sum = sum + name + value
func checksumConfigMaps(configMaps []v1.ConfigMap) string {
ruleFiles := map[string]string{}
for _, cm := range configMaps {
for filename, file := range cm.Data {
ruleFiles[filename] = file
}
}
sortedKeys := sortKeyesOfStringMap(ruleFiles)
sum := ""
for _, name := range sortedKeys {
sum += name + ruleFiles[name]
}
return fmt.Sprintf("%x", sha256.Sum256([]byte(sum)))

118
pkg/prometheus/rules_test.go Normal file
View file

@ -0,0 +1,118 @@
// Copyright 2016 The prometheus-operator Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package prometheus
import (
"strings"
"testing"
monitoringv1 "github.com/coreos/prometheus-operator/pkg/client/monitoring/v1"
"k8s.io/api/core/v1"
)
func TestMakeRulesConfigMaps(t *testing.T) {
t.Run("ShouldReturnAtLeastOneConfigMap", shouldReturnAtLeastOneConfigMap)
t.Run("ShouldErrorOnTooLargeRuleFile", shouldErrorOnTooLargeRuleFile)
t.Run("ShouldSplitUpLargeSmallIntoTwo", shouldSplitUpLargeSmallIntoTwo)
}
// makeRulesConfigMaps should return at least one config map even if it is empty
// when there are no rules. Otherwise adding a rule to a Prometheus without rules
// would change the statefulset definition and thereby force Prometheus to
// restart.
func shouldReturnAtLeastOneConfigMap(t *testing.T) {
p := &monitoringv1.Prometheus{}
ruleFiles := map[string]string{}
configMaps, err := makeRulesConfigMaps(p, ruleFiles)
if err != nil {
t.Fatalf("expected no error but got: %v", err.Error())
}
if len(configMaps) != 1 {
t.Fatalf("expected one config maps but got %v", len(configMaps))
}
}
func shouldErrorOnTooLargeRuleFile(t *testing.T) {
expectedError := "rule file 'my-rule-file' is too large for a single Kubernetes config map"
p := &monitoringv1.Prometheus{}
ruleFiles := map[string]string{}
ruleFiles["my-rule-file"] = strings.Repeat("a", v1.MaxSecretSize+1)
_, err := makeRulesConfigMaps(p, ruleFiles)
if err == nil || err.Error() != expectedError {
t.Fatalf("expected makeRulesConfigMaps to return error '%v' but got '%v'", expectedError, err)
}
}
func shouldSplitUpLargeSmallIntoTwo(t *testing.T) {
p := &monitoringv1.Prometheus{}
ruleFiles := map[string]string{}
ruleFiles["my-rule-file-1"] = strings.Repeat("a", v1.MaxSecretSize)
ruleFiles["my-rule-file-2"] = "a"
configMaps, err := makeRulesConfigMaps(p, ruleFiles)
if err != nil {
t.Fatalf("expected no error but got: %v", err)
}
if len(configMaps) != 2 {
t.Fatalf("expected rule files to be split up into two config maps, but got '%v' instead", len(configMaps))
}
if configMaps[0].Data["my-rule-file-1"] != ruleFiles["my-rule-file-1"] &&
configMaps[1].Data["my-rule-file-2"] != ruleFiles["my-rule-file-2"] {
t.Fatal("expected config map data to match rule file content")
}
}
func TestChecksumConfigMaps(t *testing.T) {
configMapsAsc := []v1.ConfigMap{
v1.ConfigMap{
Data: map[string]string{
"key1a": "value1a",
"key1b": "value1b",
},
},
v1.ConfigMap{
Data: map[string]string{
"key2a": "value2a",
"key2b": "value2b",
},
},
}
configMapsDesc := []v1.ConfigMap{
v1.ConfigMap{
Data: map[string]string{
"key2b": "value2b",
"key2a": "value2a",
},
},
v1.ConfigMap{
Data: map[string]string{
"key1b": "value1b",
"key1a": "value1a",
},
},
}
if checksumConfigMaps(configMapsAsc) != checksumConfigMaps(configMapsDesc) {
t.Fatal("expected two config map slices with the same keys and values in different order to have same checksum")
}
}

80
pkg/prometheus/statefulset.go
View file

@ -79,6 +79,7 @@ func makeStatefulSet(
p monitoringv1.Prometheus,
previousPodManagementPolicy appsv1.PodManagementPolicyType,
config *Config,
ruleConfigMapNames []string,
inputChecksum string,
) (*appsv1.StatefulSet, error) {
// TODO(fabxc): is this the right point to inject defaults?
@ -132,7 +133,7 @@ func makeStatefulSet(
}
}
spec, err := makeStatefulSetSpec(p, config)
spec, err := makeStatefulSetSpec(p, config, ruleConfigMapNames)
if err != nil {
return nil, errors.Wrap(err, "make StatefulSet spec")
}
@ -281,7 +282,7 @@ func makeStatefulSetService(p *monitoringv1.Prometheus, config Config) *v1.Servi
return svc
}
func makeStatefulSetSpec(p monitoringv1.Prometheus, c *Config) (*appsv1.StatefulSetSpec, error) {
func makeStatefulSetSpec(p monitoringv1.Prometheus, c *Config, ruleConfigMapNames []string) (*appsv1.StatefulSetSpec, error) {
// Prometheus may take quite long to shut down to checkpoint existing data.
// Allow up to 10 minutes for clean termination.
terminationGracePeriod := int64(600)
@ -416,16 +417,19 @@ func makeStatefulSetSpec(p monitoringv1.Prometheus, c *Config) (*appsv1.Stateful
EmptyDir: &v1.EmptyDirVolumeSource{},
},
},
{
Name: "rules",
}
for _, name := range ruleConfigMapNames {
volumes = append(volumes, v1.Volume{
Name: name,
VolumeSource: v1.VolumeSource{
ConfigMap: &v1.ConfigMapVolumeSource{
LocalObjectReference: v1.LocalObjectReference{
Name: prometheusRuleConfigMapName(p.Name),
Name: name,
},
},
},
},
})
}
volName := volumeName(p.Name)
@ -441,10 +445,6 @@ func makeStatefulSetSpec(p monitoringv1.Prometheus, c *Config) (*appsv1.Stateful
ReadOnly: true,
MountPath: confOutDir,
},
{
Name: "rules",
MountPath: "/etc/prometheus/rules",
},
{
Name: volName,
MountPath: storageDir,
@ -452,6 +452,13 @@ func makeStatefulSetSpec(p monitoringv1.Prometheus, c *Config) (*appsv1.Stateful
},
}
for _, name := range ruleConfigMapNames {
promVolumeMounts = append(promVolumeMounts, v1.VolumeMount{
Name: name,
MountPath: rulesDir + "/" + name,
})
}
for _, s := range p.Spec.Secrets {
volumes = append(volumes, v1.Volume{
Name: "secret-" + s,
@ -473,10 +480,6 @@ func makeStatefulSetSpec(p monitoringv1.Prometheus, c *Config) (*appsv1.Stateful
Name: "config",
MountPath: confDir,
},
{
Name: "rules",
MountPath: "/etc/prometheus/rules",
},
{
Name: "config-out",
MountPath: confOutDir,
@ -558,6 +561,34 @@ func makeStatefulSetSpec(p monitoringv1.Prometheus, c *Config) (*appsv1.Stateful
additionalContainers := p.Spec.Containers
if len(ruleConfigMapNames) != 0 {
container := v1.Container{
Name: "rules-configmap-reloader",
Image: c.ConfigReloaderImage,
Args: []string{
fmt.Sprintf("--webhook-url=%s", localReloadURL),
},
VolumeMounts: []v1.VolumeMount{},
Resources: v1.ResourceRequirements{
Limits: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("5m"),
v1.ResourceMemory: resource.MustParse("10Mi"),
},
},
}
for _, name := range ruleConfigMapNames {
mountPath := rulesDir + "/" + name
container.VolumeMounts = append(container.VolumeMounts, v1.VolumeMount{
Name: name,
MountPath: mountPath,
})
container.Args = append(container.Args, fmt.Sprintf("--volume-dir=%s", mountPath))
}
additionalContainers = append(additionalContainers, container)
}
if p.Spec.Thanos != nil {
thanosBaseImage := c.ThanosDefaultBaseImage
if p.Spec.Thanos.BaseImage != nil {
@ -683,27 +714,6 @@ func makeStatefulSetSpec(p monitoringv1.Prometheus, c *Config) (*appsv1.Stateful
},
},
},
{
Name: "alerting-rule-files-configmap-reloader",
Image: c.ConfigReloaderImage,
Args: []string{
fmt.Sprintf("--webhook-url=%s", localReloadURL),
fmt.Sprintf("--volume-dir=%s", "/etc/prometheus/rules"),
},
VolumeMounts: []v1.VolumeMount{
{
Name: "rules",
ReadOnly: true,
MountPath: "/etc/prometheus/rules",
},
},
Resources: v1.ResourceRequirements{
Limits: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("5m"),
v1.ResourceMemory: resource.MustParse("10Mi"),
},
},
},
}, additionalContainers...),
SecurityContext: securityContext,
ServiceAccountName: p.Spec.ServiceAccountName,

37
pkg/prometheus/statefulset_test.go
View file

@ -48,7 +48,7 @@ func TestStatefulSetLabelingAndAnnotations(t *testing.T) {
Labels: labels,
Annotations: annotations,
},
}, "", defaultTestConfig, "")
}, "", defaultTestConfig, nil, "")
require.NoError(t, err)
@ -72,7 +72,7 @@ func TestPodLabelsAnnotations(t *testing.T) {
Labels: labels,
},
},
}, "", defaultTestConfig, "")
}, "", defaultTestConfig, nil, "")
require.NoError(t, err)
if _, ok := sset.Spec.Template.ObjectMeta.Labels["testlabel"]; !ok {
t.Fatal("Pod labes are not properly propagated")
@ -112,7 +112,7 @@ func TestStatefulSetPVC(t *testing.T) {
VolumeClaimTemplate: pvc,
},
},
}, "", defaultTestConfig, "")
}, "", defaultTestConfig, nil, "")
require.NoError(t, err)
ssetPvc := sset.Spec.VolumeClaimTemplates[0]
@ -144,7 +144,7 @@ func TestStatefulSetEmptyDir(t *testing.T) {
EmptyDir: &emptyDir,
},
},
}, "", defaultTestConfig, "")
}, "", defaultTestConfig, nil, "")
require.NoError(t, err)
ssetVolumes := sset.Spec.Template.Spec.Volumes
@ -166,17 +166,20 @@ func TestStatefulSetVolumeInitial(t *testing.T) {
ReadOnly: true,
MountPath: "/etc/prometheus/config_out",
SubPath: "",
}, {
Name: "rules",
ReadOnly: false,
MountPath: "/etc/prometheus/rules",
SubPath: "",
}, {
},
{
Name: "prometheus-volume-init-test-db",
ReadOnly: false,
MountPath: "/prometheus",
SubPath: "",
}, {
},
{
Name: "rules-configmap-one",
ReadOnly: false,
MountPath: "/etc/prometheus/rules/rules-configmap-one",
SubPath: "",
},
{
Name: "secret-test-secret1",
ReadOnly: true,
MountPath: "/etc/prometheus/secrets/test-secret1",
@ -201,11 +204,11 @@ func TestStatefulSetVolumeInitial(t *testing.T) {
},
},
{
Name: "rules",
Name: "rules-configmap-one",
VolumeSource: v1.VolumeSource{
ConfigMap: &v1.ConfigMapVolumeSource{
LocalObjectReference: v1.LocalObjectReference{
Name: "prometheus-volume-init-test-rulefiles",
Name: "rules-configmap-one",
},
},
},
@ -239,7 +242,7 @@ func TestStatefulSetVolumeInitial(t *testing.T) {
"test-secret1",
},
},
}, "", defaultTestConfig, "")
}, "", defaultTestConfig, []string{"rules-configmap-one"}, "")
require.NoError(t, err)
@ -265,7 +268,7 @@ func TestMemoryRequestNotAdjustedWhenLimitLarger2Gi(t *testing.T) {
},
},
},
}, "", defaultTestConfig, "")
}, "", defaultTestConfig, nil, "")
if err != nil {
t.Fatalf("Unexpected error while making StatefulSet: %v", err)
}
@ -292,7 +295,7 @@ func TestMemoryRequestAdjustedWhenOnlyLimitGiven(t *testing.T) {
},
},
},
}, "", defaultTestConfig, "")
}, "", defaultTestConfig, nil, "")
if err != nil {
t.Fatalf("Unexpected error while making StatefulSet: %v", err)
}
@ -314,7 +317,7 @@ func TestListenLocal(t *testing.T) {
Spec: monitoringv1.PrometheusSpec{
ListenLocal: true,
},
}, "", defaultTestConfig, "")
}, "", defaultTestConfig, nil, "")
if err != nil {
t.Fatalf("Unexpected error while making StatefulSet: %v", err)
}

171
test/e2e/prometheus_test.go
View file

@ -22,6 +22,7 @@ import (
"log"
"reflect"
"sort"
"strconv"
"strings"
"testing"
"time"
@ -333,6 +334,8 @@ func TestPrometheusAdditionalScrapeConfig(t *testing.T) {
}
func TestPrometheusAdditionalAlertManagerConfig(t *testing.T) {
t.Parallel()
ctx := framework.NewTestCtx(t)
defer ctx.Cleanup(t)
ns := ctx.CreateNamespace(t, framework.KubeClient)
@ -468,50 +471,6 @@ func TestPrometheusReloadRules(t *testing.T) {
}
}
// With Prometheus Operator v0.20.0 the 'RuleSelector' field in the Prometheus
// CRD Spec is deprecated. We need to ensure to still support it until the field
// is removed. Any value in 'RuleSelector' should just be copied to the new
// field 'RuleFileSelector'.
func TestPrometheusDeprecatedRuleSelectorField(t *testing.T) {
t.Parallel()
ctx := framework.NewTestCtx(t)
defer ctx.Cleanup(t)
ns := ctx.CreateNamespace(t, framework.KubeClient)
ctx.SetupPrometheusRBAC(t, ns, framework.KubeClient)
name := "test"
firtAlertName := "firstAlert"
_, err := framework.MakeAndCreateFiringRule(ns, name, firtAlertName)
if err != nil {
t.Fatal(err)
}
p := framework.MakeBasicPrometheus(ns, name, name, 1)
p.Spec.EvaluationInterval = "1s"
p.Spec.RuleSelector = &metav1.LabelSelector{
MatchLabels: map[string]string{
"role": "rulefile",
},
}
if err := framework.CreatePrometheusAndWaitUntilReady(ns, p); err != nil {
t.Fatal(err)
}
pSVC := framework.MakePrometheusService(p.Name, "not-relevant", v1.ServiceTypeClusterIP)
if finalizerFn, err := testFramework.CreateServiceAndWaitUntilReady(framework.KubeClient, ns, pSVC); err != nil {
t.Fatal(errors.Wrap(err, "creating Prometheus service failed"))
} else {
ctx.AddFinalizerFn(finalizerFn)
}
err = framework.WaitForPrometheusFiringAlert(p.Namespace, pSVC.Name, firtAlertName)
if err != nil {
t.Fatal(err)
}
}
func TestPrometheusRuleConfigMapMigration(t *testing.T) {
t.Parallel()
@ -541,7 +500,9 @@ groups:
`, alertName),
},
}
framework.KubeClient.CoreV1().ConfigMaps(ns).Create(&cm)
if _, err := framework.KubeClient.CoreV1().ConfigMaps(ns).Create(&cm); err != nil {
t.Fatalf("failed to create legacy rule config map: %v", err.Error())
}
p := framework.MakeBasicPrometheus(ns, name, name, 1)
p.Spec.RuleSelector = &metav1.LabelSelector{
@ -569,7 +530,7 @@ groups:
}
}
func TestPrometheusMultipleRuleFilesSameNS(t *testing.T) {
func TestPrometheusMultiplePrometheusRulesSameNS(t *testing.T) {
t.Parallel()
ctx := framework.NewTestCtx(t)
@ -608,7 +569,7 @@ func TestPrometheusMultipleRuleFilesSameNS(t *testing.T) {
}
}
func TestPrometheusMultipleRuleFilesDifferentNS(t *testing.T) {
func TestPrometheusMultiplePrometheusRulesDifferentNS(t *testing.T) {
t.Parallel()
ctx := framework.NewTestCtx(t)
@ -627,7 +588,10 @@ func TestPrometheusMultipleRuleFilesDifferentNS(t *testing.T) {
ruleFilesNamespaceSelector := map[string]string{"prometheus": rootNS}
for _, file := range ruleFiles {
testFramework.AddLabelsToNamespace(framework.KubeClient, file.ns, ruleFilesNamespaceSelector)
err := testFramework.AddLabelsToNamespace(framework.KubeClient, file.ns, ruleFilesNamespaceSelector)
if err != nil {
t.Fatal(err)
}
}
for _, file := range ruleFiles {
@ -661,6 +625,91 @@ func TestPrometheusMultipleRuleFilesDifferentNS(t *testing.T) {
}
}
func TestPrometheusRulesExceedingConfigMapLimit(t *testing.T) {
t.Parallel()
ctx := framework.NewTestCtx(t)
defer ctx.Cleanup(t)
ns := ctx.CreateNamespace(t, framework.KubeClient)
ctx.SetupPrometheusRBAC(t, ns, framework.KubeClient)
prometheusRules := []monitoringv1.PrometheusRule{}
for i := 0; i < 2; i++ {
rule := generateHugePrometheusRule(ns, strconv.Itoa(i))
err := framework.CreateRule(ns, rule)
if err != nil {
t.Fatal(err)
}
prometheusRules = append(prometheusRules, rule)
}
name := "test"
p := framework.MakeBasicPrometheus(ns, name, name, 1)
p.Spec.EvaluationInterval = "1s"
if err := framework.CreatePrometheusAndWaitUntilReady(ns, p); err != nil {
t.Fatal(err)
}
pSVC := framework.MakePrometheusService(p.Name, "not-relevant", v1.ServiceTypeClusterIP)
if finalizerFn, err := testFramework.CreateServiceAndWaitUntilReady(framework.KubeClient, ns, pSVC); err != nil {
t.Fatal(errors.Wrap(err, "creating Prometheus service failed"))
} else {
ctx.AddFinalizerFn(finalizerFn)
}
// Make sure both rule files ended up in the Prometheus Pod
for i := range prometheusRules {
err := framework.WaitForPrometheusFiringAlert(ns, pSVC.Name, "my-alert-"+strconv.Itoa(i))
if err != nil {
t.Fatal(err)
}
}
for i := range prometheusRules {
_, err := framework.WaitForConfigMapExist(ns, "prometheus-"+p.Name+"-rulefiles-"+strconv.Itoa(i))
if err != nil {
t.Fatal(err)
}
}
err := framework.DeleteRule(ns, prometheusRules[1].Name)
if err != nil {
t.Fatal(err)
}
_, err = framework.WaitForConfigMapExist(ns, "prometheus-"+p.Name+"-rulefiles-0")
if err != nil {
t.Fatal(err)
}
err = framework.WaitForConfigMapNotExist(ns, "prometheus-"+p.Name+"-rulefiles-1")
if err != nil {
t.Fatal(err)
}
}
// generateHugePrometheusRule returns a Prometheus rule instance that would fill
// more than half of the space of a Kubernetes config map.
func generateHugePrometheusRule(ns, identifier string) monitoringv1.PrometheusRule {
alertName := "my-alert"
groups := []monitoringv1.RuleGroup{
monitoringv1.RuleGroup{
Name: alertName,
Rules: []monitoringv1.Rule{},
},
}
// Approximating that each rule is 50 bytes and that the config map limit is 1024 * 1024 bytes
for i := 0; i < 15000; i++ {
groups[0].Rules = append(groups[0].Rules, monitoringv1.Rule{
Alert: alertName + "-" + identifier,
Expr: "vector(1)",
})
}
rule := framework.MakeBasicRule(ns, "prometheus-rule-"+identifier, groups)
return rule
}
// Make sure the Prometheus operator only updates the Prometheus config secret
// and the Prometheus rules configmap on relevant changes
func TestPrometheusOnlyUpdatedOnRelevantChanges(t *testing.T) {
@ -703,9 +752,12 @@ func TestPrometheusOnlyUpdatedOnRelevantChanges(t *testing.T) {
KubeClient.
CoreV1().
ConfigMaps(ns).
Get("prometheus-"+prometheusName+"-rulefiles", metav1.GetOptions{})
Get("prometheus-"+prometheusName+"-rulefiles-0", metav1.GetOptions{})
},
MaxExpectedChanges: 1,
// The Prometheus Operator first creates the config map for the
// given Prometheus stateful set and then updates it with the matching
// Prometheus rules.
MaxExpectedChanges: 2,
},
{
Name: "configurationSecret",
@ -773,10 +825,27 @@ func TestPrometheusOnlyUpdatedOnRelevantChanges(t *testing.T) {
}
}()
alertName := "my-alert"
if _, err := framework.MakeAndCreateFiringRule(ns, "my-prometheus-rule", alertName); err != nil {
t.Fatal(err)
}
if err := framework.CreatePrometheusAndWaitUntilReady(ns, prometheus); err != nil {
t.Fatal(err)
}
pSVC := framework.MakePrometheusService(prometheus.Name, "not-relevant", v1.ServiceTypeClusterIP)
if finalizerFn, err := testFramework.CreateServiceAndWaitUntilReady(framework.KubeClient, ns, pSVC); err != nil {
t.Fatal(errors.Wrap(err, "creating Prometheus service failed"))
} else {
testCTX.AddFinalizerFn(finalizerFn)
}
err := framework.WaitForPrometheusFiringAlert(prometheus.Namespace, pSVC.Name, alertName)
if err != nil {
t.Fatal(err)
}
if err := framework.DeletePrometheusAndWaitUntilGone(ns, name); err != nil {
t.Fatal(err)
}
@ -820,7 +889,7 @@ func TestPrometheusWhenDeleteCRDCleanUpViaOwnerReference(t *testing.T) {
t.Fatal(err)
}
configMapName := fmt.Sprintf("prometheus-%v-rulefiles", p.Name)
configMapName := fmt.Sprintf("prometheus-%v-rulefiles-0", p.Name)
_, err := framework.WaitForConfigMapExist(ns, configMapName)
if err != nil {

6
test/framework/config_map.go
View file

@ -21,6 +21,8 @@ import (
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"
"github.com/pkg/errors"
)
func (f *Framework) WaitForConfigMapExist(ns, name string) (*v1.ConfigMap, error) {
@ -42,7 +44,7 @@ func (f *Framework) WaitForConfigMapExist(ns, name string) (*v1.ConfigMap, error
return true, nil
})
return configMap, err
return configMap, errors.Wrapf(err, "waiting for config map '%v' in namespace '%v'", name, ns)
}
func (f *Framework) WaitForConfigMapNotExist(ns, name string) error {
@ -63,5 +65,5 @@ func (f *Framework) WaitForConfigMapNotExist(ns, name string) error {
return false, nil
})
return err
return errors.Wrapf(err, "waiting for config map '%v' in namespace '%v' to not exist", name, ns)
}

16
test/framework/prometheus.go
View file

@ -18,7 +18,6 @@ import (
"bytes"
"encoding/json"
"fmt"
"log"
"time"
"k8s.io/api/core/v1"
@ -215,19 +214,22 @@ func (f *Framework) UpdatePrometheusAndWaitUntilReady(ns string, p *monitoringv1
}
func (f *Framework) WaitForPrometheusReady(p *monitoringv1.Prometheus, timeout time.Duration) error {
var pollErr error
err := wait.Poll(2*time.Second, timeout, func() (bool, error) {
st, _, err := prometheus.PrometheusStatus(f.KubeClient, p)
if err != nil {
log.Print(err)
st, _, pollErr := prometheus.PrometheusStatus(f.KubeClient, p)
if pollErr != nil {
return false, nil
}
if st.UpdatedReplicas == *p.Spec.Replicas {
return true, nil
} else {
return false, nil
}
return false, nil
})
return errors.Wrapf(err, "waiting for Prometheus %v/%v", p.Namespace, p.Name)
return errors.Wrapf(pollErr, "waiting for Prometheus %v/%v: %v", p.Namespace, p.Name, err)
}
func (f *Framework) DeletePrometheusAndWaitUntilGone(ns, name string) error {

9
test/framework/rulefile.go → test/framework/prometheus_rule.go
View file

@ -92,3 +92,12 @@ func (f *Framework) UpdateRule(ns string, ar monitoringv1.PrometheusRule) error
return nil
}
func (f *Framework) DeleteRule(ns string, r string) error {
err := f.MonClientV1.PrometheusRules(ns).Delete(r, &metav1.DeleteOptions{})
if err != nil {
return fmt.Errorf("deleteing %v Prometheus rule in namespace %v failed: %v", r, ns, err.Error())
}
return nil
}