1
0
Fork 0
mirror of https://github.com/kyverno/kyverno.git synced 2025-03-13 19:28:55 +00:00

feat: add a circuit breaker for updaterequests (#10382)

* feat: add generator abstraction

Signed-off-by: ShutingZhao <shuting@nirmata.com>

* feat: replace urgenerator

Signed-off-by: ShutingZhao <shuting@nirmata.com>

* fix: ko build

Signed-off-by: ShutingZhao <shuting@nirmata.com>

* feat: load threshold from kyverno configmap

Signed-off-by: ShutingZhao <shuting@nirmata.com>

* feat: add metadata client to get ur count

Signed-off-by: ShutingZhao <shuting@nirmata.com>

* feat: add helm option to preserve configmap settings during upgrade

Signed-off-by: ShutingZhao <shuting@nirmata.com>

* feat: add helm option to preserve configmap settings during upgrade 2

Signed-off-by: ShutingZhao <shuting@nirmata.com>

* chore: rename imports

Signed-off-by: ShutingZhao <shuting@nirmata.com>

* chore: update codegen manifests

Signed-off-by: ShutingZhao <shuting@nirmata.com>

* fix: handle nil value

Signed-off-by: ShutingZhao <shuting@nirmata.com>

* fix: linter issue

Signed-off-by: ShutingZhao <shuting@nirmata.com>

* chore: update threshold to 1000

Signed-off-by: ShutingZhao <shuting@nirmata.com>

---------

Signed-off-by: ShutingZhao <shuting@nirmata.com>
This commit is contained in:
shuting 2024-06-11 16:54:51 +08:00 committed by GitHub
parent b9db2c176d
commit 9e5c297dcf
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
17 changed files with 335 additions and 21 deletions

View file

@ -34,19 +34,7 @@ annotations:
# valid kinds are: added, changed, deprecated, removed, fixed and security
artifacthub.io/changes: |
- kind: added
description: Add profiling support
- kind: added
description: Add global nodeSelector
- kind: added
description: Add podLabels to the post-upgrade hook
- kind: added
description: Add podLabels to the pre-delete hook
- kind: added
description: Add cronjob ttl support
- kind: fixed
description: Ensure CA certificate config maps are created when data is provided
- kind: added
description: Add global tolerations
description: Add a key to preserve configmap settings during upgrade
dependencies:
- name: grafana
version: v0.0.0

View file

@ -284,6 +284,7 @@ The chart values are organised per component.
| Key | Type | Default | Description |
|-----|------|---------|-------------|
| config.create | bool | `true` | Create the configmap. |
| config.preserve | bool | `true` | Preserve the configmap settings during upgrade. |
| config.name | string | `nil` | The configmap name (required if `create` is `false`). |
| config.annotations | object | `{}` | Additional annotations to add to the configmap. |
| config.enableDefaultRegistryMutation | bool | `true` | Enable registry mutation for container images. Enabled by default. |

View file

@ -6,10 +6,13 @@ metadata:
namespace: {{ template "kyverno.namespace" . }}
labels:
{{- include "kyverno.config.labels" . | nindent 4 }}
{{- with .Values.config.annotations }}
annotations:
{{- with .Values.annotations }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- end }}
{{- if .Values.config.preserve }}
helm.sh/resource-policy: "keep"
{{- end }}
data:
enableDefaultRegistryMutation: {{ .Values.config.enableDefaultRegistryMutation | quote }}
{{- with .Values.config.defaultRegistry }}

View file

@ -0,0 +1,129 @@
{{- if .Values.config.preserve -}}
{{- if not .Values.templating.enabled -}}
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: {{ template "kyverno.fullname" . }}:remove-configmap
namespace: {{ template "kyverno.namespace" . }}
labels:
{{- include "kyverno.hooks.labels" . | nindent 4 }}
annotations:
helm.sh/hook: pre-delete
helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded,hook-failed
helm.sh/hook-weight: "0"
rules:
- apiGroups:
- ""
resources:
- configmaps
verbs:
- list
- get
- delete
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: {{ template "kyverno.fullname" . }}:remove-configmap
namespace: {{ template "kyverno.namespace" . }}
labels:
{{- include "kyverno.hooks.labels" . | nindent 4 }}
annotations:
helm.sh/hook: pre-delete
helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded,hook-failed
helm.sh/hook-weight: "0"
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: {{ template "kyverno.fullname" . }}:remove-configmap
namespace: {{ template "kyverno.namespace" . }}
subjects:
- kind: ServiceAccount
name: {{ template "kyverno.fullname" . }}-remove-configmap
namespace: {{ template "kyverno.namespace" . }}
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: {{ template "kyverno.fullname" . }}-remove-configmap
namespace: {{ template "kyverno.namespace" . }}
labels:
{{- include "kyverno.hooks.labels" . | nindent 4 }}
annotations:
helm.sh/hook: pre-delete
helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
helm.sh/hook-weight: "0"
---
apiVersion: batch/v1
kind: Job
metadata:
name: {{ template "kyverno.fullname" . }}-remove-configmap
namespace: {{ template "kyverno.namespace" . }}
labels:
{{- include "kyverno.hooks.labels" . | nindent 4 }}
annotations:
helm.sh/hook: pre-delete
helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded,hook-failed
helm.sh/hook-weight: "10"
spec:
backoffLimit: 2
template:
metadata:
{{- with .Values.webhooksCleanup.podAnnotations }}
annotations:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.webhooksCleanup.podLabels }}
labels:
{{- toYaml . | nindent 8 }}
{{- end }}
spec:
serviceAccount: {{ template "kyverno.fullname" . }}-remove-configmap
{{- with .Values.webhooksCleanup.podSecurityContext }}
securityContext:
{{- tpl (toYaml .) $ | nindent 8 }}
{{- end }}
restartPolicy: Never
{{- with .Values.webhooksCleanup.imagePullSecrets }}
imagePullSecrets:
{{- tpl (toYaml .) $ | nindent 8 }}
{{- end }}
containers:
- name: kubectl
image: {{ (include "kyverno.image" (dict "globalRegistry" .Values.global.image.registry "image" .Values.webhooksCleanup.image "defaultTag" (default .Chart.AppVersion .Values.webhooksCleanup.image.tag))) | quote }}
imagePullPolicy: {{ .Values.webhooksCleanup.image.pullPolicy }}
command:
- /bin/bash
- '-c'
- |-
set -euo pipefail
kubectl delete cm -n {{ template "kyverno.namespace" . }} {{ template "kyverno.config.configMapName" . }}
{{- with .Values.webhooksCleanup.securityContext }}
securityContext:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.webhooksCleanup.tolerations }}
tolerations:
{{- tpl (toYaml .) $ | nindent 8 }}
{{- end }}
{{- with .Values.webhooksCleanup.nodeSelector | default .Values.global.nodeSelector }}
nodeSelector:
{{- tpl (toYaml .) $ | nindent 8 }}
{{- end }}
{{- if or .Values.webhooksCleanup.podAntiAffinity .Values.webhooksCleanup.podAffinity .Values.webhooksCleanup.nodeAffinity }}
affinity:
{{- with .Values.webhooksCleanup.podAntiAffinity }}
podAntiAffinity:
{{- tpl (toYaml .) $ | nindent 10 }}
{{- end }}
{{- with .Values.webhooksCleanup.podAffinity }}
podAffinity:
{{- tpl (toYaml .) $ | nindent 10 }}
{{- end }}
{{- with .Values.webhooksCleanup.nodeAffinity }}
nodeAffinity:
{{- tpl (toYaml .) $ | nindent 10 }}
{{- end }}
{{- end }}
{{- end -}}
{{- end -}}

View file

@ -10,6 +10,7 @@ metadata:
annotations:
helm.sh/hook: pre-delete
helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded,hook-failed
helm.sh/hook-weight: "100"
spec:
backoffLimit: 2
template:

View file

@ -177,6 +177,9 @@ config:
# -- Create the configmap.
create: true
# -- Preserve the configmap settings during upgrade.
preserve: true
# -- (string) The configmap name (required if `create` is `false`).
name: ~

View file

@ -26,6 +26,7 @@ import (
"github.com/kyverno/kyverno/pkg/logging"
"github.com/kyverno/kyverno/pkg/metrics"
"github.com/kyverno/kyverno/pkg/policy"
"github.com/kyverno/kyverno/pkg/utils/generator"
kubeutils "github.com/kyverno/kyverno/pkg/utils/kube"
apiserver "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
kubeinformers "k8s.io/client-go/informers"
@ -52,6 +53,7 @@ func createrLeaderControllers(
eventGenerator event.Interface,
jp jmespath.Interface,
backgroundScanInterval time.Duration,
urGenerator generator.UpdateRequestGenerator,
) ([]internal.Controller, error) {
policyCtrl, err := policy.NewPolicyController(
kyvernoClient,
@ -67,6 +69,7 @@ func createrLeaderControllers(
backgroundScanInterval,
metricsConfig,
jp,
urGenerator,
)
if err != nil {
return nil, err
@ -117,6 +120,7 @@ func main() {
internal.WithKyvernoDynamicClient(),
internal.WithEventsClient(),
internal.WithApiServerClient(),
internal.WithMetadataClient(),
internal.WithFlagSets(flagset),
)
// parse flags
@ -157,6 +161,7 @@ func main() {
eventGenerator,
event.Workers,
)
urGenerator := generator.NewUpdateRequestGenerator(setup.Configuration, setup.MetadataClient)
gcstore := store.New()
gceController := internal.NewController(
globalcontextcontroller.ControllerName,
@ -224,6 +229,7 @@ func main() {
eventGenerator,
setup.Jp,
bgscanInterval,
urGenerator,
)
if err != nil {
logger.Error(err, "failed to create leader controllers")

View file

@ -34,6 +34,7 @@ import (
"github.com/kyverno/kyverno/pkg/policycache"
"github.com/kyverno/kyverno/pkg/tls"
"github.com/kyverno/kyverno/pkg/toggle"
"github.com/kyverno/kyverno/pkg/utils/generator"
kubeutils "github.com/kyverno/kyverno/pkg/utils/kube"
runtimeutils "github.com/kyverno/kyverno/pkg/utils/runtime"
"github.com/kyverno/kyverno/pkg/validatingadmissionpolicy"
@ -290,6 +291,7 @@ func main() {
internal.WithKyvernoDynamicClient(),
internal.WithEventsClient(),
internal.WithApiServerClient(),
internal.WithMetadataClient(),
internal.WithFlagSets(flagset),
)
// parse flags
@ -501,10 +503,12 @@ func main() {
setup.Logger.Error(err, "failed to initialize leader election")
os.Exit(1)
}
urGenerator := generator.NewUpdateRequestGenerator(setup.Configuration, setup.MetadataClient)
// create webhooks server
urgen := webhookgenerate.NewGenerator(
setup.KyvernoClient,
kyvernoInformer.Kyverno().V1beta1().UpdateRequests(),
urGenerator,
)
policyHandlers := webhookspolicy.NewHandlers(
setup.KyvernoDynamicClient,

View file

@ -72,6 +72,8 @@ metadata:
app.kubernetes.io/instance: kyverno
app.kubernetes.io/part-of: kyverno
app.kubernetes.io/version: latest
annotations:
helm.sh/resource-policy: "keep"
data:
enableDefaultRegistryMutation: "true"
defaultRegistry: "docker.io"

View file

@ -96,8 +96,11 @@ const (
webhookAnnotations = "webhookAnnotations"
webhookLabels = "webhookLabels"
matchConditions = "matchConditions"
updateRequestThreshold = "updateRequestThreshold"
)
const UpdateRequestThreshold = 1000
var (
// kyvernoNamespace is the Kyverno namespace
kyvernoNamespace = osutils.GetEnvWithFallback("KYVERNO_NAMESPACE", "kyverno")
@ -177,6 +180,8 @@ type Configuration interface {
Load(*corev1.ConfigMap)
// OnChanged adds a callback to be invoked when the configuration is reloaded
OnChanged(func())
// GetUpdateRequestThreshold gets the threshold limit for the total number of updaterequests
GetUpdateRequestThreshold() int64
}
// configuration stores the configuration
@ -194,6 +199,7 @@ type configuration struct {
matchConditions []admissionregistrationv1.MatchCondition
mux sync.RWMutex
callbacks []func()
updateRequestThreshold int64
}
type match struct {
@ -322,6 +328,12 @@ func (cd *configuration) GetMatchConditions() []admissionregistrationv1.MatchCon
return cd.matchConditions
}
func (cd *configuration) GetUpdateRequestThreshold() int64 {
cd.mux.RLock()
defer cd.mux.RUnlock()
return cd.updateRequestThreshold
}
func (cd *configuration) Load(cm *corev1.ConfigMap) {
if cm != nil {
cd.load(cm)
@ -352,6 +364,7 @@ func (cd *configuration) load(cm *corev1.ConfigMap) {
cd.matchConditions = nil
// load filters
cd.filters = parseKinds(data[resourceFilters])
cd.updateRequestThreshold = UpdateRequestThreshold
logger.Info("filters configured", "filters", cd.filters)
// load defaultRegistry
defaultRegistry, ok := data[defaultRegistry]
@ -482,6 +495,19 @@ func (cd *configuration) load(cm *corev1.ConfigMap) {
logger.Info("matchConditions configured")
}
}
threshold, ok := data[updateRequestThreshold]
if !ok {
logger.Info("enableDefaultRegistryMutation not set")
} else {
logger := logger.WithValues("enableDefaultRegistryMutation", enableDefaultRegistryMutation)
urThreshold, err := strconv.ParseInt(threshold, 10, 64)
if err != nil {
logger.Error(err, "enableDefaultRegistryMutation is not a boolean")
} else {
cd.updateRequestThreshold = urThreshold
logger.Info("enableDefaultRegistryMutation configured")
}
}
}
func (cd *configuration) unload() {

View file

@ -112,11 +112,15 @@ func (pc *policyController) syncDataRulechanges(policy kyvernov1.PolicyInterface
labels := downstream.GetLabels()
trigger := generateutils.TriggerFromLabels(labels)
ur := newUR(policy, trigger, rule.Name, kyvernov1beta1.Generate, deleteDownstream)
created, err := pc.kyvernoClient.KyvernoV1beta1().UpdateRequests(config.KyvernoNamespace()).Create(context.TODO(), ur, metav1.CreateOptions{})
created, err := pc.urGenerator.Generate(context.TODO(), pc.kyvernoClient, ur, pc.log)
if err != nil {
errorList = append(errorList, err)
continue
}
if created == nil {
continue
}
updated := created.DeepCopy()
updated.Status = newURStatus(downstream)
_, err = pc.kyvernoClient.KyvernoV1beta1().UpdateRequests(config.KyvernoNamespace()).UpdateStatus(context.TODO(), updated, metav1.UpdateOptions{})

View file

@ -24,6 +24,7 @@ import (
"github.com/kyverno/kyverno/pkg/metrics"
datautils "github.com/kyverno/kyverno/pkg/utils/data"
engineutils "github.com/kyverno/kyverno/pkg/utils/engine"
"github.com/kyverno/kyverno/pkg/utils/generator"
kubeutils "github.com/kyverno/kyverno/pkg/utils/kube"
policyvalidation "github.com/kyverno/kyverno/pkg/validation/policy"
apierrors "k8s.io/apimachinery/pkg/api/errors"
@ -88,6 +89,8 @@ type policyController struct {
metricsConfig metrics.MetricsConfigManager
jp jmespath.Interface
urGenerator generator.UpdateRequestGenerator
}
// NewPolicyController create a new PolicyController
@ -105,6 +108,7 @@ func NewPolicyController(
reconcilePeriod time.Duration,
metricsConfig metrics.MetricsConfigManager,
jp jmespath.Interface,
urGenerator generator.UpdateRequestGenerator,
) (*policyController, error) {
// Event broad caster
eventInterface := client.GetEventsInterface()
@ -131,6 +135,7 @@ func NewPolicyController(
metricsConfig: metricsConfig,
log: log,
jp: jp,
urGenerator: urGenerator,
}
pc.pLister = pInformer.Lister()
@ -410,10 +415,13 @@ func (pc *policyController) handleUpdateRequest(ur *kyvernov1beta1.UpdateRequest
}
pc.log.V(2).Info("creating new UR for generate")
created, err := pc.kyvernoClient.KyvernoV1beta1().UpdateRequests(config.KyvernoNamespace()).Create(context.TODO(), ur, metav1.CreateOptions{})
created, err := pc.urGenerator.Generate(context.TODO(), pc.kyvernoClient, ur, pc.log)
if err != nil {
return false, err
}
if created == nil {
continue
}
updated := created.DeepCopy()
updated.Status.State = kyvernov1beta1.Pending
_, err = pc.kyvernoClient.KyvernoV1beta1().UpdateRequests(config.KyvernoNamespace()).UpdateStatus(context.TODO(), updated, metav1.UpdateOptions{})

View file

@ -0,0 +1,34 @@
package generator
import (
"context"
"github.com/go-logr/logr"
reportsv1 "github.com/kyverno/kyverno/api/reports/v1"
"github.com/kyverno/kyverno/pkg/client/clientset/versioned"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
type ClusterEphemeralReportGenerator = Generator[*reportsv1.ClusterEphemeralReport]
type clusterephemeralreportsgenerator struct {
// threshold config.Configuration
threshold int
count int
}
func NewClusterEphemeralReportGenerator() ClusterEphemeralReportGenerator {
return &clusterephemeralreportsgenerator{
threshold: 10,
count: 0,
}
}
func (g *clusterephemeralreportsgenerator) Generate(ctx context.Context, client versioned.Interface, resource *reportsv1.ClusterEphemeralReport, _ logr.Logger) (*reportsv1.ClusterEphemeralReport, error) {
if g.count >= g.threshold {
return nil, nil
}
report, err := client.ReportsV1().ClusterEphemeralReports().Create(ctx, resource, metav1.CreateOptions{})
return report, err
}

View file

@ -0,0 +1,34 @@
package generator
import (
"context"
"github.com/go-logr/logr"
reportsv1 "github.com/kyverno/kyverno/api/reports/v1"
"github.com/kyverno/kyverno/pkg/client/clientset/versioned"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
type EphemeralReportGenerator = Generator[*reportsv1.EphemeralReport]
type ephemeralreportsgenerator struct {
// threshold config.Configuration
threshold int
count int
}
func NewEphemeralReportGenerator() EphemeralReportGenerator {
return &ephemeralreportsgenerator{
threshold: 10,
count: 0,
}
}
func (g *ephemeralreportsgenerator) Generate(ctx context.Context, client versioned.Interface, resource *reportsv1.EphemeralReport, _ logr.Logger) (*reportsv1.EphemeralReport, error) {
if g.count >= g.threshold {
return nil, nil
}
report, err := client.ReportsV1().EphemeralReports(resource.GetNamespace()).Create(ctx, resource, metav1.CreateOptions{})
return report, err
}

View file

@ -0,0 +1,12 @@
package generator
import (
"context"
"github.com/go-logr/logr"
"github.com/kyverno/kyverno/pkg/client/clientset/versioned"
)
type Generator[T any] interface {
Generate(context.Context, versioned.Interface, T, logr.Logger) (T, error)
}

View file

@ -0,0 +1,53 @@
package generator
import (
"context"
"errors"
"github.com/go-logr/logr"
"github.com/kyverno/kyverno/api/kyverno/v1beta1"
"github.com/kyverno/kyverno/pkg/client/clientset/versioned"
configutils "github.com/kyverno/kyverno/pkg/config"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/metadata"
)
type UpdateRequestGenerator = Generator[*v1beta1.UpdateRequest]
type updaterequestsgenerator struct {
config configutils.Configuration
metaClient metadata.Interface
}
func NewUpdateRequestGenerator(config configutils.Configuration, metaClient metadata.Interface) UpdateRequestGenerator {
return &updaterequestsgenerator{
config: config,
metaClient: metaClient,
}
}
func (g *updaterequestsgenerator) Generate(ctx context.Context, client versioned.Interface, resource *v1beta1.UpdateRequest, log logr.Logger) (*v1beta1.UpdateRequest, error) {
objects, err := g.metaClient.Resource(
schema.GroupVersionResource{
Group: "kyverno.io",
Version: "v1beta1",
Resource: "updaterequests",
},
).List(ctx, metav1.ListOptions{})
if err != nil {
return nil, err
}
count := len(objects.Items)
threshold := g.config.GetUpdateRequestThreshold()
if int64(count) >= threshold {
log.Error(errors.New("UpdateRequest creation skipped"),
"the number of updaterequests exceeds the threshold, please adjust updateRequestThreshold in the Kyverno configmap",
"current count", count, "threshold", threshold)
return nil, nil
}
created, err := client.KyvernoV1beta1().UpdateRequests(configutils.KyvernoNamespace()).Create(ctx, resource, metav1.CreateOptions{})
return created, err
}

View file

@ -11,6 +11,7 @@ import (
kyvernov1beta1informers "github.com/kyverno/kyverno/pkg/client/informers/externalversions/kyverno/v1beta1"
kyvernov1beta1listers "github.com/kyverno/kyverno/pkg/client/listers/kyverno/v1beta1"
"github.com/kyverno/kyverno/pkg/config"
generatorutils "github.com/kyverno/kyverno/pkg/utils/generator"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
)
@ -27,13 +28,16 @@ type generator struct {
// listers
urLister kyvernov1beta1listers.UpdateRequestNamespaceLister
urGenerator generatorutils.UpdateRequestGenerator
}
// NewGenerator returns a new instance of UpdateRequest resource generator
func NewGenerator(client versioned.Interface, urInformer kyvernov1beta1informers.UpdateRequestInformer) Generator {
func NewGenerator(client versioned.Interface, urInformer kyvernov1beta1informers.UpdateRequestInformer, urGenerator generatorutils.UpdateRequestGenerator) Generator {
return &generator{
client: client,
urLister: urInformer.Lister().UpdateRequests(config.KyvernoNamespace()),
client: client,
urLister: urInformer.Lister().UpdateRequests(config.KyvernoNamespace()),
urGenerator: urGenerator,
}
}
@ -78,10 +82,12 @@ func (g *generator) tryApplyResource(ctx context.Context, urSpec kyvernov1beta1.
},
Spec: urSpec,
}
created, err := g.client.KyvernoV1beta1().UpdateRequests(config.KyvernoNamespace()).Create(ctx, &ur, metav1.CreateOptions{})
created, err := g.urGenerator.Generate(ctx, g.client, &ur, l)
if err != nil {
l.V(4).Error(err, "failed to create UpdateRequest, retrying", "name", ur.GetGenerateName(), "namespace", ur.GetNamespace())
return err
} else if created == nil {
return nil
}
updated := created.DeepCopy()
updated.Status.State = kyvernov1beta1.Pending