mirrors/kyverno
mirrors/kyverno
1
0
Fork 0
mirror of https://github.com/kyverno/kyverno.git synced 2025-03-13 19:28:55 +00:00
Code Activity

Dynamic webhooks (#2425)

Browse source 
* support k8s 1.22, update admissionregistration.k8s.io/v1beta1  to admissionregistration.k8s.io/v1

Signed-off-by: ShutingZhao <shutting06@gmail.com>

* - add failurePolicy to policy spec; - fix typo

Signed-off-by: ShutingZhao <shutting06@gmail.com>

* - add schema validation for failurePolicy; - add a printer column

Signed-off-by: ShutingZhao <shutting06@gmail.com>

* set default failure policy to fail if not defined

Signed-off-by: ShutingZhao <shutting06@gmail.com>

* resolve conflicts

Signed-off-by: ShutingZhao <shutting06@gmail.com>

* fix missing type for printerColumn

Signed-off-by: ShutingZhao <shutting06@gmail.com>

* refactor policy controller

Signed-off-by: ShutingZhao <shutting06@gmail.com>

* add webhook config manager

Signed-off-by: ShutingZhao <shutting06@gmail.com>

* - build webhook objects per policy update; - add fail webhook to default webhook configurations

Signed-off-by: ShutingZhao <shutting06@gmail.com>

* fix panic on policy update

Signed-off-by: ShutingZhao <shutting06@gmail.com>

* build default webhook: match empty if autoUpdateWebhooks is enabled, otherwise match all

Signed-off-by: ShutingZhao <shutting06@gmail.com>

* - set default webhook configs rule to empty; - handle policy deletion

Signed-off-by: ShutingZhao <shutting06@gmail.com>

* reset webhook config if policies with a specific failurePolicy are cleaned up

Signed-off-by: ShutingZhao <shutting06@gmail.com>

* handle wildcard pocliy

Signed-off-by: ShutingZhao <shutting06@gmail.com>

* update default webhook timeout to 10s

Signed-off-by: ShutingZhao <shutting06@gmail.com>

* cleanups

Signed-off-by: ShutingZhao <shutting06@gmail.com>

* added webhook informer to re-create it immediately if missing

Signed-off-by: ShutingZhao <shutting06@gmail.com>

* update tag webhookTimeoutSeconds description

Signed-off-by: ShutingZhao <shutting06@gmail.com>

* fix e2e tests

Signed-off-by: ShutingZhao <shutting06@gmail.com>

* fix linter issue

Signed-off-by: ShutingZhao <shutting06@gmail.com>

* correct metric endpoint

Signed-off-by: ShutingZhao <shutting06@gmail.com>

* add pol.generate.kind to webhooks

Signed-off-by: ShutingZhao <shutting06@gmail.com>
This commit is contained in:
shuting 2021-10-05 00:15:09 -07:00 • committed by GitHub
parent aba2e58f09
commit b10947b975
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
34 changed files with 1318 additions and 390 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
charts/kyverno/templates/crds.yaml
View file

@ -34,6 +34,7 @@ spec:
type: string
- jsonPath: .spec.failurePolicy
name: Failure Policy
priority: 1
type: string
- jsonPath: .status.ready
name: Ready
@ -1709,10 +1710,7 @@ spec:
in a policy report. Optional. The default value is "audit".
type: string
webhookTimeoutSeconds:
description: WebhookTimeoutSeconds specifies the webhook timeout for
this policy. After the timeout passes, the admission request will
fail based on the failure policy. The default timeout is 3s, the
value must be between 1 and 30 seconds. Default to 10 seconds.
description: WebhookTimeoutSeconds specifies the maximum time in seconds allowed to apply this policy. After the configured time expires, the admission request may fail, or may simply ignore the policy results, based on the failure policy. The default timeout is 10s, the value must be between 1 and 30 seconds.
format: int32
type: integer
type: object
@ -3339,6 +3337,7 @@ spec:
type: string
- jsonPath: .spec.failurePolicy
name: Failure Policy
priority: 1
type: string
- jsonPath: .status.ready
name: Ready
@ -5015,10 +5014,7 @@ spec:
in a policy report. Optional. The default value is "audit".
type: string
webhookTimeoutSeconds:
description: WebhookTimeoutSeconds specifies the webhook timeout for
this policy. After the timeout passes, the admission request will
fail based on the failure policy. The default timeout is 3s, the
value must be between 1 and 30 seconds. Default to 10 seconds.
description: WebhookTimeoutSeconds specifies the maximum time in seconds allowed to apply this policy. After the configured time expires, the admission request may fail, or may simply ignore the policy results, based on the failure policy. The default timeout is 10s, the value must be between 1 and 30 seconds.
format: int32
type: integer
type: object

2
charts/kyverno/values.yaml
View file

@ -32,7 +32,7 @@ testImage:
repository:
# testImage.tag defaults to "latest" if omitted
tag:
# testImage.pullPolicy defaults to image.pullPolicy if ommitted
# testImage.pullPolicy defaults to image.pullPolicy if omitted
pullPolicy:
replicaCount: 1

17
cmd/kyverno/main.go
View file

@ -10,8 +10,6 @@ import (
"strings"
"time"
"github.com/kyverno/kyverno/pkg/cosign"
"github.com/prometheus/client_golang/prometheus/promhttp"
kubeinformers "k8s.io/client-go/informers"
"k8s.io/klog/v2"
@ -23,6 +21,7 @@ import (
kyvernoinformer "github.com/kyverno/kyverno/pkg/client/informers/externalversions"
"github.com/kyverno/kyverno/pkg/common"
"github.com/kyverno/kyverno/pkg/config"
"github.com/kyverno/kyverno/pkg/cosign"
dclient "github.com/kyverno/kyverno/pkg/dclient"
event "github.com/kyverno/kyverno/pkg/event"
"github.com/kyverno/kyverno/pkg/generate"
@ -59,6 +58,7 @@ var (
genWorkers int
profile bool
disableMetricsExport bool
autoUpdateWebhooks bool
policyControllerResyncPeriod time.Duration
imagePullSecrets string
imageSignatureRepository string
@ -71,7 +71,8 @@ func main() {
flag.StringVar(&filterK8sResources, "filterK8sResources", "", "Resource in format [kind,namespace,name] where policy is not evaluated by the admission webhook. For example, --filterK8sResources \"[Deployment, kyverno, kyverno],[Events, *, *]\"")
flag.StringVar(&excludeGroupRole, "excludeGroupRole", "", "")
flag.StringVar(&excludeUsername, "excludeUsername", "", "")
flag.IntVar(&webhookTimeout, "webhooktimeout", 3, "Timeout for webhook configurations")
// deprecated
flag.IntVar(&webhookTimeout, "webhooktimeout", int(webhookconfig.DefaultWebhookTimeout), "Timeout for webhook configurations. Deprecated and will be removed in 1.6.0.")
flag.IntVar(&genWorkers, "gen-workers", 10, "Workers for generate controller")
flag.StringVar(&kubeconfig, "kubeconfig", "", "Path to a kubeconfig. Only required if out-of-cluster.")
flag.StringVar(&serverIP, "serverIP", "", "IP address where Kyverno controller runs. Only required if out-of-cluster.")
@ -82,6 +83,7 @@ func main() {
flag.DurationVar(&policyControllerResyncPeriod, "background-scan", time.Hour, "Perform background scan every given interval, e.g., 30s, 15m, 1h.")
flag.StringVar(&imagePullSecrets, "imagePullSecrets", "", "Secret resource names for image registry access credentials.")
flag.StringVar(&imageSignatureRepository, "imageSignatureRepository", "", "Alternate repository for image signatures. Can be overridden per rule via `verifyImages.Repository`.")
flag.BoolVar(&autoUpdateWebhooks, "auto-update-webhooks", true, "Set this flag to 'false' to disable auto-configuration of the webhook.")
if err := flag.Set("v", "2"); err != nil {
setupLog.Error(err, "failed to set log level")
@ -218,10 +220,15 @@ func main() {
webhookCfg := webhookconfig.NewRegister(
clientConfig,
client,
pclient,
rCache,
pInformer.Kyverno().V1().ClusterPolicies(),
pInformer.Kyverno().V1().Policies(),
serverIP,
int32(webhookTimeout),
debug,
autoUpdateWebhooks,
stopCh,
log.Log)
webhookMonitor, err := webhookconfig.NewMonitor(kubeClient, log.Log.WithName("WebhookMonitor"))
@ -381,7 +388,9 @@ func main() {
os.Exit(1)
}
go webhookCfg.UpdateWebhookConfigurations(configData)
if !autoUpdateWebhooks {
go webhookCfg.UpdateWebhookConfigurations(configData)
}
if registrationErr := registerWrapperRetry(); registrationErr != nil {
setupLog.Error(err, "Timeout registering admission control webhooks")
os.Exit(1)

10
definitions/crds/kyverno.io_clusterpolicies.yaml
View file

@ -27,6 +27,7 @@ spec:
type: string
- jsonPath: .spec.failurePolicy
name: Failure Policy
priority: 1
type: string
- jsonPath: .status.ready
name: Ready
@ -1709,10 +1710,11 @@ spec:
in a policy report. Optional. The default value is "audit".
type: string
webhookTimeoutSeconds:
description: WebhookTimeoutSeconds specifies the webhook timeout for
this policy. After the timeout passes, the admission request will
fail based on the failure policy. The default timeout is 3s, the
value must be between 1 and 30 seconds. Default to 10 seconds.
description: WebhookTimeoutSeconds specifies the maximum time in seconds
allowed to apply this policy. After the configured time expires,
the admission request may fail, or may simply ignore the policy
results, based on the failure policy. The default timeout is 10s,
the value must be between 1 and 30 seconds.
format: int32
type: integer
type: object

10
definitions/crds/kyverno.io_policies.yaml
View file

@ -27,6 +27,7 @@ spec:
type: string
- jsonPath: .spec.failurePolicy
name: Failure Policy
priority: 1
type: string
- jsonPath: .status.ready
name: Ready
@ -1710,10 +1711,11 @@ spec:
in a policy report. Optional. The default value is "audit".
type: string
webhookTimeoutSeconds:
description: WebhookTimeoutSeconds specifies the webhook timeout for
this policy. After the timeout passes, the admission request will
fail based on the failure policy. The default timeout is 3s, the
value must be between 1 and 30 seconds. Default to 10 seconds.
description: WebhookTimeoutSeconds specifies the maximum time in seconds
allowed to apply this policy. After the configured time expires,
the admission request may fail, or may simply ignore the policy
results, based on the failure policy. The default timeout is 10s,
the value must be between 1 and 30 seconds.
format: int32
type: integer
type: object

14
definitions/install.yaml
View file

@ -45,6 +45,7 @@ spec:
type: string
- jsonPath: .spec.failurePolicy
name: Failure Policy
priority: 1
type: string
- jsonPath: .status.ready
name: Ready
@ -1086,7 +1087,11 @@ spec:
description: ValidationFailureAction controls if a validation policy rule failure should disallow the admission review request (enforce), or allow (audit) the admission review request and report an error in a policy report. Optional. The default value is "audit".
type: string
webhookTimeoutSeconds:
description: WebhookTimeoutSeconds specifies the webhook timeout for this policy. After the timeout passes, the admission request will fail based on the failure policy. The default timeout is 3s, the value must be between 1 and 30 seconds. Default to 10 seconds.
description: WebhookTimeoutSeconds specifies the maximum time in seconds
allowed to apply this policy. After the configured time expires,
the admission request may fail, or may simply ignore the policy
results, based on the failure policy. The default timeout is 10s,
the value must be between 1 and 30 seconds.
format: int32
type: integer
type: object
@ -2312,6 +2317,7 @@ spec:
type: string
- jsonPath: .spec.failurePolicy
name: Failure Policy
priority: 1
type: string
- jsonPath: .status.ready
name: Ready
@ -3353,7 +3359,11 @@ spec:
description: ValidationFailureAction controls if a validation policy rule failure should disallow the admission review request (enforce), or allow (audit) the admission review request and report an error in a policy report. Optional. The default value is "audit".
type: string
webhookTimeoutSeconds:
description: WebhookTimeoutSeconds specifies the webhook timeout for this policy. After the timeout passes, the admission request will fail based on the failure policy. The default timeout is 3s, the value must be between 1 and 30 seconds. Default to 10 seconds.
description: WebhookTimeoutSeconds specifies the maximum time in seconds
allowed to apply this policy. After the configured time expires,
the admission request may fail, or may simply ignore the policy
results, based on the failure policy. The default timeout is 10s,
the value must be between 1 and 30 seconds.
format: int32
type: integer
type: object

14
definitions/install_debug.yaml
View file

@ -32,6 +32,7 @@ spec:
type: string
- jsonPath: .spec.failurePolicy
name: Failure Policy
priority: 1
type: string
- jsonPath: .status.ready
name: Ready
@ -1073,7 +1074,11 @@ spec:
description: ValidationFailureAction controls if a validation policy rule failure should disallow the admission review request (enforce), or allow (audit) the admission review request and report an error in a policy report. Optional. The default value is "audit".
type: string
webhookTimeoutSeconds:
description: WebhookTimeoutSeconds specifies the webhook timeout for this policy. After the timeout passes, the admission request will fail based on the failure policy. The default timeout is 3s, the value must be between 1 and 30 seconds. Default to 10 seconds.
description: WebhookTimeoutSeconds specifies the maximum time in seconds
allowed to apply this policy. After the configured time expires,
the admission request may fail, or may simply ignore the policy
results, based on the failure policy. The default timeout is 10s,
the value must be between 1 and 30 seconds.
format: int32
type: integer
type: object
@ -2271,6 +2276,7 @@ spec:
type: string
- jsonPath: .spec.failurePolicy
name: Failure Policy
priority: 1
type: string
- jsonPath: .status.ready
name: Ready
@ -3312,7 +3318,11 @@ spec:
description: ValidationFailureAction controls if a validation policy rule failure should disallow the admission review request (enforce), or allow (audit) the admission review request and report an error in a policy report. Optional. The default value is "audit".
type: string
webhookTimeoutSeconds:
description: WebhookTimeoutSeconds specifies the webhook timeout for this policy. After the timeout passes, the admission request will fail based on the failure policy. The default timeout is 3s, the value must be between 1 and 30 seconds. Default to 10 seconds.
description: WebhookTimeoutSeconds specifies the maximum time in seconds
allowed to apply this policy. After the configured time expires,
the admission request may fail, or may simply ignore the policy
results, based on the failure policy. The default timeout is 10s,
the value must be between 1 and 30 seconds.
format: int32
type: integer
type: object

2
pkg/api/kyverno/v1/clusterpolicy_types.go
View file

@ -13,7 +13,7 @@ import (
// +kubebuilder:resource:path=clusterpolicies,scope="Cluster",shortName=cpol
// +kubebuilder:printcolumn:name="Background",type="string",JSONPath=".spec.background"
// +kubebuilder:printcolumn:name="Action",type="string",JSONPath=".spec.validationFailureAction"
// +kubebuilder:printcolumn:name="Failure Policy",type="string",JSONPath=".spec.failurePolicy"
// +kubebuilder:printcolumn:name="Failure Policy",type="string",JSONPath=".spec.failurePolicy",priority=1
// +kubebuilder:printcolumn:name="Ready",type=string,JSONPath=`.status.ready`
type ClusterPolicy struct {
metav1.TypeMeta `json:",inline,omitempty" yaml:",inline,omitempty"`

9
pkg/api/kyverno/v1/policy_types.go
View file

@ -22,7 +22,7 @@ type PolicyList struct {
// +kubebuilder:subresource:status
// +kubebuilder:printcolumn:name="Background",type="string",JSONPath=".spec.background"
// +kubebuilder:printcolumn:name="Action",type="string",JSONPath=".spec.validationFailureAction"
// +kubebuilder:printcolumn:name="Failure Policy",type="string",JSONPath=".spec.failurePolicy"
// +kubebuilder:printcolumn:name="Failure Policy",type="string",JSONPath=".spec.failurePolicy",priority=1
// +kubebuilder:printcolumn:name="Ready",type=string,JSONPath=`.status.ready`
// +kubebuilder:resource:shortName=pol
type Policy struct {
@ -68,10 +68,9 @@ type Spec struct {
// +optional
SchemaValidation *bool `json:"schemaValidation,omitempty" yaml:"schemaValidation,omitempty"`
// WebhookTimeoutSeconds specifies the webhook timeout for this policy.
// After the timeout passes, the admission request will fail based on the failure policy.
// The default timeout is 3s, the value must be between 1 and 30 seconds.
// Default to 10 seconds.
// WebhookTimeoutSeconds specifies the maximum time in seconds allowed to apply this policy.
// After the configured time expires, the admission request may fail, or may simply ignore the policy results,
// based on the failure policy. The default timeout is 10s, the value must be between 1 and 30 seconds.
WebhookTimeoutSeconds *int32 `json:"webhookTimeoutSeconds,omitempty" yaml:"webhookTimeoutSeconds,omitempty"`
}

45
pkg/api/kyverno/v1/utils.go
View file

@ -38,6 +38,28 @@ func (p *ClusterPolicy) HasMutate() bool {
return false
}
// HasValidate checks for validate rule types
func (p *ClusterPolicy) HasValidate() bool {
for _, rule := range p.Spec.Rules {
if rule.HasValidate() {
return true
}
}
return false
}
// HasGenerate checks for generate rule types
func (p *ClusterPolicy) HasGenerate() bool {
for _, rule := range p.Spec.Rules {
if rule.HasGenerate() {
return true
}
}
return false
}
//HasVerifyImages checks for image verification rule types
func (p *ClusterPolicy) HasVerifyImages() bool {
for _, rule := range p.Spec.Rules {
@ -78,6 +100,29 @@ func (r Rule) HasGenerate() bool {
return !reflect.DeepEqual(r.Generation, Generation{})
}
func (r Rule) MatchKinds() []string {
matchKinds := r.MatchResources.ResourceDescription.Kinds
for _, value := range r.MatchResources.All {
matchKinds = append(matchKinds, value.ResourceDescription.Kinds...)
}
for _, value := range r.MatchResources.Any {
matchKinds = append(matchKinds, value.ResourceDescription.Kinds...)
}
return matchKinds
}
func (r Rule) ExcludeKinds() []string {
excludeKinds := r.ExcludeResources.ResourceDescription.Kinds
for _, value := range r.ExcludeResources.All {
excludeKinds = append(excludeKinds, value.ResourceDescription.Kinds...)
}
for _, value := range r.ExcludeResources.Any {
excludeKinds = append(excludeKinds, value.ResourceDescription.Kinds...)
}
return excludeKinds
}
// DeserializeAnyPattern deserialize apiextensions.JSON to []interface{}
func (in *Validation) DeserializeAnyPattern() ([]interface{}, error) {
if in.AnyPattern == nil {

5
pkg/config/dynamicconfig.go
View file

@ -144,8 +144,6 @@ func NewConfigData(rclient kubernetes.Interface, cmInformer informers.ConfigMapI
cd.restrictDevelopmentUsername = []string{"minikube-user", "kubernetes-admin"}
//TODO: this has been added to backward support command line arguments
// will be removed in future and the configuration will be set only via configmaps
if filterK8sResources != "" {
cd.log.Info("init configuration from commandline arguments for filterK8sResources")
cd.initFilters(filterK8sResources)
@ -320,8 +318,6 @@ func (cd *ConfigData) load(cm v1.ConfigMap) (reconcilePolicyReport, updateWebhoo
return
}
//TODO: this has been added to backward support command line arguments
// will be removed in future and the configuration will be set only via configmaps
func (cd *ConfigData) initFilters(filters string) {
logger := cd.log
// parse and load the configuration
@ -380,7 +376,6 @@ func parseKinds(list string) []k8Resource {
element = strings.Trim(element, "[")
element = strings.Trim(element, "]")
elements := strings.Split(element, ",")
//TODO: wildcards for namespace and name
if len(elements) == 0 {
continue
}

2
pkg/dclient/client.go
View file

@ -76,8 +76,6 @@ func (c *Client) NewDynamicSharedInformerFactory(defaultResync time.Duration) dy
}
//GetEventsInterface provides typed interface for events
//TODO: can we use dynamic client to fetch the typed interface
// or generate a kube client value to access the interface
func (c *Client) GetEventsInterface() (event.EventInterface, error) {
return c.kclient.CoreV1().Events(""), nil
}

3
pkg/engine/mutate/patches.go
View file

@ -52,7 +52,6 @@ func ProcessPatches(log logr.Logger, ruleName string, mutation kyverno.Mutation,
continue
}
patchResource, err := applyPatch(resourceRaw, patchRaw)
// TODO: continue on error if one of the patches fails, will add the failure event in such case
if err != nil && patch.Operation == "remove" {
log.Error(err, "failed to process JSON path or patch is a 'remove' operation")
continue
@ -79,7 +78,7 @@ func ProcessPatches(log logr.Logger, ruleName string, mutation kyverno.Mutation,
}
err = patchedResource.UnmarshalJSON(resourceRaw)
if err != nil {
logger.Error(err, "failed to unmmarshal resource")
logger.Error(err, "failed to unmarshal resource")
resp.Status = response.RuleStatusFail
resp.Message = fmt.Sprintf("failed to process JSON patches: %v", err)
return resp, resource

2
pkg/engine/mutation.go
View file

@ -116,7 +116,7 @@ func Mutate(policyContext *PolicyContext) (resp *response.EngineResponse) {
if *ruleCopy, err = variables.SubstituteAllInRule(logger, ctx, *ruleCopy); err != nil {
ruleResp := response.RuleResponse{
Name: ruleCopy.Name,
Type: utils.Validation.String(),
Type: utils.Mutation.String(),
Message: fmt.Sprintf("variable substitution failed: %s", err.Error()),
Status: response.RuleStatusPass,
}

3
pkg/engine/validate/pattern.go
View file

@ -42,10 +42,8 @@ func ValidateValueWithPattern(log logr.Logger, value, pattern interface{}) bool
case nil:
return validateValueWithNilPattern(log, value)
case map[string]interface{}:
// TODO: check if this is ever called?
return validateValueWithMapPattern(log, value, typedPattern)
case []interface{}:
// TODO: check if this is ever called?
log.Info("arrays are not supported as patterns")
return false
default:
@ -57,7 +55,6 @@ func ValidateValueWithPattern(log logr.Logger, value, pattern interface{}) bool
func validateValueWithMapPattern(log logr.Logger, value interface{}, typedPattern map[string]interface{}) bool {
// verify the type of the resource value is map[string]interface,
// we only check for existence of object, not the equality of content and value
//TODO: check if adding
_, ok := value.(map[string]interface{})
if !ok {
log.Info("Expected type map[string]interface{}", "type", fmt.Sprintf("%T", value), "value", value)

1
pkg/event/controller.go
View file

@ -188,7 +188,6 @@ func (gen *Generator) syncHandler(key Info) error {
var err error
switch key.Kind {
case "ClusterPolicy":
//TODO: policy is clustered resource so wont need namespace
robj, err = gen.cpLister.Get(key.Name)
if err != nil {
logger.Error(err, "failed to get cluster policy", "name", key.Name)

1
pkg/generate/generate_controller.go
View file

@ -64,7 +64,6 @@ type Controller struct {
// dynamic shared informer factory
dynamicInformer dynamicinformer.DynamicSharedInformerFactory
//TODO: list of generic informers
// only support Namespaces for re-evaluation on resource updates
nsInformer informers.GenericInformer
log logr.Logger

2
pkg/metrics/metrics.go
View file

@ -47,7 +47,7 @@ func NewPromConfig(metricsConfigData *config.MetricsConfigData, log logr.Logger)
)
policyRuleInfoLabels := []string{
"policy_validation_mode", "policy_type", "policy_background_mode", "policy_namespace", "policy_name", "rule_name", "rule_type",
"policy_validation_mode", "policy_type", "policy_background_mode", "policy_namespace", "policy_name", "rule_name", "rule_type", "status_ready",
}
policyRuleInfoMetric := prom.NewGaugeVec(
prom.GaugeOpts{

19
pkg/metrics/policyruleinfo/policyRuleInfo.go
View file

@ -15,6 +15,7 @@ func (pc PromConfig) registerPolicyRuleInfoMetric(
policyNamespace, policyName, ruleName string,
ruleType metrics.RuleType,
metricChangeType PolicyRuleInfoMetricChangeType,
ready bool,
) error {
var metricValue float64
switch metricChangeType {
@ -40,6 +41,11 @@ func (pc PromConfig) registerPolicyRuleInfoMetric(
policyNamespace = "-"
}
status := "false"
if ready {
status = "true"
}
pc.Metrics.PolicyRuleInfo.With(prom.Labels{
"policy_validation_mode": string(policyValidationMode),
"policy_type": string(policyType),
@ -48,6 +54,7 @@ func (pc PromConfig) registerPolicyRuleInfoMetric(
"policy_name": policyName,
"rule_name": ruleName,
"rule_type": string(ruleType),
"status_ready": status,
}).Set(metricValue)
return nil
@ -64,12 +71,13 @@ func (pc PromConfig) AddPolicy(policy interface{}) error {
policyType := metrics.Cluster
policyNamespace := "" // doesn't matter for cluster policy
policyName := inputPolicy.ObjectMeta.Name
ready := inputPolicy.Status.Ready
// registering the metrics on a per-rule basis
for _, rule := range inputPolicy.Spec.Rules {
ruleName := rule.Name
ruleType := metrics.ParseRuleType(rule)
if err = pc.registerPolicyRuleInfoMetric(policyValidationMode, policyType, policyBackgroundMode, policyNamespace, policyName, ruleName, ruleType, PolicyRuleCreated); err != nil {
if err = pc.registerPolicyRuleInfoMetric(policyValidationMode, policyType, policyBackgroundMode, policyNamespace, policyName, ruleName, ruleType, PolicyRuleCreated, ready); err != nil {
return err
}
}
@ -83,12 +91,13 @@ func (pc PromConfig) AddPolicy(policy interface{}) error {
policyType := metrics.Namespaced
policyNamespace := inputPolicy.ObjectMeta.Namespace
policyName := inputPolicy.ObjectMeta.Name
ready := inputPolicy.Status.Ready
// registering the metrics on a per-rule basis
for _, rule := range inputPolicy.Spec.Rules {
ruleName := rule.Name
ruleType := metrics.ParseRuleType(rule)
if err = pc.registerPolicyRuleInfoMetric(policyValidationMode, policyType, policyBackgroundMode, policyNamespace, policyName, ruleName, ruleType, PolicyRuleCreated); err != nil {
if err = pc.registerPolicyRuleInfoMetric(policyValidationMode, policyType, policyBackgroundMode, policyNamespace, policyName, ruleName, ruleType, PolicyRuleCreated, ready); err != nil {
return err
}
}
@ -112,8 +121,9 @@ func (pc PromConfig) RemovePolicy(policy interface{}) error {
policyName := inputPolicy.ObjectMeta.Name
ruleName := rule.Name
ruleType := metrics.ParseRuleType(rule)
ready := inputPolicy.Status.Ready
if err = pc.registerPolicyRuleInfoMetric(policyValidationMode, policyType, policyBackgroundMode, policyNamespace, policyName, ruleName, ruleType, PolicyRuleDeleted); err != nil {
if err = pc.registerPolicyRuleInfoMetric(policyValidationMode, policyType, policyBackgroundMode, policyNamespace, policyName, ruleName, ruleType, PolicyRuleDeleted, ready); err != nil {
return err
}
}
@ -130,8 +140,9 @@ func (pc PromConfig) RemovePolicy(policy interface{}) error {
policyName := inputPolicy.ObjectMeta.Name
ruleName := rule.Name
ruleType := metrics.ParseRuleType(rule)
ready := inputPolicy.Status.Ready
if err = pc.registerPolicyRuleInfoMetric(policyValidationMode, policyType, policyBackgroundMode, policyNamespace, policyName, ruleName, ruleType, PolicyRuleDeleted); err != nil {
if err = pc.registerPolicyRuleInfoMetric(policyValidationMode, policyType, policyBackgroundMode, policyNamespace, policyName, ruleName, ruleType, PolicyRuleDeleted, ready); err != nil {
return err
}
}

12
pkg/policy/existing.go
View file

@ -29,15 +29,8 @@ func (pc *PolicyController) processExistingResources(policy *kyverno.ClusterPoli
continue
}
match := rule.MatchResources
for _, value := range match.Any {
pc.processExistingKinds(value.ResourceDescription.Kinds, policy, rule, logger)
}
for _, value := range match.All {
pc.processExistingKinds(value.ResourceDescription.Kinds, policy, rule, logger)
}
pc.processExistingKinds(match.Kinds, policy, rule, logger)
matchKinds := rule.MatchKinds()
pc.processExistingKinds(matchKinds, policy, rule, logger)
}
}
@ -161,7 +154,6 @@ type resourceManager interface {
}
//Drop drop the cache after every rebuild interval mins
//TODO: or drop based on the size
func (rm *ResourceManager) Drop() {
timeSince := time.Since(rm.time)
if timeSince > time.Duration(rm.rebuildTime)*time.Second {

126
pkg/policy/metrics.go Normal file
View file

@ -0,0 +1,126 @@
package policy
import (
"reflect"
"github.com/go-logr/logr"
kyverno "github.com/kyverno/kyverno/pkg/api/kyverno/v1"
policyChangesMetric "github.com/kyverno/kyverno/pkg/metrics/policychanges"
policyRuleInfoMetric "github.com/kyverno/kyverno/pkg/metrics/policyruleinfo"
)
func (pc *PolicyController) registerPolicyRuleInfoMetricAddPolicy(logger logr.Logger, p *kyverno.ClusterPolicy) {
err := policyRuleInfoMetric.ParsePromConfig(*pc.promConfig).AddPolicy(p)
if err != nil {
logger.Error(err, "error occurred while registering kyverno_policy_rule_info_total metrics for the above policy's creation", "name", p.Name)
}
}
func (pc *PolicyController) registerPolicyRuleInfoMetricUpdatePolicy(logger logr.Logger, oldP, curP *kyverno.ClusterPolicy) {
// removing the old rules associated metrics
err := policyRuleInfoMetric.ParsePromConfig(*pc.promConfig).RemovePolicy(oldP)
if err != nil {
logger.Error(err, "error occurred while registering kyverno_policy_rule_info_total metrics for the above policy's updation", "name", oldP.Name)
}
// adding the new rules associated metrics
err = policyRuleInfoMetric.ParsePromConfig(*pc.promConfig).AddPolicy(curP)
if err != nil {
logger.Error(err, "error occurred while registering kyverno_policy_rule_info_total metrics for the above policy's updation", "name", oldP.Name)
}
}
func (pc *PolicyController) registerPolicyRuleInfoMetricDeletePolicy(logger logr.Logger, p *kyverno.ClusterPolicy) {
err := policyRuleInfoMetric.ParsePromConfig(*pc.promConfig).RemovePolicy(p)
if err != nil {
logger.Error(err, "error occurred while registering kyverno_policy_rule_info_total metrics for the above policy's deletion", "name", p.Name)
}
}
func (pc *PolicyController) registerPolicyChangesMetricAddPolicy(logger logr.Logger, p *kyverno.ClusterPolicy) {
err := policyChangesMetric.ParsePromConfig(*pc.promConfig).RegisterPolicy(p, policyChangesMetric.PolicyCreated)
if err != nil {
logger.Error(err, "error occurred while registering kyverno_policy_changes_total metrics for the above policy's creation", "name", p.Name)
}
}
func (pc *PolicyController) registerPolicyChangesMetricUpdatePolicy(logger logr.Logger, oldP, curP *kyverno.ClusterPolicy) {
if reflect.DeepEqual((*oldP).Spec, (*curP).Spec) {
return
}
err := policyChangesMetric.ParsePromConfig(*pc.promConfig).RegisterPolicy(oldP, policyChangesMetric.PolicyUpdated)
if err != nil {
logger.Error(err, "error occurred while registering kyverno_policy_changes_total metrics for the above policy's updation", "name", oldP.Name)
}
// curP will require a new kyverno_policy_changes_total metric if the above update involved change in the following fields:
if curP.Spec.Background != oldP.Spec.Background || curP.Spec.ValidationFailureAction != oldP.Spec.ValidationFailureAction {
err = policyChangesMetric.ParsePromConfig(*pc.promConfig).RegisterPolicy(curP, policyChangesMetric.PolicyUpdated)
if err != nil {
logger.Error(err, "error occurred while registering kyverno_policy_changes_total metrics for the above policy's updation", "name", curP.Name)
}
}
}
func (pc *PolicyController) registerPolicyChangesMetricDeletePolicy(logger logr.Logger, p *kyverno.ClusterPolicy) {
err := policyChangesMetric.ParsePromConfig(*pc.promConfig).RegisterPolicy(p, policyChangesMetric.PolicyDeleted)
if err != nil {
logger.Error(err, "error occurred while registering kyverno_policy_changes_total metrics for the above policy's deletion", "name", p.Name)
}
}
func (pc *PolicyController) registerPolicyRuleInfoMetricDeleteNsPolicy(logger logr.Logger, p *kyverno.Policy) {
err := policyRuleInfoMetric.ParsePromConfig(*pc.promConfig).RemovePolicy(p)
if err != nil {
logger.Error(err, "error occurred while registering kyverno_policy_rule_info_total metrics for the above policy's deletion", "name", p.Name)
}
}
func (pc *PolicyController) registerPolicyChangesMetricAddNsPolicy(logger logr.Logger, p *kyverno.Policy) {
err := policyChangesMetric.ParsePromConfig(*pc.promConfig).RegisterPolicy(p, policyChangesMetric.PolicyCreated)
if err != nil {
logger.Error(err, "error occurred while registering kyverno_policy_changes_total metrics for the above policy's creation", "name", p.Name)
}
}
func (pc *PolicyController) registerPolicyChangesMetricUpdateNsPolicy(logger logr.Logger, oldP, curP *kyverno.Policy) {
if reflect.DeepEqual((*oldP).Spec, (*curP).Spec) {
return
}
err := policyChangesMetric.ParsePromConfig(*pc.promConfig).RegisterPolicy(oldP, policyChangesMetric.PolicyUpdated)
if err != nil {
logger.Error(err, "error occurred while registering kyverno_policy_changes_total metrics for the above policy's updation", "name", oldP.Name)
}
// curP will require a new kyverno_policy_changes_total metric if the above update involved change in the following fields:
if curP.Spec.Background != oldP.Spec.Background || curP.Spec.ValidationFailureAction != oldP.Spec.ValidationFailureAction {
err = policyChangesMetric.ParsePromConfig(*pc.promConfig).RegisterPolicy(curP, policyChangesMetric.PolicyUpdated)
if err != nil {
logger.Error(err, "error occurred while registering kyverno_policy_changes_total metrics for the above policy's updation", "name", curP.Name)
}
}
}
func (pc *PolicyController) registerPolicyChangesMetricDeleteNsPolicy(logger logr.Logger, p *kyverno.Policy) {
err := policyChangesMetric.ParsePromConfig(*pc.promConfig).RegisterPolicy(p, policyChangesMetric.PolicyDeleted)
if err != nil {
logger.Error(err, "error occurred while registering kyverno_policy_changes_total metrics for the above policy's deletion", "name", p.Name)
}
}
func (pc *PolicyController) registerPolicyRuleInfoMetricAddNsPolicy(logger logr.Logger, p *kyverno.Policy) {
err := policyRuleInfoMetric.ParsePromConfig(*pc.promConfig).AddPolicy(p)
if err != nil {
logger.Error(err, "error occurred while registering kyverno_policy_rule_info_total metrics for the above policy's creation", "name", p.Name)
}
}
func (pc *PolicyController) registerPolicyRuleInfoMetricUpdateNsPolicy(logger logr.Logger, oldP, curP *kyverno.Policy) {
// removing the old rules associated metrics
err := policyRuleInfoMetric.ParsePromConfig(*pc.promConfig).RemovePolicy(oldP)
if err != nil {
logger.Error(err, "error occurred while registering kyverno_policy_rule_info_total metrics for the above policy's updation", "name", oldP.Name)
}
// adding the new rules associated metrics
err = policyRuleInfoMetric.ParsePromConfig(*pc.promConfig).AddPolicy(curP)
if err != nil {
logger.Error(err, "error occurred while registering kyverno_policy_rule_info_total metrics for the above policy's updation", "name", oldP.Name)
}
}

120
pkg/policy/validate_controller.go → pkg/policy/policy_controller.go
View file

@ -21,7 +21,6 @@ import (
"github.com/kyverno/kyverno/pkg/event"
"github.com/kyverno/kyverno/pkg/kyverno/common"
"github.com/kyverno/kyverno/pkg/metrics"
policyRuleInfoMetric "github.com/kyverno/kyverno/pkg/metrics/policyruleinfo"
pm "github.com/kyverno/kyverno/pkg/policymutation"
"github.com/kyverno/kyverno/pkg/policyreport"
"github.com/kyverno/kyverno/pkg/resourcecache"
@ -40,8 +39,6 @@ import (
"k8s.io/client-go/tools/cache"
"k8s.io/client-go/tools/record"
"k8s.io/client-go/util/workqueue"
policyChangesMetric "github.com/kyverno/kyverno/pkg/metrics/policychanges"
)
const (
@ -176,7 +173,6 @@ func NewPolicyController(
// resource manager
// rebuild after 300 seconds/ 5 mins
//TODO: pass the time in seconds instead of converting it internally
pc.rm = NewResourceManager(30)
return &pc, nil
@ -197,64 +193,6 @@ func (pc *PolicyController) canBackgroundProcess(p *kyverno.ClusterPolicy) bool
return true
}
func (pc *PolicyController) registerPolicyRuleInfoMetricAddPolicy(logger logr.Logger, p *kyverno.ClusterPolicy) {
err := policyRuleInfoMetric.ParsePromConfig(*pc.promConfig).AddPolicy(p)
if err != nil {
logger.Error(err, "error occurred while registering kyverno_policy_rule_info_total metrics for the above policy's creation", "name", p.Name)
}
}
func (pc *PolicyController) registerPolicyRuleInfoMetricUpdatePolicy(logger logr.Logger, oldP, curP *kyverno.ClusterPolicy) {
// removing the old rules associated metrics
err := policyRuleInfoMetric.ParsePromConfig(*pc.promConfig).RemovePolicy(oldP)
if err != nil {
logger.Error(err, "error occurred while registering kyverno_policy_rule_info_total metrics for the above policy's updation", "name", oldP.Name)
}
// adding the new rules associated metrics
err = policyRuleInfoMetric.ParsePromConfig(*pc.promConfig).AddPolicy(curP)
if err != nil {
logger.Error(err, "error occurred while registering kyverno_policy_rule_info_total metrics for the above policy's updation", "name", oldP.Name)
}
}
func (pc *PolicyController) registerPolicyRuleInfoMetricDeletePolicy(logger logr.Logger, p *kyverno.ClusterPolicy) {
err := policyRuleInfoMetric.ParsePromConfig(*pc.promConfig).RemovePolicy(p)
if err != nil {
logger.Error(err, "error occurred while registering kyverno_policy_rule_info_total metrics for the above policy's deletion", "name", p.Name)
}
}
func (pc *PolicyController) registerPolicyChangesMetricAddPolicy(logger logr.Logger, p *kyverno.ClusterPolicy) {
err := policyChangesMetric.ParsePromConfig(*pc.promConfig).RegisterPolicy(p, policyChangesMetric.PolicyCreated)
if err != nil {
logger.Error(err, "error occurred while registering kyverno_policy_changes_total metrics for the above policy's creation", "name", p.Name)
}
}
func (pc *PolicyController) registerPolicyChangesMetricUpdatePolicy(logger logr.Logger, oldP, curP *kyverno.ClusterPolicy) {
if reflect.DeepEqual((*oldP).Spec, (*curP).Spec) {
return
}
err := policyChangesMetric.ParsePromConfig(*pc.promConfig).RegisterPolicy(oldP, policyChangesMetric.PolicyUpdated)
if err != nil {
logger.Error(err, "error occurred while registering kyverno_policy_changes_total metrics for the above policy's updation", "name", oldP.Name)
}
// curP will require a new kyverno_policy_changes_total metric if the above update involved change in the following fields:
if curP.Spec.Background != oldP.Spec.Background || curP.Spec.ValidationFailureAction != oldP.Spec.ValidationFailureAction {
err = policyChangesMetric.ParsePromConfig(*pc.promConfig).RegisterPolicy(curP, policyChangesMetric.PolicyUpdated)
if err != nil {
logger.Error(err, "error occurred while registering kyverno_policy_changes_total metrics for the above policy's updation", "name", curP.Name)
}
}
}
func (pc *PolicyController) registerPolicyChangesMetricDeletePolicy(logger logr.Logger, p *kyverno.ClusterPolicy) {
err := policyChangesMetric.ParsePromConfig(*pc.promConfig).RegisterPolicy(p, policyChangesMetric.PolicyDeleted)
if err != nil {
logger.Error(err, "error occurred while registering kyverno_policy_changes_total metrics for the above policy's deletion", "name", p.Name)
}
}
func (pc *PolicyController) addPolicy(obj interface{}) {
logger := pc.log
p := obj.(*kyverno.ClusterPolicy)
@ -353,64 +291,6 @@ func (pc *PolicyController) deletePolicy(obj interface{}) {
}
}
func (pc *PolicyController) registerPolicyRuleInfoMetricAddNsPolicy(logger logr.Logger, p *kyverno.Policy) {
err := policyRuleInfoMetric.ParsePromConfig(*pc.promConfig).AddPolicy(p)
if err != nil {
logger.Error(err, "error occurred while registering kyverno_policy_rule_info_total metrics for the above policy's creation", "name", p.Name)
}
}
func (pc *PolicyController) registerPolicyRuleInfoMetricUpdateNsPolicy(logger logr.Logger, oldP, curP *kyverno.Policy) {
// removing the old rules associated metrics
err := policyRuleInfoMetric.ParsePromConfig(*pc.promConfig).RemovePolicy(oldP)
if err != nil {
logger.Error(err, "error occurred while registering kyverno_policy_rule_info_total metrics for the above policy's updation", "name", oldP.Name)
}
// adding the new rules associated metrics
err = policyRuleInfoMetric.ParsePromConfig(*pc.promConfig).AddPolicy(curP)
if err != nil {
logger.Error(err, "error occurred while registering kyverno_policy_rule_info_total metrics for the above policy's updation", "name", oldP.Name)
}
}
func (pc *PolicyController) registerPolicyRuleInfoMetricDeleteNsPolicy(logger logr.Logger, p *kyverno.Policy) {
err := policyRuleInfoMetric.ParsePromConfig(*pc.promConfig).RemovePolicy(p)
if err != nil {
logger.Error(err, "error occurred while registering kyverno_policy_rule_info_total metrics for the above policy's deletion", "name", p.Name)
}
}
func (pc *PolicyController) registerPolicyChangesMetricAddNsPolicy(logger logr.Logger, p *kyverno.Policy) {
err := policyChangesMetric.ParsePromConfig(*pc.promConfig).RegisterPolicy(p, policyChangesMetric.PolicyCreated)
if err != nil {
logger.Error(err, "error occurred while registering kyverno_policy_changes_total metrics for the above policy's creation", "name", p.Name)
}
}
func (pc *PolicyController) registerPolicyChangesMetricUpdateNsPolicy(logger logr.Logger, oldP, curP *kyverno.Policy) {
if reflect.DeepEqual((*oldP).Spec, (*curP).Spec) {
return
}
err := policyChangesMetric.ParsePromConfig(*pc.promConfig).RegisterPolicy(oldP, policyChangesMetric.PolicyUpdated)
if err != nil {
logger.Error(err, "error occurred while registering kyverno_policy_changes_total metrics for the above policy's updation", "name", oldP.Name)
}
// curP will require a new kyverno_policy_changes_total metric if the above update involved change in the following fields:
if curP.Spec.Background != oldP.Spec.Background || curP.Spec.ValidationFailureAction != oldP.Spec.ValidationFailureAction {
err = policyChangesMetric.ParsePromConfig(*pc.promConfig).RegisterPolicy(curP, policyChangesMetric.PolicyUpdated)
if err != nil {
logger.Error(err, "error occurred while registering kyverno_policy_changes_total metrics for the above policy's updation", "name", curP.Name)
}
}
}
func (pc *PolicyController) registerPolicyChangesMetricDeleteNsPolicy(logger logr.Logger, p *kyverno.Policy) {
err := policyChangesMetric.ParsePromConfig(*pc.promConfig).RegisterPolicy(p, policyChangesMetric.PolicyDeleted)
if err != nil {
logger.Error(err, "error occurred while registering kyverno_policy_changes_total metrics for the above policy's deletion", "name", p.Name)
}
}
func (pc *PolicyController) addNsPolicy(obj interface{}) {
logger := pc.log
p := obj.(*kyverno.Policy)

50
pkg/policymutation/policymutation.go
View file

@ -38,6 +38,11 @@ func GenerateJSONPatchesForDefaults(policy *kyverno.ClusterPolicy, log logr.Logg
updateMsgs = append(updateMsgs, updateMsg)
}
if patch, updateMsg := defaultFailurePolicy(policy, log); patch != nil {
patches = append(patches, patch)
updateMsgs = append(updateMsgs, updateMsg)
}
patch, errs := GeneratePodControllerRule(*policy, log)
if len(errs) > 0 {
var errMsgs []string
@ -307,6 +312,33 @@ func defaultvalidationFailureAction(policy *kyverno.ClusterPolicy, log logr.Logg
return nil, ""
}
func defaultFailurePolicy(policy *kyverno.ClusterPolicy, log logr.Logger) ([]byte, string) {
// set failurePolicy to Fail if not present
failurePolicy := string(kyverno.Fail)
if policy.Spec.FailurePolicy == nil {
log.V(4).Info("setting default value", "spec.failurePolicy", failurePolicy)
jsonPatch := struct {
Path string `json:"path"`
Op string `json:"op"`
Value string `json:"value"`
}{
"/spec/failurePolicy",
"add",
string(kyverno.Fail),
}
patchByte, err := json.Marshal(jsonPatch)
if err != nil {
log.Error(err, "failed to set default value", "spec.failurePolicy", failurePolicy)
return nil, ""
}
log.V(3).Info("generated JSON Patch to set default", "spec.failurePolicy", failurePolicy)
return patchByte, fmt.Sprintf("default failurePolicy to '%s'", failurePolicy)
}
return nil, ""
}
// podControllersKey annotation could be:
// scenario A: not exist, set default to "all", which generates on all pod controllers
@ -534,20 +566,10 @@ func generateRuleForControllers(rule kyverno.Rule, controllers string, log logr.
match := rule.MatchResources
exclude := rule.ExcludeResources
matchResourceDescriptionsKinds := match.ResourceDescription.Kinds
for _, value := range match.All {
matchResourceDescriptionsKinds = append(matchResourceDescriptionsKinds, value.ResourceDescription.Kinds...)
}
for _, value := range match.Any {
matchResourceDescriptionsKinds = append(matchResourceDescriptionsKinds, value.ResourceDescription.Kinds...)
}
excludeResourceDescriptionsKinds := exclude.ResourceDescription.Kinds
for _, value := range exclude.All {
excludeResourceDescriptionsKinds = append(excludeResourceDescriptionsKinds, value.ResourceDescription.Kinds...)
}
for _, value := range exclude.Any {
excludeResourceDescriptionsKinds = append(excludeResourceDescriptionsKinds, value.ResourceDescription.Kinds...)
}
matchResourceDescriptionsKinds := rule.MatchKinds()
excludeResourceDescriptionsKinds := rule.ExcludeKinds()
if !utils.ContainsString(matchResourceDescriptionsKinds, "Pod") ||
(len(excludeResourceDescriptionsKinds) != 0 && !utils.ContainsString(excludeResourceDescriptionsKinds, "Pod")) {
return kyvernoRule{}

8
pkg/signal/signal_windows.go
View file

@ -1,8 +0,0 @@
package signal
//TODO: how to pick files based on OS compilation ?
// import (
// "os"
// )
// var shutdownSignals = []os.Signal{os.Interrupt}

1
pkg/testrunner/scenario.go
View file

@ -421,7 +421,6 @@ func loadObjects(t *testing.T, path string) []k8sRuntime.Object {
continue
}
t.Log(gvk)
//TODO: add more details
t.Logf("loaded object %s", gvk.Kind)
resources = append(resources, obj)
}

15
pkg/webhookconfig/certmanager.go
View file

@ -9,7 +9,6 @@ import (
"github.com/go-logr/logr"
"github.com/kyverno/kyverno/pkg/common"
"github.com/kyverno/kyverno/pkg/config"
"github.com/kyverno/kyverno/pkg/tls"
ktls "github.com/kyverno/kyverno/pkg/tls"
v1 "k8s.io/api/core/v1"
informerv1 "k8s.io/client-go/informers/core/v1"
@ -29,14 +28,14 @@ type Interface interface {
GetTLSPemPair() (*ktls.PemPair, error)
}
type certManager struct {
renewer *tls.CertRenewer
renewer *ktls.CertRenewer
secretInformer informerv1.SecretInformer
secretQueue chan bool
stopCh <-chan struct{}
log logr.Logger
}
func NewCertManager(secretInformer informerv1.SecretInformer, kubeClient kubernetes.Interface, certRenewer *tls.CertRenewer, log logr.Logger, stopCh <-chan struct{}) (Interface, error) {
func NewCertManager(secretInformer informerv1.SecretInformer, kubeClient kubernetes.Interface, certRenewer *ktls.CertRenewer, log logr.Logger, stopCh <-chan struct{}) (Interface, error) {
manager := &certManager{
renewer: certRenewer,
secretInformer: secretInformer,
@ -59,7 +58,7 @@ func (m *certManager) addSecretFunc(obj interface{}) {
return
}
val, ok := secret.GetAnnotations()[tls.SelfSignedAnnotation]
val, ok := secret.GetAnnotations()[ktls.SelfSignedAnnotation]
if !ok || val != "true" {
return
}
@ -74,7 +73,7 @@ func (m *certManager) updateSecretFunc(oldObj interface{}, newObj interface{}) {
return
}
val, ok := new.GetAnnotations()[tls.SelfSignedAnnotation]
val, ok := new.GetAnnotations()[ktls.SelfSignedAnnotation]
if !ok || val != "true" {
return
}
@ -127,7 +126,7 @@ func (m *certManager) Run(stopCh <-chan struct{}) {
})
m.log.Info("start managing certificate")
certsRenewalTicker := time.NewTicker(tls.CertRenewalInterval)
certsRenewalTicker := time.NewTicker(ktls.CertRenewalInterval)
defer certsRenewalTicker.Stop()
for {
@ -137,7 +136,7 @@ func (m *certManager) Run(stopCh <-chan struct{}) {
if err != nil {
m.log.Error(err, "failed to validate cert")
if !strings.Contains(err.Error(), tls.ErrorsNotFound) {
if !strings.Contains(err.Error(), ktls.ErrorsNotFound) {
continue
}
}
@ -157,7 +156,7 @@ func (m *certManager) Run(stopCh <-chan struct{}) {
if err != nil {
m.log.Error(err, "failed to validate cert")
if !strings.Contains(err.Error(), tls.ErrorsNotFound) {
if !strings.Contains(err.Error(), ktls.ErrorsNotFound) {
continue
}
}

130
pkg/webhookconfig/common.go
View file

@ -2,10 +2,11 @@ package webhookconfig
import (
"io/ioutil"
"reflect"
"github.com/kyverno/kyverno/pkg/config"
"github.com/kyverno/kyverno/pkg/tls"
admregapi "k8s.io/api/admissionregistration/v1beta1"
admregapi "k8s.io/api/admissionregistration/v1"
apps "k8s.io/api/apps/v1"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@ -95,76 +96,66 @@ func (wrc *Register) GetKubePolicyDeployment() (*apps.Deployment, *unstructured.
}
// debug mutating webhook
func generateDebugMutatingWebhook(name, url string, caData []byte, validate bool, timeoutSeconds int32, resources []string, apiGroups, apiVersions string, operationTypes []admregapi.OperationType) admregapi.MutatingWebhook {
func generateDebugMutatingWebhook(name, url string, caData []byte, validate bool, timeoutSeconds int32, rule admregapi.Rule, operationTypes []admregapi.OperationType, failurePolicy admregapi.FailurePolicyType) admregapi.MutatingWebhook {
sideEffect := admregapi.SideEffectClassNoneOnDryRun
failurePolicy := admregapi.Ignore
reinvocationPolicy := admregapi.NeverReinvocationPolicy
return admregapi.MutatingWebhook{
w := admregapi.MutatingWebhook{
ReinvocationPolicy: &reinvocationPolicy,
Name: name,
ClientConfig: admregapi.WebhookClientConfig{
URL: &url,
CABundle: caData,
},
SideEffects: &sideEffect,
Rules: []admregapi.RuleWithOperations{
{
Operations: operationTypes,
Rule: admregapi.Rule{
APIGroups: []string{
apiGroups,
},
APIVersions: []string{
apiVersions,
},
Resources: resources,
},
},
},
SideEffects: &sideEffect,
AdmissionReviewVersions: []string{"v1beta1"},
TimeoutSeconds: &timeoutSeconds,
FailurePolicy: &failurePolicy,
}
if !reflect.DeepEqual(rule, admregapi.Rule{}) {
w.Rules = []admregapi.RuleWithOperations{
{
Operations: operationTypes,
Rule: rule,
},
}
}
return w
}
func generateDebugValidatingWebhook(name, url string, caData []byte, validate bool, timeoutSeconds int32, resources []string, apiGroups, apiVersions string, operationTypes []admregapi.OperationType) admregapi.ValidatingWebhook {
func generateDebugValidatingWebhook(name, url string, caData []byte, validate bool, timeoutSeconds int32, rule admregapi.Rule, operationTypes []admregapi.OperationType, failurePolicy admregapi.FailurePolicyType) admregapi.ValidatingWebhook {
sideEffect := admregapi.SideEffectClassNoneOnDryRun
failurePolicy := admregapi.Ignore
return admregapi.ValidatingWebhook{
w := admregapi.ValidatingWebhook{
Name: name,
ClientConfig: admregapi.WebhookClientConfig{
URL: &url,
CABundle: caData,
},
SideEffects: &sideEffect,
Rules: []admregapi.RuleWithOperations{
{
Operations: operationTypes,
Rule: admregapi.Rule{
APIGroups: []string{
apiGroups,
},
APIVersions: []string{
apiVersions,
},
Resources: resources,
},
},
},
SideEffects: &sideEffect,
AdmissionReviewVersions: []string{"v1beta1"},
TimeoutSeconds: &timeoutSeconds,
FailurePolicy: &failurePolicy,
}
if !reflect.DeepEqual(rule, admregapi.Rule{}) {
w.Rules = []admregapi.RuleWithOperations{
{
Operations: operationTypes,
Rule: rule,
},
}
}
return w
}
// mutating webhook
func generateMutatingWebhook(name, servicePath string, caData []byte, validation bool, timeoutSeconds int32, resources []string, apiGroups, apiVersions string, operationTypes []admregapi.OperationType) admregapi.MutatingWebhook {
func generateMutatingWebhook(name, servicePath string, caData []byte, validation bool, timeoutSeconds int32, rule admregapi.Rule, operationTypes []admregapi.OperationType, failurePolicy admregapi.FailurePolicyType) admregapi.MutatingWebhook {
sideEffect := admregapi.SideEffectClassNoneOnDryRun
failurePolicy := admregapi.Ignore
reinvocationPolicy := admregapi.NeverReinvocationPolicy
reinvocationPolicy := admregapi.IfNeededReinvocationPolicy
return admregapi.MutatingWebhook{
w := admregapi.MutatingWebhook{
ReinvocationPolicy: &reinvocationPolicy,
Name: name,
ClientConfig: admregapi.WebhookClientConfig{
@ -175,32 +166,27 @@ func generateMutatingWebhook(name, servicePath string, caData []byte, validation
},
CABundle: caData,
},
SideEffects: &sideEffect,
Rules: []admregapi.RuleWithOperations{
{
Operations: operationTypes,
Rule: admregapi.Rule{
APIGroups: []string{
apiGroups,
},
APIVersions: []string{
apiVersions,
},
Resources: resources,
},
},
},
SideEffects: &sideEffect,
AdmissionReviewVersions: []string{"v1beta1"},
TimeoutSeconds: &timeoutSeconds,
FailurePolicy: &failurePolicy,
}
if !reflect.DeepEqual(rule, admregapi.Rule{}) {
w.Rules = []admregapi.RuleWithOperations{
{
Operations: operationTypes,
Rule: rule,
},
}
}
return w
}
// validating webhook
func generateValidatingWebhook(name, servicePath string, caData []byte, validation bool, timeoutSeconds int32, resources []string, apiGroups, apiVersions string, operationTypes []admregapi.OperationType) admregapi.ValidatingWebhook {
func generateValidatingWebhook(name, servicePath string, caData []byte, validation bool, timeoutSeconds int32, rule admregapi.Rule, operationTypes []admregapi.OperationType, failurePolicy admregapi.FailurePolicyType) admregapi.ValidatingWebhook {
sideEffect := admregapi.SideEffectClassNoneOnDryRun
failurePolicy := admregapi.Ignore
return admregapi.ValidatingWebhook{
w := admregapi.ValidatingWebhook{
Name: name,
ClientConfig: admregapi.WebhookClientConfig{
Service: &admregapi.ServiceReference{
@ -210,23 +196,19 @@ func generateValidatingWebhook(name, servicePath string, caData []byte, validati
},
CABundle: caData,
},
SideEffects: &sideEffect,
Rules: []admregapi.RuleWithOperations{
{
Operations: operationTypes,
Rule: admregapi.Rule{
APIGroups: []string{
apiGroups,
},
APIVersions: []string{
apiVersions,
},
Resources: resources,
},
},
},
SideEffects: &sideEffect,
AdmissionReviewVersions: []string{"v1beta1"},
TimeoutSeconds: &timeoutSeconds,
FailurePolicy: &failurePolicy,
}
if !reflect.DeepEqual(rule, admregapi.Rule{}) {
w.Rules = []admregapi.RuleWithOperations{
{
Operations: operationTypes,
Rule: rule,
},
}
}
return w
}

755
pkg/webhookconfig/configmanager.go Normal file
View file

@ -0,0 +1,755 @@
package webhookconfig
import (
"context"
"fmt"
"reflect"
"strings"
"sync/atomic"
"time"
"github.com/go-logr/logr"
kyverno "github.com/kyverno/kyverno/pkg/api/kyverno/v1"
kyvernoclient "github.com/kyverno/kyverno/pkg/client/clientset/versioned"
kyvernoinformer "github.com/kyverno/kyverno/pkg/client/informers/externalversions/kyverno/v1"
kyvernolister "github.com/kyverno/kyverno/pkg/client/listers/kyverno/v1"
"github.com/kyverno/kyverno/pkg/common"
"github.com/kyverno/kyverno/pkg/config"
client "github.com/kyverno/kyverno/pkg/dclient"
"github.com/kyverno/kyverno/pkg/resourcecache"
"github.com/kyverno/kyverno/pkg/utils"
"github.com/pkg/errors"
admregapi "k8s.io/api/admissionregistration/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime/schema"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/client-go/tools/cache"
"k8s.io/client-go/util/workqueue"
)
var DefaultWebhookTimeout int64 = 10
// webhookConfigManager manges the webhook configuration dynamically
// it is NOT multi-thread safe
type webhookConfigManager struct {
client *client.Client
kyvernoClient *kyvernoclient.Clientset
pInformer kyvernoinformer.ClusterPolicyInformer
npInformer kyvernoinformer.PolicyInformer
// pLister can list/get policy from the shared informer's store
pLister kyvernolister.ClusterPolicyLister
// npLister can list/get namespace policy from the shared informer's store
npLister kyvernolister.PolicyLister
// pListerSynced returns true if the cluster policy store has been synced at least once
pListerSynced cache.InformerSynced
// npListerSynced returns true if the namespace policy store has been synced at least once
npListerSynced cache.InformerSynced
resCache resourcecache.ResourceCache
mutateInformer cache.SharedIndexInformer
validateInformer cache.SharedIndexInformer
mutateInformerSynced cache.InformerSynced
validateInformerSynced cache.InformerSynced
queue workqueue.RateLimitingInterface
// wildcardPolicy indicates the number of policies that matches all kinds (*) defined
wildcardPolicy int64
createDefaultWebhook chan<- string
stopCh <-chan struct{}
log logr.Logger
}
type manage interface {
start()
}
func newWebhookConfigManager(
client *client.Client,
kyvernoClient *kyvernoclient.Clientset,
pInformer kyvernoinformer.ClusterPolicyInformer,
npInformer kyvernoinformer.PolicyInformer,
resCache resourcecache.ResourceCache,
createDefaultWebhook chan<- string,
stopCh <-chan struct{},
log logr.Logger) manage {
m := &webhookConfigManager{
client: client,
kyvernoClient: kyvernoClient,
pInformer: pInformer,
npInformer: npInformer,
resCache: resCache,
queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "configmanager"),
wildcardPolicy: 0,
createDefaultWebhook: createDefaultWebhook,
stopCh: stopCh,
log: log,
}
m.pLister = pInformer.Lister()
m.npLister = npInformer.Lister()
m.pListerSynced = pInformer.Informer().HasSynced
m.npListerSynced = npInformer.Informer().HasSynced
mutateCache, _ := m.resCache.GetGVRCache(kindMutating)
m.mutateInformer = mutateCache.GetInformer()
m.mutateInformerSynced = mutateCache.GetInformer().HasSynced
validateCache, _ := m.resCache.GetGVRCache(kindValidating)
m.validateInformer = validateCache.GetInformer()
m.validateInformerSynced = validateCache.GetInformer().HasSynced
return m
}
func (m *webhookConfigManager) handleErr(err error, key interface{}) {
logger := m.log
if err == nil {
m.queue.Forget(key)
return
}
if m.queue.NumRequeues(key) < 3 {
logger.Error(err, "failed to sync policy", "key", key)
m.queue.AddRateLimited(key)
return
}
utilruntime.HandleError(err)
logger.V(2).Info("dropping policy out of queue", "key", key)
m.queue.Forget(key)
}
func (m *webhookConfigManager) addClusterPolicy(obj interface{}) {
p := obj.(*kyverno.ClusterPolicy)
if hasWildcard(p) {
atomic.AddInt64(&m.wildcardPolicy, int64(1))
}
m.enqueue(p)
}
func (m *webhookConfigManager) updateClusterPolicy(old, cur interface{}) {
oldP := old.(*kyverno.ClusterPolicy)
curP := cur.(*kyverno.ClusterPolicy)
if reflect.DeepEqual(oldP.Spec, curP.Spec) {
return
}
if hasWildcard(oldP) && !hasWildcard(curP) {
atomic.AddInt64(&m.wildcardPolicy, ^int64(0))
} else if !hasWildcard(oldP) && hasWildcard(curP) {
atomic.AddInt64(&m.wildcardPolicy, int64(1))
}
m.enqueue(curP)
}
func (m *webhookConfigManager) deleteClusterPolicy(obj interface{}) {
p, ok := obj.(*kyverno.ClusterPolicy)
if !ok {
tombstone, ok := obj.(cache.DeletedFinalStateUnknown)
if !ok {
utilruntime.HandleError(fmt.Errorf("error decoding object, invalid type"))
return
}
p, ok = tombstone.Obj.(*kyverno.ClusterPolicy)
if !ok {
utilruntime.HandleError(fmt.Errorf("error decoding object tombstone, invalid type"))
return
}
m.log.V(4).Info("Recovered deleted ClusterPolicy '%s' from tombstone", "name", p.GetName())
}
if hasWildcard(p) {
atomic.AddInt64(&m.wildcardPolicy, ^int64(0))
}
m.enqueue(p)
}
func (m *webhookConfigManager) addPolicy(obj interface{}) {
p := obj.(*kyverno.Policy)
if hasWildcard(p) {
atomic.AddInt64(&m.wildcardPolicy, int64(1))
}
pol := kyverno.ClusterPolicy(*p)
m.enqueue(&pol)
}
func (m *webhookConfigManager) updatePolicy(old, cur interface{}) {
oldP := old.(*kyverno.Policy)
curP := cur.(*kyverno.Policy)
if reflect.DeepEqual(oldP.Spec, curP.Spec) {
return
}
if hasWildcard(oldP) && !hasWildcard(curP) {
atomic.AddInt64(&m.wildcardPolicy, ^int64(0))
} else if !hasWildcard(oldP) && hasWildcard(curP) {
atomic.AddInt64(&m.wildcardPolicy, int64(1))
}
pol := kyverno.ClusterPolicy(*curP)
m.enqueue(&pol)
}
func (m *webhookConfigManager) deletePolicy(obj interface{}) {
p, ok := obj.(*kyverno.Policy)
if !ok {
tombstone, ok := obj.(cache.DeletedFinalStateUnknown)
if !ok {
utilruntime.HandleError(fmt.Errorf("error decoding object, invalid type"))
return
}
p, ok = tombstone.Obj.(*kyverno.Policy)
if !ok {
utilruntime.HandleError(fmt.Errorf("error decoding object tombstone, invalid type"))
return
}
m.log.V(4).Info("Recovered deleted ClusterPolicy '%s' from tombstone", "name", p.GetName())
}
if hasWildcard(p) {
atomic.AddInt64(&m.wildcardPolicy, ^int64(0))
}
pol := kyverno.ClusterPolicy(*p)
m.enqueue(&pol)
}
func (m *webhookConfigManager) deleteWebhook(obj interface{}) {
m.log.WithName("deleteWebhook").Info("resource webhook configuration was deleted, recreating...")
if webhook, ok := obj.(*unstructured.Unstructured); ok {
k := webhook.GetKind()
if (k == kindMutating && webhook.GetName() == config.MutatingWebhookConfigurationName) ||
(k == kindValidating && webhook.GetName() == config.ValidatingWebhookConfigurationName) {
m.enqueueAllPolicies()
}
}
}
func (m *webhookConfigManager) enqueueAllPolicies() {
logger := m.log.WithName("enqueueAllPolicies")
cpols, err := m.listPolicies("")
if err != nil {
logger.Error(err, "unabled to list clusterpolicies")
}
for _, cpol := range cpols {
m.enqueue(cpol)
logger.V(4).Info("added CLusterPolicy to the queue", "name", cpol.GetName())
}
nsCache, ok := m.resCache.GetGVRCache("Namespace")
if !ok {
nsCache, err = m.resCache.CreateGVKInformer("Namespace")
if err != nil {
logger.Error(err, "unabled to create Namespace listser")
return
}
}
namespaces, err := nsCache.Lister().List(labels.Everything())
if err != nil {
logger.Error(err, "unabled to list namespaces")
return
}
for _, ns := range namespaces {
pols, err := m.listPolicies(ns.GetName())
if err != nil {
logger.Error(err, "unabled to list policies", "namespace", ns.GetName())
}
for _, p := range pols {
m.enqueue(p)
logger.V(4).Info("added Policy to the queue", "namespace", p.GetName(), "name", p.GetName())
}
}
}
func (m *webhookConfigManager) enqueue(policy *kyverno.ClusterPolicy) {
logger := m.log
key, err := cache.MetaNamespaceKeyFunc(policy)
if err != nil {
logger.Error(err, "failed to enqueue policy")
return
}
m.queue.Add(key)
}
// start is a blocking call to configure webhook
func (m *webhookConfigManager) start() {
defer utilruntime.HandleCrash()
defer m.queue.ShutDown()
m.log.Info("starting")
defer m.log.Info("shutting down")
if !cache.WaitForCacheSync(m.stopCh, m.pListerSynced, m.npListerSynced, m.mutateInformerSynced, m.validateInformerSynced) {
m.log.Info("failed to sync informer cache")
return
}
m.pInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
AddFunc: m.addClusterPolicy,
UpdateFunc: m.updateClusterPolicy,
DeleteFunc: m.deleteClusterPolicy,
})
m.npInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
AddFunc: m.addPolicy,
UpdateFunc: m.updatePolicy,
DeleteFunc: m.deletePolicy,
})
m.mutateInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{
DeleteFunc: m.deleteWebhook,
})
m.validateInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{
DeleteFunc: m.deleteWebhook,
})
for m.processNextWorkItem() {
}
}
func (m *webhookConfigManager) processNextWorkItem() bool {
key, quit := m.queue.Get()
if quit {
return false
}
defer m.queue.Done(key)
err := m.sync(key.(string))
m.handleErr(err, key)
return true
}
func (m *webhookConfigManager) sync(key string) error {
logger := m.log.WithName("sync")
startTime := time.Now()
logger.V(4).Info("started syncing policy", "key", key, "startTime", startTime)
defer func() {
logger.V(4).Info("finished syncing policy", "key", key, "processingTime", time.Since(startTime).String())
}()
namespace, name, err := cache.SplitMetaNamespaceKey(key)
if err != nil {
logger.Info("invalid resource key", "key", key)
return nil
}
return m.reconcileWebhook(namespace, name)
}
func (m *webhookConfigManager) reconcileWebhook(namespace, name string) error {
logger := m.log.WithName("reconcileWebhook").WithValues("namespace", namespace, "policy", name)
policy, err := m.getPolicy(namespace, name)
if err != nil && !apierrors.IsNotFound(err) {
return errors.Wrapf(err, "unable to get policy object %s/%s", namespace, name)
}
webhooks, err := m.buildWebhooks(namespace)
if err != nil {
return err
}
if err := m.updateWebhookConfig(webhooks); err != nil {
return errors.Wrapf(err, "failed to update webhook configurations for policy %s/%s", namespace, name)
}
// DELETION of the policy
if policy == nil {
return nil
}
if err := m.updateStatus(policy); err != nil {
return errors.Wrapf(err, "failed to update policy status %s/%s", namespace, name)
}
logger.Info("policy is ready to serve admission requests")
return nil
}
func (m *webhookConfigManager) getPolicy(namespace, name string) (*kyverno.ClusterPolicy, error) {
// TODO: test default/policy
if namespace == "" {
return m.pLister.Get(name)
}
nsPolicy, err := m.npLister.Policies(namespace).Get(name)
if err == nil && nsPolicy != nil {
p := kyverno.ClusterPolicy(*nsPolicy)
return &p, err
}
return nil, err
}
func (m *webhookConfigManager) listPolicies(namespace string) ([]*kyverno.ClusterPolicy, error) {
if namespace != "" {
polList, err := m.npLister.Policies(namespace).List(labels.Everything())
if err != nil {
return nil, errors.Wrapf(err, "failed to list Policy")
}
policies := make([]*kyverno.ClusterPolicy, len(polList))
for i, pol := range polList {
p := kyverno.ClusterPolicy(*pol)
policies[i] = &p
}
return policies, nil
}
cpolList, err := m.pLister.List(labels.Everything())
if err != nil {
return nil, errors.Wrapf(err, "failed to list ClusterPolicy")
}
return cpolList, nil
}
const (
apiGroups string = "apiGroups"
apiVersions string = "apiVersions"
resources string = "resources"
)
// webhook is the instance that aggregates the GVK of existing policies
// based on kind, failurePolicy and webhookTimeout
type webhook struct {
kind string
maxWebhookTimeout int64
failurePolicy kyverno.FailurePolicyType
// rule represents the same rule struct of the webhook using a map object
// https://github.com/kubernetes/api/blob/master/admissionregistration/v1/types.go#L25
rule map[string]interface{}
}
func (m *webhookConfigManager) buildWebhooks(namespace string) (res []*webhook, err error) {
mutateIgnore := newWebhook(kindMutating, DefaultWebhookTimeout, kyverno.Ignore)
mutateFail := newWebhook(kindMutating, DefaultWebhookTimeout, kyverno.Fail)
validateIgnore := newWebhook(kindValidating, DefaultWebhookTimeout, kyverno.Ignore)
validateFail := newWebhook(kindValidating, DefaultWebhookTimeout, kyverno.Fail)
if atomic.LoadInt64(&m.wildcardPolicy) != 0 {
for _, w := range []*webhook{mutateIgnore, mutateFail, validateIgnore, validateFail} {
setWildcardConfig(w)
}
m.log.V(4).WithName("buildWebhooks").Info("warning: found wildcard policy, setting webhook configurations to accept admission requests of all kinds")
return append(res, mutateIgnore, mutateFail, validateIgnore, validateFail), nil
}
policies, err := m.listPolicies(namespace)
if err != nil {
return nil, errors.Wrap(err, "unable to list current policies")
}
for _, p := range policies {
if p.HasValidate() || p.HasGenerate() {
if p.Spec.FailurePolicy != nil && *p.Spec.FailurePolicy == kyverno.Ignore {
m.mergeWebhook(validateIgnore, p)
} else {
m.mergeWebhook(validateFail, p)
}
}
if p.HasMutate() || p.HasGenerate() {
if p.Spec.FailurePolicy != nil && *p.Spec.FailurePolicy == kyverno.Ignore {
m.mergeWebhook(mutateIgnore, p)
} else {
m.mergeWebhook(mutateFail, p)
}
}
}
res = append(res, mutateIgnore, mutateFail, validateIgnore, validateFail)
return res, nil
}
func (m *webhookConfigManager) updateWebhookConfig(webhooks []*webhook) error {
logger := m.log.WithName("updateWebhookConfig")
webhooksMap := make(map[string]interface{}, len(webhooks))
for _, w := range webhooks {
key := webhookKey(w.kind, string(w.failurePolicy))
webhooksMap[key] = w
}
var errs []string
if err := m.compareAndUpdateWebhook(kindMutating, getResourceMutatingWebhookConfigName(""), webhooksMap); err != nil {
logger.V(4).Info("failed to update mutatingwebhookconfigurations", "error", err.Error())
errs = append(errs, err.Error())
}
if err := m.compareAndUpdateWebhook(kindValidating, getResourceValidatingWebhookConfigName(""), webhooksMap); err != nil {
logger.V(4).Info("failed to update validatingwebhookconfigurations", "error", err.Error())
errs = append(errs, err.Error())
}
if len(errs) != 0 {
return errors.New(strings.Join(errs, "\n"))
}
return nil
}
func (m *webhookConfigManager) getWebhook(webhookKind, webhookName string) (resourceWebhook *unstructured.Unstructured, err error) {
get := func() error {
webhookCache, _ := m.resCache.GetGVRCache(webhookKind)
resourceWebhook, err = webhookCache.Lister().Get(webhookName)
if err != nil && !apierrors.IsNotFound(err) {
return errors.Wrapf(err, "unable to get %s/%s", webhookKind, webhookName)
} else if apierrors.IsNotFound(err) {
m.createDefaultWebhook <- webhookKind
return err
}
return nil
}
retryGetWebhook := common.RetryFunc(time.Second, 10*time.Second, get, m.log)
if err := retryGetWebhook(); err != nil {
return nil, err
}
return resourceWebhook, nil
}
func (m *webhookConfigManager) compareAndUpdateWebhook(webhookKind, webhookName string, webhooksMap map[string]interface{}) error {
logger := m.log.WithName("compareAndUpdateWebhook").WithValues("kind", webhookKind, "name", webhookName)
resourceWebhook, err := m.getWebhook(webhookKind, webhookName)
if err != nil {
return err
}
webhooksUntyped, _, err := unstructured.NestedSlice(resourceWebhook.UnstructuredContent(), "webhooks")
if err != nil {
return errors.Wrapf(err, "unable to fetch tag webhooks for %s/%s", webhookKind, webhookName)
}
newWebooks := make([]interface{}, len(webhooksUntyped))
copy(newWebooks, webhooksUntyped)
var changed bool
for i, webhookUntyed := range webhooksUntyped {
existingWebhook, ok := webhookUntyed.(map[string]interface{})
if !ok {
logger.Error(errors.New("type mismatched"), "expected map[string]interface{}, got %T", webhooksUntyped)
continue
}
failurePolicy, _, err := unstructured.NestedString(existingWebhook, "failurePolicy")
if err != nil {
logger.Error(errors.New("type mismatched"), "expected string, got %T", failurePolicy)
continue
}
rules, _, err := unstructured.NestedSlice(existingWebhook, "rules")
if err != nil {
logger.Error(err, "type mismatched, expected []interface{}, got %T", rules)
continue
}
newWebhook := webhooksMap[webhookKey(webhookKind, failurePolicy)]
w, ok := newWebhook.(*webhook)
if !ok {
logger.Error(errors.New("type mismatched"), "expected *webhook, got %T", newWebooks)
continue
}
if !reflect.DeepEqual(rules, []interface{}{w.rule}) {
changed = true
tmpRules, ok := newWebooks[i].(map[string]interface{})["rules"].([]interface{})
if !ok {
// init operations
ops := []string{string(admregapi.Create), string(admregapi.Update), string(admregapi.Delete), string(admregapi.Connect)}
if webhookKind == kindMutating {
ops = []string{string(admregapi.Create), string(admregapi.Update)}
}
tmpRules = []interface{}{map[string]interface{}{}}
if err = unstructured.SetNestedStringSlice(tmpRules[0].(map[string]interface{}), ops, "operations"); err != nil {
return errors.Wrapf(err, "unable to set webhooks[%d].rules[0].%s", i, apiGroups)
}
}
if w.rule == nil || reflect.DeepEqual(w.rule, map[string]interface{}{}) {
// zero kyverno policy with the current failurePolicy, reset webhook rules to empty
newWebooks[i].(map[string]interface{})["rules"] = []interface{}{}
continue
}
if err = unstructured.SetNestedStringSlice(tmpRules[0].(map[string]interface{}), w.rule[apiGroups].([]string), apiGroups); err != nil {
return errors.Wrapf(err, "unable to set webhooks[%d].rules[0].%s", i, apiGroups)
}
if err = unstructured.SetNestedStringSlice(tmpRules[0].(map[string]interface{}), w.rule[apiVersions].([]string), apiVersions); err != nil {
return errors.Wrapf(err, "unable to set webhooks[%d].rules[0].%s", i, apiVersions)
}
if err = unstructured.SetNestedStringSlice(tmpRules[0].(map[string]interface{}), w.rule[resources].([]string), resources); err != nil {
return errors.Wrapf(err, "unable to set webhooks[%d].rules[0].%s", i, resources)
}
newWebooks[i].(map[string]interface{})["rules"] = tmpRules
}
if err = unstructured.SetNestedField(newWebooks[i].(map[string]interface{}), w.maxWebhookTimeout, "timeoutSeconds"); err != nil {
return errors.Wrapf(err, "unable to set webhooks[%d].timeoutSeconds to %v", i, w.maxWebhookTimeout)
}
}
if changed {
logger.V(4).Info("webhook configuration has been changed, updating")
if err := unstructured.SetNestedSlice(resourceWebhook.UnstructuredContent(), newWebooks, "webhooks"); err != nil {
return errors.Wrap(err, "unable to set new webhooks")
}
if _, err := m.client.UpdateResource(resourceWebhook.GetAPIVersion(), resourceWebhook.GetKind(), "", resourceWebhook, false); err != nil {
return errors.Wrapf(err, "unable to update %s/%s: %s", resourceWebhook.GetAPIVersion(), resourceWebhook.GetKind(), resourceWebhook.GetName())
}
logger.V(4).Info("successfully updated the webhook configuration")
}
return nil
}
func (m *webhookConfigManager) updateStatus(policy *kyverno.ClusterPolicy) error {
policyCopy := policy.DeepCopy()
policyCopy.Status.Ready = true
if policy.GetNamespace() == "" {
_, err := m.kyvernoClient.KyvernoV1().ClusterPolicies().UpdateStatus(context.TODO(), policyCopy, v1.UpdateOptions{})
return err
}
_, err := m.kyvernoClient.KyvernoV1().Policies(policyCopy.GetNamespace()).UpdateStatus(context.TODO(), (*kyverno.Policy)(policyCopy), v1.UpdateOptions{})
return err
}
// mergeWebhook merges the matching kinds of the policy to webhook.rule
func (m *webhookConfigManager) mergeWebhook(dst *webhook, policy *kyverno.ClusterPolicy) {
matchedGVK := make([]string, 0)
for _, rule := range policy.Spec.Rules {
matchedGVK = append(matchedGVK, rule.MatchKinds()...)
if rule.HasGenerate() {
matchedGVK = append(matchedGVK, rule.Generation.ResourceSpec.Kind)
}
}
gvkMap := make(map[string]int)
gvrList := make([]schema.GroupVersionResource, 0)
for _, gvk := range matchedGVK {
if _, ok := gvkMap[gvk]; !ok {
gvkMap[gvk] = 1
// note: webhook stores GVR in its rules while policy stores GVK in its rules definition
gv, k := common.GetKindFromGVK(gvk)
_, gvr, err := m.client.DiscoveryClient.FindResource(gv, k)
if err != nil {
continue
}
gvrList = append(gvrList, gvr)
}
}
var groups, versions, rsrcs []string
if val, ok := dst.rule[apiGroups]; ok {
groups = make([]string, len(val.([]string)))
copy(groups, val.([]string))
}
if val, ok := dst.rule[apiVersions]; ok {
versions = make([]string, len(val.([]string)))
copy(versions, val.([]string))
}
if val, ok := dst.rule[resources]; ok {
rsrcs = make([]string, len(val.([]string)))
copy(rsrcs, val.([]string))
}
for _, gvr := range gvrList {
groups = append(groups, gvr.Group)
versions = append(versions, gvr.Version)
rsrcs = append(rsrcs, gvr.Resource)
}
dst.rule[apiGroups] = removeDuplicates(groups)
dst.rule[apiVersions] = removeDuplicates(versions)
dst.rule[resources] = removeDuplicates(rsrcs)
if policy.Spec.WebhookTimeoutSeconds != nil {
if dst.maxWebhookTimeout < int64(*policy.Spec.WebhookTimeoutSeconds) {
dst.maxWebhookTimeout = int64(*policy.Spec.WebhookTimeoutSeconds)
}
}
}
func removeDuplicates(items []string) (res []string) {
set := make(map[string]int)
for _, item := range items {
if _, ok := set[item]; !ok {
set[item] = 1
res = append(res, item)
}
}
return
}
func newWebhook(kind string, timeout int64, failurePolicy kyverno.FailurePolicyType) *webhook {
return &webhook{
kind: kind,
maxWebhookTimeout: timeout,
failurePolicy: failurePolicy,
rule: make(map[string]interface{}),
}
}
func webhookKey(webhookKind, failurePolicy string) string {
return strings.Join([]string{webhookKind, failurePolicy}, "/")
}
func hasWildcard(policy interface{}) bool {
if p, ok := policy.(*kyverno.ClusterPolicy); ok {
for _, rule := range p.Spec.Rules {
if kinds := rule.MatchKinds(); utils.ContainsString(kinds, "*") {
return true
}
}
}
if p, ok := policy.(*kyverno.Policy); ok {
for _, rule := range p.Spec.Rules {
if kinds := rule.MatchKinds(); utils.ContainsString(kinds, "*") {
return true
}
}
}
return false
}
func setWildcardConfig(w *webhook) {
w.rule[apiGroups] = []string{"*"}
w.rule[apiVersions] = []string{"*"}
w.rule[resources] = []string{"*/*"}
}

17
pkg/webhookconfig/monitor.go
View file

@ -74,7 +74,7 @@ func (t *Monitor) SetTime(tm time.Time) {
// Run runs the checker and verify the resource update
func (t *Monitor) Run(register *Register, certRenewer *tls.CertRenewer, eventGen event.Interface, stopCh <-chan struct{}) {
logger := t.log
logger := t.log.WithName("webhookMonitor")
logger.V(4).Info("starting webhook monitor", "interval", idleCheckInterval.String())
status := newStatusControl(register, eventGen, t.log.WithName("WebhookStatusControl"))
@ -82,8 +82,23 @@ func (t *Monitor) Run(register *Register, certRenewer *tls.CertRenewer, eventGen
ticker := time.NewTicker(tickerInterval)
defer ticker.Stop()
createDefaultWebhook := register.createDefaultWebhook
for {
select {
case webhookKind := <-createDefaultWebhook:
logger.Info("received recreation request for resource webhook")
if webhookKind == kindMutating {
err := register.createResourceMutatingWebhookConfiguration(register.readCaData())
if err != nil {
logger.Error(err, "failed to create default MutatingWebhookConfiguration for resources, the webhook will be reconciled", "interval", tickerInterval)
}
} else if webhookKind == kindValidating {
err := register.createResourceValidatingWebhookConfiguration(register.readCaData())
if err != nil {
logger.Error(err, "failed to create default ValidatingWebhookConfiguration for resources, the webhook will be reconciled", "interval", tickerInterval)
}
}
case <-ticker.C:
err := registerWebhookIfNotPresent(register, t.log.WithName("registerWebhookIfNotPresent"))

46
pkg/webhookconfig/policy.go
View file

@ -4,11 +4,11 @@ import (
"fmt"
"github.com/kyverno/kyverno/pkg/config"
admregapi "k8s.io/api/admissionregistration/v1beta1"
admregapi "k8s.io/api/admissionregistration/v1"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func (wrc *Register) contructPolicyValidatingWebhookConfig(caData []byte) *admregapi.ValidatingWebhookConfiguration {
func (wrc *Register) constructPolicyValidatingWebhookConfig(caData []byte) *admregapi.ValidatingWebhookConfiguration {
return &admregapi.ValidatingWebhookConfiguration{
ObjectMeta: v1.ObjectMeta{
@ -24,16 +24,19 @@ func (wrc *Register) contructPolicyValidatingWebhookConfig(caData []byte) *admre
caData,
true,
wrc.timeoutSeconds,
[]string{"clusterpolicies/*", "policies/*"},
"kyverno.io",
"v1",
admregapi.Rule{
Resources: []string{"clusterpolicies/*", "policies/*"},
APIGroups: []string{"kyverno.io"},
APIVersions: []string{"v1"},
},
[]admregapi.OperationType{admregapi.Create, admregapi.Update},
admregapi.Ignore,
),
},
}
}
func (wrc *Register) contructDebugPolicyValidatingWebhookConfig(caData []byte) *admregapi.ValidatingWebhookConfiguration {
func (wrc *Register) constructDebugPolicyValidatingWebhookConfig(caData []byte) *admregapi.ValidatingWebhookConfiguration {
logger := wrc.log
url := fmt.Sprintf("https://%s%s", wrc.serverIP, config.PolicyValidatingWebhookServicePath)
logger.V(4).Info("Debug PolicyValidatingWebhookConfig is registered with url ", "url", url)
@ -49,16 +52,19 @@ func (wrc *Register) contructDebugPolicyValidatingWebhookConfig(caData []byte) *
caData,
true,
wrc.timeoutSeconds,
[]string{"clusterpolicies/*", "policies/*"},
"kyverno.io",
"v1",
admregapi.Rule{
Resources: []string{"clusterpolicies/*", "policies/*"},
APIGroups: []string{"kyverno.io"},
APIVersions: []string{"v1"},
},
[]admregapi.OperationType{admregapi.Create, admregapi.Update},
admregapi.Ignore,
),
},
}
}
func (wrc *Register) contructPolicyMutatingWebhookConfig(caData []byte) *admregapi.MutatingWebhookConfiguration {
func (wrc *Register) constructPolicyMutatingWebhookConfig(caData []byte) *admregapi.MutatingWebhookConfiguration {
return &admregapi.MutatingWebhookConfiguration{
ObjectMeta: v1.ObjectMeta{
Name: config.PolicyMutatingWebhookConfigurationName,
@ -73,16 +79,19 @@ func (wrc *Register) contructPolicyMutatingWebhookConfig(caData []byte) *admrega
caData,
true,
wrc.timeoutSeconds,
[]string{"clusterpolicies/*", "policies/*"},
"kyverno.io",
"v1",
admregapi.Rule{
Resources: []string{"clusterpolicies/*", "policies/*"},
APIGroups: []string{"kyverno.io"},
APIVersions: []string{"v1"},
},
[]admregapi.OperationType{admregapi.Create, admregapi.Update},
admregapi.Ignore,
),
},
}
}
func (wrc *Register) contructDebugPolicyMutatingWebhookConfig(caData []byte) *admregapi.MutatingWebhookConfiguration {
func (wrc *Register) constructDebugPolicyMutatingWebhookConfig(caData []byte) *admregapi.MutatingWebhookConfiguration {
logger := wrc.log
url := fmt.Sprintf("https://%s%s", wrc.serverIP, config.PolicyMutatingWebhookServicePath)
logger.V(4).Info("Debug PolicyMutatingWebhookConfig is registered with url ", "url", url)
@ -98,10 +107,13 @@ func (wrc *Register) contructDebugPolicyMutatingWebhookConfig(caData []byte) *ad
caData,
true,
wrc.timeoutSeconds,
[]string{"clusterpolicies/*", "policies/*"},
"kyverno.io",
"v1",
admregapi.Rule{
Resources: []string{"clusterpolicies/*", "policies/*"},
APIGroups: []string{"kyverno.io"},
APIVersions: []string{"v1"},
},
[]admregapi.OperationType{admregapi.Create, admregapi.Update},
admregapi.Ignore,
),
},
}

124
pkg/webhookconfig/registration.go
View file

@ -8,12 +8,14 @@ import (
"time"
"github.com/go-logr/logr"
kyvernoclient "github.com/kyverno/kyverno/pkg/client/clientset/versioned"
kyvernoinformer "github.com/kyverno/kyverno/pkg/client/informers/externalversions/kyverno/v1"
"github.com/kyverno/kyverno/pkg/config"
client "github.com/kyverno/kyverno/pkg/dclient"
"github.com/kyverno/kyverno/pkg/resourcecache"
"github.com/kyverno/kyverno/pkg/tls"
"github.com/pkg/errors"
admregapi "k8s.io/api/admissionregistration/v1beta1"
admregapi "k8s.io/api/admissionregistration/v1"
corev1 "k8s.io/api/core/v1"
errorsapi "k8s.io/apimachinery/pkg/api/errors"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@ -34,36 +36,54 @@ const (
// 4. Resource Mutation
// 5. Webhook Status Mutation
type Register struct {
client *client.Client
clientConfig *rest.Config
resCache resourcecache.ResourceCache
serverIP string // when running outside a cluster
timeoutSeconds int32
log logr.Logger
debug bool
client *client.Client
clientConfig *rest.Config
resCache resourcecache.ResourceCache
serverIP string // when running outside a cluster
timeoutSeconds int32
log logr.Logger
debug bool
autoUpdateWebhooks bool
UpdateWebhookChan chan bool
UpdateWebhookChan chan bool
createDefaultWebhook chan string
// manage implements methods to manage webhook configurations
manage
}
// NewRegister creates new Register instance
func NewRegister(
clientConfig *rest.Config,
client *client.Client,
kyvernoClient *kyvernoclient.Clientset,
resCache resourcecache.ResourceCache,
pInformer kyvernoinformer.ClusterPolicyInformer,
npInformer kyvernoinformer.PolicyInformer,
serverIP string,
webhookTimeout int32,
debug bool,
autoUpdateWebhooks bool,
stopCh <-chan struct{},
log logr.Logger) *Register {
return &Register{
clientConfig: clientConfig,
client: client,
resCache: resCache,
serverIP: serverIP,
timeoutSeconds: webhookTimeout,
log: log.WithName("Register"),
debug: debug,
UpdateWebhookChan: make(chan bool),
register := &Register{
clientConfig: clientConfig,
client: client,
resCache: resCache,
serverIP: serverIP,
timeoutSeconds: webhookTimeout,
log: log.WithName("Register"),
debug: debug,
autoUpdateWebhooks: autoUpdateWebhooks,
UpdateWebhookChan: make(chan bool),
createDefaultWebhook: make(chan string),
}
if register.autoUpdateWebhooks {
register.manage = newWebhookConfigManager(client, kyvernoClient, pInformer, npInformer, resCache, register.createDefaultWebhook, stopCh, log.WithName("WebhookConfigManager"))
}
return register
}
// Register clean up the old webhooks and re-creates admission webhooks configs on cluster
@ -109,6 +129,9 @@ func (wrc *Register) Register() error {
return fmt.Errorf("%s", strings.Join(errors, ","))
}
if wrc.autoUpdateWebhooks {
go wrc.manage.start()
}
return nil
}
@ -121,19 +144,19 @@ func (wrc *Register) Check() error {
return err
}
if _, err := mutatingCache.Lister().Get(wrc.getResourceMutatingWebhookConfigName()); err != nil {
if _, err := mutatingCache.Lister().Get(getResourceMutatingWebhookConfigName(wrc.serverIP)); err != nil {
return err
}
if _, err := validatingCache.Lister().Get(wrc.getResourceValidatingWebhookConfigName()); err != nil {
if _, err := validatingCache.Lister().Get(getResourceValidatingWebhookConfigName(wrc.serverIP)); err != nil {
return err
}
if _, err := mutatingCache.Lister().Get(wrc.getPolicyMutatingWebhookConfigurationName()); err != nil {
if _, err := mutatingCache.Lister().Get(getPolicyMutatingWebhookConfigurationName(wrc.serverIP)); err != nil {
return err
}
if _, err := validatingCache.Lister().Get(wrc.getPolicyValidatingWebhookConfigurationName()); err != nil {
if _, err := validatingCache.Lister().Get(getPolicyValidatingWebhookConfigurationName(wrc.serverIP)); err != nil {
return err
}
@ -151,10 +174,11 @@ func (wrc *Register) Remove(cleanUp chan<- struct{}) {
wrc.removeSecrets()
}
// +deprecated
// UpdateWebhookConfigurations updates resource webhook configurations dynamically
// base on the UPDATEs of Kyverno init-config ConfigMap
//
// it currently updates namespaceSelector only, can be extend to update other fieids
// it currently updates namespaceSelector only, can be extend to update other fields
func (wrc *Register) UpdateWebhookConfigurations(configHandler config.Interface) {
logger := wrc.log.WithName("UpdateWebhookConfigurations")
for {
@ -178,17 +202,17 @@ func (wrc *Register) UpdateWebhookConfigurations(configHandler config.Interface)
}
if err := wrc.updateResourceMutatingWebhookConfiguration(nsSelector); err != nil {
logger.Error(err, "unable to update mutatingWebhookConfigurations", "name", wrc.getResourceMutatingWebhookConfigName())
logger.Error(err, "unable to update mutatingWebhookConfigurations", "name", getResourceMutatingWebhookConfigName(wrc.serverIP))
go func() { wrc.UpdateWebhookChan <- true }()
} else {
logger.Info("successfully updated mutatingWebhookConfigurations", "name", wrc.getResourceMutatingWebhookConfigName())
logger.Info("successfully updated mutatingWebhookConfigurations", "name", getResourceMutatingWebhookConfigName(wrc.serverIP))
}
if err := wrc.updateResourceValidatingWebhookConfiguration(nsSelector); err != nil {
logger.Error(err, "unable to update validatingWebhookConfigurations", "name", wrc.getResourceValidatingWebhookConfigName())
logger.Error(err, "unable to update validatingWebhookConfigurations", "name", getResourceValidatingWebhookConfigName(wrc.serverIP))
go func() { wrc.UpdateWebhookChan <- true }()
} else {
logger.Info("successfully updated validatingWebhookConfigurations", "name", wrc.getResourceValidatingWebhookConfigName())
logger.Info("successfully updated validatingWebhookConfigurations", "name", getResourceValidatingWebhookConfigName(wrc.serverIP))
}
}
}
@ -302,9 +326,9 @@ func (wrc *Register) createPolicyValidatingWebhookConfiguration(caData []byte) e
var config *admregapi.ValidatingWebhookConfiguration
if wrc.serverIP != "" {
config = wrc.contructDebugPolicyValidatingWebhookConfig(caData)
config = wrc.constructDebugPolicyValidatingWebhookConfig(caData)
} else {
config = wrc.contructPolicyValidatingWebhookConfig(caData)
config = wrc.constructPolicyValidatingWebhookConfig(caData)
}
if _, err := wrc.client.CreateResource("", kindValidating, "", *config, false); err != nil {
@ -324,9 +348,9 @@ func (wrc *Register) createPolicyMutatingWebhookConfiguration(caData []byte) err
var config *admregapi.MutatingWebhookConfiguration
if wrc.serverIP != "" {
config = wrc.contructDebugPolicyMutatingWebhookConfig(caData)
config = wrc.constructDebugPolicyMutatingWebhookConfig(caData)
} else {
config = wrc.contructPolicyMutatingWebhookConfig(caData)
config = wrc.constructPolicyMutatingWebhookConfig(caData)
}
// create mutating webhook configuration resource
@ -387,7 +411,7 @@ func (wrc *Register) removeWebhookConfigurations() {
func (wrc *Register) removePolicyMutatingWebhookConfiguration(wg *sync.WaitGroup) {
defer wg.Done()
mutatingConfig := wrc.getPolicyMutatingWebhookConfigurationName()
mutatingConfig := getPolicyMutatingWebhookConfigurationName(wrc.serverIP)
logger := wrc.log.WithValues("kind", kindMutating, "name", mutatingConfig)
@ -412,9 +436,9 @@ func (wrc *Register) removePolicyMutatingWebhookConfiguration(wg *sync.WaitGroup
logger.Info("webhook configuration deleted")
}
func (wrc *Register) getPolicyMutatingWebhookConfigurationName() string {
func getPolicyMutatingWebhookConfigurationName(serverIP string) string {
var mutatingConfig string
if wrc.serverIP != "" {
if serverIP != "" {
mutatingConfig = config.PolicyMutatingWebhookConfigurationDebugName
} else {
mutatingConfig = config.PolicyMutatingWebhookConfigurationName
@ -425,7 +449,7 @@ func (wrc *Register) getPolicyMutatingWebhookConfigurationName() string {
func (wrc *Register) removePolicyValidatingWebhookConfiguration(wg *sync.WaitGroup) {
defer wg.Done()
validatingConfig := wrc.getPolicyValidatingWebhookConfigurationName()
validatingConfig := getPolicyValidatingWebhookConfigurationName(wrc.serverIP)
logger := wrc.log.WithValues("kind", kindValidating, "name", validatingConfig)
if mutateCache, ok := wrc.resCache.GetGVRCache("ValidatingWebhookConfiguration"); ok {
@ -450,9 +474,9 @@ func (wrc *Register) removePolicyValidatingWebhookConfiguration(wg *sync.WaitGro
logger.Info("webhook configuration deleted")
}
func (wrc *Register) getPolicyValidatingWebhookConfigurationName() string {
func getPolicyValidatingWebhookConfigurationName(serverIP string) string {
var validatingConfig string
if wrc.serverIP != "" {
if serverIP != "" {
validatingConfig = config.PolicyValidatingWebhookConfigurationDebugName
} else {
validatingConfig = config.PolicyValidatingWebhookConfigurationName
@ -475,10 +499,13 @@ func (wrc *Register) constructVerifyMutatingWebhookConfig(caData []byte) *admreg
caData,
true,
wrc.timeoutSeconds,
[]string{"deployments/*"},
"apps",
"v1",
admregapi.Rule{
Resources: []string{"deployments/*"},
APIGroups: []string{"apps"},
APIVersions: []string{"v1"},
},
[]admregapi.OperationType{admregapi.Update},
admregapi.Ignore,
),
},
}
@ -499,10 +526,13 @@ func (wrc *Register) constructDebugVerifyMutatingWebhookConfig(caData []byte) *a
caData,
true,
wrc.timeoutSeconds,
[]string{"deployments/*"},
"apps",
"v1",
admregapi.Rule{
Resources: []string{"deployments/*"},
APIGroups: []string{"apps"},
APIVersions: []string{"v1"},
},
[]admregapi.OperationType{admregapi.Update},
admregapi.Ignore,
),
},
}
@ -597,7 +627,7 @@ func (wrc *Register) checkEndpoint() error {
}
if podIp == "" {
return fmt.Errorf("Pod is not assigned to any node yet")
return fmt.Errorf("pod is not assigned to any node yet")
}
for _, subset := range endpoint.Subsets {
@ -616,7 +646,7 @@ func (wrc *Register) checkEndpoint() error {
// clean up old webhook configurations, if any
wrc.removeWebhookConfigurations()
err = fmt.Errorf("Endpoint not ready")
err = fmt.Errorf("endpoint not ready")
wrc.log.V(3).Info(err.Error(), "ns", config.KyvernoNamespace, "name", config.KyvernoServiceName)
return err
}
@ -624,7 +654,7 @@ func (wrc *Register) checkEndpoint() error {
func (wrc *Register) updateResourceValidatingWebhookConfiguration(nsSelector map[string]interface{}) error {
validatingCache, _ := wrc.resCache.GetGVRCache(kindValidating)
resourceValidating, err := validatingCache.Lister().Get(wrc.getResourceValidatingWebhookConfigName())
resourceValidating, err := validatingCache.Lister().Get(getResourceValidatingWebhookConfigName(wrc.serverIP))
if err != nil {
return errors.Wrapf(err, "unable to get validatingWebhookConfigurations")
}
@ -660,7 +690,7 @@ func (wrc *Register) updateResourceValidatingWebhookConfiguration(nsSelector map
func (wrc *Register) updateResourceMutatingWebhookConfiguration(nsSelector map[string]interface{}) error {
mutatingCache, _ := wrc.resCache.GetGVRCache(kindMutating)
resourceMutating, err := mutatingCache.Lister().Get(wrc.getResourceMutatingWebhookConfigName())
resourceMutating, err := mutatingCache.Lister().Get(getResourceMutatingWebhookConfigName(wrc.serverIP))
if err != nil {
return errors.Wrapf(err, "unable to get mutatingWebhookConfigurations")
}

117
pkg/webhookconfig/resource.go
View file

@ -5,12 +5,23 @@ import (
"sync"
"github.com/kyverno/kyverno/pkg/config"
admregapi "k8s.io/api/admissionregistration/v1beta1"
"k8s.io/apimachinery/pkg/api/errors"
admregapi "k8s.io/api/admissionregistration/v1"
errorsapi "k8s.io/apimachinery/pkg/api/errors"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func (wrc *Register) defaultResourceWebhookRule() admregapi.Rule {
if wrc.autoUpdateWebhooks {
return admregapi.Rule{}
}
return admregapi.Rule{
Resources: []string{"*/*"},
APIGroups: []string{"*"},
APIVersions: []string{"*"},
}
}
func (wrc *Register) constructDefaultDebugMutatingWebhookConfig(caData []byte) *admregapi.MutatingWebhookConfiguration {
logger := wrc.log
url := fmt.Sprintf("https://%s%s", wrc.serverIP, config.MutatingWebhookServicePath)
@ -21,32 +32,30 @@ func (wrc *Register) constructDefaultDebugMutatingWebhookConfig(caData []byte) *
},
Webhooks: []admregapi.MutatingWebhook{
generateDebugMutatingWebhook(
config.MutatingWebhookName,
config.MutatingWebhookName+"-ignore",
url,
caData,
true,
wrc.timeoutSeconds,
[]string{"*/*"},
"*",
"*",
wrc.defaultResourceWebhookRule(),
[]admregapi.OperationType{admregapi.Create, admregapi.Update},
admregapi.Ignore,
),
generateDebugMutatingWebhook(
config.MutatingWebhookName+"-fail",
url,
caData,
true,
wrc.timeoutSeconds,
wrc.defaultResourceWebhookRule(),
[]admregapi.OperationType{admregapi.Create, admregapi.Update},
admregapi.Fail,
),
},
}
}
func (wrc *Register) constructDefaultMutatingWebhookConfig(caData []byte) *admregapi.MutatingWebhookConfiguration {
webhookCfg := generateMutatingWebhook(
config.MutatingWebhookName,
config.MutatingWebhookServicePath,
caData, false, wrc.timeoutSeconds,
[]string{"*/*"}, "*", "*",
[]admregapi.OperationType{admregapi.Create, admregapi.Update})
reinvoke := admregapi.IfNeededReinvocationPolicy
webhookCfg.ReinvocationPolicy = &reinvoke
return &admregapi.MutatingWebhookConfiguration{
ObjectMeta: v1.ObjectMeta{
Name: config.MutatingWebhookConfigurationName,
@ -54,13 +63,34 @@ func (wrc *Register) constructDefaultMutatingWebhookConfig(caData []byte) *admre
wrc.constructOwner(),
},
},
Webhooks: []admregapi.MutatingWebhook{webhookCfg},
Webhooks: []admregapi.MutatingWebhook{
generateMutatingWebhook(
config.MutatingWebhookName+"-ignore",
config.MutatingWebhookServicePath,
caData,
false,
wrc.timeoutSeconds,
wrc.defaultResourceWebhookRule(),
[]admregapi.OperationType{admregapi.Create, admregapi.Update},
admregapi.Ignore,
),
generateMutatingWebhook(
config.MutatingWebhookName+"-fail",
config.MutatingWebhookServicePath,
caData,
false,
wrc.timeoutSeconds,
wrc.defaultResourceWebhookRule(),
[]admregapi.OperationType{admregapi.Create, admregapi.Update},
admregapi.Fail,
),
},
}
}
//getResourceMutatingWebhookConfigName returns the webhook configuration name
func (wrc *Register) getResourceMutatingWebhookConfigName() string {
if wrc.serverIP != "" {
func getResourceMutatingWebhookConfigName(serverIP string) string {
if serverIP != "" {
return config.MutatingWebhookConfigurationDebugName
}
return config.MutatingWebhookConfigurationName
@ -69,7 +99,7 @@ func (wrc *Register) getResourceMutatingWebhookConfigName() string {
func (wrc *Register) removeResourceMutatingWebhookConfiguration(wg *sync.WaitGroup) {
defer wg.Done()
configName := wrc.getResourceMutatingWebhookConfigName()
configName := getResourceMutatingWebhookConfigName(wrc.serverIP)
logger := wrc.log.WithValues("kind", kindMutating, "name", configName)
if mutateCache, ok := wrc.resCache.GetGVRCache("MutatingWebhookConfiguration"); ok {
@ -81,7 +111,7 @@ func (wrc *Register) removeResourceMutatingWebhookConfiguration(wg *sync.WaitGro
// delete webhook configuration
err := wrc.client.DeleteResource("", kindMutating, "", configName, false)
if errors.IsNotFound(err) {
if errorsapi.IsNotFound(err) {
logger.V(4).Info("webhook configuration not found")
return
}
@ -103,15 +133,24 @@ func (wrc *Register) constructDefaultDebugValidatingWebhookConfig(caData []byte)
},
Webhooks: []admregapi.ValidatingWebhook{
generateDebugValidatingWebhook(
config.ValidatingWebhookName,
config.ValidatingWebhookName+"-ignore",
url,
caData,
true,
wrc.timeoutSeconds,
[]string{"*/*"},
"*",
"*",
wrc.defaultResourceWebhookRule(),
[]admregapi.OperationType{admregapi.Create, admregapi.Update, admregapi.Delete, admregapi.Connect},
admregapi.Ignore,
),
generateDebugValidatingWebhook(
config.ValidatingWebhookName+"-fail",
url,
caData,
true,
wrc.timeoutSeconds,
wrc.defaultResourceWebhookRule(),
[]admregapi.OperationType{admregapi.Create, admregapi.Update, admregapi.Delete, admregapi.Connect},
admregapi.Fail,
),
},
}
@ -127,23 +166,32 @@ func (wrc *Register) constructDefaultValidatingWebhookConfig(caData []byte) *adm
},
Webhooks: []admregapi.ValidatingWebhook{
generateValidatingWebhook(
config.ValidatingWebhookName,
config.ValidatingWebhookName+"-ignore",
config.ValidatingWebhookServicePath,
caData,
false,
wrc.timeoutSeconds,
[]string{"*/*"},
"*",
"*",
wrc.defaultResourceWebhookRule(),
[]admregapi.OperationType{admregapi.Create, admregapi.Update, admregapi.Delete, admregapi.Connect},
admregapi.Ignore,
),
generateValidatingWebhook(
config.ValidatingWebhookName+"-fail",
config.ValidatingWebhookServicePath,
caData,
false,
wrc.timeoutSeconds,
wrc.defaultResourceWebhookRule(),
[]admregapi.OperationType{admregapi.Create, admregapi.Update, admregapi.Delete, admregapi.Connect},
admregapi.Fail,
),
},
}
}
// getResourceValidatingWebhookConfigName returns the webhook configuration name
func (wrc *Register) getResourceValidatingWebhookConfigName() string {
if wrc.serverIP != "" {
func getResourceValidatingWebhookConfigName(serverIP string) string {
if serverIP != "" {
return config.ValidatingWebhookConfigurationDebugName
}
@ -153,7 +201,7 @@ func (wrc *Register) getResourceValidatingWebhookConfigName() string {
func (wrc *Register) removeResourceValidatingWebhookConfiguration(wg *sync.WaitGroup) {
defer wg.Done()
configName := wrc.getResourceValidatingWebhookConfigName()
configName := getResourceValidatingWebhookConfigName(wrc.serverIP)
logger := wrc.log.WithValues("kind", kindValidating, "name", configName)
if mutateCache, ok := wrc.resCache.GetGVRCache("ValidatingWebhookConfiguration"); ok {
@ -164,7 +212,7 @@ func (wrc *Register) removeResourceValidatingWebhookConfiguration(wg *sync.WaitG
}
err := wrc.client.DeleteResource("", kindValidating, "", configName, false)
if errors.IsNotFound(err) {
if errorsapi.IsNotFound(err) {
logger.V(5).Info("webhook configuration not found")
return
}
@ -175,5 +223,4 @@ func (wrc *Register) removeResourceValidatingWebhookConfiguration(wg *sync.WaitG
}
logger.Info("webhook configuration deleted")
return
}

2
pkg/webhooks/generate/generate.go
View file

@ -116,8 +116,6 @@ func retryApplyResource(client *kyvernoclient.Clientset, grSpec kyverno.Generate
gr.SetNamespace(config.KyvernoNamespace)
// Initial state "Pending"
// TODO: status is not updated
// gr.Status.State = kyverno.Pending
// generate requests created in kyverno namespace
isExist := false
if action == v1beta1.Create || action == v1beta1.Update {

12
test/e2e/common/common.go
View file

@ -34,7 +34,7 @@ func CallMetrics() (string, error) {
func ProcessMetrics(newStr, e2ePolicyName string) error {
splitByNewLine := strings.Split(newStr, "\n")
for _, lineSplitByNewLine := range splitByNewLine {
// kyverno_policy_rule_info_total{policy_background_mode=\"false\",policy_name=\"gen-cluster-policy\",policy_namespace=\"-\",policy_type=\"cluster\",policy_validation_mode=\"audit\",rule_name=\"gen-cluster-role\",rule_type=\"generate\"} 1
// kyverno_policy_rule_info_total{policy_background_mode=\"false\",policy_name=\"gen-cluster-policy\",policy_namespace=\"-\",policy_type=\"cluster\",policy_validation_mode=\"audit\",rule_name=\"gen-cluster-role\",rule_type=\"generate\",status_ready="false"} 1
if !strings.HasPrefix(lineSplitByNewLine, "kyverno_policy_rule_info_total{") {
continue
}
@ -48,10 +48,18 @@ func ProcessMetrics(newStr, e2ePolicyName string) error {
if strings.HasPrefix(lineSplitByComma, "policy_name=") {
splitByQuote := strings.Split(lineSplitByComma, "\"")
policyName := splitByQuote[1]
if policyName == e2ePolicyName {
if policyName != e2ePolicyName {
continue
}
}
if strings.HasPrefix(lineSplitByComma, "status_ready=") {
splitByQuote := strings.Split(lineSplitByComma, "\"")
status := splitByQuote[1]
if status == "true" {
return nil
}
}
}
}