diff --git a/cmd/kyverno/main.go b/cmd/kyverno/main.go index b55b339d84..d8fcffc808 100644 --- a/cmd/kyverno/main.go +++ b/cmd/kyverno/main.go @@ -95,6 +95,7 @@ var ( reportsChunkSize int backgroundScanWorkers int logFormat string + dumpPayload bool // DEPRECATED: remove in 1.9 splitPolicyReport bool ) @@ -102,6 +103,7 @@ var ( func parseFlags() error { logging.Init(nil) flag.StringVar(&logFormat, "loggingFormat", logging.TextFormat, "This determines the output format of the logger.") + flag.BoolVar(&dumpPayload, "dumpPayload", false, "Set this flag to activate/deactivate debug mode.") flag.IntVar(&webhookTimeout, "webhookTimeout", webhookcontroller.DefaultWebhookTimeout, "Timeout for webhook configurations.") flag.IntVar(&genWorkers, "genWorkers", 10, "Workers for generate controller.") flag.IntVar(&maxQueuedEvents, "maxQueuedEvents", 1000, "Maximum events to be queued.") @@ -747,6 +749,9 @@ func main() { policyHandlers, resourceHandlers, configuration, + webhooks.DebugModeOptions{ + DumpPayload: dumpPayload, + }, func() ([]byte, []byte, error) { secret, err := secretLister.Secrets(config.KyvernoNamespace()).Get(tls.GenerateTLSPairSecretName()) if err != nil { diff --git a/pkg/utils/util.go b/pkg/utils/util.go index 8064fe5f1d..7b5955da51 100644 --- a/pkg/utils/util.go +++ b/pkg/utils/util.go @@ -216,6 +216,64 @@ func NormalizeSecret(resource *unstructured.Unstructured) (unstructured.Unstruct return *resource, nil } +// RedactSecret masks keys of data and metadata.annotation fields of Secrets. +func RedactSecret(resource *unstructured.Unstructured) (unstructured.Unstructured, error) { + var secret *corev1.Secret + data, err := json.Marshal(resource.Object) + if err != nil { + return *resource, err + } + err = json.Unmarshal(data, &secret) + if err != nil { + return *resource, errors.Wrap(err, "unable to convert object to secret") + } + stringSecret := struct { + Data map[string]string `json:"string_data"` + *corev1.Secret + }{ + Data: make(map[string]string), + Secret: secret, + } + for key := range secret.Data { + secret.Data[key] = []byte("**REDACTED**") + stringSecret.Data[key] = string(secret.Data[key]) + } + for key := range secret.Annotations { + secret.Annotations[key] = "**REDACTED**" + } + updateSecret := map[string]interface{}{} + raw, err := json.Marshal(stringSecret) + if err != nil { + return *resource, nil + } + err = json.Unmarshal(raw, &updateSecret) + if err != nil { + return *resource, errors.Wrap(err, "unable to convert object from secret") + } + if secret.Data != nil { + v := updateSecret["string_data"].(map[string]interface{}) + err = unstructured.SetNestedMap(resource.Object, v, "data") + if err != nil { + return *resource, errors.Wrap(err, "failed to set secret.data") + } + } + if secret.Annotations != nil { + metadata, err := ToMap(resource.Object["metadata"]) + if err != nil { + return *resource, errors.Wrap(err, "unable to convert metadata to map") + } + updatedMeta := updateSecret["metadata"].(map[string]interface{}) + if err != nil { + return *resource, errors.Wrap(err, "unable to convert object from secret") + } + err = unstructured.SetNestedMap(metadata, updatedMeta["annotations"].(map[string]interface{}), "annotations") + if err != nil { + return *resource, errors.Wrap(err, "failed to set secret.annotations") + } + } + return *resource, nil +} + // HigherThanKubernetesVersion compare Kubernetes client version to user given version func HigherThanKubernetesVersion(client discovery.ServerVersionInterface, log logr.Logger, major, minor, patch int) bool { logger := log.WithName("CompareKubernetesVersion") diff --git a/pkg/webhooks/debug_test.go b/pkg/webhooks/debug_test.go new file mode 100644 index 0000000000..72f89ad27d --- /dev/null +++ b/pkg/webhooks/debug_test.go @@ -0,0 +1,176 @@ +package webhooks + +import ( + "encoding/json" + "testing" + + "github.com/kyverno/kyverno/pkg/utils" + "gotest.tools/assert" + admissionv1 "k8s.io/api/admission/v1" +) + +func Test_RedactPayload(t *testing.T) { + tc := []struct { + name string + requestPayload []byte + }{ + { + name: "request payload with nil old object", + requestPayload: []byte(`{ + "uid":"631a230b-b949-468d-b9ae-927fdd76217e", + "kind":{ + "group":"", + "version":"v1", + "kind":"Secret" + }, + "resource":{ + "group":"", + "version":"v1", + "resource":"secrets" + }, + "requestKind":{ + "group":"", + "version":"v1", + "kind":"Secret" + }, + "requestResource":{ + "group":"", + "version":"v1", + "resource":"secrets" + }, + "name":"mysecret2", + "namespace":"default", + "operation":"CREATE", + "userInfo":{ + "username":"kubernetes-admin", + "groups":["system:masters","system:authenticated"] + }, + "object":{ + "kind":"Secret", + "apiVersion":"v1", + "metadata":{ + "name":"mysecret2", + "namespace":"default", + "uid":"de6f1564-295d-4c57-a10b-f37358414a81", + "creationTimestamp":"2022-10-20T15:17:56Z", + "labels":{ + "purpose":"production" + }, + "annotations":{ + "kubectl.kubernetes.io/last-applied-configuration":"{\"apiVersion\":\"v1\",\"data\":{\"password\":\"MWYyZDFlMmU2N2Rm\",\"username\":\"YWRtaW4=\"},\"kind\":\"Secret\",\"metadata\":{\"annotations\":{},\"labels\":{\"purpose\":\"production\"},\"name\":\"mysecret2\",\"namespace\":\"default\"}}\n"},"managedFields":[{"manager":"kubectl-client-side-apply","operation":"Update","apiVersion":"v1","time":"2022-10-20T15:17:56Z","fieldsType":"FieldsV1","fieldsV1":{"f:data":{".":{},"f:password":{},"f:username":{}},"f:metadata":{"f:annotations":{".":{},"f:kubectl.kubernetes.io/last-applied-configuration":{}},"f:labels":{".":{},"f:purpose":{}}},"f:type":{}}}]}, + "data":{ + "password":"MWYyZDFlMmU2N2Rm", + "username":"YWRtaW4=" + }, + "type":"Opaque" + }, + "oldObject":null, + "dryRun":false, + "options":{ + "kind":"CreateOptions", + "apiVersion":"meta.k8s.io/v1", + "fieldManager":"kubectl-client-side-apply", + "fieldValidation":"Strict" + } + }`), + }, + { + name: "request payload with non nil old object", + requestPayload: []byte(`{ + "uid":"631a230b-b949-468d-b9ae-927fdd76217e", + "kind":{ + "group":"", + "version":"v1", + "kind":"Secret" + }, + "resource":{ + "group":"", + "version":"v1", + "resource":"secrets" + }, + "requestKind":{ + "group":"", + "version":"v1", + "kind":"Secret" + }, + "requestResource":{ + "group":"", + "version":"v1", + "resource":"secrets" + }, + "name":"mysecret2", + "namespace":"default", + "operation":"CREATE", + "userInfo":{ + "username":"kubernetes-admin", + "groups":["system:masters","system:authenticated"] + }, + "object": null, + "oldObject":{ + "kind":"Secret", + "apiVersion":"v1", + "metadata":{ + "name":"mysecret2", + "namespace":"default", + "uid":"de6f1564-295d-4c57-a10b-f37358414a81", + "creationTimestamp":"2022-10-20T15:17:56Z", + "labels":{ + "purpose":"production" + }, + "annotations":{ + "kubectl.kubernetes.io/last-applied-configuration":"{\"apiVersion\":\"v1\",\"data\":{\"password\":\"MWYyZDFlMmU2N2Rm\",\"username\":\"YWRtaW4=\"},\"kind\":\"Secret\",\"metadata\":{\"annotations\":{},\"labels\":{\"purpose\":\"production\"},\"name\":\"mysecret2\",\"namespace\":\"default\"}}\n"},"managedFields":[{"manager":"kubectl-client-side-apply","operation":"Update","apiVersion":"v1","time":"2022-10-20T15:17:56Z","fieldsType":"FieldsV1","fieldsV1":{"f:data":{".":{},"f:password":{},"f:username":{}},"f:metadata":{"f:annotations":{".":{},"f:kubectl.kubernetes.io/last-applied-configuration":{}},"f:labels":{".":{},"f:purpose":{}}},"f:type":{}}}]}, + "data":{ + "password":"MWYyZDFlMmU2N2Rm", + "username":"YWRtaW4=" + }, + "type":"Opaque" + }, + "dryRun":false, + "options":{ + "kind":"CreateOptions", + "apiVersion":"meta.k8s.io/v1", + "fieldManager":"kubectl-client-side-apply", + "fieldValidation":"Strict" + } + }`), + }, + } + + for _, c := range tc { + t.Run(c.name, func(t *testing.T) { + req := new(admissionv1.AdmissionRequest) + err := json.Unmarshal(c.requestPayload, req) + assert.NilError(t, err) + payload, err := newAdmissionRequestPayload(req) + assert.NilError(t, err) + if payload.Object.Object != nil { + data, err := utils.ToMap(payload.Object.Object["data"]) + assert.NilError(t, err) + for _, v := range data { + assert.Assert(t, v == "**REDACTED**") + } + metadata, err := utils.ToMap(payload.Object.Object["metadata"]) + assert.NilError(t, err) + annotations, err := utils.ToMap(metadata["annotations"]) + assert.NilError(t, err) + for _, v := range annotations { + assert.Assert(t, v == "**REDACTED**") + } + } + if payload.OldObject.Object != nil { + data, err := utils.ToMap(payload.OldObject.Object["data"]) + assert.NilError(t, err) + for _, v := range data { + assert.Assert(t, v == "**REDACTED**") + } + metadata, err := utils.ToMap(payload.OldObject.Object["metadata"]) + assert.NilError(t, err) + annotations, err := utils.ToMap(metadata["annotations"]) + assert.NilError(t, err) + for _, v := range annotations { + assert.Assert(t, v == "**REDACTED**") + } + } + }) + } +} diff --git a/pkg/webhooks/server.go b/pkg/webhooks/server.go index e9ac61f8f7..29bfd362b5 100644 --- a/pkg/webhooks/server.go +++ b/pkg/webhooks/server.go @@ -5,12 +5,14 @@ import ( "crypto/tls" "fmt" "net/http" + "strings" "time" "github.com/go-logr/logr" "github.com/julienschmidt/httprouter" kyvernov1 "github.com/kyverno/kyverno/api/kyverno/v1" "github.com/kyverno/kyverno/pkg/config" + engineutils "github.com/kyverno/kyverno/pkg/engine/utils" "github.com/kyverno/kyverno/pkg/logging" "github.com/kyverno/kyverno/pkg/toggle" "github.com/kyverno/kyverno/pkg/utils" @@ -20,12 +22,90 @@ import ( "github.com/kyverno/kyverno/pkg/webhooks/handlers" admissionv1 "k8s.io/api/admission/v1" admissionregistrationv1 "k8s.io/api/admissionregistration/v1" + authenticationv1 "k8s.io/api/authentication/v1" coordinationv1 "k8s.io/api/coordination/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" ) +// DebugModeOptions holds the options to configure debug mode +type DebugModeOptions struct { + // DumpPayload is used to activate/deactivate debug mode. + DumpPayload bool +} + +// AdmissionRequestPayload holds a copy of the AdmissionRequest payload +type AdmissionRequestPayload struct { + UID types.UID `json:"uid"` + Kind metav1.GroupVersionKind `json:"kind"` + Resource metav1.GroupVersionResource `json:"resource"` + SubResource string `json:"subResource,omitempty"` + RequestKind *metav1.GroupVersionKind `json:"requestKind,omitempty"` + RequestResource *metav1.GroupVersionResource `json:"requestResource,omitempty"` + RequestSubResource string `json:"requestSubResource,omitempty"` + Name string `json:"name,omitempty"` + Namespace string `json:"namespace,omitempty"` + Operation string `json:"operation"` + UserInfo authenticationv1.UserInfo `json:"userInfo"` + Object unstructured.Unstructured `json:"object,omitempty"` + OldObject unstructured.Unstructured `json:"oldObject,omitempty"` + DryRun *bool `json:"dryRun,omitempty"` + Options unstructured.Unstructured `json:"options,omitempty"` +} + +func newAdmissionRequestPayload(rq *admissionv1.AdmissionRequest) (*AdmissionRequestPayload, error) { + newResource, oldResource, err := utils.ExtractResources(nil, rq) + if err != nil { + return nil, err + } + options := new(unstructured.Unstructured) + if rq.Options.Raw != nil { + options, err = engineutils.ConvertToUnstructured(rq.Options.Raw) + if err != nil { + return nil, err + } + } + return redactPayload(&AdmissionRequestPayload{ + UID: rq.UID, + Kind: rq.Kind, + Resource: rq.Resource, + SubResource: rq.SubResource, + RequestKind: rq.RequestKind, + RequestResource: rq.RequestResource, + RequestSubResource: rq.RequestSubResource, + Name: rq.Name, + Namespace: rq.Namespace, + Operation: string(rq.Operation), + UserInfo: rq.UserInfo, + Object: newResource, + OldObject: oldResource, + DryRun: rq.DryRun, + Options: *options, + }) +} + +func redactPayload(payload *AdmissionRequestPayload) (*AdmissionRequestPayload, error) { + if strings.EqualFold(payload.Kind.Kind, "Secret") { + if payload.Object.Object != nil { + obj, err := utils.RedactSecret(&payload.Object) + if err != nil { + return nil, err + } + payload.Object = obj + } + if payload.OldObject.Object != nil { + oldObj, err := utils.RedactSecret(&payload.OldObject) + if err != nil { + return nil, err + } + payload.OldObject = oldObj + } + } + return payload, nil +} + type Server interface { // Run TLS server in separate thread and returns control immediately Run(<-chan struct{}) @@ -65,6 +145,7 @@ func NewServer( policyHandlers PolicyHandlers, resourceHandlers ResourceHandlers, configuration config.Configuration, + debugModeOpts DebugModeOptions, tlsProvider TlsProvider, mwcClient controllerutils.DeleteClient[*admissionregistrationv1.MutatingWebhookConfiguration], vwcClient controllerutils.DeleteClient[*admissionregistrationv1.ValidatingWebhookConfiguration], @@ -75,11 +156,11 @@ func NewServer( resourceLogger := logger.WithName("resource") policyLogger := logger.WithName("policy") verifyLogger := logger.WithName("verify") - registerWebhookHandlers(resourceLogger.WithName("mutate"), mux, config.MutatingWebhookServicePath, configuration, resourceHandlers.Mutate) - registerWebhookHandlers(resourceLogger.WithName("validate"), mux, config.ValidatingWebhookServicePath, configuration, resourceHandlers.Validate) - mux.HandlerFunc("POST", config.PolicyMutatingWebhookServicePath, admission(policyLogger.WithName("mutate"), filter(configuration, policyHandlers.Mutate))) - mux.HandlerFunc("POST", config.PolicyValidatingWebhookServicePath, admission(policyLogger.WithName("validate"), filter(configuration, policyHandlers.Validate))) - mux.HandlerFunc("POST", config.VerifyMutatingWebhookServicePath, admission(verifyLogger.WithName("mutate"), handlers.Verify())) + registerWebhookHandlers(resourceLogger.WithName("mutate"), mux, config.MutatingWebhookServicePath, configuration, resourceHandlers.Mutate, debugModeOpts) + registerWebhookHandlers(resourceLogger.WithName("validate"), mux, config.ValidatingWebhookServicePath, configuration, resourceHandlers.Validate, debugModeOpts) + mux.HandlerFunc("POST", config.PolicyMutatingWebhookServicePath, admission(policyLogger.WithName("mutate"), filter(configuration, policyHandlers.Mutate), debugModeOpts)) + mux.HandlerFunc("POST", config.PolicyValidatingWebhookServicePath, admission(policyLogger.WithName("validate"), filter(configuration, policyHandlers.Validate), debugModeOpts)) + mux.HandlerFunc("POST", config.VerifyMutatingWebhookServicePath, admission(verifyLogger.WithName("mutate"), handlers.Verify(), DebugModeOptions{})) mux.HandlerFunc("GET", config.LivenessServicePath, handlers.Probe(runtime.IsLive)) mux.HandlerFunc("GET", config.ReadinessServicePath, handlers.Probe(runtime.IsReady)) return &server{ @@ -167,21 +248,43 @@ func (s *server) cleanup(ctx context.Context) { close(s.cleanUp) } -func protect(inner handlers.AdmissionHandler) handlers.AdmissionHandler { +func dumpPayload(logger logr.Logger, request *admissionv1.AdmissionRequest, response *admissionv1.AdmissionResponse) { + reqPayload, err := newAdmissionRequestPayload(request) + if err != nil { + logger.Error(err, "Failed to extract resources") + } else { + logger.Info("Logging admission request and response payload ", "AdmissionRequest", reqPayload, "AdmissionResponse", response) + } +} + +func dump(inner handlers.AdmissionHandler, debugModeOpts DebugModeOptions) handlers.AdmissionHandler { + // debug mode not enabled, no need to add debug middleware + if !debugModeOpts.DumpPayload { + return inner + } return func(logger logr.Logger, request *admissionv1.AdmissionRequest, startTime time.Time) *admissionv1.AdmissionResponse { - if toggle.ProtectManagedResources.Enabled() { - newResource, oldResource, err := utils.ExtractResources(nil, request) - if err != nil { - logger.Error(err, "Failed to extract resources") - return admissionutils.ResponseFailure(err.Error()) - } - for _, resource := range []unstructured.Unstructured{newResource, oldResource} { - resLabels := resource.GetLabels() - if resLabels[kyvernov1.LabelAppManagedBy] == kyvernov1.ValueKyvernoApp { - if request.UserInfo.Username != fmt.Sprintf("system:serviceaccount:%s:%s", config.KyvernoNamespace(), config.KyvernoServiceAccountName()) { - logger.Info("Access to the resource not authorized, this is a kyverno managed resource and should be altered only by kyverno") - return admissionutils.ResponseFailure("A kyverno managed resource can only be modified by kyverno") - } + response := inner(logger, request, startTime) + dumpPayload(logger, request, response) + return response + } +} + +func protect(inner handlers.AdmissionHandler) handlers.AdmissionHandler { + if !toggle.ProtectManagedResources.Enabled() { + return inner + } + return func(logger logr.Logger, request *admissionv1.AdmissionRequest, startTime time.Time) *admissionv1.AdmissionResponse { + newResource, oldResource, err := utils.ExtractResources(nil, request) + if err != nil { + logger.Error(err, "Failed to extract resources") + return admissionutils.ResponseFailure(err.Error()) + } + for _, resource := range []unstructured.Unstructured{newResource, oldResource} { + resLabels := resource.GetLabels() + if resLabels[kyvernov1.LabelAppManagedBy] == kyvernov1.ValueKyvernoApp { + if request.UserInfo.Username != fmt.Sprintf("system:serviceaccount:%s:%s", config.KyvernoNamespace(), config.KyvernoServiceAccountName()) { + logger.Info("Access to the resource not authorized, this is a kyverno managed resource and should be altered only by kyverno") + return admissionutils.ResponseFailure("A kyverno managed resource can only be modified by kyverno") } } } @@ -193,8 +296,8 @@ func filter(configuration config.Configuration, inner handlers.AdmissionHandler) return handlers.Filter(configuration, inner) } -func admission(logger logr.Logger, inner handlers.AdmissionHandler) http.HandlerFunc { - return handlers.Admission(logger, protect(inner)) +func admission(logger logr.Logger, inner handlers.AdmissionHandler, debugModeOpts DebugModeOptions) http.HandlerFunc { + return handlers.Admission(logger, dump(protect(inner), debugModeOpts)) } func registerWebhookHandlers( @@ -203,23 +306,24 @@ func registerWebhookHandlers( basePath string, configuration config.Configuration, handlerFunc func(logr.Logger, *admissionv1.AdmissionRequest, string, time.Time) *admissionv1.AdmissionResponse, + debugModeOpts DebugModeOptions, ) { mux.HandlerFunc("POST", basePath, admission(logger, filter( configuration, func(logger logr.Logger, request *admissionv1.AdmissionRequest, startTime time.Time) *admissionv1.AdmissionResponse { return handlerFunc(logger, request, "all", startTime) - })), + }), debugModeOpts), ) mux.HandlerFunc("POST", basePath+"/fail", admission(logger, filter( configuration, func(logger logr.Logger, request *admissionv1.AdmissionRequest, startTime time.Time) *admissionv1.AdmissionResponse { return handlerFunc(logger, request, "fail", startTime) - })), + }), debugModeOpts), ) mux.HandlerFunc("POST", basePath+"/ignore", admission(logger, filter( configuration, func(logger logr.Logger, request *admissionv1.AdmissionRequest, startTime time.Time) *admissionv1.AdmissionResponse { return handlerFunc(logger, request, "ignore", startTime) - })), + }), debugModeOpts), ) }