feat: enable/disable Debug mode which shows entire AdmissionReview payload (#5024)

* work in progress PR

Signed-off-by: damilola olayinka <holayinkajr@gmail.com>

* add custom request struct

Signed-off-by: damilola olayinka <holayinkajr@gmail.com>

* pass debug mode option through constructor and replace logger with klogr

Signed-off-by: damilola olayinka <holayinkajr@gmail.com>

* make changes

Signed-off-by: damilola olayinka <holayinkajr@gmail.com>

* cleanup

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* fix linter

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* add another test case

Signed-off-by: damilola olayinka <holayinkajr@gmail.com>

* removed unused function

Signed-off-by: damilola olayinka <holayinkajr@gmail.com>

* fix linter

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

Signed-off-by: damilola olayinka <holayinkajr@gmail.com>
Signed-off-by: Charles-Edouard Brétéché <charled.breteche@gmail.com>
Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>
Co-authored-by: Charles-Edouard Brétéché <charled.breteche@gmail.com>
Co-authored-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

cmd/kyverno/main.go

@@ -95,6 +95,7 @@ var (
 	reportsChunkSize           int
 	backgroundScanWorkers      int
 	logFormat                  string
+	dumpPayload                bool
 	// DEPRECATED: remove in 1.9
 	splitPolicyReport bool
 )
@@ -102,6 +103,7 @@ var (
 func parseFlags() error {
 	logging.Init(nil)
 	flag.StringVar(&logFormat, "loggingFormat", logging.TextFormat, "This determines the output format of the logger.")
+	flag.BoolVar(&dumpPayload, "dumpPayload", false, "Set this flag to activate/deactivate debug mode.")
 	flag.IntVar(&webhookTimeout, "webhookTimeout", webhookcontroller.DefaultWebhookTimeout, "Timeout for webhook configurations.")
 	flag.IntVar(&genWorkers, "genWorkers", 10, "Workers for generate controller.")
 	flag.IntVar(&maxQueuedEvents, "maxQueuedEvents", 1000, "Maximum events to be queued.")
@@ -747,6 +749,9 @@ func main() {
 		policyHandlers,
 		resourceHandlers,
 		configuration,
+		webhooks.DebugModeOptions{
+			DumpPayload: dumpPayload,
+		},
 		func() ([]byte, []byte, error) {
 			secret, err := secretLister.Secrets(config.KyvernoNamespace()).Get(tls.GenerateTLSPairSecretName())
 			if err != nil {

pkg/utils/util.go

@@ -216,6 +216,64 @@ func NormalizeSecret(resource *unstructured.Unstructured) (unstructured.Unstruct
 	return *resource, nil
 }
 
+// RedactSecret masks keys of data and metadata.annotation fields of Secrets.
+func RedactSecret(resource *unstructured.Unstructured) (unstructured.Unstructured, error) {
+	var secret *corev1.Secret
+	data, err := json.Marshal(resource.Object)
+	if err != nil {
+		return *resource, err
+	}
+	err = json.Unmarshal(data, &secret)
+	if err != nil {
+		return *resource, errors.Wrap(err, "unable to convert object to secret")
+	}
+	stringSecret := struct {
+		Data map[string]string `json:"string_data"`
+		*corev1.Secret
+	}{
+		Data:   make(map[string]string),
+		Secret: secret,
+	}
+	for key := range secret.Data {
+		secret.Data[key] = []byte("**REDACTED**")
+		stringSecret.Data[key] = string(secret.Data[key])
+	}
+	for key := range secret.Annotations {
+		secret.Annotations[key] = "**REDACTED**"
+	}
+	updateSecret := map[string]interface{}{}
+	raw, err := json.Marshal(stringSecret)
+	if err != nil {
+		return *resource, nil
+	}
+	err = json.Unmarshal(raw, &updateSecret)
+	if err != nil {
+		return *resource, errors.Wrap(err, "unable to convert object from secret")
+	}
+	if secret.Data != nil {
+		v := updateSecret["string_data"].(map[string]interface{})
+		err = unstructured.SetNestedMap(resource.Object, v, "data")
+		if err != nil {
+			return *resource, errors.Wrap(err, "failed to set secret.data")
+		}
+	}
+	if secret.Annotations != nil {
+		metadata, err := ToMap(resource.Object["metadata"])
+		if err != nil {
+			return *resource, errors.Wrap(err, "unable to convert metadata to map")
+		}
+		updatedMeta := updateSecret["metadata"].(map[string]interface{})
+		if err != nil {
+			return *resource, errors.Wrap(err, "unable to convert object from secret")
+		}
+		err = unstructured.SetNestedMap(metadata, updatedMeta["annotations"].(map[string]interface{}), "annotations")
+		if err != nil {
+			return *resource, errors.Wrap(err, "failed to set secret.annotations")
+		}
+	}
+	return *resource, nil
+}
+
 // HigherThanKubernetesVersion compare Kubernetes client version to user given version
 func HigherThanKubernetesVersion(client discovery.ServerVersionInterface, log logr.Logger, major, minor, patch int) bool {
 	logger := log.WithName("CompareKubernetesVersion")

pkg/webhooks/debug_test.go

@@ -0,0 +1,176 @@
+package webhooks
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/kyverno/kyverno/pkg/utils"
+	"gotest.tools/assert"
+	admissionv1 "k8s.io/api/admission/v1"
+)
+
+func Test_RedactPayload(t *testing.T) {
+	tc := []struct {
+		name           string
+		requestPayload []byte
+	}{
+		{
+			name: "request payload with nil old object",
+			requestPayload: []byte(`{
+				"uid":"631a230b-b949-468d-b9ae-927fdd76217e",
+				"kind":{
+					"group":"",
+					"version":"v1",
+					"kind":"Secret"
+				},
+				"resource":{
+					"group":"",
+					"version":"v1",
+					"resource":"secrets"
+				},
+				"requestKind":{
+					"group":"",
+					"version":"v1",
+					"kind":"Secret"
+				},
+				"requestResource":{
+					"group":"",
+					"version":"v1",
+					"resource":"secrets"
+				},
+				"name":"mysecret2",
+				"namespace":"default",
+				"operation":"CREATE",
+				"userInfo":{
+					"username":"kubernetes-admin",
+					"groups":["system:masters","system:authenticated"]
+				},
+				"object":{
+					"kind":"Secret",
+					"apiVersion":"v1",
+					"metadata":{
+						"name":"mysecret2",
+						"namespace":"default",
+						"uid":"de6f1564-295d-4c57-a10b-f37358414a81",
+						"creationTimestamp":"2022-10-20T15:17:56Z",
+						"labels":{
+							"purpose":"production"
+						},
+						"annotations":{
+							"kubectl.kubernetes.io/last-applied-configuration":"{\"apiVersion\":\"v1\",\"data\":{\"password\":\"MWYyZDFlMmU2N2Rm\",\"username\":\"YWRtaW4=\"},\"kind\":\"Secret\",\"metadata\":{\"annotations\":{},\"labels\":{\"purpose\":\"production\"},\"name\":\"mysecret2\",\"namespace\":\"default\"}}\n"},"managedFields":[{"manager":"kubectl-client-side-apply","operation":"Update","apiVersion":"v1","time":"2022-10-20T15:17:56Z","fieldsType":"FieldsV1","fieldsV1":{"f:data":{".":{},"f:password":{},"f:username":{}},"f:metadata":{"f:annotations":{".":{},"f:kubectl.kubernetes.io/last-applied-configuration":{}},"f:labels":{".":{},"f:purpose":{}}},"f:type":{}}}]},
+					"data":{
+						"password":"MWYyZDFlMmU2N2Rm",
+						"username":"YWRtaW4="
+					},
+					"type":"Opaque"
+				},
+				"oldObject":null,
+				"dryRun":false,
+				"options":{
+					"kind":"CreateOptions",
+					"apiVersion":"meta.k8s.io/v1",
+					"fieldManager":"kubectl-client-side-apply",
+					"fieldValidation":"Strict"
+				}
+			}`),
+		},
+		{
+			name: "request payload with non nil old object",
+			requestPayload: []byte(`{
+				"uid":"631a230b-b949-468d-b9ae-927fdd76217e",
+				"kind":{
+					"group":"",
+					"version":"v1",
+					"kind":"Secret"
+				},
+				"resource":{
+					"group":"",
+					"version":"v1",
+					"resource":"secrets"
+				},
+				"requestKind":{
+					"group":"",
+					"version":"v1",
+					"kind":"Secret"
+				},
+				"requestResource":{
+					"group":"",
+					"version":"v1",
+					"resource":"secrets"
+				},
+				"name":"mysecret2",
+				"namespace":"default",
+				"operation":"CREATE",
+				"userInfo":{
+					"username":"kubernetes-admin",
+					"groups":["system:masters","system:authenticated"]
+				},
+				"object": null,
+				"oldObject":{
+					"kind":"Secret",
+					"apiVersion":"v1",
+					"metadata":{
+						"name":"mysecret2",
+						"namespace":"default",
+						"uid":"de6f1564-295d-4c57-a10b-f37358414a81",
+						"creationTimestamp":"2022-10-20T15:17:56Z",
+						"labels":{
+							"purpose":"production"
+						},
+						"annotations":{
+							"kubectl.kubernetes.io/last-applied-configuration":"{\"apiVersion\":\"v1\",\"data\":{\"password\":\"MWYyZDFlMmU2N2Rm\",\"username\":\"YWRtaW4=\"},\"kind\":\"Secret\",\"metadata\":{\"annotations\":{},\"labels\":{\"purpose\":\"production\"},\"name\":\"mysecret2\",\"namespace\":\"default\"}}\n"},"managedFields":[{"manager":"kubectl-client-side-apply","operation":"Update","apiVersion":"v1","time":"2022-10-20T15:17:56Z","fieldsType":"FieldsV1","fieldsV1":{"f:data":{".":{},"f:password":{},"f:username":{}},"f:metadata":{"f:annotations":{".":{},"f:kubectl.kubernetes.io/last-applied-configuration":{}},"f:labels":{".":{},"f:purpose":{}}},"f:type":{}}}]},
+					"data":{
+						"password":"MWYyZDFlMmU2N2Rm",
+						"username":"YWRtaW4="
+					},
+					"type":"Opaque"
+				},
+				"dryRun":false,
+				"options":{
+					"kind":"CreateOptions",
+					"apiVersion":"meta.k8s.io/v1",
+					"fieldManager":"kubectl-client-side-apply",
+					"fieldValidation":"Strict"
+				}
+			}`),
+		},
+	}
+
+	for _, c := range tc {
+		t.Run(c.name, func(t *testing.T) {
+			req := new(admissionv1.AdmissionRequest)
+			err := json.Unmarshal(c.requestPayload, req)
+			assert.NilError(t, err)
+			payload, err := newAdmissionRequestPayload(req)
+			assert.NilError(t, err)
+			if payload.Object.Object != nil {
+				data, err := utils.ToMap(payload.Object.Object["data"])
+				assert.NilError(t, err)
+				for _, v := range data {
+					assert.Assert(t, v == "**REDACTED**")
+				}
+				metadata, err := utils.ToMap(payload.Object.Object["metadata"])
+				assert.NilError(t, err)
+				annotations, err := utils.ToMap(metadata["annotations"])
+				assert.NilError(t, err)
+				for _, v := range annotations {
+					assert.Assert(t, v == "**REDACTED**")
+				}
+			}
+			if payload.OldObject.Object != nil {
+				data, err := utils.ToMap(payload.OldObject.Object["data"])
+				assert.NilError(t, err)
+				for _, v := range data {
+					assert.Assert(t, v == "**REDACTED**")
+				}
+				metadata, err := utils.ToMap(payload.OldObject.Object["metadata"])
+				assert.NilError(t, err)
+				annotations, err := utils.ToMap(metadata["annotations"])
+				assert.NilError(t, err)
+				for _, v := range annotations {
+					assert.Assert(t, v == "**REDACTED**")
+				}
+			}
+		})
+	}
+}

pkg/webhooks/server.go

@@ -5,12 +5,14 @@ import (
 	"crypto/tls"
 	"fmt"
 	"net/http"
+	"strings"
 	"time"
 
 	"github.com/go-logr/logr"
 	"github.com/julienschmidt/httprouter"
 	kyvernov1 "github.com/kyverno/kyverno/api/kyverno/v1"
 	"github.com/kyverno/kyverno/pkg/config"
+	engineutils "github.com/kyverno/kyverno/pkg/engine/utils"
 	"github.com/kyverno/kyverno/pkg/logging"
 	"github.com/kyverno/kyverno/pkg/toggle"
 	"github.com/kyverno/kyverno/pkg/utils"
@@ -20,12 +22,90 @@ import (
 	"github.com/kyverno/kyverno/pkg/webhooks/handlers"
 	admissionv1 "k8s.io/api/admission/v1"
 	admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
+	authenticationv1 "k8s.io/api/authentication/v1"
 	coordinationv1 "k8s.io/api/coordination/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/types"
 )
 
+// DebugModeOptions holds the options to configure debug mode
+type DebugModeOptions struct {
+	// DumpPayload is used to activate/deactivate debug mode.
+	DumpPayload bool
+}
+
+// AdmissionRequestPayload holds a copy of the AdmissionRequest payload
+type AdmissionRequestPayload struct {
+	UID                types.UID                    `json:"uid"`
+	Kind               metav1.GroupVersionKind      `json:"kind"`
+	Resource           metav1.GroupVersionResource  `json:"resource"`
+	SubResource        string                       `json:"subResource,omitempty"`
+	RequestKind        *metav1.GroupVersionKind     `json:"requestKind,omitempty"`
+	RequestResource    *metav1.GroupVersionResource `json:"requestResource,omitempty"`
+	RequestSubResource string                       `json:"requestSubResource,omitempty"`
+	Name               string                       `json:"name,omitempty"`
+	Namespace          string                       `json:"namespace,omitempty"`
+	Operation          string                       `json:"operation"`
+	UserInfo           authenticationv1.UserInfo    `json:"userInfo"`
+	Object             unstructured.Unstructured    `json:"object,omitempty"`
+	OldObject          unstructured.Unstructured    `json:"oldObject,omitempty"`
+	DryRun             *bool                        `json:"dryRun,omitempty"`
+	Options            unstructured.Unstructured    `json:"options,omitempty"`
+}
+
+func newAdmissionRequestPayload(rq *admissionv1.AdmissionRequest) (*AdmissionRequestPayload, error) {
+	newResource, oldResource, err := utils.ExtractResources(nil, rq)
+	if err != nil {
+		return nil, err
+	}
+	options := new(unstructured.Unstructured)
+	if rq.Options.Raw != nil {
+		options, err = engineutils.ConvertToUnstructured(rq.Options.Raw)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return redactPayload(&AdmissionRequestPayload{
+		UID:                rq.UID,
+		Kind:               rq.Kind,
+		Resource:           rq.Resource,
+		SubResource:        rq.SubResource,
+		RequestKind:        rq.RequestKind,
+		RequestResource:    rq.RequestResource,
+		RequestSubResource: rq.RequestSubResource,
+		Name:               rq.Name,
+		Namespace:          rq.Namespace,
+		Operation:          string(rq.Operation),
+		UserInfo:           rq.UserInfo,
+		Object:             newResource,
+		OldObject:          oldResource,
+		DryRun:             rq.DryRun,
+		Options:            *options,
+	})
+}
+
+func redactPayload(payload *AdmissionRequestPayload) (*AdmissionRequestPayload, error) {
+	if strings.EqualFold(payload.Kind.Kind, "Secret") {
+		if payload.Object.Object != nil {
+			obj, err := utils.RedactSecret(&payload.Object)
+			if err != nil {
+				return nil, err
+			}
+			payload.Object = obj
+		}
+		if payload.OldObject.Object != nil {
+			oldObj, err := utils.RedactSecret(&payload.OldObject)
+			if err != nil {
+				return nil, err
+			}
+			payload.OldObject = oldObj
+		}
+	}
+	return payload, nil
+}
+
 type Server interface {
 	// Run TLS server in separate thread and returns control immediately
 	Run(<-chan struct{})
@@ -65,6 +145,7 @@ func NewServer(
 	policyHandlers PolicyHandlers,
 	resourceHandlers ResourceHandlers,
 	configuration config.Configuration,
+	debugModeOpts DebugModeOptions,
 	tlsProvider TlsProvider,
 	mwcClient controllerutils.DeleteClient[*admissionregistrationv1.MutatingWebhookConfiguration],
 	vwcClient controllerutils.DeleteClient[*admissionregistrationv1.ValidatingWebhookConfiguration],
@@ -75,11 +156,11 @@ func NewServer(
 	resourceLogger := logger.WithName("resource")
 	policyLogger := logger.WithName("policy")
 	verifyLogger := logger.WithName("verify")
-	registerWebhookHandlers(resourceLogger.WithName("mutate"), mux, config.MutatingWebhookServicePath, configuration, resourceHandlers.Mutate)
+	registerWebhookHandlers(resourceLogger.WithName("mutate"), mux, config.MutatingWebhookServicePath, configuration, resourceHandlers.Mutate, debugModeOpts)
-	registerWebhookHandlers(resourceLogger.WithName("validate"), mux, config.ValidatingWebhookServicePath, configuration, resourceHandlers.Validate)
+	registerWebhookHandlers(resourceLogger.WithName("validate"), mux, config.ValidatingWebhookServicePath, configuration, resourceHandlers.Validate, debugModeOpts)
-	mux.HandlerFunc("POST", config.PolicyMutatingWebhookServicePath, admission(policyLogger.WithName("mutate"), filter(configuration, policyHandlers.Mutate)))
+	mux.HandlerFunc("POST", config.PolicyMutatingWebhookServicePath, admission(policyLogger.WithName("mutate"), filter(configuration, policyHandlers.Mutate), debugModeOpts))
-	mux.HandlerFunc("POST", config.PolicyValidatingWebhookServicePath, admission(policyLogger.WithName("validate"), filter(configuration, policyHandlers.Validate)))
+	mux.HandlerFunc("POST", config.PolicyValidatingWebhookServicePath, admission(policyLogger.WithName("validate"), filter(configuration, policyHandlers.Validate), debugModeOpts))
-	mux.HandlerFunc("POST", config.VerifyMutatingWebhookServicePath, admission(verifyLogger.WithName("mutate"), handlers.Verify()))
+	mux.HandlerFunc("POST", config.VerifyMutatingWebhookServicePath, admission(verifyLogger.WithName("mutate"), handlers.Verify(), DebugModeOptions{}))
 	mux.HandlerFunc("GET", config.LivenessServicePath, handlers.Probe(runtime.IsLive))
 	mux.HandlerFunc("GET", config.ReadinessServicePath, handlers.Probe(runtime.IsReady))
 	return &server{
@@ -167,21 +248,43 @@ func (s *server) cleanup(ctx context.Context) {
 	close(s.cleanUp)
 }
 
-func protect(inner handlers.AdmissionHandler) handlers.AdmissionHandler {
+func dumpPayload(logger logr.Logger, request *admissionv1.AdmissionRequest, response *admissionv1.AdmissionResponse) {
+	reqPayload, err := newAdmissionRequestPayload(request)
+	if err != nil {
+		logger.Error(err, "Failed to extract resources")
+	} else {
+		logger.Info("Logging admission request and response payload ", "AdmissionRequest", reqPayload, "AdmissionResponse", response)
+	}
+}
+
+func dump(inner handlers.AdmissionHandler, debugModeOpts DebugModeOptions) handlers.AdmissionHandler {
+	// debug mode not enabled, no need to add debug middleware
+	if !debugModeOpts.DumpPayload {
+		return inner
+	}
 	return func(logger logr.Logger, request *admissionv1.AdmissionRequest, startTime time.Time) *admissionv1.AdmissionResponse {
-		if toggle.ProtectManagedResources.Enabled() {
+		response := inner(logger, request, startTime)
-			newResource, oldResource, err := utils.ExtractResources(nil, request)
+		dumpPayload(logger, request, response)
-			if err != nil {
+		return response
-				logger.Error(err, "Failed to extract resources")
+	}
-				return admissionutils.ResponseFailure(err.Error())
+}
-			}
+
-			for _, resource := range []unstructured.Unstructured{newResource, oldResource} {
+func protect(inner handlers.AdmissionHandler) handlers.AdmissionHandler {
-				resLabels := resource.GetLabels()
+	if !toggle.ProtectManagedResources.Enabled() {
-				if resLabels[kyvernov1.LabelAppManagedBy] == kyvernov1.ValueKyvernoApp {
+		return inner
-					if request.UserInfo.Username != fmt.Sprintf("system:serviceaccount:%s:%s", config.KyvernoNamespace(), config.KyvernoServiceAccountName()) {
+	}
-						logger.Info("Access to the resource not authorized, this is a kyverno managed resource and should be altered only by kyverno")
+	return func(logger logr.Logger, request *admissionv1.AdmissionRequest, startTime time.Time) *admissionv1.AdmissionResponse {
-						return admissionutils.ResponseFailure("A kyverno managed resource can only be modified by kyverno")
+		newResource, oldResource, err := utils.ExtractResources(nil, request)
-					}
+		if err != nil {
+			logger.Error(err, "Failed to extract resources")
+			return admissionutils.ResponseFailure(err.Error())
+		}
+		for _, resource := range []unstructured.Unstructured{newResource, oldResource} {
+			resLabels := resource.GetLabels()
+			if resLabels[kyvernov1.LabelAppManagedBy] == kyvernov1.ValueKyvernoApp {
+				if request.UserInfo.Username != fmt.Sprintf("system:serviceaccount:%s:%s", config.KyvernoNamespace(), config.KyvernoServiceAccountName()) {
+					logger.Info("Access to the resource not authorized, this is a kyverno managed resource and should be altered only by kyverno")
+					return admissionutils.ResponseFailure("A kyverno managed resource can only be modified by kyverno")
 				}
 			}
 		}
@@ -193,8 +296,8 @@ func filter(configuration config.Configuration, inner handlers.AdmissionHandler)
 	return handlers.Filter(configuration, inner)
 }
 
-func admission(logger logr.Logger, inner handlers.AdmissionHandler) http.HandlerFunc {
+func admission(logger logr.Logger, inner handlers.AdmissionHandler, debugModeOpts DebugModeOptions) http.HandlerFunc {
-	return handlers.Admission(logger, protect(inner))
+	return handlers.Admission(logger, dump(protect(inner), debugModeOpts))
 }
 
 func registerWebhookHandlers(
@@ -203,23 +306,24 @@ func registerWebhookHandlers(
 	basePath string,
 	configuration config.Configuration,
 	handlerFunc func(logr.Logger, *admissionv1.AdmissionRequest, string, time.Time) *admissionv1.AdmissionResponse,
+	debugModeOpts DebugModeOptions,
 ) {
 	mux.HandlerFunc("POST", basePath, admission(logger, filter(
 		configuration,
 		func(logger logr.Logger, request *admissionv1.AdmissionRequest, startTime time.Time) *admissionv1.AdmissionResponse {
 			return handlerFunc(logger, request, "all", startTime)
-		})),
+		}), debugModeOpts),
 	)
 	mux.HandlerFunc("POST", basePath+"/fail", admission(logger, filter(
 		configuration,
 		func(logger logr.Logger, request *admissionv1.AdmissionRequest, startTime time.Time) *admissionv1.AdmissionResponse {
 			return handlerFunc(logger, request, "fail", startTime)
-		})),
+		}), debugModeOpts),
 	)
 	mux.HandlerFunc("POST", basePath+"/ignore", admission(logger, filter(
 		configuration,
 		func(logger logr.Logger, request *admissionv1.AdmissionRequest, startTime time.Time) *admissionv1.AdmissionResponse {
 			return handlerFunc(logger, request, "ignore", startTime)
-		})),
+		}), debugModeOpts),
 	)
 }