1
0
Fork 0
mirror of https://github.com/kyverno/kyverno.git synced 2025-03-29 02:45:06 +00:00

feat: enable/disable Debug mode which shows entire AdmissionReview payload (#5024)

* work in progress PR

Signed-off-by: damilola olayinka <holayinkajr@gmail.com>

* add custom request struct

Signed-off-by: damilola olayinka <holayinkajr@gmail.com>

* pass debug mode option through constructor and replace logger with klogr

Signed-off-by: damilola olayinka <holayinkajr@gmail.com>

* make changes

Signed-off-by: damilola olayinka <holayinkajr@gmail.com>

* cleanup

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* fix linter

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* add another test case

Signed-off-by: damilola olayinka <holayinkajr@gmail.com>

* removed unused function

Signed-off-by: damilola olayinka <holayinkajr@gmail.com>

* fix linter

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

Signed-off-by: damilola olayinka <holayinkajr@gmail.com>
Signed-off-by: Charles-Edouard Brétéché <charled.breteche@gmail.com>
Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>
Co-authored-by: Charles-Edouard Brétéché <charled.breteche@gmail.com>
Co-authored-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>
This commit is contained in:
yinka 2022-10-21 17:17:49 +01:00 committed by GitHub
parent af787b9fe6
commit 822dbdc011
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 367 additions and 24 deletions

View file

@ -95,6 +95,7 @@ var (
reportsChunkSize int
backgroundScanWorkers int
logFormat string
dumpPayload bool
// DEPRECATED: remove in 1.9
splitPolicyReport bool
)
@ -102,6 +103,7 @@ var (
func parseFlags() error {
logging.Init(nil)
flag.StringVar(&logFormat, "loggingFormat", logging.TextFormat, "This determines the output format of the logger.")
flag.BoolVar(&dumpPayload, "dumpPayload", false, "Set this flag to activate/deactivate debug mode.")
flag.IntVar(&webhookTimeout, "webhookTimeout", webhookcontroller.DefaultWebhookTimeout, "Timeout for webhook configurations.")
flag.IntVar(&genWorkers, "genWorkers", 10, "Workers for generate controller.")
flag.IntVar(&maxQueuedEvents, "maxQueuedEvents", 1000, "Maximum events to be queued.")
@ -747,6 +749,9 @@ func main() {
policyHandlers,
resourceHandlers,
configuration,
webhooks.DebugModeOptions{
DumpPayload: dumpPayload,
},
func() ([]byte, []byte, error) {
secret, err := secretLister.Secrets(config.KyvernoNamespace()).Get(tls.GenerateTLSPairSecretName())
if err != nil {

View file

@ -216,6 +216,64 @@ func NormalizeSecret(resource *unstructured.Unstructured) (unstructured.Unstruct
return *resource, nil
}
// RedactSecret masks keys of data and metadata.annotation fields of Secrets.
func RedactSecret(resource *unstructured.Unstructured) (unstructured.Unstructured, error) {
var secret *corev1.Secret
data, err := json.Marshal(resource.Object)
if err != nil {
return *resource, err
}
err = json.Unmarshal(data, &secret)
if err != nil {
return *resource, errors.Wrap(err, "unable to convert object to secret")
}
stringSecret := struct {
Data map[string]string `json:"string_data"`
*corev1.Secret
}{
Data: make(map[string]string),
Secret: secret,
}
for key := range secret.Data {
secret.Data[key] = []byte("**REDACTED**")
stringSecret.Data[key] = string(secret.Data[key])
}
for key := range secret.Annotations {
secret.Annotations[key] = "**REDACTED**"
}
updateSecret := map[string]interface{}{}
raw, err := json.Marshal(stringSecret)
if err != nil {
return *resource, nil
}
err = json.Unmarshal(raw, &updateSecret)
if err != nil {
return *resource, errors.Wrap(err, "unable to convert object from secret")
}
if secret.Data != nil {
v := updateSecret["string_data"].(map[string]interface{})
err = unstructured.SetNestedMap(resource.Object, v, "data")
if err != nil {
return *resource, errors.Wrap(err, "failed to set secret.data")
}
}
if secret.Annotations != nil {
metadata, err := ToMap(resource.Object["metadata"])
if err != nil {
return *resource, errors.Wrap(err, "unable to convert metadata to map")
}
updatedMeta := updateSecret["metadata"].(map[string]interface{})
if err != nil {
return *resource, errors.Wrap(err, "unable to convert object from secret")
}
err = unstructured.SetNestedMap(metadata, updatedMeta["annotations"].(map[string]interface{}), "annotations")
if err != nil {
return *resource, errors.Wrap(err, "failed to set secret.annotations")
}
}
return *resource, nil
}
// HigherThanKubernetesVersion compare Kubernetes client version to user given version
func HigherThanKubernetesVersion(client discovery.ServerVersionInterface, log logr.Logger, major, minor, patch int) bool {
logger := log.WithName("CompareKubernetesVersion")

176
pkg/webhooks/debug_test.go Normal file
View file

@ -0,0 +1,176 @@
package webhooks
import (
"encoding/json"
"testing"
"github.com/kyverno/kyverno/pkg/utils"
"gotest.tools/assert"
admissionv1 "k8s.io/api/admission/v1"
)
func Test_RedactPayload(t *testing.T) {
tc := []struct {
name string
requestPayload []byte
}{
{
name: "request payload with nil old object",
requestPayload: []byte(`{
"uid":"631a230b-b949-468d-b9ae-927fdd76217e",
"kind":{
"group":"",
"version":"v1",
"kind":"Secret"
},
"resource":{
"group":"",
"version":"v1",
"resource":"secrets"
},
"requestKind":{
"group":"",
"version":"v1",
"kind":"Secret"
},
"requestResource":{
"group":"",
"version":"v1",
"resource":"secrets"
},
"name":"mysecret2",
"namespace":"default",
"operation":"CREATE",
"userInfo":{
"username":"kubernetes-admin",
"groups":["system:masters","system:authenticated"]
},
"object":{
"kind":"Secret",
"apiVersion":"v1",
"metadata":{
"name":"mysecret2",
"namespace":"default",
"uid":"de6f1564-295d-4c57-a10b-f37358414a81",
"creationTimestamp":"2022-10-20T15:17:56Z",
"labels":{
"purpose":"production"
},
"annotations":{
"kubectl.kubernetes.io/last-applied-configuration":"{\"apiVersion\":\"v1\",\"data\":{\"password\":\"MWYyZDFlMmU2N2Rm\",\"username\":\"YWRtaW4=\"},\"kind\":\"Secret\",\"metadata\":{\"annotations\":{},\"labels\":{\"purpose\":\"production\"},\"name\":\"mysecret2\",\"namespace\":\"default\"}}\n"},"managedFields":[{"manager":"kubectl-client-side-apply","operation":"Update","apiVersion":"v1","time":"2022-10-20T15:17:56Z","fieldsType":"FieldsV1","fieldsV1":{"f:data":{".":{},"f:password":{},"f:username":{}},"f:metadata":{"f:annotations":{".":{},"f:kubectl.kubernetes.io/last-applied-configuration":{}},"f:labels":{".":{},"f:purpose":{}}},"f:type":{}}}]},
"data":{
"password":"MWYyZDFlMmU2N2Rm",
"username":"YWRtaW4="
},
"type":"Opaque"
},
"oldObject":null,
"dryRun":false,
"options":{
"kind":"CreateOptions",
"apiVersion":"meta.k8s.io/v1",
"fieldManager":"kubectl-client-side-apply",
"fieldValidation":"Strict"
}
}`),
},
{
name: "request payload with non nil old object",
requestPayload: []byte(`{
"uid":"631a230b-b949-468d-b9ae-927fdd76217e",
"kind":{
"group":"",
"version":"v1",
"kind":"Secret"
},
"resource":{
"group":"",
"version":"v1",
"resource":"secrets"
},
"requestKind":{
"group":"",
"version":"v1",
"kind":"Secret"
},
"requestResource":{
"group":"",
"version":"v1",
"resource":"secrets"
},
"name":"mysecret2",
"namespace":"default",
"operation":"CREATE",
"userInfo":{
"username":"kubernetes-admin",
"groups":["system:masters","system:authenticated"]
},
"object": null,
"oldObject":{
"kind":"Secret",
"apiVersion":"v1",
"metadata":{
"name":"mysecret2",
"namespace":"default",
"uid":"de6f1564-295d-4c57-a10b-f37358414a81",
"creationTimestamp":"2022-10-20T15:17:56Z",
"labels":{
"purpose":"production"
},
"annotations":{
"kubectl.kubernetes.io/last-applied-configuration":"{\"apiVersion\":\"v1\",\"data\":{\"password\":\"MWYyZDFlMmU2N2Rm\",\"username\":\"YWRtaW4=\"},\"kind\":\"Secret\",\"metadata\":{\"annotations\":{},\"labels\":{\"purpose\":\"production\"},\"name\":\"mysecret2\",\"namespace\":\"default\"}}\n"},"managedFields":[{"manager":"kubectl-client-side-apply","operation":"Update","apiVersion":"v1","time":"2022-10-20T15:17:56Z","fieldsType":"FieldsV1","fieldsV1":{"f:data":{".":{},"f:password":{},"f:username":{}},"f:metadata":{"f:annotations":{".":{},"f:kubectl.kubernetes.io/last-applied-configuration":{}},"f:labels":{".":{},"f:purpose":{}}},"f:type":{}}}]},
"data":{
"password":"MWYyZDFlMmU2N2Rm",
"username":"YWRtaW4="
},
"type":"Opaque"
},
"dryRun":false,
"options":{
"kind":"CreateOptions",
"apiVersion":"meta.k8s.io/v1",
"fieldManager":"kubectl-client-side-apply",
"fieldValidation":"Strict"
}
}`),
},
}
for _, c := range tc {
t.Run(c.name, func(t *testing.T) {
req := new(admissionv1.AdmissionRequest)
err := json.Unmarshal(c.requestPayload, req)
assert.NilError(t, err)
payload, err := newAdmissionRequestPayload(req)
assert.NilError(t, err)
if payload.Object.Object != nil {
data, err := utils.ToMap(payload.Object.Object["data"])
assert.NilError(t, err)
for _, v := range data {
assert.Assert(t, v == "**REDACTED**")
}
metadata, err := utils.ToMap(payload.Object.Object["metadata"])
assert.NilError(t, err)
annotations, err := utils.ToMap(metadata["annotations"])
assert.NilError(t, err)
for _, v := range annotations {
assert.Assert(t, v == "**REDACTED**")
}
}
if payload.OldObject.Object != nil {
data, err := utils.ToMap(payload.OldObject.Object["data"])
assert.NilError(t, err)
for _, v := range data {
assert.Assert(t, v == "**REDACTED**")
}
metadata, err := utils.ToMap(payload.OldObject.Object["metadata"])
assert.NilError(t, err)
annotations, err := utils.ToMap(metadata["annotations"])
assert.NilError(t, err)
for _, v := range annotations {
assert.Assert(t, v == "**REDACTED**")
}
}
})
}
}

View file

@ -5,12 +5,14 @@ import (
"crypto/tls"
"fmt"
"net/http"
"strings"
"time"
"github.com/go-logr/logr"
"github.com/julienschmidt/httprouter"
kyvernov1 "github.com/kyverno/kyverno/api/kyverno/v1"
"github.com/kyverno/kyverno/pkg/config"
engineutils "github.com/kyverno/kyverno/pkg/engine/utils"
"github.com/kyverno/kyverno/pkg/logging"
"github.com/kyverno/kyverno/pkg/toggle"
"github.com/kyverno/kyverno/pkg/utils"
@ -20,12 +22,90 @@ import (
"github.com/kyverno/kyverno/pkg/webhooks/handlers"
admissionv1 "k8s.io/api/admission/v1"
admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
authenticationv1 "k8s.io/api/authentication/v1"
coordinationv1 "k8s.io/api/coordination/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/types"
)
// DebugModeOptions holds the options to configure debug mode
type DebugModeOptions struct {
// DumpPayload is used to activate/deactivate debug mode.
DumpPayload bool
}
// AdmissionRequestPayload holds a copy of the AdmissionRequest payload
type AdmissionRequestPayload struct {
UID types.UID `json:"uid"`
Kind metav1.GroupVersionKind `json:"kind"`
Resource metav1.GroupVersionResource `json:"resource"`
SubResource string `json:"subResource,omitempty"`
RequestKind *metav1.GroupVersionKind `json:"requestKind,omitempty"`
RequestResource *metav1.GroupVersionResource `json:"requestResource,omitempty"`
RequestSubResource string `json:"requestSubResource,omitempty"`
Name string `json:"name,omitempty"`
Namespace string `json:"namespace,omitempty"`
Operation string `json:"operation"`
UserInfo authenticationv1.UserInfo `json:"userInfo"`
Object unstructured.Unstructured `json:"object,omitempty"`
OldObject unstructured.Unstructured `json:"oldObject,omitempty"`
DryRun *bool `json:"dryRun,omitempty"`
Options unstructured.Unstructured `json:"options,omitempty"`
}
func newAdmissionRequestPayload(rq *admissionv1.AdmissionRequest) (*AdmissionRequestPayload, error) {
newResource, oldResource, err := utils.ExtractResources(nil, rq)
if err != nil {
return nil, err
}
options := new(unstructured.Unstructured)
if rq.Options.Raw != nil {
options, err = engineutils.ConvertToUnstructured(rq.Options.Raw)
if err != nil {
return nil, err
}
}
return redactPayload(&AdmissionRequestPayload{
UID: rq.UID,
Kind: rq.Kind,
Resource: rq.Resource,
SubResource: rq.SubResource,
RequestKind: rq.RequestKind,
RequestResource: rq.RequestResource,
RequestSubResource: rq.RequestSubResource,
Name: rq.Name,
Namespace: rq.Namespace,
Operation: string(rq.Operation),
UserInfo: rq.UserInfo,
Object: newResource,
OldObject: oldResource,
DryRun: rq.DryRun,
Options: *options,
})
}
func redactPayload(payload *AdmissionRequestPayload) (*AdmissionRequestPayload, error) {
if strings.EqualFold(payload.Kind.Kind, "Secret") {
if payload.Object.Object != nil {
obj, err := utils.RedactSecret(&payload.Object)
if err != nil {
return nil, err
}
payload.Object = obj
}
if payload.OldObject.Object != nil {
oldObj, err := utils.RedactSecret(&payload.OldObject)
if err != nil {
return nil, err
}
payload.OldObject = oldObj
}
}
return payload, nil
}
type Server interface {
// Run TLS server in separate thread and returns control immediately
Run(<-chan struct{})
@ -65,6 +145,7 @@ func NewServer(
policyHandlers PolicyHandlers,
resourceHandlers ResourceHandlers,
configuration config.Configuration,
debugModeOpts DebugModeOptions,
tlsProvider TlsProvider,
mwcClient controllerutils.DeleteClient[*admissionregistrationv1.MutatingWebhookConfiguration],
vwcClient controllerutils.DeleteClient[*admissionregistrationv1.ValidatingWebhookConfiguration],
@ -75,11 +156,11 @@ func NewServer(
resourceLogger := logger.WithName("resource")
policyLogger := logger.WithName("policy")
verifyLogger := logger.WithName("verify")
registerWebhookHandlers(resourceLogger.WithName("mutate"), mux, config.MutatingWebhookServicePath, configuration, resourceHandlers.Mutate)
registerWebhookHandlers(resourceLogger.WithName("validate"), mux, config.ValidatingWebhookServicePath, configuration, resourceHandlers.Validate)
mux.HandlerFunc("POST", config.PolicyMutatingWebhookServicePath, admission(policyLogger.WithName("mutate"), filter(configuration, policyHandlers.Mutate)))
mux.HandlerFunc("POST", config.PolicyValidatingWebhookServicePath, admission(policyLogger.WithName("validate"), filter(configuration, policyHandlers.Validate)))
mux.HandlerFunc("POST", config.VerifyMutatingWebhookServicePath, admission(verifyLogger.WithName("mutate"), handlers.Verify()))
registerWebhookHandlers(resourceLogger.WithName("mutate"), mux, config.MutatingWebhookServicePath, configuration, resourceHandlers.Mutate, debugModeOpts)
registerWebhookHandlers(resourceLogger.WithName("validate"), mux, config.ValidatingWebhookServicePath, configuration, resourceHandlers.Validate, debugModeOpts)
mux.HandlerFunc("POST", config.PolicyMutatingWebhookServicePath, admission(policyLogger.WithName("mutate"), filter(configuration, policyHandlers.Mutate), debugModeOpts))
mux.HandlerFunc("POST", config.PolicyValidatingWebhookServicePath, admission(policyLogger.WithName("validate"), filter(configuration, policyHandlers.Validate), debugModeOpts))
mux.HandlerFunc("POST", config.VerifyMutatingWebhookServicePath, admission(verifyLogger.WithName("mutate"), handlers.Verify(), DebugModeOptions{}))
mux.HandlerFunc("GET", config.LivenessServicePath, handlers.Probe(runtime.IsLive))
mux.HandlerFunc("GET", config.ReadinessServicePath, handlers.Probe(runtime.IsReady))
return &server{
@ -167,21 +248,43 @@ func (s *server) cleanup(ctx context.Context) {
close(s.cleanUp)
}
func protect(inner handlers.AdmissionHandler) handlers.AdmissionHandler {
func dumpPayload(logger logr.Logger, request *admissionv1.AdmissionRequest, response *admissionv1.AdmissionResponse) {
reqPayload, err := newAdmissionRequestPayload(request)
if err != nil {
logger.Error(err, "Failed to extract resources")
} else {
logger.Info("Logging admission request and response payload ", "AdmissionRequest", reqPayload, "AdmissionResponse", response)
}
}
func dump(inner handlers.AdmissionHandler, debugModeOpts DebugModeOptions) handlers.AdmissionHandler {
// debug mode not enabled, no need to add debug middleware
if !debugModeOpts.DumpPayload {
return inner
}
return func(logger logr.Logger, request *admissionv1.AdmissionRequest, startTime time.Time) *admissionv1.AdmissionResponse {
if toggle.ProtectManagedResources.Enabled() {
newResource, oldResource, err := utils.ExtractResources(nil, request)
if err != nil {
logger.Error(err, "Failed to extract resources")
return admissionutils.ResponseFailure(err.Error())
}
for _, resource := range []unstructured.Unstructured{newResource, oldResource} {
resLabels := resource.GetLabels()
if resLabels[kyvernov1.LabelAppManagedBy] == kyvernov1.ValueKyvernoApp {
if request.UserInfo.Username != fmt.Sprintf("system:serviceaccount:%s:%s", config.KyvernoNamespace(), config.KyvernoServiceAccountName()) {
logger.Info("Access to the resource not authorized, this is a kyverno managed resource and should be altered only by kyverno")
return admissionutils.ResponseFailure("A kyverno managed resource can only be modified by kyverno")
}
response := inner(logger, request, startTime)
dumpPayload(logger, request, response)
return response
}
}
func protect(inner handlers.AdmissionHandler) handlers.AdmissionHandler {
if !toggle.ProtectManagedResources.Enabled() {
return inner
}
return func(logger logr.Logger, request *admissionv1.AdmissionRequest, startTime time.Time) *admissionv1.AdmissionResponse {
newResource, oldResource, err := utils.ExtractResources(nil, request)
if err != nil {
logger.Error(err, "Failed to extract resources")
return admissionutils.ResponseFailure(err.Error())
}
for _, resource := range []unstructured.Unstructured{newResource, oldResource} {
resLabels := resource.GetLabels()
if resLabels[kyvernov1.LabelAppManagedBy] == kyvernov1.ValueKyvernoApp {
if request.UserInfo.Username != fmt.Sprintf("system:serviceaccount:%s:%s", config.KyvernoNamespace(), config.KyvernoServiceAccountName()) {
logger.Info("Access to the resource not authorized, this is a kyverno managed resource and should be altered only by kyverno")
return admissionutils.ResponseFailure("A kyverno managed resource can only be modified by kyverno")
}
}
}
@ -193,8 +296,8 @@ func filter(configuration config.Configuration, inner handlers.AdmissionHandler)
return handlers.Filter(configuration, inner)
}
func admission(logger logr.Logger, inner handlers.AdmissionHandler) http.HandlerFunc {
return handlers.Admission(logger, protect(inner))
func admission(logger logr.Logger, inner handlers.AdmissionHandler, debugModeOpts DebugModeOptions) http.HandlerFunc {
return handlers.Admission(logger, dump(protect(inner), debugModeOpts))
}
func registerWebhookHandlers(
@ -203,23 +306,24 @@ func registerWebhookHandlers(
basePath string,
configuration config.Configuration,
handlerFunc func(logr.Logger, *admissionv1.AdmissionRequest, string, time.Time) *admissionv1.AdmissionResponse,
debugModeOpts DebugModeOptions,
) {
mux.HandlerFunc("POST", basePath, admission(logger, filter(
configuration,
func(logger logr.Logger, request *admissionv1.AdmissionRequest, startTime time.Time) *admissionv1.AdmissionResponse {
return handlerFunc(logger, request, "all", startTime)
})),
}), debugModeOpts),
)
mux.HandlerFunc("POST", basePath+"/fail", admission(logger, filter(
configuration,
func(logger logr.Logger, request *admissionv1.AdmissionRequest, startTime time.Time) *admissionv1.AdmissionResponse {
return handlerFunc(logger, request, "fail", startTime)
})),
}), debugModeOpts),
)
mux.HandlerFunc("POST", basePath+"/ignore", admission(logger, filter(
configuration,
func(logger logr.Logger, request *admissionv1.AdmissionRequest, startTime time.Time) *admissionv1.AdmissionResponse {
return handlerFunc(logger, request, "ignore", startTime)
})),
}), debugModeOpts),
)
}