Update to use gvk to store OpenAPI schema (#1906)

* bump swagger doc to 1.21.0

Signed-off-by: Shuting Zhao <shutting06@gmail.com>

* stores openapi schema by gvk

Signed-off-by: Shuting Zhao <shutting06@gmail.com>

* fix schema validation in CLI

Signed-off-by: Shuting Zhao <shutting06@gmail.com>

* add missing resource lists

Signed-off-by: Shuting Zhao <shutting06@gmail.com>

* add e2e tests

Signed-off-by: Shuting Zhao <shutting06@gmail.com>

* address review doc comments

Signed-off-by: Shuting Zhao <shutting06@gmail.com>

go.mod

@@ -14,6 +14,7 @@ require (
 	github.com/go-git/go-billy/v5 v5.0.0
 	github.com/go-git/go-git/v5 v5.2.0
 	github.com/go-logr/logr v0.4.0
+	github.com/google/uuid v1.1.2
 	github.com/googleapis/gnostic v0.5.4
 	github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af
 	github.com/julienschmidt/httprouter v1.3.0

pkg/openapi/crdSync.go

@@ -64,7 +64,11 @@ func NewCRDSync(client *client.Client, controller *Controller) *crdSync {
 }
 
 func (c *crdSync) Run(workers int, stopCh <-chan struct{}) {
-	newDoc, err := c.client.DiscoveryClient.OpenAPISchema()
+	if err := c.updateInClusterKindToAPIVersions(); err != nil {
+		log.Log.Error(err, "failed to update in-cluster api versions")
+	}
+
+	newDoc, err := c.client.DiscoveryClient.DiscoveryCache().OpenAPISchema()
 	if err != nil {
 		log.Log.Error(err, "cannot get OpenAPI schema")
 	}
@@ -85,7 +89,7 @@ func (c *crdSync) Run(workers int, stopCh <-chan struct{}) {
 func (c *crdSync) sync() {
 	crds, err := c.client.GetDynamicInterface().Resource(runtimeSchema.GroupVersionResource{
 		Group:    "apiextensions.k8s.io",
-		Version:  "v1beta1",
+		Version:  "v1",
 		Resource: "customresourcedefinitions",
 	}).List(context.TODO(), v1.ListOptions{})
 	if err != nil {
@@ -98,11 +102,30 @@ func (c *crdSync) sync() {
 	for _, crd := range crds.Items {
 		c.controller.ParseCRD(crd)
 	}
+
+	if err := c.updateInClusterKindToAPIVersions(); err != nil {
+		log.Log.Error(err, "sync failed, unable to update in-cluster api versions")
+	}
+}
+
+func (c *crdSync) updateInClusterKindToAPIVersions() error {
+	_, apiResourceLists, err := c.client.DiscoveryClient.DiscoveryCache().ServerGroupsAndResources()
+	if err != nil {
+		return fmt.Errorf("unable to fetch apiResourceLists: %v", err)
+	}
+
+	preferredAPIResourcesLists, err := c.client.DiscoveryClient.DiscoveryCache().ServerPreferredResources()
+	if err != nil {
+		return fmt.Errorf("unable to fetch apiResourceLists: %v", err)
+	}
+
+	c.controller.updateKindToAPIVersions(apiResourceLists, preferredAPIResourcesLists)
+	return nil
 }
 
 func (o *Controller) deleteCRDFromPreviousSync() {
 	for _, crd := range o.crdList {
-		o.kindToDefinitionName.Remove(crd)
+		o.gvkToDefinitionName.Remove(crd)
 		o.definitions.Remove(crd)
 	}
 
@@ -160,7 +183,7 @@ func (o *Controller) ParseCRD(crd unstructured.Unstructured) {
 	}
 
 	o.crdList = append(o.crdList, crdName)
-	o.kindToDefinitionName.Set(crdName, crdName)
+	o.gvkToDefinitionName.Set(crdName, crdName)
 	o.definitions.Set(crdName, parsedSchema)
 }
 

pkg/openapi/validation.go

@@ -1,6 +1,7 @@
 package openapi
 
 import (
+	"encoding/json"
 	"errors"
 	"fmt"
 	"strconv"
@@ -12,8 +13,10 @@ import (
 	data "github.com/kyverno/kyverno/api"
 	v1 "github.com/kyverno/kyverno/pkg/api/kyverno/v1"
 	"github.com/kyverno/kyverno/pkg/engine"
+	"github.com/kyverno/kyverno/pkg/utils"
 	cmap "github.com/orcaman/concurrent-map"
 	"gopkg.in/yaml.v3"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/kube-openapi/pkg/util/proto"
 	"k8s.io/kube-openapi/pkg/util/proto/validation"
@@ -24,13 +27,28 @@ type concurrentMap struct{ cmap.ConcurrentMap }
 
 // Controller represents OpenAPIController
 type Controller struct {
-	// definitions holds the kind - *openapiv2.Schema map
+	// definitions holds the map of {definitionName: *openapiv2.Schema}
 	definitions concurrentMap
-	// kindToDefinitionName holds the kind - definition map
-	// i.e. - Namespace: io.k8s.api.core.v1.Namespace
-	kindToDefinitionName concurrentMap
-	crdList              []string
-	models               proto.Models
+
+	// kindToDefinitionName holds the map of {(group/version/)kind: definitionName}
+	// i.e. with k8s 1.20.2
+	// - Ingress: io.k8s.api.networking.v1.Ingress (preferred version)
+	// - networking.k8s.io/v1/Ingress: io.k8s.api.networking.v1.Ingress
+	// - networking.k8s.io/v1beta1/Ingress: io.k8s.api.networking.v1beta1.Ingress
+	// - extension/v1beta1/Ingress: io.k8s.api.extensions.v1beta1.Ingress
+	gvkToDefinitionName concurrentMap
+
+	crdList []string
+	models  proto.Models
+
+	// kindToAPIVersions stores the Kind and all its available apiVersions, {kind: apiVersions}
+	kindToAPIVersions concurrentMap
+}
+
+// apiVersions stores all available gvks for a kind, a gvk is "/" seperated string
+type apiVersions struct {
+	serverPreferredGVK string
+	gvks               []string
 }
 
 func newConcurrentMap() concurrentMap {
@@ -58,10 +76,18 @@ func (m concurrentMap) GetSchema(key string) *openapiv2.Schema {
 // NewOpenAPIController initializes a new instance of OpenAPIController
 func NewOpenAPIController() (*Controller, error) {
 	controller := &Controller{
-		definitions:          newConcurrentMap(),
-		kindToDefinitionName: newConcurrentMap(),
+		definitions:         newConcurrentMap(),
+		gvkToDefinitionName: newConcurrentMap(),
+		kindToAPIVersions:   newConcurrentMap(),
 	}
 
+	apiResourceLists, preferredAPIResourcesLists, err := getAPIResourceLists()
+	if err != nil {
+		return nil, err
+	}
+
+	controller.updateKindToAPIVersions(apiResourceLists, preferredAPIResourcesLists)
+
 	defaultDoc, err := getSchemaDocument()
 	if err != nil {
 		return nil, err
@@ -81,10 +107,15 @@ func (o *Controller) ValidatePolicyFields(policy v1.ClusterPolicy) error {
 }
 
 // ValidateResource ...
-func (o *Controller) ValidateResource(patchedResource unstructured.Unstructured, kind string) error {
+func (o *Controller) ValidateResource(patchedResource unstructured.Unstructured, apiVersion, kind string) error {
 	var err error
 
-	kind = o.kindToDefinitionName.GetKind(kind)
+	gvk := kind
+	if apiVersion != "" {
+		gvk = apiVersion + "/" + kind
+	}
+
+	kind = o.gvkToDefinitionName.GetKind(gvk)
 	schema := o.models.LookupModel(kind)
 	if schema == nil {
 		// Check if kind is a CRD
@@ -121,7 +152,7 @@ func (o *Controller) ValidatePolicyMutation(policy v1.ClusterPolicy) error {
 	for kind, rules := range kindToRules {
 		newPolicy := *policy.DeepCopy()
 		newPolicy.Spec.Rules = rules
-		k := o.kindToDefinitionName.GetKind(kind)
+		k := o.gvkToDefinitionName.GetKind(kind)
 		resource, _ := o.generateEmptyResource(o.definitions.GetSchema(k)).(map[string]interface{})
 		if resource == nil || len(resource) == 0 {
 			log.Log.V(2).Info("unable to validate resource. OpenApi definition not found", "kind", kind)
@@ -136,7 +167,7 @@ func (o *Controller) ValidatePolicyMutation(policy v1.ClusterPolicy) error {
 			return err
 		}
 
-		err = o.ValidateResource(*patchedResource.DeepCopy(), kind)
+		err = o.ValidateResource(*patchedResource.DeepCopy(), "", kind)
 		if err != nil {
 			return err
 		}
@@ -147,9 +178,24 @@ func (o *Controller) ValidatePolicyMutation(policy v1.ClusterPolicy) error {
 
 func (o *Controller) useOpenAPIDocument(doc *openapiv2.Document) error {
 	for _, definition := range doc.GetDefinitions().AdditionalProperties {
-		o.definitions.Set(definition.GetName(), definition.GetValue())
-		path := strings.Split(definition.GetName(), ".")
-		o.kindToDefinitionName.Set(path[len(path)-1], definition.GetName())
+		definitionName := definition.GetName()
+		o.definitions.Set(definitionName, definition.GetValue())
+
+		gvk, preferredGVK, err := o.getGVKByDefinitionName(definitionName)
+		if err != nil {
+			log.Log.V(3).Info("unable to cache OpenAPISchema", "definitionName", definitionName, "reason", err.Error())
+			continue
+		}
+
+		if preferredGVK {
+			paths := strings.Split(definitionName, ".")
+			kind := paths[len(paths)-1]
+			o.gvkToDefinitionName.Set(kind, definitionName)
+		}
+
+		if gvk != "" {
+			o.gvkToDefinitionName.Set(gvk, definitionName)
+		}
 	}
 
 	var err error
@@ -161,6 +207,90 @@ func (o *Controller) useOpenAPIDocument(doc *openapiv2.Document) error {
 	return nil
 }
 
+func (o *Controller) getGVKByDefinitionName(definitionName string) (gvk string, preferredGVK bool, err error) {
+	paths := strings.Split(definitionName, ".")
+	kind := paths[len(paths)-1]
+	versions, ok := o.kindToAPIVersions.Get(kind)
+	if !ok {
+		// the kind here is the sub-resource of a K8s Kind, i.e. CronJobStatus
+		// such cases are skipped in schema validation
+		return
+	}
+
+	versionsTyped, ok := versions.(apiVersions)
+	if !ok {
+		return "", preferredGVK, fmt.Errorf("type mismatched, expected apiVersions, got %T", versions)
+	}
+
+	if matchGVK(definitionName, versionsTyped.serverPreferredGVK) {
+		preferredGVK = true
+	}
+
+	for _, gvk := range versionsTyped.gvks {
+		if matchGVK(definitionName, gvk) {
+			return gvk, preferredGVK, nil
+		}
+	}
+
+	return "", preferredGVK, fmt.Errorf("gvk not found by the given definition name %s, %v", definitionName, versionsTyped.gvks)
+}
+
+// matchGVK is a helper function that checks if the
+// given GVK matches the definition name
+func matchGVK(definitionName, gvk string) bool {
+	paths := strings.Split(definitionName, ".")
+
+	gvkMap := make(map[string]bool)
+	for _, p := range paths {
+		gvkMap[p] = true
+	}
+
+	gvkList := strings.Split(gvk, "/")
+	// group can be a dot-seperated string
+	// here we allow at most 1 missing element in group elements, except for Ingress
+	// as a specific element could be missing in apiDocs name
+	// io.k8s.api.rbac.v1.Role - rbac.authorization.k8s.io/v1/Role
+	missingMoreThanOneElement := false
+	for i, element := range gvkList {
+		if i == 0 {
+			items := strings.Split(element, ".")
+			for _, item := range items {
+				_, ok := gvkMap[item]
+				if !ok {
+					if gvkList[len(gvkList)-1] == "Ingress" {
+						return false
+					}
+
+					if missingMoreThanOneElement {
+						return false
+					}
+					missingMoreThanOneElement = true
+				}
+			}
+			continue
+		}
+
+		_, ok := gvkMap[element]
+		if !ok {
+			return false
+		}
+	}
+
+	return true
+}
+
+// updateKindToAPIVersions sets kindToAPIVersions with static manifests
+func (c *Controller) updateKindToAPIVersions(apiResourceLists, preferredAPIResourcesLists []*metav1.APIResourceList) {
+	tempKindToAPIVersions := getAllAPIVersions(apiResourceLists)
+	tempKindToAPIVersions = setPreferredVersions(tempKindToAPIVersions, preferredAPIResourcesLists)
+
+	c.kindToAPIVersions = newConcurrentMap()
+	for key, value := range tempKindToAPIVersions {
+		c.kindToAPIVersions.Set(key, value)
+	}
+
+}
+
 func getSchemaDocument() (*openapiv2.Document, error) {
 	var spec yaml.Node
 	err := yaml.Unmarshal([]byte(data.SwaggerDoc), &spec)
@@ -313,3 +443,84 @@ func getAnyValue(any *openapiv2.Any) []byte {
 
 	return nil
 }
+
+// getAllAPIVersions gets all available versions for a kind
+// returns a map which stores all kinds with its versions
+func getAllAPIVersions(apiResourceLists []*metav1.APIResourceList) map[string]apiVersions {
+	tempKindToAPIVersions := make(map[string]apiVersions)
+
+	for _, apiResourceList := range apiResourceLists {
+		lastKind := ""
+		for _, apiResource := range apiResourceList.APIResources {
+			if apiResource.Kind == lastKind {
+				continue
+			}
+
+			version, ok := tempKindToAPIVersions[apiResource.Kind]
+			if !ok {
+				tempKindToAPIVersions[apiResource.Kind] = apiVersions{}
+			}
+
+			gvk := strings.Join([]string{apiResourceList.GroupVersion, apiResource.Kind}, "/")
+			version.gvks = append(version.gvks, gvk)
+			tempKindToAPIVersions[apiResource.Kind] = version
+			lastKind = apiResource.Kind
+		}
+	}
+
+	return tempKindToAPIVersions
+}
+
+// setPreferredVersions sets the serverPreferredGVK of the given apiVersions map
+func setPreferredVersions(kindToAPIVersions map[string]apiVersions, preferredAPIResourcesLists []*metav1.APIResourceList) map[string]apiVersions {
+	tempKindToAPIVersionsCopied := copyKindToAPIVersions(kindToAPIVersions)
+
+	for kind, versions := range tempKindToAPIVersionsCopied {
+		for _, preferredAPIResourcesList := range preferredAPIResourcesLists {
+			for _, resource := range preferredAPIResourcesList.APIResources {
+				preferredGV := preferredAPIResourcesList.GroupVersion
+				preferredGVK := preferredGV + "/" + resource.Kind
+
+				if utils.ContainsString(versions.gvks, preferredGVK) {
+					v := kindToAPIVersions[kind]
+
+					// if a Kind belongs to multiple groups, the first group/version
+					// returned from discovery docs is used as preferred version
+					// https://github.com/kubernetes/kubernetes/issues/94761#issuecomment-691982480
+					if v.serverPreferredGVK != "" {
+						continue
+					}
+
+					v.serverPreferredGVK = strings.Join([]string{preferredGV, kind}, "/")
+					kindToAPIVersions[kind] = v
+				}
+			}
+		}
+	}
+
+	return kindToAPIVersions
+}
+
+func copyKindToAPIVersions(old map[string]apiVersions) map[string]apiVersions {
+	new := make(map[string]apiVersions, len(old))
+	for key, value := range old {
+		new[key] = value
+	}
+	return new
+}
+
+func getAPIResourceLists() ([]*metav1.APIResourceList, []*metav1.APIResourceList, error) {
+	var apiResourceLists []*metav1.APIResourceList
+	err := json.Unmarshal([]byte(data.APIResourceLists), &apiResourceLists)
+	if err != nil {
+		return nil, nil, fmt.Errorf("unable to load apiResourceLists: %v", err)
+	}
+
+	var preferredAPIResourcesLists []*metav1.APIResourceList
+	err = json.Unmarshal([]byte(data.APIResourceLists), &preferredAPIResourcesLists)
+	if err != nil {
+		return nil, nil, fmt.Errorf("unable to load preferredAPIResourcesLists: %v", err)
+	}
+
+	return apiResourceLists, preferredAPIResourcesLists, nil
+}

pkg/openapi/validation_test.go

@@ -5,6 +5,7 @@ import (
 	"testing"
 
 	v1 "github.com/kyverno/kyverno/pkg/api/kyverno/v1"
+	"gotest.tools/assert"
 )
 
 func Test_ValidateMutationPolicy(t *testing.T) {
@@ -70,3 +71,89 @@ func Test_addDefaultFieldsToSchema(t *testing.T) {
 	addingDefaultFieldsToSchema([]byte(`null`))
 	addingDefaultFieldsToSchema(nil)
 }
+
+func Test_matchGVK(t *testing.T) {
+	testCases := []struct {
+		definitionName string
+		gvk            string
+		match          bool
+	}{
+		{
+			"io.k8s.api.networking.v1.Ingress",
+			"networking.k8s.io/v1/Ingress",
+			true,
+		},
+		{
+			"io.wgpolicyk8s.v1alpha1.PolicyReport",
+			"wgpolicyk8s.io/v1alpha1/PolicyReport",
+			true,
+		},
+		{
+			"io.k8s.api.rbac.v1.RoleBinding",
+			"rbac.authorization.k8s.io/v1/RoleBinding",
+			true,
+		},
+		{
+			"io.k8s.api.rbac.v1beta1.ClusterRoleBinding",
+			"rbac.authorization.k8s.io/v1beta1/ClusterRoleBinding",
+			true,
+		},
+		{
+			"io.k8s.api.rbac.v1.Role",
+			"rbac.authorization.k8s.io/v1/Role",
+			true,
+		},
+		{
+			"io.k8s.api.rbac.v1.ClusterRole",
+			"rbac.authorization.k8s.io/v1/ClusterRole",
+			true,
+		},
+		{
+			"io.k8s.api.flowcontrol.v1beta1.FlowSchema",
+			"flowcontrol.apiserver.k8s.io/v1beta1/FlowSchema",
+			true,
+		},
+		{
+			"io.k8s.api.policy.v1beta1.Eviction",
+			"v1/Eviction",
+			true,
+		},
+		{
+			"io.k8s.api.rbac.v1beta1.ClusterRole",
+			"rbac.authorization.k8s.io/v1beta1/ClusterRole",
+			true,
+		},
+	}
+
+	for i, test := range testCases {
+		res := matchGVK(test.definitionName, test.gvk)
+		assert.Equal(t, res, test.match, "test #%d failed", i)
+	}
+}
+
+// this test covers all supported Ingress in 1.20 cluster
+// networking.k8s.io/v1/Ingress
+// networking.k8s.io/v1beta1/Ingress
+// extensions/v1beta1/Ingress
+func Test_Ingress(t *testing.T) {
+	o, err := NewOpenAPIController()
+	assert.NilError(t, err)
+
+	versions, ok := o.kindToAPIVersions.Get("Ingress")
+	assert.Equal(t, true, ok)
+	versionsTyped := versions.(apiVersions)
+	assert.Equal(t, versionsTyped.serverPreferredGVK, "networking.k8s.io/v1/Ingress")
+	assert.Equal(t, len(versionsTyped.gvks), 3)
+
+	definitionName, _ := o.gvkToDefinitionName.Get("Ingress")
+	assert.Equal(t, definitionName, "io.k8s.api.networking.v1.Ingress")
+
+	definitionName, _ = o.gvkToDefinitionName.Get("networking.k8s.io/v1/Ingress")
+	assert.Equal(t, definitionName, "io.k8s.api.networking.v1.Ingress")
+
+	definitionName, _ = o.gvkToDefinitionName.Get("networking.k8s.io/v1beta1/Ingress")
+	assert.Equal(t, definitionName, "io.k8s.api.networking.v1beta1.Ingress")
+
+	definitionName, _ = o.gvkToDefinitionName.Get("extensions/v1beta1/Ingress")
+	assert.Equal(t, definitionName, "io.k8s.api.extensions.v1beta1.Ingress")
+}

pkg/webhooks/checker.go

@@ -5,7 +5,7 @@ import (
 )
 
 func (ws *WebhookServer) verifyHandler(request *v1beta1.AdmissionRequest) *v1beta1.AdmissionResponse {
-	logger := ws.log.WithValues("action", "verify", "uid", request.UID, "kind", request.Kind, "namespace", request.Namespace, "name", request.Name, "operation", request.Operation)
+	logger := ws.log.WithValues("action", "verify", "uid", request.UID, "kind", request.Kind, "namespace", request.Namespace, "name", request.Name, "operation", request.Operation, "gvk", request.Kind.String())
 	logger.V(4).Info("incoming request")
 	return &v1beta1.AdmissionResponse{
 		Allowed: true,

pkg/webhooks/generation.go

@@ -34,7 +34,7 @@ import (
 
 //HandleGenerate handles admission-requests for policies with generate rules
 func (ws *WebhookServer) HandleGenerate(request *v1beta1.AdmissionRequest, policies []*kyverno.ClusterPolicy, ctx *context.Context, userRequestInfo kyverno.RequestInfo, dynamicConfig config.Interface) {
-	logger := ws.log.WithValues("action", "generation", "uid", request.UID, "kind", request.Kind, "namespace", request.Namespace, "name", request.Name, "operation", request.Operation)
+	logger := ws.log.WithValues("action", "generation", "uid", request.UID, "kind", request.Kind, "namespace", request.Namespace, "name", request.Name, "operation", request.Operation, "gvk", request.Kind.String())
 	logger.V(4).Info("incoming request")
 	var engineResponses []*response.EngineResponse
 	if request.Operation == v1beta1.Create || request.Operation == v1beta1.Update {
@@ -100,7 +100,7 @@ func (ws *WebhookServer) HandleGenerate(request *v1beta1.AdmissionRequest, polic
 
 //handleUpdate handles admission-requests for update
 func (ws *WebhookServer) handleUpdate(request *v1beta1.AdmissionRequest, policies []*kyverno.ClusterPolicy) {
-	logger := ws.log.WithValues("action", "generation", "uid", request.UID, "kind", request.Kind, "namespace", request.Namespace, "name", request.Name, "operation", request.Operation)
+	logger := ws.log.WithValues("action", "generation", "uid", request.UID, "kind", request.Kind, "namespace", request.Namespace, "name", request.Name, "operation", request.Operation, "gvk", request.Kind.String())
 	resource, err := enginutils.ConvertToUnstructured(request.OldObject.Raw)
 	if err != nil {
 		logger.Error(err, "failed to convert object resource to unstructured format")
@@ -293,7 +293,7 @@ func stripNonPolicyFields(obj, newRes map[string]interface{}, logger logr.Logger
 
 //HandleDelete handles admission-requests for delete
 func (ws *WebhookServer) handleDelete(request *v1beta1.AdmissionRequest) {
-	logger := ws.log.WithValues("action", "generation", "uid", request.UID, "kind", request.Kind, "namespace", request.Namespace, "name", request.Name, "operation", request.Operation)
+	logger := ws.log.WithValues("action", "generation", "uid", request.UID, "kind", request.Kind, "namespace", request.Namespace, "name", request.Name, "operation", request.Operation, "gvk", request.Kind.String())
 	resource, err := enginutils.ConvertToUnstructured(request.OldObject.Raw)
 	if err != nil {
 		logger.Error(err, "failed to convert object resource to unstructured format")

pkg/webhooks/mutation.go

@@ -35,7 +35,7 @@ func (ws *WebhookServer) HandleMutation(
 		resourceName = request.Namespace + "/" + resourceName
 	}
 
-	logger := ws.log.WithValues("action", "mutate", "resource", resourceName, "operation", request.Operation)
+	logger := ws.log.WithValues("action", "mutate", "resource", resourceName, "operation", request.Operation, "gvk", request.Kind.String())
 
 	var patches [][]byte
 	var engineResponses []*response.EngineResponse
@@ -73,7 +73,7 @@ func (ws *WebhookServer) HandleMutation(
 			continue
 		}
 
-		err := ws.openAPIController.ValidateResource(*engineResponse.PatchedResource.DeepCopy(), engineResponse.PatchedResource.GetKind())
+		err := ws.openAPIController.ValidateResource(*engineResponse.PatchedResource.DeepCopy(), engineResponse.PatchedResource.GetAPIVersion(), engineResponse.PatchedResource.GetKind())
 		if err != nil {
 			logger.V(4).Info("validation error", "policy", policy.Name, "error", err.Error())
 			continue

pkg/webhooks/policymutation.go

@@ -15,7 +15,7 @@ import (
 )
 
 func (ws *WebhookServer) policyMutation(request *v1beta1.AdmissionRequest) *v1beta1.AdmissionResponse {
-	logger := ws.log.WithValues("action", "policy mutation", "uid", request.UID, "kind", request.Kind, "namespace", request.Namespace, "name", request.Name, "operation", request.Operation)
+	logger := ws.log.WithValues("action", "policy mutation", "uid", request.UID, "kind", request.Kind, "namespace", request.Namespace, "name", request.Name, "operation", request.Operation, "gvk", request.Kind.String())
 	var policy *kyverno.ClusterPolicy
 	raw := request.Object.Raw
 

pkg/webhooks/policyvalidation.go

@@ -13,7 +13,7 @@ import (
 
 //HandlePolicyValidation performs the validation check on policy resource
 func (ws *WebhookServer) policyValidation(request *v1beta1.AdmissionRequest) *v1beta1.AdmissionResponse {
-	logger := ws.log.WithValues("action", "policy validation", "uid", request.UID, "kind", request.Kind, "namespace", request.Namespace, "name", request.Name, "operation", request.Operation)
+	logger := ws.log.WithValues("action", "policy validation", "uid", request.UID, "kind", request.Kind, "namespace", request.Namespace, "name", request.Name, "operation", request.Operation, "gvk", request.Kind.String())
 	var policy *kyverno.ClusterPolicy
 
 	if err := json.Unmarshal(request.Object.Raw, &policy); err != nil {

pkg/webhooks/server.go

@@ -296,7 +296,7 @@ func writeResponse(rw http.ResponseWriter, admissionReview *v1beta1.AdmissionRev
 
 // ResourceMutation mutates resource
 func (ws *WebhookServer) ResourceMutation(request *v1beta1.AdmissionRequest) *v1beta1.AdmissionResponse {
-	logger := ws.log.WithName("ResourceMutation").WithValues("uid", request.UID, "kind", request.Kind.Kind, "namespace", request.Namespace, "name", request.Name, "operation", request.Operation)
+	logger := ws.log.WithName("ResourceMutation").WithValues("uid", request.UID, "kind", request.Kind.Kind, "namespace", request.Namespace, "name", request.Name, "operation", request.Operation, "gvk", request.Kind.String())
 
 	if excludeKyvernoResources(request.Kind.Kind) {
 		return &v1beta1.AdmissionResponse{

pkg/webhooks/validation.go

@@ -51,7 +51,7 @@ func HandleValidation(
 		resourceName = request.Namespace + "/" + resourceName
 	}
 
-	logger := log.WithValues("action", "validate", "resource", resourceName, "operation", request.Operation)
+	logger := log.WithValues("action", "validate", "resource", resourceName, "operation", request.Operation, "gvk", request.Kind.String())
 
 	// Get new and old resource
 	newR, oldR, err := utils.ExtractResources(patchedResource, request)

test/e2e/mutate/config.go

@@ -20,3 +20,47 @@ var MutateTests = []struct {
 		Data:     configMapMutationWithContextLabelSelectionYaml,
 	},
 }
+
+var ingressTests = struct {
+	testNamesapce string
+	cpol          []byte
+	tests         []struct {
+		testName                          string
+		group, version, rsc, resourceName string
+		resource                          []byte
+	}
+}{
+	testNamesapce: "test-ingress",
+	cpol:          mutateIngressCpol,
+	tests: []struct {
+		testName                          string
+		group, version, rsc, resourceName string
+		resource                          []byte
+	}{
+		{
+			testName:     "test-networking-v1-ingress",
+			group:        "networking.k8s.io",
+			version:      "v1",
+			rsc:          "ingresses",
+			resourceName: "kuard-v1",
+			resource:     ingressNetworkingV1,
+		},
+		// the following two tests can be removed after 1.22 cluster
+		{
+			testName:     "test-networking-v1beta1-ingress",
+			group:        "networking.k8s.io",
+			version:      "v1beta1",
+			rsc:          "ingresses",
+			resourceName: "kuard-v1beta1",
+			resource:     ingressNetworkingV1beta1,
+		},
+		{
+			testName:     "test-extensions-v1beta1-ingress",
+			group:        "extensions",
+			version:      "v1beta1",
+			rsc:          "ingresses",
+			resourceName: "kuard-extensions",
+			resource:     ingressExtensionV1beta1,
+		},
+	},
+}

test/e2e/mutate/mutate_test.go

@@ -10,10 +10,11 @@ import (
 	"github.com/kyverno/kyverno/test/e2e"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 )
 
 var (
-	// Cluster Polict GVR
+	// Cluster Policy GVR
 	clPolGVR = e2e.GetGVR("kyverno.io", "v1", "clusterpolicies")
 	// Namespace GVR
 	nsGVR = e2e.GetGVR("", "v1", "namespaces")
@@ -57,7 +58,7 @@ func Test_Mutate_Sets(t *testing.T) {
 
 		// Create Namespace
 		By(fmt.Sprintf("Creating Namespace %s", clPolNS))
-		_, err = e2eClient.CreateClusteredResourceYaml(nsGVR, namespaceYaml)
+		_, err = e2eClient.CreateClusteredResourceYaml(nsGVR, newNamespaceYaml("test-mutate"))
 		Expect(err).NotTo(HaveOccurred())
 
 		// Create source CM
@@ -107,3 +108,67 @@ func Test_Mutate_Sets(t *testing.T) {
 	}
 
 }
+
+func Test_Mutate_Ingress(t *testing.T) {
+	RegisterTestingT(t)
+	if os.Getenv("E2E") == "" {
+		t.Skip("Skipping E2E Test")
+	}
+
+	// Generate E2E Client
+	e2eClient, err := e2e.NewE2EClient()
+	Expect(err).To(BeNil())
+
+	nspace := ingressTests.testNamesapce
+	By(fmt.Sprintf("Cleaning Cluster Policies"))
+	e2eClient.CleanClusterPolicies(clPolGVR)
+
+	By(fmt.Sprintf("Deleting Namespace : %s", nspace))
+	e2eClient.DeleteClusteredResource(nsGVR, nspace)
+
+	// Wait Till Deletion of Namespace
+	err = e2e.GetWithRetry(time.Duration(1), 15, func() error {
+		_, err := e2eClient.GetClusteredResource(nsGVR, nspace)
+		if err != nil {
+			return nil
+		}
+		return errors.New("Deleting Namespace")
+	})
+	Expect(err).To(BeNil())
+
+	By(fmt.Sprintf("Creating mutate ClusterPolicy "))
+	_, err = e2eClient.CreateClusteredResourceYaml(clPolGVR, ingressTests.cpol)
+	Expect(err).NotTo(HaveOccurred())
+
+	By(fmt.Sprintf("Creating Namespace %s", nspace))
+	_, err = e2eClient.CreateClusteredResourceYaml(nsGVR, newNamespaceYaml(nspace))
+	Expect(err).NotTo(HaveOccurred())
+
+	for _, test := range ingressTests.tests {
+		By(fmt.Sprintf("\n\nStart testing %s", test.testName))
+
+		gvr := e2e.GetGVR(test.group, test.version, test.rsc)
+		By(fmt.Sprintf("Creating Ingress %v in %s", gvr, nspace))
+		_, err = e2eClient.CreateNamespacedResourceYaml(gvr, nspace, test.resource)
+		Expect(err).NotTo(HaveOccurred())
+
+		By(fmt.Sprintf("Verifying Ingress %v in the Namespace : %s", gvr, nspace))
+		var mutatedResource *unstructured.Unstructured
+		err = e2e.GetWithRetry(time.Duration(1), 15, func() error {
+			mutatedResource, err = e2eClient.GetNamespacedResource(gvr, nspace, test.resourceName)
+			if err != nil {
+				return err
+			}
+			return nil
+		})
+		Expect(err).To(BeNil())
+
+		By(fmt.Sprintf("Comparing patched field"))
+		rules, ok, err := unstructured.NestedSlice(mutatedResource.UnstructuredContent(), "spec", "rules")
+		Expect(err).To(BeNil())
+		Expect(ok).To(BeTrue())
+		rule := rules[0].(map[string]interface{})
+		host := rule["host"].(string)
+		Expect(host).To(Equal("kuard.mycompany.com"))
+	}
+}

test/e2e/mutate/resources.go

@@ -1,12 +1,17 @@
 package mutate
 
-// Namespace Description
-var namespaceYaml = []byte(`
+import "fmt"
+
+func newNamespaceYaml(name string) []byte {
+	ns := fmt.Sprintf(`
 apiVersion: v1
 kind: Namespace
 metadata:
-  name: test-mutate
-`)
+  name: %s
+`, name)
+
+	return []byte(ns)
+}
 
 // Cluster Policy to copy the copy me label from one configmap to the target
 var configMapMutationYaml = []byte(`
@@ -108,3 +113,95 @@ data:
   data.yaml: |
     some: data
 `)
+
+var mutateIngressCpol = []byte(`
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: mutate-ingress-host
+spec:
+  rules:
+  - name: mutate-rules-host
+    match:
+      resources:
+        kinds:
+        - Ingress
+        namespaces:
+        - test-ingress
+    mutate:
+      patchesJson6902: |-
+        - op: replace
+          path: /spec/rules/0/host
+          value: {{request.object.spec.rules[0].host}}.mycompany.com
+`)
+
+var ingressNetworkingV1 = []byte(`
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: kuard-v1
+  namespace: test-ingress
+  labels:
+    app: kuard
+spec:
+  rules:
+  - host: kuard
+    http:
+      paths:
+      - backend:
+          service: 
+            name: kuard
+            port: 
+              number: 8080
+        path: /
+        pathType: ImplementationSpecific
+  tls:
+  - hosts:
+    - kuard
+`)
+
+var ingressNetworkingV1beta1 = []byte(`
+apiVersion: networking.k8s.io/v1beta1
+kind: Ingress
+metadata:
+  labels:
+    app: kuard
+  name: kuard-v1beta1
+  namespace: test-ingress
+spec:
+  rules:
+  - host: kuard
+    http:
+      paths:
+      - backend:
+          serviceName: kuard
+          servicePort: 8080
+        path: /
+        pathType: ImplementationSpecific
+  tls:
+  - hosts:
+    - kuard
+`)
+
+var ingressExtensionV1beta1 = []byte(`
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+  labels:
+    app: kuard
+  name: kuard-extensions
+  namespace: test-ingress
+spec:
+  rules:
+  - host: kuard
+    http:
+      paths:
+      - backend:
+          serviceName: kuard
+          servicePort: 8080
+        path: /
+        pathType: ImplementationSpecific
+  tls:
+  - hosts:
+    - kuard
+`)