diff --git a/cmd/cli/kubectl-kyverno/apply/apply_command.go b/cmd/cli/kubectl-kyverno/apply/apply_command.go index b1af3be4c0..8ebb793b12 100644 --- a/cmd/cli/kubectl-kyverno/apply/apply_command.go +++ b/cmd/cli/kubectl-kyverno/apply/apply_command.go @@ -185,7 +185,7 @@ func (c *ApplyCommandConfig) applyCommandHelper() (rc *common.ResultCounts, reso return rc, resources, skipInvalidPolicies, pvInfos, err } - openAPIController, err := openapi.NewOpenAPIController() + openApiManager, err := openapi.NewOpenAPIManager() if err != nil { return rc, resources, skipInvalidPolicies, pvInfos, sanitizederror.NewWithError("failed to initialize openAPIController", err) } @@ -322,7 +322,7 @@ func (c *ApplyCommandConfig) applyCommandHelper() (rc *common.ResultCounts, reso skipInvalidPolicies.invalid = make([]string, 0) for _, policy := range mutatedPolicies { - _, err := policy2.Validate(policy, nil, true, openAPIController) + _, err := policy2.Validate(policy, nil, true, openApiManager) if err != nil { log.Log.Error(err, "policy validation error") if strings.HasPrefix(err.Error(), "variable 'element.name'") { diff --git a/cmd/cli/kubectl-kyverno/test/test_command.go b/cmd/cli/kubectl-kyverno/test/test_command.go index 248f42a5ca..b4d31dd688 100644 --- a/cmd/cli/kubectl-kyverno/test/test_command.go +++ b/cmd/cli/kubectl-kyverno/test/test_command.go @@ -362,7 +362,7 @@ func testCommandExecute(dirPath []string, fileName string, gitBranch string, tes tf.enabled = false } - openAPIController, err := openapi.NewOpenAPIController() + openAPIController, err := openapi.NewOpenAPIManager() if err != nil { return rc, fmt.Errorf("unable to create open api controller, %w", err) } @@ -480,7 +480,7 @@ func testCommandExecute(dirPath []string, fileName string, gitBranch string, tes return rc, nil } -func getLocalDirTestFiles(fs billy.Filesystem, path, fileName string, rc *resultCounts, testFiles *int, openAPIController *openapi.Controller, tf *testFilter, failOnly, removeColor bool) []error { +func getLocalDirTestFiles(fs billy.Filesystem, path, fileName string, rc *resultCounts, testFiles *int, openApiManager *openapi.Manager, tf *testFilter, failOnly, removeColor bool) []error { var errors []error files, err := os.ReadDir(path) @@ -489,7 +489,7 @@ func getLocalDirTestFiles(fs billy.Filesystem, path, fileName string, rc *result } for _, file := range files { if file.IsDir() { - getLocalDirTestFiles(fs, filepath.Join(path, file.Name()), fileName, rc, testFiles, openAPIController, tf, failOnly, removeColor) + getLocalDirTestFiles(fs, filepath.Join(path, file.Name()), fileName, rc, testFiles, openApiManager, tf, failOnly, removeColor) continue } if file.Name() == fileName { @@ -505,7 +505,7 @@ func getLocalDirTestFiles(fs billy.Filesystem, path, fileName string, rc *result errors = append(errors, sanitizederror.NewWithError("failed to convert json", err)) continue } - if err := applyPoliciesFromPath(fs, valuesBytes, false, path, rc, openAPIController, tf, failOnly, removeColor); err != nil { + if err := applyPoliciesFromPath(fs, valuesBytes, false, path, rc, openApiManager, tf, failOnly, removeColor); err != nil { errors = append(errors, sanitizederror.NewWithError(fmt.Sprintf("failed to apply test command from file %s", file.Name()), err)) continue } @@ -819,7 +819,7 @@ func getFullPath(paths []string, policyResourcePath string, isGit bool) []string return paths } -func applyPoliciesFromPath(fs billy.Filesystem, policyBytes []byte, isGit bool, policyResourcePath string, rc *resultCounts, openAPIController *openapi.Controller, tf *testFilter, failOnly, removeColor bool) (err error) { +func applyPoliciesFromPath(fs billy.Filesystem, policyBytes []byte, isGit bool, policyResourcePath string, rc *resultCounts, openAPIController *openapi.Manager, tf *testFilter, failOnly, removeColor bool) (err error) { engineResponses := make([]*response.EngineResponse, 0) var dClient dclient.Interface values := &Test{} diff --git a/cmd/kyverno/main.go b/cmd/kyverno/main.go index bc0569011a..7dcc204316 100644 --- a/cmd/kyverno/main.go +++ b/cmd/kyverno/main.go @@ -303,7 +303,7 @@ func createNonLeaderControllers( configuration config.Configuration, policyCache policycache.Cache, eventGenerator event.Interface, - manager *openapi.Controller, + manager *openapi.Manager, ) ([]controller, func() error) { policyCacheController := policycachecontroller.NewController( policyCache, @@ -558,7 +558,7 @@ func main() { logger.Error(err, "failed to initialize configuration") os.Exit(1) } - openApiManager, err := openapi.NewOpenAPIController() + openApiManager, err := openapi.NewOpenAPIManager() if err != nil { logger.Error(err, "Failed to create openapi manager") os.Exit(1) diff --git a/pkg/openapi/crdSync.go b/pkg/openapi/crdSync.go index dd028dcabb..a3734f3608 100644 --- a/pkg/openapi/crdSync.go +++ b/pkg/openapi/crdSync.go @@ -2,29 +2,24 @@ package openapi import ( "context" - "encoding/json" "fmt" "strings" "time" - "github.com/google/gnostic/compiler" - openapiv2 "github.com/google/gnostic/openapiv2" "github.com/kyverno/kyverno/pkg/clients/dclient" "github.com/kyverno/kyverno/pkg/logging" "github.com/kyverno/kyverno/pkg/metrics" util "github.com/kyverno/kyverno/pkg/utils" "github.com/pkg/errors" - "gopkg.in/yaml.v3" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" runtimeSchema "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/discovery" ) type crdSync struct { - client dclient.Interface - controller *Controller + client dclient.Interface + manager *Manager } const ( @@ -59,14 +54,14 @@ var crdDefinitionNew struct { } // NewCRDSync ... -func NewCRDSync(client dclient.Interface, controller *Controller) *crdSync { - if controller == nil { - panic(fmt.Errorf("nil controller sent into crd sync")) +func NewCRDSync(client dclient.Interface, mgr *Manager) *crdSync { + if mgr == nil { + panic(fmt.Errorf("nil manager sent into crd sync")) } return &crdSync{ - controller: controller, - client: client, + manager: mgr, + client: client, } } @@ -80,7 +75,7 @@ func (c *crdSync) Run(ctx context.Context, workers int) { logging.Error(err, "cannot get OpenAPI schema") } - err = c.controller.useOpenAPIDocument(newDoc) + err = c.manager.useOpenAPIDocument(newDoc) if err != nil { logging.Error(err, "Could not set custom OpenAPI document") } @@ -105,10 +100,10 @@ func (c *crdSync) sync() { return } - c.controller.deleteCRDFromPreviousSync() + c.manager.deleteCRDFromPreviousSync() for _, crd := range crds.Items { - c.controller.ParseCRD(crd) + c.manager.ParseCRD(crd) } if err := c.updateInClusterKindToAPIVersions(); err != nil { @@ -120,7 +115,7 @@ func (c *crdSync) sync() { logging.Error(err, "cannot get OpenAPI schema") } - err = c.controller.useOpenAPIDocument(newDoc) + err = c.manager.useOpenAPIDocument(newDoc) if err != nil { logging.Error(err, "Could not set custom OpenAPI document") } @@ -138,117 +133,10 @@ func (c *crdSync) updateInClusterKindToAPIVersions() error { return errors.Wrapf(err, "fetching API server preferreds resources") } - c.controller.updateKindToAPIVersions(apiResourceLists, preferredAPIResourcesLists) + c.manager.updateKindToAPIVersions(apiResourceLists, preferredAPIResourcesLists) return nil } -func (o *Controller) deleteCRDFromPreviousSync() { - for _, crd := range o.crdList { - o.gvkToDefinitionName.Remove(crd) - o.definitions.Remove(crd) - } - - o.crdList = make([]string, 0) -} - -// ParseCRD loads CRD to the cache -func (o *Controller) ParseCRD(crd unstructured.Unstructured) { - var err error - - crdRaw, _ := json.Marshal(crd.Object) - _ = json.Unmarshal(crdRaw, &crdDefinitionPrior) - - openV3schema := crdDefinitionPrior.Spec.Validation.OpenAPIV3Schema - crdName := crdDefinitionPrior.Spec.Names.Kind - - if openV3schema == nil { - _ = json.Unmarshal(crdRaw, &crdDefinitionNew) - for _, crdVersion := range crdDefinitionNew.Spec.Versions { - if crdVersion.Storage { - openV3schema = crdVersion.Schema.OpenAPIV3Schema - crdName = crdDefinitionNew.Spec.Names.Kind - break - } - } - } - - if openV3schema == nil { - logging.V(4).Info("skip adding schema, CRD has no properties", "name", crdName) - return - } - - schemaRaw, _ := json.Marshal(openV3schema) - if len(schemaRaw) < 1 { - logging.V(4).Info("failed to parse crd schema", "name", crdName) - return - } - - schemaRaw, err = addingDefaultFieldsToSchema(crdName, schemaRaw) - if err != nil { - logging.Error(err, "failed to parse crd schema", "name", crdName) - return - } - - var schema yaml.Node - _ = yaml.Unmarshal(schemaRaw, &schema) - - parsedSchema, err := openapiv2.NewSchema(&schema, compiler.NewContext("schema", &schema, nil)) - if err != nil { - v3valueFound := isOpenV3Error(err) - if !v3valueFound { - logging.Error(err, "failed to parse crd schema", "name", crdName) - } - return - } - - o.crdList = append(o.crdList, crdName) - o.gvkToDefinitionName.Set(crdName, crdName) - o.definitions.Set(crdName, parsedSchema) -} - -func isOpenV3Error(err error) bool { - unsupportedValues := []string{"anyOf", "allOf", "not"} - v3valueFound := false - for _, value := range unsupportedValues { - if !strings.Contains(err.Error(), fmt.Sprintf("has invalid property: %s", value)) { - v3valueFound = true - break - } - } - return v3valueFound -} - -// addingDefaultFieldsToSchema will add any default missing fields like apiVersion, metadata -func addingDefaultFieldsToSchema(crdName string, schemaRaw []byte) ([]byte, error) { - var schema struct { - Properties map[string]interface{} `json:"properties"` - } - _ = json.Unmarshal(schemaRaw, &schema) - - if len(schema.Properties) < 1 { - logging.V(6).Info("crd schema has no properties", "name", crdName) - return schemaRaw, nil - } - - if schema.Properties["apiVersion"] == nil { - apiVersionDefRaw := `{"description":"APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources","type":"string"}` - apiVersionDef := make(map[string]interface{}) - _ = json.Unmarshal([]byte(apiVersionDefRaw), &apiVersionDef) - schema.Properties["apiVersion"] = apiVersionDef - } - - if schema.Properties["metadata"] == nil { - metadataDefRaw := `{"$ref":"#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta","description":"Standard object's metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata"}` - metadataDef := make(map[string]interface{}) - _ = json.Unmarshal([]byte(metadataDefRaw), &metadataDef) - schema.Properties["metadata"] = metadataDef - } - - schemaWithDefaultFields, _ := json.Marshal(schema) - - return schemaWithDefaultFields, nil -} - func (c *crdSync) CheckSync(ctx context.Context) { crds, err := c.client.GetDynamicInterface().Resource(runtimeSchema.GroupVersionResource{ Group: "apiextensions.k8s.io", @@ -259,7 +147,7 @@ func (c *crdSync) CheckSync(ctx context.Context) { logging.Error(err, "could not fetch crd's from server") return } - if len(c.controller.crdList) != len(crds.Items) { + if len(c.manager.crdList) != len(crds.Items) { c.sync() } } diff --git a/pkg/openapi/validation.go b/pkg/openapi/manager.go similarity index 50% rename from pkg/openapi/validation.go rename to pkg/openapi/manager.go index 44685cbd7b..108e19137e 100644 --- a/pkg/openapi/validation.go +++ b/pkg/openapi/manager.go @@ -3,18 +3,14 @@ package openapi import ( "encoding/json" "fmt" - "strconv" "strings" - "sync" "github.com/google/gnostic/compiler" openapiv2 "github.com/google/gnostic/openapiv2" kyvernov1 "github.com/kyverno/kyverno/api/kyverno/v1" - "github.com/kyverno/kyverno/data" "github.com/kyverno/kyverno/pkg/autogen" "github.com/kyverno/kyverno/pkg/engine" "github.com/kyverno/kyverno/pkg/logging" - "github.com/kyverno/kyverno/pkg/utils" cmap "github.com/orcaman/concurrent-map/v2" "github.com/pkg/errors" "gopkg.in/yaml.v3" @@ -24,14 +20,11 @@ import ( "k8s.io/kube-openapi/pkg/util/proto/validation" ) -// type concurrentMap struct{ cmap.ConcurrentMap } - type ValidateInterface interface { ValidateResource(resource unstructured.Unstructured, apiVersion, kind string) error } -// Controller represents OpenAPIController -type Controller struct { +type Manager struct { // definitions holds the map of {definitionName: *openapiv2.Schema} definitions cmap.ConcurrentMap[*openapiv2.Schema] @@ -56,9 +49,9 @@ type apiVersions struct { gvks []string } -// NewOpenAPIController initializes a new instance of OpenAPIController -func NewOpenAPIController() (*Controller, error) { - controller := &Controller{ +// NewOpenAPIManager initializes a new instance of openapi schema manager +func NewOpenAPIManager() (*Manager, error) { + mgr := &Manager{ definitions: cmap.New[*openapiv2.Schema](), gvkToDefinitionName: cmap.New[string](), kindToAPIVersions: cmap.New[apiVersions](), @@ -69,23 +62,23 @@ func NewOpenAPIController() (*Controller, error) { return nil, err } - controller.updateKindToAPIVersions(apiResourceLists, preferredAPIResourcesLists) + mgr.updateKindToAPIVersions(apiResourceLists, preferredAPIResourcesLists) defaultDoc, err := getSchemaDocument() if err != nil { return nil, err } - err = controller.useOpenAPIDocument(defaultDoc) + err = mgr.useOpenAPIDocument(defaultDoc) if err != nil { return nil, err } - return controller, nil + return mgr, nil } // ValidateResource ... -func (o *Controller) ValidateResource(patchedResource unstructured.Unstructured, apiVersion, kind string) error { +func (o *Manager) ValidateResource(patchedResource unstructured.Unstructured, apiVersion, kind string) error { var err error gvk := kind @@ -117,7 +110,7 @@ func (o *Controller) ValidateResource(patchedResource unstructured.Unstructured, } // ValidatePolicyMutation ... -func (o *Controller) ValidatePolicyMutation(policy kyvernov1.PolicyInterface) error { +func (o *Manager) ValidatePolicyMutation(policy kyvernov1.PolicyInterface) error { kindToRules := make(map[string][]kyvernov1.Rule) for _, rule := range autogen.ComputeRules(policy) { if rule.HasMutate() { @@ -158,7 +151,7 @@ func (o *Controller) ValidatePolicyMutation(policy kyvernov1.PolicyInterface) er return nil } -func (o *Controller) useOpenAPIDocument(doc *openapiv2.Document) error { +func (o *Manager) useOpenAPIDocument(doc *openapiv2.Document) error { for _, definition := range doc.GetDefinitions().AdditionalProperties { definitionName := definition.GetName() @@ -190,7 +183,7 @@ func (o *Controller) useOpenAPIDocument(doc *openapiv2.Document) error { return nil } -func (o *Controller) getGVKByDefinitionName(definitionName string) (gvk string, preferredGVK bool, err error) { +func (o *Manager) getGVKByDefinitionName(definitionName string) (gvk string, preferredGVK bool, err error) { paths := strings.Split(definitionName, ".") kind := paths[len(paths)-1] versions, ok := o.kindToAPIVersions.Get(kind) @@ -213,64 +206,8 @@ func (o *Controller) getGVKByDefinitionName(definitionName string) (gvk string, return "", preferredGVK, fmt.Errorf("gvk not found by the given definition name %s, %v", definitionName, versions.gvks) } -func parseGVK(str string) (group, apiVersion, kind string) { - if strings.Count(str, "/") == 0 { - return "", "", str - } - splitString := strings.Split(str, "/") - if strings.Count(str, "/") == 1 { - return "", splitString[0], splitString[1] - } - return splitString[0], splitString[1], splitString[2] -} - -func groupMatches(gvkMap map[string]bool, group, kind string) bool { - if group == "" { - ok := gvkMap["core"] - if ok { - return true - } - } else { - elements := strings.Split(group, ".") - ok := gvkMap[elements[0]] - if ok { - return true - } - } - return false -} - -// matchGVK is a helper function that checks if the -// given GVK matches the definition name - -func matchGVK(definitionName, gvk string) bool { - paths := strings.Split(definitionName, ".") - - gvkMap := make(map[string]bool) - for _, p := range paths { - gvkMap[p] = true - } - - group, version, kind := parseGVK(gvk) - - ok := gvkMap[kind] - if !ok { - return false - } - ok = gvkMap[version] - if !ok { - return false - } - - if !groupMatches(gvkMap, group, kind) { - return false - } - - return true -} - // updateKindToAPIVersions sets kindToAPIVersions with static manifests -func (c *Controller) updateKindToAPIVersions(apiResourceLists, preferredAPIResourcesLists []*metav1.APIResourceList) { +func (c *Manager) updateKindToAPIVersions(apiResourceLists, preferredAPIResourcesLists []*metav1.APIResourceList) { tempKindToAPIVersions := getAllAPIVersions(apiResourceLists) tempKindToAPIVersions = setPreferredVersions(tempKindToAPIVersions, preferredAPIResourcesLists) @@ -280,19 +217,8 @@ func (c *Controller) updateKindToAPIVersions(apiResourceLists, preferredAPIResou } } -func getSchemaDocument() (*openapiv2.Document, error) { - var spec yaml.Node - err := yaml.Unmarshal([]byte(data.SwaggerDoc), &spec) - if err != nil { - return nil, err - } - - root := spec.Content[0] - return openapiv2.NewDocument(root, compiler.NewContext("$root", root, nil)) -} - // For crd, we do not store definition in document -func (o *Controller) getCRDSchema(kind string) (proto.Schema, error) { +func (o *Manager) getCRDSchema(kind string) (proto.Schema, error) { if kind == "" { return nil, errors.New("invalid kind") } @@ -312,7 +238,7 @@ func (o *Controller) getCRDSchema(kind string) (proto.Schema, error) { return (existingDefinitions).ParseSchema(definition, &path) } -func (o *Controller) generateEmptyResource(kindSchema *openapiv2.Schema) interface{} { +func (o *Manager) generateEmptyResource(kindSchema *openapiv2.Schema) interface{} { types := kindSchema.GetType().GetValue() if kindSchema.GetXRef() != "" { @@ -347,169 +273,66 @@ func (o *Controller) generateEmptyResource(kindSchema *openapiv2.Schema) interfa return nil } -func getArrayValue(kindSchema *openapiv2.Schema, o *Controller) interface{} { - var array []interface{} - for _, schema := range kindSchema.GetItems().GetSchema() { - array = append(array, o.generateEmptyResource(schema)) +func (o *Manager) deleteCRDFromPreviousSync() { + for _, crd := range o.crdList { + o.gvkToDefinitionName.Remove(crd) + o.definitions.Remove(crd) } - return array + o.crdList = make([]string, 0) } -func getObjectValue(kindSchema *openapiv2.Schema, o *Controller) interface{} { - props := make(map[string]interface{}) - properties := kindSchema.GetProperties().GetAdditionalProperties() - if len(properties) == 0 { - return props - } +// ParseCRD loads CRD to the cache +func (o *Manager) ParseCRD(crd unstructured.Unstructured) { + var err error - var wg sync.WaitGroup - var mutex sync.Mutex - wg.Add(len(properties)) - for _, property := range properties { - go func(property *openapiv2.NamedSchema) { - prop := o.generateEmptyResource(property.GetValue()) - mutex.Lock() - props[property.GetName()] = prop - mutex.Unlock() - wg.Done() - }(property) - } - wg.Wait() - return props -} + crdRaw, _ := json.Marshal(crd.Object) + _ = json.Unmarshal(crdRaw, &crdDefinitionPrior) -func getBoolValue(kindSchema *openapiv2.Schema) bool { - if d := kindSchema.GetDefault(); d != nil { - v := getAnyValue(d) - return string(v) == "true" - } + openV3schema := crdDefinitionPrior.Spec.Validation.OpenAPIV3Schema + crdName := crdDefinitionPrior.Spec.Names.Kind - if e := kindSchema.GetExample(); e != nil { - v := getAnyValue(e) - return string(v) == "true" - } - - return false -} - -func getNumericValue(kindSchema *openapiv2.Schema) int64 { - if d := kindSchema.GetDefault(); d != nil { - v := getAnyValue(d) - val, _ := strconv.Atoi(string(v)) - return int64(val) - } - - if e := kindSchema.GetExample(); e != nil { - v := getAnyValue(e) - val, _ := strconv.Atoi(string(v)) - return int64(val) - } - - return int64(0) -} - -func getStringValue(kindSchema *openapiv2.Schema) string { - if d := kindSchema.GetDefault(); d != nil { - v := getAnyValue(d) - return string(v) - } - - if e := kindSchema.GetExample(); e != nil { - v := getAnyValue(e) - return string(v) - } - - return "" -} - -func getAnyValue(any *openapiv2.Any) []byte { - if any != nil { - if val := any.GetValue(); val != nil { - return val.GetValue() - } - } - - return nil -} - -// getAllAPIVersions gets all available versions for a kind -// returns a map which stores all kinds with its versions -func getAllAPIVersions(apiResourceLists []*metav1.APIResourceList) map[string]apiVersions { - tempKindToAPIVersions := make(map[string]apiVersions) - - for _, apiResourceList := range apiResourceLists { - lastKind := "" - for _, apiResource := range apiResourceList.APIResources { - if apiResource.Kind == lastKind { - continue - } - - version, ok := tempKindToAPIVersions[apiResource.Kind] - if !ok { - tempKindToAPIVersions[apiResource.Kind] = apiVersions{} - } - - gvk := strings.Join([]string{apiResourceList.GroupVersion, apiResource.Kind}, "/") - version.gvks = append(version.gvks, gvk) - tempKindToAPIVersions[apiResource.Kind] = version - lastKind = apiResource.Kind - } - } - - return tempKindToAPIVersions -} - -// setPreferredVersions sets the serverPreferredGVK of the given apiVersions map -func setPreferredVersions(kindToAPIVersions map[string]apiVersions, preferredAPIResourcesLists []*metav1.APIResourceList) map[string]apiVersions { - tempKindToAPIVersionsCopied := copyKindToAPIVersions(kindToAPIVersions) - - for kind, versions := range tempKindToAPIVersionsCopied { - for _, preferredAPIResourcesList := range preferredAPIResourcesLists { - for _, resource := range preferredAPIResourcesList.APIResources { - preferredGV := preferredAPIResourcesList.GroupVersion - preferredGVK := preferredGV + "/" + resource.Kind - - if utils.ContainsString(versions.gvks, preferredGVK) { - v := kindToAPIVersions[kind] - - // if a Kind belongs to multiple groups, the first group/version - // returned from discovery docs is used as preferred version - // https://github.com/kubernetes/kubernetes/issues/94761#issuecomment-691982480 - if v.serverPreferredGVK != "" { - continue - } - - v.serverPreferredGVK = strings.Join([]string{preferredGV, kind}, "/") - kindToAPIVersions[kind] = v - } + if openV3schema == nil { + _ = json.Unmarshal(crdRaw, &crdDefinitionNew) + for _, crdVersion := range crdDefinitionNew.Spec.Versions { + if crdVersion.Storage { + openV3schema = crdVersion.Schema.OpenAPIV3Schema + crdName = crdDefinitionNew.Spec.Names.Kind + break } } } - return kindToAPIVersions -} - -func copyKindToAPIVersions(old map[string]apiVersions) map[string]apiVersions { - new := make(map[string]apiVersions, len(old)) - for key, value := range old { - new[key] = value + if openV3schema == nil { + logging.V(4).Info("skip adding schema, CRD has no properties", "name", crdName) + return } - return new -} -func getAPIResourceLists() ([]*metav1.APIResourceList, []*metav1.APIResourceList, error) { - var apiResourceLists []*metav1.APIResourceList - err := json.Unmarshal([]byte(data.APIResourceLists), &apiResourceLists) + schemaRaw, _ := json.Marshal(openV3schema) + if len(schemaRaw) < 1 { + logging.V(4).Info("failed to parse crd schema", "name", crdName) + return + } + + schemaRaw, err = addingDefaultFieldsToSchema(crdName, schemaRaw) if err != nil { - return nil, nil, fmt.Errorf("unable to load apiResourceLists: %v", err) + logging.Error(err, "failed to parse crd schema", "name", crdName) + return } - var preferredAPIResourcesLists []*metav1.APIResourceList - err = json.Unmarshal([]byte(data.APIResourceLists), &preferredAPIResourcesLists) + var schema yaml.Node + _ = yaml.Unmarshal(schemaRaw, &schema) + + parsedSchema, err := openapiv2.NewSchema(&schema, compiler.NewContext("schema", &schema, nil)) if err != nil { - return nil, nil, fmt.Errorf("unable to load preferredAPIResourcesLists: %v", err) + v3valueFound := isOpenV3Error(err) + if !v3valueFound { + logging.Error(err, "failed to parse crd schema", "name", crdName) + } + return } - return apiResourceLists, preferredAPIResourcesLists, nil + o.crdList = append(o.crdList, crdName) + o.gvkToDefinitionName.Set(crdName, crdName) + o.definitions.Set(crdName, parsedSchema) } diff --git a/pkg/openapi/validation_test.go b/pkg/openapi/manager_test.go similarity index 99% rename from pkg/openapi/validation_test.go rename to pkg/openapi/manager_test.go index 853a59ca1c..2a98031d15 100644 --- a/pkg/openapi/validation_test.go +++ b/pkg/openapi/manager_test.go @@ -41,7 +41,7 @@ func Test_ValidateMutationPolicy(t *testing.T) { }, } - o, _ := NewOpenAPIController() + o, _ := NewOpenAPIManager() for i, tc := range tcs { policy := v1.ClusterPolicy{} @@ -165,7 +165,7 @@ func Test_matchGVK(t *testing.T) { // networking.k8s.io/v1beta1/Ingress // extensions/v1beta1/Ingress func Test_Ingress(t *testing.T) { - o, err := NewOpenAPIController() + o, err := NewOpenAPIManager() assert.NilError(t, err) versions, ok := o.kindToAPIVersions.Get("Ingress") diff --git a/pkg/openapi/utils.go b/pkg/openapi/utils.go new file mode 100644 index 0000000000..1b4f43183f --- /dev/null +++ b/pkg/openapi/utils.go @@ -0,0 +1,292 @@ +package openapi + +import ( + "encoding/json" + "fmt" + "strconv" + "strings" + "sync" + + "github.com/google/gnostic/compiler" + openapiv2 "github.com/google/gnostic/openapiv2" + "github.com/kyverno/kyverno/data" + "github.com/kyverno/kyverno/pkg/logging" + "github.com/kyverno/kyverno/pkg/utils" + "gopkg.in/yaml.v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func parseGVK(str string) (group, apiVersion, kind string) { + if strings.Count(str, "/") == 0 { + return "", "", str + } + splitString := strings.Split(str, "/") + if strings.Count(str, "/") == 1 { + return "", splitString[0], splitString[1] + } + return splitString[0], splitString[1], splitString[2] +} + +func groupMatches(gvkMap map[string]bool, group, kind string) bool { + if group == "" { + ok := gvkMap["core"] + if ok { + return true + } + } else { + elements := strings.Split(group, ".") + ok := gvkMap[elements[0]] + if ok { + return true + } + } + return false +} + +// matchGVK is a helper function that checks if the given GVK matches the definition name +func matchGVK(definitionName, gvk string) bool { + paths := strings.Split(definitionName, ".") + + gvkMap := make(map[string]bool) + for _, p := range paths { + gvkMap[p] = true + } + + group, version, kind := parseGVK(gvk) + + ok := gvkMap[kind] + if !ok { + return false + } + ok = gvkMap[version] + if !ok { + return false + } + + if !groupMatches(gvkMap, group, kind) { + return false + } + + return true +} + +func getSchemaDocument() (*openapiv2.Document, error) { + var spec yaml.Node + err := yaml.Unmarshal([]byte(data.SwaggerDoc), &spec) + if err != nil { + return nil, err + } + + root := spec.Content[0] + return openapiv2.NewDocument(root, compiler.NewContext("$root", root, nil)) +} + +func getArrayValue(kindSchema *openapiv2.Schema, o *Manager) interface{} { + var array []interface{} + for _, schema := range kindSchema.GetItems().GetSchema() { + array = append(array, o.generateEmptyResource(schema)) + } + + return array +} + +func getObjectValue(kindSchema *openapiv2.Schema, o *Manager) interface{} { + props := make(map[string]interface{}) + properties := kindSchema.GetProperties().GetAdditionalProperties() + if len(properties) == 0 { + return props + } + + var wg sync.WaitGroup + var mutex sync.Mutex + wg.Add(len(properties)) + for _, property := range properties { + go func(property *openapiv2.NamedSchema) { + prop := o.generateEmptyResource(property.GetValue()) + mutex.Lock() + props[property.GetName()] = prop + mutex.Unlock() + wg.Done() + }(property) + } + wg.Wait() + return props +} + +func getBoolValue(kindSchema *openapiv2.Schema) bool { + if d := kindSchema.GetDefault(); d != nil { + v := getAnyValue(d) + return string(v) == "true" + } + + if e := kindSchema.GetExample(); e != nil { + v := getAnyValue(e) + return string(v) == "true" + } + + return false +} + +func getNumericValue(kindSchema *openapiv2.Schema) int64 { + if d := kindSchema.GetDefault(); d != nil { + v := getAnyValue(d) + val, _ := strconv.Atoi(string(v)) + return int64(val) + } + + if e := kindSchema.GetExample(); e != nil { + v := getAnyValue(e) + val, _ := strconv.Atoi(string(v)) + return int64(val) + } + + return int64(0) +} + +func getStringValue(kindSchema *openapiv2.Schema) string { + if d := kindSchema.GetDefault(); d != nil { + v := getAnyValue(d) + return string(v) + } + + if e := kindSchema.GetExample(); e != nil { + v := getAnyValue(e) + return string(v) + } + + return "" +} + +func getAnyValue(any *openapiv2.Any) []byte { + if any != nil { + if val := any.GetValue(); val != nil { + return val.GetValue() + } + } + + return nil +} + +// getAllAPIVersions gets all available versions for a kind +// returns a map which stores all kinds with its versions +func getAllAPIVersions(apiResourceLists []*metav1.APIResourceList) map[string]apiVersions { + tempKindToAPIVersions := make(map[string]apiVersions) + + for _, apiResourceList := range apiResourceLists { + lastKind := "" + for _, apiResource := range apiResourceList.APIResources { + if apiResource.Kind == lastKind { + continue + } + + version, ok := tempKindToAPIVersions[apiResource.Kind] + if !ok { + tempKindToAPIVersions[apiResource.Kind] = apiVersions{} + } + + gvk := strings.Join([]string{apiResourceList.GroupVersion, apiResource.Kind}, "/") + version.gvks = append(version.gvks, gvk) + tempKindToAPIVersions[apiResource.Kind] = version + lastKind = apiResource.Kind + } + } + + return tempKindToAPIVersions +} + +// setPreferredVersions sets the serverPreferredGVK of the given apiVersions map +func setPreferredVersions(kindToAPIVersions map[string]apiVersions, preferredAPIResourcesLists []*metav1.APIResourceList) map[string]apiVersions { + tempKindToAPIVersionsCopied := copyKindToAPIVersions(kindToAPIVersions) + + for kind, versions := range tempKindToAPIVersionsCopied { + for _, preferredAPIResourcesList := range preferredAPIResourcesLists { + for _, resource := range preferredAPIResourcesList.APIResources { + preferredGV := preferredAPIResourcesList.GroupVersion + preferredGVK := preferredGV + "/" + resource.Kind + + if utils.ContainsString(versions.gvks, preferredGVK) { + v := kindToAPIVersions[kind] + + // if a Kind belongs to multiple groups, the first group/version + // returned from discovery docs is used as preferred version + // https://github.com/kubernetes/kubernetes/issues/94761#issuecomment-691982480 + if v.serverPreferredGVK != "" { + continue + } + + v.serverPreferredGVK = strings.Join([]string{preferredGV, kind}, "/") + kindToAPIVersions[kind] = v + } + } + } + } + + return kindToAPIVersions +} + +func copyKindToAPIVersions(old map[string]apiVersions) map[string]apiVersions { + new := make(map[string]apiVersions, len(old)) + for key, value := range old { + new[key] = value + } + return new +} + +func getAPIResourceLists() ([]*metav1.APIResourceList, []*metav1.APIResourceList, error) { + var apiResourceLists []*metav1.APIResourceList + err := json.Unmarshal([]byte(data.APIResourceLists), &apiResourceLists) + if err != nil { + return nil, nil, fmt.Errorf("unable to load apiResourceLists: %v", err) + } + + var preferredAPIResourcesLists []*metav1.APIResourceList + err = json.Unmarshal([]byte(data.APIResourceLists), &preferredAPIResourcesLists) + if err != nil { + return nil, nil, fmt.Errorf("unable to load preferredAPIResourcesLists: %v", err) + } + + return apiResourceLists, preferredAPIResourcesLists, nil +} + +func isOpenV3Error(err error) bool { + unsupportedValues := []string{"anyOf", "allOf", "not"} + v3valueFound := false + for _, value := range unsupportedValues { + if !strings.Contains(err.Error(), fmt.Sprintf("has invalid property: %s", value)) { + v3valueFound = true + break + } + } + return v3valueFound +} + +// addingDefaultFieldsToSchema will add any default missing fields like apiVersion, metadata +func addingDefaultFieldsToSchema(crdName string, schemaRaw []byte) ([]byte, error) { + var schema struct { + Properties map[string]interface{} `json:"properties"` + } + _ = json.Unmarshal(schemaRaw, &schema) + + if len(schema.Properties) < 1 { + logging.V(6).Info("crd schema has no properties", "name", crdName) + return schemaRaw, nil + } + + if schema.Properties["apiVersion"] == nil { + apiVersionDefRaw := `{"description":"APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources","type":"string"}` + apiVersionDef := make(map[string]interface{}) + _ = json.Unmarshal([]byte(apiVersionDefRaw), &apiVersionDef) + schema.Properties["apiVersion"] = apiVersionDef + } + + if schema.Properties["metadata"] == nil { + metadataDefRaw := `{"$ref":"#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta","description":"Standard object's metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata"}` + metadataDef := make(map[string]interface{}) + _ = json.Unmarshal([]byte(metadataDefRaw), &metadataDef) + schema.Properties["metadata"] = metadataDef + } + + schemaWithDefaultFields, _ := json.Marshal(schema) + + return schemaWithDefaultFields, nil +} diff --git a/pkg/policy/validate.go b/pkg/policy/validate.go index 12dffbf977..4dca0a77a0 100644 --- a/pkg/policy/validate.go +++ b/pkg/policy/validate.go @@ -79,13 +79,13 @@ func validateJSONPatchPathForForwardSlash(patch string) error { } // Validate checks the policy and rules declarations for required configurations -func Validate(policy kyvernov1.PolicyInterface, client dclient.Interface, mock bool, openAPIController *openapi.Controller) (*admissionv1.AdmissionResponse, error) { +func Validate(policy kyvernov1.PolicyInterface, client dclient.Interface, mock bool, openApiManager *openapi.Manager) (*admissionv1.AdmissionResponse, error) { namespaced := policy.IsNamespaced() spec := policy.GetSpec() background := spec.BackgroundProcessingEnabled() onPolicyUpdate := spec.GetMutateExistingOnPolicyUpdate() if !mock { - openapi.NewCRDSync(client, openAPIController).CheckSync(context.TODO()) + openapi.NewCRDSync(client, openApiManager).CheckSync(context.TODO()) } var errs field.ErrorList @@ -359,7 +359,7 @@ func Validate(policy kyvernov1.PolicyInterface, client dclient.Interface, mock b } if spec.SchemaValidation == nil || *spec.SchemaValidation { - if err := openAPIController.ValidatePolicyMutation(policy); err != nil { + if err := openApiManager.ValidatePolicyMutation(policy); err != nil { return nil, err } } diff --git a/pkg/policy/validate_test.go b/pkg/policy/validate_test.go index b4a8e01def..5675f32a0f 100644 --- a/pkg/policy/validate_test.go +++ b/pkg/policy/validate_test.go @@ -345,7 +345,7 @@ func Test_Validate_Policy(t *testing.T) { } }`) - openAPIController, _ := openapi.NewOpenAPIController() + openAPIController, _ := openapi.NewOpenAPIManager() var policy *kyverno.ClusterPolicy err := json.Unmarshal(rawPolicy, &policy) assert.NilError(t, err) @@ -496,7 +496,7 @@ func Test_Validate_ErrorFormat(t *testing.T) { err := json.Unmarshal(rawPolicy, &policy) assert.NilError(t, err) - openAPIController, _ := openapi.NewOpenAPIController() + openAPIController, _ := openapi.NewOpenAPIManager() _, err = Validate(policy, nil, true, openAPIController) assert.Assert(t, err != nil) } @@ -898,7 +898,7 @@ func Test_Validate_Kind(t *testing.T) { err := json.Unmarshal(rawPolicy, &policy) assert.NilError(t, err) - openAPIController, _ := openapi.NewOpenAPIController() + openAPIController, _ := openapi.NewOpenAPIManager() _, err = Validate(policy, nil, true, openAPIController) assert.Assert(t, err != nil) } @@ -947,7 +947,7 @@ func Test_Validate_Any_Kind(t *testing.T) { err := json.Unmarshal(rawPolicy, &policy) assert.NilError(t, err) - openAPIController, _ := openapi.NewOpenAPIController() + openAPIController, _ := openapi.NewOpenAPIManager() _, err = Validate(policy, nil, true, openAPIController) assert.Assert(t, err != nil) } @@ -1075,7 +1075,7 @@ func Test_Wildcards_Kind(t *testing.T) { err := json.Unmarshal(rawPolicy, &policy) assert.NilError(t, err) - openAPIController, _ := openapi.NewOpenAPIController() + openAPIController, _ := openapi.NewOpenAPIManager() _, err = Validate(policy, nil, true, openAPIController) assert.Assert(t, err != nil) } @@ -1125,7 +1125,7 @@ func Test_Namespced_Policy(t *testing.T) { err := json.Unmarshal(rawPolicy, &policy) assert.NilError(t, err) - openAPIController, _ := openapi.NewOpenAPIController() + openAPIController, _ := openapi.NewOpenAPIManager() _, err = Validate(policy, nil, true, openAPIController) assert.Assert(t, err != nil) } @@ -1173,7 +1173,7 @@ func Test_patchesJson6902_Policy(t *testing.T) { err := json.Unmarshal(rawPolicy, &policy) assert.NilError(t, err) - openAPIController, _ := openapi.NewOpenAPIController() + openAPIController, _ := openapi.NewOpenAPIManager() _, err = Validate(policy, nil, true, openAPIController) assert.NilError(t, err) } @@ -1221,7 +1221,7 @@ func Test_deny_exec(t *testing.T) { err = json.Unmarshal(rawPolicy, &policy) assert.NilError(t, err) - openAPIController, _ := openapi.NewOpenAPIController() + openAPIController, _ := openapi.NewOpenAPIManager() _, err = Validate(policy, nil, true, openAPIController) assert.NilError(t, err) } @@ -1266,7 +1266,7 @@ func Test_existing_resource_policy(t *testing.T) { err = json.Unmarshal(rawPolicy, &policy) assert.NilError(t, err) - openAPIController, _ := openapi.NewOpenAPIController() + openAPIController, _ := openapi.NewOpenAPIManager() _, err = Validate(policy, nil, true, openAPIController) assert.NilError(t, err) } @@ -1322,7 +1322,7 @@ func Test_PodControllerAutoGenExclusion_All_Controllers_Policy(t *testing.T) { err := json.Unmarshal(rawPolicy, &policy) assert.NilError(t, err) - openAPIController, _ := openapi.NewOpenAPIController() + openAPIController, _ := openapi.NewOpenAPIManager() res, err := Validate(policy, nil, true, openAPIController) assert.NilError(t, err) assert.Assert(t, res == nil) @@ -1379,7 +1379,7 @@ func Test_PodControllerAutoGenExclusion_Not_All_Controllers_Policy(t *testing.T) err := json.Unmarshal(rawPolicy, &policy) assert.NilError(t, err) - openAPIController, _ := openapi.NewOpenAPIController() + openAPIController, _ := openapi.NewOpenAPIManager() res, err := Validate(policy, nil, true, openAPIController) if res != nil { assert.Assert(t, res.Warnings != nil) @@ -1438,7 +1438,7 @@ func Test_PodControllerAutoGenExclusion_None_Policy(t *testing.T) { err := json.Unmarshal(rawPolicy, &policy) assert.NilError(t, err) - openAPIController, _ := openapi.NewOpenAPIController() + openAPIController, _ := openapi.NewOpenAPIManager() res, err := Validate(policy, nil, true, openAPIController) if res != nil { assert.Assert(t, res.Warnings != nil) diff --git a/pkg/webhooks/policy/handlers.go b/pkg/webhooks/policy/handlers.go index 9cecb32f11..c6bdefb607 100644 --- a/pkg/webhooks/policy/handlers.go +++ b/pkg/webhooks/policy/handlers.go @@ -17,14 +17,14 @@ import ( ) type handlers struct { - client dclient.Interface - openAPIController *openapi.Controller + client dclient.Interface + openApiManager *openapi.Manager } -func NewHandlers(client dclient.Interface, openAPIController *openapi.Controller) webhooks.PolicyHandlers { +func NewHandlers(client dclient.Interface, openAPIController *openapi.Manager) webhooks.PolicyHandlers { return &handlers{ - client: client, - openAPIController: openAPIController, + client: client, + openApiManager: openAPIController, } } @@ -38,7 +38,7 @@ func (h *handlers) Validate(logger logr.Logger, request *admissionv1.AdmissionRe logger.Error(err, "failed to unmarshal policies from admission request") return admissionutils.ResponseWithMessage(true, fmt.Sprintf("failed to validate policy, check kyverno controller logs for details: %v", err)) } - response, err := policyvalidate.Validate(policy, h.client, false, h.openAPIController) + response, err := policyvalidate.Validate(policy, h.client, false, h.openApiManager) if err != nil { logger.Error(err, "policy validation errors") return admissionutils.ResponseWithMessage(false, err.Error())