1
0
Fork 0
mirror of https://github.com/kyverno/kyverno.git synced 2025-04-15 16:56:56 +00:00

refactor: openapi controller part 1 (#4901)

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>
Co-authored-by: Vyankatesh Kudtarkar <vyankateshkd@gmail.com>
This commit is contained in:
Charles-Edouard Brétéché 2022-10-12 13:38:48 +02:00 committed by GitHub
parent d25dccbd9c
commit de67a507cd
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 394 additions and 391 deletions

View file

@ -185,7 +185,7 @@ func (c *ApplyCommandConfig) applyCommandHelper() (rc *common.ResultCounts, reso
return rc, resources, skipInvalidPolicies, pvInfos, err
}
openAPIController, err := openapi.NewOpenAPIController()
openApiManager, err := openapi.NewOpenAPIManager()
if err != nil {
return rc, resources, skipInvalidPolicies, pvInfos, sanitizederror.NewWithError("failed to initialize openAPIController", err)
}
@ -322,7 +322,7 @@ func (c *ApplyCommandConfig) applyCommandHelper() (rc *common.ResultCounts, reso
skipInvalidPolicies.invalid = make([]string, 0)
for _, policy := range mutatedPolicies {
_, err := policy2.Validate(policy, nil, true, openAPIController)
_, err := policy2.Validate(policy, nil, true, openApiManager)
if err != nil {
log.Log.Error(err, "policy validation error")
if strings.HasPrefix(err.Error(), "variable 'element.name'") {

View file

@ -362,7 +362,7 @@ func testCommandExecute(dirPath []string, fileName string, gitBranch string, tes
tf.enabled = false
}
openAPIController, err := openapi.NewOpenAPIController()
openAPIController, err := openapi.NewOpenAPIManager()
if err != nil {
return rc, fmt.Errorf("unable to create open api controller, %w", err)
}
@ -480,7 +480,7 @@ func testCommandExecute(dirPath []string, fileName string, gitBranch string, tes
return rc, nil
}
func getLocalDirTestFiles(fs billy.Filesystem, path, fileName string, rc *resultCounts, testFiles *int, openAPIController *openapi.Controller, tf *testFilter, failOnly, removeColor bool) []error {
func getLocalDirTestFiles(fs billy.Filesystem, path, fileName string, rc *resultCounts, testFiles *int, openApiManager *openapi.Manager, tf *testFilter, failOnly, removeColor bool) []error {
var errors []error
files, err := os.ReadDir(path)
@ -489,7 +489,7 @@ func getLocalDirTestFiles(fs billy.Filesystem, path, fileName string, rc *result
}
for _, file := range files {
if file.IsDir() {
getLocalDirTestFiles(fs, filepath.Join(path, file.Name()), fileName, rc, testFiles, openAPIController, tf, failOnly, removeColor)
getLocalDirTestFiles(fs, filepath.Join(path, file.Name()), fileName, rc, testFiles, openApiManager, tf, failOnly, removeColor)
continue
}
if file.Name() == fileName {
@ -505,7 +505,7 @@ func getLocalDirTestFiles(fs billy.Filesystem, path, fileName string, rc *result
errors = append(errors, sanitizederror.NewWithError("failed to convert json", err))
continue
}
if err := applyPoliciesFromPath(fs, valuesBytes, false, path, rc, openAPIController, tf, failOnly, removeColor); err != nil {
if err := applyPoliciesFromPath(fs, valuesBytes, false, path, rc, openApiManager, tf, failOnly, removeColor); err != nil {
errors = append(errors, sanitizederror.NewWithError(fmt.Sprintf("failed to apply test command from file %s", file.Name()), err))
continue
}
@ -819,7 +819,7 @@ func getFullPath(paths []string, policyResourcePath string, isGit bool) []string
return paths
}
func applyPoliciesFromPath(fs billy.Filesystem, policyBytes []byte, isGit bool, policyResourcePath string, rc *resultCounts, openAPIController *openapi.Controller, tf *testFilter, failOnly, removeColor bool) (err error) {
func applyPoliciesFromPath(fs billy.Filesystem, policyBytes []byte, isGit bool, policyResourcePath string, rc *resultCounts, openAPIController *openapi.Manager, tf *testFilter, failOnly, removeColor bool) (err error) {
engineResponses := make([]*response.EngineResponse, 0)
var dClient dclient.Interface
values := &Test{}

View file

@ -303,7 +303,7 @@ func createNonLeaderControllers(
configuration config.Configuration,
policyCache policycache.Cache,
eventGenerator event.Interface,
manager *openapi.Controller,
manager *openapi.Manager,
) ([]controller, func() error) {
policyCacheController := policycachecontroller.NewController(
policyCache,
@ -558,7 +558,7 @@ func main() {
logger.Error(err, "failed to initialize configuration")
os.Exit(1)
}
openApiManager, err := openapi.NewOpenAPIController()
openApiManager, err := openapi.NewOpenAPIManager()
if err != nil {
logger.Error(err, "Failed to create openapi manager")
os.Exit(1)

View file

@ -2,29 +2,24 @@ package openapi
import (
"context"
"encoding/json"
"fmt"
"strings"
"time"
"github.com/google/gnostic/compiler"
openapiv2 "github.com/google/gnostic/openapiv2"
"github.com/kyverno/kyverno/pkg/clients/dclient"
"github.com/kyverno/kyverno/pkg/logging"
"github.com/kyverno/kyverno/pkg/metrics"
util "github.com/kyverno/kyverno/pkg/utils"
"github.com/pkg/errors"
"gopkg.in/yaml.v3"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
runtimeSchema "k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/discovery"
)
type crdSync struct {
client dclient.Interface
controller *Controller
client dclient.Interface
manager *Manager
}
const (
@ -59,14 +54,14 @@ var crdDefinitionNew struct {
}
// NewCRDSync ...
func NewCRDSync(client dclient.Interface, controller *Controller) *crdSync {
if controller == nil {
panic(fmt.Errorf("nil controller sent into crd sync"))
func NewCRDSync(client dclient.Interface, mgr *Manager) *crdSync {
if mgr == nil {
panic(fmt.Errorf("nil manager sent into crd sync"))
}
return &crdSync{
controller: controller,
client: client,
manager: mgr,
client: client,
}
}
@ -80,7 +75,7 @@ func (c *crdSync) Run(ctx context.Context, workers int) {
logging.Error(err, "cannot get OpenAPI schema")
}
err = c.controller.useOpenAPIDocument(newDoc)
err = c.manager.useOpenAPIDocument(newDoc)
if err != nil {
logging.Error(err, "Could not set custom OpenAPI document")
}
@ -105,10 +100,10 @@ func (c *crdSync) sync() {
return
}
c.controller.deleteCRDFromPreviousSync()
c.manager.deleteCRDFromPreviousSync()
for _, crd := range crds.Items {
c.controller.ParseCRD(crd)
c.manager.ParseCRD(crd)
}
if err := c.updateInClusterKindToAPIVersions(); err != nil {
@ -120,7 +115,7 @@ func (c *crdSync) sync() {
logging.Error(err, "cannot get OpenAPI schema")
}
err = c.controller.useOpenAPIDocument(newDoc)
err = c.manager.useOpenAPIDocument(newDoc)
if err != nil {
logging.Error(err, "Could not set custom OpenAPI document")
}
@ -138,117 +133,10 @@ func (c *crdSync) updateInClusterKindToAPIVersions() error {
return errors.Wrapf(err, "fetching API server preferreds resources")
}
c.controller.updateKindToAPIVersions(apiResourceLists, preferredAPIResourcesLists)
c.manager.updateKindToAPIVersions(apiResourceLists, preferredAPIResourcesLists)
return nil
}
func (o *Controller) deleteCRDFromPreviousSync() {
for _, crd := range o.crdList {
o.gvkToDefinitionName.Remove(crd)
o.definitions.Remove(crd)
}
o.crdList = make([]string, 0)
}
// ParseCRD loads CRD to the cache
func (o *Controller) ParseCRD(crd unstructured.Unstructured) {
var err error
crdRaw, _ := json.Marshal(crd.Object)
_ = json.Unmarshal(crdRaw, &crdDefinitionPrior)
openV3schema := crdDefinitionPrior.Spec.Validation.OpenAPIV3Schema
crdName := crdDefinitionPrior.Spec.Names.Kind
if openV3schema == nil {
_ = json.Unmarshal(crdRaw, &crdDefinitionNew)
for _, crdVersion := range crdDefinitionNew.Spec.Versions {
if crdVersion.Storage {
openV3schema = crdVersion.Schema.OpenAPIV3Schema
crdName = crdDefinitionNew.Spec.Names.Kind
break
}
}
}
if openV3schema == nil {
logging.V(4).Info("skip adding schema, CRD has no properties", "name", crdName)
return
}
schemaRaw, _ := json.Marshal(openV3schema)
if len(schemaRaw) < 1 {
logging.V(4).Info("failed to parse crd schema", "name", crdName)
return
}
schemaRaw, err = addingDefaultFieldsToSchema(crdName, schemaRaw)
if err != nil {
logging.Error(err, "failed to parse crd schema", "name", crdName)
return
}
var schema yaml.Node
_ = yaml.Unmarshal(schemaRaw, &schema)
parsedSchema, err := openapiv2.NewSchema(&schema, compiler.NewContext("schema", &schema, nil))
if err != nil {
v3valueFound := isOpenV3Error(err)
if !v3valueFound {
logging.Error(err, "failed to parse crd schema", "name", crdName)
}
return
}
o.crdList = append(o.crdList, crdName)
o.gvkToDefinitionName.Set(crdName, crdName)
o.definitions.Set(crdName, parsedSchema)
}
func isOpenV3Error(err error) bool {
unsupportedValues := []string{"anyOf", "allOf", "not"}
v3valueFound := false
for _, value := range unsupportedValues {
if !strings.Contains(err.Error(), fmt.Sprintf("has invalid property: %s", value)) {
v3valueFound = true
break
}
}
return v3valueFound
}
// addingDefaultFieldsToSchema will add any default missing fields like apiVersion, metadata
func addingDefaultFieldsToSchema(crdName string, schemaRaw []byte) ([]byte, error) {
var schema struct {
Properties map[string]interface{} `json:"properties"`
}
_ = json.Unmarshal(schemaRaw, &schema)
if len(schema.Properties) < 1 {
logging.V(6).Info("crd schema has no properties", "name", crdName)
return schemaRaw, nil
}
if schema.Properties["apiVersion"] == nil {
apiVersionDefRaw := `{"description":"APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources","type":"string"}`
apiVersionDef := make(map[string]interface{})
_ = json.Unmarshal([]byte(apiVersionDefRaw), &apiVersionDef)
schema.Properties["apiVersion"] = apiVersionDef
}
if schema.Properties["metadata"] == nil {
metadataDefRaw := `{"$ref":"#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta","description":"Standard object's metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata"}`
metadataDef := make(map[string]interface{})
_ = json.Unmarshal([]byte(metadataDefRaw), &metadataDef)
schema.Properties["metadata"] = metadataDef
}
schemaWithDefaultFields, _ := json.Marshal(schema)
return schemaWithDefaultFields, nil
}
func (c *crdSync) CheckSync(ctx context.Context) {
crds, err := c.client.GetDynamicInterface().Resource(runtimeSchema.GroupVersionResource{
Group: "apiextensions.k8s.io",
@ -259,7 +147,7 @@ func (c *crdSync) CheckSync(ctx context.Context) {
logging.Error(err, "could not fetch crd's from server")
return
}
if len(c.controller.crdList) != len(crds.Items) {
if len(c.manager.crdList) != len(crds.Items) {
c.sync()
}
}

View file

@ -3,18 +3,14 @@ package openapi
import (
"encoding/json"
"fmt"
"strconv"
"strings"
"sync"
"github.com/google/gnostic/compiler"
openapiv2 "github.com/google/gnostic/openapiv2"
kyvernov1 "github.com/kyverno/kyverno/api/kyverno/v1"
"github.com/kyverno/kyverno/data"
"github.com/kyverno/kyverno/pkg/autogen"
"github.com/kyverno/kyverno/pkg/engine"
"github.com/kyverno/kyverno/pkg/logging"
"github.com/kyverno/kyverno/pkg/utils"
cmap "github.com/orcaman/concurrent-map/v2"
"github.com/pkg/errors"
"gopkg.in/yaml.v3"
@ -24,14 +20,11 @@ import (
"k8s.io/kube-openapi/pkg/util/proto/validation"
)
// type concurrentMap struct{ cmap.ConcurrentMap }
type ValidateInterface interface {
ValidateResource(resource unstructured.Unstructured, apiVersion, kind string) error
}
// Controller represents OpenAPIController
type Controller struct {
type Manager struct {
// definitions holds the map of {definitionName: *openapiv2.Schema}
definitions cmap.ConcurrentMap[*openapiv2.Schema]
@ -56,9 +49,9 @@ type apiVersions struct {
gvks []string
}
// NewOpenAPIController initializes a new instance of OpenAPIController
func NewOpenAPIController() (*Controller, error) {
controller := &Controller{
// NewOpenAPIManager initializes a new instance of openapi schema manager
func NewOpenAPIManager() (*Manager, error) {
mgr := &Manager{
definitions: cmap.New[*openapiv2.Schema](),
gvkToDefinitionName: cmap.New[string](),
kindToAPIVersions: cmap.New[apiVersions](),
@ -69,23 +62,23 @@ func NewOpenAPIController() (*Controller, error) {
return nil, err
}
controller.updateKindToAPIVersions(apiResourceLists, preferredAPIResourcesLists)
mgr.updateKindToAPIVersions(apiResourceLists, preferredAPIResourcesLists)
defaultDoc, err := getSchemaDocument()
if err != nil {
return nil, err
}
err = controller.useOpenAPIDocument(defaultDoc)
err = mgr.useOpenAPIDocument(defaultDoc)
if err != nil {
return nil, err
}
return controller, nil
return mgr, nil
}
// ValidateResource ...
func (o *Controller) ValidateResource(patchedResource unstructured.Unstructured, apiVersion, kind string) error {
func (o *Manager) ValidateResource(patchedResource unstructured.Unstructured, apiVersion, kind string) error {
var err error
gvk := kind
@ -117,7 +110,7 @@ func (o *Controller) ValidateResource(patchedResource unstructured.Unstructured,
}
// ValidatePolicyMutation ...
func (o *Controller) ValidatePolicyMutation(policy kyvernov1.PolicyInterface) error {
func (o *Manager) ValidatePolicyMutation(policy kyvernov1.PolicyInterface) error {
kindToRules := make(map[string][]kyvernov1.Rule)
for _, rule := range autogen.ComputeRules(policy) {
if rule.HasMutate() {
@ -158,7 +151,7 @@ func (o *Controller) ValidatePolicyMutation(policy kyvernov1.PolicyInterface) er
return nil
}
func (o *Controller) useOpenAPIDocument(doc *openapiv2.Document) error {
func (o *Manager) useOpenAPIDocument(doc *openapiv2.Document) error {
for _, definition := range doc.GetDefinitions().AdditionalProperties {
definitionName := definition.GetName()
@ -190,7 +183,7 @@ func (o *Controller) useOpenAPIDocument(doc *openapiv2.Document) error {
return nil
}
func (o *Controller) getGVKByDefinitionName(definitionName string) (gvk string, preferredGVK bool, err error) {
func (o *Manager) getGVKByDefinitionName(definitionName string) (gvk string, preferredGVK bool, err error) {
paths := strings.Split(definitionName, ".")
kind := paths[len(paths)-1]
versions, ok := o.kindToAPIVersions.Get(kind)
@ -213,64 +206,8 @@ func (o *Controller) getGVKByDefinitionName(definitionName string) (gvk string,
return "", preferredGVK, fmt.Errorf("gvk not found by the given definition name %s, %v", definitionName, versions.gvks)
}
func parseGVK(str string) (group, apiVersion, kind string) {
if strings.Count(str, "/") == 0 {
return "", "", str
}
splitString := strings.Split(str, "/")
if strings.Count(str, "/") == 1 {
return "", splitString[0], splitString[1]
}
return splitString[0], splitString[1], splitString[2]
}
func groupMatches(gvkMap map[string]bool, group, kind string) bool {
if group == "" {
ok := gvkMap["core"]
if ok {
return true
}
} else {
elements := strings.Split(group, ".")
ok := gvkMap[elements[0]]
if ok {
return true
}
}
return false
}
// matchGVK is a helper function that checks if the
// given GVK matches the definition name
func matchGVK(definitionName, gvk string) bool {
paths := strings.Split(definitionName, ".")
gvkMap := make(map[string]bool)
for _, p := range paths {
gvkMap[p] = true
}
group, version, kind := parseGVK(gvk)
ok := gvkMap[kind]
if !ok {
return false
}
ok = gvkMap[version]
if !ok {
return false
}
if !groupMatches(gvkMap, group, kind) {
return false
}
return true
}
// updateKindToAPIVersions sets kindToAPIVersions with static manifests
func (c *Controller) updateKindToAPIVersions(apiResourceLists, preferredAPIResourcesLists []*metav1.APIResourceList) {
func (c *Manager) updateKindToAPIVersions(apiResourceLists, preferredAPIResourcesLists []*metav1.APIResourceList) {
tempKindToAPIVersions := getAllAPIVersions(apiResourceLists)
tempKindToAPIVersions = setPreferredVersions(tempKindToAPIVersions, preferredAPIResourcesLists)
@ -280,19 +217,8 @@ func (c *Controller) updateKindToAPIVersions(apiResourceLists, preferredAPIResou
}
}
func getSchemaDocument() (*openapiv2.Document, error) {
var spec yaml.Node
err := yaml.Unmarshal([]byte(data.SwaggerDoc), &spec)
if err != nil {
return nil, err
}
root := spec.Content[0]
return openapiv2.NewDocument(root, compiler.NewContext("$root", root, nil))
}
// For crd, we do not store definition in document
func (o *Controller) getCRDSchema(kind string) (proto.Schema, error) {
func (o *Manager) getCRDSchema(kind string) (proto.Schema, error) {
if kind == "" {
return nil, errors.New("invalid kind")
}
@ -312,7 +238,7 @@ func (o *Controller) getCRDSchema(kind string) (proto.Schema, error) {
return (existingDefinitions).ParseSchema(definition, &path)
}
func (o *Controller) generateEmptyResource(kindSchema *openapiv2.Schema) interface{} {
func (o *Manager) generateEmptyResource(kindSchema *openapiv2.Schema) interface{} {
types := kindSchema.GetType().GetValue()
if kindSchema.GetXRef() != "" {
@ -347,169 +273,66 @@ func (o *Controller) generateEmptyResource(kindSchema *openapiv2.Schema) interfa
return nil
}
func getArrayValue(kindSchema *openapiv2.Schema, o *Controller) interface{} {
var array []interface{}
for _, schema := range kindSchema.GetItems().GetSchema() {
array = append(array, o.generateEmptyResource(schema))
func (o *Manager) deleteCRDFromPreviousSync() {
for _, crd := range o.crdList {
o.gvkToDefinitionName.Remove(crd)
o.definitions.Remove(crd)
}
return array
o.crdList = make([]string, 0)
}
func getObjectValue(kindSchema *openapiv2.Schema, o *Controller) interface{} {
props := make(map[string]interface{})
properties := kindSchema.GetProperties().GetAdditionalProperties()
if len(properties) == 0 {
return props
}
// ParseCRD loads CRD to the cache
func (o *Manager) ParseCRD(crd unstructured.Unstructured) {
var err error
var wg sync.WaitGroup
var mutex sync.Mutex
wg.Add(len(properties))
for _, property := range properties {
go func(property *openapiv2.NamedSchema) {
prop := o.generateEmptyResource(property.GetValue())
mutex.Lock()
props[property.GetName()] = prop
mutex.Unlock()
wg.Done()
}(property)
}
wg.Wait()
return props
}
crdRaw, _ := json.Marshal(crd.Object)
_ = json.Unmarshal(crdRaw, &crdDefinitionPrior)
func getBoolValue(kindSchema *openapiv2.Schema) bool {
if d := kindSchema.GetDefault(); d != nil {
v := getAnyValue(d)
return string(v) == "true"
}
openV3schema := crdDefinitionPrior.Spec.Validation.OpenAPIV3Schema
crdName := crdDefinitionPrior.Spec.Names.Kind
if e := kindSchema.GetExample(); e != nil {
v := getAnyValue(e)
return string(v) == "true"
}
return false
}
func getNumericValue(kindSchema *openapiv2.Schema) int64 {
if d := kindSchema.GetDefault(); d != nil {
v := getAnyValue(d)
val, _ := strconv.Atoi(string(v))
return int64(val)
}
if e := kindSchema.GetExample(); e != nil {
v := getAnyValue(e)
val, _ := strconv.Atoi(string(v))
return int64(val)
}
return int64(0)
}
func getStringValue(kindSchema *openapiv2.Schema) string {
if d := kindSchema.GetDefault(); d != nil {
v := getAnyValue(d)
return string(v)
}
if e := kindSchema.GetExample(); e != nil {
v := getAnyValue(e)
return string(v)
}
return ""
}
func getAnyValue(any *openapiv2.Any) []byte {
if any != nil {
if val := any.GetValue(); val != nil {
return val.GetValue()
}
}
return nil
}
// getAllAPIVersions gets all available versions for a kind
// returns a map which stores all kinds with its versions
func getAllAPIVersions(apiResourceLists []*metav1.APIResourceList) map[string]apiVersions {
tempKindToAPIVersions := make(map[string]apiVersions)
for _, apiResourceList := range apiResourceLists {
lastKind := ""
for _, apiResource := range apiResourceList.APIResources {
if apiResource.Kind == lastKind {
continue
}
version, ok := tempKindToAPIVersions[apiResource.Kind]
if !ok {
tempKindToAPIVersions[apiResource.Kind] = apiVersions{}
}
gvk := strings.Join([]string{apiResourceList.GroupVersion, apiResource.Kind}, "/")
version.gvks = append(version.gvks, gvk)
tempKindToAPIVersions[apiResource.Kind] = version
lastKind = apiResource.Kind
}
}
return tempKindToAPIVersions
}
// setPreferredVersions sets the serverPreferredGVK of the given apiVersions map
func setPreferredVersions(kindToAPIVersions map[string]apiVersions, preferredAPIResourcesLists []*metav1.APIResourceList) map[string]apiVersions {
tempKindToAPIVersionsCopied := copyKindToAPIVersions(kindToAPIVersions)
for kind, versions := range tempKindToAPIVersionsCopied {
for _, preferredAPIResourcesList := range preferredAPIResourcesLists {
for _, resource := range preferredAPIResourcesList.APIResources {
preferredGV := preferredAPIResourcesList.GroupVersion
preferredGVK := preferredGV + "/" + resource.Kind
if utils.ContainsString(versions.gvks, preferredGVK) {
v := kindToAPIVersions[kind]
// if a Kind belongs to multiple groups, the first group/version
// returned from discovery docs is used as preferred version
// https://github.com/kubernetes/kubernetes/issues/94761#issuecomment-691982480
if v.serverPreferredGVK != "" {
continue
}
v.serverPreferredGVK = strings.Join([]string{preferredGV, kind}, "/")
kindToAPIVersions[kind] = v
}
if openV3schema == nil {
_ = json.Unmarshal(crdRaw, &crdDefinitionNew)
for _, crdVersion := range crdDefinitionNew.Spec.Versions {
if crdVersion.Storage {
openV3schema = crdVersion.Schema.OpenAPIV3Schema
crdName = crdDefinitionNew.Spec.Names.Kind
break
}
}
}
return kindToAPIVersions
}
func copyKindToAPIVersions(old map[string]apiVersions) map[string]apiVersions {
new := make(map[string]apiVersions, len(old))
for key, value := range old {
new[key] = value
if openV3schema == nil {
logging.V(4).Info("skip adding schema, CRD has no properties", "name", crdName)
return
}
return new
}
func getAPIResourceLists() ([]*metav1.APIResourceList, []*metav1.APIResourceList, error) {
var apiResourceLists []*metav1.APIResourceList
err := json.Unmarshal([]byte(data.APIResourceLists), &apiResourceLists)
schemaRaw, _ := json.Marshal(openV3schema)
if len(schemaRaw) < 1 {
logging.V(4).Info("failed to parse crd schema", "name", crdName)
return
}
schemaRaw, err = addingDefaultFieldsToSchema(crdName, schemaRaw)
if err != nil {
return nil, nil, fmt.Errorf("unable to load apiResourceLists: %v", err)
logging.Error(err, "failed to parse crd schema", "name", crdName)
return
}
var preferredAPIResourcesLists []*metav1.APIResourceList
err = json.Unmarshal([]byte(data.APIResourceLists), &preferredAPIResourcesLists)
var schema yaml.Node
_ = yaml.Unmarshal(schemaRaw, &schema)
parsedSchema, err := openapiv2.NewSchema(&schema, compiler.NewContext("schema", &schema, nil))
if err != nil {
return nil, nil, fmt.Errorf("unable to load preferredAPIResourcesLists: %v", err)
v3valueFound := isOpenV3Error(err)
if !v3valueFound {
logging.Error(err, "failed to parse crd schema", "name", crdName)
}
return
}
return apiResourceLists, preferredAPIResourcesLists, nil
o.crdList = append(o.crdList, crdName)
o.gvkToDefinitionName.Set(crdName, crdName)
o.definitions.Set(crdName, parsedSchema)
}

View file

@ -41,7 +41,7 @@ func Test_ValidateMutationPolicy(t *testing.T) {
},
}
o, _ := NewOpenAPIController()
o, _ := NewOpenAPIManager()
for i, tc := range tcs {
policy := v1.ClusterPolicy{}
@ -165,7 +165,7 @@ func Test_matchGVK(t *testing.T) {
// networking.k8s.io/v1beta1/Ingress
// extensions/v1beta1/Ingress
func Test_Ingress(t *testing.T) {
o, err := NewOpenAPIController()
o, err := NewOpenAPIManager()
assert.NilError(t, err)
versions, ok := o.kindToAPIVersions.Get("Ingress")

292
pkg/openapi/utils.go Normal file
View file

@ -0,0 +1,292 @@
package openapi
import (
"encoding/json"
"fmt"
"strconv"
"strings"
"sync"
"github.com/google/gnostic/compiler"
openapiv2 "github.com/google/gnostic/openapiv2"
"github.com/kyverno/kyverno/data"
"github.com/kyverno/kyverno/pkg/logging"
"github.com/kyverno/kyverno/pkg/utils"
"gopkg.in/yaml.v3"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func parseGVK(str string) (group, apiVersion, kind string) {
if strings.Count(str, "/") == 0 {
return "", "", str
}
splitString := strings.Split(str, "/")
if strings.Count(str, "/") == 1 {
return "", splitString[0], splitString[1]
}
return splitString[0], splitString[1], splitString[2]
}
func groupMatches(gvkMap map[string]bool, group, kind string) bool {
if group == "" {
ok := gvkMap["core"]
if ok {
return true
}
} else {
elements := strings.Split(group, ".")
ok := gvkMap[elements[0]]
if ok {
return true
}
}
return false
}
// matchGVK is a helper function that checks if the given GVK matches the definition name
func matchGVK(definitionName, gvk string) bool {
paths := strings.Split(definitionName, ".")
gvkMap := make(map[string]bool)
for _, p := range paths {
gvkMap[p] = true
}
group, version, kind := parseGVK(gvk)
ok := gvkMap[kind]
if !ok {
return false
}
ok = gvkMap[version]
if !ok {
return false
}
if !groupMatches(gvkMap, group, kind) {
return false
}
return true
}
func getSchemaDocument() (*openapiv2.Document, error) {
var spec yaml.Node
err := yaml.Unmarshal([]byte(data.SwaggerDoc), &spec)
if err != nil {
return nil, err
}
root := spec.Content[0]
return openapiv2.NewDocument(root, compiler.NewContext("$root", root, nil))
}
func getArrayValue(kindSchema *openapiv2.Schema, o *Manager) interface{} {
var array []interface{}
for _, schema := range kindSchema.GetItems().GetSchema() {
array = append(array, o.generateEmptyResource(schema))
}
return array
}
func getObjectValue(kindSchema *openapiv2.Schema, o *Manager) interface{} {
props := make(map[string]interface{})
properties := kindSchema.GetProperties().GetAdditionalProperties()
if len(properties) == 0 {
return props
}
var wg sync.WaitGroup
var mutex sync.Mutex
wg.Add(len(properties))
for _, property := range properties {
go func(property *openapiv2.NamedSchema) {
prop := o.generateEmptyResource(property.GetValue())
mutex.Lock()
props[property.GetName()] = prop
mutex.Unlock()
wg.Done()
}(property)
}
wg.Wait()
return props
}
func getBoolValue(kindSchema *openapiv2.Schema) bool {
if d := kindSchema.GetDefault(); d != nil {
v := getAnyValue(d)
return string(v) == "true"
}
if e := kindSchema.GetExample(); e != nil {
v := getAnyValue(e)
return string(v) == "true"
}
return false
}
func getNumericValue(kindSchema *openapiv2.Schema) int64 {
if d := kindSchema.GetDefault(); d != nil {
v := getAnyValue(d)
val, _ := strconv.Atoi(string(v))
return int64(val)
}
if e := kindSchema.GetExample(); e != nil {
v := getAnyValue(e)
val, _ := strconv.Atoi(string(v))
return int64(val)
}
return int64(0)
}
func getStringValue(kindSchema *openapiv2.Schema) string {
if d := kindSchema.GetDefault(); d != nil {
v := getAnyValue(d)
return string(v)
}
if e := kindSchema.GetExample(); e != nil {
v := getAnyValue(e)
return string(v)
}
return ""
}
func getAnyValue(any *openapiv2.Any) []byte {
if any != nil {
if val := any.GetValue(); val != nil {
return val.GetValue()
}
}
return nil
}
// getAllAPIVersions gets all available versions for a kind
// returns a map which stores all kinds with its versions
func getAllAPIVersions(apiResourceLists []*metav1.APIResourceList) map[string]apiVersions {
tempKindToAPIVersions := make(map[string]apiVersions)
for _, apiResourceList := range apiResourceLists {
lastKind := ""
for _, apiResource := range apiResourceList.APIResources {
if apiResource.Kind == lastKind {
continue
}
version, ok := tempKindToAPIVersions[apiResource.Kind]
if !ok {
tempKindToAPIVersions[apiResource.Kind] = apiVersions{}
}
gvk := strings.Join([]string{apiResourceList.GroupVersion, apiResource.Kind}, "/")
version.gvks = append(version.gvks, gvk)
tempKindToAPIVersions[apiResource.Kind] = version
lastKind = apiResource.Kind
}
}
return tempKindToAPIVersions
}
// setPreferredVersions sets the serverPreferredGVK of the given apiVersions map
func setPreferredVersions(kindToAPIVersions map[string]apiVersions, preferredAPIResourcesLists []*metav1.APIResourceList) map[string]apiVersions {
tempKindToAPIVersionsCopied := copyKindToAPIVersions(kindToAPIVersions)
for kind, versions := range tempKindToAPIVersionsCopied {
for _, preferredAPIResourcesList := range preferredAPIResourcesLists {
for _, resource := range preferredAPIResourcesList.APIResources {
preferredGV := preferredAPIResourcesList.GroupVersion
preferredGVK := preferredGV + "/" + resource.Kind
if utils.ContainsString(versions.gvks, preferredGVK) {
v := kindToAPIVersions[kind]
// if a Kind belongs to multiple groups, the first group/version
// returned from discovery docs is used as preferred version
// https://github.com/kubernetes/kubernetes/issues/94761#issuecomment-691982480
if v.serverPreferredGVK != "" {
continue
}
v.serverPreferredGVK = strings.Join([]string{preferredGV, kind}, "/")
kindToAPIVersions[kind] = v
}
}
}
}
return kindToAPIVersions
}
func copyKindToAPIVersions(old map[string]apiVersions) map[string]apiVersions {
new := make(map[string]apiVersions, len(old))
for key, value := range old {
new[key] = value
}
return new
}
func getAPIResourceLists() ([]*metav1.APIResourceList, []*metav1.APIResourceList, error) {
var apiResourceLists []*metav1.APIResourceList
err := json.Unmarshal([]byte(data.APIResourceLists), &apiResourceLists)
if err != nil {
return nil, nil, fmt.Errorf("unable to load apiResourceLists: %v", err)
}
var preferredAPIResourcesLists []*metav1.APIResourceList
err = json.Unmarshal([]byte(data.APIResourceLists), &preferredAPIResourcesLists)
if err != nil {
return nil, nil, fmt.Errorf("unable to load preferredAPIResourcesLists: %v", err)
}
return apiResourceLists, preferredAPIResourcesLists, nil
}
func isOpenV3Error(err error) bool {
unsupportedValues := []string{"anyOf", "allOf", "not"}
v3valueFound := false
for _, value := range unsupportedValues {
if !strings.Contains(err.Error(), fmt.Sprintf("has invalid property: %s", value)) {
v3valueFound = true
break
}
}
return v3valueFound
}
// addingDefaultFieldsToSchema will add any default missing fields like apiVersion, metadata
func addingDefaultFieldsToSchema(crdName string, schemaRaw []byte) ([]byte, error) {
var schema struct {
Properties map[string]interface{} `json:"properties"`
}
_ = json.Unmarshal(schemaRaw, &schema)
if len(schema.Properties) < 1 {
logging.V(6).Info("crd schema has no properties", "name", crdName)
return schemaRaw, nil
}
if schema.Properties["apiVersion"] == nil {
apiVersionDefRaw := `{"description":"APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources","type":"string"}`
apiVersionDef := make(map[string]interface{})
_ = json.Unmarshal([]byte(apiVersionDefRaw), &apiVersionDef)
schema.Properties["apiVersion"] = apiVersionDef
}
if schema.Properties["metadata"] == nil {
metadataDefRaw := `{"$ref":"#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta","description":"Standard object's metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata"}`
metadataDef := make(map[string]interface{})
_ = json.Unmarshal([]byte(metadataDefRaw), &metadataDef)
schema.Properties["metadata"] = metadataDef
}
schemaWithDefaultFields, _ := json.Marshal(schema)
return schemaWithDefaultFields, nil
}

View file

@ -79,13 +79,13 @@ func validateJSONPatchPathForForwardSlash(patch string) error {
}
// Validate checks the policy and rules declarations for required configurations
func Validate(policy kyvernov1.PolicyInterface, client dclient.Interface, mock bool, openAPIController *openapi.Controller) (*admissionv1.AdmissionResponse, error) {
func Validate(policy kyvernov1.PolicyInterface, client dclient.Interface, mock bool, openApiManager *openapi.Manager) (*admissionv1.AdmissionResponse, error) {
namespaced := policy.IsNamespaced()
spec := policy.GetSpec()
background := spec.BackgroundProcessingEnabled()
onPolicyUpdate := spec.GetMutateExistingOnPolicyUpdate()
if !mock {
openapi.NewCRDSync(client, openAPIController).CheckSync(context.TODO())
openapi.NewCRDSync(client, openApiManager).CheckSync(context.TODO())
}
var errs field.ErrorList
@ -359,7 +359,7 @@ func Validate(policy kyvernov1.PolicyInterface, client dclient.Interface, mock b
}
if spec.SchemaValidation == nil || *spec.SchemaValidation {
if err := openAPIController.ValidatePolicyMutation(policy); err != nil {
if err := openApiManager.ValidatePolicyMutation(policy); err != nil {
return nil, err
}
}

View file

@ -345,7 +345,7 @@ func Test_Validate_Policy(t *testing.T) {
}
}`)
openAPIController, _ := openapi.NewOpenAPIController()
openAPIController, _ := openapi.NewOpenAPIManager()
var policy *kyverno.ClusterPolicy
err := json.Unmarshal(rawPolicy, &policy)
assert.NilError(t, err)
@ -496,7 +496,7 @@ func Test_Validate_ErrorFormat(t *testing.T) {
err := json.Unmarshal(rawPolicy, &policy)
assert.NilError(t, err)
openAPIController, _ := openapi.NewOpenAPIController()
openAPIController, _ := openapi.NewOpenAPIManager()
_, err = Validate(policy, nil, true, openAPIController)
assert.Assert(t, err != nil)
}
@ -898,7 +898,7 @@ func Test_Validate_Kind(t *testing.T) {
err := json.Unmarshal(rawPolicy, &policy)
assert.NilError(t, err)
openAPIController, _ := openapi.NewOpenAPIController()
openAPIController, _ := openapi.NewOpenAPIManager()
_, err = Validate(policy, nil, true, openAPIController)
assert.Assert(t, err != nil)
}
@ -947,7 +947,7 @@ func Test_Validate_Any_Kind(t *testing.T) {
err := json.Unmarshal(rawPolicy, &policy)
assert.NilError(t, err)
openAPIController, _ := openapi.NewOpenAPIController()
openAPIController, _ := openapi.NewOpenAPIManager()
_, err = Validate(policy, nil, true, openAPIController)
assert.Assert(t, err != nil)
}
@ -1075,7 +1075,7 @@ func Test_Wildcards_Kind(t *testing.T) {
err := json.Unmarshal(rawPolicy, &policy)
assert.NilError(t, err)
openAPIController, _ := openapi.NewOpenAPIController()
openAPIController, _ := openapi.NewOpenAPIManager()
_, err = Validate(policy, nil, true, openAPIController)
assert.Assert(t, err != nil)
}
@ -1125,7 +1125,7 @@ func Test_Namespced_Policy(t *testing.T) {
err := json.Unmarshal(rawPolicy, &policy)
assert.NilError(t, err)
openAPIController, _ := openapi.NewOpenAPIController()
openAPIController, _ := openapi.NewOpenAPIManager()
_, err = Validate(policy, nil, true, openAPIController)
assert.Assert(t, err != nil)
}
@ -1173,7 +1173,7 @@ func Test_patchesJson6902_Policy(t *testing.T) {
err := json.Unmarshal(rawPolicy, &policy)
assert.NilError(t, err)
openAPIController, _ := openapi.NewOpenAPIController()
openAPIController, _ := openapi.NewOpenAPIManager()
_, err = Validate(policy, nil, true, openAPIController)
assert.NilError(t, err)
}
@ -1221,7 +1221,7 @@ func Test_deny_exec(t *testing.T) {
err = json.Unmarshal(rawPolicy, &policy)
assert.NilError(t, err)
openAPIController, _ := openapi.NewOpenAPIController()
openAPIController, _ := openapi.NewOpenAPIManager()
_, err = Validate(policy, nil, true, openAPIController)
assert.NilError(t, err)
}
@ -1266,7 +1266,7 @@ func Test_existing_resource_policy(t *testing.T) {
err = json.Unmarshal(rawPolicy, &policy)
assert.NilError(t, err)
openAPIController, _ := openapi.NewOpenAPIController()
openAPIController, _ := openapi.NewOpenAPIManager()
_, err = Validate(policy, nil, true, openAPIController)
assert.NilError(t, err)
}
@ -1322,7 +1322,7 @@ func Test_PodControllerAutoGenExclusion_All_Controllers_Policy(t *testing.T) {
err := json.Unmarshal(rawPolicy, &policy)
assert.NilError(t, err)
openAPIController, _ := openapi.NewOpenAPIController()
openAPIController, _ := openapi.NewOpenAPIManager()
res, err := Validate(policy, nil, true, openAPIController)
assert.NilError(t, err)
assert.Assert(t, res == nil)
@ -1379,7 +1379,7 @@ func Test_PodControllerAutoGenExclusion_Not_All_Controllers_Policy(t *testing.T)
err := json.Unmarshal(rawPolicy, &policy)
assert.NilError(t, err)
openAPIController, _ := openapi.NewOpenAPIController()
openAPIController, _ := openapi.NewOpenAPIManager()
res, err := Validate(policy, nil, true, openAPIController)
if res != nil {
assert.Assert(t, res.Warnings != nil)
@ -1438,7 +1438,7 @@ func Test_PodControllerAutoGenExclusion_None_Policy(t *testing.T) {
err := json.Unmarshal(rawPolicy, &policy)
assert.NilError(t, err)
openAPIController, _ := openapi.NewOpenAPIController()
openAPIController, _ := openapi.NewOpenAPIManager()
res, err := Validate(policy, nil, true, openAPIController)
if res != nil {
assert.Assert(t, res.Warnings != nil)

View file

@ -17,14 +17,14 @@ import (
)
type handlers struct {
client dclient.Interface
openAPIController *openapi.Controller
client dclient.Interface
openApiManager *openapi.Manager
}
func NewHandlers(client dclient.Interface, openAPIController *openapi.Controller) webhooks.PolicyHandlers {
func NewHandlers(client dclient.Interface, openAPIController *openapi.Manager) webhooks.PolicyHandlers {
return &handlers{
client: client,
openAPIController: openAPIController,
client: client,
openApiManager: openAPIController,
}
}
@ -38,7 +38,7 @@ func (h *handlers) Validate(logger logr.Logger, request *admissionv1.AdmissionRe
logger.Error(err, "failed to unmarshal policies from admission request")
return admissionutils.ResponseWithMessage(true, fmt.Sprintf("failed to validate policy, check kyverno controller logs for details: %v", err))
}
response, err := policyvalidate.Validate(policy, h.client, false, h.openAPIController)
response, err := policyvalidate.Validate(policy, h.client, false, h.openApiManager)
if err != nil {
logger.Error(err, "policy validation errors")
return admissionutils.ResponseWithMessage(false, err.Error())