From e5f5ef9db15609f31bb9cea168bc93a032ceea86 Mon Sep 17 00:00:00 2001 From: NoSkillGirl Date: Mon, 31 Aug 2020 17:33:01 +0530 Subject: [PATCH] split multi yaml in crd --- pkg/kyverno/common/common.go | 251 ++++++++++---------------------- pkg/kyverno/validate/command.go | 19 +-- 2 files changed, 83 insertions(+), 187 deletions(-) diff --git a/pkg/kyverno/common/common.go b/pkg/kyverno/common/common.go index 51c7170aaf..cc2362660b 100644 --- a/pkg/kyverno/common/common.go +++ b/pkg/kyverno/common/common.go @@ -11,10 +11,11 @@ import ( "os" "path/filepath" "regexp" - "strings" yaml_v2 "sigs.k8s.io/yaml" + "strings" "errors" + jsonpatch "github.com/evanphx/json-patch" "github.com/go-logr/logr" v1 "github.com/nirmata/kyverno/pkg/api/kyverno/v1" @@ -217,7 +218,62 @@ func MutatePolicy(policy *v1.ClusterPolicy, logger logr.Logger) (*v1.ClusterPoli -func ConvertFileToUnstructed(path string) (*unstructured.Unstructured, error) { +// ConvertFileToUnstructed - converting file to unstructured +func ConvertFileToUnstructed(crdPaths []string) (crds []*unstructured.Unstructured, err error) { + crds, err = GetCRDs(crdPaths) + if err != nil { + if !sanitizedError.IsErrorSanitized(err) { + return nil, sanitizedError.NewWithError((fmt.Sprintf("failed to parse %v path/s.", crdPaths)), err) + } + return nil, err + } + return crds, nil +} + +// GetCRDs - Extracting the crds from multiple YAML +func GetCRDs(paths []string) (unstructuredCrds []*unstructured.Unstructured, err error) { + log := log.Log + for _, path := range paths { + path = filepath.Clean(path) + + fileDesc, err := os.Stat(path) + if err != nil { + log.Error(err, "failed to describe file") + return nil, err + } + + if fileDesc.IsDir() { + files, err := ioutil.ReadDir(path) + if err != nil { + return nil, sanitizedError.NewWithError(fmt.Sprintf("failed to parse %v", path), err) + } + + listOfFiles := make([]string, 0) + for _, file := range files { + listOfFiles = append(listOfFiles, filepath.Join(path, file.Name())) + } + + policiesFromDir, err := GetCRDs(listOfFiles) + if err != nil { + log.Error(err, fmt.Sprintf("failed to extract crds from %v", listOfFiles)) + return nil, sanitizedError.NewWithError(("failed to extract crds"), err) + } + + unstructuredCrds = append(unstructuredCrds, policiesFromDir...) + } else { + getCRDs, err := GetCRD(path) + if err != nil { + fmt.Printf("failed to extract crds: %s\n", err) + } + + unstructuredCrds = append(unstructuredCrds, getCRDs...) + } + } + return nil, nil +} + +// GetCRD - Extracts crds from a YAML +func GetCRD(path string) (unstructuredCrds []*unstructured.Unstructured, err error) { log := log.Log path = filepath.Clean(path) @@ -231,185 +287,30 @@ func ConvertFileToUnstructed(path string) (*unstructured.Unstructured, error) { return nil, errors.New("path should be a file") } - bytes, err := ioutil.ReadFile(path) + yamlbytes, err := ioutil.ReadFile(path) if err != nil { log.Error(err, "failed to read file", "file", path) return nil, err } var u unstructured.Unstructured - err = yaml_v2.Unmarshal(bytes, &u) + buf := bytes.NewBuffer(yamlbytes) + reader := yaml.NewYAMLReader(bufio.NewReader(buf)) - if err != nil { - log.Error(err, "failed to convert file into unstructured object", "file", path) - return nil, err + for { + // Read one YAML document at a time, until io.EOF is returned + b, err := reader.Read() + if err == io.EOF || len(b) == 0 { + break + } else if err != nil { + log.Error(err, "unable to read yaml") + } + err = yaml_v2.Unmarshal(b, &u) + if err != nil { + log.Error(err, "failed to convert file into unstructured object", "file", path) + return nil, err + } + unstructuredCrds = append(unstructuredCrds, &u) } - return &u, nil + return unstructuredCrds, nil } - - -//func ValidatePolicyAgainstCrd(policy *v1.ClusterPolicy, path string) error { -// log := log.Log -// path = filepath.Clean(path) -// -// fileDesc, err := os.Stat(path) -// if err != nil { -// log.Error(err, "failed to describe crd file") -// return err -// } -// -// if fileDesc.IsDir() { -// return errors.New("crd path should be a file") -// } -// -// bytes, err := ioutil.ReadFile(path) -// if err != nil { -// log.Error(err, "failed to crd read file") -// return err -// } -// -// var crd unstructured.Unstructured -// err = yaml_v2.Unmarshal(bytes, &crd) -// -// if err != nil { -// return err -// } -// log.Info("coming till here .................. 5") -// -// // crdDefinitionPrior represents CRDs version prior to 1.16 -// var crdDefinitionPrior struct { -// Spec struct { -// Names struct { -// Kind string `json:"kind"` -// } `json:"names"` -// Validation struct { -// OpenAPIV3Schema interface{} `json:"openAPIV3Schema"` -// } `json:"validation"` -// } `json:"spec"` -// } -// -// // crdDefinitionNew represents CRDs version 1.16+ -// var crdDefinitionNew struct { -// Spec struct { -// Names struct { -// Kind string `json:"kind"` -// } `json:"names"` -// Versions []struct { -// Schema struct { -// OpenAPIV3Schema interface{} `json:"openAPIV3Schema"` -// } `json:"schema"` -// Storage bool `json:"storage"` -// } `json:"versions"` -// } `json:"spec"` -// } -// -// log.Info("coming till here .................. 6") -// crdRaw, _ := json.Marshal(crd.Object) -// _ = json.Unmarshal(crdRaw, &crdDefinitionPrior) -// -// log.Info("coming till here .................. 7") -// openV3schema := crdDefinitionPrior.Spec.Validation.OpenAPIV3Schema -// crdName := crdDefinitionPrior.Spec.Names.Kind -// fmt.Println(crdName) -// -// log.Info("coming till here .................. 8") -// -// if openV3schema == nil { -// log.Info("coming till here .................. coming into openV3Schema = nil") -// _ = json.Unmarshal(crdRaw, &crdDefinitionNew) -// for _, crdVersion := range crdDefinitionNew.Spec.Versions { -// if crdVersion.Storage { -// openV3schema = crdVersion.Schema.OpenAPIV3Schema -// crdName = crdDefinitionNew.Spec.Names.Kind -// break -// } -// } -// } -// -// log.Info("coming till here .................. 9") -// log.Info("crd", "openV3schema", openV3schema) -// -// schemaRaw, _ := json.Marshal(openV3schema) -// if len(schemaRaw) < 1 { -// //log.Log.V(3).Info("could not parse crd schema", "name", crdName) -// return err -// } -// log.Info("coming till here .................. 10") -// -// //schemaRaw, err = addingDefaultFieldsToSchema(schemaRaw) -// //if err != nil { -// // //log.Log.Error(err, "could not parse crd schema", "name", crdName) -// // //return err -// //} -// log.Info("coming till here .................. 11") -// -// schema := new(spec.Schema) -// _ = json.Unmarshal(schemaRaw, schema) -// -// -// // strfmt.Default is the registry of recognized formats -// err = validate.AgainstSchema(schema, policy, strfmt.Default) -// if err != nil { -// fmt.Printf("JSON does not validate against schema: %v", err) -// } else { -// fmt.Printf("OK") -// } -// log.Info("coming till here .................. 14") -// -// //var schema yaml_v2.MapSlice -// //_ = yaml_v2.Unmarshal(schemaRaw, &schema) -// // -// //parsedSchema, err := openapi_v2.NewSchema(schema, compiler.NewContext("schema", nil)) -// //if err != nil { -// // //log.Log.Error(err, "could not parse crd schema", "name", crdName) -// // return -// //} -// -// -// -// //var spec yaml_v2.MapSlice -// //err := yaml_v2.Unmarshal([]byte(data.SwaggerDoc), &spec) -// //if err != nil { -// // return err -// //} -// // -// //crdDoc, err := openapi_v2.NewDocument(spec, compiler.NewContext("$root", nil)) -// //if err != nil { -// // return err -// //} -// // -// //crdDoc -// -// return nil -//} -// -//// addingDefaultFieldsToSchema will add any default missing fields like apiVersion, metadata -//func addingDefaultFieldsToSchema(schemaRaw []byte) ([]byte, error) { -// var schema struct { -// Properties map[string]interface{} `json:"properties"` -// } -// _ = json.Unmarshal(schemaRaw, &schema) -// -// if len(schema.Properties) < 1 { -// return nil, errors.New("crd schema has no properties") -// } -// -// if schema.Properties["apiVersion"] == nil { -// apiVersionDefRaw := `{"description":"APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources","type":"string"}` -// apiVersionDef := make(map[string]interface{}) -// _ = json.Unmarshal([]byte(apiVersionDefRaw), &apiVersionDef) -// schema.Properties["apiVersion"] = apiVersionDef -// } -// -// if schema.Properties["metadata"] == nil { -// metadataDefRaw := `{"$ref":"#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta","description":"Standard object's metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata"}` -// metadataDef := make(map[string]interface{}) -// _ = json.Unmarshal([]byte(metadataDefRaw), &metadataDef) -// schema.Properties["metadata"] = metadataDef -// } -// -// schemaWithDefaultFields, _ := json.Marshal(schema) -// -// return schemaWithDefaultFields, nil -//} - diff --git a/pkg/kyverno/validate/command.go b/pkg/kyverno/validate/command.go index ab24694a89..aab75b68dd 100644 --- a/pkg/kyverno/validate/command.go +++ b/pkg/kyverno/validate/command.go @@ -4,9 +4,10 @@ import ( "encoding/json" "errors" "fmt" - "github.com/nirmata/kyverno/pkg/utils" "os" + "github.com/nirmata/kyverno/pkg/utils" + "github.com/nirmata/kyverno/pkg/kyverno/common" "github.com/nirmata/kyverno/pkg/kyverno/sanitizedError" @@ -51,17 +52,11 @@ func Command() *cobra.Command { // if CRD's are passed, add these to OpenAPIController if len(crdPaths) > 0 { - //err := common.ValidatePolicyAgainstCrd(policy, crdPath) - //if err != nil { - // log.Log.Error(err, "policy "+policy.Name+" is invalid") - // //os.Exit(1) - // return err - //} - for _, path := range crdPaths { - crd, err := common.ConvertFileToUnstructed(path) - if err != nil { - log.Error(err, "crd is invalid", "file", path) - } + crds, err := common.ConvertFileToUnstructed(crdPaths) + if err != nil { + log.Error(err, "crd is invalid", "file", crdPaths) + } + for _, crd := range crds { openAPIController.ParseCRD(*crd) } }