Handling Multi Yaml (Policies and Resources)

pkg/kyverno/apply/command.go

@@ -2,10 +2,9 @@ package apply
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io/ioutil"
-	"os"
-	"path/filepath"
 	"regexp"
 	"time"
 
@@ -13,16 +12,13 @@ import (
 
 	"github.com/nirmata/kyverno/pkg/utils"
 
-	"github.com/nirmata/kyverno/pkg/openapi"
-
+	"github.com/nirmata/kyverno/pkg/kyverno/common"
 	"github.com/nirmata/kyverno/pkg/kyverno/sanitizedError"
 
 	policy2 "github.com/nirmata/kyverno/pkg/policy"
 
 	"k8s.io/apimachinery/pkg/runtime/schema"
 
-	"k8s.io/apimachinery/pkg/util/yaml"
-
 	"github.com/nirmata/kyverno/pkg/engine"
 
 	engineutils "github.com/nirmata/kyverno/pkg/engine/utils"
@@ -64,16 +60,7 @@ func Command() *cobra.Command {
 				return sanitizedError.New(fmt.Sprintf("Specify path to resource file or cluster name"))
 			}
 
-			policies, err := getPolicies(policyPaths)
-			if err != nil {
-				if !sanitizedError.IsErrorSanitized(err) {
-					return sanitizedError.New("Could not parse policy paths")
-				} else {
-					return err
-				}
-			}
-
-			openAPIController, err := openapi.NewOpenAPIController()
+			policies, openAPIController, err := common.GetPoliciesValidation(policyPaths)
 			if err != nil {
 				return err
 			}
@@ -108,7 +95,7 @@ func Command() *cobra.Command {
 			for i, policy := range policies {
 				for j, resource := range resources {
 					if !(j == 0 && i == 0) {
-						fmt.Printf("\n\n=======================================================================\n")
+						fmt.Printf("\n\n==========================================================================================\n")
 					}
 
 					err = applyPolicyOnResource(policy, resource)
@@ -154,12 +141,14 @@ func getResources(policies []*v1.ClusterPolicy, resourcePaths []string, dClient
 	}
 
 	for _, resourcePath := range resourcePaths {
-		resource, err := getResource(resourcePath)
+		getResources, err := getResource(resourcePath)
 		if err != nil {
 			return nil, err
 		}
 
-		resources = append(resources, resource)
+		for _, resource := range getResources {
+			resources = append(resources, resource)
+		}
 	}
 
 	return resources, nil
@@ -188,129 +177,64 @@ func getResourcesOfTypeFromCluster(resourceTypes []string, dClient *client.Clien
 	return resources, nil
 }
 
-func getPoliciesInDir(path string) ([]*v1.ClusterPolicy, error) {
-	var policies []*v1.ClusterPolicy
+func getResource(path string) ([]*unstructured.Unstructured, error) {
 
-	files, err := ioutil.ReadDir(path)
-	if err != nil {
-		return nil, err
-	}
-
-	for _, file := range files {
-		if file.IsDir() {
-			policiesFromDir, err := getPoliciesInDir(filepath.Join(path, file.Name()))
-			if err != nil {
-				return nil, err
-			}
-
-			policies = append(policies, policiesFromDir...)
-		} else {
-			policy, err := getPolicy(filepath.Join(path, file.Name()))
-			if err != nil {
-				return nil, err
-			}
-
-			policies = append(policies, policy)
-		}
-	}
-
-	return policies, nil
-}
-
-func getPolicies(paths []string) ([]*v1.ClusterPolicy, error) {
-	var policies = make([]*v1.ClusterPolicy, 0, len(paths))
-	for _, path := range paths {
-		path = filepath.Clean(path)
-
-		fileDesc, err := os.Stat(path)
-		if err != nil {
-			return nil, err
-		}
-
-		if fileDesc.IsDir() {
-			policiesFromDir, err := getPoliciesInDir(path)
-			if err != nil {
-				return nil, err
-			}
-
-			policies = append(policies, policiesFromDir...)
-		} else {
-			policy, err := getPolicy(path)
-			if err != nil {
-				return nil, err
-			}
-
-			policies = append(policies, policy)
-		}
-	}
-
-	for i := range policies {
-		setFalse := false
-		policies[i].Spec.Background = &setFalse
-	}
-
-	return policies, nil
-}
-
-func getPolicy(path string) (*v1.ClusterPolicy, error) {
-	policy := &v1.ClusterPolicy{}
+	resources := make([]*unstructured.Unstructured, 0)
+	getResourceErrors := make([]error, 0)
 
 	file, err := ioutil.ReadFile(path)
-	if err != nil {
-		return nil, fmt.Errorf("failed to load file: %v", err)
-	}
-
-	policyBytes, err := yaml.ToJSON(file)
 	if err != nil {
 		return nil, err
 	}
 
-	if err := json.Unmarshal(policyBytes, policy); err != nil {
-		return nil, sanitizedError.New(fmt.Sprintf("failed to decode policy in %s", path))
+	files := common.SplitYAMLDocuments(file)
+
+	for _, resourceYaml := range files {
+
+		decode := scheme.Codecs.UniversalDeserializer().Decode
+		resourceObject, metaData, err := decode(resourceYaml, nil, nil)
+		if err != nil {
+			getResourceErrors = append(getResourceErrors, err)
+			continue
+		}
+
+		resourceUnstructured, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&resourceObject)
+		if err != nil {
+			getResourceErrors = append(getResourceErrors, err)
+			continue
+		}
+
+		resourceJSON, err := json.Marshal(resourceUnstructured)
+		if err != nil {
+			getResourceErrors = append(getResourceErrors, err)
+			continue
+		}
+
+		resource, err := engineutils.ConvertToUnstructured(resourceJSON)
+		if err != nil {
+			getResourceErrors = append(getResourceErrors, err)
+			continue
+		}
+
+		resource.SetGroupVersionKind(*metaData)
+
+		if resource.GetNamespace() == "" {
+			resource.SetNamespace("default")
+		}
+
+		resources = append(resources, resource)
 	}
 
-	if policy.TypeMeta.Kind != "ClusterPolicy" {
-		return nil, sanitizedError.New(fmt.Sprintf("resource %v is not a cluster policy", policy.Name))
+	var getErrString string
+	for _, getResourceError := range getResourceErrors {
+		getErrString = getErrString + getResourceError.Error() + "\n"
 	}
 
-	return policy, nil
-}
-
-func getResource(path string) (*unstructured.Unstructured, error) {
-
-	resourceYaml, err := ioutil.ReadFile(path)
-	if err != nil {
-		return nil, err
+	if getErrString != "" {
+		return nil, errors.New(getErrString)
 	}
 
-	decode := scheme.Codecs.UniversalDeserializer().Decode
-	resourceObject, metaData, err := decode(resourceYaml, nil, nil)
-	if err != nil {
-		return nil, err
-	}
-
-	resourceUnstructured, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&resourceObject)
-	if err != nil {
-		return nil, err
-	}
-
-	resourceJSON, err := json.Marshal(resourceUnstructured)
-	if err != nil {
-		return nil, err
-	}
-
-	resource, err := engineutils.ConvertToUnstructured(resourceJSON)
-	if err != nil {
-		return nil, err
-	}
-
-	resource.SetGroupVersionKind(*metaData)
-
-	if resource.GetNamespace() == "" {
-		resource.SetNamespace("default")
-	}
-
-	return resource, nil
+	return resources, nil
 }
 
 func applyPolicyOnResource(policy *v1.ClusterPolicy, resource *unstructured.Unstructured) error {

pkg/kyverno/common/common.go

@@ -0,0 +1,149 @@
+package common
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	v1 "github.com/nirmata/kyverno/pkg/api/kyverno/v1"
+	"github.com/nirmata/kyverno/pkg/kyverno/sanitizedError"
+	"github.com/nirmata/kyverno/pkg/openapi"
+	"k8s.io/apimachinery/pkg/util/yaml"
+)
+
+//GetPolicies - Extracting the policies from multiple YAML
+func GetPolicies(paths []string) ([]*v1.ClusterPolicy, error) {
+	var policies []*v1.ClusterPolicy
+	for _, path := range paths {
+		path = filepath.Clean(path)
+
+		fileDesc, err := os.Stat(path)
+		if err != nil {
+			return nil, err
+		}
+
+		if fileDesc.IsDir() {
+			files, err := ioutil.ReadDir(path)
+			if err != nil {
+				return nil, err
+			}
+
+			listOfFiles := make([]string, 0)
+			for _, file := range files {
+				listOfFiles = append(listOfFiles, filepath.Join(path, file.Name()))
+			}
+
+			policiesFromDir, err := GetPolicies(listOfFiles)
+			if err != nil {
+				return nil, err
+			}
+
+			policies = append(policies, policiesFromDir...)
+		} else {
+			getPolicies, getErrors := GetPolicy(path)
+			var errString string
+			for _, err := range getErrors {
+				if err != nil {
+					errString = errString + err.Error() + "\n"
+				}
+			}
+
+			if errString != "" {
+				return nil, errors.New(errString)
+			}
+
+			for _, policy := range getPolicies {
+				policies = append(policies, policy)
+			}
+		}
+	}
+
+	for i := range policies {
+		setFalse := false
+		policies[i].Spec.Background = &setFalse
+	}
+
+	return policies, nil
+}
+
+// GetPolicy - Extracts policies from a YAML
+func GetPolicy(path string) ([]*v1.ClusterPolicy, []error) {
+	clusterPolicies := make([]*v1.ClusterPolicy, 0)
+	errors := make([]error, 0)
+
+	file, err := ioutil.ReadFile(path)
+	if err != nil {
+		errors = append(errors, fmt.Errorf("failed to load file: %v", err))
+		return clusterPolicies, errors
+	}
+
+	policies := SplitYAMLDocuments(file)
+
+	for _, thisPolicyBytes := range policies {
+		policyBytes, err := yaml.ToJSON(thisPolicyBytes)
+		if err != nil {
+			return clusterPolicies, errors
+		}
+
+		policy := &v1.ClusterPolicy{}
+		if err := json.Unmarshal(policyBytes, policy); err != nil {
+			errors = append(errors, sanitizedError.New(fmt.Sprintf("failed to decode policy in %s", path)))
+			continue
+		}
+
+		if policy.TypeMeta.Kind != "ClusterPolicy" {
+			errors = append(errors, sanitizedError.New(fmt.Sprintf("resource %v is not a cluster policy", policy.Name)))
+			continue
+		}
+		clusterPolicies = append(clusterPolicies, policy)
+	}
+
+	return clusterPolicies, errors
+}
+
+// SplitYAMLDocuments reads the YAML bytes per-document, unmarshals the TypeMeta information from each document
+// and returns a map between the GroupVersionKind of the document and the document bytes
+func SplitYAMLDocuments(yamlBytes []byte) [][]byte {
+	policies := make([][]byte, 0)
+	buf := bytes.NewBuffer(yamlBytes)
+	reader := yaml.NewYAMLReader(bufio.NewReader(buf))
+	for {
+
+		// Read one YAML document at a time, until io.EOF is returned
+		b, err := reader.Read()
+		if err == io.EOF {
+			break
+		} else if err != nil {
+			fmt.Println(err)
+		}
+		if len(b) == 0 {
+			break
+		}
+		policies = append(policies, b)
+	}
+	return policies
+}
+
+//GetPoliciesValidation - validating policies
+func GetPoliciesValidation(policyPaths []string) ([]*v1.ClusterPolicy, *openapi.Controller, error) {
+	policies, err := GetPolicies(policyPaths)
+	if err != nil {
+		if !sanitizedError.IsErrorSanitized(err) {
+			return nil, nil, sanitizedError.New("Could not parse policy paths")
+		} else {
+			return nil, nil, err
+		}
+	}
+
+	openAPIController, err := openapi.NewOpenAPIController()
+	if err != nil {
+		return nil, nil, err
+	}
+	return policies, openAPIController, nil
+}

pkg/kyverno/validate/command.go

@@ -1,23 +1,16 @@
 package validate
 
 import (
-	"encoding/json"
 	"fmt"
-	"io/ioutil"
-	"os"
-	"path/filepath"
 
 	"github.com/nirmata/kyverno/pkg/utils"
 
-	"github.com/nirmata/kyverno/pkg/openapi"
-
+	"github.com/nirmata/kyverno/pkg/kyverno/common"
 	"github.com/nirmata/kyverno/pkg/kyverno/sanitizedError"
 
 	policyvalidate "github.com/nirmata/kyverno/pkg/policy"
 
-	v1 "github.com/nirmata/kyverno/pkg/api/kyverno/v1"
 	"github.com/spf13/cobra"
-	"k8s.io/apimachinery/pkg/util/yaml"
 	log "sigs.k8s.io/controller-runtime/pkg/log"
 )
 
@@ -36,16 +29,7 @@ func Command() *cobra.Command {
 				}
 			}()
 
-			policies, err := getPolicies(policyPaths)
-			if err != nil {
-				if !sanitizedError.IsErrorSanitized(err) {
-					return sanitizedError.New("Could not parse policy paths")
-				} else {
-					return err
-				}
-			}
-
-			openAPIController, err := openapi.NewOpenAPIController()
+			policies, openAPIController, err := common.GetPoliciesValidation(policyPaths)
 			if err != nil {
 				return err
 			}
@@ -62,94 +46,5 @@ func Command() *cobra.Command {
 			return nil
 		},
 	}
-
 	return cmd
 }
-
-func getPoliciesInDir(path string) ([]*v1.ClusterPolicy, error) {
-	var policies []*v1.ClusterPolicy
-
-	files, err := ioutil.ReadDir(path)
-	if err != nil {
-		return nil, err
-	}
-
-	for _, file := range files {
-		if file.IsDir() {
-			policiesFromDir, err := getPoliciesInDir(filepath.Join(path, file.Name()))
-			if err != nil {
-				return nil, err
-			}
-
-			policies = append(policies, policiesFromDir...)
-		} else {
-			policy, err := getPolicy(filepath.Join(path, file.Name()))
-			if err != nil {
-				return nil, err
-			}
-
-			policies = append(policies, policy)
-		}
-	}
-
-	return policies, nil
-}
-
-func getPolicies(paths []string) ([]*v1.ClusterPolicy, error) {
-	var policies = make([]*v1.ClusterPolicy, 0, len(paths))
-	for _, path := range paths {
-		path = filepath.Clean(path)
-
-		fileDesc, err := os.Stat(path)
-		if err != nil {
-			return nil, err
-		}
-
-		if fileDesc.IsDir() {
-			policiesFromDir, err := getPoliciesInDir(path)
-			if err != nil {
-				return nil, err
-			}
-
-			policies = append(policies, policiesFromDir...)
-		} else {
-			policy, err := getPolicy(path)
-			if err != nil {
-				return nil, err
-			}
-
-			policies = append(policies, policy)
-		}
-	}
-
-	for i := range policies {
-		setFalse := false
-		policies[i].Spec.Background = &setFalse
-	}
-
-	return policies, nil
-}
-
-func getPolicy(path string) (*v1.ClusterPolicy, error) {
-	policy := &v1.ClusterPolicy{}
-
-	file, err := ioutil.ReadFile(path)
-	if err != nil {
-		return nil, fmt.Errorf("failed to load file: %v", err)
-	}
-
-	policyBytes, err := yaml.ToJSON(file)
-	if err != nil {
-		return nil, err
-	}
-
-	if err := json.Unmarshal(policyBytes, policy); err != nil {
-		return nil, sanitizedError.New(fmt.Sprintf("failed to decode policy in %s", path))
-	}
-
-	if policy.TypeMeta.Kind != "ClusterPolicy" {
-		return nil, sanitizedError.New(fmt.Sprintf("resource %v is not a cluster policy", policy.Name))
-	}
-
-	return policy, nil
-}