diff --git a/.github/workflows/conformance.yaml b/.github/workflows/conformance.yaml new file mode 100644 index 0000000000..3f5631534a --- /dev/null +++ b/.github/workflows/conformance.yaml @@ -0,0 +1,21 @@ +name: Conformance tests +on: + pull_request: + branches: + - 'main' + - 'release*' + +jobs: + run-conformace: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # pin@v3 + - name: Unshallow + run: git fetch --prune --unshallow + - name: Setup go + uses: actions/setup-go@268d8c0ca0432bb2cf416faae41297df9d262d7f # pin@v3 + with: + go-version: ~1.18.6 + - name: Kyverno conformance tests + run: go run ./test/conformance/main.go diff --git a/Makefile b/Makefile index 28b3c90800..1acf257957 100644 --- a/Makefile +++ b/Makefile @@ -51,11 +51,13 @@ KUSTOMIZE := $(TOOLS_DIR)/kustomize KUSTOMIZE_VERSION := latest GOIMPORTS := $(TOOLS_DIR)/goimports GOIMPORTS_VERSION := latest +HELM := $(TOOLS_DIR)/helm +HELM_VERSION := v3.10.1 HELM_DOCS := $(TOOLS_DIR)/helm-docs HELM_DOCS_VERSION := v1.11.0 KO := $(TOOLS_DIR)/ko KO_VERSION := main #e93dbee8540f28c45ec9a2b8aec5ef8e43123966 -TOOLS := $(KIND) $(CONTROLLER_GEN) $(CLIENT_GEN) $(LISTER_GEN) $(INFORMER_GEN) $(OPENAPI_GEN) $(GEN_CRD_API_REFERENCE_DOCS) $(GO_ACC) $(KUSTOMIZE) $(GOIMPORTS) $(HELM_DOCS) $(KO) +TOOLS := $(KIND) $(CONTROLLER_GEN) $(CLIENT_GEN) $(LISTER_GEN) $(INFORMER_GEN) $(OPENAPI_GEN) $(GEN_CRD_API_REFERENCE_DOCS) $(GO_ACC) $(KUSTOMIZE) $(GOIMPORTS) $(HELM) $(HELM_DOCS) $(KO) ifeq ($(GOOS), darwin) SED := gsed else @@ -102,6 +104,10 @@ $(GOIMPORTS): @echo Install goimports... >&2 @GOBIN=$(TOOLS_DIR) go install golang.org/x/tools/cmd/goimports@$(GOIMPORTS_VERSION) +$(HELM): + @echo Install helm... >&2 + @GOBIN=$(TOOLS_DIR) go install helm.sh/helm/v3/cmd/helm@$(HELM_VERSION) + $(HELM_DOCS): @echo Install helm-docs... >&2 @GOBIN=$(TOOLS_DIR) go install github.com/norwoodj/helm-docs/cmd/helm-docs@$(HELM_DOCS_VERSION) @@ -694,9 +700,9 @@ kind-load-kyverno: $(KIND) image-build-kyverno ## Build kyverno image and load i kind-load-all: kind-load-kyvernopre kind-load-kyverno ## Build images and load them in kind cluster .PHONY: kind-deploy-kyverno -kind-deploy-kyverno: kind-load-all ## Build images, load them in kind cluster and deploy kyverno helm chart +kind-deploy-kyverno: $(HELM) kind-load-all ## Build images, load them in kind cluster and deploy kyverno helm chart @echo Install kyverno chart... >&2 - @helm upgrade --install kyverno --namespace kyverno --wait --create-namespace ./charts/kyverno \ + @$(HELM) upgrade --install kyverno --namespace kyverno --wait --create-namespace ./charts/kyverno \ --set image.repository=$(LOCAL_KYVERNO_IMAGE) \ --set image.tag=$(IMAGE_TAG_DEV) \ --set initImage.repository=$(LOCAL_KYVERNOPRE_IMAGE) \ @@ -707,14 +713,14 @@ kind-deploy-kyverno: kind-load-all ## Build images, load them in kind cluster an @kubectl rollout restart deployment -n kyverno kyverno .PHONY: kind-deploy-kyverno-policies -kind-deploy-kyverno-policies: ## Deploy kyverno-policies helm chart +kind-deploy-kyverno-policies: $(HELM) ## Deploy kyverno-policies helm chart @echo Install kyverno-policies chart... >&2 - @helm upgrade --install kyverno-policies --namespace kyverno --create-namespace ./charts/kyverno-policies + @$(HELM) upgrade --install kyverno-policies --namespace kyverno --create-namespace ./charts/kyverno-policies .PHONY: kind-deploy-metrics-server -kind-deploy-metrics-server: ## Deploy metrics-server helm chart +kind-deploy-metrics-server: $(HELM) ## Deploy metrics-server helm chart @echo Install metrics-server chart... >&2 - @helm upgrade --install metrics-server --repo https://charts.bitnami.com/bitnami metrics-server -n kube-system \ + @$(HELM) upgrade --install metrics-server --repo https://charts.bitnami.com/bitnami metrics-server -n kube-system \ --set extraArgs={--kubelet-insecure-tls=true} \ --set apiService.create=true @@ -722,9 +728,9 @@ kind-deploy-metrics-server: ## Deploy metrics-server helm chart kind-deploy-all: kind-deploy-metrics-server | kind-deploy-kyverno kind-deploy-kyverno-policies ## Build images, load them in kind cluster and deploy helm charts .PHONY: kind-deploy-reporter -kind-deploy-reporter: ## Deploy policy-reporter helm chart +kind-deploy-reporter: $(HELM) ## Deploy policy-reporter helm chart @echo Install policy-reporter chart... >&2 - @helm upgrade --install policy-reporter --repo https://kyverno.github.io/policy-reporter policy-reporter -n policy-reporter \ + @$(HELM) upgrade --install policy-reporter --repo https://kyverno.github.io/policy-reporter policy-reporter -n policy-reporter \ --set ui.enabled=true \ --set kyvernoPlugin.enabled=true \ --create-namespace diff --git a/pkg/policy/background.go b/pkg/policy/background.go index 95f872aaac..75f54ccf3a 100644 --- a/pkg/policy/background.go +++ b/pkg/policy/background.go @@ -2,32 +2,40 @@ package policy import ( "fmt" - "strings" + "regexp" kyvernov1 "github.com/kyverno/kyverno/api/kyverno/v1" "github.com/kyverno/kyverno/pkg/autogen" ) +var forbidden = []*regexp.Regexp{ + regexp.MustCompile(`[^\.](serviceAccountName)\b`), + regexp.MustCompile(`[^\.](serviceAccountNamespace)\b`), + regexp.MustCompile(`[^\.](request.userInfo)\b`), + regexp.MustCompile(`[^\.](request.roles)\b`), + regexp.MustCompile(`[^\.](request.clusterRoles)\b`), +} + // ContainsUserVariables returns error if variable that does not start from request.object func containsUserVariables(policy kyvernov1.PolicyInterface, vars [][]string) error { - for _, rule := range policy.GetSpec().Rules { - if rule.IsMutateExisting() { - return nil - } - } - - for _, s := range vars { - if strings.Contains(s[0], "userInfo") { - return fmt.Errorf("variable %s is not allowed", s[0]) - } - } rules := autogen.ComputeRules(policy) for idx := range rules { if err := hasUserMatchExclude(idx, &rules[idx]); err != nil { return err } } - + for _, rule := range policy.GetSpec().Rules { + if rule.IsMutateExisting() { + return nil + } + } + for _, s := range vars { + for _, banned := range forbidden { + if banned.Match([]byte(s[0])) { + return fmt.Errorf("variable %s is not allowed", s[0]) + } + } + } return nil } diff --git a/pkg/policy/background_test.go b/pkg/policy/background_test.go index 3b5f0b9ab9..5e36016e5a 100644 --- a/pkg/policy/background_test.go +++ b/pkg/policy/background_test.go @@ -132,5 +132,5 @@ func Test_Validation_invalid_backgroundPolicy(t *testing.T) { err := json.Unmarshal(rawPolicy, &policy) assert.NilError(t, err) err = ValidateVariables(&policy, true) - assert.ErrorContains(t, err, "variable serviceAccountName must match") + assert.ErrorContains(t, err, "variable \"{{serviceAccountName}} is not allowed") } diff --git a/pkg/policy/validate.go b/pkg/policy/validate.go index 7202287635..a14f771ddf 100644 --- a/pkg/policy/validate.go +++ b/pkg/policy/validate.go @@ -413,20 +413,14 @@ func Validate(policy kyvernov1.PolicyInterface, client dclient.Interface, mock b func ValidateVariables(p kyvernov1.PolicyInterface, backgroundMode bool) error { vars := hasVariables(p) - if len(vars) == 0 { - return nil - } - - if err := hasInvalidVariables(p, backgroundMode); err != nil { - return fmt.Errorf("policy contains invalid variables: %s", err.Error()) - } - if backgroundMode { if err := containsUserVariables(p, vars); err != nil { return fmt.Errorf("only select variables are allowed in background mode. Set spec.background=false to disable background mode for this policy rule: %s ", err) } } - + if err := hasInvalidVariables(p, backgroundMode); err != nil { + return fmt.Errorf("policy contains invalid variables: %s", err.Error()) + } return nil } diff --git a/test/cli/test/admission_user_info/disallow_latest_tag.yaml b/test/cli/test/admission_user_info/disallow_latest_tag.yaml index db47bfef31..b06132a013 100644 --- a/test/cli/test/admission_user_info/disallow_latest_tag.yaml +++ b/test/cli/test/admission_user_info/disallow_latest_tag.yaml @@ -10,6 +10,7 @@ metadata: a specific version of an application pod. spec: validationFailureAction: audit + background: false rules: - name: require-image-tag match: diff --git a/test/conformance/main.go b/test/conformance/main.go new file mode 100644 index 0000000000..35a36b91a2 --- /dev/null +++ b/test/conformance/main.go @@ -0,0 +1,166 @@ +package main + +import ( + "bytes" + "errors" + "flag" + "fmt" + "io/ioutil" + "log" + "os" + "os/exec" + "strings" + + "go.uber.org/multierr" + "gopkg.in/yaml.v3" +) + +type CommandExpectation struct { + ExitCode *int + StdOut *string + StdErr *string +} + +func (x CommandExpectation) Verify(stdout []byte, stderr []byte, err error) error { + exitcode := 0 + if err != nil { + exitError := err.(*exec.ExitError) + exitcode = exitError.ExitCode() + } + if x.ExitCode != nil { + if exitcode != *x.ExitCode { + return errors.New(fmt.Sprint("unexpected exit code\n expected: ", *x.ExitCode, "\n actual: ", exitcode)) + } + } + if x.StdOut != nil { + if trim(*x.StdOut, "\n", " ") != trim(string(stdout), "\n", " ") { + return errors.New(fmt.Sprint("unexpected stdout\n expected: ", *x.StdOut, "\n actual: ", string(stdout))) + } + } + if x.StdErr != nil { + if trim(*x.StdErr, "\n", " ") != trim(string(stderr), "\n", " ") { + return errors.New(fmt.Sprint("unexpected stderr\n expected: ", *x.StdErr, "\n actual: ", string(stderr))) + } + } + return nil +} + +type KubectlTest struct { + Args []string + Expect *CommandExpectation +} + +func (kt KubectlTest) Run(name string) error { + stdout, stderr, err := runCommand("kubectl", kt.Args...) + if kt.Expect != nil { + return kt.Expect.Verify(stdout, stderr, err) + } + return nil +} + +type Test struct { + Description string + Kubectl *KubectlTest +} + +func (t Test) Run(name string) error { + if t.Kubectl != nil { + return t.Kubectl.Run(name) + } + return errors.New("no test defined") +} + +func trim(in string, s ...string) string { + for _, s := range s { + in = strings.TrimSuffix(in, s) + } + return in +} + +func runCommand(name string, arg ...string) ([]byte, []byte, error) { + cmd := exec.Command(name, arg...) + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + err := cmd.Run() + return stdout.Bytes(), stderr.Bytes(), err +} + +func stdCommand(name string, arg ...string) *exec.Cmd { + cmd := exec.Command(name, arg...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd +} + +func makeCluster() error { + cmd := stdCommand("make", "kind-create-cluster", "kind-deploy-kyverno") + if err := cmd.Run(); err != nil { + return err + } + return nil +} + +func makeDeleteCluster() error { + cmd := stdCommand("make", "kind-delete-cluster") + if err := cmd.Run(); err != nil { + return err + } + return nil +} + +func loadTests() (map[string][]Test, error) { + data, err := ioutil.ReadFile("./test/conformance/tests.yaml") + if err != nil { + return nil, err + } + tests := map[string][]Test{} + if err := yaml.Unmarshal(data, tests); err != nil { + return nil, err + } + return tests, nil +} + +func main() { + var createCluster bool + var deleteCluster bool + flag.BoolVar(&createCluster, "create-cluster", true, "Set this flag to 'false', to use an existing cluster.") + flag.BoolVar(&deleteCluster, "delete-cluster", true, "Set this flag to 'false', to not delete the created cluster.") + flag.Parse() + + tests, err := loadTests() + if err != nil { + log.Fatal(err) + } + for cluster, tests := range tests { + runner := func(name string, tests []Test) error { + if err := os.Setenv("KIND_NAME", name); err != nil { + return err + } + if createCluster { + if err := makeCluster(); err != nil { + return err + } + if deleteCluster { + defer func(name string) { + if err := makeDeleteCluster(); err != nil { + log.Fatal(err) + } + }(name) + } + } + var errs []error + for _, test := range tests { + log.Println("Running test ", test.Description, " ...") + if err := test.Run(name); err != nil { + log.Println("FAILED: ", err) + errs = append(errs, err) + } + } + return multierr.Combine(errs...) + } + if err := runner(cluster, tests); err != nil { + log.Fatal(err) + } + } +} diff --git a/test/conformance/manifests/should-fail/background-userinfo-1.yaml b/test/conformance/manifests/should-fail/background-userinfo-1.yaml new file mode 100644 index 0000000000..b207a0b884 --- /dev/null +++ b/test/conformance/manifests/should-fail/background-userinfo-1.yaml @@ -0,0 +1,20 @@ +apiVersion: kyverno.io/v1 +kind: ClusterPolicy +metadata: + name: background-userinfo-1 +spec: + validationFailureAction: audit + background: true + rules: + - name: ns-vars + match: + any: + - resources: + kinds: + - Pod + validate: + message: The `owner` label is required for all Namespaces. + pattern: + metadata: + labels: + foo: "{{request.roles}}" diff --git a/test/conformance/manifests/should-fail/background-userinfo-2.yaml b/test/conformance/manifests/should-fail/background-userinfo-2.yaml new file mode 100644 index 0000000000..8d1ee8ee8c --- /dev/null +++ b/test/conformance/manifests/should-fail/background-userinfo-2.yaml @@ -0,0 +1,21 @@ +apiVersion: kyverno.io/v1 +kind: ClusterPolicy +metadata: + name: background-userinfo-2 +spec: + validationFailureAction: audit + background: true + rules: + - name: ns-clusterroles-old + match: + resources: + kinds: + - Pod + clusterRoles: + - foo-admin + validate: + message: The `owner` label is required for all Namespaces. + pattern: + metadata: + labels: + owner: "?*" diff --git a/test/conformance/manifests/should-fail/background-userinfo-3.yaml b/test/conformance/manifests/should-fail/background-userinfo-3.yaml new file mode 100644 index 0000000000..820bc7595e --- /dev/null +++ b/test/conformance/manifests/should-fail/background-userinfo-3.yaml @@ -0,0 +1,20 @@ +apiVersion: kyverno.io/v1 +kind: ClusterPolicy +metadata: + name: validate-labels +spec: + validationFailureAction: audit + background: true + rules: + - name: ns-vars + match: + any: + - resources: + kinds: + - Pod + validate: + message: The `owner` label is required for all Namespaces. + pattern: + metadata: + labels: + owner: "{{request.userInfo}}" diff --git a/test/conformance/manifests/should-fail/background-userinfo-4.yaml b/test/conformance/manifests/should-fail/background-userinfo-4.yaml new file mode 100644 index 0000000000..23ab95ecd3 --- /dev/null +++ b/test/conformance/manifests/should-fail/background-userinfo-4.yaml @@ -0,0 +1,20 @@ +apiVersion: kyverno.io/v1 +kind: ClusterPolicy +metadata: + name: validate-labels +spec: + validationFailureAction: audit + background: true + rules: + - name: ns-vars + match: + any: + - resources: + kinds: + - Pod + validate: + message: The `owner` label is required for all Namespaces. + pattern: + metadata: + labels: + baz: "{{serviceAccountName}}" diff --git a/test/conformance/tests.yaml b/test/conformance/tests.yaml new file mode 100644 index 0000000000..c86d850947 --- /dev/null +++ b/test/conformance/tests.yaml @@ -0,0 +1,51 @@ +should-fail: + - description: Policy with backgound enabled and referencing user infos should be rejected + kubectl: + args: + - create + - -f + - test/conformance/manifests/should-fail/background-userinfo-1.yaml + expect: + exitcode: 1 + stderr: >- + Error from server: error when creating "test/conformance/manifests/should-fail/background-userinfo-1.yaml": + admission webhook "validate-policy.kyverno.svc" denied the request: only select variables are allowed in background mode. + Set spec.background=false to disable background mode for this policy rule: variable "{{request.roles}} is not allowed + - description: Policy with backgound enabled and referencing user infos should be rejected + kubectl: + args: + - create + - -f + - test/conformance/manifests/should-fail/background-userinfo-2.yaml + expect: + exitcode: 1 + stderr: >- + Error from server: error when creating "test/conformance/manifests/should-fail/background-userinfo-2.yaml": + admission webhook "validate-policy.kyverno.svc" denied the request: + only select variables are allowed in background mode. + Set spec.background=false to disable background mode for this policy rule: + invalid variable used at path: spec/rules[0]/match/clusterRoles + - description: Policy with backgound enabled and referencing user infos should be rejected + kubectl: + args: + - create + - -f + - test/conformance/manifests/should-fail/background-userinfo-3.yaml + expect: + exitcode: 1 + stderr: >- + Error from server: error when creating "test/conformance/manifests/should-fail/background-userinfo-3.yaml": + admission webhook "validate-policy.kyverno.svc" denied the request: only select variables are allowed in background mode. + Set spec.background=false to disable background mode for this policy rule: variable "{{request.userInfo}} is not allowed + - description: Policy with backgound enabled and referencing user infos should be rejected + kubectl: + args: + - create + - -f + - test/conformance/manifests/should-fail/background-userinfo-4.yaml + expect: + exitcode: 1 + stderr: >- + Error from server: error when creating "test/conformance/manifests/should-fail/background-userinfo-4.yaml": + admission webhook "validate-policy.kyverno.svc" denied the request: only select variables are allowed in background mode. + Set spec.background=false to disable background mode for this policy rule: variable "{{serviceAccountName}} is not allowed