feat: add simple conformance tests (#5073)

* feat: add simple conformance tests
Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>
Co-authored-by: Vyankatesh Kudtarkar <vyankateshkd@gmail.com>

.github/workflows/conformance.yaml

@@ -0,0 +1,21 @@
+name: Conformance tests
+on:
+  pull_request:
+    branches:
+      - 'main'
+      - 'release*'
+
+jobs:
+  run-conformace:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # pin@v3
+      - name: Unshallow
+        run: git fetch --prune --unshallow
+      - name: Setup go
+        uses: actions/setup-go@268d8c0ca0432bb2cf416faae41297df9d262d7f # pin@v3
+        with:
+          go-version: ~1.18.6
+      - name: Kyverno conformance tests
+        run: go run ./test/conformance/main.go

Makefile

@@ -51,11 +51,13 @@ KUSTOMIZE                          := $(TOOLS_DIR)/kustomize
 KUSTOMIZE_VERSION                  := latest
 GOIMPORTS                          := $(TOOLS_DIR)/goimports
 GOIMPORTS_VERSION                  := latest
+HELM                               := $(TOOLS_DIR)/helm
+HELM_VERSION                       := v3.10.1
 HELM_DOCS                          := $(TOOLS_DIR)/helm-docs
 HELM_DOCS_VERSION                  := v1.11.0
 KO                                 := $(TOOLS_DIR)/ko
 KO_VERSION                         := main #e93dbee8540f28c45ec9a2b8aec5ef8e43123966
-TOOLS                              := $(KIND) $(CONTROLLER_GEN) $(CLIENT_GEN) $(LISTER_GEN) $(INFORMER_GEN) $(OPENAPI_GEN) $(GEN_CRD_API_REFERENCE_DOCS) $(GO_ACC) $(KUSTOMIZE) $(GOIMPORTS) $(HELM_DOCS) $(KO)
+TOOLS                              := $(KIND) $(CONTROLLER_GEN) $(CLIENT_GEN) $(LISTER_GEN) $(INFORMER_GEN) $(OPENAPI_GEN) $(GEN_CRD_API_REFERENCE_DOCS) $(GO_ACC) $(KUSTOMIZE) $(GOIMPORTS) $(HELM) $(HELM_DOCS) $(KO)
 ifeq ($(GOOS), darwin)
 SED                                := gsed
 else
@@ -102,6 +104,10 @@ $(GOIMPORTS):
 	@echo Install goimports... >&2
 	@GOBIN=$(TOOLS_DIR) go install golang.org/x/tools/cmd/goimports@$(GOIMPORTS_VERSION)
 
+$(HELM):
+	@echo Install helm... >&2
+	@GOBIN=$(TOOLS_DIR) go install helm.sh/helm/v3/cmd/helm@$(HELM_VERSION)
+
 $(HELM_DOCS):
 	@echo Install helm-docs... >&2
 	@GOBIN=$(TOOLS_DIR) go install github.com/norwoodj/helm-docs/cmd/helm-docs@$(HELM_DOCS_VERSION)
@@ -694,9 +700,9 @@ kind-load-kyverno: $(KIND) image-build-kyverno ## Build kyverno image and load i
 kind-load-all: kind-load-kyvernopre kind-load-kyverno ## Build images and load them in kind cluster
 
 .PHONY: kind-deploy-kyverno
-kind-deploy-kyverno: kind-load-all ## Build images, load them in kind cluster and deploy kyverno helm chart
+kind-deploy-kyverno: $(HELM) kind-load-all ## Build images, load them in kind cluster and deploy kyverno helm chart
 	@echo Install kyverno chart... >&2
-	@helm upgrade --install kyverno --namespace kyverno --wait --create-namespace ./charts/kyverno \
+	@$(HELM) upgrade --install kyverno --namespace kyverno --wait --create-namespace ./charts/kyverno \
 		--set image.repository=$(LOCAL_KYVERNO_IMAGE) \
 		--set image.tag=$(IMAGE_TAG_DEV) \
 		--set initImage.repository=$(LOCAL_KYVERNOPRE_IMAGE) \
@@ -707,14 +713,14 @@ kind-deploy-kyverno: kind-load-all ## Build images, load them in kind cluster an
 	@kubectl rollout restart deployment -n kyverno kyverno
 
 .PHONY: kind-deploy-kyverno-policies
-kind-deploy-kyverno-policies: ## Deploy kyverno-policies helm chart
+kind-deploy-kyverno-policies: $(HELM) ## Deploy kyverno-policies helm chart
 	@echo Install kyverno-policies chart... >&2
-	@helm upgrade --install kyverno-policies --namespace kyverno --create-namespace ./charts/kyverno-policies
+	@$(HELM) upgrade --install kyverno-policies --namespace kyverno --create-namespace ./charts/kyverno-policies
 
 .PHONY: kind-deploy-metrics-server
-kind-deploy-metrics-server: ## Deploy metrics-server helm chart
+kind-deploy-metrics-server: $(HELM) ## Deploy metrics-server helm chart
 	@echo Install metrics-server chart... >&2
-	@helm upgrade --install metrics-server --repo https://charts.bitnami.com/bitnami metrics-server -n kube-system \
+	@$(HELM) upgrade --install metrics-server --repo https://charts.bitnami.com/bitnami metrics-server -n kube-system \
 		--set extraArgs={--kubelet-insecure-tls=true} \
 		--set apiService.create=true
 
@@ -722,9 +728,9 @@ kind-deploy-metrics-server: ## Deploy metrics-server helm chart
 kind-deploy-all: kind-deploy-metrics-server | kind-deploy-kyverno kind-deploy-kyverno-policies ## Build images, load them in kind cluster and deploy helm charts
 
 .PHONY: kind-deploy-reporter
-kind-deploy-reporter: ## Deploy policy-reporter helm chart
+kind-deploy-reporter: $(HELM) ## Deploy policy-reporter helm chart
 	@echo Install policy-reporter chart... >&2
-	@helm upgrade --install policy-reporter --repo https://kyverno.github.io/policy-reporter policy-reporter -n policy-reporter \
+	@$(HELM) upgrade --install policy-reporter --repo https://kyverno.github.io/policy-reporter policy-reporter -n policy-reporter \
 		--set ui.enabled=true \
 		--set kyvernoPlugin.enabled=true \
 		--create-namespace

pkg/policy/background.go

@@ -2,32 +2,40 @@ package policy
 
 import (
 	"fmt"
-	"strings"
+	"regexp"
 
 	kyvernov1 "github.com/kyverno/kyverno/api/kyverno/v1"
 	"github.com/kyverno/kyverno/pkg/autogen"
 )
 
+var forbidden = []*regexp.Regexp{
+	regexp.MustCompile(`[^\.](serviceAccountName)\b`),
+	regexp.MustCompile(`[^\.](serviceAccountNamespace)\b`),
+	regexp.MustCompile(`[^\.](request.userInfo)\b`),
+	regexp.MustCompile(`[^\.](request.roles)\b`),
+	regexp.MustCompile(`[^\.](request.clusterRoles)\b`),
+}
+
 // ContainsUserVariables returns error if variable that does not start from request.object
 func containsUserVariables(policy kyvernov1.PolicyInterface, vars [][]string) error {
-	for _, rule := range policy.GetSpec().Rules {
-		if rule.IsMutateExisting() {
-			return nil
-		}
-	}
-
-	for _, s := range vars {
-		if strings.Contains(s[0], "userInfo") {
-			return fmt.Errorf("variable %s is not allowed", s[0])
-		}
-	}
 	rules := autogen.ComputeRules(policy)
 	for idx := range rules {
 		if err := hasUserMatchExclude(idx, &rules[idx]); err != nil {
 			return err
 		}
 	}
-
+	for _, rule := range policy.GetSpec().Rules {
+		if rule.IsMutateExisting() {
+			return nil
+		}
+	}
+	for _, s := range vars {
+		for _, banned := range forbidden {
+			if banned.Match([]byte(s[0])) {
+				return fmt.Errorf("variable %s is not allowed", s[0])
+			}
+		}
+	}
 	return nil
 }
 

pkg/policy/background_test.go

@@ -132,5 +132,5 @@ func Test_Validation_invalid_backgroundPolicy(t *testing.T) {
 	err := json.Unmarshal(rawPolicy, &policy)
 	assert.NilError(t, err)
 	err = ValidateVariables(&policy, true)
-	assert.ErrorContains(t, err, "variable serviceAccountName must match")
+	assert.ErrorContains(t, err, "variable \"{{serviceAccountName}} is not allowed")
 }

pkg/policy/validate.go

@@ -413,20 +413,14 @@ func Validate(policy kyvernov1.PolicyInterface, client dclient.Interface, mock b
 
 func ValidateVariables(p kyvernov1.PolicyInterface, backgroundMode bool) error {
 	vars := hasVariables(p)
-	if len(vars) == 0 {
-		return nil
-	}
-
-	if err := hasInvalidVariables(p, backgroundMode); err != nil {
-		return fmt.Errorf("policy contains invalid variables: %s", err.Error())
-	}
-
 	if backgroundMode {
 		if err := containsUserVariables(p, vars); err != nil {
 			return fmt.Errorf("only select variables are allowed in background mode. Set spec.background=false to disable background mode for this policy rule: %s ", err)
 		}
 	}
-
+	if err := hasInvalidVariables(p, backgroundMode); err != nil {
+		return fmt.Errorf("policy contains invalid variables: %s", err.Error())
+	}
 	return nil
 }
 

test/cli/test/admission_user_info/disallow_latest_tag.yaml

@@ -10,6 +10,7 @@ metadata:
       a specific version of an application pod.      
 spec:
   validationFailureAction: audit
+  background: false
   rules:
   - name: require-image-tag
     match:

test/conformance/main.go

@@ -0,0 +1,166 @@
+package main
+
+import (
+	"bytes"
+	"errors"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"os/exec"
+	"strings"
+
+	"go.uber.org/multierr"
+	"gopkg.in/yaml.v3"
+)
+
+type CommandExpectation struct {
+	ExitCode *int
+	StdOut   *string
+	StdErr   *string
+}
+
+func (x CommandExpectation) Verify(stdout []byte, stderr []byte, err error) error {
+	exitcode := 0
+	if err != nil {
+		exitError := err.(*exec.ExitError)
+		exitcode = exitError.ExitCode()
+	}
+	if x.ExitCode != nil {
+		if exitcode != *x.ExitCode {
+			return errors.New(fmt.Sprint("unexpected exit code\n  expected: ", *x.ExitCode, "\n  actual:   ", exitcode))
+		}
+	}
+	if x.StdOut != nil {
+		if trim(*x.StdOut, "\n", " ") != trim(string(stdout), "\n", " ") {
+			return errors.New(fmt.Sprint("unexpected stdout\n  expected: ", *x.StdOut, "\n  actual:   ", string(stdout)))
+		}
+	}
+	if x.StdErr != nil {
+		if trim(*x.StdErr, "\n", " ") != trim(string(stderr), "\n", " ") {
+			return errors.New(fmt.Sprint("unexpected stderr\n  expected: ", *x.StdErr, "\n  actual:   ", string(stderr)))
+		}
+	}
+	return nil
+}
+
+type KubectlTest struct {
+	Args   []string
+	Expect *CommandExpectation
+}
+
+func (kt KubectlTest) Run(name string) error {
+	stdout, stderr, err := runCommand("kubectl", kt.Args...)
+	if kt.Expect != nil {
+		return kt.Expect.Verify(stdout, stderr, err)
+	}
+	return nil
+}
+
+type Test struct {
+	Description string
+	Kubectl     *KubectlTest
+}
+
+func (t Test) Run(name string) error {
+	if t.Kubectl != nil {
+		return t.Kubectl.Run(name)
+	}
+	return errors.New("no test defined")
+}
+
+func trim(in string, s ...string) string {
+	for _, s := range s {
+		in = strings.TrimSuffix(in, s)
+	}
+	return in
+}
+
+func runCommand(name string, arg ...string) ([]byte, []byte, error) {
+	cmd := exec.Command(name, arg...)
+	var stdout, stderr bytes.Buffer
+	cmd.Stdout = &stdout
+	cmd.Stderr = &stderr
+	err := cmd.Run()
+	return stdout.Bytes(), stderr.Bytes(), err
+}
+
+func stdCommand(name string, arg ...string) *exec.Cmd {
+	cmd := exec.Command(name, arg...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	return cmd
+}
+
+func makeCluster() error {
+	cmd := stdCommand("make", "kind-create-cluster", "kind-deploy-kyverno")
+	if err := cmd.Run(); err != nil {
+		return err
+	}
+	return nil
+}
+
+func makeDeleteCluster() error {
+	cmd := stdCommand("make", "kind-delete-cluster")
+	if err := cmd.Run(); err != nil {
+		return err
+	}
+	return nil
+}
+
+func loadTests() (map[string][]Test, error) {
+	data, err := ioutil.ReadFile("./test/conformance/tests.yaml")
+	if err != nil {
+		return nil, err
+	}
+	tests := map[string][]Test{}
+	if err := yaml.Unmarshal(data, tests); err != nil {
+		return nil, err
+	}
+	return tests, nil
+}
+
+func main() {
+	var createCluster bool
+	var deleteCluster bool
+	flag.BoolVar(&createCluster, "create-cluster", true, "Set this flag to 'false', to use an existing cluster.")
+	flag.BoolVar(&deleteCluster, "delete-cluster", true, "Set this flag to 'false', to not delete the created cluster.")
+	flag.Parse()
+
+	tests, err := loadTests()
+	if err != nil {
+		log.Fatal(err)
+	}
+	for cluster, tests := range tests {
+		runner := func(name string, tests []Test) error {
+			if err := os.Setenv("KIND_NAME", name); err != nil {
+				return err
+			}
+			if createCluster {
+				if err := makeCluster(); err != nil {
+					return err
+				}
+				if deleteCluster {
+					defer func(name string) {
+						if err := makeDeleteCluster(); err != nil {
+							log.Fatal(err)
+						}
+					}(name)
+				}
+			}
+			var errs []error
+			for _, test := range tests {
+				log.Println("Running test ", test.Description, " ...")
+				if err := test.Run(name); err != nil {
+					log.Println("FAILED: ", err)
+					errs = append(errs, err)
+				}
+			}
+			return multierr.Combine(errs...)
+		}
+		if err := runner(cluster, tests); err != nil {
+			log.Fatal(err)
+		}
+	}
+}

test/conformance/manifests/should-fail/background-userinfo-1.yaml

@@ -0,0 +1,20 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: background-userinfo-1
+spec:
+  validationFailureAction: audit
+  background: true
+  rules:
+  - name: ns-vars
+    match:
+      any:
+      - resources:
+          kinds:
+            - Pod
+    validate:
+      message: The `owner` label is required for all Namespaces.
+      pattern:
+        metadata:
+          labels:
+            foo: "{{request.roles}}"

test/conformance/manifests/should-fail/background-userinfo-2.yaml

@@ -0,0 +1,21 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: background-userinfo-2
+spec:
+  validationFailureAction: audit
+  background: true
+  rules:
+  - name: ns-clusterroles-old
+    match:
+      resources:
+        kinds:
+          - Pod
+      clusterRoles:
+        - foo-admin
+    validate:
+      message: The `owner` label is required for all Namespaces.
+      pattern:
+        metadata:
+          labels:
+            owner: "?*"

test/conformance/manifests/should-fail/background-userinfo-3.yaml

@@ -0,0 +1,20 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: validate-labels
+spec:
+  validationFailureAction: audit
+  background: true
+  rules:
+  - name: ns-vars
+    match:
+      any:
+      - resources:
+          kinds:
+            - Pod
+    validate:
+      message: The `owner` label is required for all Namespaces.
+      pattern:
+        metadata:
+          labels:
+            owner: "{{request.userInfo}}"

test/conformance/manifests/should-fail/background-userinfo-4.yaml

@@ -0,0 +1,20 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: validate-labels
+spec:
+  validationFailureAction: audit
+  background: true
+  rules:
+  - name: ns-vars
+    match:
+      any:
+      - resources:
+          kinds:
+            - Pod
+    validate:
+      message: The `owner` label is required for all Namespaces.
+      pattern:
+        metadata:
+          labels:
+            baz: "{{serviceAccountName}}"

test/conformance/tests.yaml

@@ -0,0 +1,51 @@
+should-fail:
+  - description: Policy with backgound enabled and referencing user infos should be rejected
+    kubectl:
+      args:
+        - create
+        - -f 
+        - test/conformance/manifests/should-fail/background-userinfo-1.yaml
+      expect:
+        exitcode: 1
+        stderr: >-
+          Error from server: error when creating "test/conformance/manifests/should-fail/background-userinfo-1.yaml":
+          admission webhook "validate-policy.kyverno.svc" denied the request: only select variables are allowed in background mode.
+          Set spec.background=false to disable background mode for this policy rule: variable "{{request.roles}} is not allowed 
+  - description: Policy with backgound enabled and referencing user infos should be rejected
+    kubectl:
+      args:
+        - create
+        - -f 
+        - test/conformance/manifests/should-fail/background-userinfo-2.yaml
+      expect:
+        exitcode: 1
+        stderr: >-
+          Error from server: error when creating "test/conformance/manifests/should-fail/background-userinfo-2.yaml":
+          admission webhook "validate-policy.kyverno.svc" denied the request:
+          only select variables are allowed in background mode.
+          Set spec.background=false to disable background mode for this policy rule:
+          invalid variable used at path: spec/rules[0]/match/clusterRoles
+  - description: Policy with backgound enabled and referencing user infos should be rejected
+    kubectl:
+      args:
+        - create
+        - -f 
+        - test/conformance/manifests/should-fail/background-userinfo-3.yaml
+      expect:
+        exitcode: 1
+        stderr: >-
+          Error from server: error when creating "test/conformance/manifests/should-fail/background-userinfo-3.yaml":
+          admission webhook "validate-policy.kyverno.svc" denied the request: only select variables are allowed in background mode.
+          Set spec.background=false to disable background mode for this policy rule: variable "{{request.userInfo}} is not allowed 
+  - description: Policy with backgound enabled and referencing user infos should be rejected
+    kubectl:
+      args:
+        - create
+        - -f 
+        - test/conformance/manifests/should-fail/background-userinfo-4.yaml
+      expect:
+        exitcode: 1
+        stderr: >-
+          Error from server: error when creating "test/conformance/manifests/should-fail/background-userinfo-4.yaml":
+          admission webhook "validate-policy.kyverno.svc" denied the request: only select variables are allowed in background mode.
+          Set spec.background=false to disable background mode for this policy rule: variable "{{serviceAccountName}} is not allowed