clean conformance (#5089)

* clean conformance

Signed-off-by: Chip Zoller <chipzoller@gmail.com>

* adjust names

Signed-off-by: Chip Zoller <chipzoller@gmail.com>

Signed-off-by: Chip Zoller <chipzoller@gmail.com>
Signed-off-by: Charles-Edouard Brétéché <charled.breteche@gmail.com>
Co-authored-by: Vyankatesh Kudtarkar <vyankateshkd@gmail.com>
Co-authored-by: Charles-Edouard Brétéché <charled.breteche@gmail.com>

test/conformance/manifests/generate/foo.yaml

@@ -0,0 +1 @@
+# placeholder

test/conformance/manifests/mutate/foo.yaml

@@ -0,0 +1 @@
+# placeholder

test/conformance/manifests/validate/fail/background-match-clusterroles.yaml

@@ -0,0 +1,22 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: background-match-clusterroles
+spec:
+  validationFailureAction: audit
+  background: true
+  rules:
+  - name: ns-clusterroles
+    match:
+      any:
+      - resources:
+          kinds:
+            - Pod
+        clusterRoles:
+          - foo-admin
+    validate:
+      message: The `owner` label is required for all Namespaces.
+      pattern:
+        metadata:
+          labels:
+            owner: "?*"

test/conformance/manifests/validate/fail/background-match-roles.yaml

@@ -1,18 +1,19 @@
 apiVersion: kyverno.io/v1
 kind: ClusterPolicy
 metadata:
-  name: background-userinfo-2
+  name: background-match-roles
 spec:
   validationFailureAction: audit
   background: true
   rules:
-  - name: ns-clusterroles-old
+  - name: ns-roles
     match:
-      resources:
+      any:
-        kinds:
+      - resources:
-          - Pod
+          kinds:
-      clusterRoles:
+            - Pod
-        - foo-admin
+        roles:
+          - foo-role
     validate:
       message: The `owner` label is required for all Namespaces.
       pattern:

test/conformance/manifests/validate/fail/background-vars-roles.yaml

@@ -1,12 +1,12 @@
 apiVersion: kyverno.io/v1
 kind: ClusterPolicy
 metadata:
-  name: background-userinfo-1
+  name: background-vars-roles
 spec:
   validationFailureAction: audit
   background: true
   rules:
-  - name: ns-vars
+  - name: ns-vars-roles
     match:
       any:
       - resources:

test/conformance/manifests/validate/fail/background-vars-serviceaccountname.yaml

@@ -1,12 +1,12 @@
 apiVersion: kyverno.io/v1
 kind: ClusterPolicy
 metadata:
-  name: validate-labels
+  name: background-vars-serviceaccountname
 spec:
   validationFailureAction: audit
   background: true
   rules:
-  - name: ns-vars
+  - name: ns-vars-serviceaccountname
     match:
       any:
       - resources:

test/conformance/manifests/validate/fail/background-vars-userinfo.yaml

@@ -1,12 +1,12 @@
 apiVersion: kyverno.io/v1
 kind: ClusterPolicy
 metadata:
-  name: validate-labels
+  name: background-vars-userinfo
 spec:
   validationFailureAction: audit
   background: true
   rules:
-  - name: ns-vars
+  - name: ns-vars-userinfo
     match:
       any:
       - resources:

test/conformance/manifests/verifyImages/foo.yaml

@@ -0,0 +1 @@
+# placeholder

test/conformance/tests.yaml

@@ -1,52 +1,64 @@
-should-fail:
+validate-fail:
-  - description: Policy with backgound enabled and referencing user infos should be rejected
+  - description: Policy with background enabled and referencing clusterRoles in match/exclude statements should be rejected
     kubectl:
       args:
         - create
         - -f 
-        - test/conformance/manifests/should-fail/background-userinfo-1.yaml
+        - test/conformance/manifests/validate/fail/background-match-clusterroles.yaml
       expect:
         exitcode: 1
         stderr: >-
-          Error from server: error when creating "test/conformance/manifests/should-fail/background-userinfo-1.yaml":
+          Error from server: error when creating "test/conformance/manifests/validate/fail/background-match-clusterroles.yaml":
           admission webhook "validate-policy.kyverno.svc" denied the request: only select variables are allowed in background mode.
           Set spec.background=false to disable background mode for this policy rule: variable {{request.roles}} is not allowed 
-  - description: Policy with backgound enabled and referencing user infos should be rejected
+  - description: Policy with background enabled and referencing roles in match/exclude statements should be rejected
     kubectl:
       args:
         - create
         - -f 
-        - test/conformance/manifests/should-fail/background-userinfo-2.yaml
+        - test/conformance/manifests/validate/fail/background-match-roles.yaml
       expect:
         exitcode: 1
         stderr: >-
-          Error from server: error when creating "test/conformance/manifests/should-fail/background-userinfo-2.yaml":
+          Error from server: error when creating "test/conformance/manifests/validate/fail/background-match-roles.yaml":
           admission webhook "validate-policy.kyverno.svc" denied the request:
           only select variables are allowed in background mode.
           Set spec.background=false to disable background mode for this policy rule:
-          invalid variable used at path: spec/rules[0]/match/clusterRoles
+          invalid variable used at path: spec/rules[0]/match/any[0]/roles
-  - description: Policy with backgound enabled and referencing user infos should be rejected
+  - description: Policy with background enabled and referencing the var request.roles should be rejected.
     kubectl:
       args:
         - create
         - -f 
-        - test/conformance/manifests/should-fail/background-userinfo-3.yaml
+        - test/conformance/manifests/validate/fail/background-vars-roles.yaml
       expect:
         exitcode: 1
         stderr: >-
-          Error from server: error when creating "test/conformance/manifests/should-fail/background-userinfo-3.yaml":
+          Error from server: error when creating "test/conformance/manifests/validate/fail/background-vars-roles.yaml":
+          admission webhook "validate-policy.kyverno.svc" denied the request: only select variables are allowed in background mode.
+          Set spec.background=false to disable background mode for this policy rule: variable "{{request.roles}} is not allowed 
+  - description: Policy with background enabled and referencing the var request.userInfo should be rejected.
+    kubectl:
+      args:
+        - create
+        - -f 
+        - test/conformance/manifests/validate/fail/background-vars-userinfo.yaml
+      expect:
+        exitcode: 1
+        stderr: >-
+          Error from server: error when creating "test/conformance/manifests/validate/fail/background-vars-userinfo.yaml":
           admission webhook "validate-policy.kyverno.svc" denied the request: only select variables are allowed in background mode.
           Set spec.background=false to disable background mode for this policy rule: variable {{request.userInfo}} is not allowed 
-  - description: Policy with backgound enabled and referencing user infos should be rejected
+  - description: Policy with background enabled and referencing the var request.serviceaccountname should be rejected.
     kubectl:
       args:
         - create
         - -f 
-        - test/conformance/manifests/should-fail/background-userinfo-4.yaml
+        - test/conformance/manifests/validate/fail/background-vars-serviceaccountname.yaml
       expect:
         exitcode: 1
         stderr: >-
-          Error from server: error when creating "test/conformance/manifests/should-fail/background-userinfo-4.yaml":
+          Error from server: error when creating "test/conformance/manifests/validate/fail/background-vars-serviceaccountname.yaml":
           admission webhook "validate-policy.kyverno.svc" denied the request: only select variables are allowed in background mode.
           Set spec.background=false to disable background mode for this policy rule: variable {{serviceAccountName}} is not allowed 
   - description: Best practice policies should create fine