feat add chainsaw tests for pod security and exceptions (#10664)

* feat add chainsaw tests for pod security and exceptions

Signed-off-by: Mariam Fahmy <mariam.fahmy@nirmata.com>

* fix: enable ProcMountType in the kind config

Signed-off-by: Mariam Fahmy <mariam.fahmy@nirmata.com>

---------

Signed-off-by: Mariam Fahmy <mariam.fahmy@nirmata.com>

go.mod

@@ -390,5 +390,5 @@ replace (
 	github.com/prometheus/client_golang v1.19.0 => github.com/prometheus/client_golang v1.18.0
 	github.com/prometheus/common v0.48.0 => github.com/prometheus/common v0.44.0
 	github.com/sigstore/cosign/v2 v2.2.4 => github.com/kyverno/cosign/v2 v2.2.4-deps-fix
-	k8s.io/pod-security-admission v0.30.1 => github.com/YTGhost/pod-security-admission v0.22.0-beta.0.0.20240603173423-11663473ae49
+	k8s.io/pod-security-admission v0.30.1 => github.com/kyverno/pod-security-admission v0.0.0-20240715131510-7fb54a8d376d
 )

go.sum

@@ -88,8 +88,6 @@ github.com/ProtonMail/go-crypto v1.0.0 h1:LRuvITjQWX+WIfr930YHG2HNfjR1uOfyf5vE0k
 github.com/ProtonMail/go-crypto v1.0.0/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0=
 github.com/ThalesIgnite/crypto11 v1.2.5 h1:1IiIIEqYmBvUYFeMnHqRft4bwf/O36jryEUpY+9ef8E=
 github.com/ThalesIgnite/crypto11 v1.2.5/go.mod h1:ILDKtnCKiQ7zRoNxcp36Y1ZR8LBPmR2E23+wTQe/MlE=
-github.com/YTGhost/pod-security-admission v0.22.0-beta.0.0.20240603173423-11663473ae49 h1:dwYC6vA5cR+2YFeEasNoJGVj/NrsXCSIWww3qQxeQFY=
-github.com/YTGhost/pod-security-admission v0.22.0-beta.0.0.20240603173423-11663473ae49/go.mod h1:wJpTzOGwDdTbVbIqwBuAX7io1eDQIuW/UfMaK5/Xzn0=
 github.com/agnivade/levenshtein v1.1.1 h1:QY8M92nrzkmr798gCo3kmMyqXFzdQVpxLlGPRBij0P8=
 github.com/agnivade/levenshtein v1.1.1/go.mod h1:veldBMzWxcCG2ZvUTKD2kJNRdCk5hVbJomOvKkmgYbo=
 github.com/alessio/shellescape v1.4.1 h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVKJUX0=
@@ -613,6 +611,8 @@ github.com/kyverno/kyverno-json v0.0.3 h1:EImI/YV41dG4hDQer/W0qMZHfxqul1yiHrBEXx
 github.com/kyverno/kyverno-json v0.0.3/go.mod h1:KUgXPXwUh0Sm/UgtHPomZAfEX8v79I3B5RZbUlzNihg=
 github.com/kyverno/pkg/ext v0.0.0-20240418121121-df8add26c55c h1:lAolpR9H8BwM5lRRvgCQ8JowswyxZRH+fgtIQzHFVCk=
 github.com/kyverno/pkg/ext v0.0.0-20240418121121-df8add26c55c/go.mod h1:02vxM0GNXz9+B/i6+rMfWAIwibUuAH+qFsd73IFskgQ=
+github.com/kyverno/pod-security-admission v0.0.0-20240715131510-7fb54a8d376d h1:JNgsQw8TtxEeGA3lkra0qMG+B4fMhUwZiMRdJ8NQah4=
+github.com/kyverno/pod-security-admission v0.0.0-20240715131510-7fb54a8d376d/go.mod h1:wJpTzOGwDdTbVbIqwBuAX7io1eDQIuW/UfMaK5/Xzn0=
 github.com/lensesio/tableprinter v0.0.0-20201125135848-89e81fc956e7 h1:k/1ku0yehLCPqERCHkIHMDqDg1R02AcCScRuHbamU3s=
 github.com/lensesio/tableprinter v0.0.0-20201125135848-89e81fc956e7/go.mod h1:YR/zYthNdWfO8+0IOyHDcIDBBBS2JMnYUIwSsnwmRqU=
 github.com/letsencrypt/boulder v0.0.0-20240127020530-97a19b18d21e h1:7QjzPboPE+0pVMsZP1sz1mN26m6vew78YmcIZz1FMrg=

pkg/pss/evaluate_test.go

@@ -7408,7 +7408,7 @@ var restricted_runAsNonRoot = []testCase{
 				]
 			}
 		}`),
-		allowed: true,
+		allowed: false,
 	},
 	{
 		name: "restricted_runAsNonRoot_defines_all_violate_spec_true_container_true_spec_level_allowed_positive",

scripts/config/kind/default.yaml

@@ -37,3 +37,4 @@ nodes:
 featureGates:
   "JobPodFailurePolicy": true
   "PodDisruptionConditions": true
+  "ProcMountType": true

test/conformance/chainsaw/exceptions/psa-run-as-non-root/README.md

@@ -0,0 +1,32 @@
+## Description
+
+This test creates an exception for the init containers to set the `runAsNonRoot` to false
+
+## Expected Behavior
+
+1. Create a policy that applies the restricted profile.
+
+2. Create an exception for the init containters to set the `runAsNonRoot` to false.
+
+3. Create a pod with the following characteristics:
+   - The pod has an init container that sets the `runAsNonRoot` field to `false`.
+   - The pod has a container that doesn't set the `runAsNonRoot` field.
+
+   It is expected that the pod will be blocked with a message reporting the violation of the container. The init container is already excluded by the exception.
+
+3. Create a pod with the following characteristics:
+    - The pod has an init container that sets the `runAsNonRoot` field to `true`.
+    - The pod has a container that doesn't set the `runAsNonRoot` field.
+    
+    It is expected that the pod will be blocked with a message reporting the violation of the container.
+
+4. Create a pod with the following characteristics:
+    - The pod has an init container that sets the `runAsNonRoot` field to `false`.
+    - The pod has a container that doesn't set the `runAsNonRoot` field.
+    - `runAsNonRoot` is set to `true` in the pod spec.
+
+    It is expected that the pod will be created successfully.
+
+## Reference Issue(s)
+
+#10581

test/conformance/chainsaw/exceptions/psa-run-as-non-root/bad-pod-01.yaml

@@ -0,0 +1,39 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    run: test-pod
+  name: test-pod
+  namespace: default
+spec:
+  containers:
+  - image: nginx
+    name: test-pod
+    resources:
+      limits:
+        cpu: "2"
+        memory: 4Gi
+      requests:
+        cpu: 50m
+        memory: 256Mi
+    securityContext:
+      allowPrivilegeEscalation: false
+  initContainers:
+  - args:
+    - istio-iptables
+    env:
+    - name: TERMINATION_DRAIN_DURATION_SECONDS
+      value: "30"
+    image: some.registry/istio/proxyv2:1.18.7
+    imagePullPolicy: IfNotPresent
+    name: istio-init
+    resources:
+      limits:
+        cpu: "2"
+        memory: 1Gi
+      requests:
+        cpu: 10m
+        memory: 40Mi
+    securityContext:
+      allowPrivilegeEscalation: false
+      runAsNonRoot: false

test/conformance/chainsaw/exceptions/psa-run-as-non-root/bad-pod-02.yaml

@@ -0,0 +1,38 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    run: test-pod
+  name: test-pod
+spec:
+  containers:
+  - image: nginx
+    name: test-pod
+    resources:
+      limits:
+        cpu: "2"
+        memory: 4Gi
+      requests:
+        cpu: 50m
+        memory: 256Mi
+    securityContext:
+      allowPrivilegeEscalation: false
+  initContainers:
+  - args:
+    - istio-iptables
+    env:
+    - name: TERMINATION_DRAIN_DURATION_SECONDS
+      value: "30"
+    image: some.registry/istio/proxyv2:1.18.7
+    imagePullPolicy: IfNotPresent
+    name: istio-init
+    resources:
+      limits:
+        cpu: "2"
+        memory: 1Gi
+      requests:
+        cpu: 10m
+        memory: 40Mi
+    securityContext:
+      allowPrivilegeEscalation: false
+      runAsNonRoot: true

test/conformance/chainsaw/exceptions/psa-run-as-non-root/chainsaw-test.yaml

@@ -0,0 +1,35 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  creationTimestamp: null
+  name: psa-run-as-non-root
+spec:
+  steps:
+  - name: step-01
+    try:
+    - apply:
+        file: policy.yaml
+    - assert:
+        file: policy-assert.yaml
+  - name: step-02
+    try:
+    - apply:
+        file: exception.yaml
+  - name: step-03
+    try:
+    - script:
+        content: kubectl apply -f bad-pod-01.yaml
+        check:
+          ($error != null): true
+          (contains($stderr, 'spec.containers[0].securityContext.runAsNonRoot')): true
+  - name: step-04
+    try:
+    - script:
+        content: kubectl apply -f bad-pod-02.yaml
+        check:
+          ($error != null): true
+          (contains($stderr, 'spec.containers[0].securityContext.runAsNonRoot')): true
+  - name: step-05
+    try:
+    - apply:
+        file: good-pod.yaml

test/conformance/chainsaw/exceptions/psa-run-as-non-root/exception.yaml

@@ -0,0 +1,21 @@
+apiVersion: kyverno.io/v2
+kind: PolicyException
+metadata:
+  name: pod-security-exception
+spec:
+  exceptions:
+  - policyName: psp-restricted-limited
+    ruleNames:
+    - restricted
+  match:
+    any:
+    - resources:
+        kinds:
+        - Pod
+  podSecurity:
+  - controlName: Running as Non-root
+    images:
+    - '*/istio/proxyv2*'
+    restrictedField: spec.initContainers[*].securityContext.runAsNonRoot
+    values:
+    - "false"

test/conformance/chainsaw/exceptions/psa-run-as-non-root/good-pod.yaml

@@ -0,0 +1,40 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    run: test-pod
+  name: test-pod
+spec:
+  securityContext:
+    runAsNonRoot: true
+  containers:
+  - image: nginx
+    name: test-pod
+    resources:
+      limits:
+        cpu: "2"
+        memory: 4Gi
+      requests:
+        cpu: 50m
+        memory: 256Mi
+    securityContext:
+      allowPrivilegeEscalation: false
+  initContainers:
+  - args:
+    - istio-iptables
+    env:
+    - name: TERMINATION_DRAIN_DURATION_SECONDS
+      value: "30"
+    image: some.registry/istio/proxyv2:1.18.7
+    imagePullPolicy: IfNotPresent
+    name: istio-init
+    resources:
+      limits:
+        cpu: "2"
+        memory: 1Gi
+      requests:
+        cpu: 10m
+        memory: 40Mi
+    securityContext:
+      allowPrivilegeEscalation: false
+      runAsNonRoot: false

test/conformance/chainsaw/exceptions/psa-run-as-non-root/policy-assert.yaml

@@ -0,0 +1,9 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: psp-restricted-limited
+status:
+  conditions:
+  - reason: Succeeded
+    status: "True"
+    type: Ready

test/conformance/chainsaw/exceptions/psa-run-as-non-root/policy.yaml

@@ -0,0 +1,31 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: psp-restricted-limited
+  annotations:
+    pod-policies.kyverno.io/autogen-controllers: none
+spec:
+  background: true
+  validationFailureAction: Enforce
+  rules:
+  - name: restricted
+    match:
+      any:
+      - resources:
+          kinds:
+          - Pod
+          namespaces:
+          - default
+    validate:
+      podSecurity:
+        level: restricted
+        version: v1.29
+        exclude:
+        - controlName: Volume Types
+        - controlName: Seccomp
+        - controlName: Seccomp
+          images:
+          - '*'
+        - controlName: Capabilities
+          images:
+          - "*"

test/conformance/chainsaw/validate/clusterpolicy/cornercases/psa-run-as-non-root/README.md

@@ -0,0 +1,30 @@
+## Description
+
+This test ensures that pods whose container don't set the `runAsNonRoot` field but init container sets the field to `false` are blocked by the `psa-run-as-non-root` policy with messages reporting both violations.
+
+## Expected Behavior
+
+1. Create a policy that applies the restricted profile.
+
+2. Create a pod with the following characteristics:
+   - The pod has an init container that sets the `runAsNonRoot` field to `false`.
+   - The pod has a container that doesn't set the `runAsNonRoot` field.
+
+   It is expected that the pod will be blocked with a message reporting both violations.
+
+3. Create a pod with the following characteristics:
+    - The pod has an init container that sets the `runAsNonRoot` field to `true`.
+    - The pod has a container that doesn't set the `runAsNonRoot` field.
+    
+    It is expected that the pod will be blocked with a message reporting the violation of the container.
+
+4. Create a pod with the following characteristics:
+    - The pod has an init container that sets the `runAsNonRoot` field to `true`.
+    - The pod has a container that doesn't set the `runAsNonRoot` field.
+    - `runAsNonRoot` is set to `true` in the pod spec.
+
+    It is expected that the pod will be created successfully.
+
+## Reference Issue(s)
+
+#10581

test/conformance/chainsaw/validate/clusterpolicy/cornercases/psa-run-as-non-root/bad-pod-01.yaml

@@ -0,0 +1,39 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    run: test-pod
+  name: test-pod
+  namespace: default
+spec:
+  containers:
+  - image: nginx
+    name: test-pod
+    resources:
+      limits:
+        cpu: "2"
+        memory: 4Gi
+      requests:
+        cpu: 50m
+        memory: 256Mi
+    securityContext:
+      allowPrivilegeEscalation: false
+  initContainers:
+  - args:
+    - istio-iptables
+    env:
+    - name: TERMINATION_DRAIN_DURATION_SECONDS
+      value: "30"
+    image: some.registry/istio/proxyv2:1.18.7
+    imagePullPolicy: IfNotPresent
+    name: istio-init
+    resources:
+      limits:
+        cpu: "2"
+        memory: 1Gi
+      requests:
+        cpu: 10m
+        memory: 40Mi
+    securityContext:
+      allowPrivilegeEscalation: false
+      runAsNonRoot: false

test/conformance/chainsaw/validate/clusterpolicy/cornercases/psa-run-as-non-root/bad-pod-02.yaml

@@ -0,0 +1,38 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    run: test-pod
+  name: test-pod
+spec:
+  containers:
+  - image: nginx
+    name: test-pod
+    resources:
+      limits:
+        cpu: "2"
+        memory: 4Gi
+      requests:
+        cpu: 50m
+        memory: 256Mi
+    securityContext:
+      allowPrivilegeEscalation: false
+  initContainers:
+  - args:
+    - istio-iptables
+    env:
+    - name: TERMINATION_DRAIN_DURATION_SECONDS
+      value: "30"
+    image: some.registry/istio/proxyv2:1.18.7
+    imagePullPolicy: IfNotPresent
+    name: istio-init
+    resources:
+      limits:
+        cpu: "2"
+        memory: 1Gi
+      requests:
+        cpu: 10m
+        memory: 40Mi
+    securityContext:
+      allowPrivilegeEscalation: false
+      runAsNonRoot: true

test/conformance/chainsaw/validate/clusterpolicy/cornercases/psa-run-as-non-root/chainsaw-test.yaml

@@ -0,0 +1,32 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  creationTimestamp: null
+  name: psa-run-as-non-root
+spec:
+  steps:
+  - name: step-01
+    try:
+    - apply:
+        file: policy.yaml
+    - assert:
+        file: policy-assert.yaml
+  - name: step-02
+    try:
+    - script:
+        content: kubectl apply -f bad-pod-01.yaml
+        check:
+          ($error != null): true
+          (contains($stderr, 'spec.initContainers[0].securityContext.runAsNonRoot')): true
+          (contains($stderr, 'spec.containers[0].securityContext.runAsNonRoot')): true
+  - name: step-03
+    try:
+    - script:
+        content: kubectl apply -f bad-pod-02.yaml
+        check:
+          ($error != null): true
+          (contains($stderr, 'spec.containers[0].securityContext.runAsNonRoot')): true
+  - name: step-04
+    try:
+    - apply:
+        file: good-pod.yaml

test/conformance/chainsaw/validate/clusterpolicy/cornercases/psa-run-as-non-root/good-pod.yaml

@@ -0,0 +1,40 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    run: test-pod
+  name: test-pod
+spec:
+  securityContext:
+    runAsNonRoot: true
+  containers:
+  - image: nginx
+    name: test-pod
+    resources:
+      limits:
+        cpu: "2"
+        memory: 4Gi
+      requests:
+        cpu: 50m
+        memory: 256Mi
+    securityContext:
+      allowPrivilegeEscalation: false
+  initContainers:
+  - args:
+    - istio-iptables
+    env:
+    - name: TERMINATION_DRAIN_DURATION_SECONDS
+      value: "30"
+    image: some.registry/istio/proxyv2:1.18.7
+    imagePullPolicy: IfNotPresent
+    name: istio-init
+    resources:
+      limits:
+        cpu: "2"
+        memory: 1Gi
+      requests:
+        cpu: 10m
+        memory: 40Mi
+    securityContext:
+      allowPrivilegeEscalation: false
+      runAsNonRoot: true

test/conformance/chainsaw/validate/clusterpolicy/cornercases/psa-run-as-non-root/policy-assert.yaml

@@ -0,0 +1,9 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: psp-restricted-limited
+status:
+  conditions:
+  - reason: Succeeded
+    status: "True"
+    type: Ready

test/conformance/chainsaw/validate/clusterpolicy/cornercases/psa-run-as-non-root/policy.yaml

@@ -0,0 +1,31 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: psp-restricted-limited
+  annotations:
+    pod-policies.kyverno.io/autogen-controllers: none
+spec:
+  background: true
+  validationFailureAction: Enforce
+  rules:
+  - name: restricted
+    match:
+      any:
+      - resources:
+          kinds:
+          - Pod
+          namespaces:
+          - default
+    validate:
+      podSecurity:
+        level: restricted
+        version: v1.29
+        exclude:
+        - controlName: Volume Types
+        - controlName: Seccomp
+        - controlName: Seccomp
+          images:
+          - '*'
+        - controlName: Capabilities
+          images:
+          - "*"