Evaluate one version of each pod security standard (#10924)

The original logic for evaluating pod security standards took two steps
for each defined check:

1. If the policy author requested the latest version of the standard,
   find the newest version of the check and evaluate the pod against it,
   adding any failure to the final results.

2. Otherwise, evaluate the pod against *each version of the check* whose
   minimum version is below the requested version, adding any failures
   to the final results.

This second step can be problematic, as new PSS versions may permit a
broader range of values for a restricted field compared to old versions.
As a concrete example, versioned podSecurity rules don't permit some of
the newer sysctls allowed by Kubernetes v1.27 and v1.29, since Kyverno
still evaluates v1.0 of the check.

With this change, Kyverno identifies the highest version of the check
that the podSecurity rule allows, and only executes that version of the
check against the pod. Since the "latest" version is special-cased to
compare newer than all non-latest versions, no special logic is required
in that case.

I've added unit tests for several combinations of sysctl and policy
version, especially to check that policy v1.27 permits the new sysctl
allowed in v1.27 but not the sysctls allowed in v1.29. I've also taken
the liberty of changing `assert.Assert` to `assert.Check`, to collect
multiple failures from a single unit test run.

Signed-off-by: Alex Hamlin <alexanderh@qualtrics.com>

pkg/pss/evaluate.go

@@ -23,47 +23,26 @@ var (
 // Evaluate Pod's specified containers only and get PSSCheckResults
 func evaluatePSS(level *api.LevelVersion, pod corev1.Pod) (results []pssutils.PSSCheckResult) {
 	checks := policy.DefaultChecks()
-	var latestVersionCheck policy.VersionedCheck
 	for _, check := range checks {
 		if level.Level == api.LevelBaseline && check.Level != level.Level {
 			continue
 		}
 
-		latestVersionCheck = check.Versions[0]
+		selectedCheck := check.Versions[0]
 		for i := 1; i < len(check.Versions); i++ {
-			vc := check.Versions[i]
+			nextCheck := check.Versions[i]
-			if !vc.MinimumVersion.Older(latestVersionCheck.MinimumVersion) {
+			if !level.Version.Older(nextCheck.MinimumVersion) && selectedCheck.MinimumVersion.Older(nextCheck.MinimumVersion) {
-				latestVersionCheck = vc
+				selectedCheck = nextCheck
 			}
 		}
 
-		if level.Version == api.LatestVersion() {
+		checkResult := selectedCheck.CheckPod(&pod.ObjectMeta, &pod.Spec, policy.WithFieldErrors())
-			checkResult := latestVersionCheck.CheckPod(&pod.ObjectMeta, &pod.Spec, policy.WithFieldErrors())
+		if !checkResult.Allowed {
-			if !checkResult.Allowed {
+			results = append(results, pssutils.PSSCheckResult{
-				results = append(results, pssutils.PSSCheckResult{
+				ID:               string(check.ID),
-					ID:               string(check.ID),
+				CheckResult:      checkResult,
-					CheckResult:      checkResult,
+				RestrictedFields: GetRestrictedFields(check),
-					RestrictedFields: GetRestrictedFields(check),
+			})
-				})
-			}
-		}
-
-		for _, versionCheck := range check.Versions {
-			// the latest check returned twice, skip duplicate application
-			if level.Version == api.LatestVersion() {
-				continue
-			} else if level.Version != api.LatestVersion() && level.Version.Older(versionCheck.MinimumVersion) {
-				continue
-			}
-			checkResult := versionCheck.CheckPod(&pod.ObjectMeta, &pod.Spec, policy.WithFieldErrors())
-			// Append only if the checkResult is not already in pssCheckResult
-			if !checkResult.Allowed {
-				results = append(results, pssutils.PSSCheckResult{
-					ID:               string(check.ID),
-					CheckResult:      checkResult,
-					RestrictedFields: GetRestrictedFields(check),
-				})
-			}
 		}
 	}
 	return results

pkg/pss/evaluate_test.go

@@ -55,7 +55,7 @@ func Test_EvaluatePod(t *testing.T) {
 				fmt.Printf("failed check result: %v\n", result)
 			}
 		}
-		assert.Assert(t, allowed == test.allowed, fmt.Sprintf("test \"%s\" fails", test.name))
+		assert.Check(t, allowed == test.allowed, fmt.Sprintf("test \"%s\" fails", test.name))
 	}
 }
 
@@ -5480,6 +5480,161 @@ var baseline_sysctls = []testCase{
 		}`),
 		allowed: true,
 	},
+	{
+		name: "baseline_sysctls_new_v1.27_policy_v1.0_allowed_negative",
+		rawRule: []byte(`
+		{
+			"level": "baseline",
+			"version": "v1.0"
+		}`),
+		rawPod: []byte(`
+		{
+			"kind": "Pod",
+			"metadata": {
+				"name": "test"
+			},
+			"spec": {
+				"securityContext": {
+					"sysctls": [
+						{
+							"name": "net.ipv4.ip_local_reserved_ports"
+						}
+					]
+				},
+				"containers": [
+					{
+						"name": "nginx",
+						"image": "nginx"
+					}
+				]
+			}
+		}`),
+		allowed: false,
+	},
+	{
+		name: "baseline_sysctls_new_v1.27_policy_v1.27_allowed_positive",
+		rawRule: []byte(`
+		{
+			"level": "baseline",
+			"version": "v1.27"
+		}`),
+		rawPod: []byte(`
+		{
+			"kind": "Pod",
+			"metadata": {
+				"name": "test"
+			},
+			"spec": {
+				"securityContext": {
+					"sysctls": [
+						{
+							"name": "net.ipv4.ip_local_reserved_ports"
+						}
+					]
+				},
+				"containers": [
+					{
+						"name": "nginx",
+						"image": "nginx"
+					}
+				]
+			}
+		}`),
+		allowed: true,
+	},
+	{
+		name: "baseline_sysctls_new_v1.29_policy_v1.27_allowed_negative",
+		rawRule: []byte(`
+		{
+			"level": "baseline",
+			"version": "v1.27"
+		}`),
+		rawPod: []byte(`
+		{
+			"kind": "Pod",
+			"metadata": {
+				"name": "test"
+			},
+			"spec": {
+				"securityContext": {
+					"sysctls": [
+						{
+							"name": "net.ipv4.tcp_keepalive_time"
+						}
+					]
+				},
+				"containers": [
+					{
+						"name": "nginx",
+						"image": "nginx"
+					}
+				]
+			}
+		}`),
+		allowed: false,
+	},
+	{
+		name: "baseline_sysctls_new_v1.29_policy_v1.29_allowed_positive",
+		rawRule: []byte(`
+		{
+			"level": "baseline",
+			"version": "v1.29"
+		}`),
+		rawPod: []byte(`
+		{
+			"kind": "Pod",
+			"metadata": {
+				"name": "test"
+			},
+			"spec": {
+				"securityContext": {
+					"sysctls": [
+						{
+							"name": "net.ipv4.tcp_keepalive_time"
+						}
+					]
+				},
+				"containers": [
+					{
+						"name": "nginx",
+						"image": "nginx"
+					}
+				]
+			}
+		}`),
+		allowed: true,
+	},
+	{
+		name: "baseline_sysctls_new_v1.29_policy_latest_allowed_positive",
+		rawRule: []byte(`
+		{
+			"level": "baseline",
+			"version": "latest"
+		}`),
+		rawPod: []byte(`
+		{
+			"kind": "Pod",
+			"metadata": {
+				"name": "test"
+			},
+			"spec": {
+				"securityContext": {
+					"sysctls": [
+						{
+							"name": "net.ipv4.tcp_keepalive_time"
+						}
+					]
+				},
+				"containers": [
+					{
+						"name": "nginx",
+						"image": "nginx"
+					}
+				]
+			}
+		}`),
+		allowed: true,
+	},
 	{
 		name: "baseline_sysctls_multiple_sysctls_pass_v1.24",
 		rawRule: []byte(`