chore: validate test files are up to date (#8342)

* chore: validate test files are up to date

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* makefile

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* codegen

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

---------

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>
Co-authored-by: Mariam Fahmy <mariam.fahmy@nirmata.com>

.github/workflows/conformance.yaml

@@ -723,3 +723,28 @@ jobs:
           else
             echo "Helm secret size ($SIZE bytes) is below the max allowed ($MAX_ALLOWED bytes)"
           fi
+
+  check-tests:
+    runs-on: ubuntu-latest
+    permissions:
+      packages: read
+    needs: prepare-cli
+    steps:
+      - name: Checkout
+        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
+      - name: Download kyverno CLI archive
+        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
+        with:
+          name: kubectl-kyverno
+      - name: Install Kyverno CLI
+        shell: bash
+        run: |
+          set -e
+          chmod +x kubectl-kyverno && mv kubectl-kyverno ./cmd/cli/kubectl-kyverno/kyverno
+          echo "$PWD/cmd/cli/kubectl-kyverno" >> $GITHUB_PATH
+      - name: Fix test files
+        shell: bash
+        run: |
+          set -e
+          KYVERNO_EXPERIMENTAL=true kyverno fix test ./test/cli --save
+          make verify-cli-tests

Makefile

@@ -518,6 +518,11 @@ codegen-cli-docs: $(CLI_BIN) ## Generate CLI docs
 	@rm -rf docs/user/cli && mkdir -p docs/user/cli
 	@KYVERNO_EXPERIMENTAL=true $(CLI_BIN) docs -o docs/user/cli --autogenTag=false
 
+.PHONY: codegen-cli-tests
+codegen-cli-tests: $(CLI_BIN) ## Fix CLI test files
+	@echo Fix CLI test files... >&2
+	@KYVERNO_EXPERIMENTAL=true $(CLI_BIN) fix test ./test/cli --save
+
 .PHONY: codegen-docs-all
 codegen-docs-all: codegen-helm-docs codegen-cli-docs codegen-api-docs  ## Generate all docs
 
@@ -649,6 +654,13 @@ verify-manifests: codegen-manifest-all ## Check manifests are up to date
 	@echo 'To correct this, locally run "make codegen-manifest-all", commit the changes, and re-run tests.' >&2
 	@git diff --quiet --exit-code ${INSTALL_MANIFEST_PATH}
 
+.PHONY: verify-cli-tests
+verify-cli-tests: ## Check CLI test files are up to date
+	@echo Checking CLI test files are up to date... >&2
+	@git --no-pager diff test/cli
+	@echo 'If this test fails, it is because the git diff is non-empty after running "make codegen-cli-tests".' >&2
+	@echo 'To correct this, locally run "make codegen-cli-tests", commit the changes, and re-run tests.' >&2
+	@git diff --quiet --exit-code test/cli
 
 .PHONY: verify-codegen
 verify-codegen: verify-crds verify-client verify-deepcopy verify-docs verify-helm verify-manifests ## Verify all generated code and docs are up to date

test/cli/test/validating-admission-policies/disallow-host-path/kyverno-test.yaml

@@ -1,16 +1,18 @@
 name: disallow-host-path-test
 policies:
-  - disallow-host-path.yaml
+- disallow-host-path.yaml
 resources:
-  - deployments.yaml
+- deployments.yaml
 results:
-  - policy: disallow-host-path
+- isValidatingAdmissionPolicy: true
-    resource: deployment-pass
+  kind: Deployment
-    isValidatingAdmissionPolicy: true
+  policy: disallow-host-path
-    kind: Deployment
+  resources:
-    result: pass
+  - deployment-pass
-  - policy: disallow-host-path
+  result: pass
-    resource: deployment-fail
+- isValidatingAdmissionPolicy: true
-    isValidatingAdmissionPolicy: true
+  kind: Deployment
-    kind: Deployment
+  policy: disallow-host-path
-    result: fail
+  resources:
+  - deployment-fail
+  result: fail