chore: split kuttl tests (#6423)

* fix: makefile build/kind targets and add target to save built images to files

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* try in ci

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* prepare for kuttl

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* load

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* install

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* makefile

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* makefile

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* chore: split kuttl tests

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* config

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* more test suites

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

* config

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>

---------

Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>
Signed-off-by: Charles-Edouard Brétéché <charled.breteche@gmail.com>

.github/workflows/conformance.yaml

@@ -34,7 +34,23 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        k8s-version: [v1.24.7, v1.25.3, v1.26.0]
+        k8s-version:
+          - v1.24.7
+          - v1.25.3
+          - v1.26.0
+        tests:
+          - autogen
+          - cleanup
+          - events
+          - exceptions
+          - generate
+          - mutate
+          - rangeoperators
+          - rbac
+          - reports
+          - validate
+          - verifyImages
+          - webhooks
     runs-on: ubuntu-latest
     needs: prepare-images
     steps:
@@ -59,7 +75,9 @@ jobs:
       - name: Wait for kyverno ready
         uses: ./.github/actions/kyverno-wait-ready
       - name: Test with kuttl
-        run: make test-kuttl
+        run: |
+          ./.tools/kubectl-kuttl test ./test/conformance/kuttl/${{ matrix.tests }} \
+            --config ./test/conformance/kuttl/_config/common.yaml
       - name: Debug failure
         if: failure()
         uses: ./.github/actions/kyverno-logs

test/conformance/kuttl/_config/common.yaml

@@ -0,0 +1,7 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestSuite
+startKIND: false
+timeout: 90
+parallel: 1
+fullName: true
+skipTestRegex: '_.+'