feat: add support for custom sigstore using TUF (#8385)

* feat; add support for custom sigstore using TUF Signed-off-by: Vishal Choudhary <sendtovishalchoudhary@gmail.com> * feat: add kuttl test Signed-off-by: Vishal Choudhary <sendtovishalchoudhary@gmail.com> * feat: add commit hash Signed-off-by: Vishal Choudhary <sendtovishalchoudhary@gmail.com> * feat: add kyverno.yaml Signed-off-by: Vishal Choudhary <sendtovishalchoudhary@gmail.com> * feat: update kyverno deployment Signed-off-by: Vishal Choudhary <sendtovishalchoudhary@gmail.com> * feat: update ordering Signed-off-by: Vishal Choudhary <sendtovishalchoudhary@gmail.com> * feat: update deployment Signed-off-by: Vishal Choudhary <sendtovishalchoudhary@gmail.com> * feat: update create image step Signed-off-by: Vishal Choudhary <sendtovishalchoudhary@gmail.com> * feat: remove wait step Signed-off-by: Vishal Choudhary <sendtovishalchoudhary@gmail.com> * feat: install crane Signed-off-by: Vishal Choudhary <sendtovishalchoudhary@gmail.com> * feat: set sha on install crane Signed-off-by: Vishal Choudhary <sendtovishalchoudhary@gmail.com> * feat: add cosign installer Signed-off-by: Vishal Choudhary <sendtovishalchoudhary@gmail.com> * feat: update custom deployment Signed-off-by: Vishal Choudhary <sendtovishalchoudhary@gmail.com> * feat: helm chart linting Signed-off-by: Vishal Choudhary <sendtovishalchoudhary@gmail.com> * feat: update Chart.yaml Signed-off-by: Vishal Choudhary <sendtovishalchoudhary@gmail.com> * fix: helm values liniting error Signed-off-by: Vishal Choudhary <sendtovishalchoudhary@gmail.com> * feat: remove step Signed-off-by: Vishal Choudhary <sendtovishalchoudhary@gmail.com> * feat: kind-deploy-kyverno Signed-off-by: Vishal Choudhary <sendtovishalchoudhary@gmail.com> * feat: create configmap in kyverno namespace Signed-off-by: Vishal Choudhary <sendtovishalchoudhary@gmail.com> * feat: update policy Signed-off-by: Vishal Choudhary <sendtovishalchoudhary@gmail.com> * feat: create kyverno ns Signed-off-by: Vishal Choudhary <sendtovishalchoudhary@gmail.com> * feat: use envfrom Signed-off-by: Vishal Choudhary <sendtovishalchoudhary@gmail.com> * fix: indentation Signed-off-by: Vishal Choudhary <sendtovishalchoudhary@gmail.com> * feat: update tuf root Signed-off-by: Vishal Choudhary <sendtovishalchoudhary@gmail.com> * feat: add sigstore volume Signed-off-by: Vishal Choudhary <sendtovishalchoudhary@gmail.com> * feat: nit Signed-off-by: Vishal Choudhary <sendtovishalchoudhary@gmail.com> * feat: remove tuf root Signed-off-by: Vishal Choudhary <sendtovishalchoudhary@gmail.com> * feat: use default tuf instead :( Signed-off-by: Vishal Choudhary <sendtovishalchoudhary@gmail.com> * feat: update Create kind cluster Signed-off-by: Vishal Choudhary <sendtovishalchoudhary@gmail.com> * feat: remove root Signed-off-by: Vishal Choudhary <sendtovishalchoudhary@gmail.com> * feat: update impl Signed-off-by: Vishal Choudhary <sendtovishalchoudhary@gmail.com> * feat: nit Signed-off-by: Vishal Choudhary <sendtovishalchoudhary@gmail.com> * feat: use custom test Signed-off-by: Vishal Choudhary <sendtovishalchoudhary@gmail.com> * feat: remove force Signed-off-by: Vishal Choudhary <sendtovishalchoudhary@gmail.com> * feat: cosign initialize Signed-off-by: Vishal Choudhary <sendtovishalchoudhary@gmail.com> * feat: add yes flag Signed-off-by: Vishal Choudhary <sendtovishalchoudhary@gmail.com> * update manifest Signed-off-by: Vishal Choudhary <sendtovishalchoudhary@gmail.com> * feat: move tuf to features Signed-off-by: Vishal Choudhary <sendtovishalchoudhary@gmail.com> * feat: update comments Signed-off-by: Vishal Choudhary <sendtovishalchoudhary@gmail.com> * chore: helmchart generate Signed-off-by: Vishal Choudhary <sendtovishalchoudhary@gmail.com> * feat: trailing white space Signed-off-by: Vishal Choudhary <sendtovishalchoudhary@gmail.com> * feat: remove old fields Signed-off-by: Vishal Choudhary <sendtovishalchoudhary@gmail.com> * feat: decouple env config map from tuf Signed-off-by: Vishal Choudhary <sendtovishalchoudhary@gmail.com> * change the way we pass flags Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com> * fix: re add envConfigMap Signed-off-by: Vishal Choudhary <sendtovishalchoudhary@gmail.com> * fix env vars Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com> * remove envConfigMap Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com> * fix Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com> --------- Signed-off-by: Vishal Choudhary <sendtovishalchoudhary@gmail.com> Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com> Co-authored-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>
2024-12-14 11:57:48 +00:00 · 2023-09-18 18:46:08 +05:30 · 2023-09-18 18:46:08 +05:30 · e2469415b7
commit e2469415b7
parent 382754c055
16 changed files with 255 additions and 0 deletions
--- a/.github/workflows/conformance.yaml
+++ b/.github/workflows/conformance.yaml
@ -499,6 +499,89 @@ jobs:
        if: failure()
        uses: ./.github/actions/kyverno-logs

+  # runs conformance test suites with configuration:
+  custom-sigstore:
+    runs-on: ubuntu-latest
+    permissions:
+      packages: read
+    strategy:
+      fail-fast: false
+      matrix:
+        config:
+          - name: custom-sigstore
+            values:
+              - standard
+              - custom-sigstore
+        k8s-version:
+          - name: v1.25
+            version: v1.25.11
+          - name: v1.26
+            version: v1.26.6
+          - name: v1.27
+            version: v1.27.3
+          - name: v1.28
+            version: v1.28.0
+        tests:
+          - custom-sigstore
+    needs: prepare-images
+    name: ${{ matrix.k8s-version.name }} - ${{ matrix.config.name }} - ${{ matrix.tests }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
+      - name: Setup build env
+        uses: ./.github/actions/setup-build-env
+        timeout-minutes: 10
+        with:
+          build-cache-key: run-conformance
+      - name: Create kind cluster and setup Sigstore Scaffolding
+        uses: sigstore/scaffolding/actions/setup@9fb4937ae18ed8456d725e99cb2871d309673022
+      - name: Create TUF values config map
+        run: |
+          kubectl create namespace kyverno
+          kubectl -n kyverno create configmap tufvalues --from-literal=TUF_MIRROR=$TUF_MIRROR --from-literal=FULCIO_URL=$FULCIO_URL --from-literal=REKOR_URL=$REKOR_URL --from-literal=CTLOG_URL=$CTLOG_URL --from-literal=ISSUER_URL=$ISSUER_URL
+          kubectl -n tuf-system get secrets tuf-root -oyaml | sed 's/namespace: .*/namespace: kyverno/' | kubectl create -f -
+      - name: Download kyverno images archive
+        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
+        with:
+          name: kyverno.tar
+      - name: Load kyverno images archive in kind cluster
+        shell: bash
+        run: |
+          set -e
+          make kind-load-image-archive
+      - name: Install kyverno
+        shell: bash
+        run: |
+          set -e
+          export USE_CONFIG=${{ join(matrix.config.values, ',') }}
+          make kind-deploy-kyverno
+      - name: Install crane
+        uses: imjasonh/setup-crane@00c9e93efa4e1138c9a7a5c594acd6c75a2fbf0c
+      - name: Install Cosign
+        uses: sigstore/cosign-installer@11086d25041f77fe8fe7b9ea4e48e3b9192b8f19
+      - name: Create test image
+        shell: bash
+        run: |
+          DIGEST=$(crane digest cgr.dev/chainguard/static)   
+          IMAGE_NAME=$(uuidgen | tr "[:upper:]" "[:lower:]")
+          TEST_IMAGE_URL=ttl.sh/${IMAGE_NAME}:1h
+          crane copy cgr.dev/chainguard/static@$DIGEST $TEST_IMAGE_URL
+          cosign initialize --mirror $TUF_MIRROR --root $TUF_MIRROR/root.json
+          COSIGN_EXPERIMENTAL=1 cosign sign --rekor-url $REKOR_URL --fulcio-url $FULCIO_URL $TEST_IMAGE_URL --identity-token `curl -s $ISSUER_URL` -y
+          echo "TEST_IMAGE_URL=$TEST_IMAGE_URL" >> $GITHUB_ENV
+      - name: Wait for kyverno ready
+        uses: ./.github/actions/kyverno-wait-ready
+      - name: Test with kuttl
+        shell: bash
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          set -e
+          ./.tools/kubectl-kuttl test ./test/conformance/kuttl/${{ matrix.tests }} --config ./test/conformance/kuttl/_config/common.yaml
+      - name: Debug failure
+        if: failure()
+        uses: ./.github/actions/kyverno-logs
+
  # runs conformance test suites with configuration:
  default:
    runs-on: ubuntu-latest
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@ -4,6 +4,7 @@

 ### Note

+- Added `--tufRoot` and `--tufMirror` flags to configure tuf for custom sigstore deployments.
 - Remove description from deprecated fields in CRDs
 - Remove CLI `kyverno test manifest ...` commands (replaced by `kyverno create ...`).
 - Added `--caSecretName` and `--tlsSecretName` flags to control names of certificate related secrets.
--- a/charts/kyverno/Chart.yaml
+++ b/charts/kyverno/Chart.yaml
@ -68,3 +68,5 @@ annotations:
      description: match conditions support in webhooks
    - kind: fixed
      description: missing image pull policy missing in a couple of deployments
+    - kind: added
+      description: added TUF flags for custom sigstore deployments
--- a/charts/kyverno/README.md
+++ b/charts/kyverno/README.md
@ -315,6 +315,8 @@ The chart values are organised per component.
 | features.registryClient.credentialHelpers | list | `["default","google","amazon","azure","github"]` | Enable registry client helpers |
 | features.reports.chunkSize | int | `1000` | Reports chunk size |
 | features.ttlController.reconciliationInterval | string | `"1m"` | Reconciliation interval for the label based cleanup manager |
+| features.tuf.root | string | `nil` | Tuf root |
+| features.tuf.mirror | string | `nil` | Tuf mirror |

 ### Admission controller

--- a/charts/kyverno/templates/_helpers.tpl
+++ b/charts/kyverno/templates/_helpers.tpl
@ -74,6 +74,14 @@
 {{- with .ttlController -}}
  {{- $flags = append $flags (print "--ttlReconciliationInterval=" .reconciliationInterval) -}}
 {{- end -}}
+{{- with .tuf -}}
+  {{- with .mirror -}}
+    {{- $flags = append $flags (print "--tufMirror=" .) -}}
+  {{- end -}}
+  {{- with .root -}}
+    {{- $flags = append $flags (print "--tufRoot=" .) -}}
+  {{- end -}}
+{{- end -}}
 {{- with $flags -}}
  {{- toYaml . -}}
 {{- end -}}
--- a/charts/kyverno/templates/admission-controller/deployment.yaml
+++ b/charts/kyverno/templates/admission-controller/deployment.yaml
@ -166,6 +166,7 @@ spec:
              "policyExceptions"
              "protectManagedResources"
              "registryClient"
+              "tuf"
            ) | nindent 12 }}
            {{- range $key, $value := .Values.admissionController.container.extraArgs }}
            {{- if $value }}
--- a/charts/kyverno/templates/reports-controller/deployment.yaml
+++ b/charts/kyverno/templates/reports-controller/deployment.yaml
@ -121,6 +121,7 @@ spec:
              "policyExceptions"
              "reports"
              "registryClient"
+              "tuf"
            ) | nindent 12 }}
            {{- range $key, $value := .Values.reportsController.extraArgs }}
            {{- if $value }}
--- a/charts/kyverno/values.yaml
+++ b/charts/kyverno/values.yaml
@ -447,6 +447,11 @@ features:
  ttlController:
    # -- Reconciliation interval for the label based cleanup manager
    reconciliationInterval: 1m
+  tuf:
+    # -- Tuf root
+    root:
+    # -- Tuf mirror
+    mirror:

 # Cleanup cronjobs to prevent internal resources from stacking up in the cluster
 cleanupJobs:
--- a/cmd/internal/flag.go
+++ b/cmd/internal/flag.go
@ -8,6 +8,7 @@ import (
 	"github.com/kyverno/kyverno/pkg/leaderelection"
 	"github.com/kyverno/kyverno/pkg/logging"
 	"github.com/kyverno/kyverno/pkg/toggle"
+	"github.com/sigstore/sigstore/pkg/tuf"
 )

 var (
@ -38,6 +39,8 @@ var (
 	enableConfigMapCaching bool
 	// cosign
 	imageSignatureRepository string
+	tufMirror                string
+	tufRoot                  string
 	// registry client
 	imagePullSecrets          string
 	allowInsecureRegistry     bool
@ -98,6 +101,8 @@ func initDeferredLoadingFlags() {

 func initCosignFlags() {
 	flag.StringVar(&imageSignatureRepository, "imageSignatureRepository", "", "(DEPRECATED, will be removed in 1.12) Alternate repository for image signatures. Can be overridden per rule via `verifyImages.Repository`.")
+	flag.StringVar(&tufMirror, "tufMirror", tuf.DefaultRemoteRoot, "Alternate TUF mirror for sigstore. If left blank, public sigstore one is used for cosign verification..")
+	flag.StringVar(&tufRoot, "tufRoot", "", "Alternate TUF root.json for sigstore. If left blank, public sigstore one is used for cosign verification.")
 }

 func initRegistryClientFlags() {
--- a/cmd/internal/setup.go
+++ b/cmd/internal/setup.go
@ -72,6 +72,9 @@ func Setup(config Configuration, name string, skipResourceFilters bool) (context
 	if config.UsesImageVerifyCache() {
 		imageVerifyCache = setupImageVerifyCache(ctx, logger)
 	}
+	if config.UsesCosign() {
+		setupSigstoreTUF(ctx, logger)
+	}
 	var leaderElectionClient kubeclient.UpstreamInterface
 	if config.UsesLeaderElection() {
 		leaderElectionClient = createKubernetesClient(logger, kubeclient.WithMetrics(metricsManager, metrics.KubeClient), kubeclient.WithTracing())
--- a/cmd/internal/tuf.go
+++ b/cmd/internal/tuf.go
@ -0,0 +1,27 @@
+package internal
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/go-logr/logr"
+	"github.com/sigstore/cosign/v2/pkg/blob"
+	"github.com/sigstore/sigstore/pkg/tuf"
+)
+
+func setupSigstoreTUF(ctx context.Context, logger logr.Logger) {
+	logger = logger.WithName("sigstore-tuf").WithValues("tufroot", tufRoot, "tufmirror", tufMirror)
+	logger.Info("setup tuf client for sigstore...")
+	var tufRootBytes []byte
+	var err error
+	if tufRoot != "" {
+		tufRootBytes, err = blob.LoadFileOrURL(tufRoot)
+		if err != nil {
+			checkError(logger, err, fmt.Sprintf("Failed to read alternate TUF root file %s : %v", tufRoot, err))
+		}
+	}
+	logger.Info("Initializing TUF root")
+	if err := tuf.Initialize(ctx, tufMirror, tufRootBytes); err != nil {
+		checkError(logger, err, fmt.Sprintf("Failed to initialize TUF client from %s : %v", tufRoot, err))
+	}
+}
--- a/scripts/config/custom-sigstore/kyverno.yaml
+++ b/scripts/config/custom-sigstore/kyverno.yaml
@ -0,0 +1,61 @@
+features:
+  tuf:
+    root: "$(TUF_MIRROR)/root.json"
+    mirror: "$(TUF_MIRROR)"
+
+admissionController:
+  container:
+    extraEnvVars:
+      - name: TUF_MIRROR
+        valueFrom:
+          configMapKeyRef:
+            name: tufvalues
+            key: TUF_MIRROR
+      - name: FULCIO_URL
+        valueFrom:
+          configMapKeyRef:
+            name: tufvalues
+            key: FULCIO_URL
+      - name: REKOR_URL
+        valueFrom:
+          configMapKeyRef:
+            name: tufvalues
+            key: REKOR_URL
+      - name: CTLOG_URL
+        valueFrom:
+          configMapKeyRef:
+            name: tufvalues
+            key: CTLOG_URL
+      - name: ISSUER_URL
+        valueFrom:
+          configMapKeyRef:
+            name: tufvalues
+            key: ISSUER_URL
+
+reportsController:
+  extraEnvVars:
+    - name: TUF_MIRROR
+      valueFrom:
+        configMapKeyRef:
+          name: tufvalues
+          key: TUF_MIRROR
+    - name: FULCIO_URL
+      valueFrom:
+        configMapKeyRef:
+          name: tufvalues
+          key: FULCIO_URL
+    - name: REKOR_URL
+      valueFrom:
+        configMapKeyRef:
+          name: tufvalues
+          key: REKOR_URL
+    - name: CTLOG_URL
+      valueFrom:
+        configMapKeyRef:
+          name: tufvalues
+          key: CTLOG_URL
+    - name: ISSUER_URL
+      valueFrom:
+        configMapKeyRef:
+          name: tufvalues
+          key: ISSUER_URL
--- a/test/conformance/kuttl/custom-sigstore/standard/basic/01-manifest.yaml
+++ b/test/conformance/kuttl/custom-sigstore/standard/basic/01-manifest.yaml
@ -0,0 +1,38 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: test-custom-sigstore
+---
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: basic-sigstore-test-policy
+spec:
+  validationFailureAction: Enforce
+  background: false
+  webhookTimeoutSeconds: 30
+  failurePolicy: Fail
+  rules:
+  - name: keyed-basic-rule
+    match:
+      any:
+      - resources:
+          kinds:
+          - Pod
+    context:
+      - name: tufvalues
+        configMap:
+          name: tufvalues
+          namespace: kyverno
+    verifyImages:
+    - imageReferences:
+      - "ttl.sh/*"
+      attestors:
+      - count: 1
+        entries:
+        - keyless:
+            issuer: "https://kubernetes.default.svc.cluster.local"
+            subject: "*"
+            rekor:
+              url: "{{ tufvalues.data.REKOR_URL }}"
+      required: true
--- a/test/conformance/kuttl/custom-sigstore/standard/basic/02-assert.yaml
+++ b/test/conformance/kuttl/custom-sigstore/standard/basic/02-assert.yaml
@ -0,0 +1,9 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: basic-sigstore-test-policy
+status:
+  conditions:
+  - reason: Succeeded
+    status: "True"
+    type: Ready
--- a/test/conformance/kuttl/custom-sigstore/standard/basic/03-goodpod.yaml
+++ b/test/conformance/kuttl/custom-sigstore/standard/basic/03-goodpod.yaml
@ -0,0 +1,4 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - command: kubectl -n test-custom-sigstore run test-sigstore --image=$TEST_IMAGE_URL
--- a/test/conformance/kuttl/custom-sigstore/standard/basic/04-assert.yaml
+++ b/test/conformance/kuttl/custom-sigstore/standard/basic/04-assert.yaml
@ -0,0 +1,5 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: test-sigstore
+  namespace: test-custom-sigstore