mirrors/kyverno
mirrors/kyverno
1
0
Fork 0
mirror of https://github.com/kyverno/kyverno.git synced 2025-03-28 10:28:36 +00:00
Code Activity

added verify image ristretto cache implementation (#7969)

Browse source 
* updated flags

Signed-off-by: hackeramitkumar <amit9116260192@gmail.com>

* added ristretto_cache impl

Signed-off-by: hackeramitkumar <amit9116260192@gmail.com>

* added bufferSize

Signed-off-by: hackeramitkumar <amit9116260192@gmail.com>

* small nits

Signed-off-by: hackeramitkumar <amit9116260192@gmail.com>

* made cache as private member

Signed-off-by: hackeramitkumar <amit9116260192@gmail.com>

* made cache as private member

Signed-off-by: hackeramitkumar <amit9116260192@gmail.com>

* added logger.withValues

Signed-off-by: hackeramitkumar <amit9116260192@gmail.com>

* added verify image cache

Signed-off-by: hackeramitkumar <amit9116260192@gmail.com>

* small nits

Signed-off-by: hackeramitkumar <amit9116260192@gmail.com>

* added cache tests

Signed-off-by: hackeramitkumar <amit9116260192@gmail.com>

* fixed lint issue

Signed-off-by: hackeramitkumar <amit9116260192@gmail.com>

* added chaged policy test

Signed-off-by: hackeramitkumar <amit9116260192@gmail.com>

* cache time should be entered in minutes

Signed-off-by: hackeramitkumar <amit9116260192@gmail.com>

* removed cache.wait()

Signed-off-by: hackeramitkumar <amit9116260192@gmail.com>

* small nits

Signed-off-by: hackeramitkumar <amit9116260192@gmail.com>

* removed client.go logs and added in imageVerifier

Signed-off-by: hackeramitkumar <amit9116260192@gmail.com>

* added level to the logs

Signed-off-by: hackeramitkumar <amit9116260192@gmail.com>

* added notary image cache verification

Signed-off-by: hackeramitkumar <amit9116260192@gmail.com>

* replace intVar by flag.DurationVar()

Signed-off-by: hackeramitkumar <amit9116260192@gmail.com>

* removed lock from cache clinet

Signed-off-by: hackeramitkumar <amit9116260192@gmail.com>

* updated cosign tests

Signed-off-by: hackeramitkumar <amit9116260192@gmail.com>

* added execution latencies comparision

Signed-off-by: hackeramitkumar <amit9116260192@gmail.com>

* added assert.Error()

Signed-off-by: hackeramitkumar <amit9116260192@gmail.com>

* added error assertion util

Signed-off-by: hackeramitkumar <amit9116260192@gmail.com>

* added error log

Signed-off-by: hackeramitkumar <amit9116260192@gmail.com>

* Update pkg/engine/internal/imageverifier.go

Signed-off-by: shuting <shutting06@gmail.com>

* lint fixes

Signed-off-by: hackeramitkumar <amit9116260192@gmail.com>

* removed logs from unit tests

Signed-off-by: hackeramitkumar <amit9116260192@gmail.com>

* added ristretto_cache impl

Signed-off-by: hackeramitkumar <amit9116260192@gmail.com>

* removed cache.wait()

Signed-off-by: hackeramitkumar <amit9116260192@gmail.com>

* small nits

Signed-off-by: hackeramitkumar <amit9116260192@gmail.com>

* added asssertions in tests

Signed-off-by: hackeramitkumar <amit9116260192@gmail.com>

* fixed conflicts

Signed-off-by: hackeramitkumar <amit9116260192@gmail.com>

* lint fix

Signed-off-by: hackeramitkumar <amit9116260192@gmail.com>

* renamed variabls

Signed-off-by: hackeramitkumar <amit9116260192@gmail.com>

---------

Signed-off-by: hackeramitkumar <amit9116260192@gmail.com>
Signed-off-by: shuting <shutting06@gmail.com>
Co-authored-by: shuting <shutting06@gmail.com>
Co-authored-by: shuting <shuting@nirmata.com>
This commit is contained in:
Amit kumar 2023-08-30 12:56:40 +05:30 • committed by GitHub
parent 62634af6aa
commit 6d8ae16afa
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 454 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
cmd/internal/flag.go
View file

@ -46,7 +46,7 @@ var (
leaderElectionRetryPeriod time.Duration
// image verify cache
imageVerifyCacheEnabled bool
imageVerifyCacheTTLDuration int64
imageVerifyCacheTTLDuration time.Duration
imageVerifyCacheMaxSize int64
)
@ -108,8 +108,8 @@ func initRegistryClientFlags() {
func initImageVerifyCacheFlags() {
flag.BoolVar(&imageVerifyCacheEnabled, "imageVerifyCacheEnabled", true, "Whether to use a TTL cache for storing verified images.")
flag.Int64Var(&imageVerifyCacheMaxSize, "imageVerifyCacheMaxSize", 0, "Max size limit for the TTL cache, 0 means no size limit.")
flag.Int64Var(&imageVerifyCacheTTLDuration, "imageVerifyCacheTTLDuration", 0, "Max TTL value for a cache, 0 means no TTL.")
flag.Int64Var(&imageVerifyCacheMaxSize, "imageVerifyCacheMaxSize", 1000, "Max size limit for the TTL cache, 0 means default 1000 size limit.")
flag.DurationVar(&imageVerifyCacheTTLDuration, "imageVerifyCacheTTLDuration", 60*time.Minute, "Max TTL value for a cache, 0 means default 1 hour TTL.")
}
func initLeaderElectionFlags() {

3
cmd/internal/imageverifycache.go
View file

@ -2,7 +2,6 @@ package internal
import (
"context"
"time"
"github.com/go-logr/logr"
"github.com/kyverno/kyverno/pkg/imageverifycache"
@ -15,7 +14,7 @@ func setupImageVerifyCache(ctx context.Context, logger logr.Logger) imageverifyc
imageverifycache.WithLogger(logger),
imageverifycache.WithCacheEnableFlag(imageVerifyCacheEnabled),
imageverifycache.WithMaxSize(imageVerifyCacheMaxSize),
imageverifycache.WithTTLDuration(time.Duration(imageVerifyCacheTTLDuration)),
imageverifycache.WithTTLDuration(imageVerifyCacheTTLDuration),
}
imageVerifyCache, err := imageverifycache.New(opts...)
checkError(logger, err, "failed to create image verify cache client")

1
go.mod
View file

@ -13,6 +13,7 @@ require (
github.com/cenkalti/backoff v2.2.1+incompatible
github.com/chrismellard/docker-credential-acr-env v0.0.0-20230304212654-82a0ddb27589
github.com/cyphar/filepath-securejoin v0.2.3
github.com/dgraph-io/ristretto v0.1.1
github.com/distribution/distribution v2.8.2+incompatible
github.com/evanphx/json-patch/v5 v5.6.0
github.com/fatih/color v1.15.0

2
go.sum
View file

@ -386,6 +386,7 @@ github.com/denis-tingajkin/go-header v0.4.2/go.mod h1:eLRHAVXzE5atsKAnNRDB90WHCF
github.com/depcheck-test/depcheck-test v0.0.0-20220607135614-199033aaa936 h1:foGzavPWwtoyBvjWyKJYDYsyzy+23iBV7NKTwdk+LRY=
github.com/dgraph-io/badger/v3 v3.2103.5 h1:ylPa6qzbjYRQMU6jokoj4wzcaweHylt//CH0AKt0akg=
github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8=
github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA=
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y=
@ -1899,6 +1900,7 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

1
pkg/engine/handlers/mutation/mutate_image.go
View file

@ -54,6 +54,7 @@ func NewMutateImageHandler(
configuration: configuration,
rclientFactory: rclientFactory,
ivm: ivm,
ivCache: ivCache,
images: ruleImages,
imageSignatureRepository: imageSignatureRepository,
}, nil

381
pkg/engine/image_verify_test.go
View file

@ -6,6 +6,7 @@ import (
"fmt"
"strings"
"testing"
"time"
"github.com/go-logr/logr"
"github.com/kyverno/kyverno/api/kyverno"
@ -940,3 +941,383 @@ func Test_ParsePEMDelimited(t *testing.T) {
assert.Equal(t, len(verifiedImages.Data), 1)
assert.Equal(t, verifiedImages.IsVerified(image), true)
}
func testImageVerifyCache(
ivCache imageverifycache.Client,
ctx context.Context,
rclient registryclient.Client,
cmResolver engineapi.ConfigmapResolver,
pContext engineapi.PolicyContext,
cfg config.Configuration,
) (engineapi.EngineResponse, engineapi.ImageVerificationMetadata) {
e := NewEngine(
cfg,
metricsCfg,
jp,
nil,
factories.DefaultRegistryClientFactory(adapters.RegistryClient(rclient), nil),
ivCache,
factories.DefaultContextLoaderFactory(cmResolver),
nil,
"",
)
return e.VerifyAndPatchImages(
ctx,
pContext,
)
}
func errorAssertionUtil(t *testing.T, image string, ivm engineapi.ImageVerificationMetadata, er engineapi.EngineResponse) {
assert.Equal(t, len(er.PolicyResponse.Rules), 1)
assert.Equal(t, er.PolicyResponse.Rules[0].Status(), engineapi.RuleStatusPass)
assert.Equal(t, ivm.IsEmpty(), false)
assert.Equal(t, ivm.IsVerified(image), true)
}
var testUpdatedPolicyGood = `{
"apiVersion": "kyverno.io/v1",
"kind": "ClusterPolicy",
"metadata": {
"name": "attest"
},
"spec": {
"rules": [
{
"name": "attest-testing",
"match": {
"resources": {
"kinds": [
"Pod"
]
}
},
"verifyImages": [
{
"image": "*",
"key": "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEHMmDjK65krAyDaGaeyWNzgvIu155JI50B2vezCw8+3CVeE0lJTL5dbL3OP98Za0oAEBJcOxky8Riy/XcmfKZbw==\n-----END PUBLIC KEY-----",
"attestations": [
{
"predicateType": "https://example.com/CodeReview/v1",
"attestors": [
{
"entries": [
{
"keys": {
"publicKeys": "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEHMmDjK65krAyDaGaeyWNzgvIu155JI50B2vezCw8+3CVeE0lJTL5dbL3OP98Za0oAEBJcOxky8Riy/XcmfKZbw==\n-----END PUBLIC KEY-----",
"rekor": {
"url": "https://rekor.sigstore.dev",
"ignoreSCT": true,
"ignoreTlog": true
}
}
}
]
}
],
"conditions": [
{
"all": [
{
"key": "{{ repo.uri }}",
"operator": "Equals",
"value": "https://github.com/example/my-project"
},
{
"key": "{{ repo.branch }}",
"operator": "Equals",
"value": "main"
}
]
}
]
}
]
}
]
}
]
}
}`
func Test_ImageVerifyCacheCosign(t *testing.T) {
opts := []imageverifycache.Option{
imageverifycache.WithCacheEnableFlag(true),
imageverifycache.WithMaxSize(1000),
imageverifycache.WithTTLDuration(24 * time.Hour),
}
imageVerifyCache, err := imageverifycache.New(opts...)
assert.NilError(t, err)
policyContext := buildContext(t, testPolicyGood, testResource, "")
image := "ghcr.io/jimbugwadia/pause2:latest"
err = cosign.SetMock(image, attestationPayloads)
assert.NilError(t, err)
start := time.Now()
er, ivm := testImageVerifyCache(imageVerifyCache, context.TODO(), registryclient.NewOrDie(), nil, policyContext, cfg)
firstOperationTime := time.Since(start)
errorAssertionUtil(t, image, ivm, er)
start = time.Now()
er, ivm = testImageVerifyCache(imageVerifyCache, context.TODO(), registryclient.NewOrDie(), nil, policyContext, cfg)
secondOperationTime := time.Since(start)
errorAssertionUtil(t, image, ivm, er)
assert.Check(t, secondOperationTime < firstOperationTime/10, "cache entry is valid, so image verification should be from cache.", firstOperationTime, secondOperationTime)
}
func Test_ImageVerifyCacheExpiredCosign(t *testing.T) {
opts := []imageverifycache.Option{
imageverifycache.WithCacheEnableFlag(true),
imageverifycache.WithMaxSize(1000),
imageverifycache.WithTTLDuration(5 * time.Second),
}
imageVerifyCache, err := imageverifycache.New(opts...)
assert.NilError(t, err)
policyContext := buildContext(t, testPolicyGood, testResource, "")
image := "ghcr.io/jimbugwadia/pause2:latest"
err = cosign.SetMock(image, attestationPayloads)
assert.NilError(t, err)
start := time.Now()
er, ivm := testImageVerifyCache(imageVerifyCache, context.TODO(), registryclient.NewOrDie(), nil, policyContext, cfg)
firstOperationTime := time.Since(start)
errorAssertionUtil(t, image, ivm, er)
time.Sleep(5 * time.Second)
start = time.Now()
er, ivm = testImageVerifyCache(imageVerifyCache, context.TODO(), registryclient.NewOrDie(), nil, policyContext, cfg)
secondOperationTime := time.Since(start)
errorAssertionUtil(t, image, ivm, er)
assert.Check(t, secondOperationTime > firstOperationTime/10 && secondOperationTime < firstOperationTime*10, "cache entry is expired, so image verification should not be from cache.")
}
func Test_changePolicyCacheVerificationCosign(t *testing.T) {
opts := []imageverifycache.Option{
imageverifycache.WithCacheEnableFlag(true),
imageverifycache.WithMaxSize(1000),
imageverifycache.WithTTLDuration(60 * time.Minute),
}
imageVerifyCache, err := imageverifycache.New(opts...)
assert.NilError(t, err)
policyContext := buildContext(t, testPolicyGood, testResource, "")
image := "ghcr.io/jimbugwadia/pause2:latest"
err = cosign.SetMock(image, attestationPayloads)
assert.NilError(t, err)
start := time.Now()
er, ivm := testImageVerifyCache(imageVerifyCache, context.TODO(), registryclient.NewOrDie(), nil, policyContext, cfg)
firstOperationTime := time.Since(start)
errorAssertionUtil(t, image, ivm, er)
policyContext = buildContext(t, testUpdatedPolicyGood, testResource, "")
start = time.Now()
er, ivm = testImageVerifyCache(imageVerifyCache, context.TODO(), registryclient.NewOrDie(), nil, policyContext, cfg)
secondOperationTime := time.Since(start)
errorAssertionUtil(t, image, ivm, er)
assert.Check(t, secondOperationTime > firstOperationTime/10 && secondOperationTime < firstOperationTime*10, "cache entry not found, so image verification should not be from cache.")
}
var verifyImageNotaryPolicy = `{
"apiVersion": "kyverno.io/v2beta1",
"kind": "ClusterPolicy",
"metadata": {
"name": "check-image-notary"
},
"spec": {
"validationFailureAction": "Enforce",
"webhookTimeoutSeconds": 30,
"failurePolicy": "Fail",
"rules": [
{
"name": "verify-signature-notary",
"match": {
"any": [
{
"resources": {
"kinds": [
"Pod"
]
}
}
]
},
"verifyImages": [
{
"type": "Notary",
"imageReferences": [
"ghcr.io/kyverno/test-verify-image*"
],
"attestors": [
{
"count": 1,
"entries": [
{
"certificates": {
"cert": "-----BEGIN CERTIFICATE-----\nMIIDTTCCAjWgAwIBAgIJAPI+zAzn4s0xMA0GCSqGSIb3DQEBCwUAMEwxCzAJBgNV\nBAYTAlVTMQswCQYDVQQIDAJXQTEQMA4GA1UEBwwHU2VhdHRsZTEPMA0GA1UECgwG\nTm90YXJ5MQ0wCwYDVQQDDAR0ZXN0MB4XDTIzMDUyMjIxMTUxOFoXDTMzMDUxOTIx\nMTUxOFowTDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMRAwDgYDVQQHDAdTZWF0\ndGxlMQ8wDQYDVQQKDAZOb3RhcnkxDTALBgNVBAMMBHRlc3QwggEiMA0GCSqGSIb3\nDQEBAQUAA4IBDwAwggEKAoIBAQDNhTwv+QMk7jEHufFfIFlBjn2NiJaYPgL4eBS+\nb+o37ve5Zn9nzRppV6kGsa161r9s2KkLXmJrojNy6vo9a6g6RtZ3F6xKiWLUmbAL\nhVTCfYw/2n7xNlVMjyyUpE+7e193PF8HfQrfDFxe2JnX5LHtGe+X9vdvo2l41R6m\nIia04DvpMdG4+da2tKPzXIuLUz/FDb6IODO3+qsqQLwEKmmUee+KX+3yw8I6G1y0\nVp0mnHfsfutlHeG8gazCDlzEsuD4QJ9BKeRf2Vrb0ywqNLkGCbcCWF2H5Q80Iq/f\nETVO9z88R7WheVdEjUB8UrY7ZMLdADM14IPhY2Y+tLaSzEVZAgMBAAGjMjAwMAkG\nA1UdEwQCMAAwDgYDVR0PAQH/BAQDAgeAMBMGA1UdJQQMMAoGCCsGAQUFBwMDMA0G\nCSqGSIb3DQEBCwUAA4IBAQBX7x4Ucre8AIUmXZ5PUK/zUBVOrZZzR1YE8w86J4X9\nkYeTtlijf9i2LTZMfGuG0dEVFN4ae3CCpBst+ilhIndnoxTyzP+sNy4RCRQ2Y/k8\nZq235KIh7uucq96PL0qsF9s2RpTKXxyOGdtp9+HO0Ty5txJE2txtLDUIVPK5WNDF\nByCEQNhtHgN6V20b8KU2oLBZ9vyB8V010dQz0NRTDLhkcvJig00535/LUylECYAJ\n5/jn6XKt6UYCQJbVNzBg/YPGc1RF4xdsGVDBben/JXpeGEmkdmXPILTKd9tZ5TC0\nuOKpF5rWAruB5PCIrquamOejpXV9aQA/K2JQDuc0mcKz\n-----END CERTIFICATE-----"
}
}
]
}
]
}
]
}
]
}
}`
var verifyImageNotaryUpdatedPolicy = `{
"apiVersion": "kyverno.io/v2beta1",
"kind": "ClusterPolicy",
"metadata": {
"name": "check-image-notary"
},
"spec": {
"validationFailureAction": "Enforce",
"webhookTimeoutSeconds": 30,
"failurePolicy": "Fail",
"rules": [
{
"name": "verify-signature-notary-1",
"match": {
"any": [
{
"resources": {
"kinds": [
"Pod"
]
}
}
]
},
"verifyImages": [
{
"type": "Notary",
"imageReferences": [
"ghcr.io/kyverno/test-verify-image*"
],
"attestors": [
{
"count": 1,
"entries": [
{
"certificates": {
"cert": "-----BEGIN CERTIFICATE-----\nMIIDTTCCAjWgAwIBAgIJAPI+zAzn4s0xMA0GCSqGSIb3DQEBCwUAMEwxCzAJBgNV\nBAYTAlVTMQswCQYDVQQIDAJXQTEQMA4GA1UEBwwHU2VhdHRsZTEPMA0GA1UECgwG\nTm90YXJ5MQ0wCwYDVQQDDAR0ZXN0MB4XDTIzMDUyMjIxMTUxOFoXDTMzMDUxOTIx\nMTUxOFowTDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMRAwDgYDVQQHDAdTZWF0\ndGxlMQ8wDQYDVQQKDAZOb3RhcnkxDTALBgNVBAMMBHRlc3QwggEiMA0GCSqGSIb3\nDQEBAQUAA4IBDwAwggEKAoIBAQDNhTwv+QMk7jEHufFfIFlBjn2NiJaYPgL4eBS+\nb+o37ve5Zn9nzRppV6kGsa161r9s2KkLXmJrojNy6vo9a6g6RtZ3F6xKiWLUmbAL\nhVTCfYw/2n7xNlVMjyyUpE+7e193PF8HfQrfDFxe2JnX5LHtGe+X9vdvo2l41R6m\nIia04DvpMdG4+da2tKPzXIuLUz/FDb6IODO3+qsqQLwEKmmUee+KX+3yw8I6G1y0\nVp0mnHfsfutlHeG8gazCDlzEsuD4QJ9BKeRf2Vrb0ywqNLkGCbcCWF2H5Q80Iq/f\nETVO9z88R7WheVdEjUB8UrY7ZMLdADM14IPhY2Y+tLaSzEVZAgMBAAGjMjAwMAkG\nA1UdEwQCMAAwDgYDVR0PAQH/BAQDAgeAMBMGA1UdJQQMMAoGCCsGAQUFBwMDMA0G\nCSqGSIb3DQEBCwUAA4IBAQBX7x4Ucre8AIUmXZ5PUK/zUBVOrZZzR1YE8w86J4X9\nkYeTtlijf9i2LTZMfGuG0dEVFN4ae3CCpBst+ilhIndnoxTyzP+sNy4RCRQ2Y/k8\nZq235KIh7uucq96PL0qsF9s2RpTKXxyOGdtp9+HO0Ty5txJE2txtLDUIVPK5WNDF\nByCEQNhtHgN6V20b8KU2oLBZ9vyB8V010dQz0NRTDLhkcvJig00535/LUylECYAJ\n5/jn6XKt6UYCQJbVNzBg/YPGc1RF4xdsGVDBben/JXpeGEmkdmXPILTKd9tZ5TC0\nuOKpF5rWAruB5PCIrquamOejpXV9aQA/K2JQDuc0mcKz\n-----END CERTIFICATE-----"
}
}
]
}
]
}
]
}
]
}
}`
var verifyImageNotaryResource = `{
"apiVersion": "v1",
"kind": "Pod",
"metadata": {
"creationTimestamp": null,
"labels": {
"run": "test"
},
"name": "test",
"namespace": "default"
},
"spec": {
"containers": [
{
"image": "ghcr.io/kyverno/test-verify-image:signed",
"name": "test",
"resources": {}
}
],
"dnsPolicy": "ClusterFirst",
"restartPolicy": "Always"
},
"status": {}
}`
func Test_ImageVerifyCacheNotary(t *testing.T) {
opts := []imageverifycache.Option{
imageverifycache.WithCacheEnableFlag(true),
imageverifycache.WithMaxSize(1000),
imageverifycache.WithTTLDuration(24 * time.Hour),
}
imageVerifyCache, err := imageverifycache.New(opts...)
assert.NilError(t, err)
image := "ghcr.io/kyverno/test-verify-image:signed"
policyContext := buildContext(t, verifyImageNotaryPolicy, verifyImageNotaryResource, "")
start := time.Now()
er, ivm := testImageVerifyCache(imageVerifyCache, context.TODO(), registryclient.NewOrDie(), nil, policyContext, cfg)
firstOperationTime := time.Since(start)
errorAssertionUtil(t, image, ivm, er)
start = time.Now()
er, ivm = testImageVerifyCache(imageVerifyCache, context.TODO(), registryclient.NewOrDie(), nil, policyContext, cfg)
secondOperationTime := time.Since(start)
errorAssertionUtil(t, image, ivm, er)
assert.Check(t, secondOperationTime < firstOperationTime/10, "cache entry is valid, so image verification should be from cache.", firstOperationTime, secondOperationTime)
}
func Test_ImageVerifyCacheExpiredNotary(t *testing.T) {
opts := []imageverifycache.Option{
imageverifycache.WithCacheEnableFlag(true),
imageverifycache.WithMaxSize(1000),
imageverifycache.WithTTLDuration(5 * time.Second),
}
imageVerifyCache, err := imageverifycache.New(opts...)
assert.NilError(t, err)
image := "ghcr.io/kyverno/test-verify-image:signed"
policyContext := buildContext(t, verifyImageNotaryPolicy, verifyImageNotaryResource, "")
assert.NilError(t, err)
start := time.Now()
er, ivm := testImageVerifyCache(imageVerifyCache, context.TODO(), registryclient.NewOrDie(), nil, policyContext, cfg)
firstOperationTime := time.Since(start)
errorAssertionUtil(t, image, ivm, er)
time.Sleep(5 * time.Second)
start = time.Now()
er, ivm = testImageVerifyCache(imageVerifyCache, context.TODO(), registryclient.NewOrDie(), nil, policyContext, cfg)
secondOperationTime := time.Since(start)
errorAssertionUtil(t, image, ivm, er)
assert.Check(t, secondOperationTime > firstOperationTime/10 && secondOperationTime < firstOperationTime*10, "cache entry is expired, so image verification should not be from cache.")
}
func Test_changePolicyCacheVerificationNotary(t *testing.T) {
opts := []imageverifycache.Option{
imageverifycache.WithCacheEnableFlag(true),
imageverifycache.WithMaxSize(1000),
imageverifycache.WithTTLDuration(60 * time.Minute),
}
imageVerifyCache, err := imageverifycache.New(opts...)
assert.NilError(t, err)
image := "ghcr.io/kyverno/test-verify-image:signed"
policyContext := buildContext(t, verifyImageNotaryPolicy, verifyImageNotaryResource, "")
start := time.Now()
er, ivm := testImageVerifyCache(imageVerifyCache, context.TODO(), registryclient.NewOrDie(), nil, policyContext, cfg)
firstOperationTime := time.Since(start)
errorAssertionUtil(t, image, ivm, er)
policyContext = buildContext(t, verifyImageNotaryUpdatedPolicy, verifyImageNotaryResource, "")
start = time.Now()
er, ivm = testImageVerifyCache(imageVerifyCache, context.TODO(), registryclient.NewOrDie(), nil, policyContext, cfg)
secondOperationTime := time.Since(start)
errorAssertionUtil(t, image, ivm, er)
assert.Check(t, secondOperationTime > firstOperationTime/10 && secondOperationTime < firstOperationTime*10, "cache entry not found, so image verification should not be from cache.")
}

28
pkg/engine/internal/imageverifier.go
View file

@ -7,6 +7,7 @@ import (
"fmt"
"net"
"strings"
"time"
"github.com/go-logr/logr"
"github.com/kyverno/kyverno/api/kyverno"
@ -243,8 +244,33 @@ func (iv *ImageVerifier) Verify(
iv.ivm.Add(image, true)
continue
}
start := time.Now()
found, err := iv.ivCache.Get(ctx, iv.policyContext.Policy(), iv.rule.Name, image)
if err != nil {
iv.logger.Error(err, "error occurred during cache get")
}
ruleResp, digest := iv.verifyImage(ctx, imageVerify, imageInfo, cfg)
var ruleResp *engineapi.RuleResponse
var digest string
if found {
iv.logger.V(2).Info("cache entry found", "namespace", iv.policyContext.Policy().GetNamespace(), "policy", iv.policyContext.Policy().GetName(), "ruleName", iv.rule.Name, "imageRef", image)
ruleResp = engineapi.RulePass(iv.rule.Name, engineapi.ImageVerify, "verified from cache")
digest = imageInfo.Digest
} else {
iv.logger.V(2).Info("cache entry not found", "namespace", iv.policyContext.Policy().GetNamespace(), "policy", iv.policyContext.Policy().GetName(), "ruleName", iv.rule.Name, "imageRef", image)
ruleResp, digest = iv.verifyImage(ctx, imageVerify, imageInfo, cfg)
if ruleResp != nil && ruleResp.Status() == engineapi.RuleStatusPass {
setted, err := iv.ivCache.Set(ctx, iv.policyContext.Policy(), iv.rule.Name, image)
if err != nil {
iv.logger.Error(err, "error occurred during cache set")
} else {
if setted {
iv.logger.V(4).Info("successfully set cache", "namespace", iv.policyContext.Policy().GetNamespace(), "policy", iv.policyContext.Policy().GetName(), "ruleName", iv.rule.Name, "imageRef", image)
}
}
}
}
iv.logger.V(4).Info("time taken by the image verify operation", "duration", time.Since(start))
if imageVerify.MutateDigest {
patch, retrievedDigest, err := iv.handleMutateDigest(ctx, digest, imageInfo)

52
pkg/imageverifycache/client.go
View file

@ -2,19 +2,24 @@ package imageverifycache
import (
"context"
"sync"
"time"
"github.com/dgraph-io/ristretto"
"github.com/go-logr/logr"
kyvernov1 "github.com/kyverno/kyverno/api/kyverno/v1"
)
const (
defaultTTL = 1 * time.Hour
deafultMaxSize = 1000
)
type cache struct {
logger logr.Logger
isCacheEnabled bool
maxSize int64
ttl time.Duration
lock sync.Mutex
cache *ristretto.Cache
}
type Option = func(*cache) error
@ -26,7 +31,16 @@ func New(options ...Option) (Client, error) {
return nil, err
}
}
config := ristretto.Config{
MaxCost: cache.maxSize,
NumCounters: 10 * cache.maxSize,
BufferItems: 64,
}
rcache, err := ristretto.NewCache(&config)
if err != nil {
return nil, err
}
cache.cache = rcache
return cache, nil
}
@ -55,6 +69,9 @@ func WithCacheEnableFlag(b bool) Option {
func WithMaxSize(s int64) Option {
return func(c *cache) error {
if s == 0 {
s = deafultMaxSize
}
c.maxSize = s
return nil
}
@ -62,32 +79,39 @@ func WithMaxSize(s int64) Option {
func WithTTLDuration(t time.Duration) Option {
return func(c *cache) error {
if t == 0 {
t = defaultTTL
}
c.ttl = t
return nil
}
}
func (c *cache) Set(ctx context.Context, policy kyvernov1.PolicyInterface, ruleName string, imageRef string) (bool, error) {
c.lock.Lock()
defer c.lock.Unlock()
func generateKey(policy kyvernov1.PolicyInterface, ruleName string, imageRef string) string {
return string(policy.GetUID()) + ";" + policy.GetResourceVersion() + ";" + ruleName + ";" + imageRef
}
c.logger.Info("Setting cache", "policy", policy.GetName(), "ruleName", ruleName, "imageRef", imageRef)
func (c *cache) Set(ctx context.Context, policy kyvernov1.PolicyInterface, ruleName string, imageRef string) (bool, error) {
if !c.isCacheEnabled {
return false, nil
}
c.logger.Info("Successfully set cache", "policy", policy.GetName(), "ruleName", ruleName, "imageRef", imageRef)
key := generateKey(policy, ruleName, imageRef)
stored := c.cache.SetWithTTL(key, nil, 1, c.ttl)
if stored {
return true, nil
}
return false, nil
}
func (c *cache) Get(ctx context.Context, policy kyvernov1.PolicyInterface, ruleName string, imageRef string) (bool, error) {
c.lock.Lock()
defer c.lock.Unlock()
c.logger.Info("Searching in cache", "policy", policy.GetName(), "ruleName", ruleName, "imageRef", imageRef)
if !c.isCacheEnabled {
return false, nil
}
c.logger.Info("Cache entry not found", "policy", policy.GetName(), "ruleName", ruleName, "imageRef", imageRef)
c.logger.Info("Cache entry found", "policy", policy.GetName(), "ruleName", ruleName, "imageRef", imageRef)
key := generateKey(policy, ruleName, imageRef)
_, found := c.cache.Get(key)
if found {
return true, nil
}
return false, nil
}