mirror of
https://github.com/kyverno/kyverno.git
synced 2025-03-28 02:18:15 +00:00
feat: use assertion trees in cli test command (#9380)
* feat: use assertion trees in cli test command Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com> * assert / error Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com> * output Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com> * check for nil Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com> --------- Signed-off-by: Charles-Edouard Brétéché <charles.edouard@nirmata.com>
This commit is contained in:
Charles-Edouard Brétéché
GitHub
parent
451d362104
commit
b7eea2faa9
5 changed files with 210 additions and 19 deletions
|
|
@ -1,6 +1,7 @@
|
|||
package v1alpha1
|
||||
|
||||
import (
|
||||
"github.com/kyverno/kyverno-json/pkg/apis/v1alpha1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
|
||||
|
|
@ -32,6 +33,31 @@ type Test struct {
|
|||
// Results are the results to be checked in the test
|
||||
Results []TestResult `json:"results,omitempty"`
|
||||
|
||||
// Checks are the verifications to be checked in the test
|
||||
Checks []CheckResult `json:"checks,omitempty"`
|
||||
|
||||
// Values are the values to be used in the test
|
||||
Values *ValuesSpec `json:"values,omitempty"`
|
||||
}
|
||||
|
||||
type CheckResult struct {
|
||||
// Match tells how to match relevant rule responses
|
||||
Match CheckMatch `json:"match,omitempty"`
|
||||
|
||||
// Assert contains assertion to be performed on the relevant rule responses
|
||||
Assert v1alpha1.Any `json:"assert"`
|
||||
|
||||
// Error contains negative assertion to be performed on the relevant rule responses
|
||||
Error v1alpha1.Any `json:"error"`
|
||||
}
|
||||
|
||||
type CheckMatch struct {
|
||||
// Resource filters engine responses
|
||||
Resource *v1alpha1.Any `json:"resource,omitempty"`
|
||||
|
||||
// Policy filters engine responses
|
||||
Policy *v1alpha1.Any `json:"policy,omitempty"`
|
||||
|
||||
// Rule filters rule responses
|
||||
Rule *v1alpha1.Any `json:"rule,omitempty"`
|
||||
}
|
||||
|
|
|
|||
|
|
@ -101,7 +101,7 @@ func testCommandExecute(
|
|||
}
|
||||
}
|
||||
rc := &resultCounts{}
|
||||
var table table.Table
|
||||
var fullTable table.Table
|
||||
for _, test := range tests {
|
||||
if test.Err == nil {
|
||||
deprecations.CheckTest(out, test.Path, test.Test)
|
||||
|
|
@ -121,11 +121,24 @@ func testCommandExecute(
|
|||
return fmt.Errorf("failed to run test (%w)", err)
|
||||
}
|
||||
fmt.Fprintln(out, " Checking results ...")
|
||||
t, err := printTestResult(out, filteredResults, responses, rc, failOnly, detailedResults, test.Fs, resourcePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to print test result (%w)", err)
|
||||
var resultsTable table.Table
|
||||
{
|
||||
err := printTestResult(out, filteredResults, responses, rc, &resultsTable, test.Fs, resourcePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to print test result (%w)", err)
|
||||
}
|
||||
}
|
||||
table.AddFailed(t.RawRows...)
|
||||
{
|
||||
err := printCheckResult(out, test.Test.Checks, responses, rc, &resultsTable)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to print test result (%w)", err)
|
||||
}
|
||||
}
|
||||
fullTable.AddFailed(resultsTable.RawRows...)
|
||||
printer := table.NewTablePrinter(out)
|
||||
fmt.Fprintln(out)
|
||||
printer.Print(resultsTable.Rows(detailedResults))
|
||||
fmt.Fprintln(out)
|
||||
}
|
||||
}
|
||||
if !failOnly {
|
||||
|
|
@ -136,7 +149,7 @@ func testCommandExecute(
|
|||
fmt.Fprintln(out)
|
||||
if rc.Fail > 0 {
|
||||
if !failOnly {
|
||||
printFailedTestResult(out, table, detailedResults)
|
||||
printFailedTestResult(out, fullTable, detailedResults)
|
||||
}
|
||||
return fmt.Errorf("%d tests failed", rc.Fail)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,30 +1,174 @@
|
|||
package test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/go-git/go-billy/v5"
|
||||
"github.com/kyverno/kyverno-json/pkg/engine/assert"
|
||||
policyreportv1alpha2 "github.com/kyverno/kyverno/api/policyreport/v1alpha2"
|
||||
"github.com/kyverno/kyverno/cmd/cli/kubectl-kyverno/apis/v1alpha1"
|
||||
"github.com/kyverno/kyverno/cmd/cli/kubectl-kyverno/output/color"
|
||||
"github.com/kyverno/kyverno/cmd/cli/kubectl-kyverno/output/table"
|
||||
engineapi "github.com/kyverno/kyverno/pkg/engine/api"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
)
|
||||
|
||||
func printCheckResult(
|
||||
out io.Writer,
|
||||
checks []v1alpha1.CheckResult,
|
||||
responses []engineapi.EngineResponse,
|
||||
rc *resultCounts,
|
||||
resultsTable *table.Table,
|
||||
) error {
|
||||
testCount := 1
|
||||
for _, check := range checks {
|
||||
// filter engine responses
|
||||
matchingEngineResponses := responses
|
||||
// 1. by resource
|
||||
if check.Match.Resource != nil {
|
||||
var filtered []engineapi.EngineResponse
|
||||
for _, response := range matchingEngineResponses {
|
||||
errs, err := assert.Validate(context.Background(), check.Match.Resource.Value, response.Resource.UnstructuredContent(), nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(errs) == 0 {
|
||||
filtered = append(filtered, response)
|
||||
}
|
||||
}
|
||||
matchingEngineResponses = filtered
|
||||
}
|
||||
// 2. by policy
|
||||
if check.Match.Policy != nil {
|
||||
var filtered []engineapi.EngineResponse
|
||||
for _, response := range matchingEngineResponses {
|
||||
data, err := runtime.DefaultUnstructuredConverter.ToUnstructured(response.Policy().MetaObject())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
errs, err := assert.Validate(context.Background(), check.Match.Policy.Value, data, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(errs) == 0 {
|
||||
filtered = append(filtered, response)
|
||||
}
|
||||
}
|
||||
matchingEngineResponses = filtered
|
||||
}
|
||||
for _, response := range matchingEngineResponses {
|
||||
// filter rule responses
|
||||
matchingRuleResponses := response.PolicyResponse.Rules
|
||||
if check.Match.Rule != nil {
|
||||
var filtered []engineapi.RuleResponse
|
||||
for _, response := range matchingRuleResponses {
|
||||
data := map[string]any{
|
||||
"name": response.Name(),
|
||||
}
|
||||
errs, err := assert.Validate(context.Background(), check.Match.Rule.Value, data, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(errs) == 0 {
|
||||
filtered = append(filtered, response)
|
||||
}
|
||||
}
|
||||
matchingRuleResponses = filtered
|
||||
}
|
||||
for _, rule := range matchingRuleResponses {
|
||||
// perform check
|
||||
data := map[string]any{
|
||||
"name": rule.Name(),
|
||||
"ruleType": rule.RuleType(),
|
||||
"message": rule.Message(),
|
||||
"status": string(rule.Status()),
|
||||
// generatedResource unstructured.Unstructured
|
||||
// patchedTarget *unstructured.Unstructured
|
||||
// patchedTargetParentResourceGVR metav1.GroupVersionResource
|
||||
// patchedTargetSubresourceName string
|
||||
// podSecurityChecks contains pod security checks (only if this is a pod security rule)
|
||||
"podSecurityChecks": rule.PodSecurityChecks(),
|
||||
"exception ": rule.Exception(),
|
||||
}
|
||||
if check.Assert.Value != nil {
|
||||
errs, err := assert.Validate(context.Background(), check.Assert.Value, data, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
row := table.Row{
|
||||
RowCompact: table.RowCompact{
|
||||
ID: testCount,
|
||||
Policy: color.Policy("", response.Policy().GetName()),
|
||||
Rule: color.Rule(rule.Name()),
|
||||
Resource: color.Resource(response.Resource.GetKind(), response.Resource.GetNamespace(), response.Resource.GetName()),
|
||||
IsFailure: len(errs) != 0,
|
||||
},
|
||||
Message: rule.Message(),
|
||||
}
|
||||
if len(errs) == 0 {
|
||||
row.Result = color.ResultPass()
|
||||
row.Reason = "Ok"
|
||||
if rule.Status() == engineapi.RuleStatusSkip {
|
||||
rc.Skip++
|
||||
} else {
|
||||
rc.Pass++
|
||||
}
|
||||
} else {
|
||||
row.Result = color.ResultFail()
|
||||
row.Reason = errs.ToAggregate().Error()
|
||||
rc.Fail++
|
||||
}
|
||||
resultsTable.Add(row)
|
||||
testCount++
|
||||
}
|
||||
if check.Error.Value != nil {
|
||||
errs, err := assert.Validate(context.Background(), check.Error.Value, data, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
row := table.Row{
|
||||
RowCompact: table.RowCompact{
|
||||
ID: testCount,
|
||||
Policy: color.Policy("", response.Policy().GetName()),
|
||||
Rule: color.Rule(rule.Name()),
|
||||
Resource: color.Resource(response.Resource.GetKind(), response.Resource.GetNamespace(), response.Resource.GetName()),
|
||||
IsFailure: len(errs) != 0,
|
||||
},
|
||||
Message: rule.Message(),
|
||||
}
|
||||
if len(errs) != 0 {
|
||||
row.Result = color.ResultPass()
|
||||
row.Reason = errs.ToAggregate().Error()
|
||||
if rule.Status() == engineapi.RuleStatusSkip {
|
||||
rc.Skip++
|
||||
} else {
|
||||
rc.Pass++
|
||||
}
|
||||
} else {
|
||||
row.Result = color.ResultFail()
|
||||
row.Reason = "The assertion succeeded but was expected to fail"
|
||||
rc.Fail++
|
||||
}
|
||||
resultsTable.Add(row)
|
||||
testCount++
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func printTestResult(
|
||||
out io.Writer,
|
||||
tests []v1alpha1.TestResult,
|
||||
responses []engineapi.EngineResponse,
|
||||
rc *resultCounts,
|
||||
failOnly bool,
|
||||
detailedResults bool,
|
||||
resultsTable *table.Table,
|
||||
fs billy.Filesystem,
|
||||
resoucePath string,
|
||||
) (table.Table, error) {
|
||||
printer := table.NewTablePrinter(out)
|
||||
var resultsTable table.Table
|
||||
var countDeprecatedResource int
|
||||
) error {
|
||||
testCount := 1
|
||||
for _, test := range tests {
|
||||
// lookup matching engine responses (without the resource name)
|
||||
|
|
@ -36,7 +180,6 @@ func printTestResult(
|
|||
if test.Resources != nil {
|
||||
resources = append(resources, test.Resources...)
|
||||
} else if test.Resource != "" {
|
||||
countDeprecatedResource++
|
||||
resources = append(resources, test.Resource)
|
||||
}
|
||||
for _, resource := range resources {
|
||||
|
|
@ -116,10 +259,7 @@ func printTestResult(
|
|||
}
|
||||
}
|
||||
}
|
||||
fmt.Fprintln(out)
|
||||
printer.Print(resultsTable.Rows(detailedResults))
|
||||
fmt.Fprintln(out)
|
||||
return resultsTable, nil
|
||||
return nil
|
||||
}
|
||||
|
||||
func printFailedTestResult(out io.Writer, resultsTable table.Table, detailedResults bool) {
|
||||
|
|
|
|||
6
go.mod
6
go.mod
|
|
@ -30,6 +30,7 @@ require (
|
|||
github.com/julienschmidt/httprouter v1.3.0
|
||||
github.com/kataras/tablewriter v0.0.0-20180708051242-e063d29b7c23
|
||||
github.com/kyverno/go-jmespath v0.4.1-0.20231124160150-95e59c162877
|
||||
github.com/kyverno/kyverno-json v0.0.1
|
||||
github.com/lensesio/tableprinter v0.0.0-20201125135848-89e81fc956e7
|
||||
github.com/notaryproject/notation-core-go v1.0.2
|
||||
github.com/notaryproject/notation-go v1.0.1
|
||||
|
|
@ -92,6 +93,11 @@ require (
|
|||
|
||||
require github.com/open-policy-agent/gatekeeper/v3 v3.14.0 // indirect
|
||||
|
||||
require (
|
||||
github.com/jmespath-community/go-jmespath v1.1.2-0.20231004164315-78945398586a // indirect
|
||||
github.com/smarty/assertions v1.15.1 // indirect
|
||||
)
|
||||
|
||||
require (
|
||||
cloud.google.com/go/compute v1.23.3 // indirect
|
||||
cloud.google.com/go/compute/metadata v0.2.3 // indirect
|
||||
|
|
|
|||
10
go.sum
10
go.sum
|
|
@ -547,6 +547,8 @@ github.com/jellydator/ttlcache/v3 v3.1.1 h1:RCgYJqo3jgvhl+fEWvjNW8thxGWsgxi+TPhR
|
|||
github.com/jellydator/ttlcache/v3 v3.1.1/go.mod h1:hi7MGFdMAwZna5n2tuvh63DvFLzVKySzCVW6+0gA2n4=
|
||||
github.com/jinzhu/copier v0.4.0 h1:w3ciUoD19shMCRargcpm0cm91ytaBhDvuRpz1ODO/U8=
|
||||
github.com/jinzhu/copier v0.4.0/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg=
|
||||
github.com/jmespath-community/go-jmespath v1.1.2-0.20231004164315-78945398586a h1:8W5d74FhEWTJPnFwpDDxbwUK3pPLUbY4RlfN2uzTTSE=
|
||||
github.com/jmespath-community/go-jmespath v1.1.2-0.20231004164315-78945398586a/go.mod h1:4gOyFJsR/Gk+05RgTKYrifT7tBPWD8Lubtb5jRrfy9I=
|
||||
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
|
||||
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
|
||||
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
|
||||
|
|
@ -588,6 +590,8 @@ github.com/kyverno/go-jmespath v0.4.1-0.20231124160150-95e59c162877 h1:XOLJNGX/q
|
|||
github.com/kyverno/go-jmespath v0.4.1-0.20231124160150-95e59c162877/go.mod h1:yzDHaKovQy16rjN4kFnjF+IdNoN4p1ndw+va6+B8zUU=
|
||||
github.com/kyverno/go-jmespath/internal/testify v1.5.2-0.20230630133209-945021c749d9 h1:lL311dF3a2aeNibJj8v+uhFU3XkvRHZmCtAdSPOrQYY=
|
||||
github.com/kyverno/go-jmespath/internal/testify v1.5.2-0.20230630133209-945021c749d9/go.mod h1:XRxUGHIiCy1WYma1CdfdO1WOhIe8dLPTENaZr5D1ex4=
|
||||
github.com/kyverno/kyverno-json v0.0.1 h1:2d3k1M0YCWRz9r5fdHkIMesChPbmtSYqR6qk+2s05b0=
|
||||
github.com/kyverno/kyverno-json v0.0.1/go.mod h1:7lNc9nnrNYC1Pbn/Qd5acyoRXa6sqBrZulc6Rg64q7w=
|
||||
github.com/lensesio/tableprinter v0.0.0-20201125135848-89e81fc956e7 h1:k/1ku0yehLCPqERCHkIHMDqDg1R02AcCScRuHbamU3s=
|
||||
github.com/lensesio/tableprinter v0.0.0-20201125135848-89e81fc956e7/go.mod h1:YR/zYthNdWfO8+0IOyHDcIDBBBS2JMnYUIwSsnwmRqU=
|
||||
github.com/letsencrypt/boulder v0.0.0-20240122173420-ce5632b480f0 h1:Ixlk3bGcKTpJeRujQ/YsO7aKq4CIHvBLodPcAzp/QZg=
|
||||
|
|
@ -781,11 +785,13 @@ github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 h1:JIAuq3EE
|
|||
github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog=
|
||||
github.com/smallstep/assert v0.0.0-20200723003110-82e2b9b3b262 h1:unQFBIznI+VYD1/1fApl1A+9VcBk+9dcqGfnePY87LY=
|
||||
github.com/smallstep/assert v0.0.0-20200723003110-82e2b9b3b262/go.mod h1:MyOHs9Po2fbM1LHej6sBUT8ozbxmMOFG+E+rx/GSGuc=
|
||||
github.com/smarty/assertions v1.15.1 h1:812oFiXI+G55vxsFf+8bIZ1ux30qtkdqzKbEFwyX3Tk=
|
||||
github.com/smarty/assertions v1.15.1/go.mod h1:yABtdzeQs6l1brC900WlRNwj6ZR55d7B+E8C6HtKdec=
|
||||
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
|
||||
github.com/smartystreets/assertions v1.1.0 h1:MkTeG1DMwsrdH7QtLXy5W+fUxWq+vmb6cLmyJ7aRtF0=
|
||||
github.com/smartystreets/assertions v1.1.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
|
||||
github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
|
||||
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
|
||||
github.com/smartystreets/goconvey v1.8.1 h1:qGjIddxOk4grTu9JPOU31tVfq3cNdBlNa5sSznIX1xY=
|
||||
github.com/smartystreets/goconvey v1.8.1/go.mod h1:+/u4qLyY6x1jReYOp7GOM2FSt8aP9CzCZL03bI28W60=
|
||||
github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js=
|
||||
github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0=
|
||||
github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue