Add a kyverno jp command to test jmespath expressions (#3169)

* Add a kyverno jp command to test jmespath expressions

Signed-off-by: Sambhav Kothari <sambhavs.email@gmail.com>

* Auto-generate custom function docs

Signed-off-by: Sambhav Kothari <sambhavs.email@gmail.com>

Co-authored-by: Jim Bugwadia <jim@nirmata.com>

pkg/engine/jmespath/functions.go

@@ -26,6 +26,7 @@ var (
 	JpArray       = gojmespath.JpArray
 	JpArrayString = gojmespath.JpArrayString
 	JpAny         = gojmespath.JpAny
+	JpBool        = gojmespath.JpType("bool")
 )
 
 type (
@@ -70,215 +71,301 @@ const zeroDivisionError = errorPrefix + "Zero divisor passed"
 const undefinedQuoError = errorPrefix + "Undefined quotient"
 const nonIntModuloError = errorPrefix + "Non-integer argument(s) passed for modulo"
 
-func getFunctions() []*gojmespath.FunctionEntry {
-	return []*gojmespath.FunctionEntry{
+type FunctionEntry struct {
+	Entry      *gojmespath.FunctionEntry
+	Note       string
+	ReturnType []JpType
+}
+
+func (f *FunctionEntry) String() string {
+	args := []string{}
+	for _, a := range f.Entry.Arguments {
+		aTypes := []string{}
+		for _, t := range a.Types {
+			aTypes = append(aTypes, string(t))
+		}
+		args = append(args, strings.Join(aTypes, "|"))
+	}
+	returnArgs := []string{}
+	for _, ra := range f.ReturnType {
+		returnArgs = append(returnArgs, string(ra))
+	}
+	output := fmt.Sprintf("%s(%s) %s", f.Entry.Name, strings.Join(args, ", "), strings.Join(returnArgs, ","))
+	if f.Note != "" {
+		output += fmt.Sprintf(" (%s)", f.Note)
+	}
+	return output
+}
+
+func GetFunctions() []*FunctionEntry {
+	return []*FunctionEntry{
 		{
-			Name: compare,
-			Arguments: []ArgSpec{
-				{Types: []JpType{JpString}},
-				{Types: []JpType{JpString}},
+			Entry: &gojmespath.FunctionEntry{Name: compare,
+				Arguments: []ArgSpec{
+					{Types: []JpType{JpString}},
+					{Types: []JpType{JpString}},
+				},
+				Handler: jpfCompare,
 			},
-			Handler: jpfCompare,
+			ReturnType: []JpType{JpBool},
 		},
 		{
-			Name: equalFold,
-			Arguments: []ArgSpec{
-				{Types: []JpType{JpString}},
-				{Types: []JpType{JpString}},
+			Entry: &gojmespath.FunctionEntry{Name: equalFold,
+				Arguments: []ArgSpec{
+					{Types: []JpType{JpString}},
+					{Types: []JpType{JpString}},
+				},
+				Handler: jpfEqualFold,
 			},
-			Handler: jpfEqualFold,
+			ReturnType: []JpType{JpBool},
 		},
 		{
-			Name: replace,
-			Arguments: []ArgSpec{
-				{Types: []JpType{JpString}},
-				{Types: []JpType{JpString}},
-				{Types: []JpType{JpString}},
-				{Types: []JpType{JpNumber}},
+			Entry: &gojmespath.FunctionEntry{Name: replace,
+				Arguments: []ArgSpec{
+					{Types: []JpType{JpString}},
+					{Types: []JpType{JpString}},
+					{Types: []JpType{JpString}},
+					{Types: []JpType{JpNumber}},
+				},
+				Handler: jpfReplace,
 			},
-			Handler: jpfReplace,
+			ReturnType: []JpType{JpString},
 		},
 		{
-			Name: replaceAll,
-			Arguments: []ArgSpec{
-				{Types: []JpType{JpString}},
-				{Types: []JpType{JpString}},
-				{Types: []JpType{JpString}},
+			Entry: &gojmespath.FunctionEntry{Name: replaceAll,
+				Arguments: []ArgSpec{
+					{Types: []JpType{JpString}},
+					{Types: []JpType{JpString}},
+					{Types: []JpType{JpString}},
+				},
+				Handler: jpfReplaceAll,
 			},
-			Handler: jpfReplaceAll,
+			ReturnType: []JpType{JpString},
 		},
 		{
-			Name: toUpper,
-			Arguments: []ArgSpec{
-				{Types: []JpType{JpString}},
+			Entry: &gojmespath.FunctionEntry{Name: toUpper,
+				Arguments: []ArgSpec{
+					{Types: []JpType{JpString}},
+				},
+				Handler: jpfToUpper,
 			},
-			Handler: jpfToUpper,
+			ReturnType: []JpType{JpString},
 		},
 		{
-			Name: toLower,
-			Arguments: []ArgSpec{
-				{Types: []JpType{JpString}},
+			Entry: &gojmespath.FunctionEntry{Name: toLower,
+				Arguments: []ArgSpec{
+					{Types: []JpType{JpString}},
+				},
+				Handler: jpfToLower,
 			},
-			Handler: jpfToLower,
+			ReturnType: []JpType{JpString},
 		},
 		{
-			Name: trim,
-			Arguments: []ArgSpec{
-				{Types: []JpType{JpString}},
-				{Types: []JpType{JpString}},
+			Entry: &gojmespath.FunctionEntry{Name: trim,
+				Arguments: []ArgSpec{
+					{Types: []JpType{JpString}},
+					{Types: []JpType{JpString}},
+				},
+				Handler: jpfTrim,
 			},
-			Handler: jpfTrim,
+			ReturnType: []JpType{JpString},
 		},
 		{
-			Name: split,
-			Arguments: []ArgSpec{
-				{Types: []JpType{JpString}},
-				{Types: []JpType{JpString}},
+			Entry: &gojmespath.FunctionEntry{Name: split,
+				Arguments: []ArgSpec{
+					{Types: []JpType{JpString}},
+					{Types: []JpType{JpString}},
+				},
+				Handler: jpfSplit,
 			},
-			Handler: jpfSplit,
+			ReturnType: []JpType{JpArrayString},
 		},
 		{
-			Name: regexReplaceAll,
-			Arguments: []ArgSpec{
-				{Types: []JpType{JpString}},
-				{Types: []JpType{JpString, JpNumber}},
-				{Types: []JpType{JpString, JpNumber}},
+			Entry: &gojmespath.FunctionEntry{Name: regexReplaceAll,
+				Arguments: []ArgSpec{
+					{Types: []JpType{JpString}},
+					{Types: []JpType{JpString, JpNumber}},
+					{Types: []JpType{JpString, JpNumber}},
+				},
+				Handler: jpRegexReplaceAll,
 			},
-			Handler: jpRegexReplaceAll,
+			ReturnType: []JpType{JpString},
+			Note:       "converts all parameters to string",
 		},
 		{
-			Name: regexReplaceAllLiteral,
-			Arguments: []ArgSpec{
-				{Types: []JpType{JpString}},
-				{Types: []JpType{JpString, JpNumber}},
-				{Types: []JpType{JpString, JpNumber}},
+			Entry: &gojmespath.FunctionEntry{Name: regexReplaceAllLiteral,
+				Arguments: []ArgSpec{
+					{Types: []JpType{JpString}},
+					{Types: []JpType{JpString, JpNumber}},
+					{Types: []JpType{JpString, JpNumber}},
+				},
+				Handler: jpRegexReplaceAllLiteral,
 			},
-			Handler: jpRegexReplaceAllLiteral,
+			ReturnType: []JpType{JpString},
+			Note:       "converts all parameters to string",
 		},
 		{
-			Name: regexMatch,
-			Arguments: []ArgSpec{
-				{Types: []JpType{JpString}},
-				{Types: []JpType{JpString, JpNumber}},
+			Entry: &gojmespath.FunctionEntry{Name: regexMatch,
+				Arguments: []ArgSpec{
+					{Types: []JpType{JpString}},
+					{Types: []JpType{JpString, JpNumber}},
+				},
+				Handler: jpRegexMatch,
 			},
-			Handler: jpRegexMatch,
 		},
 		{
-			Name: patternMatch,
-			Arguments: []ArgSpec{
-				{Types: []JpType{JpString}},
-				{Types: []JpType{JpString, JpNumber}},
+			Entry: &gojmespath.FunctionEntry{Name: patternMatch,
+				Arguments: []ArgSpec{
+					{Types: []JpType{JpString}},
+					{Types: []JpType{JpString, JpNumber}},
+				},
+				Handler: jpPatternMatch,
 			},
-			Handler: jpPatternMatch,
+			ReturnType: []JpType{JpBool},
+			Note:       "'*' matches zero or more alphanumeric characters, '?' matches a single alphanumeric character",
 		},
 		{
 			// Validates if label (param1) would match pod/host/etc labels (param2)
-			Name: labelMatch,
-			Arguments: []ArgSpec{
-				{Types: []JpType{JpObject}},
-				{Types: []JpType{JpObject}},
+			Entry: &gojmespath.FunctionEntry{Name: labelMatch,
+				Arguments: []ArgSpec{
+					{Types: []JpType{JpObject}},
+					{Types: []JpType{JpObject}},
+				},
+				Handler: jpLabelMatch,
 			},
-			Handler: jpLabelMatch,
+			ReturnType: []JpType{JpBool},
+			Note:       "object arguments must be enclosed in backticks; ex. `{{request.object.spec.template.metadata.labels}}`",
 		},
 		{
-			Name: add,
-			Arguments: []ArgSpec{
-				{Types: []JpType{JpAny}},
-				{Types: []JpType{JpAny}},
+			Entry: &gojmespath.FunctionEntry{Name: add,
+				Arguments: []ArgSpec{
+					{Types: []JpType{JpAny}},
+					{Types: []JpType{JpAny}},
+				},
+				Handler: jpAdd,
 			},
-			Handler: jpAdd,
+			ReturnType: []JpType{JpAny},
 		},
 		{
-			Name: subtract,
-			Arguments: []ArgSpec{
-				{Types: []JpType{JpAny}},
-				{Types: []JpType{JpAny}},
+			Entry: &gojmespath.FunctionEntry{Name: subtract,
+				Arguments: []ArgSpec{
+					{Types: []JpType{JpAny}},
+					{Types: []JpType{JpAny}},
+				},
+				Handler: jpSubtract,
 			},
-			Handler: jpSubtract,
+			ReturnType: []JpType{JpAny},
 		},
 		{
-			Name: multiply,
-			Arguments: []ArgSpec{
-				{Types: []JpType{JpAny}},
-				{Types: []JpType{JpAny}},
+			Entry: &gojmespath.FunctionEntry{Name: multiply,
+				Arguments: []ArgSpec{
+					{Types: []JpType{JpAny}},
+					{Types: []JpType{JpAny}},
+				},
+				Handler: jpMultiply,
 			},
-			Handler: jpMultiply,
+			ReturnType: []JpType{JpAny},
 		},
 		{
-			Name: divide,
-			Arguments: []ArgSpec{
-				{Types: []JpType{JpAny}},
-				{Types: []JpType{JpAny}},
+			Entry: &gojmespath.FunctionEntry{Name: divide,
+				Arguments: []ArgSpec{
+					{Types: []JpType{JpAny}},
+					{Types: []JpType{JpAny}},
+				},
+				Handler: jpDivide,
 			},
-			Handler: jpDivide,
+			ReturnType: []JpType{JpAny},
+			Note:       "divisor must be non zero",
 		},
 		{
-			Name: modulo,
-			Arguments: []ArgSpec{
-				{Types: []JpType{JpAny}},
-				{Types: []JpType{JpAny}},
+			Entry: &gojmespath.FunctionEntry{Name: modulo,
+				Arguments: []ArgSpec{
+					{Types: []JpType{JpAny}},
+					{Types: []JpType{JpAny}},
+				},
+				Handler: jpModulo,
 			},
-			Handler: jpModulo,
+			ReturnType: []JpType{JpAny},
+			Note:       "divisor must be non-zero, arguments must be integers",
 		},
 		{
-			Name: base64Decode,
-			Arguments: []ArgSpec{
-				{Types: []JpType{JpString}},
+			Entry: &gojmespath.FunctionEntry{Name: base64Decode,
+				Arguments: []ArgSpec{
+					{Types: []JpType{JpString}},
+				},
+				Handler: jpBase64Decode,
 			},
-			Handler: jpBase64Decode,
+			ReturnType: []JpType{JpString},
 		},
 		{
-			Name: base64Encode,
-			Arguments: []ArgSpec{
-				{Types: []JpType{JpString}},
+			Entry: &gojmespath.FunctionEntry{Name: base64Encode,
+				Arguments: []ArgSpec{
+					{Types: []JpType{JpString}},
+				},
+				Handler: jpBase64Encode,
 			},
-			Handler: jpBase64Encode,
+			ReturnType: []JpType{JpString},
 		},
 		{
-			Name: timeSince,
-			Arguments: []ArgSpec{
-				{Types: []JpType{JpString}},
-				{Types: []JpType{JpString}},
-				{Types: []JpType{JpString}},
+			Entry: &gojmespath.FunctionEntry{Name: timeSince,
+				Arguments: []ArgSpec{
+					{Types: []JpType{JpString}},
+					{Types: []JpType{JpString}},
+					{Types: []JpType{JpString}},
+				},
+				Handler: jpTimeSince,
 			},
-			Handler: jpTimeSince,
+			ReturnType: []JpType{JpString},
 		},
 		{
-			Name: pathCanonicalize,
-			Arguments: []ArgSpec{
-				{Types: []JpType{JpString}},
+			Entry: &gojmespath.FunctionEntry{Name: pathCanonicalize,
+				Arguments: []ArgSpec{
+					{Types: []JpType{JpString}},
+				},
+				Handler: jpPathCanonicalize,
 			},
-			Handler: jpPathCanonicalize,
+			ReturnType: []JpType{JpString},
 		},
 		{
-			Name: truncate,
-			Arguments: []ArgSpec{
-				{Types: []JpType{JpString}},
-				{Types: []JpType{JpNumber}},
+			Entry: &gojmespath.FunctionEntry{Name: truncate,
+				Arguments: []ArgSpec{
+					{Types: []JpType{JpString}},
+					{Types: []JpType{JpNumber}},
+				},
+				Handler: jpTruncate,
 			},
-			Handler: jpTruncate,
+			ReturnType: []JpType{JpString},
+			Note:       "length argument must be enclosed in backticks; ex. \"{{request.object.metadata.name | truncate(@, `9`)}}\"",
 		},
 		{
-			Name: semverCompare,
-			Arguments: []ArgSpec{
-				{Types: []JpType{JpString}},
-				{Types: []JpType{JpString}},
+			Entry: &gojmespath.FunctionEntry{Name: semverCompare,
+				Arguments: []ArgSpec{
+					{Types: []JpType{JpString}},
+					{Types: []JpType{JpString}},
+				},
+				Handler: jpSemverCompare,
 			},
-			Handler: jpSemverCompare,
+			ReturnType: []JpType{JpBool},
 		},
 		{
-			Name: parseJson,
-			Arguments: []ArgSpec{
-				{Types: []JpType{JpString}},
+			Entry: &gojmespath.FunctionEntry{Name: parseJson,
+				Arguments: []ArgSpec{
+					{Types: []JpType{JpString}},
+				},
+				Handler: jpParseJson,
 			},
-			Handler: jpParseJson,
+			ReturnType: []JpType{JpAny},
+			Note:       "decodes a valid JSON encoded string to the appropriate type. Opposite of `to_string` function",
 		},
 		{
-			Name: parseYAML,
-			Arguments: []ArgSpec{
-				{Types: []JpType{JpString}},
+			Entry: &gojmespath.FunctionEntry{Name: parseYAML,
+				Arguments: []ArgSpec{
+					{Types: []JpType{JpString}},
+				},
+				Handler: jpParseYAML,
 			},
-			Handler: jpParseYAML,
+			ReturnType: []JpType{JpAny},
+			Note:       "decodes a valid YAML encoded string to the appropriate type provided it can be represented as JSON",
 		},
 	}
 

pkg/engine/jmespath/new.go

@@ -10,8 +10,8 @@ func New(query string) (*gojmespath.JMESPath, error) {
 		return nil, err
 	}
 
-	for _, function := range getFunctions() {
-		jp.Register(function)
+	for _, function := range GetFunctions() {
+		jp.Register(function.Entry)
 	}
 
 	return jp, nil

pkg/kyverno/jp/jp_command.go

@@ -0,0 +1,122 @@
+package jp
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	gojmespath "github.com/jmespath/go-jmespath"
+	jmespath "github.com/kyverno/kyverno/pkg/engine/jmespath"
+	"github.com/spf13/cobra"
+)
+
+// Command returns jp command
+func Command() *cobra.Command {
+	var compact, unquoted, ast, listFunctions bool
+	var filename, exprFile string
+	cmd := &cobra.Command{
+		Use:          "jp",
+		Short:        "Provides a command-line interface to JMESPath, enhanced with Kyverno specific custom functions",
+		SilenceUsage: true,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			if listFunctions {
+				printFunctionList()
+				return nil
+			}
+			// The following function has been adapted from
+			// https://github.com/jmespath/jp/blob/54882e03bd277fc4475a677fab1d35eaa478b839/jp.go
+			var expression string
+			if exprFile != "" {
+				byteExpr, err := ioutil.ReadFile(filepath.Clean(exprFile))
+				if err != nil {
+					return fmt.Errorf("error opening expression file: %w", err)
+				}
+				expression = string(byteExpr)
+			} else {
+				if len(args) == 0 {
+					return fmt.Errorf("must provide at least one argument")
+				}
+				expression = args[0]
+			}
+			if ast {
+				parser := gojmespath.NewParser()
+				parsed, err := parser.Parse(expression)
+				if err != nil {
+					if syntaxError, ok := err.(gojmespath.SyntaxError); ok {
+						return fmt.Errorf("%w\n%s",
+							syntaxError,
+							syntaxError.HighlightLocation())
+					}
+					return err
+				}
+				fmt.Printf("%s", parsed)
+				return nil
+			}
+			var input interface{}
+			var jsonParser *json.Decoder
+			if filename != "" {
+				f, err := ioutil.ReadFile(filename)
+				if err != nil {
+					return fmt.Errorf("error opening input file: %w", err)
+				}
+				if err := json.Unmarshal(f, &input); err != nil {
+					return fmt.Errorf("error parsing input json: %w", err)
+				}
+			} else {
+				jsonParser = json.NewDecoder(os.Stdin)
+				if err := jsonParser.Decode(&input); err != nil {
+					return fmt.Errorf("error parsing input json: %w", err)
+				}
+			}
+			jp, err := jmespath.New(expression)
+			if err != nil {
+				return fmt.Errorf("failed to compile JMESPath: %s, error: %v", expression, err)
+			}
+			result, err := jp.Search(input)
+			if err != nil {
+				if syntaxError, ok := err.(gojmespath.SyntaxError); ok {
+					return fmt.Errorf("%s\n%s",
+						syntaxError,
+						syntaxError.HighlightLocation())
+				}
+				return fmt.Errorf("error evaluating JMESPath expression: %w", err)
+			}
+			converted, isString := result.(string)
+			if unquoted && isString {
+				fmt.Println(converted)
+			} else {
+				var toJSON []byte
+				if compact {
+					toJSON, err = json.Marshal(result)
+				} else {
+					toJSON, err = json.MarshalIndent(result, "", "  ")
+				}
+				if err != nil {
+					return fmt.Errorf("error marshalling result to JSON: %w", err)
+				}
+				fmt.Println(string(toJSON))
+			}
+			return nil
+		},
+	}
+	cmd.Flags().BoolVarP(&compact, "compact", "c", false, "Produce compact JSON output that omits nonessential whitespace")
+	cmd.Flags().BoolVarP(&listFunctions, "list-functions", "l", false, "Output a list of custom JMESPath functions in Kyverno")
+	cmd.Flags().BoolVarP(&unquoted, "unquoted", "u", false, "If the final result is a string, it will be printed without quotes")
+	cmd.Flags().BoolVar(&ast, "ast", false, "Only print the AST of the parsed expression.  Do not rely on this output, only useful for debugging purposes")
+	cmd.Flags().StringVarP(&exprFile, "expr-file", "e", "", "Read JMESPath expression from the specified file")
+	cmd.Flags().StringVarP(&filename, "filename", "f", "", "Read input JSON from a file instead of stdin")
+	return cmd
+}
+
+func printFunctionList() {
+	functions := []string{}
+	for _, function := range jmespath.GetFunctions() {
+		functions = append(functions, string(function.String()))
+	}
+	sort.Strings(functions)
+	fmt.Println(strings.Join(functions, "\n"))
+}

pkg/kyverno/main.go

@@ -5,6 +5,7 @@ import (
 	"os"
 
 	"github.com/kyverno/kyverno/pkg/kyverno/apply"
+	"github.com/kyverno/kyverno/pkg/kyverno/jp"
 	"github.com/kyverno/kyverno/pkg/kyverno/test"
 	"github.com/kyverno/kyverno/pkg/kyverno/validate"
 	"github.com/kyverno/kyverno/pkg/kyverno/version"
@@ -28,6 +29,7 @@ func CLI() {
 		apply.Command(),
 		validate.Command(),
 		test.Command(),
+		jp.Command(),
 	}
 
 	cli.AddCommand(commands...)