Add support for elasticSearch typeless API (#387)

* Add support for elasticSearch typeless API

Signed-off-by: guipal <guillermo.palacio@docplanner.com>

charts/policy-reporter/config.yaml

@@ -30,13 +30,14 @@ elasticsearch:
   skipTLS: {{ .Values.target.elasticsearch.skipTLS }}
   username: {{ .Values.target.elasticsearch.username | quote }}
   password: {{ .Values.target.elasticsearch.password | quote }}
-  apiKey: {{ .Values.target.elasticsearch.password | quote }}
+  apiKey: {{ .Values.target.elasticsearch.apiKey | quote }}
   secretRef: {{ .Values.target.elasticsearch.secretRef | quote }}
   mountedSecret: {{ .Values.target.elasticsearch.mountedSecret | quote }}
   index: {{ .Values.target.elasticsearch.index | default "policy-reporter" | quote }}
   rotation: {{ .Values.target.elasticsearch.rotation | default "daily" | quote }}
   minimumPriority: {{ .Values.target.elasticsearch.minimumPriority | quote }}
   skipExistingOnStartup: {{ .Values.target.elasticsearch.skipExistingOnStartup }}
+  typelessApi: {{ .Values.target.elasticsearch.typelessApi }}
   {{- with .Values.target.elasticsearch.sources }}
   sources:
     {{- toYaml . | nindent 4 }}

charts/policy-reporter/values.yaml

@@ -377,6 +377,8 @@ target:
     sources: []
     # Skip already existing PolicyReportResults on startup
     skipExistingOnStartup: true
+    # https://www.elastic.co/blog/moving-from-types-to-typeless-apis-in-elasticsearch-7-0 keeping as false for retrocompatibility.
+    typelessApi: false
     # Added as additional properties to each elasticsearch event
     customFields: {}
     # filter results send by namespaces, policies and priorities

pkg/config/config.go

@@ -110,6 +110,7 @@ type Elasticsearch struct {
 	Password          string           `mapstructure:"password"`
 	ApiKey            string           `mapstructure:"apiKey"`
 	Channels          []*Elasticsearch `mapstructure:"channels"`
+	TypelessApi       bool             `mapstructure:"typelessApi"`
 }
 
 // Slack configuration

pkg/config/target_factory.go

@@ -411,6 +411,7 @@ func (f *TargetFactory) createElasticsearchClient(config, parent *Elasticsearch)
 	setFallback(&config.ApiKey, parent.ApiKey)
 	setFallback(&config.Index, parent.Index, "policy-reporter")
 	setFallback(&config.Rotation, parent.Rotation, elasticsearch.Daily)
+	setBool(&config.TypelessApi, parent.TypelessApi)
 
 	config.MapBaseParent(parent.TargetBaseOptions)
 
@@ -426,6 +427,7 @@ func (f *TargetFactory) createElasticsearchClient(config, parent *Elasticsearch)
 		Index:         config.Index,
 		CustomFields:  config.CustomFields,
 		HTTPClient:    http.NewClient(config.Certificate, config.SkipTLS),
+		TypelessApi:   config.TypelessApi,
 	})
 }
 
@@ -827,6 +829,9 @@ func (f *TargetFactory) mapSecretValues(config any, ref, mountedSecret string) {
 		if values.ApiKey != "" {
 			c.ApiKey = values.ApiKey
 		}
+		if values.TypelessApi != false {
+			c.TypelessApi = values.TypelessApi
+		}
 
 	case *S3:
 		if values.AccessKeyID != "" {

pkg/config/target_factory_test.go

@@ -40,6 +40,7 @@ func newFakeClient() v1.SecretInterface {
 			"credentials":     []byte(`{"token": "token", "type": "authorized_user"}`),
 			"database":        []byte("database"),
 			"dsn":             []byte(""),
+			"typelessApi":     []byte("false"),
 		},
 	}).CoreV1().Secrets("default")
 }
@@ -58,6 +59,7 @@ func mountSecret() {
 		Credentials:     `{"token": "token", "type": "authorized_user"}`,
 		Database:        "database",
 		DSN:             "",
+		TypelessApi:     false,
 	}
 	file, _ := json.MarshalIndent(secretValues, "", " ")
 	_ = os.WriteFile(mountedSecret, file, 0o644)
@@ -339,6 +341,11 @@ func Test_GetValuesFromSecret(t *testing.T) {
 		if apiKey != "apiKey" {
 			t.Errorf("Expected apiKey from secret, got %s", apiKey)
 		}
+
+		typelessApi := client.FieldByName("typelessApi").Bool()
+		if typelessApi != false {
+			t.Errorf("Expected typelessApi false value from secret, got %t", typelessApi)
+		}
 	})
 
 	t.Run("Get Discord values from Secret", func(t *testing.T) {
@@ -651,6 +658,11 @@ func Test_GetValuesFromMountedSecret(t *testing.T) {
 		if apiKey != "apiKey" {
 			t.Errorf("Expected apiKey from secret, got %s", apiKey)
 		}
+
+		typelessApi := client.FieldByName("typelessApi").Bool()
+		if typelessApi != false {
+			t.Errorf("Expected typelessApi false value from secret, got %t", typelessApi)
+		}
 	})
 
 	t.Run("Get Discord values from MountedSecret", func(t *testing.T) {

pkg/kubernetes/secrets/client.go

@@ -3,6 +3,8 @@ package secrets
 import (
 	"context"
 
+	"strconv"
+
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -25,6 +27,7 @@ type Values struct {
 	Credentials     string `json:"credentials,omitempty"`
 	Database        string `json:"database,omitempty"`
 	DSN             string `json:"dsn,omitempty"`
+	TypelessApi     bool   `json:"typelessApi,omitempty"`
 }
 
 type Client interface {
@@ -124,6 +127,13 @@ func (c *k8sClient) Get(ctx context.Context, name string) (Values, error) {
 		values.Credentials = string(credentials)
 	}
 
+	if typelessApi, ok := secret.Data["typelessApi"]; ok {
+		values.TypelessApi, err = strconv.ParseBool(string(typelessApi))
+		if err != nil {
+			values.TypelessApi = false
+		}
+	}
+
 	return values, nil
 }
 

pkg/kubernetes/secrets/client_test.go

@@ -34,6 +34,7 @@ func newFakeClient() v1.SecretInterface {
 			"accountID":       []byte("accountID"),
 			"database":        []byte("database"),
 			"dsn":             []byte("dsn"),
+			"typelessApi":     []byte("false"),
 		},
 	}).CoreV1().Secrets("default")
 }
@@ -94,6 +95,10 @@ func Test_Client(t *testing.T) {
 		if values.DSN != "dsn" {
 			t.Errorf("Unexpected DSN: %s", values.DSN)
 		}
+
+		if values.TypelessApi {
+			t.Errorf("Unexpected TypelessApi: %t", values.TypelessApi)
+		}
 	})
 
 	t.Run("Get values from not existing secret", func(t *testing.T) {

pkg/target/elasticsearch/elasticsearch.go

@@ -19,6 +19,8 @@ type Options struct {
 	Rotation     string
 	CustomFields map[string]string
 	HTTPClient   http.Client
+	// https://www.elastic.co/blog/moving-from-types-to-typeless-apis-in-elasticsearch-7-0
+	TypelessApi bool
 }
 
 // Rotation Enum
@@ -42,19 +44,28 @@ type client struct {
 	rotation     Rotation
 	customFields map[string]string
 	client       http.Client
+	// https://www.elastic.co/blog/moving-from-types-to-typeless-apis-in-elasticsearch-7-0
+	typelessApi bool
 }
 
 func (e *client) Send(result v1alpha2.PolicyReportResult) {
 	var host string
+	var apiSuffix string
+	if e.typelessApi {
+		apiSuffix = "_doc"
+	} else {
+		apiSuffix = "event"
+	}
+
 	switch e.rotation {
 	case None:
-		host = e.host + "/" + e.index + "/event"
+		host = e.host + "/" + e.index + "/" + apiSuffix
 	case Annually:
-		host = e.host + "/" + e.index + "-" + time.Now().Format("2006") + "/event"
+		host = e.host + "/" + e.index + "-" + time.Now().Format("2006") + "/" + apiSuffix
 	case Monthly:
-		host = e.host + "/" + e.index + "-" + time.Now().Format("2006.01") + "/event"
+		host = e.host + "/" + e.index + "-" + time.Now().Format("2006.01") + "/" + apiSuffix
 	default:
-		host = e.host + "/" + e.index + "-" + time.Now().Format("2006.01.02") + "/event"
+		host = e.host + "/" + e.index + "-" + time.Now().Format("2006.01.02") + "/" + apiSuffix
 	}
 
 	if len(e.customFields) > 0 {
@@ -98,5 +109,6 @@ func NewClient(options Options) target.Client {
 		options.Rotation,
 		options.CustomFields,
 		options.HTTPClient,
+		options.TypelessApi,
 	}
 }