From c73670a8f9ec2d579dd64c122c416bc1ad37f876 Mon Sep 17 00:00:00 2001 From: Guillermo Palacio Date: Mon, 8 Jan 2024 15:03:50 +0100 Subject: [PATCH] Add support for elasticSearch typeless API (#387) * Add support for elasticSearch typeless API Signed-off-by: guipal --- charts/policy-reporter/config.yaml | 3 ++- charts/policy-reporter/values.yaml | 2 ++ pkg/config/config.go | 1 + pkg/config/target_factory.go | 5 +++++ pkg/config/target_factory_test.go | 12 ++++++++++++ pkg/kubernetes/secrets/client.go | 10 ++++++++++ pkg/kubernetes/secrets/client_test.go | 5 +++++ pkg/target/elasticsearch/elasticsearch.go | 20 ++++++++++++++++---- 8 files changed, 53 insertions(+), 5 deletions(-) diff --git a/charts/policy-reporter/config.yaml b/charts/policy-reporter/config.yaml index 30277ccd..ec913d16 100644 --- a/charts/policy-reporter/config.yaml +++ b/charts/policy-reporter/config.yaml @@ -30,13 +30,14 @@ elasticsearch: skipTLS: {{ .Values.target.elasticsearch.skipTLS }} username: {{ .Values.target.elasticsearch.username | quote }} password: {{ .Values.target.elasticsearch.password | quote }} - apiKey: {{ .Values.target.elasticsearch.password | quote }} + apiKey: {{ .Values.target.elasticsearch.apiKey | quote }} secretRef: {{ .Values.target.elasticsearch.secretRef | quote }} mountedSecret: {{ .Values.target.elasticsearch.mountedSecret | quote }} index: {{ .Values.target.elasticsearch.index | default "policy-reporter" | quote }} rotation: {{ .Values.target.elasticsearch.rotation | default "daily" | quote }} minimumPriority: {{ .Values.target.elasticsearch.minimumPriority | quote }} skipExistingOnStartup: {{ .Values.target.elasticsearch.skipExistingOnStartup }} + typelessApi: {{ .Values.target.elasticsearch.typelessApi }} {{- with .Values.target.elasticsearch.sources }} sources: {{- toYaml . | nindent 4 }} diff --git a/charts/policy-reporter/values.yaml b/charts/policy-reporter/values.yaml index f49c8a1e..3432529e 100644 --- a/charts/policy-reporter/values.yaml +++ b/charts/policy-reporter/values.yaml @@ -377,6 +377,8 @@ target: sources: [] # Skip already existing PolicyReportResults on startup skipExistingOnStartup: true + # https://www.elastic.co/blog/moving-from-types-to-typeless-apis-in-elasticsearch-7-0 keeping as false for retrocompatibility. + typelessApi: false # Added as additional properties to each elasticsearch event customFields: {} # filter results send by namespaces, policies and priorities diff --git a/pkg/config/config.go b/pkg/config/config.go index 4d5f8f63..293ce221 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -110,6 +110,7 @@ type Elasticsearch struct { Password string `mapstructure:"password"` ApiKey string `mapstructure:"apiKey"` Channels []*Elasticsearch `mapstructure:"channels"` + TypelessApi bool `mapstructure:"typelessApi"` } // Slack configuration diff --git a/pkg/config/target_factory.go b/pkg/config/target_factory.go index b45fe379..d38fe43c 100644 --- a/pkg/config/target_factory.go +++ b/pkg/config/target_factory.go @@ -411,6 +411,7 @@ func (f *TargetFactory) createElasticsearchClient(config, parent *Elasticsearch) setFallback(&config.ApiKey, parent.ApiKey) setFallback(&config.Index, parent.Index, "policy-reporter") setFallback(&config.Rotation, parent.Rotation, elasticsearch.Daily) + setBool(&config.TypelessApi, parent.TypelessApi) config.MapBaseParent(parent.TargetBaseOptions) @@ -426,6 +427,7 @@ func (f *TargetFactory) createElasticsearchClient(config, parent *Elasticsearch) Index: config.Index, CustomFields: config.CustomFields, HTTPClient: http.NewClient(config.Certificate, config.SkipTLS), + TypelessApi: config.TypelessApi, }) } @@ -827,6 +829,9 @@ func (f *TargetFactory) mapSecretValues(config any, ref, mountedSecret string) { if values.ApiKey != "" { c.ApiKey = values.ApiKey } + if values.TypelessApi != false { + c.TypelessApi = values.TypelessApi + } case *S3: if values.AccessKeyID != "" { diff --git a/pkg/config/target_factory_test.go b/pkg/config/target_factory_test.go index 1d772d2c..a25bf272 100644 --- a/pkg/config/target_factory_test.go +++ b/pkg/config/target_factory_test.go @@ -40,6 +40,7 @@ func newFakeClient() v1.SecretInterface { "credentials": []byte(`{"token": "token", "type": "authorized_user"}`), "database": []byte("database"), "dsn": []byte(""), + "typelessApi": []byte("false"), }, }).CoreV1().Secrets("default") } @@ -58,6 +59,7 @@ func mountSecret() { Credentials: `{"token": "token", "type": "authorized_user"}`, Database: "database", DSN: "", + TypelessApi: false, } file, _ := json.MarshalIndent(secretValues, "", " ") _ = os.WriteFile(mountedSecret, file, 0o644) @@ -339,6 +341,11 @@ func Test_GetValuesFromSecret(t *testing.T) { if apiKey != "apiKey" { t.Errorf("Expected apiKey from secret, got %s", apiKey) } + + typelessApi := client.FieldByName("typelessApi").Bool() + if typelessApi != false { + t.Errorf("Expected typelessApi false value from secret, got %t", typelessApi) + } }) t.Run("Get Discord values from Secret", func(t *testing.T) { @@ -651,6 +658,11 @@ func Test_GetValuesFromMountedSecret(t *testing.T) { if apiKey != "apiKey" { t.Errorf("Expected apiKey from secret, got %s", apiKey) } + + typelessApi := client.FieldByName("typelessApi").Bool() + if typelessApi != false { + t.Errorf("Expected typelessApi false value from secret, got %t", typelessApi) + } }) t.Run("Get Discord values from MountedSecret", func(t *testing.T) { diff --git a/pkg/kubernetes/secrets/client.go b/pkg/kubernetes/secrets/client.go index 8ef329d4..723afa6b 100644 --- a/pkg/kubernetes/secrets/client.go +++ b/pkg/kubernetes/secrets/client.go @@ -3,6 +3,8 @@ package secrets import ( "context" + "strconv" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -25,6 +27,7 @@ type Values struct { Credentials string `json:"credentials,omitempty"` Database string `json:"database,omitempty"` DSN string `json:"dsn,omitempty"` + TypelessApi bool `json:"typelessApi,omitempty"` } type Client interface { @@ -124,6 +127,13 @@ func (c *k8sClient) Get(ctx context.Context, name string) (Values, error) { values.Credentials = string(credentials) } + if typelessApi, ok := secret.Data["typelessApi"]; ok { + values.TypelessApi, err = strconv.ParseBool(string(typelessApi)) + if err != nil { + values.TypelessApi = false + } + } + return values, nil } diff --git a/pkg/kubernetes/secrets/client_test.go b/pkg/kubernetes/secrets/client_test.go index 38c28df1..d0384c36 100644 --- a/pkg/kubernetes/secrets/client_test.go +++ b/pkg/kubernetes/secrets/client_test.go @@ -34,6 +34,7 @@ func newFakeClient() v1.SecretInterface { "accountID": []byte("accountID"), "database": []byte("database"), "dsn": []byte("dsn"), + "typelessApi": []byte("false"), }, }).CoreV1().Secrets("default") } @@ -94,6 +95,10 @@ func Test_Client(t *testing.T) { if values.DSN != "dsn" { t.Errorf("Unexpected DSN: %s", values.DSN) } + + if values.TypelessApi { + t.Errorf("Unexpected TypelessApi: %t", values.TypelessApi) + } }) t.Run("Get values from not existing secret", func(t *testing.T) { diff --git a/pkg/target/elasticsearch/elasticsearch.go b/pkg/target/elasticsearch/elasticsearch.go index f2ab9ed7..47a57642 100644 --- a/pkg/target/elasticsearch/elasticsearch.go +++ b/pkg/target/elasticsearch/elasticsearch.go @@ -19,6 +19,8 @@ type Options struct { Rotation string CustomFields map[string]string HTTPClient http.Client + // https://www.elastic.co/blog/moving-from-types-to-typeless-apis-in-elasticsearch-7-0 + TypelessApi bool } // Rotation Enum @@ -42,19 +44,28 @@ type client struct { rotation Rotation customFields map[string]string client http.Client + // https://www.elastic.co/blog/moving-from-types-to-typeless-apis-in-elasticsearch-7-0 + typelessApi bool } func (e *client) Send(result v1alpha2.PolicyReportResult) { var host string + var apiSuffix string + if e.typelessApi { + apiSuffix = "_doc" + } else { + apiSuffix = "event" + } + switch e.rotation { case None: - host = e.host + "/" + e.index + "/event" + host = e.host + "/" + e.index + "/" + apiSuffix case Annually: - host = e.host + "/" + e.index + "-" + time.Now().Format("2006") + "/event" + host = e.host + "/" + e.index + "-" + time.Now().Format("2006") + "/" + apiSuffix case Monthly: - host = e.host + "/" + e.index + "-" + time.Now().Format("2006.01") + "/event" + host = e.host + "/" + e.index + "-" + time.Now().Format("2006.01") + "/" + apiSuffix default: - host = e.host + "/" + e.index + "-" + time.Now().Format("2006.01.02") + "/event" + host = e.host + "/" + e.index + "-" + time.Now().Format("2006.01.02") + "/" + apiSuffix } if len(e.customFields) > 0 { @@ -98,5 +109,6 @@ func NewClient(options Options) target.Client { options.Rotation, options.CustomFields, options.HTTPClient, + options.TypelessApi, } }