From bd689f1bb8fce98b3724b0a61a60ec8d4d951867 Mon Sep 17 00:00:00 2001 From: Dries De Peuter Date: Tue, 11 Apr 2023 08:43:47 +0200 Subject: [PATCH] feat: Add recursive merge and debounce --- go.mod | 10 ++++++ go.sum | 17 +++++++++ server/main.go | 72 ++++++++++++++++--------------------- server/model.go | 86 ++++++++++++++++++++++++++++++++++++++++++++ server/model_test.go | 29 +++++++++++++++ 5 files changed, 172 insertions(+), 42 deletions(-) create mode 100644 server/model.go create mode 100644 server/model_test.go diff --git a/go.mod b/go.mod index 3067001..e65c2ac 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,13 @@ require ( ) require ( + github.com/hashicorp/errwrap v1.0.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 // indirect +) + +require ( + github.com/bep/debounce v1.2.1 github.com/davecgh/go-spew v1.1.1 // indirect github.com/emicklei/go-restful/v3 v3.9.0 // indirect github.com/go-logr/logr v1.2.3 // indirect @@ -22,6 +29,7 @@ require ( github.com/google/gnostic v0.5.7-v3refs // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/google/gofuzz v1.1.0 // indirect + github.com/hashicorp/go-multierror v1.1.1 github.com/imdario/mergo v0.3.6 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -29,7 +37,9 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/samber/lo v1.38.1 github.com/spf13/pflag v1.0.5 // indirect + github.com/stretchr/testify v1.8.2 golang.org/x/net v0.7.0 // indirect golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b // indirect golang.org/x/sys v0.5.0 // indirect diff --git a/go.sum b/go.sum index cc8372c..a4e905f 100644 --- a/go.sum +++ b/go.sum @@ -33,6 +33,8 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/bep/debounce v1.2.1 h1:v67fRdBA9UQu2NhLFXrSg0Brw7CexQekrBwDMM8bzeY= +github.com/bep/debounce v1.2.1/go.mod h1:H8yggRPQKLUhUoqrJC1bO2xNya7vanpDl7xR3ISbCJ0= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= @@ -124,6 +126,10 @@ github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= @@ -154,17 +160,26 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/samber/lo v1.38.1 h1:j2XEAqXKb09Am4ebOg31SpvzUTTs6EN3VfgeLUhPdXM= +github.com/samber/lo v1.38.1/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -189,6 +204,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 h1:3MTrJm4PyNL9NBqvYDSj3DHl46qQakyfqfWo4jgfaEM= +golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17/go.mod h1:lgLbSvA5ygNOMpwM/9anMpWVlVJ7Z+cHWq/eFuinpGE= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= diff --git a/server/main.go b/server/main.go index e72564f..f0c2eb8 100644 --- a/server/main.go +++ b/server/main.go @@ -22,6 +22,7 @@ import ( "k8s.io/client-go/util/homedir" klog "k8s.io/klog/v2" + "github.com/bep/debounce" "k8s.io/client-go/tools/clientcmd" ) @@ -134,6 +135,8 @@ func loop(ctx context.Context, clientset *kubernetes.Clientset) { os.Exit(1) } + debounced := debounce.New(500 * time.Millisecond) + for event := range watch.ResultChan() { svc, ok := event.Object.(*v1.Service) if !ok { @@ -141,16 +144,19 @@ func loop(ctx context.Context, clientset *kubernetes.Clientset) { } klog.Infof("Change detected on %s", svc.GetName()) - reg, err := discoverData(clientset, namespace) - if err != nil { - klog.Error(err) - continue - } + debounced(func() { + reg, err := discoverData(clientset, namespace) + if err != nil { + klog.Error(err) + return + } + + klog.Info("Writing configmap") + if err := updateConfigMap(ctx, clientset, reg); err != nil { + klog.Error(err) + } + }) - klog.Info("Writing configmap") - if err := updateConfigMap(ctx, clientset, reg); err != nil { - klog.Error(err) - } } } @@ -164,20 +170,13 @@ func discoverData(clientset *kubernetes.Clientset, ns string) (wkRegistry, error if err != nil { return reg, err } - r := regexp.MustCompile(`^well-known.stenic.io/(.+)$`) for _, svc := range svcs.Items { for name, value := range svc.ObjectMeta.Annotations { - // Skip any non - if !r.MatchString(name) { + name = resolveName(name) + if name == "" { continue } - m := r.FindStringSubmatch(name) - if len(m) != 2 { - klog.Warningf("failed to resolve name: %s", name) - continue - } - name = m[1] if _, ok := reg[name]; !ok { reg[name] = make(wkData, 0) @@ -189,17 +188,24 @@ func discoverData(clientset *kubernetes.Clientset, ns string) (wkRegistry, error klog.Error(err) } - for k, v := range d { - if _, ok := reg[name][k]; ok { - klog.Warningf("key \"%s\" already exists", k) - } - reg[name][k] = v - } + reg[name].append(d) } } return reg, nil } +func resolveName(name string) string { + r := regexp.MustCompile(`^well-known.stenic.io/(.+)$`) + if !r.MatchString(name) { + return "" + } + m := r.FindStringSubmatch(name) + if len(m) != 2 { + return "" + } + return m[1] +} + func updateConfigMap(ctx context.Context, client kubernetes.Interface, reg wkRegistry) error { cm := &v1.ConfigMap{Data: reg.encode()} cm.Namespace = namespace @@ -226,21 +232,3 @@ func updateConfigMap(ctx context.Context, client kubernetes.Interface, reg wkReg klog.Infof("Updated ConfigMap %s/%s\n", cm.GetNamespace(), cm.GetName()) return nil } - -type wkData map[string]interface{} -type wkRegistry map[string]wkData - -func (reg wkRegistry) encode() map[string]string { - d := make(map[string]string, len(reg)) - - for name, data := range reg { - file, err := json.MarshalIndent(data, "", " ") - if err != nil { - klog.Error(err) - } else { - d[name+".json"] = string(file) - } - } - - return d -} diff --git a/server/model.go b/server/model.go new file mode 100644 index 0000000..1f6764e --- /dev/null +++ b/server/model.go @@ -0,0 +1,86 @@ +package main + +import ( + "encoding/json" + "reflect" + + klog "k8s.io/klog/v2" +) + +type wkData map[string]interface{} + +func (d wkData) append(data map[string]interface{}) { + for k, v := range data { + if _, ok := d[k]; ok { + d[k] = mergeStructs(d[k], v) + } else { + d[k] = v + } + } +} + +type wkRegistry map[string]wkData + +func (reg wkRegistry) encode() map[string]string { + d := make(map[string]string, len(reg)) + + for name, data := range reg { + file, err := json.MarshalIndent(data, "", " ") + if err != nil { + klog.Error(err) + } else { + d[name+".json"] = string(file) + } + } + + return d +} + +func mergeMapsRecursive(x1, x2 interface{}) interface{} { + switch x1 := x1.(type) { + case map[string]interface{}: + x2, ok := x2.(map[string]interface{}) + if !ok { + return x1 + } + for k, v2 := range x2 { + if v1, ok := x1[k]; ok { + x1[k] = mergeMapsRecursive(v1, v2) + } else { + x1[k] = v2 + } + } + case nil: + // merge(nil, map[string]interface{...}) -> map[string]interface{...} + x2, ok := x2.(map[string]interface{}) + if ok { + return x2 + } + } + return x1 +} + +func mergeStructs(x1, x2 interface{}) interface{} { + if reflect.TypeOf(x1) != reflect.TypeOf(x2) { + return x1 + } + + switch x1 := x1.(type) { + case []interface{}: + x1 = append(x1, x2.([]interface{})...) + case string: + x1 = x2.(string) + case map[string]interface{}: + x2 := x2.(map[string]interface{}) + for k, v2 := range x2 { + if v1, ok := x1[k]; ok { + x1[k] = mergeStructs(v1, v2) + } else { + x1[k] = v2 + } + } + default: + klog.Warningf("unknown type: %T", x1) + } + return x1 +} diff --git a/server/model_test.go b/server/model_test.go new file mode 100644 index 0000000..e32edf9 --- /dev/null +++ b/server/model_test.go @@ -0,0 +1,29 @@ +package main + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_wkData_append(t *testing.T) { + tests := []struct { + name string + d wkData + data map[string]interface{} + expected wkData + }{ + {"empty", wkData{}, wkData{"a": 1}, wkData{"a": 1}}, + {"append", wkData{"a": 1}, wkData{"b": 2}, wkData{"a": 1, "b": 2}}, + {"existing", wkData{"a": "a", "b": "b"}, wkData{"a": "aa"}, wkData{"a": "a", "b": "b"}}, + {"nested", wkData{"a": map[string]interface{}{"nest": "value"}}, wkData{"b": 2}, wkData{"b": 2, "a": map[string]interface{}{"nest": "value"}}}, + {"nestedExist", wkData{"a": map[string]interface{}{"nest": "value"}}, wkData{"a": 2}, wkData{"a": map[string]interface{}{"nest": "value"}}}, + {"nestedExistMerge", wkData{"a": map[string]interface{}{"nest": "value"}}, wkData{"a": map[string]interface{}{"nest2": "value2"}}, wkData{"a": map[string]interface{}{"nest": "value", "nest2": "value2"}}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.d.append(tt.data) + assert.Equal(t, tt.expected, tt.d, "expected: %v, got: %v", tt.expected, tt.d) + }) + } +}