feat: Serve data instead of using the nginx container

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.20 AS build-server
+FROM golang:1.22 AS build-server
 
 WORKDIR /workspace/server
 # Copy the Go Modules manifests
@@ -19,16 +19,21 @@ RUN CGO_ENABLED=0 GOOS=linux go build -a -o well-known ./
 
 FROM alpine AS downloader
 
-RUN wget -O /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.5/dumb-init_1.2.5_x86_64
+ARG TARGETPLATFORM
-RUN chmod +x /usr/local/bin/dumb-init
+ARG TINI_VERSION=v0.19.0
+RUN if [ "$TARGETPLATFORM" = "linux/amd64" ]; then ARCHITECTURE=amd64; elif [ "$TARGETPLATFORM" = "linux/arm/v7" ]; then ARCHITECTURE=arm; elif [ "$TARGETPLATFORM" = "linux/arm64" ]; then ARCHITECTURE=arm64; else ARCHITECTURE=amd64; fi \
+    && wget -O /usr/local/bin/tini https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-static-${ARCHITECTURE}
+RUN chmod +x /usr/local/bin/tini
+
+#
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details
 FROM gcr.io/distroless/static:nonroot
 WORKDIR /app
 
-COPY --from=downloader /usr/local/bin/dumb-init /app/dumb-init
+COPY --from=downloader /usr/local/bin/tini /app/tini
 COPY --from=build-server /workspace/server/well-known /app/well-known
 USER 65532:65532
 
-ENTRYPOINT ["/app/dumb-init", "--", "/app/well-known"]
+ENTRYPOINT ["/app/tini", "--", "/app/well-known"]

charts/well-known/ci/pluto-values.yaml

@@ -6,4 +6,4 @@ autoscaling:
 
 networkpolicies:
   enabled: true
-  kubeApiServerCIDR: 1.2.3.4/32
+  kubeApiServerCIDR: 1.2.3.4/32

charts/well-known/templates/configmap.yaml

@@ -1,50 +0,0 @@
-kind: ConfigMap
-apiVersion: v1
-metadata:
-  name: {{ include "well-known.fullname" . }}
-  namespace: {{ .Release.Namespace }}
-  labels:
-    {{- include "well-known.labels" . | nindent 4 }}
-data:
-  default.conf: |
-    server {
-      listen       8080;
-      server_name  _;
-      
-    {{- if not .Values.webserver.config.accessLogEnabled }}
-      access_log off;
-    {{- end }}
-
-      location /.well-known/ {
-        default_type application/json;
-        root   /usr/share/nginx/html;
-        try_files $uri $uri/ $uri.json $uri/index.json =404;
-      }
-
-      error_page 400 404 405 =200 @40*_json;
-      location @40*_json {
-        default_type application/json;
-        return 200 '{"code":"1", "message": "Not Found"}';
-      }
-
-      error_page 500 502 503 504 =200 @50*_json;
-      location @50*_json {
-        default_type application/json;
-        return 200 '{"code":"1", "message": "Unknown Error"}';
-      }
-    }
-    server {
-      listen 8082;
-      server_name localhost;
-      root /usr/share/nginx/html;
-
-      access_log off;
-      allow 127.0.0.1;
-      deny all;
-
-      location /healthz {
-        allow 127.0.0.1;
-        stub_status;
-        server_tokens on;
-      }
-    }

charts/well-known/templates/deployment.yaml

@@ -30,36 +30,6 @@ spec:
       securityContext:
         {{- toYaml .Values.podSecurityContext | nindent 8 }}
       containers:
-        - name: webserver
-          securityContext:
-            {{- toYaml .Values.securityContext | nindent 12 }}
-          image: "{{ .Values.webserver.image.repository }}:{{ .Values.webserver.image.tag }}"
-          imagePullPolicy: {{ .Values.webserver.image.pullPolicy }}
-          ports:
-            - name: http
-              containerPort: 8080
-              protocol: TCP
-            - name: probe
-              containerPort: 8082
-              protocol: TCP
-          livenessProbe:
-            httpGet:
-              path: /healthz
-              port: probe
-          readinessProbe:
-            httpGet:
-              path: /healthz
-              port: probe
-          volumeMounts:
-            - name: config
-              mountPath: /etc/nginx/conf.d/default.conf
-              subPath: default.conf
-            - name: data
-              mountPath: /usr/share/nginx/html/.well-known
-            - mountPath: /tmp
-              name: tmp-volume
-          resources:
-            {{- toYaml .Values.webserver.resources | nindent 12 }}
         - name: {{ .Chart.Name }}
           securityContext:
             {{- toYaml .Values.securityContext | nindent 12 }}
@@ -76,6 +46,9 @@ spec:
                   apiVersion: v1
                   fieldPath: metadata.name
           ports:
+            - name: http
+              containerPort: 8080
+              protocol: TCP
             - name: probe
               containerPort: 8081
               protocol: TCP
@@ -101,13 +74,3 @@ spec:
       tolerations:
         {{- toYaml . | nindent 8 }}
       {{- end }}
-      volumes:
-        - name: config
-          configMap:
-            name: {{ include "well-known.fullname" . }}
-        - name: data
-          configMap:
-            name: {{ include "well-known.fullname" . }}-data
-            optional: true
-        - name: tmp-volume
-          emptyDir: {}

charts/well-known/values.yaml

@@ -18,22 +18,7 @@ resources:
     cpu: 20m
     memory: 32Mi
 
-webserver:
+podDisruptionBudget:
-  image:
-    repository: nginxinc/nginx-unprivileged
-    pullPolicy: Always
-    tag: "1.25"
-  resources:
-    limits:
-      cpu: 50m
-      memory: 24Mi
-    requests:
-      cpu: 10m
-      memory: 10Mi
-  config:
-    accessLogEnabled: false
-
-podDisruptionBudget: 
   maxUnavailable: 1
 
 imagePullSecrets: []
@@ -93,8 +78,9 @@ autoscaling:
 
 networkpolicies:
   enabled: false
-  kubeApi: [] # kubectl get svc -n default kubernetes -oyaml
+  kubeApi: []
-    # - addresses: 
+    # kubectl get svc -n default kubernetes -oyaml
+    # - addresses:
     #     - 10.0.0.153
     #     - 10.0.0.90
     #   ports:

go.mod

@@ -1,6 +1,6 @@
 module well-known
 
-go 1.20
+go 1.22
 
 require (
 	k8s.io/api v0.28.3

server/main.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"context"
-	"encoding/json"
 	"flag"
 	"net/http"
 	"os"
@@ -11,8 +10,6 @@ import (
 	"syscall"
 	"time"
 
-	v1 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
@@ -21,19 +18,21 @@ import (
 	"k8s.io/client-go/util/homedir"
 	klog "k8s.io/klog/v2"
 
-	"github.com/bep/debounce"
 	"k8s.io/client-go/tools/clientcmd"
-
-	"github.com/davegardnerisme/deephash"
 )
 
-var kubeconfig string
+var (
-var namespace string
+	kubeconfig    string
-var cmName string
+	namespace     string
-var id string
+	cmName        string
-var leaseLockName string
+	id            string
+	leaseLockName string
 
-func main() {
+	serverPort string
+	healthPort string
+)
+
+func parseFlags() {
 	klog.InitFlags(nil)
 	if home := homedir.HomeDir(); home != "" {
 		flag.StringVar(&kubeconfig, "kubeconfig", filepath.Join(home, ".kube", "config"), "(optional) absolute path to the kubeconfig file")
@@ -44,9 +43,16 @@ func main() {
 	flag.StringVar(&cmName, "configmap", "well-known-generated", "")
 	flag.StringVar(&id, "id", os.Getenv("POD_NAME"), "the holder identity name")
 	flag.StringVar(&leaseLockName, "lease-lock-name", "well-known", "the lease lock resource name")
+	flag.StringVar(&serverPort, "server-port", "8080", "server port")
+	flag.StringVar(&healthPort, "health-port", "8081", "health port")
 	flag.Parse()
 
-	// creates the in-cluster config
+	if id == "" {
+		klog.Fatal("id is required")
+	}
+}
+
+func getClientset() *kubernetes.Clientset {
 	config, err := rest.InClusterConfig()
 	if err != nil {
 		config, err = clientcmd.BuildConfigFromFlags("", kubeconfig)
@@ -60,6 +66,13 @@ func main() {
 		klog.Fatal(err)
 	}
 
+	return clientset
+}
+
+func main() {
+	// Parse flags
+	parseFlags()
+
 	// use a Go context so we can tell the leaderelection code when we
 	// want to step down
 	ctx, cancel := context.WithCancel(context.Background())
@@ -76,34 +89,41 @@ func main() {
 		cancel()
 	}()
 
-	// we use the Lease lock type since edits to Leases are less common
+	// Connect to the cluster
-	// and fewer objects in the cluster watch "all Leases".
+	clientset := getClientset()
-	lock := &resourcelock.LeaseLock{
-		LeaseMeta: metav1.ObjectMeta{
-			Name:      leaseLockName,
-			Namespace: namespace,
-		},
-		Client: clientset.CoordinationV1(),
-		LockConfig: resourcelock.ResourceLockConfig{
-			Identity: id,
-		},
-	}
 
+	wks := NewWellKnownService(clientset, namespace, cmName)
+
+	// Start the server
 	go func() {
-		http.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
+		klog.Infof("Running /.well-known/{id} endpoint on :%s", serverPort)
-			w.WriteHeader(http.StatusOK)
+		if err := http.ListenAndServe(":"+serverPort, GetServer(wks)); err != nil {
-		})
-
-		klog.Info("Running /healthz endpoint on :8081")
-		if err := http.ListenAndServe(":8081", nil); err != nil {
 			klog.Error(err)
 			os.Exit(1)
 		}
 	}()
 
-	// start the leader election code loop
+	// Start the health server
+	go func() {
+		klog.Infof("Running /healthz endpoint on :%s", healthPort)
+		if err := http.ListenAndServe(":"+healthPort, GetHealthServer()); err != nil {
+			klog.Error(err)
+			os.Exit(1)
+		}
+	}()
+
+	// Start the leader election code loop
 	leaderelection.RunOrDie(ctx, leaderelection.LeaderElectionConfig{
-		Lock:            lock,
+		Lock: &resourcelock.LeaseLock{
+			LeaseMeta: metav1.ObjectMeta{
+				Name:      leaseLockName,
+				Namespace: namespace,
+			},
+			Client: clientset.CoordinationV1(),
+			LockConfig: resourcelock.ResourceLockConfig{
+				Identity: id,
+			},
+		},
 		ReleaseOnCancel: true,
 		LeaseDuration:   60 * time.Second,
 		RenewDeadline:   15 * time.Second,
@@ -112,7 +132,7 @@ func main() {
 			OnStartedLeading: func(ctx context.Context) {
 				for {
 					// ensure that we keep observing
-					loop(ctx, clientset)
+					wks.DiscoveryLoop(ctx)
 				}
 			},
 			OnStoppedLeading: func() {
@@ -128,106 +148,3 @@ func main() {
 		},
 	})
 }
-
-func loop(ctx context.Context, clientset *kubernetes.Clientset) {
-	watch, err := clientset.
-		CoreV1().
-		Services(namespace).
-		Watch(ctx, metav1.ListOptions{})
-	if err != nil {
-		klog.Error(err)
-		os.Exit(1)
-	}
-
-	debounced := debounce.New(500 * time.Millisecond)
-	hash := []byte{}
-
-	for event := range watch.ResultChan() {
-		svc, ok := event.Object.(*v1.Service)
-		if !ok {
-			continue
-		}
-		klog.V(1).Infof("Change detected on %s", svc.GetName())
-
-		debounced(func() {
-			reg, err := discoverData(clientset, namespace)
-			if err != nil {
-				klog.Error(err)
-				return
-			}
-
-			newHash := deephash.Hash(reg)
-			if string(hash) == string(newHash) {
-				klog.V(1).Info("No changes detected")
-				return
-			}
-			hash = newHash
-
-			klog.Info("Writing configmap")
-			if err := updateConfigMap(ctx, clientset, reg); err != nil {
-				klog.Error(err)
-			}
-		})
-	}
-}
-
-func discoverData(clientset *kubernetes.Clientset, ns string) (wkRegistry, error) {
-	reg := make(wkRegistry, 0)
-
-	svcs, err := clientset.
-		CoreV1().
-		Services(namespace).
-		List(context.Background(), metav1.ListOptions{})
-	if err != nil {
-		return reg, err
-	}
-
-	for _, svc := range svcs.Items {
-		for name, value := range svc.ObjectMeta.Annotations {
-			name = resolveName(name)
-			if name == "" {
-				continue
-			}
-
-			if _, ok := reg[name]; !ok {
-				reg[name] = make(wkData, 0)
-			}
-
-			var d map[string]interface{}
-			err := json.Unmarshal([]byte(value), &d)
-			if err != nil {
-				klog.Error(err)
-			}
-
-			reg[name].append(d)
-		}
-	}
-	return reg, nil
-}
-
-func updateConfigMap(ctx context.Context, client kubernetes.Interface, reg wkRegistry) error {
-	cm := &v1.ConfigMap{Data: reg.encode()}
-	cm.Namespace = namespace
-	cm.Name = cmName
-
-	_, err := client.
-		CoreV1().
-		ConfigMaps(namespace).
-		Update(ctx, cm, metav1.UpdateOptions{})
-	if errors.IsNotFound(err) {
-		_, err = client.
-			CoreV1().
-			ConfigMaps(namespace).
-			Create(ctx, cm, metav1.CreateOptions{})
-		if err == nil {
-			klog.Infof("Created ConfigMap %s/%s\n", cm.GetNamespace(), cm.GetName())
-		}
-		return err
-	} else if err != nil {
-		klog.Error(err)
-		return err
-	}
-
-	klog.Infof("Updated ConfigMap %s/%s\n", cm.GetNamespace(), cm.GetName())
-	return nil
-}

server/server.go

@@ -0,0 +1,58 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+)
+
+type WellKnownGetter interface {
+	GetData(ctx context.Context) (*wkRegistry, error)
+}
+
+func GetHealthServer() http.Handler {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+		w.Write([]byte("OK"))
+	})
+	return mux
+}
+
+func GetServer(wks WellKnownGetter) http.Handler {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/.well-known/{id}", func(w http.ResponseWriter, r *http.Request) {
+		reg, err := wks.GetData(r.Context())
+		if err != nil {
+			w.WriteHeader(http.StatusInternalServerError)
+			w.Write([]byte("Failed to fetch well-known records"))
+			return
+		}
+
+		if reg == nil {
+			w.WriteHeader(http.StatusNotFound)
+			w.Write([]byte("Not found"))
+			return
+		}
+
+		items := reg
+		id := r.PathValue("id")
+		if val, ok := (*items)[id]; ok {
+			b, err := json.Marshal(val)
+			if err != nil {
+				w.WriteHeader(http.StatusInternalServerError)
+				w.Write([]byte("Failed to encode"))
+				return
+			}
+
+			w.WriteHeader(http.StatusOK)
+			w.Write(b)
+			return
+		}
+
+		w.WriteHeader(http.StatusNotFound)
+		w.Write([]byte("Not found"))
+	})
+
+	return mux
+}

server/server_test.go

@@ -0,0 +1,71 @@
+package main
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+type fakeWellKnownGetter struct {
+	reg *wkRegistry
+}
+
+func (f *fakeWellKnownGetter) GetData(ctx context.Context) (*wkRegistry, error) {
+	return f.reg, nil
+}
+
+func Test_GetServer(t *testing.T) {
+	wks := &fakeWellKnownGetter{
+		reg: &wkRegistry{
+			"test": {
+				"key": "value",
+			},
+			"empty": {},
+		},
+	}
+
+	tt := []struct {
+		name     string
+		path     string
+		expected string
+		code     int
+	}{
+		{
+			name:     "existing",
+			path:     "/.well-known/test",
+			expected: `{"key":"value"}`,
+			code:     http.StatusOK,
+		},
+		{
+			name:     "non-existing",
+			path:     "/.well-known/non-existing",
+			expected: "Not found",
+			code:     http.StatusNotFound,
+		},
+		{
+			name:     "empty",
+			path:     "/.well-known/empty",
+			expected: "{}",
+			code:     http.StatusOK,
+		},
+	}
+
+	server := GetServer(wks)
+	for _, tc := range tt {
+		t.Run(tc.name, func(t *testing.T) {
+			req := httptest.NewRequest("GET", tc.path, nil)
+			w := httptest.NewRecorder()
+			server.ServeHTTP(w, req)
+
+			if w.Code != tc.code {
+				t.Errorf("Expected status code %d, got %d", tc.code, w.Code)
+			}
+
+			if w.Body.String() != tc.expected {
+				t.Errorf("Expected body %s, got %s", tc.expected, w.Body.String())
+			}
+
+		})
+	}
+}

server/wellknown.go

@@ -0,0 +1,163 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"os"
+	"time"
+
+	"github.com/bep/debounce"
+	"github.com/davegardnerisme/deephash"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+	klog "k8s.io/klog/v2"
+)
+
+var regLocal *wkRegistry
+
+type WellKnownService struct {
+	clientset *kubernetes.Clientset
+	namespace string
+	cmName    string
+
+	localCache *wkRegistry
+}
+
+func NewWellKnownService(clientset *kubernetes.Clientset, namespace string, cmName string) *WellKnownService {
+	return &WellKnownService{
+		clientset: clientset,
+		namespace: namespace,
+		cmName:    cmName,
+	}
+}
+
+func (s *WellKnownService) GetData(ctx context.Context) (*wkRegistry, error) {
+	if s.localCache != nil {
+		return s.localCache, nil
+	}
+
+	cm, err := s.clientset.CoreV1().ConfigMaps(s.namespace).Get(ctx, s.cmName, metav1.GetOptions{})
+	if errors.IsNotFound(err) {
+		return nil, nil
+	} else if err != nil {
+		return nil, err
+	}
+
+	reg := make(wkRegistry, 0)
+	for name, data := range cm.Data {
+		var d wkData
+		if err := json.Unmarshal([]byte(data), &d); err != nil {
+			klog.Error(err)
+		}
+		reg[name] = d
+	}
+
+	return &reg, nil
+}
+
+func (s *WellKnownService) UpdateConfigMap(ctx context.Context, reg wkRegistry) error {
+	s.localCache = &reg
+
+	cm := &v1.ConfigMap{Data: reg.encode()}
+	cm.Namespace = s.namespace
+	cm.Name = s.cmName
+
+	_, err := s.clientset.
+		CoreV1().
+		ConfigMaps(s.namespace).
+		Update(ctx, cm, metav1.UpdateOptions{})
+	if errors.IsNotFound(err) {
+		_, err = s.clientset.
+			CoreV1().
+			ConfigMaps(s.namespace).
+			Create(ctx, cm, metav1.CreateOptions{})
+		if err == nil {
+			klog.Infof("Created ConfigMap %s/%s\n", cm.GetNamespace(), cm.GetName())
+		}
+		return err
+	} else if err != nil {
+		klog.Error(err)
+		return err
+	}
+
+	klog.Infof("Updated ConfigMap %s/%s\n", cm.GetNamespace(), cm.GetName())
+	return nil
+}
+
+func (s *WellKnownService) DiscoveryLoop(ctx context.Context) {
+	watch, err := s.clientset.
+		CoreV1().
+		Services(s.namespace).
+		Watch(ctx, metav1.ListOptions{})
+	if err != nil {
+		klog.Error(err)
+		os.Exit(1)
+	}
+
+	debounced := debounce.New(500 * time.Millisecond)
+	hash := []byte{}
+
+	for event := range watch.ResultChan() {
+		svc, ok := event.Object.(*v1.Service)
+		if !ok {
+			continue
+		}
+		klog.V(1).Infof("Change detected on %s", svc.GetName())
+
+		debounced(func() {
+			reg, err := s.collectData(ctx)
+			if err != nil {
+				klog.Error(err)
+				return
+			}
+
+			newHash := deephash.Hash(reg)
+			if string(hash) == string(newHash) {
+				klog.V(1).Info("No changes detected")
+				return
+			}
+			hash = newHash
+
+			klog.Info("Writing configmap")
+			if err := s.UpdateConfigMap(ctx, reg); err != nil {
+				klog.Error(err)
+			}
+		})
+	}
+}
+
+func (s *WellKnownService) collectData(ctx context.Context) (wkRegistry, error) {
+	reg := make(wkRegistry, 0)
+
+	svcs, err := s.clientset.
+		CoreV1().
+		Services(s.namespace).
+		List(ctx, metav1.ListOptions{})
+	if err != nil {
+		return reg, err
+	}
+
+	for _, svc := range svcs.Items {
+		for name, value := range svc.ObjectMeta.Annotations {
+			name = resolveName(name)
+			if name == "" {
+				continue
+			}
+
+			if _, ok := reg[name]; !ok {
+				reg[name] = make(wkData, 0)
+			}
+
+			var d map[string]interface{}
+			err := json.Unmarshal([]byte(value), &d)
+			if err != nil {
+				klog.Error(err)
+			}
+
+			reg[name].append(d)
+		}
+	}
+	return reg, nil
+}