[Feature] Merge ArangoDB Usage Metrics (#1608)

CHANGELOG.md

@@ -8,6 +8,7 @@
 - (Bugfix) Fix Image Error Propagation
 - (Feature) JobScheduler Coverage
 - (Feature) JobScheduler Volumes, Probes, Lifecycle and Ports integration
+- (Feature) Merge ArangoDB Usage Metrics
 
 ## [1.2.38](https://github.com/arangodb/kube-arangodb/tree/1.2.38) (2024-02-22)
 - (Feature) Extract GRPC Server

cmd/exporter.go

@@ -1,7 +1,7 @@
 //
 // DISCLAIMER
 //
-// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+// Copyright 2016-2024 ArangoDB GmbH, Cologne, Germany
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -32,6 +32,7 @@ import (
 
 	"github.com/arangodb/kube-arangodb/pkg/exporter"
 	"github.com/arangodb/kube-arangodb/pkg/util"
+	"github.com/arangodb/kube-arangodb/pkg/util/errors"
 )
 
 var (
@@ -43,9 +44,9 @@ var (
 	exporterInput struct {
 		listenAddress string
 
-		endpoint string
-		jwtFile  string
-		timeout  time.Duration
+		endpoints []string
+		jwtFile   string
+		timeout   time.Duration
 
 		keyfile string
 	}
@@ -57,7 +58,7 @@ func init() {
 	f.StringVar(&exporterInput.listenAddress, "server.address", ":9101", "Address the exporter will listen on (IP:port)")
 	f.StringVar(&exporterInput.keyfile, "ssl.keyfile", "", "File containing TLS certificate used for the metrics server. Format equal to ArangoDB keyfiles")
 
-	f.StringVar(&exporterInput.endpoint, "arangodb.endpoint", "http://127.0.0.1:8529", "Endpoint used to reach the ArangoDB server")
+	f.StringSliceVar(&exporterInput.endpoints, "arangodb.endpoint", []string{"http://127.0.0.1:8529"}, "Endpoints used to reach the ArangoDB server")
 	f.StringVar(&exporterInput.jwtFile, "arangodb.jwt-file", "", "File containing the JWT for authentication with ArangoDB server")
 	f.DurationVar(&exporterInput.timeout, "arangodb.timeout", time.Second*15, "Timeout of statistics requests for ArangoDB")
 
@@ -83,7 +84,11 @@ func onSigterm(f func()) {
 }
 
 func cmdExporterCheckE() error {
-	p, err := exporter.NewPassthru(exporterInput.endpoint, func() (string, error) {
+	if len(exporterInput.endpoints) < 1 {
+		return errors.Errorf("Requires at least one ArangoDB Endpoint to be present")
+	}
+
+	p, err := exporter.NewPassthru(func() (string, error) {
 		if exporterInput.jwtFile == "" {
 			return "", nil
 		}
@@ -94,12 +99,12 @@ func cmdExporterCheckE() error {
 		}
 
 		return string(data), nil
-	}, false, 15*time.Second)
+	}, false, 15*time.Second, exporterInput.endpoints...)
 	if err != nil {
 		return err
 	}
 
-	mon := exporter.NewMonitor(exporterInput.endpoint, func() (string, error) {
+	mon := exporter.NewMonitor(exporterInput.endpoints[0], func() (string, error) {
 		if exporterInput.jwtFile == "" {
 			return "", nil
 		}

docs/api/ArangoDeployment.V1.md

@@ -3405,6 +3405,22 @@ Default Value: `false`
 
 ***
 
+### .spec.metrics.extensions.usageMetrics
+
+Type: `boolean` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.38/pkg/apis/deployment/v1/deployment_metrics_spec_extensions.go#L29)</sup>
+
+> [!IMPORTANT]
+> **UsageMetrics needs to be also enabled via DBServer Arguments**
+
+UsageMetrics enables ArangoDB Usage metrics scrape. Affects only DBServers in the Cluster mode.
+
+Links:
+* [Documentation](https://docs.arangodb.com/devel/develop/http-api/monitoring/metrics/#get-usage-metrics)
+
+Default Value: `false`
+
+***
+
 ### .spec.metrics.image
 
 Type: `string` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.38/pkg/apis/deployment/v1/deployment_metrics_spec.go#L86)</sup>

pkg/apis/deployment/v1/deployment_metrics_spec.go

@@ -105,6 +105,9 @@ type MetricsSpec struct {
 	ServiceMonitor *MetricsServiceMonitorSpec `json:"serviceMonitor,omitempty"`
 
 	Port *uint16 `json:"port,omitempty"`
+
+	// Extensions keeps the information about Metrics Extensions
+	Extensions *MetricsSpecExtensions `json:"extensions,omitempty"`
 }
 
 func (s *MetricsSpec) IsTLS() bool {
@@ -123,6 +126,14 @@ func (s *MetricsSpec) GetPort() uint16 {
 	return *s.Port
 }
 
+func (s *MetricsSpec) GetExtensions() *MetricsSpecExtensions {
+	if s == nil || s.Extensions == nil {
+		return nil
+	}
+
+	return s.Extensions
+}
+
 // IsEnabled returns whether metrics are enabled or not
 func (s *MetricsSpec) IsEnabled() bool {
 	return util.TypeOrDefault[bool](s.Enabled, false)

pkg/apis/deployment/v1/deployment_metrics_spec_extensions.go

@@ -0,0 +1,38 @@
+//
+// DISCLAIMER
+//
+// Copyright 2024 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package v1
+
+// MetricsSpecExtensions defines enabled extensions for MetricsExporter
+type MetricsSpecExtensions struct {
+	// UsageMetrics enables ArangoDB Usage metrics scrape. Affects only DBServers in the Cluster mode.
+	// +doc/default: false
+	// +doc/link: Documentation|https://docs.arangodb.com/devel/develop/http-api/monitoring/metrics/#get-usage-metrics
+	// +doc/important: UsageMetrics needs to be also enabled via DBServer Arguments
+	UsageMetrics *bool `json:"usageMetrics,omitempty"`
+}
+
+func (m *MetricsSpecExtensions) GetUsageMetrics() bool {
+	if m == nil || m.UsageMetrics == nil {
+		return false
+	}
+
+	return *m.UsageMetrics
+}

pkg/apis/deployment/v1/zz_generated.deepcopy.go

@@ -1958,6 +1958,11 @@ func (in *MetricsSpec) DeepCopyInto(out *MetricsSpec) {
 		*out = new(uint16)
 		**out = **in
 	}
+	if in.Extensions != nil {
+		in, out := &in.Extensions, &out.Extensions
+		*out = new(MetricsSpecExtensions)
+		(*in).DeepCopyInto(*out)
+	}
 	return
 }
 
@@ -1971,6 +1976,27 @@ func (in *MetricsSpec) DeepCopy() *MetricsSpec {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *MetricsSpecExtensions) DeepCopyInto(out *MetricsSpecExtensions) {
+	*out = *in
+	if in.UsageMetrics != nil {
+		in, out := &in.UsageMetrics, &out.UsageMetrics
+		*out = new(bool)
+		**out = **in
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MetricsSpecExtensions.
+func (in *MetricsSpecExtensions) DeepCopy() *MetricsSpecExtensions {
+	if in == nil {
+		return nil
+	}
+	out := new(MetricsSpecExtensions)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *MonitoringSpec) DeepCopyInto(out *MonitoringSpec) {
 	*out = *in

pkg/apis/deployment/v2alpha1/deployment_metrics_spec.go

@@ -105,6 +105,9 @@ type MetricsSpec struct {
 	ServiceMonitor *MetricsServiceMonitorSpec `json:"serviceMonitor,omitempty"`
 
 	Port *uint16 `json:"port,omitempty"`
+
+	// Extensions keeps the information about Metrics Extensions
+	Extensions *MetricsSpecExtensions `json:"extensions,omitempty"`
 }
 
 func (s *MetricsSpec) IsTLS() bool {
@@ -123,6 +126,14 @@ func (s *MetricsSpec) GetPort() uint16 {
 	return *s.Port
 }
 
+func (s *MetricsSpec) GetExtensions() *MetricsSpecExtensions {
+	if s == nil || s.Extensions == nil {
+		return nil
+	}
+
+	return s.Extensions
+}
+
 // IsEnabled returns whether metrics are enabled or not
 func (s *MetricsSpec) IsEnabled() bool {
 	return util.TypeOrDefault[bool](s.Enabled, false)

pkg/apis/deployment/v2alpha1/deployment_metrics_spec_extensions.go

@@ -0,0 +1,38 @@
+//
+// DISCLAIMER
+//
+// Copyright 2024 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package v2alpha1
+
+// MetricsSpecExtensions defines enabled extensions for MetricsExporter
+type MetricsSpecExtensions struct {
+	// UsageMetrics enables ArangoDB Usage metrics scrape. Affects only DBServers in the Cluster mode.
+	// +doc/default: false
+	// +doc/link: Documentation|https://docs.arangodb.com/devel/develop/http-api/monitoring/metrics/#get-usage-metrics
+	// +doc/important: UsageMetrics needs to be also enabled via DBServer Arguments
+	UsageMetrics *bool `json:"usageMetrics,omitempty"`
+}
+
+func (m *MetricsSpecExtensions) GetUsageMetrics() bool {
+	if m == nil || m.UsageMetrics == nil {
+		return false
+	}
+
+	return *m.UsageMetrics
+}

pkg/apis/deployment/v2alpha1/zz_generated.deepcopy.go

@@ -1958,6 +1958,11 @@ func (in *MetricsSpec) DeepCopyInto(out *MetricsSpec) {
 		*out = new(uint16)
 		**out = **in
 	}
+	if in.Extensions != nil {
+		in, out := &in.Extensions, &out.Extensions
+		*out = new(MetricsSpecExtensions)
+		(*in).DeepCopyInto(*out)
+	}
 	return
 }
 
@@ -1971,6 +1976,27 @@ func (in *MetricsSpec) DeepCopy() *MetricsSpec {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *MetricsSpecExtensions) DeepCopyInto(out *MetricsSpecExtensions) {
+	*out = *in
+	if in.UsageMetrics != nil {
+		in, out := &in.UsageMetrics, &out.UsageMetrics
+		*out = new(bool)
+		**out = **in
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MetricsSpecExtensions.
+func (in *MetricsSpecExtensions) DeepCopy() *MetricsSpecExtensions {
+	if in == nil {
+		return nil
+	}
+	out := new(MetricsSpecExtensions)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *MonitoringSpec) DeepCopyInto(out *MonitoringSpec) {
 	*out = *in

pkg/apis/shared/constants.go

@@ -31,6 +31,7 @@ const (
 	ArangoExporterClusterHealthEndpoint = "/_admin/cluster/health"
 	ArangoExporterInternalEndpoint      = "/_admin/metrics"
 	ArangoExporterInternalEndpointV2    = "/_admin/metrics/v2"
+	ArangoExporterUsageEndpoint         = "/_admin/usage-metrics"
 	ArangoExporterDefaultEndpoint       = "/metrics"
 
 	ArangoSyncStatusEndpoint = "/_api/version"

pkg/crd/crds/database-deployment.schema.generated.yaml

@@ -6917,6 +6917,13 @@ v1:
                   Enabled if this is set to `true`, the operator runs a sidecar container for
                   every Agent, DB-Server, Coordinator and Single server.
                 type: boolean
+              extensions:
+                description: Extensions keeps the information about Metrics Extensions
+                properties:
+                  usageMetrics:
+                    description: UsageMetrics enables ArangoDB Usage metrics scrape. Affects only DBServers in the Cluster mode.
+                    type: boolean
+                type: object
               image:
                 description: Image used for the Metrics Sidecar
                 type: string
@@ -20398,6 +20405,13 @@ v1alpha:
                   Enabled if this is set to `true`, the operator runs a sidecar container for
                   every Agent, DB-Server, Coordinator and Single server.
                 type: boolean
+              extensions:
+                description: Extensions keeps the information about Metrics Extensions
+                properties:
+                  usageMetrics:
+                    description: UsageMetrics enables ArangoDB Usage metrics scrape. Affects only DBServers in the Cluster mode.
+                    type: boolean
+                type: object
               image:
                 description: Image used for the Metrics Sidecar
                 type: string
@@ -33879,6 +33893,13 @@ v2alpha1:
                   Enabled if this is set to `true`, the operator runs a sidecar container for
                   every Agent, DB-Server, Coordinator and Single server.
                 type: boolean
+              extensions:
+                description: Extensions keeps the information about Metrics Extensions
+                properties:
+                  usageMetrics:
+                    description: UsageMetrics enables ArangoDB Usage metrics scrape. Affects only DBServers in the Cluster mode.
+                    type: boolean
+                type: object
               image:
                 description: Image used for the Metrics Sidecar
                 type: string

pkg/deployment/resources/exporter.go

@@ -1,5 +1,5 @@
 //
-// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany
+// Copyright 2016-2024 ArangoDB GmbH, Cologne, Germany
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -30,7 +30,7 @@ import (
 	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil/probes"
 )
 
-func createInternalExporterArgs(spec api.DeploymentSpec, groupSpec api.ServerGroupSpec, version driver.Version) []string {
+func createInternalExporterArgs(spec api.DeploymentSpec, group api.ServerGroup, groupSpec api.ServerGroupSpec, version driver.Version) []string {
 	tokenpath := filepath.Join(shared.ExporterJWTVolumeMountDir, constants.SecretKeyToken)
 	options := k8sutil.CreateOptionPairs(64)
 
@@ -46,8 +46,14 @@ func createInternalExporterArgs(spec api.DeploymentSpec, groupSpec api.ServerGro
 			scheme = "https"
 		}
 		options.Addf("--arangodb.endpoint", "%s://localhost:%d%s", scheme, groupSpec.GetPort(), path)
+		if spec.Metrics.GetExtensions().GetUsageMetrics() && group == api.ServerGroupDBServers {
+			options.Addf("--arangodb.endpoint", "%s://localhost:%d%s", scheme, groupSpec.GetPort(), shared.ArangoExporterUsageEndpoint)
+		}
 	} else {
 		options.Addf("--arangodb.endpoint", "http://localhost:%d%s", *port, path)
+		if spec.Metrics.GetExtensions().GetUsageMetrics() && group == api.ServerGroupDBServers {
+			options.Addf("--arangodb.endpoint", "http://localhost:%d%s", groupSpec.GetPort(), shared.ArangoExporterUsageEndpoint)
+		}
 	}
 
 	keyPath := filepath.Join(shared.TLSKeyfileVolumeMountDir, constants.SecretTLSKeyfile)

pkg/deployment/resources/internal_exporter.go

@@ -1,5 +1,5 @@
 //
-// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany
+// Copyright 2016-2024 ArangoDB GmbH, Cologne, Germany
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -50,7 +50,7 @@ func ArangodbInternalExporterContainer(image string, args []string, livenessProb
 		Command: append([]string{exePath, "exporter"}, args...),
 		Ports: []core.ContainerPort{
 			{
-				Name:          "exporter",
+				Name:          shared.ExporterContainerName,
 				ContainerPort: int32(port),
 				Protocol:      core.ProtocolTCP,
 			},

pkg/deployment/resources/pod_creator_arangod.go

@@ -561,7 +561,7 @@ func (m *MemberArangoDPod) GetRestartPolicy() core.RestartPolicy {
 func (m *MemberArangoDPod) createMetricsExporterSidecarInternalExporter() (*core.Container, error) {
 	image := m.GetContainerCreator().GetImage()
 
-	args := createInternalExporterArgs(m.spec, m.groupSpec, m.imageInfo.ArangoDBVersion)
+	args := createInternalExporterArgs(m.spec, m.group, m.groupSpec, m.imageInfo.ArangoDBVersion)
 
 	c, err := ArangodbInternalExporterContainer(image, args,
 		createExporterLivenessProbe(m.spec.IsSecure() && m.spec.Metrics.IsTLS()), m.spec.Metrics.Resources,

pkg/deployment/resources/secrets.go

@@ -426,8 +426,11 @@ var (
 		token.ClaimISS: token.ClaimISSValue,
 		"server_id":    "exporter",
 		"allowed_paths": []interface{}{"/_admin/statistics", "/_admin/statistics-description",
-			shared.ArangoExporterInternalEndpoint, shared.ArangoExporterInternalEndpointV2,
-			shared.ArangoExporterStatusEndpoint, shared.ArangoExporterClusterHealthEndpoint},
+			shared.ArangoExporterInternalEndpoint,
+			shared.ArangoExporterInternalEndpointV2,
+			shared.ArangoExporterUsageEndpoint,
+			shared.ArangoExporterStatusEndpoint,
+			shared.ArangoExporterClusterHealthEndpoint},
 	}
 )
 
@@ -449,6 +452,15 @@ func (r *Resources) ensureExporterTokenSecret(ctx context.Context, cachedStatus
 				// Failed to create secret
 				return errors.WithStack(err)
 			}
+		} else {
+			err = k8sutil.UpdateJWTFromSecret(ctx, cachedStatus.Secret().V1().Read(), secrets, tokenSecretName, secretSecretName, exporterTokenClaims)
+			if kerrors.IsAlreadyExists(err) {
+				// Secret added while we tried it also
+				return nil
+			} else if err != nil {
+				// Failed to create secret
+				return errors.WithStack(err)
+			}
 		}
 
 		return errors.Reconcile()

pkg/exporter/monitor.go

@@ -42,7 +42,7 @@ import (
 )
 
 const (
-	monitorMetricTemplate  = "arangodb_member_health{role=\"%s\",id=\"%s\"} %d \n"
+	monitorMetricTemplate  = "arangodb_member_health{role=\"%s\",id=\"%s\"} %d\n"
 	successRefreshInterval = time.Second * 120
 	failRefreshInterval    = time.Second * 15
 )
@@ -59,7 +59,8 @@ func NewMonitor(arangodbEndpoint string, auth Authentication, sslVerify bool, ti
 	}
 
 	return &monitor{
-		factory:   newHttpClientFactory(arangodbEndpoint, auth, sslVerify, timeout),
+		factory:   newHttpClientFactory(auth, sslVerify, timeout),
+		endpoint:  arangodbEndpoint,
 		healthURI: uri,
 	}
 }
@@ -67,6 +68,7 @@ func NewMonitor(arangodbEndpoint string, auth Authentication, sslVerify bool, ti
 type monitor struct {
 	factory   httpClientFactory
 	healthURI *url.URL
+	endpoint  string
 }
 
 // UpdateMonitorStatus load monitor metrics for current cluster into currentMembersStatus
@@ -102,7 +104,7 @@ func (m monitor) UpdateMonitorStatus(ctx context.Context) {
 
 // GetClusterHealth returns current ArangoDeployment cluster health status
 func (m monitor) GetClusterHealth() (*driver.ClusterHealth, error) {
-	c, req, err := m.factory()
+	c, req, err := m.factory(m.endpoint)
 	if err != nil {
 		return nil, err
 	}
@@ -130,7 +132,7 @@ func (m monitor) GetClusterHealth() (*driver.ClusterHealth, error) {
 func (m monitor) GetMemberStatus(id driver.ServerID, member driver.ServerHealth) (string, error) {
 	result := fmt.Sprintf(monitorMetricTemplate, member.Role, id, 0)
 
-	c, req, err := m.factory()
+	c, req, err := m.factory(m.endpoint)
 	if err != nil {
 		return result, err
 	}

pkg/exporter/passthru.go

@@ -1,7 +1,7 @@
 //
 // DISCLAIMER
 //
-// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+// Copyright 2016-2024 ArangoDB GmbH, Cologne, Germany
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -25,6 +25,7 @@ import (
 	"io"
 	"net/http"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/arangodb/kube-arangodb/pkg/util/errors"
@@ -32,19 +33,20 @@ import (
 
 var _ http.Handler = &passthru{}
 
-func NewPassthru(arangodbEndpoint string, auth Authentication, sslVerify bool, timeout time.Duration) (http.Handler, error) {
+func NewPassthru(auth Authentication, sslVerify bool, timeout time.Duration, endpoints ...string) (http.Handler, error) {
 	return &passthru{
-		factory: newHttpClientFactory(arangodbEndpoint, auth, sslVerify, timeout),
+		factory:   newHttpClientFactory(auth, sslVerify, timeout),
+		endpoints: endpoints,
 	}, nil
 }
 
-type httpClientFactory func() (*http.Client, *http.Request, error)
+type httpClientFactory func(endpoint string) (*http.Client, *http.Request, error)
 
-func newHttpClientFactory(arangodbEndpoint string, auth Authentication, sslVerify bool, timeout time.Duration) httpClientFactory {
-	return func() (*http.Client, *http.Request, error) {
+func newHttpClientFactory(auth Authentication, sslVerify bool, timeout time.Duration) httpClientFactory {
+	return func(endpoint string) (*http.Client, *http.Request, error) {
 		transport := &http.Transport{}
 
-		req, err := http.NewRequest("GET", arangodbEndpoint, nil)
+		req, err := http.NewRequest("GET", endpoint, nil)
 		if err != nil {
 			return nil, nil, errors.WithStack(err)
 		}
@@ -78,37 +80,79 @@ func newHttpClientFactory(arangodbEndpoint string, auth Authentication, sslVerif
 }
 
 type passthru struct {
-	factory httpClientFactory
+	endpoints []string
+	factory   httpClientFactory
 }
 
-func (p passthru) get() (*http.Response, error) {
-	c, req, err := p.factory()
+func (p passthru) get(endpoint string) (*http.Response, error) {
+	c, req, err := p.factory(endpoint)
 	if err != nil {
 		return nil, err
 	}
 	return c.Do(req)
 }
 
-func (p passthru) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
-	data, err := p.get()
+func (p passthru) read(endpoint string) (string, error) {
+	data, err := p.get(endpoint)
 
 	if err != nil {
-		// Ignore error
-		resp.WriteHeader(http.StatusInternalServerError)
-		resp.Write([]byte(err.Error()))
-		return
+		return "", err
 	}
 
 	if data.Body == nil {
-		// Ignore error
-		resp.WriteHeader(http.StatusInternalServerError)
-		resp.Write([]byte("Body is empty"))
-		return
+		return "", err
 	}
 
 	defer data.Body.Close()
 
 	response, err := io.ReadAll(data.Body)
+	if err != nil {
+		return "", err
+	}
+
+	responseStr := string(response)
+
+	// Fix Header response
+	return strings.ReplaceAll(responseStr, "guage", "gauge"), nil
+}
+
+func (p passthru) getAll() (string, error) {
+	errs := make([]error, len(p.endpoints))
+	responses := make([]string, len(p.endpoints))
+
+	var wg sync.WaitGroup
+
+	for id := range p.endpoints {
+		wg.Add(1)
+
+		go func(id int) {
+			defer wg.Done()
+			responses[id], errs[id] = p.read(p.endpoints[id])
+		}(id)
+	}
+
+	wg.Wait()
+
+	for _, err := range errs {
+		if err != nil {
+			return "", errors.WithStack(err)
+		}
+	}
+
+	response := strings.Join(responses, "\n")
+
+	// Attach monitor data
+	monitorData := currentMembersStatus.Load()
+	if monitorData != nil {
+		response = response + monitorData.(string)
+	}
+
+	return response, nil
+}
+
+func (p passthru) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+	response, err := p.getAll()
+
 	if err != nil {
 		// Ignore error
 		resp.WriteHeader(http.StatusInternalServerError)
@@ -116,19 +160,8 @@ func (p passthru) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
 		return
 	}
 
-	responseStr := string(response)
-
-	// Fix Header response
-	responseStr = strings.ReplaceAll(responseStr, "guage", "gauge")
-
-	// Attach monitor data
-	monitorData := currentMembersStatus.Load()
-	if monitorData != nil {
-		responseStr = responseStr + monitorData.(string)
-	}
-
-	resp.WriteHeader(data.StatusCode)
-	_, err = resp.Write([]byte(responseStr))
+	resp.WriteHeader(http.StatusOK)
+	_, err = resp.Write([]byte(response))
 	if err != nil {
 		// Ignore error
 		resp.WriteHeader(http.StatusInternalServerError)

pkg/util/k8sutil/secrets.go

@@ -295,6 +295,19 @@ func CreateTokenSecret(ctx context.Context, secrets secretv1.ModInterface, secre
 	return nil
 }
 
+// UpdateTokenSecret updates a secret with given name in given namespace
+// with a given token as value.
+func UpdateTokenSecret(ctx context.Context, secrets secretv1.ModInterface, secret *core.Secret, token string) error {
+	secret.Data = map[string][]byte{
+		constants.SecretKeyToken: []byte(token),
+	}
+	if _, err := secrets.Update(ctx, secret, meta.UpdateOptions{}); err != nil {
+		// Failed to update secret
+		return kerrors.NewResourceError(err, secret)
+	}
+	return nil
+}
+
 // CreateJWTFromSecret creates a JWT using the secret stored in secretSecretName and stores the
 // result in a new secret called tokenSecretName
 func CreateJWTFromSecret(ctx context.Context, cachedSecrets secretv1.ReadInterface, secrets secretv1.ModInterface, tokenSecretName, secretSecretName string, claims map[string]interface{}, ownerRef *meta.OwnerReference) error {
@@ -313,6 +326,29 @@ func CreateJWTFromSecret(ctx context.Context, cachedSecrets secretv1.ReadInterfa
 	})
 }
 
+// UpdateJWTFromSecret updates a JWT using the secret stored in secretSecretName and stores the
+// result in a new secret called tokenSecretName
+func UpdateJWTFromSecret(ctx context.Context, cachedSecrets secretv1.ReadInterface, secrets secretv1.ModInterface, tokenSecretName, secretSecretName string, claims map[string]interface{}) error {
+	current, err := cachedSecrets.Get(ctx, tokenSecretName, meta.GetOptions{})
+	if err != nil {
+		return errors.WithStack(err)
+	}
+
+	secret, err := GetTokenSecret(ctx, cachedSecrets, secretSecretName)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+
+	signedToken, err := token.New([]byte(secret), claims)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+
+	return globals.GetGlobalTimeouts().Kubernetes().RunWithTimeout(ctx, func(ctxChild context.Context) error {
+		return UpdateTokenSecret(ctxChild, secrets, current, signedToken)
+	})
+}
+
 // CreateBasicAuthSecret creates a secret with given name in given namespace
 // with a given username and password as value.
 func CreateBasicAuthSecret(ctx context.Context, secrets secretv1.ModInterface, secretName, username, password string,