[Feature] Merge ArangoDB Usage Metrics

Author: ajanikow

CHANGELOG.md

@@ -8,6 +8,7 @@
 - (Bugfix) Fix Image Error Propagation
 - (Feature) JobScheduler Coverage
 - (Feature) JobScheduler Volumes, Probes, Lifecycle and Ports integration
+- (Feature) Merge ArangoDB Usage Metrics
 
 ## [1.2.38](https://github.com/arangodb/kube-arangodb/tree/1.2.38) (2024-02-22)
 - (Feature) Extract GRPC Server

cmd/exporter.go

@@ -1,7 +1,7 @@
 //
 // DISCLAIMER
 //
-// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany
+// Copyright 2016-2024 ArangoDB GmbH, Cologne, Germany
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -32,6 +32,7 @@ import (
 
 	"github.com/arangodb/kube-arangodb/pkg/exporter"
 	"github.com/arangodb/kube-arangodb/pkg/util"
+	"github.com/arangodb/kube-arangodb/pkg/util/errors"
 )
 
 var (
@@ -43,9 +44,9 @@ var (
 	exporterInput struct {
 		listenAddress string
 
-		endpoint string
-		jwtFile  string
-		timeout  time.Duration
+		endpoints []string
+		jwtFile   string
+		timeout   time.Duration
 
 		keyfile string
 	}
@@ -57,7 +58,7 @@ func init() {
 	f.StringVar(&exporterInput.listenAddress, "server.address", ":9101", "Address the exporter will listen on (IP:port)")
 	f.StringVar(&exporterInput.keyfile, "ssl.keyfile", "", "File containing TLS certificate used for the metrics server. Format equal to ArangoDB keyfiles")
 
-	f.StringVar(&exporterInput.endpoint, "arangodb.endpoint", "http://127.0.0.1:8529", "Endpoint used to reach the ArangoDB server")
+	f.StringSliceVar(&exporterInput.endpoints, "arangodb.endpoint", []string{"http://127.0.0.1:8529"}, "Endpoints used to reach the ArangoDB server")
 	f.StringVar(&exporterInput.jwtFile, "arangodb.jwt-file", "", "File containing the JWT for authentication with ArangoDB server")
 	f.DurationVar(&exporterInput.timeout, "arangodb.timeout", time.Second*15, "Timeout of statistics requests for ArangoDB")
 
@@ -83,7 +84,11 @@ func onSigterm(f func()) {
 }
 
 func cmdExporterCheckE() error {
-	p, err := exporter.NewPassthru(exporterInput.endpoint, func() (string, error) {
+	if len(exporterInput.endpoints) < 1 {
+		return errors.Errorf("Requires at least one ArangoDB Endpoint to be present")
+	}
+
+	p, err := exporter.NewPassthru(func() (string, error) {
 		if exporterInput.jwtFile == "" {
 			return "", nil
 		}
@@ -94,12 +99,12 @@ func cmdExporterCheckE() error {
 		}
 
 		return string(data), nil
-	}, false, 15*time.Second)
+	}, false, 15*time.Second, exporterInput.endpoints...)
 	if err != nil {
 		return err
 	}
 
-	mon := exporter.NewMonitor(exporterInput.endpoint, func() (string, error) {
+	mon := exporter.NewMonitor(exporterInput.endpoints[0], func() (string, error) {
 		if exporterInput.jwtFile == "" {
 			return "", nil
 		}

pkg/apis/deployment/v1/deployment_metrics_spec.go

@@ -105,6 +105,9 @@ type MetricsSpec struct {
 	ServiceMonitor *MetricsServiceMonitorSpec `json:"serviceMonitor,omitempty"`
 
 	Port *uint16 `json:"port,omitempty"`
+
+	// Extensions keeps the information about Metrics Extensions
+	Extensions *MetricsSpecExtensions `json:"extensions,omitempty"`
 }
 
 func (s *MetricsSpec) IsTLS() bool {
@@ -123,6 +126,14 @@ func (s *MetricsSpec) GetPort() uint16 {
 	return *s.Port
 }
 
+func (s *MetricsSpec) GetExtensions() *MetricsSpecExtensions {
+	if s == nil || s.Extensions == nil {
+		return nil
+	}
+
+	return s.Extensions
+}
+
 // IsEnabled returns whether metrics are enabled or not
 func (s *MetricsSpec) IsEnabled() bool {
 	return util.TypeOrDefault[bool](s.Enabled, false)

pkg/apis/deployment/v1/deployment_metrics_spec_extensions.go

@@ -0,0 +1,37 @@
+//
+// DISCLAIMER
+//
+// Copyright 2024 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+
+package v1
+
+// MetricsSpecExtensions defines enabled extensions for MetricsExporter
+type MetricsSpecExtensions struct {
+	// UsageMetrics enables ArangoDB Usage metrics scrape. Affects only DBServers in the Cluster mode.
+	// +doc/default: false
+	// +doc/link: Documentation:https://docs.arangodb.com/devel/develop/http-api/monitoring/metrics/#get-usage-metrics
+	UsageMetrics *bool `json:"usageMetrics,omitempty"`
+}
+
+func (m *MetricsSpecExtensions) GetUsageMetrics() bool {
+	if m == nil || m.UsageMetrics == nil {
+		return false
+	}
+
+	return *m.UsageMetrics
+}

pkg/apis/deployment/v1/zz_generated.deepcopy.go

@@ -31,6 +31,7 @@ const (
 	ArangoExporterClusterHealthEndpoint = "/_admin/cluster/health"
 	ArangoExporterInternalEndpoint      = "/_admin/metrics"
 	ArangoExporterInternalEndpointV2    = "/_admin/metrics/v2"
+	ArangoExporterUsageEndpoint         = "/_admin/usage-metrics"
 	ArangoExporterDefaultEndpoint       = "/metrics"
 
 	ArangoSyncStatusEndpoint = "/_api/version"

pkg/apis/shared/constants.go

@@ -1,5 +1,5 @@
 //
-// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany
+// Copyright 2016-2024 ArangoDB GmbH, Cologne, Germany
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -30,7 +30,7 @@ import (
 	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil/probes"
 )
 
-func createInternalExporterArgs(spec api.DeploymentSpec, groupSpec api.ServerGroupSpec, version driver.Version) []string {
+func createInternalExporterArgs(spec api.DeploymentSpec, group api.ServerGroup, groupSpec api.ServerGroupSpec, version driver.Version) []string {
 	tokenpath := filepath.Join(shared.ExporterJWTVolumeMountDir, constants.SecretKeyToken)
 	options := k8sutil.CreateOptionPairs(64)
 
@@ -46,8 +46,14 @@ func createInternalExporterArgs(spec api.DeploymentSpec, groupSpec api.ServerGro
 			scheme = "https"
 		}
 		options.Addf("--arangodb.endpoint", "%s://localhost:%d%s", scheme, groupSpec.GetPort(), path)
+		if spec.Metrics.GetExtensions().GetUsageMetrics() && group == api.ServerGroupDBServers {
+			options.Addf("--arangodb.endpoint", "%s://localhost:%d%s", scheme, groupSpec.GetPort(), shared.ArangoExporterUsageEndpoint)
+		}
 	} else {
 		options.Addf("--arangodb.endpoint", "http://localhost:%d%s", *port, path)
+		if spec.Metrics.GetExtensions().GetUsageMetrics() && group == api.ServerGroupDBServers {
+			options.Addf("--arangodb.endpoint", "http://localhost:%d%s", groupSpec.GetPort(), shared.ArangoExporterUsageEndpoint)
+		}
 	}
 
 	keyPath := filepath.Join(shared.TLSKeyfileVolumeMountDir, constants.SecretTLSKeyfile)

pkg/deployment/resources/exporter.go

@@ -1,5 +1,5 @@
 //
-// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany
+// Copyright 2016-2024 ArangoDB GmbH, Cologne, Germany
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -50,7 +50,7 @@ func ArangodbInternalExporterContainer(image string, args []string, livenessProb
 		Command: append([]string{exePath, "exporter"}, args...),
 		Ports: []core.ContainerPort{
 			{
-				Name:          "exporter",
+				Name:          shared.ExporterContainerName,
 				ContainerPort: int32(port),
 				Protocol:      core.ProtocolTCP,
 			},

pkg/deployment/resources/internal_exporter.go

@@ -561,7 +561,7 @@ func (m *MemberArangoDPod) GetRestartPolicy() core.RestartPolicy {
 func (m *MemberArangoDPod) createMetricsExporterSidecarInternalExporter() (*core.Container, error) {
 	image := m.GetContainerCreator().GetImage()
 
-	args := createInternalExporterArgs(m.spec, m.groupSpec, m.imageInfo.ArangoDBVersion)
+	args := createInternalExporterArgs(m.spec, m.group, m.groupSpec, m.imageInfo.ArangoDBVersion)
 
 	c, err := ArangodbInternalExporterContainer(image, args,
 		createExporterLivenessProbe(m.spec.IsSecure() && m.spec.Metrics.IsTLS()), m.spec.Metrics.Resources,

pkg/deployment/resources/pod_creator_arangod.go

@@ -426,8 +426,11 @@ var (
 		token.ClaimISS: token.ClaimISSValue,
 		"server_id":    "exporter",
 		"allowed_paths": []interface{}{"/_admin/statistics", "/_admin/statistics-description",
-			shared.ArangoExporterInternalEndpoint, shared.ArangoExporterInternalEndpointV2,
-			shared.ArangoExporterStatusEndpoint, shared.ArangoExporterClusterHealthEndpoint},
+			shared.ArangoExporterInternalEndpoint,
+			shared.ArangoExporterInternalEndpointV2,
+			shared.ArangoExporterUsageEndpoint,
+			shared.ArangoExporterStatusEndpoint,
+			shared.ArangoExporterClusterHealthEndpoint},
 	}
 )
 
@@ -449,6 +452,15 @@ func (r *Resources) ensureExporterTokenSecret(ctx context.Context, cachedStatus
 				// Failed to create secret
 				return errors.WithStack(err)
 			}
+		} else {
+			err = k8sutil.UpdateJWTFromSecret(ctx, cachedStatus.Secret().V1().Read(), secrets, tokenSecretName, secretSecretName, exporterTokenClaims)
+			if kerrors.IsAlreadyExists(err) {
+				// Secret added while we tried it also
+				return nil
+			} else if err != nil {
+				// Failed to create secret
+				return errors.WithStack(err)
+			}
 		}
 
 		return errors.Reconcile()

pkg/deployment/resources/secrets.go

@@ -42,7 +42,7 @@ import (
 )
 
 const (
-	monitorMetricTemplate  = "arangodb_member_health{role=\"%s\",id=\"%s\"} %d \n"
+	monitorMetricTemplate  = "arangodb_member_health{role=\"%s\",id=\"%s\"} %d\n"
 	successRefreshInterval = time.Second * 120
 	failRefreshInterval    = time.Second * 15
 )
@@ -59,14 +59,16 @@ func NewMonitor(arangodbEndpoint string, auth Authentication, sslVerify bool, ti
 	}
 
 	return &monitor{
-		factory:   newHttpClientFactory(arangodbEndpoint, auth, sslVerify, timeout),
+		factory:   newHttpClientFactory(auth, sslVerify, timeout),
+		endpoint:  arangodbEndpoint,
 		healthURI: uri,
 	}
 }
 
 type monitor struct {
 	factory   httpClientFactory
 	healthURI *url.URL
+	endpoint  string
 }
 
 // UpdateMonitorStatus load monitor metrics for current cluster into currentMembersStatus
@@ -102,7 +104,7 @@ func (m monitor) UpdateMonitorStatus(ctx context.Context) {
 
 // GetClusterHealth returns current ArangoDeployment cluster health status
 func (m monitor) GetClusterHealth() (*driver.ClusterHealth, error) {
-	c, req, err := m.factory()
+	c, req, err := m.factory(m.endpoint)
 	if err != nil {
 		return nil, err
 	}
@@ -130,7 +132,7 @@ func (m monitor) GetClusterHealth() (*driver.ClusterHealth, error) {
 func (m monitor) GetMemberStatus(id driver.ServerID, member driver.ServerHealth) (string, error) {
 	result := fmt.Sprintf(monitorMetricTemplate, member.Role, id, 0)
 
-	c, req, err := m.factory()
+	c, req, err := m.factory(m.endpoint)
 	if err != nil {
 		return result, err
 	}