Skip to content

Commit d9f0570

Browse files
committed
Merge branch 'master' into tests/multi-deployment
2 parents 9225509 + 2e5b21f commit d9f0570

File tree

9 files changed

+251
-16
lines changed

9 files changed

+251
-16
lines changed
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,10 @@
11
# Metrics
22

3-
TBD
3+
The ArangoDB Kubernetes Operator (`kube-arangodb`) exposes metrics of
4+
its operations in a format that is compatible with [Prometheus](https://prometheus.io).
5+
6+
The metrics are exposed through HTTPS on port `8528` under path `/metrics`.
7+
8+
Look at [examples/metrics](https://github.com/arangodb/kube-arangodb/tree/master/examples/metrics)
9+
for examples of `Services` and `ServiceMonitors` you can use to integrate
10+
with Prometheus through the [Prometheus-Operator by CoreOS](https://github.com/coreos/prometheus-operator).
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
# This example shows how to integrate with the Prometheus Operator
2+
# to bring metrics from kube-arangodb to Prometheus.
3+
4+
apiVersion: v1
5+
kind: Service
6+
metadata:
7+
name: arango-deployment-operator
8+
labels:
9+
app: arango-deployment-operator
10+
spec:
11+
selector:
12+
app: arango-deployment-operator
13+
ports:
14+
- name: metrics
15+
port: 8528
16+
17+
---
18+
19+
apiVersion: monitoring.coreos.com/v1
20+
kind: ServiceMonitor
21+
metadata:
22+
name: arango-deployment-operator
23+
labels:
24+
team: frontend
25+
spec:
26+
selector:
27+
matchLabels:
28+
app: arango-deployment-operator
29+
endpoints:
30+
- port: metrics
31+
scheme: https
32+
tlsConfig:
33+
insecureSkipVerify: true
34+

main.go

+38-14
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ import (
4646
"github.com/arangodb/kube-arangodb/pkg/client"
4747
"github.com/arangodb/kube-arangodb/pkg/logging"
4848
"github.com/arangodb/kube-arangodb/pkg/operator"
49+
"github.com/arangodb/kube-arangodb/pkg/server"
4950
"github.com/arangodb/kube-arangodb/pkg/util/constants"
5051
"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
5152
"github.com/arangodb/kube-arangodb/pkg/util/probe"
@@ -69,12 +70,13 @@ var (
6970
Run: cmdMainRun,
7071
}
7172

72-
logLevel string
73-
cliLog = logging.NewRootLogger()
74-
logService logging.Service
75-
server struct {
76-
host string
77-
port int
73+
logLevel string
74+
cliLog = logging.NewRootLogger()
75+
logService logging.Service
76+
serverOptions struct {
77+
host string
78+
port int
79+
tlsSecretName string
7880
}
7981
operatorOptions struct {
8082
enableDeployment bool // Run deployment operator
@@ -89,8 +91,9 @@ var (
8991

9092
func init() {
9193
f := cmdMain.Flags()
92-
f.StringVar(&server.host, "server.host", defaultServerHost, "Host to listen on")
93-
f.IntVar(&server.port, "server.port", defaultServerPort, "Port to listen on")
94+
f.StringVar(&serverOptions.host, "server.host", defaultServerHost, "Host to listen on")
95+
f.IntVar(&serverOptions.port, "server.port", defaultServerPort, "Port to listen on")
96+
f.StringVar(&serverOptions.tlsSecretName, "server.tls-secret-name", "", "Name of secret containing tls.crt & tls.key for HTTPS server (if empty, self-signed certificate is used)")
9497
f.StringVar(&logLevel, "log.level", defaultLogLevel, "Set initial log level")
9598
f.BoolVar(&operatorOptions.enableDeployment, "operator.deployment", false, "Enable to run the ArangoDeployment operator")
9699
f.BoolVar(&operatorOptions.enableStorage, "operator.storage", false, "Enable to run the ArangoLocalStorage operator")
@@ -133,19 +136,40 @@ func cmdMainRun(cmd *cobra.Command, args []string) {
133136
if len(name) == 0 {
134137
cliLog.Fatal().Msgf("%s environment variable missing", constants.EnvOperatorPodName)
135138
}
139+
ip := os.Getenv(constants.EnvOperatorPodIP)
140+
if len(ip) == 0 {
141+
cliLog.Fatal().Msgf("%s environment variable missing", constants.EnvOperatorPodIP)
142+
}
136143

137144
// Get host name
138145
id, err := os.Hostname()
139146
if err != nil {
140147
cliLog.Fatal().Err(err).Msg("Failed to get hostname")
141148
}
142149

143-
http.HandleFunc("/health", probe.LivenessHandler)
144-
http.HandleFunc("/ready/deployment", deploymentProbe.ReadyHandler)
145-
http.HandleFunc("/ready/storage", storageProbe.ReadyHandler)
146-
http.Handle("/metrics", prometheus.Handler())
147-
listenAddr := net.JoinHostPort(server.host, strconv.Itoa(server.port))
148-
go http.ListenAndServe(listenAddr, nil)
150+
// Create kubernetes client
151+
kubecli, err := k8sutil.NewKubeClient()
152+
if err != nil {
153+
cliLog.Fatal().Err(err).Msg("Failed to create Kubernetes client")
154+
}
155+
156+
mux := http.NewServeMux()
157+
mux.HandleFunc("/health", probe.LivenessHandler)
158+
mux.HandleFunc("/ready/deployment", deploymentProbe.ReadyHandler)
159+
mux.HandleFunc("/ready/storage", storageProbe.ReadyHandler)
160+
mux.Handle("/metrics", prometheus.Handler())
161+
listenAddr := net.JoinHostPort(serverOptions.host, strconv.Itoa(serverOptions.port))
162+
if svr, err := server.NewServer(kubecli.CoreV1(), mux, server.Config{
163+
Address: listenAddr,
164+
TLSSecretName: serverOptions.tlsSecretName,
165+
TLSSecretNamespace: namespace,
166+
PodName: name,
167+
PodIP: ip,
168+
}); err != nil {
169+
cliLog.Fatal().Err(err).Msg("Failed to create HTTP server")
170+
} else {
171+
go svr.Run()
172+
}
149173

150174
cfg, deps, err := newOperatorConfigAndDeps(id+"-"+name, namespace, name)
151175
if err != nil {

manifests/templates/deployment/deployment.yaml

+7
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ spec:
1212
metadata:
1313
labels:
1414
name: {{ .Deployment.OperatorDeploymentName }}
15+
app: arango-deployment-operator
1516
spec:
1617
containers:
1718
- name: operator
@@ -29,18 +30,24 @@ spec:
2930
valueFrom:
3031
fieldRef:
3132
fieldPath: metadata.name
33+
- name: MY_POD_IP
34+
valueFrom:
35+
fieldRef:
36+
fieldPath: status.podIP
3237
ports:
3338
- name: metrics
3439
containerPort: 8528
3540
livenessProbe:
3641
httpGet:
3742
path: /health
3843
port: 8528
44+
scheme: HTTPS
3945
initialDelaySeconds: 5
4046
periodSeconds: 10
4147
readinessProbe:
4248
httpGet:
4349
path: /ready/deployment
4450
port: 8528
51+
scheme: HTTPS
4552
initialDelaySeconds: 5
4653
periodSeconds: 10

manifests/templates/storage/deployment.yaml

+7
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ spec:
2020
metadata:
2121
labels:
2222
name: {{ .Storage.OperatorDeploymentName }}
23+
app: arango-storage-operator
2324
spec:
2425
serviceAccountName: {{ .Storage.Operator.ServiceAccountName }}
2526
containers:
@@ -37,19 +38,25 @@ spec:
3738
valueFrom:
3839
fieldRef:
3940
fieldPath: metadata.name
41+
- name: MY_POD_IP
42+
valueFrom:
43+
fieldRef:
44+
fieldPath: status.podIP
4045
ports:
4146
- name: metrics
4247
containerPort: 8528
4348
livenessProbe:
4449
httpGet:
4550
path: /health
4651
port: 8528
52+
scheme: HTTPS
4753
initialDelaySeconds: 5
4854
periodSeconds: 10
4955
readinessProbe:
5056
httpGet:
5157
path: /ready/storage
5258
port: 8528
59+
scheme: HTTPS
5360
initialDelaySeconds: 5
5461
periodSeconds: 10
5562

pkg/deployment/deployment.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -137,7 +137,7 @@ func New(config Config, deps Dependencies, apiObject *api.ArangoDeployment) (*De
137137
}
138138
if config.AllowChaos {
139139
d.chaosMonkey = chaos.NewMonkey(deps.Log, d)
140-
d.chaosMonkey.Run(d.stopCh)
140+
go d.chaosMonkey.Run(d.stopCh)
141141
}
142142

143143
return d, nil

pkg/server/errors.go

+29
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
//
2+
// DISCLAIMER
3+
//
4+
// Copyright 2018 ArangoDB GmbH, Cologne, Germany
5+
//
6+
// Licensed under the Apache License, Version 2.0 (the "License");
7+
// you may not use this file except in compliance with the License.
8+
// You may obtain a copy of the License at
9+
//
10+
// http://www.apache.org/licenses/LICENSE-2.0
11+
//
12+
// Unless required by applicable law or agreed to in writing, software
13+
// distributed under the License is distributed on an "AS IS" BASIS,
14+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
// See the License for the specific language governing permissions and
16+
// limitations under the License.
17+
//
18+
// Copyright holder is ArangoDB GmbH, Cologne, Germany
19+
//
20+
// Author Ewout Prangsma
21+
//
22+
23+
package server
24+
25+
import "github.com/pkg/errors"
26+
27+
var (
28+
maskAny = errors.WithStack
29+
)

pkg/server/server.go

+126
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
//
2+
// DISCLAIMER
3+
//
4+
// Copyright 2018 ArangoDB GmbH, Cologne, Germany
5+
//
6+
// Licensed under the Apache License, Version 2.0 (the "License");
7+
// you may not use this file except in compliance with the License.
8+
// You may obtain a copy of the License at
9+
//
10+
// http://www.apache.org/licenses/LICENSE-2.0
11+
//
12+
// Unless required by applicable law or agreed to in writing, software
13+
// distributed under the License is distributed on an "AS IS" BASIS,
14+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
// See the License for the specific language governing permissions and
16+
// limitations under the License.
17+
//
18+
// Copyright holder is ArangoDB GmbH, Cologne, Germany
19+
//
20+
// Author Ewout Prangsma
21+
//
22+
23+
package server
24+
25+
import (
26+
"crypto/tls"
27+
"fmt"
28+
"net/http"
29+
"time"
30+
31+
certificates "github.com/arangodb-helper/go-certificates"
32+
"k8s.io/api/core/v1"
33+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
34+
corev1 "k8s.io/client-go/kubernetes/typed/core/v1"
35+
)
36+
37+
// Config settings for the Server
38+
type Config struct {
39+
Address string // Address to listen on
40+
TLSSecretName string // Name of secret containing TLS certificate
41+
TLSSecretNamespace string // Namespace of secret containing TLS certificate
42+
PodName string // Name of the Pod we're running in
43+
PodIP string // IP address of the Pod we're running in
44+
}
45+
46+
// Server is the HTTPS server for the operator.
47+
type Server struct {
48+
httpServer *http.Server
49+
}
50+
51+
// NewServer creates a new server, fetching/preparing a TLS certificate.
52+
func NewServer(cli corev1.CoreV1Interface, handler http.Handler, cfg Config) (*Server, error) {
53+
httpServer := &http.Server{
54+
Addr: cfg.Address,
55+
Handler: handler,
56+
ReadTimeout: time.Second * 30,
57+
ReadHeaderTimeout: time.Second * 15,
58+
WriteTimeout: time.Second * 30,
59+
TLSNextProto: make(map[string]func(*http.Server, *tls.Conn, http.Handler)),
60+
}
61+
62+
var cert, key string
63+
if cfg.TLSSecretName != "" && cfg.TLSSecretNamespace != "" {
64+
// Load TLS certificate from secret
65+
s, err := cli.Secrets(cfg.TLSSecretNamespace).Get(cfg.TLSSecretName, metav1.GetOptions{})
66+
if err != nil {
67+
return nil, maskAny(err)
68+
}
69+
certBytes, found := s.Data[v1.TLSCertKey]
70+
if !found {
71+
return nil, maskAny(fmt.Errorf("No %s found in secret %s", v1.TLSCertKey, cfg.TLSSecretName))
72+
}
73+
keyBytes, found := s.Data[v1.TLSPrivateKeyKey]
74+
if !found {
75+
return nil, maskAny(fmt.Errorf("No %s found in secret %s", v1.TLSPrivateKeyKey, cfg.TLSSecretName))
76+
}
77+
cert = string(certBytes)
78+
key = string(keyBytes)
79+
} else {
80+
// Secret not specified, create our own TLS certificate
81+
options := certificates.CreateCertificateOptions{
82+
CommonName: cfg.PodName,
83+
Hosts: []string{cfg.PodName, cfg.PodIP},
84+
ValidFrom: time.Now(),
85+
ValidFor: time.Hour * 24 * 365 * 10,
86+
IsCA: false,
87+
ECDSACurve: "P256",
88+
}
89+
var err error
90+
cert, key, err = certificates.CreateCertificate(options, nil)
91+
if err != nil {
92+
return nil, maskAny(err)
93+
}
94+
}
95+
tlsConfig, err := createTLSConfig(cert, key)
96+
if err != nil {
97+
return nil, maskAny(err)
98+
}
99+
tlsConfig.BuildNameToCertificate()
100+
httpServer.TLSConfig = tlsConfig
101+
102+
return &Server{
103+
httpServer: httpServer,
104+
}, nil
105+
}
106+
107+
// Run the server until the program stops.
108+
func (s *Server) Run() error {
109+
if err := s.httpServer.ListenAndServeTLS("", ""); err != nil && err != http.ErrServerClosed {
110+
return maskAny(err)
111+
}
112+
return nil
113+
}
114+
115+
// createTLSConfig creates a TLS config based on given config
116+
func createTLSConfig(cert, key string) (*tls.Config, error) {
117+
var result *tls.Config
118+
c, err := tls.X509KeyPair([]byte(cert), []byte(key))
119+
if err != nil {
120+
return nil, maskAny(err)
121+
}
122+
result = &tls.Config{
123+
Certificates: []tls.Certificate{c},
124+
}
125+
return result, nil
126+
}

pkg/util/constants/constants.go

+1
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ const (
2626
EnvOperatorNodeName = "MY_NODE_NAME"
2727
EnvOperatorPodName = "MY_POD_NAME"
2828
EnvOperatorPodNamespace = "MY_POD_NAMESPACE"
29+
EnvOperatorPodIP = "MY_POD_IP"
2930

3031
EnvArangodJWTSecret = "ARANGOD_JWT_SECRET" // Contains JWT secret for the ArangoDB cluster
3132
EnvArangoSyncJWTSecret = "ARANGOSYNC_JWT_SECRET" // Contains JWT secret for the ArangoSync masters

0 commit comments

Comments
 (0)