Skip to content

Commit 562c83d

Browse files
authored
Merge pull request #80 from arangodb/secret-hashes
Store & compare hash of secrets.
2 parents 21de95a + 9c07dcd commit 562c83d

File tree

9 files changed

+330
-1
lines changed

9 files changed

+330
-1
lines changed

pkg/apis/deployment/v1alpha/conditions.go

+4
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,10 @@ const (
3737
ConditionTypeTerminated ConditionType = "Terminated"
3838
// ConditionTypeAutoUpgrade indicates that the member has to be started with `--database.auto-upgrade` once.
3939
ConditionTypeAutoUpgrade ConditionType = "AutoUpgrade"
40+
// ConditionTypeSecretsChanged indicates that the value of one of more secrets used by
41+
// the deployment have changed. Once that is the case, the operator will no longer
42+
// touch the deployment, until the original secrets have been restored.
43+
ConditionTypeSecretsChanged ConditionType = "SecretsChanged"
4044
)
4145

4246
// Condition represents one current condition of a deployment or deployment member.

pkg/apis/deployment/v1alpha/deployment_status.go

+4
Original file line numberDiff line numberDiff line change
@@ -50,4 +50,8 @@ type DeploymentStatus struct {
5050

5151
// AcceptedSpec contains the last specification that was accepted by the operator.
5252
AcceptedSpec *DeploymentSpec `json:"accepted-spec,omitempty"`
53+
54+
// SecretHashes keeps a sha256 hash of secret values, so we can
55+
// detect changes in secret values.
56+
SecretHashes *SecretHashes `json:"secret-hashes,omitempty"`
5357
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
//
2+
// DISCLAIMER
3+
//
4+
// Copyright 2018 ArangoDB GmbH, Cologne, Germany
5+
//
6+
// Licensed under the Apache License, Version 2.0 (the "License");
7+
// you may not use this file except in compliance with the License.
8+
// You may obtain a copy of the License at
9+
//
10+
// http://www.apache.org/licenses/LICENSE-2.0
11+
//
12+
// Unless required by applicable law or agreed to in writing, software
13+
// distributed under the License is distributed on an "AS IS" BASIS,
14+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
// See the License for the specific language governing permissions and
16+
// limitations under the License.
17+
//
18+
// Copyright holder is ArangoDB GmbH, Cologne, Germany
19+
//
20+
// Author Ewout Prangsma
21+
//
22+
23+
package v1alpha
24+
25+
// SecretHashes keeps track of the value of secrets
26+
// so we can detect changes.
27+
// For each used secret, a sha256 hash is stored.
28+
type SecretHashes struct {
29+
// AuthJWT contains the hash of the auth.jwtSecretName secret
30+
AuthJWT string `json:"auth-jwt,omitempty"`
31+
// RocksDBEncryptionKey contains the hash of the rocksdb.encryption.keySecretName secret
32+
RocksDBEncryptionKey string `json:"rocksdb-encryption-key,omitempty"`
33+
// TLSCA contains the hash of the tls.caSecretName secret
34+
TLSCA string `json:"tls-ca,omitempty"`
35+
// SyncTLSCA contains the hash of the sync.tls.caSecretName secret
36+
SyncTLSCA string `json:"sync-tls-ca,omitempty"`
37+
}

pkg/apis/deployment/v1alpha/zz_generated.deepcopy.go

+25
Original file line numberDiff line numberDiff line change
@@ -265,6 +265,15 @@ func (in *DeploymentStatus) DeepCopyInto(out *DeploymentStatus) {
265265
(*in).DeepCopyInto(*out)
266266
}
267267
}
268+
if in.SecretHashes != nil {
269+
in, out := &in.SecretHashes, &out.SecretHashes
270+
if *in == nil {
271+
*out = nil
272+
} else {
273+
*out = new(SecretHashes)
274+
**out = **in
275+
}
276+
}
268277
return
269278
}
270279

@@ -442,6 +451,22 @@ func (in *RocksDBSpec) DeepCopy() *RocksDBSpec {
442451
return out
443452
}
444453

454+
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
455+
func (in *SecretHashes) DeepCopyInto(out *SecretHashes) {
456+
*out = *in
457+
return
458+
}
459+
460+
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecretHashes.
461+
func (in *SecretHashes) DeepCopy() *SecretHashes {
462+
if in == nil {
463+
return nil
464+
}
465+
out := new(SecretHashes)
466+
in.DeepCopyInto(out)
467+
return out
468+
}
469+
445470
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
446471
func (in *ServerGroupSpec) DeepCopyInto(out *ServerGroupSpec) {
447472
*out = *in

pkg/deployment/deployment.go

+1
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,7 @@ func New(config Config, deps Dependencies, apiObject *api.ArangoDeployment) (*De
120120
go d.run()
121121
go d.listenForPodEvents(d.stopCh)
122122
go d.listenForPVCEvents(d.stopCh)
123+
go d.listenForSecretEvents(d.stopCh)
123124
go d.listenForServiceEvents(d.stopCh)
124125
if apiObject.Spec.GetMode() == api.DeploymentModeCluster {
125126
ci := newClusterScalingIntegration(d)

pkg/deployment/deployment_inspector.go

+20-1
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import (
2626
"context"
2727
"time"
2828

29+
api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha"
2930
"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
3031
)
3132

@@ -37,12 +38,30 @@ import (
3738
// - once in a while
3839
// Returns the delay until this function should be called again.
3940
func (d *Deployment) inspectDeployment(lastInterval time.Duration) time.Duration {
40-
// log := d.deps.Log
41+
log := d.deps.Log
4142

4243
nextInterval := lastInterval
4344
hasError := false
4445
ctx := context.Background()
4546

47+
// Is the deployment in failed state, if so, give up.
48+
if d.status.State == api.DeploymentStateFailed {
49+
log.Debug().Msg("Deployment is in Failed state.")
50+
return nextInterval
51+
}
52+
53+
// Inspect secret hashes
54+
if err := d.resources.ValidateSecretHashes(); err != nil {
55+
hasError = true
56+
d.CreateEvent(k8sutil.NewErrorEvent("Secret hash validation failed", err, d.apiObject))
57+
}
58+
59+
// Is the deployment in a good state?
60+
if d.status.Conditions.IsTrue(api.ConditionTypeSecretsChanged) {
61+
log.Debug().Msg("Condition SecretsChanged is true. Revert secrets before we can continue")
62+
return nextInterval
63+
}
64+
4665
// Ensure we have image info
4766
if retrySoon, err := d.ensureImages(d.apiObject); err != nil {
4867
hasError = true

pkg/deployment/informers.go

+43
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,49 @@ func (d *Deployment) listenForPVCEvents(stopCh <-chan struct{}) {
112112
informer.Run(stopCh)
113113
}
114114

115+
// listenForSecretEvents keep listening for changes in Secrets's until the given channel is closed.
116+
func (d *Deployment) listenForSecretEvents(stopCh <-chan struct{}) {
117+
source := cache.NewListWatchFromClient(
118+
d.deps.KubeCli.CoreV1().RESTClient(),
119+
"secrets",
120+
d.apiObject.GetNamespace(),
121+
fields.Everything())
122+
123+
getSecret := func(obj interface{}) (*v1.Secret, bool) {
124+
secret, ok := obj.(*v1.Secret)
125+
if !ok {
126+
tombstone, ok := obj.(cache.DeletedFinalStateUnknown)
127+
if !ok {
128+
return nil, false
129+
}
130+
secret, ok = tombstone.Obj.(*v1.Secret)
131+
return secret, ok
132+
}
133+
return secret, true
134+
}
135+
136+
_, informer := cache.NewIndexerInformer(source, &v1.Secret{}, 0, cache.ResourceEventHandlerFuncs{
137+
// Note: For secrets we look at all of them because they do not have to be owned by this deployment.
138+
AddFunc: func(obj interface{}) {
139+
if _, ok := getSecret(obj); ok {
140+
d.triggerInspection()
141+
}
142+
},
143+
UpdateFunc: func(oldObj, newObj interface{}) {
144+
if _, ok := getSecret(newObj); ok {
145+
d.triggerInspection()
146+
}
147+
},
148+
DeleteFunc: func(obj interface{}) {
149+
if _, ok := getSecret(obj); ok {
150+
d.triggerInspection()
151+
}
152+
},
153+
}, cache.Indexers{})
154+
155+
informer.Run(stopCh)
156+
}
157+
115158
// listenForServiceEvents keep listening for changes in Service's until the given channel is closed.
116159
func (d *Deployment) listenForServiceEvents(stopCh <-chan struct{}) {
117160
source := cache.NewListWatchFromClient(
+177
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,177 @@
1+
//
2+
// DISCLAIMER
3+
//
4+
// Copyright 2018 ArangoDB GmbH, Cologne, Germany
5+
//
6+
// Licensed under the Apache License, Version 2.0 (the "License");
7+
// you may not use this file except in compliance with the License.
8+
// You may obtain a copy of the License at
9+
//
10+
// http://www.apache.org/licenses/LICENSE-2.0
11+
//
12+
// Unless required by applicable law or agreed to in writing, software
13+
// distributed under the License is distributed on an "AS IS" BASIS,
14+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
// See the License for the specific language governing permissions and
16+
// limitations under the License.
17+
//
18+
// Copyright holder is ArangoDB GmbH, Cologne, Germany
19+
//
20+
// Author Ewout Prangsma
21+
//
22+
23+
package resources
24+
25+
import (
26+
"crypto/sha256"
27+
"encoding/hex"
28+
"fmt"
29+
"sort"
30+
"strings"
31+
32+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
33+
34+
api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha"
35+
"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
36+
)
37+
38+
// ValidateSecretHashes checks the hash of used secrets
39+
// against the stored ones.
40+
// If a hash is different, the deployment is marked
41+
// with a SecretChangedCondition and the operator will no
42+
// touch it until this is resolved.
43+
func (r *Resources) ValidateSecretHashes() error {
44+
// validate performs a secret hash comparison for a single secret.
45+
// Return true if all is good, false when the SecretChanged condition
46+
// must be set.
47+
validate := func(secretName string, expectedHashRef *string, status *api.DeploymentStatus) (bool, error) {
48+
log := r.log.With().Str("secret-name", secretName).Logger()
49+
expectedHash := *expectedHashRef
50+
hash, err := r.getSecretHash(secretName)
51+
if expectedHash == "" {
52+
// No hash set yet, try to fill it
53+
if k8sutil.IsNotFound(err) {
54+
// Secret does not (yet) exists, do nothing
55+
return true, nil
56+
}
57+
if err != nil {
58+
log.Warn().Err(err).Msg("Failed to get secret")
59+
return true, nil // Since we do not yet have a hash, we let this go with only a warning.
60+
}
61+
// Hash fetched succesfully, store it
62+
*expectedHashRef = hash
63+
if r.context.UpdateStatus(*status); err != nil {
64+
log.Debug().Msg("Failed to save secret hash")
65+
return true, maskAny(err)
66+
}
67+
return true, nil
68+
}
69+
// Hash is set, it must match the current hash
70+
if err != nil {
71+
// Fetching error failed for other reason.
72+
log.Debug().Err(err).Msg("Failed to fetch secret hash")
73+
// This is not good, return false so SecretsChanged condition will be set.
74+
return false, nil
75+
}
76+
if hash != expectedHash {
77+
// Oops, hash has changed
78+
log.Debug().
79+
Str("expected-hash", expectedHash).
80+
Str("new-hash", hash).
81+
Msg("Secret has changed.")
82+
// This is not good, return false so SecretsChanged condition will be set.
83+
return false, nil
84+
}
85+
// All good
86+
return true, nil
87+
}
88+
89+
spec := r.context.GetSpec()
90+
log := r.log
91+
var badSecretNames []string
92+
status := r.context.GetStatus()
93+
if status.SecretHashes == nil {
94+
status.SecretHashes = &api.SecretHashes{}
95+
}
96+
hashes := status.SecretHashes
97+
if spec.IsAuthenticated() {
98+
secretName := spec.Authentication.GetJWTSecretName()
99+
if hashOK, err := validate(secretName, &hashes.AuthJWT, &status); err != nil {
100+
return maskAny(err)
101+
} else if !hashOK {
102+
badSecretNames = append(badSecretNames, secretName)
103+
}
104+
}
105+
if spec.RocksDB.IsEncrypted() {
106+
secretName := spec.RocksDB.Encryption.GetKeySecretName()
107+
if hashOK, err := validate(secretName, &hashes.RocksDBEncryptionKey, &status); err != nil {
108+
return maskAny(err)
109+
} else if !hashOK {
110+
badSecretNames = append(badSecretNames, secretName)
111+
}
112+
}
113+
if spec.IsSecure() {
114+
secretName := spec.TLS.GetCASecretName()
115+
if hashOK, err := validate(secretName, &hashes.TLSCA, &status); err != nil {
116+
return maskAny(err)
117+
} else if !hashOK {
118+
badSecretNames = append(badSecretNames, secretName)
119+
}
120+
}
121+
if spec.Sync.IsEnabled() {
122+
secretName := spec.Sync.TLS.GetCASecretName()
123+
if hashOK, err := validate(secretName, &hashes.SyncTLSCA, &status); err != nil {
124+
return maskAny(err)
125+
} else if !hashOK {
126+
badSecretNames = append(badSecretNames, secretName)
127+
}
128+
}
129+
130+
if len(badSecretNames) > 0 {
131+
// We have invalid hashes, set the SecretsChanged condition
132+
if status.Conditions.Update(api.ConditionTypeSecretsChanged, true,
133+
"Secrets have changed", fmt.Sprintf("Found %d changed secrets", len(badSecretNames))) {
134+
log.Warn().Msgf("Found %d changed secrets. Settings SecretsChanged condition", len(badSecretNames))
135+
if err := r.context.UpdateStatus(status); err != nil {
136+
log.Error().Err(err).Msg("Failed to save SecretsChanged condition")
137+
return maskAny(err)
138+
}
139+
// Add an event about this
140+
r.context.CreateEvent(k8sutil.NewSecretsChangedEvent(badSecretNames, r.context.GetAPIObject()))
141+
}
142+
} else {
143+
// All good, we van remove the SecretsChanged condition
144+
if status.Conditions.Remove(api.ConditionTypeSecretsChanged) {
145+
log.Info().Msg("Resetting SecretsChanged condition")
146+
if err := r.context.UpdateStatus(status); err != nil {
147+
log.Error().Err(err).Msg("Failed to save SecretsChanged condition")
148+
return maskAny(err)
149+
}
150+
// Add an event about this
151+
r.context.CreateEvent(k8sutil.NewSecretsRestoredEvent(r.context.GetAPIObject()))
152+
}
153+
}
154+
155+
return nil
156+
}
157+
158+
// getSecretHash fetches a secret with given name and returns a hash over its value.
159+
func (r *Resources) getSecretHash(secretName string) (string, error) {
160+
kubecli := r.context.GetKubeCli()
161+
ns := r.context.GetNamespace()
162+
s, err := kubecli.CoreV1().Secrets(ns).Get(secretName, metav1.GetOptions{})
163+
if err != nil {
164+
return "", maskAny(err)
165+
}
166+
// Create hash of value
167+
rows := make([]string, 0, len(s.Data))
168+
for k, v := range s.Data {
169+
rows = append(rows, k+"="+hex.EncodeToString(v))
170+
}
171+
// Sort so we're not detecting order differences
172+
sort.Strings(rows)
173+
data := strings.Join(rows, "\n")
174+
rawHash := sha256.Sum256([]byte(data))
175+
hash := fmt.Sprintf("%0x", rawHash)
176+
return hash, nil
177+
}

pkg/util/k8sutil/events.go

+19
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,25 @@ func NewImmutableFieldEvent(fieldName string, apiObject APIObject) *v1.Event {
7878
return event
7979
}
8080

81+
// NewSecretsChangedEvent creates an event indicating that one of more secrets have changed.
82+
func NewSecretsChangedEvent(changedSecretNames []string, apiObject APIObject) *v1.Event {
83+
event := newDeploymentEvent(apiObject)
84+
event.Type = v1.EventTypeNormal
85+
event.Reason = "Secrets changed"
86+
event.Message = fmt.Sprintf("Found %d changed secrets. You must revert them before the operator can continue. Secrets: %v", len(changedSecretNames), changedSecretNames)
87+
return event
88+
}
89+
90+
// NewSecretsRestoredEvent creates an event indicating that all secrets have been restored
91+
// to their original values.
92+
func NewSecretsRestoredEvent(apiObject APIObject) *v1.Event {
93+
event := newDeploymentEvent(apiObject)
94+
event.Type = v1.EventTypeNormal
95+
event.Reason = "Secrets restored"
96+
event.Message = "All secrets have been restored to their original value"
97+
return event
98+
}
99+
81100
// NewErrorEvent creates an even of type error.
82101
func NewErrorEvent(reason string, err error, apiObject APIObject) *v1.Event {
83102
event := newDeploymentEvent(apiObject)

0 commit comments

Comments
 (0)