Skip to content

Commit 9a4056f

Browse files
committed
Store hash of secret values, and detect changes
1 parent 14ee4f7 commit 9a4056f

File tree

7 files changed

+283
-1
lines changed

7 files changed

+283
-1
lines changed

pkg/apis/deployment/v1alpha/conditions.go

+4
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,10 @@ const (
3737
ConditionTypeTerminated ConditionType = "Terminated"
3838
// ConditionTypeAutoUpgrade indicates that the member has to be started with `--database.auto-upgrade` once.
3939
ConditionTypeAutoUpgrade ConditionType = "AutoUpgrade"
40+
// ConditionTypeSecretsChanged indicates that the value of one of more secrets used by
41+
// the deployment have changed. Once that is the case, the operator will no longer
42+
// touch the deployment, until the original secrets have been restored.
43+
ConditionTypeSecretsChanged ConditionType = "SecretsChanged"
4044
)
4145

4246
// Condition represents one current condition of a deployment or deployment member.

pkg/apis/deployment/v1alpha/deployment_status.go

+4
Original file line numberDiff line numberDiff line change
@@ -50,4 +50,8 @@ type DeploymentStatus struct {
5050

5151
// AcceptedSpec contains the last specification that was accepted by the operator.
5252
AcceptedSpec *DeploymentSpec `json:"accepted-spec,omitempty"`
53+
54+
// SecretHashes keeps a sha256 hash of secret values, so we can
55+
// detect changes in secret values.
56+
SecretHashes *SecretHashes `json:"secret-hashes,omitempty"`
5357
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
//
2+
// DISCLAIMER
3+
//
4+
// Copyright 2018 ArangoDB GmbH, Cologne, Germany
5+
//
6+
// Licensed under the Apache License, Version 2.0 (the "License");
7+
// you may not use this file except in compliance with the License.
8+
// You may obtain a copy of the License at
9+
//
10+
// http://www.apache.org/licenses/LICENSE-2.0
11+
//
12+
// Unless required by applicable law or agreed to in writing, software
13+
// distributed under the License is distributed on an "AS IS" BASIS,
14+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
// See the License for the specific language governing permissions and
16+
// limitations under the License.
17+
//
18+
// Copyright holder is ArangoDB GmbH, Cologne, Germany
19+
//
20+
// Author Ewout Prangsma
21+
//
22+
23+
package v1alpha
24+
25+
// SecretHashes keeps track of the value of secrets
26+
// so we can detect changes.
27+
// For each used secret, a sha256 hash is stored.
28+
type SecretHashes struct {
29+
// AuthJWT contains the hash of the auth.jwtSecretName secret
30+
AuthJWT string `json:"auth-jwt,omitempty"`
31+
// RocksDBEncryptionKey contains the hash of the rocksdb.encryption.keySecretName secret
32+
RocksDBEncryptionKey string `json:"rocksdb-encryption-key,omitempty"`
33+
// TLSCA contains the hash of the tls.caSecretName secret
34+
TLSCA string `json:"tls-ca,omitempty"`
35+
// SyncTLSCA contains the hash of the sync.tls.caSecretName secret
36+
SyncTLSCA string `json:"sync-tls-ca,omitempty"`
37+
}

pkg/apis/deployment/v1alpha/zz_generated.deepcopy.go

+25
Original file line numberDiff line numberDiff line change
@@ -265,6 +265,15 @@ func (in *DeploymentStatus) DeepCopyInto(out *DeploymentStatus) {
265265
(*in).DeepCopyInto(*out)
266266
}
267267
}
268+
if in.SecretHashes != nil {
269+
in, out := &in.SecretHashes, &out.SecretHashes
270+
if *in == nil {
271+
*out = nil
272+
} else {
273+
*out = new(SecretHashes)
274+
**out = **in
275+
}
276+
}
268277
return
269278
}
270279

@@ -442,6 +451,22 @@ func (in *RocksDBSpec) DeepCopy() *RocksDBSpec {
442451
return out
443452
}
444453

454+
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
455+
func (in *SecretHashes) DeepCopyInto(out *SecretHashes) {
456+
*out = *in
457+
return
458+
}
459+
460+
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecretHashes.
461+
func (in *SecretHashes) DeepCopy() *SecretHashes {
462+
if in == nil {
463+
return nil
464+
}
465+
out := new(SecretHashes)
466+
in.DeepCopyInto(out)
467+
return out
468+
}
469+
445470
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
446471
func (in *ServerGroupSpec) DeepCopyInto(out *ServerGroupSpec) {
447472
*out = *in

pkg/deployment/deployment_inspector.go

+20-1
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import (
2626
"context"
2727
"time"
2828

29+
api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha"
2930
"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
3031
)
3132

@@ -37,12 +38,30 @@ import (
3738
// - once in a while
3839
// Returns the delay until this function should be called again.
3940
func (d *Deployment) inspectDeployment(lastInterval time.Duration) time.Duration {
40-
// log := d.deps.Log
41+
log := d.deps.Log
4142

4243
nextInterval := lastInterval
4344
hasError := false
4445
ctx := context.Background()
4546

47+
// Is the deployment in failed state, if so, give up.
48+
if d.status.State == api.DeploymentStateFailed {
49+
log.Debug().Msg("Deployment is in Failed state.")
50+
return nextInterval
51+
}
52+
53+
// Inspect secret hashes
54+
if err := d.resources.ValidateSecretHashes(); err != nil {
55+
hasError = true
56+
d.CreateEvent(k8sutil.NewErrorEvent("Secret hash validation failed", err, d.apiObject))
57+
}
58+
59+
// Is the deployment in a good state?
60+
if d.status.Conditions.IsTrue(api.ConditionTypeSecretsChanged) {
61+
log.Debug().Msg("Condition SecretsChanged is true. Revert secrets before we can continue")
62+
return nextInterval
63+
}
64+
4665
// Ensure we have image info
4766
if retrySoon, err := d.ensureImages(d.apiObject); err != nil {
4867
hasError = true
+174
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,174 @@
1+
//
2+
// DISCLAIMER
3+
//
4+
// Copyright 2018 ArangoDB GmbH, Cologne, Germany
5+
//
6+
// Licensed under the Apache License, Version 2.0 (the "License");
7+
// you may not use this file except in compliance with the License.
8+
// You may obtain a copy of the License at
9+
//
10+
// http://www.apache.org/licenses/LICENSE-2.0
11+
//
12+
// Unless required by applicable law or agreed to in writing, software
13+
// distributed under the License is distributed on an "AS IS" BASIS,
14+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
// See the License for the specific language governing permissions and
16+
// limitations under the License.
17+
//
18+
// Copyright holder is ArangoDB GmbH, Cologne, Germany
19+
//
20+
// Author Ewout Prangsma
21+
//
22+
23+
package resources
24+
25+
import (
26+
"crypto/sha256"
27+
"encoding/hex"
28+
"fmt"
29+
"sort"
30+
"strings"
31+
32+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
33+
34+
api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha"
35+
"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
36+
)
37+
38+
// ValidateSecretHashes checks the hash of used secrets
39+
// against the stored ones.
40+
// If a hash is different, the deployment is marked
41+
// with a SecretChangedCondition and the operator will no
42+
// touch it until this is resolved.
43+
func (r *Resources) ValidateSecretHashes() error {
44+
// validate performs a secret hash comparison for a single secret.
45+
// Return true if all is good, false when the SecretChanged condition
46+
// must be set.
47+
validate := func(secretName string, expectedHashRef *string, status *api.DeploymentStatus) (bool, error) {
48+
log := r.log.With().Str("secret-name", secretName).Logger()
49+
expectedHash := *expectedHashRef
50+
hash, err := r.getSecretHash(secretName)
51+
if expectedHash == "" {
52+
// No hash set yet, try to fill it
53+
if k8sutil.IsNotFound(err) {
54+
// Secret does not (yet) exists, do nothing
55+
return true, nil
56+
}
57+
if err != nil {
58+
log.Warn().Err(err).Msg("Failed to get secret")
59+
return true, nil // Since we do not yet have a hash, we let this go with only a warning.
60+
}
61+
// Hash fetched succesfully, store it
62+
*expectedHashRef = hash
63+
if r.context.UpdateStatus(*status); err != nil {
64+
log.Debug().Msg("Failed to save secret hash")
65+
return true, maskAny(err)
66+
}
67+
return true, nil
68+
}
69+
// Hash is set, it must match the current hash
70+
if err != nil {
71+
// Fetching error failed for other reason.
72+
log.Debug().Err(err).Msg("Failed to fetch secret hash")
73+
// This is not good, return false so SecretsChanged condition will be set.
74+
return false, nil
75+
}
76+
if hash != expectedHash {
77+
// Oops, hash has changed
78+
log.Error().Msg("Secret has changed. You must revert it to the original value!")
79+
// This is not good, return false so SecretsChanged condition will be set.
80+
return false, nil
81+
}
82+
// All good
83+
return true, nil
84+
}
85+
86+
spec := r.context.GetSpec()
87+
log := r.log
88+
var badSecretNames []string
89+
status := r.context.GetStatus()
90+
if status.SecretHashes == nil {
91+
status.SecretHashes = &api.SecretHashes{}
92+
}
93+
hashes := status.SecretHashes
94+
if spec.IsAuthenticated() {
95+
secretName := spec.Authentication.GetJWTSecretName()
96+
if hashOK, err := validate(secretName, &hashes.AuthJWT, &status); err != nil {
97+
return maskAny(err)
98+
} else if !hashOK {
99+
badSecretNames = append(badSecretNames, secretName)
100+
}
101+
}
102+
if spec.RocksDB.IsEncrypted() {
103+
secretName := spec.RocksDB.Encryption.GetKeySecretName()
104+
if hashOK, err := validate(secretName, &hashes.RocksDBEncryptionKey, &status); err != nil {
105+
return maskAny(err)
106+
} else if !hashOK {
107+
badSecretNames = append(badSecretNames, secretName)
108+
}
109+
}
110+
if spec.IsSecure() {
111+
secretName := spec.TLS.GetCASecretName()
112+
if hashOK, err := validate(secretName, &hashes.TLSCA, &status); err != nil {
113+
return maskAny(err)
114+
} else if !hashOK {
115+
badSecretNames = append(badSecretNames, secretName)
116+
}
117+
}
118+
if spec.Sync.IsEnabled() {
119+
secretName := spec.Sync.TLS.GetCASecretName()
120+
if hashOK, err := validate(secretName, &hashes.SyncTLSCA, &status); err != nil {
121+
return maskAny(err)
122+
} else if !hashOK {
123+
badSecretNames = append(badSecretNames, secretName)
124+
}
125+
}
126+
127+
if len(badSecretNames) > 0 {
128+
// We have invalid hashes, set the SecretsChanged condition
129+
if status.Conditions.Update(api.ConditionTypeSecretsChanged, true,
130+
"Secrets have changed", fmt.Sprintf("Found %d changed secrets", len(badSecretNames))) {
131+
log.Warn().Msgf("Found %d changed secrets. Settings SecretsChanged condition", len(badSecretNames))
132+
if err := r.context.UpdateStatus(status); err != nil {
133+
log.Error().Err(err).Msg("Failed to save SecretsChanged condition")
134+
return maskAny(err)
135+
}
136+
// Add an event about this
137+
r.context.CreateEvent(k8sutil.NewSecretsChangedEvent(badSecretNames, r.context.GetAPIObject()))
138+
}
139+
} else {
140+
// All good, we van remove the SecretsChanged condition
141+
if status.Conditions.Remove(api.ConditionTypeSecretsChanged) {
142+
log.Warn().Msg("Resetting SecretsChanged condition")
143+
if err := r.context.UpdateStatus(status); err != nil {
144+
log.Error().Err(err).Msg("Failed to save SecretsChanged condition")
145+
return maskAny(err)
146+
}
147+
// Add an event about this
148+
r.context.CreateEvent(k8sutil.NewSecretsRestoredEvent(r.context.GetAPIObject()))
149+
}
150+
}
151+
152+
return nil
153+
}
154+
155+
// getSecretHash fetches a secret with given name and returns a hash over its value.
156+
func (r *Resources) getSecretHash(secretName string) (string, error) {
157+
kubecli := r.context.GetKubeCli()
158+
ns := r.context.GetNamespace()
159+
s, err := kubecli.CoreV1().Secrets(ns).Get(secretName, metav1.GetOptions{})
160+
if err != nil {
161+
return "", maskAny(err)
162+
}
163+
// Create hash of value
164+
rows := make([]string, 0, len(s.Data))
165+
for k, v := range s.Data {
166+
rows = append(rows, k+"="+hex.EncodeToString(v))
167+
}
168+
// Sort so we're not detecting order differences
169+
sort.Strings(rows)
170+
data := strings.Join(rows, "\n")
171+
rawHash := sha256.Sum256([]byte(data))
172+
hash := fmt.Sprintf("%0x", rawHash)
173+
return hash, nil
174+
}

pkg/util/k8sutil/events.go

+19
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,25 @@ func NewImmutableFieldEvent(fieldName string, apiObject APIObject) *v1.Event {
7878
return event
7979
}
8080

81+
// NewSecretsChangedEvent creates an event indicating that one of more secrets have changed.
82+
func NewSecretsChangedEvent(changedSecretNames []string, apiObject APIObject) *v1.Event {
83+
event := newDeploymentEvent(apiObject)
84+
event.Type = v1.EventTypeNormal
85+
event.Reason = "Secrets changed"
86+
event.Message = fmt.Sprintf("Found %d changed secrets. You must revert them before the operator can continue. Secrets: %v", len(changedSecretNames), changedSecretNames)
87+
return event
88+
}
89+
90+
// NewSecretsRestoredEvent creates an event indicating that all secrets have been restored
91+
// to their original values.
92+
func NewSecretsRestoredEvent(apiObject APIObject) *v1.Event {
93+
event := newDeploymentEvent(apiObject)
94+
event.Type = v1.EventTypeNormal
95+
event.Reason = "Secrets restored"
96+
event.Message = "All secrets have been restored to their original value"
97+
return event
98+
}
99+
81100
// NewErrorEvent creates an even of type error.
82101
func NewErrorEvent(reason string, err error, apiObject APIObject) *v1.Event {
83102
event := newDeploymentEvent(apiObject)

0 commit comments

Comments
 (0)