Skip to content

Commit 4dcf8f8

Browse files
authored
Merge pull request #136 from arangodb/feature/pod-finalizers
Feature: finalizers
2 parents 21fc9e6 + bd90071 commit 4dcf8f8

31 files changed

+1184
-51
lines changed
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
# Lifecycle hooks & Finalizers
2+
3+
The ArangoDB operator expects full control of the `Pods` and `PersistentVolumeClaims` it creates.
4+
Therefore it takes measures to prevent the removal of those resources
5+
until it is safe to do so.
6+
7+
To achieve this, the server containers in the `Pods` have
8+
a `preStop` hook configured and finalizers are added to the `Pods`
9+
and `PersistentVolumeClaims`.
10+
11+
The `preStop` hook executes a binary that waits until all finalizers of
12+
the current pod have been removed.
13+
Until this `preStop` hook terminates, Kubernetes will not send a `TERM` signal
14+
to the processes inside the container, which ensures that the server remains running
15+
until it is safe to stop them.
16+
17+
The operator performs all actions needed when a delete of a `Pod` or
18+
`PersistentVolumeClaims` has been triggered.
19+
E.g. for a dbserver it cleans out the server if the `Pod` and `PersistentVolumeClaim` are being deleted.
20+
21+
## Lifecycle init-container
22+
23+
Because the binary that is called in the `preStop` hook is not part of a standard
24+
ArangoDB docker image, it has to be brought into the filesystem of a `Pod`.
25+
This is done by an initial container that copies the binary to an `emptyDir` volume that
26+
is shared between the init-container and the server container.
27+
28+
## Finalizers
29+
30+
The ArangoDB operators adds the following finalizers to `Pods`.
31+
32+
- `dbserver.database.arangodb.com/drain`: Added to DBServers, removed only when the dbserver can be restarted or is completely drained
33+
- `agent.database.arangodb.com/agency-serving`: Added to Agents, removed only when enough agents are left to keep the agency serving
34+
35+
The ArangoDB operators adds the following finalizers to `PersistentVolumeClaims`.
36+
37+
- `pvc.database.arangodb.com/member-exists`: removed only when its member exists no longer exists or can be safely rebuild

lifecycle.go

+150
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
//
2+
// DISCLAIMER
3+
//
4+
// Copyright 2018 ArangoDB GmbH, Cologne, Germany
5+
//
6+
// Licensed under the Apache License, Version 2.0 (the "License");
7+
// you may not use this file except in compliance with the License.
8+
// You may obtain a copy of the License at
9+
//
10+
// http://www.apache.org/licenses/LICENSE-2.0
11+
//
12+
// Unless required by applicable law or agreed to in writing, software
13+
// distributed under the License is distributed on an "AS IS" BASIS,
14+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
// See the License for the specific language governing permissions and
16+
// limitations under the License.
17+
//
18+
// Copyright holder is ArangoDB GmbH, Cologne, Germany
19+
//
20+
// Author Ewout Prangsma
21+
//
22+
23+
package main
24+
25+
import (
26+
"io"
27+
"os"
28+
"path/filepath"
29+
"time"
30+
31+
"github.com/spf13/cobra"
32+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
33+
34+
"github.com/arangodb/kube-arangodb/pkg/util/constants"
35+
"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
36+
)
37+
38+
var (
39+
cmdLifecycle = &cobra.Command{
40+
Use: "lifecycle",
41+
Run: cmdUsage,
42+
Hidden: true,
43+
}
44+
45+
cmdLifecyclePreStop = &cobra.Command{
46+
Use: "preStop",
47+
Run: cmdLifecyclePreStopRun,
48+
Hidden: true,
49+
}
50+
cmdLifecycleCopy = &cobra.Command{
51+
Use: "copy",
52+
Run: cmdLifecycleCopyRun,
53+
Hidden: true,
54+
}
55+
56+
lifecycleCopyOptions struct {
57+
TargetDir string
58+
}
59+
)
60+
61+
func init() {
62+
cmdMain.AddCommand(cmdLifecycle)
63+
cmdLifecycle.AddCommand(cmdLifecyclePreStop)
64+
cmdLifecycle.AddCommand(cmdLifecycleCopy)
65+
66+
cmdLifecycleCopy.Flags().StringVar(&lifecycleCopyOptions.TargetDir, "target", "", "Target directory to copy the executable to")
67+
}
68+
69+
// Wait until all finalizers of the current pod have been removed.
70+
func cmdLifecyclePreStopRun(cmd *cobra.Command, args []string) {
71+
cliLog.Info().Msgf("Starting arangodb-operator, lifecycle preStop, version %s build %s", projectVersion, projectBuild)
72+
73+
// Get environment
74+
namespace := os.Getenv(constants.EnvOperatorPodNamespace)
75+
if len(namespace) == 0 {
76+
cliLog.Fatal().Msgf("%s environment variable missing", constants.EnvOperatorPodNamespace)
77+
}
78+
name := os.Getenv(constants.EnvOperatorPodName)
79+
if len(name) == 0 {
80+
cliLog.Fatal().Msgf("%s environment variable missing", constants.EnvOperatorPodName)
81+
}
82+
83+
// Create kubernetes client
84+
kubecli, err := k8sutil.NewKubeClient()
85+
if err != nil {
86+
cliLog.Fatal().Err(err).Msg("Failed to create Kubernetes client")
87+
}
88+
89+
pods := kubecli.CoreV1().Pods(namespace)
90+
recentErrors := 0
91+
for {
92+
p, err := pods.Get(name, metav1.GetOptions{})
93+
if k8sutil.IsNotFound(err) {
94+
cliLog.Warn().Msg("Pod not found")
95+
return
96+
} else if err != nil {
97+
recentErrors++
98+
cliLog.Error().Err(err).Msg("Failed to get pod")
99+
if recentErrors > 20 {
100+
cliLog.Fatal().Err(err).Msg("Too many recent errors")
101+
return
102+
}
103+
} else {
104+
// We got our pod
105+
finalizerCount := len(p.GetFinalizers())
106+
if finalizerCount == 0 {
107+
// No more finalizers, we're done
108+
cliLog.Info().Msg("All finalizers gone, we can stop now")
109+
return
110+
}
111+
cliLog.Info().Msgf("Waiting for %d more finalizers to be removed", finalizerCount)
112+
}
113+
// Wait a bit
114+
time.Sleep(time.Second)
115+
}
116+
}
117+
118+
// Copy the executable to a given place.
119+
func cmdLifecycleCopyRun(cmd *cobra.Command, args []string) {
120+
cliLog.Info().Msgf("Starting arangodb-operator, lifecycle copy, version %s build %s", projectVersion, projectBuild)
121+
122+
exePath, err := os.Executable()
123+
if err != nil {
124+
cliLog.Fatal().Err(err).Msg("Failed to get executable path")
125+
}
126+
127+
// Open source
128+
rd, err := os.Open(exePath)
129+
if err != nil {
130+
cliLog.Fatal().Err(err).Msg("Failed to open executable file")
131+
}
132+
defer rd.Close()
133+
134+
// Open target
135+
targetPath := filepath.Join(lifecycleCopyOptions.TargetDir, filepath.Base(exePath))
136+
wr, err := os.Create(targetPath)
137+
if err != nil {
138+
cliLog.Fatal().Err(err).Msg("Failed to create target file")
139+
}
140+
defer wr.Close()
141+
142+
if _, err := io.Copy(wr, rd); err != nil {
143+
cliLog.Fatal().Err(err).Msg("Failed to copy")
144+
}
145+
146+
// Set file mode
147+
if err := os.Chmod(targetPath, 0755); err != nil {
148+
cliLog.Fatal().Err(err).Msg("Failed to chmod")
149+
}
150+
}

main.go

+13-6
Original file line numberDiff line numberDiff line change
@@ -193,7 +193,7 @@ func newOperatorConfigAndDeps(id, namespace, name string) (operator.Config, oper
193193
return operator.Config{}, operator.Dependencies{}, maskAny(err)
194194
}
195195

196-
serviceAccount, err := getMyPodServiceAccount(kubecli, namespace, name)
196+
image, serviceAccount, err := getMyPodInfo(kubecli, namespace, name)
197197
if err != nil {
198198
return operator.Config{}, operator.Dependencies{}, maskAny(fmt.Errorf("Failed to get my pod's service account: %s", err))
199199
}
@@ -213,6 +213,7 @@ func newOperatorConfigAndDeps(id, namespace, name string) (operator.Config, oper
213213
Namespace: namespace,
214214
PodName: name,
215215
ServiceAccount: serviceAccount,
216+
LifecycleImage: image,
216217
EnableDeployment: operatorOptions.enableDeployment,
217218
EnableStorage: operatorOptions.enableStorage,
218219
AllowChaos: chaosOptions.allowed,
@@ -231,9 +232,10 @@ func newOperatorConfigAndDeps(id, namespace, name string) (operator.Config, oper
231232
return cfg, deps, nil
232233
}
233234

234-
// getMyPodServiceAccount looks up the service account of the pod with given name in given namespace
235-
func getMyPodServiceAccount(kubecli kubernetes.Interface, namespace, name string) (string, error) {
236-
var sa string
235+
// getMyPodInfo looks up the image & service account of the pod with given name in given namespace
236+
// Returns image, serviceAccount, error.
237+
func getMyPodInfo(kubecli kubernetes.Interface, namespace, name string) (string, string, error) {
238+
var image, sa string
237239
op := func() error {
238240
pod, err := kubecli.CoreV1().Pods(namespace).Get(name, metav1.GetOptions{})
239241
if err != nil {
@@ -244,12 +246,17 @@ func getMyPodServiceAccount(kubecli kubernetes.Interface, namespace, name string
244246
return maskAny(err)
245247
}
246248
sa = pod.Spec.ServiceAccountName
249+
image = k8sutil.ConvertImageID2Image(pod.Status.ContainerStatuses[0].ImageID)
250+
if image == "" {
251+
// Fallback in case we don't know the id.
252+
image = pod.Spec.Containers[0].Image
253+
}
247254
return nil
248255
}
249256
if err := retry.Retry(op, time.Minute*5); err != nil {
250-
return "", maskAny(err)
257+
return "", "", maskAny(err)
251258
}
252-
return sa, nil
259+
return image, sa, nil
253260
}
254261

255262
func createRecorder(log zerolog.Logger, kubecli kubernetes.Interface, name, namespace string) record.EventRecorder {

pkg/apis/deployment/v1alpha/conditions.go

+3
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,9 @@ const (
3737
ConditionTypeTerminated ConditionType = "Terminated"
3838
// ConditionTypeAutoUpgrade indicates that the member has to be started with `--database.auto-upgrade` once.
3939
ConditionTypeAutoUpgrade ConditionType = "AutoUpgrade"
40+
// ConditionTypeCleanedOut indicates that the member (dbserver) has been cleaned out.
41+
// Always check in combination with ConditionTypeTerminated.
42+
ConditionTypeCleanedOut ConditionType = "CleanedOut"
4043
// ConditionTypePodSchedulingFailure indicates that one or more pods belonging to the deployment cannot be schedule.
4144
ConditionTypePodSchedulingFailure ConditionType = "PodSchedulingFailure"
4245
// ConditionTypeSecretsChanged indicates that the value of one of more secrets used by

pkg/apis/deployment/v1alpha/deployment.go

+7-4
Original file line numberDiff line numberDiff line change
@@ -50,11 +50,14 @@ type ArangoDeployment struct {
5050

5151
// AsOwner creates an OwnerReference for the given deployment
5252
func (d *ArangoDeployment) AsOwner() metav1.OwnerReference {
53+
trueVar := true
5354
return metav1.OwnerReference{
54-
APIVersion: SchemeGroupVersion.String(),
55-
Kind: ArangoDeploymentResourceKind,
56-
Name: d.Name,
57-
UID: d.UID,
55+
APIVersion: SchemeGroupVersion.String(),
56+
Kind: ArangoDeploymentResourceKind,
57+
Name: d.Name,
58+
UID: d.UID,
59+
Controller: &trueVar,
60+
BlockOwnerDeletion: &trueVar,
5861
}
5962
}
6063

pkg/apis/deployment/v1alpha/deployment_status_members.go

+16
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,22 @@ func (ds DeploymentStatusMembers) MemberStatusByPodName(podName string) (MemberS
121121
return MemberStatus{}, 0, false
122122
}
123123

124+
// MemberStatusByPVCName returns a reference to the element in the given set of lists that has the given PVC name.
125+
// If no such element exists, nil is returned.
126+
func (ds DeploymentStatusMembers) MemberStatusByPVCName(pvcName string) (MemberStatus, ServerGroup, bool) {
127+
if result, found := ds.Single.ElementByPVCName(pvcName); found {
128+
return result, ServerGroupSingle, true
129+
}
130+
if result, found := ds.Agents.ElementByPVCName(pvcName); found {
131+
return result, ServerGroupAgents, true
132+
}
133+
if result, found := ds.DBServers.ElementByPVCName(pvcName); found {
134+
return result, ServerGroupDBServers, true
135+
}
136+
// Note: Other server groups do not have PVC's so we can skip them.
137+
return MemberStatus{}, 0, false
138+
}
139+
124140
// UpdateMemberStatus updates the given status in the given group.
125141
func (ds *DeploymentStatusMembers) UpdateMemberStatus(status MemberStatus, group ServerGroup) error {
126142
var err error

pkg/apis/deployment/v1alpha/member_status_list.go

+11
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,17 @@ func (l MemberStatusList) ElementByPodName(podName string) (MemberStatus, bool)
6363
return MemberStatus{}, false
6464
}
6565

66+
// ElementByPVCName returns the element in the given list that has the given PVC name and true.
67+
// If no such element exists, an empty element and false is returned.
68+
func (l MemberStatusList) ElementByPVCName(pvcName string) (MemberStatus, bool) {
69+
for i, x := range l {
70+
if x.PersistentVolumeClaimName == pvcName {
71+
return l[i], true
72+
}
73+
}
74+
return MemberStatus{}, false
75+
}
76+
6677
// Add a member to the list.
6778
// Returns an AlreadyExistsError if the ID of the given member already exists.
6879
func (l *MemberStatusList) Add(m MemberStatus) error {

pkg/apis/deployment/v1alpha/server_group.go

+16
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@
2222

2323
package v1alpha
2424

25+
import time "time"
26+
2527
type ServerGroup int
2628

2729
const (
@@ -85,6 +87,20 @@ func (g ServerGroup) AsRoleAbbreviated() string {
8587
}
8688
}
8789

90+
// DefaultTerminationGracePeriod returns the default period between SIGTERM & SIGKILL for a server in the given group.
91+
func (g ServerGroup) DefaultTerminationGracePeriod() time.Duration {
92+
switch g {
93+
case ServerGroupSingle:
94+
return time.Minute
95+
case ServerGroupAgents:
96+
return time.Minute
97+
case ServerGroupDBServers:
98+
return time.Hour
99+
default:
100+
return time.Second * 30
101+
}
102+
}
103+
88104
// IsArangod returns true when the groups runs servers of type `arangod`.
89105
func (g ServerGroup) IsArangod() bool {
90106
switch g {

pkg/deployment/cleanup.go

+59
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
//
2+
// DISCLAIMER
3+
//
4+
// Copyright 2018 ArangoDB GmbH, Cologne, Germany
5+
//
6+
// Licensed under the Apache License, Version 2.0 (the "License");
7+
// you may not use this file except in compliance with the License.
8+
// You may obtain a copy of the License at
9+
//
10+
// http://www.apache.org/licenses/LICENSE-2.0
11+
//
12+
// Unless required by applicable law or agreed to in writing, software
13+
// distributed under the License is distributed on an "AS IS" BASIS,
14+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
// See the License for the specific language governing permissions and
16+
// limitations under the License.
17+
//
18+
// Copyright holder is ArangoDB GmbH, Cologne, Germany
19+
//
20+
// Author Ewout Prangsma
21+
//
22+
23+
package deployment
24+
25+
import (
26+
"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
27+
)
28+
29+
// removePodFinalizers removes all finalizers from all pods owned by us.
30+
func (d *Deployment) removePodFinalizers() error {
31+
log := d.deps.Log
32+
kubecli := d.GetKubeCli()
33+
pods, err := d.GetOwnedPods()
34+
if err != nil {
35+
return maskAny(err)
36+
}
37+
for _, p := range pods {
38+
if err := k8sutil.RemovePodFinalizers(log, kubecli, &p, p.GetFinalizers()); err != nil {
39+
log.Warn().Err(err).Msg("Failed to remove pod finalizers")
40+
}
41+
}
42+
return nil
43+
}
44+
45+
// removePVCFinalizers removes all finalizers from all PVCs owned by us.
46+
func (d *Deployment) removePVCFinalizers() error {
47+
log := d.deps.Log
48+
kubecli := d.GetKubeCli()
49+
pvcs, err := d.GetOwnedPVCs()
50+
if err != nil {
51+
return maskAny(err)
52+
}
53+
for _, p := range pvcs {
54+
if err := k8sutil.RemovePVCFinalizers(log, kubecli, &p, p.GetFinalizers()); err != nil {
55+
log.Warn().Err(err).Msg("Failed to remove PVC finalizers")
56+
}
57+
}
58+
return nil
59+
}

0 commit comments

Comments
 (0)