Skip to content

Commit 7977f30

Browse files
bjee19bjee19
committed
Update zero downtime scale test and add some other review feedback
1 parent 069c13a commit 7977f30

File tree

6 files changed

+75
-30
lines changed

6 files changed

+75
-30
lines changed

‎tests/framework/crossplane.go

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,8 @@ type ExpectedNginxField struct {
3838
ValueSubstringAllowed bool
3939
}
4040

41+
const crossplaneImageName = "nginx-crossplane:latest"
42+
4143
// ValidateNginxFieldExists accepts the nginx config and the configuration for the expected field,
4244
// and returns whether or not that field exists where it should.
4345
func ValidateNginxFieldExists(conf *Payload, expFieldCfg ExpectedNginxField) error {
@@ -150,7 +152,7 @@ func injectCrossplaneContainer(
150152
ctx, cancel := context.WithTimeout(context.Background(), timeout)
151153
defer cancel()
152154

153-
image := "nginx-crossplane:latest"
155+
image := crossplaneImageName
154156
if crossplaneImageRepo != "" {
155157
image = crossplaneImageRepo + "/" + image
156158
}

‎tests/framework/resourcemanager.go

Lines changed: 41 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,8 @@ import (
4646
"k8s.io/client-go/util/retry"
4747
"sigs.k8s.io/controller-runtime/pkg/client"
4848
v1 "sigs.k8s.io/gateway-api/apis/v1"
49+
50+
ngfAPIv1alpha2 "github.com/nginx/nginx-gateway-fabric/apis/v1alpha2"
4951
)
5052

5153
// ResourceManager handles creating/updating/deleting Kubernetes resources.
@@ -188,7 +190,7 @@ func (rm *ResourceManager) Delete(resources []client.Object, opts ...client.Dele
188190
}
189191

190192
func (rm *ResourceManager) DeleteNamespace(name string) error {
191-
ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.DeleteNamespaceTimeout*2)
193+
ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.DeleteNamespaceTimeout)
192194
defer cancel()
193195

194196
ns := &core.Namespace{}
@@ -647,6 +649,44 @@ func (rm *ResourceManager) GetNGFDeployment(namespace, releaseName string) (*app
647649
return &deployment, nil
648650
}
649651

652+
func (rm *ResourceManager) getGatewayClassNginxProxy(
653+
namespace,
654+
releaseName string,
655+
) (*ngfAPIv1alpha2.NginxProxy, error) {
656+
ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.GetTimeout)
657+
defer cancel()
658+
659+
var proxy ngfAPIv1alpha2.NginxProxy
660+
proxyName := releaseName + "-proxy-config"
661+
662+
if err := rm.K8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: proxyName}, &proxy); err != nil {
663+
return nil, fmt.Errorf("error getting GatewayClass NginxProxy: %w", err)
664+
}
665+
666+
return &proxy, nil
667+
}
668+
669+
// ScaleNginxDeployment scales the Nginx Deployment to the specified number of replicas.
670+
func (rm *ResourceManager) ScaleNginxDeployment(namespace, releaseName string, replicas int32) error {
671+
ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.UpdateTimeout)
672+
defer cancel()
673+
674+
// If there is another NginxProxy which "overrides" the gateway class level one, then this won't work and
675+
// may need refactoring.
676+
proxy, err := rm.getGatewayClassNginxProxy(namespace, releaseName)
677+
if err != nil {
678+
return fmt.Errorf("error getting NginxProxy: %w", err)
679+
}
680+
681+
proxy.Spec.Kubernetes.Deployment.Replicas = &replicas
682+
683+
if err = rm.K8sClient.Update(ctx, proxy); err != nil {
684+
return fmt.Errorf("error updating NginxProxy: %w", err)
685+
}
686+
687+
return nil
688+
}
689+
650690
// GetEvents returns all Events in the specified namespace.
651691
func (rm *ResourceManager) GetEvents(namespace string) (*core.EventList, error) {
652692
ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.GetTimeout)

‎tests/framework/timeout.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ func DefaultTimeoutConfig() TimeoutConfig {
4343
CreateTimeout: 60 * time.Second,
4444
UpdateTimeout: 60 * time.Second,
4545
DeleteTimeout: 10 * time.Second,
46-
DeleteNamespaceTimeout: 60 * time.Second,
46+
DeleteNamespaceTimeout: 90 * time.Second,
4747
GetTimeout: 10 * time.Second,
4848
ManifestFetchTimeout: 10 * time.Second,
4949
RequestTimeout: 10 * time.Second,

‎tests/suite/manifests/scale/zero-downtime/values-affinity.yaml

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -7,13 +7,6 @@ nginxGateway:
77
- sleep
88
- --duration=40s
99
terminationGracePeriodSeconds: 50
10-
affinity:
11-
podAntiAffinity:
12-
requiredDuringSchedulingIgnoredDuringExecution:
13-
- topologyKey: kubernetes.io/hostname
14-
labelSelector:
15-
matchLabels:
16-
app.kubernetes.io/name: nginx-gateway
1710

1811

1912
nginx:
@@ -26,3 +19,10 @@ nginx:
2619
command:
2720
- /bin/sleep
2821
- "40"
22+
affinity:
23+
podAntiAffinity:
24+
requiredDuringSchedulingIgnoredDuringExecution:
25+
- topologyKey: kubernetes.io/hostname
26+
labelSelector:
27+
matchLabels:
28+
app.kubernetes.io/name: gateway-nginx

‎tests/suite/scale_test.go

Lines changed: 22 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -711,13 +711,13 @@ var _ = Describe("Zero downtime scale test", Ordered, Label("nfr", "zero-downtim
711711
}
712712

713713
var (
714-
outFile *os.File
715-
resultsDir string
716-
ngfDeploymentName string
717-
ns core.Namespace
718-
metricsCh chan *metricsResults
714+
outFile *os.File
715+
resultsDir string
716+
ns core.Namespace
717+
metricsCh chan *metricsResults
719718

720-
files = []string{
719+
numCoffeeAndTeaPods = 20
720+
files = []string{
721721
"scale/zero-downtime/cafe.yaml",
722722
"scale/zero-downtime/cafe-secret.yaml",
723723
"scale/zero-downtime/gateway-1.yaml",
@@ -858,12 +858,12 @@ var _ = Describe("Zero downtime scale test", Ordered, Label("nfr", "zero-downtim
858858
numReplicas int
859859
}{
860860
{
861-
name: "One NGF Pod runs per node",
861+
name: "One NGINX Pod runs per node",
862862
valuesFile: "manifests/scale/zero-downtime/values-affinity.yaml",
863863
numReplicas: 12, // equals number of nodes
864864
},
865865
{
866-
name: "Multiple NGF Pods run per node",
866+
name: "Multiple NGINX Pods run per node",
867867
valuesFile: "manifests/scale/zero-downtime/values.yaml",
868868
numReplicas: 24, // twice the number of nodes
869869
},
@@ -876,18 +876,15 @@ var _ = Describe("Zero downtime scale test", Ordered, Label("nfr", "zero-downtim
876876
cfg.nfr = true
877877
setup(cfg, "--values", test.valuesFile)
878878

879-
deploy, err := resourceManager.GetNGFDeployment(ngfNamespace, releaseName)
880-
Expect(err).ToNot(HaveOccurred())
881-
ngfDeploymentName = deploy.GetName()
882-
883879
Expect(resourceManager.Apply([]client.Object{&ns})).To(Succeed())
884880
Expect(resourceManager.ApplyFromFiles(files, ns.Name)).To(Succeed())
885881
Expect(resourceManager.WaitForAppsToBeReady(ns.Name)).To(Succeed())
886882

887883
var nginxPodNames []string
884+
var err error
888885
Eventually(
889886
func() bool {
890-
nginxPodNames, err := framework.GetReadyNginxPodNames(k8sClient, ns.Name, timeoutConfig.GetTimeout)
887+
nginxPodNames, err = framework.GetReadyNginxPodNames(k8sClient, ns.Name, timeoutConfig.GetTimeout)
891888
return len(nginxPodNames) == 1 && err == nil
892889
}).
893890
WithTimeout(timeoutConfig.CreateTimeout).
@@ -931,8 +928,8 @@ var _ = Describe("Zero downtime scale test", Ordered, Label("nfr", "zero-downtim
931928

932929
// scale NGF up one at a time
933930
for i := 2; i <= test.numReplicas; i++ {
934-
Eventually(resourceManager.ScaleDeployment).
935-
WithArguments(ngfNamespace, ngfDeploymentName, int32(i)).
931+
Eventually(resourceManager.ScaleNginxDeployment).
932+
WithArguments(ngfNamespace, releaseName, int32(i)).
936933
WithTimeout(timeoutConfig.UpdateTimeout).
937934
WithPolling(500 * time.Millisecond).
938935
Should(Succeed())
@@ -942,7 +939,7 @@ var _ = Describe("Zero downtime scale test", Ordered, Label("nfr", "zero-downtim
942939

943940
ctx, cancel := context.WithTimeout(context.Background(), timeoutConfig.UpdateTimeout)
944941

945-
Expect(resourceManager.WaitForPodsToBeReadyWithCount(ctx, ngfNamespace, i)).To(Succeed())
942+
Expect(resourceManager.WaitForPodsToBeReadyWithCount(ctx, ns.Name, i+numCoffeeAndTeaPods)).To(Succeed())
946943
Expect(resourceManager.WaitForGatewayObservedGeneration(ctx, ns.Name, "gateway", i)).To(Succeed())
947944

948945
cancel()
@@ -984,8 +981,8 @@ var _ = Describe("Zero downtime scale test", Ordered, Label("nfr", "zero-downtim
984981
// scale NGF down one at a time
985982
currentGen := test.numReplicas
986983
for i := test.numReplicas - 1; i >= 1; i-- {
987-
Eventually(resourceManager.ScaleDeployment).
988-
WithArguments(ngfNamespace, ngfDeploymentName, int32(i)).
984+
Eventually(resourceManager.ScaleNginxDeployment).
985+
WithArguments(ngfNamespace, releaseName, int32(i)).
989986
WithTimeout(timeoutConfig.UpdateTimeout).
990987
WithPolling(500 * time.Millisecond).
991988
Should(Succeed())
@@ -994,7 +991,12 @@ var _ = Describe("Zero downtime scale test", Ordered, Label("nfr", "zero-downtim
994991
Expect(resourceManager.ApplyFromFiles([]string{gatewayFile}, ns.Name)).To(Succeed())
995992
currentGen++
996993

994+
ctx, cancel := context.WithTimeout(context.Background(), timeoutConfig.UpdateTimeout)
995+
997996
time.Sleep(terminationTime)
997+
Expect(resourceManager.WaitForGatewayObservedGeneration(ctx, ns.Name, "gateway", currentGen)).To(Succeed())
998+
999+
cancel()
9981000
}
9991001

10001002
wg.Wait()
@@ -1049,7 +1051,7 @@ var _ = Describe("Zero downtime scale test", Ordered, Label("nfr", "zero-downtim
10491051
// allow traffic flow to start
10501052
time.Sleep(2 * time.Second)
10511053

1052-
Expect(resourceManager.ScaleDeployment(ngfNamespace, ngfDeploymentName, int32(test.numReplicas))).To(Succeed())
1054+
Expect(resourceManager.ScaleNginxDeployment(ngfNamespace, releaseName, int32(test.numReplicas))).To(Succeed())
10531055
Expect(resourceManager.ApplyFromFiles([]string{"scale/zero-downtime/gateway-2.yaml"}, ns.Name)).To(Succeed())
10541056
checkGatewayListeners(3)
10551057

@@ -1081,7 +1083,7 @@ var _ = Describe("Zero downtime scale test", Ordered, Label("nfr", "zero-downtim
10811083
// allow traffic flow to start
10821084
time.Sleep(2 * time.Second)
10831085

1084-
Expect(resourceManager.ScaleDeployment(ngfNamespace, ngfDeploymentName, int32(1))).To(Succeed())
1086+
Expect(resourceManager.ScaleNginxDeployment(ngfNamespace, releaseName, int32(1))).To(Succeed())
10851087
Expect(resourceManager.ApplyFromFiles([]string{"scale/zero-downtime/gateway-1.yaml"}, ns.Name)).To(Succeed())
10861088
checkGatewayListeners(2)
10871089

‎tests/suite/upgrade_test.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,7 @@ var _ = Describe("Upgrade testing", Label("nfr", "upgrade"), func() {
9191
})
9292

9393
It("upgrades NGF with zero downtime", func() {
94+
Skip("Skipping test until version 2.1.0 since 2.0.0 is a breaking change")
9495
nginxImage := *nginxImageRepository
9596
if *plusEnabled {
9697
nginxImage = *nginxPlusImageRepository

0 commit comments

Comments
 (0)