CP/DP Update non-functional tests (#3305)

Update non-functional tests for the control plane data plane split.

Problem: The non-functional tests do not work for the control plane data plane split changes.

Solution: Update non-functional tests.

Testing: Scale, Reconfiguration, Performance, and Longevity tests work. Upgrade test doesn't work, however that is sort of planned since the CP/DP split is a breaking change of NGF and thus you can't easily upgrade with zero downtime.

---------

Co-authored-by: Saylor Berman <[email protected]>

Author: bjee19

.github/workflows/nfr.yml

@@ -92,6 +92,13 @@ jobs:
           workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY }}
           service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }}
 
+      - name: Login to GAR
+        uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
+        with:
+          registry: us-docker.pkg.dev
+          username: oauth2accesstoken
+          password: ${{ steps.auth.outputs.access_token }}
+
       - name: Set up Cloud SDK
         uses: google-github-actions/setup-gcloud@77e7a554d41e2ee56fc945c52dfd3f33d12def9a # v2.1.4
         with:

charts/nginx-gateway-fabric/values.yaml

@@ -375,19 +375,19 @@ nginx:
     # -- The termination grace period of the NGINX data plane pod.
     # terminationGracePeriodSeconds: 30
 
-    # -- Tolerations for the NGINX Gateway Fabric control plane pod.
+    # -- Tolerations for the NGINX data plane pod.
     # tolerations: []
 
-    # -- The nodeSelector of the NGINX Gateway Fabric control plane pod.
+    # -- The nodeSelector of the NGINX data plane pod.
     # nodeSelector: {}
 
-    # -- The affinity of the NGINX Gateway Fabric control plane pod.
+    # -- The affinity of the NGINX data plane pod.
     # affinity: {}
 
-    # -- The topology spread constraints for the NGINX Gateway Fabric control plane pod.
+    # -- The topology spread constraints for the NGINX data plane pod.
     # topologySpreadConstraints: []
 
-    # -- extraVolumes for the NGINX Gateway Fabric control plane pod. Use in conjunction with
+    # -- extraVolumes for the NGINX data plane pod. Use in conjunction with
     # nginx.container.extraVolumeMounts mount additional volumes to the container.
     # extraVolumes: []
 

internal/mode/static/handler_test.go

@@ -249,8 +249,6 @@ var _ = Describe("eventHandler", func() {
 				handler.HandleEventBatch(context.Background(), logr.Discard(), batch)
 
 				checkUpsertEventExpectations(e)
-				Expect(fakeProvisioner.RegisterGatewayCallCount()).Should(Equal(0))
-				Expect(fakeGenerator.GenerateCallCount()).Should(Equal(0))
 				// status update should still occur for GatewayClasses
 				Eventually(
 					func() int {
@@ -305,7 +303,7 @@ var _ = Describe("eventHandler", func() {
 			Eventually(
 				func() int {
 					return fakeStatusUpdater.UpdateGroupCallCount()
-				}).Should(Equal(1))
+				}).Should(BeNumerically(">", 1))
 
 			_, name, reqs := fakeStatusUpdater.UpdateGroupArgsForCall(0)
 			Expect(name).To(Equal(groupControlPlane))
@@ -324,7 +322,7 @@ var _ = Describe("eventHandler", func() {
 			Eventually(
 				func() int {
 					return fakeStatusUpdater.UpdateGroupCallCount()
-				}).Should(Equal(1))
+				}).Should(BeNumerically(">", 1))
 
 			_, name, reqs := fakeStatusUpdater.UpdateGroupArgsForCall(0)
 			Expect(name).To(Equal(groupControlPlane))
@@ -356,7 +354,7 @@ var _ = Describe("eventHandler", func() {
 			Eventually(
 				func() int {
 					return fakeStatusUpdater.UpdateGroupCallCount()
-				}).Should(Equal(1))
+				}).Should(BeNumerically(">", 1))
 
 			_, name, reqs := fakeStatusUpdater.UpdateGroupArgsForCall(0)
 			Expect(name).To(Equal(groupControlPlane))

tests/Makefile

@@ -44,7 +44,7 @@ build-test-runner-image: ## Build conformance test runner image
 
 .PHONY: build-crossplane-image
 build-crossplane-image: ## Build the crossplane image
-	docker build --build-arg NGINX_CONF_DIR=$(NGINX_CONF_DIR) -t nginx-crossplane:latest -f framework/crossplane/Dockerfile ..
+	docker build --platform $(GOOS)/$(GOARCH) --build-arg NGINX_CONF_DIR=$(NGINX_CONF_DIR) -t nginx-crossplane:latest -f framework/crossplane/Dockerfile ..
 
 .PHONY: run-conformance-tests
 run-conformance-tests: ## Run conformance tests
@@ -105,7 +105,9 @@ sync-files-to-vm: ## Syncs your local NGF files with the NGF repo on the VM
 	./scripts/sync-files-to-vm.sh
 
 .PHONY: nfr-test
-nfr-test: check-for-plus-usage-endpoint ## Run the NFR tests on a GCP VM
+nfr-test: GOARCH=amd64
+nfr-test: check-for-plus-usage-endpoint build-crossplane-image  ## Run the NFR tests on a GCP VM
+	./scripts/push-crossplane-image.sh
 	CI=$(CI) ./scripts/run-tests-gcp-vm.sh
 
 .PHONY: start-longevity-test

tests/framework/crossplane.go

@@ -38,6 +38,8 @@ type ExpectedNginxField struct {
 	ValueSubstringAllowed bool
 }
 
+const crossplaneImageName = "nginx-crossplane:latest"
+
 // ValidateNginxFieldExists accepts the nginx config and the configuration for the expected field,
 // and returns whether or not that field exists where it should.
 func ValidateNginxFieldExists(conf *Payload, expFieldCfg ExpectedNginxField) error {
@@ -144,11 +146,17 @@ func injectCrossplaneContainer(
 	k8sClient kubernetes.Interface,
 	timeout time.Duration,
 	ngfPodName,
-	namespace string,
+	namespace,
+	crossplaneImageRepo string,
 ) error {
 	ctx, cancel := context.WithTimeout(context.Background(), timeout)
 	defer cancel()
 
+	image := crossplaneImageName
+	if crossplaneImageRepo != "" {
+		image = crossplaneImageRepo + "/" + image
+	}
+
 	pod := &core.Pod{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      ngfPodName,
@@ -160,8 +168,8 @@ func injectCrossplaneContainer(
 					TargetContainerName: "nginx",
 					EphemeralContainerCommon: core.EphemeralContainerCommon{
 						Name:            "crossplane",
-						Image:           "nginx-crossplane:latest",
-						ImagePullPolicy: "Never",
+						Image:           image,
+						ImagePullPolicy: "IfNotPresent",
 						Stdin:           true,
 						VolumeMounts: []core.VolumeMount{
 							{

tests/framework/info.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"runtime/debug"
 
+	. "github.com/onsi/ginkgo/v2"
 	core "k8s.io/api/core/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
@@ -81,3 +82,27 @@ func GetBuildInfo() (commitHash string, commitTime string, dirtyBuild string) {
 
 	return
 }
+
+// AddNginxLogsAndEventsToReport adds nginx logs and events from the namespace to the report if the spec failed.
+func AddNginxLogsAndEventsToReport(rm ResourceManager, namespace string) {
+	if CurrentSpecReport().Failed() {
+		var returnLogs string
+
+		nginxPodNames, _ := GetReadyNginxPodNames(rm.K8sClient, namespace, rm.TimeoutConfig.GetStatusTimeout)
+
+		for _, nginxPodName := range nginxPodNames {
+			returnLogs += fmt.Sprintf("Logs for Nginx Pod %s:\n", nginxPodName)
+			nginxLogs, _ := rm.GetPodLogs(
+				namespace,
+				nginxPodName,
+				&core.PodLogOptions{Container: "nginx"},
+			)
+
+			returnLogs += fmt.Sprintf("  %s\n", nginxLogs)
+		}
+		AddReportEntry("Nginx Logs", returnLogs, ReportEntryVisibilityNever)
+
+		events := GetEvents(rm, namespace)
+		AddReportEntry("Test Events", events, ReportEntryVisibilityNever)
+	}
+}

tests/framework/prometheus.go

@@ -302,119 +302,6 @@ type Bucket struct {
 	Val int
 }
 
-// GetReloadCount gets the total number of nginx reloads.
-func GetReloadCount(promInstance PrometheusInstance, ngfPodName string) (float64, error) {
-	return getFirstValueOfVector(
-		fmt.Sprintf(
-			`nginx_gateway_fabric_nginx_reloads_total{pod="%[1]s"}`,
-			ngfPodName,
-		),
-		promInstance,
-	)
-}
-
-// GetReloadCountWithStartTime gets the total number of nginx reloads from a start time to the current time.
-func GetReloadCountWithStartTime(
-	promInstance PrometheusInstance,
-	ngfPodName string,
-	startTime time.Time,
-) (float64, error) {
-	return getFirstValueOfVector(
-		fmt.Sprintf(
-			`nginx_gateway_fabric_nginx_reloads_total{pod="%[1]s"}`+
-				` - `+
-				`nginx_gateway_fabric_nginx_reloads_total{pod="%[1]s"} @ %d`,
-			ngfPodName,
-			startTime.Unix(),
-		),
-		promInstance,
-	)
-}
-
-// GetReloadErrsCountWithStartTime gets the total number of nginx reload errors from a start time to the current time.
-func GetReloadErrsCountWithStartTime(
-	promInstance PrometheusInstance,
-	ngfPodName string,
-	startTime time.Time,
-) (float64, error) {
-	return getFirstValueOfVector(
-		fmt.Sprintf(
-			`nginx_gateway_fabric_nginx_reload_errors_total{pod="%[1]s"}`+
-				` - `+
-				`nginx_gateway_fabric_nginx_reload_errors_total{pod="%[1]s"} @ %d`,
-			ngfPodName,
-			startTime.Unix(),
-		),
-		promInstance,
-	)
-}
-
-// GetReloadAvgTime gets the average time in milliseconds for nginx to reload.
-func GetReloadAvgTime(promInstance PrometheusInstance, ngfPodName string) (float64, error) {
-	return getFirstValueOfVector(
-		fmt.Sprintf(
-			`nginx_gateway_fabric_nginx_reloads_milliseconds_sum{pod="%[1]s"}`+
-				` / `+
-				`nginx_gateway_fabric_nginx_reloads_total{pod="%[1]s"}`,
-			ngfPodName,
-		),
-		promInstance,
-	)
-}
-
-// GetReloadAvgTimeWithStartTime gets the average time in milliseconds for nginx to reload using a start time
-// to the current time to calculate.
-func GetReloadAvgTimeWithStartTime(
-	promInstance PrometheusInstance,
-	ngfPodName string,
-	startTime time.Time,
-) (float64, error) {
-	return getFirstValueOfVector(
-		fmt.Sprintf(
-			`(nginx_gateway_fabric_nginx_reloads_milliseconds_sum{pod="%[1]s"}`+
-				` - `+
-				`nginx_gateway_fabric_nginx_reloads_milliseconds_sum{pod="%[1]s"} @ %[2]d)`+
-				` / `+
-				`(nginx_gateway_fabric_nginx_reloads_total{pod="%[1]s"}`+
-				` - `+
-				`nginx_gateway_fabric_nginx_reloads_total{pod="%[1]s"} @ %[2]d)`,
-			ngfPodName,
-			startTime.Unix(),
-		),
-		promInstance,
-	)
-}
-
-// GetReloadBuckets gets the Buckets in millisecond intervals for nginx reloads.
-func GetReloadBuckets(promInstance PrometheusInstance, ngfPodName string) ([]Bucket, error) {
-	return getBuckets(
-		fmt.Sprintf(
-			`nginx_gateway_fabric_nginx_reloads_milliseconds_bucket{pod="%[1]s"}`,
-			ngfPodName,
-		),
-		promInstance,
-	)
-}
-
-// GetReloadBucketsWithStartTime gets the Buckets in millisecond intervals for nginx reloads from a start time
-// to the current time.
-func GetReloadBucketsWithStartTime(
-	promInstance PrometheusInstance,
-	ngfPodName string,
-	startTime time.Time,
-) ([]Bucket, error) {
-	return getBuckets(
-		fmt.Sprintf(
-			`nginx_gateway_fabric_nginx_reloads_milliseconds_bucket{pod="%[1]s"}`+
-				` - `+
-				`nginx_gateway_fabric_nginx_reloads_milliseconds_bucket{pod="%[1]s"} @ %d`,
-			ngfPodName,
-			startTime.Unix(),
-		),
-		promInstance,
-	)
-}
-
 // GetEventsCount gets the NGF event batch processing count.
 func GetEventsCount(promInstance PrometheusInstance, ngfPodName string) (float64, error) {
 	return getFirstValueOfVector(

tests/framework/resourcemanager.go

@@ -46,6 +46,8 @@ import (
 	"k8s.io/client-go/util/retry"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	v1 "sigs.k8s.io/gateway-api/apis/v1"
+
+	ngfAPIv1alpha2 "github.com/nginx/nginx-gateway-fabric/apis/v1alpha2"
 )
 
 // ResourceManager handles creating/updating/deleting Kubernetes resources.
@@ -647,6 +649,44 @@ func (rm *ResourceManager) GetNGFDeployment(namespace, releaseName string) (*app
 	return &deployment, nil
 }
 
+func (rm *ResourceManager) getGatewayClassNginxProxy(
+	namespace,
+	releaseName string,
+) (*ngfAPIv1alpha2.NginxProxy, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.GetTimeout)
+	defer cancel()
+
+	var proxy ngfAPIv1alpha2.NginxProxy
+	proxyName := releaseName + "-proxy-config"
+
+	if err := rm.K8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: proxyName}, &proxy); err != nil {
+		return nil, err
+	}
+
+	return &proxy, nil
+}
+
+// ScaleNginxDeployment scales the Nginx Deployment to the specified number of replicas.
+func (rm *ResourceManager) ScaleNginxDeployment(namespace, releaseName string, replicas int32) error {
+	ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.UpdateTimeout)
+	defer cancel()
+
+	// If there is another NginxProxy which "overrides" the gateway class  one, then this won't work and
+	// may need refactoring.
+	proxy, err := rm.getGatewayClassNginxProxy(namespace, releaseName)
+	if err != nil {
+		return fmt.Errorf("error getting NginxProxy: %w", err)
+	}
+
+	proxy.Spec.Kubernetes.Deployment.Replicas = &replicas
+
+	if err = rm.K8sClient.Update(ctx, proxy); err != nil {
+		return fmt.Errorf("error updating NginxProxy: %w", err)
+	}
+
+	return nil
+}
+
 // GetEvents returns all Events in the specified namespace.
 func (rm *ResourceManager) GetEvents(namespace string) (*core.EventList, error) {
 	ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.GetTimeout)
@@ -843,12 +883,14 @@ func (rm *ResourceManager) WaitForGatewayObservedGeneration(
 }
 
 // GetNginxConfig uses crossplane to get the nginx configuration and convert it to JSON.
-func (rm *ResourceManager) GetNginxConfig(nginxPodName, namespace string) (*Payload, error) {
+// If the crossplane image is loaded locally on the node, crossplaneImageRepo can be empty.
+func (rm *ResourceManager) GetNginxConfig(nginxPodName, namespace, crossplaneImageRepo string) (*Payload, error) {
 	if err := injectCrossplaneContainer(
 		rm.ClientGoClient,
 		rm.TimeoutConfig.UpdateTimeout,
 		nginxPodName,
 		namespace,
+		crossplaneImageRepo,
 	); err != nil {
 		return nil, err
 	}

tests/framework/timeout.go

@@ -43,7 +43,7 @@ func DefaultTimeoutConfig() TimeoutConfig {
 		CreateTimeout:           60 * time.Second,
 		UpdateTimeout:           60 * time.Second,
 		DeleteTimeout:           10 * time.Second,
-		DeleteNamespaceTimeout:  60 * time.Second,
+		DeleteNamespaceTimeout:  90 * time.Second,
 		GetTimeout:              10 * time.Second,
 		ManifestFetchTimeout:    10 * time.Second,
 		RequestTimeout:          10 * time.Second,

tests/scripts/push-crossplane-image.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+set -eo pipefail
+
+source scripts/vars.env
+
+docker tag nginx-crossplane:latest us-docker.pkg.dev/$GKE_PROJECT/nginx-gateway-fabric/nginx-crossplane:latest
+docker push us-docker.pkg.dev/$GKE_PROJECT/nginx-gateway-fabric/nginx-crossplane:latest

tests/suite/advanced_routing_test.go

@@ -48,6 +48,7 @@ var _ = Describe("AdvancedRouting", Ordered, Label("functional", "routing"), fun
 	})
 
 	AfterAll(func() {
+		framework.AddNginxLogsAndEventsToReport(resourceManager, namespace)
 		cleanUpPortForward()
 
 		Expect(resourceManager.DeleteFromFiles(files, namespace)).To(Succeed())