Skip to content

Commit bb3f186

Browse files
committed
Add crossplane framework for testing
Problem: We want a way to verify nginx configuration reliably in our tests. This is especially useful when introducing new policies, without the desire for testing nginx functionality directly. Solution: Added a framework for getting the nginx config and passing through crossplane into a structured JSON format for easier parsing. Because we now use a local container for crossplane in our functional tests, we'll only support running these tests in a kind cluster.
1 parent bf17bd5 commit bb3f186

File tree

9 files changed

+433
-64
lines changed

9 files changed

+433
-64
lines changed

tests/Dockerfile.crossplane

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
FROM python:3.12-alpine
2+
3+
ARG NGINX_CONF_DIR
4+
5+
RUN pip install crossplane
6+
7+
COPY ${NGINX_CONF_DIR}/nginx.conf /etc/nginx/nginx.conf
8+
9+
USER 101:1001
10+
11+
ENTRYPOINT ["sh"]

tests/Makefile

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ GW_SERVICE_TYPE = NodePort## Service type to use for the gateway
1212
GW_SVC_GKE_INTERNAL = false
1313
NGF_VERSION ?= edge## NGF version to be tested
1414
PULL_POLICY = Never## Pull policy for the images
15+
NGINX_CONF_DIR = internal/mode/static/nginx/conf
1516
PROVISIONER_MANIFEST = conformance/provisioner/provisioner.yaml
1617
SUPPORTED_EXTENDED_FEATURES = HTTPRouteQueryParamMatching,HTTPRouteMethodMatching,HTTPRoutePortRedirect,HTTPRouteSchemeRedirect,HTTPRouteHostRewrite,HTTPRoutePathRewrite,GatewayPort8080,HTTPRouteResponseHeaderModification
1718
STANDARD_CONFORMANCE_PROFILES = GATEWAY-HTTP,GATEWAY-GRPC
@@ -38,6 +39,10 @@ update-go-modules: ## Update the gateway-api go modules to latest main version
3839
build-test-runner-image: ## Build conformance test runner image
3940
docker build -t $(CONFORMANCE_PREFIX):$(CONFORMANCE_TAG) -f conformance/Dockerfile .
4041

42+
.PHONY: build-crossplane-image
43+
build-crossplane-image: ## Build the crossplane image
44+
docker build --build-arg NGINX_CONF_DIR=$(NGINX_CONF_DIR) -t nginx-crossplane:latest -f Dockerfile.crossplane ..
45+
4146
.PHONY: run-conformance-tests
4247
run-conformance-tests: ## Run conformance tests
4348
kind load docker-image $(CONFORMANCE_PREFIX):$(CONFORMANCE_TAG) --name $(CLUSTER_NAME)
@@ -80,9 +85,6 @@ ifeq ($(PLUS_ENABLED),true)
8085
NGINX_PREFIX := $(NGINX_PLUS_PREFIX)
8186
endif
8287

83-
.PHONY: setup-gcp-and-run-tests
84-
setup-gcp-and-run-tests: create-gke-router create-and-setup-vm run-tests-on-vm ## Create and setup a GKE router and GCP VM for tests and run the functional tests
85-
8688
.PHONY: setup-gcp-and-run-nfr-tests
8789
setup-gcp-and-run-nfr-tests: create-gke-router create-and-setup-vm nfr-test ## Create and setup a GKE router and GCP VM for tests and run the NFR tests
8890

@@ -102,13 +104,9 @@ create-gke-router: ## Create a GKE router to allow egress traffic from private n
102104
sync-files-to-vm: ## Syncs your local NGF files with the NGF repo on the VM
103105
./scripts/sync-files-to-vm.sh
104106

105-
.PHONY: run-tests-on-vm
106-
run-tests-on-vm: ## Run the functional tests on a GCP VM
107-
./scripts/run-tests-gcp-vm.sh
108-
109107
.PHONY: nfr-test
110108
nfr-test: ## Run the NFR tests on a GCP VM
111-
NFR=true CI=$(CI) ./scripts/run-tests-gcp-vm.sh
109+
CI=$(CI) ./scripts/run-tests-gcp-vm.sh
112110

113111
.PHONY: start-longevity-test
114112
start-longevity-test: export START_LONGEVITY=true
@@ -130,7 +128,8 @@ stop-longevity-test: nfr-test ## Stop the longevity test and collects results
130128
--is-gke-internal-lb=$(GW_SVC_GKE_INTERNAL)
131129

132130
.PHONY: test
133-
test: ## Runs the functional tests on your default k8s cluster
131+
test: build-crossplane-image ## Runs the functional tests on your kind k8s cluster
132+
kind load docker-image nginx-crossplane:latest --name $(CLUSTER_NAME)
134133
go run github.com/onsi/ginkgo/v2/ginkgo --randomize-all --randomize-suites --keep-going --fail-on-pending \
135134
--trace -r -v --buildvcs --force-newlines $(GITHUB_OUTPUT) \
136135
--label-filter "functional" $(GINKGO_FLAGS) ./suite -- \

tests/README.md

Lines changed: 9 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -28,10 +28,8 @@ This directory contains the tests for NGINX Gateway Fabric. The tests are divide
2828
- [System Testing](#system-testing)
2929
- [Logging in tests](#logging-in-tests)
3030
- [Step 1 - Run the tests](#step-1---run-the-tests)
31-
- [1a - Run the functional tests locally](#1a---run-the-functional-tests-locally)
32-
- [1b - Run the tests on a GKE cluster from a GCP VM](#1b---run-the-tests-on-a-gke-cluster-from-a-gcp-vm)
33-
- [Functional Tests](#functional-tests)
34-
- [NFR tests](#nfr-tests)
31+
- [Run the functional tests locally](#run-the-functional-tests-locally)
32+
- [Run the NFR tests on a GKE cluster from a GCP VM](#run-the-nfr-tests-on-a-gke-cluster-from-a-gcp-vm)
3533
- [Longevity testing](#longevity-testing)
3634
- [Common test amendments](#common-test-amendments)
3735
- [Step 2 - Cleanup](#step-2---cleanup)
@@ -47,7 +45,7 @@ This directory contains the tests for NGINX Gateway Fabric. The tests are divide
4745
- [yq](https://github.com/mikefarah/yq/#install)
4846
- Make.
4947

50-
If running NFR tests, or running functional tests in GKE:
48+
If running NFR tests:
5149

5250
- The [gcloud CLI](https://cloud.google.com/sdk/docs/install)
5351
- A GKE cluster (if `master-authorized-networks` is enabled, please set `ADD_VM_IP_AUTH_NETWORKS=true` in your vars.env file)
@@ -59,9 +57,7 @@ All the commands below are executed from the `tests` directory. You can see all
5957

6058
### Step 1 - Create a Kubernetes cluster
6159

62-
This can be done in a cloud provider of choice, or locally using `kind`.
63-
64-
**Important**: NFR tests can only be run on a GKE cluster.
60+
**Important**: Functional/conformance tests can only be run on a `kind` cluster. NFR tests can only be run on a GKE cluster.
6561

6662
To create a local `kind` cluster:
6763

@@ -237,7 +233,7 @@ When running locally, the tests create a port-forward from your NGF Pod to local
237233
test framework. Traffic is sent over this port. If running on a GCP VM targeting a GKE cluster, the tests will create an
238234
internal LoadBalancer service which will receive the test traffic.
239235

240-
**Important**: NFR tests can only be run on a GKE cluster.
236+
**Important**: Functional tests can only be run on a `kind` cluster. NFR tests can only be run on a GKE cluster.
241237

242238
Directory structure is as follows:
243239

@@ -252,7 +248,7 @@ To log in the tests, use the `GinkgoWriter` interface described here: https://on
252248

253249
### Step 1 - Run the tests
254250

255-
#### 1a - Run the functional tests locally
251+
#### Run the functional tests locally
256252

257253
```makefile
258254
make test TAG=$(whoami)
@@ -273,9 +269,7 @@ To run the telemetry test:
273269
make test TAG=$(whoami) GINKGO_LABEL=telemetry
274270
```
275271

276-
#### 1b - Run the tests on a GKE cluster from a GCP VM
277-
278-
This step only applies if you are running the NFR tests, or would like to run the functional tests on a GKE cluster from a GCP based VM.
272+
#### Run the NFR tests on a GKE cluster from a GCP VM
279273

280274
Before running the below `make` commands, copy the `scripts/vars.env-example` file to `scripts/vars.env` and populate the
281275
required env vars. `GKE_SVC_ACCOUNT` needs to be the name of a service account that has Kubernetes admin permissions.
@@ -292,7 +286,7 @@ To just set up the VM with no router (this will not run the tests):
292286
make create-and-setup-vm
293287
```
294288

295-
Otherwise, you can set up the VM, router, and run the tests with a single command. See the options in the sections below.
289+
Otherwise, you can set up the VM, router, and run the tests with a single command. See the options below.
296290

297291
By default, the tests run using the version of NGF that was `git cloned` during the setup. If you want to make
298292
incremental changes and copy your local changes to the VM to test, you can run
@@ -301,22 +295,6 @@ incremental changes and copy your local changes to the VM to test, you can run
301295
make sync-files-to-vm
302296
```
303297

304-
#### Functional Tests
305-
306-
To set up the GCP environment with the router and VM and then run the tests, run the following command:
307-
308-
```makefile
309-
make setup-gcp-and-run-tests
310-
```
311-
312-
To use an existing VM to run the tests, run the following
313-
314-
```makefile
315-
make run-tests-on-vm
316-
```
317-
318-
#### NFR tests
319-
320298
To set up the GCP environment with the router and VM and then run the tests, run the following command:
321299

322300
```makefile
@@ -374,7 +352,7 @@ or to pass a specific flag, e.g. run a specific test, use the GINKGO_FLAGS varia
374352
make test TAG=$(whoami) GINKGO_FLAGS='-ginkgo.focus "writes the system info to a results file"'
375353
```
376354

377-
> Note: if filtering on NFR tests (or functional tests on GKE), set the filter in the appropriate field in your `vars.env` file.
355+
> Note: if filtering on NFR tests, set the filter in the appropriate field in your `vars.env` file.
378356
379357
If you are running the tests in GCP, add your required label/ flags to `scripts/var.env`.
380358

tests/framework/crossplane.go

Lines changed: 218 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,218 @@
1+
package framework
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"net/http"
7+
"strings"
8+
"time"
9+
10+
core "k8s.io/api/core/v1"
11+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
12+
"k8s.io/client-go/kubernetes"
13+
"k8s.io/client-go/kubernetes/scheme"
14+
"k8s.io/client-go/rest"
15+
"k8s.io/client-go/tools/remotecommand"
16+
)
17+
18+
// ExpectedNginxField contains an nginx directive key and value,
19+
// and the expected file, server, and location block that it should exist in.
20+
type ExpectedNginxField struct {
21+
// Key is the directive name.
22+
Key string
23+
// Value is the value for the directive. Can be the full value or a substring.
24+
Value string
25+
// File is the file name that should contain the directive. Can be a full filename or a substring.
26+
File string
27+
// Location is the location name that the directive should exist in.
28+
Location string
29+
// Servers are the server names that the directive should exist in.
30+
Servers []string
31+
// ValueSubstringAllowed allows the expected value to be a substring of the real value.
32+
// This makes it easier for cases when real values are complex file names or contain things we
33+
// don't care about, and we just want to check if a substring exists.
34+
ValueSubstringAllowed bool
35+
}
36+
37+
// ValidateNginxFieldExists accepts the nginx config and the configuration for the expected field,
38+
// and returns whether or not that field exists where it should.
39+
func ValidateNginxFieldExists(conf *Payload, expFieldCfg ExpectedNginxField) bool {
40+
for _, config := range conf.Config {
41+
if !strings.Contains(config.File, expFieldCfg.File) {
42+
continue
43+
}
44+
45+
for _, directive := range config.Parsed {
46+
if len(expFieldCfg.Servers) == 0 {
47+
if expFieldCfg.fieldFound(directive) {
48+
return true
49+
}
50+
continue
51+
}
52+
53+
for _, serverName := range expFieldCfg.Servers {
54+
if directive.Directive == "server" && getServerName(directive.Block) == serverName {
55+
for _, serverDirective := range directive.Block {
56+
if expFieldCfg.Location == "" && expFieldCfg.fieldFound(serverDirective) {
57+
return true
58+
} else if serverDirective.Directive == "location" &&
59+
fieldExistsInLocation(serverDirective, expFieldCfg) {
60+
return true
61+
}
62+
}
63+
}
64+
}
65+
}
66+
}
67+
68+
return false
69+
}
70+
71+
func getServerName(serverBlock Directives) string {
72+
for _, directive := range serverBlock {
73+
if directive.Directive == "server_name" {
74+
return directive.Args[0]
75+
}
76+
}
77+
78+
return ""
79+
}
80+
81+
func (e ExpectedNginxField) fieldFound(directive *Directive) bool {
82+
arg := strings.Join(directive.Args, " ")
83+
84+
valueMatch := arg == e.Value
85+
if e.ValueSubstringAllowed {
86+
valueMatch = strings.Contains(arg, e.Value)
87+
}
88+
89+
return directive.Directive == e.Key && valueMatch
90+
}
91+
92+
func fieldExistsInLocation(serverDirective *Directive, expFieldCfg ExpectedNginxField) bool {
93+
// location could start with '=', so get the last element which is the path
94+
loc := serverDirective.Args[len(serverDirective.Args)-1]
95+
if loc == expFieldCfg.Location {
96+
for _, locDirective := range serverDirective.Block {
97+
if expFieldCfg.fieldFound(locDirective) {
98+
return true
99+
}
100+
}
101+
}
102+
103+
return false
104+
}
105+
106+
// injectCrossplaneContainer adds an ephemeral container that contains crossplane for parsing
107+
// nginx config. It attaches to the nginx container and shares volumes with it.
108+
func injectCrossplaneContainer(
109+
k8sClient kubernetes.Interface,
110+
timeout time.Duration,
111+
ngfPodName,
112+
namespace string,
113+
) error {
114+
ctx, cancel := context.WithTimeout(context.Background(), timeout)
115+
defer cancel()
116+
117+
pod := &core.Pod{
118+
ObjectMeta: metav1.ObjectMeta{
119+
Name: ngfPodName,
120+
Namespace: namespace,
121+
},
122+
Spec: core.PodSpec{
123+
EphemeralContainers: []core.EphemeralContainer{
124+
{
125+
TargetContainerName: "nginx",
126+
EphemeralContainerCommon: core.EphemeralContainerCommon{
127+
Name: "crossplane",
128+
Image: "nginx-crossplane:latest",
129+
ImagePullPolicy: "Never",
130+
Stdin: true,
131+
VolumeMounts: []core.VolumeMount{
132+
{
133+
MountPath: "/etc/nginx/conf.d",
134+
Name: "nginx-conf",
135+
},
136+
{
137+
MountPath: "/etc/nginx/stream-conf.d",
138+
Name: "nginx-stream-conf",
139+
},
140+
{
141+
MountPath: "/etc/nginx/module-includes",
142+
Name: "module-includes",
143+
},
144+
{
145+
MountPath: "/etc/nginx/secrets",
146+
Name: "nginx-secrets",
147+
},
148+
{
149+
MountPath: "/etc/nginx/includes",
150+
Name: "nginx-includes",
151+
},
152+
},
153+
},
154+
},
155+
},
156+
},
157+
}
158+
159+
podClient := k8sClient.CoreV1().Pods(namespace)
160+
if _, err := podClient.UpdateEphemeralContainers(ctx, ngfPodName, pod, metav1.UpdateOptions{}); err != nil {
161+
return fmt.Errorf("error adding ephemeral container: %w", err)
162+
}
163+
164+
return nil
165+
}
166+
167+
// createCrossplaneExecutor creates the executor for the crossplane command.
168+
func createCrossplaneExecutor(
169+
k8sClient kubernetes.Interface,
170+
k8sConfig *rest.Config,
171+
ngfPodName,
172+
namespace string,
173+
) (remotecommand.Executor, error) {
174+
cmd := []string{"crossplane", "parse", "/etc/nginx/nginx.conf"}
175+
opts := &core.PodExecOptions{
176+
Command: cmd,
177+
Container: "crossplane",
178+
Stdout: true,
179+
Stderr: true,
180+
}
181+
182+
req := k8sClient.CoreV1().RESTClient().Post().
183+
Resource("pods").
184+
SubResource("exec").
185+
Name(ngfPodName).
186+
Namespace(namespace).
187+
VersionedParams(opts, scheme.ParameterCodec)
188+
189+
exec, err := remotecommand.NewSPDYExecutor(k8sConfig, http.MethodPost, req.URL())
190+
if err != nil {
191+
return nil, fmt.Errorf("error creating executor: %w", err)
192+
}
193+
194+
return exec, nil
195+
}
196+
197+
// The following types are copied from https://github.com/nginxinc/nginx-go-crossplane,
198+
// with unnecessary fields stripped out.
199+
type Payload struct {
200+
Config []Config `json:"config"`
201+
}
202+
203+
type Config struct {
204+
File string `json:"file"`
205+
Parsed Directives `json:"parsed"`
206+
}
207+
208+
type Directive struct {
209+
Comment *string `json:"comment,omitempty"`
210+
Directive string `json:"directive"`
211+
File string `json:"file,omitempty"`
212+
Args []string `json:"args"`
213+
Includes []int `json:"includes,omitempty"`
214+
Block Directives `json:"block,omitempty"`
215+
Line int `json:"line"`
216+
}
217+
218+
type Directives []*Directive

0 commit comments

Comments
 (0)