Skip to content

Commit d627248

Browse files
sjbermansjberman
committed
Run conformance tests nightly on main branch
In order to ensure we are staying ahead of Gateway API changes, we'll now run conformance tests off of the main branch on a nightly schedule. Devs can also run from the main branch locally by setting the GW_API_VERSION variable.
1 parent 5bf78d8 commit d627248

File tree

7 files changed

+152
-89
lines changed

7 files changed

+152
-89
lines changed

‎.github/workflows/ci.yml

-86
Original file line numberDiff line numberDiff line change
@@ -147,92 +147,6 @@ jobs:
147147
path: ${{ github.workspace }}/dist
148148
key: nginx-kubernetes-gateway-${{ github.run_id }}-${{ github.run_number }}
149149

150-
conformance-tests:
151-
name: Gateway Conformance Tests
152-
runs-on: ubuntu-22.04
153-
needs: vars
154-
if: ${{ github.ref_type != 'tag' }}
155-
steps:
156-
- name: Checkout Repository
157-
uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
158-
159-
- name: Setup Golang Environment
160-
uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1
161-
with:
162-
go-version-file: go.mod
163-
164-
- name: Docker Buildx
165-
uses: docker/setup-buildx-action@4c0219f9ac95b02789c1075625400b2acbff50b1 # v2.9.1
166-
167-
- name: Docker meta
168-
id: meta
169-
uses: docker/metadata-action@818d4b7b91585d195f67373fd9cb0332e31a7175 # v4.6.0
170-
with:
171-
images: |
172-
name=ghcr.io/nginxinc/nginx-kubernetes-gateway
173-
tags: |
174-
type=semver,pattern={{version}}
175-
type=edge
176-
type=ref,event=pr
177-
type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }}
178-
179-
- name: Prepare NKG files
180-
run: |
181-
nkg_prefix=$(echo ${{ steps.meta.outputs.tags }} | cut -d ":" -f 1)
182-
nkg_tag=$(echo ${{ steps.meta.outputs.tags }} | cut -d ":" -f 2)
183-
make update-nkg-manifest NKG_PREFIX=${nkg_prefix} NKG_TAG=${nkg_tag}
184-
working-directory: ./conformance
185-
186-
- name: Build binary
187-
uses: goreleaser/goreleaser-action@336e29918d653399e599bfca99fadc1d7ffbc9f7 # v4.3.0
188-
with:
189-
version: latest
190-
args: ${{ startsWith(github.ref, 'refs/tags/') && 'release' || 'build --snapshot' }} --clean
191-
env:
192-
GOPATH: ${{ needs.vars.outputs.go_path }}
193-
194-
- name: Build Docker Image
195-
uses: docker/build-push-action@2eb1c1961a95fc15694676618e422e8ba1d63825 # v4.1.1
196-
with:
197-
file: build/Dockerfile
198-
tags: ${{ steps.meta.outputs.tags }}
199-
context: "."
200-
target: goreleaser
201-
load: true
202-
cache-from: type=gha
203-
cache-to: type=gha,mode=max
204-
pull: true
205-
206-
- name: Build Test Docker Image
207-
uses: docker/build-push-action@2eb1c1961a95fc15694676618e422e8ba1d63825 # v4.1.1
208-
with:
209-
file: conformance/tests/Dockerfile
210-
tags: conformance-test-runner:${{ github.sha }}
211-
context: "."
212-
load: true
213-
cache-from: type=gha
214-
cache-to: type=gha,mode=max
215-
pull: true
216-
217-
- name: Deploy Kubernetes
218-
id: k8s
219-
run: |
220-
make create-kind-cluster KIND_KUBE_CONFIG=kube-${{ github.run_id }}
221-
echo "KUBECONFIG=kube-${{ github.run_id }}" >> "$GITHUB_ENV"
222-
working-directory: ./conformance
223-
224-
- name: Setup conformance tests
225-
run: |
226-
nkg_prefix=$(echo ${{ steps.meta.outputs.tags }} | cut -d ":" -f 1)
227-
nkg_tag=$(echo ${{ steps.meta.outputs.tags }} | cut -d ":" -f 2)
228-
make install-nkg-local-no-build NKG_PREFIX=${nkg_prefix} NKG_TAG=${nkg_tag}
229-
working-directory: ./conformance
230-
231-
- name: Run conformance tests
232-
run: |
233-
make run-conformance-tests TAG=${{ github.sha }}
234-
working-directory: ./conformance
235-
236150
build:
237151
name: Build Image
238152
runs-on: ubuntu-22.04

‎.github/workflows/conformance.yml

+113
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
name: Conformance Testing
2+
3+
on:
4+
push:
5+
branches:
6+
- main
7+
- release-*
8+
pull_request:
9+
branches:
10+
- main
11+
- release-*
12+
types:
13+
- opened
14+
- reopened
15+
- synchronize
16+
schedule:
17+
- cron: '0 4 * * *'
18+
19+
concurrency:
20+
group: ${{ github.ref_name }}-conformance
21+
cancel-in-progress: true
22+
23+
permissions:
24+
contents: read
25+
26+
jobs:
27+
conformance-tests:
28+
name: Gateway Conformance Tests
29+
runs-on: ubuntu-22.04
30+
steps:
31+
- name: Checkout Repository
32+
uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
33+
34+
- name: Setup Golang Environment
35+
uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1
36+
with:
37+
go-version-file: go.mod
38+
39+
- name: Set GOPATH
40+
run: echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV
41+
42+
- name: Docker Buildx
43+
uses: docker/setup-buildx-action@4c0219f9ac95b02789c1075625400b2acbff50b1 # v2.9.1
44+
45+
- name: Docker meta
46+
id: meta
47+
uses: docker/metadata-action@818d4b7b91585d195f67373fd9cb0332e31a7175 # v4.6.0
48+
with:
49+
images: |
50+
name=ghcr.io/nginxinc/nginx-kubernetes-gateway
51+
tags: |
52+
type=semver,pattern={{version}}
53+
type=edge
54+
type=ref,event=pr
55+
type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }}
56+
57+
- name: Prepare NKG files
58+
run: |
59+
nkg_prefix=$(echo ${{ steps.meta.outputs.tags }} | cut -d ":" -f 1)
60+
nkg_tag=$(echo ${{ steps.meta.outputs.tags }} | cut -d ":" -f 2)
61+
make update-nkg-manifest NKG_PREFIX=${nkg_prefix} NKG_TAG=${nkg_tag}
62+
working-directory: ./conformance
63+
64+
- name: Build binary
65+
uses: goreleaser/goreleaser-action@336e29918d653399e599bfca99fadc1d7ffbc9f7 # v4.3.0
66+
with:
67+
version: latest
68+
args: build --snapshot --clean
69+
70+
- name: Build Docker Image
71+
uses: docker/build-push-action@2eb1c1961a95fc15694676618e422e8ba1d63825 # v4.1.1
72+
with:
73+
file: build/Dockerfile
74+
tags: ${{ steps.meta.outputs.tags }}
75+
context: "."
76+
target: goreleaser
77+
load: true
78+
cache-from: type=gha
79+
cache-to: type=gha,mode=max
80+
pull: true
81+
82+
- name: Build Test Docker Image
83+
uses: docker/build-push-action@2eb1c1961a95fc15694676618e422e8ba1d63825 # v4.1.1
84+
with:
85+
file: conformance/tests/Dockerfile
86+
tags: conformance-test-runner:${{ github.sha }}
87+
context: "."
88+
load: true
89+
cache-from: type=gha
90+
cache-to: type=gha,mode=max
91+
pull: true
92+
93+
- name: Deploy Kubernetes
94+
id: k8s
95+
run: |
96+
make create-kind-cluster KIND_KUBE_CONFIG=${{ github.workspace }}/kube-${{ github.run_id }}
97+
echo "KUBECONFIG=${{ github.workspace }}/kube-${{ github.run_id }}" >> "$GITHUB_ENV"
98+
working-directory: ./conformance
99+
100+
- name: Setup conformance tests
101+
run: |
102+
nkg_prefix=$(echo ${{ steps.meta.outputs.tags }} | cut -d ":" -f 1)
103+
nkg_tag=$(echo ${{ steps.meta.outputs.tags }} | cut -d ":" -f 2)
104+
if [ ${{ github.event_name }} == "schedule" ]; then
105+
export GW_API_VERSION=main
106+
fi
107+
make install-nkg-local-no-build NKG_PREFIX=${nkg_prefix} NKG_TAG=${nkg_tag}
108+
working-directory: ./conformance
109+
110+
- name: Run conformance tests
111+
run: |
112+
make run-conformance-tests TAG=${{ github.sha }}
113+
working-directory: ./conformance

‎Makefile

+1-1
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ deps: ## Add missing and remove unused modules, verify deps and download them to
5656
create-kind-cluster: ## Create a kind cluster
5757
$(eval KIND_IMAGE=$(shell grep -m1 'FROM kindest/node' <conformance/tests/Dockerfile | awk -F'[ ]' '{print $$2}'))
5858
kind create cluster --image $(KIND_IMAGE)
59-
kind export kubeconfig --kubeconfig $(KIND_KUBE_CONFIG_FOLDER)/config
59+
kind export kubeconfig --kubeconfig $(KIND_KUBE_CONFIG)
6060

6161
.PHONY: delete-kind-cluster
6262
delete-kind-cluster: ## Delete kind cluster

‎conformance/Makefile

+3-2
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
NKG_TAG = edge
22
NKG_PREFIX = nginx-kubernetes-gateway
3+
GW_API_VERSION ?= 0.7.1
34
GATEWAY_CLASS = nginx
45
SUPPORTED_FEATURES = HTTPRoute,HTTPRouteQueryParamMatching,HTTPRouteMethodMatching,HTTPRoutePortRedirect,HTTPRouteSchemeRedirect,GatewayClassObservedGenerationBump
56
KIND_KUBE_CONFIG=$${HOME}/.kube/kind/config
@@ -42,7 +43,7 @@ load-images: preload-nginx-container ## Load NKG and NGINX containers on configu
4243

4344
.PHONY: prepare-nkg-dependencies
4445
prepare-nkg-dependencies: ## Install NKG dependencies on configured kind cluster
45-
kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v0.7.1/standard-install.yaml
46+
./scripts/install-gateway.sh $(GW_API_VERSION)
4647
kubectl wait --for=condition=available --timeout=60s deployment gateway-api-admission-server -n gateway-system
4748
kubectl apply -f ../deploy/manifests/namespace.yaml
4849
kubectl create configmap njs-modules --from-file=../internal/mode/static/nginx/modules/src/httpmatches.js -n nginx-gateway
@@ -83,7 +84,7 @@ cleanup-conformance-tests: ## Clean up conformance tests fixtures
8384

8485
.PHONY: uninstall-nkg
8586
uninstall-nkg: ## Uninstall NKG on configured kind cluster
86-
kubectl delete -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v0.7.1/standard-install.yaml
87+
./scripts/uninstall-gateway.sh $(GW_API_VERSION)
8788
kubectl delete -f ../deploy/manifests/rbac.yaml
8889
kubectl delete -f ../deploy/manifests/namespace.yaml
8990
kubectl delete clusterrole nginx-gateway-provisioner

‎conformance/README.md

+1
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ update-nkg-manifest Update the NKG deployment manifest image name and
4343
| PREFIX | conformance-test-runner | The prefix for the conformance test image |
4444
| NKG_TAG | edge | The tag for the locally built NKG image |
4545
| NKG_PREFIX | nginx-kubernetes-gateway | The prefix for the locally built NKG image |
46+
| GW_API_VERSION | 0.7.1 | Tag for the Gateway API version to check out. Set to `main` to get the latest version |
4647
| KIND_KUBE_CONFIG | ~/.kube/kind/config | The location of the kubeconfig |
4748
| GATEWAY_CLASS | nginx | The gateway class that should be used for the tests |
4849
| SUPPORTED_FEATURES | HTTPRoute,HTTPRouteQueryParamMatching, HTTPRouteMethodMatching,HTTPRoutePortRedirect, HTTPRouteSchemeRedirect | The supported features that should be tested by the conformance tests. Ensure the list is comma separated with no spaces. |

‎conformance/scripts/install-gateway.sh

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
#!/bin/bash
2+
3+
if [ -z $1 ]; then
4+
echo "gateway API version argument not set; exiting"
5+
exit 1
6+
fi
7+
8+
if [ $1 == "main" ]; then
9+
temp_dir=$(mktemp -d)
10+
cd ${temp_dir}
11+
curl -s https://codeload.github.com/kubernetes-sigs/gateway-api/tar.gz/main | tar -xz --strip=2 gateway-api-main/config
12+
kubectl apply -f webhook
13+
kubectl apply -f crd/standard
14+
rm -rf ${temp_dir}
15+
else
16+
kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v$1/standard-install.yaml
17+
fi

‎conformance/scripts/uninstall-gateway.sh

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
#!/bin/bash
2+
3+
if [ -z $1 ]; then
4+
echo "gatway API version argument not set; exiting"
5+
exit 1
6+
fi
7+
8+
if [ $1 == "main" ]; then
9+
temp_dir=$(mktemp -d)
10+
cd ${temp_dir}
11+
curl -s https://codeload.github.com/kubernetes-sigs/gateway-api/tar.gz/main | tar -xz --strip=2 gateway-api-main/config
12+
kubectl delete -f webhook
13+
kubectl delete -f crd/standard
14+
rm -rf ${temp_dir}
15+
else
16+
kubectl delete -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v$1/standard-install.yaml
17+
fi

0 commit comments

Comments
 (0)