Run conformance tests in the CI pipeline (#792)

* Add conformance tests to the CI pipeline

Author: ciarams87

.github/workflows/ci.yml

@@ -144,10 +144,96 @@ jobs:
           path: ${{ github.workspace }}/dist
           key: nginx-kubernetes-gateway-${{ github.run_id }}-${{ github.run_number }}
 
+  conformance-tests:
+    name: Gateway Conformance Tests
+    runs-on: ubuntu-22.04
+    needs: vars
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+
+      - name: Setup Golang Environment
+        uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1
+        with:
+          go-version-file: go.mod
+
+      - name: Docker Buildx
+        uses: docker/setup-buildx-action@ecf95283f03858871ff00b787d79c419715afc34 # v2.7.0
+      
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@818d4b7b91585d195f67373fd9cb0332e31a7175 # v4.6.0
+        with:
+          images: |
+            name=ghcr.io/nginxinc/nginx-kubernetes-gateway
+          tags: |
+            type=semver,pattern={{version}}
+            type=edge
+            type=ref,event=pr
+            type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }}
+
+      - name: Prepare NKG files
+        run: |
+          nkg_prefix=$(echo ${{ steps.meta.outputs.tags }} | cut -d ":" -f 1)
+          nkg_tag=$(echo ${{ steps.meta.outputs.tags }} | cut -d ":" -f 2)
+          make update-nkg-manifest NKG_PREFIX=${nkg_prefix} NKG_TAG=${nkg_tag}
+        working-directory: ./conformance
+    
+      - name: Build binary
+        uses: goreleaser/goreleaser-action@336e29918d653399e599bfca99fadc1d7ffbc9f7 # v4.3.0
+        with:
+          version: latest
+          args: ${{ startsWith(github.ref, 'refs/tags/') && 'release' || 'build --snapshot' }} --clean
+        env:
+          GOPATH: ${{ needs.vars.outputs.go_path }}
+
+      - name: Build Docker Image
+        uses: docker/build-push-action@2eb1c1961a95fc15694676618e422e8ba1d63825 # v4.1.1
+        with:
+          file: build/Dockerfile
+          tags: ${{ steps.meta.outputs.tags }}
+          context: "."
+          target: goreleaser
+          load: true
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          pull: true
+      
+      - name: Build Test Docker Image
+        uses: docker/build-push-action@2eb1c1961a95fc15694676618e422e8ba1d63825 # v4.1.1
+        with:
+          file: conformance/tests/Dockerfile
+          tags: conformance-test-runner:${{ github.sha }}
+          context: "."
+          load: true
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          pull: true
+
+      - name: Deploy Kubernetes
+        id: k8s
+        run: |
+          make create-kind-cluster KIND_KUBE_CONFIG=kube-${{ github.run_id }}
+          echo "KUBECONFIG=kube-${{ github.run_id }}" >> "$GITHUB_ENV"
+        working-directory: ./conformance
+
+      - name: Setup conformance tests
+        run: |
+          nkg_prefix=$(echo ${{ steps.meta.outputs.tags }} | cut -d ":" -f 1)
+          nkg_tag=$(echo ${{ steps.meta.outputs.tags }} | cut -d ":" -f 2)
+          make install-nkg-local-no-build NKG_PREFIX=${nkg_prefix} NKG_TAG=${nkg_tag}
+        working-directory: ./conformance
+
+      - name: Run conformance tests
+        run: |
+          make run-conformance-tests TAG=${{ github.sha }}
+        working-directory: ./conformance
+        continue-on-error: true
+
   build:
     name: Build Image
     runs-on: ubuntu-22.04
-    needs: [vars, binary]
+    needs: [vars, binary, conformance-tests]
     steps:
       - name: Checkout Repository
         uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3

Makefile

@@ -46,7 +46,8 @@ deps: ## Add missing and remove unused modules, verify deps and download them to
 
 .PHONY: create-kind-cluster
 create-kind-cluster: ## Create a kind cluster
-	kind create cluster --image kindest/node:v1.27.1
+	$(eval KIND_IMAGE=$(shell grep -m1 'FROM kindest/node' <conformance/tests/Dockerfile | awk -F'[ ]' '{print $$2}'))
+	kind create cluster --image $(KIND_IMAGE)
 	kind export kubeconfig --kubeconfig $(KIND_KUBE_CONFIG_FOLDER)/config
 
 .PHONY: delete-kind-cluster

conformance/Makefile

@@ -2,7 +2,7 @@ NKG_TAG = edge
 NKG_PREFIX = nginx-kubernetes-gateway
 GATEWAY_CLASS = nginx
 SUPPORTED_FEATURES = HTTPRoute,HTTPRouteQueryParamMatching,HTTPRouteMethodMatching,HTTPRoutePortRedirect,HTTPRouteSchemeRedirect
-KIND_KUBE_CONFIG_FOLDER = $${HOME}/.kube/kind
+KIND_KUBE_CONFIG=$${HOME}/.kube/kind/config
 TAG = latest
 PREFIX = conformance-test-runner
 NKG_DEPLOYMENT_MANIFEST=../deploy/manifests/deployment.yaml
@@ -19,18 +19,25 @@ build-test-runner-image: ## Build conformance test runner image
 
 .PHONY: create-kind-cluster
 create-kind-cluster: ## Create a kind cluster
-	kind create cluster --image kindest/node:v1.27.1
-	kind export kubeconfig --kubeconfig $(KIND_KUBE_CONFIG_FOLDER)/config
+	$(eval KIND_IMAGE=$(shell grep -m1 'FROM kindest/node' <tests/Dockerfile | awk -F'[ ]' '{print $$2}'))
+	kind create cluster --image $(KIND_IMAGE)
+	kind export kubeconfig --kubeconfig $(KIND_KUBE_CONFIG)
 
 .PHONY: preload-nginx-container
 preload-nginx-container: ## Preload NGINX container on configured kind cluster
 	docker pull $(NGINX_IMAGE)
 	kind load docker-image $(NGINX_IMAGE)
 
-.PHONY: build-and-load-images
-build-and-load-images: preload-nginx-container ## Build NKG container and load it and NGINX container on configured kind cluster
+.PHONY: update-nkg-manifest
+update-nkg-manifest: ## Update the NKG deployment manifest image name and imagePullPolicy
 	yq -i 'with(.spec.template.spec.containers[0]; .image = "$(NKG_PREFIX):$(NKG_TAG)" | .imagePullPolicy = "Never")' $(NKG_DEPLOYMENT_MANIFEST)
+
+.PHONY: build-nkg-image
+build-nkg-image: update-nkg-manifest ## Build NKG container and load it and NGINX container on configured kind cluster
 	cd .. && make PREFIX=$(NKG_PREFIX) TAG=$(NKG_TAG) container
+
+.PHONY: load-images
+load-images: preload-nginx-container ## Load NKG and NGINX containers on configured kind cluster
 	kind load docker-image $(NKG_PREFIX):$(NKG_TAG)
 
 .PHONY: prepare-nkg-dependencies
@@ -44,11 +51,17 @@ prepare-nkg-dependencies: ## Install NKG dependencies on configured kind cluster
 	kubectl apply -f ../deploy/manifests/gatewayclass.yaml
 	kubectl apply -f ../deploy/manifests/service/nodeport.yaml
 
-.PHONY: install-nkg-local-build
-install-nkg-local-build: build-and-load-images prepare-nkg-dependencies ## Install NKG from local build with provisioner on configured kind cluster
+.PHONY: deploy-updated-provisioner
+deploy-updated-provisioner: ## Update provisioner manifest and deploy to the configured kind cluster
 	yq '(select(di != 3))' provisioner/provisioner.yaml | kubectl apply -f -
 	yq '(select(.spec.template.spec.containers[].image) | .spec.template.spec.containers[].image="$(NKG_PREFIX):$(NKG_TAG)" | .spec.template.spec.containers[].imagePullPolicy = "Never")' provisioner/provisioner.yaml | kubectl apply -f -
 
+.PHONY: install-nkg-local-build
+install-nkg-local-build: build-nkg-image load-images prepare-nkg-dependencies deploy-updated-provisioner ## Install NKG from local build with provisioner on configured kind cluster
+
+.PHONY: install-nkg-local-build
+install-nkg-local-no-build: load-images prepare-nkg-dependencies deploy-updated-provisioner ## Install NKG from local build with provisioner on configured kind cluster but do not build the NKG image
+
 .PHONY: install-nkg-edge
 install-nkg-edge: preload-nginx-container prepare-nkg-dependencies ## Install NKG with provisioner from edge on configured kind cluster
 	kubectl apply -f provisioner/provisioner.yaml

conformance/README.md

@@ -14,19 +14,23 @@ List available commands:
 ```bash
 $ make
 
-build-and-load-images          Build NKG container and load it and NGINX container on configured kind cluster
+build-nkg-image                Build NKG container and load it and NGINX container on configured kind cluster
 build-test-runner-image        Build conformance test runner image
 cleanup-conformance-tests      Clean up conformance tests fixtures
 create-kind-cluster            Create a kind cluster
 delete-kind-cluster            Delete kind cluster
+deploy-updated-provisioner     Update provisioner manifest and deploy to the configured kind cluster
 help                           Display this help
 install-nkg-edge               Install NKG with provisioner from edge on configured kind cluster
 install-nkg-local-build        Install NKG from local build with provisioner on configured kind cluster
+install-nkg-local-no-build     Install NKG from local build with provisioner on configured kind cluster but do not build the NKG image
+load-images                    Load NKG and NGINX containers on configured kind cluster
 preload-nginx-container        Preload NGINX container on configured kind cluster
 prepare-nkg-dependencies       Install NKG dependencies on configured kind cluster
 run-conformance-tests          Run conformance tests
 undo-image-update              Undo the NKG image name and tag in deployment manifest
 uninstall-nkg                  Uninstall NKG on configured kind cluster
+update-nkg-manifest            Update the NKG deployment manifest image name and imagePullPolicy
 ```
 
 **Note:** The following variables are configurable when running the below `make` commands:
@@ -37,7 +41,7 @@ uninstall-nkg                  Uninstall NKG on configured kind cluster
 | PREFIX | conformance-test-runner | The prefix for the conformance test image |
 | NKG_TAG  | edge  | The tag for the locally built NKG image |
 | NKG_PREFIX | nginx-kubernetes-gateway  | The prefix for the locally built NKG image |
-| KIND_KUBE_CONFIG_FOLDER | ~/.kube/kind  | The location of the kubeconfig folder |
+| KIND_KUBE_CONFIG |  ~/.kube/kind/config | The location of the kubeconfig |
 | GATEWAY_CLASS | nginx | The gateway class that should be used for the tests |
 | SUPPORTED_FEATURES | HTTPRoute,HTTPRouteQueryParamMatching, HTTPRouteMethodMatching,HTTPRoutePortRedirect, HTTPRouteSchemeRedirect | The supported features that should be tested by the conformance tests. Ensure the list is comma separated with no spaces. |
 | EXEMPT_FEATURES | ReferenceGrant | The features that should not be tested by the conformance tests |
@@ -55,10 +59,26 @@ $ make create-kind-cluster
 ```bash
 $ make install-nkg-local-build
 ```
-
-#### *Option 2* Install Nginx Kubernetes Gateway from edge to configured kind cluster
-Instead of the above command, you can skip the build NKG image step and prepare the environment to instead
-use the `edge` image
+#### *Option 2* Install Nginx Kubernetes Gateway from local already built image to configured kind cluster
+```bash
+$ make install-nkg-local-no-build
+```
+**Note:** You can optionally skip the actual *build* step. However, if choosing 
+this option, the following step *must* be completed manually *before* the build step:
+ * Set NKG_PREFIX=<nkg_repo_name> NKG_TAG=<nkg_image_tag> to preferred values.
+ * Navigate to `deploy/manifests` and update values in `deployment.yaml` as specified in below code-block.
+ * Save the changes.
+ ```
+ .
+ ..
+ containers:
+ - image: <nkg_repo_name>:<nkg_image_tag>
+   imagePullPolicy: Never
+ ..
+ .
+ ```
+#### *Option 3* Install Nginx Kubernetes Gateway from edge to configured kind cluster
+You can also skip the build NKG image step and prepare the environment to instead use the `edge` image
 
 ```bash
 $ make install-nkg-edge

conformance/tests/Dockerfile

@@ -1,4 +1,6 @@
 # syntax=docker/dockerfile:1.5
+# this is here so we can grab the latest version of kind and have dependabot keep it up to date
+FROM kindest/node:v1.27.3
 
 FROM golang:1.20