CP/DP Split: Support configuring NodePorts (#3343)

Problem: Now that the control plane provisions the NGINX Service, users can't set specific NodePorts values.

Solution: Allow users to specify NodePorts in the helm chart (globally) and in the NginxProxy resource.

Author: sjberman

apis/v1alpha2/nginxproxy_types.go

@@ -533,6 +533,13 @@ type ServiceSpec struct {
 	//
 	// +optional
 	LoadBalancerSourceRanges []string `json:"loadBalancerSourceRanges,omitempty"`
+
+	// NodePorts are the list of NodePorts to expose on the NGINX data plane service.
+	// Each NodePort MUST map to a Gateway listener port, otherwise it will be ignored.
+	// The default NodePort range enforced by Kubernetes is 30000-32767.
+	//
+	// +optional
+	NodePorts []NodePort `json:"nodePorts,omitempty"`
 }
 
 // ServiceType describes ingress method for the Service.
@@ -569,3 +576,18 @@ const (
 	// (dropping the traffic if there are no local endpoints).
 	ExternalTrafficPolicyLocal ExternalTrafficPolicy = ExternalTrafficPolicy(corev1.ServiceExternalTrafficPolicyLocal)
 )
+
+// NodePort creates a port on each node on which the NGINX data plane service is exposed. The NodePort MUST
+// map to a Gateway listener port, otherwise it will be ignored. If not specified, Kubernetes allocates a NodePort
+// automatically if required. The default NodePort range enforced by Kubernetes is 30000-32767.
+type NodePort struct {
+	// Port is the NodePort to expose.
+	// kubebuilder:validation:Minimum=1
+	// kubebuilder:validation:Maximum=65535
+	Port int32 `json:"port"`
+
+	// ListenerPort is the Gateway listener port that this NodePort maps to.
+	// kubebuilder:validation:Minimum=1
+	// kubebuilder:validation:Maximum=65535
+	ListenerPort int32 `json:"listenerPort"`
+}

apis/v1alpha2/zz_generated.deepcopy.go

@@ -258,19 +258,24 @@ The following table lists the configurable parameters of the NGINX Gateway Fabri
 | `certGenerator.overwrite` | Overwrite existing TLS Secrets on startup. | bool | `false` |
 | `certGenerator.serverTLSSecretName` | The name of the Secret containing TLS CA, certificate, and key for the NGINX Gateway Fabric control plane to securely communicate with the NGINX Agent. Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway). | string | `"server-tls"` |
 | `clusterDomain` | The DNS cluster domain of your Kubernetes cluster. | string | `"cluster.local"` |
-| `nginx` | The nginx section contains the configuration for all NGINX data plane deployments installed by the NGINX Gateway Fabric control plane. | object | `{"config":{},"container":{},"debug":false,"image":{"pullPolicy":"Always","repository":"ghcr.io/nginx/nginx-gateway-fabric/nginx","tag":"edge"},"imagePullSecret":"","imagePullSecrets":[],"kind":"deployment","plus":false,"pod":{},"replicas":1,"service":{"externalTrafficPolicy":"Local","type":"LoadBalancer"},"usage":{"caSecretName":"","clientSSLSecretName":"","endpoint":"","resolver":"","secretName":"nplus-license","skipVerify":false}}` |
-| `nginx.config` | The configuration for the data plane that is contained in the NginxProxy resource. | object | `{}` |
-| `nginx.container` | The container configuration for the NGINX container. | object | `{}` |
+| `nginx` | The nginx section contains the configuration for all NGINX data plane deployments installed by the NGINX Gateway Fabric control plane. | object | `{"config":{},"container":{},"debug":false,"image":{"pullPolicy":"Always","repository":"ghcr.io/nginx/nginx-gateway-fabric/nginx","tag":"edge"},"imagePullSecret":"","imagePullSecrets":[],"kind":"deployment","plus":false,"pod":{},"replicas":1,"service":{"annotations":{},"externalTrafficPolicy":"Local","loadBalancerClass":"","loadBalancerIP":"","loadBalancerSourceRanges":[],"nodePorts":[],"type":"LoadBalancer"},"usage":{"caSecretName":"","clientSSLSecretName":"","endpoint":"","resolver":"","secretName":"nplus-license","skipVerify":false}}` |
+| `nginx.config` | The configuration for the data plane that is contained in the NginxProxy resource. This is applied globally to all Gateways managed by this instance of NGINX Gateway Fabric. | object | `{}` |
+| `nginx.container` | The container configuration for the NGINX container. This is applied globally to all Gateways managed by this instance of NGINX Gateway Fabric. | object | `{}` |
 | `nginx.debug` | Enable debugging for NGINX. Uses the nginx-debug binary. The NGINX error log level should be set to debug in the NginxProxy resource. | bool | `false` |
 | `nginx.image.repository` | The NGINX image to use. | string | `"ghcr.io/nginx/nginx-gateway-fabric/nginx"` |
 | `nginx.imagePullSecret` | The name of the secret containing docker registry credentials. Secret must exist in the same namespace as the helm release. The control plane will copy this secret into any namespace where NGINX is deployed. | string | `""` |
 | `nginx.imagePullSecrets` | A list of secret names containing docker registry credentials. Secrets must exist in the same namespace as the helm release. The control plane will copy these secrets into any namespace where NGINX is deployed. | list | `[]` |
 | `nginx.kind` | The kind of NGINX deployment. | string | `"deployment"` |
 | `nginx.plus` | Is NGINX Plus image being used. | bool | `false` |
-| `nginx.pod` | The pod configuration for the NGINX data plane pod. | object | `{}` |
+| `nginx.pod` | The pod configuration for the NGINX data plane pod. This is applied globally to all Gateways managed by this instance of NGINX Gateway Fabric. | object | `{}` |
 | `nginx.replicas` | The number of replicas of the NGINX Deployment. | int | `1` |
-| `nginx.service` | The service configuration for the NGINX data plane. | object | `{"externalTrafficPolicy":"Local","type":"LoadBalancer"}` |
+| `nginx.service` | The service configuration for the NGINX data plane. This is applied globally to all Gateways managed by this instance of NGINX Gateway Fabric. | object | `{"annotations":{},"externalTrafficPolicy":"Local","loadBalancerClass":"","loadBalancerIP":"","loadBalancerSourceRanges":[],"nodePorts":[],"type":"LoadBalancer"}` |
+| `nginx.service.annotations` | The annotations of the NGINX data plane service. | object | `{}` |
 | `nginx.service.externalTrafficPolicy` | The externalTrafficPolicy of the service. The value Local preserves the client source IP. | string | `"Local"` |
+| `nginx.service.loadBalancerClass` | LoadBalancerClass is the class of the load balancer implementation this Service belongs to. Requires nginx.service.type set to LoadBalancer. | string | `""` |
+| `nginx.service.loadBalancerIP` | The static IP address for the load balancer. Requires nginx.service.type set to LoadBalancer. | string | `""` |
+| `nginx.service.loadBalancerSourceRanges` | The IP ranges (CIDR) that are allowed to access the load balancer. Requires nginx.service.type set to LoadBalancer. | list | `[]` |
+| `nginx.service.nodePorts` | A list of NodePorts to expose on the NGINX data plane service. Each NodePort MUST map to a Gateway listener port, otherwise it will be ignored. The default NodePort range enforced by Kubernetes is 30000-32767. | list | `[]` |
 | `nginx.service.type` | The type of service to create for the NGINX data plane. | string | `"LoadBalancer"` |
 | `nginx.usage.caSecretName` | The name of the Secret containing the NGINX Instance Manager CA certificate. Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway). | string | `""` |
 | `nginx.usage.clientSSLSecretName` | The name of the Secret containing the client certificate and key for authenticating with NGINX Instance Manager. Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway). | string | `""` |

charts/nginx-gateway-fabric/README.md

@@ -91,3 +91,16 @@ Expand leader election lock name.
 {{- printf "%s-%s" (include "nginx-gateway.fullname" .) "leader-election" -}}
 {{- end -}}
 {{- end -}}
+
+{{/*
+Filters out empty fields from a struct.
+*/}}
+{{- define "filterEmptyFields" -}}
+{{- $result := dict }}
+{{- range $key, $value := . }}
+  {{- if and (not (empty $value)) (not (and (kindIs "slice" $value) (eq (len $value) 0))) }}
+    {{- $result = merge $result (dict $key $value) }}
+  {{- end }}
+{{- end }}
+{{- $result | toYaml }}
+{{- end }}

charts/nginx-gateway-fabric/templates/_helpers.tpl

@@ -29,5 +29,7 @@ spec:
     {{- end }}
     {{- if .Values.nginx.service }}
     service:
-      {{- toYaml .Values.nginx.service | nindent 6 }}
+      {{- with .Values.nginx.service }}
+      {{- include "filterEmptyFields" . | nindent 6 }}
+      {{- end }}
     {{- end }}

charts/nginx-gateway-fabric/templates/nginxproxy.yaml

@@ -53,7 +53,7 @@
       "description": "The nginx section contains the configuration for all NGINX data plane deployments\ninstalled by the NGINX Gateway Fabric control plane.",
       "properties": {
         "config": {
-          "description": "The configuration for the data plane that is contained in the NginxProxy resource.",
+          "description": "The configuration for the data plane that is contained in the NginxProxy resource. This is applied globally to all Gateways\nmanaged by this instance of NGINX Gateway Fabric.",
           "properties": {
             "disableHTTP2": {
               "description": "DisableHTTP2 defines if http2 should be disabled for all servers.",
@@ -266,7 +266,7 @@
           "type": "object"
         },
         "container": {
-          "description": "The container configuration for the NGINX container.",
+          "description": "The container configuration for the NGINX container. This is applied globally to all Gateways managed by this\ninstance of NGINX Gateway Fabric.",
           "required": [],
           "title": "container",
           "type": "object"
@@ -341,7 +341,7 @@
           "type": "boolean"
         },
         "pod": {
-          "description": "The pod configuration for the NGINX data plane pod.",
+          "description": "The pod configuration for the NGINX data plane pod. This is applied globally to all Gateways managed by this\ninstance of NGINX Gateway Fabric.",
           "required": [],
           "title": "pod",
           "type": "object"
@@ -354,8 +354,14 @@
           "type": "integer"
         },
         "service": {
-          "description": "The service configuration for the NGINX data plane.",
+          "description": "The service configuration for the NGINX data plane. This is applied globally to all Gateways managed by this\ninstance of NGINX Gateway Fabric.",
           "properties": {
+            "annotations": {
+              "description": "The annotations of the NGINX data plane service.",
+              "required": [],
+              "title": "annotations",
+              "type": "object"
+            },
             "externalTrafficPolicy": {
               "default": "Local",
               "description": "The externalTrafficPolicy of the service. The value Local preserves the client source IP.",
@@ -366,6 +372,53 @@
               "required": [],
               "title": "externalTrafficPolicy"
             },
+            "loadBalancerClass": {
+              "default": "",
+              "description": "LoadBalancerClass is the class of the load balancer implementation this Service belongs to.\nRequires nginx.service.type set to LoadBalancer.",
+              "required": [],
+              "title": "loadBalancerClass",
+              "type": "string"
+            },
+            "loadBalancerIP": {
+              "default": "",
+              "description": "The static IP address for the load balancer. Requires nginx.service.type set to LoadBalancer.",
+              "required": [],
+              "title": "loadBalancerIP",
+              "type": "string"
+            },
+            "loadBalancerSourceRanges": {
+              "description": "The IP ranges (CIDR) that are allowed to access the load balancer. Requires nginx.service.type set to LoadBalancer.",
+              "items": {
+                "required": []
+              },
+              "required": [],
+              "title": "loadBalancerSourceRanges",
+              "type": "array"
+            },
+            "nodePorts": {
+              "description": "A list of NodePorts to expose on the NGINX data plane service. Each NodePort MUST map to a Gateway listener port,\notherwise it will be ignored. The default NodePort range enforced by Kubernetes is 30000-32767.",
+              "items": {
+                "properties": {
+                  "listenerPort": {
+                    "maximum": 65535,
+                    "minimum": 1,
+                    "required": [],
+                    "type": "integer"
+                  },
+                  "port": {
+                    "maximum": 65535,
+                    "minimum": 1,
+                    "required": [],
+                    "type": "integer"
+                  }
+                },
+                "required": [],
+                "type": "object"
+              },
+              "required": [],
+              "title": "nodePorts",
+              "type": "array"
+            },
             "type": {
               "default": "LoadBalancer",
               "description": "The type of service to create for the NGINX data plane.",

charts/nginx-gateway-fabric/values.schema.json

@@ -367,10 +367,12 @@ nginx:
   #             value:
   #               type: string
   # @schema
-  # -- The configuration for the data plane that is contained in the NginxProxy resource.
+  # -- The configuration for the data plane that is contained in the NginxProxy resource. This is applied globally to all Gateways
+  # managed by this instance of NGINX Gateway Fabric.
   config: {}
 
-  # -- The pod configuration for the NGINX data plane pod.
+  # -- The pod configuration for the NGINX data plane pod. This is applied globally to all Gateways managed by this
+  # instance of NGINX Gateway Fabric.
   pod: {}
     # -- The termination grace period of the NGINX data plane pod.
     # terminationGracePeriodSeconds: 30
@@ -391,7 +393,8 @@ nginx:
     # nginx.container.extraVolumeMounts mount additional volumes to the container.
     # extraVolumes: []
 
-  # -- The container configuration for the NGINX container.
+  # -- The container configuration for the NGINX container. This is applied globally to all Gateways managed by this
+  # instance of NGINX Gateway Fabric.
   container: {}
     # -- The resource requirements of the NGINX container.
     # resources: {}
@@ -402,7 +405,8 @@ nginx:
     # -- extraVolumeMounts are the additional volume mounts for the NGINX container.
     # extraVolumeMounts: []
 
-  # -- The service configuration for the NGINX data plane.
+  # -- The service configuration for the NGINX data plane. This is applied globally to all Gateways managed by this
+  # instance of NGINX Gateway Fabric.
   service:
     # @schema
     # enum:
@@ -422,17 +426,39 @@ nginx:
     externalTrafficPolicy: Local
 
     # -- The annotations of the NGINX data plane service.
-    # annotations: {}
+    annotations: {}
 
     # -- The static IP address for the load balancer. Requires nginx.service.type set to LoadBalancer.
-    # loadBalancerIP: ""
+    loadBalancerIP: ""
 
     # -- LoadBalancerClass is the class of the load balancer implementation this Service belongs to.
     # Requires nginx.service.type set to LoadBalancer.
-    # loadBalancerClass: ""
+    loadBalancerClass: ""
 
     # -- The IP ranges (CIDR) that are allowed to access the load balancer. Requires nginx.service.type set to LoadBalancer.
-    # loadBalancerSourceRanges: []
+    loadBalancerSourceRanges: []
+
+    # @schema
+    # type: array
+    # items:
+    #   type: object
+    #   properties:
+    #     port:
+    #       type: integer
+    #       required: true
+    #       minimum: 1
+    #       maximum: 65535
+    #     listenerPort:
+    #       type: integer
+    #       required: true
+    #       minimum: 1
+    #       maximum: 65535
+    # @schema
+    # -- A list of NodePorts to expose on the NGINX data plane service. Each NodePort MUST map to a Gateway listener port,
+    # otherwise it will be ignored. The default NodePort range enforced by Kubernetes is 30000-32767.
+    nodePorts: []
+    # - port: 30025
+    #   listenerPort: 80
 
   # -- Enable debugging for NGINX. Uses the nginx-debug binary. The NGINX error log level should be set to debug in the NginxProxy resource.
   debug: false

charts/nginx-gateway-fabric/values.yaml

@@ -3498,6 +3498,36 @@ spec:
                         items:
                           type: string
                         type: array
+                      nodePorts:
+                        description: |-
+                          NodePorts are the list of NodePorts to expose on the NGINX data plane service.
+                          Each NodePort MUST map to a Gateway listener port, otherwise it will be ignored.
+                          The default NodePort range enforced by Kubernetes is 30000-32767.
+                        items:
+                          description: |-
+                            NodePort creates a port on each node on which the NGINX data plane service is exposed. The NodePort MUST
+                            map to a Gateway listener port, otherwise it will be ignored. If not specified, Kubernetes allocates a NodePort
+                            automatically if required. The default NodePort range enforced by Kubernetes is 30000-32767.
+                          properties:
+                            listenerPort:
+                              description: |-
+                                ListenerPort is the Gateway listener port that this NodePort maps to.
+                                kubebuilder:validation:Minimum=1
+                                kubebuilder:validation:Maximum=65535
+                              format: int32
+                              type: integer
+                            port:
+                              description: |-
+                                Port is the NodePort to expose.
+                                kubebuilder:validation:Minimum=1
+                                kubebuilder:validation:Maximum=65535
+                              format: int32
+                              type: integer
+                          required:
+                          - listenerPort
+                          - port
+                          type: object
+                        type: array
                       type:
                         default: LoadBalancer
                         description: ServiceType describes ingress method for the