Update nginx template for TLS passthrough

Problem: nginx configuration templates didn't support TLS passthrough

Solution: I added a template setup fro stream servers

Author: sarthyparty

charts/nginx-gateway-fabric/templates/deployment.yaml

@@ -118,6 +118,8 @@ spec:
         volumeMounts:
         - name: nginx-conf
           mountPath: /etc/nginx/conf.d
+        - name: nginx-stream-conf
+          mountPath: /etc/nginx/stream-conf.d
         - name: module-includes
           mountPath: /etc/nginx/module-includes
         - name: nginx-secrets
@@ -153,6 +155,8 @@ spec:
         volumeMounts:
         - name: nginx-conf
           mountPath: /etc/nginx/conf.d
+        - name: nginx-stream-conf
+          mountPath: /etc/nginx/stream-conf.d
         - name: module-includes
           mountPath: /etc/nginx/module-includes
         - name: nginx-secrets
@@ -189,6 +193,8 @@ spec:
       volumes:
       - name: nginx-conf
         emptyDir: {}
+      - name: nginx-stream-conf
+        emptyDir: {}
       - name: module-includes
         emptyDir: {}
       - name: nginx-secrets

internal/mode/static/nginx/conf/nginx.conf

@@ -36,3 +36,7 @@ http {
     }
   }
 }
+
+stream {
+	include /etc/nginx/stream-conf.d/*.conf;
+}

internal/mode/static/nginx/config/base_http_config_template.go

@@ -2,4 +2,20 @@ package config
 
 const baseHTTPTemplateText = `
 {{- if .HTTP2 }}http2 on;{{ end }}
+
+# Set $gw_api_compliant_host variable to the value of $http_host unless $http_host is empty, then set it to the value
+# of $host. We prefer $http_host because it contains the original value of the host header, which is required by the
+# Gateway API. However, in an HTTP/1.0 request, it's possible that $http_host can be empty. In this case, we will use
+# the value of $host. See http://nginx.org/en/docs/http/ngx_http_core_module.html#var_host.
+map $http_host $gw_api_compliant_host {
+    '' $host;
+    default $http_host;
+}
+
+# Set $connection_header variable to upgrade when the $http_upgrade header is set, otherwise, set it to close. This
+# allows support for websocket connections. See https://nginx.org/en/docs/http/websocket.html.
+map $http_upgrade $connection_upgrade {
+    default upgrade;
+    '' close;
+}
 `

internal/mode/static/nginx/config/generator.go

@@ -17,6 +17,9 @@ const (
 	// httpFolder is the folder where NGINX HTTP configuration files are stored.
 	httpFolder = configFolder + "/conf.d"
 
+	// streamFolder is the folder where NGINX HTTP configuration files are stored.
+	streamFolder = configFolder + "/stream-conf.d"
+
 	// modulesIncludesFolder is the folder where the included "load_module" file is stored.
 	modulesIncludesFolder = configFolder + "/module-includes"
 
@@ -29,6 +32,9 @@ const (
 	// httpConfigFile is the path to the configuration file with HTTP configuration.
 	httpConfigFile = httpFolder + "/http.conf"
 
+	// streamConfigFile is the path to the configuration file with Stream configuration.
+	streamConfigFile = streamFolder + "/stream.conf"
+
 	// configVersionFile is the path to the config version configuration file.
 	configVersionFile = httpFolder + "/config-version.conf"
 
@@ -157,6 +163,9 @@ func (g GeneratorImpl) getExecuteFuncs() []executeFunc {
 		executeSplitClients,
 		executeMaps,
 		executeTelemetry,
+		executeStreamServers,
+		g.executeStreamUpstreams,
+		executeStreamMaps,
 	}
 }
 

internal/mode/static/nginx/config/maps.go

@@ -1,6 +1,7 @@
 package config
 
 import (
+	"fmt"
 	"strings"
 	gotemplate "text/template"
 
@@ -21,6 +22,48 @@ func executeMaps(conf dataplane.Configuration) []executeResult {
 	return []executeResult{result}
 }
 
+func executeStreamMaps(conf dataplane.Configuration) []executeResult {
+	maps := createStreamMaps(conf)
+
+	result := executeResult{
+		dest: streamConfigFile,
+		data: helpers.MustExecuteTemplate(mapsTemplate, maps),
+	}
+
+	return []executeResult{result}
+}
+
+func createStreamMaps(conf dataplane.Configuration) []*http.Map {
+	var maps []*http.Map
+	portsToMap := make(map[int32]*http.Map)
+
+	for _, t := range conf.TLSServers {
+		streamMap, ok := portsToMap[t.Port]
+
+		if !ok {
+			m := http.Map{
+				Source:   "$ssl_preread_server_name",
+				Variable: "$dest" + fmt.Sprint(t.Port),
+				Parameters: []http.MapParameter{
+					{
+						Value:  t.Hostname,
+						Result: "unix:/var/lib/nginx/" + t.Hostname + fmt.Sprint(t.Port) + ".sock",
+					},
+				},
+			}
+			maps = append(maps, &m)
+			portsToMap[t.Port] = &m
+		} else {
+			streamMap.Parameters = append(streamMap.Parameters, http.MapParameter{
+				Value:  t.Hostname,
+				Result: "unix:/var/lib/nginx/" + t.Hostname + fmt.Sprint(t.Port) + ".sock",
+			})
+		}
+	}
+
+	return maps
+}
+
 func buildAddHeaderMaps(servers []dataplane.VirtualServer) []http.Map {
 	addHeaderNames := make(map[string]struct{})
 

internal/mode/static/nginx/config/maps_template.go

@@ -8,20 +8,4 @@ map {{ $m.Source }} {{ $m.Variable }} {
     {{ end }}
 }
 {{- end }}
-
-# Set $gw_api_compliant_host variable to the value of $http_host unless $http_host is empty, then set it to the value
-# of $host. We prefer $http_host because it contains the original value of the host header, which is required by the
-# Gateway API. However, in an HTTP/1.0 request, it's possible that $http_host can be empty. In this case, we will use
-# the value of $host. See http://nginx.org/en/docs/http/ngx_http_core_module.html#var_host.
-map $http_host $gw_api_compliant_host {
-    '' $host;
-    default $http_host;
-}
-
-# Set $connection_header variable to upgrade when the $http_upgrade header is set, otherwise, set it to close. This
-# allows support for websocket connections. See https://nginx.org/en/docs/http/websocket.html.
-map $http_upgrade $connection_upgrade {
-    default upgrade;
-    '' close;
-}
 `

internal/mode/static/nginx/config/maps_test.go

@@ -189,3 +189,85 @@ func TestBuildAddHeaderMaps(t *testing.T) {
 
 	g.Expect(maps).To(ConsistOf(expectedMap))
 }
+
+func TestExecuteStreamMaps(t *testing.T) {
+	g := NewWithT(t)
+	conf := dataplane.Configuration{
+		TLSServers: []dataplane.Layer4Server{
+			{
+				Hostname:     "example.com",
+				Port:         8081,
+				UpstreamName: "backend1",
+			},
+			{
+				Hostname:     "example.com",
+				Port:         8080,
+				UpstreamName: "backend1",
+			},
+			{
+				Hostname:     "cafe.example.com",
+				Port:         8080,
+				UpstreamName: "backend2",
+			},
+		},
+	}
+
+	expSubStrings := map[string]int{
+		"example.com unix:/var/lib/nginx/example.com8081.sock;":           1,
+		"example.com unix:/var/lib/nginx/example.com8080.sock;":           1,
+		"cafe.example.com unix:/var/lib/nginx/cafe.example.com8080.sock;": 1,
+	}
+
+	type assertion func(g *WithT, data string)
+
+	expectedResults := map[string]assertion{
+		streamConfigFile: func(g *WithT, data string) {
+			for expSubStr, expCount := range expSubStrings {
+				g.Expect(strings.Count(data, expSubStr)).To(Equal(expCount))
+			}
+		},
+	}
+
+	results := executeStreamMaps(conf)
+	g.Expect(results).To(HaveLen(len(expectedResults)))
+
+	for _, res := range results {
+		g.Expect(expectedResults).To(HaveKey(res.dest), "executeStreamServers returned unexpected result destination")
+
+		assertData := expectedResults[res.dest]
+		assertData(g, string(res.data))
+	}
+}
+
+func TestCreateStreamMaps(t *testing.T) {
+	g := NewWithT(t)
+	conf := dataplane.Configuration{
+		TLSServers: []dataplane.Layer4Server{
+			{
+				Hostname:     "example.com",
+				Port:         8081,
+				UpstreamName: "backend1",
+			},
+			{
+				Hostname:     "example.com",
+				Port:         8080,
+				UpstreamName: "backend1",
+			},
+			{
+				Hostname:     "cafe.example.com",
+				Port:         8080,
+				UpstreamName: "backend2",
+			},
+		},
+	}
+
+	maps := createStreamMaps(conf)
+	g.Expect(maps).To(HaveLen(2))
+
+	g.Expect(maps[0].Parameters).To(HaveLen(1))
+	g.Expect(maps[1].Parameters).To(HaveLen(2))
+
+	g.Expect(maps[0].Parameters[0].Result).To(Equal("unix:/var/lib/nginx/example.com8081.sock"))
+	g.Expect(maps[1].Parameters[0].Result).To(Equal("unix:/var/lib/nginx/example.com8080.sock"))
+	g.Expect(maps[0].Parameters[0].Result).To(Equal("unix:/var/lib/nginx/cafe.example.com8080.sock"))
+}

internal/mode/static/nginx/config/stream/config.go

@@ -0,0 +1,9 @@
+package stream
+
+// Server holds all configuration for a stream server.
+type Server struct {
+	Listen      string
+	Destination string
+	SSLPreread  bool
+	ProxyPass   bool
+}

internal/mode/static/nginx/config/stream_servers.go

@@ -0,0 +1,56 @@
+package config
+
+import (
+	"fmt"
+	gotemplate "text/template"
+
+	"github.com/nginxinc/nginx-gateway-fabric/internal/framework/helpers"
+	"github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/nginx/config/stream"
+	"github.com/nginxinc/nginx-gateway-fabric/internal/mode/static/state/dataplane"
+)
+
+var streamServersTemplate = gotemplate.Must(gotemplate.New("servers").Parse(streamServersTemplateText))
+
+func executeStreamServers(conf dataplane.Configuration) []executeResult {
+	streamServers := createStreamServers(conf)
+
+	streamServerResult := executeResult{
+		dest: streamConfigFile,
+		data: helpers.MustExecuteTemplate(streamServersTemplate, streamServers),
+	}
+
+	result := []executeResult{
+		streamServerResult,
+	}
+
+	return result
+}
+
+func createStreamServers(conf dataplane.Configuration) []stream.Server {
+	streamServers := make([]stream.Server, 0)
+	for _, t := range conf.TLSServers {
+		streamServers = append(streamServers, stream.Server{
+			Listen:      "unix:/var/lib/nginx/" + t.Hostname + fmt.Sprint(t.Port) + ".sock",
+			Destination: t.UpstreamName,
+			ProxyPass:   true,
+			SSLPreread:  false,
+		})
+	}
+
+	ports := make(map[int32]bool)
+
+	for _, t := range conf.TLSServers {
+		if ports[t.Port] {
+			continue
+		}
+		ports[t.Port] = true
+		streamServers = append(streamServers, stream.Server{
+			Listen:      fmt.Sprint(t.Port),
+			Destination: "$dest" + fmt.Sprint(t.Port),
+			ProxyPass:   false,
+			SSLPreread:  true,
+		})
+	}
+
+	return streamServers
+}

internal/mode/static/nginx/config/stream_servers_template.go

@@ -0,0 +1,19 @@
+package config
+
+const streamServersTemplateText = `
+{{- range $s := . }}
+server {
+	listen {{ $s.Listen }};
+
+	{{- if $s.ProxyPass }}
+	proxy_pass {{ $s.Destination }};
+	{{- else }}
+	pass {{ $s.Destination }};
+	{{- end }}
+
+	{{- if $s.SSLPreread }}
+	ssl_preread on;
+	{{- end }}
+}
+{{- end }}
+`