CP/DP Split: Add agent/nginx container and deployment

.yamllint.yaml

@@ -2,8 +2,7 @@
 ignore:
   - charts/nginx-gateway-fabric/templates
   - config/crd/bases/
-  - deploy/crds.yaml
-  - deploy/*nginx-plus
+  - deploy
   - site/static
 
 rules:

build/Dockerfile.nginx

@@ -1,18 +1,41 @@
 # syntax=docker/dockerfile:1.12
+# TODO(sberman): the commented out lines are for when we use the published agent release
+# FROM scratch AS nginx-files
+
+# # the following links can be replaced with local files if needed, i.e. ADD --chown=101:1001 <local_file> <container_file>
+# ADD --link --chown=101:1001 https://cs.nginx.com/static/keys/nginx_signing.rsa.pub nginx_signing.rsa.pub
+
+FROM golang:alpine AS builder
+
+WORKDIR /tmp
+
+RUN apk add --no-cache git make \
+    && git clone https://github.com/nginx/agent.git \
+    && cd agent \
+    && git checkout v3 \
+    && make build
+
 FROM nginx:1.27.3-alpine-otel
 
 ARG NJS_DIR
 ARG NGINX_CONF_DIR
 ARG BUILD_AGENT
 
-RUN apk add --no-cache libcap \
+# RUN --mount=type=bind,from=nginx-files,src=nginx_signing.rsa.pub,target=/etc/apk/keys/nginx_signing.rsa.pub \
+#     printf "%s\n" "http://packages.nginx.org/nginx-agent/alpine/v$(egrep -o '^[0-9]+\.[0-9]+' /etc/alpine-release)/main" >> /etc/apk/repositories \
+#     && apk add --no-cache nginx-agent
+
+RUN apk add --no-cache libcap bash \
     && mkdir -p /usr/lib/nginx/modules \
-	&& setcap 'cap_net_bind_service=+ep' /usr/sbin/nginx \
-	&& setcap -v 'cap_net_bind_service=+ep' /usr/sbin/nginx \
+    && setcap 'cap_net_bind_service=+ep' /usr/sbin/nginx \
+    && setcap -v 'cap_net_bind_service=+ep' /usr/sbin/nginx \
     && setcap 'cap_net_bind_service=+ep' /usr/sbin/nginx-debug \
     && setcap -v 'cap_net_bind_service=+ep' /usr/sbin/nginx-debug \
     && apk del libcap
 
+COPY --from=builder /tmp/agent/build/nginx-agent /usr/bin/nginx-agent
+
+COPY build/entrypoint.sh /agent/entrypoint.sh
 COPY ${NJS_DIR}/httpmatches.js /usr/lib/nginx/modules/njs/httpmatches.js
 COPY ${NGINX_CONF_DIR}/nginx.conf /etc/nginx/nginx.conf
 COPY ${NGINX_CONF_DIR}/grpc-error-locations.conf /etc/nginx/grpc-error-locations.conf
@@ -24,4 +47,4 @@ LABEL org.nginx.ngf.image.build.agent="${BUILD_AGENT}"
 
 USER 101:1001
 
-CMD ["sh", "-c", "rm -rf /var/run/nginx/*.sock && nginx -g 'daemon off;'"]
+ENTRYPOINT ["/agent/entrypoint.sh"]

build/Dockerfile.nginxplus

@@ -4,6 +4,15 @@ FROM scratch AS nginx-files
 # the following links can be replaced with local files if needed, i.e. ADD --chown=101:1001 <local_file> <container_file>
 ADD --link --chown=101:1001 https://cs.nginx.com/static/keys/nginx_signing.rsa.pub nginx_signing.rsa.pub
 
+FROM golang:alpine AS builder
+
+WORKDIR /tmp
+
+RUN apk add --no-cache git make \
+    && git clone https://github.com/nginx/agent.git \
+    && cd agent \
+    && git checkout v3 \
+    && make build
 
 FROM alpine:3.20
 
@@ -18,7 +27,7 @@ RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/apk/cert.pem,mode=0644 \
     addgroup -g 1001 -S nginx \
     && adduser -S -D -H -u 101 -h /var/cache/nginx -s /sbin/nologin -G nginx -g nginx nginx \
     && printf "%s\n" "https://pkgs.nginx.com/plus/${NGINX_PLUS_VERSION}/alpine/v$(grep -E -o '^[0-9]+\.[0-9]+' /etc/alpine-release)/main" >> /etc/apk/repositories \
-    && apk add --no-cache nginx-plus nginx-plus-module-njs nginx-plus-module-otel libcap \
+    && apk add --no-cache nginx-plus nginx-plus-module-njs nginx-plus-module-otel libcap bash \
     && mkdir -p /usr/lib/nginx/modules \
 	&& setcap 'cap_net_bind_service=+ep' /usr/sbin/nginx \
 	&& setcap -v 'cap_net_bind_service=+ep' /usr/sbin/nginx \
@@ -29,6 +38,9 @@ RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/apk/cert.pem,mode=0644 \
     && ln -sf /dev/stdout /var/log/nginx/access.log \
     && ln -sf /dev/stderr /var/log/nginx/error.log
 
+COPY --from=builder /tmp/agent/build/nginx-agent /usr/bin/nginx-agent
+
+COPY build/entrypoint.sh /agent/entrypoint.sh
 COPY ${NJS_DIR}/httpmatches.js /usr/lib/nginx/modules/njs/httpmatches.js
 COPY ${NGINX_CONF_DIR}/nginx-plus.conf /etc/nginx/nginx.conf
 COPY ${NGINX_CONF_DIR}/grpc-error-locations.conf /etc/nginx/grpc-error-locations.conf
@@ -40,4 +52,4 @@ USER 101:1001
 
 LABEL org.nginx.ngf.image.build.agent="${BUILD_AGENT}"
 
-CMD ["sh", "-c", "rm -rf /var/run/nginx/*.sock && nginx -g 'daemon off;'"]
+ENTRYPOINT ["/agent/entrypoint.sh"]

build/entrypoint.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+
+set -euxo pipefail
+
+handle_term() {
+    echo "received TERM signal"
+    echo "stopping nginx-agent ..."
+    kill -TERM "${agent_pid}" 2>/dev/null
+    echo "stopping nginx ..."
+    kill -TERM "${nginx_pid}" 2>/dev/null
+}
+
+trap 'handle_term' TERM
+
+rm -rf /var/run/nginx/*.sock
+
+# Launch nginx
+echo "starting nginx ..."
+/usr/sbin/nginx -g "daemon off;" &
+
+nginx_pid=$!
+
+SECONDS=0
+
+while ! ps -ef | grep "nginx: master process" | grep -v grep; do
+    if ((SECONDS > 5)); then
+        echo "couldn't find nginx master process"
+        exit 1
+    fi
+done
+
+# start nginx-agent, pass args
+echo "starting nginx-agent ..."
+nginx-agent "$@" &
+
+agent_pid=$!
+
+if [ $? != 0 ]; then
+    echo "couldn't start the agent, please check the log file"
+    exit 1
+fi
+
+wait_term() {
+    wait ${agent_pid}
+    trap - TERM
+    kill -QUIT "${nginx_pid}" 2>/dev/null
+    echo "waiting for nginx to stop..."
+    wait ${nginx_pid}
+}
+
+wait_term
+
+echo "nginx-agent process has stopped, exiting."

charts/nginx-gateway-fabric/templates/deployment.yaml

@@ -117,6 +117,8 @@ spec:
         {{- toYaml .Values.nginxGateway.resources | nindent 10 }}
         {{- end }}
         ports:
+        - name: agent-grpc
+          containerPort: 8443
         {{- if .Values.metrics.enable }}
         - name: metrics
           containerPort: {{ .Values.metrics.port }}

charts/nginx-gateway-fabric/templates/service.yaml

@@ -14,7 +14,7 @@ spec:
   selector:
     {{- include "nginx-gateway.selectorLabels" . | nindent 4 }}
   ports:
-  - name: grpc
+  - name: agent-grpc
     port: 443
     protocol: TCP
-    targetPort: 443
+    targetPort: 8443

charts/nginx-gateway-fabric/templates/tmp-nginx-agent-conf.yaml

@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: nginx-agent-config
+  namespace: {{ .Release.Namespace }}
+data:
+  nginx-agent.conf: |-
+    command:
+      server:
+        host: {{ include "nginx-gateway.fullname" . }}.{{ .Release.Namespace }}.svc
+        port: 443
+    allowed_directories:
+    - /etc/nginx
+    - /usr/share/nginx
+    - /var/run/nginx
+    features:
+    - connection
+    log:
+      level: debug

charts/nginx-gateway-fabric/templates/tmp-nginx-deployment.yaml

@@ -0,0 +1,186 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: tmp-nginx-deployment
+  namespace: {{ .Release.Namespace }}
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: tmp-nginx-deployment
+      app.kubernetes.io/instance: {{ .Release.Name }}
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: tmp-nginx-deployment
+        app.kubernetes.io/instance: {{ .Release.Name }}
+    spec:
+      initContainers:
+      - name: sleep # wait for a bit for control plane to be ready
+        image: {{ .Values.nginxGateway.image.repository }}:{{ default .Chart.AppVersion .Values.nginxGateway.image.tag }}
+        imagePullPolicy: {{ .Values.nginxGateway.image.pullPolicy }}
+        command:
+        - /usr/bin/gateway
+        - sleep
+        - --duration=15s
+      - name: init
+        image: {{ .Values.nginxGateway.image.repository }}:{{ default .Chart.AppVersion .Values.nginxGateway.image.tag }}
+        imagePullPolicy: {{ .Values.nginxGateway.image.pullPolicy }}
+        command:
+        - /usr/bin/gateway
+        - initialize
+        - --source
+        - /includes/main.conf
+        {{- if .Values.nginx.plus }}
+        - --source
+        - /includes/mgmt.conf
+        - --nginx-plus
+        {{- end }}
+        - --destination
+        - /etc/nginx/main-includes
+        env:
+          - name: POD_UID
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.uid
+        securityContext:
+          seccompProfile:
+            type: RuntimeDefault
+          capabilities:
+            drop:
+            - ALL
+          readOnlyRootFilesystem: true
+          runAsUser: 102
+          runAsGroup: 1001
+        volumeMounts:
+        - name: nginx-includes-bootstrap
+          mountPath: /includes
+        - name: nginx-main-includes
+          mountPath: /etc/nginx/main-includes
+      containers:
+      - image: {{ .Values.nginx.image.repository }}:{{ .Values.nginx.image.tag | default .Chart.AppVersion }}
+        imagePullPolicy: {{ .Values.nginx.image.pullPolicy }}
+        name: nginx
+        {{- if .Values.nginx.lifecycle }}
+        lifecycle:
+        {{- toYaml .Values.nginx.lifecycle | nindent 10 }}
+        {{- end }}
+        ports:
+        - containerPort: 80
+          name: http
+        - containerPort: 443
+          name: https
+        securityContext:
+          seccompProfile:
+            type: RuntimeDefault
+          allowPrivilegeEscalation: {{ .Values.nginx.securityContext.allowPrivilegeEscalation }}
+          capabilities:
+            add:
+            - NET_BIND_SERVICE
+            drop:
+            - ALL
+          readOnlyRootFilesystem: true
+          runAsUser: 101
+          runAsGroup: 1001
+        volumeMounts:
+        - name: nginx-agent
+          mountPath: /etc/nginx-agent
+        - name: nginx-conf
+          mountPath: /etc/nginx/conf.d
+        - name: nginx-stream-conf
+          mountPath: /etc/nginx/stream-conf.d
+        - name: nginx-main-includes
+          mountPath: /etc/nginx/main-includes
+        - name: nginx-secrets
+          mountPath: /etc/nginx/secrets
+        - name: nginx-run
+          mountPath: /var/run/nginx
+        - name: nginx-cache
+          mountPath: /var/cache/nginx
+        - name: nginx-includes
+          mountPath: /etc/nginx/includes
+        {{- if .Values.nginx.plus }}
+        - name: nginx-lib
+          mountPath: /var/lib/nginx/state
+          {{- if .Values.nginx.usage.secretName }}
+        - name: nginx-plus-license
+          mountPath: /etc/nginx/license.jwt
+          subPath: license.jwt
+          {{- end }}
+          {{- if or .Values.nginx.usage.caSecretName .Values.nginx.usage.clientSSLSecretName }}
+        - name: nginx-plus-usage-certs
+          mountPath: /etc/nginx/certs-bootstrap/
+          {{- end }}
+        {{- end }}
+        {{- with .Values.nginx.extraVolumeMounts -}}
+        {{ toYaml . | nindent 8 }}
+        {{- end }}
+        {{- if .Values.nginx.debug }}
+        command:
+          - "/bin/sh"
+        args:
+          - "-c"
+          - "rm -rf /var/run/nginx/*.sock && nginx-debug -g 'daemon off;'"
+        {{- end }}
+      terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds }}
+      {{- if .Values.affinity }}
+      affinity:
+      {{- toYaml .Values.affinity | nindent 8 }}
+      {{- end }}
+      serviceAccountName: {{ include "nginx-gateway.serviceAccountName" . }}
+      securityContext:
+        fsGroup: 1001
+        runAsNonRoot: true
+      {{- if .Values.tolerations }}
+      tolerations:
+      {{- toYaml .Values.tolerations | nindent 6 }}
+      {{- end }}
+      {{- if .Values.nodeSelector }}
+      nodeSelector:
+      {{- toYaml .Values.nodeSelector | nindent 8 }}
+      {{- end }}
+      volumes:
+      - name: nginx-agent
+        configMap:
+          name: nginx-agent-config
+      - name: nginx-conf
+        emptyDir: {}
+      - name: nginx-stream-conf
+        emptyDir: {}
+      - name: nginx-main-includes
+        emptyDir: {}
+      - name: nginx-secrets
+        emptyDir: {}
+      - name: nginx-run
+        emptyDir: {}
+      - name: nginx-cache
+        emptyDir: {}
+      - name: nginx-includes
+        emptyDir: {}
+      - name: nginx-includes-bootstrap
+        configMap:
+          name: nginx-includes-bootstrap
+      {{- if .Values.nginx.plus }}
+      - name: nginx-lib
+        emptyDir: {}
+        {{- if .Values.nginx.usage.secretName }}
+      - name: nginx-plus-license
+        secret:
+          secretName: {{ .Values.nginx.usage.secretName }}
+        {{- end }}
+        {{- if or .Values.nginx.usage.caSecretName .Values.nginx.usage.clientSSLSecretName }}
+      - name: nginx-plus-usage-certs
+        projected:
+          sources:
+          {{- if .Values.nginx.usage.caSecretName }}
+          - secret:
+              name: {{ .Values.nginx.usage.caSecretName }}
+          {{- end }}
+          {{- if .Values.nginx.usage.clientSSLSecretName }}
+          - secret:
+              name: {{ .Values.nginx.usage.clientSSLSecretName }}
+          {{- end }}
+        {{- end }}
+      {{- end }}
+      {{- with .Values.extraVolumes -}}
+      {{ toYaml . | nindent 6 }}
+      {{- end }}