Add init container for setting up base config (#2649)

Problem: We are starting to introduce configuration options that exist in the main context. However, that
configuration won't be written until the control plane writes it to nginx, meaning it doesn't exist on
nginx startup. Therefore nginx uses its default configuration for a brief time, which is incorrect.

We want to be able to provide this configuration on startup.

Solution: Using an init container, we can mount a ConfigMap containing the dynamic base config, and copy it
to the proper location in the filesystem before nginx starts. We can't mount the ConfigMap directly to the proper location
because it would be read-only, preventing our control plane from writing to it.

This allows us to bootstrap the user config into nginx on startup, while also allowing our control plane to overwrite it
if the user ever changes the config after the fact.

Removed logic that cleared out nginx files on startup because it would erase this bootstrap config, and it wasn't really
needed since we delete nginx files when we write config anyway.

Also fixed an issue where the log level was not honored when no Gateway resources existed.

Author: sjberman

charts/nginx-gateway-fabric/templates/deployment.yaml

@@ -29,6 +29,33 @@ spec:
         {{- end }}
       {{- end }}
     spec:
+      initContainers:
+      - name: copy-nginx-config
+        image: {{ .Values.nginxGateway.image.repository }}:{{ default .Chart.AppVersion .Values.nginxGateway.image.tag }}
+        imagePullPolicy: {{ .Values.nginxGateway.image.pullPolicy }}
+        command:
+        - /usr/bin/gateway
+        - copy
+        - --source
+        - /includes/main.conf
+        - --destination
+        - /etc/nginx/main-includes/main.conf
+        securityContext:
+          seccompProfile:
+            type: RuntimeDefault
+          capabilities:
+            add:
+            - KILL # Set because the binary has CAP_KILL for the main controller process. Not used by init.
+            drop:
+            - ALL
+          readOnlyRootFilesystem: true
+          runAsUser: 102
+          runAsGroup: 1001
+        volumeMounts:
+        - name: nginx-includes-configmap
+          mountPath: /includes
+        - name: nginx-main-includes
+          mountPath: /etc/nginx/main-includes
       containers:
       - args:
         - static-mode
@@ -223,6 +250,9 @@ spec:
         emptyDir: {}
       - name: nginx-includes
         emptyDir: {}
+      - name: nginx-includes-configmap
+        configMap:
+          name: nginx-includes
       {{- with .Values.extraVolumes -}}
       {{ toYaml . | nindent 6 }}
       {{- end }}

charts/nginx-gateway-fabric/templates/include-configmap.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: nginx-includes
+  namespace: {{ .Release.Namespace }}
+  labels:
+  {{- include "nginx-gateway.labels" . | nindent 4 }}
+data:
+  main.conf: |
+    {{- if and .Values.nginx.config .Values.nginx.config.logging .Values.nginx.config.logging.errorLevel }}
+    error_log stderr {{ .Values.nginx.config.logging.errorLevel }};
+    {{ else }}
+    error_log stderr info;
+    {{- end }}

charts/nginx-gateway-fabric/templates/scc.yaml

@@ -32,6 +32,7 @@ seccompProfiles:
 volumes:
 - emptyDir
 - secret
+- configMap
 users:
 - {{ printf "system:serviceaccount:%s:%s" .Release.Namespace (include "nginx-gateway.serviceAccountName" .) }}
 allowedCapabilities:

cmd/gateway/commands.go

@@ -3,6 +3,7 @@ package main
 import (
 	"errors"
 	"fmt"
+	"io"
 	"os"
 	"runtime/debug"
 	"strconv"
@@ -481,6 +482,63 @@ func createSleepCommand() *cobra.Command {
 	return cmd
 }
 
+func createCopyCommand() *cobra.Command {
+	// flag names
+	const srcFlag = "source"
+	const destFlag = "destination"
+	// flag values
+	var src, dest string
+
+	cmd := &cobra.Command{
+		Use:   "copy",
+		Short: "Copy a file to a destination",
+		RunE: func(_ *cobra.Command, _ []string) error {
+			if len(src) == 0 {
+				return errors.New("source must not be empty")
+			}
+			if len(dest) == 0 {
+				return errors.New("destination must not be empty")
+			}
+
+			srcFile, err := os.Open(src)
+			if err != nil {
+				return fmt.Errorf("error opening source file: %w", err)
+			}
+			defer srcFile.Close()
+
+			destFile, err := os.Create(dest)
+			if err != nil {
+				return fmt.Errorf("error creating destination file: %w", err)
+			}
+			defer destFile.Close()
+
+			if _, err := io.Copy(destFile, srcFile); err != nil {
+				return fmt.Errorf("error copying file contents: %w", err)
+			}
+
+			return nil
+		},
+	}
+
+	cmd.Flags().StringVar(
+		&src,
+		srcFlag,
+		"",
+		"The source file to be copied",
+	)
+
+	cmd.Flags().StringVar(
+		&dest,
+		destFlag,
+		"",
+		"The destination for the source file to be copied to",
+	)
+
+	cmd.MarkFlagsRequiredTogether(srcFlag, destFlag)
+
+	return cmd
+}
+
 func parseFlags(flags *pflag.FlagSet) ([]string, []string) {
 	var flagKeys, flagValues []string
 

cmd/gateway/commands_test.go

@@ -437,6 +437,51 @@ func TestSleepCmdFlagValidation(t *testing.T) {
 	}
 }
 
+func TestCopyCmdFlagValidation(t *testing.T) {
+	t.Parallel()
+	tests := []flagTestCase{
+		{
+			name: "valid flags",
+			args: []string{
+				"--source=/my/file",
+				"--destination=dest/file",
+			},
+			wantErr: false,
+		},
+		{
+			name:    "omitted flags",
+			args:    nil,
+			wantErr: false,
+		},
+		{
+			name: "source set without destination",
+			args: []string{
+				"--source=/my/file",
+			},
+			wantErr: true,
+			expectedErrPrefix: "if any flags in the group [source destination] are set they must all be set; " +
+				"missing [destination]",
+		},
+		{
+			name: "destination set without source",
+			args: []string{
+				"--destination=/dest/file",
+			},
+			wantErr: true,
+			expectedErrPrefix: "if any flags in the group [source destination] are set they must all be set; " +
+				"missing [source]",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			t.Parallel()
+			cmd := createCopyCommand()
+			testFlag(t, cmd, test)
+		})
+	}
+}
+
 func TestParseFlags(t *testing.T) {
 	t.Parallel()
 	g := NewWithT(t)

cmd/gateway/main.go

@@ -23,6 +23,7 @@ func main() {
 	rootCmd.AddCommand(
 		createStaticModeCommand(),
 		createProvisionerModeCommand(),
+		createCopyCommand(),
 		createSleepCommand(),
 	)
 

config/tests/static-deployment.yaml

@@ -21,6 +21,33 @@ spec:
         app.kubernetes.io/name: nginx-gateway
         app.kubernetes.io/instance: nginx-gateway
     spec:
+      initContainers:
+      - name: copy-nginx-config
+        image: ghcr.io/nginxinc/nginx-gateway-fabric:edge
+        imagePullPolicy: Always
+        command:
+        - /usr/bin/gateway
+        - copy
+        - --source
+        - /includes/main.conf
+        - --destination
+        - /etc/nginx/main-includes/main.conf
+        securityContext:
+          seccompProfile:
+            type: RuntimeDefault
+          capabilities:
+            add:
+            - KILL # Set because the binary has CAP_KILL for the main controller process. Not used by init.
+            drop:
+            - ALL
+          readOnlyRootFilesystem: true
+          runAsUser: 102
+          runAsGroup: 1001
+        volumeMounts:
+        - name: nginx-includes-configmap
+          mountPath: /includes
+        - name: nginx-main-includes
+          mountPath: /etc/nginx/main-includes
       containers:
       - args:
         - static-mode
@@ -137,3 +164,6 @@ spec:
         emptyDir: {}
       - name: nginx-includes
         emptyDir: {}
+      - name: nginx-includes-configmap
+        configMap:
+          name: nginx-includes

deploy/aws-nlb/deploy.yaml

@@ -143,6 +143,19 @@ subjects:
   namespace: nginx-gateway
 ---
 apiVersion: v1
+data:
+  main.conf: |
+    error_log stderr info;
+kind: ConfigMap
+metadata:
+  labels:
+    app.kubernetes.io/instance: nginx-gateway
+    app.kubernetes.io/name: nginx-gateway
+    app.kubernetes.io/version: edge
+  name: nginx-includes
+  namespace: nginx-gateway
+---
+apiVersion: v1
 kind: Service
 metadata:
   annotations:
@@ -290,6 +303,33 @@ spec:
           name: nginx-cache
         - mountPath: /etc/nginx/includes
           name: nginx-includes
+      initContainers:
+      - command:
+        - /usr/bin/gateway
+        - copy
+        - --source
+        - /includes/main.conf
+        - --destination
+        - /etc/nginx/main-includes/main.conf
+        image: ghcr.io/nginxinc/nginx-gateway-fabric:edge
+        imagePullPolicy: Always
+        name: copy-nginx-config
+        securityContext:
+          capabilities:
+            add:
+            - KILL
+            drop:
+            - ALL
+          readOnlyRootFilesystem: true
+          runAsGroup: 1001
+          runAsUser: 102
+          seccompProfile:
+            type: RuntimeDefault
+        volumeMounts:
+        - mountPath: /includes
+          name: nginx-includes-configmap
+        - mountPath: /etc/nginx/main-includes
+          name: nginx-main-includes
       securityContext:
         fsGroup: 1001
         runAsNonRoot: true
@@ -311,6 +351,9 @@ spec:
         name: nginx-cache
       - emptyDir: {}
         name: nginx-includes
+      - configMap:
+          name: nginx-includes
+        name: nginx-includes-configmap
 ---
 apiVersion: gateway.networking.k8s.io/v1
 kind: GatewayClass

deploy/azure/deploy.yaml

@@ -143,6 +143,19 @@ subjects:
   namespace: nginx-gateway
 ---
 apiVersion: v1
+data:
+  main.conf: |
+    error_log stderr info;
+kind: ConfigMap
+metadata:
+  labels:
+    app.kubernetes.io/instance: nginx-gateway
+    app.kubernetes.io/name: nginx-gateway
+    app.kubernetes.io/version: edge
+  name: nginx-includes
+  namespace: nginx-gateway
+---
+apiVersion: v1
 kind: Service
 metadata:
   labels:
@@ -287,6 +300,33 @@ spec:
           name: nginx-cache
         - mountPath: /etc/nginx/includes
           name: nginx-includes
+      initContainers:
+      - command:
+        - /usr/bin/gateway
+        - copy
+        - --source
+        - /includes/main.conf
+        - --destination
+        - /etc/nginx/main-includes/main.conf
+        image: ghcr.io/nginxinc/nginx-gateway-fabric:edge
+        imagePullPolicy: Always
+        name: copy-nginx-config
+        securityContext:
+          capabilities:
+            add:
+            - KILL
+            drop:
+            - ALL
+          readOnlyRootFilesystem: true
+          runAsGroup: 1001
+          runAsUser: 102
+          seccompProfile:
+            type: RuntimeDefault
+        volumeMounts:
+        - mountPath: /includes
+          name: nginx-includes-configmap
+        - mountPath: /etc/nginx/main-includes
+          name: nginx-main-includes
       nodeSelector:
         kubernetes.io/os: linux
       securityContext:
@@ -310,6 +350,9 @@ spec:
         name: nginx-cache
       - emptyDir: {}
         name: nginx-includes
+      - configMap:
+          name: nginx-includes
+        name: nginx-includes-configmap
 ---
 apiVersion: gateway.networking.k8s.io/v1
 kind: GatewayClass