nixos/etc: handle mountpoints on top of /etc when switching

The activation script that remounts the /etc overlay now handles other
mount points on top of /etc by bind mounting them to the new temporary
/etc overlay and then atomically revealing it.

nixos/modules/system/etc/etc.nix

@@ -247,6 +247,30 @@ in
           --options lowerdir=$tmpMetadataMount::${config.system.build.etcBasedir},${etcOverlayOptions} \
           $tmpEtcMount
 
+        # Before moving the new /etc overlay under the old /etc, we have to
+        # move mounts on top of /etc to the new /etc mountpoint.
+        findmnt /etc --submounts --list --noheading --kernel --output TARGET | while read -r mountPoint; do
+          if [[ "$mountPoint" = "/etc" ]]; then
+            continue
+          fi
+
+          tmpMountPoint="$tmpEtcMount/''${mountPoint:5}"
+            ${if config.system.etc.overlay.mutable then ''
+              if [[ -f "$mountPoint" ]]; then
+                touch "$tmpMountPoint"
+              elif [[ -d "$mountPoint" ]]; then
+                mkdir -p "$tmpMountPoint"
+              fi
+            '' else ''
+              if [[ ! -e "$tmpMountPoint" ]]; then
+                echo "Skipping undeclared mountpoint in environment.etc: $mountPoint"
+                continue
+              fi
+            ''
+          }
+          mount --bind "$mountPoint" "$tmpMountPoint"
+        done
+
         # Move the new temporary /etc mount underneath the current /etc mount.
         #
         # This should eventually use util-linux to perform this move beneath,
@@ -255,8 +279,7 @@ in
         ${pkgs.move-mount-beneath}/bin/move-mount --move --beneath $tmpEtcMount /etc
 
         # Unmount the top /etc mount to atomically reveal the new mount.
-        umount /etc
+        umount --recursive /etc
-
       fi
     '' else ''
       # Set up the statically computed bits of /etc.

nixos/tests/activation/etc-overlay-immutable.nix

@@ -15,6 +15,11 @@
     boot.kernelPackages = pkgs.linuxPackages_latest;
     time.timeZone = "Utc";
 
+    environment.etc = {
+      "mountpoint/.keep".text = "keep";
+      "filemount".text = "keep";
+    };
+
     specialisation.new-generation.configuration = {
       environment.etc."newgen".text = "newgen";
     };
@@ -33,8 +38,17 @@
     with subtest("switching to a new generation"):
       machine.fail("stat /etc/newgen")
 
+      machine.succeed("mount -t tmpfs tmpfs /etc/mountpoint")
+      machine.succeed("touch /etc/mountpoint/extra-file")
+      machine.succeed("mount --bind /dev/null /etc/filemount")
+
       machine.succeed("/run/current-system/specialisation/new-generation/bin/switch-to-configuration switch")
 
       assert machine.succeed("cat /etc/newgen") == "newgen"
+
+      print(machine.succeed("findmnt /etc/mountpoint"))
+      print(machine.succeed("ls /etc/mountpoint"))
+      print(machine.succeed("stat /etc/mountpoint/extra-file"))
+      print(machine.succeed("findmnt /etc/filemount"))
   '';
 }

nixos/tests/activation/etc-overlay-mutable.nix

@@ -28,9 +28,22 @@
       machine.fail("stat /etc/newgen")
       machine.succeed("echo -n 'mutable' > /etc/mutable")
 
+      # Directory
+      machine.succeed("mkdir /etc/mountpoint")
+      machine.succeed("mount -t tmpfs tmpfs /etc/mountpoint")
+      machine.succeed("touch /etc/mountpoint/extra-file")
+
+      # File
+      machine.succeed("touch /etc/filemount")
+      machine.succeed("mount --bind /dev/null /etc/filemount")
+
       machine.succeed("/run/current-system/specialisation/new-generation/bin/switch-to-configuration switch")
 
       assert machine.succeed("cat /etc/newgen") == "newgen"
       assert machine.succeed("cat /etc/mutable") == "mutable"
+
+      print(machine.succeed("findmnt /etc/mountpoint"))
+      print(machine.succeed("stat /etc/mountpoint/extra-file"))
+      print(machine.succeed("findmnt /etc/filemount"))
   '';
 }