Merge pull request #181746 from hercules-ci/nixosTest-erofs

nixos/qemu-vm: Use disposable EROFS for store when writableStore = false

nixos/modules/virtualisation/includes-to-excludes.py

@@ -0,0 +1,86 @@
+
+# Convert a list of strings to a regex that matches everything but those strings
+# ... and it had to be a POSIX regex; no negative lookahead :(
+# This is a workaround for erofs supporting only exclude regex, not an include list
+
+import sys
+import re
+from collections import defaultdict
+
+# We can configure this script to match in different ways if we need to.
+# The regex got too long for the argument list, so we had to truncate the
+# hashes and use MATCH_STRING_PREFIX. That's less accurate, and might pick up some
+# garbage like .lock files, but only if the sandbox doesn't hide those. Even
+# then it should be harmless.
+
+# Produce the negation of ^a$
+MATCH_EXACTLY = ".+"
+# Produce the negation of ^a
+MATCH_STRING_PREFIX = "//X" # //X should be epsilon regex instead. Not supported??
+# Produce the negation of ^a/?
+MATCH_SUBPATHS = "[^/].*$"
+
+# match_end = MATCH_SUBPATHS
+match_end = MATCH_STRING_PREFIX
+# match_end = MATCH_EXACTLY
+
+def chars_to_inverted_class(letters):
+    assert len(letters) > 0
+    letters = list(letters)
+
+    s = "[^"
+
+    if "]" in letters:
+        s += "]"
+        letters.remove("]")
+
+    final = ""
+    if "-" in letters:
+        final = "-"
+        letters.remove("-")
+
+    s += "".join(letters)
+
+    s += final
+
+    s += "]"
+
+    return s
+
+# There's probably at least one bug in here, but it seems to works well enough
+# for filtering store paths.
+def strings_to_inverted_regex(strings):
+    s = "("
+
+    # Match anything that starts with the wrong character
+
+    chars = defaultdict(list)
+
+    for item in strings:
+        if item != "":
+            chars[item[0]].append(item[1:])
+
+    if len(chars) == 0:
+        s += match_end
+    else:
+        s += chars_to_inverted_class(chars)
+
+    # Now match anything that starts with the right char, but then goes wrong
+
+    for char, sub in chars.items():
+        s += "|(" + re.escape(char) + strings_to_inverted_regex(sub) + ")"
+
+    s += ")"
+    return s
+
+if __name__ == "__main__":
+    stdin_lines = []
+    for line in sys.stdin:
+        if line.strip() != "":
+            stdin_lines.append(line.strip())
+
+    print("^" + strings_to_inverted_regex(stdin_lines))
+
+# Test:
+# (echo foo; echo fo/; echo foo/; echo foo/ba/r; echo b; echo az; echo az/; echo az/a; echo ab; echo ab/a; echo ab/; echo abc; echo abcde; echo abb; echo ac; echo b) | grep -vE "$((echo ab; echo az; echo foo;) | python includes-to-excludes.py | tee /dev/stderr )"
+# should print ab, az, foo and their subpaths

nixos/modules/virtualisation/qemu-vm.nix

@@ -17,6 +17,8 @@ let
 
   cfg = config.virtualisation;
 
+  opt = options.virtualisation;
+
   qemu = cfg.qemu.package;
 
   consoles = lib.concatMapStringsSep " " (c: "console=${c}") cfg.qemu.consoles;
@@ -122,11 +124,32 @@ let
           TMPDIR=$(mktemp -d nix-vm.XXXXXXXXXX --tmpdir)
       fi
 
-      ${lib.optionalString cfg.useNixStoreImage
+      ${lib.optionalString (cfg.useNixStoreImage)
-      ''
+        (if cfg.writableStore
-        # Create a writable copy/snapshot of the store image.
+          then ''
-        ${qemu}/bin/qemu-img create -f qcow2 -F qcow2 -b ${storeImage}/nixos.qcow2 "$TMPDIR"/store.img
+            # Create a writable copy/snapshot of the store image.
-      ''}
+            ${qemu}/bin/qemu-img create -f qcow2 -F qcow2 -b ${storeImage}/nixos.qcow2 "$TMPDIR"/store.img
+          ''
+          else ''
+            (
+              cd ${builtins.storeDir}
+              ${pkgs.erofs-utils}/bin/mkfs.erofs \
+                --force-uid=0 \
+                --force-gid=0 \
+                -U eb176051-bd15-49b7-9e6b-462e0b467019 \
+                -T 0 \
+                --exclude-regex="$(
+                  <${pkgs.closureInfo { rootPaths = [ config.system.build.toplevel regInfo ]; }}/store-paths \
+                    sed -e 's^.*/^^g' \
+                  | cut -c -10 \
+                  | ${pkgs.python3}/bin/python ${./includes-to-excludes.py} )" \
+                "$TMPDIR"/store.img \
+                . \
+                </dev/null >/dev/null
+            )
+          ''
+        )
+      }
 
       # Create a directory for exchanging data with the VM.
       mkdir -p "$TMPDIR/xchg"
@@ -746,6 +769,26 @@ in
           }
         ]));
 
+    warnings =
+      optional (
+        cfg.writableStore &&
+        cfg.useNixStoreImage &&
+        opt.writableStore.highestPrio > lib.modules.defaultPriority)
+        ''
+          You have enabled ${opt.useNixStoreImage} = true,
+          without setting ${opt.writableStore} = false.
+
+          This causes a store image to be written to the store, which is
+          costly, especially for the binary cache, and because of the need
+          for more frequent garbage collection.
+
+          If you really need this combination, you can set ${opt.writableStore}
+          explicitly to false, incur the cost and make this warning go away.
+          Otherwise, we recommend
+
+            ${opt.writableStore} = false;
+        '';
+
     # Note [Disk layout with `useBootLoader`]
     #
     # If `useBootLoader = true`, we configure 2 drives:
@@ -769,6 +812,8 @@ in
     );
     boot.loader.grub.gfxmodeBios = with cfg.resolution; "${toString x}x${toString y}";
 
+    boot.initrd.kernelModules = optionals (cfg.useNixStoreImage && !cfg.writableStore) [ "erofs" ];
+
     boot.initrd.extraUtilsCommands = lib.mkIf (cfg.useDefaultFilesystems && !config.boot.initrd.systemd.enable)
       ''
         # We need mke2fs in the initrd.
@@ -905,6 +950,7 @@ in
         name = "nix-store";
         file = ''"$TMPDIR"/store.img'';
         deviceExtraOpts.bootindex = if cfg.useBootLoader then "3" else "2";
+        driveExtraOpts.format = if cfg.writableStore then "qcow2" else "raw";
       }])
       (mkIf cfg.useBootLoader [
         # The order of this list determines the device names, see

nixos/tests/discourse.nix

@@ -30,6 +30,7 @@ import ./make-test-python.nix (
         virtualisation.memorySize = 2048;
         virtualisation.cores = 4;
         virtualisation.useNixStoreImage = true;
+        virtualisation.writableStore = false;
 
         imports = [ common/user-account.nix ];
 

nixos/tests/gitlab.nix

@@ -38,6 +38,8 @@ in {
       virtualisation.memorySize = if pkgs.stdenv.is64bit then 4096 else 2047;
       virtualisation.cores = 4;
       virtualisation.useNixStoreImage = true;
+      virtualisation.writableStore = false;
+
       systemd.services.gitlab.serviceConfig.Restart = mkForce "no";
       systemd.services.gitlab-workhorse.serviceConfig.Restart = mkForce "no";
       systemd.services.gitaly.serviceConfig.Restart = mkForce "no";