From afc60d017b238d0aadbaf0c77ff844388e70640b Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 16 Jul 2022 18:20:16 +0200 Subject: [PATCH 1/4] nixos/qemu-vm: Use disposable EROFS for store when writableStore = false This avoids putting a large disk image in the store (and possibly in a binary cache), while improving runtime performance. Assuming you're running an SSD, and/or with plenty of cache (?) it is feasible to preempt the virtualization overhead before VM start, in single-digit seconds. For some tests that perform many reads on the store, the improved performance of EROFS is sufficient that not only the image creation overhead is compensated for, but is actually faster. Stats for nixosTests.gitlab: Baseline without useNixStoreImage: >1000s Baseline with useNixStoreImage without writableStore = false ext4 image in store: 277 seconds + significant image build time and/or disk space Disposable erofs image: 249 seconds _including_ image build time Custom erofs overlay on 9p host store: 391 seconds; presumably because the overlay still performs too many 9p accesses, or perhaps some other overhead. This solution had no obvious performance advantage, while requiring extra options to work, so it was discarded. --- .../virtualisation/includes-to-excludes.py | 86 +++++++++++++++++++ nixos/modules/virtualisation/qemu-vm.nix | 34 ++++++-- 2 files changed, 115 insertions(+), 5 deletions(-) create mode 100644 nixos/modules/virtualisation/includes-to-excludes.py diff --git a/nixos/modules/virtualisation/includes-to-excludes.py b/nixos/modules/virtualisation/includes-to-excludes.py new file mode 100644 index 000000000000..05ef9c0f23b9 --- /dev/null +++ b/nixos/modules/virtualisation/includes-to-excludes.py @@ -0,0 +1,86 @@ + +# Convert a list of strings to a regex that matches everything but those strings +# ... and it had to be a POSIX regex; no negative lookahead :( +# This is a workaround for erofs supporting only exclude regex, not an include list + +import sys +import re +from collections import defaultdict + +# We can configure this script to match in different ways if we need to. +# The regex got too long for the argument list, so we had to truncate the +# hashes and use MATCH_STRING_PREFIX. That's less accurate, and might pick up some +# garbage like .lock files, but only if the sandbox doesn't hide those. Even +# then it should be harmless. + +# Produce the negation of ^a$ +MATCH_EXACTLY = ".+" +# Produce the negation of ^a +MATCH_STRING_PREFIX = "//X" # //X should be epsilon regex instead. Not supported?? +# Produce the negation of ^a/? +MATCH_SUBPATHS = "[^/].*$" + +# match_end = MATCH_SUBPATHS +match_end = MATCH_STRING_PREFIX +# match_end = MATCH_EXACTLY + +def chars_to_inverted_class(letters): + assert len(letters) > 0 + letters = list(letters) + + s = "[^" + + if "]" in letters: + s += "]" + letters.remove("]") + + final = "" + if "-" in letters: + final = "-" + letters.remove("-") + + s += "".join(letters) + + s += final + + s += "]" + + return s + +# There's probably at least one bug in here, but it seems to works well enough +# for filtering store paths. +def strings_to_inverted_regex(strings): + s = "(" + + # Match anything that starts with the wrong character + + chars = defaultdict(list) + + for item in strings: + if item != "": + chars[item[0]].append(item[1:]) + + if len(chars) == 0: + s += match_end + else: + s += chars_to_inverted_class(chars) + + # Now match anything that starts with the right char, but then goes wrong + + for char, sub in chars.items(): + s += "|(" + re.escape(char) + strings_to_inverted_regex(sub) + ")" + + s += ")" + return s + +if __name__ == "__main__": + stdin_lines = [] + for line in sys.stdin: + if line.strip() != "": + stdin_lines.append(line.strip()) + + print("^" + strings_to_inverted_regex(stdin_lines)) + +# Test: +# (echo foo; echo fo/; echo foo/; echo foo/ba/r; echo b; echo az; echo az/; echo az/a; echo ab; echo ab/a; echo ab/; echo abc; echo abcde; echo abb; echo ac; echo b) | grep -vE "$((echo ab; echo az; echo foo;) | python includes-to-excludes.py | tee /dev/stderr )" +# should print ab, az, foo and their subpaths diff --git a/nixos/modules/virtualisation/qemu-vm.nix b/nixos/modules/virtualisation/qemu-vm.nix index e87f540fd57c..3fafbbc55dd6 100644 --- a/nixos/modules/virtualisation/qemu-vm.nix +++ b/nixos/modules/virtualisation/qemu-vm.nix @@ -122,11 +122,32 @@ let TMPDIR=$(mktemp -d nix-vm.XXXXXXXXXX --tmpdir) fi - ${lib.optionalString cfg.useNixStoreImage - '' - # Create a writable copy/snapshot of the store image. - ${qemu}/bin/qemu-img create -f qcow2 -F qcow2 -b ${storeImage}/nixos.qcow2 "$TMPDIR"/store.img - ''} + ${lib.optionalString (cfg.useNixStoreImage) + (if cfg.writableStore + then '' + # Create a writable copy/snapshot of the store image. + ${qemu}/bin/qemu-img create -f qcow2 -F qcow2 -b ${storeImage}/nixos.qcow2 "$TMPDIR"/store.img + '' + else '' + ( + cd ${builtins.storeDir} + ${pkgs.erofs-utils}/bin/mkfs.erofs \ + --force-uid=0 \ + --force-gid=0 \ + -U eb176051-bd15-49b7-9e6b-462e0b467019 \ + -T 0 \ + --exclude-regex="$( + <${pkgs.closureInfo { rootPaths = [ config.system.build.toplevel regInfo ]; }}/store-paths \ + sed -e 's^.*/^^g' \ + | cut -c -10 \ + | ${pkgs.python3}/bin/python ${./includes-to-excludes.py} )" \ + "$TMPDIR"/store.img \ + . \ + /dev/null + ) + '' + ) + } # Create a directory for exchanging data with the VM. mkdir -p "$TMPDIR/xchg" @@ -769,6 +790,8 @@ in ); boot.loader.grub.gfxmodeBios = with cfg.resolution; "${toString x}x${toString y}"; + boot.initrd.kernelModules = optionals (cfg.useNixStoreImage && !cfg.writableStore) [ "erofs" ]; + boot.initrd.extraUtilsCommands = lib.mkIf (cfg.useDefaultFilesystems && !config.boot.initrd.systemd.enable) '' # We need mke2fs in the initrd. @@ -905,6 +928,7 @@ in name = "nix-store"; file = ''"$TMPDIR"/store.img''; deviceExtraOpts.bootindex = if cfg.useBootLoader then "3" else "2"; + driveExtraOpts.format = if cfg.writableStore then "qcow2" else "raw"; }]) (mkIf cfg.useBootLoader [ # The order of this list determines the device names, see From 67ebd123ff53ed57eb26294f5005572e981d0ed1 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 16 Jul 2022 18:45:21 +0200 Subject: [PATCH 2/4] nixos/tests/gitlab: Optimize with EROFS --- nixos/tests/gitlab.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nixos/tests/gitlab.nix b/nixos/tests/gitlab.nix index 4f7d3f07f065..d9d75d1cbd89 100644 --- a/nixos/tests/gitlab.nix +++ b/nixos/tests/gitlab.nix @@ -38,6 +38,8 @@ in { virtualisation.memorySize = if pkgs.stdenv.is64bit then 4096 else 2047; virtualisation.cores = 4; virtualisation.useNixStoreImage = true; + virtualisation.writableStore = false; + systemd.services.gitlab.serviceConfig.Restart = mkForce "no"; systemd.services.gitlab-workhorse.serviceConfig.Restart = mkForce "no"; systemd.services.gitaly.serviceConfig.Restart = mkForce "no"; From 7ba6f74c1c0de034049456c3704e9e48fc483bda Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 16 Jul 2022 18:46:31 +0200 Subject: [PATCH 3/4] nixos/tests/discourse: Optimize with EROFS --- nixos/tests/discourse.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/nixos/tests/discourse.nix b/nixos/tests/discourse.nix index cfac5f84a62f..35ca083c6c4e 100644 --- a/nixos/tests/discourse.nix +++ b/nixos/tests/discourse.nix @@ -30,6 +30,7 @@ import ./make-test-python.nix ( virtualisation.memorySize = 2048; virtualisation.cores = 4; virtualisation.useNixStoreImage = true; + virtualisation.writableStore = false; imports = [ common/user-account.nix ]; From 438f6f17de68353f9d453ef113c40ce6816191cd Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 16 Jul 2022 19:02:49 +0200 Subject: [PATCH 4/4] nixos/qemu-vm: Warn when wasting space --- nixos/modules/virtualisation/qemu-vm.nix | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/nixos/modules/virtualisation/qemu-vm.nix b/nixos/modules/virtualisation/qemu-vm.nix index 3fafbbc55dd6..13030f9bd5f6 100644 --- a/nixos/modules/virtualisation/qemu-vm.nix +++ b/nixos/modules/virtualisation/qemu-vm.nix @@ -17,6 +17,8 @@ let cfg = config.virtualisation; + opt = options.virtualisation; + qemu = cfg.qemu.package; consoles = lib.concatMapStringsSep " " (c: "console=${c}") cfg.qemu.consoles; @@ -767,6 +769,26 @@ in } ])); + warnings = + optional ( + cfg.writableStore && + cfg.useNixStoreImage && + opt.writableStore.highestPrio > lib.modules.defaultPriority) + '' + You have enabled ${opt.useNixStoreImage} = true, + without setting ${opt.writableStore} = false. + + This causes a store image to be written to the store, which is + costly, especially for the binary cache, and because of the need + for more frequent garbage collection. + + If you really need this combination, you can set ${opt.writableStore} + explicitly to false, incur the cost and make this warning go away. + Otherwise, we recommend + + ${opt.writableStore} = false; + ''; + # Note [Disk layout with `useBootLoader`] # # If `useBootLoader = true`, we configure 2 drives: