From 92ee9598125f78f4c625f7bb49746a9830d2c85c Mon Sep 17 00:00:00 2001 From: Adam Stephens Date: Fri, 27 Oct 2023 12:56:52 -0400 Subject: [PATCH 1/5] distrobuilder: 2.1 -> 3.0 --- .../virtualization/distrobuilder/default.nix | 24 ++++--------------- 1 file changed, 4 insertions(+), 20 deletions(-) diff --git a/pkgs/tools/virtualization/distrobuilder/default.nix b/pkgs/tools/virtualization/distrobuilder/default.nix index a1a0cfde0fe7..71fbb1722ee4 100644 --- a/pkgs/tools/virtualization/distrobuilder/default.nix +++ b/pkgs/tools/virtualization/distrobuilder/default.nix @@ -8,7 +8,6 @@ , gnutar , squashfsTools , debootstrap -, fetchpatch }: let @@ -22,35 +21,20 @@ let in buildGoModule rec { pname = "distrobuilder"; - version = "2.1"; + version = "3.0"; - vendorHash = "sha256-yRMsf8KfpNmVUX4Rn4ZPLUPFZCT/g78MKAfgbFDPVkE="; + vendorHash = "sha256-pFrEkZnrcx0d3oM1klQrNHH+MiLvO4V1uFQdE0kXUqM="; src = fetchFromGitHub { owner = "lxc"; repo = "distrobuilder"; - rev = "distrobuilder-${version}"; - sha256 = "sha256-t3ECLtb0tvIzTWgjmVQDFza+kcm3abTZZMSGYjvw1qQ="; + rev = "refs/tags/distrobuilder-${version}"; + sha256 = "sha256-JfME9VaqaQnrhnzhSLGUy9uU+tki1hXdnwqBUD/5XH0="; fetchSubmodules = false; }; buildInputs = bins; - patches = [ - # go.mod update: needed to to include a newer lxd which contains - # https://github.com/canonical/lxd/commit/d83f061a21f509d42b7a334b97403d2a019a7b52 - # which is needed to fix the build w/glibc-2.36. - (fetchpatch { - url = "https://github.com/lxc/distrobuilder/commit/5346bcc77dd7f141a36a8da851f016d0b929835e.patch"; - sha256 = "sha256-H6cSbY0v/FThx72AvoAvUCs2VCYN/PQ0W4H82mQQ3SI="; - }) - # Fixup to keep it building after go.mod update. - (fetchpatch { - url = "https://github.com/lxc/distrobuilder/commit/2c8cbfbf603e7446efce9f30812812336ccf4f2c.patch"; - sha256 = "sha256-qqofghcHGosR2qycGb02c8rwErFyRRhsRKdQfyah8Ds="; - }) - ]; - # tests require a local keyserver (mkg20001/nixpkgs branch distrobuilder-with-tests) but gpg is currently broken in tests doCheck = false; From e14da5c1b828e370eed56c3169bc01bbab7ff712 Mon Sep 17 00:00:00 2001 From: Adam Stephens Date: Fri, 27 Oct 2023 12:57:37 -0400 Subject: [PATCH 2/5] distrobuilder: expose lxc systemd generator, patched for nixos --- .../virtualization/distrobuilder/default.nix | 3 + .../distrobuilder/generator.nix | 19 +++ .../distrobuilder/nixos-generator.patch | 113 ++++++++++++++++++ 3 files changed, 135 insertions(+) create mode 100644 pkgs/tools/virtualization/distrobuilder/generator.nix create mode 100644 pkgs/tools/virtualization/distrobuilder/nixos-generator.patch diff --git a/pkgs/tools/virtualization/distrobuilder/default.nix b/pkgs/tools/virtualization/distrobuilder/default.nix index 71fbb1722ee4..f87d685979de 100644 --- a/pkgs/tools/virtualization/distrobuilder/default.nix +++ b/pkgs/tools/virtualization/distrobuilder/default.nix @@ -8,6 +8,7 @@ , gnutar , squashfsTools , debootstrap +, callPackage }: let @@ -35,6 +36,8 @@ buildGoModule rec { buildInputs = bins; + passthru.generator = callPackage ./generator.nix { inherit src version; }; + # tests require a local keyserver (mkg20001/nixpkgs branch distrobuilder-with-tests) but gpg is currently broken in tests doCheck = false; diff --git a/pkgs/tools/virtualization/distrobuilder/generator.nix b/pkgs/tools/virtualization/distrobuilder/generator.nix new file mode 100644 index 000000000000..e514a7df2e08 --- /dev/null +++ b/pkgs/tools/virtualization/distrobuilder/generator.nix @@ -0,0 +1,19 @@ +{ stdenvNoCC, lib, src, version, makeWrapper, coreutils, findutils, gnugrep, systemd }: + +stdenvNoCC.mkDerivation { + name = "distrobuilder-nixos-generator"; + + inherit src version; + + patches = [ + ./nixos-generator.patch + ]; + + dontBuild = true; + nativeBuildInputs = [ makeWrapper ]; + + installPhase = '' + install -D -m 0555 distrobuilder/lxc.generator $out/lib/systemd/system-generators/lxc + wrapProgram $out/lib/systemd/system-generators/lxc --prefix PATH : ${lib.makeBinPath [coreutils findutils gnugrep systemd]}:${systemd}/lib/systemd + ''; +} diff --git a/pkgs/tools/virtualization/distrobuilder/nixos-generator.patch b/pkgs/tools/virtualization/distrobuilder/nixos-generator.patch new file mode 100644 index 000000000000..6194f33e1918 --- /dev/null +++ b/pkgs/tools/virtualization/distrobuilder/nixos-generator.patch @@ -0,0 +1,113 @@ +diff --git a/distrobuilder/lxc.generator b/distrobuilder/lxc.generator +index 0ad81d1..69dbfe7 100644 +--- a/distrobuilder/lxc.generator ++++ b/distrobuilder/lxc.generator +@@ -25,16 +25,6 @@ is_incus_vm() { + [ -e /dev/virtio-ports/org.linuxcontainers.incus ] + } + +-# is_in_path succeeds if the given file exists in on of the paths +-is_in_path() { +- # Don't use $PATH as that may not include all relevant paths +- for path in /bin /sbin /usr/bin /usr/sbin /usr/local/bin /usr/local/sbin; do +- [ -e "${path}/$1" ] && return 0 +- done +- +- return 1 +-} +- + ## Fix functions + # fix_ro_paths avoids udevd issues with /sys and /proc being writable + fix_ro_paths() { +@@ -45,35 +35,6 @@ BindReadOnlyPaths=/sys /proc + EOF + } + +-# fix_nm_link_state forces the network interface to a DOWN state ahead of NetworkManager starting up +-fix_nm_link_state() { +- [ -e "/sys/class/net/$1" ] || return 0 +- ip_path= +- if [ -f /sbin/ip ]; then +- ip_path=/sbin/ip +- elif [ -f /bin/ip ]; then +- ip_path=/bin/ip +- else +- return 0 +- fi +- cat <<-EOF > /run/systemd/system/network-device-down.service +-[Unit] +-Description=Turn off network device +-Before=NetworkManager.service +-Before=systemd-networkd.service +-[Service] +-# do not turn off if there is a default route to 169.254.0.1, i.e. the device is a routed nic +-ExecCondition=/bin/sh -c '! /usr/bin/grep -qs 00000000.0100FEA9 /proc/net/route' +-ExecStart=-${ip_path} link set $1 down +-Type=oneshot +-RemainAfterExit=true +-[Install] +-WantedBy=default.target +-EOF +- mkdir -p /run/systemd/system/default.target.wants +- ln -sf /run/systemd/system/network-device-down.service /run/systemd/system/default.target.wants/network-device-down.service +-} +- + # fix_systemd_override_unit generates a unit specific override + fix_systemd_override_unit() { + dropin_dir="/run/systemd/${1}.d" +@@ -112,16 +73,7 @@ fix_systemd_mask() { + # fix_systemd_udev_trigger overrides the systemd-udev-trigger.service to match the latest version + # of the file which uses "ExecStart=-" instead of "ExecStart=". + fix_systemd_udev_trigger() { +- cmd= +- if [ -f /usr/bin/udevadm ]; then +- cmd=/usr/bin/udevadm +- elif [ -f /sbin/udevadm ]; then +- cmd=/sbin/udevadm +- elif [ -f /bin/udevadm ]; then +- cmd=/bin/udevadm +- else +- return 0 +- fi ++ cmd=udevadm + + mkdir -p /run/systemd/system/systemd-udev-trigger.service.d + cat <<-EOF > /run/systemd/system/systemd-udev-trigger.service.d/zzz-lxc-override.conf +@@ -145,24 +97,12 @@ EOF + } + + ## Main logic +-# Nothing to do in Incus VM but deployed in case it is later converted to a container +-is_incus_vm || is_lxd_vm && exit 0 + + # Exit immediately if not an Incus/LXC container + is_lxc_container || exit 0 + +-# Check for NetworkManager +-nm_exists=0 +- +-is_in_path NetworkManager && nm_exists=1 +- + # Determine systemd version +-for path in /usr/lib/systemd/systemd /lib/systemd/systemd; do +- [ -x "${path}" ] || continue +- +- systemd_version="$("${path}" --version | head -n1 | cut -d' ' -f2)" +- break +-done ++systemd_version="$(systemd --version | head -n1 | cut -d' ' -f2)" + + # Determine distro name and release + ID="" +@@ -222,11 +162,6 @@ ACTION=="add|change|move", ENV{ID_NET_DRIVER}=="veth", ENV{INTERFACE}=="eth[0-9] + EOF + fi + +-# Workarounds for NetworkManager in containers +-if [ "${nm_exists}" -eq 1 ]; then +- fix_nm_link_state eth0 +-fi +- + # Allow masking units created by the lxc system-generator. + for d in /etc/systemd/system /usr/lib/systemd/system /lib/systemd/system; do + if ! [ -d "${d}" ]; then From 2cd9619801c0ad8a223ad6bd7915695b440abbb6 Mon Sep 17 00:00:00 2001 From: Adam Stephens Date: Fri, 27 Oct 2023 12:58:50 -0400 Subject: [PATCH 3/5] nixos/lxc-container: use lxc systemd generator --- .../modules/virtualisation/lxc-container.nix | 52 +++---------------- nixos/tests/incus/container.nix | 28 ++++++++++ .../virtualization/distrobuilder/default.nix | 8 ++- 3 files changed, 43 insertions(+), 45 deletions(-) diff --git a/nixos/modules/virtualisation/lxc-container.nix b/nixos/modules/virtualisation/lxc-container.nix index 61d7c4cb73fe..3471db974a15 100644 --- a/nixos/modules/virtualisation/lxc-container.nix +++ b/nixos/modules/virtualisation/lxc-container.nix @@ -1,26 +1,14 @@ { lib, config, pkgs, ... }: -let - cfg = config.virtualisation.lxc; -in { +{ imports = [ ./lxc-instance-common.nix + + (lib.mkRemovedOptionModule [ "virtualisation" "lxc" "nestedContainer" ] "") + (lib.mkRemovedOptionModule [ "virtualisation" "lxc" "privilegedContainer" ] "") ]; - options = { - virtualisation.lxc = { - nestedContainer = lib.mkEnableOption (lib.mdDoc '' - Whether this container is configured as a nested container. On LXD containers this is recommended - for all containers and is enabled with `security.nesting = true`. - ''); - - privilegedContainer = lib.mkEnableOption (lib.mdDoc '' - Whether this LXC container will be running as a privileged container or not. If set to `true` then - additional configuration will be applied to the `systemd` instance running within the container as - recommended by [distrobuilder](https://linuxcontainers.org/distrobuilder/introduction/). - ''); - }; - }; + options = { }; config = { boot.isContainer = true; @@ -85,34 +73,10 @@ in { ${pkgs.coreutils}/bin/ln -fs "$1/init" /sbin/init ''; - systemd.additionalUpstreamSystemUnits = lib.mkIf cfg.nestedContainer ["systemd-udev-trigger.service"]; + # networkd depends on this, but systemd module disables this for containers + systemd.additionalUpstreamSystemUnits = ["systemd-udev-trigger.service"]; - # Add the overrides from lxd distrobuilder - # https://github.com/lxc/distrobuilder/blob/05978d0d5a72718154f1525c7d043e090ba7c3e0/distrobuilder/main.go#L630 - systemd.packages = [ - (pkgs.writeTextFile { - name = "systemd-lxc-service-overrides"; - destination = "/etc/systemd/system/service.d/zzz-lxc-service.conf"; - text = '' - [Service] - ProcSubset=all - ProtectProc=default - ProtectControlGroups=no - ProtectKernelTunables=no - NoNewPrivileges=no - LoadCredential= - '' + lib.optionalString cfg.privilegedContainer '' - # Additional settings for privileged containers - ProtectHome=no - ProtectSystem=no - PrivateDevices=no - PrivateTmp=no - ProtectKernelLogs=no - ProtectKernelModules=no - ReadWritePaths= - ''; - }) - ]; + systemd.packages = [ pkgs.distrobuilder.generator ]; system.activationScripts.installInitScript = lib.mkForce '' ln -fs $systemConfig/init /sbin/init diff --git a/nixos/tests/incus/container.nix b/nixos/tests/incus/container.nix index 79b9e2fbabdc..49a22c08aad1 100644 --- a/nixos/tests/incus/container.nix +++ b/nixos/tests/incus/container.nix @@ -73,5 +73,33 @@ in meminfo = machine.succeed("incus exec container grep -- MemTotal /proc/meminfo").strip() meminfo_bytes = " ".join(meminfo.split(' ')[-2:]) assert meminfo_bytes == "125000 kB", f"Wrong amount of memory reported from /proc/meminfo, want: '125000 kB', got: '{meminfo_bytes}'" + + with subtest("lxc-container generator configures plain container"): + machine.execute("incus delete --force container") + machine.succeed("incus launch nixos container") + with machine.nested("Waiting for instance to start and be usable"): + retry(instance_is_up) + + machine.succeed("incus exec container test -- -e /run/systemd/system/service.d/zzz-lxc-service.conf") + + with subtest("lxc-container generator configures nested container"): + machine.execute("incus delete --force container") + machine.succeed("incus launch nixos container --config security.nesting=true") + with machine.nested("Waiting for instance to start and be usable"): + retry(instance_is_up) + + machine.fail("incus exec container test -- -e /run/systemd/system/service.d/zzz-lxc-service.conf") + target = machine.succeed("incus exec container readlink -- -f /run/systemd/system/systemd-binfmt.service").strip() + assert target == "/dev/null", "lxc generator did not correctly mask /run/systemd/system/systemd-binfmt.service" + + with subtest("lxc-container generator configures privileged container"): + machine.execute("incus delete --force container") + machine.succeed("incus launch nixos container --config security.privileged=true") + with machine.nested("Waiting for instance to start and be usable"): + retry(instance_is_up) + # give generator an extra second to run + machine.sleep(1) + + machine.succeed("incus exec container test -- -e /run/systemd/system/service.d/zzz-lxc-service.conf") ''; }) diff --git a/pkgs/tools/virtualization/distrobuilder/default.nix b/pkgs/tools/virtualization/distrobuilder/default.nix index f87d685979de..e5444d4a5b1e 100644 --- a/pkgs/tools/virtualization/distrobuilder/default.nix +++ b/pkgs/tools/virtualization/distrobuilder/default.nix @@ -9,6 +9,7 @@ , squashfsTools , debootstrap , callPackage +, nixosTests }: let @@ -36,7 +37,6 @@ buildGoModule rec { buildInputs = bins; - passthru.generator = callPackage ./generator.nix { inherit src version; }; # tests require a local keyserver (mkg20001/nixpkgs branch distrobuilder-with-tests) but gpg is currently broken in tests doCheck = false; @@ -50,6 +50,12 @@ buildGoModule rec { wrapProgram $out/bin/distrobuilder --prefix PATH ":" ${lib.makeBinPath bins} ''; + passthru = { + tests.incus = nixosTests.incus.container; + + generator = callPackage ./generator.nix { inherit src version; }; + }; + meta = with lib; { description = "System container image builder for LXC and LXD"; homepage = "https://github.com/lxc/distrobuilder"; From 0fdef67e07d3297e626b42f361c155e38f447a62 Mon Sep 17 00:00:00 2001 From: Adam Stephens Date: Wed, 1 Nov 2023 23:23:42 -0400 Subject: [PATCH 4/5] nixos/lxc-container: add adamcstephens as maintainer --- nixos/modules/virtualisation/lxc-container.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nixos/modules/virtualisation/lxc-container.nix b/nixos/modules/virtualisation/lxc-container.nix index 3471db974a15..4db4df02fe8c 100644 --- a/nixos/modules/virtualisation/lxc-container.nix +++ b/nixos/modules/virtualisation/lxc-container.nix @@ -1,6 +1,8 @@ { lib, config, pkgs, ... }: { + meta.maintainers = with lib.maintainers; [ adamcstephens ]; + imports = [ ./lxc-instance-common.nix From a3f8732797755e3e0107538eeb8a5c81e93f0815 Mon Sep 17 00:00:00 2001 From: Adam Stephens Date: Wed, 1 Nov 2023 23:23:56 -0400 Subject: [PATCH 5/5] distrobuilder: add adamcstephens as maintainer --- pkgs/tools/virtualization/distrobuilder/default.nix | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkgs/tools/virtualization/distrobuilder/default.nix b/pkgs/tools/virtualization/distrobuilder/default.nix index e5444d4a5b1e..e6b867139165 100644 --- a/pkgs/tools/virtualization/distrobuilder/default.nix +++ b/pkgs/tools/virtualization/distrobuilder/default.nix @@ -56,12 +56,12 @@ buildGoModule rec { generator = callPackage ./generator.nix { inherit src version; }; }; - meta = with lib; { + meta = { description = "System container image builder for LXC and LXD"; homepage = "https://github.com/lxc/distrobuilder"; - license = licenses.asl20; - maintainers = with maintainers; [ megheaiulian ]; - platforms = platforms.linux; + license = lib.licenses.asl20; + maintainers = with lib.maintainers; [ megheaiulian adamcstephens ]; + platforms = lib.platforms.linux; mainProgram = "distrobuilder"; }; }