From 3c65f3d84651fd1b120db0d26bc140f6275aafec Mon Sep 17 00:00:00 2001 From: Emily Date: Mon, 30 Dec 2024 19:42:51 +0000 Subject: [PATCH 01/13] flattenReferencesGraph: avoid `lib.fileset` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I am not sure `nix-gitignore.gitignoreSource` fares any better with chroot stores, but so far it’s always been `lib.fileset` that has caused issues with the installer ISO, and a blanket ban on `lib.fileset` seems like the simplest remedy unless it turns out that one of the other source filtering helpers causes issues. This could probably be omitted entirely as it’s just an aid for development convenience, anyway. This reverts commit ba5a5fac7bb68edfb1de329ca037ebe0d4094560. This reverts commit 9ae0f7757760d269c0f3865aed98a3dfaafd6d3d. --- pkgs/by-name/fl/flattenReferencesGraph/package.nix | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/pkgs/by-name/fl/flattenReferencesGraph/package.nix b/pkgs/by-name/fl/flattenReferencesGraph/package.nix index 9adb9a320e0c..c3c7edb17609 100644 --- a/pkgs/by-name/fl/flattenReferencesGraph/package.nix +++ b/pkgs/by-name/fl/flattenReferencesGraph/package.nix @@ -1,25 +1,19 @@ { callPackage, - lib, + nix-gitignore, python3Packages, }: let - inherit (lib) fileset; helpers = callPackage ./helpers.nix { }; pythonPackages = python3Packages; + in pythonPackages.buildPythonApplication { version = "0.1.0"; pname = "flatten-references-graph"; - src = fileset.toSource { - root = ./src; - fileset = fileset.unions [ - ./src/.flake8 - ./src/flatten_references_graph - ./src/setup.py - ]; - }; + # Note: this uses only ./src/.gitignore + src = nix-gitignore.gitignoreSource [ ] ./src; propagatedBuildInputs = with pythonPackages; [ igraph From 2eae7d63e29c74afb9bca17746528dc62ba5486d Mon Sep 17 00:00:00 2001 From: Emily Date: Mon, 30 Dec 2024 20:09:11 +0000 Subject: [PATCH 02/13] nixpkgs-manual: avoid `lib.fileset` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I don’t love this, and I’m not convinced it doesn’t have the same pitfalls as filesets, but see the `flattenReferencesGraph` commit for reasoning. It may be better to ban the relevant builtins entirely and just move things into subdirectories when needed, but I didn’t want to do more surgery to this part of the tree than necessary to solve the immediate problem. --- doc/doc-support/lib-function-docs.nix | 7 +++---- doc/doc-support/package.nix | 26 ++++++++++++++++---------- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/doc/doc-support/lib-function-docs.nix b/doc/doc-support/lib-function-docs.nix index 51b4601975c3..9a2f0667dd60 100644 --- a/doc/doc-support/lib-function-docs.nix +++ b/doc/doc-support/lib-function-docs.nix @@ -90,10 +90,7 @@ stdenvNoCC.mkDerivation { name = "nixpkgs-lib-docs"; - src = lib.fileset.toSource { - root = ../..; - fileset = ../../lib; - }; + src = ../../lib; nativeBuildInputs = [ nixdoc @@ -101,6 +98,8 @@ stdenvNoCC.mkDerivation { ]; installPhase = '' + cd .. + export NIX_STATE_DIR=$(mktemp -d) nix-instantiate --eval --strict --json ${./lib-function-locations.nix} \ --arg nixpkgsPath "./." \ diff --git a/doc/doc-support/package.nix b/doc/doc-support/package.nix index c2bbdf09e533..bc5566f3cbd3 100644 --- a/doc/doc-support/package.nix +++ b/doc/doc-support/package.nix @@ -25,16 +25,22 @@ stdenvNoCC.mkDerivation ( nativeBuildInputs = [ nixos-render-docs ]; - src = lib.fileset.toSource { - root = ../.; - fileset = lib.fileset.unions [ - (lib.fileset.fileFilter (file: file.hasExt "md" || file.hasExt "md.in") ../.) - ../style.css - ../anchor-use.js - ../anchor.min.js - ../manpage-urls.json - ../redirects.json - ]; + src = lib.cleanSourceWith { + src = ../.; + filter = + path: type: + type == "directory" + || lib.hasSuffix ".md" path + || lib.hasSuffix ".md.in" path + || lib.elem path ( + map toString [ + ../style.css + ../anchor-use.js + ../anchor.min.js + ../manpage-urls.json + ../redirects.json + ] + ); }; postPatch = '' From f6ce575a03505ef90dbb864009058ac8d1392c3f Mon Sep 17 00:00:00 2001 From: Emily Date: Tue, 31 Dec 2024 00:14:30 +0000 Subject: [PATCH 03/13] nixos/test-driver: avoid `lib.fileset` --- nixos/lib/test-driver/default.nix | 12 +----------- .../lib/test-driver/nixos-test-driver-docstrings.nix | 2 +- .../lib/test-driver/{ => src}/extract-docstrings.py | 0 nixos/lib/test-driver/{ => src}/pyproject.toml | 0 .../test-driver/{ => src}/test_driver/__init__.py | 0 .../lib/test-driver/{ => src}/test_driver/driver.py | 0 .../lib/test-driver/{ => src}/test_driver/logger.py | 0 .../lib/test-driver/{ => src}/test_driver/machine.py | 0 .../{ => src}/test_driver/polling_condition.py | 0 nixos/lib/test-driver/{ => src}/test_driver/py.typed | 0 nixos/lib/test-driver/{ => src}/test_driver/qmp.py | 0 nixos/lib/test-driver/{ => src}/test_driver/vlan.py | 0 12 files changed, 2 insertions(+), 12 deletions(-) rename nixos/lib/test-driver/{ => src}/extract-docstrings.py (100%) rename nixos/lib/test-driver/{ => src}/pyproject.toml (100%) rename nixos/lib/test-driver/{ => src}/test_driver/__init__.py (100%) rename nixos/lib/test-driver/{ => src}/test_driver/driver.py (100%) rename nixos/lib/test-driver/{ => src}/test_driver/logger.py (100%) rename nixos/lib/test-driver/{ => src}/test_driver/machine.py (100%) rename nixos/lib/test-driver/{ => src}/test_driver/polling_condition.py (100%) rename nixos/lib/test-driver/{ => src}/test_driver/py.typed (100%) rename nixos/lib/test-driver/{ => src}/test_driver/qmp.py (100%) rename nixos/lib/test-driver/{ => src}/test_driver/vlan.py (100%) diff --git a/nixos/lib/test-driver/default.nix b/nixos/lib/test-driver/default.nix index 35471f74949c..f22744806d48 100644 --- a/nixos/lib/test-driver/default.nix +++ b/nixos/lib/test-driver/default.nix @@ -14,22 +14,12 @@ extraPythonPackages ? (_: [ ]), nixosTests, }: -let - fs = lib.fileset; -in python3Packages.buildPythonApplication { pname = "nixos-test-driver"; version = "1.1"; pyproject = true; - src = fs.toSource { - root = ./.; - fileset = fs.unions [ - ./pyproject.toml - ./test_driver - ./extract-docstrings.py - ]; - }; + src = ./src; build-system = with python3Packages; [ setuptools diff --git a/nixos/lib/test-driver/nixos-test-driver-docstrings.nix b/nixos/lib/test-driver/nixos-test-driver-docstrings.nix index b76da9ac5fb2..b6450685e047 100644 --- a/nixos/lib/test-driver/nixos-test-driver-docstrings.nix +++ b/nixos/lib/test-driver/nixos-test-driver-docstrings.nix @@ -11,6 +11,6 @@ in runCommand "nixos-test-driver-docstrings" env '' mkdir $out - python3 ${./extract-docstrings.py} ${./test_driver/machine.py} \ + python3 ${./src/extract-docstrings.py} ${./src/test_driver/machine.py} \ > $out/machine-methods.md '' diff --git a/nixos/lib/test-driver/extract-docstrings.py b/nixos/lib/test-driver/src/extract-docstrings.py similarity index 100% rename from nixos/lib/test-driver/extract-docstrings.py rename to nixos/lib/test-driver/src/extract-docstrings.py diff --git a/nixos/lib/test-driver/pyproject.toml b/nixos/lib/test-driver/src/pyproject.toml similarity index 100% rename from nixos/lib/test-driver/pyproject.toml rename to nixos/lib/test-driver/src/pyproject.toml diff --git a/nixos/lib/test-driver/test_driver/__init__.py b/nixos/lib/test-driver/src/test_driver/__init__.py similarity index 100% rename from nixos/lib/test-driver/test_driver/__init__.py rename to nixos/lib/test-driver/src/test_driver/__init__.py diff --git a/nixos/lib/test-driver/test_driver/driver.py b/nixos/lib/test-driver/src/test_driver/driver.py similarity index 100% rename from nixos/lib/test-driver/test_driver/driver.py rename to nixos/lib/test-driver/src/test_driver/driver.py diff --git a/nixos/lib/test-driver/test_driver/logger.py b/nixos/lib/test-driver/src/test_driver/logger.py similarity index 100% rename from nixos/lib/test-driver/test_driver/logger.py rename to nixos/lib/test-driver/src/test_driver/logger.py diff --git a/nixos/lib/test-driver/test_driver/machine.py b/nixos/lib/test-driver/src/test_driver/machine.py similarity index 100% rename from nixos/lib/test-driver/test_driver/machine.py rename to nixos/lib/test-driver/src/test_driver/machine.py diff --git a/nixos/lib/test-driver/test_driver/polling_condition.py b/nixos/lib/test-driver/src/test_driver/polling_condition.py similarity index 100% rename from nixos/lib/test-driver/test_driver/polling_condition.py rename to nixos/lib/test-driver/src/test_driver/polling_condition.py diff --git a/nixos/lib/test-driver/test_driver/py.typed b/nixos/lib/test-driver/src/test_driver/py.typed similarity index 100% rename from nixos/lib/test-driver/test_driver/py.typed rename to nixos/lib/test-driver/src/test_driver/py.typed diff --git a/nixos/lib/test-driver/test_driver/qmp.py b/nixos/lib/test-driver/src/test_driver/qmp.py similarity index 100% rename from nixos/lib/test-driver/test_driver/qmp.py rename to nixos/lib/test-driver/src/test_driver/qmp.py diff --git a/nixos/lib/test-driver/test_driver/vlan.py b/nixos/lib/test-driver/src/test_driver/vlan.py similarity index 100% rename from nixos/lib/test-driver/test_driver/vlan.py rename to nixos/lib/test-driver/src/test_driver/vlan.py From 6ff076b1b121970ed13a38ffee0f58b540d417e8 Mon Sep 17 00:00:00 2001 From: Emily Date: Mon, 30 Dec 2024 19:24:54 +0000 Subject: [PATCH 04/13] tests.makeBinaryWrapper: avoid `lib.fileset` --- .../make-binary-wrapper/{ => add-flags}/add-flags.c | 0 .../{ => add-flags}/add-flags.cmdline | 0 .../make-binary-wrapper/{ => add-flags}/add-flags.env | 0 pkgs/test/make-binary-wrapper/{ => argv0}/argv0.c | 0 .../make-binary-wrapper/{ => argv0}/argv0.cmdline | 0 pkgs/test/make-binary-wrapper/{ => argv0}/argv0.env | 0 pkgs/test/make-binary-wrapper/{ => basic}/basic.c | 0 .../make-binary-wrapper/{ => basic}/basic.cmdline | 0 pkgs/test/make-binary-wrapper/{ => basic}/basic.env | 0 pkgs/test/make-binary-wrapper/{ => chdir}/chdir.c | 0 .../make-binary-wrapper/{ => chdir}/chdir.cmdline | 0 pkgs/test/make-binary-wrapper/{ => chdir}/chdir.env | 0 .../{ => combination}/combination.c | 0 .../{ => combination}/combination.cmdline | 0 .../{ => combination}/combination.env | 0 pkgs/test/make-binary-wrapper/default.nix | 11 +---------- pkgs/test/make-binary-wrapper/{ => env}/env.c | 0 pkgs/test/make-binary-wrapper/{ => env}/env.cmdline | 0 pkgs/test/make-binary-wrapper/{ => env}/env.env | 0 .../{ => inherit-argv0}/inherit-argv0.c | 0 .../{ => inherit-argv0}/inherit-argv0.cmdline | 0 .../{ => inherit-argv0}/inherit-argv0.env | 0 .../{ => invalid-env}/invalid-env.c | 0 .../{ => invalid-env}/invalid-env.cmdline | 0 .../{ => overlength-strings}/overlength-strings.c | 0 .../overlength-strings.cmdline | 0 .../{ => overlength-strings}/overlength-strings.env | 0 pkgs/test/make-binary-wrapper/{ => prefix}/prefix.c | 0 .../make-binary-wrapper/{ => prefix}/prefix.cmdline | 0 pkgs/test/make-binary-wrapper/{ => prefix}/prefix.env | 0 pkgs/test/make-binary-wrapper/{ => suffix}/suffix.c | 0 .../make-binary-wrapper/{ => suffix}/suffix.cmdline | 0 pkgs/test/make-binary-wrapper/{ => suffix}/suffix.env | 0 33 files changed, 1 insertion(+), 10 deletions(-) rename pkgs/test/make-binary-wrapper/{ => add-flags}/add-flags.c (100%) rename pkgs/test/make-binary-wrapper/{ => add-flags}/add-flags.cmdline (100%) rename pkgs/test/make-binary-wrapper/{ => add-flags}/add-flags.env (100%) rename pkgs/test/make-binary-wrapper/{ => argv0}/argv0.c (100%) rename pkgs/test/make-binary-wrapper/{ => argv0}/argv0.cmdline (100%) rename pkgs/test/make-binary-wrapper/{ => argv0}/argv0.env (100%) rename pkgs/test/make-binary-wrapper/{ => basic}/basic.c (100%) rename pkgs/test/make-binary-wrapper/{ => basic}/basic.cmdline (100%) rename pkgs/test/make-binary-wrapper/{ => basic}/basic.env (100%) rename pkgs/test/make-binary-wrapper/{ => chdir}/chdir.c (100%) rename pkgs/test/make-binary-wrapper/{ => chdir}/chdir.cmdline (100%) rename pkgs/test/make-binary-wrapper/{ => chdir}/chdir.env (100%) rename pkgs/test/make-binary-wrapper/{ => combination}/combination.c (100%) rename pkgs/test/make-binary-wrapper/{ => combination}/combination.cmdline (100%) rename pkgs/test/make-binary-wrapper/{ => combination}/combination.env (100%) rename pkgs/test/make-binary-wrapper/{ => env}/env.c (100%) rename pkgs/test/make-binary-wrapper/{ => env}/env.cmdline (100%) rename pkgs/test/make-binary-wrapper/{ => env}/env.env (100%) rename pkgs/test/make-binary-wrapper/{ => inherit-argv0}/inherit-argv0.c (100%) rename pkgs/test/make-binary-wrapper/{ => inherit-argv0}/inherit-argv0.cmdline (100%) rename pkgs/test/make-binary-wrapper/{ => inherit-argv0}/inherit-argv0.env (100%) rename pkgs/test/make-binary-wrapper/{ => invalid-env}/invalid-env.c (100%) rename pkgs/test/make-binary-wrapper/{ => invalid-env}/invalid-env.cmdline (100%) rename pkgs/test/make-binary-wrapper/{ => overlength-strings}/overlength-strings.c (100%) rename pkgs/test/make-binary-wrapper/{ => overlength-strings}/overlength-strings.cmdline (100%) rename pkgs/test/make-binary-wrapper/{ => overlength-strings}/overlength-strings.env (100%) rename pkgs/test/make-binary-wrapper/{ => prefix}/prefix.c (100%) rename pkgs/test/make-binary-wrapper/{ => prefix}/prefix.cmdline (100%) rename pkgs/test/make-binary-wrapper/{ => prefix}/prefix.env (100%) rename pkgs/test/make-binary-wrapper/{ => suffix}/suffix.c (100%) rename pkgs/test/make-binary-wrapper/{ => suffix}/suffix.cmdline (100%) rename pkgs/test/make-binary-wrapper/{ => suffix}/suffix.env (100%) diff --git a/pkgs/test/make-binary-wrapper/add-flags.c b/pkgs/test/make-binary-wrapper/add-flags/add-flags.c similarity index 100% rename from pkgs/test/make-binary-wrapper/add-flags.c rename to pkgs/test/make-binary-wrapper/add-flags/add-flags.c diff --git a/pkgs/test/make-binary-wrapper/add-flags.cmdline b/pkgs/test/make-binary-wrapper/add-flags/add-flags.cmdline similarity index 100% rename from pkgs/test/make-binary-wrapper/add-flags.cmdline rename to pkgs/test/make-binary-wrapper/add-flags/add-flags.cmdline diff --git a/pkgs/test/make-binary-wrapper/add-flags.env b/pkgs/test/make-binary-wrapper/add-flags/add-flags.env similarity index 100% rename from pkgs/test/make-binary-wrapper/add-flags.env rename to pkgs/test/make-binary-wrapper/add-flags/add-flags.env diff --git a/pkgs/test/make-binary-wrapper/argv0.c b/pkgs/test/make-binary-wrapper/argv0/argv0.c similarity index 100% rename from pkgs/test/make-binary-wrapper/argv0.c rename to pkgs/test/make-binary-wrapper/argv0/argv0.c diff --git a/pkgs/test/make-binary-wrapper/argv0.cmdline b/pkgs/test/make-binary-wrapper/argv0/argv0.cmdline similarity index 100% rename from pkgs/test/make-binary-wrapper/argv0.cmdline rename to pkgs/test/make-binary-wrapper/argv0/argv0.cmdline diff --git a/pkgs/test/make-binary-wrapper/argv0.env b/pkgs/test/make-binary-wrapper/argv0/argv0.env similarity index 100% rename from pkgs/test/make-binary-wrapper/argv0.env rename to pkgs/test/make-binary-wrapper/argv0/argv0.env diff --git a/pkgs/test/make-binary-wrapper/basic.c b/pkgs/test/make-binary-wrapper/basic/basic.c similarity index 100% rename from pkgs/test/make-binary-wrapper/basic.c rename to pkgs/test/make-binary-wrapper/basic/basic.c diff --git a/pkgs/test/make-binary-wrapper/basic.cmdline b/pkgs/test/make-binary-wrapper/basic/basic.cmdline similarity index 100% rename from pkgs/test/make-binary-wrapper/basic.cmdline rename to pkgs/test/make-binary-wrapper/basic/basic.cmdline diff --git a/pkgs/test/make-binary-wrapper/basic.env b/pkgs/test/make-binary-wrapper/basic/basic.env similarity index 100% rename from pkgs/test/make-binary-wrapper/basic.env rename to pkgs/test/make-binary-wrapper/basic/basic.env diff --git a/pkgs/test/make-binary-wrapper/chdir.c b/pkgs/test/make-binary-wrapper/chdir/chdir.c similarity index 100% rename from pkgs/test/make-binary-wrapper/chdir.c rename to pkgs/test/make-binary-wrapper/chdir/chdir.c diff --git a/pkgs/test/make-binary-wrapper/chdir.cmdline b/pkgs/test/make-binary-wrapper/chdir/chdir.cmdline similarity index 100% rename from pkgs/test/make-binary-wrapper/chdir.cmdline rename to pkgs/test/make-binary-wrapper/chdir/chdir.cmdline diff --git a/pkgs/test/make-binary-wrapper/chdir.env b/pkgs/test/make-binary-wrapper/chdir/chdir.env similarity index 100% rename from pkgs/test/make-binary-wrapper/chdir.env rename to pkgs/test/make-binary-wrapper/chdir/chdir.env diff --git a/pkgs/test/make-binary-wrapper/combination.c b/pkgs/test/make-binary-wrapper/combination/combination.c similarity index 100% rename from pkgs/test/make-binary-wrapper/combination.c rename to pkgs/test/make-binary-wrapper/combination/combination.c diff --git a/pkgs/test/make-binary-wrapper/combination.cmdline b/pkgs/test/make-binary-wrapper/combination/combination.cmdline similarity index 100% rename from pkgs/test/make-binary-wrapper/combination.cmdline rename to pkgs/test/make-binary-wrapper/combination/combination.cmdline diff --git a/pkgs/test/make-binary-wrapper/combination.env b/pkgs/test/make-binary-wrapper/combination/combination.env similarity index 100% rename from pkgs/test/make-binary-wrapper/combination.env rename to pkgs/test/make-binary-wrapper/combination/combination.env diff --git a/pkgs/test/make-binary-wrapper/default.nix b/pkgs/test/make-binary-wrapper/default.nix index 036b70f7594e..6f086de299a0 100644 --- a/pkgs/test/make-binary-wrapper/default.nix +++ b/pkgs/test/make-binary-wrapper/default.nix @@ -20,16 +20,7 @@ let runCommand "make-binary-wrapper-test-${testname}" env '' mkdir -p tmp/foo # for the chdir test - source=${ - lib.fileset.toSource { - root = ./.; - fileset = lib.fileset.unions [ - (./. + "/${testname}.cmdline") - (./. + "/${testname}.c") - (lib.fileset.maybeMissing (./. + "/${testname}.env")) - ]; - } - } + source=${./${testname}} params=$(<"$source/${testname}.cmdline") eval "makeCWrapper /send/me/flags $params" > wrapper.c diff --git a/pkgs/test/make-binary-wrapper/env.c b/pkgs/test/make-binary-wrapper/env/env.c similarity index 100% rename from pkgs/test/make-binary-wrapper/env.c rename to pkgs/test/make-binary-wrapper/env/env.c diff --git a/pkgs/test/make-binary-wrapper/env.cmdline b/pkgs/test/make-binary-wrapper/env/env.cmdline similarity index 100% rename from pkgs/test/make-binary-wrapper/env.cmdline rename to pkgs/test/make-binary-wrapper/env/env.cmdline diff --git a/pkgs/test/make-binary-wrapper/env.env b/pkgs/test/make-binary-wrapper/env/env.env similarity index 100% rename from pkgs/test/make-binary-wrapper/env.env rename to pkgs/test/make-binary-wrapper/env/env.env diff --git a/pkgs/test/make-binary-wrapper/inherit-argv0.c b/pkgs/test/make-binary-wrapper/inherit-argv0/inherit-argv0.c similarity index 100% rename from pkgs/test/make-binary-wrapper/inherit-argv0.c rename to pkgs/test/make-binary-wrapper/inherit-argv0/inherit-argv0.c diff --git a/pkgs/test/make-binary-wrapper/inherit-argv0.cmdline b/pkgs/test/make-binary-wrapper/inherit-argv0/inherit-argv0.cmdline similarity index 100% rename from pkgs/test/make-binary-wrapper/inherit-argv0.cmdline rename to pkgs/test/make-binary-wrapper/inherit-argv0/inherit-argv0.cmdline diff --git a/pkgs/test/make-binary-wrapper/inherit-argv0.env b/pkgs/test/make-binary-wrapper/inherit-argv0/inherit-argv0.env similarity index 100% rename from pkgs/test/make-binary-wrapper/inherit-argv0.env rename to pkgs/test/make-binary-wrapper/inherit-argv0/inherit-argv0.env diff --git a/pkgs/test/make-binary-wrapper/invalid-env.c b/pkgs/test/make-binary-wrapper/invalid-env/invalid-env.c similarity index 100% rename from pkgs/test/make-binary-wrapper/invalid-env.c rename to pkgs/test/make-binary-wrapper/invalid-env/invalid-env.c diff --git a/pkgs/test/make-binary-wrapper/invalid-env.cmdline b/pkgs/test/make-binary-wrapper/invalid-env/invalid-env.cmdline similarity index 100% rename from pkgs/test/make-binary-wrapper/invalid-env.cmdline rename to pkgs/test/make-binary-wrapper/invalid-env/invalid-env.cmdline diff --git a/pkgs/test/make-binary-wrapper/overlength-strings.c b/pkgs/test/make-binary-wrapper/overlength-strings/overlength-strings.c similarity index 100% rename from pkgs/test/make-binary-wrapper/overlength-strings.c rename to pkgs/test/make-binary-wrapper/overlength-strings/overlength-strings.c diff --git a/pkgs/test/make-binary-wrapper/overlength-strings.cmdline b/pkgs/test/make-binary-wrapper/overlength-strings/overlength-strings.cmdline similarity index 100% rename from pkgs/test/make-binary-wrapper/overlength-strings.cmdline rename to pkgs/test/make-binary-wrapper/overlength-strings/overlength-strings.cmdline diff --git a/pkgs/test/make-binary-wrapper/overlength-strings.env b/pkgs/test/make-binary-wrapper/overlength-strings/overlength-strings.env similarity index 100% rename from pkgs/test/make-binary-wrapper/overlength-strings.env rename to pkgs/test/make-binary-wrapper/overlength-strings/overlength-strings.env diff --git a/pkgs/test/make-binary-wrapper/prefix.c b/pkgs/test/make-binary-wrapper/prefix/prefix.c similarity index 100% rename from pkgs/test/make-binary-wrapper/prefix.c rename to pkgs/test/make-binary-wrapper/prefix/prefix.c diff --git a/pkgs/test/make-binary-wrapper/prefix.cmdline b/pkgs/test/make-binary-wrapper/prefix/prefix.cmdline similarity index 100% rename from pkgs/test/make-binary-wrapper/prefix.cmdline rename to pkgs/test/make-binary-wrapper/prefix/prefix.cmdline diff --git a/pkgs/test/make-binary-wrapper/prefix.env b/pkgs/test/make-binary-wrapper/prefix/prefix.env similarity index 100% rename from pkgs/test/make-binary-wrapper/prefix.env rename to pkgs/test/make-binary-wrapper/prefix/prefix.env diff --git a/pkgs/test/make-binary-wrapper/suffix.c b/pkgs/test/make-binary-wrapper/suffix/suffix.c similarity index 100% rename from pkgs/test/make-binary-wrapper/suffix.c rename to pkgs/test/make-binary-wrapper/suffix/suffix.c diff --git a/pkgs/test/make-binary-wrapper/suffix.cmdline b/pkgs/test/make-binary-wrapper/suffix/suffix.cmdline similarity index 100% rename from pkgs/test/make-binary-wrapper/suffix.cmdline rename to pkgs/test/make-binary-wrapper/suffix/suffix.cmdline diff --git a/pkgs/test/make-binary-wrapper/suffix.env b/pkgs/test/make-binary-wrapper/suffix/suffix.env similarity index 100% rename from pkgs/test/make-binary-wrapper/suffix.env rename to pkgs/test/make-binary-wrapper/suffix/suffix.env From f6cc18e0e46a4d3c9cace01eb698ac9c26cbed78 Mon Sep 17 00:00:00 2001 From: Emily Date: Tue, 31 Dec 2024 00:42:35 +0000 Subject: [PATCH 05/13] tests.haskell.cabalSdist: avoid `lib.fileset` --- pkgs/test/haskell/cabalSdist/default.nix | 18 +++--------------- .../cabalSdist/{local => }/generated.nix | 5 +++-- 2 files changed, 6 insertions(+), 17 deletions(-) rename pkgs/test/haskell/cabalSdist/{local => }/generated.nix (64%) diff --git a/pkgs/test/haskell/cabalSdist/default.nix b/pkgs/test/haskell/cabalSdist/default.nix index d9dc2bada6e2..147602126ded 100644 --- a/pkgs/test/haskell/cabalSdist/default.nix +++ b/pkgs/test/haskell/cabalSdist/default.nix @@ -6,18 +6,7 @@ }: let - src = lib.fileset.toSource { - root = ./local; - fileset = lib.fileset.unions [ - ./local/app - ./local/CHANGELOG.md - ./local/local.cabal - ]; - }; - # This prevents the source from depending on the formatting of the ./local/generated.nix file - localRaw = haskell.lib.compose.overrideSrc { - inherit src; - } (haskellPackages.callPackage ./local/generated.nix { }); + localRaw = haskellPackages.callPackage ./generated.nix { }; in lib.recurseIntoAttrs rec { @@ -28,7 +17,6 @@ lib.recurseIntoAttrs rec { localFromCabalSdist = haskellPackages.buildFromCabalSdist localRaw; - # NOTE: ./local refers to the "./." path in `./local/generated.nix`. # This test makes sure that localHasNoDirectReference can actually fail if # it doesn't do anything. If this test fails, either the test setup was broken, # or Haskell packaging has changed the way `src` is treated in such a way that @@ -39,7 +27,7 @@ lib.recurseIntoAttrs rec { drvPath = builtins.unsafeDiscardOutputDependency localRaw.drvPath; } '' - grep ${src} $drvPath >/dev/null + grep ${localRaw.src} $drvPath >/dev/null touch $out ''; @@ -49,7 +37,7 @@ lib.recurseIntoAttrs rec { drvPath = builtins.unsafeDiscardOutputDependency localFromCabalSdist.drvPath; } '' - grep -v ${src} $drvPath >/dev/null + grep -v ${localRaw.src} $drvPath >/dev/null touch $out ''; } diff --git a/pkgs/test/haskell/cabalSdist/local/generated.nix b/pkgs/test/haskell/cabalSdist/generated.nix similarity index 64% rename from pkgs/test/haskell/cabalSdist/local/generated.nix rename to pkgs/test/haskell/cabalSdist/generated.nix index f0f52a03e0d0..0cf12d6d565d 100644 --- a/pkgs/test/haskell/cabalSdist/local/generated.nix +++ b/pkgs/test/haskell/cabalSdist/generated.nix @@ -1,4 +1,4 @@ -# nix run ../../../../..#cabal2nix -- ./. +# nix run ../../../../..#cabal2nix -- ./local { mkDerivation, base, @@ -7,10 +7,11 @@ mkDerivation { pname = "local"; version = "0.1.0.0"; - src = ./.; # also referred to as ./local in the test; these are the same path constants + src = ./local; isLibrary = false; isExecutable = true; executableHaskellDepends = [ base ]; description = "Nixpkgs test case"; license = lib.licenses.mit; + mainProgram = "local"; } From 77def225cb60ae7c35ae9c82034aababed016f50 Mon Sep 17 00:00:00 2001 From: Emily Date: Tue, 31 Dec 2024 00:52:44 +0000 Subject: [PATCH 06/13] tests.haskell.setBuildTarget: avoid `lib.fileset` --- pkgs/test/haskell/setBuildTarget/default.nix | 14 +++----------- pkgs/test/haskell/setBuildTarget/{ => src}/Bar.hs | 0 pkgs/test/haskell/setBuildTarget/{ => src}/Foo.hs | 0 .../test/haskell/setBuildTarget/{ => src}/Setup.hs | 0 .../{ => src}/haskell-setBuildTarget.cabal | 0 5 files changed, 3 insertions(+), 11 deletions(-) rename pkgs/test/haskell/setBuildTarget/{ => src}/Bar.hs (100%) rename pkgs/test/haskell/setBuildTarget/{ => src}/Foo.hs (100%) rename pkgs/test/haskell/setBuildTarget/{ => src}/Setup.hs (100%) rename pkgs/test/haskell/setBuildTarget/{ => src}/haskell-setBuildTarget.cabal (100%) diff --git a/pkgs/test/haskell/setBuildTarget/default.nix b/pkgs/test/haskell/setBuildTarget/default.nix index 012bce8541e6..68f9b25dba75 100644 --- a/pkgs/test/haskell/setBuildTarget/default.nix +++ b/pkgs/test/haskell/setBuildTarget/default.nix @@ -1,7 +1,7 @@ { pkgs, haskellPackages }: let - # This can be regenerated by running `cabal2nix .` in the current directory. + # This can be regenerated by running `cabal2nix ./src` in the current directory. pkgDef = { mkDerivation, @@ -11,19 +11,11 @@ let mkDerivation { pname = "haskell-setBuildTarget"; version = "0.1.0.0"; - src = lib.fileset.toSource { - root = ./.; - fileset = lib.fileset.unions [ - ./haskell-setBuildTarget.cabal - ./Bar.hs - ./Foo.hs - ./Setup.hs - ]; - }; + src = ./src; isLibrary = false; isExecutable = true; executableHaskellDepends = [ base ]; - license = lib.licenses.bsd3; + license = "unknown"; }; drv = haskellPackages.callPackage pkgDef { }; diff --git a/pkgs/test/haskell/setBuildTarget/Bar.hs b/pkgs/test/haskell/setBuildTarget/src/Bar.hs similarity index 100% rename from pkgs/test/haskell/setBuildTarget/Bar.hs rename to pkgs/test/haskell/setBuildTarget/src/Bar.hs diff --git a/pkgs/test/haskell/setBuildTarget/Foo.hs b/pkgs/test/haskell/setBuildTarget/src/Foo.hs similarity index 100% rename from pkgs/test/haskell/setBuildTarget/Foo.hs rename to pkgs/test/haskell/setBuildTarget/src/Foo.hs diff --git a/pkgs/test/haskell/setBuildTarget/Setup.hs b/pkgs/test/haskell/setBuildTarget/src/Setup.hs similarity index 100% rename from pkgs/test/haskell/setBuildTarget/Setup.hs rename to pkgs/test/haskell/setBuildTarget/src/Setup.hs diff --git a/pkgs/test/haskell/setBuildTarget/haskell-setBuildTarget.cabal b/pkgs/test/haskell/setBuildTarget/src/haskell-setBuildTarget.cabal similarity index 100% rename from pkgs/test/haskell/setBuildTarget/haskell-setBuildTarget.cabal rename to pkgs/test/haskell/setBuildTarget/src/haskell-setBuildTarget.cabal From 44475aafe3c1f67f7a1f304632c9271c61589a02 Mon Sep 17 00:00:00 2001 From: Emily Date: Tue, 31 Dec 2024 01:02:53 +0000 Subject: [PATCH 07/13] tests.testers.shellcheck: avoid `lib.fileset` --- .../testers/shellcheck/{ => src}/example.sh | 0 pkgs/build-support/testers/shellcheck/tester.nix | 11 ++++++----- pkgs/build-support/testers/shellcheck/tests.nix | 12 ++---------- 3 files changed, 8 insertions(+), 15 deletions(-) rename pkgs/build-support/testers/shellcheck/{ => src}/example.sh (100%) diff --git a/pkgs/build-support/testers/shellcheck/example.sh b/pkgs/build-support/testers/shellcheck/src/example.sh similarity index 100% rename from pkgs/build-support/testers/shellcheck/example.sh rename to pkgs/build-support/testers/shellcheck/src/example.sh diff --git a/pkgs/build-support/testers/shellcheck/tester.nix b/pkgs/build-support/testers/shellcheck/tester.nix index 131a64a85157..caa4f51f9927 100644 --- a/pkgs/build-support/testers/shellcheck/tester.nix +++ b/pkgs/build-support/testers/shellcheck/tester.nix @@ -2,6 +2,7 @@ { lib, stdenv, + runCommand, shellcheck, }: @@ -10,7 +11,7 @@ # Tests: ./tests.nix { src }: let - inherit (lib) fileset pathType isPath; + inherit (lib) pathType isPath; in stdenv.mkDerivation { name = "run-shellcheck"; @@ -18,10 +19,10 @@ stdenv.mkDerivation { if isPath src && pathType src == "regular" # note that for strings this would have been IFD, which we prefer to avoid then - fileset.toSource { - root = dirOf src; - fileset = src; - } + runCommand "testers-shellcheck-src" { } '' + mkdir $out + cp ${src} $out + '' else src; nativeBuildInputs = [ shellcheck ]; diff --git a/pkgs/build-support/testers/shellcheck/tests.nix b/pkgs/build-support/testers/shellcheck/tests.nix index 0128670dbe70..4a9ebdf9ffb1 100644 --- a/pkgs/build-support/testers/shellcheck/tests.nix +++ b/pkgs/build-support/testers/shellcheck/tests.nix @@ -6,9 +6,6 @@ testers, runCommand, }: -let - inherit (lib) fileset; -in lib.recurseIntoAttrs { example-dir = @@ -16,12 +13,7 @@ lib.recurseIntoAttrs { { failure = testers.testBuildFailure ( testers.shellcheck { - src = fileset.toSource { - root = ./.; - fileset = fileset.unions [ - ./example.sh - ]; - }; + src = ./src; } ); } @@ -37,7 +29,7 @@ lib.recurseIntoAttrs { { failure = testers.testBuildFailure ( testers.shellcheck { - src = ./example.sh; + src = ./src/example.sh; } ); } From ba42e09bf2dca362e84500f1169b921164bbb8de Mon Sep 17 00:00:00 2001 From: Emily Date: Tue, 31 Dec 2024 00:03:59 +0000 Subject: [PATCH 08/13] yarn2nix-moretea: avoid `lib.fileset` --- .../yarn2nix-moretea/{yarn2nix => }/default.nix | 14 +++----------- .../{yarn2nix => }/internal/fixup_bin.js | 0 .../{yarn2nix => }/internal/fixup_yarn_lock.js | 0 .../{yarn2nix => }/nix/expectShFunctions.sh | 0 .../tools/yarn2nix-moretea/{yarn2nix => }/yarn.nix | 0 pkgs/top-level/all-packages.nix | 2 +- 6 files changed, 4 insertions(+), 12 deletions(-) rename pkgs/development/tools/yarn2nix-moretea/{yarn2nix => }/default.nix (98%) rename pkgs/development/tools/yarn2nix-moretea/{yarn2nix => }/internal/fixup_bin.js (100%) rename pkgs/development/tools/yarn2nix-moretea/{yarn2nix => }/internal/fixup_yarn_lock.js (100%) rename pkgs/development/tools/yarn2nix-moretea/{yarn2nix => }/nix/expectShFunctions.sh (100%) rename pkgs/development/tools/yarn2nix-moretea/{yarn2nix => }/yarn.nix (100%) diff --git a/pkgs/development/tools/yarn2nix-moretea/yarn2nix/default.nix b/pkgs/development/tools/yarn2nix-moretea/default.nix similarity index 98% rename from pkgs/development/tools/yarn2nix-moretea/yarn2nix/default.nix rename to pkgs/development/tools/yarn2nix-moretea/default.nix index 3cc5e31c3c15..94048c28f88b 100644 --- a/pkgs/development/tools/yarn2nix-moretea/yarn2nix/default.nix +++ b/pkgs/development/tools/yarn2nix-moretea/default.nix @@ -398,15 +398,7 @@ in rec { }); yarn2nix = mkYarnPackage { - src = lib.fileset.toSource { - root = ./.; - fileset = lib.fileset.unions [ - ./bin - ./lib - ./package.json - ./yarn.lock - ]; - }; + src = ./yarn2nix; # yarn2nix is the only package that requires the yarnNix option. # All the other projects can auto-generate that file. @@ -415,7 +407,7 @@ in rec { # Using the filter above and importing package.json from the filtered # source results in an error in restricted mode. To circumvent this, # we import package.json from the unfiltered source - packageJSON = ./package.json; + packageJSON = ./yarn2nix/package.json; yarnFlags = defaultYarnFlags ++ [ "--ignore-scripts" "--production=true" ]; @@ -446,7 +438,7 @@ in rec { mkdir -p $out/lib mkdir -p $out/bin - cp ${./lib/urlToName.js} $out/lib/urlToName.js + cp ${./yarn2nix/lib/urlToName.js} $out/lib/urlToName.js cp ${./internal/fixup_yarn_lock.js} $out/bin/fixup_yarn_lock patchShebangs $out diff --git a/pkgs/development/tools/yarn2nix-moretea/yarn2nix/internal/fixup_bin.js b/pkgs/development/tools/yarn2nix-moretea/internal/fixup_bin.js similarity index 100% rename from pkgs/development/tools/yarn2nix-moretea/yarn2nix/internal/fixup_bin.js rename to pkgs/development/tools/yarn2nix-moretea/internal/fixup_bin.js diff --git a/pkgs/development/tools/yarn2nix-moretea/yarn2nix/internal/fixup_yarn_lock.js b/pkgs/development/tools/yarn2nix-moretea/internal/fixup_yarn_lock.js similarity index 100% rename from pkgs/development/tools/yarn2nix-moretea/yarn2nix/internal/fixup_yarn_lock.js rename to pkgs/development/tools/yarn2nix-moretea/internal/fixup_yarn_lock.js diff --git a/pkgs/development/tools/yarn2nix-moretea/yarn2nix/nix/expectShFunctions.sh b/pkgs/development/tools/yarn2nix-moretea/nix/expectShFunctions.sh similarity index 100% rename from pkgs/development/tools/yarn2nix-moretea/yarn2nix/nix/expectShFunctions.sh rename to pkgs/development/tools/yarn2nix-moretea/nix/expectShFunctions.sh diff --git a/pkgs/development/tools/yarn2nix-moretea/yarn2nix/yarn.nix b/pkgs/development/tools/yarn2nix-moretea/yarn.nix similarity index 100% rename from pkgs/development/tools/yarn2nix-moretea/yarn2nix/yarn.nix rename to pkgs/development/tools/yarn2nix-moretea/yarn.nix diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index f2d42a1fdc98..6761f05b422d 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -5679,7 +5679,7 @@ with pkgs; yarn-berry = callPackage ../development/tools/yarn-berry { }; - yarn2nix-moretea = callPackage ../development/tools/yarn2nix-moretea/yarn2nix { pkgs = pkgs.__splicedPackages; }; + yarn2nix-moretea = callPackage ../development/tools/yarn2nix-moretea { pkgs = pkgs.__splicedPackages; }; inherit (yarn2nix-moretea) yarn2nix From 4e30014c062cec5059552c0dfc4a4656d7d59dd4 Mon Sep 17 00:00:00 2001 From: Emily Date: Mon, 30 Dec 2024 23:17:08 +0000 Subject: [PATCH 09/13] shopify-cli: avoid `lib.fileset` --- .../sh/shopify-cli/{ => manifests}/package-lock.json | 0 pkgs/by-name/sh/shopify-cli/{ => manifests}/package.json | 0 pkgs/by-name/sh/shopify-cli/package.nix | 8 +------- pkgs/by-name/sh/shopify-cli/update.sh | 5 +++++ 4 files changed, 6 insertions(+), 7 deletions(-) rename pkgs/by-name/sh/shopify-cli/{ => manifests}/package-lock.json (100%) rename pkgs/by-name/sh/shopify-cli/{ => manifests}/package.json (100%) diff --git a/pkgs/by-name/sh/shopify-cli/package-lock.json b/pkgs/by-name/sh/shopify-cli/manifests/package-lock.json similarity index 100% rename from pkgs/by-name/sh/shopify-cli/package-lock.json rename to pkgs/by-name/sh/shopify-cli/manifests/package-lock.json diff --git a/pkgs/by-name/sh/shopify-cli/package.json b/pkgs/by-name/sh/shopify-cli/manifests/package.json similarity index 100% rename from pkgs/by-name/sh/shopify-cli/package.json rename to pkgs/by-name/sh/shopify-cli/manifests/package.json diff --git a/pkgs/by-name/sh/shopify-cli/package.nix b/pkgs/by-name/sh/shopify-cli/package.nix index b49e430de8be..a45f65ce7506 100644 --- a/pkgs/by-name/sh/shopify-cli/package.nix +++ b/pkgs/by-name/sh/shopify-cli/package.nix @@ -6,13 +6,7 @@ buildNpmPackage { pname = "shopify"; version = version; - src = lib.fileset.toSource { - root = ./.; - fileset = with lib.fileset; unions [ - ./package.json - ./package-lock.json - ]; - }; + src = ./manifests; npmDepsHash = "sha256-QhbOKOs/0GEOeySG4uROzgtD4o7C+6tS/TAaPcmC3xk="; dontNpmBuild = true; diff --git a/pkgs/by-name/sh/shopify-cli/update.sh b/pkgs/by-name/sh/shopify-cli/update.sh index 6421f59a3a17..a63a81b59057 100755 --- a/pkgs/by-name/sh/shopify-cli/update.sh +++ b/pkgs/by-name/sh/shopify-cli/update.sh @@ -18,6 +18,8 @@ if [[ "$UPDATE_NIX_OLD_VERSION" == "$version" ]]; then exit 0 fi +pushd manifests + # Update the package.json sed -i "s|$UPDATE_NIX_OLD_VERSION|$version|g" package.json @@ -25,6 +27,9 @@ sed -i "s|$UPDATE_NIX_OLD_VERSION|$version|g" package.json rm -f package-lock.json npm i --package-lock-only npm_hash=$(prefetch-npm-deps package-lock.json) + +popd + sed -i "s|npmDepsHash = \".*\";|npmDepsHash = \"$npm_hash\";|" package.nix popd From 34a6c331e79081db4d133fc6692105552bbeb52b Mon Sep 17 00:00:00 2001 From: Emily Date: Mon, 30 Dec 2024 21:48:15 +0000 Subject: [PATCH 10/13] purescm: avoid `lib.fileset` --- pkgs/by-name/pu/purescm/README.rst | 4 ++-- .../pu/purescm/{ => manifests}/.gitignore | 0 .../purescm/{ => manifests}/package-lock.json | 0 .../pu/purescm/{ => manifests}/package.json | 0 pkgs/by-name/pu/purescm/package.nix | 19 +++---------------- 5 files changed, 5 insertions(+), 18 deletions(-) rename pkgs/by-name/pu/purescm/{ => manifests}/.gitignore (100%) rename pkgs/by-name/pu/purescm/{ => manifests}/package-lock.json (100%) rename pkgs/by-name/pu/purescm/{ => manifests}/package.json (100%) diff --git a/pkgs/by-name/pu/purescm/README.rst b/pkgs/by-name/pu/purescm/README.rst index 635d4b46d811..ff34f3913009 100644 --- a/pkgs/by-name/pu/purescm/README.rst +++ b/pkgs/by-name/pu/purescm/README.rst @@ -14,6 +14,6 @@ Suggested additional ``buildInputs`` To update this package ====================== -#. Bump the ``./package.json`` version pin -#. Run ``nix-shell -p nodejs --command "npm i --package-lock-only"`` +#. Bump the ``./manifests/package.json`` version pin +#. Run ``(cd manifests && nix-shell -p nodejs --command "npm i --package-lock-only")`` #. Update ``npmDeps.hash`` in the ``package.nix`` diff --git a/pkgs/by-name/pu/purescm/.gitignore b/pkgs/by-name/pu/purescm/manifests/.gitignore similarity index 100% rename from pkgs/by-name/pu/purescm/.gitignore rename to pkgs/by-name/pu/purescm/manifests/.gitignore diff --git a/pkgs/by-name/pu/purescm/package-lock.json b/pkgs/by-name/pu/purescm/manifests/package-lock.json similarity index 100% rename from pkgs/by-name/pu/purescm/package-lock.json rename to pkgs/by-name/pu/purescm/manifests/package-lock.json diff --git a/pkgs/by-name/pu/purescm/package.json b/pkgs/by-name/pu/purescm/manifests/package.json similarity index 100% rename from pkgs/by-name/pu/purescm/package.json rename to pkgs/by-name/pu/purescm/manifests/package.json diff --git a/pkgs/by-name/pu/purescm/package.nix b/pkgs/by-name/pu/purescm/package.nix index faee196e3eb9..9dd16d489a0c 100644 --- a/pkgs/by-name/pu/purescm/package.nix +++ b/pkgs/by-name/pu/purescm/package.nix @@ -8,7 +8,7 @@ let inherit (lib) fileset; - packageLock = builtins.fromJSON (builtins.readFile ./package-lock.json); + packageLock = builtins.fromJSON (builtins.readFile ./manifests/package-lock.json); pname = "purescm"; version = packageLock.packages."node_modules/${pname}".version; @@ -16,24 +16,11 @@ let package = buildNpmPackage { inherit pname version; - src = fileset.toSource { - root = ./.; - fileset = fileset.unions [ - ./package.json - ./package-lock.json - ./.gitignore - ]; - }; + src = ./manifests; dontNpmBuild = true; npmDeps = fetchNpmDeps { - src = fileset.toSource { - root = ./.; - fileset = fileset.unions [ - ./package-lock.json - ./package.json - ]; - }; + src = ./manifests; hash = "sha256-ljeFcLvIET77Q0OR6O5Ok1fGnaxaKaoywpcy2aHq/6o="; }; From bc419f5949c49e1779da9ffecb201e42fc819b71 Mon Sep 17 00:00:00 2001 From: Emily Date: Mon, 30 Dec 2024 21:40:19 +0000 Subject: [PATCH 11/13] python312Packages.waitress-django: avoid `lib.fileset` --- .../python-modules/waitress-django/default.nix | 12 +----------- .../waitress-django/{ => src}/setup.py | 3 +-- 2 files changed, 2 insertions(+), 13 deletions(-) rename pkgs/development/python-modules/waitress-django/{ => src}/setup.py (76%) diff --git a/pkgs/development/python-modules/waitress-django/default.nix b/pkgs/development/python-modules/waitress-django/default.nix index 9358f20e1a33..cb0c46abb611 100644 --- a/pkgs/development/python-modules/waitress-django/default.nix +++ b/pkgs/development/python-modules/waitress-django/default.nix @@ -6,22 +6,12 @@ waitress, }: -let - fs = lib.fileset; -in - buildPythonPackage { pname = "waitress-django"; version = "1.0.0"; pyproject = true; - src = fs.toSource { - root = ./.; - fileset = fs.unions [ - ./setup.py - ./src - ]; - }; + src = ./src; build-system = [ setuptools ]; diff --git a/pkgs/development/python-modules/waitress-django/setup.py b/pkgs/development/python-modules/waitress-django/src/setup.py similarity index 76% rename from pkgs/development/python-modules/waitress-django/setup.py rename to pkgs/development/python-modules/waitress-django/src/setup.py index 16fde78f42a9..65f51237b589 100644 --- a/pkgs/development/python-modules/waitress-django/setup.py +++ b/pkgs/development/python-modules/waitress-django/src/setup.py @@ -7,6 +7,5 @@ setup( name = "waitress-django" , description = "A waitress WSGI server serving django" , author = "Bas van Dijk" , author_email = "v.dijk.bas@gmail.com" - , package_dir = {"" : "src"} - , scripts = ["src/waitress-serve-django"] + , scripts = ["waitress-serve-django"] ) From a38901aa8539613589598f59f567b9001199badd Mon Sep 17 00:00:00 2001 From: Emily Date: Mon, 30 Dec 2024 20:00:27 +0000 Subject: [PATCH 12/13] cudaPackages.saxpy: avoid `lib.fileset` --- pkgs/development/cuda-modules/saxpy/default.nix | 9 +-------- .../cuda-modules/saxpy/{ => src}/CMakeLists.txt | 0 pkgs/development/cuda-modules/saxpy/{ => src}/saxpy.cu | 0 3 files changed, 1 insertion(+), 8 deletions(-) rename pkgs/development/cuda-modules/saxpy/{ => src}/CMakeLists.txt (100%) rename pkgs/development/cuda-modules/saxpy/{ => src}/saxpy.cu (100%) diff --git a/pkgs/development/cuda-modules/saxpy/default.nix b/pkgs/development/cuda-modules/saxpy/default.nix index 57b6120ccea9..c1d1711fc997 100644 --- a/pkgs/development/cuda-modules/saxpy/default.nix +++ b/pkgs/development/cuda-modules/saxpy/default.nix @@ -18,19 +18,12 @@ let libcublas ; inherit (lib) getDev getLib getOutput; - fs = lib.fileset; in backendStdenv.mkDerivation { pname = "saxpy"; version = "unstable-2023-07-11"; - src = fs.toSource { - root = ./.; - fileset = fs.unions [ - ./CMakeLists.txt - ./saxpy.cu - ]; - }; + src = ./src; __structuredAttrs = true; strictDeps = true; diff --git a/pkgs/development/cuda-modules/saxpy/CMakeLists.txt b/pkgs/development/cuda-modules/saxpy/src/CMakeLists.txt similarity index 100% rename from pkgs/development/cuda-modules/saxpy/CMakeLists.txt rename to pkgs/development/cuda-modules/saxpy/src/CMakeLists.txt diff --git a/pkgs/development/cuda-modules/saxpy/saxpy.cu b/pkgs/development/cuda-modules/saxpy/src/saxpy.cu similarity index 100% rename from pkgs/development/cuda-modules/saxpy/saxpy.cu rename to pkgs/development/cuda-modules/saxpy/src/saxpy.cu From 8725e466ef2bcc5be69106c16dbf69b9ef989273 Mon Sep 17 00:00:00 2001 From: Emily Date: Mon, 30 Dec 2024 15:36:05 +0000 Subject: [PATCH 13/13] release: forbid use of `lib.fileset` in Nixpkgs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Due to Nix bug , `builtins.filterSource` and chroot stores interact in a confusing and broken way that breaks `lib.fileset`. This means that uses of the API inside Nixpkgs keep breaking the NixOS installer, blocking the channel. The resulting error messages are inscrutable (they look like “the installer test is trying to download `curl`…?” and eventually bottom out in a derivation that has the wrong `outPath` because of the chroot store causing an incorrect `lib.fileset` result). Whenever this happens, someone (well, in practice K900 or I) has to bisect the change that introduced it and remove the use of `lib.fileset`. This has happened at least three times in the past four months (I believe I might actually be missing one here, but these are the ones I remember and could easily dig up): * * * The options I see here are: 1. Forbid use of `lib.fileset` within Nixpkgs until the Nix bug is fixed. This is the approach taken here. External users of Nixpkgs can continue to use the API as normal, but using it from within something that affects any release jobset `outPath`s will cause an evaluation failure with a hopefully‐helpful error message. 2. Forbid `lib.fileset` and also all of the other library APIs that use `builtins.filterSource`. I’m happy to do this, but so far none of those have broken the installer, so I decided to start small and worry about the others if they end up causing a problem in practice. 3. Forbid `builtins.filterSource` directly. This is hard and would require more invasive `builtins.scopedImport` crimes to do at evaluation time. I think this would realistically have to be done in something like nixpkgs-vet instead and I didn’t have much luck shoehorning a check like this into that codebase when I tried. 4. Fix the Nix bug. This would be great! But also it doesn’t seem to be happening any time soon, it seems difficult to fix in a way that doesn’t subtly break compatibility with the previous semantics, and arguably the fix would need backporting all the way back to 2.3 given our minimum version policy. 5. Do nothing; have people continue to innocuously use `lib.fileset` throughout Nixpkgs, breaking the installer whenever one of them sneaks in to that closure, causing the channel to be blocked and requiring expensive bisections to narrow down the inscrutable test failure to the package using `lib.fileset`, which then needs moving back off it. This sucks for the people who keep having to track it down, holds back important channel bumps, and the criteria for when it’s okay to use `lib.fileset` are not realistically possible to teach to all contributors. I'd be happy to work on (2) as an alternative; (3) would be difficult and seems like overkill, (4) is not really something I trust myself to do and wouldn’t address the immediate problem, and (5) isn’t sustainable. I think that the current approach here is the best trade‐off for now, as `lib.fileset` seems to be the only prominent user of the `builtins.filterSource` API that works with full store paths, exposing it to the Nix bug. It’s unfortunate to lose the nice API, but since we can’t rely on it to produce correct results and the channels keep getting blocked as a result, I don’t think we really have an alternative right now. --- pkgs/top-level/default.nix | 23 +++++++++++++++++- pkgs/top-level/release-cross.nix | 5 +++- pkgs/top-level/release-cuda.nix | 2 ++ pkgs/top-level/release-lib.nix | 5 +++- pkgs/top-level/release-outpaths.nix | 2 ++ pkgs/top-level/release-python.nix | 2 ++ pkgs/top-level/release-small.nix | 2 ++ .../release-unfree-redistributable.nix | 2 ++ pkgs/top-level/release.nix | 24 +++++++++++-------- 9 files changed, 54 insertions(+), 13 deletions(-) diff --git a/pkgs/top-level/default.nix b/pkgs/top-level/default.nix index 5c224802d5bf..fd443ac773ff 100644 --- a/pkgs/top-level/default.nix +++ b/pkgs/top-level/default.nix @@ -27,6 +27,10 @@ , # Allow a configuration attribute set to be passed in as an argument. config ? {} +, # Temporary hack to let Nixpkgs forbid internal use of `lib.fileset` + # until is fixed. + __allowFileset ? true + , # List of overlays layers used to extend Nixpkgs. overlays ? [] @@ -47,7 +51,24 @@ let # Rename the function arguments crossSystem0 = crossSystem; in let - lib = import ../../lib; + pristineLib = import ../../lib; + + lib = + if __allowFileset then + pristineLib + else + pristineLib.extend (_: _: { + fileset = abort '' + + The use of `lib.fileset` is currently forbidden in Nixpkgs due to the + upstream Nix bug . This + causes difficult‐to‐debug errors when combined with chroot stores, + such as in the NixOS installer. + + For packages that require source to be vendored inside Nixpkgs, + please use a subdirectory of the package instead. + ''; + }); inherit (lib) throwIfNot; diff --git a/pkgs/top-level/release-cross.nix b/pkgs/top-level/release-cross.nix index 1ce8f8f709e2..492a7503a03d 100644 --- a/pkgs/top-level/release-cross.nix +++ b/pkgs/top-level/release-cross.nix @@ -17,7 +17,10 @@ , # Strip most of attributes when evaluating to spare memory usage scrubJobs ? true , # Attributes passed to nixpkgs. Don't build packages marked as unfree. - nixpkgsArgs ? { config = { allowUnfree = false; inHydra = true; }; } + nixpkgsArgs ? { + config = { allowUnfree = false; inHydra = true; }; + __allowFileset = false; + } }: let diff --git a/pkgs/top-level/release-cuda.nix b/pkgs/top-level/release-cuda.nix index 945f0e6d2954..a19bc6f8ecaa 100644 --- a/pkgs/top-level/release-cuda.nix +++ b/pkgs/top-level/release-cuda.nix @@ -43,6 +43,8 @@ in "${variant}Support" = true; inHydra = true; }; + + __allowFileset = false; }, ... }@args: diff --git a/pkgs/top-level/release-lib.nix b/pkgs/top-level/release-lib.nix index c7581c47d341..a2821386ce3e 100644 --- a/pkgs/top-level/release-lib.nix +++ b/pkgs/top-level/release-lib.nix @@ -3,7 +3,10 @@ , packageSet ? (import ../..) , scrubJobs ? true , # Attributes passed to nixpkgs. Don't build packages marked as unfree. - nixpkgsArgs ? { config = { allowUnfree = false; inHydra = true; }; } + nixpkgsArgs ? { + config = { allowUnfree = false; inHydra = true; }; + __allowFileset = false; + } }: let diff --git a/pkgs/top-level/release-outpaths.nix b/pkgs/top-level/release-outpaths.nix index 0b70ea631244..e6caabec04e9 100644 --- a/pkgs/top-level/release-outpaths.nix +++ b/pkgs/top-level/release-outpaths.nix @@ -57,6 +57,8 @@ let inHydra = true; }; + + __allowFileset = false; }; }; recurseIntoAttrs = attrs: attrs // { recurseForDerivations = true; }; diff --git a/pkgs/top-level/release-python.nix b/pkgs/top-level/release-python.nix index a6222acaad53..161a6ddb68c3 100644 --- a/pkgs/top-level/release-python.nix +++ b/pkgs/top-level/release-python.nix @@ -15,6 +15,8 @@ allowUnfree = false; inHydra = true; }; + + __allowFileset = false; }, }: diff --git a/pkgs/top-level/release-small.nix b/pkgs/top-level/release-small.nix index 42cf6e79b732..511cef6a0d8a 100644 --- a/pkgs/top-level/release-small.nix +++ b/pkgs/top-level/release-small.nix @@ -19,6 +19,8 @@ allowUnfree = false; inHydra = true; }; + + __allowFileset = false; }, }: diff --git a/pkgs/top-level/release-unfree-redistributable.nix b/pkgs/top-level/release-unfree-redistributable.nix index 96384e9fb020..45940c4d02cc 100644 --- a/pkgs/top-level/release-unfree-redistributable.nix +++ b/pkgs/top-level/release-unfree-redistributable.nix @@ -37,6 +37,8 @@ cudaSupport = true; inHydra = true; }; + + __allowFileset = false; }, # We only build the full package set on infrequently releasing channels. full ? false, diff --git a/pkgs/top-level/release.nix b/pkgs/top-level/release.nix index 0140f0bd9d6a..78bdf1491125 100644 --- a/pkgs/top-level/release.nix +++ b/pkgs/top-level/release.nix @@ -28,16 +28,20 @@ # Strip most of attributes when evaluating to spare memory usage , scrubJobs ? true # Attributes passed to nixpkgs. Don't build packages marked as unfree. -, nixpkgsArgs ? { config = { - allowUnfree = false; - inHydra = true; - # Exceptional unsafe packages that we still build and distribute, - # so users choosing to allow don't have to rebuild them every time. - permittedInsecurePackages = [ - "olm-3.2.16" # see PR #347899 - "kanidm_1_3-1.3.3" - ]; - }; } +, nixpkgsArgs ? { + config = { + allowUnfree = false; + inHydra = true; + # Exceptional unsafe packages that we still build and distribute, + # so users choosing to allow don't have to rebuild them every time. + permittedInsecurePackages = [ + "olm-3.2.16" # see PR #347899 + "kanidm_1_3-1.3.3" + ]; + }; + + __allowFileset = false; + } # This flag, if set to true, will inhibit the use of `mapTestOn` # and `release-lib.packagePlatforms`. Generally, it causes the