From 5672d3d8b835b510a72b4280c97e8db0765ebef4 Mon Sep 17 00:00:00 2001 From: h7x4 Date: Wed, 1 Nov 2023 19:47:51 +0100 Subject: [PATCH 01/42] nixos/quicktun: clean up module --- .../modules/services/networking/quicktun.nix | 146 ++++++++++++------ 1 file changed, 102 insertions(+), 44 deletions(-) diff --git a/nixos/modules/services/networking/quicktun.nix b/nixos/modules/services/networking/quicktun.nix index 7aed972adc88..2d44659f2080 100644 --- a/nixos/modules/services/networking/quicktun.nix +++ b/nixos/modules/services/networking/quicktun.nix @@ -1,94 +1,153 @@ -{ config, pkgs, lib, ... }: +{ options, config, pkgs, lib, ... }: let + inherit (lib) mkOption mdDoc types mkIf; + opt = options.services.quicktun; cfg = config.services.quicktun; - in - -with lib; - { options = { - services.quicktun = mkOption { default = { }; - description = lib.mdDoc "QuickTun tunnels"; - type = types.attrsOf (types.submodule { + description = mdDoc '' + QuickTun tunnels. + + See for more information about available options. + ''; + type = types.attrsOf (types.submodule ({ name, ... }: let + qtcfg = cfg.${name}; + in { options = { tunMode = mkOption { - type = types.int; - default = 0; - example = 1; - description = lib.mdDoc ""; + type = with types; coercedTo bool (b: if b then 1 else 0) (ints.between 0 1); + default = false; + example = true; + description = mdDoc "Whether to operate in tun (IP) or tap (Ethernet) mode."; }; remoteAddress = mkOption { type = types.str; + default = "0.0.0.0"; example = "tunnel.example.com"; - description = lib.mdDoc ""; + description = mdDoc '' + IP address or hostname of the remote end (use `0.0.0.0` for a floating/dynamic remote endpoint). + ''; }; localAddress = mkOption { - type = types.str; + type = with types; nullOr str; + default = null; example = "0.0.0.0"; - description = lib.mdDoc ""; + description = mdDoc "IP address or hostname of the local end."; }; localPort = mkOption { - type = types.int; + type = types.port; default = 2998; - description = lib.mdDoc ""; + description = mdDoc "Local UDP port."; }; remotePort = mkOption { - type = types.int; - default = 2998; - description = lib.mdDoc ""; + type = types.port; + default = qtcfg.localPort; + defaultText = lib.literalExpression "config.services.quicktun..localPort"; + description = mdDoc " remote UDP port"; }; remoteFloat = mkOption { - type = types.int; - default = 0; - description = lib.mdDoc ""; + type = with types; coercedTo bool (b: if b then 1 else 0) (ints.between 0 1); + default = false; + example = true; + description = mdDoc '' + Whether to allow the remote address and port to change when properly encrypted packets are received. + ''; }; protocol = mkOption { - type = types.str; + type = types.enum [ "raw" "nacl0" "nacltai" "salty" ]; default = "nacltai"; - description = lib.mdDoc ""; + description = mdDoc "Which protocol to use."; }; privateKey = mkOption { - type = types.str; - description = lib.mdDoc ""; + type = with types; nullOr str; + default = null; + description = mdDoc '' + Local secret key in hexadecimal form. + + ::: {.warning} + This option is deprecated. Please use {var}`services.quicktun..privateKeyFile` instead. + ::: + + ::: {.note} + Not needed when {var}`services.quicktun..protocol` is set to `raw`. + ::: + ''; + }; + + privateKeyFile = mkOption { + type = with types; nullOr path; + # This is a hack to deprecate `privateKey` without using `mkChangedModuleOption` + default = if qtcfg.privateKey == null then null else pkgs.writeText "quickttun-key-${name}" qtcfg.privateKey; + defaultText = "null"; + description = mdDoc '' + Path to file containing local secret key in binary or hexadecimal form. + + ::: {.note} + Not needed when {var}`services.quicktun..protocol` is set to `raw`. + ::: + ''; }; publicKey = mkOption { - type = types.str; - description = lib.mdDoc ""; + type = with types; nullOr str; + default = null; + description = mdDoc '' + Remote public key in hexadecimal form. + + ::: {.note} + Not needed when {var}`services.quicktun..protocol` is set to `raw`. + ::: + ''; }; timeWindow = mkOption { - type = types.int; + type = types.ints.unsigned; default = 5; - description = lib.mdDoc ""; + description = mdDoc '' + Allowed time window for first received packet in seconds (positive number allows packets from history) + ''; }; upScript = mkOption { - type = types.lines; - default = ""; - description = lib.mdDoc ""; + type = with types; nullOr lines; + default = null; + description = mdDoc '' + Run specified command or script after the tunnel device has been opened. + ''; }; }; - }); + })); }; - }; - config = mkIf (cfg != []) { - systemd.services = foldr (a: b: a // b) {} ( - mapAttrsToList (name: qtcfg: { + config = { + warnings = lib.pipe cfg [ + (lib.mapAttrsToList (name: value: if value.privateKey != null then name else null)) + (builtins.filter (n: n != null)) + (map (n: " - services.quicktun.${n}.privateKey")) + (services: lib.optional (services != [ ]) '' + `services.quicktun..privateKey` is deprecated. + Please use `services.quicktun..privateKeyFile` instead. + + Offending options: + ${lib.concatStringsSep "\n" services} + '') + ]; + + systemd.services = lib.mkMerge ( + lib.mapAttrsToList (name: qtcfg: { "quicktun-${name}" = { wantedBy = [ "multi-user.target" ]; after = [ "network.target" ]; @@ -96,14 +155,14 @@ with lib; INTERFACE = name; TUN_MODE = toString qtcfg.tunMode; REMOTE_ADDRESS = qtcfg.remoteAddress; - LOCAL_ADDRESS = qtcfg.localAddress; + LOCAL_ADDRESS = mkIf (qtcfg.localAddress != null) (qtcfg.localAddress); LOCAL_PORT = toString qtcfg.localPort; REMOTE_PORT = toString qtcfg.remotePort; REMOTE_FLOAT = toString qtcfg.remoteFloat; - PRIVATE_KEY = qtcfg.privateKey; - PUBLIC_KEY = qtcfg.publicKey; + PRIVATE_KEY_FILE = mkIf (qtcfg.privateKeyFile != null) qtcfg.privateKeyFile; + PUBLIC_KEY = mkIf (qtcfg.publicKey != null) qtcfg.publicKey; TIME_WINDOW = toString qtcfg.timeWindow; - TUN_UP_SCRIPT = pkgs.writeScript "quicktun-${name}-up.sh" qtcfg.upScript; + TUN_UP_SCRIPT = mkIf (qtcfg.upScript != null) (pkgs.writeScript "quicktun-${name}-up.sh" qtcfg.upScript); SUID = "nobody"; }; serviceConfig = { @@ -114,5 +173,4 @@ with lib; }) cfg ); }; - } From 78f663bc0b6d4e24401164f07113533f119d2fa7 Mon Sep 17 00:00:00 2001 From: h7x4 Date: Wed, 1 Nov 2023 19:48:51 +0100 Subject: [PATCH 02/42] nixos/quicktun: add test --- nixos/tests/all-tests.nix | 1 + nixos/tests/quicktun.nix | 18 ++++++++++++++++++ pkgs/tools/networking/quicktun/default.nix | 4 +++- 3 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 nixos/tests/quicktun.nix diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index 4a3f4a331ca8..79dc341eb4f1 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -692,6 +692,7 @@ in { qgis-ltr = handleTest ./qgis.nix { qgisPackage = pkgs.qgis-ltr; }; qownnotes = handleTest ./qownnotes.nix {}; quake3 = handleTest ./quake3.nix {}; + quicktun = handleTest ./quicktun.nix {}; quorum = handleTest ./quorum.nix {}; rabbitmq = handleTest ./rabbitmq.nix {}; radarr = handleTest ./radarr.nix {}; diff --git a/nixos/tests/quicktun.nix b/nixos/tests/quicktun.nix new file mode 100644 index 000000000000..a5a632457117 --- /dev/null +++ b/nixos/tests/quicktun.nix @@ -0,0 +1,18 @@ +import ./make-test-python.nix ({ pkgs, lib, ... }: +{ + name = "quicktun"; + meta.maintainers = with lib.maintainers; [ h7x4 ]; + + nodes = { + machine = { ... }: { + services.quicktun."test-tunnel" = { + protocol = "raw"; + }; + }; + }; + + testScript = '' + start_all() + machine.wait_for_unit("quicktun-test-tunnel.service") + ''; +}) diff --git a/pkgs/tools/networking/quicktun/default.nix b/pkgs/tools/networking/quicktun/default.nix index b997aad0c2d6..b5321725cca9 100644 --- a/pkgs/tools/networking/quicktun/default.nix +++ b/pkgs/tools/networking/quicktun/default.nix @@ -1,4 +1,4 @@ -{ lib, stdenv, fetchFromGitHub, libsodium }: +{ lib, stdenv, fetchFromGitHub, libsodium, nixosTests }: stdenv.mkDerivation { pname = "quicktun"; @@ -22,6 +22,8 @@ stdenv.mkDerivation { install -vD out/quicktun* -t $out/bin ''; + passthru.tests.quicktun = nixosTests.quicktun; + meta = with lib; { broken = stdenv.isDarwin; description = "Very simple, yet secure VPN software"; From 1a409a92aaf5b223dae7447c20daf174faf17dd7 Mon Sep 17 00:00:00 2001 From: h7x4 Date: Wed, 1 Nov 2023 19:49:38 +0100 Subject: [PATCH 03/42] quicktun: add h7x4 as maintainer --- pkgs/tools/networking/quicktun/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/tools/networking/quicktun/default.nix b/pkgs/tools/networking/quicktun/default.nix index b5321725cca9..2c1799387df8 100644 --- a/pkgs/tools/networking/quicktun/default.nix +++ b/pkgs/tools/networking/quicktun/default.nix @@ -28,7 +28,7 @@ stdenv.mkDerivation { broken = stdenv.isDarwin; description = "Very simple, yet secure VPN software"; homepage = "http://wiki.ucis.nl/QuickTun"; - maintainers = [ ]; + maintainers = with maintainers; [ h7x4 ]; platforms = platforms.unix; license = licenses.bsd2; }; From e730fe9bf6b7c290583f61477e05eebfc88e32fc Mon Sep 17 00:00:00 2001 From: "R. Ryantm" Date: Sat, 9 Dec 2023 07:19:44 +0000 Subject: [PATCH 04/42] janus-gateway: 1.2.0 -> 1.2.1 --- pkgs/servers/janus-gateway/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/servers/janus-gateway/default.nix b/pkgs/servers/janus-gateway/default.nix index d62be801a4ca..a8af63754277 100644 --- a/pkgs/servers/janus-gateway/default.nix +++ b/pkgs/servers/janus-gateway/default.nix @@ -15,13 +15,13 @@ in stdenv.mkDerivation rec { pname = "janus-gateway"; - version = "1.2.0"; + version = "1.2.1"; src = fetchFromGitHub { owner = "meetecho"; repo = pname; rev = "v${version}"; - sha256 = "sha256-YbY7wcd8YHcPo5w4n54gjOtepYLbboLsrLij7oYzhco="; + sha256 = "sha256-Bqb4UO4R5CnV8+2OthGrEVORzH+k+zgzI4UsvwRHgk8="; }; nativeBuildInputs = [ autoreconfHook pkg-config gengetopt ]; From 8d09a79fa4a5d79e64e71030232f2e49146e689c Mon Sep 17 00:00:00 2001 From: Andrew Marshall Date: Mon, 11 Dec 2023 10:35:40 -0500 Subject: [PATCH 05/42] davinci-resolve{,-studio}: 18.6.3 -> 18.6.4 --- pkgs/applications/video/davinci-resolve/default.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkgs/applications/video/davinci-resolve/default.nix b/pkgs/applications/video/davinci-resolve/default.nix index 5b44ddc042a3..951d46d783b0 100644 --- a/pkgs/applications/video/davinci-resolve/default.nix +++ b/pkgs/applications/video/davinci-resolve/default.nix @@ -31,7 +31,7 @@ let davinci = ( stdenv.mkDerivation rec { pname = "davinci-resolve${lib.optionalString studioVariant "-studio"}"; - version = "18.6.3"; + version = "18.6.4"; nativeBuildInputs = [ (appimage-run.override { buildFHSEnv = buildFHSEnvChroot; } ) @@ -52,8 +52,8 @@ let outputHashAlgo = "sha256"; outputHash = if studioVariant - then "sha256-OX8PyMhfl0jRdXBNsjlwkCAh8XUNJv8HEbmyAdjIv18=" - else "sha256-PNzdVxGgXIHM2vi3ChHx67TQBFlCYBOZCiFkDi/RSu4="; + then "sha256-Us8DsxdGwBxUL+yUHT9DNJFIV7EO+J9CSN2Juyf8VQ4=" + else "sha256-yPdfmS42ID7MOTB3XlGXfOqp46kRLR8martJ9gWqDjA="; impureEnvVars = lib.fetchers.proxyImpureEnvVars; From b93b449f792e74f79e4b7baa80f35b611b18c151 Mon Sep 17 00:00:00 2001 From: "R. Ryantm" Date: Mon, 11 Dec 2023 22:49:20 +0000 Subject: [PATCH 06/42] git-machete: 3.20.0 -> 3.22.0 --- pkgs/applications/version-management/git-machete/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/applications/version-management/git-machete/default.nix b/pkgs/applications/version-management/git-machete/default.nix index a1f45d65db35..776ec66d3a77 100644 --- a/pkgs/applications/version-management/git-machete/default.nix +++ b/pkgs/applications/version-management/git-machete/default.nix @@ -12,13 +12,13 @@ buildPythonApplication rec { pname = "git-machete"; - version = "3.20.0"; + version = "3.22.0"; src = fetchFromGitHub { owner = "virtuslab"; repo = pname; rev = "v${version}"; - hash = "sha256-6TntyAkDIcCVcAsNdAlgvKYO7Db0oMDWKW92rMRIDI4="; + hash = "sha256-2oEpBNMHj4qpkPp8rXEMsRRiRQeC30hQCQh7d8bOLUU="; }; nativeBuildInputs = [ installShellFiles ]; From 7e705f15e2cd939a4673b28159ce6df017aa1cd6 Mon Sep 17 00:00:00 2001 From: "R. Ryantm" Date: Tue, 12 Dec 2023 01:01:24 +0000 Subject: [PATCH 07/42] gridtracker: 1.23.1202 -> 1.23.1207 --- pkgs/applications/radio/gridtracker/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/applications/radio/gridtracker/default.nix b/pkgs/applications/radio/gridtracker/default.nix index d3a6d104dc26..c961e1bd2d49 100644 --- a/pkgs/applications/radio/gridtracker/default.nix +++ b/pkgs/applications/radio/gridtracker/default.nix @@ -2,13 +2,13 @@ stdenv.mkDerivation rec { pname = "gridtracker"; - version = "1.23.1202"; + version = "1.23.1207"; src = fetchFromGitLab { owner = "gridtracker.org"; repo = "gridtracker"; rev = "v${version}"; - sha256 = "sha256-XrHA+h6qAYyZjp/C7+oS3eAvX0ptD+T4UKFzM2ROBCw="; + sha256 = "sha256-r7H+fds8FbSLDxPQqn0XUPC6loLgsaNX+DBqJJ96/d4="; }; nativeBuildInputs = [ wrapGAppsHook ]; From 8e4e53729cabcc329b3c2a0075373c29e425cf6f Mon Sep 17 00:00:00 2001 From: "R. Ryantm" Date: Wed, 13 Dec 2023 03:53:55 +0000 Subject: [PATCH 08/42] ani-cli: 4.6 -> 4.7 --- pkgs/applications/video/ani-cli/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/applications/video/ani-cli/default.nix b/pkgs/applications/video/ani-cli/default.nix index 389d00dc1a41..375a930b0294 100644 --- a/pkgs/applications/video/ani-cli/default.nix +++ b/pkgs/applications/video/ani-cli/default.nix @@ -21,13 +21,13 @@ assert withMpv || withVlc || withIina; stdenvNoCC.mkDerivation rec { pname = "ani-cli"; - version = "4.6"; + version = "4.7"; src = fetchFromGitHub { owner = "pystardust"; repo = "ani-cli"; rev = "v${version}"; - hash = "sha256-ahyCD4QsYyb3xtNK03HITeF0+hJFIHZ+PAjisuS/Kdo="; + hash = "sha256-Ll4bHKrDZukoQX35iiMI6rMSgiTC6wp7fHUnOMPagOA="; }; nativeBuildInputs = [ makeWrapper ]; From c0a56eb2e801c3938a6e6c1dc70ec2e8a676adad Mon Sep 17 00:00:00 2001 From: "R. Ryantm" Date: Wed, 13 Dec 2023 13:25:29 +0000 Subject: [PATCH 09/42] cargo-run-bin: 1.5.0 -> 1.6.0 --- pkgs/development/tools/rust/cargo-run-bin/default.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkgs/development/tools/rust/cargo-run-bin/default.nix b/pkgs/development/tools/rust/cargo-run-bin/default.nix index 91d914422277..facc1566a990 100644 --- a/pkgs/development/tools/rust/cargo-run-bin/default.nix +++ b/pkgs/development/tools/rust/cargo-run-bin/default.nix @@ -5,14 +5,14 @@ rustPlatform.buildRustPackage rec { pname = "cargo-run-bin"; - version = "1.5.0"; + version = "1.6.0"; src = fetchCrate { inherit pname version; - hash = "sha256-FPkZk5qKHrRR3V8s04yLgOVOKj+Rln3Cu/VW2bnr2fE="; + hash = "sha256-PB44m39TDH1z8N3DrxAlZ/FKOdZmpe+U84tbmBBP9VQ="; }; - cargoHash = "sha256-aFHuIEDpGCel1FC7D0hTUmzHbEj7wVarsE0wNZ/3Khw="; + cargoHash = "sha256-FMlirUr3c8QhnTmTHvfNPff7PYlWSl83vCGLOLbyaR4="; # multiple impurities in tests doCheck = false; From 9b11ad66d63e808e174eaa864788fd7413c07982 Mon Sep 17 00:00:00 2001 From: "R. Ryantm" Date: Wed, 13 Dec 2023 14:34:05 +0000 Subject: [PATCH 10/42] cargo-xwin: 0.14.9 -> 0.16.2 --- pkgs/by-name/ca/cargo-xwin/package.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkgs/by-name/ca/cargo-xwin/package.nix b/pkgs/by-name/ca/cargo-xwin/package.nix index 0b759f520975..6052d780ced3 100644 --- a/pkgs/by-name/ca/cargo-xwin/package.nix +++ b/pkgs/by-name/ca/cargo-xwin/package.nix @@ -7,16 +7,16 @@ rustPlatform.buildRustPackage rec { pname = "cargo-xwin"; - version = "0.14.9"; + version = "0.16.2"; src = fetchFromGitHub { owner = "rust-cross"; repo = "cargo-xwin"; rev = "v${version}"; - hash = "sha256-y2hlzewDWYxkKhr77JB3lkYo5hexcdmPiCRbXLtnolM="; + hash = "sha256-EZM1TeWUnoRcsF6m6mDNCoUR2WWe7ohqT3wNWnq0kQY="; }; - cargoHash = "sha256-uIFjWgoNCU5kUX4i1Law/YE0TmFLOi6V3Y4b9BpQlI4="; + cargoHash = "sha256-MEBMXP7a/w2aN6RuWrm16PsnIPw6+8k5jI2yRnwBy0s="; buildInputs = lib.optionals stdenv.isDarwin [ darwin.apple_sdk.frameworks.Security From 9301c212ccd5ffd4b215947a451605a8f3497657 Mon Sep 17 00:00:00 2001 From: Sergei Trofimovich Date: Wed, 13 Dec 2023 18:20:38 +0000 Subject: [PATCH 11/42] lesspipe: 2.10 -> 2.11 Changes: https://github.com/wofr06/lesspipe/compare/v2.10...v2.11 --- pkgs/tools/misc/lesspipe/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/tools/misc/lesspipe/default.nix b/pkgs/tools/misc/lesspipe/default.nix index fcd46e0e85fd..f478aaea0519 100644 --- a/pkgs/tools/misc/lesspipe/default.nix +++ b/pkgs/tools/misc/lesspipe/default.nix @@ -2,13 +2,13 @@ stdenv.mkDerivation rec { pname = "lesspipe"; - version = "2.10"; + version = "2.11"; src = fetchFromGitHub { owner = "wofr06"; repo = "lesspipe"; rev = "v${version}"; - sha256 = "sha256-fLDB0rUo1kfPs0Xy2s1gG5ZsRjk1h1yYqjXkQC4qPf0="; + hash = "sha256-jJrKiRdrargk0JzcPWxBZGyOpMfTIONHG8HNRecazVo="; }; nativeBuildInputs = [ perl makeWrapper ]; From 8eb10ebc26996dd1f003b4604c3108bf2d6b37d3 Mon Sep 17 00:00:00 2001 From: Kirill Radzikhovskyy Date: Thu, 14 Dec 2023 05:49:33 +1100 Subject: [PATCH 12/42] juce: 7.0.7 -> 7.0.9 --- pkgs/development/misc/juce/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/development/misc/juce/default.nix b/pkgs/development/misc/juce/default.nix index 6c409a18e7aa..ced9acc27c4f 100644 --- a/pkgs/development/misc/juce/default.nix +++ b/pkgs/development/misc/juce/default.nix @@ -20,13 +20,13 @@ stdenv.mkDerivation (finalAttrs: { pname = "juce"; - version = "7.0.7"; + version = "7.0.9"; src = fetchFromGitHub { owner = "juce-framework"; repo = "juce"; rev = finalAttrs.version; - hash = "sha256-r+Wf/skPDexm3rsrVBoWrygKvV9HGlCQd7r0iHr9avM="; + hash = "sha256-k8cNTPH9OgOav4dsSLqrd5PlJ1rqO0PLt6Lwmumc2Gg="; }; patches = [ From 411fd474e120d9d6f7640c1f3894cfd665866320 Mon Sep 17 00:00:00 2001 From: Alyssa Ross Date: Thu, 14 Dec 2023 00:49:36 +0100 Subject: [PATCH 13/42] libtiff: drop maintainership When I added myself as a maintainer here, I thought it would be way less work than it turns out to be, because I didn't realise how vulnerability-prone libtiff is. I basically haven't been maintaining it at all, so let's reflect reality. --- pkgs/development/libraries/libtiff/default.nix | 1 - 1 file changed, 1 deletion(-) diff --git a/pkgs/development/libraries/libtiff/default.nix b/pkgs/development/libraries/libtiff/default.nix index 22b0f8af2043..8e735369e076 100644 --- a/pkgs/development/libraries/libtiff/default.nix +++ b/pkgs/development/libraries/libtiff/default.nix @@ -85,7 +85,6 @@ stdenv.mkDerivation rec { description = "Library and utilities for working with the TIFF image file format"; homepage = "https://libtiff.gitlab.io/libtiff"; changelog = "https://libtiff.gitlab.io/libtiff/v${version}.html"; - maintainers = with maintainers; [ qyliss ]; license = licenses.libtiff; platforms = platforms.unix; }; From 3ce5a64da2b1fc42670920084abfd256f0696285 Mon Sep 17 00:00:00 2001 From: "R. Ryantm" Date: Thu, 14 Dec 2023 00:35:46 +0000 Subject: [PATCH 14/42] consul: 1.16.3 -> 1.17.0 --- pkgs/servers/consul/default.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkgs/servers/consul/default.nix b/pkgs/servers/consul/default.nix index 2d449c531cc8..dee27e5877dc 100644 --- a/pkgs/servers/consul/default.nix +++ b/pkgs/servers/consul/default.nix @@ -2,7 +2,7 @@ buildGoModule rec { pname = "consul"; - version = "1.16.3"; + version = "1.17.0"; rev = "v${version}"; # Note: Currently only release tags are supported, because they have the Consul UI @@ -17,7 +17,7 @@ buildGoModule rec { owner = "hashicorp"; repo = pname; inherit rev; - hash = "sha256-XxT+66CNuDeVBoaNhlgET5bJYB/KDCjcO0RDmyI6S9o="; + hash = "sha256-fAcgO7r0GrL2GrsX7flezhbQMcg+YBH6Lrn7BW2XMwM="; }; passthru.tests.consul = nixosTests.consul; @@ -26,7 +26,7 @@ buildGoModule rec { # has a split module structure in one repo subPackages = ["." "connect/certgen"]; - vendorHash = "sha256-WNvdHT915GSTFhZZfoi/MCHAjzBVYkhUiPNPw5GDT4s="; + vendorHash = "sha256-xxREyw7xgx9Zp7nua1yq39TioWvRQXOhWqYaK6eJaOc="; doCheck = false; From b8e4d555b4936adf93510c9e333798eecf32b5e7 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Thu, 14 Dec 2023 02:07:50 +0100 Subject: [PATCH 15/42] tests.nixpkgs-check-by-name: Minor refactor, allow more simultaneous problems This makes it such that these two errors can both be thrown for a single package: - The attribute value not being a derivation - The attribute not being a proper callPackage The tests had to be adjusted to only throw the error they were testing for --- pkgs/test/nixpkgs-check-by-name/src/eval.rs | 42 +++++++++++-------- .../override-no-call-package/all-packages.nix | 2 +- .../tests/override-no-file/all-packages.nix | 2 +- 3 files changed, 26 insertions(+), 20 deletions(-) diff --git a/pkgs/test/nixpkgs-check-by-name/src/eval.rs b/pkgs/test/nixpkgs-check-by-name/src/eval.rs index 161d013374e7..08dc243359d5 100644 --- a/pkgs/test/nixpkgs-check-by-name/src/eval.rs +++ b/pkgs/test/nixpkgs-check-by-name/src/eval.rs @@ -116,8 +116,18 @@ pub fn check_values( let absolute_package_file = nixpkgs_path.join(&relative_package_file); if let Some(attribute_info) = actual_files.get(package_name) { - let valid = match &attribute_info.variant { - AttributeVariant::AutoCalled => true, + let check_result = if !attribute_info.is_derivation { + NixpkgsProblem::NonDerivation { + relative_package_file: relative_package_file.clone(), + package_name: package_name.clone(), + } + .into() + } else { + Success(()) + }; + + check_result.and(match &attribute_info.variant { + AttributeVariant::AutoCalled => Success(()), AttributeVariant::CallPackage { path, empty_arg } => { let correct_file = if let Some(call_package_path) = path { absolute_package_file == *call_package_path @@ -131,26 +141,22 @@ pub fn check_values( } else { true }; - correct_file && non_empty + if correct_file && non_empty { + Success(()) + } else { + NixpkgsProblem::WrongCallPackage { + relative_package_file: relative_package_file.clone(), + package_name: package_name.clone(), + } + .into() + } } - AttributeVariant::Other => false, - }; - - if !valid { - NixpkgsProblem::WrongCallPackage { + AttributeVariant::Other => NixpkgsProblem::WrongCallPackage { relative_package_file: relative_package_file.clone(), package_name: package_name.clone(), } - .into() - } else if !attribute_info.is_derivation { - NixpkgsProblem::NonDerivation { - relative_package_file: relative_package_file.clone(), - package_name: package_name.clone(), - } - .into() - } else { - Success(()) - } + .into(), + }) } else { NixpkgsProblem::UndefinedAttr { relative_package_file: relative_package_file.clone(), diff --git a/pkgs/test/nixpkgs-check-by-name/tests/override-no-call-package/all-packages.nix b/pkgs/test/nixpkgs-check-by-name/tests/override-no-call-package/all-packages.nix index 4fad280ae1c7..853c3a87db56 100644 --- a/pkgs/test/nixpkgs-check-by-name/tests/override-no-call-package/all-packages.nix +++ b/pkgs/test/nixpkgs-check-by-name/tests/override-no-call-package/all-packages.nix @@ -1,3 +1,3 @@ self: super: { - nonDerivation = null; + nonDerivation = self.someDrv; } diff --git a/pkgs/test/nixpkgs-check-by-name/tests/override-no-file/all-packages.nix b/pkgs/test/nixpkgs-check-by-name/tests/override-no-file/all-packages.nix index 4c521d2d4468..dc07f69b40ee 100644 --- a/pkgs/test/nixpkgs-check-by-name/tests/override-no-file/all-packages.nix +++ b/pkgs/test/nixpkgs-check-by-name/tests/override-no-file/all-packages.nix @@ -1,3 +1,3 @@ self: super: { - nonDerivation = self.callPackage ({ }: { }) { }; + nonDerivation = self.callPackage ({ someDrv }: someDrv) { }; } From e98d22851b67a6125683f80735e4bc1042252aef Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Thu, 14 Dec 2023 03:03:04 +0100 Subject: [PATCH 16/42] tests.nixpkgs-check-by-name: Introduce result_map Convenience function to run another validation over a successful validation result. This will be usable in more locations in future commits, making the code nicer. --- pkgs/test/nixpkgs-check-by-name/src/main.rs | 9 ++------- pkgs/test/nixpkgs-check-by-name/src/validation.rs | 9 +++++++++ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/pkgs/test/nixpkgs-check-by-name/src/main.rs b/pkgs/test/nixpkgs-check-by-name/src/main.rs index 4cabf8f446f5..567da00333e6 100644 --- a/pkgs/test/nixpkgs-check-by-name/src/main.rs +++ b/pkgs/test/nixpkgs-check-by-name/src/main.rs @@ -86,14 +86,9 @@ pub fn check_nixpkgs( ); Success(()) } else { - match check_structure(&nixpkgs_path)? { - Failure(errors) => Failure(errors), - Success(package_names) => + check_structure(&nixpkgs_path)?.result_map(|package_names| // Only if we could successfully parse the structure, we do the evaluation checks - { - eval::check_values(version, &nixpkgs_path, package_names, eval_accessible_paths)? - } - } + eval::check_values(version, &nixpkgs_path, package_names, eval_accessible_paths))? }; match check_result { diff --git a/pkgs/test/nixpkgs-check-by-name/src/validation.rs b/pkgs/test/nixpkgs-check-by-name/src/validation.rs index e72793851521..b14bfb92eb2e 100644 --- a/pkgs/test/nixpkgs-check-by-name/src/validation.rs +++ b/pkgs/test/nixpkgs-check-by-name/src/validation.rs @@ -58,6 +58,15 @@ impl Validation { Success(value) => Success(f(value)), } } + + /// Map a `Validation` to a `Result` by applying a function `A -> Result` + /// only if there is a `Success` value + pub fn result_map(self, f: impl FnOnce(A) -> Result) -> Result { + match self { + Failure(err) => Ok(Failure(err)), + Success(value) => f(value), + } + } } impl Validation<()> { From a6ba4cae311698ea907ee239785f75d76bd01e4b Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Thu, 14 Dec 2023 02:47:44 +0100 Subject: [PATCH 17/42] tests.nixpkgs-check-by-name: Intermediate refactor This prepares the code base for the removal of the `--version` flag, to be replaced with a flag that can specify a base version to compare the main Nixpkgs against, in order to have gradual transitions to stricter checks. This refactoring does: - Introduce the `version` module that can house the logic to increase strictness, with a `version::Nixpkgs` struct that contains the strictness conformity of a single Nixpkgs version - Make the check return `version::Nixpkgs` - Handle the behavior of the still-existing `--version` flag with `version::Nixpkgs` - Introduce an intermediate `process` function to handle the top-level logic, especially useful in the next commit --- pkgs/test/nixpkgs-check-by-name/src/eval.rs | 44 +++++++----- pkgs/test/nixpkgs-check-by-name/src/main.rs | 64 +++++++++++------ .../test/nixpkgs-check-by-name/src/version.rs | 71 +++++++++++++++++++ 3 files changed, 139 insertions(+), 40 deletions(-) create mode 100644 pkgs/test/nixpkgs-check-by-name/src/version.rs diff --git a/pkgs/test/nixpkgs-check-by-name/src/eval.rs b/pkgs/test/nixpkgs-check-by-name/src/eval.rs index 08dc243359d5..face1117f643 100644 --- a/pkgs/test/nixpkgs-check-by-name/src/eval.rs +++ b/pkgs/test/nixpkgs-check-by-name/src/eval.rs @@ -1,7 +1,7 @@ use crate::nixpkgs_problem::NixpkgsProblem; use crate::structure; use crate::validation::{self, Validation::Success}; -use crate::Version; +use crate::version; use std::path::Path; use anyhow::Context; @@ -39,11 +39,10 @@ enum AttributeVariant { /// of the form `callPackage { ... }`. /// See the `eval.nix` file for how this is achieved on the Nix side pub fn check_values( - version: Version, nixpkgs_path: &Path, package_names: Vec, eval_accessible_paths: Vec<&Path>, -) -> validation::Result<()> { +) -> validation::Result { // Write the list of packages we need to check into a temporary JSON file. // This can then get read by the Nix evaluation. let attrs_file = NamedTempFile::new().context("Failed to create a temporary file")?; @@ -110,8 +109,8 @@ pub fn check_values( String::from_utf8_lossy(&result.stdout) ))?; - Ok(validation::sequence_(package_names.iter().map( - |package_name| { + Ok( + validation::sequence(package_names.iter().map(|package_name| { let relative_package_file = structure::relative_file_for_package(package_name); let absolute_package_file = nixpkgs_path.join(&relative_package_file); @@ -126,23 +125,25 @@ pub fn check_values( Success(()) }; - check_result.and(match &attribute_info.variant { - AttributeVariant::AutoCalled => Success(()), + let check_result = check_result.and(match &attribute_info.variant { + AttributeVariant::AutoCalled => Success(version::Attribute { + empty_non_auto_called: version::EmptyNonAutoCalled::Valid, + }), AttributeVariant::CallPackage { path, empty_arg } => { let correct_file = if let Some(call_package_path) = path { absolute_package_file == *call_package_path } else { false }; - // Only check for the argument to be non-empty if the version is V1 or - // higher - let non_empty = if version >= Version::V1 { - !empty_arg - } else { - true - }; - if correct_file && non_empty { - Success(()) + + if correct_file { + Success(version::Attribute { + empty_non_auto_called: if *empty_arg { + version::EmptyNonAutoCalled::Invalid + } else { + version::EmptyNonAutoCalled::Valid + }, + }) } else { NixpkgsProblem::WrongCallPackage { relative_package_file: relative_package_file.clone(), @@ -156,7 +157,9 @@ pub fn check_values( package_name: package_name.clone(), } .into(), - }) + }); + + check_result.map(|value| (package_name.clone(), value)) } else { NixpkgsProblem::UndefinedAttr { relative_package_file: relative_package_file.clone(), @@ -164,6 +167,9 @@ pub fn check_values( } .into() } - }, - ))) + })) + .map(|elems| version::Nixpkgs { + attributes: elems.into_iter().collect(), + }), + ) } diff --git a/pkgs/test/nixpkgs-check-by-name/src/main.rs b/pkgs/test/nixpkgs-check-by-name/src/main.rs index 567da00333e6..981d1134c85a 100644 --- a/pkgs/test/nixpkgs-check-by-name/src/main.rs +++ b/pkgs/test/nixpkgs-check-by-name/src/main.rs @@ -4,6 +4,7 @@ mod references; mod structure; mod utils; mod validation; +mod version; use crate::structure::check_structure; use crate::validation::Validation::Failure; @@ -39,7 +40,7 @@ pub enum Version { fn main() -> ExitCode { let args = Args::parse(); - match check_nixpkgs(&args.nixpkgs, args.version, vec![], &mut io::stderr()) { + match process(&args.nixpkgs, args.version, vec![], &mut io::stderr()) { Ok(true) => { eprintln!("{}", "Validated successfully".green()); ExitCode::SUCCESS @@ -55,7 +56,7 @@ fn main() -> ExitCode { } } -/// Checks whether the pkgs/by-name structure in Nixpkgs is valid. +/// Does the actual work. This is the abstraction used both by `main` and the tests. /// /// # Arguments /// - `nixpkgs_path`: The path to the Nixpkgs to check @@ -68,28 +69,23 @@ fn main() -> ExitCode { /// - `Err(e)` if an I/O-related error `e` occurred. /// - `Ok(false)` if there are problems, all of which will be written to `error_writer`. /// - `Ok(true)` if there are no problems -pub fn check_nixpkgs( +pub fn process( nixpkgs_path: &Path, version: Version, eval_accessible_paths: Vec<&Path>, error_writer: &mut W, ) -> anyhow::Result { - let nixpkgs_path = nixpkgs_path.canonicalize().context(format!( - "Nixpkgs path {} could not be resolved", - nixpkgs_path.display() - ))?; - - let check_result = if !nixpkgs_path.join(utils::BASE_SUBPATH).exists() { - eprintln!( - "Given Nixpkgs path does not contain a {} subdirectory, no check necessary.", - utils::BASE_SUBPATH - ); - Success(()) - } else { - check_structure(&nixpkgs_path)?.result_map(|package_names| - // Only if we could successfully parse the structure, we do the evaluation checks - eval::check_values(version, &nixpkgs_path, package_names, eval_accessible_paths))? - }; + let nixpkgs_result = check_nixpkgs(nixpkgs_path, eval_accessible_paths)?; + let check_result = nixpkgs_result.result_map(|nixpkgs_version| { + let empty_non_auto_called_base = match version { + Version::V0 => version::EmptyNonAutoCalled::Invalid, + Version::V1 => version::EmptyNonAutoCalled::Valid, + }; + Ok(version::Nixpkgs::compare( + &empty_non_auto_called_base, + nixpkgs_version, + )) + })?; match check_result { Failure(errors) => { @@ -102,9 +98,35 @@ pub fn check_nixpkgs( } } +/// Checks whether the pkgs/by-name structure in Nixpkgs is valid, +/// and returns to which degree it's valid for checks with increased strictness. +pub fn check_nixpkgs( + nixpkgs_path: &Path, + eval_accessible_paths: Vec<&Path>, +) -> validation::Result { + Ok({ + let nixpkgs_path = nixpkgs_path.canonicalize().context(format!( + "Nixpkgs path {} could not be resolved", + nixpkgs_path.display() + ))?; + + if !nixpkgs_path.join(utils::BASE_SUBPATH).exists() { + eprintln!( + "Given Nixpkgs path does not contain a {} subdirectory, no check necessary.", + utils::BASE_SUBPATH + ); + Success(version::Nixpkgs::default()) + } else { + check_structure(&nixpkgs_path)?.result_map(|package_names| + // Only if we could successfully parse the structure, we do the evaluation checks + eval::check_values(&nixpkgs_path, package_names, eval_accessible_paths))? + } + }) +} + #[cfg(test)] mod tests { - use crate::check_nixpkgs; + use crate::process; use crate::utils; use crate::Version; use anyhow::Context; @@ -195,7 +217,7 @@ mod tests { // We don't want coloring to mess up the tests let writer = temp_env::with_var("NO_COLOR", Some("1"), || -> anyhow::Result<_> { let mut writer = vec![]; - check_nixpkgs(&path, Version::V1, vec![&extra_nix_path], &mut writer) + process(&path, Version::V1, vec![&extra_nix_path], &mut writer) .context(format!("Failed test case {name}"))?; Ok(writer) })?; diff --git a/pkgs/test/nixpkgs-check-by-name/src/version.rs b/pkgs/test/nixpkgs-check-by-name/src/version.rs new file mode 100644 index 000000000000..ab146270241e --- /dev/null +++ b/pkgs/test/nixpkgs-check-by-name/src/version.rs @@ -0,0 +1,71 @@ +use crate::nixpkgs_problem::NixpkgsProblem; +use crate::structure; +use crate::validation::{self, Validation, Validation::Success}; +use std::collections::HashMap; + +/// The check version conformity of a Nixpkgs path: +/// When the strictness of the check increases, this structure should be extended to distinguish +/// between parts that are still valid, and ones that aren't valid anymore. +#[derive(Default)] +pub struct Nixpkgs { + /// The package attributes tracked in `pkgs/by-name` + pub attributes: HashMap, +} + +impl Nixpkgs { + /// Compares two Nixpkgs versions against each other, returning validation errors only if the + /// `from` version satisfied the stricter checks, while the `to` version doesn't satisfy them + /// anymore. + pub fn compare(empty_non_auto_called_from: &EmptyNonAutoCalled, to: Self) -> Validation<()> { + validation::sequence_( + // We only loop over the current attributes, + // we don't need to check ones that were removed + to.attributes.into_iter().map(|(name, attr_to)| { + Attribute::compare(&name, empty_non_auto_called_from, &attr_to) + }), + ) + } +} + +/// The check version conformity of an attribute defined by `pkgs/by-name` +pub struct Attribute { + pub empty_non_auto_called: EmptyNonAutoCalled, +} + +impl Attribute { + pub fn compare( + name: &str, + empty_non_auto_called_from: &EmptyNonAutoCalled, + to: &Self, + ) -> Validation<()> { + EmptyNonAutoCalled::compare(name, empty_non_auto_called_from, &to.empty_non_auto_called) + } +} + +/// Whether an attribute conforms to the new strictness check that +/// `callPackage ... {}` is not allowed anymore in `all-package.nix` +#[derive(PartialEq, PartialOrd)] +pub enum EmptyNonAutoCalled { + /// The attribute is not valid anymore with the new check + Invalid, + /// The attribute is still valid with the new check + Valid, +} + +impl EmptyNonAutoCalled { + fn compare( + name: &str, + empty_non_auto_called_from: &EmptyNonAutoCalled, + to: &Self, + ) -> Validation<()> { + if to >= empty_non_auto_called_from { + Success(()) + } else { + NixpkgsProblem::WrongCallPackage { + relative_package_file: structure::relative_file_for_package(name), + package_name: name.to_owned(), + } + .into() + } + } +} From d487a975ccb27302d1095ab56cd3c104712452c7 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Thu, 14 Dec 2023 03:11:14 +0100 Subject: [PATCH 18/42] tests.nixpkgs-check-by-name: Gradual migration from base Nixpkgs This implements the option for a gradual migration to stricter checks. For now this is only done for the check against empty non-auto-called callPackage arguments, but in the future this can be used to ensure all new packages make use of `pkgs/by-name`. This is implemented by adding a `--base ` flag, which then compares the base nixpkgs against the main nixpkgs version, making sure that there are no regressions. The `--version` flag is removed. While it was implemented, it was never used in CI, so this is fine. --- pkgs/test/nixpkgs-check-by-name/README.md | 13 ++-- pkgs/test/nixpkgs-check-by-name/src/eval.rs | 2 +- pkgs/test/nixpkgs-check-by-name/src/main.rs | 64 +++++++++---------- .../test/nixpkgs-check-by-name/src/version.rs | 25 ++++---- 4 files changed, 47 insertions(+), 57 deletions(-) diff --git a/pkgs/test/nixpkgs-check-by-name/README.md b/pkgs/test/nixpkgs-check-by-name/README.md index 146cea0a64ba..b098658fce4c 100644 --- a/pkgs/test/nixpkgs-check-by-name/README.md +++ b/pkgs/test/nixpkgs-check-by-name/README.md @@ -8,16 +8,10 @@ This is part of the implementation of [RFC 140](https://github.com/NixOS/rfcs/pu This API may be changed over time if the CI workflow making use of it is adjusted to deal with the change appropriately. -- Command line: `nixpkgs-check-by-name ` +- Command line: `nixpkgs-check-by-name [--base ] ` - Arguments: + - ``: The path to the Nixpkgs to check against - ``: The path to the Nixpkgs to check - - `--version `: The version of the checks to perform. - - Possible values: - - `v0` (default) - - `v1` - - See [validation](#validity-checks) for the differences. - Exit code: - `0`: If the [validation](#validity-checks) is successful - `1`: If the [validation](#validity-checks) is not successful @@ -42,7 +36,8 @@ These checks are performed by this tool: ### Nix evaluation checks - `pkgs.${name}` is defined as `callPackage pkgs/by-name/${shard}/${name}/package.nix args` for some `args`. - - **Only after --version v1**: If `pkgs.${name}` is not auto-called from `pkgs/by-name`, `args` must not be empty + - If `pkgs.${name}` is not auto-called from `pkgs/by-name`, `args` must not be empty, + with the exception that if `BASE_NIXPKGS` also has a definition for the same package with empty `args`, it's allowed - `pkgs.lib.isDerivation pkgs.${name}` is `true`. ## Development diff --git a/pkgs/test/nixpkgs-check-by-name/src/eval.rs b/pkgs/test/nixpkgs-check-by-name/src/eval.rs index face1117f643..927e446b452f 100644 --- a/pkgs/test/nixpkgs-check-by-name/src/eval.rs +++ b/pkgs/test/nixpkgs-check-by-name/src/eval.rs @@ -41,7 +41,7 @@ enum AttributeVariant { pub fn check_values( nixpkgs_path: &Path, package_names: Vec, - eval_accessible_paths: Vec<&Path>, + eval_accessible_paths: &Vec<&Path>, ) -> validation::Result { // Write the list of packages we need to check into a temporary JSON file. // This can then get read by the Nix evaluation. diff --git a/pkgs/test/nixpkgs-check-by-name/src/main.rs b/pkgs/test/nixpkgs-check-by-name/src/main.rs index 981d1134c85a..53c24845cb20 100644 --- a/pkgs/test/nixpkgs-check-by-name/src/main.rs +++ b/pkgs/test/nixpkgs-check-by-name/src/main.rs @@ -9,8 +9,9 @@ mod version; use crate::structure::check_structure; use crate::validation::Validation::Failure; use crate::validation::Validation::Success; +use crate::version::Nixpkgs; use anyhow::Context; -use clap::{Parser, ValueEnum}; +use clap::Parser; use colored::Colorize; use std::io; use std::path::{Path, PathBuf}; @@ -22,25 +23,20 @@ use std::process::ExitCode; pub struct Args { /// Path to nixpkgs nixpkgs: PathBuf, - /// The version of the checks - /// Increasing this may cause failures for a Nixpkgs that succeeded before - /// TODO: Remove default once Nixpkgs CI passes this argument - #[arg(long, value_enum, default_value_t = Version::V0)] - version: Version, -} -/// The version of the checks -#[derive(Debug, Clone, PartialEq, PartialOrd, ValueEnum)] -pub enum Version { - /// Initial version - V0, - /// Empty argument check - V1, + /// Path to the base Nixpkgs to compare against + #[arg(long)] + base: Option, } fn main() -> ExitCode { let args = Args::parse(); - match process(&args.nixpkgs, args.version, vec![], &mut io::stderr()) { + match process( + args.base.as_deref(), + &args.nixpkgs, + &vec![], + &mut io::stderr(), + ) { Ok(true) => { eprintln!("{}", "Validated successfully".green()); ExitCode::SUCCESS @@ -59,7 +55,8 @@ fn main() -> ExitCode { /// Does the actual work. This is the abstraction used both by `main` and the tests. /// /// # Arguments -/// - `nixpkgs_path`: The path to the Nixpkgs to check +/// - `base_nixpkgs`: The path to the base Nixpkgs to compare against +/// - `main_nixpkgs`: The path to the main Nixpkgs to check /// - `eval_accessible_paths`: /// Extra paths that need to be accessible to evaluate Nixpkgs using `restrict-eval`. /// This is used to allow the tests to access the mock-nixpkgs.nix file @@ -70,21 +67,23 @@ fn main() -> ExitCode { /// - `Ok(false)` if there are problems, all of which will be written to `error_writer`. /// - `Ok(true)` if there are no problems pub fn process( - nixpkgs_path: &Path, - version: Version, - eval_accessible_paths: Vec<&Path>, + base_nixpkgs: Option<&Path>, + main_nixpkgs: &Path, + eval_accessible_paths: &Vec<&Path>, error_writer: &mut W, ) -> anyhow::Result { - let nixpkgs_result = check_nixpkgs(nixpkgs_path, eval_accessible_paths)?; - let check_result = nixpkgs_result.result_map(|nixpkgs_version| { - let empty_non_auto_called_base = match version { - Version::V0 => version::EmptyNonAutoCalled::Invalid, - Version::V1 => version::EmptyNonAutoCalled::Valid, - }; - Ok(version::Nixpkgs::compare( - &empty_non_auto_called_base, - nixpkgs_version, - )) + let main_result = check_nixpkgs(main_nixpkgs, eval_accessible_paths)?; + let check_result = main_result.result_map(|nixpkgs_version| { + if let Some(base) = base_nixpkgs { + check_nixpkgs(base, eval_accessible_paths)?.result_map(|base_nixpkgs_version| { + Ok(Nixpkgs::compare(base_nixpkgs_version, nixpkgs_version)) + }) + } else { + Ok(Nixpkgs::compare( + version::Nixpkgs::default(), + nixpkgs_version, + )) + } })?; match check_result { @@ -94,7 +93,7 @@ pub fn process( } Ok(false) } - Success(_) => Ok(true), + Success(()) => Ok(true), } } @@ -102,7 +101,7 @@ pub fn process( /// and returns to which degree it's valid for checks with increased strictness. pub fn check_nixpkgs( nixpkgs_path: &Path, - eval_accessible_paths: Vec<&Path>, + eval_accessible_paths: &Vec<&Path>, ) -> validation::Result { Ok({ let nixpkgs_path = nixpkgs_path.canonicalize().context(format!( @@ -128,7 +127,6 @@ pub fn check_nixpkgs( mod tests { use crate::process; use crate::utils; - use crate::Version; use anyhow::Context; use std::fs; use std::path::Path; @@ -217,7 +215,7 @@ mod tests { // We don't want coloring to mess up the tests let writer = temp_env::with_var("NO_COLOR", Some("1"), || -> anyhow::Result<_> { let mut writer = vec![]; - process(&path, Version::V1, vec![&extra_nix_path], &mut writer) + process(None, &path, &vec![&extra_nix_path], &mut writer) .context(format!("Failed test case {name}"))?; Ok(writer) })?; diff --git a/pkgs/test/nixpkgs-check-by-name/src/version.rs b/pkgs/test/nixpkgs-check-by-name/src/version.rs index ab146270241e..7f83bdf3ff67 100644 --- a/pkgs/test/nixpkgs-check-by-name/src/version.rs +++ b/pkgs/test/nixpkgs-check-by-name/src/version.rs @@ -16,12 +16,12 @@ impl Nixpkgs { /// Compares two Nixpkgs versions against each other, returning validation errors only if the /// `from` version satisfied the stricter checks, while the `to` version doesn't satisfy them /// anymore. - pub fn compare(empty_non_auto_called_from: &EmptyNonAutoCalled, to: Self) -> Validation<()> { + pub fn compare(from: Self, to: Self) -> Validation<()> { validation::sequence_( // We only loop over the current attributes, // we don't need to check ones that were removed to.attributes.into_iter().map(|(name, attr_to)| { - Attribute::compare(&name, empty_non_auto_called_from, &attr_to) + Attribute::compare(&name, from.attributes.get(&name), &attr_to) }), ) } @@ -33,12 +33,12 @@ pub struct Attribute { } impl Attribute { - pub fn compare( - name: &str, - empty_non_auto_called_from: &EmptyNonAutoCalled, - to: &Self, - ) -> Validation<()> { - EmptyNonAutoCalled::compare(name, empty_non_auto_called_from, &to.empty_non_auto_called) + pub fn compare(name: &str, optional_from: Option<&Self>, to: &Self) -> Validation<()> { + EmptyNonAutoCalled::compare( + name, + optional_from.map(|x| &x.empty_non_auto_called), + &to.empty_non_auto_called, + ) } } @@ -53,12 +53,9 @@ pub enum EmptyNonAutoCalled { } impl EmptyNonAutoCalled { - fn compare( - name: &str, - empty_non_auto_called_from: &EmptyNonAutoCalled, - to: &Self, - ) -> Validation<()> { - if to >= empty_non_auto_called_from { + fn compare(name: &str, optional_from: Option<&Self>, to: &Self) -> Validation<()> { + let from = optional_from.unwrap_or(&Self::Valid); + if to >= from { Success(()) } else { NixpkgsProblem::WrongCallPackage { From bb08bfc2d37f9c1dbaf6f585fded65bf77a67e61 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Thu, 14 Dec 2023 04:01:54 +0100 Subject: [PATCH 19/42] tests.nixpkgs-check-by-name: Test for gradual transition This implements the ability to test gradual transitions in check strictness, and adds one such test for the empty non-auto-called arguments check. --- pkgs/test/nixpkgs-check-by-name/README.md | 4 ++++ pkgs/test/nixpkgs-check-by-name/src/main.rs | 9 ++++++++- .../tests/override-empty-arg-gradual/all-packages.nix | 3 +++ .../override-empty-arg-gradual/base/all-packages.nix | 3 +++ .../tests/override-empty-arg-gradual/base/default.nix | 1 + .../base/pkgs/by-name/no/nonDerivation/package.nix | 1 + .../tests/override-empty-arg-gradual/default.nix | 1 + .../tests/override-empty-arg-gradual/expected | 0 .../pkgs/by-name/no/nonDerivation/package.nix | 1 + 9 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg-gradual/all-packages.nix create mode 100644 pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg-gradual/base/all-packages.nix create mode 100644 pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg-gradual/base/default.nix create mode 100644 pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg-gradual/base/pkgs/by-name/no/nonDerivation/package.nix create mode 100644 pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg-gradual/default.nix create mode 100644 pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg-gradual/expected create mode 100644 pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg-gradual/pkgs/by-name/no/nonDerivation/package.nix diff --git a/pkgs/test/nixpkgs-check-by-name/README.md b/pkgs/test/nixpkgs-check-by-name/README.md index b098658fce4c..c906eaffc974 100644 --- a/pkgs/test/nixpkgs-check-by-name/README.md +++ b/pkgs/test/nixpkgs-check-by-name/README.md @@ -81,6 +81,10 @@ Tests are declared in [`./tests`](./tests) as subdirectories imitating Nixpkgs w allowing the simulation of package overrides to the real [`pkgs/top-level/all-packages.nix`](../../top-level/all-packages.nix`). The default is an empty overlay. +- `base` (optional): + Contains another subdirectory imitating Nixpkgs with potentially any of the above structures. + This will be used as the `--base` argument, allowing tests of gradual transitions. + - `expected` (optional): A file containing the expected standard output. The default is expecting an empty standard output. diff --git a/pkgs/test/nixpkgs-check-by-name/src/main.rs b/pkgs/test/nixpkgs-check-by-name/src/main.rs index 53c24845cb20..bf3bfb193f18 100644 --- a/pkgs/test/nixpkgs-check-by-name/src/main.rs +++ b/pkgs/test/nixpkgs-check-by-name/src/main.rs @@ -212,10 +212,17 @@ mod tests { fn test_nixpkgs(name: &str, path: &Path, expected_errors: &str) -> anyhow::Result<()> { let extra_nix_path = Path::new("tests/mock-nixpkgs.nix"); + let base_path = path.join("base"); + let base_nixpkgs = if base_path.exists() { + Some(base_path.as_path()) + } else { + None + }; + // We don't want coloring to mess up the tests let writer = temp_env::with_var("NO_COLOR", Some("1"), || -> anyhow::Result<_> { let mut writer = vec![]; - process(None, &path, &vec![&extra_nix_path], &mut writer) + process(base_nixpkgs, &path, &vec![&extra_nix_path], &mut writer) .context(format!("Failed test case {name}"))?; Ok(writer) })?; diff --git a/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg-gradual/all-packages.nix b/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg-gradual/all-packages.nix new file mode 100644 index 000000000000..d369dd7228dc --- /dev/null +++ b/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg-gradual/all-packages.nix @@ -0,0 +1,3 @@ +self: super: { + nonDerivation = self.callPackage ./pkgs/by-name/no/nonDerivation/package.nix { }; +} diff --git a/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg-gradual/base/all-packages.nix b/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg-gradual/base/all-packages.nix new file mode 100644 index 000000000000..d369dd7228dc --- /dev/null +++ b/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg-gradual/base/all-packages.nix @@ -0,0 +1,3 @@ +self: super: { + nonDerivation = self.callPackage ./pkgs/by-name/no/nonDerivation/package.nix { }; +} diff --git a/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg-gradual/base/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg-gradual/base/default.nix new file mode 100644 index 000000000000..2875ea6327ef --- /dev/null +++ b/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg-gradual/base/default.nix @@ -0,0 +1 @@ +import ../../mock-nixpkgs.nix { root = ./.; } diff --git a/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg-gradual/base/pkgs/by-name/no/nonDerivation/package.nix b/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg-gradual/base/pkgs/by-name/no/nonDerivation/package.nix new file mode 100644 index 000000000000..a1b92efbbadb --- /dev/null +++ b/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg-gradual/base/pkgs/by-name/no/nonDerivation/package.nix @@ -0,0 +1 @@ +{ someDrv }: someDrv diff --git a/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg-gradual/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg-gradual/default.nix new file mode 100644 index 000000000000..af25d1450122 --- /dev/null +++ b/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg-gradual/default.nix @@ -0,0 +1 @@ +import ../mock-nixpkgs.nix { root = ./.; } diff --git a/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg-gradual/expected b/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg-gradual/expected new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg-gradual/pkgs/by-name/no/nonDerivation/package.nix b/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg-gradual/pkgs/by-name/no/nonDerivation/package.nix new file mode 100644 index 000000000000..a1b92efbbadb --- /dev/null +++ b/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg-gradual/pkgs/by-name/no/nonDerivation/package.nix @@ -0,0 +1 @@ +{ someDrv }: someDrv From 98ecb3df5195f584af56e2828e959ad9e3c78855 Mon Sep 17 00:00:00 2001 From: "R. Ryantm" Date: Thu, 14 Dec 2023 03:33:36 +0000 Subject: [PATCH 20/42] crocoddyl: 2.0.1 -> 2.0.2 --- pkgs/development/libraries/crocoddyl/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/development/libraries/crocoddyl/default.nix b/pkgs/development/libraries/crocoddyl/default.nix index a25ca5b97dff..efba13612298 100644 --- a/pkgs/development/libraries/crocoddyl/default.nix +++ b/pkgs/development/libraries/crocoddyl/default.nix @@ -11,14 +11,14 @@ stdenv.mkDerivation (finalAttrs: { pname = "crocoddyl"; - version = "2.0.1"; + version = "2.0.2"; src = fetchFromGitHub { owner = "loco-3d"; repo = finalAttrs.pname; rev = "v${finalAttrs.version}"; fetchSubmodules = true; - hash = "sha256-h7rzLSvmWOZCP8rvmUEhFeMEiPhojfbvkt+fNKpgoXo="; + hash = "sha256-MsAXHfxLNlIK/PbtVTjvBN1Jk3dyGEkfpj3/98nExj4="; }; strictDeps = true; From 4f759938271ae84cf5ca082faf7cfa5bf2f97ea2 Mon Sep 17 00:00:00 2001 From: "R. Ryantm" Date: Thu, 14 Dec 2023 08:23:36 +0000 Subject: [PATCH 21/42] ddev: 1.22.5 -> 1.22.6 --- pkgs/applications/virtualization/ddev/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/applications/virtualization/ddev/default.nix b/pkgs/applications/virtualization/ddev/default.nix index 5ea3aa54b2ef..3709b8dbd901 100644 --- a/pkgs/applications/virtualization/ddev/default.nix +++ b/pkgs/applications/virtualization/ddev/default.nix @@ -2,13 +2,13 @@ buildGoModule rec { pname = "ddev"; - version = "1.22.5"; + version = "1.22.6"; src = fetchFromGitHub { owner = "ddev"; repo = "ddev"; rev = "v${version}"; - hash = "sha256-s4uRS/BIRjVVN3u+ocy2RcwSnvJLtWpkvxtvgumuWtk="; + hash = "sha256-i+uubmCQwJALt7YRuANpEN2AAn9i6880MaXkayIZ82g="; }; vendorHash = null; From d1e9869ebd5951a4c7defbab5866505edb6d400c Mon Sep 17 00:00:00 2001 From: Sergei Trofimovich Date: Thu, 14 Dec 2023 20:42:57 +0000 Subject: [PATCH 22/42] iwd: 2.10 -> 2.11 Changes: - https://git.kernel.org/pub/scm/network/wireless/iwd.git/tree/ChangeLog?h=2.11 --- pkgs/os-specific/linux/iwd/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/os-specific/linux/iwd/default.nix b/pkgs/os-specific/linux/iwd/default.nix index 84813723fb05..bea03fc1e8a2 100644 --- a/pkgs/os-specific/linux/iwd/default.nix +++ b/pkgs/os-specific/linux/iwd/default.nix @@ -14,12 +14,12 @@ stdenv.mkDerivation rec { pname = "iwd"; - version = "2.10"; + version = "2.11"; src = fetchgit { url = "https://git.kernel.org/pub/scm/network/wireless/iwd.git"; rev = version; - hash = "sha256-zePFmcQRFjcH6KToTpBFMQzGY+Eq7jijfn0R/MMKGrw="; + hash = "sha256-kE9GBVTKNpgEuE9jQ7k85OhEAN3VWgjmAgifvZfq46I="; }; # Revert test that's broken on aarch64 From 82c73a8d726d7f8e13276f0303655257d0e0fa58 Mon Sep 17 00:00:00 2001 From: Luflosi Date: Mon, 4 Dec 2023 09:51:08 +0100 Subject: [PATCH 23/42] nixos/tests/kubo: disable broken FUSE test The FUSE functionality is completely broken in Kubo v0.24.0. See https://github.com/ipfs/kubo/issues/10242. --- nixos/tests/kubo/default.nix | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nixos/tests/kubo/default.nix b/nixos/tests/kubo/default.nix index 629922fc366d..d8c0c69dc1fb 100644 --- a/nixos/tests/kubo/default.nix +++ b/nixos/tests/kubo/default.nix @@ -1,5 +1,7 @@ { recurseIntoAttrs, runTest }: recurseIntoAttrs { kubo = runTest ./kubo.nix; - kubo-fuse = runTest ./kubo-fuse.nix; + # The FUSE functionality is completely broken since Kubo v0.24.0 + # See https://github.com/ipfs/kubo/issues/10242 + # kubo-fuse = runTest ./kubo-fuse.nix; } From 60781d9b2d23f1ca8c18d7d0582e5a2a54d32932 Mon Sep 17 00:00:00 2001 From: Luflosi Date: Sun, 3 Dec 2023 00:50:53 +0100 Subject: [PATCH 24/42] nixos/tests/kubo: test socket activation for the Gateway Add a new test to check that accessing the Gateway socket also starts the daemon via socket activation. --- nixos/tests/kubo/kubo.nix | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/nixos/tests/kubo/kubo.nix b/nixos/tests/kubo/kubo.nix index 7965ad277385..b8222c652b33 100644 --- a/nixos/tests/kubo/kubo.nix +++ b/nixos/tests/kubo/kubo.nix @@ -46,6 +46,13 @@ f"ipfs --api /unix/run/ipfs.sock cat /ipfs/{ipfs_hash.strip()} | grep fnord2" ) + machine.stop_job("ipfs") + + with subtest("Socket activation for the Gateway"): + machine.succeed( + f"curl 'http://127.0.0.1:8080/ipfs/{ipfs_hash.strip()}' | grep fnord2" + ) + with subtest("Setting dataDir works properly with the hardened systemd unit"): machine.succeed("test -e /mnt/ipfs/config") machine.succeed("test ! -e /var/lib/ipfs/") From 05b4972db0bf91462b1e01b0c1ad234ff29c37d7 Mon Sep 17 00:00:00 2001 From: Luflosi Date: Thu, 14 Dec 2023 23:14:34 +0100 Subject: [PATCH 25/42] kubo: migrate to by-name --- .../networking/kubo/default.nix => by-name/ku/kubo/package.nix} | 0 .../networking => by-name/ku}/kubo/test-repoVersion.nix | 0 pkgs/top-level/all-packages.nix | 2 -- 3 files changed, 2 deletions(-) rename pkgs/{applications/networking/kubo/default.nix => by-name/ku/kubo/package.nix} (100%) rename pkgs/{applications/networking => by-name/ku}/kubo/test-repoVersion.nix (100%) diff --git a/pkgs/applications/networking/kubo/default.nix b/pkgs/by-name/ku/kubo/package.nix similarity index 100% rename from pkgs/applications/networking/kubo/default.nix rename to pkgs/by-name/ku/kubo/package.nix diff --git a/pkgs/applications/networking/kubo/test-repoVersion.nix b/pkgs/by-name/ku/kubo/test-repoVersion.nix similarity index 100% rename from pkgs/applications/networking/kubo/test-repoVersion.nix rename to pkgs/by-name/ku/kubo/test-repoVersion.nix diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 065be3200f7b..3902a037d765 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -9902,8 +9902,6 @@ with pkgs; kubergrunt = callPackage ../applications/networking/cluster/kubergrunt { }; - kubo = callPackage ../applications/networking/kubo { }; - kubo-migrator-all-fs-repo-migrations = callPackage ../applications/networking/kubo-migrator/all-migrations.nix { }; kubo-migrator-unwrapped = callPackage ../applications/networking/kubo-migrator/unwrapped.nix { }; kubo-migrator = callPackage ../applications/networking/kubo-migrator { }; From 16a8b378b7e69e44cac785116cef061463b7f070 Mon Sep 17 00:00:00 2001 From: Luflosi Date: Thu, 14 Dec 2023 20:05:42 +0100 Subject: [PATCH 26/42] kubo: 0.24.0 -> 0.25.0 https://github.com/ipfs/kubo/releases/tag/v0.25.0 --- pkgs/by-name/ku/kubo/package.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/by-name/ku/kubo/package.nix b/pkgs/by-name/ku/kubo/package.nix index 117fbc850d06..ae20f84e8170 100644 --- a/pkgs/by-name/ku/kubo/package.nix +++ b/pkgs/by-name/ku/kubo/package.nix @@ -7,7 +7,7 @@ buildGoModule rec { pname = "kubo"; - version = "0.24.0"; # When updating, also check if the repo version changed and adjust repoVersion below + version = "0.25.0"; # When updating, also check if the repo version changed and adjust repoVersion below rev = "v${version}"; passthru.repoVersion = "15"; # Also update kubo-migrator when changing the repo version @@ -15,7 +15,7 @@ buildGoModule rec { # Kubo makes changes to its source tarball that don't match the git source. src = fetchurl { url = "https://github.com/ipfs/kubo/releases/download/${rev}/kubo-source.tar.gz"; - hash = "sha256-stSjLvg8G1EiXon3Qby4wLgbhX7Aaj9pnxcvE32/42k="; + hash = "sha256-+Mk3rDdtjhETmdaOOSXEFdLTJ0nX9G3qUxctsu5vrSc="; }; # tarball contains multiple files/directories From 53b43ce0e322eb4fc8daaad8a5a597155d42379a Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Fri, 15 Dec 2023 00:47:34 +0100 Subject: [PATCH 27/42] tests.nixpkgs-check-by-name: Fix and document behavior without --base Previously, not passing `--base` would enforce the most strict checks. While there's currently no actual violation of these stricter checks, this does not match the previous behavior. This won't matter once CI passes `--base`, the code handling the optionality can be removed then. --- pkgs/test/nixpkgs-check-by-name/README.md | 9 +++++++-- pkgs/test/nixpkgs-check-by-name/src/main.rs | 16 +++++++++------- pkgs/test/nixpkgs-check-by-name/src/version.rs | 17 +++++++++++++++-- .../tests/override-empty-arg/base/default.nix | 1 + .../base/pkgs/by-name/README.md | 1 + 5 files changed, 33 insertions(+), 11 deletions(-) create mode 100644 pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg/base/default.nix create mode 100644 pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg/base/pkgs/by-name/README.md diff --git a/pkgs/test/nixpkgs-check-by-name/README.md b/pkgs/test/nixpkgs-check-by-name/README.md index c906eaffc974..8ed23204deca 100644 --- a/pkgs/test/nixpkgs-check-by-name/README.md +++ b/pkgs/test/nixpkgs-check-by-name/README.md @@ -10,8 +10,13 @@ This API may be changed over time if the CI workflow making use of it is adjuste - Command line: `nixpkgs-check-by-name [--base ] ` - Arguments: - - ``: The path to the Nixpkgs to check against - - ``: The path to the Nixpkgs to check + - ``: The path to the Nixpkgs to check. + - ``: The path to the Nixpkgs to use as the base to compare `` against. + This allows the strictness of checks to increase over time by only preventing _new_ violations from being introduced, + while allowing violations that already existed. + + If not specified, all violations of stricter checks are allowed. + However, this flag will become required once CI passes it. - Exit code: - `0`: If the [validation](#validity-checks) is successful - `1`: If the [validation](#validity-checks) is not successful diff --git a/pkgs/test/nixpkgs-check-by-name/src/main.rs b/pkgs/test/nixpkgs-check-by-name/src/main.rs index bf3bfb193f18..91e1992a52c9 100644 --- a/pkgs/test/nixpkgs-check-by-name/src/main.rs +++ b/pkgs/test/nixpkgs-check-by-name/src/main.rs @@ -75,14 +75,16 @@ pub fn process( let main_result = check_nixpkgs(main_nixpkgs, eval_accessible_paths)?; let check_result = main_result.result_map(|nixpkgs_version| { if let Some(base) = base_nixpkgs { - check_nixpkgs(base, eval_accessible_paths)?.result_map(|base_nixpkgs_version| { - Ok(Nixpkgs::compare(base_nixpkgs_version, nixpkgs_version)) - }) + check_nixpkgs(base, eval_accessible_paths, error_writer)?.result_map( + |base_nixpkgs_version| { + Ok(Nixpkgs::compare( + Some(base_nixpkgs_version), + nixpkgs_version, + )) + }, + ) } else { - Ok(Nixpkgs::compare( - version::Nixpkgs::default(), - nixpkgs_version, - )) + Ok(Nixpkgs::compare(None, nixpkgs_version)) } })?; diff --git a/pkgs/test/nixpkgs-check-by-name/src/version.rs b/pkgs/test/nixpkgs-check-by-name/src/version.rs index 7f83bdf3ff67..c5cee95e0d53 100644 --- a/pkgs/test/nixpkgs-check-by-name/src/version.rs +++ b/pkgs/test/nixpkgs-check-by-name/src/version.rs @@ -16,12 +16,25 @@ impl Nixpkgs { /// Compares two Nixpkgs versions against each other, returning validation errors only if the /// `from` version satisfied the stricter checks, while the `to` version doesn't satisfy them /// anymore. - pub fn compare(from: Self, to: Self) -> Validation<()> { + pub fn compare(optional_from: Option, to: Self) -> Validation<()> { validation::sequence_( // We only loop over the current attributes, // we don't need to check ones that were removed to.attributes.into_iter().map(|(name, attr_to)| { - Attribute::compare(&name, from.attributes.get(&name), &attr_to) + let attr_from = if let Some(from) = &optional_from { + from.attributes.get(&name) + } else { + // This pretends that if there's no base version to compare against, all + // attributes existed without conforming to the new strictness check for + // backwards compatibility. + // TODO: Remove this case. This is only needed because the `--base` + // argument is still optional, which doesn't need to be once CI is updated + // to pass it. + Some(&Attribute { + empty_non_auto_called: EmptyNonAutoCalled::Invalid, + }) + }; + Attribute::compare(&name, attr_from, &attr_to) }), ) } diff --git a/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg/base/default.nix b/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg/base/default.nix new file mode 100644 index 000000000000..2875ea6327ef --- /dev/null +++ b/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg/base/default.nix @@ -0,0 +1 @@ +import ../../mock-nixpkgs.nix { root = ./.; } diff --git a/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg/base/pkgs/by-name/README.md b/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg/base/pkgs/by-name/README.md new file mode 100644 index 000000000000..b0d2b34e338a --- /dev/null +++ b/pkgs/test/nixpkgs-check-by-name/tests/override-empty-arg/base/pkgs/by-name/README.md @@ -0,0 +1 @@ +(this is just here so the directory can get tracked by git) From 413dd9c03e37562c61cd89799e5eb8a88c7bb42a Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Fri, 15 Dec 2023 01:02:02 +0100 Subject: [PATCH 28/42] tests.nixpkgs-check-by-name: Minor improvements from review --- pkgs/test/nixpkgs-check-by-name/src/eval.rs | 8 ++--- pkgs/test/nixpkgs-check-by-name/src/main.rs | 32 ++++++++++--------- .../test/nixpkgs-check-by-name/src/version.rs | 2 ++ .../tests/no-by-name/expected | 1 + 4 files changed, 24 insertions(+), 19 deletions(-) create mode 100644 pkgs/test/nixpkgs-check-by-name/tests/no-by-name/expected diff --git a/pkgs/test/nixpkgs-check-by-name/src/eval.rs b/pkgs/test/nixpkgs-check-by-name/src/eval.rs index 927e446b452f..20652d9ede26 100644 --- a/pkgs/test/nixpkgs-check-by-name/src/eval.rs +++ b/pkgs/test/nixpkgs-check-by-name/src/eval.rs @@ -41,7 +41,7 @@ enum AttributeVariant { pub fn check_values( nixpkgs_path: &Path, package_names: Vec, - eval_accessible_paths: &Vec<&Path>, + eval_accessible_paths: &[&Path], ) -> validation::Result { // Write the list of packages we need to check into a temporary JSON file. // This can then get read by the Nix evaluation. @@ -110,11 +110,11 @@ pub fn check_values( ))?; Ok( - validation::sequence(package_names.iter().map(|package_name| { - let relative_package_file = structure::relative_file_for_package(package_name); + validation::sequence(package_names.into_iter().map(|package_name| { + let relative_package_file = structure::relative_file_for_package(&package_name); let absolute_package_file = nixpkgs_path.join(&relative_package_file); - if let Some(attribute_info) = actual_files.get(package_name) { + if let Some(attribute_info) = actual_files.get(&package_name) { let check_result = if !attribute_info.is_derivation { NixpkgsProblem::NonDerivation { relative_package_file: relative_package_file.clone(), diff --git a/pkgs/test/nixpkgs-check-by-name/src/main.rs b/pkgs/test/nixpkgs-check-by-name/src/main.rs index 91e1992a52c9..ee73ffbd0f8d 100644 --- a/pkgs/test/nixpkgs-check-by-name/src/main.rs +++ b/pkgs/test/nixpkgs-check-by-name/src/main.rs @@ -31,12 +31,7 @@ pub struct Args { fn main() -> ExitCode { let args = Args::parse(); - match process( - args.base.as_deref(), - &args.nixpkgs, - &vec![], - &mut io::stderr(), - ) { + match process(args.base.as_deref(), &args.nixpkgs, &[], &mut io::stderr()) { Ok(true) => { eprintln!("{}", "Validated successfully".green()); ExitCode::SUCCESS @@ -69,10 +64,10 @@ fn main() -> ExitCode { pub fn process( base_nixpkgs: Option<&Path>, main_nixpkgs: &Path, - eval_accessible_paths: &Vec<&Path>, + eval_accessible_paths: &[&Path], error_writer: &mut W, ) -> anyhow::Result { - let main_result = check_nixpkgs(main_nixpkgs, eval_accessible_paths)?; + let main_result = check_nixpkgs(main_nixpkgs, eval_accessible_paths, error_writer)?; let check_result = main_result.result_map(|nixpkgs_version| { if let Some(base) = base_nixpkgs { check_nixpkgs(base, eval_accessible_paths, error_writer)?.result_map( @@ -99,11 +94,17 @@ pub fn process( } } -/// Checks whether the pkgs/by-name structure in Nixpkgs is valid, -/// and returns to which degree it's valid for checks with increased strictness. -pub fn check_nixpkgs( +/// Checks whether the pkgs/by-name structure in Nixpkgs is valid. +/// +/// This does not include checks that depend on the base version of Nixpkgs to compare against, +/// which is used for checks that were only introduced later and increased strictness. +/// +/// Instead a `version::Nixpkgs` is returned, whose `compare` method allows comparing the +/// result of this function for the base Nixpkgs against the one for the main Nixpkgs. +pub fn check_nixpkgs( nixpkgs_path: &Path, - eval_accessible_paths: &Vec<&Path>, + eval_accessible_paths: &[&Path], + error_writer: &mut W, ) -> validation::Result { Ok({ let nixpkgs_path = nixpkgs_path.canonicalize().context(format!( @@ -112,10 +113,11 @@ pub fn check_nixpkgs( ))?; if !nixpkgs_path.join(utils::BASE_SUBPATH).exists() { - eprintln!( + writeln!( + error_writer, "Given Nixpkgs path does not contain a {} subdirectory, no check necessary.", utils::BASE_SUBPATH - ); + )?; Success(version::Nixpkgs::default()) } else { check_structure(&nixpkgs_path)?.result_map(|package_names| @@ -224,7 +226,7 @@ mod tests { // We don't want coloring to mess up the tests let writer = temp_env::with_var("NO_COLOR", Some("1"), || -> anyhow::Result<_> { let mut writer = vec![]; - process(base_nixpkgs, &path, &vec![&extra_nix_path], &mut writer) + process(base_nixpkgs, &path, &[&extra_nix_path], &mut writer) .context(format!("Failed test case {name}"))?; Ok(writer) })?; diff --git a/pkgs/test/nixpkgs-check-by-name/src/version.rs b/pkgs/test/nixpkgs-check-by-name/src/version.rs index c5cee95e0d53..c82f537c504b 100644 --- a/pkgs/test/nixpkgs-check-by-name/src/version.rs +++ b/pkgs/test/nixpkgs-check-by-name/src/version.rs @@ -16,6 +16,8 @@ impl Nixpkgs { /// Compares two Nixpkgs versions against each other, returning validation errors only if the /// `from` version satisfied the stricter checks, while the `to` version doesn't satisfy them /// anymore. + /// This enables a gradual transition from weaker to stricter checks, by only allowing PRs to + /// increase strictness. pub fn compare(optional_from: Option, to: Self) -> Validation<()> { validation::sequence_( // We only loop over the current attributes, diff --git a/pkgs/test/nixpkgs-check-by-name/tests/no-by-name/expected b/pkgs/test/nixpkgs-check-by-name/tests/no-by-name/expected new file mode 100644 index 000000000000..ddcb2df46e5f --- /dev/null +++ b/pkgs/test/nixpkgs-check-by-name/tests/no-by-name/expected @@ -0,0 +1 @@ +Given Nixpkgs path does not contain a pkgs/by-name subdirectory, no check necessary. From 79618ff8cbfb69b6eb70dbc591b42cee2fed974e Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Fri, 15 Dec 2023 02:14:48 +0100 Subject: [PATCH 29/42] tests.nixpkgs-check-by-name: Improve docs, introduce "ratchet" term --- pkgs/test/nixpkgs-check-by-name/README.md | 30 +++++++++---- pkgs/test/nixpkgs-check-by-name/src/eval.rs | 19 ++++---- pkgs/test/nixpkgs-check-by-name/src/main.rs | 34 +++++++------- .../src/{version.rs => ratchet.rs} | 44 ++++++++++--------- 4 files changed, 73 insertions(+), 54 deletions(-) rename pkgs/test/nixpkgs-check-by-name/src/{version.rs => ratchet.rs} (61%) diff --git a/pkgs/test/nixpkgs-check-by-name/README.md b/pkgs/test/nixpkgs-check-by-name/README.md index 8ed23204deca..7e8d39104e48 100644 --- a/pkgs/test/nixpkgs-check-by-name/README.md +++ b/pkgs/test/nixpkgs-check-by-name/README.md @@ -10,13 +10,15 @@ This API may be changed over time if the CI workflow making use of it is adjuste - Command line: `nixpkgs-check-by-name [--base ] ` - Arguments: - - ``: The path to the Nixpkgs to check. - - ``: The path to the Nixpkgs to use as the base to compare `` against. - This allows the strictness of checks to increase over time by only preventing _new_ violations from being introduced, - while allowing violations that already existed. + - ``: + The path to the Nixpkgs to check. + For PRs, this should be set to a checkout of the PR branch. + - ``: + The path to the Nixpkgs to use as the [ratchet check](#ratchet-checks) base. + For PRs, this should be set to a checkout of the PRs base branch. - If not specified, all violations of stricter checks are allowed. - However, this flag will become required once CI passes it. + If not specified, no ratchet checks will be performed. + However, this flag will become required once CI uses it. - Exit code: - `0`: If the [validation](#validity-checks) is successful - `1`: If the [validation](#validity-checks) is not successful @@ -41,10 +43,20 @@ These checks are performed by this tool: ### Nix evaluation checks - `pkgs.${name}` is defined as `callPackage pkgs/by-name/${shard}/${name}/package.nix args` for some `args`. - - If `pkgs.${name}` is not auto-called from `pkgs/by-name`, `args` must not be empty, - with the exception that if `BASE_NIXPKGS` also has a definition for the same package with empty `args`, it's allowed - `pkgs.lib.isDerivation pkgs.${name}` is `true`. +### Ratchet checks + +Furthermore, this tool implements certain [ratchet](https://qntm.org/ratchet) checks. +This allows gradually phasing out deprecated patterns without breaking the base branch or having to migrate it all at once. +It works by not allowing new instances of the pattern to be introduced, but allowing already existing instances. +The existing instances are coming from ``, which is then checked against `` for new instances. +Ratchets should be removed eventually once the pattern is not used anymore. + +The current ratchets are: + +- If `pkgs.${name}` is not auto-called from `pkgs/by-name`, the `args` in its `callPackage` must not be empty, + ## Development Enter the development environment in this directory either automatically with `direnv` or with @@ -88,7 +100,7 @@ Tests are declared in [`./tests`](./tests) as subdirectories imitating Nixpkgs w - `base` (optional): Contains another subdirectory imitating Nixpkgs with potentially any of the above structures. - This will be used as the `--base` argument, allowing tests of gradual transitions. + This is used for [ratchet checks](#ratchet-checks). - `expected` (optional): A file containing the expected standard output. diff --git a/pkgs/test/nixpkgs-check-by-name/src/eval.rs b/pkgs/test/nixpkgs-check-by-name/src/eval.rs index 20652d9ede26..cd8c70472cf2 100644 --- a/pkgs/test/nixpkgs-check-by-name/src/eval.rs +++ b/pkgs/test/nixpkgs-check-by-name/src/eval.rs @@ -1,7 +1,7 @@ use crate::nixpkgs_problem::NixpkgsProblem; +use crate::ratchet; use crate::structure; use crate::validation::{self, Validation::Success}; -use crate::version; use std::path::Path; use anyhow::Context; @@ -42,7 +42,7 @@ pub fn check_values( nixpkgs_path: &Path, package_names: Vec, eval_accessible_paths: &[&Path], -) -> validation::Result { +) -> validation::Result { // Write the list of packages we need to check into a temporary JSON file. // This can then get read by the Nix evaluation. let attrs_file = NamedTempFile::new().context("Failed to create a temporary file")?; @@ -126,8 +126,8 @@ pub fn check_values( }; let check_result = check_result.and(match &attribute_info.variant { - AttributeVariant::AutoCalled => Success(version::Attribute { - empty_non_auto_called: version::EmptyNonAutoCalled::Valid, + AttributeVariant::AutoCalled => Success(ratchet::Package { + empty_non_auto_called: ratchet::EmptyNonAutoCalled::Valid, }), AttributeVariant::CallPackage { path, empty_arg } => { let correct_file = if let Some(call_package_path) = path { @@ -137,11 +137,12 @@ pub fn check_values( }; if correct_file { - Success(version::Attribute { + Success(ratchet::Package { + // Empty arguments for non-auto-called packages are not allowed anymore. empty_non_auto_called: if *empty_arg { - version::EmptyNonAutoCalled::Invalid + ratchet::EmptyNonAutoCalled::Invalid } else { - version::EmptyNonAutoCalled::Valid + ratchet::EmptyNonAutoCalled::Valid }, }) } else { @@ -168,8 +169,8 @@ pub fn check_values( .into() } })) - .map(|elems| version::Nixpkgs { - attributes: elems.into_iter().collect(), + .map(|elems| ratchet::Nixpkgs { + packages: elems.into_iter().collect(), }), ) } diff --git a/pkgs/test/nixpkgs-check-by-name/src/main.rs b/pkgs/test/nixpkgs-check-by-name/src/main.rs index ee73ffbd0f8d..01f7d4b71982 100644 --- a/pkgs/test/nixpkgs-check-by-name/src/main.rs +++ b/pkgs/test/nixpkgs-check-by-name/src/main.rs @@ -1,15 +1,14 @@ mod eval; mod nixpkgs_problem; +mod ratchet; mod references; mod structure; mod utils; mod validation; -mod version; use crate::structure::check_structure; use crate::validation::Validation::Failure; use crate::validation::Validation::Success; -use crate::version::Nixpkgs; use anyhow::Context; use clap::Parser; use colored::Colorize; @@ -21,10 +20,14 @@ use std::process::ExitCode; #[derive(Parser, Debug)] #[command(about)] pub struct Args { - /// Path to nixpkgs + /// Path to the main Nixpkgs to check. + /// For PRs, this should be set to a checkout of the PR branch. nixpkgs: PathBuf, - /// Path to the base Nixpkgs to compare against + /// Path to the base Nixpkgs to run ratchet checks against. + /// For PRs, this should be set to a checkout of the PRs base branch. + /// If not specified, no ratchet checks will be performed. + /// However, this flag will become required once CI uses it. #[arg(long)] base: Option, } @@ -50,8 +53,8 @@ fn main() -> ExitCode { /// Does the actual work. This is the abstraction used both by `main` and the tests. /// /// # Arguments -/// - `base_nixpkgs`: The path to the base Nixpkgs to compare against -/// - `main_nixpkgs`: The path to the main Nixpkgs to check +/// - `base_nixpkgs`: Path to the base Nixpkgs to run ratchet checks against. +/// - `main_nixpkgs`: Path to the main Nixpkgs to check. /// - `eval_accessible_paths`: /// Extra paths that need to be accessible to evaluate Nixpkgs using `restrict-eval`. /// This is used to allow the tests to access the mock-nixpkgs.nix file @@ -67,19 +70,22 @@ pub fn process( eval_accessible_paths: &[&Path], error_writer: &mut W, ) -> anyhow::Result { + // Check the main Nixpkgs first let main_result = check_nixpkgs(main_nixpkgs, eval_accessible_paths, error_writer)?; let check_result = main_result.result_map(|nixpkgs_version| { + // If the main Nixpkgs doesn't have any problems, run the ratchet checks against the base + // Nixpkgs if let Some(base) = base_nixpkgs { check_nixpkgs(base, eval_accessible_paths, error_writer)?.result_map( |base_nixpkgs_version| { - Ok(Nixpkgs::compare( + Ok(ratchet::Nixpkgs::compare( Some(base_nixpkgs_version), nixpkgs_version, )) }, ) } else { - Ok(Nixpkgs::compare(None, nixpkgs_version)) + Ok(ratchet::Nixpkgs::compare(None, nixpkgs_version)) } })?; @@ -96,16 +102,14 @@ pub fn process( /// Checks whether the pkgs/by-name structure in Nixpkgs is valid. /// -/// This does not include checks that depend on the base version of Nixpkgs to compare against, -/// which is used for checks that were only introduced later and increased strictness. -/// -/// Instead a `version::Nixpkgs` is returned, whose `compare` method allows comparing the -/// result of this function for the base Nixpkgs against the one for the main Nixpkgs. +/// This does not include ratchet checks, see ../README.md#ratchet-checks +/// Instead a `ratchet::Nixpkgs` value is returned, whose `compare` method allows performing the +/// ratchet check against another result. pub fn check_nixpkgs( nixpkgs_path: &Path, eval_accessible_paths: &[&Path], error_writer: &mut W, -) -> validation::Result { +) -> validation::Result { Ok({ let nixpkgs_path = nixpkgs_path.canonicalize().context(format!( "Nixpkgs path {} could not be resolved", @@ -118,7 +122,7 @@ pub fn check_nixpkgs( "Given Nixpkgs path does not contain a {} subdirectory, no check necessary.", utils::BASE_SUBPATH )?; - Success(version::Nixpkgs::default()) + Success(ratchet::Nixpkgs::default()) } else { check_structure(&nixpkgs_path)?.result_map(|package_names| // Only if we could successfully parse the structure, we do the evaluation checks diff --git a/pkgs/test/nixpkgs-check-by-name/src/version.rs b/pkgs/test/nixpkgs-check-by-name/src/ratchet.rs similarity index 61% rename from pkgs/test/nixpkgs-check-by-name/src/version.rs rename to pkgs/test/nixpkgs-check-by-name/src/ratchet.rs index c82f537c504b..c12f1ead2540 100644 --- a/pkgs/test/nixpkgs-check-by-name/src/version.rs +++ b/pkgs/test/nixpkgs-check-by-name/src/ratchet.rs @@ -1,30 +1,28 @@ +//! This module implements the ratchet checks, see ../README.md#ratchet-checks +//! +//! Each type has a `compare` method that validates the ratchet checks for that item. + use crate::nixpkgs_problem::NixpkgsProblem; use crate::structure; use crate::validation::{self, Validation, Validation::Success}; use std::collections::HashMap; -/// The check version conformity of a Nixpkgs path: -/// When the strictness of the check increases, this structure should be extended to distinguish -/// between parts that are still valid, and ones that aren't valid anymore. +/// The ratchet value for the entirety of Nixpkgs. #[derive(Default)] pub struct Nixpkgs { - /// The package attributes tracked in `pkgs/by-name` - pub attributes: HashMap, + /// The ratchet values for each package in `pkgs/by-name` + pub packages: HashMap, } impl Nixpkgs { - /// Compares two Nixpkgs versions against each other, returning validation errors only if the - /// `from` version satisfied the stricter checks, while the `to` version doesn't satisfy them - /// anymore. - /// This enables a gradual transition from weaker to stricter checks, by only allowing PRs to - /// increase strictness. + /// Validates the ratchet checks for Nixpkgs pub fn compare(optional_from: Option, to: Self) -> Validation<()> { validation::sequence_( // We only loop over the current attributes, // we don't need to check ones that were removed - to.attributes.into_iter().map(|(name, attr_to)| { + to.packages.into_iter().map(|(name, attr_to)| { let attr_from = if let Some(from) = &optional_from { - from.attributes.get(&name) + from.packages.get(&name) } else { // This pretends that if there's no base version to compare against, all // attributes existed without conforming to the new strictness check for @@ -32,22 +30,24 @@ impl Nixpkgs { // TODO: Remove this case. This is only needed because the `--base` // argument is still optional, which doesn't need to be once CI is updated // to pass it. - Some(&Attribute { + Some(&Package { empty_non_auto_called: EmptyNonAutoCalled::Invalid, }) }; - Attribute::compare(&name, attr_from, &attr_to) + Package::compare(&name, attr_from, &attr_to) }), ) } } -/// The check version conformity of an attribute defined by `pkgs/by-name` -pub struct Attribute { +/// The ratchet value for a single package in `pkgs/by-name` +pub struct Package { + /// The ratchet value for the check for non-auto-called empty arguments pub empty_non_auto_called: EmptyNonAutoCalled, } -impl Attribute { +impl Package { + /// Validates the ratchet checks for a single package defined in `pkgs/by-name` pub fn compare(name: &str, optional_from: Option<&Self>, to: &Self) -> Validation<()> { EmptyNonAutoCalled::compare( name, @@ -57,17 +57,19 @@ impl Attribute { } } -/// Whether an attribute conforms to the new strictness check that -/// `callPackage ... {}` is not allowed anymore in `all-package.nix` +/// The ratchet value of a single package in `pkgs/by-name` +/// for the non-auto-called empty argument check of a single. +/// +/// This checks that packages defined in `pkgs/by-name` cannot be overridden +/// with an empty second argument like `callPackage ... { }`. #[derive(PartialEq, PartialOrd)] pub enum EmptyNonAutoCalled { - /// The attribute is not valid anymore with the new check Invalid, - /// The attribute is still valid with the new check Valid, } impl EmptyNonAutoCalled { + /// Validates the non-auto-called empty argument ratchet check for a single package defined in `pkgs/by-name` fn compare(name: &str, optional_from: Option<&Self>, to: &Self) -> Validation<()> { let from = optional_from.unwrap_or(&Self::Valid); if to >= from { From 74e8b38dbe809022d096b11b87ba33a68ba0d374 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Fri, 15 Dec 2023 02:23:05 +0100 Subject: [PATCH 30/42] tests.nixpkgs-check-by-name: Move interface description into code This would be duplicated otherwise --- pkgs/test/nixpkgs-check-by-name/README.md | 26 +++++---------------- pkgs/test/nixpkgs-check-by-name/src/main.rs | 14 ++++++++++- 2 files changed, 19 insertions(+), 21 deletions(-) diff --git a/pkgs/test/nixpkgs-check-by-name/README.md b/pkgs/test/nixpkgs-check-by-name/README.md index 7e8d39104e48..640e744546a7 100644 --- a/pkgs/test/nixpkgs-check-by-name/README.md +++ b/pkgs/test/nixpkgs-check-by-name/README.md @@ -4,28 +4,14 @@ This directory implements a program to check the [validity](#validity-checks) of It is being used by [this GitHub Actions workflow](../../../.github/workflows/check-by-name.yml). This is part of the implementation of [RFC 140](https://github.com/NixOS/rfcs/pull/140). -## API +## Interface -This API may be changed over time if the CI workflow making use of it is adjusted to deal with the change appropriately. +The interface of the tool is shown with `--help`: +``` +cargo run -- --help +``` -- Command line: `nixpkgs-check-by-name [--base ] ` -- Arguments: - - ``: - The path to the Nixpkgs to check. - For PRs, this should be set to a checkout of the PR branch. - - ``: - The path to the Nixpkgs to use as the [ratchet check](#ratchet-checks) base. - For PRs, this should be set to a checkout of the PRs base branch. - - If not specified, no ratchet checks will be performed. - However, this flag will become required once CI uses it. -- Exit code: - - `0`: If the [validation](#validity-checks) is successful - - `1`: If the [validation](#validity-checks) is not successful - - `2`: If an unexpected I/O error occurs -- Standard error: - - Informative messages - - Detected problems if validation is not successful +The interface may be changed over time only if the CI workflow making use of it is adjusted to deal with the change appropriately. ## Validity checks diff --git a/pkgs/test/nixpkgs-check-by-name/src/main.rs b/pkgs/test/nixpkgs-check-by-name/src/main.rs index 01f7d4b71982..18c950d0a6eb 100644 --- a/pkgs/test/nixpkgs-check-by-name/src/main.rs +++ b/pkgs/test/nixpkgs-check-by-name/src/main.rs @@ -17,8 +17,20 @@ use std::path::{Path, PathBuf}; use std::process::ExitCode; /// Program to check the validity of pkgs/by-name +/// +/// This CLI interface may be changed over time if the CI workflow making use of +/// it is adjusted to deal with the change appropriately. +/// +/// Exit code: +/// - `0`: If the validation is successful +/// - `1`: If the validation is not successful +/// - `2`: If an unexpected I/O error occurs +/// +/// Standard error: +/// - Informative messages +/// - Detected problems if validation is not successful #[derive(Parser, Debug)] -#[command(about)] +#[command(about, verbatim_doc_comment)] pub struct Args { /// Path to the main Nixpkgs to check. /// For PRs, this should be set to a checkout of the PR branch. From dbbbf2470e021dc723f42ee91e87dac3b0c8fcd9 Mon Sep 17 00:00:00 2001 From: "R. Ryantm" Date: Fri, 15 Dec 2023 15:55:09 +0000 Subject: [PATCH 31/42] grpc-gateway: 2.18.0 -> 2.18.1 --- pkgs/development/tools/grpc-gateway/default.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkgs/development/tools/grpc-gateway/default.nix b/pkgs/development/tools/grpc-gateway/default.nix index 08452c98c286..52a4b4295a6a 100644 --- a/pkgs/development/tools/grpc-gateway/default.nix +++ b/pkgs/development/tools/grpc-gateway/default.nix @@ -2,16 +2,16 @@ buildGoModule rec { pname = "grpc-gateway"; - version = "2.18.0"; + version = "2.18.1"; src = fetchFromGitHub { owner = "grpc-ecosystem"; repo = "grpc-gateway"; rev = "v${version}"; - sha256 = "sha256-FVb3x/wZ0jLI17lXAT/lcUpJiq3ZcvWjFnORynOvfmY="; + sha256 = "sha256-mbRceXqc7UmrhM2Y6JJIUvMf9YxMFMjRW7VvEa8/xHs="; }; - vendorHash = "sha256-SV2ZO8Y9yt6iyw9VvNY0xpqZIzLrTyHYYpIpzcEVsLY="; + vendorHash = "sha256-zVojs4q8TytJY3myKvLdACnMFJ0iK9Cfn+aZ4d/j34s="; meta = with lib; { description = From fc2d26939d2839d65553e8ca605f2440a45fc387 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Fri, 15 Dec 2023 17:27:26 +0100 Subject: [PATCH 32/42] tests.nixpkgs-check-by-name: Improve check clarity --- pkgs/test/nixpkgs-check-by-name/README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pkgs/test/nixpkgs-check-by-name/README.md b/pkgs/test/nixpkgs-check-by-name/README.md index 640e744546a7..19865ca0952b 100644 --- a/pkgs/test/nixpkgs-check-by-name/README.md +++ b/pkgs/test/nixpkgs-check-by-name/README.md @@ -19,7 +19,7 @@ These checks are performed by this tool: ### File structure checks - `pkgs/by-name` must only contain subdirectories of the form `${shard}/${name}`, called _package directories_. -- The `name`'s of package directories must be unique when lowercased +- The `name`'s of package directories must be unique when lowercased. - `name` is a string only consisting of the ASCII characters `a-z`, `A-Z`, `0-9`, `-` or `_`. - `shard` is the lowercased first two letters of `name`, expressed in Nix: `shard = toLower (substring 0 2 name)`. - Each package directory must contain a `package.nix` file and may contain arbitrary other files. @@ -28,8 +28,8 @@ These checks are performed by this tool: - Each package directory must not refer to files outside itself using symlinks or Nix path expressions. ### Nix evaluation checks -- `pkgs.${name}` is defined as `callPackage pkgs/by-name/${shard}/${name}/package.nix args` for some `args`. -- `pkgs.lib.isDerivation pkgs.${name}` is `true`. +- For each package directory, the `pkgs.${name}` attribute must be defined as `callPackage pkgs/by-name/${shard}/${name}/package.nix args` for some `args`. +- For each package directory, `pkgs.lib.isDerivation pkgs.${name}` must be `true`. ### Ratchet checks @@ -41,7 +41,8 @@ Ratchets should be removed eventually once the pattern is not used anymore. The current ratchets are: -- If `pkgs.${name}` is not auto-called from `pkgs/by-name`, the `args` in its `callPackage` must not be empty, +- New manual definitions of `pkgs.${name}` (e.g. in `pkgs/top-level/all-packages.nix`) with `args = { }` + (see [nix evaluation checks](#nix-evaluation-checks)) must not be introduced. ## Development From dbb599f2e4337aa80cf71462f04bd32216d8bb90 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Fri, 15 Dec 2023 18:49:29 +0100 Subject: [PATCH 33/42] workflows/check-by-name: Cancel on merge conflicts --- .github/workflows/check-by-name.yml | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/check-by-name.yml b/.github/workflows/check-by-name.yml index 294775fa1c8e..5e3e65641f82 100644 --- a/.github/workflows/check-by-name.yml +++ b/.github/workflows/check-by-name.yml @@ -8,8 +8,9 @@ on: # Using pull_request_target instead of pull_request avoids having to approve first time contributors pull_request_target -# The tool doesn't need any permissions, it only outputs success or not based on the checkout -permissions: {} +permissions: + # We need this permission to cancel the workflow run if there's a merge conflict + actions: write jobs: check: @@ -62,7 +63,14 @@ jobs: if [[ "$mergeable" == "true" ]]; then echo "The PR can be merged, checking the merge commit $mergedSha" else - echo "The PR cannot be merged, it has a merge conflict" + echo "The PR cannot be merged, it has a merge conflict, cancelling the workflow.." + gh api \ + --method POST \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + /repos/"$GITHUB_REPOSITORY"/actions/runs/"$GITHUB_RUN_ID"/cancel + sleep 60 + # If it's still not canceled after a minute, something probably went wrong, just exit exit 1 fi echo "mergedSha=$mergedSha" >> "$GITHUB_ENV" From e4129a6cad6e0e599e4c81522f513d2c06d195b5 Mon Sep 17 00:00:00 2001 From: Joerie de Gram Date: Fri, 15 Dec 2023 19:24:27 +0100 Subject: [PATCH 34/42] winbox: switch to wineWowPackages.stable In a9bf124 (PR #273232) winePackages.staging was upgraded from 8.20 to 9.0-rc1 after which WinBox refuses to run This change downgrades wine to the stable release (8.20) --- pkgs/top-level/all-packages.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index ff762e7deb25..ff9ffbc5923c 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -2145,7 +2145,7 @@ with pkgs; vrrtest = callPackage ../tools/video/vrrtest { }; winbox = callPackage ../tools/admin/winbox { - wine = wineWowPackages.staging; + wine = wineWowPackages.stable; }; wtwitch = callPackage ../tools/video/wtwitch { }; From 919401f9635b3dd6fb0a76d8d43d75a3fd63dae7 Mon Sep 17 00:00:00 2001 From: embr Date: Fri, 15 Dec 2023 18:00:20 +0100 Subject: [PATCH 35/42] bozohttpd: Update source URL The domain has changed, but it's the same tarball with the same hash. --- pkgs/servers/http/bozohttpd/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/servers/http/bozohttpd/default.nix b/pkgs/servers/http/bozohttpd/default.nix index 5fd9f350b117..038548bb16fa 100644 --- a/pkgs/servers/http/bozohttpd/default.nix +++ b/pkgs/servers/http/bozohttpd/default.nix @@ -27,7 +27,7 @@ stdenv.mkDerivation rec { # bozohttpd is developed in-tree in pkgsrc, canonical hashes can be found at: # http://cvsweb.netbsd.org/bsdweb.cgi/pkgsrc/www/bozohttpd/distinfo src = fetchurl { - url = "http://www.eterna.com.au/${pname}/${pname}-${version}.tar.bz2"; + url = "http://eterna23.net/${pname}/${pname}-${version}.tar.bz2"; hash = "sha512-J1uPqzzy5sWXIWgsrpUtuV2lvTsfIGgCQMbPEClGNpP2/soEf77146PnUotAt7LoeypW/YALYS5nmhbySJDltg=="; }; From 700959c8eea7f2c18a3adee5c40cfba81b69ea7c Mon Sep 17 00:00:00 2001 From: Pol Dellaiera Date: Fri, 15 Dec 2023 19:48:49 +0100 Subject: [PATCH 36/42] nixos/winbox: init --- nixos/modules/module-list.nix | 1 + nixos/modules/programs/winbox.nix | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+) create mode 100644 nixos/modules/programs/winbox.nix diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix index c0bd0b1600b9..11c344f99903 100644 --- a/nixos/modules/module-list.nix +++ b/nixos/modules/module-list.nix @@ -278,6 +278,7 @@ ./programs/wayland/waybar.nix ./programs/wayland/wayfire.nix ./programs/weylus.nix + ./programs/winbox.nix ./programs/wireshark.nix ./programs/xastir.nix ./programs/wshowkeys.nix diff --git a/nixos/modules/programs/winbox.nix b/nixos/modules/programs/winbox.nix new file mode 100644 index 000000000000..6af299d52009 --- /dev/null +++ b/nixos/modules/programs/winbox.nix @@ -0,0 +1,23 @@ +{ config, lib, pkgs, ... }: + +let + cfg = config.programs.winbox; +in +{ + options.programs.winbox = { + enable = lib.mkEnableOption ("MikroTik Winbox"); + package = lib.mkPackageOption pkgs "winbox" { }; + + openFirewall = lib.mkOption { + description = '' + Whether to open ports for the MikroTik Neighbor Discovery protocol. Required for Winbox neighbor discovery. + ''; + default = false; + type = lib.types.bool; + }; + }; + + config = lib.mkIf cfg.enable { + networking.firewall.allowedUDPPorts = lib.optionals cfg.openFirewall [ 5678 ]; + }; +} From eb9018d59225a5c3044922946c0cd6f56e8e2c53 Mon Sep 17 00:00:00 2001 From: "R. Ryantm" Date: Fri, 15 Dec 2023 19:42:56 +0000 Subject: [PATCH 37/42] httplib: 0.14.1 -> 0.14.2 --- pkgs/development/libraries/httplib/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/development/libraries/httplib/default.nix b/pkgs/development/libraries/httplib/default.nix index a887ea55330f..22c81d57e9df 100644 --- a/pkgs/development/libraries/httplib/default.nix +++ b/pkgs/development/libraries/httplib/default.nix @@ -7,13 +7,13 @@ stdenv.mkDerivation rec { pname = "httplib"; - version = "0.14.1"; + version = "0.14.2"; src = fetchFromGitHub { owner = "yhirose"; repo = "cpp-httplib"; rev = "v${version}"; - hash = "sha256-JBs2FvcdAvxysYhzakP0wU/mUCWfKZ8dk5ROWL5sej0="; + hash = "sha256-JfxeXHo34MKtAkMO3pNWiPorh3f8s4SVrdAaydVYdrY="; }; nativeBuildInputs = [ cmake ]; From c71de1b26252bbcabd49f736996cdcc64eeb386c Mon Sep 17 00:00:00 2001 From: "R. RyanTM" Date: Fri, 15 Dec 2023 13:02:46 -0800 Subject: [PATCH 38/42] cf-terraforming: 0.16.1 -> 0.17.0 (#274375) --- pkgs/tools/misc/cf-terraforming/default.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkgs/tools/misc/cf-terraforming/default.nix b/pkgs/tools/misc/cf-terraforming/default.nix index e6858863fc4e..d3c1490d528e 100644 --- a/pkgs/tools/misc/cf-terraforming/default.nix +++ b/pkgs/tools/misc/cf-terraforming/default.nix @@ -2,16 +2,16 @@ buildGoModule rec { pname = "cf-terraforming"; - version = "0.16.1"; + version = "0.17.0"; src = fetchFromGitHub { owner = "cloudflare"; repo = "cf-terraforming"; rev = "v${version}"; - sha256 = "sha256-C046oNN+tGvVIakCGJKKQBNHX+L1naaMDIk7tNGNjeQ="; + sha256 = "sha256-XFL9BfTpZaIsNeJLT3wEPFBvyO/VxvFjpApjjxvaGKw="; }; - vendorHash = "sha256-bfxF0qlEbZDczEuFhckqsG00/IzuM18ut/AQ9EMwdh0="; + vendorHash = "sha256-j4J6VAXT+CdU4WkISBunJn6B25CDdaarhFnnLaC4cBE="; ldflags = [ "-X github.com/cloudflare/cf-terraforming/internal/app/cf-terraforming/cmd.versionString=${version}" ]; # The test suite insists on downloading a binary release of Terraform from From 271235e3891e6825fd7dbe94d45e229e41283b3a Mon Sep 17 00:00:00 2001 From: Tom Vincent Date: Mon, 4 Dec 2023 18:25:15 +0000 Subject: [PATCH 39/42] signal-desktop: re-enable wayland --- .../networking/instant-messengers/signal-desktop/generic.nix | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkgs/applications/networking/instant-messengers/signal-desktop/generic.nix b/pkgs/applications/networking/instant-messengers/signal-desktop/generic.nix index 68c5552b251c..15db7d09ac42 100644 --- a/pkgs/applications/networking/instant-messengers/signal-desktop/generic.nix +++ b/pkgs/applications/networking/instant-messengers/signal-desktop/generic.nix @@ -159,8 +159,7 @@ in stdenv.mkDerivation rec { preFixup = '' gappsWrapperArgs+=( --prefix LD_LIBRARY_PATH : "${lib.makeLibraryPath [ stdenv.cc.cc pipewire ] }" - # Currently crashes see https://github.com/NixOS/nixpkgs/issues/222043 - #--add-flags "\''${NIXOS_OZONE_WL:+\''${WAYLAND_DISPLAY:+--ozone-platform-hint=auto --enable-features=WaylandWindowDecorations}}" + --add-flags "\''${NIXOS_OZONE_WL:+\''${WAYLAND_DISPLAY:+--ozone-platform-hint=auto --enable-features=WaylandWindowDecorations}}" --suffix PATH : ${lib.makeBinPath [ xdg-utils ]} ) From d4fcb44dcc6153ef982bd7fa87dca97bd3142796 Mon Sep 17 00:00:00 2001 From: Luflosi Date: Sat, 2 Dec 2023 23:52:09 +0100 Subject: [PATCH 40/42] nixos/kubo: fix potential panic on startup This fixes a panic of the kubo daemon which could occur under certain conditions when the daemon was starting. It was caused by the `ipfs.service` unit not depending on the `ipfs-api.socket` and `ipfs-gateway.socket` units with `Wants=`. This allows the `ipfs.service` to be started manually or by `nixos-rebuild` without the sockets being set up before that. When that happens, the daemon won't know about these sockets and will only use what is set in `services.kubo.settings.Addresses.Gateway` and `services.kubo.settings.Addresses.API`. By default the `API` is an empty list in NixOS though. The daemon doesn't like this at all and panics on startup, see https://github.com/ipfs/kubo/issues/10056. With this commit, starting `ipfs.service` will first set up the two sockets before starting the actual service. Adding the `Sockets=` option implicitly adds a `Wants=` for the sockets and this is exactly what we need. See https://www.freedesktop.org/software/systemd/man/latest/systemd.service.html#Implicit%20Dependencies . This can be checked with `systemctl show ipfs.service`. This should probably be upstreamed to the unit file in the Kubo repo. The problem can be reproduced in the following way: - Add `services.kubo.enable = true` to `/etc/nixos/configuration.nix` - `sudo nixos-rebuild switch` (this may already fail, not sure why it's not deterministic for me) - `sudo systemctl stop ipfs-api.socket` - `sudo systemctl stop ipfs-gateway.socket` - `sudo systemctl stop ipfs.service` - `sudo systemctl start ipfs.service` Fixes #248447. --- nixos/modules/services/network-filesystems/kubo.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nixos/modules/services/network-filesystems/kubo.nix b/nixos/modules/services/network-filesystems/kubo.nix index fbf9b32a2b25..e0b3fb0f36a7 100644 --- a/nixos/modules/services/network-filesystems/kubo.nix +++ b/nixos/modules/services/network-filesystems/kubo.nix @@ -361,6 +361,8 @@ in Group = cfg.group; StateDirectory = ""; ReadWritePaths = optionals (!cfg.autoMount) [ "" cfg.dataDir ]; + # Make sure the socket units are started before ipfs.service + Sockets = [ "ipfs-gateway.socket" "ipfs-api.socket" ]; } // optionalAttrs (cfg.serviceFdlimit != null) { LimitNOFILE = cfg.serviceFdlimit; }; } // optionalAttrs (!cfg.startWhenNeeded) { wantedBy = [ "default.target" ]; From 2925a9ef304ca42050efc7d0589b9cebd9e8173b Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Sat, 16 Dec 2023 00:37:41 +0200 Subject: [PATCH 41/42] hjson-go: 4.3.1 -> 4.4.0 --- .../hjson-go/default.nix => by-name/hj/hjson-go/package.nix} | 4 ++-- pkgs/top-level/all-packages.nix | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) rename pkgs/{development/tools/hjson-go/default.nix => by-name/hj/hjson-go/package.nix} (86%) diff --git a/pkgs/development/tools/hjson-go/default.nix b/pkgs/by-name/hj/hjson-go/package.nix similarity index 86% rename from pkgs/development/tools/hjson-go/default.nix rename to pkgs/by-name/hj/hjson-go/package.nix index 00af9d409f81..0e625e2985b6 100644 --- a/pkgs/development/tools/hjson-go/default.nix +++ b/pkgs/by-name/hj/hjson-go/package.nix @@ -5,13 +5,13 @@ buildGoModule rec { pname = "hjson-go"; - version = "4.3.1"; + version = "4.4.0"; src = fetchFromGitHub { owner = "hjson"; repo = pname; rev = "v${version}"; - hash = "sha256-ox6/PY7Nx282bUekLoXezWfKDiDzCBUZMa5/nu2qG40="; + hash = "sha256-fonPxk/9ue8LzHTdKpuHJcucQoMl4P6gq+tbjS8Ui7Q="; }; vendorHash = null; diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 875a17fc36dd..538485059581 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -40440,8 +40440,6 @@ with pkgs; hjson = with python3Packages; toPythonApplication hjson; - hjson-go = callPackage ../development/tools/hjson-go { }; - epkowa = callPackage ../misc/drivers/epkowa { }; utsushi = callPackage ../misc/drivers/utsushi { }; From 386647bebf6b1430c783790a513a51769f478252 Mon Sep 17 00:00:00 2001 From: Yureka Date: Sat, 16 Dec 2023 00:06:12 +0100 Subject: [PATCH 42/42] lsp-plugins: apply patch "Fix aarch64 msmatrix code" (#274504) This fixes a critical issue in the dsp code path used on Apple Silicon machines --- pkgs/applications/audio/lsp-plugins/default.nix | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pkgs/applications/audio/lsp-plugins/default.nix b/pkgs/applications/audio/lsp-plugins/default.nix index 43e5338cb9bb..e6db737c4016 100644 --- a/pkgs/applications/audio/lsp-plugins/default.nix +++ b/pkgs/applications/audio/lsp-plugins/default.nix @@ -1,4 +1,4 @@ -{ lib, stdenv, fetchurl, pkg-config, makeWrapper +{ lib, stdenv, fetchurl, fetchpatch, pkg-config, makeWrapper , libsndfile, jack2 , libGLU, libGL, lv2, cairo , ladspaH, php, libXrandr }: @@ -11,6 +11,14 @@ stdenv.mkDerivation rec { url = "https://github.com/sadko4u/${pname}/releases/download/${version}/${pname}-src-${version}.tar.gz"; sha256 = "sha256-eJO+1fCNzqjTdGrPlhIrHc3UimkJOydRqTq49IN+Iwo="; }; + patches = [ + (fetchpatch { + url = "https://github.com/lsp-plugins/lsp-dsp-lib/commit/58c3f985f009c84347fa91236f164a9e47aafa93.patch"; + stripLen = 1; + extraPrefix = "modules/lsp-dsp-lib/"; + hash = "sha256-pCLucLijXOgp69xNjSRCRxgVoQziT0YiHLnQGbkefqE="; + }) + ]; outputs = [ "out" "dev" "doc" ];