fetchurl: Add hook for rewriting/filtering URLs

This allows on-the-fly rewriting of URLs before they are passed from
fetchurl (or fetchurlBoot) to curl.

The intended use is to allow inserting company-internal mirrors, or
working around company firewalls and similar network restrictions,
without having to extensively patch across all of nixpkgs. Instead,
users can pass a function in their nixpkgs that performs the necessary
URL rewrites.

Co-authored-by: Alexander Bantyev <balsoft@balsoft.ru>

doc/release-notes/rl-2511.section.md

@@ -21,7 +21,7 @@
 
 <!-- To avoid merge conflicts, consider adding your item at an arbitrary place in the list instead. -->
 
-- Create the first release note entry in this section!
+- Added `rewriteURL` attribute to the nixpkgs `config`, to allow for rewriting the URLs downloaded by `fetchurl`.
 
 ## Nixpkgs Library {#sec-nixpkgs-release-25.11-lib}
 

pkgs/build-support/fetchurl/boot.nix

@@ -2,7 +2,10 @@ let
   mirrors = import ./mirrors.nix;
 in
 
-{ system }:
+{
+  rewriteURL,
+  system,
+}:
 
 {
   url ? builtins.head urls,
@@ -28,7 +31,15 @@ import <nix/fetchurl.nix> {
     # Handle mirror:// URIs. Since <nix/fetchurl.nix> currently
     # supports only one URI, use the first listed mirror.
     let
-      m = builtins.match "mirror://([a-z]+)/(.*)" url;
+      url_ =
+        let
+          u = rewriteURL url;
+        in
+        if builtins.isString u then
+          u
+        else
+          throw "rewriteURL deleted the only URL passed to fetchurlBoot (was ${url})";
+      m = builtins.match "mirror://([a-z]+)/(.*)" url_;
     in
-    if m == null then url else builtins.head (mirrors.${builtins.elemAt m 0}) + (builtins.elemAt m 1);
+    if m == null then url_ else builtins.head (mirrors.${builtins.elemAt m 0}) + (builtins.elemAt m 1);
 }

pkgs/build-support/fetchurl/default.nix

@@ -6,6 +6,7 @@
   stdenvNoCC,
   curl, # Note that `curl' may be `null', in case of the native stdenvNoCC.
   cacert ? null,
+  rewriteURL,
 }:
 
 let
@@ -122,7 +123,7 @@ in
 }@args:
 
 let
-  urls_ =
+  preRewriteUrls =
     if urls != [ ] && url == "" then
       (
         if lib.isList urls then urls else throw "`urls` is not a list: ${lib.generators.toPretty { } urls}"
@@ -137,6 +138,12 @@ let
     else
       throw "fetchurl requires either `url` or `urls` to be set: ${lib.generators.toPretty { } args}";
 
+  urls_ =
+    let
+      u = lib.lists.filter (url: lib.isString url) (map rewriteURL preRewriteUrls);
+    in
+    if u == [ ] then throw "urls is empty after rewriteURL (was ${toString preRewriteUrls})" else u;
+
   hash_ =
     if
       with lib.lists;

pkgs/stdenv/darwin/default.nix

@@ -181,6 +181,7 @@ let
           inherit lib;
           stdenvNoCC = prevStage.ccWrapperStdenv or thisStdenv;
           curl = bootstrapTools;
+          inherit (config) rewriteURL;
         };
 
         inherit cc;

pkgs/stdenv/freebsd/default.nix

@@ -399,6 +399,7 @@ let
       fetchurlBoot = import ../../build-support/fetchurl {
         inherit lib stdenvNoCC;
         inherit (prevStage) curl;
+        inherit (config) rewriteURL;
       };
       stdenv = import ../generic {
         inherit

pkgs/stdenv/linux/default.nix

@@ -190,6 +190,7 @@ let
 
         fetchurlBoot = import ../../build-support/fetchurl/boot.nix {
           inherit system;
+          inherit (config) rewriteURL;
         };
 
         cc =

pkgs/top-level/all-packages.nix

@@ -626,6 +626,7 @@ with pkgs;
       makeOverridable (import ../build-support/fetchurl) {
         inherit lib stdenvNoCC buildPackages;
         inherit cacert;
+        inherit (config) rewriteURL;
         curl = buildPackages.curlMinimal.override (old: rec {
           # break dependency cycles
           fetchurl = stdenv.fetchurlBoot;

pkgs/top-level/config.nix

@@ -211,6 +211,27 @@ let
         Whether to check that the `meta` attribute of derivations are correct during evaluation time.
       '';
     };
+
+    rewriteURL = mkOption {
+      type = types.functionTo (types.nullOr types.str);
+      description = ''
+        A hook to rewrite/filter URLs before they are fetched.
+
+        The function is passed the URL as a string, and is expected to return a new URL, or null if the given URL should not be attempted.
+
+        This function is applied _prior_ to resolving mirror:// URLs.
+
+        The intended use is to allow URL rewriting to insert company-internal mirrors, or work around company firewalls and similar network restrictions.
+      '';
+      default = lib.id;
+      defaultText = literalExpression "(url: url)";
+      example = literalExpression ''
+        {
+          # Use Nix like it's 2024! ;-)
+          rewriteURL = url: "https://web.archive.org/web/2024/''${url}";
+        }
+      '';
+    };
   };
 
 in