diff --git a/lib/default.nix b/lib/default.nix index 6f8530296044..d45e6de0c33f 100644 --- a/lib/default.nix +++ b/lib/default.nix @@ -433,6 +433,8 @@ let pathHasContext canCleanSource pathIsGitRepo + revOrTag + repoRevToName ; inherit (self.modules) evalModules diff --git a/lib/sources.nix b/lib/sources.nix index e165439998a1..47376d0d0369 100644 --- a/lib/sources.nix +++ b/lib/sources.nix @@ -3,7 +3,7 @@ # Tested in lib/tests/sources.sh let - inherit (builtins) + inherit (lib.strings) match split storeDir @@ -403,6 +403,101 @@ let }; }; + # urlToName : (URL | Path | String) -> String + # + # Transform a URL (or path, or string) into a clean package name. + urlToName = + url: + let + inherit (lib.strings) stringLength; + base = baseNameOf (lib.removeSuffix "/" (lib.last (lib.splitString ":" (toString url)))); + # chop away one git or archive-related extension + removeExt = + name: + let + matchExt = match "(.*)\\.(git|tar|zip|gz|tgz|bz|tbz|bz2|tbz2|lzma|txz|xz|zstd)$" name; + in + if matchExt != null then lib.head matchExt else name; + # apply function f to string x while the result shrinks + shrink = + f: x: + let + v = f x; + in + if stringLength v < stringLength x then shrink f v else x; + in + shrink removeExt base; + + # shortRev : (String | Integer) -> String + # + # Given a package revision (like "refs/tags/v12.0"), produce a short revision ("12.0"). + shortRev = + rev: + let + baseRev = baseNameOf (toString rev); + matchHash = match "[a-f0-9]+" baseRev; + matchVer = match "([A-Za-z]+[-_. ]?)*(v)?([0-9.]+.*)" baseRev; + in + if matchHash != null then + builtins.substring 0 7 baseRev + else if matchVer != null then + lib.last matchVer + else + baseRev; + + # revOrTag : String -> String -> String + # + # Turn git `rev` and `tag` pair into a revision usable in `repoRevToName*`. + revOrTag = + rev: tag: + if tag != null then + tag + else if rev != null then + rev + else + "HEAD"; + + # repoRevToNameFull : (URL | Path | String) -> (String | Integer | null) -> (String | null) -> String + # + # See `repoRevToName` below. + repoRevToNameFull = + repo_: rev_: suffix_: + let + repo = urlToName repo_; + rev = if rev_ != null then "-${shortRev rev_}" else ""; + suffix = if suffix_ != null then "-${suffix_}" else ""; + in + "${repo}${rev}${suffix}-source"; + + # repoRevToName : String -> (URL | Path | String) -> (String | Integer | null) -> String -> String + # + # Produce derivation.name attribute for a given repository URL/path/name and (optionally) its revision/version tag. + # + # This is used by fetch(zip|git|FromGitHub|hg|svn|etc) to generate discoverable + # /nix/store paths. + # + # This uses a different implementation depending on the `pretty` argument: + # "source" -> name everything as "source" + # "versioned" -> name everything as "${repo}-${rev}-source" + # "full" -> name everything as "${repo}-${rev}-${fetcher}-source" + repoRevToName = + kind: + # match on `kind` first to minimize the thunk + if kind == "source" then + ( + repo: rev: suffix: + "source" + ) + else if kind == "versioned" then + ( + repo: rev: suffix: + repoRevToNameFull repo rev null + ) + else if kind == "full" then + repoRevToNameFull + else + throw "repoRevToName: invalid kind"; + in { @@ -431,6 +526,11 @@ in pathHasContext canCleanSource + urlToName + shortRev + revOrTag + repoRevToName + sourceByRegex sourceFilesBySuffices diff --git a/pkgs/build-support/fetchbitbucket/default.nix b/pkgs/build-support/fetchbitbucket/default.nix index 9e64954bebff..b98fc21de7ba 100644 --- a/pkgs/build-support/fetchbitbucket/default.nix +++ b/pkgs/build-support/fetchbitbucket/default.nix @@ -1,11 +1,15 @@ -{ fetchzip, lib }: +{ + lib, + repoRevToNameMaybe, + fetchzip, +}: lib.makeOverridable ( { owner, repo, rev, - name ? "source", + name ? repoRevToNameMaybe repo rev "bitbucket", ... # For hash agility }@args: fetchzip ( diff --git a/pkgs/build-support/fetchgit/default.nix b/pkgs/build-support/fetchgit/default.nix index 9fd433ff6d66..374d55b0456c 100644 --- a/pkgs/build-support/fetchgit/default.nix +++ b/pkgs/build-support/fetchgit/default.nix @@ -5,29 +5,26 @@ git-lfs, cacert, }: + let urlToName = url: rev: let - inherit (lib) removeSuffix splitString last; - base = last (splitString ":" (baseNameOf (removeSuffix "/" url))); - - matched = builtins.match "(.*)\\.git" base; - - short = builtins.substring 0 7 rev; - - appendShort = lib.optionalString ((builtins.match "[a-f0-9]*" rev) != null) "-${short}"; + shortRev = lib.sources.shortRev rev; + appendShort = lib.optionalString ((builtins.match "[a-f0-9]*" rev) != null) "-${shortRev}"; in - "${if matched == null then base else builtins.head matched}${appendShort}"; + "${lib.sources.urlToName url}${appendShort}"; in + lib.makeOverridable ( lib.fetchers.withNormalizedHash { } ( # NOTE Please document parameter additions or changes in - # doc/build-helpers/fetchers.chapter.md + # ../../../doc/build-helpers/fetchers.chapter.md { url, tag ? null, rev ? null, + name ? urlToName url (lib.revOrTag rev tag), leaveDotGit ? deepClone || fetchTags, outputHash ? lib.fakeHash, outputHashAlgo ? null, @@ -36,7 +33,6 @@ lib.makeOverridable ( branchName ? null, sparseCheckout ? [ ], nonConeMode ? false, - name ? null, nativeBuildInputs ? [ ], # Shell code executed before the file has been fetched. This, in # particular, can do things like set NIX_PREFETCH_GIT_CHECKOUT_HOOK to @@ -108,7 +104,7 @@ lib.makeOverridable ( "Please provide directories/patterns for sparse checkout as a list of strings. Passing a (multi-line) string is not supported any more." else stdenvNoCC.mkDerivation { - name = if name != null then name else urlToName url revWithTag; + inherit name; builder = ./builder.sh; fetcher = ./nix-prefetch-git; diff --git a/pkgs/build-support/fetchgithub/default.nix b/pkgs/build-support/fetchgithub/default.nix index be387e0fc484..2b3ab060418a 100644 --- a/pkgs/build-support/fetchgithub/default.nix +++ b/pkgs/build-support/fetchgithub/default.nix @@ -1,5 +1,6 @@ { lib, + repoRevToNameMaybe, fetchgit, fetchzip, }: @@ -10,7 +11,7 @@ lib.makeOverridable ( repo, tag ? null, rev ? null, - name ? "source", + name ? repoRevToNameMaybe repo (lib.revOrTag rev tag) "github", fetchSubmodules ? false, leaveDotGit ? null, deepClone ? false, diff --git a/pkgs/build-support/fetchgitiles/default.nix b/pkgs/build-support/fetchgitiles/default.nix index 409f2f4e2bd6..af30fa3da36a 100644 --- a/pkgs/build-support/fetchgitiles/default.nix +++ b/pkgs/build-support/fetchgitiles/default.nix @@ -1,11 +1,15 @@ -{ fetchzip, lib }: +{ + fetchzip, + repoRevToNameMaybe, + lib, +}: lib.makeOverridable ( { url, rev ? null, tag ? null, - name ? "source", + name ? repoRevToNameMaybe url (lib.revOrTag rev tag) "gitiles", ... }@args: diff --git a/pkgs/build-support/fetchgitlab/default.nix b/pkgs/build-support/fetchgitlab/default.nix index c060352d1a9f..822c3991fa29 100644 --- a/pkgs/build-support/fetchgitlab/default.nix +++ b/pkgs/build-support/fetchgitlab/default.nix @@ -1,5 +1,6 @@ { lib, + repoRevToNameMaybe, fetchgit, fetchzip, }: @@ -11,9 +12,9 @@ lib.makeOverridable ( repo, rev ? null, tag ? null, + name ? repoRevToNameMaybe repo (lib.revOrTag rev tag) "gitlab", protocol ? "https", domain ? "gitlab.com", - name ? "source", group ? null, fetchSubmodules ? false, leaveDotGit ? false, diff --git a/pkgs/build-support/fetchrepoorcz/default.nix b/pkgs/build-support/fetchrepoorcz/default.nix index bfa0ff4fa31a..83fd043e7088 100644 --- a/pkgs/build-support/fetchrepoorcz/default.nix +++ b/pkgs/build-support/fetchrepoorcz/default.nix @@ -1,10 +1,14 @@ -{ fetchzip }: +{ + lib, + repoRevToNameMaybe, + fetchzip, +}: # gitweb example, snapshot support is optional in gitweb { repo, rev, - name ? "source", + name ? repoRevToNameMaybe repo rev "repoorcz", ... # For hash agility }@args: fetchzip ( diff --git a/pkgs/build-support/fetchsavannah/default.nix b/pkgs/build-support/fetchsavannah/default.nix index e6828311c22f..a9d172382b8a 100644 --- a/pkgs/build-support/fetchsavannah/default.nix +++ b/pkgs/build-support/fetchsavannah/default.nix @@ -1,11 +1,15 @@ -{ fetchzip, lib }: +{ + lib, + repoRevToNameMaybe, + fetchzip, +}: lib.makeOverridable ( # cgit example, snapshot support is optional in cgit { repo, rev, - name ? "source", + name ? repoRevToNameMaybe repo rev "savannah", ... # For hash agility }@args: fetchzip ( diff --git a/pkgs/build-support/fetchsourcehut/default.nix b/pkgs/build-support/fetchsourcehut/default.nix index a8f645c351b3..fb36805733b8 100644 --- a/pkgs/build-support/fetchsourcehut/default.nix +++ b/pkgs/build-support/fetchsourcehut/default.nix @@ -1,8 +1,9 @@ { + lib, + repoRevToNameMaybe, fetchgit, fetchhg, fetchzip, - lib, }: let @@ -18,9 +19,9 @@ makeOverridable ( owner, repo, rev, + name ? repoRevToNameMaybe repo rev "sourcehut", domain ? "sr.ht", vc ? "git", - name ? "source", fetchSubmodules ? false, ... # For hash agility }@args: diff --git a/pkgs/build-support/fetchzip/default.nix b/pkgs/build-support/fetchzip/default.nix index dfd632192586..50be0c38f1e5 100644 --- a/pkgs/build-support/fetchzip/default.nix +++ b/pkgs/build-support/fetchzip/default.nix @@ -7,6 +7,7 @@ { lib, + repoRevToNameMaybe, fetchurl, withUnzip ? true, unzip, @@ -14,9 +15,9 @@ }: { - name ? "source", url ? "", urls ? [ ], + name ? repoRevToNameMaybe (if url != "" then url else builtins.head urls) null "unpacked", nativeBuildInputs ? [ ], postFetch ? "", extraPostFetch ? "", diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 32b032e90a1c..bea1ddd8acc8 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -574,6 +574,9 @@ with pkgs; stdenv = if stdenv.hostPlatform.isDarwin then llvmPackages_18.stdenv else stdenv; }; + # this is used by most `fetch*` functions + repoRevToNameMaybe = lib.repoRevToName config.fetchedSourceNameDefault; + fetchpatch = callPackage ../build-support/fetchpatch { # 0.3.4 would change hashes: https://github.com/NixOS/nixpkgs/issues/25154 diff --git a/pkgs/top-level/config.nix b/pkgs/top-level/config.nix index 9b2ae12f595b..6122425266d6 100644 --- a/pkgs/top-level/config.nix +++ b/pkgs/top-level/config.nix @@ -59,6 +59,52 @@ let default = false; }; + fetchedSourceNameDefault = mkOption { + type = types.uniq ( + types.enum [ + "source" + "versioned" + "full" + ] + ); + default = "source"; + description = '' + This controls the default derivation `name` attribute set by the + `fetch*` (`fetchzip`, `fetchFromGitHub`, etc) functions. + + Possible values and the resulting `.name`: + + - `"source"` -> `"source"` + - `"versioned"` -> `"''${repo}-''${rev}-source"` + - `"full"` -> `"''${repo}-''${rev}-''${fetcherName}-source"` + + The default `"source"` is the best choice for minimal rebuilds, it + will ignore any non-hash changes (like branches being renamed, source + URLs changing, etc) at the cost of `/nix/store` being easily + cache-poisoned (see [NixOS/nix#969](https://github.com/NixOS/nix/issues/969)). + + Setting this to `"versioned"` greatly helps with discoverability of + sources in `/nix/store` and makes cache-poisoning of `/nix/store` much + harder, at the cost of a single mass-rebuild for all `src` + derivations, and an occasional rebuild when a source changes some of + its non-hash attributes. + + Setting this to `"full"` is similar to setting it to `"versioned"`, + but the use of `fetcherName` in the derivation name will force a + rebuild when `src` switches between `fetch*` functions, thus forcing + `nix` to check new derivation's `outputHash`, which is useful for + debugging. + + Also, `"full"` is useful for easy collection and tracking of + statistics of where the packages you use are hosted. + + If you are a developer, you should probably set this to at + least`"versioned"`. + + Changing the default will cause a mass rebuild. + ''; + }; + doCheckByDefault = mkMassRebuild { feature = "run `checkPhase` by default"; };