movefasta/nixpkgs
1
0
Fork 0
mirror of https://github.com/NixOS/nixpkgs.git synced 2025-06-13 21:25:30 +03:00
Code Issues Projects Releases Packages Wiki Activity

maintainers/scripts/rebuild-amount.sh: report parallelism, add example, cleanup (#137695)

Browse source 
* maintainers/scripts/rebuild-amount.sh: report parallelism setting

This slightly helps discovering parallelism support.

* maintainers/scripts/rebuild-amount.sh: add basic example

It's not clear if script should be used against HEAD or HEAD^.
Let's have a copyable example.

* maintainers/scripts/rebuild-amount.sh: convert tabs to four spaces

One notable change is here-document conversion.
Before:

  <TAB>cat <<-FOO
  <TAB>...
  <TAB>FOO

After:

  <sp><sp><sp><sp>cat <<FOO
  <sp><sp><sp><sp>...
  FOO

Note seemingly misaligned 'FOO': '-FOO' understands leading tabs,
but not spaces.
This commit is contained in:
Sergei Trofimovich 2021-09-24 08:06:57 +00:00 • committed by GitHub
parent 1922e87924
commit 39e8ec2db6
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 77 additions and 70 deletions
Download patch file Download diff file Expand all files Collapse all files

147
maintainers/scripts/rebuild-amount.sh
View file

@ -4,123 +4,130 @@ set -e
# --print: avoid dependency on environment # --print: avoid dependency on environment
optPrint= optPrint=
if [ "$1" == "--print" ]; then if [ "$1" == "--print" ]; then
optPrint=true optPrint=true
shift shift
fi fi
if [ "$#" != 1 ] && [ "$#" != 2 ]; then if [ "$#" != 1 ] && [ "$#" != 2 ]; then
cat <<-EOF cat <<EOF
Usage: $0 [--print] commit-spec [commit-spec] Usage: $0 [--print] from-commit-spec [to-commit-spec]
You need to be in a git-controlled nixpkgs tree. You need to be in a git-controlled nixpkgs tree.
The current state of the tree will be used if the second commit is missing. The current state of the tree will be used if the second commit is missing.
EOF
exit 1 Examples:
effect of latest commit:
$ $0 HEAD^
$ $0 --print HEAD^
effect of the whole patch series for 'staging' branch:
$ $0 origin/staging staging
EOF
exit 1
fi fi
# A slightly hacky way to get the config. # A slightly hacky way to get the config.
parallel="$(echo 'config.rebuild-amount.parallel or false' | nix-repl . 2>/dev/null \ parallel="$(echo 'config.rebuild-amount.parallel or false' | nix-repl . 2>/dev/null \
| grep -v '^\(nix-repl.*\)\?$' | tail -n 1 || true)" | grep -v '^\(nix-repl.*\)\?$' | tail -n 1 || true)"
echo "Estimating rebuild amount by counting changed Hydra jobs." echo "Estimating rebuild amount by counting changed Hydra jobs (parallel=${parallel:-unset})."
toRemove=() toRemove=()
cleanup() { cleanup() {
rm -rf "${toRemove[@]}" rm -rf "${toRemove[@]}"
} }
trap cleanup EXIT SIGINT SIGQUIT ERR trap cleanup EXIT SIGINT SIGQUIT ERR
MKTEMP='mktemp --tmpdir nix-rebuild-amount-XXXXXXXX' MKTEMP='mktemp --tmpdir nix-rebuild-amount-XXXXXXXX'
nixexpr() { nixexpr() {
cat <<-EONIX cat <<EONIX
let let
lib = import $1/lib; lib = import $1/lib;
hydraJobs = import $1/pkgs/top-level/release.nix hydraJobs = import $1/pkgs/top-level/release.nix
# Compromise: accuracy vs. resources needed for evaluation. # Compromise: accuracy vs. resources needed for evaluation.
{ supportedSystems = cfg.systems or [ "x86_64-linux" "x86_64-darwin" ]; }; { supportedSystems = cfg.systems or [ "x86_64-linux" "x86_64-darwin" ]; };
cfg = (import $1 {}).config.rebuild-amount or {}; cfg = (import $1 {}).config.rebuild-amount or {};
recurseIntoAttrs = attrs: attrs // { recurseForDerivations = true; }; recurseIntoAttrs = attrs: attrs // { recurseForDerivations = true; };
# hydraJobs leaves recurseForDerivations as empty attrmaps; # hydraJobs leaves recurseForDerivations as empty attrmaps;
# that would break nix-env and we also need to recurse everywhere. # that would break nix-env and we also need to recurse everywhere.
tweak = lib.mapAttrs tweak = lib.mapAttrs
(name: val: (name: val:
if name == "recurseForDerivations" then true if name == "recurseForDerivations" then true
else if lib.isAttrs val && val.type or null != "derivation" else if lib.isAttrs val && val.type or null != "derivation"
then recurseIntoAttrs (tweak val) then recurseIntoAttrs (tweak val)
else val else val
); );
# Some of these contain explicit references to platform(s) we want to avoid; # Some of these contain explicit references to platform(s) we want to avoid;
# some even (transitively) depend on ~/.nixpkgs/config.nix (!) # some even (transitively) depend on ~/.nixpkgs/config.nix (!)
blacklist = [ blacklist = [
"tarball" "metrics" "manual" "tarball" "metrics" "manual"
"darwin-tested" "unstable" "stdenvBootstrapTools" "darwin-tested" "unstable" "stdenvBootstrapTools"
"moduleSystem" "lib-tests" # these just confuse the output "moduleSystem" "lib-tests" # these just confuse the output
]; ];
in in
tweak (builtins.removeAttrs hydraJobs blacklist) tweak (builtins.removeAttrs hydraJobs blacklist)
EONIX EONIX
} }
# Output packages in tree $2 that weren't in $1. # Output packages in tree $2 that weren't in $1.
# Changing the output hash or name is taken as a change. # Changing the output hash or name is taken as a change.
# Extra nix-env parameters can be in $3 # Extra nix-env parameters can be in $3
newPkgs() { newPkgs() {
# We use files instead of pipes, as running multiple nix-env processes # We use files instead of pipes, as running multiple nix-env processes
# could eat too much memory for a standard 4GiB machine. # could eat too much memory for a standard 4GiB machine.
local -a list local -a list
for i in 1 2; do for i in 1 2; do
local l="$($MKTEMP)" local l="$($MKTEMP)"
list[$i]="$l" list[$i]="$l"
toRemove+=("$l") toRemove+=("$l")
local expr="$($MKTEMP)" local expr="$($MKTEMP)"
toRemove+=("$expr") toRemove+=("$expr")
nixexpr "${!i}" > "$expr" nixexpr "${!i}" > "$expr"
nix-env -f "$expr" -qaP --no-name --out-path --show-trace $3 \ nix-env -f "$expr" -qaP --no-name --out-path --show-trace $3 \
| sort > "${list[$i]}" & | sort > "${list[$i]}" &
if [ "$parallel" != "true" ]; then if [ "$parallel" != "true" ]; then
wait wait
fi fi
done done
wait wait
comm -13 "${list[@]}" comm -13 "${list[@]}"
} }
# Prepare nixpkgs trees. # Prepare nixpkgs trees.
declare -a tree declare -a tree
for i in 1 2; do for i in 1 2; do
if [ -n "${!i}" ]; then # use the given commit if [ -n "${!i}" ]; then # use the given commit
dir="$($MKTEMP -d)" dir="$($MKTEMP -d)"
tree[$i]="$dir" tree[$i]="$dir"
toRemove+=("$dir") toRemove+=("$dir")
git clone --shared --no-checkout --quiet . "${tree[$i]}" git clone --shared --no-checkout --quiet . "${tree[$i]}"
(cd "${tree[$i]}" && git checkout --quiet "${!i}") (cd "${tree[$i]}" && git checkout --quiet "${!i}")
else #use the current tree else #use the current tree
tree[$i]="$(pwd)" tree[$i]="$(pwd)"
fi fi
done done
newlist="$($MKTEMP)" newlist="$($MKTEMP)"
toRemove+=("$newlist") toRemove+=("$newlist")
# Notes: # Notes:
# - the evaluation is done on x86_64-linux, like on Hydra. # - the evaluation is done on x86_64-linux, like on Hydra.
# - using $newlist file so that newPkgs() isn't in a sub-shell (because of toRemove) # - using $newlist file so that newPkgs() isn't in a sub-shell (because of toRemove)
newPkgs "${tree[1]}" "${tree[2]}" '--argstr system "x86_64-linux"' > "$newlist" newPkgs "${tree[1]}" "${tree[2]}" '--argstr system "x86_64-linux"' > "$newlist"
# Hacky: keep only the last word of each attribute path and sort. # Hacky: keep only the last word of each attribute path and sort.
sed -n 's/\([^. ]*\.\)*\([^. ]*\) .*$/\2/p' < "$newlist" \ sed -n 's/\([^. ]*\.\)*\([^. ]*\) .*$/\2/p' < "$newlist" \
| sort | uniq -c | sort | uniq -c
if [ -n "$optPrint" ]; then if [ -n "$optPrint" ]; then
echo echo
cat "$newlist" cat "$newlist"
fi fi