workflows/eval: diff outpaths immediately

This moves the diff of outpaths into the outpaths job, mainly as a
preparation to allow future improvements. For example, this will allow
running the purity release checks only on changed outpaths instead of
the whole eval.

This also removes the inefficiency introduced in the last commit about
uploading the intermediate paths twice. Now, only the diff is passed on.

Also, technically, the diff is now run in parallel across 4 jobs. This
should be *slightly* faster than before, where outpaths from all systems
were combined first and then diffed. It's probably only a few seconds,
though.

ci/eval/compare/default.nix

@@ -7,8 +7,7 @@
   python3,
 }:
 {
-  beforeDir,
-  afterDir,
+  combinedDir,
   touchedFilesJson,
   githubAuthorId,
   byName ? false,
@@ -66,7 +65,6 @@ let
       Example: { name = "python312Packages.numpy"; platform = "x86_64-linux"; }
   */
   inherit (import ./utils.nix { inherit lib; })
-    diff
     groupByKernel
     convertToPackagePlatformAttrs
     groupByPlatform
@@ -74,22 +72,10 @@ let
     getLabels
     ;
 
-  getAttrs =
-    dir:
-    let
-      raw = builtins.readFile "${dir}/outpaths.json";
-      # The file contains Nix paths; we need to ignore them for evaluation purposes,
-      # else there will be a "is not allowed to refer to a store path" error.
-      data = builtins.unsafeDiscardStringContext raw;
-    in
-    builtins.fromJSON data;
-  beforeAttrs = getAttrs beforeDir;
-  afterAttrs = getAttrs afterDir;
-
   # Attrs
   # - keys: "added", "changed" and "removed"
   # - values: lists of `packagePlatformPath`s
-  diffAttrs = diff beforeAttrs afterAttrs;
+  diffAttrs = builtins.fromJSON (builtins.readFile "${combinedDir}/combined-diff.json");
 
   rebuilds = diffAttrs.added ++ diffAttrs.changed;
   rebuildsPackagePlatformAttrs = convertToPackagePlatformAttrs rebuilds;
@@ -149,8 +135,8 @@ runCommand "compare"
     maintainers = builtins.toJSON maintainers;
     passAsFile = [ "maintainers" ];
     env = {
-      BEFORE_DIR = "${beforeDir}";
-      AFTER_DIR = "${afterDir}";
+      BEFORE_DIR = "${combinedDir}/before";
+      AFTER_DIR = "${combinedDir}/after";
     };
   }
   ''

ci/eval/compare/utils.nix

@@ -93,32 +93,6 @@ rec {
     in
     uniqueStrings (builtins.map (p: p.name) packagePlatformAttrs);
 
-  /*
-    Computes the key difference between two attrs
-
-    {
-      added: [ <keys only in the second object> ],
-      removed: [ <keys only in the first object> ],
-      changed: [ <keys with different values between the two objects> ],
-    }
-  */
-  diff =
-    let
-      filterKeys = cond: attrs: lib.attrNames (lib.filterAttrs cond attrs);
-    in
-    old: new: {
-      added = filterKeys (n: _: !(old ? ${n})) new;
-      removed = filterKeys (n: _: !(new ? ${n})) old;
-      changed = filterKeys (
-        n: v:
-        # Filter out attributes that don't exist anymore
-        (new ? ${n})
-
-        # Filter out attributes that are the same as the new value
-        && (v != (new.${n}))
-      ) old;
-    };
-
   /*
     Group a list of `packagePlatformAttr`s by platforms
 

ci/eval/default.nix

@@ -191,9 +191,11 @@ let
         cat "$chunkOutputDir"/result/* | jq -s 'add | map_values(.outputs)' > $out/${evalSystem}/paths.json
       '';
 
+  diff = callPackage ./diff.nix { };
+
   combine =
     {
-      evalDir,
+      diffDir,
     }:
     runCommand "combined-eval"
       {
@@ -205,12 +207,22 @@ let
         mkdir -p $out
 
         # Combine output paths from all systems
-        cat ${evalDir}/*/paths.json | jq -s add > $out/outpaths.json
+        cat ${diffDir}/*/diff.json | jq -s '
+          reduce .[] as $item ({}; {
+            added: (.added + $item.added),
+            changed: (.changed + $item.changed),
+            removed: (.removed + $item.removed)
+          })
+        ' > $out/combined-diff.json
 
-        mkdir -p $out/stats
+        mkdir -p $out/before/stats
+        for d in ${diffDir}/before/*; do
+          cp -r "$d"/stats-by-chunk $out/before/stats/$(basename "$d")
+        done
 
-        for d in ${evalDir}/*; do
-          cp -r "$d"/stats-by-chunk $out/stats/$(basename "$d")
+        mkdir -p $out/after/stats
+        for d in ${diffDir}/after/*; do
+          cp -r "$d"/stats-by-chunk $out/after/stats/$(basename "$d")
         done
       '';
 
@@ -225,18 +237,26 @@ let
       quickTest ? false,
     }:
     let
-      evals = symlinkJoin {
-        name = "evals";
+      diffs = symlinkJoin {
+        name = "diffs";
         paths = map (
           evalSystem:
-          singleSystem {
-            inherit quickTest evalSystem chunkSize;
+          let
+            eval = singleSystem {
+              inherit quickTest evalSystem chunkSize;
+            };
+          in
+          diff {
+            inherit evalSystem;
+            # Local "full" evaluation doesn't do a real diff.
+            beforeDir = eval;
+            afterDir = eval;
           }
         ) evalSystems;
       };
     in
     combine {
-      evalDir = evals;
+      diffDir = diffs;
     };
 
 in
@@ -244,6 +264,7 @@ in
   inherit
     attrpathsSuperset
     singleSystem
+    diff
     combine
     compare
     # The above three are used by separate VMs in a GitHub workflow,

ci/eval/diff.nix

@@ -0,0 +1,61 @@
+{
+  lib,
+  runCommand,
+  writeText,
+}:
+
+{
+  beforeDir,
+  afterDir,
+  evalSystem,
+}:
+
+let
+  /*
+    Computes the key difference between two attrs
+
+    {
+      added: [ <keys only in the second object> ],
+      removed: [ <keys only in the first object> ],
+      changed: [ <keys with different values between the two objects> ],
+    }
+  */
+  diff =
+    let
+      filterKeys = cond: attrs: lib.attrNames (lib.filterAttrs cond attrs);
+    in
+    old: new: {
+      added = filterKeys (n: _: !(old ? ${n})) new;
+      removed = filterKeys (n: _: !(new ? ${n})) old;
+      changed = filterKeys (
+        n: v:
+        # Filter out attributes that don't exist anymore
+        (new ? ${n})
+
+        # Filter out attributes that are the same as the new value
+        && (v != (new.${n}))
+      ) old;
+    };
+
+  getAttrs =
+    dir:
+    let
+      raw = builtins.readFile "${dir}/${evalSystem}/paths.json";
+      # The file contains Nix paths; we need to ignore them for evaluation purposes,
+      # else there will be a "is not allowed to refer to a store path" error.
+      data = builtins.unsafeDiscardStringContext raw;
+    in
+    builtins.fromJSON data;
+
+  beforeAttrs = getAttrs beforeDir;
+  afterAttrs = getAttrs afterDir;
+  diffAttrs = diff beforeAttrs afterAttrs;
+  diffJson = writeText "diff.json" (builtins.toJSON diffAttrs);
+in
+runCommand "diff" { } ''
+  mkdir -p $out/${evalSystem}
+
+  cp -r ${beforeDir} $out/before
+  cp -r ${afterDir} $out/after
+  cp ${diffJson} $out/${evalSystem}/diff.json
+''