From 0344bd7f882b1c485f025239be420cb50d5b0d86 Mon Sep 17 00:00:00 2001
From: Silvan Mosberger <silvan.mosberger@tweag.io>
Date: Tue, 11 Feb 2025 21:12:31 +0100
Subject: [PATCH 1/2] ci/eval: Refactor to cleanly separate stderr

Previously stderr was mixed with the time stats

This allows checking stderr in the next commit
---
 ci/eval/default.nix | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/ci/eval/default.nix b/ci/eval/default.nix
index 927dec5a908d..fe4a8027383f 100644
--- a/ci/eval/default.nix
+++ b/ci/eval/default.nix
@@ -87,7 +87,8 @@ let
         export NIX_SHOW_STATS_PATH="$outputDir/stats/$myChunk"
         echo "Chunk $myChunk on $system start"
         set +e
-        command time -f "Chunk $myChunk on $system done [%MKB max resident, %Es elapsed] %C" \
+        command time -o "$outputDir/timestats/$myChunk" \
+          -f "Chunk $myChunk on $system done [%MKB max resident, %Es elapsed] %C" \
           nix-env -f "${nixpkgs}/pkgs/top-level/release-attrpaths-parallel.nix" \
           --option restrict-eval true \
           --option allow-import-from-derivation false \
@@ -102,9 +103,12 @@ let
           --arg includeBroken ${lib.boolToString includeBroken} \
           -I ${nixpkgs} \
           -I ${attrpathFile} \
-          > "$outputDir/result/$myChunk"
+          > "$outputDir/result/$myChunk" \
+          2> "$outputDir/stderr/$myChunk"
         exitCode=$?
         set -e
+        cat "$outputDir/stderr/$myChunk"
+        cat "$outputDir/timestats/$myChunk"
         if (( exitCode != 0 )); then
           echo "Evaluation failed with exit code $exitCode"
           # This immediately halts all xargs processes
@@ -164,7 +168,7 @@ let
         ''}
 
         chunkOutputDir=$(mktemp -d)
-        mkdir "$chunkOutputDir"/{result,stats}
+        mkdir "$chunkOutputDir"/{result,stats,timestats,stderr}
 
         seq -w 0 "$seq_end" |
           command time -f "%e" -o "$out/total-time" \

From 7c62a764b4872d6a1de0bb564d9dd8d9b1a2ac56 Mon Sep 17 00:00:00 2001
From: Silvan Mosberger <silvan.mosberger@tweag.io>
Date: Tue, 11 Feb 2025 21:16:58 +0100
Subject: [PATCH 2/2] ci/eval: Fail on non-empty stderr

Just like the channel eval requires.
---
 ci/eval/default.nix | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/ci/eval/default.nix b/ci/eval/default.nix
index fe4a8027383f..377582b96068 100644
--- a/ci/eval/default.nix
+++ b/ci/eval/default.nix
@@ -113,6 +113,9 @@ let
           echo "Evaluation failed with exit code $exitCode"
           # This immediately halts all xargs processes
           kill $PPID
+        elif [[ -s "$outputDir/stderr/$myChunk" ]]; then
+          echo "Nixpkgs on $system evaluated with warnings, aborting"
+          kill $PPID
         fi
       '';
     in