diff --git a/ci/eval/default.nix b/ci/eval/default.nix
index fe4a8027383f..377582b96068 100644
--- a/ci/eval/default.nix
+++ b/ci/eval/default.nix
@@ -113,6 +113,9 @@ let
           echo "Evaluation failed with exit code $exitCode"
           # This immediately halts all xargs processes
           kill $PPID
+        elif [[ -s "$outputDir/stderr/$myChunk" ]]; then
+          echo "Nixpkgs on $system evaluated with warnings, aborting"
+          kill $PPID
         fi
       '';
     in