movefasta/nixpkgs
1
0
Fork 0
mirror of https://github.com/NixOS/nixpkgs.git synced 2025-06-11 20:25:32 +03:00
Code Issues Projects Releases Packages Wiki Activity

Rename scrubDrv -> hydraJob and make it more effective

Browse source 
It now strictly evaluates all remaining attributes, preventing
unevaluated thunks that cannot be garbage-collected. It's also applied
to all jobs in Nixpkgs' release.nix.

This reduces hydra-eval-jobs' memory consumption on the 14.12
release-combined jobset from 5.1 GB to 2.0 GB.
This commit is contained in:
Eelco Dolstra 2015-03-20 19:23:55 +01:00
parent 738cf42639
commit 0461f35894
5 changed files with 73 additions and 40 deletions
Download patch file Download diff file Expand all files Collapse all files

35
lib/customisation.nix
View file

@ -105,6 +105,7 @@ rec {
let f = if builtins.isFunction fn then fn else import fn; in let f = if builtins.isFunction fn then fn else import fn; in
makeOverridable f ((builtins.intersectAttrs (builtins.functionArgs f) autoArgs) // args); makeOverridable f ((builtins.intersectAttrs (builtins.functionArgs f) autoArgs) // args);
/* Add attributes to each output of a derivation without changing the derivation itself */ /* Add attributes to each output of a derivation without changing the derivation itself */
addPassthru = drv: passthru: addPassthru = drv: passthru:
let let
@ -122,4 +123,38 @@ rec {
outputsList = map outputToAttrListElement outputs; outputsList = map outputToAttrListElement outputs;
in commonAttrs.${drv.outputName}; in commonAttrs.${drv.outputName};
/* Strip a derivation of all non-essential attributes, returning
only those needed by hydra-eval-jobs. Also strictly evaluate the
result to ensure that there are no thunks kept alive to prevent
garbage collection. */
hydraJob = drv:
let
outputs = drv.outputs or ["out"];
commonAttrs =
{ inherit (drv) name system meta; inherit outputs; }
// lib.optionalAttrs (drv._hydraAggregate or false) {
_hydraAggregate = true;
constituents = map hydraJob (lib.flatten drv.constituents);
}
// (lib.listToAttrs outputsList);
makeOutput = outputName:
let output = drv.${outputName}; in
{ name = outputName;
value = commonAttrs // {
outPath = output.outPath;
drvPath = output.drvPath;
type = "derivation";
inherit outputName;
};
};
outputsList = map makeOutput outputs;
drv' = (lib.head outputsList).value;
in lib.deepSeq drv' drv';
} }

4
nixos/release-combined.nix
View file

@ -31,7 +31,7 @@ in rec {
nixpkgs = nixpkgsSrc; nixpkgs = nixpkgsSrc;
})) [ "unstable" ]; })) [ "unstable" ];
tested = pkgs.releaseTools.aggregate { tested = pkgs.lib.hydraJob (pkgs.releaseTools.aggregate {
name = "nixos-${nixos.channel.version}"; name = "nixos-${nixos.channel.version}";
meta = { meta = {
description = "Release-critical builds for the NixOS channel"; description = "Release-critical builds for the NixOS channel";
@ -85,6 +85,6 @@ in rec {
nixpkgs.tarball nixpkgs.tarball
(all nixpkgs.emacs) (all nixpkgs.emacs)
]; ];
}; });
} }

4
nixos/release-small.nix
View file

@ -79,7 +79,7 @@ in rec {
vim; vim;
}; };
tested = pkgs.releaseTools.aggregate { tested = lib.hydraJob (pkgs.releaseTools.aggregate {
name = "nixos-${nixos.channel.version}"; name = "nixos-${nixos.channel.version}";
meta = { meta = {
description = "Release-critical builds for the NixOS channel"; description = "Release-critical builds for the NixOS channel";
@ -88,6 +88,6 @@ in rec {
constituents = constituents =
let all = x: map (system: x.${system}) supportedSystems; in let all = x: map (system: x.${system}) supportedSystems; in
[ nixpkgs.tarball ] ++ lib.collect lib.isDerivation nixos; [ nixpkgs.tarball ] ++ lib.collect lib.isDerivation nixos;
}; });
} }

68
nixos/release.nix
View file

@ -3,22 +3,20 @@
, supportedSystems ? [ "x86_64-linux" "i686-linux" ] , supportedSystems ? [ "x86_64-linux" "i686-linux" ]
}: }:
with import ../lib;
let let
version = builtins.readFile ../.version; version = builtins.readFile ../.version;
versionSuffix = versionSuffix =
(if stableBranch then "." else "pre") + "${toString nixpkgs.revCount}.${nixpkgs.shortRev}"; (if stableBranch then "." else "pre") + "${toString nixpkgs.revCount}.${nixpkgs.shortRev}";
forAllSystems = pkgs.lib.genAttrs supportedSystems; forAllSystems = genAttrs supportedSystems;
scrubDrv = drv: let res = { inherit (drv) drvPath outPath type name system meta; outputName = "out"; out = res; }; in res; callTest = fn: args: forAllSystems (system: hydraJob (import fn ({ inherit system; } // args)));
callTest = fn: args: forAllSystems (system: scrubDrv (import fn ({ inherit system; } // args)));
pkgs = import nixpkgs { system = "x86_64-linux"; }; pkgs = import nixpkgs { system = "x86_64-linux"; };
lib = pkgs.lib;
versionModule = versionModule =
{ system.nixosVersionSuffix = versionSuffix; { system.nixosVersionSuffix = versionSuffix;
@ -42,10 +40,10 @@ let
in in
# Declare the ISO as a build product so that it shows up in Hydra. # Declare the ISO as a build product so that it shows up in Hydra.
scrubDrv (runCommand "nixos-iso-${config.system.nixosVersion}" hydraJob (runCommand "nixos-iso-${config.system.nixosVersion}"
{ meta = { { meta = {
description = "NixOS installation CD (${description}) - ISO image for ${system}"; description = "NixOS installation CD (${description}) - ISO image for ${system}";
maintainers = map (x: lib.getAttr x lib.maintainers) maintainers; maintainers = map (x: lib.maintainers.${x}) maintainers;
}; };
inherit iso; inherit iso;
passthru = { inherit config; }; passthru = { inherit config; };
@ -74,7 +72,7 @@ let
tarball // tarball //
{ meta = { { meta = {
description = "NixOS system tarball for ${system} - ${stdenv.platform.name}"; description = "NixOS system tarball for ${system} - ${stdenv.platform.name}";
maintainers = map (x: lib.getAttr x lib.maintainers) maintainers; maintainers = map (x: lib.maintainers.${x}) maintainers;
}; };
inherit config; inherit config;
}; };
@ -83,12 +81,12 @@ let
makeClosure = module: buildFromConfig module (config: config.system.build.toplevel); makeClosure = module: buildFromConfig module (config: config.system.build.toplevel);
buildFromConfig = module: sel: forAllSystems (system: scrubDrv (sel (import ./lib/eval-config.nix { buildFromConfig = module: sel: forAllSystems (system: hydraJob (sel (import ./lib/eval-config.nix {
inherit system; inherit system;
modules = [ module versionModule ] ++ lib.singleton modules = [ module versionModule ] ++ singleton
({ config, lib, ... }: ({ config, lib, ... }:
{ fileSystems."/".device = lib.mkDefault "/dev/sda1"; { fileSystems."/".device = mkDefault "/dev/sda1";
boot.loader.grub.device = lib.mkDefault "/dev/sda"; boot.loader.grub.device = mkDefault "/dev/sda";
}); });
}).config)); }).config));
@ -175,10 +173,10 @@ in rec {
in in
# Declare the OVA as a build product so that it shows up in Hydra. # Declare the OVA as a build product so that it shows up in Hydra.
scrubDrv (runCommand "nixos-ova-${config.system.nixosVersion}-${system}" hydraJob (runCommand "nixos-ova-${config.system.nixosVersion}-${system}"
{ meta = { { meta = {
description = "NixOS VirtualBox appliance (${system})"; description = "NixOS VirtualBox appliance (${system})";
maintainers = lib.maintainers.eelco; maintainers = maintainers.eelco;
}; };
ova = config.system.build.virtualBoxOVA; ova = config.system.build.virtualBoxOVA;
} }
@ -195,9 +193,9 @@ in rec {
dummy = forAllSystems (system: pkgs.runCommand "dummy" dummy = forAllSystems (system: pkgs.runCommand "dummy"
{ toplevel = (import lib/eval-config.nix { { toplevel = (import lib/eval-config.nix {
inherit system; inherit system;
modules = lib.singleton ({ config, pkgs, ... }: modules = singleton ({ config, pkgs, ... }:
{ fileSystems."/".device = lib.mkDefault "/dev/sda1"; { fileSystems."/".device = mkDefault "/dev/sda1";
boot.loader.grub.device = lib.mkDefault "/dev/sda"; boot.loader.grub.device = mkDefault "/dev/sda";
}); });
}).config.system.build.toplevel; }).config.system.build.toplevel;
} }
@ -242,34 +240,34 @@ in rec {
tests.avahi = callTest tests/avahi.nix {}; tests.avahi = callTest tests/avahi.nix {};
tests.bittorrent = callTest tests/bittorrent.nix {}; tests.bittorrent = callTest tests/bittorrent.nix {};
tests.blivet = callTest tests/blivet.nix {}; tests.blivet = callTest tests/blivet.nix {};
tests.cadvisor = scrubDrv (import tests/cadvisor.nix { system = "x86_64-linux"; }); tests.cadvisor = hydraJob (import tests/cadvisor.nix { system = "x86_64-linux"; });
tests.chromium = callTest tests/chromium.nix {}; tests.chromium = callTest tests/chromium.nix {};
#tests.cjdns = callTest tests/cjdns.nix {}; #tests.cjdns = callTest tests/cjdns.nix {};
tests.containers = callTest tests/containers.nix {}; tests.containers = callTest tests/containers.nix {};
tests.docker = scrubDrv (import tests/docker.nix { system = "x86_64-linux"; }); tests.docker = hydraJob (import tests/docker.nix { system = "x86_64-linux"; });
tests.dockerRegistry = scrubDrv (import tests/docker-registry.nix { system = "x86_64-linux"; }); tests.dockerRegistry = hydraJob (import tests/docker-registry.nix { system = "x86_64-linux"; });
tests.etcd = scrubDrv (import tests/etcd.nix { system = "x86_64-linux"; }); tests.etcd = hydraJob (import tests/etcd.nix { system = "x86_64-linux"; });
tests.firefox = callTest tests/firefox.nix {}; tests.firefox = callTest tests/firefox.nix {};
tests.firewall = callTest tests/firewall.nix {}; tests.firewall = callTest tests/firewall.nix {};
tests.fleet = scrubDrv (import tests/fleet.nix { system = "x86_64-linux"; }); tests.fleet = hydraJob (import tests/fleet.nix { system = "x86_64-linux"; });
#tests.gitlab = callTest tests/gitlab.nix {}; #tests.gitlab = callTest tests/gitlab.nix {};
tests.gnome3 = callTest tests/gnome3.nix {}; tests.gnome3 = callTest tests/gnome3.nix {};
tests.i3wm = callTest tests/i3wm.nix {}; tests.i3wm = callTest tests/i3wm.nix {};
tests.installer.grub1 = forAllSystems (system: scrubDrv (import tests/installer.nix { inherit system; }).grub1.test); tests.installer.grub1 = forAllSystems (system: hydraJob (import tests/installer.nix { inherit system; }).grub1.test);
tests.installer.lvm = forAllSystems (system: scrubDrv (import tests/installer.nix { inherit system; }).lvm.test); tests.installer.lvm = forAllSystems (system: hydraJob (import tests/installer.nix { inherit system; }).lvm.test);
tests.installer.rebuildCD = forAllSystems (system: scrubDrv (import tests/installer.nix { inherit system; }).rebuildCD.test); tests.installer.rebuildCD = forAllSystems (system: hydraJob (import tests/installer.nix { inherit system; }).rebuildCD.test);
tests.installer.separateBoot = forAllSystems (system: scrubDrv (import tests/installer.nix { inherit system; }).separateBoot.test); tests.installer.separateBoot = forAllSystems (system: hydraJob (import tests/installer.nix { inherit system; }).separateBoot.test);
tests.installer.simple = forAllSystems (system: scrubDrv (import tests/installer.nix { inherit system; }).simple.test); tests.installer.simple = forAllSystems (system: hydraJob (import tests/installer.nix { inherit system; }).simple.test);
tests.installer.simpleLabels = forAllSystems (system: scrubDrv (import tests/installer.nix { inherit system; }).simpleLabels.test); tests.installer.simpleLabels = forAllSystems (system: hydraJob (import tests/installer.nix { inherit system; }).simpleLabels.test);
tests.installer.simpleProvided = forAllSystems (system: scrubDrv (import tests/installer.nix { inherit system; }).simpleProvided.test); tests.installer.simpleProvided = forAllSystems (system: hydraJob (import tests/installer.nix { inherit system; }).simpleProvided.test);
tests.installer.btrfsSimple = forAllSystems (system: scrubDrv (import tests/installer.nix { inherit system; }).btrfsSimple.test); tests.installer.btrfsSimple = forAllSystems (system: hydraJob (import tests/installer.nix { inherit system; }).btrfsSimple.test);
tests.installer.btrfsSubvols = forAllSystems (system: scrubDrv (import tests/installer.nix { inherit system; }).btrfsSubvols.test); tests.installer.btrfsSubvols = forAllSystems (system: hydraJob (import tests/installer.nix { inherit system; }).btrfsSubvols.test);
tests.installer.btrfsSubvolDefault = forAllSystems (system: scrubDrv (import tests/installer.nix { inherit system; }).btrfsSubvolDefault.test); tests.installer.btrfsSubvolDefault = forAllSystems (system: hydraJob (import tests/installer.nix { inherit system; }).btrfsSubvolDefault.test);
tests.influxdb = callTest tests/influxdb.nix {}; tests.influxdb = callTest tests/influxdb.nix {};
tests.ipv6 = callTest tests/ipv6.nix {}; tests.ipv6 = callTest tests/ipv6.nix {};
tests.jenkins = callTest tests/jenkins.nix {}; tests.jenkins = callTest tests/jenkins.nix {};
tests.kde4 = callTest tests/kde4.nix {}; tests.kde4 = callTest tests/kde4.nix {};
tests.kubernetes = scrubDrv (import tests/kubernetes.nix { system = "x86_64-linux"; }); tests.kubernetes = hydraJob (import tests/kubernetes.nix { system = "x86_64-linux"; });
tests.latestKernel.login = callTest tests/login.nix { latestKernel = true; }; tests.latestKernel.login = callTest tests/login.nix { latestKernel = true; };
tests.login = callTest tests/login.nix {}; tests.login = callTest tests/login.nix {};
#tests.logstash = callTest tests/logstash.nix {}; #tests.logstash = callTest tests/logstash.nix {};
@ -301,7 +299,7 @@ in rec {
tests.nfs3 = callTest tests/nfs.nix { version = 3; }; tests.nfs3 = callTest tests/nfs.nix { version = 3; };
tests.nsd = callTest tests/nsd.nix {}; tests.nsd = callTest tests/nsd.nix {};
tests.openssh = callTest tests/openssh.nix {}; tests.openssh = callTest tests/openssh.nix {};
tests.panamax = scrubDrv (import tests/panamax.nix { system = "x86_64-linux"; }); tests.panamax = hydraJob (import tests/panamax.nix { system = "x86_64-linux"; });
tests.peerflix = callTest tests/peerflix.nix {}; tests.peerflix = callTest tests/peerflix.nix {};
tests.printing = callTest tests/printing.nix {}; tests.printing = callTest tests/printing.nix {};
tests.proxy = callTest tests/proxy.nix {}; tests.proxy = callTest tests/proxy.nix {};

2
pkgs/top-level/release-lib.nix
View file

@ -44,7 +44,7 @@ rec {
a derivation for each supported platform, i.e. ‘{ x86_64-linux = a derivation for each supported platform, i.e. ‘{ x86_64-linux =
f pkgs_x86_64_linux; i686-linux = f pkgs_i686_linux; ... }’. */ f pkgs_x86_64_linux; i686-linux = f pkgs_i686_linux; ... }’. */
testOn = systems: f: genAttrs testOn = systems: f: genAttrs
(filter (x: elem x supportedSystems) systems) (system: f (pkgsFor system)); (filter (x: elem x supportedSystems) systems) (system: hydraJob (f (pkgsFor system)));
/* Similar to the testOn function, but with an additional /* Similar to the testOn function, but with an additional