Python: the pythonModule attribute

Python libraries or modules now have an attribute `pythonModule = interpreter;` to indicate
they provide Python modules for the specified `interpreter`.

The package set provides the following helper functions:

- hasPythonModule: Check whether a derivation provides a Python module.
- requiredPythonModules: Recurse into a list of Python modules, returning all Python modules that are required.
- makePythonPath: Create a PYTHONPATH from a list of Python modules.

Also included in this commit is:
- disabledIf: Helper function for disabling non-buildPythonPackage functions.

pkgs/development/interpreters/python/build-python-package.nix

@@ -5,7 +5,12 @@
 
 { lib
 , python
-, mkPythonDerivation
+, wrapPython
+, setuptools
+, unzip
+, ensureNewerSourcesHook
+, pythonModule
+, namePrefix
 , bootstrapped-pip
 , flit
 }:
@@ -15,6 +20,9 @@ let
   flit-specific = import ./build-python-package-flit.nix { inherit python flit; };
   wheel-specific = import ./build-python-package-wheel.nix { };
   common = import ./build-python-package-common.nix { inherit python bootstrapped-pip; };
+  mkPythonDerivation = import ./mk-python-derivation.nix {
+    inherit lib python wrapPython setuptools unzip ensureNewerSourcesHook pythonModule namePrefix;
+  };
 in
 
 {

pkgs/development/interpreters/python/cpython/2.7/default.nix

@@ -201,7 +201,7 @@ in stdenv.mkDerivation {
     in rec {
       inherit libPrefix sitePackages x11Support hasDistutilsCxxPatch;
       executable = libPrefix;
-      buildEnv = callPackage ../../wrapper.nix { python = self; };
+      buildEnv = callPackage ../../wrapper.nix { python = self; inherit (pythonPackages) requiredPythonModules; };
       withPackages = import ../../with-packages.nix { inherit buildEnv pythonPackages;};
       pkgs = pythonPackages;
       isPy2 = true;

pkgs/development/interpreters/python/cpython/3.4/default.nix

@@ -160,7 +160,7 @@ in stdenv.mkDerivation {
   in rec {
     inherit libPrefix sitePackages x11Support;
     executable = "${libPrefix}m";
-    buildEnv = callPackage ../../wrapper.nix { python = self; };
+    buildEnv = callPackage ../../wrapper.nix { python = self; inherit (pythonPackages) requiredPythonModules; };
     withPackages = import ../../with-packages.nix { inherit buildEnv pythonPackages;};
     pkgs = pythonPackages;
     isPy3 = true;

pkgs/development/interpreters/python/cpython/3.5/default.nix

@@ -154,7 +154,7 @@ in stdenv.mkDerivation {
   in rec {
     inherit libPrefix sitePackages x11Support;
     executable = "${libPrefix}m";
-    buildEnv = callPackage ../../wrapper.nix { python = self; };
+    buildEnv = callPackage ../../wrapper.nix { python = self; inherit (pythonPackages) requiredPythonModules; };
     withPackages = import ../../with-packages.nix { inherit buildEnv pythonPackages;};
     pkgs = pythonPackages;
     isPy3 = true;

pkgs/development/interpreters/python/cpython/3.6/default.nix

@@ -153,7 +153,7 @@ in stdenv.mkDerivation {
   in rec {
     inherit libPrefix sitePackages x11Support;
     executable = "${libPrefix}m";
-    buildEnv = callPackage ../../wrapper.nix { python = self; };
+    buildEnv = callPackage ../../wrapper.nix { python = self; inherit (pythonPackages) requiredPythonModules; };
     withPackages = import ../../with-packages.nix { inherit buildEnv pythonPackages;};
     pkgs = pythonPackages;
     isPy3 = true;

pkgs/development/interpreters/python/mk-python-derivation.nix

@@ -6,13 +6,13 @@
 , setuptools
 , unzip
 , ensureNewerSourcesHook
+# Whether the derivation provides a Python module or not.
+, pythonModule
+, namePrefix
 }:
 
 { name ? "${attrs.pname}-${attrs.version}"
 
-# by default prefix `name` e.g. "python3.3-${name}"
-, namePrefix ? python.libPrefix + "-"
-
 # Dependencies for building the package
 , buildInputs ? []
 
@@ -54,7 +54,7 @@ if disabled
 then throw "${name} not supported for interpreter ${python.executable}"
 else
 
-python.stdenv.mkDerivation (builtins.removeAttrs attrs ["disabled" "checkInputs"] // {
+python.stdenv.mkDerivation (builtins.removeAttrs attrs ["disabled" "checkInputs" "pythonModule"] // {
 
   name = namePrefix + name;
 
@@ -83,6 +83,7 @@ python.stdenv.mkDerivation (builtins.removeAttrs attrs ["disabled" "checkInputs"
 
   passthru = {
     inherit python; # The python interpreter
+    inherit pythonModule;
   } // passthru;
 
   meta = with lib.maintainers; {

pkgs/development/interpreters/python/pypy/2.7/default.nix

@@ -137,7 +137,7 @@ in stdenv.mkDerivation rec {
     inherit zlibSupport libPrefix sitePackages;
     executable = "pypy";
     isPypy = true;
-    buildEnv = callPackage ../../wrapper.nix { python = self; };
+    buildEnv = callPackage ../../wrapper.nix { python = self; inherit (pythonPackages) requiredPythonModules; };
     interpreter = "${self}/bin/${executable}";
     withPackages = import ../../with-packages.nix { inherit buildEnv pythonPackages;};
     pkgs = pythonPackages;

pkgs/development/interpreters/python/wrapper.nix

@@ -2,13 +2,14 @@
 , extraLibs ? []
 , extraOutputsToInstall ? []
 , postBuild ? ""
-, ignoreCollisions ? false }:
+, ignoreCollisions ? false
+, requiredPythonModules
+, }:
 
 # Create a python executable that knows about additional packages.
 let
-  recursivePthLoader = import ../../python-modules/recursive-pth-loader/default.nix { stdenv = stdenv; python = python; };
   env = let
-    paths = stdenv.lib.closePropagation (extraLibs ++ [ python recursivePthLoader ] ) ;
+    paths = requiredPythonModules (extraLibs ++ [ python ] ) ;
   in buildEnv {
     name = "${python.name}-env";
 

pkgs/top-level/python-packages.nix

@@ -31,10 +31,9 @@ let
 
   callPackage = pkgs.newScope self;
 
-  bootstrapped-pip = callPackage ../development/python-modules/bootstrapped-pip { };
+  namePrefix = python.libPrefix + "-";
 
-  mkPythonDerivation = makeOverridable( callPackage ../development/interpreters/python/mk-python-derivation.nix {
+  bootstrapped-pip = callPackage ../development/python-modules/bootstrapped-pip { };
-  });
 
   # Derivations built with `buildPythonPackage` can already be overriden with `override`, `overrideAttrs`, and `overrideDerivation`.
   # This function introduces `overridePythonAttrs` and it overrides the call to `buildPythonPackage`.
@@ -52,13 +51,20 @@ let
       }
       else ff;
 
-  buildPythonPackage = makeOverridablePythonPackage (callPackage ../development/interpreters/python/build-python-package.nix {
+  buildPythonPackage = makeOverridablePythonPackage ( makeOverridable (callPackage ../development/interpreters/python/build-python-package.nix {
-    inherit mkPythonDerivation;
     inherit bootstrapped-pip;
     flit = self.flit;
-  });
+    # We want Python libraries to be named like e.g. "python3.6-${name}"
+    inherit namePrefix;
+    pythonModule = python;
+  }));
 
-  buildPythonApplication = args: buildPythonPackage ({namePrefix="";} // args );
+  buildPythonApplication = makeOverridablePythonPackage ( makeOverridable (callPackage ../development/interpreters/python/build-python-package.nix {
+    inherit bootstrapped-pip;
+    flit = self.flit;
+    namePrefix = "";
+    pythonModule = false;
+  }));
 
   graphiteVersion = "1.0.2";
 
@@ -80,10 +86,40 @@ let
         else throw "Unsupported kind ${kind}");
     in fetcher (builtins.removeAttrs attrs ["format"]) );
 
+  # Check whether a derivation provides a Python module.
+  hasPythonModule = drv: (hasAttr "pythonModule" drv) && ( (getAttr "pythonModule" drv) == python);
+
+  # Get list of required Python modules given a list of derivations.
+  requiredPythonModules = drvs: let
+    filterNull = list: filter (x: !isNull x) list;
+    conditionalGetRecurse = attr: condition: drv: let f = conditionalGetRecurse attr condition; in
+      (if (condition drv) then unique [drv]++(concatMap f (filterNull(getAttr attr drv))) else []);
+    _required = drv: conditionalGetRecurse "propagatedBuildInputs" hasPythonModule drv;
+  in [python] ++ (unique (concatMap _required (filterNull drvs)));
+
+  # Create a PYTHONPATH from a list of derivations. This function recurses into the items to find derivations
+  # providing Python modules.
+  makePythonPath = drvs: stdenv.lib.makeSearchPath python.sitePackages (requiredPythonModules drvs);
+
+  # Convert derivation to a Python module.
+  toPythonModule = drv:
+    drv.overrideAttrs( oldAttrs: {
+      # Use passthru in order to prevent rebuilds when possible.
+      passthru = (oldAttrs.passthru or {})// {
+        name = namePrefix + oldAttrs.name;
+        pythonModule = python;
+        pythonPath = [ ]; # Deprecated, for compatibility.
+      };
+    });
+
+  disabledIf = x: drv:
+    if x then throw "${removePrefix namePrefix (drv.pname or drv.name)} not supported for interpreter ${python.executable}" else drv;
+
 in {
 
-  inherit python bootstrapped-pip pythonAtLeast pythonOlder isPy26 isPy27 isPy33 isPy34 isPy35 isPy36 isPyPy isPy3k mkPythonDerivation buildPythonPackage buildPythonApplication;
+  inherit python bootstrapped-pip pythonAtLeast pythonOlder isPy26 isPy27 isPy33 isPy34 isPy35 isPy36 isPyPy isPy3k buildPythonPackage buildPythonApplication;
   inherit fetchPypi callPackage;
+  inherit hasPythonModule requiredPythonModules makePythonPath disabledIf;
 
   # helpers