Split buildPythonPackage into setup hooks

This commit splits the `buildPythonPackage` into multiple setup hooks.

Generally, Python packages are built from source to wheels using `setuptools`.
The wheels are then installed with `pip`. Tests were often called with
`python setup.py test` but this is less common nowadays. Most projects
now use a different entry point for running tests, typically `pytest`
or `nosetests`.

Since the wheel format was introduced more tools were built to generate these,
e.g. `flit`. Since PEP 517 is provisionally accepted, defining a build-system
independent format (`pyproject.toml`), `pip` can now use that format to
execute the correct build-system.

In the past I've added support for PEP 517 (`pyproject`) to the Python
builder, resulting in a now rather large builder. Furthermore, it was not possible
to reuse components elsewhere. Therefore, the builder is now split into multiple
setup hooks.

The `setuptoolsCheckHook` is included now by default but in time it should
be removed from `buildPythonPackage` to make it easier to use another hook
(curently one has to pass in `dontUseSetuptoolsCheck`).

pkgs/development/interpreters/python/hooks/default.nix

@@ -0,0 +1,95 @@
+# Hooks for building Python packages.
+{ python
+, callPackage
+, makeSetupHook
+}:
+
+let
+  pythonInterpreter = python.pythonForBuild.interpreter;
+  pythonSitePackages = python.sitePackages;
+  pythonCheckInterpreter = python.interpreter;
+  setuppy = ../run_setup.py;
+in rec {
+
+  flitBuildHook = callPackage ({ flit }:
+    makeSetupHook {
+      name = "flit-build-hook";
+      deps = [ flit ];
+      substitutions = {
+        inherit pythonInterpreter;
+      };
+    } ./flit-build-hook.sh) {};
+
+  pipBuildHook = callPackage ({ pip }:
+    makeSetupHook {
+      name = "pip-build-hook.sh";
+      deps = [ pip ];
+      substitutions = {
+        inherit pythonInterpreter pythonSitePackages;
+      };
+    } ./pip-build-hook.sh) {};
+
+  pipInstallHook = callPackage ({ pip }:
+    makeSetupHook {
+      name = "pip-install-hook";
+      deps = [ pip ];
+      substitutions = {
+        inherit pythonInterpreter pythonSitePackages;
+      };
+    } ./pip-install-hook.sh) {};
+
+  pytestCheckHook = callPackage ({ pytest }:
+    makeSetupHook {
+      name = "pytest-check-hook";
+      deps = [ pytest ];
+      substitutions = {
+        inherit pythonCheckInterpreter;
+      };
+    } ./pytest-check-hook.sh) {};
+
+  pythonCatchConflictsHook = callPackage ({ setuptools }:
+    makeSetupHook {
+      name = "python-catch-conflicts-hook";
+      deps = [ setuptools ];
+      substitutions = {
+        inherit pythonInterpreter;
+        catchConflicts=../catch_conflicts/catch_conflicts.py;
+      };
+    } ./python-catch-conflicts-hook.sh) {};
+
+  pythonImportsCheckHook = callPackage ({}:
+    makeSetupHook {
+      name = "python-imports-check-hook.sh";
+      substitutions = {
+        inherit pythonCheckInterpreter;
+      };
+    } ./python-imports-check-hook.sh) {};
+
+  pythonRemoveBinBytecodeHook = callPackage ({ }:
+    makeSetupHook {
+      name = "python-remove-bin-bytecode-hook";
+    } ./python-remove-bin-bytecode-hook.sh) {};
+
+  setuptoolsBuildHook = callPackage ({ setuptools, wheel }:
+    makeSetupHook {
+      name = "setuptools-setup-hook";
+      deps = [ setuptools wheel ];
+      substitutions = {
+        inherit pythonInterpreter pythonSitePackages setuppy;
+      };
+    } ./setuptools-build-hook.sh) {};
+
+  setuptoolsCheckHook = callPackage ({ setuptools }:
+    makeSetupHook {
+      name = "setuptools-check-hook";
+      deps = [ setuptools ];
+      substitutions = {
+        inherit pythonCheckInterpreter setuppy;
+      };
+    } ./setuptools-check-hook.sh) {};
+
+  wheelUnpackHook = callPackage ({ }:
+    makeSetupHook {
+      name = "wheel-unpack-hook.sh";
+    } ./wheel-unpack-hook.sh) {};
+}

pkgs/development/interpreters/python/hooks/flit-build-hook.sh

@@ -0,0 +1,15 @@
+# Setup hook for flit
+echo "Sourcing flit-build-hook"
+
+flitBuildPhase () {
+    echo "Executing flitBuildPhase"
+    preBuild
+    @pythonInterpreter@ -m flit build --format wheel
+    postBuild
+    echo "Finished executing flitBuildPhase"
+}
+
+if [ -z "$dontUseFlitBuild" ] && [ -z "$buildPhase" ]; then
+    echo "Using flitBuildPhase"
+    buildPhase=flitBuildPhase
+fi

pkgs/development/interpreters/python/hooks/pip-build-hook.sh

@@ -0,0 +1,42 @@
+# Setup hook to use for pip projects
+echo "Sourcing pip-build-hook"
+
+pipBuildPhase() {
+    echo "Executing pipBuildPhase"
+    runHook preBuild
+
+    mkdir -p dist
+    echo "Creating a wheel..."
+    @pythonInterpreter@ -m pip wheel --no-index --no-deps --no-clean --no-build-isolation --wheel-dir dist "$options" .
+    echo "Finished creating a wheel..."
+
+    runHook postBuild
+    echo "Finished executing pipBuildPhase"
+}
+
+pipShellHook() {
+    echo "Executing pipShellHook"
+    runHook preShellHook
+
+    # Long-term setup.py should be dropped.
+    if [ -e pyproject.toml ]; then
+      tmp_path=$(mktemp -d)
+      export PATH="$tmp_path/bin:$PATH"
+      export PYTHONPATH="$tmp_path/@pythonSitePackages@:$PYTHONPATH"
+      mkdir -p "$tmp_path/@pythonSitePackages@"
+      @pythonInterpreter@ -m pip install -e . --prefix "$tmp_path" >&2
+    fi
+
+    runHook postShellHook
+    echo "Finished executing pipShellHook"
+}
+
+if [ -z "$dontUsePipBuild" ] && [ -z "$buildPhase" ]; then
+    echo "Using pipBuildPhase"
+    buildPhase=pipBuildPhase
+fi
+
+if [ -z "$shellHook" ]; then
+    echo "Using pipShellHook"
+    shellHook=pipShellHook
+fi

pkgs/development/interpreters/python/hooks/pip-install-hook.sh

@@ -0,0 +1,24 @@
+# Setup hook for pip.
+echo "Sourcing pip-install-hook"
+
+declare -a pipInstallFlags
+
+pipInstallPhase() {
+    echo "Executing pipInstallPhase"
+    runHook preInstall
+
+    mkdir -p "$out/@pythonSitePackages@"
+    export PYTHONPATH="$out/@pythonSitePackages@:$PYTHONPATH"
+
+    pushd dist || return 1
+    @pythonInterpreter@ -m pip install ./*.whl --no-index --prefix="$out" --no-cache $pipInstallFlags --build tmpbuild
+    popd || return 1
+
+    runHook postInstall
+    echo "Finished executing pipInstallPhase"
+}
+
+if [ -z "$dontUsePipInstall" ] && [ -z "$installPhase" ]; then
+    echo "Using pipInstallPhase"
+    installPhase=pipInstallPhase
+fi

pkgs/development/interpreters/python/hooks/pytest-check-hook.sh

@@ -0,0 +1,49 @@
+# Setup hook for pytest
+echo "Sourcing pytest-check-hook"
+
+declare -ar disabledTests
+
+function _concatSep {
+    local result
+    local sep="$1"
+    local -n arr=$2
+    for index in ${!arr[*]}; do
+        if [ $index -eq 0 ]; then
+            result="${arr[index]}"
+        else
+            result+=" $sep ${arr[index]}"
+        fi
+    done
+    echo "$result"
+}
+
+function _pytestComputeDisabledTestsString () {
+    declare -a tests
+    local tests=($1)
+    local prefix="not "
+    prefixed=( "${tests[@]/#/$prefix}" )
+    result=$(_concatSep "and" prefixed)
+    echo "$result"
+}
+
+function pytestCheckPhase() {
+    echo "Executing pytestCheckPhase"
+    runHook preCheck
+
+    # Compose arguments
+    args=" -m pytest"
+    if [ -n "$disabledTests" ]; then
+        disabledTestsString=$(_pytestComputeDisabledTestsString "${disabledTests[@]}")
+      args+=" -k \""$disabledTestsString"\""
+    fi
+    args+=" ${pytestFlagsArray[@]}"
+    eval "@pythonCheckInterpreter@ $args"
+
+    runHook postCheck
+    echo "Finished executing pytestCheckPhase"
+}
+
+if [ -z "$dontUsePytestCheck" ] && [ -z "$installCheckPhase" ]; then
+    echo "Using pytestCheckPhase"
+    preDistPhases+=" pytestCheckPhase"
+fi

pkgs/development/interpreters/python/hooks/python-catch-conflicts-hook.sh

@@ -0,0 +1,10 @@
+# Setup hook for detecting conflicts in Python packages
+echo "Sourcing python-catch-conflicts-hook.sh"
+
+pythonCatchConflictsPhase() {
+    @pythonInterpreter@ @catchConflicts@
+}
+
+if [ -z "$dontUsePythonCatchConflicts" ]; then
+    preDistPhases+=" pythonCatchConflictsPhase"
+fi

pkgs/development/interpreters/python/hooks/python-imports-check-hook.sh

@@ -0,0 +1,16 @@
+# Setup hook for checking whether Python imports succeed
+echo "Sourcing python-imports-check-hook.sh"
+
+pythonImportsCheckPhase () {
+    echo "Executing pythonImportsCheckPhase"
+
+    if [ -n "$pythonImportsCheck" ]; then
+        echo "Check whether the following modules can be imported: $pythonImportsCheck"
+        cd $out && eval "@pythonCheckInterpreter@ -c 'import os; import importlib; list(map(lambda mod: importlib.import_module(mod), os.environ[\"pythonImportsCheck\"].split()))'"
+    fi
+}
+
+if [ -z "$dontUsePythonImportsCheck" ]; then
+    echo "Using pythonImportsCheckPhase"
+    preDistPhases+=" pythonImportsCheckPhase"
+fi

pkgs/development/interpreters/python/hooks/python-remove-bin-bytecode-hook.sh

@@ -0,0 +1,17 @@
+# Setup hook for detecting conflicts in Python packages
+echo "Sourcing python-remove-bin-bytecode-hook.sh"
+
+# Check if we have two packages with the same name in the closure and fail.
+# If this happens, something went wrong with the dependencies specs.
+# Intentionally kept in a subdirectory, see catch_conflicts/README.md.
+
+pythonRemoveBinBytecodePhase () {
+    if [ -d "$out/bin" ]; then
+      rm -rf "$out/bin/__pycache__"                 # Python 3
+      find "$out/bin" -type f -name "*.pyc" -delete # Python 2
+    fi
+}
+
+if [ -z "$dontUsePythonRemoveBinBytecode" ]; then
+    preDistPhases+=" pythonRemoveBinBytecodePhase"
+fi

pkgs/development/interpreters/python/hooks/setuptools-build-hook.sh

@@ -0,0 +1,47 @@
+# Setup hook for setuptools.
+echo "Sourcing setuptools-build-hook"
+
+setuptoolsBuildPhase() {
+    echo "Executing setuptoolsBuildPhase"
+    local args
+    runHook preBuild
+
+    cp -f @setuppy@ nix_run_setup
+    args=""
+    if [ -n "$setupPyGlobalFlags" ]; then
+        args+="$setupPyGlobalFlags"
+    fi
+    if [ -n "$setupPyBuildFlags" ]; then
+        args+="build_ext $setupPyBuildFlags"
+    fi
+    eval "@pythonInterpreter@ nix_run_setup $args bdist_wheel"
+
+    runHook postBuild
+    echo "Finished executing setuptoolsInstallPhase"
+}
+
+setuptoolsShellHook() {
+    echo "Executing setuptoolsShellHook"
+    runHook preShellHook
+
+    if test -e setup.py; then
+        tmp_path=$(mktemp -d)
+        export PATH="$tmp_path/bin:$PATH"
+        export PYTHONPATH="@pythonSitePackages@:$PYTHONPATH"
+        mkdir -p "$tmp_path/@pythonSitePackages@"
+        eval "@pythonInterpreter@ -m pip -e . --prefix $tmp_path >&2"
+    fi
+
+    runHook postShellHook
+    echo "Finished executing setuptoolsShellHook"
+}
+
+if [ -z "$dontUseSetuptoolsBuild" ] && [ -z "$buildPhase" ]; then
+    echo "Using setuptoolsBuildPhase"
+    buildPhase=setuptoolsBuildPhase
+fi
+
+if [ -z "$dontUseSetuptoolsShellHook" ] && [ -z "$shellHook" ]; then
+    echo "Using setuptoolsShellHook"
+    shellHook=setuptoolsShellHook
+fi

pkgs/development/interpreters/python/hooks/setuptools-check-hook.sh

@@ -0,0 +1,18 @@
+# Setup hook for setuptools.
+echo "Sourcing setuptools-check-hook"
+
+setuptoolsCheckPhase() {
+    echo "Executing setuptoolsCheckPhase"
+    runHook preCheck
+
+    cp -f @setuppy@ nix_run_setup
+    @pythonCheckInterpreter@ nix_run_setup test
+
+    runHook postCheck
+    echo "Finished executing setuptoolsCheckPhase"
+}
+
+if [ -z "$dontUseSetuptoolsCheck" ] && [ -z "$installCheckPhase" ]; then
+    echo "Using setuptoolsCheckPhase"
+    preDistPhases+=" setuptoolsCheckPhase"
+fi

pkgs/development/interpreters/python/hooks/wheel-unpack-hook.sh

@@ -0,0 +1,18 @@
+# Setup hook to use in case a wheel is fetched
+echo "Sourcing wheel setup hook"
+
+wheelUnpackPhase(){
+    echo "Executing wheelUnpackPhase"
+    runHook preUnpack
+
+    mkdir -p dist
+    cp "$src" "dist/$(stripHash "$src")"
+
+#     runHook postUnpack # Calls find...?
+    echo "Finished executing wheelUnpackPhase"
+}
+
+if [ -z "$dontUseWheelUnpack" ] && [ -z "$unpackPhase" ]; then
+    echo "Using wheelUnpackPhase"
+    unpackPhase=wheelUnpackPhase
+fi