From 01624e1ac29ee0854cb63d0c1efb6d791c1441d4 Mon Sep 17 00:00:00 2001 From: Nikolay Amiantov Date: Mon, 15 Aug 2016 12:35:32 +0300 Subject: [PATCH 1/6] wrapPythonProgram: use site.addsitedir instead of PYTHONPATH --- doc/languages-frameworks/python.md | 7 +++++-- pkgs/development/python-modules/generic/wrap.sh | 2 +- pkgs/top-level/python-packages.nix | 14 +++++++++++++- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/doc/languages-frameworks/python.md b/doc/languages-frameworks/python.md index 434a9b156e78..89659c2166b1 100644 --- a/doc/languages-frameworks/python.md +++ b/doc/languages-frameworks/python.md @@ -503,9 +503,12 @@ and can be used as: The `buildPythonPackage` mainly does four things: -* In the `buildPhase`, it calls `${python.interpreter} setup.py bdist_wheel` to build a wheel binary zipfile. +* In the `buildPhase`, it calls `${python.interpreter} setup.py bdist_wheel` to + build a wheel binary zipfile. * In the `installPhase`, it installs the wheel file using `pip install *.whl`. -* In the `postFixup` phase, the `wrapPythonPrograms` bash function is called to wrap all programs in the `$out/bin/*` directory to include `$PYTHONPATH` and `$PATH` environment variables. +* In the `postFixup` phase, the `wrapPythonPrograms` bash function is called to + wrap all programs in the `$out/bin/*` directory to include `$PATH` + environment variable and add dependent libraries to script's `sys.path`. * In the `installCheck` phase, `${python.interpreter} setup.py test` is ran. As in Perl, dependencies on other Python packages can be specified in the diff --git a/pkgs/development/python-modules/generic/wrap.sh b/pkgs/development/python-modules/generic/wrap.sh index efbb1e737876..d0c49c36c4b5 100644 --- a/pkgs/development/python-modules/generic/wrap.sh +++ b/pkgs/development/python-modules/generic/wrap.sh @@ -42,12 +42,12 @@ wrapPythonProgramsIn() { # The magicalSedExpression will invoke a "$(basename "$f")", so # if you change $f to something else, be sure to also change it # in pkgs/top-level/python-packages.nix! + # It also uses $program_PYTHONPATH. sed -i "$f" -re '@magicalSedExpression@' # wrapProgram creates the executable shell script described # above. The script will set PYTHONPATH and PATH variables.! # (see pkgs/build-support/setup-hooks/make-wrapper.sh) local -a wrap_args=("$f" - --prefix PYTHONPATH ':' "$program_PYTHONPATH" --prefix PATH ':' "$program_PATH:$dir/bin") # Add any additional arguments provided by makeWrapperArgs diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index d39c8d85cdd2..8f0b85af8c57 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -59,13 +59,25 @@ in modules // { } ''; + # This preamble does two things: + # * Sets argv[0] to the original application's name; otherwise it would be .foo-wrapped. + # Python doesn't support `exec -a`. + # * Adds all required libraries to sys.path via `site.addsitedir`. It also handles *.pth files. + preamble = '' + import sys + import site + import functools + sys.argv[0] = '"'$(basename "$f")'"' + functools.reduce(lambda k, p: site.addsitedir(p, k), ['"$([ -n "$program_PYTHONPATH" ] && (echo "'$program_PYTHONPATH'" | sed "s|:|','|g") || true)"'], site._init_pathinfo()) + ''; + in '' 1 { /^#!/!b; :r /\\$/{N;br} /__future__|^ *(#.*)?$/{n;br} ${concatImapStrings mkStringSkipper quoteVariants} - /^ *[^# ]/i import sys; sys.argv[0] = '"'$(basename "$f")'"' + /^ *[^# ]/i ${replaceStrings ["\n"] [";"] preamble} } ''; } From 4ad459e3e0ad7366fa01a4c6592adf14e4edf62c Mon Sep 17 00:00:00 2001 From: Nikolay Amiantov Date: Wed, 17 Aug 2016 02:51:55 +0300 Subject: [PATCH 2/6] wrapPythonProgram: split into several functions --- .../python-modules/generic/wrap.sh | 44 +++++++++++++------ 1 file changed, 30 insertions(+), 14 deletions(-) diff --git a/pkgs/development/python-modules/generic/wrap.sh b/pkgs/development/python-modules/generic/wrap.sh index d0c49c36c4b5..f4b2d44993ca 100644 --- a/pkgs/development/python-modules/generic/wrap.sh +++ b/pkgs/development/python-modules/generic/wrap.sh @@ -4,16 +4,12 @@ wrapPythonPrograms() { wrapPythonProgramsIn $out "$out $pythonPath" } -# Transforms any binaries generated by the setup.py script, replacing them -# with an executable shell script which will set some environment variables -# and then call into the original binary (which has been given a .wrapped -# suffix). -wrapPythonProgramsIn() { - local dir="$1" - local pythonPath="$2" +# Builds environment variables like PYTHONPATH and PATH walking through closure +# of dependencies. +buildPythonPath() { + local pythonPath="$1" local python="@executable@" local path - local f # Create an empty table of python paths (see doc on _addToPythonPath # for how this is used). Build up the program_PATH and program_PYTHONPATH @@ -24,6 +20,30 @@ wrapPythonProgramsIn() { for path in $pythonPath; do _addToPythonPath $path done +} + +# Patches a Python script so that it has correct libraries path and executable +# name. +patchPythonScript() { + local f="$1" + + # The magicalSedExpression will invoke a "$(basename "$f")", so + # if you change $f to something else, be sure to also change it + # in pkgs/top-level/python-packages.nix! + # It also uses $program_PYTHONPATH. + sed -i "$f" -re '@magicalSedExpression@' +} + +# Transforms any binaries generated by the setup.py script, replacing them +# with an executable shell script which will set some environment variables +# and then call into the original binary (which has been given a .wrapped +# suffix). +wrapPythonProgramsIn() { + local dir="$1" + local pythonPath="$2" + local f + + buildPythonPath "$pythonPath" # Find all regular files in the output directory that are executable. for f in $(find "$dir" -type f -perm -0100); do @@ -39,16 +59,12 @@ wrapPythonProgramsIn() { # dont wrap EGG-INFO scripts since they are called from python if echo "$f" | grep -qv EGG-INFO/scripts; then echo "wrapping \`$f'..." - # The magicalSedExpression will invoke a "$(basename "$f")", so - # if you change $f to something else, be sure to also change it - # in pkgs/top-level/python-packages.nix! - # It also uses $program_PYTHONPATH. - sed -i "$f" -re '@magicalSedExpression@' + patchPythonScript "$f" # wrapProgram creates the executable shell script described # above. The script will set PYTHONPATH and PATH variables.! # (see pkgs/build-support/setup-hooks/make-wrapper.sh) local -a wrap_args=("$f" - --prefix PATH ':' "$program_PATH:$dir/bin") + --prefix PATH ':' "$program_PATH") # Add any additional arguments provided by makeWrapperArgs # argument to buildPythonPackage. From adafdb88f55e94f28f46d57c4432cf3dd57ebc60 Mon Sep 17 00:00:00 2001 From: Nikolay Amiantov Date: Wed, 17 Aug 2016 04:31:35 +0300 Subject: [PATCH 3/6] wrapPythonProgram: fix magical sed expression * Treat #! as all other comments; * Treat comma at the end as a multi-line marker; * Skip lines with a space at start (they are continuing previous lines); * Don't allow spaces in front of quotes or the first real statement. --- pkgs/top-level/python-packages.nix | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 8f0b85af8c57..2674da51f777 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -53,7 +53,7 @@ in modules // { isSingle = elem quote [ "\"" "'\"'\"'" ]; endQuote = if isSingle then "[^\\\\]${quote}" else quote; in '' - /^ *[a-z]?${quote}/ { + /^[a-z]?${quote}/ { /${quote}${quote}|${quote}.*${endQuote}/{n;br} :${label}; n; /^${quote}/{n;br}; /${endQuote}/{n;br}; b${label} } @@ -73,11 +73,11 @@ in modules // { in '' 1 { - /^#!/!b; :r - /\\$/{N;br} - /__future__|^ *(#.*)?$/{n;br} + :r + /\\$|,$/{N;br} + /__future__|^ |^ *(#.*)?$/{n;br} ${concatImapStrings mkStringSkipper quoteVariants} - /^ *[^# ]/i ${replaceStrings ["\n"] [";"] preamble} + /^[^# ]/i ${replaceStrings ["\n"] [";"] preamble} } ''; } From 837cf416cee05e1651ae56ef9b65454645497d8d Mon Sep 17 00:00:00 2001 From: Nikolay Amiantov Date: Wed, 17 Aug 2016 06:18:07 +0300 Subject: [PATCH 4/6] wrapPythonProgram: exclude Python itself from PYTHONPATH --- pkgs/development/python-modules/generic/wrap.sh | 2 ++ pkgs/top-level/python-packages.nix | 1 + 2 files changed, 3 insertions(+) diff --git a/pkgs/development/python-modules/generic/wrap.sh b/pkgs/development/python-modules/generic/wrap.sh index f4b2d44993ca..ca73a473ed56 100644 --- a/pkgs/development/python-modules/generic/wrap.sh +++ b/pkgs/development/python-modules/generic/wrap.sh @@ -17,6 +17,8 @@ buildPythonPath() { declare -A pythonPathsSeen=() program_PYTHONPATH= program_PATH= + pythonPathsSeen["@python@"]=1 + addToSearchPath program_PATH @python@/bin for path in $pythonPath; do _addToPythonPath $path done diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 2674da51f777..9e04f3c459c8 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -42,6 +42,7 @@ in modules // { { deps = pkgs.makeWrapper; substitutions.libPrefix = python.libPrefix; substitutions.executable = python.interpreter; + substitutions.python = python; substitutions.magicalSedExpression = let # Looks weird? Of course, it's between single quoted shell strings. # NOTE: Order DOES matter here, so single character quotes need to be From 826c398318fa3d597da1d3803261ef950b39d05c Mon Sep 17 00:00:00 2001 From: Nikolay Amiantov Date: Wed, 17 Aug 2016 04:34:25 +0300 Subject: [PATCH 5/6] pythonPackages.scfbuild: remove workaround --- pkgs/top-level/python-packages.nix | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 9e04f3c459c8..0a2db37ace1f 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -8534,20 +8534,7 @@ in modules // { installPhase = '' mkdir -p $out/${python.sitePackages} cp -r scfbuild $out/${python.sitePackages} - # Workaround for #16133 - mkdir -p $out/bin - - cat >$out/bin/scfbuild < Date: Wed, 17 Aug 2016 03:05:55 +0300 Subject: [PATCH 6/6] gajim: use wrapPython --- .../instant-messengers/gajim/default.nix | 26 +++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/pkgs/applications/networking/instant-messengers/gajim/default.nix b/pkgs/applications/networking/instant-messengers/gajim/default.nix index e7ebcf149e19..8d9cf434e5d2 100644 --- a/pkgs/applications/networking/instant-messengers/gajim/default.nix +++ b/pkgs/applications/networking/instant-messengers/gajim/default.nix @@ -39,7 +39,6 @@ stdenv.mkDerivation rec { postPatch = '' sed -i -e '0,/^[^#]/ { /^[^#]/i export \\\ - PYTHONPATH="'"$PYTHONPATH\''${PYTHONPATH:+:}\$PYTHONPATH"'" \\\ GST_PLUGIN_PATH="'"\$GST_PLUGIN_PATH''${GST_PLUGIN_PATH:+:}${"" }$GST_PLUGIN_PATH"'" }' scripts/gajim.in @@ -53,21 +52,38 @@ stdenv.mkDerivation rec { ''; buildInputs = [ - python intltool pkgconfig libX11 + python libX11 + ] ++ optionals enableJingle [ farstream gst_plugins_bad libnice ]; + + nativeBuildInputs = [ + pythonPackages.wrapPython intltool pkgconfig + ]; + + propagatedBuildInputs = [ pythonPackages.pygobject pythonPackages.pyGtkGlade pythonPackages.sqlite3 pythonPackages.pyasn1 pythonPackages.pyxdg pythonPackages.nbxmpp pythonPackages.pyopenssl pythonPackages.dbus-python - ] ++ optionals enableJingle [ farstream gst_plugins_bad libnice ] - ++ optional enableE2E pythonPackages.pycrypto + ] ++ optional enableE2E pythonPackages.pycrypto ++ optional enableRST pythonPackages.docutils ++ optional enableNotifications pythonPackages.notify ++ extraPythonPackages pythonPackages; - postInstall = '' + postFixup = '' install -m 644 -t "$out/share/gajim/icons/hicolor" \ "icons/hicolor/index.theme" + + buildPythonPath "$out" + + for i in $out/bin/*; do + name="$(basename "$i")" + if [ "$name" = "gajim-history-manager" ]; then + name="history_manager" + fi + + patchPythonScript "$out/share/gajim/src/$name.py" + done ''; enableParallelBuilding = true;