Merge #16391: texlive: 2015 -> 2016

Mirroring isn't finalized, but we'll have to fix that on the go.

nixos/doc/manual/release-notes/rl-1509.xml

@@ -471,7 +471,7 @@ in
   <listitem> <para>
     A newly packaged TeX Live 2015 is provided in <literal>pkgs.texlive</literal>,
     split into 6500 nix packages. For basic user documentation see
-    <link xlink:href="https://github.com/NixOS/nixpkgs/blob/release-15.09/pkgs/tools/typesetting/tex/texlive-new/default.nix#L1"
+    <link xlink:href="https://github.com/NixOS/nixpkgs/blob/release-15.09/pkgs/tools/typesetting/tex/texlive/default.nix#L1"
       >the source</link>.
     Beware of <link xlink:href="https://github.com/NixOS/nixpkgs/issues/9757"
       >an issue</link> when installing a too large package set.

pkgs/tools/typesetting/tex/dblatex/default.nix

@@ -4,7 +4,7 @@
 , tex ? texlive.combine { # satisfy all packages that ./configure mentions
     inherit (texlive) scheme-basic epstopdf anysize appendix changebar
       fancybox fancyvrb float footmisc listings jknapltx/*for mathrsfs.sty*/
-      multirow overpic pdfpages rotating stmaryrd subfigure titlesec wasysym
+      multirow overpic pdfpages graphics stmaryrd subfigure titlesec wasysym
       # pkgs below don't seem requested by dblatex, but our manual fails without them
       ec zapfding symbol eepic times rsfs cs tex4ht courier helvetic ly1;
   }

pkgs/tools/typesetting/tex/texlive/bin.nix

@@ -14,13 +14,15 @@
 let
   withSystemLibs = map (libname: "--with-system-${libname}");
 
-  year = "2015";
+  year = "2016";
   version = year; # keep names simple for now
 
   common = rec {
     src = fetchurl {
-      url = "ftp://tug.org/historic/systems/texlive/${year}/texlive-20150521-source.tar.xz";
+      url = # "ftp://tug.org/historic/systems/texlive/${year}/"
-      sha256 = "ed9bcd7bdce899c3c27c16a8c5c3017c4f09e1d7fd097038351b72497e9d4669";
+        "http://lipa.ms.mff.cuni.cz/~cunav5am/nix/texlive-2016" # FIXME: a proper mirror
+        + "/texlive-${year}0523b-source.tar.xz";
+      sha256 = "1v91vahxlxkdra0qz3f132vvx5d9cx2jy84yl1hkch0agyj2rcx8";
     };
 
     configureFlags = [
@@ -75,8 +77,8 @@ core = stdenv.mkDerivation rec {
   preConfigure = ''
     rm -r libs/{cairo,freetype2,gd,gmp,graphite2,harfbuzz,icu,libpaper,libpng} \
       libs/{mpfr,pixman,poppler,potrace,xpdf,zlib,zziplib}
-    mkdir Work
+    mkdir WorkDir
-    cd Work
+    cd WorkDir
   '';
   configureScript = "../configure";
 
@@ -85,7 +87,7 @@ core = stdenv.mkDerivation rec {
     ++ map (what: "--disable-${what}") [
       "dvisvgm" "dvipng" # ghostscript dependency
       "luatex" "luajittex" "mp" "pmp" "upmp" "mf" # cairo would bring in X and more
-      "xetex" "bibtexu" "bibtex8" "bibtex-x" # ICU isn't small
+      "xetex" "bibtexu" "bibtex8" "bibtex-x" "upmendex" # ICU isn't small
     ]
     ++ [ "--without-system-harfbuzz" "--without-system-icu" ] # bogus configure
     ;
@@ -125,7 +127,7 @@ core = stdenv.mkDerivation rec {
 
 
 inherit (core-big) metafont metapost luatex xetex;
-core-big = stdenv.mkDerivation {
+core-big = stdenv.mkDerivation { #TODO: upmendex
   name = "texlive-core-big.bin-${version}";
 
   inherit (common) src;
@@ -138,7 +140,7 @@ core-big = stdenv.mkDerivation {
     ++ withSystemLibs [ "kpathsea" "ptexenc" "cairo" "harfbuzz" "icu" "graphite2" ]
     ++ map (prog: "--disable-${prog}") # don't build things we already have
       [ "tex" "ptex" "eptex" "uptex" "euptex" "aleph" "pdftex"
-        "web-progs" "synctex" "luajittex" # luajittex is mostly not needed, see:
+        "web-progs" "synctex" "luajittex" "mfluajit" # luajittex is mostly not needed, see:
         # http://tex.stackexchange.com/questions/97999/when-to-use-luajittex-in-favour-of-luatex
       ];
 
@@ -146,7 +148,7 @@ core-big = stdenv.mkDerivation {
 
   # we use static libtexlua, because it's only used by a single binary
   postConfigure = ''
-    mkdir ./Work && cd ./Work
+    mkdir ./WorkDir && cd ./WorkDir
     for path in libs/{teckit,lua52} texk/web2c; do
       (
         if [[ "$path" == "libs/lua52" ]]; then
@@ -264,7 +266,7 @@ xindy = stdenv.mkDerivation {
   prePatch = "cd utils/xindy";
   # hardcode clisp location
   postPatch = ''
-    substituteInPlace xindy-?.?.?/user-commands/xindy.in \
+    substituteInPlace xindy-*/user-commands/xindy.in \
       --replace "our \$clisp = ( \$is_windows ? 'clisp.exe' : 'clisp' ) ;" \
                 "our \$clisp = '$(type -P clisp)';"
   '';

pkgs/tools/typesetting/tex/texlive/combine.nix

@@ -28,7 +28,7 @@ let
       ++ lib.optional (lib.any pkgNeedsPython splitBin.wrong) ruby;
   };
 
-  mkUniquePkgs = pkgs: fastUnique (a: b: a < b)
+  mkUniquePkgs = pkgs: fastUnique (a: b: a < b) # highlighting hack: >
     # here we deal with those dummy packages needed for hyphenation filtering
     (map (p: if lib.isDerivation p then builtins.toPath p else "") pkgs);
 
@@ -58,6 +58,17 @@ in buildEnv {
       '')
       pkgList.bin
     +
+
+    # Patch texlinks.sh back to 2015 version;
+    # otherwise some bin/ links break, e.g. xe(la)tex.
+  ''
+    (
+      cd "$out/share/texmf/scripts/texlive"
+      local target="$(readlink texlinks.sh)"
+      rm texlinks.sh && cp "$target" texlinks.sh
+      patch --verbose -R texlinks.sh < '${./texlinks.diff}'
+    )
+  '' +
   ''
     export PATH="$out/bin:$out/share/texmf/scripts/texlive:${perl}/bin:$PATH"
     export TEXMFCNF="$out/share/texmf/web2c"

pkgs/tools/typesetting/tex/texlive/default.nix

@@ -21,7 +21,8 @@ let
   # map: name -> fixed-output hash
   # sha1 in base32 was chosen as a compromise between security and length
   # warning: the following generator command takes lots of resources
-  # nix-build -Q -A texlive.scheme-full.pkgs | ./fixHashes.sh > ./fixedHashes.nix
+  # nix-build ../../../../.. -Q -A texlive.scheme-full.pkgs | ./fixHashes.sh > ./fixedHashes-new.nix
+  # mv ./fixedHashes{-new,}.nix
   fixedHashes = lib.optionalAttrs useFixedHashes (import ./fixedHashes.nix);
 
   # function for creating a working environment from a set of TL packages
@@ -32,22 +33,13 @@ let
 
   # the set of TeX Live packages, collections, and schemes; using upstream naming
   tl = let
-    /* curl ftp://tug.ctan.org/pub/tex/historic/systems/texlive/2015/tlnet-final/tlpkg/texlive.tlpdb.xz \
+    /* # beware: the URL below changes contents continuously
+      curl http://mirror.ctan.org/tex-archive/systems/texlive/tlnet/tlpkg/texlive.tlpdb.xz \
         | xzcat | uniq -u | sed -rn -f ./tl2nix.sed > ./pkgs.nix */
-    orig = import ./pkgs.nix tl;
+    orig = import ./pkgs.nix tl; # XXX XXX XXX FIXME: the file is probably too big now XXX XXX XXX XXX XXX XXX
     clean = orig // {
       # overrides of texlive.tlpdb
 
-      tetex = orig.tetex // { # 2015.08.27 as we need version with mktexlsr.pl
-        # TODO: official hashed mirror
-        urlPrefix = "http://lipa.ms.mff.cuni.cz/~cunav5am/nix";
-        md5.run = "4b4c0208124dfc9c8244c24421946d36";
-        md5.doc = "983f5e5b5f4e407760b4ec176cf6a58f";
-        version = "3.0"; # it's the same
-        postUnpack = "cd $out && patch -p2 < ${./texlinks.patch} || true";
-        # TODO: postUnpack per tlType instead of these hacks
-      };
-
       dvidvi = orig.dvidvi // {
         hasRunfiles = false; # only contains docs that's in bin.core.doc already
       };
@@ -55,6 +47,10 @@ let
         hasRunfiles = false; # only *.po for tlmgr
       };
 
+      xdvi = orig.xdvi // { # it seems to need it to transform fonts
+        deps = (orig.xdvi.deps or {}) // { inherit (tl) metafont; };
+      };
+
       # remove dependency-heavy packages from the basic collections
       collection-basic = orig.collection-basic // {
         deps = removeAttrs orig.collection-basic.deps [ "luatex" "metafont" "xdvi" ];
@@ -62,9 +58,15 @@ let
       latex = orig.latex // {
         deps = removeAttrs orig.latex.deps [ "luatex" ];
       };
-
+      # add them elsewhere so that collections cover all packages
-      xdvi = orig.xdvi // { # it seems to need it to transform fonts
+      collection-luatex = orig.collection-luatex // {
-        deps = (orig.xdvi.deps or {}) // { inherit (tl) metafont; };
+        deps = orig.collection-luatex.deps // { inherit (tl) luatex; };
+      };
+      collection-metapost = orig.collection-metapost // {
+        deps = orig.collection-metapost.deps // { inherit (tl) metafont; };
+      };
+      collection-genericextra = orig.collection-genericextra // {
+        deps = orig.collection-genericextra.deps // { inherit (tl) xdvi; };
       };
     }; # overrides
 
@@ -78,13 +80,10 @@ let
       version = attrs.version or bin.texliveYear;
       mkPkgV = tlType: let
         pkg = attrs // {
-          md5 = attrs.md5.${tlType};
+          sha512 = attrs.sha512.${tlType};
           inherit pname tlType version;
         };
-        in mkPkgs {
+        in mkPkg pkg;
-          inherit (pkg) pname tlType version;
-          pkgList = [ pkg ];
-        };
     in {
       # TL pkg contains lists of packages: runtime files, docs, sources, binaries
       pkgs =
@@ -93,54 +92,56 @@ let
             # the fake derivations are used for filtering of hyphenation patterns
           else { inherit pname version; tlType = "run"; }
         )]
-        ++ lib.optional (attrs.md5 ? "doc") (mkPkgV "doc")
+        ++ lib.optional (attrs.sha512 ? "doc") (mkPkgV "doc")
-        ++ lib.optional (attrs.md5 ? "source") (mkPkgV "source")
+        ++ lib.optional (attrs.sha512 ? "source") (mkPkgV "source")
         ++ lib.optional (bin ? ${pname})
             ( bin.${pname} // { inherit pname; tlType = "bin"; } )
         ++ combinePkgs (attrs.deps or {});
     };
 
-  # the basename used by upstream (without ".tar.xz" suffix)
+  # create a derivation that contains an unpacked upstream TL package
-  mkUrlName = { pname, tlType, ... }:
+  mkPkg = { pname, tlType, version, sha512, postUnpack ? "", stripPrefix ? 1, ... }@args:
-    pname + lib.optionalString (tlType != "run") ".${tlType}";
-
-  # command to unpack a single TL package
-  unpackPkg =
-    { # url ? null, urlPrefix ? null
-      md5, pname, tlType, postUnpack ? "", stripPrefix ? 1, ...
-    }@args: let
-      url = args.url or "${urlPrefix}/${mkUrlName args}.tar.xz";
-      urlPrefix = args.urlPrefix or
-        ("${mirror}/pub/tex/historic/systems/texlive/${bin.texliveYear}/tlnet-final/archive");
-      # beware: standard mirrors http://mirror.ctan.org/ don't have releases
-      mirror = "http://ftp.math.utah.edu"; # ftp://tug.ctan.org no longer works, although same IP
-    in
-      rec {
-        src = fetchurl { inherit url md5; };
-        unpackCmd =  ''
-          tar -xf '${src}' \
-            '--strip-components=${toString stripPrefix}' \
-            -C "$out" --anchored --exclude=tlpkg --keep-old-files
-        '' + postUnpack;
-      };
-
-  # create a derivation that contains unpacked upstream TL packages
-  mkPkgs = { pname, tlType, version, pkgList }@args:
-      /* TODOs:
-          - "historic" isn't mirrored; posted a question at #287
-          - maybe cache (some) collections? (they don't overlap)
-      */
     let
-      tlName = "${mkUrlName args}-${version}";
+      # the basename used by upstream (without ".tar.xz" suffix)
+      urlName = pname + lib.optionalString (tlType != "run") ".${tlType}";
+      tlName = urlName + "-${version}";
       fixedHash = fixedHashes.${tlName} or null; # be graceful about missing hashes
-      pkgs = map unpackPkg (fastUnique (a: b: a.md5 < b.md5) pkgList);
+
-    in runCommand "texlive-${tlName}"
+      url = args.url or "${urlPrefix}/${urlName}.tar.xz";
+      urlPrefix = args.urlPrefix or
+        http://lipa.ms.mff.cuni.cz/~cunav5am/nix/texlive-2016;
+      # XXX XXX XXX FIXME: mirror the snapshot XXX XXX XXX XXX XXX XXX XXX XXX XXX XXX XXX XXX XXX XXX
+      #  ("${mirror}/pub/tex/historic/systems/texlive/${bin.texliveYear}/tlnet-final/archive");
+      #mirror = "http://ftp.math.utah.edu";
+      src = fetchurl { inherit url sha512; };
+
+      passthru = {
+        inherit pname tlType version;
+      } // lib.optionalAttrs (sha512 != "") { inherit src; };
+      unpackCmd = file: ''
+        tar -xf ${file} \
+          '--strip-components=${toString stripPrefix}' \
+          -C "$out" --anchored --exclude=tlpkg --keep-old-files
+      '' + postUnpack;
+
+    in if sha512 == "" then
+      # hash stripped from pkgs.nix to save space -> fetch&unpack in a single step
+      fetchurl {
+        inherit url;
+        sha1 = if fixedHash == null then throw "TeX Live package ${tlName} is missing hash!"
+          else fixedHash;
+        name = tlName;
+        recursiveHash = true;
+        downloadToTemp = true;
+        postFetch = ''mkdir "$out";'' + unpackCmd "$downloadedFile";
+        # TODO: perhaps override preferHashedMirrors and allowSubstitutes
+      }
+        // passthru
+
+    else runCommand "texlive-${tlName}"
       ( { # lots of derivations, not meant to be cached
           preferLocalBuild = true; allowSubstitutes = false;
-          passthru = {
+          inherit passthru;
-            inherit pname tlType version;
-            srcs = map (pkg: pkg.src) pkgs;
-          };
         } // lib.optionalAttrs (fixedHash != null) {
           outputHash = fixedHash;
           outputHashAlgo = "sha1";
@@ -149,7 +150,7 @@ let
       )
       ( ''
           mkdir "$out"
-        '' + lib.concatMapStrings (pkg: pkg.unpackCmd) pkgs
+        '' + unpackCmd "'${src}'"
       );
 
   # combine a set of TL packages into a single TL meta-package

pkgs/tools/typesetting/tex/texlive/texlinks.diff

@@ -0,0 +1,9 @@
+@@ -157,7 +157,7 @@
+       verbose_echo "skipped ConTeXtish symlink $src -> $dest (special case)"
+       ;;
+     *)
+-      test ! -f "$src" &&
++      test "x$src" != "x`(ls -ld $src | awk '{print $NF}') 2>/dev/null`" &&
+         rm -f "$src"
+   
+       if test -f "$src"; then

pkgs/tools/typesetting/tex/texlive/tl2nix.sed

@@ -12,10 +12,10 @@ s/^name (.*)/name "\1"/
 /^name /s/^name (.*)/\1 = {/p
 /^$/,1i};
 
-# extract md5 for runfiles
+# extract hashes of *.tar.xz
-s/^containermd5 (.*)/  md5.run = "\1";/p
+s/^containerchecksum (.*)/  sha512.run = "\1";/p
-s/^doccontainermd5 (.*)/  md5.doc = "\1";/p
+s/^doccontainerchecksum (.*)/  sha512.doc = "\1";/p
-s/^srccontainermd5 (.*)/  md5.source = "\1";/p
+s/^srccontainerchecksum (.*)/  sha512.source = "\1";/p
 /^runfiles /i\  hasRunfiles = true;
 
 # number of path components to strip, defaulting to 1 ("texmf-dist/")

pkgs/top-level/all-packages.nix

@@ -17450,9 +17450,9 @@ in
 
   texFunctions = callPackage ../tools/typesetting/tex/nix pkgs;
 
-  # All the new TeX Live is inside. See description in default.nix.
+  # TeX Live; see http://nixos.org/nixpkgs/manual/#sec-language-texlive
   texlive = recurseIntoAttrs
-    (callPackage ../tools/typesetting/tex/texlive-new { });
+    (callPackage ../tools/typesetting/tex/texlive { });
 
   ib-tws = callPackage ../applications/office/ib/tws { jdk=oraclejdk8; };