typst: add initial support for typst packages (#369283)

doc/languages-frameworks/index.md

@@ -98,6 +98,7 @@ scheme.section.md
 swift.section.md
 tcl.section.md
 texlive.section.md
+typst.section.md
 vim.section.md
 neovim.section.md
 ```

doc/languages-frameworks/typst.section.md

@@ -0,0 +1,62 @@
+# Typst {#typst}
+
+Typst can be configured to include packages from [Typst Universe](https://typst.app/universe/) or custom packages.
+
+## Custom Environment {#typst-custom-environment}
+
+You can create a custom Typst environment with a selected set of packages from **Typst Universe** using the following code. It is also possible to specify a Typst package with a specific version (e.g., `cetz_0_3_0`). A package without a version number will always refer to its latest version.
+
+```nix
+typst.withPackages (p: with p; [
+  polylux_0_4_0
+  cetz_0_3_0
+])
+```
+
+### Handling Outdated Package Hashes {#typst-handling-outdated-package-hashes}
+
+Since **Typst Universe** does not provide a way to fetch a package with a specific hash, the package hashes in `nixpkgs` can sometimes be outdated. To resolve this issue, you can manually override the package source using the following approach:
+
+```nix
+typst.withPackages.override (old: {
+  typstPackages = old.typstPackages.extend (_: previous: {
+    polylux_0_4_0 = previous.polylux_0_4_0.overrideAttrs (oldPolylux: {
+      src = oldPolylux.src.overrideAttrs {
+        outputHash = YourUpToDatePolyluxHash;
+      };
+    });
+  });
+}) (p: with p; [
+  polylux_0_4_0
+  cetz_0_3_0
+])
+```
+
+## Custom Packages {#typst-custom-packages}
+
+`Nixpkgs` provides a helper function, `buildTypstPackage`, to build custom Typst packages that can be used within the Typst environment. However, all dependencies of the custom package must be explicitly specified in `typstDeps`.
+
+Here's how to define a custom Typst package:
+
+```nix
+{ buildTypstPackage, typstPackages, fetchzip }:
+
+buildTypstPackage (finalAttrs: {
+  pname = "my-typst-package";
+  version = "0.0.1";
+  src = fetchzip { ... };
+  typstDeps = with typstPackages; [ cetz_0_3_0 ];
+})
+```
+
+### Package Scope and Usage {#typst-package-scope-and-usage}
+
+By default, every custom package is scoped under `@preview`, as shown below:
+
+```typst
+#import "@preview/my-typst-package:0.0.1": *
+```
+
+Since `@preview` is intended for packages from **Typst Universe**, it is recommended to use this approach **only for temporary or experimental modifications over existing packages** from **Typst Universe**.
+
+On the other hand, **local packages**, packages scoped under `@local`, are **not** considered part of the Typst environment. This means that local packages must be manually linked to the Typst compiler if needed.

doc/redirects.json

@@ -413,6 +413,24 @@
   "tester-testEqualArrayOrMap-return": [
     "index.html#tester-testEqualArrayOrMap-return"
   ],
+  "typst": [
+    "index.html#typst",
+    "doc/languages-frameworks/typst.section.md#typst"
+  ],
+  "typst-custom-environment": [
+    "index.html#typst-custom-environment",
+    "doc/languages-frameworks/typst.section.md#typst-custom-environment"
+  ],
+  "typst-custom-packages": [
+    "index.html#typst-custom-packages",
+    "doc/languages-frameworks/typst.section.md#typst-custom-packages"
+  ],
+  "typst-handling-outdated-package-hashes": [
+    "index.html#typst-handling-outdated-package-hashes"
+  ],
+  "typst-package-scope-and-usage": [
+    "index.html#typst-package-scope-and-usage"
+  ],
   "variables-specifying-dependencies": [
     "index.html#variables-specifying-dependencies"
   ],

maintainers/maintainer-list.nix

@@ -4284,6 +4284,11 @@
     name = "CherryKitten";
     keys = [ { fingerprint = "264C FA1A 194C 585D F822  F673 C01A 7CBB A617 BD5F"; } ];
   };
+  cherrypiejam = {
+    github = "cherrypiejam";
+    githubId = 46938348;
+    name = "Gongqi Huang";
+  };
   chessai = {
     email = "chessai1996@gmail.com";
     github = "chessai";

maintainers/scripts/update-typst-packages.py

@@ -0,0 +1,226 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -p "python3.withPackages (p: with p; [ tomli tomli-w packaging license-expression])" -i python3
+
+# This file is formatted with `ruff format`.
+
+import os
+import re
+import tomli
+import tomli_w
+import subprocess
+import concurrent.futures
+import argparse
+import tempfile
+import tarfile
+from string import punctuation
+from packaging.version import Version
+from urllib import request
+from collections import OrderedDict
+
+
+class TypstPackage:
+    def __init__(self, **kwargs):
+        self.pname = kwargs["pname"]
+        self.version = kwargs["version"]
+        self.meta = kwargs["meta"]
+        self.path = kwargs["path"]
+        self.repo = (
+            None
+            if "repository" not in self.meta["package"]
+            else self.meta["package"]["repository"]
+        )
+        self.description = self.meta["package"]["description"].rstrip(punctuation)
+        self.license = self.meta["package"]["license"]
+        self.params = "" if "params" not in kwargs else kwargs["params"]
+        self.deps = [] if "deps" not in kwargs else kwargs["deps"]
+
+    @classmethod
+    def package_name_full(cls, package_name, version):
+        version_number = map(lambda x: int(x), version.split("."))
+        version_nix = "_".join(map(lambda x: str(x), version_number))
+        return "_".join((package_name, version_nix))
+
+    def license_tokens(self):
+        import license_expression as le
+
+        try:
+            # FIXME: ad hoc conversion
+            exception_list = [("EUPL-1.2+", "EUPL-1.2")]
+
+            def sanitize_license_string(license_string, lookups):
+                if not lookups:
+                    return license_string
+                return sanitize_license_string(
+                    license_string.replace(lookups[0][0], lookups[0][1]), lookups[1:]
+                )
+
+            sanitized = sanitize_license_string(self.license, exception_list)
+            licensing = le.get_spdx_licensing()
+            parsed = licensing.parse(sanitized, validate=True)
+            return [s.key for s in licensing.license_symbols(parsed)]
+        except le.ExpressionError as e:
+            print(
+                f'Failed to parse license string "{self.license}" because of {str(e)}'
+            )
+            exit(1)
+
+    def source(self):
+        url = f"https://packages.typst.org/preview/{self.pname}-{self.version}.tar.gz"
+        cmd = [
+            "nix",
+            "store",
+            "prefetch-file",
+            "--unpack",
+            "--hash-type",
+            "sha256",
+            "--refresh",
+            "--extra-experimental-features",
+            "nix-command",
+        ]
+        result = subprocess.run(cmd + [url], capture_output=True, text=True)
+        hash = re.search(r"hash\s+\'(sha256-.{44})\'", result.stderr).groups()[0]
+        return url, hash
+
+    def to_name_full(self):
+        return self.package_name_full(self.pname, self.version)
+
+    def to_attrs(self):
+        deps = set()
+        excludes = list(map(
+            lambda e: os.path.join(self.path, e),
+            self.meta["package"]["exclude"] if "exclude" in self.meta["package"] else [],
+        ))
+        for root, _, files in os.walk(self.path):
+            for file in filter(lambda f: f.split(".")[-1] == "typ", files):
+                file_path = os.path.join(root, file)
+                if file_path in excludes:
+                    continue
+                with open(file_path, "r") as f:
+                    deps.update(
+                        set(
+                            re.findall(
+                                r"^\s*#import\s+\"@preview/([\w|-]+):(\d+.\d+.\d+)\"",
+                                f.read(),
+                                re.MULTILINE,
+                            )
+                        )
+                    )
+        self.deps = list(
+            filter(lambda p: p[0] != self.pname or p[1] != self.version, deps)
+        )
+        source_url, source_hash = self.source()
+
+        return dict(
+            url=source_url,
+            hash=source_hash,
+            typstDeps=[
+                self.package_name_full(p, v)
+                for p, v in sorted(self.deps, key=lambda x: (x[0], Version(x[1])))
+            ],
+            description=self.description,
+            license=self.license_tokens(),
+        ) | (dict(homepage=self.repo) if self.repo else dict())
+
+
+def generate_typst_packages(preview_dir, output_file):
+    package_tree = dict()
+
+    print("Parsing metadata... from", preview_dir)
+    for p in os.listdir(preview_dir):
+        package_dir = os.path.join(preview_dir, p)
+        for v in os.listdir(package_dir):
+            package_version_dir = os.path.join(package_dir, v)
+            with open(
+                os.path.join(package_version_dir, "typst.toml"), "rb"
+            ) as meta_file:
+                try:
+                    package = TypstPackage(
+                        pname=p,
+                        version=v,
+                        meta=tomli.load(meta_file),
+                        path=package_version_dir,
+                    )
+                    if package.pname in package_tree:
+                        package_tree[package.pname][v] = package
+                    else:
+                        package_tree[package.pname] = dict({v: package})
+                except tomli.TOMLDecodeError:
+                    print("Invalid typst.toml:", package_version_dir)
+
+    with open(output_file, "wb") as typst_packages:
+
+        def generate_package(pname, package_subtree):
+            sorted_keys = sorted(package_subtree.keys(), key=Version, reverse=True)
+            print(f"Generating metadata for {pname}")
+            return {
+                pname: OrderedDict(
+                    (k, package_subtree[k].to_attrs()) for k in sorted_keys
+                )
+            }
+
+        with concurrent.futures.ThreadPoolExecutor(max_workers=100) as executor:
+            sorted_packages = sorted(package_tree.items(), key=lambda x: x[0])
+            futures = list()
+            for pname, psubtree in sorted_packages:
+                futures.append(executor.submit(generate_package, pname, psubtree))
+            packages = OrderedDict(
+                (package, subtree)
+                for future in futures
+                for package, subtree in future.result().items()
+            )
+            print(f"Writing metadata... to {output_file}")
+            tomli_w.dump(packages, typst_packages)
+
+
+def main(args):
+    PREVIEW_DIR = "packages/preview"
+    TYPST_PACKAGE_TARBALL_URL = (
+        "https://github.com/typst/packages/archive/refs/heads/main.tar.gz"
+    )
+
+    directory = args.directory
+    if not directory:
+        tempdir = tempfile.mkdtemp()
+        print(tempdir)
+        typst_tarball = os.path.join(tempdir, "main.tar.gz")
+
+        print(
+            "Downloading Typst packages source from {} to {}".format(
+                TYPST_PACKAGE_TARBALL_URL, typst_tarball
+            )
+        )
+        with request.urlopen(
+            request.Request(TYPST_PACKAGE_TARBALL_URL), timeout=15.0
+        ) as response:
+            if response.status == 200:
+                with open(typst_tarball, "wb+") as f:
+                    f.write(response.read())
+            else:
+                print("Download failed")
+                exit(1)
+        with tarfile.open(typst_tarball) as tar:
+            tar.extractall(path=tempdir, filter="data")
+        directory = os.path.join(tempdir, "packages-main")
+    directory = os.path.abspath(directory)
+
+    generate_typst_packages(
+        os.path.join(directory, PREVIEW_DIR),
+        args.output,
+    )
+
+    exit(0)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-d", "--directory", help="Local Typst Universe repository", default=None
+    )
+    parser.add_argument(
+        "-o",
+        "--output",
+        help="Output file",
+        default=os.path.join(os.path.abspath("."), "typst-packages-from-universe.toml"),
+    )
+    args = parser.parse_args()
+    main(args)

pkgs/build-support/build-typst-package.nix

@@ -0,0 +1,60 @@
+{
+  lib,
+  stdenvNoCC,
+}:
+
+/**
+  `buildTypstPackage` is a helper builder for typst packages.
+
+  # Inputs
+
+    `attrs`
+    : attrs for stdenvNoCC.mkDerivation + typstDeps (a list of `buildTypstPackage` derivations)
+
+  # Example
+  ```nix
+  { buildTypstPackage, typstPackages }:
+
+  buildTypstPackage {
+    pname = "example";
+    version = "0.0.1";
+    src = ./.;
+    typstDeps = with typstPackages; [ oxifmt ];
+  }
+  ```
+*/
+
+lib.extendMkDerivation {
+  constructDrv = stdenvNoCC.mkDerivation;
+
+  excludeDrvArgNames = [
+    "typstDeps"
+  ];
+
+  extendDrvArgs =
+    finalAttrs:
+    {
+      typstDeps ? [ ],
+      ...
+    }@attrs:
+    {
+      name = "typst-package-${finalAttrs.pname}-${finalAttrs.version}";
+
+      installPhase =
+        let
+          outDir = "$out/lib/typst-packages/${finalAttrs.pname}/${finalAttrs.version}";
+        in
+        ''
+          runHook preInstall
+          mkdir -p ${outDir}
+          cp -r . ${outDir}
+          runHook postInstall
+        '';
+
+      propagatedBuildInputs = typstDeps;
+
+      passthru = {
+        inherit typstDeps;
+      };
+    };
+}

pkgs/by-name/ty/typst/package.nix

@@ -7,6 +7,7 @@
   openssl,
   nix-update-script,
   versionCheckHook,
+  callPackage,
 }:
 
 rustPlatform.buildRustPackage (finalAttrs: {
@@ -56,7 +57,11 @@ rustPlatform.buildRustPackage (finalAttrs: {
   nativeInstallCheckInputs = [ versionCheckHook ];
   versionCheckProgramArg = "--version";
 
-  passthru.updateScript = nix-update-script { };
+  passthru = {
+    updateScript = nix-update-script { };
+    packages = callPackage ./typst-packages.nix { };
+    withPackages = callPackage ./with-packages.nix { };
+  };
 
   meta = {
     changelog = "https://github.com/typst/typst/releases/tag/v${finalAttrs.version}";

pkgs/by-name/ty/typst/typst-packages.nix

@@ -0,0 +1,52 @@
+{
+  lib,
+  callPackage,
+}:
+
+let
+  toPackageName = name: version: "${name}_${lib.replaceStrings [ "." ] [ "_" ] version}";
+in
+lib.makeExtensible (
+  final:
+  lib.recurseIntoAttrs (
+    lib.foldlAttrs (
+      packageSet: pname: versionSet:
+      packageSet
+      // (lib.foldlAttrs (
+        subPackageSet: version: packageSpec:
+        subPackageSet
+        // {
+          ${toPackageName pname version} = callPackage (
+            {
+              lib,
+              buildTypstPackage,
+              fetchzip,
+            }:
+            buildTypstPackage (finalAttrs: {
+              inherit pname version;
+
+              src = fetchzip {
+                inherit (packageSpec) hash;
+                url = "https://packages.typst.org/preview/${finalAttrs.pname}-${finalAttrs.version}.tar.gz";
+                stripRoot = false;
+              };
+
+              typstDeps = builtins.filter (x: x != null) (
+                lib.map (d: (lib.attrsets.attrByPath [ d ] null final)) packageSpec.typstDeps
+              );
+
+              meta = {
+                inherit (packageSpec) description;
+                maintainers = with lib.maintainers; [ cherrypiejam ];
+                license = lib.map (lib.flip lib.getAttr lib.licensesSpdx) packageSpec.license;
+              } // (if packageSpec ? "homepage" then { inherit (packageSpec) homepage; } else { });
+            })
+          ) { };
+        }
+      ) { } versionSet)
+      // {
+        ${pname} = final.${toPackageName pname (lib.last (lib.attrNames versionSet))};
+      }
+    ) { } (lib.importTOML ./typst-packages-from-universe.toml)
+  )
+)

pkgs/by-name/ty/typst/with-packages.nix

@@ -0,0 +1,33 @@
+{
+  lib,
+  buildEnv,
+  typstPackages,
+  makeBinaryWrapper,
+  typst,
+}:
+
+lib.makeOverridable (
+  { ... }@typstPkgs:
+  f:
+  buildEnv {
+    name = "${typst.name}-env";
+
+    paths = lib.foldl' (acc: p: acc ++ lib.singleton p ++ p.propagatedBuildInputs) [ ] (f typstPkgs);
+
+    pathsToLink = [ "/lib/typst-packages" ];
+
+    nativeBuildInputs = [ makeBinaryWrapper ];
+
+    postBuild = ''
+      export TYPST_LIB_DIR="$out/lib/typst/packages"
+      mkdir -p $TYPST_LIB_DIR
+
+      mv $out/lib/typst-packages $TYPST_LIB_DIR/preview
+
+      cp -r ${typst}/share $out/share
+      mkdir -p $out/bin
+
+      makeWrapper "${lib.getExe typst}" "$out/bin/typst" --set TYPST_PACKAGE_CACHE_PATH $TYPST_LIB_DIR
+    '';
+  }
+) typstPackages

pkgs/top-level/all-packages.nix

@@ -16442,6 +16442,10 @@ with pkgs;
     inherit (darwin.apple_sdk.frameworks) Security;
   };
 
+  buildTypstPackage = callPackage ../build-support/build-typst-package.nix { };
+
+  typstPackages = typst.packages;
+
   ueberzug = with python3Packages; toPythonApplication ueberzug;
 
   ueberzugpp = callPackage ../by-name/ue/ueberzugpp/package.nix {