maubot: add plugins & plugins update script

pkgs/tools/networking/maubot/default.nix

@@ -1,6 +1,7 @@
 { lib
 , fetchPypi
 , fetchpatch
+, callPackage
 , runCommand
 , python3
 , encryptionSupport ? true
@@ -88,15 +89,6 @@ let
       rm $out/example-config.yaml
     '';
 
-    passthru = {
-      inherit python;
-      tests = {
-        simple = runCommand "${pname}-tests" { } ''
-          ${maubot}/bin/mbc --help > $out
-        '';
-      };
-    };
-
     # Setuptools is trying to do python -m maubot test
     dontUseSetuptoolsCheck = true;
 
@@ -104,6 +96,35 @@ let
       "maubot"
     ];
 
+    passthru = let
+      wrapper = callPackage ./wrapper.nix {
+        unwrapped = maubot;
+        python3 = python;
+      };
+    in
+    {
+      tests = {
+        simple = runCommand "${pname}-tests" { } ''
+          ${maubot}/bin/mbc --help > $out
+        '';
+      };
+
+      inherit python;
+
+      plugins = callPackage ./plugins {
+        maubot = maubot;
+        python3 = python;
+      };
+
+      withPythonPackages = pythonPackages: wrapper { inherit pythonPackages; };
+
+      # This adds the plugins to lib/maubot-plugins
+      withPlugins = plugins: wrapper { inherit plugins; };
+
+      # This changes example-config.yaml in module directory
+      withBaseConfig = baseConfig: wrapper { inherit baseConfig; };
+    };
+
     meta = with lib; {
       description = "A plugin-based Matrix bot system written in Python";
       homepage = "https://maubot.xyz/";

pkgs/tools/networking/maubot/plugins/default.nix

@@ -0,0 +1,64 @@
+{ lib
+, fetchgit
+, fetchFromGitHub
+, fetchFromGitLab
+, fetchFromGitea
+, stdenvNoCC
+, callPackage
+, maubot
+, python3
+, poetry
+, formats
+}:
+
+let
+  # pname: plugin id (example: xyz.maubot.echo)
+  # version: plugin version
+  # other attributes are passed directly to stdenv.mkDerivation (you at least need src)
+  buildMaubotPlugin = attrs@{ version, pname, base_config ? null, ... }:
+    stdenvNoCC.mkDerivation (builtins.removeAttrs attrs [ "base_config" ] // {
+      pluginName = "${pname}-v${version}.mbp";
+      nativeBuildInputs = (attrs.nativeBuildInputs or [ ]) ++ [ maubot ];
+      buildPhase = ''
+        runHook preBuild
+
+        mbc build
+
+        runHook postBuild
+      '';
+
+      postPatch = lib.optionalString (base_config != null) ''
+        [ -e base-config.yaml ] || (echo "base-config.yaml doesn't exist, can't override it" && exit 1)
+        cp "${if builtins.isPath base_config || lib.isDerivation base_config then base_config
+          else if builtins.isString base_config then builtins.toFile "base-config.yaml" base_config
+          else (formats.yaml { }).generate "base-config.yaml" base_config}" base-config.yaml
+      '' + attrs.postPatch or "";
+
+      installPhase = ''
+        runHook preInstall
+
+        mkdir -p $out/lib/maubot-plugins
+        install -m 444 $pluginName $out/lib/maubot-plugins
+
+        runHook postInstall
+      '';
+    });
+
+  generated = import ./generated.nix {
+    inherit lib fetchgit fetchFromGitHub fetchFromGitLab
+      fetchFromGitea python3 poetry buildMaubotPlugin;
+  };
+in
+generated // {
+  inherit buildMaubotPlugin;
+
+  allOfficialPlugins =
+    builtins.filter
+      (x: x.isOfficial && !x.meta.broken)
+      (builtins.attrValues generated);
+
+  allPlugins =
+    builtins.filter
+      (x: !x.meta.broken)
+      (builtins.attrValues generated);
+}

pkgs/tools/networking/maubot/plugins/generated.nix

@@ -0,0 +1,74 @@
+{ lib
+, fetchgit
+, fetchFromGitHub
+, fetchFromGitLab
+, fetchFromGitea
+, python3
+, poetry
+, buildMaubotPlugin
+}:
+
+let
+  json = builtins.fromJSON (builtins.readFile ./generated.json);
+in
+
+lib.flip builtins.mapAttrs json (name: entry:
+let
+  inherit (entry) manifest;
+
+  resolveDeps = deps: map
+    (name:
+      let
+        packageName = builtins.head (builtins.match "([^~=<>]*).*" name);
+        lower = lib.toLower packageName;
+        dash = builtins.replaceStrings ["_"] ["-"] packageName;
+        lowerDash = builtins.replaceStrings ["_"] ["-"] lower;
+      in
+        python3.pkgs.${packageName}
+        or python3.pkgs.${lower}
+        or python3.pkgs.${dash}
+        or python3.pkgs.${lowerDash}
+        or null)
+    (builtins.filter (x: x != "maubot" && x != null) deps);
+
+  reqDeps = resolveDeps (lib.toList (manifest.dependencies or null));
+  optDeps = resolveDeps (lib.toList (manifest.soft_dependencies or null));
+in
+
+lib.makeOverridable buildMaubotPlugin (entry.attrs // {
+  pname = manifest.id;
+  inherit (manifest) version;
+
+  src =
+    if entry?github then fetchFromGitHub entry.github
+    else if entry?git then fetchgit entry.git
+    else if entry?gitlab then fetchFromGitLab entry.gitlab
+    else if entry?gitea then fetchFromGitea entry.gitea
+    else throw "Invalid generated entry for ${manifest.id}: missing source";
+
+  propagatedBuildInputs = builtins.filter (x: x != null) (reqDeps ++ optDeps);
+
+  passthru.isOfficial = entry.isOfficial or false;
+
+  meta = entry.attrs.meta // {
+    license =
+      let
+        spdx = entry.attrs.meta.license or manifest.license or "unfree";
+        spdxLicenses = builtins.listToAttrs
+          (map (x: lib.nameValuePair x.spdxId x) (builtins.filter (x: x?spdxId) (builtins.attrValues lib.licenses)));
+      in
+      spdxLicenses.${spdx};
+    broken = builtins.any (x: x == null) reqDeps;
+  };
+} // lib.optionalAttrs (entry.isPoetry or false) {
+  nativeBuildInputs = [
+    poetry
+    (python3.withPackages (p: with p; [ toml ruamel-yaml isort ]))
+  ];
+
+  preBuild = lib.optionalString (entry?attrs.preBuild) (entry.attrs.preBuild + "\n") + ''
+    export HOME=$(mktemp -d)
+    [[ ! -d scripts ]] || patchShebangs --build scripts
+    make maubot.yaml
+  '';
+}))

pkgs/tools/networking/maubot/plugins/update.py

@@ -0,0 +1,200 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i python3 -p git nurl "(python3.withPackages (ps: with ps; [ toml gitpython requests ruamel-yaml ]))"
+
+import git
+import json
+import os
+import subprocess
+import ruamel.yaml
+import sys
+import toml
+import zipfile
+
+from typing import Dict, List
+
+HOSTNAMES = {
+    'git.skeg1.se': 'gitlab',
+    'edugit.org': 'gitlab',
+    'codeberg.org': 'gitea',
+}
+PLUGINS: Dict[str, dict] = {}
+
+yaml = ruamel.yaml.YAML(typ='safe')
+
+TMP = os.environ.get('TEMPDIR', '/tmp')
+
+def process_repo(path: str, official: bool):
+    global PLUGINS
+    with open(path, 'rt') as f:
+        data = yaml.load(f)
+    name, repourl, license, desc = data['name'], data['repo'], data['license'], data['description']
+    origurl = repourl
+    if '/' in name or ' ' in name:
+        name = os.path.split(path)[-1].removesuffix('.yaml')
+    name = name.replace('_', '-')
+    if name in PLUGINS.keys():
+        raise ValueError(f'Duplicate plugin {name}, refusing to continue')
+    repodir = os.path.join(TMP, 'maubot-plugins', name)
+    plugindir = repodir
+    if '/tree/' in repourl:
+        repourl, rev_path = repourl.split('/tree/')
+        rev, subdir = rev_path.strip('/').split('/')
+        plugindir = os.path.join(plugindir, subdir)
+    else:
+        rev = None
+        subdir = None
+
+    if repourl.startswith('http:'):
+        repourl = 'https' + repourl[4:]
+    repourl = repourl.rstrip('/')
+    if not os.path.exists(repodir):
+        print('Fetching', name)
+        repo = git.Repo.clone_from(repourl + '.git', repodir)
+    else:
+        repo = git.Repo(repodir)
+    tags = sorted(repo.tags, key=lambda t: t.commit.committed_datetime)
+    tags = list(filter(lambda x: 'rc' not in str(x), tags))
+    if tags:
+        repo.git.checkout(tags[-1])
+        rev = str(tags[-1])
+    else:
+        rev = str(repo.commit('HEAD'))
+    ret: dict = {'attrs':{}}
+    if subdir:
+        ret['attrs']['postPatch'] = f'cd {subdir}'
+    domain, query = repourl.removeprefix('https://').split('/', 1)
+    hash = subprocess.run([
+        'nurl',
+        '--hash',
+        f'file://{repodir}',
+        rev
+    ], capture_output=True, check=True).stdout.decode('utf-8')
+    ret['attrs']['meta'] = {
+        'description': desc,
+        'homepage': origurl,
+    }
+    if domain.endswith('github.com'):
+        owner, repo = query.split('/')
+        ret['github'] = {
+            'owner': owner,
+            'repo': repo,
+            'rev': rev,
+            'hash': hash,
+        }
+        ret['attrs']['meta']['downloadPage'] = f'{repourl}/releases'
+        ret['attrs']['meta']['changelog'] = f'{repourl}/releases'
+        repobase = f'{repourl}/blob/{rev}'
+    elif HOSTNAMES.get(domain, 'gitea' if 'gitea.' in domain or 'forgejo.' in domain else None) == 'gitea':
+        owner, repo = query.split('/')
+        ret['gitea'] = {
+            'domain': domain,
+            'owner': owner,
+            'repo': repo,
+            'rev': rev,
+            'hash': hash,
+        }
+        repobase = f'{repourl}/src/commit/{rev}'
+        ret['attrs']['meta']['downloadPage'] = f'{repourl}/releases'
+        ret['attrs']['meta']['changelog'] = f'{repourl}/releases'
+    elif HOSTNAMES.get(domain, 'gitlab' if 'gitlab.' in domain else None) == 'gitlab':
+        owner, repo = query.split('/')
+        ret['gitlab'] = {
+            'owner': owner,
+            'repo': repo,
+            'rev': rev,
+            'hash': hash,
+        }
+        if domain != 'gitlab.com':
+            ret['gitlab']['domain'] = domain
+        repobase = f'{repourl}/-/blob/{rev}'
+    else:
+        raise ValueError(f'Is {domain} Gitea or Gitlab, or something else? Please specify in the Python script!')
+    if os.path.exists(os.path.join(plugindir, 'CHANGELOG.md')):
+        ret['attrs']['meta']['changelog'] = f'{repobase}/CHANGELOG.md'
+    if os.path.exists(os.path.join(plugindir, 'maubot.yaml')):
+        with open(os.path.join(plugindir, 'maubot.yaml'), 'rt') as f:
+            ret['manifest'] = yaml.load(f)
+    elif os.path.exists(os.path.join(plugindir, 'pyproject.toml')):
+        ret['isPoetry'] = True
+        with open(os.path.join(plugindir, 'pyproject.toml'), 'rt') as f:
+            data = toml.load(f)
+        deps = []
+        for key, val in data['tool']['poetry'].get('dependencies', {}).items():
+            if key in ['maubot', 'mautrix', 'python']:
+                continue
+            reqs = []
+            for req in val.split(','):
+                reqs.extend(poetry_to_pep(req))
+            deps.append(key + ', '.join(reqs))
+        ret['manifest'] = data['tool']['maubot']
+        ret['manifest']['id'] = data['tool']['poetry']['name']
+        ret['manifest']['version'] = data['tool']['poetry']['version']
+        ret['manifest']['license'] = data['tool']['poetry']['license']
+        if deps:
+            ret['manifest']['dependencies'] = deps
+    else:
+        raise ValueError(f'No maubot.yaml or pyproject.toml found in {repodir}')
+    # normalize non-spdx-conformant licenses this way
+    # (and fill out missing license info)
+    if 'license' not in ret['manifest'] or ret['manifest']['license'] in ['GPLv3', 'AGPL 3.0']:
+        ret['attrs']['meta']['license'] = license
+    elif ret['manifest']['license'] != license:
+        print(f"Warning: licenses for {repourl} don't match! {ret['manifest']['license']} != {license}")
+    if official:
+        ret['isOfficial'] = official
+    PLUGINS[name] = ret
+
+def next_incomp(ver_s: str) -> str:
+    ver = ver_s.split('.')
+    zero = False
+    for i in range(len(ver)):
+        try:
+            seg = int(ver[i])
+        except ValueError:
+            if zero:
+                ver = ver[:i]
+                break
+            continue
+        if zero:
+            ver[i] = '0'
+        elif seg:
+            ver[i] = str(seg + 1)
+            zero = True
+    return '.'.join(ver)
+
+def poetry_to_pep(ver_req: str) -> List[str]:
+    if '*' in ver_req:
+        raise NotImplementedError('Wildcard poetry versions not implemented!')
+    if ver_req.startswith('^'):
+        return ['>=' + ver_req[1:], '<' + next_incomp(ver_req[1:])]
+    if ver_req.startswith('~'):
+        return ['~=' + ver_req[1:]]
+    return [ver_req]
+
+def main():
+    cache_path = os.path.join(TMP, 'maubot-plugins')
+    if not os.path.exists(cache_path):
+        os.makedirs(cache_path)
+        git.Repo.clone_from('https://github.com/maubot/plugins.maubot.xyz', os.path.join(cache_path, '_repo'))
+    else:
+        pass
+
+    repodir = os.path.join(cache_path, '_repo')
+
+    for suffix, official in (('official', True), ('thirdparty', False)):
+        directory = os.path.join(repodir, 'data', 'plugins', suffix)
+        for plugin_name in os.listdir(directory):
+            process_repo(os.path.join(directory, plugin_name), official)
+
+    if os.path.isdir('pkgs/tools/networking/maubot/plugins'):
+        generated = 'pkgs/tools/networking/maubot/plugins/generated.json'
+    else:
+        script_dir = os.path.dirname(os.path.realpath(__file__))
+        generated = os.path.join(script_dir, 'generated.json')
+
+    with open(generated, 'wt') as file:
+        json.dump(PLUGINS, file, indent='  ', separators=(',', ': '), sort_keys=True)
+        file.write('\n')
+
+if __name__ == '__main__':
+    main()

pkgs/tools/networking/maubot/wrapper.nix

@@ -0,0 +1,70 @@
+{ lib
+, symlinkJoin
+, runCommand
+, unwrapped
+, python3
+, formats
+}:
+
+let wrapper = { pythonPackages ? (_: [ ]), plugins ? (_: [ ]), baseConfig ? null }:
+  let
+    plugins' = plugins unwrapped.plugins;
+    extraPythonPackages = builtins.concatLists (map (p: p.propagatedBuildInputs or [ ]) plugins');
+  in
+  symlinkJoin {
+    name = "${unwrapped.pname}-with-plugins-${unwrapped.version}";
+
+    inherit unwrapped;
+    paths = lib.optional (baseConfig != null) unwrapped ++ plugins';
+    pythonPath = lib.optional (baseConfig == null) unwrapped ++ pythonPackages python3.pkgs ++ extraPythonPackages;
+
+    nativeBuildInputs = [ python3.pkgs.wrapPython ];
+
+    postBuild = ''
+      rm -f $out/nix-support/propagated-build-inputs
+      rmdir $out/nix-support || true
+      ${lib.optionalString (baseConfig != null) ''
+        rm $out/${python3.sitePackages}/maubot/example-config.yaml
+        substituteAll ${(formats.yaml { }).generate "example-config.yaml" (lib.recursiveUpdate baseConfig {
+          plugin_directories = lib.optionalAttrs (plugins' != []) {
+            load = [ "@out@/lib/maubot-plugins" ] ++ (baseConfig.plugin_directories.load or []);
+          };
+          # Normally it should be set to false by default to take it from package
+          # root, but aiohttp doesn't follow symlinks when serving static files
+          # unless follow_symlinks=True is passed. Instead of patching maubot, use
+          # this non-invasive approach
+          # XXX: would patching maubot be better? See:
+          # https://github.com/maubot/maubot/blob/75879cfb9370aade6fa0e84e1dde47222625139a/maubot/server.py#L106
+          server.override_resource_path =
+            if builtins.isNull (baseConfig.server.override_resource_path or null)
+            then "${unwrapped}/${python3.sitePackages}/maubot/management/frontend/build"
+            else baseConfig.server.override_resource_path;
+        })})} $out/${python3.sitePackages}/maubot/example-config.yaml
+        rm -rf $out/bin
+      ''}
+      mkdir -p $out/bin
+      cp $unwrapped/bin/.mbc-wrapped $out/bin/mbc
+      cp $unwrapped/bin/.maubot-wrapped $out/bin/maubot
+      wrapPythonProgramsIn "$out/bin" "${lib.optionalString (baseConfig != null) "$out "}$pythonPath"
+    '';
+
+    passthru = {
+      inherit unwrapped;
+      python = python3;
+      withPythonPackages = filter: wrapper {
+        pythonPackages = pkgs: pythonPackages pkgs ++ filter pkgs;
+        inherit plugins baseConfig;
+      };
+      withPlugins = filter: wrapper {
+        plugins = pkgs: plugins pkgs ++ filter pkgs;
+        inherit pythonPackages baseConfig;
+      };
+      withBaseConfig = baseConfig: wrapper {
+        inherit baseConfig pythonPackages plugins;
+      };
+    };
+
+    meta.priority = (unwrapped.meta.priority or 0) - 1;
+  };
+in
+wrapper