pluginupdate.py: GITHUB_API_TOKEN -> GITHUB_TOKEN; nvim-treesitter/update.py: source from nurr; vimPlugins.nvim-treesitter: update grammars (#405955)

doc/languages-frameworks/vim.section.md

@@ -177,7 +177,7 @@ Finally, there are some plugins that are also packaged in nodePackages because t
 Run the update script with a GitHub API token that has at least `public_repo` access. Running the script without the token is likely to result in rate-limiting (429 errors). For steps on creating an API token, please refer to [GitHub's token documentation](https://docs.github.com/en/free-pro-team@latest/github/authenticating-to-github/creating-a-personal-access-token).
 
 ```sh
-nix-shell -p vimPluginsUpdater --run 'vim-plugins-updater --github-token=mytoken' # or set GITHUB_API_TOKEN environment variable
+nix-shell -p vimPluginsUpdater --run 'vim-plugins-updater --github-token=mytoken' # or set GITHUB_TOKEN environment variable
 ```
 
 Alternatively, set the number of processes to a lower count to avoid rate-limiting.

maintainers/scripts/pluginupdate-py/pluginupdate.py

@@ -558,7 +558,16 @@ class Editor:
         }
 
         for plugin_desc, plugin, redirect in fetched:
-            result[plugin.normalized_name] = (plugin_desc, plugin, redirect)
+            # Check if plugin is a Plugin object and has normalized_name attribute
+            if isinstance(plugin, Plugin) and hasattr(plugin, 'normalized_name'):
+                result[plugin.normalized_name] = (plugin_desc, plugin, redirect)
+            elif isinstance(plugin, Exception):
+                # For exceptions, we can't determine the normalized_name
+                # Just log the error and continue
+                log.error(f"Error fetching plugin {plugin_desc.name}: {plugin!r}")
+            else:
+                # For unexpected types, log the issue
+                log.error(f"Unexpected plugin type for {plugin_desc.name}: {type(plugin)}")
 
         return list(result.values())
 
@@ -615,9 +624,9 @@ class Editor:
             "--github-token",
             "-t",
             type=str,
-            default=os.getenv("GITHUB_API_TOKEN"),
+            default=os.getenv("GITHUB_TOKEN"),
             help="""Allows to set --proc to higher values.
-            Uses GITHUB_API_TOKEN environment variables as the default value.""",
+            Uses GITHUB_TOKEN environment variables as the default value.""",
         )
         common.add_argument(
             "--no-commit",

pkgs/applications/editors/vim/plugins/utils/nvim-treesitter/update-shell.nix

@@ -5,19 +5,16 @@
 with pkgs;
 
 let
-  inherit (vimPlugins) nvim-treesitter;
-
-  neovim = pkgs.neovim.override {
-    configure.packages.all.start = [ nvim-treesitter ];
-  };
+  pythonWithPackages = python3.withPackages (
+    ps: with ps; [
+      requests
+    ]
+  );
 in
 
 mkShell {
   packages = [
-    neovim
     nurl
-    python3
+    pythonWithPackages
   ];
-
-  NVIM_TREESITTER = nvim-treesitter;
 }

pkgs/applications/editors/vim/plugins/utils/nvim-treesitter/update.py

@@ -5,104 +5,128 @@ import json
 import logging
 import os
 import subprocess
-import sys
 from concurrent.futures import ThreadPoolExecutor
 
+import requests
+
 log = logging.getLogger("vim-updater")
 
+NURR_JSON_URL = "https://raw.githubusercontent.com/nvim-neorocks/nurr/main/tree-sitter-parsers.json"
 
-def generate_grammar(lang, rev, cfg):
-    """Generate grammar for a language"""
-    info = cfg["install_info"]
-    url = info["url"]
+def generate_grammar(lang, parser_info):
+    """Generate grammar for a language based on the parser info"""
+    try:
+        if "install_info" not in parser_info:
+            log.warning(f"Parser {lang} does not have install_info, skipping")
+            return ""
 
-    generated = f"""  {lang} = buildGrammar {{
+        install_info = parser_info["install_info"]
+
+        url = install_info["url"]
+        rev = install_info["revision"]
+
+        generated = f"""  {lang} = buildGrammar {{
     language = "{lang}";
     version = "0.0.0+rev={rev[:7]}";
     src = """
 
-    generated += subprocess.check_output(["nurl", url, rev, "--indent=4"], text=True)
-    generated += ";"
+        generated += subprocess.check_output(["nurl", url, rev, "--indent=4"], text=True)
+        generated += ";"
 
-    location = info.get("location")
-    if location:
-        generated += f"""
+        location = install_info.get("location", "")
+        if location:
+            generated += f"""
     location = "{location}";"""
 
-    if info.get("requires_generate_from_grammar"):
-        generated += """
+        if install_info.get("generate", False):
+            generated += """
     generate = true;"""
 
-    generated += f"""
+        generated += f"""
     meta.homepage = "{url}";
   }};
 """
 
-    return generated
+        return generated
+    except Exception as e:
+        log.error(f"Error generating grammar for {lang}: {e}")
+        return ""
 
 
-def update_grammars(nvim_treesitter_dir: str):
-    """
-    The lockfile contains just revisions so we start neovim to dump the
-    grammar information in a better format
-    """
-    # the lockfile
-    cmd = [
-        "nvim",
-        "--headless",
-        "-u",
-        "NONE",
-        "--cmd",
-        f"set rtp^={nvim_treesitter_dir}",
-        "+lua io.write(vim.json.encode(require('nvim-treesitter.parsers').get_parser_configs()))",
-        "+quit!",
-    ]
-    log.debug("Running command: %s", ' '.join(cmd))
-    configs = json.loads(subprocess.check_output(cmd))
+def fetch_nurr_parsers():
+    """Fetch the parser information from nurr repository"""
+    log.info("Fetching parser data from %s", NURR_JSON_URL)
+
+    headers = {}
+    github_token = os.environ.get("GITHUB_TOKEN")
+    if github_token:
+        log.info("Using GITHUB_TOKEN for authentication")
+        headers["Authorization"] = f"token {github_token}"
+    else:
+        log.warning("No GITHUB_TOKEN found. GitHub API requests may be rate-limited.")
+
+    response = requests.get(NURR_JSON_URL, headers=headers, timeout=30)
+    response.raise_for_status()
+    data = response.json()
+
+    try:
+        parsers = data["parsers"]
+    except KeyError:
+        raise ValueError(
+            "Unexpected response from NURR:\n" + json.dumps(data, indent=2)
+        )
+    log.info(f"Successfully fetched {len(parsers)} parsers")
+    return parsers
+
+
+def process_parser_info(parser_info):
+    """Process a single parser info entry and generate grammar for it"""
+    try:
+        lang = parser_info["lang"]
+        return generate_grammar(lang, parser_info)
+    except Exception as e:
+        log.error(f"Error processing parser: {e}")
+        return ""
+
+
+def update_grammars():
+    """Update grammar definitions using nurr's parser information"""
+    parsers_info = fetch_nurr_parsers()
 
     generated_file = """# generated by pkgs/applications/editors/vim/plugins/utils/nvim-treesitter/update.py
+# Using parser data from https://github.com/nvim-neorocks/nurr/blob/main/tree-sitter-parsers.json
 
 {
   buildGrammar,
   """
 
-    # Get the output and format it properly
     nurl_output = subprocess.check_output(["nurl", "-Ls", ","], text=True).strip()
-    # Add proper indentation (2 spaces) to the comma-separated list
     indented_output = nurl_output.replace(",", ",\n  ")
     generated_file += indented_output
-
     generated_file += """,
 }:
 
 {
 """
 
-    lockfile_path = os.path.join(nvim_treesitter_dir, "lockfile.json")
-    log.debug("Opening %s", lockfile_path)
-    with open(lockfile_path) as lockfile_fd:
-        lockfile = json.load(lockfile_fd)
-
-        def _generate_grammar(item):
-            lang, lock = item
-            cfg = configs.get(lang)
-            if not cfg:
-                return ""
-            return generate_grammar(lang, lock["revision"], cfg)
-
-        for generated in ThreadPoolExecutor(max_workers=5).map(
-            _generate_grammar, lockfile.items()
-        ):
+    # Process parsers in parallel for better performance
+    with ThreadPoolExecutor(max_workers=5) as executor:
+        for generated in executor.map(process_parser_info, parsers_info):
             generated_file += generated
 
-        generated_file += "}\n"
+    generated_file += "}\n"
     return generated_file
 
 
 if __name__ == "__main__":
-    generated = update_grammars(sys.argv[1])
+    logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s')
+
+    generated = update_grammars()
     output_path = os.path.join(
         os.path.dirname(__file__),
         "../../nvim-treesitter/generated.nix"
     )
-    open(output_path, "w").write(generated)
+    log.info("Writing output to %s", output_path)
+    with open(output_path, "w") as f:
+        f.write(generated)
+    log.info("Successfully updated grammar definitions")

pkgs/applications/editors/vim/plugins/utils/update.py

@@ -19,10 +19,8 @@
 #
 
 import inspect
-import json
 import logging
 import os
-import subprocess
 import textwrap
 from pathlib import Path
 from typing import List, Tuple
@@ -136,20 +134,7 @@ class VimEditor(pluginupdate.Editor):
         # TODO this should probably be skipped when running outside a nixpkgs checkout
         if self.nvim_treesitter_updated:
             print("updating nvim-treesitter grammars")
-            cmd = [
-                "nix",
-                "build",
-                "vimPlugins.nvim-treesitter.src",
-                "-f",
-                self.nixpkgs,
-                "--print-out-paths",
-            ]
-            log.debug("Running command: %s", " ".join(cmd))
-            nvim_treesitter_dir = subprocess.check_output(
-                cmd, text=True, timeout=90
-            ).strip()
-
-            generated = treesitter.update_grammars(nvim_treesitter_dir)
+            generated = treesitter.update_grammars()
             treesitter_generated_nix_path = os.path.join(
                 NIXPKGS_NVIMTREESITTER_FOLDER, "generated.nix"
             )

pkgs/applications/editors/vim/plugins/utils/updater.nix

@@ -25,6 +25,7 @@ buildPythonApplication {
 
   pythonPath = [
     python3Packages.gitpython
+    python3Packages.requests
   ];
 
   dontUnpack = true;