diff --git a/README.md b/README.md index 5ec9406..b05a4c6 100644 --- a/README.md +++ b/README.md @@ -116,7 +116,7 @@ usage: ros2nix [-h] [--output-dir OUTPUT_DIR] [--fetch] [--use-per-package-src] [--patches | --no-patches] [--distro DISTRO] [--src-param SRC_PARAM] [--source-root SOURCE_ROOT] - [--do-check] [--extra-build-inputs DEP1,DEP2,...] + [--no-cache] [--do-check] [--extra-build-inputs DEP1,DEP2,...] [--extra-propagated-build-inputs DEP1,DEP2,...] [--extra-check-inputs DEP1,DEP2,...] [--extra-native-build-inputs DEP1,DEP2,...] [--flake] @@ -172,6 +172,8 @@ options: Set sourceRoot attribute value in the generated Nix expression. Substring '{package_name}' gets replaced with the package name. (default: None) + --no-cache Don't use cache of git checkout sha265 hashes across + generation runs. (default: False) --do-check Set doCheck attribute to true (default: False) --extra-build-inputs DEP1,DEP2,... Additional dependencies to add to the generated Nix diff --git a/ros2nix/ros2nix.py b/ros2nix/ros2nix.py index a0dc3bf..b653ef2 100755 --- a/ros2nix/ros2nix.py +++ b/ros2nix/ros2nix.py @@ -15,6 +15,7 @@ import re import subprocess import sys from contextlib import contextmanager +from pathlib import Path from textwrap import dedent, indent from typing import Iterable, Set, List @@ -26,6 +27,23 @@ from superflore.utils import err, ok, resolve_dep, warn from .nix_expression import NixExpression, NixLicense +# Copied from https://github.com/srstevenson/xdg-base-dirs +# Copyright © Scott Stevenson +# Less than 10 lines, no need to mention full ISC license here. +def _path_from_env(variable: str, default: Path) -> Path: + if (value := os.environ.get(variable)) and (path := Path(value)).is_absolute(): + return path + return default + + +def xdg_cache_home() -> Path: + """Return a Path corresponding to XDG_CACHE_HOME.""" + return _path_from_env("XDG_CACHE_HOME", Path.home() / ".cache") + + +cache_file = xdg_cache_home() / "ros2nix" / "git-cache.json" + + def resolve_dependencies(deps: Iterable[str]) -> Set[str]: return set(itertools.chain.from_iterable(map(resolve_dependency, deps))) @@ -280,6 +298,11 @@ def ros2nix(args): help="Set sourceRoot attribute value in the generated Nix expression. " "Substring '{package_name}' gets replaced with the package name.", ) + parser.add_argument( + "--no-cache", + action="store_true", + help="Don't use cache of git checkout sha265 hashes across generation runs.", + ) parser.add_argument( "--do-check", action="store_true", @@ -357,6 +380,9 @@ def ros2nix(args): expressions: dict[str, str] = {} git_cache = {} + if not args.no_cache and os.path.exists(cache_file): + with open(cache_file) as f: + git_cache = json.load(f) patch_filenames = set() for source in args.source: @@ -414,19 +440,23 @@ def ros2nix(args): cwd=srcdir, shell=True).decode().strip() if args.use_per_package_src: - # we need to get merge_base again to filter out applied patches from the package git hash - merge_base = merge_base_to_upstream(head) + # Set head to point to the last commit the subdirectory was changed. This is + # not strictly necessary, but it will increase hit rate of git_cache. + merge_base = merge_base_to_upstream(head) # filter out locally applied patches head = check_output(f"git rev-list {merge_base} -1 -- .".split()) - if not args.use_per_package_src and toplevel in git_cache: # only use cache if not using separate checkout per package - info = git_cache[toplevel] - upstream_rev = info["rev"] - else: - # Latest commit present in the upstream repo. If - # the local repository doesn't have additional - # commits, it is the same as HEAD. Should work - # even with detached HEAD. - upstream_rev = merge_base_to_upstream(head) + def cache_key(url, prefix): + if args.use_per_package_src: + return f"{url}?dir={prefix}" + return url + + # Latest commit present in the upstream repo. If + # the local repository doesn't have additional + # commits, it is the same as HEAD. Should work + # even with detached HEAD. + upstream_rev = merge_base_to_upstream(head) + info = git_cache.get(cache_key(url, prefix)) + if info is None or info["rev"] != upstream_rev: info = json.loads( subprocess.check_output( ["nix-prefetch-git", "--quiet"] @@ -438,7 +468,7 @@ def ros2nix(args): + [toplevel, upstream_rev], ).decode() ) - git_cache[toplevel] = info + git_cache[cache_key(url, prefix)] = {k : info[k] for k in ["rev", "sha256"]} match = re.match("https://github.com/(?P[^/]*)/(?P.*?)(.git|/.*)?$", url) sparse_checkout = f"""sparseCheckout = ["{prefix}"]; @@ -568,6 +598,11 @@ def ros2nix(args): generate_default(args) # TODO generate also release.nix (for testing/CI)? + if not args.no_cache: + os.makedirs(os.path.dirname(cache_file), exist_ok=True) + with open(cache_file, "w") as f: + json.dump(git_cache, f) + if args.compare and compare_failed: err("Some files are not up-to-date") return 2