From 411bd08ac84045732164f118b0791cc7846544cd Mon Sep 17 00:00:00 2001 From: Michal Sojka Date: Sun, 25 Aug 2024 17:08:47 +0200 Subject: [PATCH] Extend package generator to work for me --- ros2nix/nix_expression.py | 70 +++++++++------ ros2nix/ros2nix.py | 178 +++++++++++++++++++++++--------------- 2 files changed, 154 insertions(+), 94 deletions(-) diff --git a/ros2nix/nix_expression.py b/ros2nix/nix_expression.py index e096e6e..6c4e5db 100644 --- a/ros2nix/nix_expression.py +++ b/ros2nix/nix_expression.py @@ -26,7 +26,7 @@ from operator import attrgetter import os from textwrap import dedent from time import gmtime, strftime -from typing import Iterable, Set +from typing import Iterable, Set, Optional import urllib.parse from superflore.utils import get_license @@ -85,7 +85,6 @@ class NixLicense: class NixExpression: def __init__(self, name: str, version: str, - src_url: str, src_sha256: str, description: str, licenses: Iterable[NixLicense], distro_name: str, build_type: str, @@ -93,15 +92,20 @@ class NixExpression: propagated_build_inputs: Set[str] = set(), check_inputs: Set[str] = set(), native_build_inputs: Set[str] = set(), - propagated_native_build_inputs: Set[str] = set() + propagated_native_build_inputs: Set[str] = set(), + src_param: Optional[str] = None, + src_url: Optional[str] = None, src_sha256: Optional[str] = None, + source_root: Optional[str] = None, ) -> None: self.name = name self.version = version self.src_url = src_url self.src_sha256 = src_sha256 + self.src_param = src_param # fetchurl's naming logic cannot account for URL parameters self.src_name = os.path.basename( urllib.parse.urlparse(self.src_url).path) + self.source_root = source_root self.description = description self.licenses = licenses @@ -123,51 +127,65 @@ class NixExpression: def _to_nix_parameter(dep: str) -> str: return dep.split('.')[0] - def get_text(self, distributor: str, license_name: str) -> str: + def get_text(self, distributor: Optional[str], license_name: Optional[str]) -> str: """ Generate the Nix expression, given the distributor line and the license text. """ ret = [] - ret += dedent(''' - # Copyright {} {} - # Distributed under the terms of the {} license - ''').format( - strftime("%Y", gmtime()), distributor, - license_name) + if distributor or license_name: + ret += dedent(''' + # Copyright {} {} + # Distributed under the terms of the {} license + + ''').format( + strftime("%Y", gmtime()), distributor, + license_name) + + args = [ "lib", "buildRosPackage" ] + + assert bool(self.src_url or self.src_name or self.src_sha256) ^ bool(self.src_param) + + if self.src_param: + src = self.src_param + args.append(self.src_param) + else: + src = f'''fetchurl {{ + url = "{self.src_url}"; + name = "{self.src_name}"; + sha256 = "{self.src_sha256}"; + }}''' + args.append("fetchurl") + + args.extend(sorted(set(map(self._to_nix_parameter, + self.build_inputs | + self.propagated_build_inputs | + self.check_inputs | + self.native_build_inputs | + self.propagated_native_build_inputs)))) + ret += '{ ' + ', '.join(args) + ' }:' - ret += '{ lib, buildRosPackage, fetchurl, ' + \ - ', '.join(sorted(set(map(self._to_nix_parameter, - self.build_inputs | - self.propagated_build_inputs | - self.check_inputs | - self.native_build_inputs | - self.propagated_native_build_inputs))) - ) + ' }:' ret += dedent(''' buildRosPackage {{ pname = "ros-{distro_name}-{name}"; version = "{version}"; - src = fetchurl {{ - url = "{src_url}"; - name = "{src_name}"; - sha256 = "{src_sha256}"; - }}; + src = {src}; buildType = "{build_type}"; ''').format( distro_name=self.distro_name, name=self.name, version=self.version, - src_url=self.src_url, - src_name=self.src_name, - src_sha256=self.src_sha256, + src=src, build_type=self.build_type) + if self.source_root: + ret += f' sourceRoot = "{self.source_root}";\n' + if self.build_inputs: ret += " buildInputs = {};\n" \ .format(self._to_nix_list(sorted(self.build_inputs))) diff --git a/ros2nix/ros2nix.py b/ros2nix/ros2nix.py index e4fbe00..0544761 100755 --- a/ros2nix/ros2nix.py +++ b/ros2nix/ros2nix.py @@ -3,14 +3,16 @@ # Copyright 2019-2024 Ben Wolsieffer # Copyright 2024 Michal Sojka +import os import argparse import itertools +import subprocess from catkin_pkg.package import parse_package_string from rosinstall_generator.distro import get_distro from superflore.PackageMetadata import PackageMetadata from superflore.exceptions import UnresolvedDependency from superflore.generators.nix.nix_package import NixPackage -from superflore.generators.nix.nix_expression import NixExpression, NixLicense +from nix_expression import NixExpression, NixLicense from superflore.utils import (download_file, get_distro_condition_context, get_distros, get_pkg_version, info, resolve_dep, retry_on_exception, warn) @@ -19,9 +21,6 @@ from superflore.utils import err from superflore.utils import ok from superflore.utils import warn -org = "Open Source Robotics Foundation" # TODO change -org_license = "BSD" # TODO change - def resolve_dependencies(deps: Iterable[str]) -> Set[str]: return set(itertools.chain.from_iterable( @@ -50,83 +49,126 @@ def get_dependencies_as_set(pkg, dep_type): return set([d.name for d in deps[dep_type] if d.evaluated_condition is not False]) +def get_output_file_name(pkg, args): + if args.output_as_ros_pkg_name: + fn = f"{pkg.name}.nix" + elif args.output_as_nix_pkg_name: + fn = f"{NixPackage.normalize_name(pkg.name)}.nix" + else: + fn = args.output + + return os.path.join(args.output_dir, fn) + def main(args): - parser = argparse.ArgumentParser() - parser.add_argument("xml", help="Path to package.xml") + parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument("source", nargs="+", help="Path to package.xml") # TODO or a directory containing package.xml or an ") + + group = parser.add_mutually_exclusive_group() + group.add_argument("--output", default="package.nix", help="Output filename") + group.add_argument("--output-as-ros-pkg-name", action="store_true", help="Name output file based on ROS package name, e.g., package_name.nix") + group.add_argument("--output-as-nix-pkg-name", action="store_true", help="Name output file based on Nix package name, e.g., package-name.nix") + + parser.add_argument("--output-dir", default=".", help="Directory to store output files in") + + parser.add_argument("--distro", default="rolling", + help="ROS distro (used as a context for evaluation of conditions in package.xml and in the name of the Nix expression)") + parser.add_argument("--src-param", + help="Parameter name in arguments of the generated function to be used as a src attribute") + parser.add_argument("--source-root", + help="sourceRoot attribute value in the generated Nix expression. Substring '{package_name}' gets replaced with package name.") + + parser.add_argument("--nixfmt", action="store_true", help="Format the resulting expression with nixfmt") + + parser.add_argument("--copyright-holder") + parser.add_argument("--license", help="License of the generated Nix expression, e.g. 'BSD'") + args = parser.parse_args() - try: - distro_name = "humble" + for source in args.source: + try: + with open(source, 'r') as f: + package_xml = f.read() - with open(args.xml, 'r') as f: - package_xml = f.read() + pkg = parse_package_string(package_xml) + pkg.evaluate_conditions(NixPackage._get_condition_context(args.distro)) - pkg = parse_package_string(package_xml) - pkg.evaluate_conditions(NixPackage._get_condition_context(distro_name)) + buildtool_deps = get_dependencies_as_set(pkg, "buildtool") + buildtool_export_deps = get_dependencies_as_set(pkg, "buildtool_export") + build_deps = get_dependencies_as_set(pkg, "build") + build_export_deps = get_dependencies_as_set(pkg, "build_export") + exec_deps = get_dependencies_as_set(pkg, "exec") + test_deps = get_dependencies_as_set(pkg, "test") - buildtool_deps = get_dependencies_as_set(pkg, "buildtool") - buildtool_export_deps = get_dependencies_as_set(pkg, "buildtool_export") - build_deps = get_dependencies_as_set(pkg, "build") - build_export_deps = get_dependencies_as_set(pkg, "build_export") - exec_deps = get_dependencies_as_set(pkg, "exec") - test_deps = get_dependencies_as_set(pkg, "test") + # buildtool_depends are added to buildInputs and nativeBuildInputs. + # Some (such as CMake) have binaries that need to run at build time + # (and therefore need to be in nativeBuildInputs. Others (such as + # ament_cmake_*) need to be added to CMAKE_PREFIX_PATH and therefore + # need to be in buildInputs. There is no easy way to distinguish these + # two cases, so they are added to both, which generally works fine. + build_inputs = set(resolve_dependencies( + build_deps | buildtool_deps)) + propagated_build_inputs = resolve_dependencies( + exec_deps | build_export_deps | buildtool_export_deps) + build_inputs -= propagated_build_inputs - # buildtool_depends are added to buildInputs and nativeBuildInputs. - # Some (such as CMake) have binaries that need to run at build time - # (and therefore need to be in nativeBuildInputs. Others (such as - # ament_cmake_*) need to be added to CMAKE_PREFIX_PATH and therefore - # need to be in buildInputs. There is no easy way to distinguish these - # two cases, so they are added to both, which generally works fine. - build_inputs = set(resolve_dependencies( - build_deps | buildtool_deps)) - propagated_build_inputs = resolve_dependencies( - exec_deps | build_export_deps | buildtool_export_deps) - build_inputs -= propagated_build_inputs + check_inputs = resolve_dependencies(test_deps) + check_inputs -= build_inputs - check_inputs = resolve_dependencies(test_deps) - check_inputs -= build_inputs + native_build_inputs = resolve_dependencies( + buildtool_deps | buildtool_export_deps) - native_build_inputs = resolve_dependencies( - buildtool_deps | buildtool_export_deps) + kwargs = {} - derivation = NixExpression( - name=NixPackage.normalize_name(pkg.name), - version=pkg.version, - src_url="src_uri", # TODO - src_sha256="src_sha256", - description=pkg.description, - licenses=map(NixLicense, pkg.licenses), - distro_name=distro_name, - build_type=pkg.get_build_type(), - build_inputs=build_inputs, - propagated_build_inputs=propagated_build_inputs, - check_inputs=check_inputs, - native_build_inputs=native_build_inputs) + if args.src_param: + kwargs["src_param"] = args.src_param + else: + kwargs["src_url"] = "src_uri", # TODO + kwargs["src_sha256"] = "src_sha256", - except Exception as e: - err('Failed to generate derivation for package {}!'.format(pkg)) - raise e + if args.source_root: + kwargs["source_root"] = args.source_root.replace('{package_name}', pkg.name) - try: - derivation_text = derivation.get_text(org, org_license) - except UnresolvedDependency: - err("'Failed to resolve required dependencies for package {}!" - .format(pkg)) - unresolved = unresolved_dependencies - for dep in unresolved: - err(" unresolved: \"{}\"".format(dep)) - return None, unresolved, None - except Exception as e: - err('Failed to generate derivation for package {}!'.format(pkg)) - raise e + derivation = NixExpression( + name=NixPackage.normalize_name(pkg.name), + version=pkg.version, + description=pkg.description, + licenses=map(NixLicense, pkg.licenses), + distro_name=args.distro, + build_type=pkg.get_build_type(), + build_inputs=build_inputs, + propagated_build_inputs=propagated_build_inputs, + check_inputs=check_inputs, + native_build_inputs=native_build_inputs, **kwargs) - ok(f"Successfully generated derivation for package '{pkg.name}'.") - try: - with open('package.nix', "w") as recipe_file: - recipe_file.write(derivation_text) - except Exception as e: - err("Failed to write derivation to disk!") - raise e + except Exception as e: + err('Failed to generate derivation for package {}!'.format(pkg)) + raise e + + try: + derivation_text = derivation.get_text(args.copyright_holder, args.license) + except UnresolvedDependency: + err("'Failed to resolve required dependencies for package {}!" + .format(pkg)) + unresolved = unresolved_dependencies + for dep in unresolved: + err(" unresolved: \"{}\"".format(dep)) + return None, unresolved, None + except Exception as e: + err('Failed to generate derivation for package {}!'.format(pkg)) + raise e + + if args.nixfmt: + nixfmt = subprocess.Popen(["nixfmt"], stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True) + derivation_text, _ = nixfmt.communicate(input=derivation_text) + + try: + output_file_name = get_output_file_name(pkg, args) + with open(output_file_name, "w") as recipe_file: + recipe_file.write(derivation_text) + ok(f"Successfully generated derivation for package '{pkg.name}' as '{output_file_name}'.") + except Exception as e: + err("Failed to write derivation to disk!") + raise e if __name__ == '__main__':