Extend package generator to work for me

ros2nix/nix_expression.py

@@ -26,7 +26,7 @@ from operator import attrgetter
 import os
 from textwrap import dedent
 from time import gmtime, strftime
-from typing import Iterable, Set
+from typing import Iterable, Set, Optional
 import urllib.parse
 
 from superflore.utils import get_license
@@ -85,7 +85,6 @@ class NixLicense:
 
 class NixExpression:
     def __init__(self, name: str, version: str,
-                 src_url: str, src_sha256: str,
                  description: str, licenses: Iterable[NixLicense],
                  distro_name: str,
                  build_type: str,
@@ -93,15 +92,20 @@ class NixExpression:
                  propagated_build_inputs: Set[str] = set(),
                  check_inputs: Set[str] = set(),
                  native_build_inputs: Set[str] = set(),
-                 propagated_native_build_inputs: Set[str] = set()
+                 propagated_native_build_inputs: Set[str] = set(),
+                 src_param: Optional[str] = None,
+                 src_url: Optional[str] = None, src_sha256: Optional[str] = None,
+                 source_root: Optional[str] = None,
                  ) -> None:
         self.name = name
         self.version = version
         self.src_url = src_url
         self.src_sha256 = src_sha256
+        self.src_param = src_param
         # fetchurl's naming logic cannot account for URL parameters
         self.src_name = os.path.basename(
             urllib.parse.urlparse(self.src_url).path)
+        self.source_root = source_root
 
         self.description = description
         self.licenses = licenses
@@ -123,51 +127,65 @@ class NixExpression:
     def _to_nix_parameter(dep: str) -> str:
         return dep.split('.')[0]
 
-    def get_text(self, distributor: str, license_name: str) -> str:
+    def get_text(self, distributor: Optional[str], license_name: Optional[str]) -> str:
         """
         Generate the Nix expression, given the distributor line
         and the license text.
         """
 
         ret = []
-        ret += dedent('''
-        # Copyright {} {}
-        # Distributed under the terms of the {} license
 
-        ''').format(
+        if distributor or license_name:
-            strftime("%Y", gmtime()), distributor,
+            ret += dedent('''
-            license_name)
+            # Copyright {} {}
+            # Distributed under the terms of the {} license
+
+            ''').format(
+                strftime("%Y", gmtime()), distributor,
+                license_name)
+
+        args = [ "lib", "buildRosPackage" ]
+
+        assert bool(self.src_url or self.src_name or self.src_sha256) ^ bool(self.src_param)
+
+        if self.src_param:
+            src = self.src_param
+            args.append(self.src_param)
+        else:
+            src = f'''fetchurl {{
+                url = "{self.src_url}";
+                name = "{self.src_name}";
+                sha256 = "{self.src_sha256}";
+              }}'''
+            args.append("fetchurl")
+
+        args.extend(sorted(set(map(self._to_nix_parameter,
+                                   self.build_inputs |
+                                   self.propagated_build_inputs |
+                                   self.check_inputs |
+                                   self.native_build_inputs |
+                                   self.propagated_native_build_inputs))))
+        ret += '{ ' + ', '.join(args) + ' }:'
 
-        ret += '{ lib, buildRosPackage, fetchurl, ' + \
-               ', '.join(sorted(set(map(self._to_nix_parameter,
-                                        self.build_inputs |
-                                        self.propagated_build_inputs |
-                                        self.check_inputs |
-                                        self.native_build_inputs |
-                                        self.propagated_native_build_inputs)))
-                         ) + ' }:'
 
         ret += dedent('''
         buildRosPackage {{
           pname = "ros-{distro_name}-{name}";
           version = "{version}";
 
-          src = fetchurl {{
+          src = {src};
-            url = "{src_url}";
-            name = "{src_name}";
-            sha256 = "{src_sha256}";
-          }};
 
           buildType = "{build_type}";
         ''').format(
             distro_name=self.distro_name,
             name=self.name,
             version=self.version,
-            src_url=self.src_url,
+            src=src,
-            src_name=self.src_name,
-            src_sha256=self.src_sha256,
             build_type=self.build_type)
 
+        if self.source_root:
+            ret += f'  sourceRoot = "{self.source_root}";\n'
+
         if self.build_inputs:
             ret += "  buildInputs = {};\n" \
                 .format(self._to_nix_list(sorted(self.build_inputs)))

ros2nix/ros2nix.py

@@ -3,14 +3,16 @@
 # Copyright 2019-2024 Ben Wolsieffer <benwolsieffer@gmail.com>
 # Copyright 2024 Michal Sojka <michal.sojka@cvut.cz>
 
+import os
 import argparse
 import itertools
+import subprocess
 from catkin_pkg.package import parse_package_string
 from rosinstall_generator.distro import get_distro
 from superflore.PackageMetadata import PackageMetadata
 from superflore.exceptions import UnresolvedDependency
 from superflore.generators.nix.nix_package import NixPackage
-from superflore.generators.nix.nix_expression import NixExpression, NixLicense
+from nix_expression import NixExpression, NixLicense
 from superflore.utils import (download_file, get_distro_condition_context,
                               get_distros, get_pkg_version, info, resolve_dep,
                               retry_on_exception, warn)
@@ -19,9 +21,6 @@ from superflore.utils import err
 from superflore.utils import ok
 from superflore.utils import warn
 
-org = "Open Source Robotics Foundation" # TODO change
-org_license = "BSD"                     # TODO change
-
 
 def resolve_dependencies(deps: Iterable[str]) -> Set[str]:
     return set(itertools.chain.from_iterable(
@@ -50,83 +49,126 @@ def get_dependencies_as_set(pkg, dep_type):
     return set([d.name for d in deps[dep_type] if d.evaluated_condition is not False])
 
 
+def get_output_file_name(pkg, args):
+    if args.output_as_ros_pkg_name:
+        fn = f"{pkg.name}.nix"
+    elif args.output_as_nix_pkg_name:
+        fn = f"{NixPackage.normalize_name(pkg.name)}.nix"
+    else:
+        fn = args.output
+
+    return os.path.join(args.output_dir, fn)
+
 def main(args):
-    parser = argparse.ArgumentParser()
+    parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-    parser.add_argument("xml", help="Path to package.xml")
+    parser.add_argument("source", nargs="+", help="Path to package.xml") # TODO or a directory containing package.xml or an ")
+
+    group = parser.add_mutually_exclusive_group()
+    group.add_argument("--output", default="package.nix", help="Output filename")
+    group.add_argument("--output-as-ros-pkg-name", action="store_true", help="Name output file based on ROS package name, e.g., package_name.nix")
+    group.add_argument("--output-as-nix-pkg-name", action="store_true", help="Name output file based on Nix package name, e.g., package-name.nix")
+
+    parser.add_argument("--output-dir", default=".", help="Directory to store output files in")
+
+    parser.add_argument("--distro", default="rolling",
+                        help="ROS distro (used as a context for evaluation of conditions in package.xml and in the name of the Nix expression)")
+    parser.add_argument("--src-param",
+                        help="Parameter name in arguments of the generated function to be used as a src attribute")
+    parser.add_argument("--source-root",
+                        help="sourceRoot attribute value in the generated Nix expression. Substring '{package_name}' gets replaced with package name.")
+
+    parser.add_argument("--nixfmt", action="store_true", help="Format the resulting expression with nixfmt")
+
+    parser.add_argument("--copyright-holder")
+    parser.add_argument("--license", help="License of the generated Nix expression, e.g. 'BSD'")
+
     args = parser.parse_args()
 
-    try:
+    for source in args.source:
-        distro_name = "humble"
+        try:
+            with open(source, 'r') as f:
+                package_xml = f.read()
 
-        with open(args.xml, 'r') as f:
+            pkg = parse_package_string(package_xml)
-            package_xml = f.read()
+            pkg.evaluate_conditions(NixPackage._get_condition_context(args.distro))
 
-        pkg = parse_package_string(package_xml)
+            buildtool_deps = get_dependencies_as_set(pkg, "buildtool")
-        pkg.evaluate_conditions(NixPackage._get_condition_context(distro_name))
+            buildtool_export_deps = get_dependencies_as_set(pkg, "buildtool_export")
+            build_deps = get_dependencies_as_set(pkg, "build")
+            build_export_deps = get_dependencies_as_set(pkg, "build_export")
+            exec_deps = get_dependencies_as_set(pkg, "exec")
+            test_deps = get_dependencies_as_set(pkg, "test")
 
-        buildtool_deps = get_dependencies_as_set(pkg, "buildtool")
+            # buildtool_depends are added to buildInputs and nativeBuildInputs.
-        buildtool_export_deps = get_dependencies_as_set(pkg, "buildtool_export")
+            # Some (such as CMake) have binaries that need to run at build time
-        build_deps = get_dependencies_as_set(pkg, "build")
+            # (and therefore need to be in nativeBuildInputs. Others (such as
-        build_export_deps = get_dependencies_as_set(pkg, "build_export")
+            # ament_cmake_*) need to be added to CMAKE_PREFIX_PATH and therefore
-        exec_deps = get_dependencies_as_set(pkg, "exec")
+            # need to be in buildInputs. There is no easy way to distinguish these
-        test_deps = get_dependencies_as_set(pkg, "test")
+            # two cases, so they are added to both, which generally works fine.
+            build_inputs = set(resolve_dependencies(
+                build_deps | buildtool_deps))
+            propagated_build_inputs = resolve_dependencies(
+                exec_deps | build_export_deps | buildtool_export_deps)
+            build_inputs -= propagated_build_inputs
 
-        # buildtool_depends are added to buildInputs and nativeBuildInputs.
+            check_inputs = resolve_dependencies(test_deps)
-        # Some (such as CMake) have binaries that need to run at build time
+            check_inputs -= build_inputs
-        # (and therefore need to be in nativeBuildInputs. Others (such as
-        # ament_cmake_*) need to be added to CMAKE_PREFIX_PATH and therefore
-        # need to be in buildInputs. There is no easy way to distinguish these
-        # two cases, so they are added to both, which generally works fine.
-        build_inputs = set(resolve_dependencies(
-            build_deps | buildtool_deps))
-        propagated_build_inputs = resolve_dependencies(
-            exec_deps | build_export_deps | buildtool_export_deps)
-        build_inputs -= propagated_build_inputs
 
-        check_inputs = resolve_dependencies(test_deps)
+            native_build_inputs = resolve_dependencies(
-        check_inputs -= build_inputs
+                buildtool_deps | buildtool_export_deps)
 
-        native_build_inputs = resolve_dependencies(
+            kwargs = {}
-            buildtool_deps | buildtool_export_deps)
 
-        derivation = NixExpression(
+            if args.src_param:
-            name=NixPackage.normalize_name(pkg.name),
+                kwargs["src_param"] = args.src_param
-            version=pkg.version,
+            else:
-            src_url="src_uri",  # TODO
+                kwargs["src_url"] = "src_uri",  # TODO
-            src_sha256="src_sha256",
+                kwargs["src_sha256"] = "src_sha256",
-            description=pkg.description,
-            licenses=map(NixLicense, pkg.licenses),
-            distro_name=distro_name,
-            build_type=pkg.get_build_type(),
-            build_inputs=build_inputs,
-            propagated_build_inputs=propagated_build_inputs,
-            check_inputs=check_inputs,
-            native_build_inputs=native_build_inputs)
 
-    except Exception as e:
+            if args.source_root:
-        err('Failed to generate derivation for package {}!'.format(pkg))
+                kwargs["source_root"] = args.source_root.replace('{package_name}', pkg.name)
-        raise e
 
-    try:
+            derivation = NixExpression(
-        derivation_text = derivation.get_text(org, org_license)
+                name=NixPackage.normalize_name(pkg.name),
-    except UnresolvedDependency:
+                version=pkg.version,
-        err("'Failed to resolve required dependencies for package {}!"
+                description=pkg.description,
-            .format(pkg))
+                licenses=map(NixLicense, pkg.licenses),
-        unresolved = unresolved_dependencies
+                distro_name=args.distro,
-        for dep in unresolved:
+                build_type=pkg.get_build_type(),
-            err(" unresolved: \"{}\"".format(dep))
+                build_inputs=build_inputs,
-        return None, unresolved, None
+                propagated_build_inputs=propagated_build_inputs,
-    except Exception as e:
+                check_inputs=check_inputs,
-        err('Failed to generate derivation for package {}!'.format(pkg))
+                native_build_inputs=native_build_inputs, **kwargs)
-        raise e
 
-    ok(f"Successfully generated derivation for package '{pkg.name}'.")
+        except Exception as e:
-    try:
+            err('Failed to generate derivation for package {}!'.format(pkg))
-        with open('package.nix', "w") as recipe_file:
+            raise e
-            recipe_file.write(derivation_text)
+
-    except Exception as e:
+        try:
-        err("Failed to write derivation to disk!")
+            derivation_text = derivation.get_text(args.copyright_holder, args.license)
-        raise e
+        except UnresolvedDependency:
+            err("'Failed to resolve required dependencies for package {}!"
+                .format(pkg))
+            unresolved = unresolved_dependencies
+            for dep in unresolved:
+                err(" unresolved: \"{}\"".format(dep))
+            return None, unresolved, None
+        except Exception as e:
+            err('Failed to generate derivation for package {}!'.format(pkg))
+            raise e
+
+        if args.nixfmt:
+            nixfmt = subprocess.Popen(["nixfmt"], stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True)
+            derivation_text, _ = nixfmt.communicate(input=derivation_text)
+
+        try:
+            output_file_name = get_output_file_name(pkg, args)
+            with open(output_file_name, "w") as recipe_file:
+                recipe_file.write(derivation_text)
+                ok(f"Successfully generated derivation for package '{pkg.name}' as '{output_file_name}'.")
+        except Exception as e:
+            err("Failed to write derivation to disk!")
+            raise e
 
 
 if __name__ == '__main__':