maintainers/scripts/update.nix: Add support for auto-commiting changes

Update scripts can now declare features using

	passthru.updateScript = {
	  command = [ ../../update.sh pname ];
	  supportedFeatures = [ "commit" ];
	};

A `commit` feature means that when the update script finishes successfully,
it will print a JSON list like the following:

	[
	  {
	    "attrPath": "volume_key",
	    "oldVersion": "0.3.11",
	    "newVersion": "0.3.12",
	    "files": [
	      "/path/to/nixpkgs/pkgs/development/libraries/volume-key/default.nix"
	    ]
	  }
	]

and data from that will be used when update.nix is run with --argstr commit true
to create commits.

We will create a new git worktree for each thread in the pool and run the update
script there. Then we will commit the change and cherry pick it in the main repo,
releasing the worktree for a next change.

maintainers/scripts/update.py

@@ -1,22 +1,45 @@
 import argparse
+import contextlib
 import concurrent.futures
 import json
 import os
 import subprocess
 import sys
+import tempfile
+import threading
 
 updates = {}
 
+thread_name_prefix='UpdateScriptThread'
+
 def eprint(*args, **kwargs):
     print(*args, file=sys.stderr, **kwargs)
 
-def run_update_script(package):
+def run_update_script(package, commit):
+    if commit and 'commit' in package['supportedFeatures']:
+        thread_name = threading.current_thread().name
+        worktree, _branch, lock = temp_dirs[thread_name]
+        lock.acquire()
+        package['thread'] = thread_name
+    else:
+        worktree = None
+
     eprint(f" - {package['name']}: UPDATING ...")
 
-    subprocess.run(package['updateScript'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=True)
+    return subprocess.run(package['updateScript'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True, cwd=worktree)
 
+@contextlib.contextmanager
+def make_worktree():
+    with tempfile.TemporaryDirectory() as wt:
+        branch_name = f'update-{os.path.basename(wt)}'
+        target_directory = f'{wt}/nixpkgs'
 
-def main(max_workers, keep_going, packages):
+        subprocess.run(['git', 'worktree', 'add', '-b', branch_name, target_directory], check=True)
+        yield (target_directory, branch_name)
+        subprocess.run(['git', 'worktree', 'remove', target_directory], check=True)
+        subprocess.run(['git', 'branch', '-D', branch_name], check=True)
+
+def main(max_workers, keep_going, commit, packages):
     with open(sys.argv[1]) as f:
         packages = json.load(f)
 
@@ -31,15 +54,29 @@ def main(max_workers, keep_going, packages):
         eprint()
         eprint('Running update for:')
 
-        with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
+        with contextlib.ExitStack() as stack, concurrent.futures.ThreadPoolExecutor(max_workers=max_workers, thread_name_prefix=thread_name_prefix) as executor:
+            global temp_dirs
+
+            if commit:
+                temp_dirs = {f'{thread_name_prefix}_{str(i)}': (*stack.enter_context(make_worktree()), threading.Lock()) for i in range(max_workers)}
+
             for package in packages:
-                updates[executor.submit(run_update_script, package)] = package
+                updates[executor.submit(run_update_script, package, commit)] = package
 
             for future in concurrent.futures.as_completed(updates):
                 package = updates[future]
 
                 try:
-                    future.result()
+                    p = future.result()
+                    if commit and 'commit' in package['supportedFeatures']:
+                        thread_name = package['thread']
+                        worktree, branch, lock = temp_dirs[thread_name]
+                        changes = json.loads(p.stdout)
+                        for change in changes:
+                            subprocess.run(['git', 'add'] + change['files'], check=True, cwd=worktree)
+                            commit_message = '{attrPath}: {oldVersion} → {newVersion}'.format(**change)
+                            subprocess.run(['git', 'commit', '-m', commit_message], check=True, cwd=worktree)
+                            subprocess.run(['git', 'cherry-pick', branch], check=True)
                     eprint(f" - {package['name']}: DONE.")
                 except subprocess.CalledProcessError as e:
                     eprint(f" - {package['name']}: ERROR")
@@ -54,6 +91,9 @@ def main(max_workers, keep_going, packages):
 
                     if not keep_going:
                         sys.exit(1)
+                finally:
+                    if commit and 'commit' in package['supportedFeatures']:
+                        lock.release()
 
         eprint()
         eprint('Packages updated!')
@@ -65,13 +105,14 @@ def main(max_workers, keep_going, packages):
 parser = argparse.ArgumentParser(description='Update packages')
 parser.add_argument('--max-workers', '-j', dest='max_workers', type=int, help='Number of updates to run concurrently', nargs='?', default=4)
 parser.add_argument('--keep-going', '-k', dest='keep_going', action='store_true', help='Do not stop after first failure')
+parser.add_argument('--commit', '-c', dest='commit', action='store_true', help='Commit the changes')
 parser.add_argument('packages', help='JSON file containing the list of package names and their update scripts')
 
 if __name__ == '__main__':
     args = parser.parse_args()
 
     try:
-        main(args.max_workers, args.keep_going, args.packages)
+        main(args.max_workers, args.keep_going, args.commit, args.packages)
     except (KeyboardInterrupt, SystemExit) as e:
         for update in updates:
             update.cancel()