movefasta/nixpkgs
1
0
Fork 0
mirror of https://github.com/NixOS/nixpkgs.git synced 2025-06-10 03:23:29 +03:00
Code Issues Projects Releases Packages Wiki Activity

maintainers/scripts/update: Format with black

Browse source
This commit is contained in:
Jan Tojnar 2025-03-02 13:49:45 +01:00
parent 6313551cd0
commit b91cb89649
1 changed files with 201 additions and 59 deletions
Download patch file Download diff file Expand all files Collapse all files

260
maintainers/scripts/update.py
View file

@ -10,16 +10,20 @@ import subprocess
import sys
import tempfile
class CalledProcessError(Exception):
process: asyncio.subprocess.Process
stderr: Optional[bytes]
class UpdateFailedException(Exception):
pass
def eprint(*args, **kwargs):
print(*args, file=sys.stderr, **kwargs)
async def check_subprocess_output(*args, **kwargs):
"""
Emulate check and capture_output arguments of subprocess.run function.
@ -38,26 +42,43 @@ async def check_subprocess_output(*args, **kwargs):
return stdout
async def run_update_script(nixpkgs_root: str, merge_lock: asyncio.Lock, temp_dir: Optional[Tuple[str, str]], package: Dict, keep_going: bool):
async def run_update_script(
nixpkgs_root: str,
merge_lock: asyncio.Lock,
temp_dir: Optional[Tuple[str, str]],
package: Dict,
keep_going: bool,
):
worktree: Optional[str] = None
update_script_command = package['updateScript']
update_script_command = package["updateScript"]
if temp_dir is not None:
worktree, _branch = temp_dir
# Ensure the worktree is clean before update.
await check_subprocess_output('git', 'reset', '--hard', '--quiet', 'HEAD', cwd=worktree)
await check_subprocess_output(
"git",
"reset",
"--hard",
"--quiet",
"HEAD",
cwd=worktree,
)
# Update scripts can use $(dirname $0) to get their location but we want to run
# their clones in the git worktree, not in the main nixpkgs repo.
update_script_command = map(lambda arg: re.sub(r'^{0}'.format(re.escape(nixpkgs_root)), worktree, arg), update_script_command)
update_script_command = map(
lambda arg: re.sub(r"^{0}".format(re.escape(nixpkgs_root)), worktree, arg),
update_script_command,
)
eprint(f" - {package['name']}: UPDATING ...")
try:
update_info = await check_subprocess_output(
'env',
"env",
f"UPDATE_NIX_NAME={package['name']}",
f"UPDATE_NIX_PNAME={package['pname']}",
f"UPDATE_NIX_OLD_VERSION={package['oldVersion']}",
@ -69,50 +90,72 @@ async def run_update_script(nixpkgs_root: str, merge_lock: asyncio.Lock, temp_di
)
await merge_changes(merge_lock, package, update_info, temp_dir)
except KeyboardInterrupt as e:
eprint('Cancelling…')
eprint("Cancelling…")
raise asyncio.exceptions.CancelledError()
except CalledProcessError as e:
eprint(f" - {package['name']}: ERROR")
eprint()
eprint(f"--- SHOWING ERROR LOG FOR {package['name']} ----------------------")
eprint()
eprint(e.stderr.decode('utf-8'))
with open(f"{package['pname']}.log", 'wb') as logfile:
eprint(e.stderr.decode("utf-8"))
with open(f"{package['pname']}.log", "wb") as logfile:
logfile.write(e.stderr)
eprint()
eprint(f"--- SHOWING ERROR LOG FOR {package['name']} ----------------------")
if not keep_going:
raise UpdateFailedException(f"The update script for {package['name']} failed with exit code {e.process.returncode}")
raise UpdateFailedException(
f"The update script for {package['name']} failed with exit code {e.process.returncode}"
)
@contextlib.contextmanager
def make_worktree() -> Generator[Tuple[str, str], None, None]:
with tempfile.TemporaryDirectory() as wt:
branch_name = f'update-{os.path.basename(wt)}'
target_directory = f'{wt}/nixpkgs'
branch_name = f"update-{os.path.basename(wt)}"
target_directory = f"{wt}/nixpkgs"
subprocess.run(['git', 'worktree', 'add', '-b', branch_name, target_directory])
subprocess.run(["git", "worktree", "add", "-b", branch_name, target_directory])
try:
yield (target_directory, branch_name)
finally:
subprocess.run(['git', 'worktree', 'remove', '--force', target_directory])
subprocess.run(['git', 'branch', '-D', branch_name])
subprocess.run(["git", "worktree", "remove", "--force", target_directory])
subprocess.run(["git", "branch", "-D", branch_name])
async def commit_changes(name: str, merge_lock: asyncio.Lock, worktree: str, branch: str, changes: List[Dict]) -> None:
async def commit_changes(
name: str,
merge_lock: asyncio.Lock,
worktree: str,
branch: str,
changes: List[Dict],
) -> None:
for change in changes:
# Git can only handle a single index operation at a time
async with merge_lock:
await check_subprocess_output('git', 'add', *change['files'], cwd=worktree)
commit_message = '{attrPath}: {oldVersion} -> {newVersion}'.format(**change)
if 'commitMessage' in change:
commit_message = change['commitMessage']
elif 'commitBody' in change:
commit_message = commit_message + '\n\n' + change['commitBody']
await check_subprocess_output('git', 'commit', '--quiet', '-m', commit_message, cwd=worktree)
await check_subprocess_output('git', 'cherry-pick', branch)
await check_subprocess_output("git", "add", *change["files"], cwd=worktree)
commit_message = "{attrPath}: {oldVersion} -> {newVersion}".format(**change)
if "commitMessage" in change:
commit_message = change["commitMessage"]
elif "commitBody" in change:
commit_message = commit_message + "\n\n" + change["commitBody"]
await check_subprocess_output(
"git",
"commit",
"--quiet",
"-m",
commit_message,
cwd=worktree,
)
await check_subprocess_output("git", "cherry-pick", branch)
async def check_changes(package: Dict, worktree: str, update_info: str):
if 'commit' in package['supportedFeatures']:
async def check_changes(
package: Dict,
worktree: str,
update_info: str,
):
if "commit" in package["supportedFeatures"]:
changes = json.loads(update_info)
else:
changes = [{}]
@ -120,54 +163,93 @@ async def check_changes(package: Dict, worktree: str, update_info: str):
# Try to fill in missing attributes when there is just a single change.
if len(changes) == 1:
# Dynamic data from updater take precedence over static data from passthru.updateScript.
if 'attrPath' not in changes[0]:
if "attrPath" not in changes[0]:
# update.nix is always passing attrPath
changes[0]['attrPath'] = package['attrPath']
changes[0]["attrPath"] = package["attrPath"]
if 'oldVersion' not in changes[0]:
if "oldVersion" not in changes[0]:
# update.nix is always passing oldVersion
changes[0]['oldVersion'] = package['oldVersion']
changes[0]["oldVersion"] = package["oldVersion"]
if 'newVersion' not in changes[0]:
attr_path = changes[0]['attrPath']
obtain_new_version_output = await check_subprocess_output('nix-instantiate', '--expr', f'with import ./. {{}}; lib.getVersion {attr_path}', '--eval', '--strict', '--json', stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, cwd=worktree)
changes[0]['newVersion'] = json.loads(obtain_new_version_output.decode('utf-8'))
if "newVersion" not in changes[0]:
attr_path = changes[0]["attrPath"]
obtain_new_version_output = await check_subprocess_output(
"nix-instantiate",
"--expr",
f"with import ./. {{}}; lib.getVersion {attr_path}",
"--eval",
"--strict",
"--json",
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
cwd=worktree,
)
changes[0]["newVersion"] = json.loads(
obtain_new_version_output.decode("utf-8")
)
if 'files' not in changes[0]:
changed_files_output = await check_subprocess_output('git', 'diff', '--name-only', 'HEAD', stdout=asyncio.subprocess.PIPE, cwd=worktree)
if "files" not in changes[0]:
changed_files_output = await check_subprocess_output(
"git",
"diff",
"--name-only",
"HEAD",
stdout=asyncio.subprocess.PIPE,
cwd=worktree,
)
changed_files = changed_files_output.splitlines()
changes[0]['files'] = changed_files
changes[0]["files"] = changed_files
if len(changed_files) == 0:
return []
return changes
async def merge_changes(merge_lock: asyncio.Lock, package: Dict, update_info: str, temp_dir: Optional[Tuple[str, str]]) -> None:
async def merge_changes(
merge_lock: asyncio.Lock,
package: Dict,
update_info: str,
temp_dir: Optional[Tuple[str, str]],
) -> None:
if temp_dir is not None:
worktree, branch = temp_dir
changes = await check_changes(package, worktree, update_info)
if len(changes) > 0:
await commit_changes(package['name'], merge_lock, worktree, branch, changes)
await commit_changes(package["name"], merge_lock, worktree, branch, changes)
else:
eprint(f" - {package['name']}: DONE, no changes.")
else:
eprint(f" - {package['name']}: DONE.")
async def updater(nixpkgs_root: str, temp_dir: Optional[Tuple[str, str]], merge_lock: asyncio.Lock, packages_to_update: asyncio.Queue[Optional[Dict]], keep_going: bool, commit: bool):
async def updater(
nixpkgs_root: str,
temp_dir: Optional[Tuple[str, str]],
merge_lock: asyncio.Lock,
packages_to_update: asyncio.Queue[Optional[Dict]],
keep_going: bool,
commit: bool,
):
while True:
package = await packages_to_update.get()
if package is None:
# A sentinel received, we are done.
return
if not ('commit' in package['supportedFeatures'] or 'attrPath' in package):
if not ("commit" in package["supportedFeatures"] or "attrPath" in package):
temp_dir = None
await run_update_script(nixpkgs_root, merge_lock, temp_dir, package, keep_going)
async def start_updates(max_workers: int, keep_going: bool, commit: bool, packages: List[Dict]):
async def start_updates(
max_workers: int,
keep_going: bool,
commit: bool,
packages: List[Dict],
):
merge_lock = asyncio.Lock()
packages_to_update: asyncio.Queue[Optional[Dict]] = asyncio.Queue()
@ -177,8 +259,13 @@ async def start_updates(max_workers: int, keep_going: bool, commit: bool, packag
# Do not create more workers than there are packages.
num_workers = min(max_workers, len(packages))
nixpkgs_root_output = await check_subprocess_output('git', 'rev-parse', '--show-toplevel', stdout=asyncio.subprocess.PIPE)
nixpkgs_root = nixpkgs_root_output.decode('utf-8').strip()
nixpkgs_root_output = await check_subprocess_output(
"git",
"rev-parse",
"--show-toplevel",
stdout=asyncio.subprocess.PIPE,
)
nixpkgs_root = nixpkgs_root_output.decode("utf-8").strip()
# Set up temporary directories when using auto-commit.
for i in range(num_workers):
@ -196,7 +283,19 @@ async def start_updates(max_workers: int, keep_going: bool, commit: bool, packag
# Prepare updater workers for each temp_dir directory.
# At most `num_workers` instances of `run_update_script` will be running at one time.
updaters = asyncio.gather(*[updater(nixpkgs_root, temp_dir, merge_lock, packages_to_update, keep_going, commit) for temp_dir in temp_dirs])
updaters = asyncio.gather(
*[
updater(
nixpkgs_root,
temp_dir,
merge_lock,
packages_to_update,
keep_going,
commit,
)
for temp_dir in temp_dirs
]
)
try:
# Start updater workers.
@ -210,43 +309,86 @@ async def start_updates(max_workers: int, keep_going: bool, commit: bool, packag
eprint(e)
sys.exit(1)
def main(max_workers: int, keep_going: bool, commit: bool, packages_path: str, skip_prompt: bool) -> None:
def main(
max_workers: int,
keep_going: bool,
commit: bool,
packages_path: str,
skip_prompt: bool,
) -> None:
with open(packages_path) as f:
packages = json.load(f)
eprint()
eprint('Going to be running update for following packages:')
eprint("Going to be running update for following packages:")
for package in packages:
eprint(f" - {package['name']}")
eprint()
confirm = '' if skip_prompt else input('Press Enter key to continue...')
confirm = "" if skip_prompt else input("Press Enter key to continue...")
if confirm == '':
if confirm == "":
eprint()
eprint('Running update for:')
eprint("Running update for:")
asyncio.run(start_updates(max_workers, keep_going, commit, packages))
eprint()
eprint('Packages updated!')
eprint("Packages updated!")
sys.exit()
else:
eprint('Aborting!')
eprint("Aborting!")
sys.exit(130)
parser = argparse.ArgumentParser(description='Update packages')
parser.add_argument('--max-workers', '-j', dest='max_workers', type=int, help='Number of updates to run concurrently', nargs='?', default=4)
parser.add_argument('--keep-going', '-k', dest='keep_going', action='store_true', help='Do not stop after first failure')
parser.add_argument('--commit', '-c', dest='commit', action='store_true', help='Commit the changes')
parser.add_argument('packages', help='JSON file containing the list of package names and their update scripts')
parser.add_argument('--skip-prompt', '-s', dest='skip_prompt', action='store_true', help='Do not stop for prompts')
if __name__ == '__main__':
parser = argparse.ArgumentParser(description="Update packages")
parser.add_argument(
"--max-workers",
"-j",
dest="max_workers",
type=int,
help="Number of updates to run concurrently",
nargs="?",
default=4,
)
parser.add_argument(
"--keep-going",
"-k",
dest="keep_going",
action="store_true",
help="Do not stop after first failure",
)
parser.add_argument(
"--commit",
"-c",
dest="commit",
action="store_true",
help="Commit the changes",
)
parser.add_argument(
"packages",
help="JSON file containing the list of package names and their update scripts",
)
parser.add_argument(
"--skip-prompt",
"-s",
dest="skip_prompt",
action="store_true",
help="Do not stop for prompts",
)
if __name__ == "__main__":
args = parser.parse_args()
try:
main(args.max_workers, args.keep_going, args.commit, args.packages, args.skip_prompt)
main(
args.max_workers,
args.keep_going,
args.commit,
args.packages,
args.skip_prompt,
)
except KeyboardInterrupt as e:
# Let’s cancel outside of the main loop too.
sys.exit(130)