diff --git a/pkgs/os-specific/bsd/freebsd/pkgs/source.nix b/pkgs/os-specific/bsd/freebsd/pkgs/source.nix index 9e1c4032ad61..4aee3af27ffa 100644 --- a/pkgs/os-specific/bsd/freebsd/pkgs/source.nix +++ b/pkgs/os-specific/bsd/freebsd/pkgs/source.nix @@ -1,16 +1,12 @@ { fetchFromGitHub, sourceData }: -# Using fetchFromGitHub from their mirror because it's a lot faster than their git server -# If you want you could fetchgit from "https://git.FreeBSD.org/src.git" instead. +# Using fetchFromGitHub from their mirror because we cannot use git during bootstrap +# If you want you could fetchurl from "https://cgit.FreeBSD.org/src" instead. # The update script still pulls directly from git.freebsd.org +# Note that hashes with `forceFetchGit = true` may be different due to `export-ignore` +# directives in gitattributes files. fetchFromGitHub { owner = "freebsd"; repo = "freebsd-src"; inherit (sourceData) rev hash; - - # The GitHub export excludes some files in the git source - # that were marked `export-ignore`. - # A normal git checkout will keep those files, - # matching the update script - forceFetchGit = true; } diff --git a/pkgs/os-specific/bsd/freebsd/update.py b/pkgs/os-specific/bsd/freebsd/update.py index 533a871a4b04..f6a7911e7e75 100755 --- a/pkgs/os-specific/bsd/freebsd/update.py +++ b/pkgs/os-specific/bsd/freebsd/update.py @@ -11,12 +11,13 @@ import pandas import re import subprocess import sys +import tarfile import tempfile import typing import urllib.request _QUERY_VERSION_PATTERN = re.compile('^([A-Z]+)="(.+)"$') -_RELEASE_PATCH_PATTERN = re.compile('^RELEASE-p([0-9]+)$') +_RELEASE_PATCH_PATTERN = re.compile("^RELEASE-p([0-9]+)$") BASE_DIR = os.path.dirname(os.path.abspath(__file__)) MIN_VERSION = packaging.version.Version("13.0.0") MAIN_BRANCH = "main" @@ -32,7 +33,7 @@ BRANCH_PATTERN = re.compile( def request_supported_refs() -> list[str]: # Looks pretty shady but I think this should work with every version of the page in the last 20 years - r = re.compile("^h\d$", re.IGNORECASE) + r = re.compile(r"^h\d$", re.IGNORECASE) soup = bs4.BeautifulSoup( urllib.request.urlopen("https://www.freebsd.org/security"), features="lxml" ) @@ -45,11 +46,11 @@ def request_supported_refs() -> list[str]: return list(df["Branch"]) -def query_version(repo: git.Repo) -> dict[str, typing.Any]: +def query_version(work_dir: str) -> dict[str, typing.Any]: # This only works on FreeBSD 13 and later text = ( subprocess.check_output( - ["bash", os.path.join(repo.working_dir, "sys", "conf", "newvers.sh"), "-v"] + ["bash", os.path.join(work_dir, "sys", "conf", "newvers.sh"), "-v"] ) .decode("utf-8") .strip() @@ -86,18 +87,23 @@ def handle_commit( print(f"{ref_name}: revision still {rev.hexsha}, skipping") return old_versions[ref_name] - repo.git.checkout(rev) - print(f"{ref_name}: checked out {rev.hexsha}") + tar_content = io.BytesIO() + repo.archive(tar_content, rev, format="tar") + tar_content.seek(0) - full_hash = ( - subprocess.check_output(["nix", "hash", "path", "--sri", repo.working_dir]) - .decode("utf-8") - .strip() - ) - print(f"{ref_name}: hash is {full_hash}") + with tempfile.TemporaryDirectory(dir="/tmp") as work_dir: + file = tarfile.TarFile(fileobj=tar_content) + file.extractall(path=work_dir, filter="data") - version = query_version(repo) - print(f"{ref_name}: version is {version['version']}") + full_hash = ( + subprocess.check_output(["nix", "hash", "path", "--sri", work_dir]) + .decode("utf-8") + .strip() + ) + print(f"{ref_name}: hash is {full_hash}") + + version = query_version(work_dir) + print(f"{ref_name}: version is {version['version']}") return { "rev": rev.hexsha, @@ -115,30 +121,19 @@ def main() -> None: print(f"Selected temporary directory {temp_dir.name}") if len(sys.argv) >= 2: - orig_repo = git.Repo(sys.argv[1]) - print(f"Fetching updates on {orig_repo.git_dir}") - orig_repo.remote("origin").fetch() + repo = git.Repo(sys.argv[1]) + print(f"Fetching updates on {repo.git_dir}") + repo.remote("origin").fetch() else: print("Cloning source repo") - orig_repo = git.Repo.clone_from( - "https://git.FreeBSD.org/src.git", to_path=os.path.join(temp_dir.name, "orig") + repo = git.Repo.clone_from( + "https://git.FreeBSD.org/src.git", to_path=temp_dir.name ) supported_refs = request_supported_refs() print(f"Supported refs are: {' '.join(supported_refs)}") - print("Doing git crimes, do not run `git worktree prune` until after script finishes!") - workdir = os.path.join(temp_dir.name, "work") - git.cmd.Git(orig_repo.git_dir).worktree("add", "--orphan", workdir) - - # Have to create object before removing .git otherwise it will complain - repo = git.Repo(workdir) - repo.git.set_persistent_git_options(git_dir=repo.git_dir) - # Remove so that nix hash doesn't see the file - os.remove(os.path.join(workdir, ".git")) - - print(f"Working in directory {repo.working_dir} with git directory {repo.git_dir}") - + print(f"git directory {repo.git_dir}") try: with open(os.path.join(BASE_DIR, "versions.json"), "r") as f: @@ -191,10 +186,10 @@ def main() -> None: ) versions[fullname] = result - with open(os.path.join(BASE_DIR, "versions.json"), "w") as out: json.dump(versions, out, sort_keys=True, indent=2) out.write("\n") -if __name__ == '__main__': + +if __name__ == "__main__": main() diff --git a/pkgs/os-specific/bsd/freebsd/versions.json b/pkgs/os-specific/bsd/freebsd/versions.json index e8c3c7ec80c5..eefea201ebe9 100644 --- a/pkgs/os-specific/bsd/freebsd/versions.json +++ b/pkgs/os-specific/bsd/freebsd/versions.json @@ -1,15 +1,15 @@ { "main": { - "hash": "sha256-xsgY5Ex/B5ngOTa5OZRauSaSYvET5lWI7veJRrSq1oY=", + "hash": "sha256-f69U2FX+3GQimdUJunWApFseeKJhzjgNw4h26ZBPiT0=", "ref": "main", "refType": "branch", - "rev": "c5773d366ecc5271b9bd6e5506c00fb3520f19ae", + "rev": "9357c694e8dca627c25b15529e8435b2ab3dd48b", "supported": false, "version": { "branch": "CURRENT", "major": 15, "minor": 0, - "reldate": "1500035", + "reldate": "1500036", "release": "15.0-CURRENT", "revision": "15.0", "type": "FreeBSD", @@ -161,7 +161,7 @@ } }, "release/14.2.0": { - "hash": "sha256-qZkeuUZbuPOvXZBgP5x6Hii1YN7XdDJzwZeYacIR5BI=", + "hash": "sha256-q5/0pKRlmzZUN+VMwUImkve+NTT1+O68YlwgXp2/aQM=", "ref": "release/14.2.0", "refType": "tag", "rev": "c8918d6c7412fce87922e9bd7e4f5c7d7ca96eb7", @@ -308,7 +308,7 @@ "ref": "releng/14.1", "refType": "branch", "rev": "f389e68ca980b7e053a34d9eddde89b4c2a1ee6c", - "supported": true, + "supported": false, "version": { "branch": "RELEASE-p8", "major": 14, @@ -322,7 +322,7 @@ } }, "releng/14.2": { - "hash": "sha256-XP8BFnXvziaC9wOJj8q31UZXFqCUE7WQ5FdJHEZWGbg=", + "hash": "sha256-iSeTif/cEkqhIQ5hSNF+Rx7dORU13Bc0Dk8Zv4TYWtA=", "ref": "releng/14.2", "refType": "branch", "rev": "ac2cbb46b5f1efa7f7b5d4eb15631337329ec5b2", @@ -340,10 +340,10 @@ } }, "stable/13": { - "hash": "sha256-J9SJKeR6Den3Sep2o4r0cqIDd2V5gbY0Ow9eP69Ny0o=", + "hash": "sha256-rex3CUXyyNDz/TouNqlzZFo6DAwSOKXTdC5HaRQoKKc=", "ref": "stable/13", "refType": "branch", - "rev": "a8431b47adae8f8b731206dc38d82b2245ad245e", + "rev": "64f5a71c1cb79c09e50a37ddbe958224bb64add4", "supported": true, "version": { "branch": "STABLE", @@ -357,10 +357,10 @@ } }, "stable/14": { - "hash": "sha256-tleB6J5Cg1SIN2LCfvV3Cfp4Lxx65UHmiILpin6UYGY=", + "hash": "sha256-s+pj8LttAnKNNohUsJNOJRFmLg6x06tlkJn3xQnFQiY=", "ref": "stable/14", "refType": "branch", - "rev": "6e510d8fbaf8d91da235fe28250cd48124edda9f", + "rev": "e6a470ffcbd708cf404472bb871c2cb76eaa7b39", "supported": true, "version": { "branch": "STABLE",